diff --git a/ENVEXAMPLE b/ENVEXAMPLE index d17b1fc5..a387785f 100644 --- a/ENVEXAMPLE +++ b/ENVEXAMPLE @@ -1,12 +1,12 @@ # App Setup NODE_ENV=production # Set to 'development' or 'production' as required -JWT_SECRET=your_jwt_secret_key # Replace with a secure JWT secret key +JWT_SECRET=a9Z$kLq7^f03GzNw!bP9dH4xV6sT2yXl3O8vR@uYq3 # Replace with a secure JWT secret key DB_NAME=maxun # Your PostgreSQL database name DB_USER=postgres # PostgreSQL username DB_PASSWORD=postgres # PostgreSQL password DB_HOST=postgres # Host for PostgreSQL in Docker DB_PORT=5432 # Port for PostgreSQL (default: 5432) -ENCRYPTION_KEY=your_encryption_key # Key for encrypting sensitive data (passwords and proxies) +ENCRYPTION_KEY=f4d5e6a7b8c9d0e1f23456789abcdef01234567890abcdef123456789abcdef0 # Key for encrypting sensitive data (passwords and proxies) MINIO_ENDPOINT=minio # MinIO endpoint in Docker MINIO_PORT=9000 # Port for MinIO (default: 9000) MINIO_ACCESS_KEY=minio_access_key # MinIO access key diff --git a/README.md b/README.md index 67638748..e072af78 100644 --- a/README.md +++ b/README.md @@ -16,10 +16,13 @@ Maxun lets you train a robot in 2 minutes and scrape the web on auto-pilot. Web

Website | - Discord | + Discord | Twitter | Join Maxun Cloud | Watch Tutorials +
+
+getmaxun%2Fmaxun | Trendshift

![maxun_demo](https://github.com/user-attachments/assets/a61ba670-e56a-4ae1-9681-0b4bd6ba9cdc) @@ -61,8 +64,8 @@ You can access the frontend at http://localhost:5173/ and backend at http://loca | Variable | Mandatory | Description | If Not Set | |-----------------------|-----------|----------------------------------------------------------------------------------------------|--------------------------------------------------------------| -| `BACKEND_URL` | Yes | URL to run backend on. | Backend won't start. If not sure, set to http://localhost:8080 | -| `VITE_BACKEND_URL` | Yes | URL to run backend on. | Backend won't start. If not sure, set to http://localhost:8080 | +| `BACKEND_URL` | Yes | URL to run backend on. | Default value: http://localhost:8080 | +| `VITE_BACKEND_URL` | Yes | URL used by frontend to connect to backend | Default value: http://localhost:8080 | | `JWT_SECRET` | Yes | Secret key used to sign and verify JSON Web Tokens (JWTs) for authentication. | JWT authentication will not work. | | `DB_NAME` | Yes | Name of the Postgres database to connect to. | Database connection will fail. | | `DB_USER` | Yes | Username for Postgres database authentication. | Database connection will fail. | diff --git a/docker-compose.yml b/docker-compose.yml index 1aeefd26..d03334e0 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -32,9 +32,10 @@ services: environment: MINIO_ROOT_USER: ${MINIO_ACCESS_KEY} MINIO_ROOT_PASSWORD: ${MINIO_SECRET_KEY} - command: server /data + command: server /data --console-address :9001 ports: - - "9000:9000" + - "9000:9000" # API port + - "9001:9001" # WebUI port volumes: - minio_data:/data diff --git a/esbuild.config.js b/esbuild.config.js new file mode 100644 index 00000000..e69de29b diff --git a/package.json b/package.json index fd966458..b25715d4 100644 --- a/package.json +++ b/package.json @@ -39,6 +39,7 @@ "ioredis": "^5.4.1", "joi": "^17.6.0", "jsonwebtoken": "^9.0.2", + "jwt-decode": "^4.0.0", "loglevel": "^1.8.0", "loglevel-plugin-remote": "^0.6.8", "maxun-core": "^0.0.3", @@ -46,7 +47,7 @@ "moment-timezone": "^0.5.45", "node-cron": "^3.0.3", "pg": "^8.13.0", - "playwright": "^1.20.1", + "playwright": "^1.48.2", "playwright-extra": "^4.3.6", "posthog-node": "^4.2.1", "prismjs": "^1.28.0", @@ -89,6 +90,7 @@ "devDependencies": { "@types/cookie-parser": "^1.4.7", "@types/express": "^4.17.13", + "@types/js-cookie": "^3.0.6", "@types/loglevel": "^1.6.3", "@types/node": "22.7.9", "@types/node-cron": "^3.0.11", @@ -102,6 +104,7 @@ "ajv": "^8.8.2", "concurrently": "^7.0.0", "cross-env": "^7.0.3", + "js-cookie": "^3.0.5", "nodemon": "^2.0.15", "ts-node": "^10.4.0", "vite": "^5.4.10" diff --git a/server/src/browser-management/classes/RemoteBrowser.ts b/server/src/browser-management/classes/RemoteBrowser.ts index cbda3942..d63bd55a 100644 --- a/server/src/browser-management/classes/RemoteBrowser.ts +++ b/server/src/browser-management/classes/RemoteBrowser.ts @@ -307,6 +307,7 @@ export class RemoteBrowser { const workflow = this.generator.AddGeneratedFlags(this.generator.getWorkflowFile()); await this.initializeNewPage(); if (this.currentPage) { + this.currentPage.setViewportSize({ height: 400, width: 900 }); const params = this.generator.getParams(); if (params) { this.interpreterSettings.params = params.reduce((acc, param) => { diff --git a/server/src/middlewares/auth.ts b/server/src/middlewares/auth.ts index 75540ad5..eebef4a6 100644 --- a/server/src/middlewares/auth.ts +++ b/server/src/middlewares/auth.ts @@ -16,12 +16,12 @@ export const requireSignIn = (req: UserRequest, res: Response, next: any) => { } verify(token, secret, (err: any, user: any) => { - console.log(err) - - if (err) return res.sendStatus(403) - - req.user = user; - - next() - }) + if (err) { + console.log('JWT verification error:', err); + return res.sendStatus(403); + } + req.user = { ...user, id: user.userId }; // Normalize to include `id` + next(); + }); + }; diff --git a/server/src/routes/auth.ts b/server/src/routes/auth.ts index 6766a356..96f4769a 100644 --- a/server/src/routes/auth.ts +++ b/server/src/routes/auth.ts @@ -1,480 +1,559 @@ -import { Router, Request, Response } from 'express'; -import User from '../models/User'; -import Robot from '../models/Robot'; -import jwt from 'jsonwebtoken'; -import { hashPassword, comparePassword } from '../utils/auth'; -import { requireSignIn } from '../middlewares/auth'; -import { genAPIKey } from '../utils/api'; -import { google } from 'googleapis'; -import { capture } from "../utils/analytics" +import { Router, Request, Response } from "express"; +import User from "../models/User"; +import Robot from "../models/Robot"; +import jwt from "jsonwebtoken"; +import { hashPassword, comparePassword } from "../utils/auth"; +import { requireSignIn } from "../middlewares/auth"; +import { genAPIKey } from "../utils/api"; +import { google } from "googleapis"; +import { capture } from "../utils/analytics"; export const router = Router(); interface AuthenticatedRequest extends Request { - user?: { id: string }; + user?: { id: string }; } -router.post('/register', async (req, res) => { - console.log('Received request at /auth/register'); - console.log('Received body:', req.body); +router.post("/register", async (req, res) => { + console.log("Received request at /auth/register"); + + try { + const { email, password } = req.body; + + if (!email) return res.status(400).send("Email is required"); + if (!password || password.length < 6) + return res + .status(400) + .send("Password is required and must be at least 6 characters"); + + let userExist = await User.findOne({ raw: true, where: { email } }); + if (userExist) return res.status(400).send("User already exists"); + + const hashedPassword = await hashPassword(password); + + let user: any; + try { - const { email, password } = req.body - - if (!email) return res.status(400).send('Email is required') - if (!password || password.length < 6) return res.status(400).send('Password is required and must be at least 6 characters') - - let userExist = await User.findOne({ raw: true, where: { email } }); - if (userExist) return res.status(400).send('User already exists') - - const hashedPassword = await hashPassword(password) - - let user: any; - - try { - user = await User.create({ email, password: hashedPassword }); - } catch ( - error: any - ) { - console.log(`Could not create user - ${error}`) - return res.status(500).send(`Could not create user - ${error.message}`) - } - - if (!process.env.JWT_SECRET) { - console.log('JWT_SECRET is not defined in the environment'); - return res.status(500).send('Internal Server Error'); - } - - const token = jwt.sign({ id: user.id }, process.env.JWT_SECRET as string, { expiresIn: '12h' }); - user.password = undefined as unknown as string - res.cookie('token', token, { - httpOnly: true - }) - capture( - 'maxun-oss-user-registered', - { - email: user.email, - userId: user.id, - registeredAt: new Date().toISOString() - } - ) - console.log(`User registered - ${user.email}`) - res.json(user) + user = await User.create({ email, password: hashedPassword }); } catch (error: any) { - console.log(`Could not register user - ${error}`) - res.status(500).send(`Could not register user - ${error.message}`) + console.log(`Could not create user - ${error}`); + return res.status(500).send(`Could not create user - ${error.message}`); } -}) -router.post('/login', async (req, res) => { - try { - const { email, password } = req.body; - if (!email || !password) return res.status(400).send('Email and password are required') - if (password.length < 6) return res.status(400).send('Password must be at least 6 characters') - - let user = await User.findOne({ raw: true, where: { email } }); - if (!user) return res.status(400).send('User does not exist'); - - const match = await comparePassword(password, user.password) - if (!match) return res.status(400).send('Invalid email or password') - - const token = jwt.sign({ id: user?.id }, process.env.JWT_SECRET as string, { expiresIn: '12h' }); - - // return user and token to client, exclude hashed password - if (user) { - user.password = undefined as unknown as string; - } - res.cookie('token', token, { - httpOnly: true - }) - res.json(user) - } catch (error: any) { - res.status(400).send(`Could not login user - ${error.message}`) - console.log(`Could not login user - ${error}`) + if (!process.env.JWT_SECRET) { + console.log("JWT_SECRET is not defined in the environment"); + return res.status(500).send("Internal Server Error"); } -}) -router.get('/logout', async (req, res) => { - try { - res.clearCookie('token') - return res.json({ message: 'Logout successful' }) - } catch (error: any) { - res.status(500).send(`Could not logout user - ${error.message}`) - } -}) - -router.get('/current-user', requireSignIn, async (req: AuthenticatedRequest, res) => { - try { - if (!req.user) { - return res.status(401).json({ ok: false, error: 'Unauthorized' }); - } - const user = await User.findByPk(req.user.id, { - attributes: { exclude: ['password'] }, - }); - if (!user) { - return res.status(404).json({ ok: false, error: 'User not found' }); - } else { - return res.status(200).json({ ok: true, user: user }); - } - } catch (error: any) { - console.error('Error in current-user route:', error); - return res.status(500).json({ ok: false, error: `Could not fetch current user: ${error.message}` }); - } + const token = jwt.sign({ id: user.id }, process.env.JWT_SECRET as string); + user.password = undefined as unknown as string; + res.cookie("token", token, { + httpOnly: true, + }); + capture("maxun-oss-user-registered", { + email: user.email, + userId: user.id, + registeredAt: new Date().toISOString(), + }); + console.log(`User registered - ${user.email}`); + res.json(user); + } catch (error: any) { + console.log(`Could not register user - ${error}`); + res.status(500).send(`Could not register user - ${error.message}`); + } }); -router.get('/user/:id', requireSignIn, async (req: AuthenticatedRequest, res) => { - try { - const { id } = req.params; - if (!id) { - return res.status(400).json({ message: 'User ID is required' }); - } +router.post("/login", async (req, res) => { + try { + const { email, password } = req.body; + if (!email || !password) + return res.status(400).send("Email and password are required"); + if (password.length < 6) + return res.status(400).send("Password must be at least 6 characters"); - const user = await User.findByPk(id, { - attributes: { exclude: ['password'] }, - }); + let user = await User.findOne({ raw: true, where: { email } }); + if (!user) return res.status(400).send("User does not exist"); - if (!user) { - return res.status(404).json({ message: 'User not found' }); - } + const match = await comparePassword(password, user.password); + if (!match) return res.status(400).send("Invalid email or password"); - return res.status(200).json({ message: 'User fetched successfully', user }); - } catch (error: any) { - return res.status(500).json({ message: 'Error fetching user', error: error.message }); + const token = jwt.sign({ id: user?.id }, process.env.JWT_SECRET as string); + + // return user and token to client, exclude hashed password + if (user) { + user.password = undefined as unknown as string; } + res.cookie("token", token, { + httpOnly: true, + }); + capture("maxun-oss-user-login", { + email: user.email, + userId: user.id, + loggedInAt: new Date().toISOString(), + }); + res.json(user); + } catch (error: any) { + res.status(400).send(`Could not login user - ${error.message}`); + console.log(`Could not login user - ${error}`); + } }); -router.post('/generate-api-key', requireSignIn, async (req: AuthenticatedRequest, res) => { +router.get("/logout", async (req, res) => { + try { + res.clearCookie("token"); + return res.json({ message: "Logout successful" }); + } catch (error: any) { + res.status(500).send(`Could not logout user - ${error.message}`); + } +}); + +router.get( + "/current-user", + requireSignIn, + async (req: AuthenticatedRequest, res) => { try { - if (!req.user) { - return res.status(401).json({ ok: false, error: 'Unauthorized' }); - } - const user = await User.findByPk(req.user.id, { - attributes: { exclude: ['password'] }, + if (!req.user) { + return res.status(401).json({ ok: false, error: "Unauthorized" }); + } + const user = await User.findByPk(req.user.id, { + attributes: { exclude: ["password"] }, + }); + if (!user) { + return res.status(404).json({ ok: false, error: "User not found" }); + } else { + return res.status(200).json({ ok: true, user: user }); + } + } catch (error: any) { + console.error("Error in current-user route:", error); + return res + .status(500) + .json({ + ok: false, + error: `Could not fetch current user: ${error.message}`, }); + } + } +); - if (!user) { - return res.status(404).json({ message: 'User not found' }); - } +router.get( + "/user/:id", + requireSignIn, + async (req: AuthenticatedRequest, res) => { + try { + const { id } = req.params; + if (!id) { + return res.status(400).json({ message: "User ID is required" }); + } - if (user.api_key) { - return res.status(400).json({ message: 'API key already exists' }); - } - const apiKey = genAPIKey(); + const user = await User.findByPk(id, { + attributes: { exclude: ["password"] }, + }); - await user.update({ api_key: apiKey }); + if (!user) { + return res.status(404).json({ message: "User not found" }); + } - capture( - 'maxun-oss-api-key-created', - { - user_id: user.id, - created_at: new Date().toISOString() - } - ) + return res + .status(200) + .json({ message: "User fetched successfully", user }); + } catch (error: any) { + return res + .status(500) + .json({ message: "Error fetching user", error: error.message }); + } + } +); - return res.status(200).json({ - message: 'API key generated successfully', - api_key: apiKey, - }); +router.post( + "/generate-api-key", + requireSignIn, + async (req: AuthenticatedRequest, res) => { + try { + if (!req.user) { + return res.status(401).json({ ok: false, error: "Unauthorized" }); + } + const user = await User.findByPk(req.user.id, { + attributes: { exclude: ["password"] }, + }); + + if (!user) { + return res.status(404).json({ message: "User not found" }); + } + + if (user.api_key) { + return res.status(400).json({ message: "API key already exists" }); + } + const apiKey = genAPIKey(); + + await user.update({ api_key: apiKey }); + + capture("maxun-oss-api-key-created", { + user_id: user.id, + created_at: new Date().toISOString(), + }); + + return res.status(200).json({ + message: "API key generated successfully", + api_key: apiKey, + }); } catch (error) { - return res.status(500).json({ message: 'Error generating API key', error }); + return res + .status(500) + .json({ message: "Error generating API key", error }); } -}); + } +); -router.get('/api-key', requireSignIn, async (req: AuthenticatedRequest, res) => { +router.get( + "/api-key", + requireSignIn, + async (req: AuthenticatedRequest, res) => { try { - if (!req.user) { - return res.status(401).json({ ok: false, error: 'Unauthorized' }); - } + if (!req.user) { + return res.status(401).json({ ok: false, error: "Unauthorized" }); + } - const user = await User.findByPk(req.user.id, { - raw: true, - attributes: ['api_key'], - }); + const user = await User.findByPk(req.user.id, { + raw: true, + attributes: ["api_key"], + }); - if (!user) { - return res.status(404).json({ message: 'User not found' }); - } + if (!user) { + return res.status(404).json({ message: "User not found" }); + } - return res.status(200).json({ - message: 'API key fetched successfully', - api_key: user.api_key || null, - }); + return res.status(200).json({ + message: "API key fetched successfully", + api_key: user.api_key || null, + }); } catch (error) { - return res.status(500).json({ message: 'Error fetching API key', error }); + return res.status(500).json({ message: "Error fetching API key", error }); } -}); - -router.delete('/delete-api-key', requireSignIn, async (req: AuthenticatedRequest, res) => { + } +); +router.delete( + "/delete-api-key", + requireSignIn, + async (req: AuthenticatedRequest, res) => { if (!req.user) { - return res.status(401).send({ error: 'Unauthorized' }); + return res.status(401).send({ error: "Unauthorized" }); } try { - const user = await User.findByPk(req.user.id, { raw: true }); + const user = await User.findByPk(req.user.id, { raw: true }); - if (!user) { - return res.status(404).json({ message: 'User not found' }); - } + if (!user) { + return res.status(404).json({ message: "User not found" }); + } - if (!user.api_key) { - return res.status(404).json({ message: 'API Key not found' }); - } + if (!user.api_key) { + return res.status(404).json({ message: "API Key not found" }); + } - await User.update({ api_key: null }, { where: { id: req.user.id } }); + await User.update({ api_key: null }, { where: { id: req.user.id } }); - capture( - 'maxun-oss-api-key-deleted', - { - user_id: user.id, - deleted_at: new Date().toISOString() - } - ) + capture("maxun-oss-api-key-deleted", { + user_id: user.id, + deleted_at: new Date().toISOString(), + }); - return res.status(200).json({ message: 'API Key deleted successfully' }); + return res.status(200).json({ message: "API Key deleted successfully" }); } catch (error: any) { - return res.status(500).json({ message: 'Error deleting API key', error: error.message }); + return res + .status(500) + .json({ message: "Error deleting API key", error: error.message }); } -}); + } +); const oauth2Client = new google.auth.OAuth2( - process.env.GOOGLE_CLIENT_ID, - process.env.GOOGLE_CLIENT_SECRET, - process.env.GOOGLE_REDIRECT_URI + process.env.GOOGLE_CLIENT_ID, + process.env.GOOGLE_CLIENT_SECRET, + process.env.GOOGLE_REDIRECT_URI ); // Step 1: Redirect to Google for authentication -router.get('/google', (req, res) => { - const { robotId } = req.query; - if (!robotId) { - return res.status(400).json({ message: 'Robot ID is required' }); - } - const scopes = [ - 'https://www.googleapis.com/auth/spreadsheets', - 'https://www.googleapis.com/auth/userinfo.email', - 'https://www.googleapis.com/auth/drive.readonly', - ]; - const url = oauth2Client.generateAuthUrl({ - access_type: 'offline', - prompt: 'consent', // Ensures you get a refresh token on first login - scope: scopes, - state: robotId.toString(), - }); - res.redirect(url); +router.get("/google", (req, res) => { + const { robotId } = req.query; + if (!robotId) { + return res.status(400).json({ message: "Robot ID is required" }); + } + const scopes = [ + "https://www.googleapis.com/auth/spreadsheets", + "https://www.googleapis.com/auth/userinfo.email", + "https://www.googleapis.com/auth/drive.readonly", + ]; + const url = oauth2Client.generateAuthUrl({ + access_type: "offline", + prompt: "consent", // Ensures you get a refresh token on first login + scope: scopes, + state: robotId.toString(), + }); + res.redirect(url); }); // Step 2: Handle Google OAuth callback -router.get('/google/callback', requireSignIn, async (req: AuthenticatedRequest, res) => { +router.get( + "/google/callback", + requireSignIn, + async (req: AuthenticatedRequest, res) => { const { code, state } = req.query; try { - if (!state) { - return res.status(400).json({ message: 'Robot ID is required' }); - } + if (!state) { + return res.status(400).json({ message: "Robot ID is required" }); + } - const robotId = state + const robotId = state; - // Get access and refresh tokens - if (typeof code !== 'string') { - return res.status(400).json({ message: 'Invalid code' }); - } - const { tokens } = await oauth2Client.getToken(code); - oauth2Client.setCredentials(tokens); + // Get access and refresh tokens + if (typeof code !== "string") { + return res.status(400).json({ message: "Invalid code" }); + } + const { tokens } = await oauth2Client.getToken(code); + oauth2Client.setCredentials(tokens); - // Get user profile from Google - const oauth2 = google.oauth2({ version: 'v2', auth: oauth2Client }); - const { data: { email } } = await oauth2.userinfo.get(); + // Get user profile from Google + const oauth2 = google.oauth2({ version: "v2", auth: oauth2Client }); + const { + data: { email }, + } = await oauth2.userinfo.get(); - if (!email) { - return res.status(400).json({ message: 'Email not found' }); - } + if (!email) { + return res.status(400).json({ message: "Email not found" }); + } - if (!req.user) { - return res.status(401).send({ error: 'Unauthorized' }); - } + if (!req.user) { + return res.status(401).send({ error: "Unauthorized" }); + } - // Get the currently authenticated user (from `requireSignIn`) - let user = await User.findOne({ where: { id: req.user.id } }); + // Get the currently authenticated user (from `requireSignIn`) + let user = await User.findOne({ where: { id: req.user.id } }); - if (!user) { - return res.status(400).json({ message: 'User not found' }); - } + if (!user) { + return res.status(400).json({ message: "User not found" }); + } - let robot = await Robot.findOne({ where: { 'recording_meta.id': robotId } }); + let robot = await Robot.findOne({ + where: { "recording_meta.id": robotId }, + }); - if (!robot) { - return res.status(400).json({ message: 'Robot not found' }); - } + if (!robot) { + return res.status(400).json({ message: "Robot not found" }); + } - robot = await robot.update({ - google_sheet_email: email, - google_access_token: tokens.access_token, - google_refresh_token: tokens.refresh_token, - }); - capture( - 'maxun-oss-google-sheet-integration-created', - { - user_id: user.id, - robot_id: robot.recording_meta.id, - created_at: new Date().toISOString() - } - ) + robot = await robot.update({ + google_sheet_email: email, + google_access_token: tokens.access_token, + google_refresh_token: tokens.refresh_token, + }); + capture("maxun-oss-google-sheet-integration-created", { + user_id: user.id, + robot_id: robot.recording_meta.id, + created_at: new Date().toISOString(), + }); - // List user's Google Sheets from their Google Drive - const drive = google.drive({ version: 'v3', auth: oauth2Client }); - const response = await drive.files.list({ - q: "mimeType='application/vnd.google-apps.spreadsheet'", // List only Google Sheets files - fields: 'files(id, name)', // Retrieve the ID and name of each file - }); + // List user's Google Sheets from their Google Drive + const drive = google.drive({ version: "v3", auth: oauth2Client }); + const response = await drive.files.list({ + q: "mimeType='application/vnd.google-apps.spreadsheet'", // List only Google Sheets files + fields: "files(id, name)", // Retrieve the ID and name of each file + }); - const files = response.data.files || []; - if (files.length === 0) { - return res.status(404).json({ message: 'No spreadsheets found.' }); - } + const files = response.data.files || []; + if (files.length === 0) { + return res.status(404).json({ message: "No spreadsheets found." }); + } - // Generate JWT token for session - const jwtToken = jwt.sign({ userId: user.id }, process.env.JWT_SECRET as string, { expiresIn: '12h' }); - res.cookie('token', jwtToken, { httpOnly: true }); + // Generate JWT token for session + const jwtToken = jwt.sign( + { userId: user.id }, + process.env.JWT_SECRET as string + ); + res.cookie("token", jwtToken, { httpOnly: true }); - res.json({ - message: 'Google authentication successful', - google_sheet_email: robot.google_sheet_email, - jwtToken, - files - }); + // res.json({ + // message: 'Google authentication successful', + // google_sheet_email: robot.google_sheet_email, + // jwtToken, + // files + // }); + + res.cookie("robot_auth_status", "success", { + httpOnly: false, + maxAge: 60000, + }); // 1-minute expiration + res.cookie("robot_auth_message", "Robot successfully authenticated", { + httpOnly: false, + maxAge: 60000, + }); + res.redirect(`http://localhost:5173`); } catch (error: any) { - res.status(500).json({ message: `Google OAuth error: ${error.message}` }); + res.status(500).json({ message: `Google OAuth error: ${error.message}` }); } -}); + } +); // Step 3: Get data from Google Sheets -router.post('/gsheets/data', requireSignIn, async (req: AuthenticatedRequest, res) => { +router.post( + "/gsheets/data", + requireSignIn, + async (req: AuthenticatedRequest, res) => { const { spreadsheetId, robotId } = req.body; if (!req.user) { - return res.status(401).send({ error: 'Unauthorized' }); + return res.status(401).send({ error: "Unauthorized" }); } const user = await User.findByPk(req.user.id, { raw: true }); if (!user) { - return res.status(400).json({ message: 'User not found' }); + return res.status(400).json({ message: "User not found" }); } - const robot = await Robot.findOne({ where: { 'recording_meta.id': robotId }, raw: true }); + const robot = await Robot.findOne({ + where: { "recording_meta.id": robotId }, + raw: true, + }); if (!robot) { - return res.status(400).json({ message: 'Robot not found' }); + return res.status(400).json({ message: "Robot not found" }); } // Set Google OAuth credentials oauth2Client.setCredentials({ - access_token: robot.google_access_token, - refresh_token: robot.google_refresh_token, + access_token: robot.google_access_token, + refresh_token: robot.google_refresh_token, }); - const sheets = google.sheets({ version: 'v4', auth: oauth2Client }); + const sheets = google.sheets({ version: "v4", auth: oauth2Client }); try { - // Fetch data from the spreadsheet (you can let the user choose a specific range too) - const sheetData = await sheets.spreadsheets.values.get({ - spreadsheetId, - range: 'Sheet1!A1:D5', // Default range, could be dynamic based on user input - }); - res.json(sheetData.data); + // Fetch data from the spreadsheet (you can let the user choose a specific range too) + const sheetData = await sheets.spreadsheets.values.get({ + spreadsheetId, + range: "Sheet1!A1:D5", // Default range, could be dynamic based on user input + }); + res.json(sheetData.data); } catch (error: any) { - res.status(500).json({ message: `Error accessing Google Sheets: ${error.message}` }); + res + .status(500) + .json({ message: `Error accessing Google Sheets: ${error.message}` }); } -}); + } +); // Step 4: Get user's Google Sheets files (new route) -router.get('/gsheets/files', requireSignIn, async (req, res) => { - try { - const robotId = req.query.robotId; - const robot = await Robot.findOne({ where: { 'recording_meta.id': robotId }, raw: true }); +router.get("/gsheets/files", requireSignIn, async (req, res) => { + try { + const robotId = req.query.robotId; + const robot = await Robot.findOne({ + where: { "recording_meta.id": robotId }, + raw: true, + }); - if (!robot) { - return res.status(400).json({ message: 'Robot not found' }); - } - - oauth2Client.setCredentials({ - access_token: robot.google_access_token, - refresh_token: robot.google_refresh_token, - }); - - // List user's Google Sheets files from their Google Drive - const drive = google.drive({ version: 'v3', auth: oauth2Client }); - const response = await drive.files.list({ - q: "mimeType='application/vnd.google-apps.spreadsheet'", - fields: 'files(id, name)', - }); - - const files = response.data.files || []; - if (files.length === 0) { - return res.status(404).json({ message: 'No spreadsheets found.' }); - } - - res.json(files); - } catch (error: any) { - console.log('Error fetching Google Sheets files:', error); - res.status(500).json({ message: `Error retrieving Google Sheets files: ${error.message}` }); + if (!robot) { + return res.status(400).json({ message: "Robot not found" }); } + + oauth2Client.setCredentials({ + access_token: robot.google_access_token, + refresh_token: robot.google_refresh_token, + }); + + // List user's Google Sheets files from their Google Drive + const drive = google.drive({ version: "v3", auth: oauth2Client }); + const response = await drive.files.list({ + q: "mimeType='application/vnd.google-apps.spreadsheet'", + fields: "files(id, name)", + }); + + const files = response.data.files || []; + if (files.length === 0) { + return res.status(404).json({ message: "No spreadsheets found." }); + } + + res.json(files); + } catch (error: any) { + console.log("Error fetching Google Sheets files:", error); + res + .status(500) + .json({ + message: `Error retrieving Google Sheets files: ${error.message}`, + }); + } }); // Step 5: Update robot's google_sheet_id when a Google Sheet is selected -router.post('/gsheets/update', requireSignIn, async (req, res) => { - const { spreadsheetId, spreadsheetName, robotId } = req.body; +router.post("/gsheets/update", requireSignIn, async (req, res) => { + const { spreadsheetId, spreadsheetName, robotId } = req.body; - if (!spreadsheetId || !robotId) { - return res.status(400).json({ message: 'Spreadsheet ID and Robot ID are required' }); + if (!spreadsheetId || !robotId) { + return res + .status(400) + .json({ message: "Spreadsheet ID and Robot ID are required" }); + } + + try { + let robot = await Robot.findOne({ + where: { "recording_meta.id": robotId }, + }); + + if (!robot) { + return res.status(404).json({ message: "Robot not found" }); } - try { - let robot = await Robot.findOne({ where: { 'recording_meta.id': robotId } }); + await robot.update({ + google_sheet_id: spreadsheetId, + google_sheet_name: spreadsheetName, + }); - if (!robot) { - return res.status(404).json({ message: 'Robot not found' }); - } - - await robot.update({ google_sheet_id: spreadsheetId, google_sheet_name: spreadsheetName }); - - res.json({ message: 'Robot updated with selected Google Sheet ID' }); - } catch (error: any) { - res.status(500).json({ message: `Error updating robot: ${error.message}` }); - } + res.json({ message: "Robot updated with selected Google Sheet ID" }); + } catch (error: any) { + res.status(500).json({ message: `Error updating robot: ${error.message}` }); + } }); -router.post('/gsheets/remove', requireSignIn, async (req: AuthenticatedRequest, res) => { +router.post( + "/gsheets/remove", + requireSignIn, + async (req: AuthenticatedRequest, res) => { const { robotId } = req.body; if (!robotId) { - return res.status(400).json({ message: 'Robot ID is required' }); + return res.status(400).json({ message: "Robot ID is required" }); } if (!req.user) { - return res.status(401).send({ error: 'Unauthorized' }); + return res.status(401).send({ error: "Unauthorized" }); } try { - let robot = await Robot.findOne({ where: { 'recording_meta.id': robotId } }); + let robot = await Robot.findOne({ + where: { "recording_meta.id": robotId }, + }); - if (!robot) { - return res.status(404).json({ message: 'Robot not found' }); - } + if (!robot) { + return res.status(404).json({ message: "Robot not found" }); + } - await robot.update({ - google_sheet_id: null, - google_sheet_name: null, - google_sheet_email: null, - google_access_token: null, - google_refresh_token: null - }); + await robot.update({ + google_sheet_id: null, + google_sheet_name: null, + google_sheet_email: null, + google_access_token: null, + google_refresh_token: null, + }); - capture( - 'maxun-oss-google-sheet-integration-removed', - { - user_id: req.user.id, - robot_id: robotId, - deleted_at: new Date().toISOString() - } - ) + capture("maxun-oss-google-sheet-integration-removed", { + user_id: req.user.id, + robot_id: robotId, + deleted_at: new Date().toISOString(), + }); - res.json({ message: 'Google Sheets integration removed successfully' }); + res.json({ message: "Google Sheets integration removed successfully" }); } catch (error: any) { - res.status(500).json({ message: `Error removing Google Sheets integration: ${error.message}` }); + res + .status(500) + .json({ + message: `Error removing Google Sheets integration: ${error.message}`, + }); } -}); + } +); diff --git a/server/src/storage/mino.ts b/server/src/storage/mino.ts index 04efaf8c..0f29c3cd 100644 --- a/server/src/storage/mino.ts +++ b/server/src/storage/mino.ts @@ -12,47 +12,48 @@ const minioClient = new Client({ minioClient.bucketExists('maxun-test') .then((exists) => { if (exists) { - console.log('MinIO was connected successfully.'); + console.log('MinIO connected successfully.'); } else { - console.log('Bucket does not exist, but MinIO was connected.'); + console.log('MinIO connected successfully.'); } }) .catch((err) => { console.error('Error connecting to MinIO:', err); }) -async function createBucketWithPolicy(bucketName: string, policy?: 'public-read' | 'private') { +async function createBucketWithPolicy(bucketName: string, policy = 'public-read') { try { const bucketExists = await minioClient.bucketExists(bucketName); if (!bucketExists) { await minioClient.makeBucket(bucketName); console.log(`Bucket ${bucketName} created successfully.`); - - if (policy === 'public-read') { - // Define a public-read policy - const policyJSON = { - Version: "2012-10-17", - Statement: [ - { - Effect: "Allow", - Principal: "", - Action: ["s3:GetObject"], - Resource: [`arn:aws:s3:::${bucketName}/*`] - } - ] - }; - await minioClient.setBucketPolicy(bucketName, JSON.stringify(policyJSON)); - console.log(`Public-read policy applied to bucket ${bucketName}.`); - } } else { console.log(`Bucket ${bucketName} already exists.`); } + + if (policy === 'public-read') { + // Apply public-read policy after confirming the bucket exists + const policyJSON = { + Version: "2012-10-17", + Statement: [ + { + Effect: "Allow", + Principal: "*", + Action: ["s3:GetObject"], + Resource: [`arn:aws:s3:::${bucketName}/*`] + } + ] + }; + await minioClient.setBucketPolicy(bucketName, JSON.stringify(policyJSON)); + console.log(`Public-read policy applied to bucket ${bucketName}.`); + } } catch (error) { console.error('Error in bucket creation or policy application:', error); } } + class BinaryOutputService { private bucketName: string; diff --git a/server/src/utils/auth.ts b/server/src/utils/auth.ts index f8313df7..ddab4bd4 100644 --- a/server/src/utils/auth.ts +++ b/server/src/utils/auth.ts @@ -6,29 +6,37 @@ export const hashPassword = (password: string): Promise => { return new Promise((resolve, reject) => { bcrypt.genSalt(12, (err, salt) => { if (err) { - reject(err) + reject(err); } bcrypt.hash(password, salt, (err, hash) => { if (err) { - reject(err) + reject(err); } - resolve(hash) - }) - }) - }) -} + resolve(hash); + }); + }); + }); +}; // password from frontend and hash from database export const comparePassword = (password: string, hash: string): Promise => { - return bcrypt.compare(password, hash) -} + return bcrypt.compare(password, hash); +}; export const encrypt = (text: string): string => { const ivLength = 16; const iv = crypto.randomBytes(ivLength); const algorithm = 'aes-256-cbc'; - const key = Buffer.from(getEnvVariable('ENCRYPTION_KEY'), 'hex'); - const cipher = crypto.createCipheriv(algorithm, key, iv); + + // Retrieve the encryption key or generate a new one if invalid or empty + let key = getEnvVariable('ENCRYPTION_KEY'); + if (!key || key.length !== 64) { // aes-256-cbc requires a 256-bit key, which is 64 hex characters + console.warn('Invalid or missing ENCRYPTION_KEY, generating a new one.'); + key = crypto.randomBytes(32).toString('hex'); // Generate a new 256-bit (32-byte) key + } + const keyBuffer = Buffer.from(key, 'hex'); + + const cipher = crypto.createCipheriv(algorithm, keyBuffer, iv); let encrypted = cipher.update(text, 'utf8', 'hex'); encrypted += cipher.final('hex'); return `${iv.toString('hex')}:${encrypted}`; @@ -37,9 +45,17 @@ export const encrypt = (text: string): string => { export const decrypt = (encryptedText: string): string => { const [iv, encrypted] = encryptedText.split(':'); const algorithm = "aes-256-cbc"; - const key = Buffer.from(getEnvVariable('ENCRYPTION_KEY'), 'hex'); - const decipher = crypto.createDecipheriv(algorithm, key, Buffer.from(iv, 'hex')); + + // Retrieve the encryption key or generate a new one if invalid or empty + let key = getEnvVariable('ENCRYPTION_KEY'); + if (!key || key.length !== 64) { // aes-256-cbc requires a 256-bit key, which is 64 hex characters + console.warn('Invalid or missing ENCRYPTION_KEY, generating a new one.'); + key = crypto.randomBytes(32).toString('hex'); // Generate a new 256-bit (32-byte) key + } + const keyBuffer = Buffer.from(key, 'hex'); + + const decipher = crypto.createDecipheriv(algorithm, keyBuffer, Buffer.from(iv, 'hex')); let decrypted = decipher.update(encrypted, 'hex', 'utf8'); decrypted += decipher.final('utf8'); return decrypted; -}; \ No newline at end of file +}; diff --git a/src/api/storage.ts b/src/api/storage.ts index e48c7091..4b2f4e80 100644 --- a/src/api/storage.ts +++ b/src/api/storage.ts @@ -24,6 +24,36 @@ export const getStoredRecordings = async (): Promise => { } }; +export const updateRecording = async (id: string, data: { name?: string; limit?: number }): Promise => { + try { + const response = await axios.put(`${apiUrl}/storage/recordings/${id}`, data); + if (response.status === 200) { + return true; + } else { + throw new Error(`Couldn't update recording with id ${id}`); + } + } catch (error: any) { + console.error(`Error updating recording: ${error.message}`); + return false; + } +}; + +export const duplicateRecording = async (id: string, targetUrl: string): Promise => { + try { + const response = await axios.post(`${apiUrl}/storage/recordings/${id}/duplicate`, { + targetUrl, + }); + if (response.status === 201) { + return response.data; // Returns the duplicated robot details + } else { + throw new Error(`Couldn't duplicate recording with id ${id}`); + } + } catch (error: any) { + console.error(`Error duplicating recording: ${error.message}`); + return null; + } +}; + export const getStoredRuns = async (): Promise => { try { const response = await axios.get(`${apiUrl}/storage/runs`); diff --git a/src/components/molecules/IntegrationSettings.tsx b/src/components/molecules/IntegrationSettings.tsx index 1150dd72..005a0e68 100644 --- a/src/components/molecules/IntegrationSettings.tsx +++ b/src/components/molecules/IntegrationSettings.tsx @@ -1,227 +1,322 @@ -import React, { useState, useEffect } from 'react'; +import React, { useState, useEffect } from "react"; import { GenericModal } from "../atoms/GenericModal"; -import { MenuItem, Typography, CircularProgress, Alert, AlertTitle, Chip } from "@mui/material"; +import { + MenuItem, + Typography, + CircularProgress, + Alert, + AlertTitle, + Chip, +} from "@mui/material"; import Button from "@mui/material/Button"; import TextField from "@mui/material/TextField"; -import axios from 'axios'; -import { useGlobalInfoStore } from '../../context/globalInfo'; -import { getStoredRecording } from '../../api/storage'; -import { apiUrl } from '../../apiConfig.js'; +import axios from "axios"; +import { useGlobalInfoStore } from "../../context/globalInfo"; +import { getStoredRecording } from "../../api/storage"; +import { apiUrl } from "../../apiConfig.js"; +import Cookies from 'js-cookie'; interface IntegrationProps { - isOpen: boolean; - handleStart: (data: IntegrationSettings) => void; - handleClose: () => void; + isOpen: boolean; + handleStart: (data: IntegrationSettings) => void; + handleClose: () => void; } export interface IntegrationSettings { - spreadsheetId: string; - spreadsheetName: string; - data: string; + spreadsheetId: string; + spreadsheetName: string; + data: string; } -export const IntegrationSettingsModal = ({ isOpen, handleStart, handleClose }: IntegrationProps) => { - const [settings, setSettings] = useState({ - spreadsheetId: '', - spreadsheetName: '', - data: '', - }); +export const IntegrationSettingsModal = ({ + isOpen, + handleStart, + handleClose, +}: IntegrationProps) => { + const [settings, setSettings] = useState({ + spreadsheetId: "", + spreadsheetName: "", + data: "", + }); - const [spreadsheets, setSpreadsheets] = useState<{ id: string, name: string }[]>([]); - const [loading, setLoading] = useState(false); - const [error, setError] = useState(null); + const [spreadsheets, setSpreadsheets] = useState< + { id: string; name: string }[] + >([]); + const [loading, setLoading] = useState(false); + const [error, setError] = useState(null); - const { recordingId, notify } = useGlobalInfoStore(); - const [recording, setRecording] = useState(null); + const { recordingId, notify } = useGlobalInfoStore(); + const [recording, setRecording] = useState(null); - const authenticateWithGoogle = () => { - window.location.href = `${apiUrl}/auth/google?robotId=${recordingId}`; - }; + const authenticateWithGoogle = () => { + window.location.href = `${apiUrl}/auth/google?robotId=${recordingId}`; + }; - const handleOAuthCallback = async () => { - try { - const response = await axios.get(`${apiUrl}/auth/google/callback`); - const { google_sheet_email, files } = response.data; - } catch (error) { - setError('Error authenticating with Google'); + const handleOAuthCallback = async () => { + try { + const response = await axios.get(`${apiUrl}/auth/google/callback`); + const { google_sheet_email, files } = response.data; + } catch (error) { + setError("Error authenticating with Google"); + } + }; + + const fetchSpreadsheetFiles = async () => { + try { + const response = await axios.get( + `${apiUrl}/auth/gsheets/files?robotId=${recordingId}`, + { + withCredentials: true, } - }; + ); + setSpreadsheets(response.data); + } catch (error: any) { + console.error( + "Error fetching spreadsheet files:", + error.response?.data?.message || error.message + ); + notify( + "error", + `Error fetching spreadsheet files: ${ + error.response?.data?.message || error.message + }` + ); + } + }; - const fetchSpreadsheetFiles = async () => { - try { - const response = await axios.get(`${apiUrl}/auth/gsheets/files?robotId=${recordingId}`, { - withCredentials: true, - }); - setSpreadsheets(response.data); - } catch (error: any) { - console.error('Error fetching spreadsheet files:', error.response?.data?.message || error.message); - notify('error', `Error fetching spreadsheet files: ${error.response?.data?.message || error.message}`); - } - }; - - const handleSpreadsheetSelect = (e: React.ChangeEvent) => { - const selectedSheet = spreadsheets.find(sheet => sheet.id === e.target.value); - if (selectedSheet) { - setSettings({ ...settings, spreadsheetId: selectedSheet.id, spreadsheetName: selectedSheet.name }); - } - }; - - const updateGoogleSheetId = async () => { - try { - const response = await axios.post( - `${apiUrl}/auth/gsheets/update`, - { spreadsheetId: settings.spreadsheetId, spreadsheetName: settings.spreadsheetName, robotId: recordingId }, - { withCredentials: true } - ); - console.log('Google Sheet ID updated:', response.data); - } catch (error: any) { - console.error('Error updating Google Sheet ID:', error.response?.data?.message || error.message); - } - }; - - const removeIntegration = async () => { - try { - await axios.post( - `${apiUrl}/auth/gsheets/remove`, - { robotId: recordingId }, - { withCredentials: true } - ); - - setRecording(null); - setSpreadsheets([]); - setSettings({ spreadsheetId: '', spreadsheetName: '', data: '' }); - } catch (error: any) { - console.error('Error removing Google Sheets integration:', error.response?.data?.message || error.message); - } - }; - - useEffect(() => { - // Check if we're on the callback URL - const urlParams = new URLSearchParams(window.location.search); - const code = urlParams.get('code'); - if (code) { - handleOAuthCallback(); - } - - const fetchRecordingInfo = async () => { - if (!recordingId) return; - const recording = await getStoredRecording(recordingId); - if (recording) { - setRecording(recording); - } - }; - - fetchRecordingInfo(); - }, [recordingId]); - - return ( - -
- Integrate with Google Sheet - - {recording && recording.google_sheet_id ? ( - <> - - Google Sheet Integrated Successfully. - Every time this robot creates a successful run, its captured data is appended to your {recording.google_sheet_name} Google Sheet. You can check the data updates here. -
- Note: The data extracted before integrating with Google Sheets will not be synced in the Google Sheet. Only the data extracted after the integration will be synced. - - - - ) : ( - <> - {!recording?.google_sheet_email ? ( - <> -

If you enable this option, every time this robot runs a task successfully, its captured data will be appended to your Google Sheet.

- - - ) : ( - <> - {recording.google_sheet_email && ( - - Authenticated as: {recording.google_sheet_email} - - )} - - {loading ? ( - - ) : error ? ( - {error} - ) : spreadsheets.length === 0 ? ( - <> -
- - -
- - ) : ( - <> - - {spreadsheets.map(sheet => ( - - {sheet.name} - - ))} - - - {settings.spreadsheetId && ( - - Selected Sheet: {spreadsheets.find(s => s.id === settings.spreadsheetId)?.name} (ID: {settings.spreadsheetId}) - - )} - - - - )} - - )} - - )} -
- + const handleSpreadsheetSelect = (e: React.ChangeEvent) => { + const selectedSheet = spreadsheets.find( + (sheet) => sheet.id === e.target.value ); + if (selectedSheet) { + setSettings({ + ...settings, + spreadsheetId: selectedSheet.id, + spreadsheetName: selectedSheet.name, + }); + } + }; + + const updateGoogleSheetId = async () => { + try { + const response = await axios.post( + `${apiUrl}/auth/gsheets/update`, + { + spreadsheetId: settings.spreadsheetId, + spreadsheetName: settings.spreadsheetName, + robotId: recordingId, + }, + { withCredentials: true } + ); + console.log("Google Sheet ID updated:", response.data); + } catch (error: any) { + console.error( + "Error updating Google Sheet ID:", + error.response?.data?.message || error.message + ); + } + }; + + const removeIntegration = async () => { + try { + await axios.post( + `${apiUrl}/auth/gsheets/remove`, + { robotId: recordingId }, + { withCredentials: true } + ); + + setRecording(null); + setSpreadsheets([]); + setSettings({ spreadsheetId: "", spreadsheetName: "", data: "" }); + } catch (error: any) { + console.error( + "Error removing Google Sheets integration:", + error.response?.data?.message || error.message + ); + } + }; + + useEffect(() => { + // Check if there is a success message in cookies + const status = Cookies.get("robot_auth_status"); + const message = Cookies.get("robot_auth_message"); + + if (status === "success" && message) { + notify("success", message); + // Clear the cookies after reading + Cookies.remove("robot_auth_status"); + Cookies.remove("robot_auth_message"); + } + + // Check if we're on the callback URL + const urlParams = new URLSearchParams(window.location.search); + const code = urlParams.get("code"); + if (code) { + handleOAuthCallback(); + } + + const fetchRecordingInfo = async () => { + if (!recordingId) return; + const recording = await getStoredRecording(recordingId); + if (recording) { + setRecording(recording); + } + }; + + fetchRecordingInfo(); + }, [recordingId]); + + return ( + +
+ + Integrate with Google Sheet{" "} + + + + {recording && recording.google_sheet_id ? ( + <> + + Google Sheet Integrated Successfully. + Every time this robot creates a successful run, its captured data + is appended to your {recording.google_sheet_name} Google Sheet. + You can check the data updates{" "} + + here + + . +
+ Note: The data extracted before integrating with + Google Sheets will not be synced in the Google Sheet. Only the + data extracted after the integration will be synced. + + + + ) : ( + <> + {!recording?.google_sheet_email ? ( + <> +

+ If you enable this option, every time this robot runs a task + successfully, its captured data will be appended to your + Google Sheet. +

+ + + ) : ( + <> + {recording.google_sheet_email && ( + + Authenticated as: {recording.google_sheet_email} + + )} + + {loading ? ( + + ) : error ? ( + {error} + ) : spreadsheets.length === 0 ? ( + <> +
+ + +
+ + ) : ( + <> + + {spreadsheets.map((sheet) => ( + + {sheet.name} + + ))} + + + {settings.spreadsheetId && ( + + Selected Sheet:{" "} + { + spreadsheets.find( + (s) => s.id === settings.spreadsheetId + )?.name + }{" "} + (ID: {settings.spreadsheetId}) + + )} + + + + )} + + )} + + )} +
+ + ); +}; + +export const modalStyle = { + top: "40%", + left: "50%", + transform: "translate(-50%, -50%)", + width: "50%", + backgroundColor: "background.paper", + p: 4, + height: "fit-content", + display: "block", + padding: "20px", }; diff --git a/src/components/molecules/NavBar.tsx b/src/components/molecules/NavBar.tsx index 69fba352..30feef51 100644 --- a/src/components/molecules/NavBar.tsx +++ b/src/components/molecules/NavBar.tsx @@ -65,21 +65,21 @@ export const NavBar: React.FC = ({ recordingName, isRecording }) => {!isRecording ? ( <> - + - + handleEditRobot(row.id, row.name, row.params || [])} handleDelete={() => { + checkRunsForRecording(row.id).then((result: boolean) => { if (result) { notify('warning', 'Cannot delete recording as it has active runs'); } }) + deleteRecordingFromStorage(row.id).then((result: boolean) => { if (result) { setRows([]); diff --git a/src/components/molecules/RobotDuplicate.tsx b/src/components/molecules/RobotDuplicate.tsx new file mode 100644 index 00000000..3fe6abb7 --- /dev/null +++ b/src/components/molecules/RobotDuplicate.tsx @@ -0,0 +1,164 @@ +import React, { useState, useEffect } from 'react'; +import { GenericModal } from "../atoms/GenericModal"; +import { TextField, Typography, Box, Button, Chip } from "@mui/material"; +import { modalStyle } from "./AddWhereCondModal"; +import { useGlobalInfoStore } from '../../context/globalInfo'; +import { duplicateRecording, getStoredRecording } from '../../api/storage'; +import { WhereWhatPair } from 'maxun-core'; +import { getUserById } from "../../api/auth"; + +interface RobotMeta { + name: string; + id: string; + createdAt: string; + pairs: number; + updatedAt: string; + params: any[]; +} + +interface RobotWorkflow { + workflow: WhereWhatPair[]; +} + +interface ScheduleConfig { + runEvery: number; + runEveryUnit: 'MINUTES' | 'HOURS' | 'DAYS' | 'WEEKS' | 'MONTHS'; + startFrom: 'SUNDAY' | 'MONDAY' | 'TUESDAY' | 'WEDNESDAY' | 'THURSDAY' | 'FRIDAY' | 'SATURDAY'; + atTimeStart?: string; + atTimeEnd?: string; + timezone: string; + lastRunAt?: Date; + nextRunAt?: Date; + cronExpression?: string; +} + +export interface RobotSettings { + id: string; + userId?: number; + recording_meta: RobotMeta; + recording: RobotWorkflow; + google_sheet_email?: string | null; + google_sheet_name?: string | null; + google_sheet_id?: string | null; + google_access_token?: string | null; + google_refresh_token?: string | null; + schedule?: ScheduleConfig | null; +} + +interface RobotSettingsProps { + isOpen: boolean; + handleStart: (settings: RobotSettings) => void; + handleClose: () => void; + initialSettings?: RobotSettings | null; + +} + +export const RobotDuplicationModal = ({ isOpen, handleStart, handleClose, initialSettings }: RobotSettingsProps) => { + const [robot, setRobot] = useState(null); + const [targetUrl, setTargetUrl] = useState(''); + const { recordingId, notify } = useGlobalInfoStore(); + + useEffect(() => { + if (isOpen) { + getRobot(); + } + }, [isOpen]); + + useEffect(() => { + // Update the targetUrl when the robot data is loaded + if (robot) { + const lastPair = robot?.recording.workflow[robot?.recording.workflow.length - 1]; + const url = lastPair?.what.find(action => action.action === "goto")?.args?.[0]; + setTargetUrl(url); + } + }, [robot]); + + const getRobot = async () => { + if (recordingId) { + const robot = await getStoredRecording(recordingId); + setRobot(robot); + } else { + notify('error', 'Could not find robot details. Please try again.'); + } + } + + // const lastPair = robot?.recording.workflow[robot?.recording.workflow.length - 1]; + + // // Find the `goto` action in `what` and retrieve its arguments + // const targetUrl = lastPair?.what.find(action => action.action === "goto")?.args?.[0]; + + const handleTargetUrlChange = (e: React.ChangeEvent) => { + setTargetUrl(e.target.value); + }; + + const handleSave = async () => { + if (!robot || !targetUrl) { + notify('error', 'Target URL is required.'); + return; + } + + console.log("handle save"); + + try { + const success = await duplicateRecording(robot.recording_meta.id, targetUrl); + + if (success) { + notify('success', 'Target URL updated successfully.'); + handleStart(robot); // Inform parent about the updated robot + handleClose(); // Close the modal + + window.location.reload(); + } else { + notify('error', 'Failed to update the Target URL. Please try again.'); + } + } catch (error) { + notify('error', 'An error occurred while updating the Target URL.'); + console.error('Error updating Target URL:', error); + } + }; + + return ( + + <> + Duplicate Robot + + { + robot && ( + <> + Robot duplication is useful to extract data from pages with the same structure. +
+ + Example: If you've created a robot for producthunt.com/topics/api, you can duplicate it to scrape similar pages + like producthunt.com/topics/database without training a robot from scratch. + +
+ + ⚠️ Ensure the new page has the same structure as the original page. + + + + + + + + ) + } + + + + ); +}; diff --git a/src/components/molecules/RobotEdit.tsx b/src/components/molecules/RobotEdit.tsx new file mode 100644 index 00000000..26317260 --- /dev/null +++ b/src/components/molecules/RobotEdit.tsx @@ -0,0 +1,179 @@ +import React, { useState, useEffect } from 'react'; +import { GenericModal } from "../atoms/GenericModal"; +import { TextField, Typography, Box, Button } from "@mui/material"; +import { modalStyle } from "./AddWhereCondModal"; +import { useGlobalInfoStore } from '../../context/globalInfo'; +import { getStoredRecording, updateRecording } from '../../api/storage'; +import { WhereWhatPair } from 'maxun-core'; + +interface RobotMeta { + name: string; + id: string; + createdAt: string; + pairs: number; + updatedAt: string; + params: any[]; +} + +interface RobotWorkflow { + workflow: WhereWhatPair[]; +} + +interface RobotEditOptions { + name: string; + limit?: number; +} + +interface ScheduleConfig { + runEvery: number; + runEveryUnit: 'MINUTES' | 'HOURS' | 'DAYS' | 'WEEKS' | 'MONTHS'; + startFrom: 'SUNDAY' | 'MONDAY' | 'TUESDAY' | 'WEDNESDAY' | 'THURSDAY' | 'FRIDAY' | 'SATURDAY'; + atTimeStart?: string; + atTimeEnd?: string; + timezone: string; + lastRunAt?: Date; + nextRunAt?: Date; + cronExpression?: string; +} + +export interface RobotSettings { + id: string; + userId?: number; + recording_meta: RobotMeta; + recording: RobotWorkflow; + google_sheet_email?: string | null; + google_sheet_name?: string | null; + google_sheet_id?: string | null; + google_access_token?: string | null; + google_refresh_token?: string | null; + schedule?: ScheduleConfig | null; +} + +interface RobotSettingsProps { + isOpen: boolean; + handleStart: (settings: RobotSettings) => void; + handleClose: () => void; + initialSettings?: RobotSettings | null; + +} + +export const RobotEditModal = ({ isOpen, handleStart, handleClose, initialSettings }: RobotSettingsProps) => { + console.log("robot edit"); + const [robot, setRobot] = useState(null); + const { recordingId, notify } = useGlobalInfoStore(); + + useEffect(() => { + if (isOpen) { + getRobot(); + } + }, [isOpen]); + + const getRobot = async () => { + if (recordingId) { + const robot = await getStoredRecording(recordingId); + setRobot(robot); + } else { + notify('error', 'Could not find robot details. Please try again.'); + } + } + + const handleRobotNameChange = (newName: string) => { + setRobot((prev) => + prev ? { ...prev, recording_meta: { ...prev.recording_meta, name: newName } } : prev + ); + }; + + const handleLimitChange = (newLimit: number) => { + setRobot((prev) => { + if (!prev) return prev; + + const updatedWorkflow = [...prev.recording.workflow]; + + if ( + updatedWorkflow.length > 0 && + updatedWorkflow[0]?.what && + updatedWorkflow[0].what.length > 0 && + updatedWorkflow[0].what[0].args && + updatedWorkflow[0].what[0].args.length > 0 && + updatedWorkflow[0].what[0].args[0] + ) { + updatedWorkflow[0].what[0].args[0].limit = newLimit; + } + + return { ...prev, recording: { ...prev.recording, workflow: updatedWorkflow } }; + }); + }; + const handleSave = async () => { + if (!robot) return; + + try { + const payload = { + name: robot.recording_meta.name, + limit: robot.recording.workflow[0]?.what[0]?.args?.[0]?.limit, + }; + + const success = await updateRecording(robot.recording_meta.id, payload); + + if (success) { + notify('success', 'Robot updated successfully.'); + handleStart(robot); // Inform parent about the updated robot + handleClose(); // Close the modal + + window.location.reload(); + } else { + notify('error', 'Failed to update the robot. Please try again.'); + } + } catch (error) { + notify('error', 'An error occurred while updating the robot.'); + console.error('Error updating robot:', error); + } + }; + + return ( + + <> + Edit Robot + + { + robot && ( + <> + handleRobotNameChange(e.target.value)} + style={{ marginBottom: '20px' }} + /> + {robot.recording.workflow?.[0]?.what?.[0]?.args?.[0]?.limit !== undefined && ( + + handleLimitChange(parseInt(e.target.value, 10) || 0) + } + style={{ marginBottom: '20px' }} + /> + )} + + + + + + + ) + } + + + + ); +}; diff --git a/src/components/molecules/RobotSettings.tsx b/src/components/molecules/RobotSettings.tsx index 98638120..ffb7a2d0 100644 --- a/src/components/molecules/RobotSettings.tsx +++ b/src/components/molecules/RobotSettings.tsx @@ -104,6 +104,7 @@ export const RobotSettingsModal = ({ isOpen, handleStart, handleClose, initialSe <> { setTab(tab); - }, [interpretationInProgress]) + }, [interpretationInProgress]); useEffect(() => { if (row.serializableOutput && Object.keys(row.serializableOutput).length > 0) { @@ -47,6 +44,29 @@ export const RunContent = ({ row, currentLog, interpretationInProgress, logEndRe } }, [row.serializableOutput]); + // Function to convert table data to CSV format + const convertToCSV = (data: any[], columns: string[]): string => { + const header = columns.join(','); + const rows = data.map(row => + columns.map(col => JSON.stringify(row[col], null, 2)).join(',') + ); + return [header, ...rows].join('\n'); + }; + + // Function to download CSV file when called + const downloadCSV = () => { + const csvContent = convertToCSV(tableData, columns); + const blob = new Blob([csvContent], { type: 'text/csv;charset=utf-8;' }); + const url = URL.createObjectURL(blob); + + const link = document.createElement("a"); + link.href = url; + link.setAttribute("download", "data.csv"); + document.body.appendChild(link); + link.click(); + document.body.removeChild(link); + }; + return ( @@ -54,7 +74,6 @@ export const RunContent = ({ row, currentLog, interpretationInProgress, logEndRe setTab(newTab)} aria-label="run-content-tabs"> - {/* */} @@ -94,16 +113,19 @@ export const RunContent = ({ row, currentLog, interpretationInProgress, logEndRe Captured Data - {Object.keys(row.serializableOutput).map((key) => { - return ( - - ) - })} + + + + Download as JSON + + + + Download as CSV + + {tableData.length > 0 ? ( @@ -139,12 +161,12 @@ export const RunContent = ({ row, currentLog, interpretationInProgress, logEndRe )} } - {row.binaryOutput - && Object.keys(row.binaryOutput).length !== 0 && + {row.binaryOutput && Object.keys(row.binaryOutput).length !== 0 &&
- Captured Screenshot + Captured Screenshot + {Object.keys(row.binaryOutput).map((key) => { try { const imageUrl = row.binaryOutput[key]; @@ -152,10 +174,10 @@ export const RunContent = ({ row, currentLog, interpretationInProgress, logEndRe - + Download Screenshot - {key} + {key} ) } catch (e) { @@ -171,4 +193,4 @@ export const RunContent = ({ row, currentLog, interpretationInProgress, logEndRe ); -} +}; diff --git a/src/components/molecules/RunsTable.tsx b/src/components/molecules/RunsTable.tsx index b1cb97ef..dcb0cdd1 100644 --- a/src/components/molecules/RunsTable.tsx +++ b/src/components/molecules/RunsTable.tsx @@ -45,6 +45,8 @@ export interface Data { // task: string; log: string; runId: string; + robotId: string; + robotMetaId: string; interpreterSettings: RunSettings; serializableOutput: any; binaryOutput: any; @@ -63,8 +65,6 @@ export const RunsTable = ( const [rowsPerPage, setRowsPerPage] = useState(10); const [rows, setRows] = useState([]); - console.log(`rows runs: ${JSON.stringify(rows)}`); - const { notify, rerenderRuns, setRerenderRuns } = useGlobalInfoStore(); const handleChangePage = (event: unknown, newPage: number) => { @@ -105,12 +105,12 @@ export const RunsTable = ( fetchRuns(); }; - // Group runs by recording name + // Group runs by robot meta id const groupedRows = rows.reduce((acc, row) => { - if (!acc[row.name]) { - acc[row.name] = []; + if (!acc[row.robotMetaId]) { + acc[row.robotMetaId] = []; } - acc[row.name].push(row); + acc[row.robotMetaId].push(row); return acc; }, {} as Record); @@ -120,10 +120,10 @@ export const RunsTable = ( All Runs - {Object.entries(groupedRows).map(([name, group]) => ( - + {Object.entries(groupedRows).map(([id, data]) => ( + }> - {name} + {data[data.length - 1].name}
@@ -142,17 +142,19 @@ export const RunsTable = ( - {group.slice(page * rowsPerPage, page * rowsPerPage + rowsPerPage).map((row) => ( - - ))} + {data + .slice(page * rowsPerPage, page * rowsPerPage + rowsPerPage) + .map((row) => ( + + ))}
diff --git a/src/components/organisms/BrowserContent.tsx b/src/components/organisms/BrowserContent.tsx index 79fe6e63..11af4f2f 100644 --- a/src/components/organisms/BrowserContent.tsx +++ b/src/components/organisms/BrowserContent.tsx @@ -1,12 +1,16 @@ -import React, { useCallback, useEffect, useState } from 'react'; +import React, { useCallback, useEffect, useState } from "react"; import styled from "styled-components"; import BrowserNavBar from "../molecules/BrowserNavBar"; import { BrowserWindow } from "./BrowserWindow"; import { useBrowserDimensionsStore } from "../../context/browserDimensions"; import { BrowserTabs } from "../molecules/BrowserTabs"; import { useSocketStore } from "../../context/socket"; -import { getCurrentTabs, getCurrentUrl, interpretCurrentRecording } from "../../api/recording"; -import { Box } from '@mui/material'; +import { + getCurrentTabs, + getCurrentUrl, + interpretCurrentRecording, +} from "../../api/recording"; +import { Box } from "@mui/material"; import { InterpretationLog } from "../molecules/InterpretationLog"; // TODO: Tab !show currentUrl after recordingUrl global state @@ -14,107 +18,125 @@ export const BrowserContent = () => { const { width } = useBrowserDimensionsStore(); const { socket } = useSocketStore(); - const [tabs, setTabs] = useState(['current']); + const [tabs, setTabs] = useState(["current"]); const [tabIndex, setTabIndex] = React.useState(0); const [showOutputData, setShowOutputData] = useState(false); - const handleChangeIndex = useCallback((index: number) => { - setTabIndex(index); - }, [tabIndex]) + const handleChangeIndex = useCallback( + (index: number) => { + setTabIndex(index); + }, + [tabIndex] + ); - const handleCloseTab = useCallback((index: number) => { - // the tab needs to be closed on the backend - socket?.emit('closeTab', { - index, - isCurrent: tabIndex === index, - }); - // change the current index as current tab gets closed - if (tabIndex === index) { - if (tabs.length > index + 1) { - handleChangeIndex(index); + const handleCloseTab = useCallback( + (index: number) => { + // the tab needs to be closed on the backend + socket?.emit("closeTab", { + index, + isCurrent: tabIndex === index, + }); + // change the current index as current tab gets closed + if (tabIndex === index) { + if (tabs.length > index + 1) { + handleChangeIndex(index); + } else { + handleChangeIndex(index - 1); + } } else { - handleChangeIndex(index - 1); + handleChangeIndex(tabIndex - 1); } - } else { - handleChangeIndex(tabIndex - 1); - } - // update client tabs - setTabs((prevState) => [ - ...prevState.slice(0, index), - ...prevState.slice(index + 1) - ]) - }, [tabs, socket, tabIndex]); + // update client tabs + setTabs((prevState) => [ + ...prevState.slice(0, index), + ...prevState.slice(index + 1), + ]); + }, + [tabs, socket, tabIndex] + ); const handleAddNewTab = useCallback(() => { // Adds new tab by pressing the plus button - socket?.emit('addTab'); + socket?.emit("addTab"); // Adds a new tab to the end of the tabs array and shifts focus - setTabs((prevState) => [...prevState, 'new tab']); + setTabs((prevState) => [...prevState, "new tab"]); handleChangeIndex(tabs.length); }, [socket, tabs]); - const handleNewTab = useCallback((tab: string) => { - // Adds a new tab to the end of the tabs array and shifts focus - setTabs((prevState) => [...prevState, tab]); - // changes focus on the new tab - same happens in the remote browser - handleChangeIndex(tabs.length); - handleTabChange(tabs.length); - }, [tabs]); + const handleNewTab = useCallback( + (tab: string) => { + // Adds a new tab to the end of the tabs array and shifts focus + setTabs((prevState) => [...prevState, tab]); + // changes focus on the new tab - same happens in the remote browser + handleChangeIndex(tabs.length); + handleTabChange(tabs.length); + }, + [tabs] + ); - const handleTabChange = useCallback((index: number) => { - // page screencast and focus needs to be changed on backend - socket?.emit('changeTab', index); - }, [socket]); + const handleTabChange = useCallback( + (index: number) => { + // page screencast and focus needs to be changed on backend + socket?.emit("changeTab", index); + }, + [socket] + ); const handleUrlChanged = (url: string) => { const parsedUrl = new URL(url); if (parsedUrl.hostname) { - const host = parsedUrl.hostname.match(/\b(?!www\.)[a-zA-Z0-9]+/g)?.join('.') + const host = parsedUrl.hostname + .match(/\b(?!www\.)[a-zA-Z0-9]+/g) + ?.join("."); if (host && host !== tabs[tabIndex]) { setTabs((prevState) => [ ...prevState.slice(0, tabIndex), host, - ...prevState.slice(tabIndex + 1) - ]) + ...prevState.slice(tabIndex + 1), + ]); } } else { - if (tabs[tabIndex] !== 'new tab') { + if (tabs[tabIndex] !== "new tab") { setTabs((prevState) => [ ...prevState.slice(0, tabIndex), - 'new tab', - ...prevState.slice(tabIndex + 1) - ]) + "new tab", + ...prevState.slice(tabIndex + 1), + ]); } } - }; - const tabHasBeenClosedHandler = useCallback((index: number) => { - handleCloseTab(index); - }, [handleCloseTab]) + const tabHasBeenClosedHandler = useCallback( + (index: number) => { + handleCloseTab(index); + }, + [handleCloseTab] + ); useEffect(() => { if (socket) { - socket.on('newTab', handleNewTab); - socket.on('tabHasBeenClosed', tabHasBeenClosedHandler); + socket.on("newTab", handleNewTab); + socket.on("tabHasBeenClosed", tabHasBeenClosedHandler); } return () => { if (socket) { - socket.off('newTab', handleNewTab); - socket.off('tabHasBeenClosed', tabHasBeenClosedHandler); + socket.off("newTab", handleNewTab); + socket.off("tabHasBeenClosed", tabHasBeenClosedHandler); } - } - }, [socket, handleNewTab]) + }; + }, [socket, handleNewTab]); useEffect(() => { - getCurrentTabs().then((response) => { - if (response) { - setTabs(response); - } - }).catch((error) => { - console.log("Fetching current url failed"); - }) - }, []) + getCurrentTabs() + .then((response) => { + if (response) { + setTabs(response); + } + }) + .catch((error) => { + console.log("Fetching current url failed"); + }); + }, [handleUrlChanged]); return (
@@ -134,7 +156,6 @@ export const BrowserContent = () => {
); -} +}; -const BrowserContentWrapper = styled.div` -`; \ No newline at end of file +const BrowserContentWrapper = styled.div``; diff --git a/src/components/organisms/MainMenu.tsx b/src/components/organisms/MainMenu.tsx index edb6ed29..dadb6731 100644 --- a/src/components/organisms/MainMenu.tsx +++ b/src/components/organisms/MainMenu.tsx @@ -3,7 +3,7 @@ import Tabs from '@mui/material/Tabs'; import Tab from '@mui/material/Tab'; import Box from '@mui/material/Box'; import { Paper, Button } from "@mui/material"; -import { AutoAwesome, FormatListBulleted, VpnKey, Usb, Article, Link, CloudQueue } from "@mui/icons-material"; +import { AutoAwesome, FormatListBulleted, VpnKey, Usb, CloudQueue, Code } from "@mui/icons-material"; import { apiUrl } from "../../apiConfig"; interface MainMenuProps { @@ -87,8 +87,8 @@ export const MainMenu = ({ value = 'recordings', handleChangeContent }: MainMenu
-