Merge pull request #93 from getmaxun/develop

feat: local docker setup
This commit is contained in:
amhsirak
2024-11-03 04:09:47 +05:30
committed by GitHub
41 changed files with 417 additions and 220 deletions

View File

@@ -1,70 +1,22 @@
# --- Base Stage ---
FROM node:18 AS base
FROM node:18-alpine
WORKDIR /app
# Copy shared package.json and install dependencies
COPY package.json package-lock.json ./
COPY maxun-core/package.json ./maxun-core/package.json
RUN npm install
# --- Backend Build Stage ---
FROM base AS backend-build
WORKDIR /app
# Copy TypeScript configs
COPY tsconfig*.json ./
COPY server/tsconfig.json ./server/
# Copy ALL source code (both frontend and backend)
COPY src ./src
# Copy backend code and maxun-core
COPY server/src ./server/src
# Copy package files
COPY package*.json ./
COPY maxun-core ./maxun-core
# Install TypeScript globally and build
RUN npm install -g typescript
RUN npm run build:server
# Install dependencies
RUN npm install
# --- Frontend Build Stage ---
FROM base AS frontend-build
WORKDIR /app
# Copy frontend code and configs
# Copy frontend source code and config
COPY src ./src
COPY index.html ./index.html
COPY public ./public
COPY index.html ./
COPY vite.config.js ./
COPY tsconfig.json ./
# Build frontend
RUN npm run build
# Expose the frontend port
EXPOSE 5173
# --- Production Stage ---
FROM nginx:alpine AS production
# Install Node.js in the production image
RUN apk add --update nodejs npm
# Copy nginx configuration
COPY nginx.conf /etc/nginx/conf.d/default.conf
# Copy built frontend
COPY --from=frontend-build /app/build /usr/share/nginx/html
COPY --from=frontend-build /app/public/img /usr/share/nginx/html/img
# Copy built backend and its dependencies
WORKDIR /app
COPY --from=backend-build /app/package*.json ./
COPY --from=backend-build /app/server/dist ./server/dist
COPY --from=backend-build /app/maxun-core ./maxun-core
COPY --from=backend-build /app/node_modules ./node_modules
# Copy start script
COPY docker-entrypoint.sh /
RUN chmod +x /docker-entrypoint.sh
EXPOSE 80 8080
# Start both nginx and node server
ENTRYPOINT ["/docker-entrypoint.sh"]
# Start the frontend using the client script
CMD ["npm", "run", "client", "--", "--host"]

View File

@@ -25,13 +25,15 @@ Maxun lets you train a robot in 2 minutes and scrape the web on auto-pilot. Web
<img src="https://static.scarf.sh/a.png?x-pxid=c12a77cc-855e-4602-8a0f-614b2d0da56a" />
# Installation
# Local Setup
### Docker
⚠️ Work In Progress. Will be available by EOD.
```
docker-compose up -d --build
```
### Local Setup
### Without Docker
1. Ensure you have Node.js, PostgreSQL, MinIO and Redis installed on your system.
2. Run the commands below:
2. Run the commands below
```
git clone https://github.com/getmaxun/maxun

View File

@@ -1,30 +1,31 @@
version: '3.8'
services:
app:
build:
context: .
dockerfile: Dockerfile
target: production
env_file: .env
ports:
- "5173:80"
- "8080:8080"
depends_on:
- db
- minio
- redis
db:
postgres:
image: postgres:13
environment:
POSTGRES_DB: ${DB_NAME}
POSTGRES_USER: ${DB_USER}
POSTGRES_PASSWORD: ${DB_PASSWORD}
POSTGRES_DB: ${DB_NAME}
ports:
- "5432:5432"
volumes:
- postgres_data:/var/lib/postgresql/data
healthcheck:
test: ["CMD-SHELL", "pg_isready -U postgres"]
interval: 10s
timeout: 5s
retries: 5
redis:
image: redis:6
environment:
REDIS_HOST: ${REDIS_HOST}
REDIS_PORT: ${REDIS_PORT}
ports:
- "6379:6379"
volumes:
- redis_data:/data
minio:
image: minio/minio
@@ -37,15 +38,41 @@ services:
volumes:
- minio_data:/data
redis:
image: redis:6
environment:
- REDIS_HOST=redis
- REDIS_PORT=6379
backend:
build:
context: .
dockerfile: server/Dockerfile
ports:
- "6379:6379"
- "8080:8080"
env_file: .env
environment:
# to ensure Playwright works in Docker
PLAYWRIGHT_BROWSERS_PATH: /ms-playwright
PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 0
# DEBUG: pw:api
# PWDEBUG: 1 # Enables debugging
CHROMIUM_FLAGS: '--disable-gpu --no-sandbox --headless=new'
volumes:
- redis_data:/data
# - /tmp/.X11-unix:/tmp/.X11-unix
- /var/run/dbus:/var/run/dbus # Add this for D-Bus support
security_opt:
- seccomp=unconfined # This might help with browser sandbox issues
# Increase shared memory size for Chromium
shm_size: '2gb'
depends_on:
- postgres
- redis
- minio
frontend:
build:
context: .
dockerfile: Dockerfile
ports:
- "5173:5173"
env_file: .env
depends_on:
- backend
volumes:
postgres_data:

View File

@@ -1,7 +1,7 @@
#!/bin/sh
# Start backend server
cd /app && npm run start:server &
cd /app && npm run start:server -- --host 0.0.0.0 &
# Start nginx
nginx -g 'daemon off;'
nginx -g 'daemon off;'

View File

@@ -8,7 +8,7 @@
name="description"
content="Web site created using Vite"
/>
<link rel="icon" type="image/png" href="public/img/maxunlogo.png">
<link rel="icon" type="image/png" href="img/maxunlogo.png">
<title>Maxun | Open Source No Code Web Data Extraction Platform</title>
</head>
<body>

View File

@@ -1,17 +1,45 @@
server {
listen 80;
location / {
root /usr/share/nginx/html;
try_files $uri $uri/ /index.html;
}
location /api {
proxy_pass http://127.0.0.1:8080;
proxy_pass http://localhost:8080;
proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection 'upgrade';
proxy_set_header Host $host;
proxy_cache_bypass $http_upgrade;
# Add timeout configurations
proxy_connect_timeout 60s;
proxy_send_timeout 60s;
proxy_read_timeout 60s;
# Add error handling
proxy_intercept_errors on;
error_page 502 503 504 /50x.html;
}
location ~ ^/(record|workflow|storage|auth|integration|proxy|api-docs) {
proxy_pass http://localhost:8080;
proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection 'keep-alive'; # Ensure connections remain open
proxy_set_header Connection 'upgrade';
proxy_set_header Host $host;
proxy_cache_bypass $http_upgrade;
# Timeout configurations
proxy_connect_timeout 60s;
proxy_send_timeout 60s;
proxy_read_timeout 60s;
# Error handling for these routes
proxy_intercept_errors on;
error_page 502 503 504 /50x.html;
}
}

View File

@@ -73,11 +73,11 @@
},
"scripts": {
"start": "concurrently -k \"npm run server\" \"npm run client\"",
"server": "./node_modules/.bin/nodemon server/src/server.ts",
"server": "cross-env NODE_OPTIONS='--max-old-space-size=8000' nodemon server/src/server.ts",
"client": "vite",
"build": "vite build",
"build:server": "tsc -p server/tsconfig.json",
"start:server": "node server/dist/server/src/server.js",
"start:server": "cross-env NODE_OPTIONS='--max-old-space-size=8000' server/dist/server/src/server.js",
"preview": "vite preview",
"lint": "./node_modules/.bin/eslint ."
},
@@ -101,6 +101,7 @@
"@vitejs/plugin-react": "^4.3.3",
"ajv": "^8.8.2",
"concurrently": "^7.0.0",
"cross-env": "^7.0.3",
"nodemon": "^2.0.15",
"ts-node": "^10.4.0",
"vite": "^5.4.10"

83
server/Dockerfile Normal file
View File

@@ -0,0 +1,83 @@
FROM mcr.microsoft.com/playwright:v1.40.0-jammy
# Set working directory
WORKDIR /app
# Install node dependencies
COPY package*.json ./
COPY maxun-core ./maxun-core
COPY src ./src
COPY server ./server
COPY tsconfig.json ./
COPY server/tsconfig.json ./server/
COPY server/start.sh ./
# Install dependencies
RUN npm install
# Install Playwright browsers and dependencies
RUN npx playwright install --with-deps chromium
# Install xvfb for display support
#RUN apt-get update && apt-get install -y xvfb
# RUN apt-get update && apt-get install -y \
# libgbm-dev \
# libxkbcommon-x11-0 \
# libxcomposite1 \
# libxdamage1 \
# libxrandr2 \
# libxshmfence1 \
# libxtst6 \
# libnss3 \
# libatk1.0-0 \
# libatk-bridge2.0-0 \
# libdrm2 \
# libxcb1 \
# libxkbcommon0 \
# fonts-noto-color-emoji \
# fonts-unifont \
# libpango-1.0-0 \
# libcairo2 \
# libasound2 \
# libglib2.0-0 \
# libdbus-1-3 \
# && rm -rf /var/lib/apt/lists/*
# Create and set permissions for chrome directories
# Create the Chromium data directory with necessary permissions
RUN mkdir -p /tmp/chromium-data-dir && \
chmod -R 777 /tmp/chromium-data-dir
# Install dependencies
RUN apt-get update && apt-get install -y \
libgbm-dev \
libnss3 \
libatk1.0-0 \
libatk-bridge2.0-0 \
libdrm2 \
libxkbcommon0 \
libglib2.0-0 \
libdbus-1-3 \
libx11-xcb1 \
libxcb1 \
libxcomposite1 \
libxcursor1 \
libxdamage1 \
libxext6 \
libxi6 \
libxtst6 \
&& rm -rf /var/lib/apt/lists/* \
&& mkdir -p /tmp/.X11-unix && chmod 1777 /tmp/.X11-unix
# Add a dbus configuration to prevent connection errors
# RUN mkdir -p /var/run/dbus
# Make the script executable
RUN chmod +x ./start.sh
# Expose the backend port
EXPOSE 8080
# Start the backend using the start script
CMD ["./start.sh"]

View File

@@ -464,13 +464,7 @@ async function createWorkflowAndStoreMetadata(id: string, userId: string) {
};
}
const browserId = createRemoteBrowserForRun({
browser: chromium,
launchOptions: {
headless: true,
proxy: proxyOptions.server ? proxyOptions : undefined,
}
}, userId);
const browserId = createRemoteBrowserForRun(userId);
const runId = uuid();
@@ -656,7 +650,7 @@ export async function handleRunRecording(id: string, userId: string) {
throw new Error('browserId or runId or userId is undefined');
}
const socket = io(`http://localhost:8080/${browserId}`, {
const socket = io(`${process.env.BACKEND_URL}/${browserId}`, {
transports: ['websocket'],
rejectUnauthorized: false
});

View File

@@ -3,6 +3,7 @@ import {
Browser,
CDPSession,
BrowserContext,
chromium,
} from 'playwright';
import { Socket } from "socket.io";
import { PlaywrightBlocker } from '@cliqz/adblocker-playwright';
@@ -91,8 +92,39 @@ export class RemoteBrowser {
* @param options remote browser options to be used when launching the browser
* @returns {Promise<void>}
*/
public initialize = async (options: RemoteBrowserOptions, userId: string): Promise<void> => {
this.browser = <Browser>(await options.browser.launch(options.launchOptions));
public initialize = async (userId: string): Promise<void> => {
// const launchOptions = {
// headless: true,
// proxy: options.launchOptions?.proxy,
// chromiumSandbox: false,
// args: [
// '--no-sandbox',
// '--disable-setuid-sandbox',
// '--headless=new',
// '--disable-gpu',
// '--disable-dev-shm-usage',
// '--disable-software-rasterizer',
// '--in-process-gpu',
// '--disable-infobars',
// '--single-process',
// '--no-zygote',
// '--disable-notifications',
// '--disable-extensions',
// '--disable-background-timer-throttling',
// ...(options.launchOptions?.args || [])
// ],
// env: {
// ...process.env,
// CHROMIUM_FLAGS: '--disable-gpu --no-sandbox --headless=new'
// }
// };
// console.log('Launch options before:', options.launchOptions);
// this.browser = <Browser>(await options.browser.launch(launchOptions));
// console.log('Launch options after:', options.launchOptions)
this.browser = <Browser>(await chromium.launch({
headless: true,
}));
const proxyConfig = await getDecryptedProxyConfig(userId);
let proxyOptions: { server: string, username?: string, password?: string } = { server: '' };
if (proxyConfig.proxy_url) {
@@ -107,6 +139,16 @@ export class RemoteBrowser {
const contextOptions: any = {
viewport: { height: 400, width: 900 },
// recordVideo: { dir: 'videos/' }
// Force reduced motion to prevent animation issues
reducedMotion: 'reduce',
// Force JavaScript to be enabled
javaScriptEnabled: true,
// Set a reasonable timeout
timeout: 50000,
// Disable hardware acceleration
forcedColors: 'none',
isMobile: false,
hasTouch: false
};
if (proxyOptions.server) {
@@ -116,9 +158,17 @@ export class RemoteBrowser {
password: proxyOptions.password ? proxyOptions.password : undefined,
};
}
const browserUserAgent = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/110.0.5481.38 Safari/537.36";
contextOptions.userAgent = browserUserAgent;
this.context = await this.browser.newContext(contextOptions);
console.log(`Context from initialize: ${JSON.stringify(this.context)}`)
this.currentPage = await this.context.newPage();
console.log(`CPage from initialize: ${JSON.stringify(this.currentPage)}`)
// await this.currentPage.setExtraHTTPHeaders({
// 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3'
// });
const blocker = await PlaywrightBlocker.fromPrebuiltAdsAndTracking(fetch);
await blocker.enableBlockingInPage(this.currentPage);
this.client = await this.currentPage.context().newCDPSession(this.currentPage);
@@ -331,6 +381,9 @@ export class RemoteBrowser {
await this.stopScreencast();
const newPage = options ? await this.browser?.newPage(options)
: await this.browser?.newPage();
await newPage?.setExtraHTTPHeaders({
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3'
});
await this.currentPage?.close();
this.currentPage = newPage;

View File

@@ -20,7 +20,7 @@ import logger from "../logger";
* @returns string
* @category BrowserManagement-Controller
*/
export const initializeRemoteBrowserForRecording = (options: RemoteBrowserOptions, userId: string): string => {
export const initializeRemoteBrowserForRecording = (userId: string): string => {
const id = getActiveBrowserId() || uuid();
createSocketConnection(
io.of(id),
@@ -34,7 +34,7 @@ export const initializeRemoteBrowserForRecording = (options: RemoteBrowserOption
} else {
const browserSession = new RemoteBrowser(socket);
browserSession.interpreter.subscribeToPausing();
await browserSession.initialize(options, userId);
await browserSession.initialize(userId);
await browserSession.registerEditorEvents();
await browserSession.subscribeToScreencast();
browserPool.addRemoteBrowser(id, browserSession, true);
@@ -52,13 +52,13 @@ export const initializeRemoteBrowserForRecording = (options: RemoteBrowserOption
* @returns string
* @category BrowserManagement-Controller
*/
export const createRemoteBrowserForRun = (options: RemoteBrowserOptions, userId: string): string => {
export const createRemoteBrowserForRun = (userId: string): string => {
const id = uuid();
createSocketConnectionForRun(
io.of(id),
async (socket: Socket) => {
const browserSession = new RemoteBrowser(socket);
await browserSession.initialize(options, userId);
await browserSession.initialize(userId);
browserPool.addRemoteBrowser(id, browserSession, true);
socket.emit('ready-for-run');
});

View File

@@ -14,6 +14,8 @@ interface AuthenticatedRequest extends Request {
}
router.post('/register', async (req, res) => {
console.log('Received request at /auth/register');
console.log('Received body:', req.body);
try {
const { email, password } = req.body
@@ -25,7 +27,21 @@ router.post('/register', async (req, res) => {
const hashedPassword = await hashPassword(password)
const user = await User.create({ email, password: hashedPassword });
let user: any;
try {
user = await User.create({ email, password: hashedPassword });
} catch (
error: any
) {
console.log(`Could not create user - ${error}`)
return res.status(500).send(`Could not create user - ${error.message}`)
}
if (!process.env.JWT_SECRET) {
console.log('JWT_SECRET is not defined in the environment');
return res.status(500).send('Internal Server Error');
}
const token = jwt.sign({ id: user.id }, process.env.JWT_SECRET as string, { expiresIn: '12h' });
user.password = undefined as unknown as string
@@ -40,8 +56,10 @@ router.post('/register', async (req, res) => {
registeredAt: new Date().toISOString()
}
)
console.log(`User registered - ${user.email}`)
res.json(user)
} catch (error: any) {
console.log(`Could not register user - ${error}`)
res.status(500).send(`Could not register user - ${error.message}`)
}
})

View File

@@ -11,14 +11,14 @@ import {
stopRunningInterpretation,
getRemoteBrowserCurrentUrl, getRemoteBrowserCurrentTabs,
} from '../browser-management/controller'
import { chromium } from 'playwright-extra';
import { chromium } from 'playwright';
import stealthPlugin from 'puppeteer-extra-plugin-stealth';
import logger from "../logger";
import { getDecryptedProxyConfig } from './proxy';
import { requireSignIn } from '../middlewares/auth';
export const router = Router();
chromium.use(stealthPlugin());
// chromium.use(stealthPlugin());
export interface AuthenticatedRequest extends Request {
@@ -56,13 +56,8 @@ router.get('/start', requireSignIn, async (req: AuthenticatedRequest, res: Respo
};
}
const id = initializeRemoteBrowserForRecording({
browser: chromium,
launchOptions: {
headless: true,
proxy: proxyOptions.server ? proxyOptions : undefined,
}
}, req.user.id);
const id = initializeRemoteBrowserForRecording(req.user.id);
console.log('id start:', id);
return res.send(id);
});
@@ -74,10 +69,8 @@ router.post('/start', requireSignIn, (req: AuthenticatedRequest, res:Response) =
if (!req.user) {
return res.status(401).send('User not authenticated');
}
const id = initializeRemoteBrowserForRecording({
browser: chromium,
launchOptions: req.body,
}, req.user.id);
const id = initializeRemoteBrowserForRecording(req.user.id);
console.log('id start POST:', id);
return res.send(id);
});

View File

@@ -158,13 +158,7 @@ router.put('/runs/:id', requireSignIn, async (req: AuthenticatedRequest, res) =>
console.log(`Proxy config for run: ${JSON.stringify(proxyOptions)}`)
const id = createRemoteBrowserForRun({
browser: chromium,
launchOptions: {
headless: true,
proxy: proxyOptions.server ? proxyOptions : undefined,
}
}, req.user.id);
const id = createRemoteBrowserForRun(req.user.id);
const runId = uuid();

View File

@@ -39,8 +39,8 @@ export const io = new Server(server);
*/
export const browserPool = new BrowserPool();
app.use(bodyParser.json({ limit: '10mb' }))
app.use(bodyParser.urlencoded({ extended: true, limit: '10mb', parameterLimit: 9000 }));
// app.use(bodyParser.json({ limit: '10mb' }))
// app.use(bodyParser.urlencoded({ extended: true, limit: '10mb', parameterLimit: 9000 }));
// parse cookies - "cookie" is true in csrfProtection
app.use(cookieParser())
@@ -62,42 +62,60 @@ readdirSync(path.join(__dirname, 'api')).forEach((r) => {
}
});
// Check if we're running in production or development
const isProduction = process.env.NODE_ENV === 'production';
const workerPath = path.resolve(__dirname, isProduction ? './worker.js' : '/worker.ts');
const workerPath = path.resolve(__dirname, isProduction ? './worker.js' : './worker.ts');
// Fork the worker process
const workerProcess = fork(workerPath, [], {
execArgv: isProduction ? ['--inspect=8081'] : ['--inspect=5859'],
});
workerProcess.on('message', (message) => {
console.log(`Message from worker: ${message}`);
});
workerProcess.on('error', (error) => {
console.error(`Error in worker: ${error}`);
});
workerProcess.on('exit', (code) => {
console.log(`Worker exited with code: ${code}`);
});
let workerProcess: any;
if (!isProduction) {
workerProcess = fork(workerPath, [], {
execArgv: ['--inspect=5859'],
});
workerProcess.on('message', (message: any) => {
console.log(`Message from worker: ${message}`);
});
workerProcess.on('error', (error: any) => {
console.error(`Error in worker: ${error}`);
});
workerProcess.on('exit', (code: any) => {
console.log(`Worker exited with code: ${code}`);
});
}
app.get('/', function (req, res) {
capture(
'maxun-oss-server-run', {
event: 'server_started',
}
event: 'server_started',
}
);
return res.send('Maxun server started 🚀');
});
server.listen(SERVER_PORT, async () => {
await connectDB();
await syncDB();
logger.log('info', `Server listening on port ${SERVER_PORT}`);
// Add CORS headers
app.use((req, res, next) => {
res.header('Access-Control-Allow-Origin', '*');
res.header('Access-Control-Allow-Methods', 'GET,PUT,POST,DELETE,OPTIONS');
res.header('Access-Control-Allow-Headers', 'Content-Type, Authorization');
if (req.method === 'OPTIONS') {
return res.sendStatus(200);
}
next();
});
server.listen(SERVER_PORT, '0.0.0.0', async () => {
try {
await connectDB();
await syncDB();
logger.log('info', `Server listening on port ${SERVER_PORT}`);
} catch (error: any) {
logger.log('error', `Failed to connect to the database: ${error.message}`);
process.exit(1); // Exit the process if DB connection fails
}
});
process.on('SIGINT', () => {
console.log('Main app shutting down...');
workerProcess.kill();
if (!isProduction) {
workerProcess.kill();
}
process.exit();
});

View File

@@ -3,10 +3,15 @@ import dotenv from 'dotenv';
import setupAssociations from '../models/associations';
dotenv.config();
const sequelize = new Sequelize(
`postgresql://${process.env.DB_USER}:${process.env.DB_PASSWORD}@${process.env.DB_HOST}:${process.env.DB_PORT}/${process.env.DB_NAME}`,
const databaseUrl = `postgresql://${process.env.DB_USER}:${process.env.DB_PASSWORD}@${process.env.DB_HOST}:${process.env.DB_PORT}/${process.env.DB_NAME}`;
// Extract the hostname using the URL constructor
const host = new URL(databaseUrl).hostname;
const sequelize = new Sequelize(databaseUrl,
{
host: process.env.DB_HOST,
host,
dialect: 'postgres',
logging: false,
}

View File

@@ -25,7 +25,7 @@ const options = {
},
],
},
apis: [path.join(__dirname, '../api/*.ts')],
apis: process.env.NODE_ENV === 'production' ? [path.join(__dirname, '../api/*.js')] : [path.join(__dirname, '../api/*.ts')]
};
const swaggerSpec = swaggerJSDoc(options);

View File

@@ -36,7 +36,7 @@ export const encrypt = (text: string): string => {
export const decrypt = (encryptedText: string): string => {
const [iv, encrypted] = encryptedText.split(':');
const algorithm = getEnvVariable('ALGORITHM');
const algorithm = "aes-256-cbc";
const key = Buffer.from(getEnvVariable('ENCRYPTION_KEY'), 'hex');
const decipher = crypto.createDecipheriv(algorithm, key, Buffer.from(iv, 'hex'));
let decrypted = decipher.update(encrypted, 'hex', 'utf8');

View File

@@ -40,13 +40,7 @@ async function createWorkflowAndStoreMetadata(id: string, userId: string) {
};
}
const browserId = createRemoteBrowserForRun({
browser: chromium,
launchOptions: {
headless: true,
proxy: proxyOptions.server ? proxyOptions : undefined,
}
}, userId);
const browserId = createRemoteBrowserForRun( userId);
const runId = uuid();
const run = await Run.create({
@@ -229,7 +223,7 @@ export async function handleRunRecording(id: string, userId: string) {
throw new Error('browserId or runId or userId is undefined');
}
const socket = io(`http://localhost:8080/${browserId}`, {
const socket = io(`${process.env.BACKEND_URL}/${browserId}`, {
transports: ['websocket'],
rejectUnauthorized: false
});

10
server/start.sh Normal file
View File

@@ -0,0 +1,10 @@
#!/bin/bash
# Start Xvfb in the background with the desired dimensions
#Xvfb :0 -screen 0 900x400x24 &
# Wait for Xvfb to start
#sleep 2
# Execute the Node.js application
exec npm run server

View File

@@ -1,6 +1,6 @@
{
"compilerOptions": {
"target": "es2018",
"target": "es2020",
"module": "commonjs",
"outDir": "./dist",
"rootDir": "../",
@@ -21,12 +21,12 @@
"include": [
"src/**/*",
"../src/shared/**/*",
"../src/helpers/**/*"
"../src/helpers/**/*",
],
"exclude": [
"node_modules",
"../src/components/**/*", // Exclude frontend components
"../src/pages/**/*", // Exclude frontend pages
"../src/app/**/*" // Exclude other frontend-specific code
"../src/app/**/*", // Exclude other frontend-specific code
]
}

View File

@@ -1,8 +1,9 @@
import { default as axios } from "axios";
import { apiUrl } from "../apiConfig"
export const getUserById = async (userId: string) => {
try {
const response = await axios.get(`http://localhost:8080/auth/user/${userId}`);
const response = await axios.get(`${apiUrl}/auth/user/${userId}`);
if (response.status === 200) {
return response.data;
} else {

View File

@@ -1,8 +1,9 @@
import { default as axios } from "axios";
import { apiUrl } from "../apiConfig";
export const handleUploadCredentials = async (fileName: string, credentials: any, spreadsheetId: string, range: string): Promise<boolean> => {
try {
const response = await axios.post('http://localhost:8080/integration/upload-credentials', { fileName, credentials: JSON.parse(credentials), spreadsheetId, range });
const response = await axios.post(`${apiUrl}/integration/upload-credentials`, { fileName, credentials: JSON.parse(credentials), spreadsheetId, range });
if (response.status === 200) {
return response.data;
} else {

View File

@@ -1,8 +1,9 @@
import { default as axios } from "axios";
import { apiUrl } from "../apiConfig";
export const sendProxyConfig = async (proxyConfig: { server_url: string, username?: string, password?: string }): Promise<boolean> => {
try {
const response = await axios.post(`http://localhost:8080/proxy/config`, proxyConfig);
const response = await axios.post(`${apiUrl}/proxy/config`, proxyConfig);
if (response.status === 200) {
return response.data;
} else {
@@ -16,7 +17,7 @@ export const sendProxyConfig = async (proxyConfig: { server_url: string, usernam
export const getProxyConfig = async (): Promise<{ proxy_url: string, auth: boolean }> => {
try {
const response = await axios.get(`http://localhost:8080/proxy/config`);
const response = await axios.get(`${apiUrl}/proxy/config`);
if (response.status === 200) {
return response.data;
} else {
@@ -30,7 +31,7 @@ export const getProxyConfig = async (): Promise<{ proxy_url: string, auth: boole
export const testProxyConfig = async (): Promise<{ success: boolean }> => {
try {
const response = await axios.get(`http://localhost:8080/proxy/test`);
const response = await axios.get(`${apiUrl}/proxy/test`);
if (response.status === 200) {
return response.data;
} else {
@@ -44,7 +45,7 @@ export const testProxyConfig = async (): Promise<{ success: boolean }> => {
export const deleteProxyConfig = async (): Promise<boolean> => {
try {
const response = await axios.delete(`http://localhost:8080/proxy/config`);
const response = await axios.delete(`${apiUrl}/proxy/config`);
if (response.status === 200) {
return response.data;
} else {

View File

@@ -1,8 +1,9 @@
import { default as axios, AxiosResponse } from "axios";
import { apiUrl } from "../apiConfig";
export const startRecording = async() : Promise<string> => {
try {
const response = await axios.get('http://localhost:8080/record/start')
const response = await axios.get(`${apiUrl}/record/start`)
if (response.status === 200) {
return response.data;
} else {
@@ -14,7 +15,7 @@ export const startRecording = async() : Promise<string> => {
};
export const stopRecording = async (id: string): Promise<void> => {
await axios.get(`http://localhost:8080/record/stop/${id}`)
await axios.get(`${apiUrl}/record/stop/${id}`)
.then((response : AxiosResponse<boolean>) => {
})
.catch((error: any) => {
@@ -23,7 +24,7 @@ export const stopRecording = async (id: string): Promise<void> => {
export const getActiveBrowserId = async(): Promise<string> => {
try {
const response = await axios.get('http://localhost:8080/record/active');
const response = await axios.get(`${apiUrl}/record/active`);
if (response.status === 200) {
return response.data;
} else {
@@ -36,7 +37,7 @@ export const getActiveBrowserId = async(): Promise<string> => {
export const interpretCurrentRecording = async(): Promise<boolean> => {
try {
const response = await axios.get('http://localhost:8080/record/interpret');
const response = await axios.get(`${apiUrl}/record/interpret`);
if (response.status === 200) {
return true;
} else {
@@ -50,7 +51,7 @@ export const interpretCurrentRecording = async(): Promise<boolean> => {
export const stopCurrentInterpretation = async(): Promise<void> => {
try {
const response = await axios.get('http://localhost:8080/record/interpret/stop');
const response = await axios.get(`${apiUrl}/record/interpret/stop`);
if (response.status === 200) {
return;
} else {
@@ -63,7 +64,7 @@ export const stopCurrentInterpretation = async(): Promise<void> => {
export const getCurrentUrl = async (): Promise<string | null> => {
try {
const response = await axios.get('http://localhost:8080/record/active/url');
const response = await axios.get(`${apiUrl}/record/active/url`);
if (response.status === 200) {
return response.data;
} else {
@@ -77,7 +78,7 @@ export const getCurrentUrl = async (): Promise<string | null> => {
export const getCurrentTabs = async (): Promise<string[] | null> => {
try {
const response = await axios.get('http://localhost:8080/record/active/tabs');
const response = await axios.get(`${apiUrl}/record/active/tabs`);
if (response.status === 200) {
return response.data;
} else {

View File

@@ -3,10 +3,11 @@ import { WorkflowFile } from "maxun-core";
import { RunSettings } from "../components/molecules/RunSettings";
import { ScheduleSettings } from "../components/molecules/ScheduleSettings";
import { CreateRunResponse, ScheduleRunResponse } from "../pages/MainPage";
import { apiUrl } from "../apiConfig";
export const getStoredRecordings = async (): Promise<string[] | null> => {
try {
const response = await axios.get('http://localhost:8080/storage/recordings');
const response = await axios.get(`${apiUrl}/storage/recordings`);
if (response.status === 200) {
return response.data;
} else {
@@ -20,7 +21,7 @@ export const getStoredRecordings = async (): Promise<string[] | null> => {
export const getStoredRuns = async (): Promise<string[] | null> => {
try {
const response = await axios.get('http://localhost:8080/storage/runs');
const response = await axios.get(`${apiUrl}/storage/runs`);
if (response.status === 200) {
return response.data;
} else {
@@ -34,7 +35,7 @@ export const getStoredRuns = async (): Promise<string[] | null> => {
export const getStoredRecording = async (id: string) => {
try {
const response = await axios.get(`http://localhost:8080/storage/recordings/${id}`);
const response = await axios.get(`${apiUrl}/storage/recordings/${id}`);
if (response.status === 200) {
return response.data;
} else {
@@ -48,7 +49,7 @@ export const getStoredRecording = async (id: string) => {
export const deleteRecordingFromStorage = async (id: string): Promise<boolean> => {
try {
const response = await axios.delete(`http://localhost:8080/storage/recordings/${id}`);
const response = await axios.delete(`${apiUrl}/storage/recordings/${id}`);
if (response.status === 200) {
return response.data;
} else {
@@ -62,7 +63,7 @@ export const deleteRecordingFromStorage = async (id: string): Promise<boolean> =
export const deleteRunFromStorage = async (id: string): Promise<boolean> => {
try {
const response = await axios.delete(`http://localhost:8080/storage/runs/${id}`);
const response = await axios.delete(`${apiUrl}/storage/runs/${id}`);
if (response.status === 200) {
return response.data;
} else {
@@ -76,7 +77,7 @@ export const deleteRunFromStorage = async (id: string): Promise<boolean> => {
export const editRecordingFromStorage = async (browserId: string, id: string): Promise<WorkflowFile | null> => {
try {
const response = await axios.put(`http://localhost:8080/workflow/${browserId}/${id}`);
const response = await axios.put(`${apiUrl}/workflow/${browserId}/${id}`);
if (response.status === 200) {
return response.data;
} else {
@@ -91,7 +92,7 @@ export const editRecordingFromStorage = async (browserId: string, id: string): P
export const createRunForStoredRecording = async (id: string, settings: RunSettings): Promise<CreateRunResponse> => {
try {
const response = await axios.put(
`http://localhost:8080/storage/runs/${id}`,
`${apiUrl}/storage/runs/${id}`,
{ ...settings });
if (response.status === 200) {
return response.data;
@@ -106,7 +107,7 @@ export const createRunForStoredRecording = async (id: string, settings: RunSetti
export const interpretStoredRecording = async (id: string): Promise<boolean> => {
try {
const response = await axios.post(`http://localhost:8080/storage/runs/run/${id}`);
const response = await axios.post(`${apiUrl}/storage/runs/run/${id}`);
if (response.status === 200) {
return response.data;
} else {
@@ -120,7 +121,7 @@ export const interpretStoredRecording = async (id: string): Promise<boolean> =>
export const notifyAboutAbort = async (id: string): Promise<boolean> => {
try {
const response = await axios.post(`http://localhost:8080/storage/runs/abort/${id}`);
const response = await axios.post(`${apiUrl}/storage/runs/abort/${id}`);
if (response.status === 200) {
return response.data;
} else {
@@ -135,7 +136,7 @@ export const notifyAboutAbort = async (id: string): Promise<boolean> => {
export const scheduleStoredRecording = async (id: string, settings: ScheduleSettings): Promise<ScheduleRunResponse> => {
try {
const response = await axios.put(
`http://localhost:8080/storage/schedule/${id}`,
`${apiUrl}/storage/schedule/${id}`,
{ ...settings });
if (response.status === 200) {
return response.data;
@@ -150,7 +151,7 @@ export const scheduleStoredRecording = async (id: string, settings: ScheduleSett
export const getSchedule = async (id: string) => {
try {
const response = await axios.get(`http://localhost:8080/storage/schedule/${id}`);
const response = await axios.get(`${apiUrl}/storage/schedule/${id}`);
if (response.status === 200) {
return response.data.schedule;
} else {
@@ -164,7 +165,7 @@ export const getSchedule = async (id: string) => {
export const deleteSchedule = async (id: string): Promise<boolean> => {
try {
const response = await axios.delete(`http://localhost:8080/storage/schedule/${id}`);
const response = await axios.delete(`${apiUrl}/storage/schedule/${id}`);
if (response.status === 200) {
return response.data;
} else {

View File

@@ -1,10 +1,11 @@
import { WhereWhatPair, WorkflowFile } from "maxun-core";
import { emptyWorkflow } from "../shared/constants";
import { default as axios, AxiosResponse } from "axios";
import { apiUrl } from "../apiConfig";
export const getActiveWorkflow = async(id: string) : Promise<WorkflowFile> => {
try {
const response = await axios.get(`http://localhost:8080/workflow/${id}`)
const response = await axios.get(`${apiUrl}/workflow/${id}`)
if (response.status === 200) {
return response.data;
} else {
@@ -18,7 +19,7 @@ export const getActiveWorkflow = async(id: string) : Promise<WorkflowFile> => {
export const getParamsOfActiveWorkflow = async(id: string) : Promise<string[]|null> => {
try {
const response = await axios.get(`http://localhost:8080/workflow/params/${id}`)
const response = await axios.get(`${apiUrl}/workflow/params/${id}`)
if (response.status === 200) {
return response.data;
} else {
@@ -32,7 +33,7 @@ export const getParamsOfActiveWorkflow = async(id: string) : Promise<string[]|nu
export const deletePair = async(index: number): Promise<WorkflowFile> => {
try {
const response = await axios.delete(`http://localhost:8080/workflow/pair/${index}`);
const response = await axios.delete(`${apiUrl}/workflow/pair/${index}`);
if (response.status === 200) {
return response.data;
} else {
@@ -46,7 +47,7 @@ export const deletePair = async(index: number): Promise<WorkflowFile> => {
export const AddPair = async(index: number, pair: WhereWhatPair): Promise<WorkflowFile> => {
try {
const response = await axios.post(`http://localhost:8080/workflow/pair/${index}`, {
const response = await axios.post(`${apiUrl}/workflow/pair/${index}`, {
pair,
}, {headers: {'Content-Type': 'application/json'}});
if (response.status === 200) {
@@ -62,7 +63,7 @@ export const AddPair = async(index: number, pair: WhereWhatPair): Promise<Workfl
export const UpdatePair = async(index: number, pair: WhereWhatPair): Promise<WorkflowFile> => {
try {
const response = await axios.put(`http://localhost:8080/workflow/pair/${index}`, {
const response = await axios.put(`${apiUrl}/workflow/pair/${index}`, {
pair,
}, {headers: {'Content-Type': 'application/json'}});
if (response.status === 200) {

1
src/apiConfig.js Normal file
View File

@@ -0,0 +1 @@
export const apiUrl = import.meta.env.VITE_BACKEND_URL;

View File

@@ -6,6 +6,7 @@ import TextField from "@mui/material/TextField";
import axios from 'axios';
import { useGlobalInfoStore } from '../../context/globalInfo';
import { getStoredRecording } from '../../api/storage';
import { apiUrl } from '../../apiConfig.js';
interface IntegrationProps {
isOpen: boolean;
handleStart: (data: IntegrationSettings) => void;
@@ -32,12 +33,12 @@ export const IntegrationSettingsModal = ({ isOpen, handleStart, handleClose }: I
const [recording, setRecording] = useState<any>(null);
const authenticateWithGoogle = () => {
window.location.href = `http://localhost:8080/auth/google?robotId=${recordingId}`;
window.location.href = `${apiUrl}/auth/google?robotId=${recordingId}`;
};
const handleOAuthCallback = async () => {
try {
const response = await axios.get(`http://localhost:8080/auth/google/callback`);
const response = await axios.get(`${apiUrl}/auth/google/callback`);
const { google_sheet_email, files } = response.data;
} catch (error) {
setError('Error authenticating with Google');
@@ -46,7 +47,7 @@ export const IntegrationSettingsModal = ({ isOpen, handleStart, handleClose }: I
const fetchSpreadsheetFiles = async () => {
try {
const response = await axios.get(`http://localhost:8080/auth/gsheets/files?robotId=${recordingId}`, {
const response = await axios.get(`${apiUrl}/auth/gsheets/files?robotId=${recordingId}`, {
withCredentials: true,
});
setSpreadsheets(response.data);
@@ -66,7 +67,7 @@ export const IntegrationSettingsModal = ({ isOpen, handleStart, handleClose }: I
const updateGoogleSheetId = async () => {
try {
const response = await axios.post(
`http://localhost:8080/auth/gsheets/update`,
`${apiUrl}/auth/gsheets/update`,
{ spreadsheetId: settings.spreadsheetId, spreadsheetName: settings.spreadsheetName, robotId: recordingId },
{ withCredentials: true }
);
@@ -79,7 +80,7 @@ export const IntegrationSettingsModal = ({ isOpen, handleStart, handleClose }: I
const removeIntegration = async () => {
try {
await axios.post(
`http://localhost:8080/auth/gsheets/remove`,
`${apiUrl}/auth/gsheets/remove`,
{ robotId: recordingId },
{ withCredentials: true }
);

View File

@@ -9,6 +9,7 @@ import { useNavigate } from 'react-router-dom';
import { AuthContext } from '../../context/auth';
import { SaveRecording } from '../molecules/SaveRecording';
import DiscordIcon from '../atoms/DiscordIcon';
import { apiUrl } from '../../apiConfig';
interface NavBarProps {
recordingName: string;
@@ -34,7 +35,7 @@ export const NavBar: React.FC<NavBarProps> = ({ recordingName, isRecording }) =>
const logout = async () => {
dispatch({ type: 'LOGOUT' });
window.localStorage.removeItem('user');
const { data } = await axios.get('http://localhost:8080/auth/logout');
const { data } = await axios.get(`${apiUrl}/auth/logout`);
notify('success', data.message);
navigate('/login');
};

View File

@@ -18,6 +18,7 @@ import { ContentCopy, Visibility, Delete } from '@mui/icons-material';
import styled from 'styled-components';
import axios from 'axios';
import { useGlobalInfoStore } from '../../context/globalInfo';
import { apiUrl } from '../../apiConfig';
const Container = styled(Box)`
display: flex;
@@ -38,7 +39,7 @@ const ApiKeyManager = () => {
useEffect(() => {
const fetchApiKey = async () => {
try {
const { data } = await axios.get('http://localhost:8080/auth/api-key');
const { data } = await axios.get(`${apiUrl}/auth/api-key`);
setApiKey(data.api_key);
} catch (error: any) {
notify('error', `Failed to fetch API Key - ${error.message}`);
@@ -53,7 +54,7 @@ const ApiKeyManager = () => {
const generateApiKey = async () => {
setLoading(true);
try {
const { data } = await axios.post('http://localhost:8080/auth/generate-api-key');
const { data } = await axios.post(`${apiUrl}/auth/generate-api-key`);
setApiKey(data.api_key);
notify('success', `Generated API Key successfully`);
} catch (error: any) {
@@ -66,7 +67,7 @@ const ApiKeyManager = () => {
const deleteApiKey = async () => {
setLoading(true);
try {
await axios.delete('http://localhost:8080/auth/delete-api-key');
await axios.delete(`${apiUrl}/auth/delete-api-key`);
setApiKey(null);
notify('success', 'API Key deleted successfully');
} catch (error: any) {

View File

@@ -2,7 +2,6 @@ import React, { useCallback, useEffect, useState } from 'react';
import { useSocketStore } from '../../context/socket';
import { Button } from '@mui/material';
import Canvas from "../atoms/canvas";
import { useBrowserDimensionsStore } from "../../context/browserDimensions";
import { Highlighter } from "../atoms/Highlighter";
import { GenericModal } from '../atoms/GenericModal';
import { useActionContext } from '../../context/browserActions';
@@ -66,7 +65,6 @@ export const BrowserWindow = () => {
const { socket } = useSocketStore();
const { notify } = useGlobalInfoStore();
//const { width, height } = useBrowserDimensionsStore();
const { getText, getList, paginationMode, paginationType, limitMode } = useActionContext();
const { addTextStep, addListStep } = useBrowserSteps();
@@ -405,4 +403,4 @@ const modalStyle = {
height: 'fit-content',
display: 'block',
padding: '20px',
};
};

View File

@@ -4,6 +4,7 @@ import Tab from '@mui/material/Tab';
import Box from '@mui/material/Box';
import { Paper, Button } from "@mui/material";
import { AutoAwesome, FormatListBulleted, VpnKey, Usb, Article, Link, CloudQueue } from "@mui/icons-material";
import { apiUrl } from "../../apiConfig";
interface MainMenuProps {
value: string;
@@ -86,7 +87,7 @@ export const MainMenu = ({ value = 'recordings', handleChangeContent }: MainMenu
</Tabs>
<hr />
<Box sx={{ display: 'flex', flexDirection: 'column', gap: '1rem', textAlign: 'left' }}>
<Button href="http://localhost:8080/api-docs/" target="_blank" rel="noopener noreferrer" sx={buttonStyles} startIcon={<Article />}>
<Button href={`${apiUrl}/api-docs/`} target="_blank" rel="noopener noreferrer" sx={buttonStyles} startIcon={<Article />}>
API Docs
</Button>
<Button href="https://forms.gle/hXjgqDvkEhPcaBW76" target="_blank" rel="noopener noreferrer" sx={buttonStyles} startIcon={<CloudQueue />}>

View File

@@ -1,6 +1,7 @@
import { useReducer, createContext, useEffect } from 'react';
import axios from 'axios';
import { useNavigate } from 'react-router-dom';
import { apiUrl } from "../apiConfig";
interface AuthProviderProps {
children: React.ReactNode;
@@ -65,7 +66,7 @@ const AuthProvider = ({ children }: AuthProviderProps) => {
if (res.status === 401 && res.config && !res.config.__isRetryRequest) {
return new Promise((resolve, reject) => {
axios
.get('http://localhost:8080/auth/logout')
.get(`${apiUrl}/auth/logout`)
.then(() => {
console.log('/401 error > logout');
dispatch({ type: 'LOGOUT' });

View File

@@ -1,7 +1,8 @@
import React, { createContext, useCallback, useContext, useMemo, useState } from 'react';
import { io, Socket } from 'socket.io-client';
import { apiUrl } from "../apiConfig";
const SERVER_ENDPOINT = 'http://localhost:8080';
const SERVER_ENDPOINT = apiUrl;
interface SocketState {
socket: Socket | null;

View File

@@ -10,6 +10,7 @@ import {
CircularProgress,
} from '@mui/material';
import { useGlobalInfoStore } from "../context/globalInfo";
import { apiUrl } from "../apiConfig";
const Login = () => {
const [form, setForm] = useState({
@@ -40,7 +41,7 @@ const Login = () => {
e.preventDefault();
setLoading(true);
try {
const { data } = await axios.post(`http://localhost:8080/auth/login`, { email, password });
const { data } = await axios.post(`${apiUrl}/auth/login`, { email, password });
dispatch({ type: 'LOGIN', payload: data });
notify('success', 'Welcome to Maxun!');
window.localStorage.setItem('user', JSON.stringify(data));

View File

@@ -13,6 +13,7 @@ import { RunSettings } from "../components/molecules/RunSettings";
import { ScheduleSettings } from "../components/molecules/ScheduleSettings";
import { IntegrationSettings } from "../components/molecules/IntegrationSettings";
import { RobotSettings } from "../components/molecules/RobotSettings";
import { apiUrl } from "../apiConfig";
interface MainPageProps {
handleEditRecording: (id: string, fileName: string) => void;
@@ -88,7 +89,7 @@ export const MainPage = ({ handleEditRecording }: MainPageProps) => {
createRunForStoredRecording(runningRecordingId, settings).then(({ browserId, runId }: CreateRunResponse) => {
setIds({ browserId, runId });
const socket =
io(`http://localhost:8080/${browserId}`, {
io(`${apiUrl}/${browserId}`, {
transports: ["websocket"],
rejectUnauthorized: false
});

View File

@@ -4,6 +4,7 @@ import axios from 'axios';
import { AuthContext } from '../context/auth';
import { TextField, Button, CircularProgress, Typography, Box, Container } from '@mui/material';
import { useGlobalInfoStore } from "../context/globalInfo";
import { apiUrl } from "../apiConfig";
const Register = () => {
const [form, setForm] = useState({
@@ -31,7 +32,7 @@ const Register = () => {
e.preventDefault();
setLoading(true);
try {
const { data } = await axios.post('http://localhost:8080/auth/register', {
const { data } = await axios.post(`${apiUrl}/auth/register`, {
email,
password,
});

View File

@@ -9,13 +9,14 @@
"strict": true,
"forceConsistentCasingInFileNames": true,
"noFallthroughCasesInSwitch": true,
"module": "commonjs",
"module": "esnext",
"moduleResolution": "node",
"resolveJsonModule": true,
"isolatedModules": true,
"noEmit": true,
"jsx": "react-jsx",
"types": ["vite/client"],
"outDir": "./build"
},
"include": ["src"]
"include": ["src", "vite-env.d.ts"]
}

7
vite-env.d.ts vendored Normal file
View File

@@ -0,0 +1,7 @@
interface ImportMetaEnv {
readonly VITE_BACKEND_URL: string;
}
interface ImportMeta {
readonly env: ImportMetaEnv;
}

View File

@@ -3,6 +3,9 @@ import react from '@vitejs/plugin-react';
export default defineConfig(() => {
return {
define: {
'import.meta.env.VITE_BACKEND_URL': JSON.stringify(process.env.VITE_BACKEND_URL),
},
build: {
outDir: 'build',
manifest: true,