diff --git a/Dockerfile b/Dockerfile index d7abca37..719e7814 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,70 +1,22 @@ -# --- Base Stage --- -FROM node:18 AS base +FROM node:18-alpine + WORKDIR /app -# Copy shared package.json and install dependencies -COPY package.json package-lock.json ./ -COPY maxun-core/package.json ./maxun-core/package.json -RUN npm install - -# --- Backend Build Stage --- -FROM base AS backend-build -WORKDIR /app - -# Copy TypeScript configs -COPY tsconfig*.json ./ -COPY server/tsconfig.json ./server/ - -# Copy ALL source code (both frontend and backend) -COPY src ./src -# Copy backend code and maxun-core -COPY server/src ./server/src +# Copy package files +COPY package*.json ./ COPY maxun-core ./maxun-core -# Install TypeScript globally and build -RUN npm install -g typescript -RUN npm run build:server +# Install dependencies +RUN npm install -# --- Frontend Build Stage --- -FROM base AS frontend-build -WORKDIR /app - -# Copy frontend code and configs +# Copy frontend source code and config COPY src ./src -COPY index.html ./index.html -COPY public ./public +COPY index.html ./ COPY vite.config.js ./ COPY tsconfig.json ./ -# Build frontend -RUN npm run build +# Expose the frontend port +EXPOSE 5173 -# --- Production Stage --- -FROM nginx:alpine AS production - -# Install Node.js in the production image -RUN apk add --update nodejs npm - -# Copy nginx configuration -COPY nginx.conf /etc/nginx/conf.d/default.conf - -# Copy built frontend -COPY --from=frontend-build /app/build /usr/share/nginx/html -COPY --from=frontend-build /app/public/img /usr/share/nginx/html/img - -# Copy built backend and its dependencies -WORKDIR /app -COPY --from=backend-build /app/package*.json ./ -COPY --from=backend-build /app/server/dist ./server/dist -COPY --from=backend-build /app/maxun-core ./maxun-core -COPY --from=backend-build /app/node_modules ./node_modules - -# Copy start script -COPY docker-entrypoint.sh / -RUN chmod +x /docker-entrypoint.sh - -EXPOSE 80 8080 - -# Start both nginx and node server -ENTRYPOINT ["/docker-entrypoint.sh"] - \ No newline at end of file +# Start the frontend using the client script +CMD ["npm", "run", "client", "--", "--host"] \ No newline at end of file diff --git a/README.md b/README.md index 2fd65ed6..dfd2a97a 100644 --- a/README.md +++ b/README.md @@ -25,13 +25,15 @@ Maxun lets you train a robot in 2 minutes and scrape the web on auto-pilot. Web -# Installation +# Local Setup ### Docker -⚠️ Work In Progress. Will be available by EOD. +``` +docker-compose up -d --build +``` -### Local Setup +### Without Docker 1. Ensure you have Node.js, PostgreSQL, MinIO and Redis installed on your system. -2. Run the commands below: +2. Run the commands below ``` git clone https://github.com/getmaxun/maxun diff --git a/docker-compose.yml b/docker-compose.yml index f36e8900..50983684 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,30 +1,31 @@ version: '3.8' services: - app: - build: - context: . - dockerfile: Dockerfile - target: production - env_file: .env - ports: - - "5173:80" - - "8080:8080" - depends_on: - - db - - minio - - redis - - db: + postgres: image: postgres:13 environment: - POSTGRES_DB: ${DB_NAME} POSTGRES_USER: ${DB_USER} POSTGRES_PASSWORD: ${DB_PASSWORD} + POSTGRES_DB: ${DB_NAME} ports: - "5432:5432" volumes: - postgres_data:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U postgres"] + interval: 10s + timeout: 5s + retries: 5 + + redis: + image: redis:6 + environment: + REDIS_HOST: ${REDIS_HOST} + REDIS_PORT: ${REDIS_PORT} + ports: + - "6379:6379" + volumes: + - redis_data:/data minio: image: minio/minio @@ -37,15 +38,41 @@ services: volumes: - minio_data:/data - redis: - image: redis:6 - environment: - - REDIS_HOST=redis - - REDIS_PORT=6379 + backend: + build: + context: . + dockerfile: server/Dockerfile ports: - - "6379:6379" + - "8080:8080" + env_file: .env + environment: + # to ensure Playwright works in Docker + PLAYWRIGHT_BROWSERS_PATH: /ms-playwright + PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 0 + # DEBUG: pw:api + # PWDEBUG: 1 # Enables debugging + CHROMIUM_FLAGS: '--disable-gpu --no-sandbox --headless=new' volumes: - - redis_data:/data + # - /tmp/.X11-unix:/tmp/.X11-unix + - /var/run/dbus:/var/run/dbus # Add this for D-Bus support + security_opt: + - seccomp=unconfined # This might help with browser sandbox issues + # Increase shared memory size for Chromium + shm_size: '2gb' + depends_on: + - postgres + - redis + - minio + + frontend: + build: + context: . + dockerfile: Dockerfile + ports: + - "5173:5173" + env_file: .env + depends_on: + - backend volumes: postgres_data: diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh index 7e36eed4..e40af108 100644 --- a/docker-entrypoint.sh +++ b/docker-entrypoint.sh @@ -1,7 +1,7 @@ #!/bin/sh # Start backend server -cd /app && npm run start:server & +cd /app && npm run start:server -- --host 0.0.0.0 & # Start nginx -nginx -g 'daemon off;' \ No newline at end of file +nginx -g 'daemon off;' diff --git a/index.html b/index.html index 8a52962b..81e42f02 100644 --- a/index.html +++ b/index.html @@ -8,7 +8,7 @@ name="description" content="Web site created using Vite" /> - + Maxun | Open Source No Code Web Data Extraction Platform diff --git a/nginx.conf b/nginx.conf index e9d636f8..1ea52146 100644 --- a/nginx.conf +++ b/nginx.conf @@ -1,17 +1,45 @@ server { listen 80; - + location / { root /usr/share/nginx/html; try_files $uri $uri/ /index.html; } location /api { - proxy_pass http://127.0.0.1:8080; + proxy_pass http://localhost:8080; proxy_http_version 1.1; proxy_set_header Upgrade $http_upgrade; proxy_set_header Connection 'upgrade'; proxy_set_header Host $host; proxy_cache_bypass $http_upgrade; + + # Add timeout configurations + proxy_connect_timeout 60s; + proxy_send_timeout 60s; + proxy_read_timeout 60s; + + # Add error handling + proxy_intercept_errors on; + error_page 502 503 504 /50x.html; + } + + location ~ ^/(record|workflow|storage|auth|integration|proxy|api-docs) { + proxy_pass http://localhost:8080; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection 'keep-alive'; # Ensure connections remain open + proxy_set_header Connection 'upgrade'; + proxy_set_header Host $host; + proxy_cache_bypass $http_upgrade; + + # Timeout configurations + proxy_connect_timeout 60s; + proxy_send_timeout 60s; + proxy_read_timeout 60s; + + # Error handling for these routes + proxy_intercept_errors on; + error_page 502 503 504 /50x.html; } } \ No newline at end of file diff --git a/package.json b/package.json index 343270b8..fd966458 100644 --- a/package.json +++ b/package.json @@ -73,11 +73,11 @@ }, "scripts": { "start": "concurrently -k \"npm run server\" \"npm run client\"", - "server": "./node_modules/.bin/nodemon server/src/server.ts", + "server": "cross-env NODE_OPTIONS='--max-old-space-size=8000' nodemon server/src/server.ts", "client": "vite", "build": "vite build", "build:server": "tsc -p server/tsconfig.json", - "start:server": "node server/dist/server/src/server.js", + "start:server": "cross-env NODE_OPTIONS='--max-old-space-size=8000' server/dist/server/src/server.js", "preview": "vite preview", "lint": "./node_modules/.bin/eslint ." }, @@ -101,6 +101,7 @@ "@vitejs/plugin-react": "^4.3.3", "ajv": "^8.8.2", "concurrently": "^7.0.0", + "cross-env": "^7.0.3", "nodemon": "^2.0.15", "ts-node": "^10.4.0", "vite": "^5.4.10" diff --git a/server/Dockerfile b/server/Dockerfile new file mode 100644 index 00000000..8e5c21bb --- /dev/null +++ b/server/Dockerfile @@ -0,0 +1,83 @@ +FROM mcr.microsoft.com/playwright:v1.40.0-jammy + +# Set working directory +WORKDIR /app + +# Install node dependencies +COPY package*.json ./ +COPY maxun-core ./maxun-core +COPY src ./src +COPY server ./server +COPY tsconfig.json ./ +COPY server/tsconfig.json ./server/ +COPY server/start.sh ./ + +# Install dependencies +RUN npm install + +# Install Playwright browsers and dependencies +RUN npx playwright install --with-deps chromium + +# Install xvfb for display support +#RUN apt-get update && apt-get install -y xvfb + +# RUN apt-get update && apt-get install -y \ +# libgbm-dev \ +# libxkbcommon-x11-0 \ +# libxcomposite1 \ +# libxdamage1 \ +# libxrandr2 \ +# libxshmfence1 \ +# libxtst6 \ +# libnss3 \ +# libatk1.0-0 \ +# libatk-bridge2.0-0 \ +# libdrm2 \ +# libxcb1 \ +# libxkbcommon0 \ +# fonts-noto-color-emoji \ +# fonts-unifont \ +# libpango-1.0-0 \ +# libcairo2 \ +# libasound2 \ +# libglib2.0-0 \ +# libdbus-1-3 \ +# && rm -rf /var/lib/apt/lists/* + +# Create and set permissions for chrome directories +# Create the Chromium data directory with necessary permissions +RUN mkdir -p /tmp/chromium-data-dir && \ + chmod -R 777 /tmp/chromium-data-dir + +# Install dependencies +RUN apt-get update && apt-get install -y \ + libgbm-dev \ + libnss3 \ + libatk1.0-0 \ + libatk-bridge2.0-0 \ + libdrm2 \ + libxkbcommon0 \ + libglib2.0-0 \ + libdbus-1-3 \ + libx11-xcb1 \ + libxcb1 \ + libxcomposite1 \ + libxcursor1 \ + libxdamage1 \ + libxext6 \ + libxi6 \ + libxtst6 \ + && rm -rf /var/lib/apt/lists/* \ + && mkdir -p /tmp/.X11-unix && chmod 1777 /tmp/.X11-unix + +# Add a dbus configuration to prevent connection errors +# RUN mkdir -p /var/run/dbus + +# Make the script executable +RUN chmod +x ./start.sh + +# Expose the backend port +EXPOSE 8080 + +# Start the backend using the start script +CMD ["./start.sh"] \ No newline at end of file diff --git a/server/src/api/record.ts b/server/src/api/record.ts index 7710f075..610f1825 100644 --- a/server/src/api/record.ts +++ b/server/src/api/record.ts @@ -464,13 +464,7 @@ async function createWorkflowAndStoreMetadata(id: string, userId: string) { }; } - const browserId = createRemoteBrowserForRun({ - browser: chromium, - launchOptions: { - headless: true, - proxy: proxyOptions.server ? proxyOptions : undefined, - } - }, userId); + const browserId = createRemoteBrowserForRun(userId); const runId = uuid(); @@ -656,7 +650,7 @@ export async function handleRunRecording(id: string, userId: string) { throw new Error('browserId or runId or userId is undefined'); } - const socket = io(`http://localhost:8080/${browserId}`, { + const socket = io(`${process.env.BACKEND_URL}/${browserId}`, { transports: ['websocket'], rejectUnauthorized: false }); diff --git a/server/src/browser-management/classes/RemoteBrowser.ts b/server/src/browser-management/classes/RemoteBrowser.ts index 5f07190f..cbda3942 100644 --- a/server/src/browser-management/classes/RemoteBrowser.ts +++ b/server/src/browser-management/classes/RemoteBrowser.ts @@ -3,6 +3,7 @@ import { Browser, CDPSession, BrowserContext, + chromium, } from 'playwright'; import { Socket } from "socket.io"; import { PlaywrightBlocker } from '@cliqz/adblocker-playwright'; @@ -91,8 +92,39 @@ export class RemoteBrowser { * @param options remote browser options to be used when launching the browser * @returns {Promise} */ - public initialize = async (options: RemoteBrowserOptions, userId: string): Promise => { - this.browser = (await options.browser.launch(options.launchOptions)); + public initialize = async (userId: string): Promise => { + // const launchOptions = { + // headless: true, + // proxy: options.launchOptions?.proxy, + // chromiumSandbox: false, + // args: [ + // '--no-sandbox', + // '--disable-setuid-sandbox', + // '--headless=new', + // '--disable-gpu', + // '--disable-dev-shm-usage', + // '--disable-software-rasterizer', + // '--in-process-gpu', + // '--disable-infobars', + // '--single-process', + // '--no-zygote', + // '--disable-notifications', + // '--disable-extensions', + // '--disable-background-timer-throttling', + // ...(options.launchOptions?.args || []) + // ], + // env: { + // ...process.env, + // CHROMIUM_FLAGS: '--disable-gpu --no-sandbox --headless=new' + // } + // }; + // console.log('Launch options before:', options.launchOptions); + // this.browser = (await options.browser.launch(launchOptions)); + + // console.log('Launch options after:', options.launchOptions) + this.browser = (await chromium.launch({ + headless: true, + })); const proxyConfig = await getDecryptedProxyConfig(userId); let proxyOptions: { server: string, username?: string, password?: string } = { server: '' }; if (proxyConfig.proxy_url) { @@ -107,6 +139,16 @@ export class RemoteBrowser { const contextOptions: any = { viewport: { height: 400, width: 900 }, // recordVideo: { dir: 'videos/' } + // Force reduced motion to prevent animation issues + reducedMotion: 'reduce', + // Force JavaScript to be enabled + javaScriptEnabled: true, + // Set a reasonable timeout + timeout: 50000, + // Disable hardware acceleration + forcedColors: 'none', + isMobile: false, + hasTouch: false }; if (proxyOptions.server) { @@ -116,9 +158,17 @@ export class RemoteBrowser { password: proxyOptions.password ? proxyOptions.password : undefined, }; } + const browserUserAgent = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/110.0.5481.38 Safari/537.36"; + + contextOptions.userAgent = browserUserAgent; this.context = await this.browser.newContext(contextOptions); + console.log(`Context from initialize: ${JSON.stringify(this.context)}`) this.currentPage = await this.context.newPage(); + console.log(`CPage from initialize: ${JSON.stringify(this.currentPage)}`) + // await this.currentPage.setExtraHTTPHeaders({ + // 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3' + // }); const blocker = await PlaywrightBlocker.fromPrebuiltAdsAndTracking(fetch); await blocker.enableBlockingInPage(this.currentPage); this.client = await this.currentPage.context().newCDPSession(this.currentPage); @@ -331,6 +381,9 @@ export class RemoteBrowser { await this.stopScreencast(); const newPage = options ? await this.browser?.newPage(options) : await this.browser?.newPage(); + await newPage?.setExtraHTTPHeaders({ + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3' + }); await this.currentPage?.close(); this.currentPage = newPage; diff --git a/server/src/browser-management/controller.ts b/server/src/browser-management/controller.ts index 1b947ae4..24a677ce 100644 --- a/server/src/browser-management/controller.ts +++ b/server/src/browser-management/controller.ts @@ -20,7 +20,7 @@ import logger from "../logger"; * @returns string * @category BrowserManagement-Controller */ -export const initializeRemoteBrowserForRecording = (options: RemoteBrowserOptions, userId: string): string => { +export const initializeRemoteBrowserForRecording = (userId: string): string => { const id = getActiveBrowserId() || uuid(); createSocketConnection( io.of(id), @@ -34,7 +34,7 @@ export const initializeRemoteBrowserForRecording = (options: RemoteBrowserOption } else { const browserSession = new RemoteBrowser(socket); browserSession.interpreter.subscribeToPausing(); - await browserSession.initialize(options, userId); + await browserSession.initialize(userId); await browserSession.registerEditorEvents(); await browserSession.subscribeToScreencast(); browserPool.addRemoteBrowser(id, browserSession, true); @@ -52,13 +52,13 @@ export const initializeRemoteBrowserForRecording = (options: RemoteBrowserOption * @returns string * @category BrowserManagement-Controller */ -export const createRemoteBrowserForRun = (options: RemoteBrowserOptions, userId: string): string => { +export const createRemoteBrowserForRun = (userId: string): string => { const id = uuid(); createSocketConnectionForRun( io.of(id), async (socket: Socket) => { const browserSession = new RemoteBrowser(socket); - await browserSession.initialize(options, userId); + await browserSession.initialize(userId); browserPool.addRemoteBrowser(id, browserSession, true); socket.emit('ready-for-run'); }); diff --git a/server/src/routes/auth.ts b/server/src/routes/auth.ts index d4104f19..6766a356 100644 --- a/server/src/routes/auth.ts +++ b/server/src/routes/auth.ts @@ -14,6 +14,8 @@ interface AuthenticatedRequest extends Request { } router.post('/register', async (req, res) => { + console.log('Received request at /auth/register'); + console.log('Received body:', req.body); try { const { email, password } = req.body @@ -25,7 +27,21 @@ router.post('/register', async (req, res) => { const hashedPassword = await hashPassword(password) - const user = await User.create({ email, password: hashedPassword }); + let user: any; + + try { + user = await User.create({ email, password: hashedPassword }); + } catch ( + error: any + ) { + console.log(`Could not create user - ${error}`) + return res.status(500).send(`Could not create user - ${error.message}`) + } + + if (!process.env.JWT_SECRET) { + console.log('JWT_SECRET is not defined in the environment'); + return res.status(500).send('Internal Server Error'); + } const token = jwt.sign({ id: user.id }, process.env.JWT_SECRET as string, { expiresIn: '12h' }); user.password = undefined as unknown as string @@ -40,8 +56,10 @@ router.post('/register', async (req, res) => { registeredAt: new Date().toISOString() } ) + console.log(`User registered - ${user.email}`) res.json(user) } catch (error: any) { + console.log(`Could not register user - ${error}`) res.status(500).send(`Could not register user - ${error.message}`) } }) diff --git a/server/src/routes/record.ts b/server/src/routes/record.ts index 1daf2598..56089118 100644 --- a/server/src/routes/record.ts +++ b/server/src/routes/record.ts @@ -11,14 +11,14 @@ import { stopRunningInterpretation, getRemoteBrowserCurrentUrl, getRemoteBrowserCurrentTabs, } from '../browser-management/controller' -import { chromium } from 'playwright-extra'; +import { chromium } from 'playwright'; import stealthPlugin from 'puppeteer-extra-plugin-stealth'; import logger from "../logger"; import { getDecryptedProxyConfig } from './proxy'; import { requireSignIn } from '../middlewares/auth'; export const router = Router(); -chromium.use(stealthPlugin()); +// chromium.use(stealthPlugin()); export interface AuthenticatedRequest extends Request { @@ -56,13 +56,8 @@ router.get('/start', requireSignIn, async (req: AuthenticatedRequest, res: Respo }; } - const id = initializeRemoteBrowserForRecording({ - browser: chromium, - launchOptions: { - headless: true, - proxy: proxyOptions.server ? proxyOptions : undefined, - } - }, req.user.id); + const id = initializeRemoteBrowserForRecording(req.user.id); + console.log('id start:', id); return res.send(id); }); @@ -74,10 +69,8 @@ router.post('/start', requireSignIn, (req: AuthenticatedRequest, res:Response) = if (!req.user) { return res.status(401).send('User not authenticated'); } - const id = initializeRemoteBrowserForRecording({ - browser: chromium, - launchOptions: req.body, - }, req.user.id); + const id = initializeRemoteBrowserForRecording(req.user.id); + console.log('id start POST:', id); return res.send(id); }); diff --git a/server/src/routes/storage.ts b/server/src/routes/storage.ts index 759e8dec..228f60de 100644 --- a/server/src/routes/storage.ts +++ b/server/src/routes/storage.ts @@ -158,13 +158,7 @@ router.put('/runs/:id', requireSignIn, async (req: AuthenticatedRequest, res) => console.log(`Proxy config for run: ${JSON.stringify(proxyOptions)}`) - const id = createRemoteBrowserForRun({ - browser: chromium, - launchOptions: { - headless: true, - proxy: proxyOptions.server ? proxyOptions : undefined, - } - }, req.user.id); + const id = createRemoteBrowserForRun(req.user.id); const runId = uuid(); diff --git a/server/src/server.ts b/server/src/server.ts index 5c7fc898..e6fee5f2 100644 --- a/server/src/server.ts +++ b/server/src/server.ts @@ -39,8 +39,8 @@ export const io = new Server(server); */ export const browserPool = new BrowserPool(); -app.use(bodyParser.json({ limit: '10mb' })) -app.use(bodyParser.urlencoded({ extended: true, limit: '10mb', parameterLimit: 9000 })); +// app.use(bodyParser.json({ limit: '10mb' })) +// app.use(bodyParser.urlencoded({ extended: true, limit: '10mb', parameterLimit: 9000 })); // parse cookies - "cookie" is true in csrfProtection app.use(cookieParser()) @@ -62,42 +62,60 @@ readdirSync(path.join(__dirname, 'api')).forEach((r) => { } }); -// Check if we're running in production or development const isProduction = process.env.NODE_ENV === 'production'; -const workerPath = path.resolve(__dirname, isProduction ? './worker.js' : '/worker.ts'); +const workerPath = path.resolve(__dirname, isProduction ? './worker.js' : './worker.ts'); -// Fork the worker process -const workerProcess = fork(workerPath, [], { - execArgv: isProduction ? ['--inspect=8081'] : ['--inspect=5859'], -}); - -workerProcess.on('message', (message) => { - console.log(`Message from worker: ${message}`); -}); -workerProcess.on('error', (error) => { - console.error(`Error in worker: ${error}`); -}); -workerProcess.on('exit', (code) => { - console.log(`Worker exited with code: ${code}`); -}); +let workerProcess: any; +if (!isProduction) { + workerProcess = fork(workerPath, [], { + execArgv: ['--inspect=5859'], + }); + workerProcess.on('message', (message: any) => { + console.log(`Message from worker: ${message}`); + }); + workerProcess.on('error', (error: any) => { + console.error(`Error in worker: ${error}`); + }); + workerProcess.on('exit', (code: any) => { + console.log(`Worker exited with code: ${code}`); + }); +} app.get('/', function (req, res) { capture( 'maxun-oss-server-run', { - event: 'server_started', - } + event: 'server_started', + } ); return res.send('Maxun server started 🚀'); }); -server.listen(SERVER_PORT, async () => { - await connectDB(); - await syncDB(); - logger.log('info', `Server listening on port ${SERVER_PORT}`); +// Add CORS headers +app.use((req, res, next) => { + res.header('Access-Control-Allow-Origin', '*'); + res.header('Access-Control-Allow-Methods', 'GET,PUT,POST,DELETE,OPTIONS'); + res.header('Access-Control-Allow-Headers', 'Content-Type, Authorization'); + if (req.method === 'OPTIONS') { + return res.sendStatus(200); + } + next(); +}); + +server.listen(SERVER_PORT, '0.0.0.0', async () => { + try { + await connectDB(); + await syncDB(); + logger.log('info', `Server listening on port ${SERVER_PORT}`); + } catch (error: any) { + logger.log('error', `Failed to connect to the database: ${error.message}`); + process.exit(1); // Exit the process if DB connection fails + } }); process.on('SIGINT', () => { console.log('Main app shutting down...'); - workerProcess.kill(); + if (!isProduction) { + workerProcess.kill(); + } process.exit(); }); diff --git a/server/src/storage/db.ts b/server/src/storage/db.ts index 6a23ef42..2f0fcde4 100644 --- a/server/src/storage/db.ts +++ b/server/src/storage/db.ts @@ -3,10 +3,15 @@ import dotenv from 'dotenv'; import setupAssociations from '../models/associations'; dotenv.config(); -const sequelize = new Sequelize( - `postgresql://${process.env.DB_USER}:${process.env.DB_PASSWORD}@${process.env.DB_HOST}:${process.env.DB_PORT}/${process.env.DB_NAME}`, + +const databaseUrl = `postgresql://${process.env.DB_USER}:${process.env.DB_PASSWORD}@${process.env.DB_HOST}:${process.env.DB_PORT}/${process.env.DB_NAME}`; + +// Extract the hostname using the URL constructor +const host = new URL(databaseUrl).hostname; + +const sequelize = new Sequelize(databaseUrl, { - host: process.env.DB_HOST, + host, dialect: 'postgres', logging: false, } diff --git a/server/src/swagger/config.ts b/server/src/swagger/config.ts index 040db66c..c9c12210 100644 --- a/server/src/swagger/config.ts +++ b/server/src/swagger/config.ts @@ -25,7 +25,7 @@ const options = { }, ], }, - apis: [path.join(__dirname, '../api/*.ts')], + apis: process.env.NODE_ENV === 'production' ? [path.join(__dirname, '../api/*.js')] : [path.join(__dirname, '../api/*.ts')] }; const swaggerSpec = swaggerJSDoc(options); diff --git a/server/src/utils/auth.ts b/server/src/utils/auth.ts index e73a4237..f8313df7 100644 --- a/server/src/utils/auth.ts +++ b/server/src/utils/auth.ts @@ -36,7 +36,7 @@ export const encrypt = (text: string): string => { export const decrypt = (encryptedText: string): string => { const [iv, encrypted] = encryptedText.split(':'); - const algorithm = getEnvVariable('ALGORITHM'); + const algorithm = "aes-256-cbc"; const key = Buffer.from(getEnvVariable('ENCRYPTION_KEY'), 'hex'); const decipher = crypto.createDecipheriv(algorithm, key, Buffer.from(iv, 'hex')); let decrypted = decipher.update(encrypted, 'hex', 'utf8'); diff --git a/server/src/workflow-management/scheduler/index.ts b/server/src/workflow-management/scheduler/index.ts index 6a98217a..cde08a29 100644 --- a/server/src/workflow-management/scheduler/index.ts +++ b/server/src/workflow-management/scheduler/index.ts @@ -40,13 +40,7 @@ async function createWorkflowAndStoreMetadata(id: string, userId: string) { }; } - const browserId = createRemoteBrowserForRun({ - browser: chromium, - launchOptions: { - headless: true, - proxy: proxyOptions.server ? proxyOptions : undefined, - } - }, userId); + const browserId = createRemoteBrowserForRun( userId); const runId = uuid(); const run = await Run.create({ @@ -229,7 +223,7 @@ export async function handleRunRecording(id: string, userId: string) { throw new Error('browserId or runId or userId is undefined'); } - const socket = io(`http://localhost:8080/${browserId}`, { + const socket = io(`${process.env.BACKEND_URL}/${browserId}`, { transports: ['websocket'], rejectUnauthorized: false }); diff --git a/server/start.sh b/server/start.sh new file mode 100644 index 00000000..a959bfd2 --- /dev/null +++ b/server/start.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +# Start Xvfb in the background with the desired dimensions +#Xvfb :0 -screen 0 900x400x24 & + +# Wait for Xvfb to start +#sleep 2 + +# Execute the Node.js application +exec npm run server \ No newline at end of file diff --git a/server/tsconfig.json b/server/tsconfig.json index 820e903e..f8d7c038 100644 --- a/server/tsconfig.json +++ b/server/tsconfig.json @@ -1,6 +1,6 @@ { "compilerOptions": { - "target": "es2018", + "target": "es2020", "module": "commonjs", "outDir": "./dist", "rootDir": "../", @@ -21,12 +21,12 @@ "include": [ "src/**/*", "../src/shared/**/*", - "../src/helpers/**/*" + "../src/helpers/**/*", ], "exclude": [ "node_modules", "../src/components/**/*", // Exclude frontend components "../src/pages/**/*", // Exclude frontend pages - "../src/app/**/*" // Exclude other frontend-specific code + "../src/app/**/*", // Exclude other frontend-specific code ] } diff --git a/src/api/auth.ts b/src/api/auth.ts index 34ebcab8..32a11f69 100644 --- a/src/api/auth.ts +++ b/src/api/auth.ts @@ -1,8 +1,9 @@ import { default as axios } from "axios"; +import { apiUrl } from "../apiConfig" export const getUserById = async (userId: string) => { try { - const response = await axios.get(`http://localhost:8080/auth/user/${userId}`); + const response = await axios.get(`${apiUrl}/auth/user/${userId}`); if (response.status === 200) { return response.data; } else { diff --git a/src/api/integration.ts b/src/api/integration.ts index 2804508b..d484abbb 100644 --- a/src/api/integration.ts +++ b/src/api/integration.ts @@ -1,8 +1,9 @@ import { default as axios } from "axios"; +import { apiUrl } from "../apiConfig"; export const handleUploadCredentials = async (fileName: string, credentials: any, spreadsheetId: string, range: string): Promise => { try { - const response = await axios.post('http://localhost:8080/integration/upload-credentials', { fileName, credentials: JSON.parse(credentials), spreadsheetId, range }); + const response = await axios.post(`${apiUrl}/integration/upload-credentials`, { fileName, credentials: JSON.parse(credentials), spreadsheetId, range }); if (response.status === 200) { return response.data; } else { diff --git a/src/api/proxy.ts b/src/api/proxy.ts index 4ca79742..e08dcfae 100644 --- a/src/api/proxy.ts +++ b/src/api/proxy.ts @@ -1,8 +1,9 @@ import { default as axios } from "axios"; +import { apiUrl } from "../apiConfig"; export const sendProxyConfig = async (proxyConfig: { server_url: string, username?: string, password?: string }): Promise => { try { - const response = await axios.post(`http://localhost:8080/proxy/config`, proxyConfig); + const response = await axios.post(`${apiUrl}/proxy/config`, proxyConfig); if (response.status === 200) { return response.data; } else { @@ -16,7 +17,7 @@ export const sendProxyConfig = async (proxyConfig: { server_url: string, usernam export const getProxyConfig = async (): Promise<{ proxy_url: string, auth: boolean }> => { try { - const response = await axios.get(`http://localhost:8080/proxy/config`); + const response = await axios.get(`${apiUrl}/proxy/config`); if (response.status === 200) { return response.data; } else { @@ -30,7 +31,7 @@ export const getProxyConfig = async (): Promise<{ proxy_url: string, auth: boole export const testProxyConfig = async (): Promise<{ success: boolean }> => { try { - const response = await axios.get(`http://localhost:8080/proxy/test`); + const response = await axios.get(`${apiUrl}/proxy/test`); if (response.status === 200) { return response.data; } else { @@ -44,7 +45,7 @@ export const testProxyConfig = async (): Promise<{ success: boolean }> => { export const deleteProxyConfig = async (): Promise => { try { - const response = await axios.delete(`http://localhost:8080/proxy/config`); + const response = await axios.delete(`${apiUrl}/proxy/config`); if (response.status === 200) { return response.data; } else { diff --git a/src/api/recording.ts b/src/api/recording.ts index ef51a476..6b816001 100644 --- a/src/api/recording.ts +++ b/src/api/recording.ts @@ -1,8 +1,9 @@ import { default as axios, AxiosResponse } from "axios"; +import { apiUrl } from "../apiConfig"; export const startRecording = async() : Promise => { try { - const response = await axios.get('http://localhost:8080/record/start') + const response = await axios.get(`${apiUrl}/record/start`) if (response.status === 200) { return response.data; } else { @@ -14,7 +15,7 @@ export const startRecording = async() : Promise => { }; export const stopRecording = async (id: string): Promise => { - await axios.get(`http://localhost:8080/record/stop/${id}`) + await axios.get(`${apiUrl}/record/stop/${id}`) .then((response : AxiosResponse) => { }) .catch((error: any) => { @@ -23,7 +24,7 @@ export const stopRecording = async (id: string): Promise => { export const getActiveBrowserId = async(): Promise => { try { - const response = await axios.get('http://localhost:8080/record/active'); + const response = await axios.get(`${apiUrl}/record/active`); if (response.status === 200) { return response.data; } else { @@ -36,7 +37,7 @@ export const getActiveBrowserId = async(): Promise => { export const interpretCurrentRecording = async(): Promise => { try { - const response = await axios.get('http://localhost:8080/record/interpret'); + const response = await axios.get(`${apiUrl}/record/interpret`); if (response.status === 200) { return true; } else { @@ -50,7 +51,7 @@ export const interpretCurrentRecording = async(): Promise => { export const stopCurrentInterpretation = async(): Promise => { try { - const response = await axios.get('http://localhost:8080/record/interpret/stop'); + const response = await axios.get(`${apiUrl}/record/interpret/stop`); if (response.status === 200) { return; } else { @@ -63,7 +64,7 @@ export const stopCurrentInterpretation = async(): Promise => { export const getCurrentUrl = async (): Promise => { try { - const response = await axios.get('http://localhost:8080/record/active/url'); + const response = await axios.get(`${apiUrl}/record/active/url`); if (response.status === 200) { return response.data; } else { @@ -77,7 +78,7 @@ export const getCurrentUrl = async (): Promise => { export const getCurrentTabs = async (): Promise => { try { - const response = await axios.get('http://localhost:8080/record/active/tabs'); + const response = await axios.get(`${apiUrl}/record/active/tabs`); if (response.status === 200) { return response.data; } else { diff --git a/src/api/storage.ts b/src/api/storage.ts index da017027..9b4b06b2 100644 --- a/src/api/storage.ts +++ b/src/api/storage.ts @@ -3,10 +3,11 @@ import { WorkflowFile } from "maxun-core"; import { RunSettings } from "../components/molecules/RunSettings"; import { ScheduleSettings } from "../components/molecules/ScheduleSettings"; import { CreateRunResponse, ScheduleRunResponse } from "../pages/MainPage"; +import { apiUrl } from "../apiConfig"; export const getStoredRecordings = async (): Promise => { try { - const response = await axios.get('http://localhost:8080/storage/recordings'); + const response = await axios.get(`${apiUrl}/storage/recordings`); if (response.status === 200) { return response.data; } else { @@ -20,7 +21,7 @@ export const getStoredRecordings = async (): Promise => { export const getStoredRuns = async (): Promise => { try { - const response = await axios.get('http://localhost:8080/storage/runs'); + const response = await axios.get(`${apiUrl}/storage/runs`); if (response.status === 200) { return response.data; } else { @@ -34,7 +35,7 @@ export const getStoredRuns = async (): Promise => { export const getStoredRecording = async (id: string) => { try { - const response = await axios.get(`http://localhost:8080/storage/recordings/${id}`); + const response = await axios.get(`${apiUrl}/storage/recordings/${id}`); if (response.status === 200) { return response.data; } else { @@ -48,7 +49,7 @@ export const getStoredRecording = async (id: string) => { export const deleteRecordingFromStorage = async (id: string): Promise => { try { - const response = await axios.delete(`http://localhost:8080/storage/recordings/${id}`); + const response = await axios.delete(`${apiUrl}/storage/recordings/${id}`); if (response.status === 200) { return response.data; } else { @@ -62,7 +63,7 @@ export const deleteRecordingFromStorage = async (id: string): Promise = export const deleteRunFromStorage = async (id: string): Promise => { try { - const response = await axios.delete(`http://localhost:8080/storage/runs/${id}`); + const response = await axios.delete(`${apiUrl}/storage/runs/${id}`); if (response.status === 200) { return response.data; } else { @@ -76,7 +77,7 @@ export const deleteRunFromStorage = async (id: string): Promise => { export const editRecordingFromStorage = async (browserId: string, id: string): Promise => { try { - const response = await axios.put(`http://localhost:8080/workflow/${browserId}/${id}`); + const response = await axios.put(`${apiUrl}/workflow/${browserId}/${id}`); if (response.status === 200) { return response.data; } else { @@ -91,7 +92,7 @@ export const editRecordingFromStorage = async (browserId: string, id: string): P export const createRunForStoredRecording = async (id: string, settings: RunSettings): Promise => { try { const response = await axios.put( - `http://localhost:8080/storage/runs/${id}`, + `${apiUrl}/storage/runs/${id}`, { ...settings }); if (response.status === 200) { return response.data; @@ -106,7 +107,7 @@ export const createRunForStoredRecording = async (id: string, settings: RunSetti export const interpretStoredRecording = async (id: string): Promise => { try { - const response = await axios.post(`http://localhost:8080/storage/runs/run/${id}`); + const response = await axios.post(`${apiUrl}/storage/runs/run/${id}`); if (response.status === 200) { return response.data; } else { @@ -120,7 +121,7 @@ export const interpretStoredRecording = async (id: string): Promise => export const notifyAboutAbort = async (id: string): Promise => { try { - const response = await axios.post(`http://localhost:8080/storage/runs/abort/${id}`); + const response = await axios.post(`${apiUrl}/storage/runs/abort/${id}`); if (response.status === 200) { return response.data; } else { @@ -135,7 +136,7 @@ export const notifyAboutAbort = async (id: string): Promise => { export const scheduleStoredRecording = async (id: string, settings: ScheduleSettings): Promise => { try { const response = await axios.put( - `http://localhost:8080/storage/schedule/${id}`, + `${apiUrl}/storage/schedule/${id}`, { ...settings }); if (response.status === 200) { return response.data; @@ -150,7 +151,7 @@ export const scheduleStoredRecording = async (id: string, settings: ScheduleSett export const getSchedule = async (id: string) => { try { - const response = await axios.get(`http://localhost:8080/storage/schedule/${id}`); + const response = await axios.get(`${apiUrl}/storage/schedule/${id}`); if (response.status === 200) { return response.data.schedule; } else { @@ -164,7 +165,7 @@ export const getSchedule = async (id: string) => { export const deleteSchedule = async (id: string): Promise => { try { - const response = await axios.delete(`http://localhost:8080/storage/schedule/${id}`); + const response = await axios.delete(`${apiUrl}/storage/schedule/${id}`); if (response.status === 200) { return response.data; } else { diff --git a/src/api/workflow.ts b/src/api/workflow.ts index a32b6cbc..03b677b1 100644 --- a/src/api/workflow.ts +++ b/src/api/workflow.ts @@ -1,10 +1,11 @@ import { WhereWhatPair, WorkflowFile } from "maxun-core"; import { emptyWorkflow } from "../shared/constants"; import { default as axios, AxiosResponse } from "axios"; +import { apiUrl } from "../apiConfig"; export const getActiveWorkflow = async(id: string) : Promise => { try { - const response = await axios.get(`http://localhost:8080/workflow/${id}`) + const response = await axios.get(`${apiUrl}/workflow/${id}`) if (response.status === 200) { return response.data; } else { @@ -18,7 +19,7 @@ export const getActiveWorkflow = async(id: string) : Promise => { export const getParamsOfActiveWorkflow = async(id: string) : Promise => { try { - const response = await axios.get(`http://localhost:8080/workflow/params/${id}`) + const response = await axios.get(`${apiUrl}/workflow/params/${id}`) if (response.status === 200) { return response.data; } else { @@ -32,7 +33,7 @@ export const getParamsOfActiveWorkflow = async(id: string) : Promise => { try { - const response = await axios.delete(`http://localhost:8080/workflow/pair/${index}`); + const response = await axios.delete(`${apiUrl}/workflow/pair/${index}`); if (response.status === 200) { return response.data; } else { @@ -46,7 +47,7 @@ export const deletePair = async(index: number): Promise => { export const AddPair = async(index: number, pair: WhereWhatPair): Promise => { try { - const response = await axios.post(`http://localhost:8080/workflow/pair/${index}`, { + const response = await axios.post(`${apiUrl}/workflow/pair/${index}`, { pair, }, {headers: {'Content-Type': 'application/json'}}); if (response.status === 200) { @@ -62,7 +63,7 @@ export const AddPair = async(index: number, pair: WhereWhatPair): Promise => { try { - const response = await axios.put(`http://localhost:8080/workflow/pair/${index}`, { + const response = await axios.put(`${apiUrl}/workflow/pair/${index}`, { pair, }, {headers: {'Content-Type': 'application/json'}}); if (response.status === 200) { diff --git a/src/apiConfig.js b/src/apiConfig.js new file mode 100644 index 00000000..8661491e --- /dev/null +++ b/src/apiConfig.js @@ -0,0 +1 @@ +export const apiUrl = import.meta.env.VITE_BACKEND_URL; \ No newline at end of file diff --git a/src/components/molecules/IntegrationSettings.tsx b/src/components/molecules/IntegrationSettings.tsx index f47ac5da..1150dd72 100644 --- a/src/components/molecules/IntegrationSettings.tsx +++ b/src/components/molecules/IntegrationSettings.tsx @@ -6,6 +6,7 @@ import TextField from "@mui/material/TextField"; import axios from 'axios'; import { useGlobalInfoStore } from '../../context/globalInfo'; import { getStoredRecording } from '../../api/storage'; +import { apiUrl } from '../../apiConfig.js'; interface IntegrationProps { isOpen: boolean; handleStart: (data: IntegrationSettings) => void; @@ -32,12 +33,12 @@ export const IntegrationSettingsModal = ({ isOpen, handleStart, handleClose }: I const [recording, setRecording] = useState(null); const authenticateWithGoogle = () => { - window.location.href = `http://localhost:8080/auth/google?robotId=${recordingId}`; + window.location.href = `${apiUrl}/auth/google?robotId=${recordingId}`; }; const handleOAuthCallback = async () => { try { - const response = await axios.get(`http://localhost:8080/auth/google/callback`); + const response = await axios.get(`${apiUrl}/auth/google/callback`); const { google_sheet_email, files } = response.data; } catch (error) { setError('Error authenticating with Google'); @@ -46,7 +47,7 @@ export const IntegrationSettingsModal = ({ isOpen, handleStart, handleClose }: I const fetchSpreadsheetFiles = async () => { try { - const response = await axios.get(`http://localhost:8080/auth/gsheets/files?robotId=${recordingId}`, { + const response = await axios.get(`${apiUrl}/auth/gsheets/files?robotId=${recordingId}`, { withCredentials: true, }); setSpreadsheets(response.data); @@ -66,7 +67,7 @@ export const IntegrationSettingsModal = ({ isOpen, handleStart, handleClose }: I const updateGoogleSheetId = async () => { try { const response = await axios.post( - `http://localhost:8080/auth/gsheets/update`, + `${apiUrl}/auth/gsheets/update`, { spreadsheetId: settings.spreadsheetId, spreadsheetName: settings.spreadsheetName, robotId: recordingId }, { withCredentials: true } ); @@ -79,7 +80,7 @@ export const IntegrationSettingsModal = ({ isOpen, handleStart, handleClose }: I const removeIntegration = async () => { try { await axios.post( - `http://localhost:8080/auth/gsheets/remove`, + `${apiUrl}/auth/gsheets/remove`, { robotId: recordingId }, { withCredentials: true } ); diff --git a/src/components/molecules/NavBar.tsx b/src/components/molecules/NavBar.tsx index b0a409b1..8dd678c5 100644 --- a/src/components/molecules/NavBar.tsx +++ b/src/components/molecules/NavBar.tsx @@ -9,6 +9,7 @@ import { useNavigate } from 'react-router-dom'; import { AuthContext } from '../../context/auth'; import { SaveRecording } from '../molecules/SaveRecording'; import DiscordIcon from '../atoms/DiscordIcon'; +import { apiUrl } from '../../apiConfig'; interface NavBarProps { recordingName: string; @@ -34,7 +35,7 @@ export const NavBar: React.FC = ({ recordingName, isRecording }) => const logout = async () => { dispatch({ type: 'LOGOUT' }); window.localStorage.removeItem('user'); - const { data } = await axios.get('http://localhost:8080/auth/logout'); + const { data } = await axios.get(`${apiUrl}/auth/logout`); notify('success', data.message); navigate('/login'); }; diff --git a/src/components/organisms/ApiKey.tsx b/src/components/organisms/ApiKey.tsx index 7999296b..d9f4c6fe 100644 --- a/src/components/organisms/ApiKey.tsx +++ b/src/components/organisms/ApiKey.tsx @@ -18,6 +18,7 @@ import { ContentCopy, Visibility, Delete } from '@mui/icons-material'; import styled from 'styled-components'; import axios from 'axios'; import { useGlobalInfoStore } from '../../context/globalInfo'; +import { apiUrl } from '../../apiConfig'; const Container = styled(Box)` display: flex; @@ -38,7 +39,7 @@ const ApiKeyManager = () => { useEffect(() => { const fetchApiKey = async () => { try { - const { data } = await axios.get('http://localhost:8080/auth/api-key'); + const { data } = await axios.get(`${apiUrl}/auth/api-key`); setApiKey(data.api_key); } catch (error: any) { notify('error', `Failed to fetch API Key - ${error.message}`); @@ -53,7 +54,7 @@ const ApiKeyManager = () => { const generateApiKey = async () => { setLoading(true); try { - const { data } = await axios.post('http://localhost:8080/auth/generate-api-key'); + const { data } = await axios.post(`${apiUrl}/auth/generate-api-key`); setApiKey(data.api_key); notify('success', `Generated API Key successfully`); } catch (error: any) { @@ -66,7 +67,7 @@ const ApiKeyManager = () => { const deleteApiKey = async () => { setLoading(true); try { - await axios.delete('http://localhost:8080/auth/delete-api-key'); + await axios.delete(`${apiUrl}/auth/delete-api-key`); setApiKey(null); notify('success', 'API Key deleted successfully'); } catch (error: any) { diff --git a/src/components/organisms/BrowserWindow.tsx b/src/components/organisms/BrowserWindow.tsx index 3cfc2b3d..cbc46731 100644 --- a/src/components/organisms/BrowserWindow.tsx +++ b/src/components/organisms/BrowserWindow.tsx @@ -2,7 +2,6 @@ import React, { useCallback, useEffect, useState } from 'react'; import { useSocketStore } from '../../context/socket'; import { Button } from '@mui/material'; import Canvas from "../atoms/canvas"; -import { useBrowserDimensionsStore } from "../../context/browserDimensions"; import { Highlighter } from "../atoms/Highlighter"; import { GenericModal } from '../atoms/GenericModal'; import { useActionContext } from '../../context/browserActions'; @@ -66,7 +65,6 @@ export const BrowserWindow = () => { const { socket } = useSocketStore(); const { notify } = useGlobalInfoStore(); - //const { width, height } = useBrowserDimensionsStore(); const { getText, getList, paginationMode, paginationType, limitMode } = useActionContext(); const { addTextStep, addListStep } = useBrowserSteps(); @@ -405,4 +403,4 @@ const modalStyle = { height: 'fit-content', display: 'block', padding: '20px', -}; \ No newline at end of file +}; diff --git a/src/components/organisms/MainMenu.tsx b/src/components/organisms/MainMenu.tsx index 12df9350..edb6ed29 100644 --- a/src/components/organisms/MainMenu.tsx +++ b/src/components/organisms/MainMenu.tsx @@ -4,6 +4,7 @@ import Tab from '@mui/material/Tab'; import Box from '@mui/material/Box'; import { Paper, Button } from "@mui/material"; import { AutoAwesome, FormatListBulleted, VpnKey, Usb, Article, Link, CloudQueue } from "@mui/icons-material"; +import { apiUrl } from "../../apiConfig"; interface MainMenuProps { value: string; @@ -86,7 +87,7 @@ export const MainMenu = ({ value = 'recordings', handleChangeContent }: MainMenu
-