vodorod/parcer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'develop' into integration_airtable

Browse Source
This commit is contained in:
Amit Chauhan
2025-01-30 15:59:40 +05:30
committed by GitHub
parent 6a854f9531 98ba9623c5
commit 5ee8a4adb0
44 changed files with 1496 additions and 704 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
server/Dockerfile
View File

@@ -25,7 +25,7 @@ RUN mkdir -p /tmp/chromium-data-dir && \
# Install dependencies
RUN apt-get update && apt-get install -y \
libgbm-dev \
libgbm1 \
libnss3 \
libatk1.0-0 \
libatk-bridge2.0-0 \
@@ -44,14 +44,8 @@ RUN apt-get update && apt-get install -y \
&& rm -rf /var/lib/apt/lists/* \
&& mkdir -p /tmp/.X11-unix && chmod 1777 /tmp/.X11-unix
# Add a dbus configuration to prevent connection errors
# RUN mkdir -p /var/run/dbus
# Make the script executable
# RUN chmod +x ./start.sh
# Expose the backend port
EXPOSE ${BACKEND_PORT:-8080}
# Start the backend using the start script
CMD ["npm", "run", "server"]
CMD ["npm", "run", "server"]

65
server/src/routes/auth.ts
View File

@@ -119,12 +119,13 @@ router.get("/logout", async (req, res) => {
router.get(
"/current-user",
requireSignIn,
async (req: AuthenticatedRequest, res) => {
async (req: Request, res) => {
const authenticatedReq = req as AuthenticatedRequest;
try {
if (!req.user) {
if (!authenticatedReq.user) {
return res.status(401).json({ ok: false, error: "Unauthorized" });
}
const user = await User.findByPk(req.user.id, {
const user = await User.findByPk(authenticatedReq.user.id, {
attributes: { exclude: ["password"] },
});
if (!user) {
@@ -147,7 +148,7 @@ router.get(
router.get(
"/user/:id",
requireSignIn,
async (req: AuthenticatedRequest, res) => {
async (req: Request, res) => {
try {
const { id } = req.params;
if (!id) {
@@ -176,12 +177,13 @@ router.get(
router.post(
"/generate-api-key",
requireSignIn,
async (req: AuthenticatedRequest, res) => {
async (req: Request, res) => {
const authenticatedReq = req as AuthenticatedRequest;
try {
if (!req.user) {
if (!authenticatedReq.user) {
return res.status(401).json({ ok: false, error: "Unauthorized" });
}
const user = await User.findByPk(req.user.id, {
const user = await User.findByPk(authenticatedReq.user.id, {
attributes: { exclude: ["password"] },
});
@@ -216,13 +218,14 @@ router.post(
router.get(
"/api-key",
requireSignIn,
async (req: AuthenticatedRequest, res) => {
async (req: Request, res) => {
const authenticatedReq = req as AuthenticatedRequest;
try {
if (!req.user) {
if (!authenticatedReq.user) {
return res.status(401).json({ ok: false, error: "Unauthorized" });
}
const user = await User.findByPk(req.user.id, {
const user = await User.findByPk(authenticatedReq.user.id, {
raw: true,
attributes: ["api_key"],
});
@@ -244,13 +247,14 @@ router.get(
router.delete(
"/delete-api-key",
requireSignIn,
async (req: AuthenticatedRequest, res) => {
if (!req.user) {
async (req: Request, res) => {
const authenticatedReq = req as AuthenticatedRequest;
if (!authenticatedReq.user) {
return res.status(401).send({ error: "Unauthorized" });
}
try {
const user = await User.findByPk(req.user.id, { raw: true });
const user = await User.findByPk(authenticatedReq.user.id, { raw: true });
if (!user) {
return res.status(404).json({ message: "User not found" });
@@ -260,7 +264,7 @@ router.delete(
return res.status(404).json({ message: "API Key not found" });
}
await User.update({ api_key: null }, { where: { id: req.user.id } });
await User.update({ api_key: null }, { where: { id: authenticatedReq.user.id } });
capture("maxun-oss-api-key-deleted", {
user_id: user.id,
@@ -306,7 +310,8 @@ router.get("/google", (req, res) => {
router.get(
"/google/callback",
requireSignIn,
async (req: AuthenticatedRequest, res) => {
async (req: Request, res) => {
const authenticatedReq = req as AuthenticatedRequest;
const { code, state } = req.query;
try {
if (!state) {
@@ -332,12 +337,12 @@ router.get(
return res.status(400).json({ message: "Email not found" });
}
if (!req.user) {
if (!authenticatedReq.user) {
return res.status(401).send({ error: "Unauthorized" });
}
// Get the currently authenticated user (from `requireSignIn`)
let user = await User.findOne({ where: { id: req.user.id } });
let user = await User.findOne({ where: { id: authenticatedReq.user.id } });
if (!user) {
return res.status(400).json({ message: "User not found" });
@@ -392,11 +397,19 @@ router.get(
httpOnly: false,
maxAge: 60000,
}); // 1-minute expiration
res.cookie("robot_auth_message", "Robot successfully authenticated", {
// res.cookie("robot_auth_message", "Robot successfully authenticated", {
// httpOnly: false,
// maxAge: 60000,
// });
res.cookie('robot_auth_robotId', robotId, {
httpOnly: false,
maxAge: 60000,
});
res.redirect(`${process.env.PUBLIC_URL}/robots/${robotId}/integrate` as string || `http://localhost:5173/robots/${robotId}/integrate`);
const baseUrl = process.env.PUBLIC_URL || "http://localhost:5173";
const redirectUrl = `${baseUrl}/robots/`;
res.redirect(redirectUrl);
} catch (error: any) {
res.status(500).json({ message: `Google OAuth error: ${error.message}` });
}
@@ -407,12 +420,13 @@ router.get(
router.post(
"/gsheets/data",
requireSignIn,
async (req: AuthenticatedRequest, res) => {
async (req: Request, res) => {
const authenticatedReq = req as AuthenticatedRequest;
const { spreadsheetId, robotId } = req.body;
if (!req.user) {
if (!authenticatedReq.user) {
return res.status(401).send({ error: "Unauthorized" });
}
const user = await User.findByPk(req.user.id, { raw: true });
const user = await User.findByPk(authenticatedReq.user.id, { raw: true });
if (!user) {
return res.status(400).json({ message: "User not found" });
@@ -524,13 +538,14 @@ router.post("/gsheets/update", requireSignIn, async (req, res) => {
router.post(
"/gsheets/remove",
requireSignIn,
async (req: AuthenticatedRequest, res) => {
async (req: Request, res) => {
const authenticatedReq = req as AuthenticatedRequest;
const { robotId } = req.body;
if (!robotId) {
return res.status(400).json({ message: "Robot ID is required" });
}
if (!req.user) {
if (!authenticatedReq.user) {
return res.status(401).send({ error: "Unauthorized" });
}
@@ -552,7 +567,7 @@ router.post(
});
capture("maxun-oss-google-sheet-integration-removed", {
user_id: req.user.id,
user_id: authenticatedReq.user.id,
robot_id: robotId,
deleted_at: new Date().toISOString(),
});

28
server/src/routes/proxy.ts
View File

@@ -12,16 +12,17 @@ interface AuthenticatedRequest extends Request {
user?: { id: string };
}
router.post('/config', requireSignIn, async (req: AuthenticatedRequest, res: Response) => {
router.post('/config', requireSignIn, async (req: Request, res: Response) => {
const { server_url, username, password } = req.body;
const authenticatedReq = req as AuthenticatedRequest;
try {
if (!req.user) {
if (!authenticatedReq.user) {
return res.status(401).json({ ok: false, error: 'Unauthorized' });
}
const user = await User.findByPk(req.user.id, {
const user = await User.findByPk(authenticatedReq.user.id, {
attributes: { exclude: ['password'] },
});
@@ -57,13 +58,14 @@ router.post('/config', requireSignIn, async (req: AuthenticatedRequest, res: Res
}
});
router.get('/test', requireSignIn, async (req: AuthenticatedRequest, res: Response) => {
router.get('/test', requireSignIn, async (req: Request, res: Response) => {
const authenticatedReq = req as AuthenticatedRequest;
try {
if (!req.user) {
if (!authenticatedReq.user) {
return res.status(401).json({ ok: false, error: 'Unauthorized' });
}
const user = await User.findByPk(req.user.id, {
const user = await User.findByPk(authenticatedReq.user.id, {
attributes: ['proxy_url', 'proxy_username', 'proxy_password'],
raw: true
});
@@ -98,13 +100,14 @@ router.get('/test', requireSignIn, async (req: AuthenticatedRequest, res: Respon
}
});
router.get('/config', requireSignIn, async (req: AuthenticatedRequest, res: Response) => {
router.get('/config', requireSignIn, async (req: Request, res: Response) => {
const authenticatedReq = req as AuthenticatedRequest;
try {
if (!req.user) {
if (!authenticatedReq.user) {
return res.status(401).json({ ok: false, error: 'Unauthorized' });
}
const user = await User.findByPk(req.user.id, {
const user = await User.findByPk(authenticatedReq.user.id, {
attributes: ['proxy_url', 'proxy_username', 'proxy_password'],
raw: true,
});
@@ -125,12 +128,13 @@ router.get('/config', requireSignIn, async (req: AuthenticatedRequest, res: Resp
}
});
router.delete('/config', requireSignIn, async (req: AuthenticatedRequest, res: Response) => {
if (!req.user) {
router.delete('/config', requireSignIn, async (req: Request, res: Response) => {
const authenticatedReq = req as AuthenticatedRequest;
if (!authenticatedReq.user) {
return res.status(401).json({ ok: false, error: 'Unauthorized' });
}
const user = await User.findByPk(req.user.id);
const user = await User.findByPk(authenticatedReq.user.id);
if (!user) {
return res.status(404).json({ message: 'User not found' });

116
server/src/routes/storage.ts
View File

@@ -18,6 +18,7 @@ import { AuthenticatedRequest } from './record';
import { computeNextRun } from '../utils/schedule';
import { capture } from "../utils/analytics";
import { tryCatch } from 'bullmq';
import { encrypt, decrypt } from '../utils/auth';
import { WorkflowFile } from 'maxun-core';
import { Page } from 'playwright';
import { airtableUpdateTasks, processAirtableUpdates } from '../workflow-management/integrations/airtable';
@@ -25,6 +26,36 @@ chromium.use(stealthPlugin());
export const router = Router();
export const decryptWorkflowActions = async (workflow: any[],): Promise<any[]> => {
// Create a deep copy to avoid mutating the original workflow
const processedWorkflow = JSON.parse(JSON.stringify(workflow));
// Process each step in the workflow
for (const step of processedWorkflow) {
if (!step.what) continue;
// Process each action in the step
for (const action of step.what) {
// Only process type and press actions
if ((action.action === 'type' || action.action === 'press') && Array.isArray(action.args) && action.args.length > 1) {
// The second argument contains the encrypted value
const encryptedValue = action.args[1];
if (typeof encryptedValue === 'string') {
try {
// Decrypt the value and update the args array
action.args[1] = await decrypt(encryptedValue);
} catch (error) {
console.error('Failed to decrypt value:', error);
// Keep the encrypted value if decryption fails
}
}
}
}
}
return processedWorkflow;
};
/**
* Logs information about recordings API.
*/
@@ -56,6 +87,13 @@ router.get('/recordings/:id', requireSignIn, async (req, res) => {
raw: true
}
);
if (data?.recording?.workflow) {
data.recording.workflow = await decryptWorkflowActions(
data.recording.workflow,
);
}
return res.send(data);
} catch (e) {
logger.log('info', 'Error while reading robots');
@@ -117,13 +155,74 @@ function formatRunResponse(run: any) {
return formattedRun;
}
interface CredentialInfo {
value: string;
type: string;
}
interface Credentials {
[key: string]: CredentialInfo;
}
function updateTypeActionsInWorkflow(workflow: any[], credentials: Credentials) {
return workflow.map(step => {
if (!step.what) return step;
const indicesToRemove = new Set<number>();
step.what.forEach((action: any, index: number) => {
if (!action.action || !action.args?.[0]) return;
if ((action.action === 'type' || action.action === 'press') && credentials[action.args[0]]) {
indicesToRemove.add(index);
if (step.what[index + 1]?.action === 'waitForLoadState') {
indicesToRemove.add(index + 1);
}
}
});
const filteredWhat = step.what.filter((_: any, index: number) => !indicesToRemove.has(index));
Object.entries(credentials).forEach(([selector, credentialInfo]) => {
const clickIndex = filteredWhat.findIndex((action: any) =>
action.action === 'click' && action.args?.[0] === selector
);
if (clickIndex !== -1) {
const chars = credentialInfo.value.split('');
chars.forEach((char, i) => {
filteredWhat.splice(clickIndex + 1 + (i * 2), 0, {
action: 'type',
args: [
selector,
encrypt(char),
credentialInfo.type
]
});
filteredWhat.splice(clickIndex + 2 + (i * 2), 0, {
action: 'waitForLoadState',
args: ['networkidle']
});
});
}
});
return {
...step,
what: filteredWhat
};
});
}
/**
* PUT endpoint to update the name and limit of a robot.
*/
router.put('/recordings/:id', requireSignIn, async (req: AuthenticatedRequest, res) => {
try {
const { id } = req.params;
const { name, limit } = req.body;
const { name, limit, credentials } = req.body;
// Validate input
if (!name && limit === undefined) {
@@ -142,17 +241,21 @@ router.put('/recordings/:id', requireSignIn, async (req: AuthenticatedRequest, r
robot.set('recording_meta', { ...robot.recording_meta, name });
}
let workflow = [...robot.recording.workflow]; // Create a copy of the workflow
if (credentials) {
workflow = updateTypeActionsInWorkflow(workflow, credentials);
}
// Update the limit
if (limit !== undefined) {
const workflow = [...robot.recording.workflow]; // Create a copy of the workflow
// Ensure the workflow structure is valid before updating
if (
workflow.length > 0 &&
workflow[0]?.what?.[0]
) {
// Create a new workflow object with the updated limit
const updatedWorkflow = workflow.map((step, index) => {
workflow = workflow.map((step, index) => {
if (index === 0) { // Assuming you want to update the first step
return {
...step,
@@ -174,14 +277,13 @@ router.put('/recordings/:id', requireSignIn, async (req: AuthenticatedRequest, r
}
return step;
});
// Replace the workflow in the recording object
robot.set('recording', { ...robot.recording, workflow: updatedWorkflow });
} else {
return res.status(400).json({ error: 'Invalid workflow structure for updating limit.' });
}
}
robot.set('recording', { ...robot.recording, workflow });
await robot.save();
const updatedRobot = await Robot.findOne({ where: { 'recording_meta.id': id } });

21
server/src/server.ts
View File

@@ -18,8 +18,12 @@ import { fork } from 'child_process';
import { capture } from "./utils/analytics";
import swaggerUi from 'swagger-ui-express';
import swaggerSpec from './swagger/config';
import session from 'express-session';
import Run from './models/Run';
const app = express();
app.use(cors({
origin: process.env.PUBLIC_URL ? process.env.PUBLIC_URL : 'http://localhost:5173',
@@ -124,8 +128,23 @@ server.listen(SERVER_PORT, '0.0.0.0', async () => {
}
});
process.on('SIGINT', () => {
process.on('SIGINT', async () => {
console.log('Main app shutting down...');
try {
await Run.update(
{
status: 'failed',
finishedAt: new Date().toLocaleString(),
log: 'Process interrupted during execution - worker shutdown'
},
{
where: { status: 'running' }
}
);
} catch (error: any) {
console.error('Error updating runs:', error);
}
if (!isProduction) {
workerProcess.kill();
}

8
server/src/storage/db.ts
View File

@@ -29,7 +29,13 @@ export const connectDB = async () => {
export const syncDB = async () => {
try {
//setupAssociations();
await sequelize.sync({ force: false }); // force: true will drop and recreate tables on every run
const isDevelopment = process.env.NODE_ENV === 'development';
// force: true will drop and recreate tables on every run
// Use `alter: true` only in development mode
await sequelize.sync({
force: false,
alter: isDevelopment
});
console.log('Database synced successfully!');
} catch (error) {
console.error('Failed to sync database:', error);

10
server/src/worker.ts
View File

@@ -67,9 +67,11 @@ async function jobCounts() {
jobCounts();
process.on('SIGINT', () => {
console.log('Worker shutting down...');
process.exit();
});
// We dont need this right now
// process.on('SIGINT', () => {
// console.log('Worker shutting down...');
// process.exit();
// });
export { workflowQueue, worker };

57
server/src/workflow-management/classes/Generator.ts
View File

@@ -39,6 +39,7 @@ interface MetaData {
pairs: number;
updatedAt: string;
params: string[],
isLogin?: boolean;
}
/**
@@ -97,6 +98,7 @@ export class WorkflowGenerator {
pairs: 0,
updatedAt: '',
params: [],
isLogin: false,
}
/**
@@ -134,9 +136,9 @@ export class WorkflowGenerator {
*/
private registerEventHandlers = (socket: Socket) => {
socket.on('save', (data) => {
const { fileName, userId } = data;
const { fileName, userId, isLogin } = data;
logger.log('debug', `Saving workflow ${fileName} for user ID ${userId}`);
this.saveNewWorkflow(fileName, userId);
this.saveNewWorkflow(fileName, userId, isLogin);
});
socket.on('new-recording', () => this.workflowRecord = {
workflow: [],
@@ -425,6 +427,40 @@ export class WorkflowGenerator {
return;
}
if ((elementInfo?.tagName === 'INPUT' || elementInfo?.tagName === 'TEXTAREA') && selector) {
// Calculate the exact position within the element
const elementPos = await page.evaluate((selector) => {
const element = document.querySelector(selector);
if (!element) return null;
const rect = element.getBoundingClientRect();
return {
x: rect.left,
y: rect.top
};
}, selector);
if (elementPos) {
const relativeX = coordinates.x - elementPos.x;
const relativeY = coordinates.y - elementPos.y;
const pair: WhereWhatPair = {
where,
what: [{
action: 'click',
args: [selector, { position: { x: relativeX, y: relativeY } }]
}]
};
if (selector) {
this.generatedData.lastUsedSelector = selector;
this.generatedData.lastAction = 'click';
}
await this.addPairToWorkflowAndNotifyClient(pair, page);
return;
}
}
//const element = await getElementMouseIsOver(page, coordinates);
//logger.log('debug', `Element: ${JSON.stringify(element, null, 2)}`);
if (selector) {
@@ -474,6 +510,10 @@ export class WorkflowGenerator {
public onKeyboardInput = async (key: string, coordinates: Coordinates, page: Page) => {
let where: WhereWhatPair["where"] = { url: this.getBestUrl(page.url()) };
const selector = await this.generateSelector(page, coordinates, ActionType.Keydown);
const elementInfo = await getElementInformation(page, coordinates, '', false);
const inputType = elementInfo?.attributes?.type || "text";
if (selector) {
where.selectors = [selector];
}
@@ -481,7 +521,7 @@ export class WorkflowGenerator {
where,
what: [{
action: 'press',
args: [selector, encrypt(key)],
args: [selector, encrypt(key), inputType],
}],
}
if (selector) {
@@ -660,7 +700,7 @@ export class WorkflowGenerator {
* @param fileName The name of the file.
* @returns {Promise<void>}
*/
public saveNewWorkflow = async (fileName: string, userId: number) => {
public saveNewWorkflow = async (fileName: string, userId: number, isLogin: boolean) => {
const recording = this.optimizeWorkflow(this.workflowRecord);
try {
this.recordingMeta = {
@@ -670,6 +710,7 @@ export class WorkflowGenerator {
pairs: recording.workflow.length,
updatedAt: new Date().toLocaleString(),
params: this.getParams() || [],
isLogin: isLogin,
}
const robot = await Robot.create({
userId,
@@ -991,6 +1032,7 @@ export class WorkflowGenerator {
let input = {
selector: '',
value: '',
type: '',
actionCounter: 0,
};
@@ -1005,7 +1047,7 @@ export class WorkflowGenerator {
// when more than one press action is present, add a type action
pair.what.splice(index - input.actionCounter, input.actionCounter, {
action: 'type',
args: [input.selector, encrypt(input.value)],
args: [input.selector, encrypt(input.value), input.type],
}, {
action: 'waitForLoadState',
args: ['networkidle'],
@@ -1033,13 +1075,14 @@ export class WorkflowGenerator {
action: 'waitForLoadState',
args: ['networkidle'],
})
input = { selector: '', value: '', actionCounter: 0 };
input = { selector: '', value: '', type: '', actionCounter: 0 };
}
} else {
pushTheOptimizedAction(pair, index);
input = {
selector: condition.args[0],
value: condition.args[1],
type: condition.args[2],
actionCounter: 1,
};
}
@@ -1048,7 +1091,7 @@ export class WorkflowGenerator {
if (input.value.length !== 0) {
pushTheOptimizedAction(pair, index);
// clear the input
input = { selector: '', value: '', actionCounter: 0 };
input = { selector: '', value: '', type: '', actionCounter: 0 };
}
}
});