vodorod/parcer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'develop' into integration_airtable

Browse Source
This commit is contained in:
Amit Chauhan
2025-01-30 15:59:40 +05:30
committed by GitHub
parent 6a854f9531 98ba9623c5
commit 5ee8a4adb0
44 changed files with 1496 additions and 704 deletions
Download Patch File Download Diff File Expand all files Collapse all files

65
server/src/routes/auth.ts
View File

@@ -119,12 +119,13 @@ router.get("/logout", async (req, res) => {
router.get(
"/current-user",
requireSignIn,
async (req: AuthenticatedRequest, res) => {
async (req: Request, res) => {
const authenticatedReq = req as AuthenticatedRequest;
try {
if (!req.user) {
if (!authenticatedReq.user) {
return res.status(401).json({ ok: false, error: "Unauthorized" });
}
const user = await User.findByPk(req.user.id, {
const user = await User.findByPk(authenticatedReq.user.id, {
attributes: { exclude: ["password"] },
});
if (!user) {
@@ -147,7 +148,7 @@ router.get(
router.get(
"/user/:id",
requireSignIn,
async (req: AuthenticatedRequest, res) => {
async (req: Request, res) => {
try {
const { id } = req.params;
if (!id) {
@@ -176,12 +177,13 @@ router.get(
router.post(
"/generate-api-key",
requireSignIn,
async (req: AuthenticatedRequest, res) => {
async (req: Request, res) => {
const authenticatedReq = req as AuthenticatedRequest;
try {
if (!req.user) {
if (!authenticatedReq.user) {
return res.status(401).json({ ok: false, error: "Unauthorized" });
}
const user = await User.findByPk(req.user.id, {
const user = await User.findByPk(authenticatedReq.user.id, {
attributes: { exclude: ["password"] },
});
@@ -216,13 +218,14 @@ router.post(
router.get(
"/api-key",
requireSignIn,
async (req: AuthenticatedRequest, res) => {
async (req: Request, res) => {
const authenticatedReq = req as AuthenticatedRequest;
try {
if (!req.user) {
if (!authenticatedReq.user) {
return res.status(401).json({ ok: false, error: "Unauthorized" });
}
const user = await User.findByPk(req.user.id, {
const user = await User.findByPk(authenticatedReq.user.id, {
raw: true,
attributes: ["api_key"],
});
@@ -244,13 +247,14 @@ router.get(
router.delete(
"/delete-api-key",
requireSignIn,
async (req: AuthenticatedRequest, res) => {
if (!req.user) {
async (req: Request, res) => {
const authenticatedReq = req as AuthenticatedRequest;
if (!authenticatedReq.user) {
return res.status(401).send({ error: "Unauthorized" });
}
try {
const user = await User.findByPk(req.user.id, { raw: true });
const user = await User.findByPk(authenticatedReq.user.id, { raw: true });
if (!user) {
return res.status(404).json({ message: "User not found" });
@@ -260,7 +264,7 @@ router.delete(
return res.status(404).json({ message: "API Key not found" });
}
await User.update({ api_key: null }, { where: { id: req.user.id } });
await User.update({ api_key: null }, { where: { id: authenticatedReq.user.id } });
capture("maxun-oss-api-key-deleted", {
user_id: user.id,
@@ -306,7 +310,8 @@ router.get("/google", (req, res) => {
router.get(
"/google/callback",
requireSignIn,
async (req: AuthenticatedRequest, res) => {
async (req: Request, res) => {
const authenticatedReq = req as AuthenticatedRequest;
const { code, state } = req.query;
try {
if (!state) {
@@ -332,12 +337,12 @@ router.get(
return res.status(400).json({ message: "Email not found" });
}
if (!req.user) {
if (!authenticatedReq.user) {
return res.status(401).send({ error: "Unauthorized" });
}
// Get the currently authenticated user (from `requireSignIn`)
let user = await User.findOne({ where: { id: req.user.id } });
let user = await User.findOne({ where: { id: authenticatedReq.user.id } });
if (!user) {
return res.status(400).json({ message: "User not found" });
@@ -392,11 +397,19 @@ router.get(
httpOnly: false,
maxAge: 60000,
}); // 1-minute expiration
res.cookie("robot_auth_message", "Robot successfully authenticated", {
// res.cookie("robot_auth_message", "Robot successfully authenticated", {
// httpOnly: false,
// maxAge: 60000,
// });
res.cookie('robot_auth_robotId', robotId, {
httpOnly: false,
maxAge: 60000,
});
res.redirect(`${process.env.PUBLIC_URL}/robots/${robotId}/integrate` as string || `http://localhost:5173/robots/${robotId}/integrate`);
const baseUrl = process.env.PUBLIC_URL || "http://localhost:5173";
const redirectUrl = `${baseUrl}/robots/`;
res.redirect(redirectUrl);
} catch (error: any) {
res.status(500).json({ message: `Google OAuth error: ${error.message}` });
}
@@ -407,12 +420,13 @@ router.get(
router.post(
"/gsheets/data",
requireSignIn,
async (req: AuthenticatedRequest, res) => {
async (req: Request, res) => {
const authenticatedReq = req as AuthenticatedRequest;
const { spreadsheetId, robotId } = req.body;
if (!req.user) {
if (!authenticatedReq.user) {
return res.status(401).send({ error: "Unauthorized" });
}
const user = await User.findByPk(req.user.id, { raw: true });
const user = await User.findByPk(authenticatedReq.user.id, { raw: true });
if (!user) {
return res.status(400).json({ message: "User not found" });
@@ -524,13 +538,14 @@ router.post("/gsheets/update", requireSignIn, async (req, res) => {
router.post(
"/gsheets/remove",
requireSignIn,
async (req: AuthenticatedRequest, res) => {
async (req: Request, res) => {
const authenticatedReq = req as AuthenticatedRequest;
const { robotId } = req.body;
if (!robotId) {
return res.status(400).json({ message: "Robot ID is required" });
}
if (!req.user) {
if (!authenticatedReq.user) {
return res.status(401).send({ error: "Unauthorized" });
}
@@ -552,7 +567,7 @@ router.post(
});
capture("maxun-oss-google-sheet-integration-removed", {
user_id: req.user.id,
user_id: authenticatedReq.user.id,
robot_id: robotId,
deleted_at: new Date().toISOString(),
});

28
server/src/routes/proxy.ts
View File

@@ -12,16 +12,17 @@ interface AuthenticatedRequest extends Request {
user?: { id: string };
}
router.post('/config', requireSignIn, async (req: AuthenticatedRequest, res: Response) => {
router.post('/config', requireSignIn, async (req: Request, res: Response) => {
const { server_url, username, password } = req.body;
const authenticatedReq = req as AuthenticatedRequest;
try {
if (!req.user) {
if (!authenticatedReq.user) {
return res.status(401).json({ ok: false, error: 'Unauthorized' });
}
const user = await User.findByPk(req.user.id, {
const user = await User.findByPk(authenticatedReq.user.id, {
attributes: { exclude: ['password'] },
});
@@ -57,13 +58,14 @@ router.post('/config', requireSignIn, async (req: AuthenticatedRequest, res: Res
}
});
router.get('/test', requireSignIn, async (req: AuthenticatedRequest, res: Response) => {
router.get('/test', requireSignIn, async (req: Request, res: Response) => {
const authenticatedReq = req as AuthenticatedRequest;
try {
if (!req.user) {
if (!authenticatedReq.user) {
return res.status(401).json({ ok: false, error: 'Unauthorized' });
}
const user = await User.findByPk(req.user.id, {
const user = await User.findByPk(authenticatedReq.user.id, {
attributes: ['proxy_url', 'proxy_username', 'proxy_password'],
raw: true
});
@@ -98,13 +100,14 @@ router.get('/test', requireSignIn, async (req: AuthenticatedRequest, res: Respon
}
});
router.get('/config', requireSignIn, async (req: AuthenticatedRequest, res: Response) => {
router.get('/config', requireSignIn, async (req: Request, res: Response) => {
const authenticatedReq = req as AuthenticatedRequest;
try {
if (!req.user) {
if (!authenticatedReq.user) {
return res.status(401).json({ ok: false, error: 'Unauthorized' });
}
const user = await User.findByPk(req.user.id, {
const user = await User.findByPk(authenticatedReq.user.id, {
attributes: ['proxy_url', 'proxy_username', 'proxy_password'],
raw: true,
});
@@ -125,12 +128,13 @@ router.get('/config', requireSignIn, async (req: AuthenticatedRequest, res: Resp
}
});
router.delete('/config', requireSignIn, async (req: AuthenticatedRequest, res: Response) => {
if (!req.user) {
router.delete('/config', requireSignIn, async (req: Request, res: Response) => {
const authenticatedReq = req as AuthenticatedRequest;
if (!authenticatedReq.user) {
return res.status(401).json({ ok: false, error: 'Unauthorized' });
}
const user = await User.findByPk(req.user.id);
const user = await User.findByPk(authenticatedReq.user.id);
if (!user) {
return res.status(404).json({ message: 'User not found' });

116
server/src/routes/storage.ts
View File

@@ -18,6 +18,7 @@ import { AuthenticatedRequest } from './record';
import { computeNextRun } from '../utils/schedule';
import { capture } from "../utils/analytics";
import { tryCatch } from 'bullmq';
import { encrypt, decrypt } from '../utils/auth';
import { WorkflowFile } from 'maxun-core';
import { Page } from 'playwright';
import { airtableUpdateTasks, processAirtableUpdates } from '../workflow-management/integrations/airtable';
@@ -25,6 +26,36 @@ chromium.use(stealthPlugin());
export const router = Router();
export const decryptWorkflowActions = async (workflow: any[],): Promise<any[]> => {
// Create a deep copy to avoid mutating the original workflow
const processedWorkflow = JSON.parse(JSON.stringify(workflow));
// Process each step in the workflow
for (const step of processedWorkflow) {
if (!step.what) continue;
// Process each action in the step
for (const action of step.what) {
// Only process type and press actions
if ((action.action === 'type' || action.action === 'press') && Array.isArray(action.args) && action.args.length > 1) {
// The second argument contains the encrypted value
const encryptedValue = action.args[1];
if (typeof encryptedValue === 'string') {
try {
// Decrypt the value and update the args array
action.args[1] = await decrypt(encryptedValue);
} catch (error) {
console.error('Failed to decrypt value:', error);
// Keep the encrypted value if decryption fails
}
}
}
}
}
return processedWorkflow;
};
/**
* Logs information about recordings API.
*/
@@ -56,6 +87,13 @@ router.get('/recordings/:id', requireSignIn, async (req, res) => {
raw: true
}
);
if (data?.recording?.workflow) {
data.recording.workflow = await decryptWorkflowActions(
data.recording.workflow,
);
}
return res.send(data);
} catch (e) {
logger.log('info', 'Error while reading robots');
@@ -117,13 +155,74 @@ function formatRunResponse(run: any) {
return formattedRun;
}
interface CredentialInfo {
value: string;
type: string;
}
interface Credentials {
[key: string]: CredentialInfo;
}
function updateTypeActionsInWorkflow(workflow: any[], credentials: Credentials) {
return workflow.map(step => {
if (!step.what) return step;
const indicesToRemove = new Set<number>();
step.what.forEach((action: any, index: number) => {
if (!action.action || !action.args?.[0]) return;
if ((action.action === 'type' || action.action === 'press') && credentials[action.args[0]]) {
indicesToRemove.add(index);
if (step.what[index + 1]?.action === 'waitForLoadState') {
indicesToRemove.add(index + 1);
}
}
});
const filteredWhat = step.what.filter((_: any, index: number) => !indicesToRemove.has(index));
Object.entries(credentials).forEach(([selector, credentialInfo]) => {
const clickIndex = filteredWhat.findIndex((action: any) =>
action.action === 'click' && action.args?.[0] === selector
);
if (clickIndex !== -1) {
const chars = credentialInfo.value.split('');
chars.forEach((char, i) => {
filteredWhat.splice(clickIndex + 1 + (i * 2), 0, {
action: 'type',
args: [
selector,
encrypt(char),
credentialInfo.type
]
});
filteredWhat.splice(clickIndex + 2 + (i * 2), 0, {
action: 'waitForLoadState',
args: ['networkidle']
});
});
}
});
return {
...step,
what: filteredWhat
};
});
}
/**
* PUT endpoint to update the name and limit of a robot.
*/
router.put('/recordings/:id', requireSignIn, async (req: AuthenticatedRequest, res) => {
try {
const { id } = req.params;
const { name, limit } = req.body;
const { name, limit, credentials } = req.body;
// Validate input
if (!name && limit === undefined) {
@@ -142,17 +241,21 @@ router.put('/recordings/:id', requireSignIn, async (req: AuthenticatedRequest, r
robot.set('recording_meta', { ...robot.recording_meta, name });
}
let workflow = [...robot.recording.workflow]; // Create a copy of the workflow
if (credentials) {
workflow = updateTypeActionsInWorkflow(workflow, credentials);
}
// Update the limit
if (limit !== undefined) {
const workflow = [...robot.recording.workflow]; // Create a copy of the workflow
// Ensure the workflow structure is valid before updating
if (
workflow.length > 0 &&
workflow[0]?.what?.[0]
) {
// Create a new workflow object with the updated limit
const updatedWorkflow = workflow.map((step, index) => {
workflow = workflow.map((step, index) => {
if (index === 0) { // Assuming you want to update the first step
return {
...step,
@@ -174,14 +277,13 @@ router.put('/recordings/:id', requireSignIn, async (req: AuthenticatedRequest, r
}
return step;
});
// Replace the workflow in the recording object
robot.set('recording', { ...robot.recording, workflow: updatedWorkflow });
} else {
return res.status(400).json({ error: 'Invalid workflow structure for updating limit.' });
}
}
robot.set('recording', { ...robot.recording, workflow });
await robot.save();
const updatedRobot = await Robot.findOne({ where: { 'recording_meta.id': id } });