vodorod/parcer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge commit 'refs/pull/363/head' of https://github.com/getmaxun/maxun into develop

Browse Source
This commit is contained in:
amhsirak
2025-01-22 18:23:33 +05:30
parent 07732107b7 76d88c982d
commit 653eca884c
13 changed files with 273 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
public/locales/de.json
View File

@@ -32,6 +32,7 @@
"new": "Roboter erstellen",
"modal": {
"title": "Geben Sie die URL ein",
"login_title": "Erfordert diese Seite eine Anmeldung?",
"label": "URL",
"button": "Aufnahme starten"
},

1
public/locales/en.json
View File

@@ -32,6 +32,7 @@
"new":"Create Robot",
"modal":{
"title":"Enter the URL",
"login_title": "Does this site require login?",
"label":"URL",
"button":"Start Recording"
},

1
public/locales/es.json
View File

@@ -32,6 +32,7 @@
"new": "Crear Robot",
"modal": {
"title": "Ingresa la URL",
"login_title": "¿Este sitio requiere inicio de sesión?",
"label": "URL",
"button": "Comenzar grabación"
},

1
public/locales/ja.json
View File

@@ -32,6 +32,7 @@
"new": "ロボットを作成",
"modal": {
"title": "URLを入力してください",
"login_title": "このサイトはログインが必要ですか?",
"label": "URL",
"button": "録画を開始"
},

1
public/locales/zh.json
View File

@@ -32,6 +32,7 @@
"new": "创建机器人",
"modal": {
"title": "输入URL",
"login_title": "此网站需要登录吗？",
"label": "URL",
"button": "开始录制"
},

7
server/src/models/Robot.ts
View File

@@ -26,6 +26,7 @@ interface RobotAttributes {
google_access_token?: string | null;
google_refresh_token?: string | null;
schedule?: ScheduleConfig | null;
isLogin?: boolean;
}
interface ScheduleConfig {
@@ -54,6 +55,7 @@ class Robot extends Model<RobotAttributes, RobotCreationAttributes> implements R
public google_access_token!: string | null;
public google_refresh_token!: string | null;
public schedule!: ScheduleConfig | null;
public isLogin!: boolean;
}
Robot.init(
@@ -99,6 +101,11 @@ Robot.init(
type: DataTypes.JSONB,
allowNull: true,
},
isLogin: {
type: DataTypes.BOOLEAN,
allowNull: false,
defaultValue: false,
},
},
{
sequelize,

113
server/src/routes/storage.ts
View File

@@ -18,12 +18,43 @@ import { AuthenticatedRequest } from './record';
import { computeNextRun } from '../utils/schedule';
import { capture } from "../utils/analytics";
import { tryCatch } from 'bullmq';
import { encrypt, decrypt } from '../utils/auth';
import { WorkflowFile } from 'maxun-core';
import { Page } from 'playwright';
chromium.use(stealthPlugin());
export const router = Router();
export const decryptWorkflowActions = async (workflow: any[],): Promise<any[]> => {
// Create a deep copy to avoid mutating the original workflow
const processedWorkflow = JSON.parse(JSON.stringify(workflow));
// Process each step in the workflow
for (const step of processedWorkflow) {
if (!step.what) continue;
// Process each action in the step
for (const action of step.what) {
// Only process type and press actions
if ((action.action === 'type' || action.action === 'press') && Array.isArray(action.args) && action.args.length > 1) {
// The second argument contains the encrypted value
const encryptedValue = action.args[1];
if (typeof encryptedValue === 'string') {
try {
// Decrypt the value and update the args array
action.args[1] = await decrypt(encryptedValue);
} catch (error) {
console.error('Failed to decrypt value:', error);
// Keep the encrypted value if decryption fails
}
}
}
}
}
return processedWorkflow;
};
/**
* Logs information about recordings API.
*/
@@ -55,6 +86,13 @@ router.get('/recordings/:id', requireSignIn, async (req, res) => {
raw: true
}
);
if (data?.recording?.workflow) {
data.recording.workflow = await decryptWorkflowActions(
data.recording.workflow,
);
}
return res.send(data);
} catch (e) {
logger.log('info', 'Error while reading robots');
@@ -116,13 +154,70 @@ function formatRunResponse(run: any) {
return formattedRun;
}
interface CredentialUpdate {
[selector: string]: string;
}
function updateTypeActionsInWorkflow(workflow: any[], credentials: CredentialUpdate) {
return workflow.map(step => {
if (!step.what) return step;
// First pass: mark indices to remove
const indicesToRemove = new Set<number>();
step.what.forEach((action: any, index: any) => {
if (!action.action || !action.args?.[0]) return;
// If it's a type/press action for a credential
if ((action.action === 'type' || action.action === 'press') && credentials[action.args[0]]) {
indicesToRemove.add(index);
// Check if next action is waitForLoadState
if (step.what[index + 1]?.action === 'waitForLoadState') {
indicesToRemove.add(index + 1);
}
}
});
// Filter out marked indices and create new what array
const filteredWhat = step.what.filter((_: any, index: any) => !indicesToRemove.has(index));
// Add new type actions after click actions
Object.entries(credentials).forEach(([selector, credential]) => {
const clickIndex = filteredWhat.findIndex((action: any) =>
action.action === 'click' && action.args?.[0] === selector
);
if (clickIndex !== -1) {
const chars = credential.split('');
chars.forEach((char, i) => {
// Add type action
filteredWhat.splice(clickIndex + 1 + (i * 2), 0, {
action: 'type',
args: [selector, encrypt(char)]
});
// Add waitForLoadState
filteredWhat.splice(clickIndex + 2 + (i * 2), 0, {
action: 'waitForLoadState',
args: ['networkidle']
});
});
}
});
return {
...step,
what: filteredWhat
};
});
}
/**
* PUT endpoint to update the name and limit of a robot.
*/
router.put('/recordings/:id', requireSignIn, async (req: AuthenticatedRequest, res) => {
try {
const { id } = req.params;
const { name, limit } = req.body;
const { name, limit, credentials } = req.body;
// Validate input
if (!name && limit === undefined) {
@@ -141,17 +236,21 @@ router.put('/recordings/:id', requireSignIn, async (req: AuthenticatedRequest, r
robot.set('recording_meta', { ...robot.recording_meta, name });
}
let workflow = [...robot.recording.workflow]; // Create a copy of the workflow
if (credentials) {
workflow = updateTypeActionsInWorkflow(workflow, credentials);
}
// Update the limit
if (limit !== undefined) {
const workflow = [...robot.recording.workflow]; // Create a copy of the workflow
// Ensure the workflow structure is valid before updating
if (
workflow.length > 0 &&
workflow[0]?.what?.[0]
) {
// Create a new workflow object with the updated limit
const updatedWorkflow = workflow.map((step, index) => {
workflow = workflow.map((step, index) => {
if (index === 0) { // Assuming you want to update the first step
return {
...step,
@@ -173,14 +272,13 @@ router.put('/recordings/:id', requireSignIn, async (req: AuthenticatedRequest, r
}
return step;
});
// Replace the workflow in the recording object
robot.set('recording', { ...robot.recording, workflow: updatedWorkflow });
} else {
return res.status(400).json({ error: 'Invalid workflow structure for updating limit.' });
}
}
robot.set('recording', { ...robot.recording, workflow });
await robot.save();
const updatedRobot = await Robot.findOne({ where: { 'recording_meta.id': id } });
@@ -248,6 +346,7 @@ router.post('/recordings/:id/duplicate', requireSignIn, async (req: Authenticate
updatedAt: currentTimestamp,
},
recording: { ...originalRobot.recording, workflow },
isLogin: originalRobot.isLogin,
google_sheet_email: null,
google_sheet_name: null,
google_sheet_id: null,

7
server/src/workflow-management/classes/Generator.ts
View File

@@ -134,9 +134,9 @@ export class WorkflowGenerator {
*/
private registerEventHandlers = (socket: Socket) => {
socket.on('save', (data) => {
const { fileName, userId } = data;
const { fileName, userId, isLogin } = data;
logger.log('debug', `Saving workflow ${fileName} for user ID ${userId}`);
this.saveNewWorkflow(fileName, userId);
this.saveNewWorkflow(fileName, userId, isLogin);
});
socket.on('new-recording', () => this.workflowRecord = {
workflow: [],
@@ -660,7 +660,7 @@ export class WorkflowGenerator {
* @param fileName The name of the file.
* @returns {Promise<void>}
*/
public saveNewWorkflow = async (fileName: string, userId: number) => {
public saveNewWorkflow = async (fileName: string, userId: number, isLogin: boolean) => {
const recording = this.optimizeWorkflow(this.workflowRecord);
try {
this.recordingMeta = {
@@ -675,6 +675,7 @@ export class WorkflowGenerator {
userId,
recording_meta: this.recordingMeta,
recording: recording,
isLogin: isLogin,
});
capture(
'maxun-oss-robot-created',

6
src/api/storage.ts
View File

@@ -5,6 +5,10 @@ import { ScheduleSettings } from "../components/robot/ScheduleSettings";
import { CreateRunResponse, ScheduleRunResponse } from "../pages/MainPage";
import { apiUrl } from "../apiConfig";
interface Credentials {
[key: string]: string;
}
export const getStoredRecordings = async (): Promise<string[] | null> => {
try {
const response = await axios.get(`${apiUrl}/storage/recordings`);
@@ -19,7 +23,7 @@ export const getStoredRecordings = async (): Promise<string[] | null> => {
}
};
export const updateRecording = async (id: string, data: { name?: string; limit?: number }): Promise<boolean> => {
export const updateRecording = async (id: string, data: { name?: string; limit?: number, credentials?: Credentials }): Promise<boolean> => {
try {
const response = await axios.put(`${apiUrl}/storage/recordings/${id}`, data);
if (response.status === 200) {

4
src/components/recorder/SaveRecording.tsx
View File

@@ -22,7 +22,7 @@ export const SaveRecording = ({ fileName }: SaveRecordingProps) => {
const [recordingName, setRecordingName] = useState<string>(fileName);
const [waitingForSave, setWaitingForSave] = useState<boolean>(false);
const { browserId, setBrowserId, notify, recordings } = useGlobalInfoStore();
const { browserId, setBrowserId, notify, recordings, isLogin } = useGlobalInfoStore();
const { socket } = useSocketStore();
const { state, dispatch } = useContext(AuthContext);
const { user } = state;
@@ -59,7 +59,7 @@ export const SaveRecording = ({ fileName }: SaveRecordingProps) => {
// releases resources and changes the view for main page by clearing the global browserId
const saveRecording = async () => {
if (user) {
const payload = { fileName: recordingName, userId: user.id };
const payload = { fileName: recordingName, userId: user.id, isLogin: isLogin };
socket?.emit('save', payload);
setWaitingForSave(true);
console.log(`Saving the recording as ${recordingName} for userId ${user.id}`);

17
src/components/robot/RecordingsTable.tsx
View File

@@ -11,7 +11,7 @@ import TableRow from '@mui/material/TableRow';
import { useEffect } from "react";
import { WorkflowFile } from "maxun-core";
import SearchIcon from '@mui/icons-material/Search';
import { IconButton, Button, Box, Typography, TextField, MenuItem, Menu, ListItemIcon, ListItemText, CircularProgress } from "@mui/material";
import { IconButton, Button, Box, Typography, TextField, MenuItem, Menu, ListItemIcon, ListItemText, CircularProgress, RadioGroup, FormControlLabel, Radio } from "@mui/material";
import { Schedule, DeleteForever, Edit, PlayCircle, Settings, Power, ContentCopy, MoreHoriz } from "@mui/icons-material";
import { useGlobalInfoStore } from "../../context/globalInfo";
import { checkRunsForRecording, deleteRecordingFromStorage, getStoredRecordings } from "../../api/storage";
@@ -85,7 +85,7 @@ export const RecordingsTable = ({ handleEditRecording, handleRunRecording, handl
},
];
const { notify, setRecordings, browserId, setBrowserId, setInitialUrl, recordingUrl, setRecordingUrl, recordingName, setRecordingName, recordingId, setRecordingId } = useGlobalInfoStore();
const { notify, setRecordings, browserId, setBrowserId, setInitialUrl, recordingUrl, setRecordingUrl, isLogin, setIsLogin, recordingName, setRecordingName, recordingId, setRecordingId } = useGlobalInfoStore();
const navigate = useNavigate();
const handleChangePage = (event: unknown, newPage: number) => {
@@ -321,6 +321,19 @@ export const RecordingsTable = ({ handleEditRecording, handleRunRecording, handl
onChange={setBrowserRecordingUrl}
style={{ marginBottom: '20px', marginTop: '20px' }}
/>
<Typography variant="h6" gutterBottom>{t('recordingtable.modal.login_title')}</Typography>
<RadioGroup
aria-labelledby="login-requirement-radio-group"
name="login-requirement"
value={isLogin ? 'yes' : 'no'}
onChange={(e) => setIsLogin(e.target.value === 'yes')}
style={{ marginBottom: '20px' }}
>
<FormControlLabel value="yes" control={<Radio />} label="Yes" />
<FormControlLabel value="no" control={<Radio />} label="No" />
</RadioGroup>
<Button
variant="contained"
color="primary"

124
src/components/robot/RobotEdit.tsx
View File

@@ -1,7 +1,8 @@
import React, { useState, useEffect } from 'react';
import { useTranslation } from 'react-i18next';
import { GenericModal } from "../ui/GenericModal";
import { TextField, Typography, Box, Button } from "@mui/material";
import { TextField, Typography, Box, Button, IconButton, InputAdornment } from "@mui/material";
import { Visibility, VisibilityOff } from '@mui/icons-material';
import { modalStyle } from "../recorder/AddWhereCondModal";
import { useGlobalInfoStore } from '../../context/globalInfo';
import { getStoredRecording, updateRecording } from '../../api/storage';
@@ -25,6 +26,14 @@ interface RobotEditOptions {
limit?: number;
}
interface Credentials {
[key: string]: string;
}
interface CredentialVisibility {
[key: string]: boolean;
}
interface ScheduleConfig {
runEvery: number;
runEveryUnit: 'MINUTES' | 'HOURS' | 'DAYS' | 'WEEKS' | 'MONTHS';
@@ -48,6 +57,7 @@ export interface RobotSettings {
google_access_token?: string | null;
google_refresh_token?: string | null;
schedule?: ScheduleConfig | null;
isLogin?: boolean;
}
interface RobotSettingsProps {
@@ -60,7 +70,17 @@ interface RobotSettingsProps {
export const RobotEditModal = ({ isOpen, handleStart, handleClose, initialSettings }: RobotSettingsProps) => {
const { t } = useTranslation();
const [robot, setRobot] = useState<RobotSettings | null>(null);
const [credentials, setCredentials] = useState<Credentials>({});
const { recordingId, notify } = useGlobalInfoStore();
const [credentialSelectors, setCredentialSelectors] = useState<string[]>([]);
const [showPasswords, setShowPasswords] = useState<CredentialVisibility>({});
const handleClickShowPassword = (selector: string) => {
setShowPasswords(prev => ({
...prev,
[selector]: !prev[selector]
}));
};
useEffect(() => {
if (isOpen) {
@@ -68,6 +88,68 @@ export const RobotEditModal = ({ isOpen, handleStart, handleClose, initialSettin
}
}, [isOpen]);
useEffect(() => {
if (robot?.recording?.workflow) {
const selectors = findCredentialSelectors(robot.recording.workflow);
setCredentialSelectors(selectors);
const initialCredentials = extractInitialCredentials(robot.recording.workflow);
setCredentials(initialCredentials);
}
}, [robot]);
const findCredentialSelectors = (workflow: WhereWhatPair[]): string[] => {
const selectors = new Set<string>();
workflow?.forEach(step => {
step.what?.forEach(action => {
if (
(action.action === 'type') &&
action.args &&
action.args[0] &&
typeof action.args[0] === 'string'
) {
selectors.add(action.args[0]);
}
});
});
return Array.from(selectors);
};
const extractInitialCredentials = (workflow: any[]): Record<string, string> => {
const credentials: Record<string, string> = {};
const isPrintableCharacter = (char: string): boolean => {
return char.length === 1 && !!char.match(/^[\x20-\x7E]$/);
};
workflow.forEach(step => {
if (!step.what) return;
step.what.forEach((action: any) => {
if (
(action.action === 'type' || action.action === 'press') &&
action.args?.length >= 2 &&
typeof action.args[1] === 'string'
) {
let currentSelector: string = action.args[0];
let character: string = action.args[1];
if (!credentials.hasOwnProperty(currentSelector)) {
credentials[currentSelector] = '';
}
if (isPrintableCharacter(character)) {
credentials[currentSelector] += character;
}
}
});
});
return credentials;
};
const getRobot = async () => {
if (recordingId) {
const robot = await getStoredRecording(recordingId);
@@ -83,6 +165,13 @@ export const RobotEditModal = ({ isOpen, handleStart, handleClose, initialSettin
);
};
const handleCredentialChange = (selector: string, value: string) => {
setCredentials(prev => ({
...prev,
[selector]: value
}));
};
const handleLimitChange = (newLimit: number) => {
setRobot((prev) => {
if (!prev) return prev;
@@ -111,6 +200,7 @@ export const RobotEditModal = ({ isOpen, handleStart, handleClose, initialSettin
const payload = {
name: robot.recording_meta.name,
limit: robot.recording.workflow[0]?.what[0]?.args?.[0]?.limit,
credentials: credentials,
};
const success = await updateRecording(robot.recording_meta.id, payload);
@@ -170,6 +260,38 @@ export const RobotEditModal = ({ isOpen, handleStart, handleClose, initialSettin
/>
)}
{(robot.isLogin || credentialSelectors.length > 0) && (
<>
<Typography variant="h6" style={{ marginBottom: '20px' }}>
{t('Login Credentials')}
</Typography>
{credentialSelectors.map((selector) => (
<TextField
key={selector}
type={showPasswords[selector] ? 'text' : 'password'}
label={`Credential for ${selector}`}
value={credentials[selector] || ''}
onChange={(e) => handleCredentialChange(selector, e.target.value)}
style={{ marginBottom: '20px' }}
InputProps={{
endAdornment: (
<InputAdornment position="end">
<IconButton
aria-label="toggle password visibility"
onClick={() => handleClickShowPassword(selector)}
edge="end"
>
{showPasswords[selector] ? <Visibility /> : <VisibilityOff />}
</IconButton>
</InputAdornment>
),
}}
/>
))}
</>
)}
<Box mt={2} display="flex" justifyContent="flex-end">
<Button variant="contained" color="primary" onClick={handleSave}>
{t('robot_edit.save')}

6
src/context/globalInfo.tsx
View File

@@ -10,6 +10,8 @@ interface GlobalInfo {
notification: AlertSnackbarProps;
notify: (severity: 'error' | 'warning' | 'info' | 'success', message: string) => void;
closeNotify: () => void;
isLogin: boolean;
setIsLogin: (isLogin: boolean) => void;
recordings: string[];
setRecordings: (recordings: string[]) => void;
rerenderRuns: boolean;
@@ -53,6 +55,7 @@ class GlobalInfoStore implements Partial<GlobalInfo> {
recordingName = '';
initialUrl = 'https://';
recordingUrl = 'https://';
isLogin = false;
currentWorkflowActionsState = {
hasScrapeListAction: false,
hasScreenshotAction: false,
@@ -75,6 +78,7 @@ export const GlobalInfoProvider = ({ children }: { children: JSX.Element }) => {
const [recordingLength, setRecordingLength] = useState<number>(globalInfoStore.recordingLength);
const [recordingId, setRecordingId] = useState<string | null>(globalInfoStore.recordingId);
const [recordingName, setRecordingName] = useState<string>(globalInfoStore.recordingName);
const [isLogin, setIsLogin] = useState<boolean>(globalInfoStore.isLogin);
const [initialUrl, setInitialUrl] = useState<string>(globalInfoStore.initialUrl);
const [recordingUrl, setRecordingUrl] = useState<string>(globalInfoStore.recordingUrl);
const [currentWorkflowActionsState, setCurrentWorkflowActionsState] = useState(globalInfoStore.currentWorkflowActionsState);
@@ -127,6 +131,8 @@ export const GlobalInfoProvider = ({ children }: { children: JSX.Element }) => {
setInitialUrl,
recordingUrl,
setRecordingUrl,
isLogin,
setIsLogin,
currentWorkflowActionsState,
setCurrentWorkflowActionsState,
shouldResetInterpretationLog,