Merge branch 'develop' into perf-v11
This commit is contained in:
@@ -103,6 +103,8 @@ You can access the frontend at http://localhost:5173/ and backend at http://loca
|
||||
| `GOOGLE_CLIENT_ID` | No | Client ID for Google OAuth, used for Google Sheet integration authentication. | Google login will not work. |
|
||||
| `GOOGLE_CLIENT_SECRET`| No | Client Secret for Google OAuth. | Google login will not work. |
|
||||
| `GOOGLE_REDIRECT_URI` | No | Redirect URI for handling Google OAuth responses. | Google login will not work. |
|
||||
| `AIRTABLE_CLIENT_ID` | No | Client ID for Airtable, used for Airtable integration authentication. | Airtable login will not work. |
|
||||
| `AIRTABLE_REDIRECT_URI` | No | Redirect URI for handling Airtable OAuth responses. | Airtable login will not work. |
|
||||
| `REDIS_HOST` | Yes | Host address of the Redis server, used by BullMQ for scheduling robots. | Redis connection will fail. |
|
||||
| `REDIS_PORT` | Yes | Port number for the Redis server. | Redis connection will fail. |
|
||||
| `MAXUN_TELEMETRY` | No | Disables telemetry to stop sending anonymous usage data. Keeping it enabled helps us understand how the product is used and assess the impact of any new changes. Please keep it enabled. | Telemetry data will not be collected. |
|
||||
|
||||
@@ -210,7 +210,6 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3,
|
||||
return Array.from(document.querySelectorAll(config.selector));
|
||||
}
|
||||
|
||||
// First handle iframe traversal if present
|
||||
if (config.selector.includes(':>>')) {
|
||||
const parts = config.selector.split(':>>').map(s => s.trim());
|
||||
let currentElements = [document];
|
||||
@@ -223,23 +222,44 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3,
|
||||
|
||||
for (const element of currentElements) {
|
||||
try {
|
||||
// For document or iframe document
|
||||
const doc = element.contentDocument || element || element.contentWindow?.document;
|
||||
if (!doc) continue;
|
||||
|
||||
// Query elements in current context
|
||||
if (part.startsWith('frame[name=') || part.startsWith('iframe[name=')) {
|
||||
const nameMatch = part.match(/\[name=['"]([^'"]+)['"]\]/);
|
||||
if (nameMatch && nameMatch[1]) {
|
||||
const frameName = nameMatch[1];
|
||||
let foundFrames = [];
|
||||
|
||||
if (doc.getElementsByName && typeof doc.getElementsByName === 'function') {
|
||||
foundFrames = Array.from(doc.getElementsByName(frameName))
|
||||
.filter(el => el.tagName === 'FRAME' || el.tagName === 'IFRAME');
|
||||
}
|
||||
|
||||
if (foundFrames.length === 0) {
|
||||
const framesBySelector = Array.from(doc.querySelectorAll(`frame[name="${frameName}"], iframe[name="${frameName}"]`));
|
||||
foundFrames = framesBySelector;
|
||||
}
|
||||
|
||||
if (isLast) {
|
||||
nextElements.push(...foundFrames);
|
||||
} else {
|
||||
nextElements.push(...foundFrames);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
const found = Array.from(doc.querySelectorAll(part));
|
||||
|
||||
if (isLast) {
|
||||
// If it's the last part, keep all matching elements
|
||||
nextElements.push(...found);
|
||||
} else {
|
||||
// If not last, only keep iframes for next iteration
|
||||
const iframes = found.filter(el => el.tagName === 'IFRAME');
|
||||
nextElements.push(...iframes);
|
||||
const frames = found.filter(el => el.tagName === 'IFRAME' || el.tagName === 'FRAME');
|
||||
nextElements.push(...frames);
|
||||
}
|
||||
} catch (error) {
|
||||
console.warn('Cannot access iframe content:', error, {
|
||||
console.warn('Cannot access iframe/frame content:', error, {
|
||||
part,
|
||||
element,
|
||||
index: i
|
||||
@@ -285,12 +305,17 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3,
|
||||
return [];
|
||||
}
|
||||
|
||||
// Modified to handle iframe context for URL resolution
|
||||
function getElementValue(element, attribute) {
|
||||
if (!element) return null;
|
||||
|
||||
// Get the base URL for resolving relative URLs
|
||||
const baseURL = element.ownerDocument?.location?.href || window.location.origin;
|
||||
let baseURL;
|
||||
try {
|
||||
baseURL = element.ownerDocument?.location?.href ||
|
||||
element.ownerDocument?.baseURI ||
|
||||
window.location.origin;
|
||||
} catch (e) {
|
||||
baseURL = window.location.origin;
|
||||
}
|
||||
|
||||
switch (attribute) {
|
||||
case 'href': {
|
||||
@@ -305,6 +330,10 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3,
|
||||
return element.innerText?.trim();
|
||||
case 'textContent':
|
||||
return element.textContent?.trim();
|
||||
case 'innerHTML':
|
||||
return element.innerHTML;
|
||||
case 'outerHTML':
|
||||
return element.outerHTML;
|
||||
default:
|
||||
return element.getAttribute(attribute) || element.innerText?.trim();
|
||||
}
|
||||
@@ -394,7 +423,7 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3,
|
||||
* @returns {Array.<Array.<Object>>} Array of arrays of scraped items, one sub-array per list
|
||||
*/
|
||||
window.scrapeList = async function ({ listSelector, fields, limit = 10 }) {
|
||||
// Enhanced query function to handle both iframe and shadow DOM
|
||||
// Enhanced query function to handle iframe, frame and shadow DOM
|
||||
const queryElement = (rootElement, selector) => {
|
||||
if (!selector.includes('>>') && !selector.includes(':>>')) {
|
||||
return rootElement.querySelector(selector);
|
||||
@@ -406,14 +435,14 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3,
|
||||
for (let i = 0; i < parts.length; i++) {
|
||||
if (!currentElement) return null;
|
||||
|
||||
// Handle iframe traversal
|
||||
if (currentElement.tagName === 'IFRAME') {
|
||||
// Handle iframe and frame traversal
|
||||
if (currentElement.tagName === 'IFRAME' || currentElement.tagName === 'FRAME') {
|
||||
try {
|
||||
const iframeDoc = currentElement.contentDocument || currentElement.contentWindow.document;
|
||||
currentElement = iframeDoc.querySelector(parts[i]);
|
||||
const frameDoc = currentElement.contentDocument || currentElement.contentWindow.document;
|
||||
currentElement = frameDoc.querySelector(parts[i]);
|
||||
continue;
|
||||
} catch (e) {
|
||||
console.warn('Cannot access iframe content:', e);
|
||||
console.warn(`Cannot access ${currentElement.tagName.toLowerCase()} content:`, e);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
@@ -456,13 +485,13 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3,
|
||||
const nextElements = [];
|
||||
|
||||
for (const element of currentElements) {
|
||||
// Handle iframe traversal
|
||||
if (element.tagName === 'IFRAME') {
|
||||
// Handle iframe and frame traversal
|
||||
if (element.tagName === 'IFRAME' || element.tagName === 'FRAME') {
|
||||
try {
|
||||
const iframeDoc = element.contentDocument || element.contentWindow.document;
|
||||
nextElements.push(...iframeDoc.querySelectorAll(part));
|
||||
const frameDoc = element.contentDocument || element.contentWindow.document;
|
||||
nextElements.push(...frameDoc.querySelectorAll(part));
|
||||
} catch (e) {
|
||||
console.warn('Cannot access iframe content:', e);
|
||||
console.warn(`Cannot access ${element.tagName.toLowerCase()} content:`, e);
|
||||
continue;
|
||||
}
|
||||
} else {
|
||||
@@ -537,8 +566,8 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3,
|
||||
return { type: 'TR', element: currentElement };
|
||||
}
|
||||
|
||||
// Handle iframe crossing
|
||||
if (currentElement.tagName === 'IFRAME') {
|
||||
// Handle iframe and frame crossing
|
||||
if (currentElement.tagName === 'IFRAME' || currentElement.tagName === 'FRAME') {
|
||||
try {
|
||||
currentElement = currentElement.contentDocument.body;
|
||||
} catch (e) {
|
||||
@@ -582,7 +611,7 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3,
|
||||
|
||||
if (current.tagName === 'TH') return true;
|
||||
|
||||
if (current.tagName === 'IFRAME') {
|
||||
if (current.tagName === 'IFRAME' || current.tagName === 'FRAME') {
|
||||
try {
|
||||
current = current.contentDocument.body;
|
||||
} catch (e) {
|
||||
@@ -638,14 +667,18 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3,
|
||||
allElements.push(...shadowHost.getElementsByTagName(baseElement.tagName));
|
||||
}
|
||||
|
||||
// Get elements from iframes
|
||||
const iframes = document.getElementsByTagName('iframe');
|
||||
for (const iframe of iframes) {
|
||||
// Get elements from iframes and frames
|
||||
const frames = [
|
||||
...Array.from(document.getElementsByTagName('iframe')),
|
||||
...Array.from(document.getElementsByTagName('frame'))
|
||||
];
|
||||
|
||||
for (const frame of frames) {
|
||||
try {
|
||||
const iframeDoc = iframe.contentDocument || iframe.contentWindow.document;
|
||||
allElements.push(...iframeDoc.getElementsByTagName(baseElement.tagName));
|
||||
const frameDoc = frame.contentDocument || frame.contentWindow.document;
|
||||
allElements.push(...frameDoc.getElementsByTagName(baseElement.tagName));
|
||||
} catch (e) {
|
||||
console.warn('Cannot access iframe content:', e);
|
||||
console.warn(`Cannot access ${frame.tagName.toLowerCase()} content:`, e);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -707,7 +740,7 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3,
|
||||
const tableData = [];
|
||||
const nonTableData = [];
|
||||
|
||||
// Process table data with both iframe and shadow DOM support
|
||||
// Process table data with support for iframes, frames, and shadow DOM
|
||||
for (let containerIndex = 0; containerIndex < containers.length; containerIndex++) {
|
||||
const container = containers[containerIndex];
|
||||
const { tableFields } = containerFields[containerIndex];
|
||||
@@ -717,14 +750,14 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3,
|
||||
const firstElement = queryElement(container, firstField.selector);
|
||||
let tableContext = firstElement;
|
||||
|
||||
// Find table context including both iframe and shadow DOM
|
||||
// Find table context including iframe, frame and shadow DOM
|
||||
while (tableContext && tableContext.tagName !== 'TABLE' && tableContext !== container) {
|
||||
if (tableContext.getRootNode() instanceof ShadowRoot) {
|
||||
tableContext = tableContext.getRootNode().host;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (tableContext.tagName === 'IFRAME') {
|
||||
if (tableContext.tagName === 'IFRAME' || tableContext.tagName === 'FRAME') {
|
||||
try {
|
||||
tableContext = tableContext.contentDocument.body;
|
||||
} catch (e) {
|
||||
@@ -747,13 +780,13 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3,
|
||||
rows.push(...tableContext.shadowRoot.getElementsByTagName('TR'));
|
||||
}
|
||||
|
||||
// Get rows from iframes
|
||||
if (tableContext.tagName === 'IFRAME') {
|
||||
// Get rows from iframes and frames
|
||||
if (tableContext.tagName === 'IFRAME' || tableContext.tagName === 'FRAME') {
|
||||
try {
|
||||
const iframeDoc = tableContext.contentDocument || tableContext.contentWindow.document;
|
||||
rows.push(...iframeDoc.getElementsByTagName('TR'));
|
||||
const frameDoc = tableContext.contentDocument || tableContext.contentWindow.document;
|
||||
rows.push(...frameDoc.getElementsByTagName('TR'));
|
||||
} catch (e) {
|
||||
console.warn('Cannot access iframe rows:', e);
|
||||
console.warn(`Cannot access ${tableContext.tagName.toLowerCase()} rows:`, e);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -823,7 +856,7 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3,
|
||||
}
|
||||
}
|
||||
|
||||
// Process non-table data with both contexts support
|
||||
// Process non-table data with all contexts support
|
||||
for (let containerIndex = 0; containerIndex < containers.length; containerIndex++) {
|
||||
if (nonTableData.length >= limit) break;
|
||||
|
||||
|
||||
@@ -286,6 +286,12 @@ export default class Interpreter extends EventEmitter {
|
||||
? arrayToObject(<any>superset[key])
|
||||
: superset[key];
|
||||
|
||||
if ((key === 'url' || key === 'selectors') &&
|
||||
Array.isArray(value) && Array.isArray(superset[key]) &&
|
||||
value.length === 0 && (superset[key] as any[]).length === 0) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (key === 'selectors' && Array.isArray(value) && Array.isArray(superset[key])) {
|
||||
return value.some(selector =>
|
||||
(superset[key] as any[]).includes(selector)
|
||||
@@ -592,33 +598,52 @@ export default class Interpreter extends EventEmitter {
|
||||
};
|
||||
|
||||
// Enhanced button finder with retry mechanism
|
||||
const findWorkingButton = async (selectors: string[], retryCount = 0): Promise<{
|
||||
button: ElementHandle | null,
|
||||
workingSelector: string | null
|
||||
const findWorkingButton = async (selectors: string[]): Promise<{
|
||||
button: ElementHandle | null,
|
||||
workingSelector: string | null,
|
||||
updatedSelectors: string[]
|
||||
}> => {
|
||||
for (const selector of selectors) {
|
||||
try {
|
||||
const button = await page.waitForSelector(selector, {
|
||||
state: 'attached',
|
||||
timeout: 10000 // Reduced timeout for faster checks
|
||||
});
|
||||
if (button) {
|
||||
debugLog('Found working selector:', selector);
|
||||
return { button, workingSelector: selector };
|
||||
let updatedSelectors = [...selectors];
|
||||
|
||||
for (let i = 0; i < selectors.length; i++) {
|
||||
const selector = selectors[i];
|
||||
let retryCount = 0;
|
||||
let selectorSuccess = false;
|
||||
|
||||
while (retryCount < MAX_RETRIES && !selectorSuccess) {
|
||||
try {
|
||||
const button = await page.waitForSelector(selector, {
|
||||
state: 'attached',
|
||||
timeout: 10000
|
||||
});
|
||||
|
||||
if (button) {
|
||||
debugLog('Found working selector:', selector);
|
||||
return {
|
||||
button,
|
||||
workingSelector: selector,
|
||||
updatedSelectors
|
||||
};
|
||||
}
|
||||
} catch (error) {
|
||||
retryCount++;
|
||||
debugLog(`Selector "${selector}" failed: attempt ${retryCount}/${MAX_RETRIES}`);
|
||||
|
||||
if (retryCount < MAX_RETRIES) {
|
||||
await page.waitForTimeout(RETRY_DELAY);
|
||||
} else {
|
||||
debugLog(`Removing failed selector "${selector}" after ${MAX_RETRIES} attempts`);
|
||||
updatedSelectors = updatedSelectors.filter(s => s !== selector);
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
debugLog(`Selector failed: ${selector}`);
|
||||
}
|
||||
}
|
||||
|
||||
// Implement retry mechanism when no selectors work
|
||||
if (selectors.length > 0 && retryCount < MAX_RETRIES) {
|
||||
debugLog(`Retry attempt ${retryCount + 1} of ${MAX_RETRIES}`);
|
||||
await page.waitForTimeout(RETRY_DELAY);
|
||||
return findWorkingButton(selectors, retryCount + 1);
|
||||
}
|
||||
|
||||
return { button: null, workingSelector: null };
|
||||
|
||||
return {
|
||||
button: null,
|
||||
workingSelector: null,
|
||||
updatedSelectors
|
||||
};
|
||||
};
|
||||
|
||||
const retryOperation = async (operation: () => Promise<boolean>, retryCount = 0): Promise<boolean> => {
|
||||
@@ -680,7 +705,10 @@ export default class Interpreter extends EventEmitter {
|
||||
await scrapeCurrentPage();
|
||||
if (checkLimit()) return allResults;
|
||||
|
||||
const { button, workingSelector } = await findWorkingButton(availableSelectors);
|
||||
const { button, workingSelector, updatedSelectors } = await findWorkingButton(availableSelectors);
|
||||
|
||||
availableSelectors = updatedSelectors;
|
||||
|
||||
if (!button || !workingSelector) {
|
||||
// Final retry for navigation when no selectors work
|
||||
const success = await retryOperation(async () => {
|
||||
@@ -697,10 +725,6 @@ export default class Interpreter extends EventEmitter {
|
||||
break;
|
||||
}
|
||||
|
||||
availableSelectors = availableSelectors.slice(
|
||||
availableSelectors.indexOf(workingSelector)
|
||||
);
|
||||
|
||||
let retryCount = 0;
|
||||
let navigationSuccess = false;
|
||||
|
||||
@@ -768,22 +792,25 @@ export default class Interpreter extends EventEmitter {
|
||||
}
|
||||
|
||||
case 'clickLoadMore': {
|
||||
await scrapeCurrentPage();
|
||||
if (checkLimit()) return allResults;
|
||||
|
||||
let loadMoreCounter = 0;
|
||||
let previousResultCount = allResults.length;
|
||||
let noNewItemsCounter = 0;
|
||||
const MAX_NO_NEW_ITEMS = 2;
|
||||
|
||||
while (true) {
|
||||
// Find working button with retry mechanism, consistent with clickNext
|
||||
const { button: loadMoreButton, workingSelector } = await findWorkingButton(availableSelectors);
|
||||
// Find working button with retry mechanism
|
||||
const { button: loadMoreButton, workingSelector, updatedSelectors } = await findWorkingButton(availableSelectors);
|
||||
|
||||
availableSelectors = updatedSelectors;
|
||||
|
||||
if (!workingSelector || !loadMoreButton) {
|
||||
debugLog('No working Load More selector found after retries');
|
||||
const finalResults = await page.evaluate((cfg) => window.scrapeList(cfg), config);
|
||||
allResults = allResults.concat(finalResults);
|
||||
return allResults;
|
||||
}
|
||||
|
||||
// Update available selectors to start from the working one
|
||||
availableSelectors = availableSelectors.slice(
|
||||
availableSelectors.indexOf(workingSelector)
|
||||
);
|
||||
|
||||
// Implement retry mechanism for clicking the button
|
||||
let retryCount = 0;
|
||||
let clickSuccess = false;
|
||||
@@ -808,6 +835,8 @@ export default class Interpreter extends EventEmitter {
|
||||
|
||||
if (clickSuccess) {
|
||||
await page.waitForTimeout(1000);
|
||||
loadMoreCounter++;
|
||||
debugLog(`Successfully clicked Load More button (${loadMoreCounter} times)`);
|
||||
}
|
||||
} catch (error) {
|
||||
debugLog(`Click attempt ${retryCount + 1} failed completely.`);
|
||||
@@ -822,8 +851,6 @@ export default class Interpreter extends EventEmitter {
|
||||
|
||||
if (!clickSuccess) {
|
||||
debugLog(`Load More clicking failed after ${MAX_RETRIES} attempts`);
|
||||
const finalResults = await page.evaluate((cfg) => window.scrapeList(cfg), config);
|
||||
allResults = allResults.concat(finalResults);
|
||||
return allResults;
|
||||
}
|
||||
|
||||
@@ -833,20 +860,34 @@ export default class Interpreter extends EventEmitter {
|
||||
await page.waitForTimeout(2000);
|
||||
|
||||
const currentHeight = await page.evaluate(() => document.body.scrollHeight);
|
||||
if (currentHeight === previousHeight) {
|
||||
debugLog('No more items loaded after Load More');
|
||||
const finalResults = await page.evaluate((cfg) => window.scrapeList(cfg), config);
|
||||
allResults = allResults.concat(finalResults);
|
||||
return allResults;
|
||||
}
|
||||
const heightChanged = currentHeight !== previousHeight;
|
||||
previousHeight = currentHeight;
|
||||
|
||||
if (config.limit && allResults.length >= config.limit) {
|
||||
allResults = allResults.slice(0, config.limit);
|
||||
break;
|
||||
await scrapeCurrentPage();
|
||||
|
||||
const currentResultCount = allResults.length;
|
||||
const newItemsAdded = currentResultCount > previousResultCount;
|
||||
|
||||
if (!newItemsAdded) {
|
||||
noNewItemsCounter++;
|
||||
debugLog(`No new items added after click (${noNewItemsCounter}/${MAX_NO_NEW_ITEMS})`);
|
||||
|
||||
if (noNewItemsCounter >= MAX_NO_NEW_ITEMS) {
|
||||
debugLog(`Stopping after ${MAX_NO_NEW_ITEMS} clicks with no new items`);
|
||||
return allResults;
|
||||
}
|
||||
} else {
|
||||
noNewItemsCounter = 0;
|
||||
previousResultCount = currentResultCount;
|
||||
}
|
||||
|
||||
if (checkLimit()) return allResults;
|
||||
|
||||
if (!heightChanged) {
|
||||
debugLog('No more items loaded after Load More');
|
||||
return allResults;
|
||||
}
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
default: {
|
||||
|
||||
@@ -507,9 +507,9 @@ async function createWorkflowAndStoreMetadata(id: string, userId: string) {
|
||||
}
|
||||
}
|
||||
|
||||
async function readyForRunHandler(browserId: string, id: string) {
|
||||
async function readyForRunHandler(browserId: string, id: string, userId: string){
|
||||
try {
|
||||
const result = await executeRun(id);
|
||||
const result = await executeRun(id, userId);
|
||||
|
||||
if (result && result.success) {
|
||||
logger.log('info', `Interpretation of ${id} succeeded`);
|
||||
@@ -517,14 +517,14 @@ async function readyForRunHandler(browserId: string, id: string) {
|
||||
return result.interpretationInfo;
|
||||
} else {
|
||||
logger.log('error', `Interpretation of ${id} failed`);
|
||||
await destroyRemoteBrowser(browserId);
|
||||
await destroyRemoteBrowser(browserId, userId);
|
||||
resetRecordingState(browserId, id);
|
||||
return null;
|
||||
}
|
||||
|
||||
} catch (error: any) {
|
||||
logger.error(`Error during readyForRunHandler: ${error.message}`);
|
||||
await destroyRemoteBrowser(browserId);
|
||||
await destroyRemoteBrowser(browserId, userId);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
@@ -546,7 +546,7 @@ function AddGeneratedFlags(workflow: WorkflowFile) {
|
||||
return copy;
|
||||
};
|
||||
|
||||
async function executeRun(id: string) {
|
||||
async function executeRun(id: string, userId: string) {
|
||||
try {
|
||||
const run = await Run.findOne({ where: { runId: id } });
|
||||
if (!run) {
|
||||
@@ -568,7 +568,7 @@ async function executeRun(id: string) {
|
||||
|
||||
plainRun.status = 'running';
|
||||
|
||||
const browser = browserPool.getRemoteBrowser(plainRun.browserId);
|
||||
const browser = browserPool.getRemoteBrowser(userId);
|
||||
if (!browser) {
|
||||
throw new Error('Could not access browser');
|
||||
}
|
||||
@@ -586,7 +586,7 @@ async function executeRun(id: string) {
|
||||
const binaryOutputService = new BinaryOutputService('maxun-run-screenshots');
|
||||
const uploadedBinaryOutput = await binaryOutputService.uploadAndStoreBinaryOutput(run, interpretationInfo.binaryOutput);
|
||||
|
||||
await destroyRemoteBrowser(plainRun.browserId);
|
||||
await destroyRemoteBrowser(plainRun.browserId, userId);
|
||||
|
||||
const updatedRun = await run.update({
|
||||
...run,
|
||||
@@ -672,12 +672,12 @@ export async function handleRunRecording(id: string, userId: string) {
|
||||
rejectUnauthorized: false
|
||||
});
|
||||
|
||||
socket.on('ready-for-run', () => readyForRunHandler(browserId, newRunId));
|
||||
socket.on('ready-for-run', () => readyForRunHandler(browserId, newRunId, userId));
|
||||
|
||||
logger.log('info', `Running Robot: ${id}`);
|
||||
|
||||
socket.on('disconnect', () => {
|
||||
cleanupSocketListeners(socket, browserId, newRunId);
|
||||
cleanupSocketListeners(socket, browserId, newRunId, userId);
|
||||
});
|
||||
|
||||
// Return the runId immediately, so the client knows the run is started
|
||||
@@ -688,8 +688,8 @@ export async function handleRunRecording(id: string, userId: string) {
|
||||
}
|
||||
}
|
||||
|
||||
function cleanupSocketListeners(socket: Socket, browserId: string, id: string) {
|
||||
socket.off('ready-for-run', () => readyForRunHandler(browserId, id));
|
||||
function cleanupSocketListeners(socket: Socket, browserId: string, id: string, userId: string) {
|
||||
socket.off('ready-for-run', () => readyForRunHandler(browserId, id, userId));
|
||||
logger.log('info', `Cleaned up listeners for browserId: ${browserId}, runId: ${id}`);
|
||||
}
|
||||
|
||||
|
||||
@@ -15,6 +15,10 @@ interface BrowserPoolInfo {
|
||||
* @default false
|
||||
*/
|
||||
active: boolean,
|
||||
/**
|
||||
* The user ID that owns this browser instance.
|
||||
*/
|
||||
userId: string,
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -29,36 +33,101 @@ interface PoolDictionary {
|
||||
|
||||
/**
|
||||
* A browser pool is a collection of remote browsers that are initialized and ready to be used.
|
||||
* Enforces a "1 User - 1 Browser" policy, while allowing multiple users to have their own browser instances.
|
||||
* Adds the possibility to add, remove and retrieve remote browsers from the pool.
|
||||
* It is possible to manage multiple browsers for creating or running a recording.
|
||||
* @category BrowserManagement
|
||||
*/
|
||||
export class BrowserPool {
|
||||
|
||||
/**
|
||||
* Holds all the instances of remote browsers.
|
||||
*/
|
||||
private pool: PoolDictionary = {};
|
||||
|
||||
/**
|
||||
* Adds a remote browser instance to the pool indexed by the id.
|
||||
* Maps user IDs to their browser IDs.
|
||||
*/
|
||||
private userToBrowserMap: Map<string, string> = new Map();
|
||||
|
||||
/**
|
||||
* Adds a remote browser instance to the pool for a specific user.
|
||||
* If the user already has a browser, the existing browser will be closed and replaced.
|
||||
*
|
||||
* @param id remote browser instance's id
|
||||
* @param browser remote browser instance
|
||||
* @param userId the user ID that owns this browser instance
|
||||
* @param active states if the browser's instance is being actively used
|
||||
* @returns true if a new browser was added, false if an existing browser was replaced
|
||||
*/
|
||||
public addRemoteBrowser = (id: string, browser: RemoteBrowser, active: boolean = false): void => {
|
||||
this.pool = {
|
||||
...this.pool,
|
||||
[id]: {
|
||||
browser,
|
||||
active,
|
||||
},
|
||||
public addRemoteBrowser = (
|
||||
id: string,
|
||||
browser: RemoteBrowser,
|
||||
userId: string,
|
||||
active: boolean = false
|
||||
): boolean => {
|
||||
// Check if user already has a browser
|
||||
const existingBrowserId = this.userToBrowserMap.get(userId);
|
||||
let replaced = false;
|
||||
|
||||
if (existingBrowserId) {
|
||||
// Close and remove the existing browser
|
||||
if (existingBrowserId !== id) {
|
||||
this.closeAndDeleteBrowser(existingBrowserId);
|
||||
replaced = true;
|
||||
} else {
|
||||
// If it's the same browser ID, just update the info
|
||||
this.pool[id] = {
|
||||
browser,
|
||||
active,
|
||||
userId,
|
||||
};
|
||||
logger.log('debug', `Updated existing browser with id: ${id} for user: ${userId}`);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
logger.log('debug', `Remote browser with id: ${id} added to the pool`);
|
||||
|
||||
// Add the new browser to the pool
|
||||
this.pool[id] = {
|
||||
browser,
|
||||
active,
|
||||
userId,
|
||||
};
|
||||
|
||||
// Update the user-to-browser mapping
|
||||
this.userToBrowserMap.set(userId, id);
|
||||
|
||||
logger.log('debug', `Remote browser with id: ${id} added to the pool for user: ${userId}`);
|
||||
return !replaced;
|
||||
};
|
||||
|
||||
/**
|
||||
* Removes the remote browser instance from the pool.
|
||||
* Note: This doesn't handle browser closing as RemoteBrowser doesn't expose a close method.
|
||||
* The caller should ensure the browser is properly closed before calling this method.
|
||||
*
|
||||
* @param id remote browser instance's id
|
||||
* @returns true if the browser was removed successfully, false otherwise
|
||||
*/
|
||||
public closeAndDeleteBrowser = (id: string): boolean => {
|
||||
if (!this.pool[id]) {
|
||||
logger.log('warn', `Remote browser with id: ${id} does not exist in the pool`);
|
||||
return false;
|
||||
}
|
||||
|
||||
// Remove the user-to-browser mapping
|
||||
const userId = this.pool[id].userId;
|
||||
if (this.userToBrowserMap.get(userId) === id) {
|
||||
this.userToBrowserMap.delete(userId);
|
||||
}
|
||||
|
||||
// Remove from pool
|
||||
delete this.pool[id];
|
||||
logger.log('debug', `Remote browser with id: ${id} removed from the pool`);
|
||||
return true;
|
||||
};
|
||||
|
||||
/**
|
||||
* Removes the remote browser instance from the pool without attempting to close it.
|
||||
*
|
||||
* @param id remote browser instance's id
|
||||
* @returns true if the browser was removed successfully, false otherwise
|
||||
*/
|
||||
@@ -67,13 +136,22 @@ export class BrowserPool {
|
||||
logger.log('warn', `Remote browser with id: ${id} does not exist in the pool`);
|
||||
return false;
|
||||
}
|
||||
delete (this.pool[id]);
|
||||
|
||||
// Remove the user-to-browser mapping
|
||||
const userId = this.pool[id].userId;
|
||||
if (this.userToBrowserMap.get(userId) === id) {
|
||||
this.userToBrowserMap.delete(userId);
|
||||
}
|
||||
|
||||
// Remove from pool
|
||||
delete this.pool[id];
|
||||
logger.log('debug', `Remote browser with id: ${id} deleted from the pool`);
|
||||
return true;
|
||||
};
|
||||
|
||||
/**
|
||||
* Returns the remote browser instance from the pool.
|
||||
*
|
||||
* @param id remote browser instance's id
|
||||
* @returns remote browser instance or undefined if it does not exist in the pool
|
||||
*/
|
||||
@@ -83,18 +161,154 @@ export class BrowserPool {
|
||||
};
|
||||
|
||||
/**
|
||||
* Returns the active browser's instance id from the pool.
|
||||
* If there is no active browser, it returns undefined.
|
||||
* If there are multiple active browsers, it returns the first one.
|
||||
* @returns the first remote active browser instance's id from the pool
|
||||
* Returns the active browser's instance id for a specific user.
|
||||
*
|
||||
* @param userId the user ID to find the browser for
|
||||
* @returns the browser ID for the user, or null if no browser exists
|
||||
*/
|
||||
public getActiveBrowserId = (): string | null => {
|
||||
public getActiveBrowserId = (userId: string): string | null => {
|
||||
const browserId = this.userToBrowserMap.get(userId);
|
||||
if (!browserId) {
|
||||
logger.log('debug', `No browser found for user: ${userId}`);
|
||||
return null;
|
||||
}
|
||||
|
||||
// Verify the browser still exists in the pool
|
||||
if (!this.pool[browserId]) {
|
||||
this.userToBrowserMap.delete(userId);
|
||||
logger.log('warn', `Browser mapping found for user: ${userId}, but browser doesn't exist in pool`);
|
||||
return null;
|
||||
}
|
||||
console.log(`Browser Id ${browserId} found for user: ${userId}`);
|
||||
return browserId;
|
||||
};
|
||||
|
||||
/**
|
||||
* Returns the user ID associated with a browser ID.
|
||||
*
|
||||
* @param browserId the browser ID to find the user for
|
||||
* @returns the user ID for the browser, or null if the browser doesn't exist
|
||||
*/
|
||||
public getUserForBrowser = (browserId: string): string | null => {
|
||||
if (!this.pool[browserId]) {
|
||||
return null;
|
||||
}
|
||||
return this.pool[browserId].userId;
|
||||
};
|
||||
|
||||
/**
|
||||
* Sets the active state of a browser.
|
||||
*
|
||||
* @param id the browser ID
|
||||
* @param active the new active state
|
||||
* @returns true if successful, false if the browser wasn't found
|
||||
*/
|
||||
public setActiveBrowser = (id: string, active: boolean): boolean => {
|
||||
if (!this.pool[id]) {
|
||||
logger.log('warn', `Remote browser with id: ${id} does not exist in the pool`);
|
||||
return false;
|
||||
}
|
||||
|
||||
this.pool[id].active = active;
|
||||
logger.log('debug', `Remote browser with id: ${id} set to ${active ? 'active' : 'inactive'}`);
|
||||
return true;
|
||||
};
|
||||
|
||||
/**
|
||||
* Returns all browser instances for a specific user.
|
||||
* Should only be one per the "1 User - 1 Browser" policy, but included for flexibility.
|
||||
*
|
||||
* @param userId the user ID to find browsers for
|
||||
* @returns an array of browser IDs belonging to the user
|
||||
*/
|
||||
public getAllBrowserIdsForUser = (userId: string): string[] => {
|
||||
const browserIds: string[] = [];
|
||||
|
||||
// Normally this would just return the one browser from the map
|
||||
const mappedBrowserId = this.userToBrowserMap.get(userId);
|
||||
if (mappedBrowserId && this.pool[mappedBrowserId]) {
|
||||
browserIds.push(mappedBrowserId);
|
||||
}
|
||||
|
||||
// But as a safeguard, also check the entire pool for any browsers assigned to this user
|
||||
// This helps detect and fix any inconsistencies in the maps
|
||||
for (const [id, info] of Object.entries(this.pool)) {
|
||||
if (info.userId === userId && !browserIds.includes(id)) {
|
||||
browserIds.push(id);
|
||||
// Fix the map if it's inconsistent
|
||||
if (!mappedBrowserId) {
|
||||
this.userToBrowserMap.set(userId, id);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return browserIds;
|
||||
};
|
||||
|
||||
/**
|
||||
* Returns the total number of browsers in the pool.
|
||||
*/
|
||||
public getPoolSize = (): number => {
|
||||
return Object.keys(this.pool).length;
|
||||
};
|
||||
|
||||
/**
|
||||
* Returns the total number of active users (users with browsers).
|
||||
*/
|
||||
public getActiveUserCount = (): number => {
|
||||
return this.userToBrowserMap.size;
|
||||
};
|
||||
|
||||
/**
|
||||
* Gets the current active browser for the system if there's only one active user.
|
||||
* This is a migration helper to support code that hasn't been updated to the user-browser model yet.
|
||||
*
|
||||
* @param currentUserId The ID of the current user, which will be prioritized if multiple browsers exist
|
||||
* @returns A browser ID if one can be determined, or null
|
||||
*/
|
||||
public getActiveBrowserForMigration = (currentUserId?: string): string | null => {
|
||||
// If a current user ID is provided and they have a browser, return that
|
||||
if (currentUserId) {
|
||||
const browserForUser = this.getActiveBrowserId(currentUserId);
|
||||
if (browserForUser) {
|
||||
return browserForUser;
|
||||
}
|
||||
}
|
||||
|
||||
// If only one user has a browser, return that
|
||||
if (this.userToBrowserMap.size === 1) {
|
||||
const userId = Array.from(this.userToBrowserMap.keys())[0];
|
||||
return this.userToBrowserMap.get(userId) || null;
|
||||
}
|
||||
|
||||
// Fall back to the first active browser if any
|
||||
for (const id of Object.keys(this.pool)) {
|
||||
if (this.pool[id].active) {
|
||||
return id;
|
||||
}
|
||||
}
|
||||
logger.log('warn', `No active browser in the pool`);
|
||||
|
||||
// If all else fails, return the first browser in the pool
|
||||
const browserIds = Object.keys(this.pool);
|
||||
return browserIds.length > 0 ? browserIds[0] : null;
|
||||
};
|
||||
|
||||
/**
|
||||
* Returns the first active browser's instance id from the pool.
|
||||
* If there is no active browser, it returns null.
|
||||
* If there are multiple active browsers, it returns the first one.
|
||||
*
|
||||
* @returns the first remote active browser instance's id from the pool
|
||||
* @deprecated Use getBrowserIdForUser instead to enforce the 1 User - 1 Browser policy
|
||||
*/
|
||||
public getActiveBrowserIdLegacy = (): string | null => {
|
||||
for (const id of Object.keys(this.pool)) {
|
||||
if (this.pool[id].active) {
|
||||
return id;
|
||||
}
|
||||
}
|
||||
// Don't log a warning since this behavior is expected in the user-browser model
|
||||
// logger.log('warn', `No active browser in the pool`);
|
||||
return null;
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -90,6 +90,12 @@ export class RemoteBrowser {
|
||||
maxRepeats: 1,
|
||||
};
|
||||
|
||||
/**
|
||||
* The user ID that owns this browser instance
|
||||
* @private
|
||||
*/
|
||||
private userId: string;
|
||||
|
||||
private lastEmittedUrl: string | null = null;
|
||||
|
||||
/**
|
||||
@@ -106,6 +112,7 @@ export class RemoteBrowser {
|
||||
private screenshotQueue: Buffer[] = [];
|
||||
private isProcessingScreenshot = false;
|
||||
private screencastInterval: NodeJS.Timeout | null = null
|
||||
private isScreencastActive: boolean = false;
|
||||
|
||||
/**
|
||||
* Initializes a new instances of the {@link Generator} and {@link WorkflowInterpreter} classes and
|
||||
@@ -113,8 +120,9 @@ export class RemoteBrowser {
|
||||
* @param socket socket.io socket instance used to communicate with the client side
|
||||
* @constructor
|
||||
*/
|
||||
public constructor(socket: Socket) {
|
||||
public constructor(socket: Socket, userId: string) {
|
||||
this.socket = socket;
|
||||
this.userId = userId;
|
||||
this.interpreter = new WorkflowInterpreter(socket);
|
||||
this.generator = new WorkflowGenerator(socket);
|
||||
}
|
||||
@@ -193,7 +201,7 @@ export class RemoteBrowser {
|
||||
const currentUrl = page.url();
|
||||
if (this.shouldEmitUrlChange(currentUrl)) {
|
||||
this.lastEmittedUrl = currentUrl;
|
||||
this.socket.emit('urlChanged', currentUrl);
|
||||
this.socket.emit('urlChanged', {url: currentUrl, userId: this.userId});
|
||||
}
|
||||
}
|
||||
});
|
||||
@@ -333,15 +341,40 @@ export class RemoteBrowser {
|
||||
* @returns void
|
||||
*/
|
||||
public registerEditorEvents = (): void => {
|
||||
this.socket.on('rerender', async () => await this.makeAndEmitScreenshot());
|
||||
this.socket.on('settings', (settings) => this.interpreterSettings = settings);
|
||||
this.socket.on('changeTab', async (tabIndex) => await this.changeTab(tabIndex));
|
||||
this.socket.on('addTab', async () => {
|
||||
// For each event, include userId to make sure events are handled for the correct browser
|
||||
logger.log('debug', `Registering editor events for user: ${this.userId}`);
|
||||
|
||||
// Listen for specific events for this user
|
||||
this.socket.on(`rerender:${this.userId}`, async () => {
|
||||
logger.debug(`Rerender event received for user ${this.userId}`);
|
||||
await this.makeAndEmitScreenshot();
|
||||
});
|
||||
|
||||
// For backward compatibility, also listen to the general event
|
||||
this.socket.on('rerender', async () => {
|
||||
logger.debug(`General rerender event received, checking if for user ${this.userId}`);
|
||||
await this.makeAndEmitScreenshot();
|
||||
});
|
||||
|
||||
this.socket.on(`settings:${this.userId}`, (settings) => {
|
||||
this.interpreterSettings = settings;
|
||||
logger.debug(`Settings updated for user ${this.userId}`);
|
||||
});
|
||||
|
||||
this.socket.on(`changeTab:${this.userId}`, async (tabIndex) => {
|
||||
logger.debug(`Tab change to ${tabIndex} requested for user ${this.userId}`);
|
||||
await this.changeTab(tabIndex);
|
||||
});
|
||||
|
||||
this.socket.on(`addTab:${this.userId}`, async () => {
|
||||
logger.debug(`New tab requested for user ${this.userId}`);
|
||||
await this.currentPage?.context().newPage();
|
||||
const lastTabIndex = this.currentPage ? this.currentPage.context().pages().length - 1 : 0;
|
||||
await this.changeTab(lastTabIndex);
|
||||
});
|
||||
this.socket.on('closeTab', async (tabInfo) => {
|
||||
|
||||
this.socket.on(`closeTab:${this.userId}`, async (tabInfo) => {
|
||||
logger.debug(`Close tab ${tabInfo.index} requested for user ${this.userId}`);
|
||||
const page = this.currentPage?.context().pages()[tabInfo.index];
|
||||
if (page) {
|
||||
if (tabInfo.isCurrent) {
|
||||
@@ -356,24 +389,52 @@ export class RemoteBrowser {
|
||||
await page.close();
|
||||
logger.log(
|
||||
'debug',
|
||||
`${tabInfo.index} page was closed, new length of pages: ${this.currentPage?.context().pages().length}`
|
||||
)
|
||||
`Tab ${tabInfo.index} was closed for user ${this.userId}, new tab count: ${this.currentPage?.context().pages().length}`
|
||||
);
|
||||
} else {
|
||||
logger.log('error', `${tabInfo.index} index out of range of pages`)
|
||||
logger.log('error', `Tab index ${tabInfo.index} out of range for user ${this.userId}`);
|
||||
}
|
||||
});
|
||||
this.socket.on('setViewportSize', async (data: { width: number, height: number }) => {
|
||||
|
||||
this.socket.on(`setViewportSize:${this.userId}`, async (data: { width: number, height: number }) => {
|
||||
const { width, height } = data;
|
||||
logger.log('debug', `Received viewport size: width=${width}, height=${height}`);
|
||||
logger.log('debug', `Viewport size change to width=${width}, height=${height} requested for user ${this.userId}`);
|
||||
|
||||
// Update the browser context's viewport dynamically
|
||||
if (this.context && this.browser) {
|
||||
this.context = await this.browser.newContext({ viewport: { width, height } });
|
||||
logger.log('debug', `Viewport size updated to width=${width}, height=${height} for the entire browser context`);
|
||||
logger.log('debug', `Viewport size updated to width=${width}, height=${height} for user ${this.userId}`);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
// For backward compatibility, also register the standard events
|
||||
this.socket.on('settings', (settings) => this.interpreterSettings = settings);
|
||||
this.socket.on('changeTab', async (tabIndex) => await this.changeTab(tabIndex));
|
||||
this.socket.on('addTab', async () => {
|
||||
await this.currentPage?.context().newPage();
|
||||
const lastTabIndex = this.currentPage ? this.currentPage.context().pages().length - 1 : 0;
|
||||
await this.changeTab(lastTabIndex);
|
||||
});
|
||||
this.socket.on('closeTab', async (tabInfo) => {
|
||||
const page = this.currentPage?.context().pages()[tabInfo.index];
|
||||
if (page) {
|
||||
if (tabInfo.isCurrent) {
|
||||
if (this.currentPage?.context().pages()[tabInfo.index + 1]) {
|
||||
await this.changeTab(tabInfo.index + 1);
|
||||
} else {
|
||||
await this.changeTab(tabInfo.index - 1);
|
||||
}
|
||||
}
|
||||
await page.close();
|
||||
}
|
||||
});
|
||||
this.socket.on('setViewportSize', async (data: { width: number, height: number }) => {
|
||||
const { width, height } = data;
|
||||
if (this.context && this.browser) {
|
||||
this.context = await this.browser.newContext({ viewport: { width, height } });
|
||||
}
|
||||
});
|
||||
};
|
||||
/**
|
||||
* Subscribes the remote browser for a screencast session
|
||||
* on [CDP](https://chromedevtools.github.io/devtools-protocol/) level,
|
||||
@@ -382,16 +443,24 @@ export class RemoteBrowser {
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
public subscribeToScreencast = async (): Promise<void> => {
|
||||
logger.log('debug', `Starting screencast for user: ${this.userId}`);
|
||||
await this.startScreencast();
|
||||
if (!this.client) {
|
||||
logger.log('warn', 'client is not initialized');
|
||||
return;
|
||||
}
|
||||
// Set flag to indicate screencast is active
|
||||
this.isScreencastActive = true;
|
||||
|
||||
this.client.on('Page.screencastFrame', ({ data: base64, sessionId }) => {
|
||||
// Only process if screencast is still active for this user
|
||||
if (!this.isScreencastActive) {
|
||||
return;
|
||||
}
|
||||
this.emitScreenshot(Buffer.from(base64, 'base64'))
|
||||
setTimeout(async () => {
|
||||
try {
|
||||
if (!this.client) {
|
||||
if (!this.client || !this.isScreencastActive) {
|
||||
logger.log('warn', 'client is not initialized');
|
||||
return;
|
||||
}
|
||||
@@ -410,6 +479,8 @@ export class RemoteBrowser {
|
||||
*/
|
||||
public async switchOff(): Promise<void> {
|
||||
try {
|
||||
this.isScreencastActive = false;
|
||||
|
||||
await this.interpreter.stopInterpretation();
|
||||
|
||||
if (this.screencastInterval) {
|
||||
@@ -553,7 +624,11 @@ export class RemoteBrowser {
|
||||
|
||||
//await this.currentPage.setViewportSize({ height: 400, width: 900 })
|
||||
this.client = await this.currentPage.context().newCDPSession(this.currentPage);
|
||||
this.socket.emit('urlChanged', this.currentPage.url());
|
||||
// Include userId in the URL change event
|
||||
this.socket.emit('urlChanged', {
|
||||
url: this.currentPage.url(),
|
||||
userId: this.userId
|
||||
});
|
||||
await this.makeAndEmitScreenshot();
|
||||
await this.subscribeToScreencast();
|
||||
} else {
|
||||
@@ -602,6 +677,8 @@ export class RemoteBrowser {
|
||||
await this.client.send('Page.startScreencast', {
|
||||
format: SCREENCAST_CONFIG.format,
|
||||
});
|
||||
// Set flag to indicate screencast is active
|
||||
this.isScreencastActive = true;
|
||||
|
||||
// Set up screencast frame handler
|
||||
this.client.on('Page.screencastFrame', async ({ data, sessionId }) => {
|
||||
@@ -627,6 +704,8 @@ export class RemoteBrowser {
|
||||
}
|
||||
|
||||
try {
|
||||
// Set flag to indicate screencast is active
|
||||
this.isScreencastActive = false;
|
||||
await this.client.send('Page.stopScreencast');
|
||||
this.screenshotQueue = [];
|
||||
this.isProcessingScreenshot = false;
|
||||
@@ -657,8 +736,11 @@ export class RemoteBrowser {
|
||||
const base64Data = optimizedScreenshot.toString('base64');
|
||||
const dataWithMimeType = `data:image/jpeg;base64,${base64Data}`;
|
||||
|
||||
this.socket.emit('screencast', dataWithMimeType);
|
||||
logger.debug('Screenshot emitted');
|
||||
// Emit with user context to ensure the frontend can identify which browser's screenshot this is
|
||||
this.socket.emit('screencast', {
|
||||
image: dataWithMimeType,
|
||||
userId: this.userId
|
||||
}); logger.debug('Screenshot emitted');
|
||||
} catch (error) {
|
||||
logger.error('Screenshot emission failed:', error);
|
||||
} finally {
|
||||
|
||||
@@ -21,23 +21,23 @@ import logger from "../logger";
|
||||
* @category BrowserManagement-Controller
|
||||
*/
|
||||
export const initializeRemoteBrowserForRecording = (userId: string): string => {
|
||||
const id = getActiveBrowserId() || uuid();
|
||||
const id = getActiveBrowserId(userId) || uuid();
|
||||
createSocketConnection(
|
||||
io.of(id),
|
||||
async (socket: Socket) => {
|
||||
// browser is already active
|
||||
const activeId = getActiveBrowserId();
|
||||
const activeId = getActiveBrowserId(userId);
|
||||
if (activeId) {
|
||||
const remoteBrowser = browserPool.getRemoteBrowser(activeId);
|
||||
remoteBrowser?.updateSocket(socket);
|
||||
await remoteBrowser?.makeAndEmitScreenshot();
|
||||
} else {
|
||||
const browserSession = new RemoteBrowser(socket);
|
||||
const browserSession = new RemoteBrowser(socket, userId);
|
||||
browserSession.interpreter.subscribeToPausing();
|
||||
await browserSession.initialize(userId);
|
||||
await browserSession.registerEditorEvents();
|
||||
await browserSession.subscribeToScreencast();
|
||||
browserPool.addRemoteBrowser(id, browserSession, true);
|
||||
browserPool.addRemoteBrowser(id, browserSession, userId);
|
||||
}
|
||||
socket.emit('loaded');
|
||||
});
|
||||
@@ -57,9 +57,9 @@ export const createRemoteBrowserForRun = (userId: string): string => {
|
||||
createSocketConnectionForRun(
|
||||
io.of(id),
|
||||
async (socket: Socket) => {
|
||||
const browserSession = new RemoteBrowser(socket);
|
||||
const browserSession = new RemoteBrowser(socket, userId);
|
||||
await browserSession.initialize(userId);
|
||||
browserPool.addRemoteBrowser(id, browserSession, true);
|
||||
browserPool.addRemoteBrowser(id, browserSession, userId);
|
||||
socket.emit('ready-for-run');
|
||||
});
|
||||
return id;
|
||||
@@ -72,7 +72,7 @@ export const createRemoteBrowserForRun = (userId: string): string => {
|
||||
* @returns {Promise<boolean>}
|
||||
* @category BrowserManagement-Controller
|
||||
*/
|
||||
export const destroyRemoteBrowser = async (id: string): Promise<boolean> => {
|
||||
export const destroyRemoteBrowser = async (id: string, userId: string): Promise<boolean> => {
|
||||
const browserSession = browserPool.getRemoteBrowser(id);
|
||||
if (browserSession) {
|
||||
logger.log('debug', `Switching off the browser with id: ${id}`);
|
||||
@@ -88,8 +88,8 @@ export const destroyRemoteBrowser = async (id: string): Promise<boolean> => {
|
||||
* @returns {string | null}
|
||||
* @category BrowserManagement-Controller
|
||||
*/
|
||||
export const getActiveBrowserId = (): string | null => {
|
||||
return browserPool.getActiveBrowserId();
|
||||
export const getActiveBrowserId = (userId: string): string | null => {
|
||||
return browserPool.getActiveBrowserId(userId);
|
||||
};
|
||||
|
||||
/**
|
||||
@@ -98,7 +98,7 @@ export const getActiveBrowserId = (): string | null => {
|
||||
* @returns {string | undefined}
|
||||
* @category BrowserManagement-Controller
|
||||
*/
|
||||
export const getRemoteBrowserCurrentUrl = (id: string): string | undefined => {
|
||||
export const getRemoteBrowserCurrentUrl = (id: string, userId: string): string | undefined => {
|
||||
return browserPool.getRemoteBrowser(id)?.getCurrentPage()?.url();
|
||||
};
|
||||
|
||||
@@ -108,7 +108,7 @@ export const getRemoteBrowserCurrentUrl = (id: string): string | undefined => {
|
||||
* @return {string[] | undefined}
|
||||
* @category BrowserManagement-Controller
|
||||
*/
|
||||
export const getRemoteBrowserCurrentTabs = (id: string): string[] | undefined => {
|
||||
export const getRemoteBrowserCurrentTabs = (id: string, userId: string): string[] | undefined => {
|
||||
return browserPool.getRemoteBrowser(id)?.getCurrentPage()?.context().pages()
|
||||
.map((page) => {
|
||||
const parsedUrl = new URL(page.url());
|
||||
@@ -126,8 +126,8 @@ export const getRemoteBrowserCurrentTabs = (id: string): string[] | undefined =>
|
||||
* @returns {Promise<void>}
|
||||
* @category BrowserManagement-Controller
|
||||
*/
|
||||
export const interpretWholeWorkflow = async () => {
|
||||
const id = getActiveBrowserId();
|
||||
export const interpretWholeWorkflow = async (userId: string) => {
|
||||
const id = getActiveBrowserId(userId);
|
||||
if (id) {
|
||||
const browser = browserPool.getRemoteBrowser(id);
|
||||
if (browser) {
|
||||
@@ -146,8 +146,8 @@ export const interpretWholeWorkflow = async () => {
|
||||
* @returns {Promise<void>}
|
||||
* @category BrowserManagement-Controller
|
||||
*/
|
||||
export const stopRunningInterpretation = async () => {
|
||||
const id = getActiveBrowserId();
|
||||
export const stopRunningInterpretation = async (userId: string) => {
|
||||
const id = getActiveBrowserId(userId);
|
||||
if (id) {
|
||||
const browser = browserPool.getRemoteBrowser(id);
|
||||
await browser?.stopCurrentInterpretation();
|
||||
|
||||
@@ -4,6 +4,8 @@
|
||||
* These functions are called by the client through socket communication.
|
||||
*/
|
||||
import { Socket } from 'socket.io';
|
||||
import { IncomingMessage } from 'http';
|
||||
import { JwtPayload } from 'jsonwebtoken';
|
||||
|
||||
import logger from "../logger";
|
||||
import { Coordinates, ScrollDeltas, KeyboardInput, DatePickerEventData } from '../types';
|
||||
@@ -13,6 +15,14 @@ import { Page } from "playwright";
|
||||
import { throttle } from "../../../src/helpers/inputHelpers";
|
||||
import { CustomActions } from "../../../src/shared/types";
|
||||
|
||||
interface AuthenticatedIncomingMessage extends IncomingMessage {
|
||||
user?: JwtPayload | string;
|
||||
}
|
||||
|
||||
interface AuthenticatedSocket extends Socket {
|
||||
request: AuthenticatedIncomingMessage;
|
||||
}
|
||||
|
||||
/**
|
||||
* A wrapper function for handling user input.
|
||||
* This function gets the active browser instance from the browser pool
|
||||
@@ -23,6 +33,7 @@ import { CustomActions } from "../../../src/shared/types";
|
||||
*
|
||||
* @param handleCallback The callback handler to be called
|
||||
* @param args - arguments to be passed to the handler
|
||||
* @param socket - socket with authenticated request
|
||||
* @category HelperFunctions
|
||||
*/
|
||||
const handleWrapper = async (
|
||||
@@ -31,9 +42,21 @@ const handleWrapper = async (
|
||||
page: Page,
|
||||
args?: any
|
||||
) => Promise<void>,
|
||||
args?: any
|
||||
args?: any,
|
||||
socket?: AuthenticatedSocket,
|
||||
) => {
|
||||
const id = browserPool.getActiveBrowserId();
|
||||
if (!socket || !socket.request || !socket.request.user || typeof socket.request.user === 'string') {
|
||||
logger.log('warn', `User not authenticated or invalid JWT payload`);
|
||||
return;
|
||||
}
|
||||
|
||||
const userId = socket.request.user.id;
|
||||
if (!userId) {
|
||||
logger.log('warn', `User ID is missing in JWT payload`);
|
||||
return;
|
||||
}
|
||||
|
||||
const id = browserPool.getActiveBrowserId(userId);
|
||||
if (id) {
|
||||
const activeBrowser = browserPool.getRemoteBrowser(id);
|
||||
if (activeBrowser?.interpreter.interpretationInProgress() && !activeBrowser.interpreter.interpretationIsPaused) {
|
||||
@@ -66,12 +89,13 @@ interface CustomActionEventData {
|
||||
|
||||
/**
|
||||
* A wrapper function for handling custom actions.
|
||||
* @param socket The socket connection
|
||||
* @param customActionEventData The custom action event data
|
||||
* @category HelperFunctions
|
||||
*/
|
||||
const onGenerateAction = async (customActionEventData: CustomActionEventData) => {
|
||||
const onGenerateAction = async (socket: AuthenticatedSocket, customActionEventData: CustomActionEventData) => {
|
||||
logger.log('debug', `Generating ${customActionEventData.action} action emitted from client`);
|
||||
await handleWrapper(handleGenerateAction, customActionEventData);
|
||||
await handleWrapper(handleGenerateAction, customActionEventData, socket);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -89,12 +113,13 @@ const handleGenerateAction =
|
||||
|
||||
/**
|
||||
* A wrapper function for handling mousedown event.
|
||||
* @param socket The socket connection
|
||||
* @param coordinates - coordinates of the mouse click
|
||||
* @category HelperFunctions
|
||||
*/
|
||||
const onMousedown = async (coordinates: Coordinates) => {
|
||||
const onMousedown = async (socket: AuthenticatedSocket, coordinates: Coordinates) => {
|
||||
logger.log('debug', 'Handling mousedown event emitted from client');
|
||||
await handleWrapper(handleMousedown, coordinates);
|
||||
await handleWrapper(handleMousedown, coordinates, socket);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -139,12 +164,13 @@ const handleMousedown = async (generator: WorkflowGenerator, page: Page, { x, y
|
||||
|
||||
/**
|
||||
* A wrapper function for handling the wheel event.
|
||||
* @param socket The socket connection
|
||||
* @param scrollDeltas - the scroll deltas of the wheel event
|
||||
* @category HelperFunctions
|
||||
*/
|
||||
const onWheel = async (scrollDeltas: ScrollDeltas) => {
|
||||
const onWheel = async (socket: AuthenticatedSocket, scrollDeltas: ScrollDeltas) => {
|
||||
logger.log('debug', 'Handling scroll event emitted from client');
|
||||
await handleWrapper(handleWheel, scrollDeltas);
|
||||
await handleWrapper(handleWheel, scrollDeltas, socket);
|
||||
};
|
||||
|
||||
/**
|
||||
@@ -165,12 +191,13 @@ const handleWheel = async (generator: WorkflowGenerator, page: Page, { deltaX, d
|
||||
|
||||
/**
|
||||
* A wrapper function for handling the mousemove event.
|
||||
* @param socket The socket connection
|
||||
* @param coordinates - the coordinates of the mousemove event
|
||||
* @category HelperFunctions
|
||||
*/
|
||||
const onMousemove = async (coordinates: Coordinates) => {
|
||||
const onMousemove = async (socket: AuthenticatedSocket, coordinates: Coordinates) => {
|
||||
logger.log('debug', 'Handling mousemove event emitted from client');
|
||||
await handleWrapper(handleMousemove, coordinates);
|
||||
await handleWrapper(handleMousemove, coordinates, socket);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -199,12 +226,13 @@ const handleMousemove = async (generator: WorkflowGenerator, page: Page, { x, y
|
||||
|
||||
/**
|
||||
* A wrapper function for handling the keydown event.
|
||||
* @param socket The socket connection
|
||||
* @param keyboardInput - the keyboard input of the keydown event
|
||||
* @category HelperFunctions
|
||||
*/
|
||||
const onKeydown = async (keyboardInput: KeyboardInput) => {
|
||||
const onKeydown = async (socket: AuthenticatedSocket, keyboardInput: KeyboardInput) => {
|
||||
logger.log('debug', 'Handling keydown event emitted from client');
|
||||
await handleWrapper(handleKeydown, keyboardInput);
|
||||
await handleWrapper(handleKeydown, keyboardInput, socket);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -235,49 +263,95 @@ const handleDateSelection = async (generator: WorkflowGenerator, page: Page, dat
|
||||
logger.log('debug', `Date ${data.value} selected`);
|
||||
}
|
||||
|
||||
const onDateSelection = async (data: DatePickerEventData) => {
|
||||
/**
|
||||
* A wrapper function for handling the date selection event.
|
||||
* @param socket The socket connection
|
||||
* @param data - the data of the date selection event
|
||||
* @category HelperFunctions
|
||||
*/
|
||||
const onDateSelection = async (socket: AuthenticatedSocket, data: DatePickerEventData) => {
|
||||
logger.log('debug', 'Handling date selection event emitted from client');
|
||||
await handleWrapper(handleDateSelection, data);
|
||||
await handleWrapper(handleDateSelection, data, socket);
|
||||
}
|
||||
|
||||
/**
|
||||
* Handles the dropdown selection event.
|
||||
* @param generator - the workflow generator {@link Generator}
|
||||
* @param page - the active page of the remote browser
|
||||
* @param data - the data of the dropdown selection event
|
||||
* @category BrowserManagement
|
||||
*/
|
||||
const handleDropdownSelection = async (generator: WorkflowGenerator, page: Page, data: { selector: string, value: string }) => {
|
||||
await generator.onDropdownSelection(page, data);
|
||||
logger.log('debug', `Dropdown value ${data.value} selected`);
|
||||
}
|
||||
|
||||
const onDropdownSelection = async (data: { selector: string, value: string }) => {
|
||||
/**
|
||||
* A wrapper function for handling the dropdown selection event.
|
||||
* @param socket The socket connection
|
||||
* @param data - the data of the dropdown selection event
|
||||
* @category HelperFunctions
|
||||
*/
|
||||
const onDropdownSelection = async (socket: AuthenticatedSocket, data: { selector: string, value: string }) => {
|
||||
logger.log('debug', 'Handling dropdown selection event emitted from client');
|
||||
await handleWrapper(handleDropdownSelection, data);
|
||||
await handleWrapper(handleDropdownSelection, data, socket);
|
||||
}
|
||||
|
||||
/**
|
||||
* Handles the time selection event.
|
||||
* @param generator - the workflow generator {@link Generator}
|
||||
* @param page - the active page of the remote browser
|
||||
* @param data - the data of the time selection event
|
||||
* @category BrowserManagement
|
||||
*/
|
||||
const handleTimeSelection = async (generator: WorkflowGenerator, page: Page, data: { selector: string, value: string }) => {
|
||||
await generator.onTimeSelection(page, data);
|
||||
logger.log('debug', `Time value ${data.value} selected`);
|
||||
}
|
||||
|
||||
const onTimeSelection = async (data: { selector: string, value: string }) => {
|
||||
/**
|
||||
* A wrapper function for handling the time selection event.
|
||||
* @param socket The socket connection
|
||||
* @param data - the data of the time selection event
|
||||
* @category HelperFunctions
|
||||
*/
|
||||
const onTimeSelection = async (socket: AuthenticatedSocket, data: { selector: string, value: string }) => {
|
||||
logger.log('debug', 'Handling time selection event emitted from client');
|
||||
await handleWrapper(handleTimeSelection, data);
|
||||
await handleWrapper(handleTimeSelection, data, socket);
|
||||
}
|
||||
|
||||
/**
|
||||
* Handles the datetime-local selection event.
|
||||
* @param generator - the workflow generator {@link Generator}
|
||||
* @param page - the active page of the remote browser
|
||||
* @param data - the data of the datetime-local selection event
|
||||
* @category BrowserManagement
|
||||
*/
|
||||
const handleDateTimeLocalSelection = async (generator: WorkflowGenerator, page: Page, data: { selector: string, value: string }) => {
|
||||
await generator.onDateTimeLocalSelection(page, data);
|
||||
logger.log('debug', `DateTime Local value ${data.value} selected`);
|
||||
}
|
||||
|
||||
const onDateTimeLocalSelection = async (data: { selector: string, value: string }) => {
|
||||
/**
|
||||
* A wrapper function for handling the datetime-local selection event.
|
||||
* @param socket The socket connection
|
||||
* @param data - the data of the datetime-local selection event
|
||||
* @category HelperFunctions
|
||||
*/
|
||||
const onDateTimeLocalSelection = async (socket: AuthenticatedSocket, data: { selector: string, value: string }) => {
|
||||
logger.log('debug', 'Handling datetime-local selection event emitted from client');
|
||||
await handleWrapper(handleDateTimeLocalSelection, data);
|
||||
await handleWrapper(handleDateTimeLocalSelection, data, socket);
|
||||
}
|
||||
|
||||
/**
|
||||
* A wrapper function for handling the keyup event.
|
||||
* @param socket The socket connection
|
||||
* @param keyboardInput - the keyboard input of the keyup event
|
||||
* @category HelperFunctions
|
||||
*/
|
||||
const onKeyup = async (keyboardInput: KeyboardInput) => {
|
||||
const onKeyup = async (socket: AuthenticatedSocket, keyboardInput: KeyboardInput) => {
|
||||
logger.log('debug', 'Handling keyup event emitted from client');
|
||||
await handleWrapper(handleKeyup, keyboardInput);
|
||||
await handleWrapper(handleKeyup, keyboardInput, socket);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -296,12 +370,13 @@ const handleKeyup = async (generator: WorkflowGenerator, page: Page, key: string
|
||||
|
||||
/**
|
||||
* A wrapper function for handling the url change event.
|
||||
* @param socket The socket connection
|
||||
* @param url - the new url of the page
|
||||
* @category HelperFunctions
|
||||
*/
|
||||
const onChangeUrl = async (url: string) => {
|
||||
const onChangeUrl = async (socket: AuthenticatedSocket, url: string) => {
|
||||
logger.log('debug', 'Handling change url event emitted from client');
|
||||
await handleWrapper(handleChangeUrl, url);
|
||||
await handleWrapper(handleChangeUrl, url, socket);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -329,11 +404,12 @@ const handleChangeUrl = async (generator: WorkflowGenerator, page: Page, url: st
|
||||
|
||||
/**
|
||||
* A wrapper function for handling the refresh event.
|
||||
* @param socket The socket connection
|
||||
* @category HelperFunctions
|
||||
*/
|
||||
const onRefresh = async () => {
|
||||
const onRefresh = async (socket: AuthenticatedSocket) => {
|
||||
logger.log('debug', 'Handling refresh event emitted from client');
|
||||
await handleWrapper(handleRefresh);
|
||||
await handleWrapper(handleRefresh, undefined, socket);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -350,11 +426,12 @@ const handleRefresh = async (generator: WorkflowGenerator, page: Page) => {
|
||||
|
||||
/**
|
||||
* A wrapper function for handling the go back event.
|
||||
* @param socket The socket connection
|
||||
* @category HelperFunctions
|
||||
*/
|
||||
const onGoBack = async () => {
|
||||
logger.log('debug', 'Handling refresh event emitted from client');
|
||||
await handleWrapper(handleGoBack);
|
||||
const onGoBack = async (socket: AuthenticatedSocket) => {
|
||||
logger.log('debug', 'Handling go back event emitted from client');
|
||||
await handleWrapper(handleGoBack, undefined, socket);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -372,11 +449,12 @@ const handleGoBack = async (generator: WorkflowGenerator, page: Page) => {
|
||||
|
||||
/**
|
||||
* A wrapper function for handling the go forward event.
|
||||
* @param socket The socket connection
|
||||
* @category HelperFunctions
|
||||
*/
|
||||
const onGoForward = async () => {
|
||||
logger.log('debug', 'Handling refresh event emitted from client');
|
||||
await handleWrapper(handleGoForward);
|
||||
const onGoForward = async (socket: AuthenticatedSocket) => {
|
||||
logger.log('debug', 'Handling go forward event emitted from client');
|
||||
await handleWrapper(handleGoForward, undefined, socket);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -394,18 +472,7 @@ const handleGoForward = async (generator: WorkflowGenerator, page: Page) => {
|
||||
|
||||
/**
|
||||
* Helper function for registering the handlers onto established websocket connection.
|
||||
* Registers:
|
||||
* - mousedownHandler
|
||||
* - wheelHandler
|
||||
* - mousemoveHandler
|
||||
* - keydownHandler
|
||||
* - keyupHandler
|
||||
* - changeUrlHandler
|
||||
* - refreshHandler
|
||||
* - goBackHandler
|
||||
* - goForwardHandler
|
||||
* - onGenerateAction
|
||||
* input handlers.
|
||||
* Registers various input handlers.
|
||||
*
|
||||
* All these handlers first generates the workflow pair data
|
||||
* and then calls the corresponding playwright's function to emulate the input.
|
||||
@@ -415,21 +482,25 @@ const handleGoForward = async (generator: WorkflowGenerator, page: Page) => {
|
||||
* @returns void
|
||||
* @category BrowserManagement
|
||||
*/
|
||||
const registerInputHandlers = (socket: Socket) => {
|
||||
socket.on("input:mousedown", onMousedown);
|
||||
socket.on("input:wheel", onWheel);
|
||||
socket.on("input:mousemove", onMousemove);
|
||||
socket.on("input:keydown", onKeydown);
|
||||
socket.on("input:keyup", onKeyup);
|
||||
socket.on("input:url", onChangeUrl);
|
||||
socket.on("input:refresh", onRefresh);
|
||||
socket.on("input:back", onGoBack);
|
||||
socket.on("input:forward", onGoForward);
|
||||
socket.on("input:date", onDateSelection);
|
||||
socket.on("input:dropdown", onDropdownSelection);
|
||||
socket.on("input:time", onTimeSelection);
|
||||
socket.on("input:datetime-local", onDateTimeLocalSelection);
|
||||
socket.on("action", onGenerateAction);
|
||||
const registerInputHandlers = (socket: Socket) => {
|
||||
// Cast to our authenticated socket type
|
||||
const authSocket = socket as AuthenticatedSocket;
|
||||
|
||||
// Register handlers with the socket
|
||||
socket.on("input:mousedown", (data) => onMousedown(authSocket, data));
|
||||
socket.on("input:wheel", (data) => onWheel(authSocket, data));
|
||||
socket.on("input:mousemove", (data) => onMousemove(authSocket, data));
|
||||
socket.on("input:keydown", (data) => onKeydown(authSocket, data));
|
||||
socket.on("input:keyup", (data) => onKeyup(authSocket, data));
|
||||
socket.on("input:url", (data) => onChangeUrl(authSocket, data));
|
||||
socket.on("input:refresh", () => onRefresh(authSocket));
|
||||
socket.on("input:back", () => onGoBack(authSocket));
|
||||
socket.on("input:forward", () => onGoForward(authSocket));
|
||||
socket.on("input:date", (data) => onDateSelection(authSocket, data));
|
||||
socket.on("input:dropdown", (data) => onDropdownSelection(authSocket, data));
|
||||
socket.on("input:time", (data) => onTimeSelection(authSocket, data));
|
||||
socket.on("input:datetime-local", (data) => onDateTimeLocalSelection(authSocket, data));
|
||||
socket.on("action", (data) => onGenerateAction(authSocket, data));
|
||||
};
|
||||
|
||||
export default registerInputHandlers;
|
||||
export default registerInputHandlers;
|
||||
@@ -182,18 +182,24 @@ router.get('/stop/:browserId', requireSignIn, async (req: AuthenticatedRequest,
|
||||
/**
|
||||
* GET endpoint for getting the id of the active remote browser.
|
||||
*/
|
||||
router.get('/active', requireSignIn, (req, res) => {
|
||||
const id = getActiveBrowserId();
|
||||
router.get('/active', requireSignIn, (req: AuthenticatedRequest, res) => {
|
||||
if (!req.user) {
|
||||
return res.status(401).send('User not authenticated');
|
||||
}
|
||||
const id = getActiveBrowserId(req.user?.id);
|
||||
return res.send(id);
|
||||
});
|
||||
|
||||
/**
|
||||
* GET endpoint for getting the current url of the active remote browser.
|
||||
*/
|
||||
router.get('/active/url', requireSignIn, (req, res) => {
|
||||
const id = getActiveBrowserId();
|
||||
router.get('/active/url', requireSignIn, (req: AuthenticatedRequest, res) => {
|
||||
if (!req.user) {
|
||||
return res.status(401).send('User not authenticated');
|
||||
}
|
||||
const id = getActiveBrowserId(req.user?.id);
|
||||
if (id) {
|
||||
const url = getRemoteBrowserCurrentUrl(id);
|
||||
const url = getRemoteBrowserCurrentUrl(id, req.user?.id);
|
||||
return res.send(url);
|
||||
}
|
||||
return res.send(null);
|
||||
@@ -202,10 +208,13 @@ router.get('/active/url', requireSignIn, (req, res) => {
|
||||
/**
|
||||
* GET endpoint for getting the current tabs of the active remote browser.
|
||||
*/
|
||||
router.get('/active/tabs', requireSignIn, (req, res) => {
|
||||
const id = getActiveBrowserId();
|
||||
router.get('/active/tabs', requireSignIn, (req: AuthenticatedRequest, res) => {
|
||||
if (!req.user) {
|
||||
return res.status(401).send('User not authenticated');
|
||||
}
|
||||
const id = getActiveBrowserId(req.user?.id);
|
||||
if (id) {
|
||||
const hosts = getRemoteBrowserCurrentTabs(id);
|
||||
const hosts = getRemoteBrowserCurrentTabs(id, req.user?.id);
|
||||
return res.send(hosts);
|
||||
}
|
||||
return res.send([]);
|
||||
@@ -219,7 +228,7 @@ router.get('/interpret', requireSignIn, async (req: AuthenticatedRequest, res) =
|
||||
if (!req.user) {
|
||||
return res.status(401).send('User not authenticated');
|
||||
}
|
||||
await interpretWholeWorkflow();
|
||||
await interpretWholeWorkflow(req.user?.id);
|
||||
return res.send('interpretation done');
|
||||
} catch (e) {
|
||||
return res.send('interpretation failed');
|
||||
@@ -233,7 +242,7 @@ router.get('/interpret/stop', requireSignIn, async (req: AuthenticatedRequest, r
|
||||
if (!req.user) {
|
||||
return res.status(401).send('User not authenticated');
|
||||
}
|
||||
await stopRunningInterpretation();
|
||||
await stopRunningInterpretation(req.user?.id);
|
||||
return res.send('interpretation stopped');
|
||||
});
|
||||
|
||||
|
||||
@@ -617,7 +617,7 @@ router.post('/runs/run/:id', requireSignIn, async (req: AuthenticatedRequest, re
|
||||
workflow, currentPage, (newPage: Page) => currentPage = newPage, plainRun.interpreterSettings);
|
||||
const binaryOutputService = new BinaryOutputService('maxun-run-screenshots');
|
||||
const uploadedBinaryOutput = await binaryOutputService.uploadAndStoreBinaryOutput(run, interpretationInfo.binaryOutput);
|
||||
await destroyRemoteBrowser(plainRun.browserId);
|
||||
await destroyRemoteBrowser(plainRun.browserId, req.user?.id);
|
||||
await run.update({
|
||||
...run,
|
||||
status: 'success',
|
||||
@@ -900,9 +900,13 @@ router.delete('/schedule/:id', requireSignIn, async (req: AuthenticatedRequest,
|
||||
/**
|
||||
* POST endpoint for aborting a current interpretation of the run.
|
||||
*/
|
||||
router.post('/runs/abort/:id', requireSignIn, async (req, res) => {
|
||||
router.post('/runs/abort/:id', requireSignIn, async (req: AuthenticatedRequest, res) => {
|
||||
try {
|
||||
const run = await Run.findOne({ where: { runId: req.params.id } });
|
||||
if (!req.user) { return res.status(401).send({ error: 'Unauthorized' }); }
|
||||
const run = await Run.findOne({ where: {
|
||||
runId: req.params.id,
|
||||
runByUserId: req.user.id,
|
||||
} });
|
||||
if (!run) {
|
||||
return res.status(404).send(false);
|
||||
}
|
||||
@@ -937,4 +941,4 @@ router.post('/runs/abort/:id', requireSignIn, async (req, res) => {
|
||||
logger.log('info', `Error while running a robot with name: ${req.params.fileName}_${req.params.runId}.json`);
|
||||
return res.send(false);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
@@ -7,6 +7,7 @@ import logger from "../logger";
|
||||
import { browserPool } from "../server";
|
||||
import { requireSignIn } from '../middlewares/auth';
|
||||
import Robot from '../models/Robot';
|
||||
import { AuthenticatedRequest } from './record';
|
||||
|
||||
export const router = Router();
|
||||
|
||||
@@ -46,8 +47,9 @@ router.get('/params/:browserId', requireSignIn, (req, res) => {
|
||||
/**
|
||||
* DELETE endpoint for deleting a pair from the generated workflow.
|
||||
*/
|
||||
router.delete('/pair/:index', requireSignIn, (req, res) => {
|
||||
const id = browserPool.getActiveBrowserId();
|
||||
router.delete('/pair/:index', requireSignIn, (req: AuthenticatedRequest, res) => {
|
||||
if (!req.user) { return res.status(401).send('User not authenticated'); }
|
||||
const id = browserPool.getActiveBrowserId(req.user?.id);
|
||||
if (id) {
|
||||
const browser = browserPool.getRemoteBrowser(id);
|
||||
if (browser) {
|
||||
@@ -62,8 +64,9 @@ router.delete('/pair/:index', requireSignIn, (req, res) => {
|
||||
/**
|
||||
* POST endpoint for adding a pair to the generated workflow.
|
||||
*/
|
||||
router.post('/pair/:index', requireSignIn, (req, res) => {
|
||||
const id = browserPool.getActiveBrowserId();
|
||||
router.post('/pair/:index', requireSignIn, (req: AuthenticatedRequest, res) => {
|
||||
if (!req.user) { return res.status(401).send('User not authenticated'); }
|
||||
const id = browserPool.getActiveBrowserId(req.user?.id);
|
||||
if (id) {
|
||||
const browser = browserPool.getRemoteBrowser(id);
|
||||
logger.log('debug', `Adding pair to workflow`);
|
||||
@@ -82,8 +85,9 @@ router.post('/pair/:index', requireSignIn, (req, res) => {
|
||||
/**
|
||||
* PUT endpoint for updating a pair in the generated workflow.
|
||||
*/
|
||||
router.put('/pair/:index', requireSignIn, (req, res) => {
|
||||
const id = browserPool.getActiveBrowserId();
|
||||
router.put('/pair/:index', requireSignIn, (req: AuthenticatedRequest, res) => {
|
||||
if (!req.user) { return res.status(401).send('User not authenticated'); }
|
||||
const id = browserPool.getActiveBrowserId(req.user?.id);
|
||||
if (id) {
|
||||
const browser = browserPool.getRemoteBrowser(id);
|
||||
logger.log('debug', `Updating pair in workflow`);
|
||||
|
||||
@@ -1,6 +1,60 @@
|
||||
import { Namespace, Socket } from 'socket.io';
|
||||
import { IncomingMessage } from 'http';
|
||||
import { verify, JwtPayload } from 'jsonwebtoken';
|
||||
import logger from "../logger";
|
||||
import registerInputHandlers from '../browser-management/inputHandlers'
|
||||
import registerInputHandlers from '../browser-management/inputHandlers';
|
||||
|
||||
interface AuthenticatedIncomingMessage extends IncomingMessage {
|
||||
user?: JwtPayload | string;
|
||||
}
|
||||
|
||||
interface AuthenticatedSocket extends Socket {
|
||||
request: AuthenticatedIncomingMessage;
|
||||
}
|
||||
|
||||
/**
|
||||
* Socket.io middleware for authentication
|
||||
* This is a socket.io specific auth handler that doesn't rely on Express middleware
|
||||
*/
|
||||
const socketAuthMiddleware = (socket: Socket, next: (err?: Error) => void) => {
|
||||
const cookies = socket.handshake.headers.cookie;
|
||||
if (!cookies) {
|
||||
return next(new Error('Authentication required'));
|
||||
}
|
||||
|
||||
const tokenMatch = cookies.split(';').find(c => c.trim().startsWith('token='));
|
||||
if (!tokenMatch) {
|
||||
return next(new Error('Authentication required'));
|
||||
}
|
||||
|
||||
const token = tokenMatch.split('=')[1];
|
||||
if (!token) {
|
||||
return next(new Error('Authentication required'));
|
||||
}
|
||||
|
||||
const secret = process.env.JWT_SECRET;
|
||||
if (!secret) {
|
||||
return next(new Error('Server configuration error'));
|
||||
}
|
||||
|
||||
verify(token, secret, (err: any, user: any) => {
|
||||
if (err) {
|
||||
logger.log('warn', 'JWT verification error:', err);
|
||||
return next(new Error('Authentication failed'));
|
||||
}
|
||||
|
||||
// Normalize payload key
|
||||
if (user.userId && !user.id) {
|
||||
user.id = user.userId;
|
||||
delete user.userId; // temporary: del the old key for clarity
|
||||
}
|
||||
|
||||
// Attach user to socket request
|
||||
const authSocket = socket as AuthenticatedSocket;
|
||||
authSocket.request.user = user;
|
||||
next();
|
||||
});
|
||||
};
|
||||
|
||||
/**
|
||||
* Opens a websocket canal for duplex data transfer and registers all handlers for this data for the recording session.
|
||||
@@ -13,6 +67,8 @@ export const createSocketConnection = (
|
||||
io: Namespace,
|
||||
callback: (socket: Socket) => void,
|
||||
) => {
|
||||
io.use(socketAuthMiddleware);
|
||||
|
||||
const onConnection = async (socket: Socket) => {
|
||||
logger.log('info', "Client connected " + socket.id);
|
||||
registerInputHandlers(socket);
|
||||
@@ -34,6 +90,8 @@ export const createSocketConnectionForRun = (
|
||||
io: Namespace,
|
||||
callback: (socket: Socket) => void,
|
||||
) => {
|
||||
io.use(socketAuthMiddleware);
|
||||
|
||||
const onConnection = async (socket: Socket) => {
|
||||
logger.log('info', "Client connected " + socket.id);
|
||||
socket.on('disconnect', () => logger.log('info', "Client disconnected " + socket.id));
|
||||
@@ -41,4 +99,4 @@ export const createSocketConnectionForRun = (
|
||||
}
|
||||
|
||||
io.on('connection', onConnection);
|
||||
};
|
||||
};
|
||||
@@ -151,8 +151,8 @@ export class WorkflowGenerator {
|
||||
workflow: [],
|
||||
});
|
||||
socket.on('activeIndex', (data) => this.generatedData.lastIndex = parseInt(data));
|
||||
socket.on('decision', async ({ pair, actionType, decision }) => {
|
||||
const id = browserPool.getActiveBrowserId();
|
||||
socket.on('decision', async ({ pair, actionType, decision, userId }) => {
|
||||
const id = browserPool.getActiveBrowserId(userId);
|
||||
if (id) {
|
||||
// const activeBrowser = browserPool.getRemoteBrowser(id);
|
||||
// const currentPage = activeBrowser?.getCurrentPage();
|
||||
@@ -826,6 +826,7 @@ export class WorkflowGenerator {
|
||||
selectors?.testIdSelector,
|
||||
selectors?.id,
|
||||
selectors?.hrefSelector,
|
||||
selectors?.relSelector,
|
||||
selectors?.accessibilitySelector,
|
||||
selectors?.attrSelector
|
||||
]
|
||||
|
||||
@@ -49,11 +49,7 @@ export async function updateGoogleSheet(robotId: string, runId: string) {
|
||||
if (plainRobot.google_sheet_email && spreadsheetId) {
|
||||
console.log(`Preparing to write data to Google Sheet for robot: ${robotId}, spreadsheetId: ${spreadsheetId}`);
|
||||
|
||||
const headers = Object.keys(data[0]);
|
||||
const rows = data.map((row: { [key: string]: any }) => Object.values(row));
|
||||
const outputData = [headers, ...rows];
|
||||
|
||||
await writeDataToSheet(robotId, spreadsheetId, outputData);
|
||||
await writeDataToSheet(robotId, spreadsheetId, data);
|
||||
console.log(`Data written to Google Sheet successfully for Robot: ${robotId} and Run: ${runId}`);
|
||||
} else {
|
||||
console.log('Google Sheets integration not configured.');
|
||||
@@ -102,7 +98,43 @@ export async function writeDataToSheet(robotId: string, spreadsheetId: string, d
|
||||
|
||||
const sheets = google.sheets({ version: 'v4', auth: oauth2Client });
|
||||
|
||||
const resource = { values: data };
|
||||
const checkResponse = await sheets.spreadsheets.values.get({
|
||||
spreadsheetId,
|
||||
range: 'Sheet1!1:1',
|
||||
});
|
||||
|
||||
if (!data || data.length === 0) {
|
||||
console.log('No data to write. Exiting early.');
|
||||
return;
|
||||
}
|
||||
|
||||
const expectedHeaders = Object.keys(data[0]);
|
||||
|
||||
const rows = data.map(item => Object.values(item));
|
||||
|
||||
const existingHeaders =
|
||||
checkResponse.data.values &&
|
||||
checkResponse.data.values[0] ?
|
||||
checkResponse.data.values[0].map(String) :
|
||||
[];
|
||||
|
||||
const isSheetEmpty = existingHeaders.length === 0;
|
||||
|
||||
const headersMatch =
|
||||
!isSheetEmpty &&
|
||||
existingHeaders.length === expectedHeaders.length &&
|
||||
expectedHeaders.every((header, index) => existingHeaders[index] === header);
|
||||
|
||||
let resource;
|
||||
|
||||
if (isSheetEmpty || !headersMatch) {
|
||||
resource = { values: [expectedHeaders, ...rows] };
|
||||
console.log('Including headers in the append operation.');
|
||||
} else {
|
||||
resource = { values: rows };
|
||||
console.log('Headers already exist and match, only appending data rows.');
|
||||
}
|
||||
|
||||
console.log('Attempting to write to spreadsheet:', spreadsheetId);
|
||||
|
||||
const response = await sheets.spreadsheets.values.append({
|
||||
|
||||
@@ -92,7 +92,7 @@ function AddGeneratedFlags(workflow: WorkflowFile) {
|
||||
return copy;
|
||||
};
|
||||
|
||||
async function executeRun(id: string) {
|
||||
async function executeRun(id: string, userId: string) {
|
||||
try {
|
||||
const run = await Run.findOne({ where: { runId: id } });
|
||||
if (!run) {
|
||||
@@ -114,7 +114,7 @@ async function executeRun(id: string) {
|
||||
|
||||
plainRun.status = 'running';
|
||||
|
||||
const browser = browserPool.getRemoteBrowser(plainRun.browserId);
|
||||
const browser = browserPool.getRemoteBrowser(userId);
|
||||
if (!browser) {
|
||||
throw new Error('Could not access browser');
|
||||
}
|
||||
@@ -132,7 +132,7 @@ async function executeRun(id: string) {
|
||||
const binaryOutputService = new BinaryOutputService('maxun-run-screenshots');
|
||||
const uploadedBinaryOutput = await binaryOutputService.uploadAndStoreBinaryOutput(run, interpretationInfo.binaryOutput);
|
||||
|
||||
await destroyRemoteBrowser(plainRun.browserId);
|
||||
await destroyRemoteBrowser(plainRun.browserId, userId);
|
||||
|
||||
await run.update({
|
||||
...run,
|
||||
@@ -207,22 +207,22 @@ async function executeRun(id: string) {
|
||||
}
|
||||
}
|
||||
|
||||
async function readyForRunHandler(browserId: string, id: string) {
|
||||
async function readyForRunHandler(browserId: string, id: string, userId: string) {
|
||||
try {
|
||||
const interpretation = await executeRun(id);
|
||||
const interpretation = await executeRun(id, userId);
|
||||
|
||||
if (interpretation) {
|
||||
logger.log('info', `Interpretation of ${id} succeeded`);
|
||||
} else {
|
||||
logger.log('error', `Interpretation of ${id} failed`);
|
||||
await destroyRemoteBrowser(browserId);
|
||||
await destroyRemoteBrowser(browserId, userId);
|
||||
}
|
||||
|
||||
resetRecordingState(browserId, id);
|
||||
|
||||
} catch (error: any) {
|
||||
logger.error(`Error during readyForRunHandler: ${error.message}`);
|
||||
await destroyRemoteBrowser(browserId);
|
||||
await destroyRemoteBrowser(browserId, userId);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -245,12 +245,12 @@ export async function handleRunRecording(id: string, userId: string) {
|
||||
rejectUnauthorized: false
|
||||
});
|
||||
|
||||
socket.on('ready-for-run', () => readyForRunHandler(browserId, newRunId));
|
||||
socket.on('ready-for-run', () => readyForRunHandler(browserId, newRunId, userId));
|
||||
|
||||
logger.log('info', `Running robot: ${id}`);
|
||||
|
||||
socket.on('disconnect', () => {
|
||||
cleanupSocketListeners(socket, browserId, newRunId);
|
||||
cleanupSocketListeners(socket, browserId, newRunId, userId);
|
||||
});
|
||||
|
||||
} catch (error: any) {
|
||||
@@ -258,8 +258,8 @@ export async function handleRunRecording(id: string, userId: string) {
|
||||
}
|
||||
}
|
||||
|
||||
function cleanupSocketListeners(socket: Socket, browserId: string, id: string) {
|
||||
socket.off('ready-for-run', () => readyForRunHandler(browserId, id));
|
||||
function cleanupSocketListeners(socket: Socket, browserId: string, id: string, userId: string) {
|
||||
socket.off('ready-for-run', () => readyForRunHandler(browserId, id, userId));
|
||||
logger.log('info', `Cleaned up listeners for browserId: ${browserId}, runId: ${id}`);
|
||||
}
|
||||
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,4 +1,4 @@
|
||||
import React, { useCallback, useEffect, useState } from 'react';
|
||||
import React, { useCallback, useContext, useEffect, useState } from 'react';
|
||||
import { useSocketStore } from '../../context/socket';
|
||||
import { Button } from '@mui/material';
|
||||
import Canvas from "../recorder/canvas";
|
||||
@@ -8,6 +8,7 @@ import { useActionContext } from '../../context/browserActions';
|
||||
import { useBrowserSteps, TextStep } from '../../context/browserSteps';
|
||||
import { useGlobalInfoStore } from '../../context/globalInfo';
|
||||
import { useTranslation } from 'react-i18next';
|
||||
import { AuthContext } from '../../context/auth';
|
||||
|
||||
interface ElementInfo {
|
||||
tagName: string;
|
||||
@@ -27,6 +28,12 @@ interface AttributeOption {
|
||||
value: string;
|
||||
}
|
||||
|
||||
interface ScreencastData {
|
||||
image: string;
|
||||
userId: string;
|
||||
}
|
||||
|
||||
|
||||
const getAttributeOptions = (tagName: string, elementInfo: ElementInfo | null): AttributeOption[] => {
|
||||
if (!elementInfo) return [];
|
||||
switch (tagName.toLowerCase()) {
|
||||
@@ -71,6 +78,9 @@ export const BrowserWindow = () => {
|
||||
const { notify } = useGlobalInfoStore();
|
||||
const { getText, getList, paginationMode, paginationType, limitMode, captureStage } = useActionContext();
|
||||
const { addTextStep, addListStep } = useBrowserSteps();
|
||||
|
||||
const { state } = useContext(AuthContext);
|
||||
const { user } = state;
|
||||
|
||||
useEffect(() => {
|
||||
if (listSelector) {
|
||||
@@ -85,7 +95,7 @@ export const BrowserWindow = () => {
|
||||
if (storedListSelector && !listSelector) {
|
||||
setListSelector(storedListSelector);
|
||||
}
|
||||
}, []);
|
||||
}, []);
|
||||
|
||||
const onMouseMove = (e: MouseEvent) => {
|
||||
if (canvasRef && canvasRef.current && highlighterData) {
|
||||
@@ -114,9 +124,15 @@ export const BrowserWindow = () => {
|
||||
}
|
||||
}, [getList, resetListState]);
|
||||
|
||||
const screencastHandler = useCallback((data: string) => {
|
||||
setScreenShot(data);
|
||||
}, [screenShot]);
|
||||
const screencastHandler = useCallback((data: string | ScreencastData) => {
|
||||
if (typeof data === 'string') {
|
||||
setScreenShot(data);
|
||||
} else if (data && typeof data === 'object' && 'image' in data) {
|
||||
if (!data.userId || data.userId === user?.id) {
|
||||
setScreenShot(data.image);
|
||||
}
|
||||
}
|
||||
}, [screenShot, user?.id]);
|
||||
|
||||
useEffect(() => {
|
||||
if (socket) {
|
||||
|
||||
@@ -11,7 +11,19 @@ body {
|
||||
padding: 0;
|
||||
scrollbar-gutter: stable;
|
||||
overflow-y: auto;
|
||||
|
||||
}
|
||||
|
||||
input:-webkit-autofill,
|
||||
input:-webkit-autofill:hover,
|
||||
input:-webkit-autofill:focus,
|
||||
textarea:-webkit-autofill,
|
||||
textarea:-webkit-autofill:hover,
|
||||
textarea:-webkit-autofill:focus,
|
||||
select:-webkit-autofill,
|
||||
select:-webkit-autofill:hover,
|
||||
select:-webkit-autofill:focus {
|
||||
-webkit-box-shadow: 0 0 0 1000px transparent inset !important;
|
||||
transition: background-color 5000s ease-in-out 0s !important;
|
||||
}
|
||||
|
||||
html {
|
||||
@@ -22,6 +34,7 @@ html {
|
||||
|
||||
a {
|
||||
color: #ff00c3;
|
||||
|
||||
&:hover {
|
||||
color: #ff00c3;
|
||||
}
|
||||
@@ -29,7 +42,7 @@ a {
|
||||
|
||||
code {
|
||||
font-family: source-code-pro, Menlo, Monaco, Consolas, 'Courier New',
|
||||
monospace;
|
||||
monospace;
|
||||
color: #ff00c3;
|
||||
}
|
||||
|
||||
@@ -44,7 +57,6 @@ code {
|
||||
align-items: center;
|
||||
overflow: hidden;
|
||||
position: relative;
|
||||
|
||||
}
|
||||
|
||||
#browser-content {
|
||||
@@ -52,13 +64,10 @@ code {
|
||||
width: 100%;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
transform: scale(1); /* Ensure no scaling */
|
||||
transform-origin: top left; /* Keep the position fixed */
|
||||
}
|
||||
|
||||
|
||||
#browser {
|
||||
|
||||
transform: scale(1);
|
||||
/* Ensure no scaling */
|
||||
transform-origin: top left;
|
||||
/* Keep the position fixed */
|
||||
}
|
||||
|
||||
#browser-window {
|
||||
@@ -163,4 +172,4 @@ code {
|
||||
height: calc(100vh - 2rem);
|
||||
margin: 1rem 55rem;
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user