vodorod/parcer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: memory optimization

Browse Source
This commit is contained in:
Rohit
2025-03-23 16:24:41 +05:30
parent 7843cda003
commit 241031471b
1 changed files with 100 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files

140
server/src/browser-management/classes/RemoteBrowser.ts
View File

@@ -22,9 +22,9 @@ import { getInjectableScript } from 'idcac-playwright';
chromium.use(stealthPlugin()); chromium.use(stealthPlugin());
const MEMORY_CONFIG = { const MEMORY_CONFIG = {
gcInterval: 60000, // 1 minute gcInterval: 20000, // Check memory more frequently (20s instead of 60s)
maxHeapSize: 2048 * 1024 * 1024, // 2GB maxHeapSize: 1536 * 1024 * 1024, // 1.5GB
heapUsageThreshold: 0.85 // 85% heapUsageThreshold: 0.7 // 70% (reduced threshold to react earlier)
}; };
const SCREENCAST_CONFIG: { const SCREENCAST_CONFIG: {
@@ -35,12 +35,12 @@ const SCREENCAST_CONFIG: {
compressionQuality: number; compressionQuality: number;
maxQueueSize: number; maxQueueSize: number;
} = { } = {
format: 'png', format: 'png',
maxWidth: 1280, maxWidth: 1280,
maxHeight: 720, maxHeight: 720,
targetFPS: 30, targetFPS: 15,
compressionQuality: 0.95, compressionQuality: 0.95,
maxQueueSize: 2 maxQueueSize: 1
}; };
/** /**
@@ -131,13 +131,23 @@ export class RemoteBrowser {
setInterval(() => { setInterval(() => {
const memoryUsage = process.memoryUsage(); const memoryUsage = process.memoryUsage();
const heapUsageRatio = memoryUsage.heapUsed / MEMORY_CONFIG.maxHeapSize; const heapUsageRatio = memoryUsage.heapUsed / MEMORY_CONFIG.maxHeapSize;
if (heapUsageRatio > MEMORY_CONFIG.heapUsageThreshold) { if (heapUsageRatio > MEMORY_CONFIG.heapUsageThreshold * 1.2) {
logger.warn('High memory usage detected, triggering cleanup'); logger.warn('Critical memory pressure detected, triggering emergency cleanup');
this.performMemoryCleanup(); this.performMemoryCleanup();
} else if (heapUsageRatio > MEMORY_CONFIG.heapUsageThreshold) {
logger.warn('High memory usage detected, triggering cleanup');
if (this.screenshotQueue.length > 0) {
this.screenshotQueue = [];
logger.info('Screenshot queue cleared due to memory pressure');
}
if (global.gc && heapUsageRatio > MEMORY_CONFIG.heapUsageThreshold * 1.1) {
global.gc();
}
} }
// Clear screenshot queue if it's too large
if (this.screenshotQueue.length > SCREENCAST_CONFIG.maxQueueSize) { if (this.screenshotQueue.length > SCREENCAST_CONFIG.maxQueueSize) {
this.screenshotQueue = this.screenshotQueue.slice(-SCREENCAST_CONFIG.maxQueueSize); this.screenshotQueue = this.screenshotQueue.slice(-SCREENCAST_CONFIG.maxQueueSize);
} }
@@ -147,24 +157,37 @@ export class RemoteBrowser {
private async performMemoryCleanup(): Promise<void> { private async performMemoryCleanup(): Promise<void> {
this.screenshotQueue = []; this.screenshotQueue = [];
this.isProcessingScreenshot = false; this.isProcessingScreenshot = false;
if (global.gc) { if (global.gc) {
global.gc(); try {
global.gc();
logger.info('Garbage collection requested');
} catch (error) {
logger.error('Error during garbage collection:', error);
}
} }
// Reset CDP session if needed
if (this.client) { if (this.client) {
try { try {
await this.stopScreencast(); await this.stopScreencast();
await new Promise(resolve => setTimeout(resolve, 500));
this.client = null; this.client = null;
if (this.currentPage) { if (this.currentPage) {
this.client = await this.currentPage.context().newCDPSession(this.currentPage); this.client = await this.currentPage.context().newCDPSession(this.currentPage);
await this.startScreencast(); await this.startScreencast();
logger.info('CDP session reset completed');
} }
} catch (error) { } catch (error) {
logger.error('Error resetting CDP session:', error); logger.error('Error resetting CDP session:', error);
} }
} }
this.socket.emit('memory-cleanup', {
userId: this.userId,
timestamp: Date.now()
});
} }
/** /**
@@ -345,6 +368,8 @@ export class RemoteBrowser {
// Still need to set up the CDP session even if blocker fails // Still need to set up the CDP session even if blocker fails
this.client = await this.currentPage.context().newCDPSession(this.currentPage); this.client = await this.currentPage.context().newCDPSession(this.currentPage);
} }
this.initializeMemoryManagement();
}; };
public updateViewportInfo = async (): Promise<void> => { public updateViewportInfo = async (): Promise<void> => {
@@ -532,24 +557,32 @@ export class RemoteBrowser {
private async optimizeScreenshot(screenshot: Buffer): Promise<Buffer> { private async optimizeScreenshot(screenshot: Buffer): Promise<Buffer> {
try { try {
// Use JPEG format with optimized settings
return await sharp(screenshot) return await sharp(screenshot)
.png({ .jpeg({
quality: Math.round(SCREENCAST_CONFIG.compressionQuality * 100), quality: Math.round(SCREENCAST_CONFIG.compressionQuality * 100),
compressionLevel: 3, progressive: true, // Better streaming performance
adaptiveFiltering: true, force: true // Force JPEG even if PNG input
force: true
}) })
.resize({ .resize({
width: SCREENCAST_CONFIG.maxWidth, width: SCREENCAST_CONFIG.maxWidth,
height: SCREENCAST_CONFIG.maxHeight, height: SCREENCAST_CONFIG.maxHeight,
fit: 'inside', fit: 'inside',
withoutEnlargement: true, withoutEnlargement: true,
kernel: sharp.kernel.mitchell kernel: 'lanczos3' // Better quality/performance balance
}) })
.toBuffer(); .toBuffer();
} catch (error) { } catch (error) {
logger.error('Screenshot optimization failed:', error); logger.error('Screenshot optimization failed:', error);
return screenshot;
// If sharp processing fails, do basic resize without sharp
try {
// Fallback to simpler processing
return screenshot;
} catch (fallbackError) {
logger.error('Fallback screenshot processing failed:', fallbackError);
return screenshot;
}
} }
} }
@@ -706,24 +739,43 @@ export class RemoteBrowser {
try { try {
await this.client.send('Page.startScreencast', { await this.client.send('Page.startScreencast', {
format: SCREENCAST_CONFIG.format, format: SCREENCAST_CONFIG.format,
quality: Math.round(SCREENCAST_CONFIG.compressionQuality * 100), quality: Math.round(SCREENCAST_CONFIG.compressionQuality * 100),
maxWidth: SCREENCAST_CONFIG.maxWidth, maxWidth: SCREENCAST_CONFIG.maxWidth,
maxHeight: SCREENCAST_CONFIG.maxHeight, maxHeight: SCREENCAST_CONFIG.maxHeight,
everyNthFrame: 1
}); });
// Set flag to indicate screencast is active
this.isScreencastActive = true; this.isScreencastActive = true;
// Set up screencast frame handler
this.client.on('Page.screencastFrame', async ({ data, sessionId }) => { this.client.on('Page.screencastFrame', async ({ data, sessionId }) => {
try { try {
if (this.screenshotQueue.length >= SCREENCAST_CONFIG.maxQueueSize && this.isProcessingScreenshot) {
await this.client?.send('Page.screencastFrameAck', { sessionId });
return;
}
const buffer = Buffer.from(data, 'base64'); const buffer = Buffer.from(data, 'base64');
await this.emitScreenshot(buffer); this.emitScreenshot(buffer);
await this.client?.send('Page.screencastFrameAck', { sessionId });
setTimeout(async () => {
try {
if (this.client) {
await this.client.send('Page.screencastFrameAck', { sessionId });
}
} catch (e) {
logger.error('Error acknowledging screencast frame:', e);
}
}, 10);
} catch (error) { } catch (error) {
logger.error('Screencast frame processing failed:', error); logger.error('Screencast frame processing failed:', error);
try {
await this.client?.send('Page.screencastFrameAck', { sessionId });
} catch (ackError) {
logger.error('Failed to acknowledge screencast frame:', ackError);
}
} }
}); });
logger.info('Screencast started successfully'); logger.info('Screencast started successfully');
} catch (error) { } catch (error) {
logger.error('Failed to start screencast:', error); logger.error('Failed to start screencast:', error);
@@ -755,26 +807,31 @@ export class RemoteBrowser {
* @returns void * @returns void
*/ */
private emitScreenshot = async (payload: Buffer, viewportSize?: { width: number, height: number }): Promise<void> => { private emitScreenshot = async (payload: Buffer, viewportSize?: { width: number, height: number }): Promise<void> => {
if (this.screenshotQueue.length > SCREENCAST_CONFIG.maxQueueSize) {
this.screenshotQueue = this.screenshotQueue.slice(-SCREENCAST_CONFIG.maxQueueSize);
}
if (this.isProcessingScreenshot) { if (this.isProcessingScreenshot) {
if (this.screenshotQueue.length < SCREENCAST_CONFIG.maxQueueSize) { if (this.screenshotQueue.length < SCREENCAST_CONFIG.maxQueueSize) {
this.screenshotQueue.push(payload); this.screenshotQueue.push(payload);
} }
return; return;
} }
this.isProcessingScreenshot = true; this.isProcessingScreenshot = true;
try { try {
const optimizationPromise = this.optimizeScreenshot(payload); const optimizationPromise = this.optimizeScreenshot(payload);
const timeoutPromise = new Promise<Buffer>((resolve) => { const timeoutPromise = new Promise<Buffer>((resolve) => {
setTimeout(() => resolve(payload), 100); setTimeout(() => resolve(payload), 150);
}); });
const optimizedScreenshot = await Promise.race([optimizationPromise, timeoutPromise]); const optimizedScreenshot = await Promise.race([optimizationPromise, timeoutPromise]);
const base64Data = optimizedScreenshot.toString('base64'); const base64Data = optimizedScreenshot.toString('base64');
const dataWithMimeType = `data:image/png;base64,${base64Data}`; const dataWithMimeType = `data:image/${SCREENCAST_CONFIG.format};base64,${base64Data}`;
payload = null as any;
this.socket.emit('screencast', { this.socket.emit('screencast', {
image: dataWithMimeType, image: dataWithMimeType,
userId: this.userId, userId: this.userId,
@@ -784,7 +841,8 @@ export class RemoteBrowser {
logger.error('Screenshot emission failed:', error); logger.error('Screenshot emission failed:', error);
try { try {
const base64Data = payload.toString('base64'); const base64Data = payload.toString('base64');
const dataWithMimeType = `data:image/png;base64,${base64Data}`; const dataWithMimeType = `data:image/jpeg;base64,${base64Data}`;
this.socket.emit('screencast', { this.socket.emit('screencast', {
image: dataWithMimeType, image: dataWithMimeType,
userId: this.userId, userId: this.userId,
@@ -795,11 +853,13 @@ export class RemoteBrowser {
} }
} finally { } finally {
this.isProcessingScreenshot = false; this.isProcessingScreenshot = false;
if (this.screenshotQueue.length > 0) { if (this.screenshotQueue.length > 0) {
const nextScreenshot = this.screenshotQueue.shift(); const nextScreenshot = this.screenshotQueue.shift();
if (nextScreenshot) { if (nextScreenshot) {
setTimeout(() => this.emitScreenshot(nextScreenshot), 1000 / SCREENCAST_CONFIG.targetFPS); setTimeout(() => {
this.emitScreenshot(nextScreenshot);
}, 1000 / SCREENCAST_CONFIG.targetFPS);
} }
} }
} }