From 31d973023741111ac033102466ed848927031c97 Mon Sep 17 00:00:00 2001 From: RohitR311 Date: Thu, 26 Dec 2024 03:17:18 +0530 Subject: [PATCH] feat: add logic to match 70% classes --- maxun-core/src/browserSide/scraper.js | 29 +++++++++++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/maxun-core/src/browserSide/scraper.js b/maxun-core/src/browserSide/scraper.js index c12267d3..e109839b 100644 --- a/maxun-core/src/browserSide/scraper.js +++ b/maxun-core/src/browserSide/scraper.js @@ -330,9 +330,34 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3, } // Get all containers that match the listSelector - const containers = Array.from(document.querySelectorAll(listSelector)); + let containers = Array.from(document.querySelectorAll(listSelector)); if (containers.length === 0) return []; + if (limit > 1 && containers.length <= 1) { + const [containerSelector, _] = listSelector.split('>').map(s => s.trim()); + const container = document.querySelector(containerSelector); + + if (container) { + const allChildren = Array.from(container.children); + + const firstMatch = document.querySelector(listSelector); + if (firstMatch) { + // Get classes from the first matching element + const firstMatchClasses = Array.from(firstMatch.classList); + + // Find similar elements by matching most of their classes + containers = allChildren.filter(element => { + const elementClasses = Array.from(element.classList); + + // Element should share at least 70% of classes with the first match + const commonClasses = firstMatchClasses.filter(cls => + elementClasses.includes(cls)); + return commonClasses.length >= Math.floor(firstMatchClasses.length * 0.7); + }); + } + } + } + // Initialize arrays to store field classifications for each container const containerFields = containers.map(() => ({ tableFields: {}, @@ -436,7 +461,7 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3, } } - // Process non-table fields across all containers + // Process non-table fields across all containers for (let containerIndex = 0; containerIndex < containers.length; containerIndex++) { if (nonTableData.length >= limit) break;