vodorod/parcer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: revert non-table scraping logic

Browse Source
This commit is contained in:
RohitR311
2024-12-25 23:09:27 +05:30
parent 50bcd4b8dd
commit 991b554464
1 changed files with 11 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files

29
maxun-core/src/browserSide/scraper.js
View File

@@ -437,26 +437,19 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3,
// Handle non-table fields
if (Object.keys(nonTableFields).length > 0) {
const firstField = Object.values(nonTableFields)[0];
const baseElements = Array.from(container.querySelectorAll(firstField.selector));
const record = {};
for (let i = 0; i < Math.min(baseElements.length, limit); i++) {
const record = {};
for (const [label, { selector, attribute }] of Object.entries(nonTableFields)) {
const elements = Array.from(parent.querySelectorAll(selector));
// Use the same index to maintain correspondence between fields
const element = elements[i];
if (element) {
record[label] = extractValue(element, attribute);
}
}
if (Object.keys(record).length > 0) {
scrapedData.push(record);
}
for (const [label, { selector, attribute }] of Object.entries(nonTableFields)) {
const element = container.querySelector(selector);
if (element) {
record[label] = extractValue(element, attribute);
}
}
if (Object.keys(record).length > 0) {
scrapedData.push(record);
}
}
});