vodorod/parcer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add loop to handle nested scraping

Browse Source
This commit is contained in:
RohitR311
2024-12-24 17:27:42 +05:30
parent b411faf681
commit 3176fa275f
1 changed files with 14 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
maxun-core/src/browserSide/scraper.js
View File

@@ -262,7 +262,6 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3,
* @returns {Array.<Array.<Object>>} Array of arrays of scraped items, one sub-array per list * @returns {Array.<Array.<Object>>} Array of arrays of scraped items, one sub-array per list
*/ */
window.scrapeList = async function ({ listSelector, fields, limit = 10 }) { window.scrapeList = async function ({ listSelector, fields, limit = 10 }) {
// Separate fields into table and non-table categories
const tableFields = {}; const tableFields = {};
const nonTableFields = {}; const nonTableFields = {};
@@ -278,19 +277,19 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3,
const scrapedData = []; const scrapedData = [];
for (const parent of parentElements) { for (const parent of parentElements) {
// First, get the number of rows we'll need by checking the first table field // Get the first field's elements to determine how many items we have
const firstTableField = Object.values(tableFields)[0]; const firstField = Object.values(fields)[0];
const tableRows = firstTableField const baseElements = Array.from(parent.querySelectorAll(firstField.selector));
? Array.from(parent.querySelectorAll(firstTableField.selector)).slice(0, limit)
: [null]; // Process each item up to the limit
for (let i = 0; i < Math.min(baseElements.length, limit); i++) {
tableRows.forEach((_, rowIndex) => {
const record = {}; const record = {};
// Table fields // Process table fields
for (const [label, { selector, attribute }] of Object.entries(tableFields)) { for (const [label, { selector, attribute }] of Object.entries(tableFields)) {
const elements = Array.from(parent.querySelectorAll(selector)); const elements = Array.from(parent.querySelectorAll(selector));
const element = elements[rowIndex]; // Use the same index to maintain correspondence between fields
const element = elements[i];
if (element) { if (element) {
let value; let value;
@@ -308,9 +307,11 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3,
} }
} }
// Non table fields // Process non-table fields
for (const [label, { selector, attribute }] of Object.entries(nonTableFields)) { for (const [label, { selector, attribute }] of Object.entries(nonTableFields)) {
const element = parent.querySelector(selector); const elements = Array.from(parent.querySelectorAll(selector));
// Use the same index to maintain correspondence between fields
const element = elements[i];
if (element) { if (element) {
let value; let value;
@@ -331,7 +332,7 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3,
if (Object.keys(record).length > 0) { if (Object.keys(record).length > 0) {
scrapedData.push(record); scrapedData.push(record);
} }
}); }
if (scrapedData.length >= limit) { if (scrapedData.length >= limit) {
scrapedData.length = limit; scrapedData.length = limit;
@@ -342,7 +343,6 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3,
return scrapedData; return scrapedData;
}; };
/** /**
* Gets all children of the elements matching the listSelector, * Gets all children of the elements matching the listSelector,
* returning their CSS selectors and innerText. * returning their CSS selectors and innerText.