vodorod/parcer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add scraping logic for tabular data in scrapeList

Browse Source
This commit is contained in:
RohitR311
2024-12-24 02:31:32 +05:30
parent feb30b9f9e
commit 5ac88c6eda
1 changed files with 63 additions and 53 deletions
Download Patch File Download Diff File Expand all files Collapse all files

116
maxun-core/src/browserSide/scraper.js
View File

@@ -262,73 +262,83 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3,
* @returns {Array.<Array.<Object>>} Array of arrays of scraped items, one sub-array per list * @returns {Array.<Array.<Object>>} Array of arrays of scraped items, one sub-array per list
*/ */
window.scrapeList = async function ({ listSelector, fields, limit = 10 }) { window.scrapeList = async function ({ listSelector, fields, limit = 10 }) {
// Separate fields into table and non-table categories
const tableFields = {};
const nonTableFields = {};
for (const [label, field] of Object.entries(fields)) {
if (['TD', 'TH', 'TR'].includes(field.tag)) {
tableFields[label] = field;
} else {
nonTableFields[label] = field;
}
}
const parentElements = Array.from(document.querySelectorAll(listSelector));
const scrapedData = []; const scrapedData = [];
while (scrapedData.length < limit) { for (const parent of parentElements) {
let parentElements = Array.from(document.querySelectorAll(listSelector)); // First, get the number of rows we'll need by checking the first table field
const firstTableField = Object.values(tableFields)[0];
// If we only got one element or none, try a more generic approach const tableRows = firstTableField
if (limit > 1 && parentElements.length <= 1) { ? Array.from(parent.querySelectorAll(firstTableField.selector)).slice(0, limit)
const [containerSelector, _] = listSelector.split('>').map(s => s.trim()); : [null];
const container = document.querySelector(containerSelector);
if (container) {
const allChildren = Array.from(container.children);
const firstMatch = document.querySelector(listSelector);
if (firstMatch) {
// Get classes from the first matching element
const firstMatchClasses = Array.from(firstMatch.classList);
// Find similar elements by matching most of their classes
parentElements = allChildren.filter(element => {
const elementClasses = Array.from(element.classList);
// Element should share at least 70% of classes with the first match tableRows.forEach((_, rowIndex) => {
const commonClasses = firstMatchClasses.filter(cls =>
elementClasses.includes(cls));
return commonClasses.length >= Math.floor(firstMatchClasses.length * 0.7);
});
}
}
}
// Iterate through each parent element
for (const parent of parentElements) {
if (scrapedData.length >= limit) break;
const record = {}; const record = {};
// For each field, select the corresponding element within the parent // Table fields
for (const [label, { selector, attribute }] of Object.entries(fields)) { for (const [label, { selector, attribute }] of Object.entries(tableFields)) {
const fieldElement = parent.querySelector(selector); const elements = Array.from(parent.querySelectorAll(selector));
const element = elements[rowIndex];
if (fieldElement) {
if (element) {
let value;
if (attribute === 'innerText') { if (attribute === 'innerText') {
record[label] = fieldElement.innerText.trim(); value = element.innerText.trim();
} else if (attribute === 'innerHTML') { } else if (attribute === 'innerHTML') {
record[label] = fieldElement.innerHTML.trim(); value = element.innerHTML.trim();
} else if (attribute === 'src') { } else if (attribute === 'src' || attribute === 'href') {
// Handle relative 'src' URLs const attrValue = element.getAttribute(attribute);
const src = fieldElement.getAttribute('src'); value = attrValue ? new URL(attrValue, window.location.origin).href : null;
record[label] = src ? new URL(src, window.location.origin).href : null;
} else if (attribute === 'href') {
// Handle relative 'href' URLs
const href = fieldElement.getAttribute('href');
record[label] = href ? new URL(href, window.location.origin).href : null;
} else { } else {
record[label] = fieldElement.getAttribute(attribute); value = element.getAttribute(attribute);
} }
record[label] = value;
} }
} }
scrapedData.push(record);
} // Non table fields
for (const [label, { selector, attribute }] of Object.entries(nonTableFields)) {
const element = parent.querySelector(selector);
if (element) {
let value;
if (attribute === 'innerText') {
value = element.innerText.trim();
} else if (attribute === 'innerHTML') {
value = element.innerHTML.trim();
} else if (attribute === 'src' || attribute === 'href') {
const attrValue = element.getAttribute(attribute);
value = attrValue ? new URL(attrValue, window.location.origin).href : null;
} else {
value = element.getAttribute(attribute);
}
record[label] = value;
}
}
if (Object.keys(record).length > 0) {
scrapedData.push(record);
}
});
// If we've processed all available elements and still haven't reached the limit, if (scrapedData.length >= limit) {
// break to avoid infinite loop scrapedData.length = limit;
if (parentElements.length === 0 || scrapedData.length >= parentElements.length) {
break; break;
} }
} }
return scrapedData; return scrapedData;
}; };