vodorod/parcer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add functionality to scrape shadowDOM elements

Browse Source
This commit is contained in:
RohitR311
2024-12-30 01:24:32 +05:30
parent 542f4d31fa
commit b60f4b73b8
1 changed files with 80 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files

58
maxun-core/src/browserSide/scraper.js
View File

@@ -189,6 +189,7 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3,
* @returns {Array.<Object.<string, string>>}
*/
window.scrapeSchema = function (lists) {
// These utility functions remain unchanged as they work perfectly
function omap(object, f, kf = (x) => x) {
return Object.fromEntries(
Object.entries(object)
@@ -203,11 +204,39 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3,
);
}
function getSeedKey(listObj) {
const maxLength = Math.max(...Object.values(omap(listObj, (x) => document.querySelectorAll(x.selector).length)));
return Object.keys(ofilter(listObj, (_, v) => document.querySelectorAll(v.selector).length === maxLength))[0];
function findElement(config) {
// If this is a shadow DOM query
if (config.shadow && config.selector.includes('>>')) {
const [hostSelector, shadowSelector] = config.selector.split('>>').map(s => s.trim());
const host = document.querySelector(hostSelector);
return host?.shadowRoot?.querySelector(shadowSelector) || null;
}
// Otherwise, use regular querySelector
return document.querySelector(config.selector);
}
function findAllElements(config) {
// If this is a shadow DOM query
if (config.shadow && config.selector.includes('>>')) {
const element = findElement(config);
return element ? [element] : [];
}
// Otherwise, use regular querySelectorAll
return Array.from(document.querySelectorAll(config.selector));
}
// Modified to use our new element finding functions
function getSeedKey(listObj) {
const maxLength = Math.max(...Object.values(
omap(listObj, (x) => findAllElements(x).length)
));
return Object.keys(
ofilter(listObj, (_, v) => findAllElements(v).length === maxLength)
)[0];
}
// This function remains unchanged as it works with DOM elements
// regardless of how they were found
function getMBEs(elements) {
return elements.map((element) => {
let candidate = element;
@@ -224,33 +253,38 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3,
}
const seedName = getSeedKey(lists);
const seedElements = Array.from(document.querySelectorAll(lists[seedName].selector));
const seedElements = findAllElements(lists[seedName]);
const MBEs = getMBEs(seedElements);
return MBEs.map((mbe) => omap(
lists,
({ selector, attribute }, key) => {
const elem = Array.from(document.querySelectorAll(selector)).find((elem) => mbe.contains(elem));
(config, key) => {
// Use our new findAllElements function
const elem = findAllElements(config)
.find((elem) => mbe.contains(elem));
if (!elem) return undefined;
switch (attribute) {
case 'href':
switch (config.attribute) {
case 'href': {
const relativeHref = elem.getAttribute('href');
return relativeHref ? new URL(relativeHref, window.location.origin).href : null;
case 'src':
}
case 'src': {
const relativeSrc = elem.getAttribute('src');
return relativeSrc ? new URL(relativeSrc, window.location.origin).href : null;
}
case 'innerText':
return elem.innerText;
case 'textContent':
return elem.textContent;
default:
return elem.innerText;
return elem.getAttribute(config.attribute) || elem.innerText;
}
},
(key) => key // Use the original key in the output
(key) => key
)) || [];
}
};
/**
* Scrapes multiple lists of similar items based on a template item.