1513 lines
48 KiB
JavaScript
1513 lines
48 KiB
JavaScript
/* eslint-disable @typescript-eslint/no-unused-vars */
|
|
|
|
const area = (element) => element.offsetHeight * element.offsetWidth;
|
|
|
|
function getBiggestElement(selector) {
|
|
const elements = Array.from(document.querySelectorAll(selector));
|
|
const biggest = elements.reduce(
|
|
(max, elem) => (
|
|
area(elem) > area(max) ? elem : max),
|
|
{ offsetHeight: 0, offsetWidth: 0 },
|
|
);
|
|
return biggest;
|
|
}
|
|
|
|
/**
|
|
* Generates structural selector (describing element by its DOM tree location).
|
|
*
|
|
* **The generated selector is not guaranteed to be unique!** (In fact, this is
|
|
* the desired behaviour in here.)
|
|
* @param {HTMLElement} element Element being described.
|
|
* @returns {string} CSS-compliant selector describing the element's location in the DOM tree.
|
|
*/
|
|
function GetSelectorStructural(element) {
|
|
// Base conditions for the recursive approach.
|
|
if (element.tagName === 'BODY') {
|
|
return 'BODY';
|
|
}
|
|
const selector = element.tagName;
|
|
if (element.parentElement) {
|
|
return `${GetSelectorStructural(element.parentElement)} > ${selector}`;
|
|
}
|
|
|
|
return selector;
|
|
}
|
|
|
|
/**
|
|
* Heuristic method to find collections of "interesting" items on the page.
|
|
* @returns {Array<HTMLElement>} A collection of interesting DOM nodes
|
|
* (online store products, plane tickets, list items... and many more?)
|
|
*/
|
|
function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3, metricType = 'size_deviation') {
|
|
const restoreScroll = (() => {
|
|
const { scrollX, scrollY } = window;
|
|
return () => {
|
|
window.scrollTo(scrollX, scrollY);
|
|
};
|
|
})();
|
|
|
|
/**
|
|
* @typedef {Array<{x: number, y: number}>} Grid
|
|
*/
|
|
|
|
/**
|
|
* Returns an array of grid-aligned {x,y} points.
|
|
* @param {number} [granularity=0.005] sets the number of generated points
|
|
* (the higher the granularity, the more points).
|
|
* @returns {Grid} Array of {x, y} objects.
|
|
*/
|
|
function getGrid(startX = 0, startY = 0, granularity = 0.005) {
|
|
const width = window.innerWidth;
|
|
const height = window.innerHeight;
|
|
|
|
const out = [];
|
|
for (let x = 0; x < width; x += 1 / granularity) {
|
|
for (let y = 0; y < height; y += 1 / granularity) {
|
|
out.push({ x: startX + x, y: startY + y });
|
|
}
|
|
}
|
|
return out;
|
|
}
|
|
|
|
let maxSelector = { selector: 'body', metric: 0 };
|
|
|
|
const updateMaximumWithPoint = (point) => {
|
|
const currentElement = document.elementFromPoint(point.x, point.y);
|
|
const selector = GetSelectorStructural(currentElement);
|
|
|
|
const elements = Array.from(document.querySelectorAll(selector))
|
|
.filter((element) => area(element) > minArea);
|
|
|
|
// If the current selector targets less than three elements,
|
|
// we consider it not interesting (would be a very underwhelming scraper)
|
|
if (elements.length < 3) {
|
|
return;
|
|
}
|
|
|
|
let metric = null;
|
|
|
|
if (metricType === 'total_area') {
|
|
metric = elements
|
|
.reduce((p, x) => p + area(x), 0);
|
|
} else if (metricType === 'size_deviation') {
|
|
// This could use a proper "statistics" approach... but meh, so far so good!
|
|
const sizes = elements
|
|
.map((element) => area(element));
|
|
|
|
metric = (1 - (Math.max(...sizes) - Math.min(...sizes)) / Math.max(...sizes));
|
|
}
|
|
|
|
if (metric > maxSelector.metric && elements.length < maxCountPerPage) {
|
|
maxSelector = { selector, metric };
|
|
}
|
|
};
|
|
|
|
for (let scroll = 0; scroll < scrolls; scroll += 1) {
|
|
window.scrollTo(0, scroll * window.innerHeight);
|
|
|
|
const grid = getGrid();
|
|
|
|
grid.forEach(updateMaximumWithPoint);
|
|
}
|
|
|
|
restoreScroll();
|
|
|
|
let out = Array.from(document.querySelectorAll(maxSelector.selector));
|
|
|
|
const different = (x, i, a) => a.findIndex((e) => e === x) === i;
|
|
// as long as we don't merge any two elements by substituing them for their parents,
|
|
// we substitute.
|
|
while (out.map((x) => x.parentElement).every(different)
|
|
&& out.forEach((x) => x.parentElement !== null)) {
|
|
out = out.map((x) => x.parentElement ?? x);
|
|
}
|
|
|
|
return out;
|
|
}
|
|
|
|
/**
|
|
* Returns a "scrape" result from the current page.
|
|
* @returns {Array<Object>} *Curated* array of scraped information (with sparse rows removed)
|
|
*/
|
|
// Wrap the entire function in an IIFE (Immediately Invoked Function Expression)
|
|
// and attach it to the window object
|
|
(function (window) {
|
|
/**
|
|
* Returns a "scrape" result from the current page.
|
|
* @returns {Array<Object>} *Curated* array of scraped information (with sparse rows removed)
|
|
*/
|
|
window.scrape = function (selector = null) {
|
|
/**
|
|
* **crudeRecords** contains uncurated rundowns of "scrapable" elements
|
|
* @type {Array<Object>}
|
|
*/
|
|
const crudeRecords = (selector
|
|
? Array.from(document.querySelectorAll(selector))
|
|
: scrapableHeuristics())
|
|
.map((record) => ({
|
|
...Array.from(record.querySelectorAll('img'))
|
|
.reduce((p, x, i) => {
|
|
let url = null;
|
|
if (x.srcset) {
|
|
const urls = x.srcset.split(', ');
|
|
[url] = urls[urls.length - 1].split(' ');
|
|
}
|
|
|
|
/**
|
|
* Contains the largest elements from `srcset` - if `srcset` is not present, contains
|
|
* URL from the `src` attribute
|
|
*
|
|
* If the `src` attribute contains a data url, imgUrl contains `undefined`.
|
|
*/
|
|
let imgUrl;
|
|
if (x.srcset) {
|
|
imgUrl = url;
|
|
} else if (x.src.indexOf('data:') === -1) {
|
|
imgUrl = x.src;
|
|
}
|
|
|
|
return ({
|
|
...p,
|
|
...(imgUrl ? { [`img_${i}`]: imgUrl } : {}),
|
|
});
|
|
}, {}),
|
|
...record.innerText.split('\n')
|
|
.reduce((p, x, i) => ({
|
|
...p,
|
|
[`record_${String(i).padStart(4, '0')}`]: x.trim(),
|
|
}), {}),
|
|
}));
|
|
|
|
return crudeRecords;
|
|
};
|
|
|
|
/**
|
|
* TODO: Simplify.
|
|
* Given an object with named lists of elements,
|
|
* groups the elements by their distance in the DOM tree.
|
|
* @param {Object.<string, {selector: string, tag: string}>} lists The named lists of HTML elements.
|
|
* @returns {Array.<Object.<string, string>>}
|
|
*/
|
|
window.scrapeSchema = function(lists) {
|
|
// Utility functions remain the same
|
|
function omap(object, f, kf = (x) => x) {
|
|
return Object.fromEntries(
|
|
Object.entries(object)
|
|
.map(([k, v]) => [kf(k), f(v)]),
|
|
);
|
|
}
|
|
|
|
function ofilter(object, f) {
|
|
return Object.fromEntries(
|
|
Object.entries(object)
|
|
.filter(([k, v]) => f(k, v)),
|
|
);
|
|
}
|
|
|
|
function findAllElements(config) {
|
|
// Regular DOM query if no special delimiters
|
|
if (!config.selector.includes('>>') && !config.selector.includes(':>>')) {
|
|
return Array.from(document.querySelectorAll(config.selector));
|
|
}
|
|
|
|
if (config.selector.includes(':>>')) {
|
|
const parts = config.selector.split(':>>').map(s => s.trim());
|
|
let currentElements = [document];
|
|
|
|
// Traverse through each part of the selector
|
|
for (let i = 0; i < parts.length; i++) {
|
|
const part = parts[i];
|
|
const nextElements = [];
|
|
const isLast = i === parts.length - 1;
|
|
|
|
for (const element of currentElements) {
|
|
try {
|
|
const doc = element.contentDocument || element || element.contentWindow?.document;
|
|
if (!doc) continue;
|
|
|
|
if (part.startsWith('frame[name=') || part.startsWith('iframe[name=')) {
|
|
const nameMatch = part.match(/\[name=['"]([^'"]+)['"]\]/);
|
|
if (nameMatch && nameMatch[1]) {
|
|
const frameName = nameMatch[1];
|
|
let foundFrames = [];
|
|
|
|
if (doc.getElementsByName && typeof doc.getElementsByName === 'function') {
|
|
foundFrames = Array.from(doc.getElementsByName(frameName))
|
|
.filter(el => el.tagName === 'FRAME' || el.tagName === 'IFRAME');
|
|
}
|
|
|
|
if (foundFrames.length === 0) {
|
|
const framesBySelector = Array.from(doc.querySelectorAll(`frame[name="${frameName}"], iframe[name="${frameName}"]`));
|
|
foundFrames = framesBySelector;
|
|
}
|
|
|
|
if (isLast) {
|
|
nextElements.push(...foundFrames);
|
|
} else {
|
|
nextElements.push(...foundFrames);
|
|
}
|
|
continue;
|
|
}
|
|
}
|
|
|
|
const found = Array.from(doc.querySelectorAll(part));
|
|
|
|
if (isLast) {
|
|
nextElements.push(...found);
|
|
} else {
|
|
const frames = found.filter(el => el.tagName === 'IFRAME' || el.tagName === 'FRAME');
|
|
nextElements.push(...frames);
|
|
}
|
|
} catch (error) {
|
|
console.warn('Cannot access iframe/frame content:', error, {
|
|
part,
|
|
element,
|
|
index: i
|
|
});
|
|
}
|
|
}
|
|
|
|
if (nextElements.length === 0) {
|
|
console.warn('No elements found for part:', part, 'at depth:', i);
|
|
return [];
|
|
}
|
|
currentElements = nextElements;
|
|
}
|
|
|
|
return currentElements;
|
|
}
|
|
|
|
// Handle shadow DOM traversal
|
|
if (config.selector.includes('>>')) {
|
|
const parts = config.selector.split('>>').map(s => s.trim());
|
|
let currentElements = [document];
|
|
|
|
for (const part of parts) {
|
|
const nextElements = [];
|
|
for (const element of currentElements) {
|
|
// Try regular DOM first
|
|
const found = Array.from(element.querySelectorAll(part));
|
|
|
|
// Then check shadow roots
|
|
for (const foundEl of found) {
|
|
if (foundEl.shadowRoot) {
|
|
nextElements.push(foundEl.shadowRoot);
|
|
} else {
|
|
nextElements.push(foundEl);
|
|
}
|
|
}
|
|
}
|
|
currentElements = nextElements;
|
|
}
|
|
return currentElements.filter(el => !(el instanceof ShadowRoot));
|
|
}
|
|
|
|
return [];
|
|
}
|
|
|
|
function getElementValue(element, attribute) {
|
|
if (!element) return null;
|
|
|
|
let baseURL;
|
|
try {
|
|
baseURL = element.ownerDocument?.location?.href ||
|
|
element.ownerDocument?.baseURI ||
|
|
window.location.origin;
|
|
} catch (e) {
|
|
baseURL = window.location.origin;
|
|
}
|
|
|
|
switch (attribute) {
|
|
case 'href': {
|
|
const relativeHref = element.getAttribute('href');
|
|
return relativeHref ? new URL(relativeHref, baseURL).href : null;
|
|
}
|
|
case 'src': {
|
|
const relativeSrc = element.getAttribute('src');
|
|
return relativeSrc ? new URL(relativeSrc, baseURL).href : null;
|
|
}
|
|
case 'innerText':
|
|
return element.innerText?.trim();
|
|
case 'textContent':
|
|
return element.textContent?.trim();
|
|
case 'innerHTML':
|
|
return element.innerHTML;
|
|
case 'outerHTML':
|
|
return element.outerHTML;
|
|
default:
|
|
return element.getAttribute(attribute) || element.innerText?.trim();
|
|
}
|
|
}
|
|
|
|
// Rest of the functions remain largely the same
|
|
function getSeedKey(listObj) {
|
|
const maxLength = Math.max(...Object.values(
|
|
omap(listObj, (x) => findAllElements(x).length)
|
|
));
|
|
return Object.keys(
|
|
ofilter(listObj, (_, v) => findAllElements(v).length === maxLength)
|
|
)[0];
|
|
}
|
|
|
|
// Find minimal bounding elements
|
|
function getMBEs(elements) {
|
|
return elements.map((element) => {
|
|
let candidate = element;
|
|
const isUniqueChild = (e) => elements
|
|
.filter((elem) => {
|
|
// Handle both iframe and shadow DOM boundaries
|
|
const sameContext = elem.getRootNode() === e.getRootNode() &&
|
|
elem.ownerDocument === e.ownerDocument;
|
|
return sameContext && e.parentNode?.contains(elem);
|
|
})
|
|
.length === 1;
|
|
|
|
while (candidate && isUniqueChild(candidate)) {
|
|
candidate = candidate.parentNode;
|
|
}
|
|
|
|
return candidate;
|
|
});
|
|
}
|
|
|
|
const seedName = getSeedKey(lists);
|
|
const seedElements = findAllElements(lists[seedName]);
|
|
const MBEs = getMBEs(seedElements);
|
|
|
|
const mbeResults = MBEs.map((mbe) => omap(
|
|
lists,
|
|
(config) => {
|
|
const elem = findAllElements(config)
|
|
.find((elem) => mbe.contains(elem));
|
|
|
|
return elem ? getElementValue(elem, config.attribute) : undefined;
|
|
},
|
|
(key) => key
|
|
)) || [];
|
|
|
|
// If MBE approach didn't find all elements, try independent scraping
|
|
if (mbeResults.some(result => Object.values(result).some(v => v === undefined))) {
|
|
// Fall back to independent scraping
|
|
const results = [];
|
|
const foundElements = new Map();
|
|
|
|
// Find all elements for each selector
|
|
Object.entries(lists).forEach(([key, config]) => {
|
|
const elements = findAllElements(config);
|
|
foundElements.set(key, elements);
|
|
});
|
|
|
|
// Create result objects for each found element
|
|
foundElements.forEach((elements, key) => {
|
|
elements.forEach((element, index) => {
|
|
if (!results[index]) {
|
|
results[index] = {};
|
|
}
|
|
results[index][key] = getElementValue(element, lists[key].attribute);
|
|
});
|
|
});
|
|
|
|
return results.filter(result => Object.keys(result).length > 0);
|
|
}
|
|
|
|
return mbeResults;
|
|
};
|
|
|
|
/**
|
|
* Scrapes multiple lists of similar items based on a template item.
|
|
* @param {Object} config - Configuration object
|
|
* @param {string} config.listSelector - Selector for the list container(s)
|
|
* @param {Object.<string, {selector: string, attribute?: string}>} config.fields - Fields to scrape
|
|
* @param {number} [config.limit] - Maximum number of items to scrape per list (optional)
|
|
* @param {boolean} [config.flexible=false] - Whether to use flexible matching for field selectors
|
|
* @returns {Array.<Array.<Object>>} Array of arrays of scraped items, one sub-array per list
|
|
*/
|
|
window.scrapeList = async function ({ listSelector, fields, limit = 10 }) {
|
|
// XPath evaluation functions
|
|
const queryInsideContext = (context, part) => {
|
|
try {
|
|
const { tagName, conditions } = parseXPathPart(part);
|
|
|
|
const candidateElements = Array.from(context.querySelectorAll(tagName));
|
|
if (candidateElements.length === 0) {
|
|
return [];
|
|
}
|
|
|
|
const matchingElements = candidateElements.filter((el) => {
|
|
return elementMatchesConditions(el, conditions);
|
|
});
|
|
|
|
return matchingElements;
|
|
} catch (err) {
|
|
console.error("Error in queryInsideContext:", err);
|
|
return [];
|
|
}
|
|
};
|
|
|
|
// Helper function to parse XPath part
|
|
const parseXPathPart = (part) => {
|
|
const tagMatch = part.match(/^([a-zA-Z0-9-]+)/);
|
|
const tagName = tagMatch ? tagMatch[1] : "*";
|
|
|
|
const conditionMatches = part.match(/\[([^\]]+)\]/g);
|
|
const conditions = conditionMatches
|
|
? conditionMatches.map((c) => c.slice(1, -1))
|
|
: [];
|
|
|
|
return { tagName, conditions };
|
|
};
|
|
|
|
// Helper function to check if element matches all conditions
|
|
const elementMatchesConditions = (element, conditions) => {
|
|
for (const condition of conditions) {
|
|
if (!elementMatchesCondition(element, condition)) {
|
|
return false;
|
|
}
|
|
}
|
|
return true;
|
|
};
|
|
|
|
// Helper function to check if element matches a single condition
|
|
const elementMatchesCondition = (element, condition) => {
|
|
condition = condition.trim();
|
|
|
|
if (/^\d+$/.test(condition)) {
|
|
return true;
|
|
}
|
|
|
|
// Handle @attribute="value"
|
|
const attrMatch = condition.match(/^@([^=]+)=["']([^"']+)["']$/);
|
|
if (attrMatch) {
|
|
const [, attr, value] = attrMatch;
|
|
const elementValue = element.getAttribute(attr);
|
|
return elementValue === value;
|
|
}
|
|
|
|
// Handle contains(@class, 'value')
|
|
const classContainsMatch = condition.match(
|
|
/^contains\(@class,\s*["']([^"']+)["']\)$/
|
|
);
|
|
if (classContainsMatch) {
|
|
const className = classContainsMatch[1];
|
|
return element.classList.contains(className);
|
|
}
|
|
|
|
// Handle contains(@attribute, 'value')
|
|
const attrContainsMatch = condition.match(
|
|
/^contains\(@([^,]+),\s*["']([^"']+)["']\)$/
|
|
);
|
|
if (attrContainsMatch) {
|
|
const [, attr, value] = attrContainsMatch;
|
|
const elementValue = element.getAttribute(attr) || "";
|
|
return elementValue.includes(value);
|
|
}
|
|
|
|
// Handle text()="value"
|
|
const textMatch = condition.match(/^text\(\)=["']([^"']+)["']$/);
|
|
if (textMatch) {
|
|
const expectedText = textMatch[1];
|
|
const elementText = element.textContent?.trim() || "";
|
|
return elementText === expectedText;
|
|
}
|
|
|
|
// Handle contains(text(), 'value')
|
|
const textContainsMatch = condition.match(
|
|
/^contains\(text\(\),\s*["']([^"']+)["']\)$/
|
|
);
|
|
if (textContainsMatch) {
|
|
const expectedText = textContainsMatch[1];
|
|
const elementText = element.textContent?.trim() || "";
|
|
return elementText.includes(expectedText);
|
|
}
|
|
|
|
// Handle count(*)=0 (element has no children)
|
|
if (condition === "count(*)=0") {
|
|
return element.children.length === 0;
|
|
}
|
|
|
|
// Handle other count conditions
|
|
const countMatch = condition.match(/^count\(\*\)=(\d+)$/);
|
|
if (countMatch) {
|
|
const expectedCount = parseInt(countMatch[1]);
|
|
return element.children.length === expectedCount;
|
|
}
|
|
|
|
return true;
|
|
};
|
|
|
|
const evaluateXPath = (document, xpath, isShadow = false) => {
|
|
try {
|
|
if (!document || !xpath) {
|
|
console.warn('Invalid document or xpath provided to evaluateXPath');
|
|
return null;
|
|
}
|
|
|
|
const result = document.evaluate(
|
|
xpath,
|
|
document,
|
|
null,
|
|
XPathResult.FIRST_ORDERED_NODE_TYPE,
|
|
null
|
|
).singleNodeValue;
|
|
|
|
if (!isShadow) {
|
|
if (result === null) {
|
|
return null;
|
|
}
|
|
return result;
|
|
}
|
|
|
|
let cleanPath = xpath;
|
|
let isIndexed = false;
|
|
|
|
const indexedMatch = xpath.match(/^\((.*?)\)\[(\d+)\](.*)$/);
|
|
if (indexedMatch) {
|
|
cleanPath = indexedMatch[1] + indexedMatch[3];
|
|
isIndexed = true;
|
|
}
|
|
|
|
const pathParts = cleanPath
|
|
.replace(/^\/\//, "")
|
|
.split("/")
|
|
.map((p) => p.trim())
|
|
.filter((p) => p.length > 0);
|
|
|
|
let currentContexts = [document];
|
|
|
|
for (let i = 0; i < pathParts.length; i++) {
|
|
const part = pathParts[i];
|
|
const nextContexts = [];
|
|
|
|
for (const ctx of currentContexts) {
|
|
const positionalMatch = part.match(/^([^[]+)\[(\d+)\]$/);
|
|
let partWithoutPosition = part;
|
|
let requestedPosition = null;
|
|
|
|
if (positionalMatch) {
|
|
partWithoutPosition = positionalMatch[1];
|
|
requestedPosition = parseInt(positionalMatch[2]);
|
|
}
|
|
|
|
const matched = queryInsideContext(ctx, partWithoutPosition);
|
|
|
|
let elementsToAdd = matched;
|
|
if (requestedPosition !== null) {
|
|
const index = requestedPosition - 1; // XPath is 1-based, arrays are 0-based
|
|
if (index >= 0 && index < matched.length) {
|
|
elementsToAdd = [matched[index]];
|
|
} else {
|
|
console.warn(
|
|
`Position ${requestedPosition} out of range (${matched.length} elements found)`
|
|
);
|
|
elementsToAdd = [];
|
|
}
|
|
}
|
|
|
|
elementsToAdd.forEach((el) => {
|
|
nextContexts.push(el);
|
|
if (el.shadowRoot) {
|
|
nextContexts.push(el.shadowRoot);
|
|
}
|
|
});
|
|
}
|
|
|
|
if (nextContexts.length === 0) {
|
|
return null;
|
|
}
|
|
|
|
currentContexts = nextContexts;
|
|
}
|
|
|
|
if (currentContexts.length > 0) {
|
|
if (isIndexed && indexedMatch) {
|
|
const requestedIndex = parseInt(indexedMatch[2]) - 1;
|
|
if (requestedIndex >= 0 && requestedIndex < currentContexts.length) {
|
|
return currentContexts[requestedIndex];
|
|
} else {
|
|
console.warn(
|
|
`Requested index ${requestedIndex + 1} out of range (${currentContexts.length} elements found)`
|
|
);
|
|
return null;
|
|
}
|
|
}
|
|
|
|
return currentContexts[0];
|
|
}
|
|
|
|
return null;
|
|
} catch (err) {
|
|
console.error("Critical XPath failure:", xpath, err);
|
|
// Return null instead of throwing to prevent crashes
|
|
return null;
|
|
}
|
|
};
|
|
|
|
const evaluateXPathAll = (rootElement, xpath) => {
|
|
try {
|
|
const ownerDoc =
|
|
rootElement.nodeType === Node.DOCUMENT_NODE
|
|
? rootElement
|
|
: rootElement.ownerDocument;
|
|
|
|
if (!ownerDoc) return [];
|
|
|
|
const result = ownerDoc.evaluate(
|
|
xpath,
|
|
rootElement,
|
|
null,
|
|
XPathResult.ORDERED_NODE_SNAPSHOT_TYPE,
|
|
null
|
|
);
|
|
|
|
const elements = [];
|
|
for (let i = 0; i < result.snapshotLength; i++) {
|
|
const node = result.snapshotItem(i);
|
|
if (node && node.nodeType === Node.ELEMENT_NODE) {
|
|
elements.push(node);
|
|
}
|
|
}
|
|
|
|
return elements;
|
|
} catch (error) {
|
|
console.warn("XPath evaluation failed:", xpath, error);
|
|
return [];
|
|
}
|
|
};
|
|
|
|
// Helper function to detect selector type
|
|
const isXPathSelector = (selector) => {
|
|
return (
|
|
selector.startsWith("//") ||
|
|
selector.startsWith("/") ||
|
|
selector.startsWith("./")
|
|
);
|
|
};
|
|
|
|
// Enhanced query function to handle iframe, frame, shadow DOM, CSS selectors, and XPath
|
|
const queryElement = (rootElement, selector) => {
|
|
if (!selector.includes(">>") && !selector.includes(":>>")) {
|
|
// Check if it's an XPath selector
|
|
if (isXPathSelector(selector)) {
|
|
return evaluateXPath(rootElement, selector);
|
|
} else {
|
|
return rootElement.querySelector(selector);
|
|
}
|
|
}
|
|
|
|
const parts = selector.split(/(?:>>|:>>)/).map((part) => part.trim());
|
|
let currentElement = rootElement;
|
|
|
|
for (let i = 0; i < parts.length; i++) {
|
|
if (!currentElement) return null;
|
|
|
|
// Handle iframe and frame traversal with enhanced safety
|
|
if (
|
|
currentElement.tagName === "IFRAME" ||
|
|
currentElement.tagName === "FRAME"
|
|
) {
|
|
try {
|
|
// Check if frame is accessible
|
|
if (!currentElement.contentDocument && !currentElement.contentWindow) {
|
|
console.warn('Frame is not accessible (cross-origin or unloaded)');
|
|
return null;
|
|
}
|
|
|
|
const frameDoc =
|
|
currentElement.contentDocument ||
|
|
currentElement.contentWindow?.document;
|
|
if (!frameDoc) {
|
|
console.warn('Frame document is not available');
|
|
return null;
|
|
}
|
|
|
|
if (isXPathSelector(parts[i])) {
|
|
currentElement = evaluateXPath(frameDoc, parts[i]);
|
|
} else {
|
|
currentElement = frameDoc.querySelector(parts[i]);
|
|
}
|
|
continue;
|
|
} catch (e) {
|
|
console.warn(
|
|
`Cannot access ${currentElement.tagName.toLowerCase()} content:`,
|
|
e
|
|
);
|
|
return null;
|
|
}
|
|
}
|
|
|
|
let nextElement = null;
|
|
|
|
// Try regular DOM first
|
|
if ("querySelector" in currentElement) {
|
|
if (isXPathSelector(parts[i])) {
|
|
nextElement = evaluateXPath(currentElement, parts[i]);
|
|
} else {
|
|
nextElement = currentElement.querySelector(parts[i]);
|
|
}
|
|
}
|
|
|
|
// Try shadow DOM if not found
|
|
if (
|
|
!nextElement &&
|
|
"shadowRoot" in currentElement &&
|
|
currentElement.shadowRoot
|
|
) {
|
|
if (isXPathSelector(parts[i])) {
|
|
nextElement = evaluateXPath(currentElement.shadowRoot, parts[i]);
|
|
} else {
|
|
nextElement = currentElement.shadowRoot.querySelector(parts[i]);
|
|
}
|
|
}
|
|
|
|
// Check children's shadow roots if still not found
|
|
if (!nextElement && "children" in currentElement) {
|
|
const children = Array.from(currentElement.children || []);
|
|
for (const child of children) {
|
|
if (child.shadowRoot) {
|
|
if (isXPathSelector(parts[i])) {
|
|
nextElement = evaluateXPath(child.shadowRoot, parts[i]);
|
|
} else {
|
|
nextElement = child.shadowRoot.querySelector(parts[i]);
|
|
}
|
|
if (nextElement) break;
|
|
}
|
|
}
|
|
}
|
|
|
|
currentElement = nextElement;
|
|
}
|
|
|
|
return currentElement;
|
|
};
|
|
|
|
// Enhanced query all function for both contexts
|
|
const queryElementAll = (rootElement, selector) => {
|
|
if (!selector.includes(">>") && !selector.includes(":>>")) {
|
|
if (isXPathSelector(selector)) {
|
|
return evaluateXPathAll(rootElement, selector);
|
|
} else {
|
|
return Array.from(rootElement.querySelectorAll(selector));
|
|
}
|
|
}
|
|
|
|
const parts = selector.split(/(?:>>|:>>)/).map((part) => part.trim());
|
|
let currentElements = [rootElement];
|
|
|
|
for (const part of parts) {
|
|
const nextElements = [];
|
|
|
|
for (const element of currentElements) {
|
|
// Handle iframe and frame traversal
|
|
if (element.tagName === "IFRAME" || element.tagName === "FRAME") {
|
|
try {
|
|
const frameDoc =
|
|
element.contentDocument || element.contentWindow.document;
|
|
if (frameDoc) {
|
|
if (isXPathSelector(part)) {
|
|
nextElements.push(...evaluateXPathAll(frameDoc, part));
|
|
} else {
|
|
nextElements.push(
|
|
...Array.from(frameDoc.querySelectorAll(part))
|
|
);
|
|
}
|
|
}
|
|
} catch (e) {
|
|
console.warn(
|
|
`Cannot access ${element.tagName.toLowerCase()} content:`,
|
|
e
|
|
);
|
|
continue;
|
|
}
|
|
} else {
|
|
// Regular DOM elements
|
|
if (element.querySelectorAll) {
|
|
if (isXPathSelector(part)) {
|
|
nextElements.push(...evaluateXPathAll(element, part));
|
|
} else {
|
|
nextElements.push(
|
|
...Array.from(element.querySelectorAll(part))
|
|
);
|
|
}
|
|
}
|
|
|
|
// Shadow DOM elements
|
|
if (element.shadowRoot) {
|
|
if (isXPathSelector(part)) {
|
|
nextElements.push(
|
|
...evaluateXPathAll(element.shadowRoot, part)
|
|
);
|
|
} else {
|
|
nextElements.push(
|
|
...Array.from(element.shadowRoot.querySelectorAll(part))
|
|
);
|
|
}
|
|
}
|
|
|
|
// Check children's shadow roots
|
|
const children = Array.from(element.children || []);
|
|
for (const child of children) {
|
|
if (child.shadowRoot) {
|
|
if (isXPathSelector(part)) {
|
|
nextElements.push(
|
|
...evaluateXPathAll(child.shadowRoot, part)
|
|
);
|
|
} else {
|
|
nextElements.push(
|
|
...Array.from(child.shadowRoot.querySelectorAll(part))
|
|
);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
currentElements = nextElements;
|
|
}
|
|
|
|
return currentElements;
|
|
};
|
|
|
|
// Enhanced value extraction with context awareness
|
|
const extractValue = (element, attribute) => {
|
|
if (!element) return null;
|
|
|
|
// Get context-aware base URL
|
|
const baseURL =
|
|
element.ownerDocument?.location?.href || window.location.origin;
|
|
|
|
// Check shadow root first
|
|
if (element.shadowRoot) {
|
|
const shadowContent = element.shadowRoot.textContent;
|
|
if (shadowContent?.trim()) {
|
|
return shadowContent.trim();
|
|
}
|
|
}
|
|
|
|
if (attribute === "innerText") {
|
|
// First try standard innerText/textContent
|
|
let textContent =
|
|
element.innerText?.trim() || element.textContent?.trim();
|
|
|
|
// If empty, check for common data attributes that might contain the text
|
|
if (!textContent) {
|
|
const dataAttributes = [
|
|
"data-600",
|
|
"data-text",
|
|
"data-label",
|
|
"data-value",
|
|
"data-content",
|
|
];
|
|
for (const attr of dataAttributes) {
|
|
const dataValue = element.getAttribute(attr);
|
|
if (dataValue && dataValue.trim()) {
|
|
textContent = dataValue.trim();
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
|
|
return textContent || null;
|
|
} else if (attribute === "innerHTML") {
|
|
return element.innerHTML?.trim() || null;
|
|
} else if (attribute === "src" || attribute === "href") {
|
|
if (attribute === "href" && element.tagName !== "A") {
|
|
const parentElement = element.parentElement;
|
|
if (parentElement && parentElement.tagName === "A") {
|
|
const parentHref = parentElement.getAttribute("href");
|
|
if (parentHref) {
|
|
try {
|
|
return new URL(parentHref, baseURL).href;
|
|
} catch (e) {
|
|
return parentHref;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
const attrValue = element.getAttribute(attribute);
|
|
const dataAttr = attrValue || element.getAttribute("data-" + attribute);
|
|
|
|
if (!dataAttr || dataAttr.trim() === "") {
|
|
if (attribute === "src") {
|
|
const style = window.getComputedStyle(element);
|
|
const bgImage = style.backgroundImage;
|
|
if (bgImage && bgImage !== "none") {
|
|
const matches = bgImage.match(/url\(['"]?([^'")]+)['"]?\)/);
|
|
return matches ? new URL(matches[1], baseURL).href : null;
|
|
}
|
|
}
|
|
return null;
|
|
}
|
|
|
|
try {
|
|
return new URL(dataAttr, baseURL).href;
|
|
} catch (e) {
|
|
console.warn("Error creating URL from", dataAttr, e);
|
|
return dataAttr;
|
|
}
|
|
}
|
|
return element.getAttribute(attribute);
|
|
};
|
|
|
|
// Enhanced table ancestor finding with context support
|
|
const findTableAncestor = (element) => {
|
|
let currentElement = element;
|
|
const MAX_DEPTH = 5;
|
|
let depth = 0;
|
|
|
|
while (currentElement && depth < MAX_DEPTH) {
|
|
// Handle shadow DOM
|
|
if (currentElement.getRootNode() instanceof ShadowRoot) {
|
|
currentElement = currentElement.getRootNode().host;
|
|
continue;
|
|
}
|
|
|
|
if (currentElement.tagName === "TD") {
|
|
return { type: "TD", element: currentElement };
|
|
} else if (currentElement.tagName === "TR") {
|
|
return { type: "TR", element: currentElement };
|
|
}
|
|
|
|
// Handle iframe and frame crossing
|
|
if (
|
|
currentElement.tagName === "IFRAME" ||
|
|
currentElement.tagName === "FRAME"
|
|
) {
|
|
try {
|
|
currentElement = currentElement.contentDocument.body;
|
|
} catch (e) {
|
|
return null;
|
|
}
|
|
} else {
|
|
currentElement = currentElement.parentElement;
|
|
}
|
|
depth++;
|
|
}
|
|
return null;
|
|
};
|
|
|
|
// Helper function to get cell index
|
|
const getCellIndex = (td) => {
|
|
if (td.getRootNode() instanceof ShadowRoot) {
|
|
const shadowRoot = td.getRootNode();
|
|
const allCells = Array.from(shadowRoot.querySelectorAll("td"));
|
|
return allCells.indexOf(td);
|
|
}
|
|
|
|
let index = 0;
|
|
let sibling = td;
|
|
while ((sibling = sibling.previousElementSibling)) {
|
|
index++;
|
|
}
|
|
return index;
|
|
};
|
|
|
|
// Helper function to check for TH elements
|
|
const hasThElement = (row, tableFields) => {
|
|
for (const [_, { selector }] of Object.entries(tableFields)) {
|
|
const element = queryElement(row, selector);
|
|
if (element) {
|
|
let current = element;
|
|
while (current && current !== row) {
|
|
if (current.getRootNode() instanceof ShadowRoot) {
|
|
current = current.getRootNode().host;
|
|
continue;
|
|
}
|
|
|
|
if (current.tagName === "TH") return true;
|
|
|
|
if (current.tagName === "IFRAME" || current.tagName === "FRAME") {
|
|
try {
|
|
current = current.contentDocument.body;
|
|
} catch (e) {
|
|
break;
|
|
}
|
|
} else {
|
|
current = current.parentElement;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
return false;
|
|
};
|
|
|
|
// Helper function to filter rows
|
|
const filterRowsBasedOnTag = (rows, tableFields) => {
|
|
for (const row of rows) {
|
|
if (hasThElement(row, tableFields)) {
|
|
return rows;
|
|
}
|
|
}
|
|
return rows.filter((row) => {
|
|
const directTH = row.getElementsByTagName("TH").length === 0;
|
|
const shadowTH = row.shadowRoot
|
|
? row.shadowRoot.querySelector("th") === null
|
|
: true;
|
|
return directTH && shadowTH;
|
|
});
|
|
};
|
|
|
|
// Class similarity comparison functions
|
|
const calculateClassSimilarity = (classList1, classList2) => {
|
|
const set1 = new Set(classList1);
|
|
const set2 = new Set(classList2);
|
|
const intersection = new Set([...set1].filter((x) => set2.has(x)));
|
|
const union = new Set([...set1, ...set2]);
|
|
return intersection.size / union.size;
|
|
};
|
|
|
|
// Enhanced similar elements finding with context support
|
|
const findSimilarElements = (baseElement, similarityThreshold = 0.7) => {
|
|
const baseClasses = Array.from(baseElement.classList);
|
|
if (baseClasses.length === 0) return [];
|
|
|
|
const allElements = [];
|
|
|
|
// Get elements from main document
|
|
allElements.push(...document.getElementsByTagName(baseElement.tagName));
|
|
|
|
// Get elements from shadow DOM
|
|
if (baseElement.getRootNode() instanceof ShadowRoot) {
|
|
const shadowHost = baseElement.getRootNode().host;
|
|
allElements.push(
|
|
...shadowHost.getElementsByTagName(baseElement.tagName)
|
|
);
|
|
}
|
|
|
|
// Get elements from iframes and frames
|
|
const frames = [
|
|
...Array.from(document.getElementsByTagName("iframe")),
|
|
...Array.from(document.getElementsByTagName("frame")),
|
|
];
|
|
|
|
for (const frame of frames) {
|
|
try {
|
|
const frameDoc =
|
|
frame.contentDocument || frame.contentWindow.document;
|
|
allElements.push(
|
|
...frameDoc.getElementsByTagName(baseElement.tagName)
|
|
);
|
|
} catch (e) {
|
|
console.warn(
|
|
`Cannot access ${frame.tagName.toLowerCase()} content:`,
|
|
e
|
|
);
|
|
}
|
|
}
|
|
|
|
return allElements.filter((element) => {
|
|
if (element === baseElement) return false;
|
|
const similarity = calculateClassSimilarity(
|
|
baseClasses,
|
|
Array.from(element.classList)
|
|
);
|
|
return similarity >= similarityThreshold;
|
|
});
|
|
};
|
|
|
|
const tryFallbackSelector = (rootElement, originalSelector) => {
|
|
let element = queryElement(rootElement, originalSelector);
|
|
|
|
if (!element && originalSelector.includes("nth-child")) {
|
|
const match = originalSelector.match(/nth-child\((\d+)\)/);
|
|
if (match) {
|
|
const position = parseInt(match[1], 10);
|
|
|
|
for (let i = position - 1; i >= 1; i--) {
|
|
const fallbackSelector = originalSelector.replace(
|
|
/nth-child\(\d+\)/,
|
|
`nth-child(${i})`
|
|
);
|
|
element = queryElement(rootElement, fallbackSelector);
|
|
if (element) break;
|
|
}
|
|
|
|
if (!element) {
|
|
const baseSelector = originalSelector.replace(
|
|
/\:nth-child\(\d+\)/,
|
|
""
|
|
);
|
|
element = queryElement(rootElement, baseSelector);
|
|
}
|
|
}
|
|
}
|
|
|
|
return element;
|
|
};
|
|
|
|
// Create indexed XPath for specific container instance
|
|
const createIndexedXPath = (
|
|
childSelector,
|
|
listSelector,
|
|
containerIndex
|
|
) => {
|
|
// Check if the child selector contains the list selector pattern
|
|
if (childSelector.includes(listSelector.replace("//", ""))) {
|
|
// Replace the list selector part with indexed version
|
|
const listPattern = listSelector.replace("//", "");
|
|
const indexedListSelector = `(${listSelector})[${containerIndex}]`;
|
|
|
|
const indexedSelector = childSelector.replace(
|
|
`//${listPattern}`,
|
|
indexedListSelector
|
|
);
|
|
|
|
return indexedSelector;
|
|
} else {
|
|
// If pattern doesn't match, create a more generic indexed selector
|
|
return `(${listSelector})[${containerIndex}]${childSelector.replace(
|
|
"//",
|
|
"/"
|
|
)}`;
|
|
}
|
|
};
|
|
|
|
// Main scraping logic with unified support for both CSS and XPath
|
|
console.log("🚀 Starting unified list data extraction");
|
|
console.log("List Selector:", listSelector);
|
|
console.log("Fields:", fields);
|
|
|
|
let containers = queryElementAll(document, listSelector);
|
|
containers = Array.from(containers);
|
|
|
|
if (containers.length === 0) {
|
|
console.warn("❌ No containers found for listSelector:", listSelector);
|
|
return [];
|
|
}
|
|
|
|
console.log(`📦 Found ${containers.length} list containers`);
|
|
|
|
// For CSS selectors, try to find similar containers if needed
|
|
if (
|
|
!isXPathSelector(listSelector) &&
|
|
limit > 1 &&
|
|
containers.length === 1
|
|
) {
|
|
const baseContainer = containers[0];
|
|
const similarContainers = findSimilarElements(baseContainer);
|
|
|
|
if (similarContainers.length > 0) {
|
|
const newContainers = similarContainers.filter(
|
|
(container) => !container.matches(listSelector)
|
|
);
|
|
containers = [...containers, ...newContainers];
|
|
}
|
|
}
|
|
|
|
const containerFields = containers.map(() => ({
|
|
tableFields: {},
|
|
nonTableFields: {},
|
|
}));
|
|
|
|
// For XPath selectors, use the new approach
|
|
if (isXPathSelector(listSelector)) {
|
|
const extractedData = [];
|
|
const containersToProcess = Math.min(containers.length, limit);
|
|
|
|
for (
|
|
let containerIndex = 0;
|
|
containerIndex < containersToProcess;
|
|
containerIndex++
|
|
) {
|
|
const record = {};
|
|
|
|
for (const [label, field] of Object.entries(fields)) {
|
|
let element = null;
|
|
|
|
if (isXPathSelector(field.selector)) {
|
|
// Create indexed absolute XPath
|
|
const indexedSelector = createIndexedXPath(
|
|
field.selector,
|
|
listSelector,
|
|
containerIndex + 1
|
|
);
|
|
element = evaluateXPath(document, indexedSelector, field.isShadow);
|
|
} else {
|
|
// Fallback for CSS selectors within XPath containers
|
|
const container = containers[containerIndex];
|
|
element = queryElement(container, field.selector);
|
|
}
|
|
|
|
if (element) {
|
|
const value = extractValue(element, field.attribute);
|
|
if (value !== null && value !== "") {
|
|
record[label] = value;
|
|
} else {
|
|
record[label] = "";
|
|
}
|
|
} else {
|
|
record[label] = "";
|
|
}
|
|
}
|
|
|
|
if (Object.values(record).some((value) => value !== "")) {
|
|
extractedData.push(record);
|
|
}
|
|
}
|
|
|
|
console.log(`📊 Total records extracted: ${extractedData.length}`);
|
|
return extractedData;
|
|
}
|
|
|
|
// For CSS selectors, use the original table-aware approach
|
|
containers.forEach((container, containerIndex) => {
|
|
for (const [label, field] of Object.entries(fields)) {
|
|
const sampleElement = queryElement(container, field.selector);
|
|
|
|
if (sampleElement) {
|
|
const ancestor = findTableAncestor(sampleElement);
|
|
if (ancestor) {
|
|
containerFields[containerIndex].tableFields[label] = {
|
|
...field,
|
|
tableContext: ancestor.type,
|
|
cellIndex:
|
|
ancestor.type === "TD" ? getCellIndex(ancestor.element) : -1,
|
|
};
|
|
} else {
|
|
containerFields[containerIndex].nonTableFields[label] = field;
|
|
}
|
|
} else {
|
|
containerFields[containerIndex].nonTableFields[label] = field;
|
|
}
|
|
}
|
|
});
|
|
|
|
const tableData = [];
|
|
const nonTableData = [];
|
|
|
|
// Process table data with support for iframes, frames, and shadow DOM
|
|
for (
|
|
let containerIndex = 0;
|
|
containerIndex < containers.length;
|
|
containerIndex++
|
|
) {
|
|
const container = containers[containerIndex];
|
|
const { tableFields } = containerFields[containerIndex];
|
|
|
|
if (Object.keys(tableFields).length > 0) {
|
|
const firstField = Object.values(tableFields)[0];
|
|
const firstElement = queryElement(container, firstField.selector);
|
|
let tableContext = firstElement;
|
|
|
|
// Find table context including iframe, frame and shadow DOM
|
|
while (
|
|
tableContext &&
|
|
tableContext.tagName !== "TABLE" &&
|
|
tableContext !== container
|
|
) {
|
|
if (tableContext.getRootNode() instanceof ShadowRoot) {
|
|
tableContext = tableContext.getRootNode().host;
|
|
continue;
|
|
}
|
|
|
|
if (
|
|
tableContext.tagName === "IFRAME" ||
|
|
tableContext.tagName === "FRAME"
|
|
) {
|
|
try {
|
|
tableContext = tableContext.contentDocument.body;
|
|
} catch (e) {
|
|
break;
|
|
}
|
|
} else {
|
|
tableContext = tableContext.parentElement;
|
|
}
|
|
}
|
|
|
|
if (tableContext) {
|
|
// Get rows from all contexts
|
|
const rows = [];
|
|
|
|
// Get rows from regular DOM
|
|
rows.push(...tableContext.getElementsByTagName("TR"));
|
|
|
|
// Get rows from shadow DOM
|
|
if (tableContext.shadowRoot) {
|
|
rows.push(...tableContext.shadowRoot.getElementsByTagName("TR"));
|
|
}
|
|
|
|
// Get rows from iframes and frames
|
|
if (
|
|
tableContext.tagName === "IFRAME" ||
|
|
tableContext.tagName === "FRAME"
|
|
) {
|
|
try {
|
|
const frameDoc =
|
|
tableContext.contentDocument ||
|
|
tableContext.contentWindow.document;
|
|
rows.push(...frameDoc.getElementsByTagName("TR"));
|
|
} catch (e) {
|
|
console.warn(
|
|
`Cannot access ${tableContext.tagName.toLowerCase()} rows:`,
|
|
e
|
|
);
|
|
}
|
|
}
|
|
|
|
const processedRows = filterRowsBasedOnTag(rows, tableFields);
|
|
|
|
for (
|
|
let rowIndex = 0;
|
|
rowIndex < Math.min(processedRows.length, limit);
|
|
rowIndex++
|
|
) {
|
|
const record = {};
|
|
const currentRow = processedRows[rowIndex];
|
|
|
|
for (const [
|
|
label,
|
|
{ selector, attribute, cellIndex },
|
|
] of Object.entries(tableFields)) {
|
|
let element = null;
|
|
|
|
if (cellIndex >= 0) {
|
|
// Get TD element considering both contexts
|
|
let td = currentRow.children[cellIndex];
|
|
|
|
// Check shadow DOM for td
|
|
if (!td && currentRow.shadowRoot) {
|
|
const shadowCells = currentRow.shadowRoot.children;
|
|
if (shadowCells && shadowCells.length > cellIndex) {
|
|
td = shadowCells[cellIndex];
|
|
}
|
|
}
|
|
|
|
if (td) {
|
|
element = queryElement(td, selector);
|
|
|
|
if (
|
|
!element &&
|
|
selector
|
|
.split(/(?:>>|:>>)/)
|
|
.pop()
|
|
.includes("td:nth-child")
|
|
) {
|
|
element = td;
|
|
}
|
|
|
|
if (!element) {
|
|
const tagOnlySelector = selector.split(".")[0];
|
|
element = queryElement(td, tagOnlySelector);
|
|
}
|
|
|
|
if (!element) {
|
|
let currentElement = td;
|
|
while (
|
|
currentElement &&
|
|
currentElement.children.length > 0
|
|
) {
|
|
let foundContentChild = false;
|
|
for (const child of currentElement.children) {
|
|
if (extractValue(child, attribute)) {
|
|
currentElement = child;
|
|
foundContentChild = true;
|
|
break;
|
|
}
|
|
}
|
|
if (!foundContentChild) break;
|
|
}
|
|
element = currentElement;
|
|
}
|
|
}
|
|
} else {
|
|
element = queryElement(currentRow, selector);
|
|
}
|
|
|
|
if (element) {
|
|
record[label] = extractValue(element, attribute);
|
|
}
|
|
}
|
|
|
|
if (Object.keys(record).length > 0) {
|
|
tableData.push(record);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// Process non-table data with all contexts support
|
|
for (
|
|
let containerIndex = 0;
|
|
containerIndex < containers.length;
|
|
containerIndex++
|
|
) {
|
|
if (nonTableData.length >= limit) break;
|
|
|
|
const container = containers[containerIndex];
|
|
const { nonTableFields } = containerFields[containerIndex];
|
|
|
|
if (Object.keys(nonTableFields).length > 0) {
|
|
const record = {};
|
|
|
|
for (const [label, { selector, attribute }] of Object.entries(
|
|
nonTableFields
|
|
)) {
|
|
// Get the last part of the selector after any context delimiter
|
|
const relativeSelector = selector.split(/(?:>>|:>>)/).slice(-1)[0];
|
|
const element = tryFallbackSelector(container, relativeSelector);
|
|
|
|
if (element) {
|
|
record[label] = extractValue(element, attribute);
|
|
}
|
|
}
|
|
|
|
if (Object.keys(record).length > 0) {
|
|
nonTableData.push(record);
|
|
}
|
|
}
|
|
}
|
|
|
|
// Merge and limit the results
|
|
const scrapedData = [...tableData, ...nonTableData];
|
|
console.log(`📊 Total records extracted: ${scrapedData.length}`);
|
|
|
|
return scrapedData;
|
|
};
|
|
|
|
/**
|
|
* Gets all children of the elements matching the listSelector,
|
|
* returning their CSS selectors and innerText.
|
|
* @param {string} listSelector - Selector for the list container(s)
|
|
* @returns {Array.<Object>} Array of objects, each containing the CSS selector and innerText of the children
|
|
*/
|
|
window.scrapeListAuto = function (listSelector) {
|
|
const lists = Array.from(document.querySelectorAll(listSelector));
|
|
|
|
const results = [];
|
|
|
|
lists.forEach(list => {
|
|
const children = Array.from(list.children);
|
|
|
|
children.forEach(child => {
|
|
const selectors = [];
|
|
let element = child;
|
|
|
|
// Traverse up to gather the CSS selector for the element
|
|
while (element && element !== document) {
|
|
let selector = element.nodeName.toLowerCase();
|
|
if (element.id) {
|
|
selector += `#${element.id}`;
|
|
selectors.push(selector);
|
|
break;
|
|
} else {
|
|
const className = element.className.trim().split(/\s+/).join('.');
|
|
if (className) {
|
|
selector += `.${className}`;
|
|
}
|
|
selectors.push(selector);
|
|
element = element.parentElement;
|
|
}
|
|
}
|
|
|
|
results.push({
|
|
selector: selectors.reverse().join(' > '),
|
|
innerText: child.innerText.trim()
|
|
});
|
|
});
|
|
});
|
|
|
|
return results;
|
|
};
|
|
|
|
})(window); |