Merge branch 'develop' into iframe
This commit is contained in:
Rohit
@@ -205,50 +205,66 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3,
|
||||
}
|
||||
|
||||
function findAllElements(config) {
|
||||
// Check if selector contains iframe notation (:>>)
|
||||
if (!config.selector.includes(':>>')) {
|
||||
return Array.from(document.querySelectorAll(config.selector));
|
||||
// Regular DOM query if no special delimiters
|
||||
if (!config.selector.includes('>>') && !config.selector.includes(':>>')) {
|
||||
return Array.from(document.querySelectorAll(config.selector));
|
||||
}
|
||||
|
||||
// For iframe traversal, split by iframe boundary marker
|
||||
const parts = config.selector.split(':>>').map(s => s.trim());
|
||||
// Split by both types of delimiters
|
||||
const parts = config.selector.split(/(?:>>|:>>)/).map(s => s.trim());
|
||||
const delimiters = config.selector.match(/(?:>>|:>>)/g) || [];
|
||||
let currentElements = [document];
|
||||
|
||||
|
||||
for (let i = 0; i < parts.length; i++) {
|
||||
const part = parts[i];
|
||||
const nextElements = [];
|
||||
|
||||
const isLast = i === parts.length - 1;
|
||||
const delimiter = delimiters[i] || '';
|
||||
const isIframeTraversal = delimiter === ':>>';
|
||||
|
||||
for (const element of currentElements) {
|
||||
try {
|
||||
let targets;
|
||||
|
||||
if (i === 0) {
|
||||
// First selector is queried from main document
|
||||
targets = Array.from(element.querySelectorAll(part))
|
||||
.filter(el => {
|
||||
// Only include iframes if not the last part
|
||||
if (i === parts.length - 1) return true;
|
||||
return el.tagName === 'IFRAME';
|
||||
});
|
||||
// First selector is queried from main document
|
||||
targets = Array.from(element.querySelectorAll(part))
|
||||
.filter(el => {
|
||||
if (isLast) return true;
|
||||
// For iframe traversal, only include iframes
|
||||
if (isIframeTraversal) return el.tagName === 'IFRAME';
|
||||
// For shadow DOM traversal, only include elements with shadow root
|
||||
return el.shadowRoot && el.shadowRoot.mode === 'open';
|
||||
});
|
||||
} else {
|
||||
// For subsequent selectors, we need to look inside iframes
|
||||
if (isIframeTraversal) {
|
||||
// Handle iframe traversal
|
||||
const iframeDocument = element.contentDocument || element.contentWindow?.document;
|
||||
if (!iframeDocument) continue;
|
||||
|
||||
|
||||
targets = Array.from(iframeDocument.querySelectorAll(part));
|
||||
|
||||
// If this isn't the last part, filter for iframes only
|
||||
if (i < parts.length - 1) {
|
||||
targets = targets.filter(el => el.tagName === 'IFRAME');
|
||||
if (!isLast) {
|
||||
targets = targets.filter(el => el.tagName === 'IFRAME');
|
||||
}
|
||||
} else {
|
||||
// Handle shadow DOM traversal
|
||||
const shadowRoot = element.shadowRoot;
|
||||
if (!shadowRoot || shadowRoot.mode !== 'open') continue;
|
||||
|
||||
targets = Array.from(shadowRoot.querySelectorAll(part));
|
||||
if (!isLast) {
|
||||
targets = targets.filter(el => el.shadowRoot && el.shadowRoot.mode === 'open');
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
nextElements.push(...targets);
|
||||
} catch (error) {
|
||||
// Handle cross-origin iframe access errors
|
||||
console.warn('Cannot access iframe content:', error);
|
||||
console.warn('Cannot access content:', error);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
if (nextElements.length === 0) return [];
|
||||
currentElements = nextElements;
|
||||
}
|
||||
@@ -291,26 +307,27 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3,
|
||||
)[0];
|
||||
}
|
||||
|
||||
// Find minimal bounding elements
|
||||
function getMBEs(elements) {
|
||||
return elements.map((element) => {
|
||||
let candidate = element;
|
||||
const isUniqueChild = (e) => elements
|
||||
.filter((elem) => {
|
||||
// Handle iframe boundaries when checking containment
|
||||
const sameDocument = elem.ownerDocument === e.ownerDocument;
|
||||
return sameDocument && e.parentNode?.contains(elem);
|
||||
})
|
||||
.length === 1;
|
||||
|
||||
while (candidate && isUniqueChild(candidate)) {
|
||||
candidate = candidate.parentNode;
|
||||
}
|
||||
|
||||
return candidate;
|
||||
let candidate = element;
|
||||
const isUniqueChild = (e) => elements
|
||||
.filter((elem) => {
|
||||
// Handle both iframe and shadow DOM boundaries
|
||||
const sameContext = elem.getRootNode() === e.getRootNode() &&
|
||||
elem.ownerDocument === e.ownerDocument;
|
||||
return sameContext && e.parentNode?.contains(elem);
|
||||
})
|
||||
.length === 1;
|
||||
|
||||
while (candidate && isUniqueChild(candidate)) {
|
||||
candidate = candidate.parentNode;
|
||||
}
|
||||
|
||||
return candidate;
|
||||
});
|
||||
}
|
||||
|
||||
// Main scraping logic remains the same
|
||||
const seedName = getSeedKey(lists);
|
||||
const seedElements = findAllElements(lists[seedName]);
|
||||
const MBEs = getMBEs(seedElements);
|
||||
@@ -364,144 +381,207 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3,
|
||||
* @returns {Array.<Array.<Object>>} Array of arrays of scraped items, one sub-array per list
|
||||
*/
|
||||
window.scrapeList = async function ({ listSelector, fields, limit = 10 }) {
|
||||
// Helper function to query elements within an iframe
|
||||
const queryIframe = (rootElement, selector) => {
|
||||
if (!selector.includes(':>>')) {
|
||||
return rootElement.querySelector(selector);
|
||||
}
|
||||
// Enhanced query function to handle both iframe and shadow DOM
|
||||
const queryElement = (rootElement, selector) => {
|
||||
if (!selector.includes('>>') && !selector.includes(':>>')) {
|
||||
return rootElement.querySelector(selector);
|
||||
}
|
||||
|
||||
const parts = selector.split(':>>').map(part => part.trim());
|
||||
let currentElement = rootElement;
|
||||
const parts = selector.split(/(?:>>|:>>)/).map(part => part.trim());
|
||||
let currentElement = rootElement;
|
||||
|
||||
for (let i = 0; i < parts.length; i++) {
|
||||
if (!currentElement) return null;
|
||||
for (let i = 0; i < parts.length; i++) {
|
||||
if (!currentElement) return null;
|
||||
|
||||
// Handle iframe content document
|
||||
if (currentElement.tagName === 'IFRAME') {
|
||||
try {
|
||||
const iframeDoc = currentElement.contentDocument || currentElement.contentWindow.document;
|
||||
currentElement = iframeDoc.querySelector(parts[i]);
|
||||
continue;
|
||||
} catch (e) {
|
||||
console.error('Cannot access iframe content:', e);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
// Handle iframe traversal
|
||||
if (currentElement.tagName === 'IFRAME') {
|
||||
try {
|
||||
const iframeDoc = currentElement.contentDocument || currentElement.contentWindow.document;
|
||||
currentElement = iframeDoc.querySelector(parts[i]);
|
||||
continue;
|
||||
} catch (e) {
|
||||
console.warn('Cannot access iframe content:', e);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
currentElement = currentElement.querySelector(parts[i]);
|
||||
}
|
||||
// Try regular DOM first
|
||||
let nextElement = currentElement.querySelector(parts[i]);
|
||||
|
||||
return currentElement;
|
||||
// Try shadow DOM if not found
|
||||
if (!nextElement && currentElement.shadowRoot) {
|
||||
nextElement = currentElement.shadowRoot.querySelector(parts[i]);
|
||||
}
|
||||
|
||||
// Check children's shadow roots if still not found
|
||||
if (!nextElement) {
|
||||
const children = Array.from(currentElement.children || []);
|
||||
for (const child of children) {
|
||||
if (child.shadowRoot) {
|
||||
nextElement = child.shadowRoot.querySelector(parts[i]);
|
||||
if (nextElement) break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
currentElement = nextElement;
|
||||
}
|
||||
|
||||
return currentElement;
|
||||
};
|
||||
|
||||
// Helper function to query all matching elements within iframes
|
||||
const queryIframeAll = (rootElement, selector) => {
|
||||
if (!selector.includes(':>>')) {
|
||||
return rootElement.querySelectorAll(selector);
|
||||
}
|
||||
// Enhanced query all function for both contexts
|
||||
const queryElementAll = (rootElement, selector) => {
|
||||
if (!selector.includes('>>') && !selector.includes(':>>')) {
|
||||
return rootElement.querySelectorAll(selector);
|
||||
}
|
||||
|
||||
const parts = selector.split(':>>').map(part => part.trim());
|
||||
let currentElements = [rootElement];
|
||||
|
||||
for (const part of parts) {
|
||||
const nextElements = [];
|
||||
|
||||
for (const element of currentElements) {
|
||||
if (element.tagName === 'IFRAME') {
|
||||
try {
|
||||
const iframeDoc = element.contentDocument || element.contentWindow.document;
|
||||
nextElements.push(...iframeDoc.querySelectorAll(part));
|
||||
} catch (e) {
|
||||
console.error('Cannot access iframe content:', e);
|
||||
continue;
|
||||
}
|
||||
} else {
|
||||
nextElements.push(...element.querySelectorAll(part));
|
||||
}
|
||||
}
|
||||
|
||||
currentElements = nextElements;
|
||||
}
|
||||
|
||||
return currentElements;
|
||||
const parts = selector.split(/(?:>>|:>>)/).map(part => part.trim());
|
||||
let currentElements = [rootElement];
|
||||
|
||||
for (const part of parts) {
|
||||
const nextElements = [];
|
||||
|
||||
for (const element of currentElements) {
|
||||
// Handle iframe traversal
|
||||
if (element.tagName === 'IFRAME') {
|
||||
try {
|
||||
const iframeDoc = element.contentDocument || element.contentWindow.document;
|
||||
nextElements.push(...iframeDoc.querySelectorAll(part));
|
||||
} catch (e) {
|
||||
console.warn('Cannot access iframe content:', e);
|
||||
continue;
|
||||
}
|
||||
} else {
|
||||
// Regular DOM elements
|
||||
if (element.querySelectorAll) {
|
||||
nextElements.push(...element.querySelectorAll(part));
|
||||
}
|
||||
|
||||
// Shadow DOM elements
|
||||
if (element.shadowRoot) {
|
||||
nextElements.push(...element.shadowRoot.querySelectorAll(part));
|
||||
}
|
||||
|
||||
// Check children's shadow roots
|
||||
const children = Array.from(element.children || []);
|
||||
for (const child of children) {
|
||||
if (child.shadowRoot) {
|
||||
nextElements.push(...child.shadowRoot.querySelectorAll(part));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
currentElements = nextElements;
|
||||
}
|
||||
|
||||
return currentElements;
|
||||
};
|
||||
|
||||
// Helper function to extract values from elements
|
||||
// Enhanced value extraction with context awareness
|
||||
function extractValue(element, attribute) {
|
||||
if (!element) return null;
|
||||
|
||||
if (attribute === 'innerText') {
|
||||
return element.innerText.trim();
|
||||
} else if (attribute === 'innerHTML') {
|
||||
return element.innerHTML.trim();
|
||||
} else if (attribute === 'src' || attribute === 'href') {
|
||||
const attrValue = element.getAttribute(attribute);
|
||||
return attrValue ? new URL(attrValue, window.location.origin).href : null;
|
||||
}
|
||||
return element.getAttribute(attribute);
|
||||
if (!element) return null;
|
||||
|
||||
// Get context-aware base URL
|
||||
const baseURL = element.ownerDocument?.location?.href || window.location.origin;
|
||||
|
||||
// Check shadow root first
|
||||
if (element.shadowRoot) {
|
||||
const shadowContent = element.shadowRoot.textContent;
|
||||
if (shadowContent?.trim()) {
|
||||
return shadowContent.trim();
|
||||
}
|
||||
}
|
||||
|
||||
if (attribute === 'innerText') {
|
||||
return element.innerText.trim();
|
||||
} else if (attribute === 'innerHTML') {
|
||||
return element.innerHTML.trim();
|
||||
} else if (attribute === 'src' || attribute === 'href') {
|
||||
const attrValue = element.getAttribute(attribute);
|
||||
return attrValue ? new URL(attrValue, baseURL).href : null;
|
||||
}
|
||||
return element.getAttribute(attribute);
|
||||
}
|
||||
|
||||
// Helper function to find table ancestor elements
|
||||
// Enhanced table ancestor finding with context support
|
||||
function findTableAncestor(element) {
|
||||
let currentElement = element;
|
||||
const MAX_DEPTH = 5;
|
||||
let depth = 0;
|
||||
|
||||
while (currentElement && depth < MAX_DEPTH) {
|
||||
if (currentElement.tagName === 'TD') {
|
||||
return { type: 'TD', element: currentElement };
|
||||
} else if (currentElement.tagName === 'TR') {
|
||||
return { type: 'TR', element: currentElement };
|
||||
}
|
||||
|
||||
// Handle iframe boundary crossing
|
||||
if (currentElement.tagName === 'IFRAME') {
|
||||
try {
|
||||
currentElement = currentElement.contentDocument.body;
|
||||
} catch (e) {
|
||||
return null;
|
||||
}
|
||||
} else {
|
||||
currentElement = currentElement.parentElement;
|
||||
}
|
||||
depth++;
|
||||
}
|
||||
return null;
|
||||
let currentElement = element;
|
||||
const MAX_DEPTH = 5;
|
||||
let depth = 0;
|
||||
|
||||
while (currentElement && depth < MAX_DEPTH) {
|
||||
// Handle shadow DOM
|
||||
if (currentElement.getRootNode() instanceof ShadowRoot) {
|
||||
currentElement = currentElement.getRootNode().host;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (currentElement.tagName === 'TD') {
|
||||
return { type: 'TD', element: currentElement };
|
||||
} else if (currentElement.tagName === 'TR') {
|
||||
return { type: 'TR', element: currentElement };
|
||||
}
|
||||
|
||||
// Handle iframe crossing
|
||||
if (currentElement.tagName === 'IFRAME') {
|
||||
try {
|
||||
currentElement = currentElement.contentDocument.body;
|
||||
} catch (e) {
|
||||
return null;
|
||||
}
|
||||
} else {
|
||||
currentElement = currentElement.parentElement;
|
||||
}
|
||||
depth++;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
// Helper function to get cell index
|
||||
function getCellIndex(td) {
|
||||
let index = 0;
|
||||
let sibling = td;
|
||||
while (sibling = sibling.previousElementSibling) {
|
||||
index++;
|
||||
}
|
||||
return index;
|
||||
if (td.getRootNode() instanceof ShadowRoot) {
|
||||
const shadowRoot = td.getRootNode();
|
||||
const allCells = Array.from(shadowRoot.querySelectorAll('td'));
|
||||
return allCells.indexOf(td);
|
||||
}
|
||||
|
||||
let index = 0;
|
||||
let sibling = td;
|
||||
while (sibling = sibling.previousElementSibling) {
|
||||
index++;
|
||||
}
|
||||
return index;
|
||||
}
|
||||
|
||||
// Helper function to check for TH elements
|
||||
function hasThElement(row, tableFields) {
|
||||
for (const [label, { selector }] of Object.entries(tableFields)) {
|
||||
const element = queryIframe(row, selector);
|
||||
if (element) {
|
||||
let current = element;
|
||||
while (current && current !== row) {
|
||||
if (current.tagName === 'TH') {
|
||||
return true;
|
||||
}
|
||||
if (current.tagName === 'IFRAME') {
|
||||
try {
|
||||
current = current.contentDocument.body;
|
||||
} catch (e) {
|
||||
break;
|
||||
}
|
||||
} else {
|
||||
current = current.parentElement;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return false;
|
||||
for (const [_, { selector }] of Object.entries(tableFields)) {
|
||||
const element = queryElement(row, selector);
|
||||
if (element) {
|
||||
let current = element;
|
||||
while (current && current !== row) {
|
||||
if (current.getRootNode() instanceof ShadowRoot) {
|
||||
current = current.getRootNode().host;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (current.tagName === 'TH') return true;
|
||||
|
||||
if (current.tagName === 'IFRAME') {
|
||||
try {
|
||||
current = current.contentDocument.body;
|
||||
} catch (e) {
|
||||
break;
|
||||
}
|
||||
} else {
|
||||
current = current.parentElement;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
// Helper function to filter rows
|
||||
@@ -511,7 +591,13 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3,
|
||||
return rows;
|
||||
}
|
||||
}
|
||||
return rows.filter(row => row.getElementsByTagName('TH').length === 0);
|
||||
// Include shadow DOM in TH search
|
||||
return rows.filter(row => {
|
||||
const directTH = row.getElementsByTagName('TH').length === 0;
|
||||
const shadowTH = row.shadowRoot ?
|
||||
row.shadowRoot.querySelector('th') === null : true;
|
||||
return directTH && shadowTH;
|
||||
});
|
||||
}
|
||||
|
||||
// Class similarity comparison functions
|
||||
@@ -523,188 +609,231 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3,
|
||||
return intersection.size / union.size;
|
||||
}
|
||||
|
||||
// Enhanced similar elements finding with context support
|
||||
function findSimilarElements(baseElement, similarityThreshold = 0.7) {
|
||||
const baseClasses = Array.from(baseElement.classList);
|
||||
if (baseClasses.length === 0) return [];
|
||||
|
||||
// Include elements from all iframes
|
||||
const allElements = [];
|
||||
const iframes = document.getElementsByTagName('iframe');
|
||||
|
||||
// Add elements from main document
|
||||
allElements.push(...document.getElementsByTagName(baseElement.tagName));
|
||||
|
||||
// Add elements from each iframe
|
||||
for (const iframe of iframes) {
|
||||
try {
|
||||
const iframeDoc = iframe.contentDocument || iframe.contentWindow.document;
|
||||
allElements.push(...iframeDoc.getElementsByTagName(baseElement.tagName));
|
||||
} catch (e) {
|
||||
console.error('Cannot access iframe content:', e);
|
||||
}
|
||||
}
|
||||
|
||||
return allElements.filter(element => {
|
||||
if (element === baseElement) return false;
|
||||
const similarity = calculateClassSimilarity(
|
||||
baseClasses,
|
||||
Array.from(element.classList)
|
||||
);
|
||||
return similarity >= similarityThreshold;
|
||||
});
|
||||
const baseClasses = Array.from(baseElement.classList);
|
||||
if (baseClasses.length === 0) return [];
|
||||
|
||||
const allElements = [];
|
||||
|
||||
// Get elements from main document
|
||||
allElements.push(...document.getElementsByTagName(baseElement.tagName));
|
||||
|
||||
// Get elements from shadow DOM
|
||||
if (baseElement.getRootNode() instanceof ShadowRoot) {
|
||||
const shadowHost = baseElement.getRootNode().host;
|
||||
allElements.push(...shadowHost.getElementsByTagName(baseElement.tagName));
|
||||
}
|
||||
|
||||
// Get elements from iframes
|
||||
const iframes = document.getElementsByTagName('iframe');
|
||||
for (const iframe of iframes) {
|
||||
try {
|
||||
const iframeDoc = iframe.contentDocument || iframe.contentWindow.document;
|
||||
allElements.push(...iframeDoc.getElementsByTagName(baseElement.tagName));
|
||||
} catch (e) {
|
||||
console.warn('Cannot access iframe content:', e);
|
||||
}
|
||||
}
|
||||
|
||||
return allElements.filter(element => {
|
||||
if (element === baseElement) return false;
|
||||
const similarity = calculateClassSimilarity(
|
||||
baseClasses,
|
||||
Array.from(element.classList)
|
||||
);
|
||||
return similarity >= similarityThreshold;
|
||||
});
|
||||
}
|
||||
|
||||
// Main scraping logic
|
||||
let containers = queryIframeAll(document, listSelector);
|
||||
// Main scraping logic with context support
|
||||
let containers = queryElementAll(document, listSelector);
|
||||
containers = Array.from(containers);
|
||||
|
||||
if (containers.length === 0) return [];
|
||||
|
||||
if (limit > 1 && containers.length === 1) {
|
||||
const baseContainer = containers[0];
|
||||
const similarContainers = findSimilarElements(baseContainer);
|
||||
|
||||
if (similarContainers.length > 0) {
|
||||
const newContainers = similarContainers.filter(container =>
|
||||
!container.matches(listSelector)
|
||||
);
|
||||
containers = [...containers, ...newContainers];
|
||||
}
|
||||
const baseContainer = containers[0];
|
||||
const similarContainers = findSimilarElements(baseContainer);
|
||||
|
||||
if (similarContainers.length > 0) {
|
||||
const newContainers = similarContainers.filter(container =>
|
||||
!container.matches(listSelector)
|
||||
);
|
||||
containers = [...containers, ...newContainers];
|
||||
}
|
||||
}
|
||||
|
||||
const containerFields = containers.map(() => ({
|
||||
tableFields: {},
|
||||
nonTableFields: {}
|
||||
tableFields: {},
|
||||
nonTableFields: {}
|
||||
}));
|
||||
|
||||
// Classify fields
|
||||
containers.forEach((container, containerIndex) => {
|
||||
for (const [label, field] of Object.entries(fields)) {
|
||||
const sampleElement = queryIframe(container, field.selector);
|
||||
|
||||
if (sampleElement) {
|
||||
const ancestor = findTableAncestor(sampleElement);
|
||||
if (ancestor) {
|
||||
containerFields[containerIndex].tableFields[label] = {
|
||||
...field,
|
||||
tableContext: ancestor.type,
|
||||
cellIndex: ancestor.type === 'TD' ? getCellIndex(ancestor.element) : -1
|
||||
};
|
||||
} else {
|
||||
containerFields[containerIndex].nonTableFields[label] = field;
|
||||
}
|
||||
for (const [label, field] of Object.entries(fields)) {
|
||||
const sampleElement = queryElement(container, field.selector);
|
||||
|
||||
if (sampleElement) {
|
||||
const ancestor = findTableAncestor(sampleElement);
|
||||
if (ancestor) {
|
||||
containerFields[containerIndex].tableFields[label] = {
|
||||
...field,
|
||||
tableContext: ancestor.type,
|
||||
cellIndex: ancestor.type === 'TD' ? getCellIndex(ancestor.element) : -1
|
||||
};
|
||||
} else {
|
||||
containerFields[containerIndex].nonTableFields[label] = field;
|
||||
}
|
||||
} else {
|
||||
containerFields[containerIndex].nonTableFields[label] = field;
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
const tableData = [];
|
||||
const nonTableData = [];
|
||||
|
||||
// Process table data
|
||||
// Process table data with both iframe and shadow DOM support
|
||||
for (let containerIndex = 0; containerIndex < containers.length; containerIndex++) {
|
||||
const container = containers[containerIndex];
|
||||
const { tableFields } = containerFields[containerIndex];
|
||||
const container = containers[containerIndex];
|
||||
const { tableFields } = containerFields[containerIndex];
|
||||
|
||||
if (Object.keys(tableFields).length > 0) {
|
||||
const firstField = Object.values(tableFields)[0];
|
||||
const firstElement = queryIframe(container, firstField.selector);
|
||||
let tableContext = firstElement;
|
||||
|
||||
while (tableContext && tableContext.tagName !== 'TABLE' && tableContext !== container) {
|
||||
if (tableContext.tagName === 'IFRAME') {
|
||||
try {
|
||||
tableContext = tableContext.contentDocument.body;
|
||||
} catch (e) {
|
||||
break;
|
||||
}
|
||||
} else {
|
||||
tableContext = tableContext.parentElement;
|
||||
}
|
||||
}
|
||||
if (Object.keys(tableFields).length > 0) {
|
||||
const firstField = Object.values(tableFields)[0];
|
||||
const firstElement = queryElement(container, firstField.selector);
|
||||
let tableContext = firstElement;
|
||||
|
||||
// Find table context including both iframe and shadow DOM
|
||||
while (tableContext && tableContext.tagName !== 'TABLE' && tableContext !== container) {
|
||||
if (tableContext.getRootNode() instanceof ShadowRoot) {
|
||||
tableContext = tableContext.getRootNode().host;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (tableContext.tagName === 'IFRAME') {
|
||||
try {
|
||||
tableContext = tableContext.contentDocument.body;
|
||||
} catch (e) {
|
||||
break;
|
||||
}
|
||||
} else {
|
||||
tableContext = tableContext.parentElement;
|
||||
}
|
||||
}
|
||||
|
||||
if (tableContext) {
|
||||
const rows = Array.from(tableContext.getElementsByTagName('TR'));
|
||||
const processedRows = filterRowsBasedOnTag(rows, tableFields);
|
||||
|
||||
for (let rowIndex = 0; rowIndex < Math.min(processedRows.length, limit); rowIndex++) {
|
||||
const record = {};
|
||||
const currentRow = processedRows[rowIndex];
|
||||
|
||||
for (const [label, { selector, attribute, cellIndex }] of Object.entries(tableFields)) {
|
||||
let element = null;
|
||||
|
||||
if (cellIndex >= 0) {
|
||||
const td = currentRow.children[cellIndex];
|
||||
if (td) {
|
||||
element = queryIframe(td, selector);
|
||||
|
||||
if (!element && selector.split(">").pop().includes('td:nth-child')) {
|
||||
element = td;
|
||||
}
|
||||
if (tableContext) {
|
||||
// Get rows from all contexts
|
||||
const rows = [];
|
||||
|
||||
// Get rows from regular DOM
|
||||
rows.push(...tableContext.getElementsByTagName('TR'));
|
||||
|
||||
// Get rows from shadow DOM
|
||||
if (tableContext.shadowRoot) {
|
||||
rows.push(...tableContext.shadowRoot.getElementsByTagName('TR'));
|
||||
}
|
||||
|
||||
// Get rows from iframes
|
||||
if (tableContext.tagName === 'IFRAME') {
|
||||
try {
|
||||
const iframeDoc = tableContext.contentDocument || tableContext.contentWindow.document;
|
||||
rows.push(...iframeDoc.getElementsByTagName('TR'));
|
||||
} catch (e) {
|
||||
console.warn('Cannot access iframe rows:', e);
|
||||
}
|
||||
}
|
||||
|
||||
const processedRows = filterRowsBasedOnTag(rows, tableFields);
|
||||
|
||||
for (let rowIndex = 0; rowIndex < Math.min(processedRows.length, limit); rowIndex++) {
|
||||
const record = {};
|
||||
const currentRow = processedRows[rowIndex];
|
||||
|
||||
for (const [label, { selector, attribute, cellIndex }] of Object.entries(tableFields)) {
|
||||
let element = null;
|
||||
|
||||
if (cellIndex >= 0) {
|
||||
// Get TD element considering both contexts
|
||||
let td = currentRow.children[cellIndex];
|
||||
|
||||
// Check shadow DOM for td
|
||||
if (!td && currentRow.shadowRoot) {
|
||||
const shadowCells = currentRow.shadowRoot.children;
|
||||
if (shadowCells && shadowCells.length > cellIndex) {
|
||||
td = shadowCells[cellIndex];
|
||||
}
|
||||
}
|
||||
|
||||
if (td) {
|
||||
element = queryElement(td, selector);
|
||||
|
||||
if (!element && selector.split(/(?:>>|:>>)/).pop().includes('td:nth-child')) {
|
||||
element = td;
|
||||
}
|
||||
|
||||
if (!element) {
|
||||
const tagOnlySelector = selector.split('.')[0];
|
||||
element = queryIframe(td, tagOnlySelector);
|
||||
}
|
||||
|
||||
if (!element) {
|
||||
let currentElement = td;
|
||||
while (currentElement && currentElement.children.length > 0) {
|
||||
let foundContentChild = false;
|
||||
for (const child of currentElement.children) {
|
||||
if (extractValue(child, attribute)) {
|
||||
currentElement = child;
|
||||
foundContentChild = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!foundContentChild) break;
|
||||
}
|
||||
element = currentElement;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
element = queryIframe(currentRow, selector);
|
||||
}
|
||||
|
||||
if (element) {
|
||||
record[label] = extractValue(element, attribute);
|
||||
}
|
||||
}
|
||||
if (!element) {
|
||||
const tagOnlySelector = selector.split('.')[0];
|
||||
element = queryElement(td, tagOnlySelector);
|
||||
}
|
||||
|
||||
if (!element) {
|
||||
let currentElement = td;
|
||||
while (currentElement && currentElement.children.length > 0) {
|
||||
let foundContentChild = false;
|
||||
for (const child of currentElement.children) {
|
||||
if (extractValue(child, attribute)) {
|
||||
currentElement = child;
|
||||
foundContentChild = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!foundContentChild) break;
|
||||
}
|
||||
element = currentElement;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
element = queryElement(currentRow, selector);
|
||||
}
|
||||
|
||||
if (element) {
|
||||
record[label] = extractValue(element, attribute);
|
||||
}
|
||||
}
|
||||
|
||||
if (Object.keys(record).length > 0) {
|
||||
tableData.push(record);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if (Object.keys(record).length > 0) {
|
||||
tableData.push(record);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Process non-table data
|
||||
// Process non-table data with both contexts support
|
||||
for (let containerIndex = 0; containerIndex < containers.length; containerIndex++) {
|
||||
if (nonTableData.length >= limit) break;
|
||||
if (nonTableData.length >= limit) break;
|
||||
|
||||
const container = containers[containerIndex];
|
||||
const { nonTableFields } = containerFields[containerIndex];
|
||||
const container = containers[containerIndex];
|
||||
const { nonTableFields } = containerFields[containerIndex];
|
||||
|
||||
if (Object.keys(nonTableFields).length > 0) {
|
||||
const record = {};
|
||||
if (Object.keys(nonTableFields).length > 0) {
|
||||
const record = {};
|
||||
|
||||
for (const [label, { selector, attribute }] of Object.entries(nonTableFields)) {
|
||||
const relativeSelector = selector.split(':>>').slice(-1)[0];
|
||||
const element = queryIframe(container, relativeSelector);
|
||||
|
||||
if (element) {
|
||||
record[label] = extractValue(element, attribute);
|
||||
}
|
||||
}
|
||||
|
||||
if (Object.keys(record).length > 0) {
|
||||
nonTableData.push(record);
|
||||
}
|
||||
}
|
||||
for (const [label, { selector, attribute }] of Object.entries(nonTableFields)) {
|
||||
// Get the last part of the selector after any context delimiter
|
||||
const relativeSelector = selector.split(/(?:>>|:>>)/).slice(-1)[0];
|
||||
const element = queryElement(container, relativeSelector);
|
||||
|
||||
if (element) {
|
||||
record[label] = extractValue(element, attribute);
|
||||
}
|
||||
}
|
||||
|
||||
if (Object.keys(record).length > 0) {
|
||||
nonTableData.push(record);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Merge and limit the results
|
||||
|
||||
@@ -403,7 +403,7 @@ export default class Interpreter extends EventEmitter {
|
||||
await this.options.serializableCallback(scrapeResults);
|
||||
},
|
||||
|
||||
scrapeSchema: async (schema: Record<string, { selector: string; tag: string, attribute: string; }>) => {
|
||||
scrapeSchema: async (schema: Record<string, { selector: string; tag: string, attribute: string; shadow: string}>) => {
|
||||
await this.ensureScriptsLoaded(page);
|
||||
|
||||
const scrapeResult = await page.evaluate((schemaObj) => window.scrapeSchema(schemaObj), schema);
|
||||
@@ -663,27 +663,42 @@ export default class Interpreter extends EventEmitter {
|
||||
if (isApplicable) {
|
||||
return actionId;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private removeIframeSelectors(workflow: Workflow) {
|
||||
private removeShadowSelectors(workflow: Workflow) {
|
||||
for (let actionId = workflow.length - 1; actionId >= 0; actionId--) {
|
||||
const step = workflow[actionId];
|
||||
|
||||
// Check if step has where and selectors
|
||||
if (step.where && Array.isArray(step.where.selectors)) {
|
||||
// Filter out selectors that contain ">>"
|
||||
step.where.selectors = step.where.selectors.filter(selector => !selector.includes(':>>'));
|
||||
step.where.selectors = step.where.selectors.filter(selector => !selector.includes('>>'));
|
||||
}
|
||||
}
|
||||
|
||||
return workflow;
|
||||
}
|
||||
|
||||
private removeSpecialSelectors(workflow: Workflow) {
|
||||
for (let actionId = workflow.length - 1; actionId >= 0; actionId--) {
|
||||
const step = workflow[actionId];
|
||||
|
||||
if (step.where && Array.isArray(step.where.selectors)) {
|
||||
// Filter out if selector has EITHER ":>>" OR ">>"
|
||||
step.where.selectors = step.where.selectors.filter(selector =>
|
||||
!(selector.includes(':>>') || selector.includes('>>'))
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
return workflow;
|
||||
}
|
||||
|
||||
private async runLoop(p: Page, workflow: Workflow) {
|
||||
let workflowCopy: Workflow = JSON.parse(JSON.stringify(workflow));
|
||||
|
||||
workflowCopy = this.removeIframeSelectors(workflowCopy);
|
||||
workflowCopy = this.removeSpecialSelectors(workflowCopy);
|
||||
|
||||
// apply ad-blocker to the current page
|
||||
try {
|
||||
|
||||
Reference in New Issue
Block a user