vodorod/parcer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

chore: lint

Browse Source
This commit is contained in:
karishmas6
2024-08-02 19:07:58 +05:30
parent 8540905192
commit 9b18293643
1 changed files with 44 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files

89
maxun-core/src/browserSide/scraper.js
View File

@@ -132,12 +132,12 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3,
*/ */
// Wrap the entire function in an IIFE (Immediately Invoked Function Expression) // Wrap the entire function in an IIFE (Immediately Invoked Function Expression)
// and attach it to the window object // and attach it to the window object
(function(window) { (function (window) {
/** /**
* Returns a "scrape" result from the current page. * Returns a "scrape" result from the current page.
* @returns {Array<Object>} *Curated* array of scraped information (with sparse rows removed) * @returns {Array<Object>} *Curated* array of scraped information (with sparse rows removed)
*/ */
window.scrape = function(selector = null) { window.scrape = function (selector = null) {
/** /**
* **crudeRecords** contains uncurated rundowns of "scrapable" elements * **crudeRecords** contains uncurated rundowns of "scrapable" elements
* @type {Array<Object>} * @type {Array<Object>}
@@ -188,49 +188,48 @@ function scrapableHeuristics(maxCountPerPage = 50, minArea = 20000, scrolls = 3,
* @param {Object.<string, object[]>} lists The named lists of HTML elements. * @param {Object.<string, object[]>} lists The named lists of HTML elements.
* @returns {Array.<Object.<string, string>>} * @returns {Array.<Object.<string, string>>}
*/ */
window.scrapeSchema = function (lists) { window.scrapeSchema = function (lists) {
function omap(object, f, kf = (x) => x) { function omap(object, f, kf = (x) => x) {
return Object.fromEntries( return Object.fromEntries(
Object.entries(object) Object.entries(object)
.map(([k, v]) => [kf(k), f(v)]), .map(([k, v]) => [kf(k), f(v)]),
); );
}
function ofilter(object, f) {
return Object.fromEntries(
Object.entries(object)
.filter(([k, v]) => f(k, v)),
);
}
function getSeedKey(listObj) {
const maxLength = Math.max(...Object.values(omap(listObj, (x) => x.length)));
return Object.keys(ofilter(listObj, (_, v) => v.length === maxLength))[0];
}
function getMBEs(elements) {
return elements.map((element) => {
let candidate = element;
const isUniqueChild = (e) => elements
.filter((elem) => e.parentNode?.contains(elem))
.length === 1;
while (candidate && isUniqueChild(candidate)) {
candidate = candidate.parentNode;
}
return candidate;
});
}
const seedName = getSeedKey(lists);
const MBEs = getMBEs(lists[seedName]);
return MBEs.map((mbe) => omap(
lists,
(listOfElements) => listOfElements.find((elem) => mbe.contains(elem))?.innerText,
));
} }
function ofilter(object, f) {
return Object.fromEntries(
Object.entries(object)
.filter(([k, v]) => f(k, v)),
);
}
function getSeedKey(listObj) {
const maxLength = Math.max(...Object.values(omap(listObj, (x) => x.length)));
return Object.keys(ofilter(listObj, (_, v) => v.length === maxLength))[0];
}
function getMBEs(elements) {
return elements.map((element) => {
let candidate = element;
const isUniqueChild = (e) => elements
.filter((elem) => e.parentNode?.contains(elem))
.length === 1;
while (candidate && isUniqueChild(candidate)) {
candidate = candidate.parentNode;
}
return candidate;
});
}
const seedName = getSeedKey(lists);
const MBEs = getMBEs(lists[seedName]);
return MBEs.map((mbe) => omap(
lists,
(listOfElements) => listOfElements.find((elem) => mbe.contains(elem))?.innerText,
));
}
})(window); })(window);