vodorod/Dorod-Sky
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

make scraping timeout configurable (#2991)

Browse Source
This commit is contained in:
LawyZheng
2025-07-19 13:18:12 +08:00
committed by GitHub
parent 0efd86fb89
commit fcd22017b7
4 changed files with 22 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
skyvern/webeye/scraper/scraper.py
View File

@@ -12,10 +12,11 @@ from playwright.async_api import ElementHandle, Frame, Locator, Page
from pydantic import BaseModel, PrivateAttr
from skyvern.config import settings
from skyvern.constants import BUILDING_ELEMENT_TREE_TIMEOUT_MS, DEFAULT_MAX_TOKENS, SKYVERN_DIR, SKYVERN_ID_ATTR
from skyvern.constants import DEFAULT_MAX_TOKENS, SKYVERN_DIR, SKYVERN_ID_ATTR
from skyvern.exceptions import FailedToTakeScreenshot, ScrapingFailed, UnknownElementTreeFormat
from skyvern.forge.sdk.api.crypto import calculate_sha256
from skyvern.forge.sdk.core import skyvern_context
from skyvern.forge.sdk.settings_manager import SettingsManager
from skyvern.forge.sdk.trace import TraceManager
from skyvern.utils.image_resizer import Resolution
from skyvern.utils.token_counter import count_tokens
@@ -765,7 +766,9 @@ class IncrementalScrapePage(ElementTreeBuilder):
return
js_script = "async () => await stopGlobalIncrementalObserver()"
await SkyvernFrame.evaluate(
frame=self.skyvern_frame.get_frame(), expression=js_script, timeout_ms=BUILDING_ELEMENT_TREE_TIMEOUT_MS
frame=self.skyvern_frame.get_frame(),
expression=js_script,
timeout_ms=SettingsManager.get_settings().BROWSER_SCRAPING_BUILDING_ELEMENT_TREE_TIMEOUT_MS,
)
async def get_incremental_elements_num(self) -> int: