From 726467e7aff00c96ce46626e4f81d71c386d99a0 Mon Sep 17 00:00:00 2001 From: jehrr <79864894+jehrr@users.noreply.github.com> Date: Thu, 22 Oct 2026 08:58:24 +0000 Subject: [PATCH] Fill results pages that stop at the grid's loading placeholders A page could stop at 13 or 19 of its 50 cards with the site's own placeholders still under them. Those placeholders fill only when they scroll into view, and scrolling to the bottom again moves nothing. When the grid holds still with placeholders showing, all three engines now scroll the results container back to the top and down a screen at a time, at most three times per page. The sidecar records nudges and placeholders_left for each page. The canary's three-page proxy walk passes --block-media so that full pages fit its 10 MB traffic cap. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CKsd7q25N19WzUZjrAdzx8 --- .github/workflows/canary.yml | 8 ++- CHANGELOG.md | 21 ++++++ TROUBLESHOOTING.md | 21 ++++-- fixtures_generated.json | 4 +- make_fixtures.py | 52 ++++++++++++++ page_flow.py | 101 +++++++++++++++++++++++--- playwright_scraper.py | 47 ++++++++++-- product_parser.py | 6 ++ puppeteer_scraper.py | 46 ++++++++++-- selenium_scraper.py | 44 ++++++++++-- smoke_test.py | 136 ++++++++++++++++++++++++++++++++++- 11 files changed, 446 insertions(+), 40 deletions(-) diff --git a/.github/workflows/canary.yml b/.github/workflows/canary.yml index 2e9a5ea..e6a01be 100644 --- a/.github/workflows/canary.yml +++ b/.github/workflows/canary.yml @@ -158,14 +158,18 @@ jobs: # Three pages, not two: one turn only proves the button was found # once; the second turn is where a cursor that does not advance shows # up. --delay 15 because the turn's /graphql POST is rate limited - # separately from the HTML. + # separately from the HTML. --block-media because a full page loads + # about 100 images (1.8 MB measured 2026-10-22, no proxy): the + # 2026-10-21 walk used 9.6 MB of the 10 MB cap with two of its three + # pages only 19 cards deep, so three full pages with images would + # not fit. image_url is not one of the columns this job checks. run: | set +e start=$(date -u -d '+45 days' +%F) end=$(date -u -d '+47 days' +%F) url="https://www.vrbo.com/search?destination=Orlando%2C%20Florida%2C%20United%20States%20of%20America®ionId=2693&sort=RECOMMENDED&startDate=${start}&endDate=${end}&adults=2" xvfb-run -a python3 playwright_scraper.py --url "$url" \ - --pages 3 --delay 15 --retries 4 --proxy-budget-mb 10 \ + --pages 3 --delay 15 --retries 4 --proxy-budget-mb 10 --block-media \ --out canary_walk --dump-html canary_walk.html echo "exit_code=$?" >> "$GITHUB_OUTPUT" diff --git a/CHANGELOG.md b/CHANGELOG.md index e5d439d..904c91d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -75,6 +75,27 @@ that is a fix, and the release notes lead with it. ### Fixed +- A results page that stops filling partway now gets the rest of its cards + in all three engines. Below the last card the grid keeps loading + placeholders (`data-stid="search-listing-placeholder"`). It fills them only + when they scroll into view, and scrolling to the bottom again moves + nothing. The 2026-10-20 and 10-21 canaries stopped this way: 6 placeholders + under 13 cards on page 1 with no proxy, and under 19 cards on pages 2 and 3 + through a residential exit. Each run ended as `cards_missing`, exit 6. + - When count and height hold still for three rounds with placeholders + showing, the scraper scrolls the container to the top and back down a + screen at a time. It does this at most three times per page. + - Measured 2026-10-22 without a proxy: a 13-card stall held for eight + rounds, and one nudge took it to 50 of 50. + - Each page's `scroll` entry in the sidecar now records `nudges` and + `placeholders_left`, and the warning says which case it was. + - New sanitized fixture `LISTING_LAZY_TAIL`, cut from the stalled canary + capture. There are offline checks for the stall, the bound, short pages, + the walk and each engine's wiring. +- CI only: the canary's three-page walk through `VRBO_PROXY` passes + `--block-media`. A full page loads about 100 images (1.8 MB measured), and + the 2026-10-21 walk had already used 9.6 MB of its 10 MB cap with two short + pages. - The pyppeteer and Selenium proxy meter counts only `http://` / `https://` requests and responses that came over the network. Chrome's own loads (the `chrome://` new-tab page, `data:`, `blob:`), cache hits (`requestServedFromCache`, `fromDiskCache`, `fromPrefetchCache`, `fromServiceWorker`) and Chrome's error page count nothing. A sibling repo's Selenium run had counted about 2.8 MB of new-tab scripts while its proxy carried about 6 KB. The pyppeteer block sent once the budget is spent now reaches the page's own event loop even when the budget runs out on another thread. ## [0.3.0] — 2026-10-15 diff --git a/TROUBLESHOOTING.md b/TROUBLESHOOTING.md index 29ab9d7..ab9e0fc 100644 --- a/TROUBLESHOOTING.md +++ b/TROUBLESHOOTING.md @@ -76,12 +76,21 @@ parse got 40, so ten cards never loaded. Almost always the scroll did not finish. First paint on this site is 3 to 18 cards and the rest arrive by scrolling an **inner** container -(`.scrollable-result-section`) — the page body never scrolls at all. If the -batch behind a scroll is throttled, the grid stops growing early. - -Same remedies as above: slow down, or use a proxy pool. Re-run with -`--dump-html` and open the snapshot to confirm the cards really were absent -rather than unparsed. +(`.scrollable-result-section`) — the page body never scrolls at all. + +Below the last card the grid keeps a block of loading placeholders +(`data-stid="search-listing-placeholder"`) and fills it only when that block +comes **into** view. Scrolling to the bottom again and again does not do that, +because nothing moves after the first time. When the grid stops with those +placeholders still showing, the scraper scrolls the container back to the +top and down again a screen at a time, up to three times. The sidecar's +`scroll` entry for each page records `nudges` and `placeholders_left`. If +`placeholders_left` is above 0, the grid never filled in even after the +nudges. If it is 0, the site served a short page. + +If it persists: slow down, or use a proxy pool. Re-run with `--dump-html` +and open the snapshot to confirm the cards really were absent rather than +unparsed. --- diff --git a/fixtures_generated.json b/fixtures_generated.json index a172644..ed7a888 100644 --- a/fixtures_generated.json +++ b/fixtures_generated.json @@ -12,6 +12,7 @@ "PROPERTY_US": "

The Point Hotel & Suites Orlando

7389 Universal Boulevard, Orlando, FL, 32819

Reviews

8.68.6 out of 10
Excellent
The current price is $81
$81
for 1 night
All fees included
Sep 28 - Sep 29
", "BROWSER_ERROR": "", "SITE_ERROR": "

Sorry, we're having a problem on our end

", + "LISTING_LAZY_TAIL": "
300+ properties

Photo gallery for Foodie Lovers Walkable Mills50 - Sleeps 10

\"Image
\"Image
\"Image
\"Image
Ad
Within Colonialtown South

Foodie Lovers Walkable Mills50 - Sleeps 10

House · 2 bedrooms · 9 beds
AC
 • 
Refrigerator
 • 
Dryer
AC, Refrigerator, Dryer
Free cancellation until Nov 21
9.09.0 out of 10
Wonderfulwonderful
4 reviews(4 reviews)
$236
The current price is $236
$472 for 2 nights
All fees included
Opens Foodie Lovers Walkable Mills50 - Sleeps 10 in new tab

Photo gallery for CasaDelaney DT Sodo district. Renovated Sleeps 16

\"Image
\"Image
\"Image
\"Image
Ad
Within Wadeview Park

CasaDelaney DT Sodo district. Renovated Sleeps 16

House · 4 bedrooms · 8 beds
AC
 • 
Refrigerator
 • 
Dryer
AC, Refrigerator, Dryer
Free cancellation until Nov 21
1010 out of 10
Exceptionalexceptional
2 reviews(2 reviews)
$1,340
The current price is $1,340
$2,680 for 2 nights
All fees included
Opens CasaDelaney DT Sodo district. Renovated Sleeps 16 in new tab
1 - 50 of 300+
", "_URLS": { "LISTING_US": "https://www.vrbo.com/search?destination=Orlando%2C+Florida%2C+United+States+of+America®ionId=2693", "LISTING_US_P2": "https://www.vrbo.com/search?destination=Orlando%2C+Florida%2C+United+States+of+America®ionId=2693", @@ -24,6 +25,7 @@ "PROPERTY_US": "https://www.vrbo.com/pdp/lo/63925107", "BROWSER_ERROR": "https://www.vrbo.com/search?destination=Orlando", "SITE_ERROR": "https://www.vrbo.com/search?destination=Orlando%2C%20Florida%2C%20United%20States%20of%20America®ionId=2693&sort=RECOMMENDED&startDate=2026-11-14&endDate=2026-11-16&adults=2", - "LISTING_UNPRICED": "https://www.vrbo.com/search?destination=Orlando%2C%20Florida%2C%20United%20States%20of%20America®ionId=2693&sort=RECOMMENDED&startDate=2026-11-14&endDate=2026-11-16&adults=2" + "LISTING_UNPRICED": "https://www.vrbo.com/search?destination=Orlando%2C%20Florida%2C%20United%20States%20of%20America®ionId=2693&sort=RECOMMENDED&startDate=2026-11-14&endDate=2026-11-16&adults=2", + "LISTING_LAZY_TAIL": "https://www.vrbo.com/search?destination=Orlando%2C%20Florida%2C%20United%20States%20of%20America®ionId=2693&sort=RECOMMENDED&startDate=2026-12-05&endDate=2026-12-07&adults=2" } } \ No newline at end of file diff --git a/make_fixtures.py b/make_fixtures.py index 3fc7363..d0ec60a 100644 --- a/make_fixtures.py +++ b/make_fixtures.py @@ -86,6 +86,10 @@ def _unpriced(card) -> bool: # exit 4, "the search matched nothing". SITE_ERROR_SOURCE = ("vrbo_site_error_2026-10-07.html", "https://www.vrbo.com/search?destination=Orlando%2C%20Florida%2C%20United%20States%20of%20America®ionId=2693&sort=RECOMMENDED&startDate=2026-11-14&endDate=2026-11-16&adults=2") +# A grid the scroll loop left at 13 of 50 cards with the site's own loading +# placeholders under them (canary, 2026-10-21, no proxy). +LAZY_TAIL_SOURCE = ("vrbo_search_lazy_tail_stall_2026-10-21.html", + "https://www.vrbo.com/search?destination=Orlando%2C%20Florida%2C%20United%20States%20of%20America®ionId=2693&sort=RECOMMENDED&startDate=2026-12-05&endDate=2026-12-07&adults=2") # A card that keeps the SLUG route must survive into LISTING_DE: it is the # fifth URL shape and only the local brands use it, so a fixture without one @@ -270,6 +274,52 @@ def build_site_error(filename, url): return trimmed +def build_lazy_tail(filename, url, keep=2): + """A grid that stopped short: `keep` real cards and the lazy tail. + + Cut from a page that the scroll loop left at 13 of 50 cards with the + site's own placeholders still under them. Keeps the placeholders exactly + as served (their parent block) so the selector is tested against the + real markup, and the first `keep` cards so the page still parses. + """ + full = (CAPTURES / filename).read_text(encoding="utf-8") + soup = BeautifulSoup(full, "html.parser") + placeholders = soup.select(P.SELECTORS["grid_placeholder"]) + if not placeholders: + raise SystemExit(f"{filename}: no grid placeholder — is this the stalled capture?") + tail = placeholders[0].parent + cards = soup.select(P.SELECTORS["item_card"])[:keep] + header = soup.select_one(P.RESULTS_HEADER_SELECTOR) + pagination = soup.select_one(P.PAGINATION_SELECTOR) + block = ((str(header) if header is not None else "") + + '
' + + "".join(str(c) for c in cards) + str(tail) + "
" + + (str(pagination) if pagination is not None else "")) + trimmed = scrub(_head(soup, full) + block + "") + small = BeautifulSoup(trimmed, "html.parser") + if len(small.select(P.SELECTORS["grid_placeholder"])) != len(placeholders): + raise SystemExit("LISTING_LAZY_TAIL: trimming changed the placeholder count") + want = {r.sku: asdict(r) for r in P.parse_products(full, url, page=1)} + got = {r.sku: asdict(r) for r in P.parse_products(trimmed, url, page=1)} + if len(got) != keep or set(got) - set(want): + raise SystemExit(f"LISTING_LAZY_TAIL: kept {len(got)} rows, not the {keep} cut") + for sku, row in got.items(): + for column, value in row.items(): + expected = want[sku][column] + if isinstance(expected, str): + expected = scrub(expected) # the rate-plan id is a long hex run + if column not in ("scraped_at", "position") and expected != value: + raise SystemExit(f"LISTING_LAZY_TAIL: trimming changed {column!r} on {sku}") + if P.detect_page_state(trimmed, 200, url) != "content": + raise SystemExit("LISTING_LAZY_TAIL: trimmed fixture does not classify as content") + leftovers = re.findall(r"\b[0-9a-fA-F]{24,}\b|duaid", trimmed) + if leftovers: + raise SystemExit(f"LISTING_LAZY_TAIL: session material left in: {leftovers[:3]}") + print(f" LISTING_LAZY_TAIL: {len(full)} -> {len(trimmed)} bytes, " + f"{len(got)} rows and {len(placeholders)} placeholders") + return trimmed + + def build_property(filename, url): full = (CAPTURES / filename).read_text(encoding="utf-8") soup = BeautifulSoup(full, "html.parser") @@ -336,6 +386,8 @@ def main(): urls["PROPERTY_US"] = PROPERTY_SOURCE[1] fixtures["SITE_ERROR"] = build_site_error(*SITE_ERROR_SOURCE) urls["SITE_ERROR"] = SITE_ERROR_SOURCE[1] + fixtures["LISTING_LAZY_TAIL"] = build_lazy_tail(*LAZY_TAIL_SOURCE) + urls["LISTING_LAZY_TAIL"] = LAZY_TAIL_SOURCE[1] # Chromium's own answer when a proxy is dead. 39 bytes and no markup at # all — Playwright does not expose the interstitial, so this is what a # caller actually gets. It must classify as blocked, which is the diff --git a/page_flow.py b/page_flow.py index e7abe63..a9a1f2d 100644 --- a/page_flow.py +++ b/page_flow.py @@ -162,16 +162,59 @@ def wait_for_count(count: Callable[[str], int], sleep: Callable[[int], None], SCROLL_MAX_ROUNDS = 20 # How many stable rounds to tolerate while the page's own counter says there # are still cards to come. Earned on the first live run of this engine: page -# 2 settled at 19 cards against a counter that said 50, because the next -# batch was in flight behind a throttled `/graphql` POST and three quiet -# rounds went by while it was. A pause is not an ending when the site has -# told us how many there are — so the stable-round heuristic is the -# termination condition only for a page that published no counter, and below -# a known target the loop is twice as patient before giving up. +# 2 settled at 19 cards against a counter that said 50. That was read at the +# time as a batch in flight behind a throttled `/graphql` POST. The 2026-10-22 +# measurement found the lazy tail instead (see PLACEHOLDER_SELECTOR): no +# request is behind those cards. A pause is still not an ending when the site +# has told us how many there are, so below a known target the loop is twice +# as patient before giving up. SCROLL_STABLE_ROUNDS_BELOW_TARGET = 6 # The next batch takes longer to arrive than a single pause. SCROLL_PAUSE_MS = 1_600 +# The grid's lazy tail. Below the last rendered card the site keeps a block +# of placeholder cards and fills it only when it scrolls INTO view. Scrolling +# the container to its bottom puts it in view once; after that every further +# "scroll to bottom" sets the same scrollTop, nothing moves, and the grid +# waits for good. Measured: the 2026-10-20 and 10-21 canaries stopped with 6 +# placeholders under 13 cards (page 1, no proxy) and 19 cards (pages 2 and 3, +# residential exit) while count and height held still; on 2026-10-22, from a +# plain connection, the same 13-card stall held for eight rounds and ONE +# nudge (back to the top, then down a screen at a time) took it to 50 of 50. +# No /graphql call is behind those cards, so waiting longer cannot help. +PLACEHOLDER_SELECTOR = SELECTORS["grid_placeholder"] +# Bounded: a nudge that does not help three times will not on the fourth. +SCROLL_MAX_NUDGES = 3 +NUDGE_TOP_PAUSE_MS = 800 +NUDGE_STEP_PAUSE_MS = 300 +NUDGE_STEP_SHARE = 0.8 # of the container's visible height +NUDGE_MAX_STEPS = 40 # a 12,000px grid in ~550px steps is ~22 + + +def nudge_grid(geometry: Callable[[], Optional[tuple]], + set_top: Callable[[int], None], + sleep: Callable[[int], None]) -> None: + """Move the lazy tail out of view and walk it back in, a screen at a time. + + `geometry` returns the container's (scrollTop, scrollHeight, + clientHeight), or None when there is no container; `set_top` sets its + scrollTop. Both are named driver operations, so this reads the same in + every engine. + """ + if not geometry(): + return + set_top(0) + sleep(NUDGE_TOP_PAUSE_MS) + for _ in range(NUDGE_MAX_STEPS): + g = geometry() + if not g: + return + top, height, client = (int(v or 0) for v in g) + if top + client >= height - 2: + return + set_top(min(height, top + max(1, int(client * NUDGE_STEP_SHARE)))) + sleep(NUDGE_STEP_PAUSE_MS) + def scroll_until_settled(count: Callable[[str], int], scroll_results: Callable[[], None], @@ -179,7 +222,9 @@ def scroll_until_settled(count: Callable[[str], int], sleep: Callable[[int], None], selector: str = READY_SELECTOR_LISTING, target: Optional[int] = None, - max_rounds: int = SCROLL_MAX_ROUNDS) -> int: + max_rounds: int = SCROLL_MAX_ROUNDS, + nudge: Optional[Callable[[], None]] = None, + trace: Optional[dict] = None) -> int: """Scroll the results container until the grid stops growing. Requires the card count AND the container height to hold still for @@ -192,19 +237,40 @@ def scroll_until_settled(count: Callable[[str], int], NOT reaching it does not end anything early, because the gap is reported rather than chased (the rank-gap arithmetic). + `nudge`, when given, is called (at most `SCROLL_MAX_NUDGES` times) when + the grid has held still for `SCROLL_STABLE_ROUNDS` rounds with the site's + own placeholders still on the page. See PLACEHOLDER_SELECTOR. Its rounds + do not come out of the ordinary budget. + + `trace`, when given, receives `nudges` and `placeholders_left`. + Returns the final card count. """ seen = count(selector) height = results_height() - stable = 0 - for _ in range(max_rounds): + stable = nudges = rounds = 0 + budget = max_rounds + (SCROLL_MAX_NUDGES * SCROLL_STABLE_ROUNDS if nudge else 0) + while rounds < budget: if target is not None and seen >= target: break scroll_results() sleep(SCROLL_PAUSE_MS) + rounds += 1 now, now_height = count(selector), results_height() if now == seen and now_height == height: stable += 1 + if (nudge is not None and nudges < SCROLL_MAX_NUDGES + and stable >= SCROLL_STABLE_ROUNDS + and count(PLACEHOLDER_SELECTOR) > 0): + nudges += 1 + logger.info("The grid stopped at %d card(s) with the site's " + "loading placeholders still showing; scrolling it " + "back through (nudge %d/%d).", now, nudges, + SCROLL_MAX_NUDGES) + nudge() + stable = 0 + seen, height = count(selector), results_height() + continue # More patience while the site's own counter says cards are # still owed. See SCROLL_STABLE_ROUNDS_BELOW_TARGET. allowed = (SCROLL_STABLE_ROUNDS_BELOW_TARGET @@ -215,9 +281,26 @@ def scroll_until_settled(count: Callable[[str], int], else: stable = 0 seen, height = now, now_height + if trace is not None: + trace["nudges"] = nudges + trace["placeholders_left"] = count(PLACEHOLDER_SELECTOR) return seen +def unsettled_reason(trace: dict) -> str: + """Why a page stopped short of its counter, in words for the log.""" + left = trace.get("placeholders_left") + if left: + return (f"the site's own loading placeholders ({left}) were still on " + f"the page after {trace.get('nudges', 0)} nudge(s), so the " + f"grid never filled in") + if left == 0: + return ("no loading placeholder was left on the page, so the site " + "served a short page or the grid was still filling") + return ("the grid was still filling when the scroll budget ran out, or " + "the site served a short page") + + # --------------------------------------------------------------------------- # Classification and the policy that follows from it # --------------------------------------------------------------------------- diff --git a/playwright_scraper.py b/playwright_scraper.py index 263ecf2..6e6df42 100644 --- a/playwright_scraper.py +++ b/playwright_scraper.py @@ -251,6 +251,31 @@ def _results_height(page) -> Optional[int]: return None +def _results_geometry(page) -> Optional[tuple]: + """(scrollTop, scrollHeight, clientHeight) of the results container.""" + try: + g = page.evaluate( + """(selector) => { + const el = document.querySelector(selector); + return el ? [el.scrollTop, el.scrollHeight, el.clientHeight] : null; + }""", SELECTORS["scroll_container"]) + return tuple(g) if g else None + except (PWError, PWTimeout): + return None + + +def _set_results_top(page, top: int) -> None: + """Set the results container's scrollTop.""" + try: + page.evaluate( + """([selector, top]) => { + const el = document.querySelector(selector); + if (el) { el.scrollTop = top; } + }""", [SELECTORS["scroll_container"], top]) + except (PWError, PWTimeout): + pass + + # The data behind a page turn comes over POST /graphql, and that endpoint is # rate limited SEPARATELY from the HTML: measured on this site, the search # page kept answering HTTP 200 while every /graphql POST behind a next-press @@ -673,25 +698,33 @@ def _scroll_the_grid(session, args, html: str, page_num: int) -> dict: """ target = page_flow.expected_cards(html) before = _count(session.page, page_flow.READY_SELECTOR_LISTING) + trace: dict = {} reached = page_flow.scroll_until_settled( lambda sel: _count(session.page, sel), lambda: _scroll_results(session.page), lambda: _results_height(session.page), session.page.wait_for_timeout, selector=page_flow.READY_SELECTOR_LISTING, - target=target) + target=target, + nudge=lambda: page_flow.nudge_grid( + lambda: _results_geometry(session.page), + lambda top: _set_results_top(session.page, top), + session.page.wait_for_timeout), + trace=trace) settled = target is None or reached >= target logger.info("Scrolled page %d: %d card(s) at first paint, %d after " - "scrolling%s.", page_num, before, reached, - f" (the page says it holds {target})" if target else "") + "scrolling%s%s.", page_num, before, reached, + f" (the page says it holds {target})" if target else "", + f", {trace['nudges']} nudge(s)" if trace.get("nudges") else "") if not settled: logger.warning( "Page %d settled at %d card(s) but its own counter says it holds " - "%d. Those %d are missing from this run — the grid was still " - "filling when the scroll budget ran out, or the site served a " - "short page.", page_num, reached, target, target - reached) + "%d. Those %d are missing from this run — %s.", + page_num, reached, target, target - reached, + page_flow.unsettled_reason(trace)) return {"first_paint": before, "reached": reached, "target": target, - "settled": settled} + "settled": settled, "nudges": trace.get("nudges", 0), + "placeholders_left": trace.get("placeholders_left")} def _fetch_one_page(session, args, pool, page_num: int, url: Optional[str]) -> PageOutcome: diff --git a/product_parser.py b/product_parser.py index 8338d8d..e2e4d82 100644 --- a/product_parser.py +++ b/product_parser.py @@ -277,6 +277,12 @@ def paginates_by_url(url: str) -> bool: # nothing here — measured, 12 window scrolls, 18 cards before and 18 # after. Scrolling THIS element reached 50 of 50 in two rounds. "scroll_container": ".scrollable-result-section", + # The grid's own "more cards are coming" placeholder at the lazy tail. + # Present on every capture that stopped short of its counter (canary + # 2026-10-20/21: 6 of them under 13 or 19 cards), absent on every full + # page. It is filled only when it comes INTO view: the scroll-to-bottom + # loop leaves it in view the whole time, so the grid waits forever. + "grid_placeholder": '[data-stid="search-listing-placeholder"]', # Two spellings of one element, and both are real (see the module # docstring). Kept as one comma selector so every read gets both. "card_price": '[data-stid="product-price-summary"], [data-test-id="price-summary"]', diff --git a/puppeteer_scraper.py b/puppeteer_scraper.py index d1d0fa2..bffe50d 100644 --- a/puppeteer_scraper.py +++ b/puppeteer_scraper.py @@ -372,6 +372,31 @@ def _results_height(session) -> Optional[int]: return None +def _results_geometry(session) -> Optional[tuple]: + """(scrollTop, scrollHeight, clientHeight) of the results container.""" + try: + g = session.bridge.run(session.page.evaluate( + """(selector) => { + const el = document.querySelector(selector); + return el ? [el.scrollTop, el.scrollHeight, el.clientHeight] : null; + }""", SELECTORS["scroll_container"])) + return tuple(g) if g else None + except Exception: # noqa: BLE001 + return None + + +def _set_results_top(session, top: int) -> None: + """Set the results container's scrollTop.""" + try: + session.bridge.run(session.page.evaluate( + """(selector, top) => { + const el = document.querySelector(selector); + if (el) { el.scrollTop = top; } + }""", SELECTORS["scroll_container"], top)) + except Exception as e: # noqa: BLE001 + logger.debug("scroll failed: %s", e) + + def _first_card_href(session) -> Optional[str]: """The first result card's link, or None. The page-turn signal.""" try: @@ -482,24 +507,33 @@ def handle_captcha_if_present(session, args, *, budget) -> bool: def _scroll_the_grid(session, args, html: str, page_num: int) -> dict: target = page_flow.expected_cards(html) before = _count(session, page_flow.READY_SELECTOR_LISTING) + trace: dict = {} reached = page_flow.scroll_until_settled( lambda sel: _count(session, sel), lambda: _scroll_results(session), lambda: _results_height(session), _sleep, selector=page_flow.READY_SELECTOR_LISTING, - target=target) + target=target, + nudge=lambda: page_flow.nudge_grid( + lambda: _results_geometry(session), + lambda top: _set_results_top(session, top), + _sleep), + trace=trace) settled = target is None or reached >= target logger.info("Scrolled page %d: %d card(s) at first paint, %d after " - "scrolling%s.", page_num, before, reached, - f" (the page says it holds {target})" if target else "") + "scrolling%s%s.", page_num, before, reached, + f" (the page says it holds {target})" if target else "", + f", {trace['nudges']} nudge(s)" if trace.get("nudges") else "") if not settled: logger.warning( "Page %d settled at %d card(s) but its own counter says it holds " - "%d. Those %d are missing from this run.", - page_num, reached, target, target - reached) + "%d. Those %d are missing from this run — %s.", + page_num, reached, target, target - reached, + page_flow.unsettled_reason(trace)) return {"first_paint": before, "reached": reached, "target": target, - "settled": settled} + "settled": settled, "nudges": trace.get("nudges", 0), + "placeholders_left": trace.get("placeholders_left")} def _fetch_one_page(session, args, pool, page_num: int, diff --git a/selenium_scraper.py b/selenium_scraper.py index 617a99d..8b6617a 100644 --- a/selenium_scraper.py +++ b/selenium_scraper.py @@ -325,6 +325,29 @@ def _results_height(session) -> Optional[int]: return None +def _results_geometry(session) -> Optional[tuple]: + """(scrollTop, scrollHeight, clientHeight) of the results container.""" + try: + g = session.driver.execute_script( + "var el = document.querySelector(arguments[0]);" + "return el ? [el.scrollTop, el.scrollHeight, el.clientHeight] : null;", + SELECTORS["scroll_container"]) + return tuple(g) if g else None + except WebDriverException: + return None + + +def _set_results_top(session, top: int) -> None: + """Set the results container's scrollTop.""" + try: + session.driver.execute_script( + "var el = document.querySelector(arguments[0]);" + "if (el) { el.scrollTop = arguments[1]; }" + "return null;", SELECTORS["scroll_container"], top) + except WebDriverException as e: + logger.debug("scroll failed: %s", e) + + def _first_card_href(session) -> Optional[str]: """The first result card's link, or None. The page-turn signal.""" try: @@ -466,24 +489,33 @@ def _scroll_the_grid(session, args, html: str, page_num: int) -> dict: """Scroll the results container until the grid stops growing.""" target = page_flow.expected_cards(html) before = _count(session, page_flow.READY_SELECTOR_LISTING) + trace: dict = {} reached = page_flow.scroll_until_settled( lambda sel: _count(session, sel), lambda: _scroll_results(session), lambda: _results_height(session), _sleep, selector=page_flow.READY_SELECTOR_LISTING, - target=target) + target=target, + nudge=lambda: page_flow.nudge_grid( + lambda: _results_geometry(session), + lambda top: _set_results_top(session, top), + _sleep), + trace=trace) settled = target is None or reached >= target logger.info("Scrolled page %d: %d card(s) at first paint, %d after " - "scrolling%s.", page_num, before, reached, - f" (the page says it holds {target})" if target else "") + "scrolling%s%s.", page_num, before, reached, + f" (the page says it holds {target})" if target else "", + f", {trace['nudges']} nudge(s)" if trace.get("nudges") else "") if not settled: logger.warning( "Page %d settled at %d card(s) but its own counter says it holds " - "%d. Those %d are missing from this run.", - page_num, reached, target, target - reached) + "%d. Those %d are missing from this run — %s.", + page_num, reached, target, target - reached, + page_flow.unsettled_reason(trace)) return {"first_paint": before, "reached": reached, "target": target, - "settled": settled} + "settled": settled, "nudges": trace.get("nudges", 0), + "placeholders_left": trace.get("placeholders_left")} def _fetch_one_page(session, args, pool, page_num: int, diff --git a/smoke_test.py b/smoke_test.py index 119244d..7ee78e9 100644 --- a/smoke_test.py +++ b/smoke_test.py @@ -650,9 +650,9 @@ def test_scroll_loop(): ok &= check(f"{engine} falls back to the window if it is absent", "document.body.scrollHeight" in src) - # A pause is not an ending. This is the loop that settled at 19 cards - # against a counter saying 50 on the first live run, because the next - # batch was in flight behind a throttled /graphql POST. + # A pause is not an ending: a grid that is still growing slowly below a + # known counter is waited for. (The stall at 19 of 50 itself turned out + # to be the lazy tail; see test_grid_lazy_tail.) counts = [3, 3, 3, 3, 3, 50, 50, 50, 50, 50, 50, 50] state = {"i": 0} @@ -699,6 +699,135 @@ def count3(_sel): return ok +class _LazyTailGrid: + """The stall measured on 2026-10-20/21/22: 13 cards and 6 placeholders, + count and height still, until the tail leaves the view and comes back. + + `helps=False` is a grid a nudge does not fill (the bound must hold). + """ + CLIENT = 683 + + def __init__(self, cards=13, target=50, helps=True, placeholders=6): + self.cards, self.target, self.helps = cards, target, helps + self.placeholders = placeholders + self.top, self.height = 4960, 5643 + self.went_top = False + self.set_tops = [] + + def count(self, selector): + if selector == page_flow.PLACEHOLDER_SELECTOR: + return self.placeholders + return self.cards + + def scroll_to_bottom(self): + self.top = self.height - self.CLIENT # same value every time: no event + + def geometry(self): + return (self.top, self.height, self.CLIENT) + + def set_top(self, top): + self.set_tops.append(top) + self.top = top + if top == 0: + self.went_top = True + elif (self.went_top and self.helps + and top + self.CLIENT >= self.height - 2): + self.cards, self.placeholders = self.target, 0 + self.height = 12837 + + +def test_grid_lazy_tail(): + group("A grid stopped at its lazy tail is scrolled back through (real capture)") + ok = True + from bs4 import BeautifulSoup + sel = SELECTORS["grid_placeholder"] + stalled = FIXTURES["LISTING_LAZY_TAIL"] + n = len(BeautifulSoup(stalled, "html.parser").select(sel)) + ok &= check(f"the stalled capture carries the site's placeholders ({n})", n == 6) + for name in ("LISTING_US", "LISTING_US_P2", "LISTING_DE", "LISTING_AU"): + n = len(BeautifulSoup(FIXTURES[name], "html.parser").select(sel)) + ok &= check(f"a served grid carries none ({name}: {n})", n == 0) + rows = rows_of("LISTING_LAZY_TAIL") + ok &= check(f"it still parses its cut cards ({len(rows)})", + len(rows) == 2 and all(r.title and r.price for r in rows)) + ok &= check("and its own counter says 48 more were owed", + page_flow.page_gap(stalled, len(rows)) == 48) + + sleep = lambda ms: None # noqa: E731 + # The canary's stall, reproduced: without a nudge it stays at 13. + g = _LazyTailGrid() + got = page_flow.scroll_until_settled(g.count, g.scroll_to_bottom, + lambda: g.height, sleep, target=50) + ok &= check(f"without a nudge the loop stops at the stall ({got})", got == 13) + + g, trace = _LazyTailGrid(), {} + got = page_flow.scroll_until_settled( + g.count, g.scroll_to_bottom, lambda: g.height, sleep, target=50, + nudge=lambda: page_flow.nudge_grid(g.geometry, g.set_top, sleep), + trace=trace) + ok &= check(f"with one it reaches the counter ({got}, {trace})", + got == 50 and trace == {"nudges": 1, "placeholders_left": 0}) + + g, trace = _LazyTailGrid(helps=False), {} + got = page_flow.scroll_until_settled( + g.count, g.scroll_to_bottom, lambda: g.height, sleep, target=50, + nudge=lambda: page_flow.nudge_grid(g.geometry, g.set_top, sleep), + trace=trace) + ok &= check(f"a nudge that never helps is bounded ({trace})", + got == 13 and trace["nudges"] == page_flow.SCROLL_MAX_NUDGES + and trace["placeholders_left"] == 6) + ok &= check("...and the log says the placeholders were still there", + "placeholders" in page_flow.unsettled_reason(trace)) + + g, trace = _LazyTailGrid(cards=19, placeholders=0), {} + page_flow.scroll_until_settled( + g.count, g.scroll_to_bottom, lambda: g.height, sleep, target=50, + nudge=lambda: page_flow.nudge_grid(g.geometry, g.set_top, sleep), + trace=trace) + ok &= check("a short page with no placeholder is not nudged", + trace == {"nudges": 0, "placeholders_left": 0}) + + g = _LazyTailGrid() + page_flow.nudge_grid(g.geometry, g.set_top, sleep) + tops = g.set_tops + ok &= check(f"a nudge goes to the top, then down a screen at a time " + f"to the bottom ({len(tops)} moves)", + tops[0] == 0 and tops == sorted(tops) + and all(b - a <= g.CLIENT for a, b in zip(tops, tops[1:])) + and len(tops) <= page_flow.NUDGE_MAX_STEPS + 1 + and tops[-1] + g.CLIENT >= 5643 - 2) + moved = [] + page_flow.nudge_grid(lambda: None, moved.append, sleep) + ok &= check("no container, no nudge", moved == []) + + # Every engine passes the nudge, through its own named operations. + import types + for engine in ENGINES: + mod = _load_engine_copy(engine) + short = engine.split("_")[0] + g = _LazyTailGrid() + patches = {"_count": lambda _s, selector: g.count(selector), + "_scroll_results": lambda _s: g.scroll_to_bottom(), + "_results_height": lambda _s: g.height, + "_results_geometry": lambda _s: g.geometry(), + "_set_results_top": lambda _s, top: g.set_top(top), + "_sleep": sleep} + missing = [k for k in patches if not hasattr(mod, k) + and k != "_sleep"] + if missing: + ok &= check(f"{short}: has the named grid operations ({missing})", False) + continue + for k, v in patches.items(): + setattr(mod, k, v) + session = types.SimpleNamespace( + page=types.SimpleNamespace(wait_for_timeout=sleep)) + out = mod._scroll_the_grid(session, types.SimpleNamespace(), stalled, 1) + ok &= check(f"{short}: a stalled grid reaches its counter ({out})", + out["reached"] == 50 and out["settled"] + and out["nudges"] == 1 and out["placeholders_left"] == 0) + return ok + + def test_throttle_is_not_completion(): group("A throttled page turn is PARTIAL, not the end of the listing") ok = True @@ -3405,6 +3534,7 @@ def main() -> int: ok &= test_retries_zero_is_refused() ok &= test_page_flow_policy() ok &= test_scroll_loop() + ok &= test_grid_lazy_tail() ok &= test_throttle_is_not_completion() ok &= test_output_contract() ok &= test_writers_and_finish_run()