fix(render): measure page height from content, not the body box (#131)

pixelshot clamped the page height to `body.getBoundingClientRect().bottom`,
which only bounds the document when the page lets the body size to its
content. Sites that pin it to the viewport — `html, body { height: 100% }`,
as Wikipedia's Vector 2022 skin does — leave the content overflowing a
one-viewport box, so a 19,951px article measured 1,568px and captured a
single tile with `complete: true` (issue #124).

The rect was also read as a document coordinate while being
viewport-relative. A URL with a `#fragment` loads already scrolled, where
the bottom edge is negative and `Math.max(bottom, 1)` floors the whole page
to 1px.

Measure the lowest edge among the body and its element children instead, and
add the scroll offset back. That reads the same as before on a self-sizing
body, so the clamp still drops the blank tail an inflated
`documentElement.scrollHeight` would otherwise buy (root padding, trailing
margin), while surviving a pinned body.

Both capture paths carried their own copy of the probe; the snippet now
lives in one module so they can't drift apart.

Verified in stock Chrome, standard and turbo probes agreeing on every page:

    page                  before    after   scrollHeight
    wikipedia (#124)        1568    19951          19951
    wikipedia #fragment        1    19951          19951
    simonwillison.net      18252    18252          18252
    ourworldindata.org      8713     8713           8713

Closes #124

Author: andylizf

render/src/pixelrag_render/backends/cdp.py

@@ -35,6 +35,8 @@
 
 from PIL import Image
 
+from .page_metrics import CONTENT_BOTTOM_JS
+
 logger = logging.getLogger("pixelrag_render.backends.cdp")
 
 VIEWPORT_W = 875
@@ -344,6 +346,7 @@ def _readiness_expr() -> str:
     Returns an async-IIFE expression resolving to the page height to tile.
     """
     return f"""(async () => {{
+        {CONTENT_BOTTOM_JS}
         await new Promise(res => {{
             if (document.readyState === 'complete') return res();
             const t = setTimeout(res, {LOAD_TIMEOUT_MS});
@@ -361,8 +364,7 @@ def _readiness_expr() -> str:
         const sh = document.documentElement.scrollHeight;
         const body = document.body;
         if (body) {{
-            const bottom = Math.ceil(body.getBoundingClientRect().bottom);
-            return Math.min(sh, Math.max(bottom, 1));
+            return Math.min(sh, Math.max(contentBottom(body), 1));
         }}
         return sh;
     }})()"""

render/src/pixelrag_render/backends/fast_cdp.py

@@ -34,6 +34,8 @@
 import urllib.request
 from pathlib import Path
 
+from .page_metrics import CONTENT_BOTTOM_JS
+
 logger = logging.getLogger("pixelrag_render.backends.fast_cdp")
 
 VIEWPORT_WIDTH = 875
@@ -58,8 +60,12 @@
     "--disable-features=Translate,MediaRouter,OptimizationHints",
 ]
 
-# JS: wait for fonts + eager images, then return scrollHeight
-_WAIT_FONTS_IMGS = """new Promise(resolve => {
+# JS: wait for fonts + eager images, then return the page height to tile
+_WAIT_FONTS_IMGS = (
+    """new Promise(resolve => {
+    """
+    + CONTENT_BOTTOM_JS
+    + """
     const waitEagerImgs = Promise.all(
         Array.from(document.images)
             .filter(i => !i.complete && i.loading !== 'lazy')
@@ -79,12 +85,13 @@
                 const sh = document.documentElement.scrollHeight;
                 const body = document.body;
                 resolve(body
-                    ? Math.min(sh, Math.max(Math.ceil(body.getBoundingClientRect().bottom), 1))
+                    ? Math.min(sh, Math.max(contentBottom(body), 1))
                     : sh);
             });
         });
     });
 })"""
+)
 
 
 # ---------------------------------------------------------------------------

render/src/pixelrag_render/backends/page_metrics.py

@@ -0,0 +1,44 @@
+"""Shared in-page measurement JS for the capture backends.
+
+Kept in one place because both backends measure page height the same way and
+must agree: the standard (``cdp``) and turbo (``fast_cdp``) paths each embed
+this snippet, and a divergence between them silently changes how much of a page
+gets captured depending on which Chrome binary is installed.
+
+No imports — a plain string constant, so either backend can use it without a
+dependency edge between them.
+"""
+
+# How tall is the document's *content*?
+#
+# `documentElement.scrollHeight` alone over-reports: padding on the root element
+# or a trailing margin inflates it, buying a run of blank tiles at the bottom of
+# every such page. So it is clamped to where the content actually ends.
+#
+# That bound has to be measured from the content, not from the body box. A body
+# is only as tall as the document when the page lets it size to its content;
+# sites that pin it to the viewport (`html, body { height: 100% }` — Wikipedia's
+# Vector 2022 skin among them) leave the content overflowing a one-viewport box,
+# and clamping to that box truncates a 20,000px article to a single tile
+# (issue #124). Taking the lowest edge among the body and its element children
+# reads the same on a self-sizing body and survives a pinned one.
+#
+# Coordinates are viewport-relative, so scroll offset is added back: a page
+# navigated to a `#fragment` lands scrolled down, where a raw rect bottom would
+# under-report by exactly the scrolled distance.
+CONTENT_BOTTOM_JS = """
+    function contentBottom(body) {
+        const offset = window.scrollY || window.pageYOffset || 0;
+        let bottom = body.getBoundingClientRect().bottom;
+        for (let el = body.firstElementChild; el; el = el.nextElementSibling) {
+            const r = el.getBoundingClientRect();
+            // Skip elements with no box at all (display:none, empty <script>);
+            // theirs is a zero rect at the origin and would not move `bottom`,
+            // but skipping keeps the intent explicit.
+            if (r.width > 0 || r.height > 0) {
+                bottom = Math.max(bottom, r.bottom);
+            }
+        }
+        return Math.ceil(bottom + offset);
+    }
+"""

tests/test_render.py

@@ -6,6 +6,7 @@
 to end rather than mocked.
 """
 
+import json
 from pathlib import Path
 
 from pixelrag_render import render_file
@@ -29,3 +30,36 @@ def test_render_local_html_to_tiles(tmp_path):
         f"no tile images produced in {tile_dir} "
         f"(contents: {[p.name for p in tile_dir.iterdir()]})"
     )
+
+
+def test_page_taller_than_a_viewport_bounded_body_is_fully_tiled(tmp_path):
+    """A page whose body is pinned to the viewport must still tile in full.
+
+    Regression for issue #124. Sites that set ``html, body { height: 100% }``
+    (Wikipedia's Vector 2022 skin among them) leave the article content
+    overflowing the body box visibly, so ``body.getBoundingClientRect()`` is one
+    viewport tall on a 20,000px page. The readiness probe used to clamp the page
+    height to that rect, capturing a single tile and reporting the viewport as
+    the page height.
+    """
+    body = "".join(f"<p>line {i:03d}</p>" for i in range(400))
+    html = tmp_path / "viewport_bounded_body.html"
+    html.write_text(
+        '<!DOCTYPE html><html style="height:100%"><body style="height:100%">'
+        f"{body}</body></html>"
+    )
+    out = tmp_path / "tiles"
+
+    dirs = render_file(html, out, tile_height=1000, viewport_width=1280)
+
+    tile_dir = Path(dirs[0])
+    manifest = json.loads((tile_dir / "tiles.json").read_text())
+    tiles = sorted(tile_dir.glob("tile_*.jpg"))
+
+    # 400 paragraphs are several viewports tall whatever the default font is;
+    # assert against the viewport rather than a brittle exact pixel count.
+    assert manifest["page_height"] > 3000, (
+        f"page_height {manifest['page_height']} is about one viewport — the "
+        "content below the fold was never measured"
+    )
+    assert len(tiles) > 1, f"expected multiple tiles, got {[t.name for t in tiles]}"