From 4c150f64a13e89be016e040c78510d1c9714f277 Mon Sep 17 00:00:00 2001 From: Jens <1742418+1cu@users.noreply.github.com> Date: Fri, 3 Jul 2026 16:40:17 +0200 Subject: [PATCH] enh: Add screen-aware viewport sizing (#1198) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## ℹ️ Description - Link to the related issue(s): PR #1193 follow-up - Adds screen-aware humanization viewport selection so randomized desktop sizes fit the available browser-reported screen area while preserving permissive config validation. ## 📋 Changes Summary - Add larger common desktop viewport defaults: `2560x1440`, `1920x1200`, `1728x1117`, and `1512x982`. - Probe browser-reported CSS-pixel screen metrics at launch time using `window.screen.availWidth` / `window.screen.availHeight`. - Filter configured viewport base sizes against available screen dimensions, apply bounded jitter, and preserve explicit `--window-size` overrides. - Omit automatic `--window-size` when no configured size fits, screen metrics are unavailable, the launch is headless, or no real display/window manager is available; explicit user `--window-size` remains preserved. - Cache successful probe metrics and bound the probe with the `chrome_remote_probe` timeout. - Update generated default config and German translations. - Add unit coverage and real-browser integration coverage for viewport probing/filtering, headless/no-display skips, probe timeout/cache behavior, and `create_browser_session()`. ### ⚙️ Type of Change Select the type(s) of change(s) included in this pull request: - [ ] 🐞 Bug fix (non-breaking change which fixes an issue) - [x] ✨ New feature (adds new functionality without breaking existing usage) - [ ] 💥 Breaking change (changes that might break existing user setups, scripts, or configurations) ## ✅ Checklist Before requesting a review, confirm the following: - [x] I have reviewed my changes to ensure they meet the project's standards. - [x] I have tested my changes and ensured that all tests pass (`pdm run test`). - [x] I have formatted the code (`pdm run format`). - [x] I have verified that linting passes (`pdm run lint`). - [x] I have updated documentation where necessary. By submitting this pull request, I confirm that you can use, modify, copy, and redistribute this contribution, under the terms of your choice. ## Summary by CodeRabbit * **New Features** * Browser sessions now support screen-aware viewport selection automatically when viewport randomization is enabled. * Expanded the default set of candidate viewport sizes used for randomized sizing. * **Bug Fixes** * Viewport sizing now filters out candidates that don’t fit within the available screen area. * If a window size is already specified, it’s preserved. * **Tests** * Added/expanded slow integration tests for screen probing and viewport fitting. * Expanded unit tests for viewport filtering, jittering, fallback behavior, and probe reuse/timeout. --- docs/config.default.yaml | 4 + src/kleinanzeigen_bot/model/config_model.py | 5 +- .../resources/translations.de.yaml | 4 + .../utils/web_scraping_mixin.py | 240 +++++++++++++- tests/integration/test_viewport_sizing.py | 245 ++++++++++++++ tests/unit/test_humanization.py | 298 +++++++++++++++++- 6 files changed, 781 insertions(+), 15 deletions(-) create mode 100644 tests/integration/test_viewport_sizing.py diff --git a/docs/config.default.yaml b/docs/config.default.yaml index 439ac46..f157f3b 100644 --- a/docs/config.default.yaml +++ b/docs/config.default.yaml @@ -379,10 +379,14 @@ humanization: # - "1920x1080" # - "1366x768" viewport_sizes: + - 2560x1440 + - 1920x1200 - 1920x1080 + - 1728x1117 - 1680x1050 - 1600x900 - 1536x864 + - 1512x982 - 1440x900 - 1366x768 diff --git a/src/kleinanzeigen_bot/model/config_model.py b/src/kleinanzeigen_bot/model/config_model.py index 014a46f..99240cc 100644 --- a/src/kleinanzeigen_bot/model/config_model.py +++ b/src/kleinanzeigen_bot/model/config_model.py @@ -370,7 +370,10 @@ class HumanizationConfig(ContextualModel): default = True, description = "pick a random window size from viewport_sizes at launch (ignored if --window-size is set manually)" ) viewport_sizes:list[str] = Field( - default_factory = lambda: ["1920x1080", "1680x1050", "1600x900", "1536x864", "1440x900", "1366x768"], + default_factory = lambda: [ + "2560x1440", "1920x1200", "1920x1080", "1728x1117", + "1680x1050", "1600x900", "1536x864", "1512x982", "1440x900", "1366x768", + ], description = "whitelist of WxH desktop window sizes to randomly choose from when randomize_viewport is enabled", examples = ['"1920x1080"', '"1366x768"'], ) diff --git a/src/kleinanzeigen_bot/resources/translations.de.yaml b/src/kleinanzeigen_bot/resources/translations.de.yaml index f55ab30..093d96a 100644 --- a/src/kleinanzeigen_bot/resources/translations.de.yaml +++ b/src/kleinanzeigen_bot/resources/translations.de.yaml @@ -823,6 +823,10 @@ kleinanzeigen_bot/utils/web_scraping_mixin.py: " -> Randomized browser window size: %s": " -> Zufällige Browser-Fenstergröße: %s" "Ignoring empty --user-data-dir= argument; falling back to configured user_data_dir.": "Ignoriere leeres --user-data-dir= Argument; verwende konfiguriertes user_data_dir." + _select_viewport_size: + "Screen-aware viewport: %s jittered to %dx%d (avail %dx%d)": "Bildschirmgerechte Fenstergröße: %s mit Zufallsabweichung auf %dx%d (verfügbar: %dx%d)" + "No configured viewport fits %dx%d screen; omitting --window-size": "Keine konfigurierte Fenstergröße passt in den verfügbaren Bildschirmbereich %dx%d; --window-size wird weggelassen" + _resolve_effective_user_data_dir: "Configured browser.user_data_dir (%s) does not match --user-data-dir argument (%s); using the argument value.": "Konfiguriertes browser.user_data_dir (%s) stimmt nicht mit --user-data-dir Argument (%s) überein; verwende Argument-Wert." diff --git a/src/kleinanzeigen_bot/utils/web_scraping_mixin.py b/src/kleinanzeigen_bot/utils/web_scraping_mixin.py index b2a511e..87f36b7 100644 --- a/src/kleinanzeigen_bot/utils/web_scraping_mixin.py +++ b/src/kleinanzeigen_bot/utils/web_scraping_mixin.py @@ -1,7 +1,7 @@ # SPDX-FileCopyrightText: © Sebastian Thomschke and contributors # SPDX-License-Identifier: AGPL-3.0-or-later # SPDX-ArtifactOfProjectHomePage: https://github.com/Second-Hand-Friends/kleinanzeigen-bot/ -import asyncio, enum, inspect, json, os, platform, secrets, shutil, subprocess # isort: skip # noqa: S404 +import asyncio, enum, inspect, json, math, os, platform, secrets, shutil, subprocess, tempfile # isort: skip # noqa: S404 from collections.abc import Awaitable, Callable, Coroutine, Iterable, Sequence from gettext import gettext as _ from pathlib import Path, PureWindowsPath @@ -50,6 +50,12 @@ _PRIMARY_SELECTOR_BUDGET_RATIO:Final[float] = 0.70 _BACKUP_SELECTOR_BUDGET_CAP_SECONDS:Final[float] = 0.75 _BACKUP_SELECTOR_BUDGET_FLOOR_SECONDS:Final[float] = 0.25 +# Viewport jitter bounds applied when the real screen is probed successfully. +# The base size is jittered uniformly within these ranges, capped by the +# available screen area so the window never exceeds what the display can show. +_VIEWPORT_JITTER_W:Final[int] = 24 +_VIEWPORT_JITTER_H:Final[int] = 16 + def _resolve_user_data_dir_paths(arg_value:str, config_value:str) -> tuple[Any, Any]: """Resolve the argument and config user_data_dir paths for comparison.""" @@ -225,6 +231,62 @@ def _build_nodriver_config(browser_path:str, browser_args:list[str], user_data_d return cfg +def _filter_viewport_sizes(sizes:list[str], avail_w:int, avail_h:int) -> list[str]: + """Return viewport sizes whose width/height both fit within the given available screen area.""" + fitting:list[str] = [] + for size in sizes: + try: + parts = size.lower().split("x") + w = int(parts[0].strip()) + h = int(parts[1].strip()) + if w <= avail_w and h <= avail_h: + fitting.append(size) + except (ValueError, IndexError): + LOG.debug("Skipping unparseable viewport size during filtering: %s", size) + continue + return fitting + + +def _is_headless_browser_arg(arg:str) -> bool: + return arg == "--headless" or arg.startswith("--headless=") + + +def _has_display_available() -> bool: + if platform.system() == "Linux" or os.environ.get("CI", "").lower() == "true": + return bool(os.environ.get("DISPLAY") or os.environ.get("WAYLAND_DISPLAY")) + return True + + +def _normalize_viewport_metrics(metrics:Any) -> tuple[int, int] | None: + if not isinstance(metrics, tuple) or len(metrics) != _KEY_VALUE_PAIR_SIZE: + return None + w, h = metrics + if isinstance(w, bool) or isinstance(h, bool): + return None + if not isinstance(w, (int, float)) or not isinstance(h, (int, float)): + return None + if not math.isfinite(w) or not math.isfinite(h): + return None + if w <= 0 or h <= 0: + return None + return int(w), int(h) + + +def _jitter_viewport(base_w:int, base_h:int, avail_w:int, avail_h:int) -> tuple[int, int]: + """Apply bounded jitter around a base viewport size. + + The random offset is clamped so the final window never exceeds the + available screen dimensions or drops below 1×1 CSS pixel. + + Returns ``(jittered_w, jittered_h)``. + """ + min_w = max(1, base_w - _VIEWPORT_JITTER_W) + max_w = min(avail_w, base_w + _VIEWPORT_JITTER_W) + min_h = max(1, base_h - _VIEWPORT_JITTER_H) + max_h = min(avail_h, base_h + _VIEWPORT_JITTER_H) + return _rng.randint(min_w, max_w), _rng.randint(min_h, max_h) + + class WebScrapingMixin: # noqa: PLR0904 def __init__(self) -> None: self.browser_config:Final[BrowserConfig] = BrowserConfig() @@ -232,6 +294,7 @@ class WebScrapingMixin: # noqa: PLR0904 self.page:Page = None # pyright: ignore[reportAttributeAccessIssue] self._default_timeout_config:TimeoutConfig | None = None self._default_humanization_config:HumanizationConfig | None = None + self._screen_metrics_cache:tuple[int, int] | None = None self.config:BotConfig = cast(BotConfig, None) def _get_humanization_config(self) -> HumanizationConfig: @@ -481,7 +544,22 @@ class WebScrapingMixin: # noqa: PLR0904 # configure and initialize new browser instance... ######################################################## - browser_args, user_data_dir_from_args = self._build_new_browser_launch_args() + # Screen-aware viewport selection — probe is skipped when the user already + # supplies --window-size or when humanization / randomization is disabled. + # Computed locally to avoid stale state across repeated sessions. + humanization = self._get_humanization_config() + selected_viewport_size:str | None = None + if ( + humanization.enabled + and humanization.randomize_viewport + and humanization.viewport_sizes + and not any(arg.startswith("--window-size") for arg in self.browser_config.arguments) + ): + selected_viewport_size = await self._select_viewport_size() + + browser_args, user_data_dir_from_args = self._build_new_browser_launch_args( + selected_viewport_size = selected_viewport_size + ) effective_user_data_dir = await self._resolve_effective_user_data_dir(user_data_dir_from_args) if not effective_user_data_dir and not is_test_environment: @@ -558,9 +636,154 @@ class WebScrapingMixin: # noqa: PLR0904 LOG.error("4. Check if any antivirus or security software is blocking the connection") raise - def _build_new_browser_launch_args(self) -> tuple[list[str], str | None]: + async def _probe_screen_metrics(self) -> tuple[int, int] | None: + """Start a temporary isolated browser to probe CSS-pixel screen dimensions. + + Returns ``(availWidth, availHeight)`` from ``window.screen``, or ``None`` if + the probe browser could not be started or the metrics are unavailable. + The probe browser uses a temporary user data directory and its own lifecycle + — it never touches ``self.browser`` or ``self.page``. + """ + if self._screen_metrics_cache is not None: + return self._screen_metrics_cache + + binary = self.browser_config.binary_location + if not binary: + return None + + tmp_dir:str | None = None + browser:Browser | None = None + try: + tmp_dir = tempfile.mkdtemp(prefix = "kbb-probe-") + + # Strip flags that would tie the probe to the user's profile, alter its + # window, or load extensions. Critical generic flags (--no-sandbox, …) + # are preserved. --disable-extensions is kept as a safety/noise-reduction + # flag, not an extension-load flag. + skip_prefixes = ("--user-data-dir=", "--profile-directory=", "--window-size", "--load-extension=") + probe_args = [a for a in self.browser_config.arguments if not any(a.startswith(p) for p in skip_prefixes)] + + cfg = _build_nodriver_config(binary, probe_args, tmp_dir) + + async def run_probe() -> Any: + nonlocal browser + browser = await nodriver.start(cfg) # type: ignore[attr-defined] + page = await browser.get("about:blank") + # Give the page a moment to stabilise before querying screen metrics. + await asyncio.sleep(0.5) + + return await page.evaluate( + "({availWidth: window.screen.availWidth, availHeight: window.screen.availHeight})", + await_promise = True, + return_by_value = True, + ) + + result = await asyncio.wait_for(run_probe(), timeout = self.effective_timeout("chrome_remote_probe")) + + # Normalize nodriver RemoteObject to a plain dict when + # return_by_value is not honoured by the runtime. + if _is_remote_object(result): + try: + remote_obj:"RemoteObject" = result # noqa: F841 + if hasattr(remote_obj, "deep_serialized_value") and remote_obj.deep_serialized_value: + result = self._convert_remote_object_value(remote_obj.deep_serialized_value.value) + elif hasattr(remote_obj, "value") and remote_obj.value is not None: + result = remote_obj.value + except Exception as exc: + LOG.debug("Metrics RemoteObject extraction failed: %s", exc) + return None + + if isinstance(result, dict): + w = result.get("availWidth") + h = result.get("availHeight") + if isinstance(w, (int, float)) and isinstance(h, (int, float)) and w > 0 and h > 0: + LOG.debug("Screen metrics probed: %dx%d", int(w), int(h)) + self._screen_metrics_cache = (int(w), int(h)) + return self._screen_metrics_cache + LOG.debug("Unexpected probe result: %r", result) + return None + except TimeoutError: + LOG.debug("Screen metrics probe timed out", exc_info = True) + return None + except Exception: + LOG.debug("Screen metrics probe failed", exc_info = True) + return None + finally: + if browser is not None: + try: + browser_pid = getattr(browser, "_process_pid", None) + stop_result = cast(Any, browser.stop)() + if inspect.isawaitable(stop_result): + await stop_result + # Mirror close_browser_session() orphan child cleanup + if isinstance(browser_pid, int) and browser_pid > 0: + self._kill_orphaned_browser_children(browser_pid) + except Exception as exc: + LOG.debug("Probe browser stop ignored: %s", exc) + if tmp_dir is not None: + try: + shutil.rmtree(tmp_dir, ignore_errors = True) + except Exception as exc: + LOG.debug("Probe temp directory cleanup ignored: %s", exc) + + async def _select_viewport_size(self) -> str | None: + """Screen-aware viewport selection. + + Returns a ``WxH`` string or ``None`` (meaning *omit --window-size*). + + Decision logic, in order: + 1. Skip headless or no-display environments. + 2. Probe the real screen via an isolated browser. + 3. If metrics are obtained, pick randomly among configured sizes that fit, + then apply bounded jitter (±24px width, ±16px height) capped by the + available screen. + 4. If nothing fits, return ``None`` — let the browser/window-manager choose. + 5. If the probe fails or reports invalid metrics, return ``None``. + """ + humanization = self._get_humanization_config() + sizes = humanization.viewport_sizes + if not sizes: + return None + + if any(_is_headless_browser_arg(arg) for arg in self.browser_config.arguments): + LOG.debug("Skipping randomized viewport sizing for headless browser args") + return None + + if not _has_display_available(): + LOG.debug("Skipping randomized viewport sizing because no real display is available") + return None + + metrics = await self._probe_screen_metrics() + normalized_metrics = _normalize_viewport_metrics(metrics) + if normalized_metrics is None: + LOG.debug("Screen metrics unavailable or invalid; omitting --window-size") + return None + + avail_w, avail_h = normalized_metrics + fitting = _filter_viewport_sizes(sizes, avail_w, avail_h) + if fitting: + chosen = _rng.choice(fitting) + parts = chosen.lower().split("x") + base_w = int(parts[0].strip()) + base_h = int(parts[1].strip()) + jw, jh = _jitter_viewport(base_w, base_h, avail_w, avail_h) + LOG.info( + "Screen-aware viewport: %s jittered to %dx%d (avail %dx%d)", + chosen, jw, jh, avail_w, avail_h, + ) + return f"{jw}x{jh}" + LOG.debug("No configured viewport fits %dx%d screen; omitting --window-size", avail_w, avail_h) + return None + + def _build_new_browser_launch_args(self, selected_viewport_size:str | None = None) -> tuple[list[str], str | None]: """Build browser launch arguments and extract user_data_dir from custom args. + Args: + selected_viewport_size: Optional ``WxH`` string from screen-aware viewport + selection. When provided, appends ``--window-size=W,H`` to the launch args. + Kept as a parameter (not instance state) to prevent stale values across + repeated sessions. + Returns (browser_args, user_data_dir_from_args). """ # default_browser_args: @ https://github.com/Second-Hand-Friends/nodriver/blob/b0d1f0a59cd16e0d9001f32f35a663129e403efd/nodriver/core/config.py @@ -604,14 +827,9 @@ class WebScrapingMixin: # noqa: PLR0904 continue browser_args.append(browser_arg) - humanization = self._get_humanization_config() - if ( - humanization.enabled - and humanization.randomize_viewport - and humanization.viewport_sizes - and not any(arg.startswith("--window-size") for arg in browser_args) - ): - width, height = _rng.choice(humanization.viewport_sizes).lower().split("x") + if selected_viewport_size: + # Selected by _select_viewport_size() — already screen-filtered. + width, height = selected_viewport_size.lower().split("x") window_size = f"--window-size={width.strip()},{height.strip()}" LOG.info(" -> Randomized browser window size: %s", window_size) browser_args.append(window_size) diff --git a/tests/integration/test_viewport_sizing.py b/tests/integration/test_viewport_sizing.py new file mode 100644 index 0000000..505b73b --- /dev/null +++ b/tests/integration/test_viewport_sizing.py @@ -0,0 +1,245 @@ +# SPDX-FileCopyrightText: © Jens Bergmann and contributors +# SPDX-License-Identifier: AGPL-3.0-or-later +# SPDX-ArtifactOfProjectHomePage: https://github.com/Second-Hand-Friends/kleinanzeigen-bot/ +"""Integration test for screen-aware viewport sizing. + +Launches a real browser, reads CSS-pixel screen dimensions, verifies that +viewport-size selection respects available screen area, and that an oversized +candidate is correctly excluded. + +.. warning:: + This test starts a **real browser window** for a few seconds. It is + marked ``itest`` and ``slow`` so it is excluded from the default (unit-only) + test run. +""" +from __future__ import annotations + +import os +import platform +import shutil +import tempfile +from unittest.mock import patch + +import pytest + +from kleinanzeigen_bot.model.config_model import Config, HumanizationConfig +from kleinanzeigen_bot.utils.web_scraping_mixin import WebScrapingMixin + +pytestmark = [pytest.mark.slow, pytest.mark.itest] + +_HTML_BLANK = "about:blank" + + +def _make_bare_config(humanization:HumanizationConfig | None = None) -> Config: + """Return a minimal ``Config`` with the given humanization settings.""" + return Config.model_validate({ + "login": {"username": "test@example.com", "password": "secret"}, # noqa: S106 + "humanization": (humanization or HumanizationConfig()).model_dump(), + }) + + +def _has_browser() -> bool: + """Return True if a compatible browser binary is available on this host.""" + try: + return bool(WebScrapingMixin().get_compatible_browser()) + except (AssertionError, RuntimeError, OSError): + return False + + +def _setup_mixin() -> WebScrapingMixin: + """Create a configured ``WebScrapingMixin`` for integration testing.""" + mixin = WebScrapingMixin() + if platform.system() == "Linux": + mixin.browser_config.arguments.append("--no-sandbox") + mixin.browser_config.binary_location = mixin.get_compatible_browser() + return mixin + + +def _display_available() -> bool: + if platform.system() == "Linux" or os.environ.get("CI", "").lower() == "true": + return bool(os.environ.get("DISPLAY") or os.environ.get("WAYLAND_DISPLAY")) + return True + + +async def _probe_or_skip_metrics(mixin:WebScrapingMixin) -> tuple[int, int]: + metrics = await mixin._probe_screen_metrics() + if metrics is None: + pytest.skip("Screen metrics unavailable") + return metrics + + +def _viewport_fits(size:str, avail_w:int, avail_h:int) -> bool: + """Return True if the WxH string fits within the given available area.""" + try: + parts = size.lower().split("x") + w = int(parts[0].strip()) + h = int(parts[1].strip()) + return w <= avail_w and h <= avail_h + except (ValueError, IndexError): + return False + + +# --------------------------------------------------------------------------- +# helper: probe screen metrics in an isolated browser +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +@pytest.mark.skipif(not _has_browser(), reason = "No compatible browser binary detected") +@pytest.mark.skipif(not _display_available(), reason = "No real display/window manager available") +async def test_probe_screen_metrics_returns_positive_dimensions() -> None: + """The probe browser must report positive CSS-pixel availWidth/availHeight.""" + mixin = _setup_mixin() + avail_w, avail_h = await _probe_or_skip_metrics(mixin) + assert isinstance(avail_w, int) + assert avail_w > 0, f"availWidth must be positive, got {avail_w!r}" + assert isinstance(avail_h, int) + assert avail_h > 0, f"availHeight must be positive, got {avail_h!r}" + # Sanity: modern displays are usually at least 720p + assert avail_h >= 720 or avail_w >= 1280, f"Screen seems too small: {avail_w}x{avail_h}" + + +# --------------------------------------------------------------------------- +# viewport selection: runtime filtering path +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +@pytest.mark.skipif(not _has_browser(), reason = "No compatible browser binary detected") +@pytest.mark.skipif(not _display_available(), reason = "No real display/window manager available") +async def test_viewport_selection_respects_screen_size() -> None: + """Configure one oversized and one fitting viewport; verify the oversized + candidate is excluded by the runtime filtering path. + """ + mixin = _setup_mixin() + avail_w, avail_h = await _probe_or_skip_metrics(mixin) + + # 2. Build a viewport list with exactly one oversized candidate (> screen) + # and one that clearly fits (0.7 × screen). + oversized = f"{avail_w + 200}x{avail_h + 100}" + fitting = f"{int(avail_w * 0.7)}x{int(avail_h * 0.7)}" + sizes = [oversized, fitting] + + # 3. Verify the static filter matches expectations. + fitting_result = [s for s in sizes if _viewport_fits(s, avail_w, avail_h)] + assert oversized not in fitting_result, f"{oversized} should be filtered out for {avail_w}x{avail_h} screen" + assert fitting in fitting_result, f"{fitting} should fit {avail_w}x{avail_h} screen" + + # 4. Configure the mixin with these sizes and run the selection logic. + mixin.config = _make_bare_config(HumanizationConfig(randomize_viewport = True, viewport_sizes = sizes)) + + with patch.object(mixin, "_probe_screen_metrics", return_value = (avail_w, avail_h)): + selected = await mixin._select_viewport_size() + assert selected is not None, "At least the fitting size should be selected" + # The selected viewport must fit within available screen when parsed. + # (Jitter is bounded by avail so this always holds.) + parts = selected.lower().split("x") + sel_w, sel_h = int(parts[0].strip()), int(parts[1].strip()) + assert sel_w <= avail_w, f"Selected width {sel_w} exceeds available {avail_w}" + assert sel_h <= avail_h, f"Selected height {sel_h} exceeds available {avail_h}" + + +@pytest.mark.asyncio +@pytest.mark.skipif(not _has_browser(), reason = "No compatible browser binary detected") +@pytest.mark.skipif(not _display_available(), reason = "No real display/window manager available") +async def test_viewport_selection_returns_none_when_all_oversized() -> None: + """When *all* configured viewports exceed the available screen, selection + must return ``None`` (meaning omit ``--window-size``). + """ + mixin = _setup_mixin() + avail_w, avail_h = await _probe_or_skip_metrics(mixin) + + # All sizes are way oversized. + sizes = [f"{avail_w + 500}x{avail_h + 500}", f"{avail_w + 1000}x{avail_h + 1000}"] + + # Static filter must return empty. + assert not any(_viewport_fits(s, avail_w, avail_h) for s in sizes) + + mixin.config = _make_bare_config(HumanizationConfig(randomize_viewport = True, viewport_sizes = sizes)) + with patch.object(mixin, "_probe_screen_metrics", return_value = (avail_w, avail_h)): + selected = await mixin._select_viewport_size() + assert selected is None, f"Expected None (no sizes fit {avail_w}x{avail_h}), got {selected!r}" + + +# --------------------------------------------------------------------------- +# full launch path: create_browser_session with screen-aware viewport +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +@pytest.mark.skipif(not _has_browser(), reason = "No compatible browser binary detected") +@pytest.mark.skipif(not _display_available(), reason = "No real display/window manager available") +async def test_create_browser_session_selects_fitting_viewport() -> None: + """Start a browser via create_browser_session() with one oversized and one + fitting viewport candidate, and verify the final window fits within the + available screen area. This exercises the complete launch path: probing, + filtering, jitter, and ``--window-size`` injection. + """ + mixin = _setup_mixin() + avail_w, avail_h = await _probe_or_skip_metrics(mixin) + + # 2. Build viewport list: one oversized (exceeds screen), one clearly fitting. + oversized = f"{avail_w + 200}x{avail_h + 100}" + fitting = f"{int(avail_w * 0.7)}x{int(avail_h * 0.7)}" + + # 3. Assign a temporary user-data-dir and configure humanization. + ud_dir = tempfile.mkdtemp(prefix = "kbb-vptest-") + mixin.browser_config.user_data_dir = ud_dir + mixin.config = _make_bare_config(HumanizationConfig( + randomize_viewport = True, + viewport_sizes = [oversized, fitting], + )) + + try: + # 4. Start the browser through the full session-creation path, reusing + # the already-probed metrics so a second flaky probe cannot fail CI. + with patch.object(mixin, "_probe_screen_metrics", return_value = (avail_w, avail_h)): + await mixin.create_browser_session() + + # 5. Navigate to about:blank and read the actual inner window size. + # Uses web_execute() which normalizes nodriver RemoteObject values. + await mixin.web_open(_HTML_BLANK) + dims = await mixin.web_execute( + "({w: window.innerWidth, h: window.innerHeight})" + ) + assert isinstance(dims, dict), f"Unexpected dimensions result: {dims!r}" + w = dims.get("w", 0) + h = dims.get("h", 0) + assert isinstance(w, (int, float)), f"Inner width has unexpected type: {type(w).__name__}" + assert w > 0, f"Inner width must be positive, got {w}" + assert isinstance(h, (int, float)), f"Inner height has unexpected type: {type(h).__name__}" + assert h > 0, f"Inner height must be positive, got {h}" + + # 6. The window must never exceed the available screen area. + assert w <= avail_w, f"Window width {w} exceeds available {avail_w}" + assert h <= avail_h, f"Window height {h} exceeds available {avail_h}" + finally: + mixin.close_browser_session() + shutil.rmtree(ud_dir, ignore_errors = True) + + +# --------------------------------------------------------------------------- +# fallback behaviour: probe failure picks smallest +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_select_viewport_size_fallback_to_smallest_on_probe_failure() -> None: + """When _probe_screen_metrics fails, _select_viewport_size must return None.""" + mixin = WebScrapingMixin() + sizes = ["2560x1440", "1920x1080", "1366x768"] + mixin.config = _make_bare_config(HumanizationConfig(randomize_viewport = True, viewport_sizes = sizes)) + + with patch.object(mixin, "_probe_screen_metrics", return_value = None): + selected = await mixin._select_viewport_size() + + assert selected is None + + +@pytest.mark.asyncio +async def test_select_viewport_size_returns_none_for_empty_sizes() -> None: + """When viewport_sizes is empty, _select_viewport_size must return None.""" + mixin = WebScrapingMixin() + mixin.config = _make_bare_config(HumanizationConfig(randomize_viewport = True, viewport_sizes = [])) + selected = await mixin._select_viewport_size() + assert selected is None diff --git a/tests/unit/test_humanization.py b/tests/unit/test_humanization.py index de7a832..2f2d664 100644 --- a/tests/unit/test_humanization.py +++ b/tests/unit/test_humanization.py @@ -4,13 +4,20 @@ """Tests for the human-like interaction behavior in WebScrapingMixin (bot-detection evasion).""" from __future__ import annotations +import asyncio from types import SimpleNamespace from unittest.mock import AsyncMock, patch import pytest from kleinanzeigen_bot.model.config_model import Config, HumanizationConfig -from kleinanzeigen_bot.utils.web_scraping_mixin import By, Element, WebScrapingMixin +from kleinanzeigen_bot.utils.web_scraping_mixin import ( + By, + Element, + WebScrapingMixin, + _filter_viewport_sizes, # noqa: PLC2701 # type: ignore[attr-defined] + _jitter_viewport, # noqa: PLC2701 # type: ignore[attr-defined] +) def make_scraper(humanization:HumanizationConfig | None = None) -> WebScrapingMixin: @@ -243,10 +250,18 @@ async def test_web_sleep_with_min_only_is_fixed_delay() -> None: # viewport randomization at launch # --------------------------------------------------------------------------- -def test_viewport_arg_appended_when_enabled() -> None: +def test_viewport_arg_appended_when_selected() -> None: + """selected_viewport_size parameter is honoured by _build_new_browser_launch_args.""" + scraper = make_scraper(HumanizationConfig(randomize_viewport = True, viewport_sizes = ["1600x900"])) + args, _ = scraper._build_new_browser_launch_args(selected_viewport_size = "1600x900") + assert "--window-size=1600,900" in args + + +def test_viewport_arg_not_appended_when_unselected() -> None: + """_build_new_browser_launch_args with no selected_viewport_size must not add --window-size.""" scraper = make_scraper(HumanizationConfig(randomize_viewport = True, viewport_sizes = ["1600x900"])) args, _ = scraper._build_new_browser_launch_args() - assert "--window-size=1600,900" in args + assert not any(a.startswith("--window-size") for a in args) def test_viewport_arg_not_appended_when_user_supplied() -> None: @@ -347,3 +362,280 @@ async def test_humanized_type_fallback_clears_and_sends_full_text() -> None: assert field.send_keys.await_args_list[0].args[0] == "a" assert field.send_keys.await_args_list[1].args[0] == "b" assert field.send_keys.await_args_list[2].args[0] == "abc" + + +# --------------------------------------------------------------------------- +# viewport-size helper functions (_filter_viewport_sizes) +# --------------------------------------------------------------------------- + + +def test_filter_viewport_sizes_all_fit() -> None: + sizes = ["1920x1080", "1366x768", "2560x1440"] + assert _filter_viewport_sizes(sizes, 2560, 1440) == sizes + + +def test_filter_viewport_sizes_some_fit() -> None: + sizes = ["1920x1080", "2560x1440", "1366x768"] + result = _filter_viewport_sizes(sizes, 1920, 1080) + assert result == ["1920x1080", "1366x768"] + + +def test_filter_viewport_sizes_none_fit() -> None: + sizes = ["2560x1440", "3840x2160"] + assert _filter_viewport_sizes(sizes, 1920, 1080) == [] + + +def test_filter_viewport_sizes_boundary_exact() -> None: + """Sizes exactly equal to the available area must fit.""" + assert _filter_viewport_sizes(["1920x1080"], 1920, 1080) == ["1920x1080"] + + +def test_filter_viewport_sizes_skips_parse_errors() -> None: + sizes = ["1920x1080", "not-valid", "1366x768"] + result = _filter_viewport_sizes(sizes, 1920, 1080) + assert result == ["1920x1080", "1366x768"] + + +def test_filter_viewport_sizes_empty_list() -> None: + assert _filter_viewport_sizes([], 1920, 1080) == [] + + +# --------------------------------------------------------------------------- +# _jitter_viewport +# --------------------------------------------------------------------------- + + +def test_jitter_viewport_basic() -> None: + """Jitter stays within ±24 width and ±16 height.""" + with patch("kleinanzeigen_bot.utils.web_scraping_mixin._rng.randint", side_effect = [1024, 600]) as mock_rand: + jw, jh = _jitter_viewport(1024, 600, 2000, 1200) + assert jw == 1024 + assert jh == 600 + assert mock_rand.call_args_list[0].args == (max(1, 1024 - 24), min(2000, 1024 + 24)) + assert mock_rand.call_args_list[1].args == (max(1, 600 - 16), min(1200, 600 + 16)) + + +def test_jitter_viewport_caps_by_avail() -> None: + """max_w / max_h are clamped to avail dimensions.""" + with patch("kleinanzeigen_bot.utils.web_scraping_mixin._rng.randint", side_effect = [1900, 1050]) as mock_rand: + jw, jh = _jitter_viewport(1920, 1080, 1920, 1080) + assert jw == 1900 + assert jh == 1050 + # base_w + 24 = 1944 but avail_w caps at 1920 + assert mock_rand.call_args_list[0].args == (max(1, 1920 - 24), 1920) + assert mock_rand.call_args_list[1].args == (max(1, 1080 - 16), 1080) + + +def test_jitter_viewport_floor_at_one() -> None: + """min_w / min_h never drop below 1, even for tiny bases.""" + with patch("kleinanzeigen_bot.utils.web_scraping_mixin._rng.randint", side_effect = [1, 1]) as mock_rand: + jw, jh = _jitter_viewport(10, 10, 1920, 1080) + assert jw == 1 + assert jh == 1 + # base_w - 24 = -14, clamped to 1 + assert mock_rand.call_args_list[0].args == (1, 10 + 24) + assert mock_rand.call_args_list[1].args == (1, 10 + 16) + + +# --------------------------------------------------------------------------- +# integration-level: _select_viewport_size with jitter +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_select_viewport_size_applies_jitter_when_metrics_known() -> None: + """When metrics are available, a fitting base is jittered before returning.""" + scraper = WebScrapingMixin() + scraper.config = Config.model_validate({ + "login": {"username": "u", "password": "p"}, # noqa: S106 + "humanization": HumanizationConfig(randomize_viewport = True, viewport_sizes = ["1600x900", "1920x1080"]).model_dump(), + }) + with ( + patch("kleinanzeigen_bot.utils.web_scraping_mixin._has_display_available", return_value = True), + patch.object(scraper, "_probe_screen_metrics", return_value = (1920, 1080)), + patch("kleinanzeigen_bot.utils.web_scraping_mixin._rng.choice", return_value = "1600x900"), + patch("kleinanzeigen_bot.utils.web_scraping_mixin._rng.randint", side_effect = [1590, 890]), + ): + result = await scraper._select_viewport_size() + # Jitter applied: base 1600x900 → ~1590x890 + assert result == "1590x890" + + +@pytest.mark.asyncio +async def test_select_viewport_size_skips_headless_without_probe() -> None: + scraper = WebScrapingMixin() + scraper.browser_config.arguments = ["--headless=new"] + scraper.config = Config.model_validate({ + "login": {"username": "u", "password": "p"}, # noqa: S106 + "humanization": HumanizationConfig(randomize_viewport = True, viewport_sizes = ["1600x900"]).model_dump(), + }) + with patch.object(scraper, "_probe_screen_metrics", new_callable = AsyncMock) as probe: + result = await scraper._select_viewport_size() + probe.assert_not_awaited() + assert result is None + + +@pytest.mark.asyncio +async def test_select_viewport_size_skips_no_display_on_linux_without_probe() -> None: + scraper = WebScrapingMixin() + scraper.config = Config.model_validate({ + "login": {"username": "u", "password": "p"}, # noqa: S106 + "humanization": HumanizationConfig(randomize_viewport = True, viewport_sizes = ["1600x900"]).model_dump(), + }) + with ( + patch("kleinanzeigen_bot.utils.web_scraping_mixin.platform.system", return_value = "Linux"), + patch.dict("kleinanzeigen_bot.utils.web_scraping_mixin.os.environ", {}, clear = True), + patch.object(scraper, "_probe_screen_metrics", new_callable = AsyncMock) as probe, + ): + result = await scraper._select_viewport_size() + probe.assert_not_awaited() + assert result is None + + +@pytest.mark.asyncio +async def test_select_viewport_size_skips_ci_without_display_without_probe() -> None: + scraper = WebScrapingMixin() + scraper.config = Config.model_validate({ + "login": {"username": "u", "password": "p"}, # noqa: S106 + "humanization": HumanizationConfig(randomize_viewport = True, viewport_sizes = ["1600x900"]).model_dump(), + }) + with ( + patch("kleinanzeigen_bot.utils.web_scraping_mixin.platform.system", return_value = "Darwin"), + patch.dict("kleinanzeigen_bot.utils.web_scraping_mixin.os.environ", {"CI": "true"}, clear = True), + patch.object(scraper, "_probe_screen_metrics", new_callable = AsyncMock) as probe, + ): + result = await scraper._select_viewport_size() + probe.assert_not_awaited() + assert result is None + + +@pytest.mark.asyncio +async def test_select_viewport_size_omits_when_none_fit() -> None: + """When all sizes exceed avail, returns None regardless of jitter.""" + scraper = WebScrapingMixin() + scraper.config = Config.model_validate({ + "login": {"username": "u", "password": "p"}, # noqa: S106 + "humanization": HumanizationConfig(randomize_viewport = True, viewport_sizes = ["2560x1440"]).model_dump(), + }) + with ( + patch("kleinanzeigen_bot.utils.web_scraping_mixin._has_display_available", return_value = True), + patch.object(scraper, "_probe_screen_metrics", return_value = (1920, 1080)), + ): + result = await scraper._select_viewport_size() + assert result is None + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "metrics", + [ + None, + (0, 1080), + (1920, 0), + (-1, 1080), + (1920, -1), + ("1920", 1080), + (1920, "1080"), + (1920,), + (1920, 1080, 1), + ({"availWidth": 1920},), + (True, 1080), + (float("nan"), 1080), + (float("inf"), 1080), + ], +) +async def test_select_viewport_size_returns_none_for_invalid_metrics(metrics:object) -> None: + scraper = WebScrapingMixin() + scraper.config = Config.model_validate({ + "login": {"username": "u", "password": "p"}, # noqa: S106 + "humanization": HumanizationConfig(randomize_viewport = True, viewport_sizes = ["2560x1440", "1366x768"]).model_dump(), + }) + with ( + patch("kleinanzeigen_bot.utils.web_scraping_mixin._has_display_available", return_value = True), + patch.object(scraper, "_probe_screen_metrics", return_value = metrics), + ): + result = await scraper._select_viewport_size() + assert result is None + + +@pytest.mark.asyncio +async def test_select_viewport_size_user_window_size_bypasses_selection() -> None: + """User-supplied --window-size in browser_config.arguments prevents _select_viewport_size + from being called at all (tested via the guard in create_browser_session).""" + scraper = WebScrapingMixin() + scraper.browser_config.binary_location = "fake-browser" + scraper.browser_config.arguments = ["--window-size=800,600"] + scraper.config = Config.model_validate({ + "login": {"username": "u", "password": "p"}, # noqa: S106 + "humanization": HumanizationConfig(randomize_viewport = True, viewport_sizes = ["1920x1080"]).model_dump(), + }) + fake_browser = SimpleNamespace(websocket_url = "ws://test") + with ( + patch.object(scraper, "_validate_chrome_version_configuration", new_callable = AsyncMock), + patch.object(scraper, "_select_viewport_size", new_callable = AsyncMock) as select_viewport, + patch.object(scraper, "_resolve_effective_user_data_dir", new_callable = AsyncMock, return_value = None), + patch.object(scraper, "_add_browser_extensions", new_callable = AsyncMock), + patch("kleinanzeigen_bot.utils.web_scraping_mixin.files.exists", new_callable = AsyncMock, return_value = True), + patch("kleinanzeigen_bot.utils.web_scraping_mixin.nodriver.start", new_callable = AsyncMock, return_value = fake_browser), + ): + await scraper.create_browser_session() + select_viewport.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_probe_screen_metrics_times_out() -> None: + scraper = WebScrapingMixin() + scraper.browser_config.binary_location = "fake-browser" + scraper.config = Config.model_validate({ + "login": {"username": "u", "password": "p"}, # noqa: S106 + "timeouts": {"chrome_remote_probe": 0.1}, + }) + + async def slow_start(_cfg:object) -> object: + await asyncio.sleep(1) + return SimpleNamespace() + + with patch("kleinanzeigen_bot.utils.web_scraping_mixin.nodriver.start", side_effect = slow_start): + result = await scraper._probe_screen_metrics() + + assert result is None + + +@pytest.mark.asyncio +async def test_probe_screen_metrics_reuses_successful_probe() -> None: + scraper = WebScrapingMixin() + scraper.browser_config.binary_location = "fake-browser" + page = SimpleNamespace(evaluate = AsyncMock(return_value = {"availWidth": 111, "availHeight": 222})) + browser = SimpleNamespace(get = AsyncMock(return_value = page), stop = lambda: None) + + with ( + patch("kleinanzeigen_bot.utils.web_scraping_mixin.asyncio.sleep", new_callable = AsyncMock), + patch("kleinanzeigen_bot.utils.web_scraping_mixin.nodriver.start", new_callable = AsyncMock, return_value = browser) as start, + ): + first = await scraper._probe_screen_metrics() + second = await scraper._probe_screen_metrics() + + assert first == (111, 222) + assert second == (111, 222) + start.assert_awaited_once() + + +# --------------------------------------------------------------------------- +# default viewport sizes +# --------------------------------------------------------------------------- + +def test_default_viewport_sizes_contain_new_entries() -> None: + cfg = HumanizationConfig() + expected_extras = {"2560x1440", "1920x1200", "1728x1117", "1512x982"} + for expected in expected_extras: + assert expected in cfg.viewport_sizes, f"Missing default viewport: {expected}" + + +def test_default_viewport_sizes_omits_4k() -> None: + cfg = HumanizationConfig() + assert "3840x2160" not in cfg.viewport_sizes + + +def test_default_viewport_sizes_count() -> None: + cfg = HumanizationConfig() + assert len(cfg.viewport_sizes) == 10