enh: Add screen-aware viewport sizing (#1198)

## ℹ️ Description

- Link to the related issue(s): PR #1193 follow-up
- Adds screen-aware humanization viewport selection so randomized
desktop sizes fit the available browser-reported screen area while
preserving permissive config validation.

## 📋 Changes Summary

- Add larger common desktop viewport defaults: `2560x1440`, `1920x1200`,
`1728x1117`, and `1512x982`.
- Probe browser-reported CSS-pixel screen metrics at launch time using
`window.screen.availWidth` / `window.screen.availHeight`.
- Filter configured viewport base sizes against available screen
dimensions, apply bounded jitter, and preserve explicit `--window-size`
overrides.
- Omit automatic `--window-size` when no configured size fits, screen
metrics are unavailable, the launch is headless, or no real
display/window manager is available; explicit user `--window-size`
remains preserved.
- Cache successful probe metrics and bound the probe with the
`chrome_remote_probe` timeout.
- Update generated default config and German translations.
- Add unit coverage and real-browser integration coverage for viewport
probing/filtering, headless/no-display skips, probe timeout/cache
behavior, and `create_browser_session()`.

### ⚙️ Type of Change

Select the type(s) of change(s) included in this pull request:
- [ ] 🐞 Bug fix (non-breaking change which fixes an issue)
- [x]  New feature (adds new functionality without breaking existing
usage)
- [ ] 💥 Breaking change (changes that might break existing user setups,
scripts, or configurations)

##  Checklist

Before requesting a review, confirm the following:
- [x] I have reviewed my changes to ensure they meet the project's
standards.
- [x] I have tested my changes and ensured that all tests pass (`pdm run
test`).
- [x] I have formatted the code (`pdm run format`).
- [x] I have verified that linting passes (`pdm run lint`).
- [x] I have updated documentation where necessary.

By submitting this pull request, I confirm that you can use, modify,
copy, and redistribute this contribution, under the terms of your
choice.

<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->
## Summary by CodeRabbit

* **New Features**
* Browser sessions now support screen-aware viewport selection
automatically when viewport randomization is enabled.
* Expanded the default set of candidate viewport sizes used for
randomized sizing.
* **Bug Fixes**
* Viewport sizing now filters out candidates that don’t fit within the
available screen area.
  * If a window size is already specified, it’s preserved.
* **Tests**
* Added/expanded slow integration tests for screen probing and viewport
fitting.
* Expanded unit tests for viewport filtering, jittering, fallback
behavior, and probe reuse/timeout.
<!-- end of auto-generated comment: release notes by coderabbit.ai -->
This commit is contained in:
Jens
2026-07-03 16:40:17 +02:00
committed by GitHub
parent 6d29055323
commit 4c150f64a1
6 changed files with 781 additions and 15 deletions

View File

@@ -379,10 +379,14 @@ humanization:
# - "1920x1080"
# - "1366x768"
viewport_sizes:
- 2560x1440
- 1920x1200
- 1920x1080
- 1728x1117
- 1680x1050
- 1600x900
- 1536x864
- 1512x982
- 1440x900
- 1366x768

View File

@@ -370,7 +370,10 @@ class HumanizationConfig(ContextualModel):
default = True, description = "pick a random window size from viewport_sizes at launch (ignored if --window-size is set manually)"
)
viewport_sizes:list[str] = Field(
default_factory = lambda: ["1920x1080", "1680x1050", "1600x900", "1536x864", "1440x900", "1366x768"],
default_factory = lambda: [
"2560x1440", "1920x1200", "1920x1080", "1728x1117",
"1680x1050", "1600x900", "1536x864", "1512x982", "1440x900", "1366x768",
],
description = "whitelist of WxH desktop window sizes to randomly choose from when randomize_viewport is enabled",
examples = ['"1920x1080"', '"1366x768"'],
)

View File

@@ -823,6 +823,10 @@ kleinanzeigen_bot/utils/web_scraping_mixin.py:
" -> Randomized browser window size: %s": " -> Zufällige Browser-Fenstergröße: %s"
"Ignoring empty --user-data-dir= argument; falling back to configured user_data_dir.": "Ignoriere leeres --user-data-dir= Argument; verwende konfiguriertes user_data_dir."
_select_viewport_size:
"Screen-aware viewport: %s jittered to %dx%d (avail %dx%d)": "Bildschirmgerechte Fenstergröße: %s mit Zufallsabweichung auf %dx%d (verfügbar: %dx%d)"
"No configured viewport fits %dx%d screen; omitting --window-size": "Keine konfigurierte Fenstergröße passt in den verfügbaren Bildschirmbereich %dx%d; --window-size wird weggelassen"
_resolve_effective_user_data_dir:
"Configured browser.user_data_dir (%s) does not match --user-data-dir argument (%s); using the argument value.": "Konfiguriertes browser.user_data_dir (%s) stimmt nicht mit --user-data-dir Argument (%s) überein; verwende Argument-Wert."

View File

@@ -1,7 +1,7 @@
# SPDX-FileCopyrightText: © Sebastian Thomschke and contributors
# SPDX-License-Identifier: AGPL-3.0-or-later
# SPDX-ArtifactOfProjectHomePage: https://github.com/Second-Hand-Friends/kleinanzeigen-bot/
import asyncio, enum, inspect, json, os, platform, secrets, shutil, subprocess # isort: skip # noqa: S404
import asyncio, enum, inspect, json, math, os, platform, secrets, shutil, subprocess, tempfile # isort: skip # noqa: S404
from collections.abc import Awaitable, Callable, Coroutine, Iterable, Sequence
from gettext import gettext as _
from pathlib import Path, PureWindowsPath
@@ -50,6 +50,12 @@ _PRIMARY_SELECTOR_BUDGET_RATIO:Final[float] = 0.70
_BACKUP_SELECTOR_BUDGET_CAP_SECONDS:Final[float] = 0.75
_BACKUP_SELECTOR_BUDGET_FLOOR_SECONDS:Final[float] = 0.25
# Viewport jitter bounds applied when the real screen is probed successfully.
# The base size is jittered uniformly within these ranges, capped by the
# available screen area so the window never exceeds what the display can show.
_VIEWPORT_JITTER_W:Final[int] = 24
_VIEWPORT_JITTER_H:Final[int] = 16
def _resolve_user_data_dir_paths(arg_value:str, config_value:str) -> tuple[Any, Any]:
"""Resolve the argument and config user_data_dir paths for comparison."""
@@ -225,6 +231,62 @@ def _build_nodriver_config(browser_path:str, browser_args:list[str], user_data_d
return cfg
def _filter_viewport_sizes(sizes:list[str], avail_w:int, avail_h:int) -> list[str]:
"""Return viewport sizes whose width/height both fit within the given available screen area."""
fitting:list[str] = []
for size in sizes:
try:
parts = size.lower().split("x")
w = int(parts[0].strip())
h = int(parts[1].strip())
if w <= avail_w and h <= avail_h:
fitting.append(size)
except (ValueError, IndexError):
LOG.debug("Skipping unparseable viewport size during filtering: %s", size)
continue
return fitting
def _is_headless_browser_arg(arg:str) -> bool:
return arg == "--headless" or arg.startswith("--headless=")
def _has_display_available() -> bool:
if platform.system() == "Linux" or os.environ.get("CI", "").lower() == "true":
return bool(os.environ.get("DISPLAY") or os.environ.get("WAYLAND_DISPLAY"))
return True
def _normalize_viewport_metrics(metrics:Any) -> tuple[int, int] | None:
if not isinstance(metrics, tuple) or len(metrics) != _KEY_VALUE_PAIR_SIZE:
return None
w, h = metrics
if isinstance(w, bool) or isinstance(h, bool):
return None
if not isinstance(w, (int, float)) or not isinstance(h, (int, float)):
return None
if not math.isfinite(w) or not math.isfinite(h):
return None
if w <= 0 or h <= 0:
return None
return int(w), int(h)
def _jitter_viewport(base_w:int, base_h:int, avail_w:int, avail_h:int) -> tuple[int, int]:
"""Apply bounded jitter around a base viewport size.
The random offset is clamped so the final window never exceeds the
available screen dimensions or drops below 1×1 CSS pixel.
Returns ``(jittered_w, jittered_h)``.
"""
min_w = max(1, base_w - _VIEWPORT_JITTER_W)
max_w = min(avail_w, base_w + _VIEWPORT_JITTER_W)
min_h = max(1, base_h - _VIEWPORT_JITTER_H)
max_h = min(avail_h, base_h + _VIEWPORT_JITTER_H)
return _rng.randint(min_w, max_w), _rng.randint(min_h, max_h)
class WebScrapingMixin: # noqa: PLR0904
def __init__(self) -> None:
self.browser_config:Final[BrowserConfig] = BrowserConfig()
@@ -232,6 +294,7 @@ class WebScrapingMixin: # noqa: PLR0904
self.page:Page = None # pyright: ignore[reportAttributeAccessIssue]
self._default_timeout_config:TimeoutConfig | None = None
self._default_humanization_config:HumanizationConfig | None = None
self._screen_metrics_cache:tuple[int, int] | None = None
self.config:BotConfig = cast(BotConfig, None)
def _get_humanization_config(self) -> HumanizationConfig:
@@ -481,7 +544,22 @@ class WebScrapingMixin: # noqa: PLR0904
# configure and initialize new browser instance...
########################################################
browser_args, user_data_dir_from_args = self._build_new_browser_launch_args()
# Screen-aware viewport selection — probe is skipped when the user already
# supplies --window-size or when humanization / randomization is disabled.
# Computed locally to avoid stale state across repeated sessions.
humanization = self._get_humanization_config()
selected_viewport_size:str | None = None
if (
humanization.enabled
and humanization.randomize_viewport
and humanization.viewport_sizes
and not any(arg.startswith("--window-size") for arg in self.browser_config.arguments)
):
selected_viewport_size = await self._select_viewport_size()
browser_args, user_data_dir_from_args = self._build_new_browser_launch_args(
selected_viewport_size = selected_viewport_size
)
effective_user_data_dir = await self._resolve_effective_user_data_dir(user_data_dir_from_args)
if not effective_user_data_dir and not is_test_environment:
@@ -558,9 +636,154 @@ class WebScrapingMixin: # noqa: PLR0904
LOG.error("4. Check if any antivirus or security software is blocking the connection")
raise
def _build_new_browser_launch_args(self) -> tuple[list[str], str | None]:
async def _probe_screen_metrics(self) -> tuple[int, int] | None:
"""Start a temporary isolated browser to probe CSS-pixel screen dimensions.
Returns ``(availWidth, availHeight)`` from ``window.screen``, or ``None`` if
the probe browser could not be started or the metrics are unavailable.
The probe browser uses a temporary user data directory and its own lifecycle
— it never touches ``self.browser`` or ``self.page``.
"""
if self._screen_metrics_cache is not None:
return self._screen_metrics_cache
binary = self.browser_config.binary_location
if not binary:
return None
tmp_dir:str | None = None
browser:Browser | None = None
try:
tmp_dir = tempfile.mkdtemp(prefix = "kbb-probe-")
# Strip flags that would tie the probe to the user's profile, alter its
# window, or load extensions. Critical generic flags (--no-sandbox, …)
# are preserved. --disable-extensions is kept as a safety/noise-reduction
# flag, not an extension-load flag.
skip_prefixes = ("--user-data-dir=", "--profile-directory=", "--window-size", "--load-extension=")
probe_args = [a for a in self.browser_config.arguments if not any(a.startswith(p) for p in skip_prefixes)]
cfg = _build_nodriver_config(binary, probe_args, tmp_dir)
async def run_probe() -> Any:
nonlocal browser
browser = await nodriver.start(cfg) # type: ignore[attr-defined]
page = await browser.get("about:blank")
# Give the page a moment to stabilise before querying screen metrics.
await asyncio.sleep(0.5)
return await page.evaluate(
"({availWidth: window.screen.availWidth, availHeight: window.screen.availHeight})",
await_promise = True,
return_by_value = True,
)
result = await asyncio.wait_for(run_probe(), timeout = self.effective_timeout("chrome_remote_probe"))
# Normalize nodriver RemoteObject to a plain dict when
# return_by_value is not honoured by the runtime.
if _is_remote_object(result):
try:
remote_obj:"RemoteObject" = result # noqa: F841
if hasattr(remote_obj, "deep_serialized_value") and remote_obj.deep_serialized_value:
result = self._convert_remote_object_value(remote_obj.deep_serialized_value.value)
elif hasattr(remote_obj, "value") and remote_obj.value is not None:
result = remote_obj.value
except Exception as exc:
LOG.debug("Metrics RemoteObject extraction failed: %s", exc)
return None
if isinstance(result, dict):
w = result.get("availWidth")
h = result.get("availHeight")
if isinstance(w, (int, float)) and isinstance(h, (int, float)) and w > 0 and h > 0:
LOG.debug("Screen metrics probed: %dx%d", int(w), int(h))
self._screen_metrics_cache = (int(w), int(h))
return self._screen_metrics_cache
LOG.debug("Unexpected probe result: %r", result)
return None
except TimeoutError:
LOG.debug("Screen metrics probe timed out", exc_info = True)
return None
except Exception:
LOG.debug("Screen metrics probe failed", exc_info = True)
return None
finally:
if browser is not None:
try:
browser_pid = getattr(browser, "_process_pid", None)
stop_result = cast(Any, browser.stop)()
if inspect.isawaitable(stop_result):
await stop_result
# Mirror close_browser_session() orphan child cleanup
if isinstance(browser_pid, int) and browser_pid > 0:
self._kill_orphaned_browser_children(browser_pid)
except Exception as exc:
LOG.debug("Probe browser stop ignored: %s", exc)
if tmp_dir is not None:
try:
shutil.rmtree(tmp_dir, ignore_errors = True)
except Exception as exc:
LOG.debug("Probe temp directory cleanup ignored: %s", exc)
async def _select_viewport_size(self) -> str | None:
"""Screen-aware viewport selection.
Returns a ``WxH`` string or ``None`` (meaning *omit --window-size*).
Decision logic, in order:
1. Skip headless or no-display environments.
2. Probe the real screen via an isolated browser.
3. If metrics are obtained, pick randomly among configured sizes that fit,
then apply bounded jitter (±24px width, ±16px height) capped by the
available screen.
4. If nothing fits, return ``None`` — let the browser/window-manager choose.
5. If the probe fails or reports invalid metrics, return ``None``.
"""
humanization = self._get_humanization_config()
sizes = humanization.viewport_sizes
if not sizes:
return None
if any(_is_headless_browser_arg(arg) for arg in self.browser_config.arguments):
LOG.debug("Skipping randomized viewport sizing for headless browser args")
return None
if not _has_display_available():
LOG.debug("Skipping randomized viewport sizing because no real display is available")
return None
metrics = await self._probe_screen_metrics()
normalized_metrics = _normalize_viewport_metrics(metrics)
if normalized_metrics is None:
LOG.debug("Screen metrics unavailable or invalid; omitting --window-size")
return None
avail_w, avail_h = normalized_metrics
fitting = _filter_viewport_sizes(sizes, avail_w, avail_h)
if fitting:
chosen = _rng.choice(fitting)
parts = chosen.lower().split("x")
base_w = int(parts[0].strip())
base_h = int(parts[1].strip())
jw, jh = _jitter_viewport(base_w, base_h, avail_w, avail_h)
LOG.info(
"Screen-aware viewport: %s jittered to %dx%d (avail %dx%d)",
chosen, jw, jh, avail_w, avail_h,
)
return f"{jw}x{jh}"
LOG.debug("No configured viewport fits %dx%d screen; omitting --window-size", avail_w, avail_h)
return None
def _build_new_browser_launch_args(self, selected_viewport_size:str | None = None) -> tuple[list[str], str | None]:
"""Build browser launch arguments and extract user_data_dir from custom args.
Args:
selected_viewport_size: Optional ``WxH`` string from screen-aware viewport
selection. When provided, appends ``--window-size=W,H`` to the launch args.
Kept as a parameter (not instance state) to prevent stale values across
repeated sessions.
Returns (browser_args, user_data_dir_from_args).
"""
# default_browser_args: @ https://github.com/Second-Hand-Friends/nodriver/blob/b0d1f0a59cd16e0d9001f32f35a663129e403efd/nodriver/core/config.py
@@ -604,14 +827,9 @@ class WebScrapingMixin: # noqa: PLR0904
continue
browser_args.append(browser_arg)
humanization = self._get_humanization_config()
if (
humanization.enabled
and humanization.randomize_viewport
and humanization.viewport_sizes
and not any(arg.startswith("--window-size") for arg in browser_args)
):
width, height = _rng.choice(humanization.viewport_sizes).lower().split("x")
if selected_viewport_size:
# Selected by _select_viewport_size() — already screen-filtered.
width, height = selected_viewport_size.lower().split("x")
window_size = f"--window-size={width.strip()},{height.strip()}"
LOG.info(" -> Randomized browser window size: %s", window_size)
browser_args.append(window_size)

View File

@@ -0,0 +1,245 @@
# SPDX-FileCopyrightText: © Jens Bergmann and contributors
# SPDX-License-Identifier: AGPL-3.0-or-later
# SPDX-ArtifactOfProjectHomePage: https://github.com/Second-Hand-Friends/kleinanzeigen-bot/
"""Integration test for screen-aware viewport sizing.
Launches a real browser, reads CSS-pixel screen dimensions, verifies that
viewport-size selection respects available screen area, and that an oversized
candidate is correctly excluded.
.. warning::
This test starts a **real browser window** for a few seconds. It is
marked ``itest`` and ``slow`` so it is excluded from the default (unit-only)
test run.
"""
from __future__ import annotations
import os
import platform
import shutil
import tempfile
from unittest.mock import patch
import pytest
from kleinanzeigen_bot.model.config_model import Config, HumanizationConfig
from kleinanzeigen_bot.utils.web_scraping_mixin import WebScrapingMixin
pytestmark = [pytest.mark.slow, pytest.mark.itest]
_HTML_BLANK = "about:blank"
def _make_bare_config(humanization:HumanizationConfig | None = None) -> Config:
"""Return a minimal ``Config`` with the given humanization settings."""
return Config.model_validate({
"login": {"username": "test@example.com", "password": "secret"}, # noqa: S106
"humanization": (humanization or HumanizationConfig()).model_dump(),
})
def _has_browser() -> bool:
"""Return True if a compatible browser binary is available on this host."""
try:
return bool(WebScrapingMixin().get_compatible_browser())
except (AssertionError, RuntimeError, OSError):
return False
def _setup_mixin() -> WebScrapingMixin:
"""Create a configured ``WebScrapingMixin`` for integration testing."""
mixin = WebScrapingMixin()
if platform.system() == "Linux":
mixin.browser_config.arguments.append("--no-sandbox")
mixin.browser_config.binary_location = mixin.get_compatible_browser()
return mixin
def _display_available() -> bool:
if platform.system() == "Linux" or os.environ.get("CI", "").lower() == "true":
return bool(os.environ.get("DISPLAY") or os.environ.get("WAYLAND_DISPLAY"))
return True
async def _probe_or_skip_metrics(mixin:WebScrapingMixin) -> tuple[int, int]:
metrics = await mixin._probe_screen_metrics()
if metrics is None:
pytest.skip("Screen metrics unavailable")
return metrics
def _viewport_fits(size:str, avail_w:int, avail_h:int) -> bool:
"""Return True if the WxH string fits within the given available area."""
try:
parts = size.lower().split("x")
w = int(parts[0].strip())
h = int(parts[1].strip())
return w <= avail_w and h <= avail_h
except (ValueError, IndexError):
return False
# ---------------------------------------------------------------------------
# helper: probe screen metrics in an isolated browser
# ---------------------------------------------------------------------------
@pytest.mark.asyncio
@pytest.mark.skipif(not _has_browser(), reason = "No compatible browser binary detected")
@pytest.mark.skipif(not _display_available(), reason = "No real display/window manager available")
async def test_probe_screen_metrics_returns_positive_dimensions() -> None:
"""The probe browser must report positive CSS-pixel availWidth/availHeight."""
mixin = _setup_mixin()
avail_w, avail_h = await _probe_or_skip_metrics(mixin)
assert isinstance(avail_w, int)
assert avail_w > 0, f"availWidth must be positive, got {avail_w!r}"
assert isinstance(avail_h, int)
assert avail_h > 0, f"availHeight must be positive, got {avail_h!r}"
# Sanity: modern displays are usually at least 720p
assert avail_h >= 720 or avail_w >= 1280, f"Screen seems too small: {avail_w}x{avail_h}"
# ---------------------------------------------------------------------------
# viewport selection: runtime filtering path
# ---------------------------------------------------------------------------
@pytest.mark.asyncio
@pytest.mark.skipif(not _has_browser(), reason = "No compatible browser binary detected")
@pytest.mark.skipif(not _display_available(), reason = "No real display/window manager available")
async def test_viewport_selection_respects_screen_size() -> None:
"""Configure one oversized and one fitting viewport; verify the oversized
candidate is excluded by the runtime filtering path.
"""
mixin = _setup_mixin()
avail_w, avail_h = await _probe_or_skip_metrics(mixin)
# 2. Build a viewport list with exactly one oversized candidate (> screen)
# and one that clearly fits (0.7 × screen).
oversized = f"{avail_w + 200}x{avail_h + 100}"
fitting = f"{int(avail_w * 0.7)}x{int(avail_h * 0.7)}"
sizes = [oversized, fitting]
# 3. Verify the static filter matches expectations.
fitting_result = [s for s in sizes if _viewport_fits(s, avail_w, avail_h)]
assert oversized not in fitting_result, f"{oversized} should be filtered out for {avail_w}x{avail_h} screen"
assert fitting in fitting_result, f"{fitting} should fit {avail_w}x{avail_h} screen"
# 4. Configure the mixin with these sizes and run the selection logic.
mixin.config = _make_bare_config(HumanizationConfig(randomize_viewport = True, viewport_sizes = sizes))
with patch.object(mixin, "_probe_screen_metrics", return_value = (avail_w, avail_h)):
selected = await mixin._select_viewport_size()
assert selected is not None, "At least the fitting size should be selected"
# The selected viewport must fit within available screen when parsed.
# (Jitter is bounded by avail so this always holds.)
parts = selected.lower().split("x")
sel_w, sel_h = int(parts[0].strip()), int(parts[1].strip())
assert sel_w <= avail_w, f"Selected width {sel_w} exceeds available {avail_w}"
assert sel_h <= avail_h, f"Selected height {sel_h} exceeds available {avail_h}"
@pytest.mark.asyncio
@pytest.mark.skipif(not _has_browser(), reason = "No compatible browser binary detected")
@pytest.mark.skipif(not _display_available(), reason = "No real display/window manager available")
async def test_viewport_selection_returns_none_when_all_oversized() -> None:
"""When *all* configured viewports exceed the available screen, selection
must return ``None`` (meaning omit ``--window-size``).
"""
mixin = _setup_mixin()
avail_w, avail_h = await _probe_or_skip_metrics(mixin)
# All sizes are way oversized.
sizes = [f"{avail_w + 500}x{avail_h + 500}", f"{avail_w + 1000}x{avail_h + 1000}"]
# Static filter must return empty.
assert not any(_viewport_fits(s, avail_w, avail_h) for s in sizes)
mixin.config = _make_bare_config(HumanizationConfig(randomize_viewport = True, viewport_sizes = sizes))
with patch.object(mixin, "_probe_screen_metrics", return_value = (avail_w, avail_h)):
selected = await mixin._select_viewport_size()
assert selected is None, f"Expected None (no sizes fit {avail_w}x{avail_h}), got {selected!r}"
# ---------------------------------------------------------------------------
# full launch path: create_browser_session with screen-aware viewport
# ---------------------------------------------------------------------------
@pytest.mark.asyncio
@pytest.mark.skipif(not _has_browser(), reason = "No compatible browser binary detected")
@pytest.mark.skipif(not _display_available(), reason = "No real display/window manager available")
async def test_create_browser_session_selects_fitting_viewport() -> None:
"""Start a browser via create_browser_session() with one oversized and one
fitting viewport candidate, and verify the final window fits within the
available screen area. This exercises the complete launch path: probing,
filtering, jitter, and ``--window-size`` injection.
"""
mixin = _setup_mixin()
avail_w, avail_h = await _probe_or_skip_metrics(mixin)
# 2. Build viewport list: one oversized (exceeds screen), one clearly fitting.
oversized = f"{avail_w + 200}x{avail_h + 100}"
fitting = f"{int(avail_w * 0.7)}x{int(avail_h * 0.7)}"
# 3. Assign a temporary user-data-dir and configure humanization.
ud_dir = tempfile.mkdtemp(prefix = "kbb-vptest-")
mixin.browser_config.user_data_dir = ud_dir
mixin.config = _make_bare_config(HumanizationConfig(
randomize_viewport = True,
viewport_sizes = [oversized, fitting],
))
try:
# 4. Start the browser through the full session-creation path, reusing
# the already-probed metrics so a second flaky probe cannot fail CI.
with patch.object(mixin, "_probe_screen_metrics", return_value = (avail_w, avail_h)):
await mixin.create_browser_session()
# 5. Navigate to about:blank and read the actual inner window size.
# Uses web_execute() which normalizes nodriver RemoteObject values.
await mixin.web_open(_HTML_BLANK)
dims = await mixin.web_execute(
"({w: window.innerWidth, h: window.innerHeight})"
)
assert isinstance(dims, dict), f"Unexpected dimensions result: {dims!r}"
w = dims.get("w", 0)
h = dims.get("h", 0)
assert isinstance(w, (int, float)), f"Inner width has unexpected type: {type(w).__name__}"
assert w > 0, f"Inner width must be positive, got {w}"
assert isinstance(h, (int, float)), f"Inner height has unexpected type: {type(h).__name__}"
assert h > 0, f"Inner height must be positive, got {h}"
# 6. The window must never exceed the available screen area.
assert w <= avail_w, f"Window width {w} exceeds available {avail_w}"
assert h <= avail_h, f"Window height {h} exceeds available {avail_h}"
finally:
mixin.close_browser_session()
shutil.rmtree(ud_dir, ignore_errors = True)
# ---------------------------------------------------------------------------
# fallback behaviour: probe failure picks smallest
# ---------------------------------------------------------------------------
@pytest.mark.asyncio
async def test_select_viewport_size_fallback_to_smallest_on_probe_failure() -> None:
"""When _probe_screen_metrics fails, _select_viewport_size must return None."""
mixin = WebScrapingMixin()
sizes = ["2560x1440", "1920x1080", "1366x768"]
mixin.config = _make_bare_config(HumanizationConfig(randomize_viewport = True, viewport_sizes = sizes))
with patch.object(mixin, "_probe_screen_metrics", return_value = None):
selected = await mixin._select_viewport_size()
assert selected is None
@pytest.mark.asyncio
async def test_select_viewport_size_returns_none_for_empty_sizes() -> None:
"""When viewport_sizes is empty, _select_viewport_size must return None."""
mixin = WebScrapingMixin()
mixin.config = _make_bare_config(HumanizationConfig(randomize_viewport = True, viewport_sizes = []))
selected = await mixin._select_viewport_size()
assert selected is None

View File

@@ -4,13 +4,20 @@
"""Tests for the human-like interaction behavior in WebScrapingMixin (bot-detection evasion)."""
from __future__ import annotations
import asyncio
from types import SimpleNamespace
from unittest.mock import AsyncMock, patch
import pytest
from kleinanzeigen_bot.model.config_model import Config, HumanizationConfig
from kleinanzeigen_bot.utils.web_scraping_mixin import By, Element, WebScrapingMixin
from kleinanzeigen_bot.utils.web_scraping_mixin import (
By,
Element,
WebScrapingMixin,
_filter_viewport_sizes, # noqa: PLC2701 # type: ignore[attr-defined]
_jitter_viewport, # noqa: PLC2701 # type: ignore[attr-defined]
)
def make_scraper(humanization:HumanizationConfig | None = None) -> WebScrapingMixin:
@@ -243,10 +250,18 @@ async def test_web_sleep_with_min_only_is_fixed_delay() -> None:
# viewport randomization at launch
# ---------------------------------------------------------------------------
def test_viewport_arg_appended_when_enabled() -> None:
def test_viewport_arg_appended_when_selected() -> None:
"""selected_viewport_size parameter is honoured by _build_new_browser_launch_args."""
scraper = make_scraper(HumanizationConfig(randomize_viewport = True, viewport_sizes = ["1600x900"]))
args, _ = scraper._build_new_browser_launch_args(selected_viewport_size = "1600x900")
assert "--window-size=1600,900" in args
def test_viewport_arg_not_appended_when_unselected() -> None:
"""_build_new_browser_launch_args with no selected_viewport_size must not add --window-size."""
scraper = make_scraper(HumanizationConfig(randomize_viewport = True, viewport_sizes = ["1600x900"]))
args, _ = scraper._build_new_browser_launch_args()
assert "--window-size=1600,900" in args
assert not any(a.startswith("--window-size") for a in args)
def test_viewport_arg_not_appended_when_user_supplied() -> None:
@@ -347,3 +362,280 @@ async def test_humanized_type_fallback_clears_and_sends_full_text() -> None:
assert field.send_keys.await_args_list[0].args[0] == "a"
assert field.send_keys.await_args_list[1].args[0] == "b"
assert field.send_keys.await_args_list[2].args[0] == "abc"
# ---------------------------------------------------------------------------
# viewport-size helper functions (_filter_viewport_sizes)
# ---------------------------------------------------------------------------
def test_filter_viewport_sizes_all_fit() -> None:
sizes = ["1920x1080", "1366x768", "2560x1440"]
assert _filter_viewport_sizes(sizes, 2560, 1440) == sizes
def test_filter_viewport_sizes_some_fit() -> None:
sizes = ["1920x1080", "2560x1440", "1366x768"]
result = _filter_viewport_sizes(sizes, 1920, 1080)
assert result == ["1920x1080", "1366x768"]
def test_filter_viewport_sizes_none_fit() -> None:
sizes = ["2560x1440", "3840x2160"]
assert _filter_viewport_sizes(sizes, 1920, 1080) == []
def test_filter_viewport_sizes_boundary_exact() -> None:
"""Sizes exactly equal to the available area must fit."""
assert _filter_viewport_sizes(["1920x1080"], 1920, 1080) == ["1920x1080"]
def test_filter_viewport_sizes_skips_parse_errors() -> None:
sizes = ["1920x1080", "not-valid", "1366x768"]
result = _filter_viewport_sizes(sizes, 1920, 1080)
assert result == ["1920x1080", "1366x768"]
def test_filter_viewport_sizes_empty_list() -> None:
assert _filter_viewport_sizes([], 1920, 1080) == []
# ---------------------------------------------------------------------------
# _jitter_viewport
# ---------------------------------------------------------------------------
def test_jitter_viewport_basic() -> None:
"""Jitter stays within ±24 width and ±16 height."""
with patch("kleinanzeigen_bot.utils.web_scraping_mixin._rng.randint", side_effect = [1024, 600]) as mock_rand:
jw, jh = _jitter_viewport(1024, 600, 2000, 1200)
assert jw == 1024
assert jh == 600
assert mock_rand.call_args_list[0].args == (max(1, 1024 - 24), min(2000, 1024 + 24))
assert mock_rand.call_args_list[1].args == (max(1, 600 - 16), min(1200, 600 + 16))
def test_jitter_viewport_caps_by_avail() -> None:
"""max_w / max_h are clamped to avail dimensions."""
with patch("kleinanzeigen_bot.utils.web_scraping_mixin._rng.randint", side_effect = [1900, 1050]) as mock_rand:
jw, jh = _jitter_viewport(1920, 1080, 1920, 1080)
assert jw == 1900
assert jh == 1050
# base_w + 24 = 1944 but avail_w caps at 1920
assert mock_rand.call_args_list[0].args == (max(1, 1920 - 24), 1920)
assert mock_rand.call_args_list[1].args == (max(1, 1080 - 16), 1080)
def test_jitter_viewport_floor_at_one() -> None:
"""min_w / min_h never drop below 1, even for tiny bases."""
with patch("kleinanzeigen_bot.utils.web_scraping_mixin._rng.randint", side_effect = [1, 1]) as mock_rand:
jw, jh = _jitter_viewport(10, 10, 1920, 1080)
assert jw == 1
assert jh == 1
# base_w - 24 = -14, clamped to 1
assert mock_rand.call_args_list[0].args == (1, 10 + 24)
assert mock_rand.call_args_list[1].args == (1, 10 + 16)
# ---------------------------------------------------------------------------
# integration-level: _select_viewport_size with jitter
# ---------------------------------------------------------------------------
@pytest.mark.asyncio
async def test_select_viewport_size_applies_jitter_when_metrics_known() -> None:
"""When metrics are available, a fitting base is jittered before returning."""
scraper = WebScrapingMixin()
scraper.config = Config.model_validate({
"login": {"username": "u", "password": "p"}, # noqa: S106
"humanization": HumanizationConfig(randomize_viewport = True, viewport_sizes = ["1600x900", "1920x1080"]).model_dump(),
})
with (
patch("kleinanzeigen_bot.utils.web_scraping_mixin._has_display_available", return_value = True),
patch.object(scraper, "_probe_screen_metrics", return_value = (1920, 1080)),
patch("kleinanzeigen_bot.utils.web_scraping_mixin._rng.choice", return_value = "1600x900"),
patch("kleinanzeigen_bot.utils.web_scraping_mixin._rng.randint", side_effect = [1590, 890]),
):
result = await scraper._select_viewport_size()
# Jitter applied: base 1600x900 → ~1590x890
assert result == "1590x890"
@pytest.mark.asyncio
async def test_select_viewport_size_skips_headless_without_probe() -> None:
scraper = WebScrapingMixin()
scraper.browser_config.arguments = ["--headless=new"]
scraper.config = Config.model_validate({
"login": {"username": "u", "password": "p"}, # noqa: S106
"humanization": HumanizationConfig(randomize_viewport = True, viewport_sizes = ["1600x900"]).model_dump(),
})
with patch.object(scraper, "_probe_screen_metrics", new_callable = AsyncMock) as probe:
result = await scraper._select_viewport_size()
probe.assert_not_awaited()
assert result is None
@pytest.mark.asyncio
async def test_select_viewport_size_skips_no_display_on_linux_without_probe() -> None:
scraper = WebScrapingMixin()
scraper.config = Config.model_validate({
"login": {"username": "u", "password": "p"}, # noqa: S106
"humanization": HumanizationConfig(randomize_viewport = True, viewport_sizes = ["1600x900"]).model_dump(),
})
with (
patch("kleinanzeigen_bot.utils.web_scraping_mixin.platform.system", return_value = "Linux"),
patch.dict("kleinanzeigen_bot.utils.web_scraping_mixin.os.environ", {}, clear = True),
patch.object(scraper, "_probe_screen_metrics", new_callable = AsyncMock) as probe,
):
result = await scraper._select_viewport_size()
probe.assert_not_awaited()
assert result is None
@pytest.mark.asyncio
async def test_select_viewport_size_skips_ci_without_display_without_probe() -> None:
scraper = WebScrapingMixin()
scraper.config = Config.model_validate({
"login": {"username": "u", "password": "p"}, # noqa: S106
"humanization": HumanizationConfig(randomize_viewport = True, viewport_sizes = ["1600x900"]).model_dump(),
})
with (
patch("kleinanzeigen_bot.utils.web_scraping_mixin.platform.system", return_value = "Darwin"),
patch.dict("kleinanzeigen_bot.utils.web_scraping_mixin.os.environ", {"CI": "true"}, clear = True),
patch.object(scraper, "_probe_screen_metrics", new_callable = AsyncMock) as probe,
):
result = await scraper._select_viewport_size()
probe.assert_not_awaited()
assert result is None
@pytest.mark.asyncio
async def test_select_viewport_size_omits_when_none_fit() -> None:
"""When all sizes exceed avail, returns None regardless of jitter."""
scraper = WebScrapingMixin()
scraper.config = Config.model_validate({
"login": {"username": "u", "password": "p"}, # noqa: S106
"humanization": HumanizationConfig(randomize_viewport = True, viewport_sizes = ["2560x1440"]).model_dump(),
})
with (
patch("kleinanzeigen_bot.utils.web_scraping_mixin._has_display_available", return_value = True),
patch.object(scraper, "_probe_screen_metrics", return_value = (1920, 1080)),
):
result = await scraper._select_viewport_size()
assert result is None
@pytest.mark.asyncio
@pytest.mark.parametrize(
"metrics",
[
None,
(0, 1080),
(1920, 0),
(-1, 1080),
(1920, -1),
("1920", 1080),
(1920, "1080"),
(1920,),
(1920, 1080, 1),
({"availWidth": 1920},),
(True, 1080),
(float("nan"), 1080),
(float("inf"), 1080),
],
)
async def test_select_viewport_size_returns_none_for_invalid_metrics(metrics:object) -> None:
scraper = WebScrapingMixin()
scraper.config = Config.model_validate({
"login": {"username": "u", "password": "p"}, # noqa: S106
"humanization": HumanizationConfig(randomize_viewport = True, viewport_sizes = ["2560x1440", "1366x768"]).model_dump(),
})
with (
patch("kleinanzeigen_bot.utils.web_scraping_mixin._has_display_available", return_value = True),
patch.object(scraper, "_probe_screen_metrics", return_value = metrics),
):
result = await scraper._select_viewport_size()
assert result is None
@pytest.mark.asyncio
async def test_select_viewport_size_user_window_size_bypasses_selection() -> None:
"""User-supplied --window-size in browser_config.arguments prevents _select_viewport_size
from being called at all (tested via the guard in create_browser_session)."""
scraper = WebScrapingMixin()
scraper.browser_config.binary_location = "fake-browser"
scraper.browser_config.arguments = ["--window-size=800,600"]
scraper.config = Config.model_validate({
"login": {"username": "u", "password": "p"}, # noqa: S106
"humanization": HumanizationConfig(randomize_viewport = True, viewport_sizes = ["1920x1080"]).model_dump(),
})
fake_browser = SimpleNamespace(websocket_url = "ws://test")
with (
patch.object(scraper, "_validate_chrome_version_configuration", new_callable = AsyncMock),
patch.object(scraper, "_select_viewport_size", new_callable = AsyncMock) as select_viewport,
patch.object(scraper, "_resolve_effective_user_data_dir", new_callable = AsyncMock, return_value = None),
patch.object(scraper, "_add_browser_extensions", new_callable = AsyncMock),
patch("kleinanzeigen_bot.utils.web_scraping_mixin.files.exists", new_callable = AsyncMock, return_value = True),
patch("kleinanzeigen_bot.utils.web_scraping_mixin.nodriver.start", new_callable = AsyncMock, return_value = fake_browser),
):
await scraper.create_browser_session()
select_viewport.assert_not_awaited()
@pytest.mark.asyncio
async def test_probe_screen_metrics_times_out() -> None:
scraper = WebScrapingMixin()
scraper.browser_config.binary_location = "fake-browser"
scraper.config = Config.model_validate({
"login": {"username": "u", "password": "p"}, # noqa: S106
"timeouts": {"chrome_remote_probe": 0.1},
})
async def slow_start(_cfg:object) -> object:
await asyncio.sleep(1)
return SimpleNamespace()
with patch("kleinanzeigen_bot.utils.web_scraping_mixin.nodriver.start", side_effect = slow_start):
result = await scraper._probe_screen_metrics()
assert result is None
@pytest.mark.asyncio
async def test_probe_screen_metrics_reuses_successful_probe() -> None:
scraper = WebScrapingMixin()
scraper.browser_config.binary_location = "fake-browser"
page = SimpleNamespace(evaluate = AsyncMock(return_value = {"availWidth": 111, "availHeight": 222}))
browser = SimpleNamespace(get = AsyncMock(return_value = page), stop = lambda: None)
with (
patch("kleinanzeigen_bot.utils.web_scraping_mixin.asyncio.sleep", new_callable = AsyncMock),
patch("kleinanzeigen_bot.utils.web_scraping_mixin.nodriver.start", new_callable = AsyncMock, return_value = browser) as start,
):
first = await scraper._probe_screen_metrics()
second = await scraper._probe_screen_metrics()
assert first == (111, 222)
assert second == (111, 222)
start.assert_awaited_once()
# ---------------------------------------------------------------------------
# default viewport sizes
# ---------------------------------------------------------------------------
def test_default_viewport_sizes_contain_new_entries() -> None:
cfg = HumanizationConfig()
expected_extras = {"2560x1440", "1920x1200", "1728x1117", "1512x982"}
for expected in expected_extras:
assert expected in cfg.viewport_sizes, f"Missing default viewport: {expected}"
def test_default_viewport_sizes_omits_4k() -> None:
cfg = HumanizationConfig()
assert "3840x2160" not in cfg.viewport_sizes
def test_default_viewport_sizes_count() -> None:
cfg = HumanizationConfig()
assert len(cfg.viewport_sizes) == 10