feat: cleanup test structure and remove BelenConf testing (#639)

tests/conftest.py

@@ -1,16 +1,26 @@
 # SPDX-FileCopyrightText: © Jens Bergmann and contributors
 # SPDX-License-Identifier: AGPL-3.0-or-later
 # SPDX-ArtifactOfProjectHomePage: https://github.com/Second-Hand-Friends/kleinanzeigen-bot/
-import json
+"""
+Shared test fixtures for the kleinanzeigen-bot test suite.
+
+This module contains fixtures that are used across multiple test files.
+Test-specific fixtures should be defined in individual test files or local conftest.py files.
+
+Fixture Organization:
+- Core fixtures: Basic test infrastructure (test_data_dir, test_bot_config, test_bot)
+- Mock fixtures: Mock objects for external dependencies (browser_mock)
+- Utility fixtures: Helper fixtures for common test scenarios (log_file_path)
+- Smoke test fixtures: Special fixtures for smoke tests (smoke_bot, DummyBrowser, etc.)
+- Test data fixtures: Shared test data (description_test_cases)
+"""
 import os
-from pathlib import Path
 from typing import Any, Final, cast
 from unittest.mock import MagicMock
 
 import pytest
 
 from kleinanzeigen_bot import KleinanzeigenBot
-from kleinanzeigen_bot.extract import AdExtractor
 from kleinanzeigen_bot.model.ad_model import Ad
 from kleinanzeigen_bot.model.config_model import Config
 from kleinanzeigen_bot.utils import loggers
@@ -22,6 +32,10 @@ LOG:Final[loggers.Logger] = loggers.get_logger("kleinanzeigen_bot")
 LOG.setLevel(loggers.DEBUG)
 
 
+# ============================================================================
+# Core Fixtures - Basic test infrastructure
+# ============================================================================
+
 @pytest.fixture
 def test_data_dir(tmp_path:str) -> str:
     """Provides a temporary directory for test data.
@@ -70,6 +84,10 @@ def test_bot(test_bot_config:Config) -> KleinanzeigenBot:
     return bot_instance
 
 
+# ============================================================================
+# Mock Fixtures - Mock objects for external dependencies
+# ============================================================================
+
 @pytest.fixture
 def browser_mock() -> MagicMock:
     """Provides a mock browser instance for testing.
@@ -80,6 +98,10 @@ def browser_mock() -> MagicMock:
     return MagicMock(spec = Browser)
 
 
+# ============================================================================
+# Utility Fixtures - Helper fixtures for common test scenarios
+# ============================================================================
+
 @pytest.fixture
 def log_file_path(test_data_dir:str) -> str:
     """Provides a temporary path for log files.
@@ -90,15 +112,9 @@ def log_file_path(test_data_dir:str) -> str:
     return os.path.join(str(test_data_dir), "test.log")
 
 
-@pytest.fixture
+# ============================================================================
-def test_extractor(browser_mock:MagicMock, test_bot_config:Config) -> AdExtractor:
+# Test Data Fixtures - Shared test data
-    """Provides a fresh AdExtractor instance for testing.
+# ============================================================================
-
-    Dependencies:
-        - browser_mock: Used to mock browser interactions
-        - test_bot_config: Used to initialize the extractor with a valid configuration
-    """
-    return AdExtractor(browser_mock, test_bot_config)
 
 
 @pytest.fixture
@@ -106,6 +122,7 @@ def description_test_cases() -> list[tuple[dict[str, Any], str, str]]:
     """Provides test cases for description prefix/suffix handling.
 
     Returns tuples of (config, raw_description, expected_description)
+    Used by test_init.py and test_extract.py for testing description processing.
     """
     return [
         # Test case 1: New flattened format
@@ -171,38 +188,19 @@ def description_test_cases() -> list[tuple[dict[str, Any], str, str]]:
     ]
 
 
-@pytest.fixture
+# ============================================================================
-def mock_web_text_responses() -> list[str]:
+# Global Setup Fixtures - Applied automatically to all tests
-    """Provides common mock responses for web_text calls."""
+# ============================================================================
-    return [
-        "Test Title",  # Title
-        "Test Description",  # Description
-        "03.02.2025"  # Creation date
-    ]
-
-
-@pytest.fixture
-def belen_conf_sample() -> dict[str, Any]:
-    """Provides sample BelenConf data for testing JavaScript evaluation.
-
-    This fixture loads the BelenConf sample data from the fixtures directory,
-    allowing tests to validate window.BelenConf evaluation without accessing
-    kleinanzeigen.de directly.
-    """
-    fixtures_dir = Path(__file__).parent / "fixtures"
-    belen_conf_path = fixtures_dir / "belen_conf_sample.json"
-
-    with open(belen_conf_path, "r", encoding = "utf-8") as f:
-        data = json.load(f)
-        return cast(dict[str, Any], data)
-
 
 @pytest.fixture(autouse = True)
 def silence_nodriver_logs() -> None:
+    """Silence nodriver logs during testing to reduce noise."""
     loggers.get_logger("nodriver").setLevel(loggers.WARNING)
 
 
-# --- Smoke test fakes and fixtures ---
+# ============================================================================
+# Smoke Test Fixtures - Special fixtures for smoke tests
+# ============================================================================
 
 class DummyBrowser:
     def __init__(self) -> None:

tests/integration/test_web_scraping_mixin_integration.py

@@ -1,20 +1,18 @@
 # SPDX-FileCopyrightText: © Sebastian Thomschke and contributors
 # SPDX-License-Identifier: AGPL-3.0-or-later
 # SPDX-ArtifactOfProjectHomePage: https://github.com/Second-Hand-Friends/kleinanzeigen-bot/
-import os
 import platform
 from typing import cast
 
 import nodriver
 import pytest
 
-from kleinanzeigen_bot.utils import loggers
 from kleinanzeigen_bot.utils.misc import ensure
 from kleinanzeigen_bot.utils.web_scraping_mixin import WebScrapingMixin
 
-if os.environ.get("CI"):
+# Configure logging for integration tests
-    loggers.get_logger("kleinanzeigen_bot").setLevel(loggers.DEBUG)
+# The main bot already handles nodriver logging via silence_nodriver_logs fixture
-    loggers.get_logger("nodriver").setLevel(loggers.DEBUG)
+# and pytest handles verbosity with -v flag automatically
 
 
 async def atest_init() -> None:
@@ -37,69 +35,3 @@ async def atest_init() -> None:
 @pytest.mark.itest
 def test_init() -> None:
     nodriver.loop().run_until_complete(atest_init())
-
-
-async def atest_belen_conf_evaluation() -> None:
-    """Test that window.BelenConf can be evaluated correctly with nodriver."""
-    web_scraping_mixin = WebScrapingMixin()
-    if platform.system() == "Linux":
-        # required for Ubuntu 24.04 or newer
-        cast(list[str], web_scraping_mixin.browser_config.arguments).append("--no-sandbox")
-
-    browser_path = web_scraping_mixin.get_compatible_browser()
-    ensure(browser_path is not None, "Browser not auto-detected")
-
-    web_scraping_mixin.close_browser_session()
-    try:
-        await web_scraping_mixin.create_browser_session()
-
-        # Navigate to a simple page that can execute JavaScript
-        html_content = (
-            "data:text/html,<html><body><script>"
-            "window.BelenConf = {test: 'data', universalAnalyticsOpts: "
-            "{dimensions: {dimension92: 'test', dimension108: 'art_s:test'}}};"
-            "</script></body></html>"
-        )
-        await web_scraping_mixin.web_open(html_content)
-        await web_scraping_mixin.web_sleep(1000, 2000)  # Wait for page to load
-
-        # Test JavaScript evaluation - this is the critical test for nodriver 0.40-0.44 issues
-        belen_conf = await web_scraping_mixin.web_execute("window.BelenConf")
-
-        # Verify the evaluation worked
-        assert belen_conf is not None, "window.BelenConf evaluation returned None"
-
-        # In nodriver 0.47+, JavaScript objects are returned as RemoteObject instances
-        # We need to check if it's either a dict (old behavior) or RemoteObject (new behavior)
-        is_dict = isinstance(belen_conf, dict)
-        is_remote_object = hasattr(belen_conf, "deep_serialized_value") and belen_conf.deep_serialized_value is not None
-
-        assert is_dict or is_remote_object, f"window.BelenConf should be a dict or RemoteObject, got {type(belen_conf)}"
-
-        if is_dict:
-            # Old behavior - direct dict access
-            assert "test" in belen_conf, "window.BelenConf should contain test data"
-            assert "universalAnalyticsOpts" in belen_conf, "window.BelenConf should contain universalAnalyticsOpts"
-        else:
-            # New behavior - RemoteObject with deep_serialized_value
-            assert hasattr(belen_conf, "deep_serialized_value"), "RemoteObject should have deep_serialized_value"
-            assert belen_conf.deep_serialized_value is not None, "deep_serialized_value should not be None"
-
-        if is_dict:
-            print(f"[OK] BelenConf evaluation successful: {list(belen_conf.keys())}")
-        else:
-            print("[OK] BelenConf evaluation successful: RemoteObject with deep_serialized_value")
-
-    finally:
-        web_scraping_mixin.close_browser_session()
-
-
-@pytest.mark.flaky(reruns = 4, reruns_delay = 5)
-@pytest.mark.itest
-def test_belen_conf_evaluation() -> None:
-    """Test that window.BelenConf JavaScript evaluation works correctly.
-
-    This test specifically validates the issue that affected nodriver 0.40-0.44
-    where window.BelenConf evaluation would fail.
-    """
-    nodriver.loop().run_until_complete(atest_belen_conf_evaluation())

tests/unit/test_extract.py

@@ -35,6 +35,17 @@ class _TestCaseDict(TypedDict):  # noqa: PYI049 Private TypedDict `...` is never
     expected:_SpecialAttributesDict
 
 
+@pytest.fixture
+def test_extractor(browser_mock:MagicMock, test_bot_config:Config) -> AdExtractor:
+    """Provides a fresh AdExtractor instance for testing.
+
+    Dependencies:
+        - browser_mock: Used to mock browser interactions
+        - test_bot_config: Used to initialize the extractor with a valid configuration
+    """
+    return AdExtractor(browser_mock, test_bot_config)
+
+
 class TestAdExtractorBasics:
     """Basic synchronous tests for AdExtractor."""
 

tests/unit/test_web_scraping_mixin_remoteobject.py

@@ -1,11 +1,7 @@
 # SPDX-FileCopyrightText: © Jens Bergmann and contributors
 # SPDX-License-Identifier: AGPL-3.0-or-later
 # SPDX-ArtifactOfProjectHomePage: https://github.com/Second-Hand-Friends/kleinanzeigen-bot/
-"""Unit tests for web_scraping_mixin.py RemoteObject handling.
+"""Unit tests for web_scraping_mixin.py RemoteObject handling."""
-
-Copyright (c) 2024, kleinanzeigen-bot contributors.
-All rights reserved.
-"""
 
 from unittest.mock import AsyncMock, Mock, patch