diff --git a/tests/conftest.py b/tests/conftest.py index fd23085..bc21b3f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,16 +1,26 @@ # SPDX-FileCopyrightText: © Jens Bergmann and contributors # SPDX-License-Identifier: AGPL-3.0-or-later # SPDX-ArtifactOfProjectHomePage: https://github.com/Second-Hand-Friends/kleinanzeigen-bot/ -import json +""" +Shared test fixtures for the kleinanzeigen-bot test suite. + +This module contains fixtures that are used across multiple test files. +Test-specific fixtures should be defined in individual test files or local conftest.py files. + +Fixture Organization: +- Core fixtures: Basic test infrastructure (test_data_dir, test_bot_config, test_bot) +- Mock fixtures: Mock objects for external dependencies (browser_mock) +- Utility fixtures: Helper fixtures for common test scenarios (log_file_path) +- Smoke test fixtures: Special fixtures for smoke tests (smoke_bot, DummyBrowser, etc.) +- Test data fixtures: Shared test data (description_test_cases) +""" import os -from pathlib import Path from typing import Any, Final, cast from unittest.mock import MagicMock import pytest from kleinanzeigen_bot import KleinanzeigenBot -from kleinanzeigen_bot.extract import AdExtractor from kleinanzeigen_bot.model.ad_model import Ad from kleinanzeigen_bot.model.config_model import Config from kleinanzeigen_bot.utils import loggers @@ -22,6 +32,10 @@ LOG:Final[loggers.Logger] = loggers.get_logger("kleinanzeigen_bot") LOG.setLevel(loggers.DEBUG) +# ============================================================================ +# Core Fixtures - Basic test infrastructure +# ============================================================================ + @pytest.fixture def test_data_dir(tmp_path:str) -> str: """Provides a temporary directory for test data. @@ -70,6 +84,10 @@ def test_bot(test_bot_config:Config) -> KleinanzeigenBot: return bot_instance +# ============================================================================ +# Mock Fixtures - Mock objects for external dependencies +# ============================================================================ + @pytest.fixture def browser_mock() -> MagicMock: """Provides a mock browser instance for testing. @@ -80,6 +98,10 @@ def browser_mock() -> MagicMock: return MagicMock(spec = Browser) +# ============================================================================ +# Utility Fixtures - Helper fixtures for common test scenarios +# ============================================================================ + @pytest.fixture def log_file_path(test_data_dir:str) -> str: """Provides a temporary path for log files. @@ -90,15 +112,9 @@ def log_file_path(test_data_dir:str) -> str: return os.path.join(str(test_data_dir), "test.log") -@pytest.fixture -def test_extractor(browser_mock:MagicMock, test_bot_config:Config) -> AdExtractor: - """Provides a fresh AdExtractor instance for testing. - - Dependencies: - - browser_mock: Used to mock browser interactions - - test_bot_config: Used to initialize the extractor with a valid configuration - """ - return AdExtractor(browser_mock, test_bot_config) +# ============================================================================ +# Test Data Fixtures - Shared test data +# ============================================================================ @pytest.fixture @@ -106,6 +122,7 @@ def description_test_cases() -> list[tuple[dict[str, Any], str, str]]: """Provides test cases for description prefix/suffix handling. Returns tuples of (config, raw_description, expected_description) + Used by test_init.py and test_extract.py for testing description processing. """ return [ # Test case 1: New flattened format @@ -171,38 +188,19 @@ def description_test_cases() -> list[tuple[dict[str, Any], str, str]]: ] -@pytest.fixture -def mock_web_text_responses() -> list[str]: - """Provides common mock responses for web_text calls.""" - return [ - "Test Title", # Title - "Test Description", # Description - "03.02.2025" # Creation date - ] - - -@pytest.fixture -def belen_conf_sample() -> dict[str, Any]: - """Provides sample BelenConf data for testing JavaScript evaluation. - - This fixture loads the BelenConf sample data from the fixtures directory, - allowing tests to validate window.BelenConf evaluation without accessing - kleinanzeigen.de directly. - """ - fixtures_dir = Path(__file__).parent / "fixtures" - belen_conf_path = fixtures_dir / "belen_conf_sample.json" - - with open(belen_conf_path, "r", encoding = "utf-8") as f: - data = json.load(f) - return cast(dict[str, Any], data) - +# ============================================================================ +# Global Setup Fixtures - Applied automatically to all tests +# ============================================================================ @pytest.fixture(autouse = True) def silence_nodriver_logs() -> None: + """Silence nodriver logs during testing to reduce noise.""" loggers.get_logger("nodriver").setLevel(loggers.WARNING) -# --- Smoke test fakes and fixtures --- +# ============================================================================ +# Smoke Test Fixtures - Special fixtures for smoke tests +# ============================================================================ class DummyBrowser: def __init__(self) -> None: diff --git a/tests/integration/test_web_scraping_mixin_integration.py b/tests/integration/test_web_scraping_mixin_integration.py index eea7522..e16c948 100644 --- a/tests/integration/test_web_scraping_mixin_integration.py +++ b/tests/integration/test_web_scraping_mixin_integration.py @@ -1,20 +1,18 @@ # SPDX-FileCopyrightText: © Sebastian Thomschke and contributors # SPDX-License-Identifier: AGPL-3.0-or-later # SPDX-ArtifactOfProjectHomePage: https://github.com/Second-Hand-Friends/kleinanzeigen-bot/ -import os import platform from typing import cast import nodriver import pytest -from kleinanzeigen_bot.utils import loggers from kleinanzeigen_bot.utils.misc import ensure from kleinanzeigen_bot.utils.web_scraping_mixin import WebScrapingMixin -if os.environ.get("CI"): - loggers.get_logger("kleinanzeigen_bot").setLevel(loggers.DEBUG) - loggers.get_logger("nodriver").setLevel(loggers.DEBUG) +# Configure logging for integration tests +# The main bot already handles nodriver logging via silence_nodriver_logs fixture +# and pytest handles verbosity with -v flag automatically async def atest_init() -> None: @@ -37,69 +35,3 @@ async def atest_init() -> None: @pytest.mark.itest def test_init() -> None: nodriver.loop().run_until_complete(atest_init()) - - -async def atest_belen_conf_evaluation() -> None: - """Test that window.BelenConf can be evaluated correctly with nodriver.""" - web_scraping_mixin = WebScrapingMixin() - if platform.system() == "Linux": - # required for Ubuntu 24.04 or newer - cast(list[str], web_scraping_mixin.browser_config.arguments).append("--no-sandbox") - - browser_path = web_scraping_mixin.get_compatible_browser() - ensure(browser_path is not None, "Browser not auto-detected") - - web_scraping_mixin.close_browser_session() - try: - await web_scraping_mixin.create_browser_session() - - # Navigate to a simple page that can execute JavaScript - html_content = ( - "data:text/html," - ) - await web_scraping_mixin.web_open(html_content) - await web_scraping_mixin.web_sleep(1000, 2000) # Wait for page to load - - # Test JavaScript evaluation - this is the critical test for nodriver 0.40-0.44 issues - belen_conf = await web_scraping_mixin.web_execute("window.BelenConf") - - # Verify the evaluation worked - assert belen_conf is not None, "window.BelenConf evaluation returned None" - - # In nodriver 0.47+, JavaScript objects are returned as RemoteObject instances - # We need to check if it's either a dict (old behavior) or RemoteObject (new behavior) - is_dict = isinstance(belen_conf, dict) - is_remote_object = hasattr(belen_conf, "deep_serialized_value") and belen_conf.deep_serialized_value is not None - - assert is_dict or is_remote_object, f"window.BelenConf should be a dict or RemoteObject, got {type(belen_conf)}" - - if is_dict: - # Old behavior - direct dict access - assert "test" in belen_conf, "window.BelenConf should contain test data" - assert "universalAnalyticsOpts" in belen_conf, "window.BelenConf should contain universalAnalyticsOpts" - else: - # New behavior - RemoteObject with deep_serialized_value - assert hasattr(belen_conf, "deep_serialized_value"), "RemoteObject should have deep_serialized_value" - assert belen_conf.deep_serialized_value is not None, "deep_serialized_value should not be None" - - if is_dict: - print(f"[OK] BelenConf evaluation successful: {list(belen_conf.keys())}") - else: - print("[OK] BelenConf evaluation successful: RemoteObject with deep_serialized_value") - - finally: - web_scraping_mixin.close_browser_session() - - -@pytest.mark.flaky(reruns = 4, reruns_delay = 5) -@pytest.mark.itest -def test_belen_conf_evaluation() -> None: - """Test that window.BelenConf JavaScript evaluation works correctly. - - This test specifically validates the issue that affected nodriver 0.40-0.44 - where window.BelenConf evaluation would fail. - """ - nodriver.loop().run_until_complete(atest_belen_conf_evaluation()) diff --git a/tests/unit/test_extract.py b/tests/unit/test_extract.py index f18b223..9f10c95 100644 --- a/tests/unit/test_extract.py +++ b/tests/unit/test_extract.py @@ -35,6 +35,17 @@ class _TestCaseDict(TypedDict): # noqa: PYI049 Private TypedDict `...` is never expected:_SpecialAttributesDict +@pytest.fixture +def test_extractor(browser_mock:MagicMock, test_bot_config:Config) -> AdExtractor: + """Provides a fresh AdExtractor instance for testing. + + Dependencies: + - browser_mock: Used to mock browser interactions + - test_bot_config: Used to initialize the extractor with a valid configuration + """ + return AdExtractor(browser_mock, test_bot_config) + + class TestAdExtractorBasics: """Basic synchronous tests for AdExtractor.""" diff --git a/tests/unit/test_web_scraping_mixin_remoteobject.py b/tests/unit/test_web_scraping_mixin_remoteobject.py index 45c850f..44bba8a 100644 --- a/tests/unit/test_web_scraping_mixin_remoteobject.py +++ b/tests/unit/test_web_scraping_mixin_remoteobject.py @@ -1,11 +1,7 @@ # SPDX-FileCopyrightText: © Jens Bergmann and contributors # SPDX-License-Identifier: AGPL-3.0-or-later # SPDX-ArtifactOfProjectHomePage: https://github.com/Second-Hand-Friends/kleinanzeigen-bot/ -"""Unit tests for web_scraping_mixin.py RemoteObject handling. - -Copyright (c) 2024, kleinanzeigen-bot contributors. -All rights reserved. -""" +"""Unit tests for web_scraping_mixin.py RemoteObject handling.""" from unittest.mock import AsyncMock, Mock, patch