mirror of
https://github.com/Second-Hand-Friends/kleinanzeigen-bot.git
synced 2026-03-12 02:31:45 +01:00
## ℹ️ Description Upgrade nodriver dependency from pinned version 0.39.0 to latest 0.47.0 to resolve browser startup issues and JavaScript evaluation problems that affected versions 0.40-0.44. - Link to the related issue(s): Resolves nodriver compatibility issues - This upgrade addresses browser startup problems and window.BelenConf evaluation failures that were blocking the use of newer nodriver versions. ## 📋 Changes Summary - Updated nodriver dependency from pinned 0.39.0 to >=0.47.0 in pyproject.toml - Fixed RemoteObject handling in web_execute method for nodriver 0.47 compatibility - Added comprehensive BelenConf test fixture with real production data structure - Added integration test to validate window.BelenConf evaluation works correctly - Added German translation for new error message - Replaced real user data with privacy-safe dummy data in test fixtures ### 🔧 Type Safety Improvements **Added explicit `str()` conversions to resolve type inference issues:** The comprehensive BelenConf test fixture contains deeply nested data structures that caused pyright's type checker to infer complex dictionary types throughout the codebase. To ensure type safety and prevent runtime errors, I added explicit `str()` conversions in key locations: - **CSRF tokens**: `str(csrf_token)` - Ensures CSRF tokens are treated as strings - **Special attributes**: `str(special_attribute_value)` - Converts special attribute values to strings - **DOM attributes**: `str(special_attr_elem.attrs.id)` - Ensures element IDs are strings - **URL handling**: `str(current_img_url)` and `str(href_attributes)` - Converts URLs and href attributes to strings - **Price values**: `str(ad_cfg.price)` - Ensures price values are strings These conversions are defensive programming measures that ensure backward compatibility and prevent type-related runtime errors, even if the underlying data structures change in the future. ### ⚙️ Type of Change - [x] ✨ New feature (adds new functionality without breaking existing usage) - [ ] 🐞 Bug fix (non-breaking change which fixes an issue) - [ ] 💥 Breaking change (changes that might break existing user setups, scripts, or configurations) ## ✅ Checklist Before requesting a review, confirm the following: - [x] I have reviewed my changes to ensure they meet the project's standards. - [x] I have tested my changes and ensured that all tests pass (`pdm run test`). - [x] I have formatted the code (`pdm run format`). - [x] I have verified that linting passes (`pdm run lint`). - [x] I have updated documentation where necessary. By submitting this pull request, I confirm that you can use, modify, copy, and redistribute this contribution, under the terms of your choice.
135 lines
5.3 KiB
Python
135 lines
5.3 KiB
Python
# SPDX-FileCopyrightText: © Jens Bergmann and contributors
|
|
# SPDX-License-Identifier: AGPL-3.0-or-later
|
|
# SPDX-ArtifactOfProjectHomePage: https://github.com/Second-Hand-Friends/kleinanzeigen-bot/
|
|
"""Unit tests for web_scraping_mixin.py RemoteObject handling.
|
|
|
|
Copyright (c) 2024, kleinanzeigen-bot contributors.
|
|
All rights reserved.
|
|
"""
|
|
|
|
from unittest.mock import AsyncMock, Mock, patch
|
|
|
|
import pytest
|
|
|
|
from kleinanzeigen_bot.utils.web_scraping_mixin import WebScrapingMixin
|
|
|
|
|
|
class TestWebExecuteRemoteObjectHandling:
|
|
"""Test web_execute method with nodriver 0.47+ RemoteObject behavior."""
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_web_execute_remoteobject_with_deep_serialized_value(self) -> None:
|
|
"""Test web_execute with RemoteObject that has deep_serialized_value."""
|
|
mixin = WebScrapingMixin()
|
|
|
|
# Mock RemoteObject with deep_serialized_value
|
|
mock_remote_object = Mock()
|
|
mock_remote_object.deep_serialized_value = Mock()
|
|
mock_remote_object.deep_serialized_value.value = [
|
|
["key1", "value1"],
|
|
["key2", "value2"]
|
|
]
|
|
|
|
# Mock the page evaluation to return our RemoteObject
|
|
with patch.object(mixin, "page") as mock_page:
|
|
mock_page.evaluate = AsyncMock(return_value = mock_remote_object)
|
|
|
|
result = await mixin.web_execute("window.test")
|
|
|
|
# Should convert the RemoteObject to a dict
|
|
assert result == {"key1": "value1", "key2": "value2"}
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_web_execute_remoteobject_with_none_deep_serialized_value(self) -> None:
|
|
"""Test web_execute with RemoteObject that has None deep_serialized_value."""
|
|
mixin = WebScrapingMixin()
|
|
|
|
# Mock RemoteObject with None deep_serialized_value
|
|
mock_remote_object = Mock()
|
|
mock_remote_object.deep_serialized_value = None
|
|
|
|
with patch.object(mixin, "page") as mock_page:
|
|
mock_page.evaluate = AsyncMock(return_value = mock_remote_object)
|
|
|
|
result = await mixin.web_execute("window.test")
|
|
|
|
# Should return the original RemoteObject
|
|
assert result is mock_remote_object
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_web_execute_remoteobject_with_non_list_serialized_data(self) -> None:
|
|
"""Test web_execute with RemoteObject that has non-list serialized data."""
|
|
mixin = WebScrapingMixin()
|
|
|
|
# Mock RemoteObject with non-list serialized data
|
|
mock_remote_object = Mock()
|
|
mock_remote_object.deep_serialized_value = Mock()
|
|
mock_remote_object.deep_serialized_value.value = {"direct": "dict"}
|
|
|
|
with patch.object(mixin, "page") as mock_page:
|
|
mock_page.evaluate = AsyncMock(return_value = mock_remote_object)
|
|
|
|
result = await mixin.web_execute("window.test")
|
|
|
|
# Should return the serialized data directly
|
|
assert result == {"direct": "dict"}
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_web_execute_remoteobject_with_none_serialized_data(self) -> None:
|
|
"""Test web_execute with RemoteObject that has None serialized data."""
|
|
mixin = WebScrapingMixin()
|
|
|
|
# Mock RemoteObject with None serialized data
|
|
mock_remote_object = Mock()
|
|
mock_remote_object.deep_serialized_value = Mock()
|
|
mock_remote_object.deep_serialized_value.value = None
|
|
|
|
with patch.object(mixin, "page") as mock_page:
|
|
mock_page.evaluate = AsyncMock(return_value = mock_remote_object)
|
|
|
|
result = await mixin.web_execute("window.test")
|
|
|
|
# Should return the original RemoteObject
|
|
assert result is mock_remote_object
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_web_execute_regular_result(self) -> None:
|
|
"""Test web_execute with regular result (no RemoteObject)."""
|
|
mixin = WebScrapingMixin()
|
|
|
|
# Mock regular result (no deep_serialized_value attribute)
|
|
mock_result = {"regular": "dict"}
|
|
|
|
with patch.object(mixin, "page") as mock_page:
|
|
mock_page.evaluate = AsyncMock(return_value = mock_result)
|
|
|
|
result = await mixin.web_execute("window.test")
|
|
|
|
# Should return the result unchanged
|
|
assert result == mock_result
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_web_execute_remoteobject_conversion_exception(self) -> None:
|
|
"""Test web_execute with RemoteObject that raises exception during conversion."""
|
|
mixin = WebScrapingMixin()
|
|
|
|
# Mock RemoteObject that will raise an exception when trying to convert to dict
|
|
mock_remote_object = Mock()
|
|
mock_deep_serialized = Mock()
|
|
|
|
# Create a list-like object that raises an exception when dict() is called on it
|
|
class ExceptionRaisingList(list[str]):
|
|
def __iter__(self) -> None: # type: ignore[override]
|
|
raise ValueError("Simulated conversion error")
|
|
|
|
mock_deep_serialized.value = ExceptionRaisingList([["key", "value"]]) # type: ignore[list-item]
|
|
mock_remote_object.deep_serialized_value = mock_deep_serialized
|
|
|
|
with patch.object(mixin, "page") as mock_page:
|
|
mock_page.evaluate = AsyncMock(return_value = mock_remote_object)
|
|
|
|
result = await mixin.web_execute("window.test")
|
|
|
|
# Should return the original RemoteObject when conversion fails
|
|
assert result is mock_remote_object
|