Files
kleinanzeigen-bot/tests/unit/test_web_scraping_mixin_remoteobject.py
Jens 36ca178574 feat: upgrade nodriver from 0.39 to 0.47 (#635)
## ℹ️ Description
Upgrade nodriver dependency from pinned version 0.39.0 to latest 0.47.0
to resolve browser startup issues and JavaScript evaluation problems
that affected versions 0.40-0.44.

- Link to the related issue(s): Resolves nodriver compatibility issues
- This upgrade addresses browser startup problems and window.BelenConf
evaluation failures that were blocking the use of newer nodriver
versions.

## 📋 Changes Summary

- Updated nodriver dependency from pinned 0.39.0 to >=0.47.0 in
pyproject.toml
- Fixed RemoteObject handling in web_execute method for nodriver 0.47
compatibility
- Added comprehensive BelenConf test fixture with real production data
structure
- Added integration test to validate window.BelenConf evaluation works
correctly
- Added German translation for new error message
- Replaced real user data with privacy-safe dummy data in test fixtures

### 🔧 Type Safety Improvements

**Added explicit `str()` conversions to resolve type inference issues:**

The comprehensive BelenConf test fixture contains deeply nested data
structures that caused pyright's type checker to infer complex
dictionary types throughout the codebase. To ensure type safety and
prevent runtime errors, I added explicit `str()` conversions in key
locations:

- **CSRF tokens**: `str(csrf_token)` - Ensures CSRF tokens are treated
as strings
- **Special attributes**: `str(special_attribute_value)` - Converts
special attribute values to strings
- **DOM attributes**: `str(special_attr_elem.attrs.id)` - Ensures
element IDs are strings
- **URL handling**: `str(current_img_url)` and `str(href_attributes)` -
Converts URLs and href attributes to strings
- **Price values**: `str(ad_cfg.price)` - Ensures price values are
strings

These conversions are defensive programming measures that ensure
backward compatibility and prevent type-related runtime errors, even if
the underlying data structures change in the future.

### ⚙️ Type of Change
- [x]  New feature (adds new functionality without breaking existing
usage)
- [ ] 🐞 Bug fix (non-breaking change which fixes an issue)
- [ ] 💥 Breaking change (changes that might break existing user setups,
scripts, or configurations)

##  Checklist
Before requesting a review, confirm the following:
- [x] I have reviewed my changes to ensure they meet the project's
standards.
- [x] I have tested my changes and ensured that all tests pass (`pdm run
test`).
- [x] I have formatted the code (`pdm run format`).
- [x] I have verified that linting passes (`pdm run lint`).
- [x] I have updated documentation where necessary.

By submitting this pull request, I confirm that you can use, modify,
copy, and redistribute this contribution, under the terms of your
choice.
2025-10-12 21:22:46 +02:00

135 lines
5.3 KiB
Python

# SPDX-FileCopyrightText: © Jens Bergmann and contributors
# SPDX-License-Identifier: AGPL-3.0-or-later
# SPDX-ArtifactOfProjectHomePage: https://github.com/Second-Hand-Friends/kleinanzeigen-bot/
"""Unit tests for web_scraping_mixin.py RemoteObject handling.
Copyright (c) 2024, kleinanzeigen-bot contributors.
All rights reserved.
"""
from unittest.mock import AsyncMock, Mock, patch
import pytest
from kleinanzeigen_bot.utils.web_scraping_mixin import WebScrapingMixin
class TestWebExecuteRemoteObjectHandling:
"""Test web_execute method with nodriver 0.47+ RemoteObject behavior."""
@pytest.mark.asyncio
async def test_web_execute_remoteobject_with_deep_serialized_value(self) -> None:
"""Test web_execute with RemoteObject that has deep_serialized_value."""
mixin = WebScrapingMixin()
# Mock RemoteObject with deep_serialized_value
mock_remote_object = Mock()
mock_remote_object.deep_serialized_value = Mock()
mock_remote_object.deep_serialized_value.value = [
["key1", "value1"],
["key2", "value2"]
]
# Mock the page evaluation to return our RemoteObject
with patch.object(mixin, "page") as mock_page:
mock_page.evaluate = AsyncMock(return_value = mock_remote_object)
result = await mixin.web_execute("window.test")
# Should convert the RemoteObject to a dict
assert result == {"key1": "value1", "key2": "value2"}
@pytest.mark.asyncio
async def test_web_execute_remoteobject_with_none_deep_serialized_value(self) -> None:
"""Test web_execute with RemoteObject that has None deep_serialized_value."""
mixin = WebScrapingMixin()
# Mock RemoteObject with None deep_serialized_value
mock_remote_object = Mock()
mock_remote_object.deep_serialized_value = None
with patch.object(mixin, "page") as mock_page:
mock_page.evaluate = AsyncMock(return_value = mock_remote_object)
result = await mixin.web_execute("window.test")
# Should return the original RemoteObject
assert result is mock_remote_object
@pytest.mark.asyncio
async def test_web_execute_remoteobject_with_non_list_serialized_data(self) -> None:
"""Test web_execute with RemoteObject that has non-list serialized data."""
mixin = WebScrapingMixin()
# Mock RemoteObject with non-list serialized data
mock_remote_object = Mock()
mock_remote_object.deep_serialized_value = Mock()
mock_remote_object.deep_serialized_value.value = {"direct": "dict"}
with patch.object(mixin, "page") as mock_page:
mock_page.evaluate = AsyncMock(return_value = mock_remote_object)
result = await mixin.web_execute("window.test")
# Should return the serialized data directly
assert result == {"direct": "dict"}
@pytest.mark.asyncio
async def test_web_execute_remoteobject_with_none_serialized_data(self) -> None:
"""Test web_execute with RemoteObject that has None serialized data."""
mixin = WebScrapingMixin()
# Mock RemoteObject with None serialized data
mock_remote_object = Mock()
mock_remote_object.deep_serialized_value = Mock()
mock_remote_object.deep_serialized_value.value = None
with patch.object(mixin, "page") as mock_page:
mock_page.evaluate = AsyncMock(return_value = mock_remote_object)
result = await mixin.web_execute("window.test")
# Should return the original RemoteObject
assert result is mock_remote_object
@pytest.mark.asyncio
async def test_web_execute_regular_result(self) -> None:
"""Test web_execute with regular result (no RemoteObject)."""
mixin = WebScrapingMixin()
# Mock regular result (no deep_serialized_value attribute)
mock_result = {"regular": "dict"}
with patch.object(mixin, "page") as mock_page:
mock_page.evaluate = AsyncMock(return_value = mock_result)
result = await mixin.web_execute("window.test")
# Should return the result unchanged
assert result == mock_result
@pytest.mark.asyncio
async def test_web_execute_remoteobject_conversion_exception(self) -> None:
"""Test web_execute with RemoteObject that raises exception during conversion."""
mixin = WebScrapingMixin()
# Mock RemoteObject that will raise an exception when trying to convert to dict
mock_remote_object = Mock()
mock_deep_serialized = Mock()
# Create a list-like object that raises an exception when dict() is called on it
class ExceptionRaisingList(list[str]):
def __iter__(self) -> None: # type: ignore[override]
raise ValueError("Simulated conversion error")
mock_deep_serialized.value = ExceptionRaisingList([["key", "value"]]) # type: ignore[list-item]
mock_remote_object.deep_serialized_value = mock_deep_serialized
with patch.object(mixin, "page") as mock_page:
mock_page.evaluate = AsyncMock(return_value = mock_remote_object)
result = await mixin.web_execute("window.test")
# Should return the original RemoteObject when conversion fails
assert result is mock_remote_object