fix: resolve nodriver 0.47+ RemoteObject compatibility issues (#645)

## ℹ️ Description
*Provide a concise summary of the changes introduced in this pull
request.*

- Link to the related issue(s): #644
- Describe the motivation and context for this change.

This PR resolves compatibility issues with nodriver 0.47+ where
page.evaluate() returns RemoteObject instances that need special
handling for proper conversion to Python objects. The update introduced
breaking changes in how JavaScript evaluation results are returned,
causing TypeError: [RemoteObject] object is not subscriptable errors.

## 📋 Changes Summary

- Fixed TypeError: [RemoteObject] object is not subscriptable in
web_request() method
- Added comprehensive RemoteObject conversion logic with
_convert_remote_object_result()
- Added _convert_remote_object_dict() for recursive nested structure
conversion
- Fixed price field concatenation issue in MODIFY mode by explicit field
clearing
- Updated web_sleep() to accept integer milliseconds instead of float
seconds
- Updated German translations for new log messages
- Fixed linting issues (E711, E712) in test assertions

### ⚙️ Type of Change
Select the type(s) of change(s) included in this pull request:
- [x] 🐞 Bug fix (non-breaking change which fixes an issue)
- [ ]  New feature (adds new functionality without breaking existing
usage)
- [ ] 💥 Breaking change (changes that might break existing user setups,
scripts, or configurations)

##  Checklist
Before requesting a review, confirm the following:
- [x] I have reviewed my changes to ensure they meet the project's
standards.
- [x] I have tested my changes and ensured that all tests pass (pdm run
test).
- [x] I have formatted the code (pdm run format).
- [x] I have verified that linting passes (pdm run lint).
- [x] I have updated documentation where necessary.

By submitting this pull request, I confirm that you can use, modify,
copy, and redistribute this contribution, under the terms of your
choice.
This commit is contained in:
Jens
2025-10-18 19:38:51 +02:00
committed by GitHub
parent 34013cb869
commit 8aee313aba
4 changed files with 429 additions and 100 deletions

View File

@@ -803,9 +803,12 @@ class KleinanzeigenBot(WebScrapingMixin):
pass
if ad_cfg.price:
if mode == AdUpdateStrategy.MODIFY:
# we have to clear the input, otherwise input gets appended
await self.web_input(By.CSS_SELECTOR,
"input#post-ad-frontend-price, input#micro-frontend-price, input#pstad-price", "")
# Clear the price field first to prevent concatenation of old and new values
# This is needed because some input fields don't clear properly with just clear_input()
price_field = await self.web_find(By.CSS_SELECTOR, "input#post-ad-frontend-price, input#micro-frontend-price, input#pstad-price")
await price_field.clear_input()
await price_field.send_keys("") # Ensure field is completely empty
await self.web_sleep(500) # Brief pause to ensure clearing is complete
await self.web_input(By.CSS_SELECTOR, "input#post-ad-frontend-price, input#micro-frontend-price, input#pstad-price", str(ad_cfg.price))
#############################

View File

@@ -391,7 +391,7 @@ kleinanzeigen_bot/utils/web_scraping_mixin.py:
"4. Check browser binary permissions: %s": "4. Überprüfen Sie die Browser-Binärdatei-Berechtigungen: %s"
"4. Check if any antivirus or security software is blocking the connection": "4. Überprüfen Sie, ob Antiviren- oder Sicherheitssoftware die Verbindung blockiert"
web_execute:
_convert_remote_object_result:
"Failed to convert RemoteObject to dict: %s": "Fehler beim Konvertieren von RemoteObject zu dict: %s"
web_check:

View File

@@ -42,6 +42,9 @@ LOG:Final[loggers.Logger] = loggers.get_logger(__name__)
# see https://api.jquery.com/category/selectors/
METACHAR_ESCAPER:Final[dict[int, str]] = str.maketrans({ch: f"\\{ch}" for ch in '!"#$%&\'()*+,./:;<=>?@[\\]^`{|}~'})
# Constants for RemoteObject handling
_REMOTE_OBJECT_TYPE_VALUE_PAIR_SIZE:Final[int] = 2
def _is_admin() -> bool:
"""Check if the current process is running with admin/root privileges."""
@@ -574,22 +577,60 @@ class WebScrapingMixin:
# Handle nodriver 0.47+ RemoteObject behavior
# If result is a RemoteObject with deep_serialized_value, convert it to a dict
if hasattr(result, "deep_serialized_value"):
return self._convert_remote_object_result(result)
return result
def _convert_remote_object_result(self, result:Any) -> Any:
"""
Converts a RemoteObject result to a regular Python object.
Handles the deep_serialized_value conversion for nodriver 0.47+ compatibility.
"""
deep_serialized = getattr(result, "deep_serialized_value", None)
if deep_serialized is not None:
if deep_serialized is None:
return result
try:
# Convert the deep_serialized_value to a regular dict
serialized_data = getattr(deep_serialized, "value", None)
if serialized_data is not None:
if serialized_data is None:
return result
if isinstance(serialized_data, list):
# Convert list of [key, value] pairs to dict
return dict(serialized_data)
# Convert list of [key, value] pairs to dict, handling nested RemoteObjects
converted_dict = {}
for key, value in serialized_data:
converted_dict[key] = self._convert_remote_object_dict(value)
return converted_dict
if isinstance(serialized_data, dict):
# Handle nested RemoteObject structures like {'type': 'number', 'value': 200}
return self._convert_remote_object_dict(serialized_data)
return serialized_data
except (AttributeError, TypeError, ValueError) as e:
LOG.warning("Failed to convert RemoteObject to dict: %s", e)
# Return the original result if conversion fails
return result
return result
def _convert_remote_object_dict(self, data:Any) -> Any:
"""
Recursively converts RemoteObject dict structures to regular Python objects.
Handles structures like {'type': 'number', 'value': 200} or {'type': 'string', 'value': 'text'}.
"""
if isinstance(data, dict):
# Check if this is a RemoteObject value structure
if "type" in data and "value" in data and len(data) == _REMOTE_OBJECT_TYPE_VALUE_PAIR_SIZE:
return data["value"]
# Recursively convert nested dicts
return {key: self._convert_remote_object_dict(value) for key, value in data.items()}
if isinstance(data, list):
# Recursively convert lists
return [self._convert_remote_object_dict(item) for item in data]
# Return primitive values as-is
return data
async def web_find(self, selector_type:By, selector_value:str, *, parent:Element | None = None, timeout:int | float = 5) -> Element:
"""
@@ -724,10 +765,10 @@ class WebScrapingMixin:
await self.page.sleep(duration / 1_000)
async def web_request(self, url:str, method:str = "GET", valid_response_codes:int | Iterable[int] = 200,
headers:dict[str, str] | None = None) -> dict[str, Any]:
headers:dict[str, str] | None = None) -> Any:
method = method.upper()
LOG.debug(" -> HTTP %s [%s]...", method, url)
response = cast(dict[str, Any], await self.page.evaluate(f"""
response = await self.web_execute(f"""
fetch("{url}", {{
method: "{method}",
redirect: "follow",
@@ -743,7 +784,7 @@ class WebScrapingMixin:
content: responseText
}}
}}))
""", await_promise = True, return_by_value = True))
""")
if isinstance(valid_response_codes, int):
valid_response_codes = [valid_response_codes]
ensure(

View File

@@ -1,8 +1,12 @@
# SPDX-FileCopyrightText: © Jens Bergmann and contributors
# SPDX-License-Identifier: AGPL-3.0-or-later
# SPDX-ArtifactOfProjectHomePage: https://github.com/Second-Hand-Friends/kleinanzeigen-bot/
"""Unit tests for web_scraping_mixin.py RemoteObject handling."""
"""Unit tests for web_scraping_mixin.py RemoteObject handling.
Tests the conversion of nodriver RemoteObject results to regular Python objects.
"""
from typing import Any
from unittest.mock import AsyncMock, Mock, patch
import pytest
@@ -14,11 +18,114 @@ class TestWebExecuteRemoteObjectHandling:
"""Test web_execute method with nodriver 0.47+ RemoteObject behavior."""
@pytest.mark.asyncio
async def test_web_execute_remoteobject_with_deep_serialized_value(self) -> None:
"""Test web_execute with RemoteObject that has deep_serialized_value."""
async def test_web_execute_with_regular_result(self) -> None:
"""Test web_execute with regular (non-RemoteObject) result."""
mixin = WebScrapingMixin()
# Mock RemoteObject with deep_serialized_value
with patch.object(mixin, "page") as mock_page:
mock_page.evaluate = AsyncMock(return_value = "regular_result")
result = await mixin.web_execute("window.test")
assert result == "regular_result"
@pytest.mark.asyncio
async def test_web_execute_with_remoteobject_result(self) -> None:
"""Test web_execute with RemoteObject result."""
mixin = WebScrapingMixin()
# Mock RemoteObject
mock_remote_object = Mock()
mock_remote_object.deep_serialized_value = Mock()
mock_remote_object.deep_serialized_value.value = {"key": "value"}
with patch.object(mixin, "page") as mock_page:
mock_page.evaluate = AsyncMock(return_value = mock_remote_object)
result = await mixin.web_execute("window.test")
assert result == {"key": "value"}
@pytest.mark.asyncio
async def test_web_execute_remoteobject_with_nested_type_value_structures(self) -> None:
"""Test web_execute with RemoteObject containing nested type/value structures."""
mixin = WebScrapingMixin()
# Mock RemoteObject with nested type/value structures
mock_remote_object = Mock()
mock_remote_object.deep_serialized_value = Mock()
mock_remote_object.deep_serialized_value.value = [
["statusCode", {"type": "number", "value": 200}],
["content", {"type": "string", "value": "success"}]
]
with patch.object(mixin, "page") as mock_page:
mock_page.evaluate = AsyncMock(return_value = mock_remote_object)
result = await mixin.web_execute("window.test")
# Should convert nested type/value structures to their values
assert result == {"statusCode": 200, "content": "success"}
@pytest.mark.asyncio
async def test_web_execute_remoteobject_with_mixed_nested_structures(self) -> None:
"""Test web_execute with RemoteObject containing mixed nested structures."""
mixin = WebScrapingMixin()
# Mock RemoteObject with mixed nested structures
mock_remote_object = Mock()
mock_remote_object.deep_serialized_value = Mock()
mock_remote_object.deep_serialized_value.value = {
"simple": "value",
"nested": {"type": "number", "value": 42},
"list": [{"type": "string", "value": "item1"}, {"type": "string", "value": "item2"}]
}
with patch.object(mixin, "page") as mock_page:
mock_page.evaluate = AsyncMock(return_value = mock_remote_object)
result = await mixin.web_execute("window.test")
# Should convert nested structures while preserving simple values
expected = {
"simple": "value",
"nested": 42,
"list": ["item1", "item2"]
}
assert result == expected
class TestConvertRemoteObjectResult:
"""Test _convert_remote_object_result method for RemoteObject conversion."""
def test_convert_remote_object_result_with_none_deep_serialized_value(self) -> None:
"""Test _convert_remote_object_result when deep_serialized_value is None."""
mixin = WebScrapingMixin()
# Mock RemoteObject with None deep_serialized_value
mock_remote_object = Mock()
mock_remote_object.deep_serialized_value = None
result = mixin._convert_remote_object_result(mock_remote_object)
assert result == mock_remote_object
def test_convert_remote_object_result_with_none_serialized_data(self) -> None:
"""Test _convert_remote_object_result when serialized_data is None."""
mixin = WebScrapingMixin()
# Mock RemoteObject with None serialized_data
mock_remote_object = Mock()
mock_remote_object.deep_serialized_value = Mock()
mock_remote_object.deep_serialized_value.value = None
result = mixin._convert_remote_object_result(mock_remote_object)
assert result == mock_remote_object
def test_convert_remote_object_result_with_list_data(self) -> None:
"""Test _convert_remote_object_result with list data."""
mixin = WebScrapingMixin()
# Mock RemoteObject with list data
mock_remote_object = Mock()
mock_remote_object.deep_serialized_value = Mock()
mock_remote_object.deep_serialized_value.value = [
@@ -26,105 +133,283 @@ class TestWebExecuteRemoteObjectHandling:
["key2", "value2"]
]
# Mock the page evaluation to return our RemoteObject
with patch.object(mixin, "page") as mock_page:
mock_page.evaluate = AsyncMock(return_value = mock_remote_object)
result = await mixin.web_execute("window.test")
# Should convert the RemoteObject to a dict
result = mixin._convert_remote_object_result(mock_remote_object)
assert result == {"key1": "value1", "key2": "value2"}
@pytest.mark.asyncio
async def test_web_execute_remoteobject_with_none_deep_serialized_value(self) -> None:
"""Test web_execute with RemoteObject that has None deep_serialized_value."""
def test_convert_remote_object_result_with_dict_data(self) -> None:
"""Test _convert_remote_object_result with dict data."""
mixin = WebScrapingMixin()
# Mock RemoteObject with None deep_serialized_value
mock_remote_object = Mock()
mock_remote_object.deep_serialized_value = None
with patch.object(mixin, "page") as mock_page:
mock_page.evaluate = AsyncMock(return_value = mock_remote_object)
result = await mixin.web_execute("window.test")
# Should return the original RemoteObject
assert result is mock_remote_object
@pytest.mark.asyncio
async def test_web_execute_remoteobject_with_non_list_serialized_data(self) -> None:
"""Test web_execute with RemoteObject that has non-list serialized data."""
mixin = WebScrapingMixin()
# Mock RemoteObject with non-list serialized data
# Mock RemoteObject with dict data
mock_remote_object = Mock()
mock_remote_object.deep_serialized_value = Mock()
mock_remote_object.deep_serialized_value.value = {"direct": "dict"}
mock_remote_object.deep_serialized_value.value = {"key": "value"}
with patch.object(mixin, "page") as mock_page:
mock_page.evaluate = AsyncMock(return_value = mock_remote_object)
result = mixin._convert_remote_object_result(mock_remote_object)
assert result == {"key": "value"}
result = await mixin.web_execute("window.test")
# Should return the serialized data directly
assert result == {"direct": "dict"}
@pytest.mark.asyncio
async def test_web_execute_remoteobject_with_none_serialized_data(self) -> None:
"""Test web_execute with RemoteObject that has None serialized data."""
def test_convert_remote_object_result_with_conversion_error(self) -> None:
"""Test _convert_remote_object_result when conversion raises an exception."""
mixin = WebScrapingMixin()
# Mock RemoteObject with None serialized data
# Mock RemoteObject that will raise an exception during conversion
mock_remote_object = Mock()
mock_remote_object.deep_serialized_value = Mock()
mock_remote_object.deep_serialized_value.value = None
mock_remote_object.deep_serialized_value.value = "invalid_data"
with patch.object(mixin, "page") as mock_page:
mock_page.evaluate = AsyncMock(return_value = mock_remote_object)
# Mock the _convert_remote_object_dict to raise an exception
with patch.object(mixin, "_convert_remote_object_dict", side_effect = ValueError("Test error")):
result = mixin._convert_remote_object_result(mock_remote_object)
# When conversion fails, it should return the original value
assert result == "invalid_data"
result = await mixin.web_execute("window.test")
# Should return the original RemoteObject
assert result is mock_remote_object
class TestConvertRemoteObjectDict:
"""Test _convert_remote_object_dict method for nested RemoteObject conversion."""
@pytest.mark.asyncio
async def test_web_execute_regular_result(self) -> None:
"""Test web_execute with regular result (no RemoteObject)."""
def test_convert_remote_object_dict_with_type_value_pair(self) -> None:
"""Test conversion of type/value pair structures."""
mixin = WebScrapingMixin()
# Mock regular result (no deep_serialized_value attribute)
mock_result = {"regular": "dict"}
# Test type/value pair
data = {"type": "number", "value": 200}
result = mixin._convert_remote_object_dict(data)
assert result == 200
with patch.object(mixin, "page") as mock_page:
mock_page.evaluate = AsyncMock(return_value = mock_result)
# Test string type/value pair
data = {"type": "string", "value": "hello"}
result = mixin._convert_remote_object_dict(data)
assert result == "hello"
result = await mixin.web_execute("window.test")
# Should return the result unchanged
assert result == mock_result
@pytest.mark.asyncio
async def test_web_execute_remoteobject_conversion_exception(self) -> None:
"""Test web_execute with RemoteObject that raises exception during conversion."""
def test_convert_remote_object_dict_with_regular_dict(self) -> None:
"""Test conversion of regular dict structures."""
mixin = WebScrapingMixin()
# Mock RemoteObject that will raise an exception when trying to convert to dict
# Test regular dict (not type/value pair)
data = {"key1": "value1", "key2": "value2"}
result = mixin._convert_remote_object_dict(data)
assert result == {"key1": "value1", "key2": "value2"}
def test_convert_remote_object_dict_with_nested_structures(self) -> None:
"""Test conversion of nested dict structures."""
mixin = WebScrapingMixin()
# Test nested structures
data = {
"simple": "value",
"nested": {"type": "number", "value": 42},
"list": [{"type": "string", "value": "item1"}, {"type": "string", "value": "item2"}]
}
result = mixin._convert_remote_object_dict(data)
expected = {
"simple": "value",
"nested": 42,
"list": ["item1", "item2"]
}
assert result == expected
def test_convert_remote_object_dict_with_list(self) -> None:
"""Test conversion of list structures."""
mixin = WebScrapingMixin()
# Test list with type/value pairs
data = [{"type": "number", "value": 1}, {"type": "string", "value": "test"}]
result = mixin._convert_remote_object_dict(data)
assert result == [1, "test"]
def test_convert_remote_object_dict_with_primitive_values(self) -> None:
"""Test conversion with primitive values."""
mixin = WebScrapingMixin()
# Test primitive values
assert mixin._convert_remote_object_dict("string") == "string"
assert mixin._convert_remote_object_dict(42) == 42
assert mixin._convert_remote_object_dict(True) is True
assert mixin._convert_remote_object_dict(None) is None
def test_convert_remote_object_dict_with_complex_nested_structures(self) -> None:
"""Test conversion with complex nested structures."""
mixin = WebScrapingMixin()
# Test complex nested structures
data = {
"response": {
"status": {"type": "number", "value": 200},
"data": [
{"type": "string", "value": "item1"},
{"type": "string", "value": "item2"}
],
"metadata": {
"count": {"type": "number", "value": 2},
"type": {"type": "string", "value": "list"}
}
}
}
result = mixin._convert_remote_object_dict(data)
expected = {
"response": {
"status": 200,
"data": ["item1", "item2"],
"metadata": {
"count": 2,
"type": "list"
}
}
}
assert result == expected
class TestWebRequestRemoteObjectHandling:
"""Test web_request method with nodriver 0.47+ RemoteObject behavior."""
@pytest.mark.asyncio
async def test_web_request_with_remoteobject_result(self) -> None:
"""Test web_request with RemoteObject result to catch subscriptability issues."""
mixin = WebScrapingMixin()
# Mock RemoteObject that simulates the exact structure returned by web_request
mock_remote_object = Mock()
mock_deep_serialized = Mock()
mock_remote_object.deep_serialized_value = Mock()
mock_remote_object.deep_serialized_value.value = {
"statusCode": 200,
"statusMessage": "OK",
"headers": {"content-type": "application/json"},
"content": '{"success": true}'
}
# Create a list-like object that raises an exception when dict() is called on it
class ExceptionRaisingList(list[str]):
def __iter__(self) -> None: # type: ignore[override]
raise ValueError("Simulated conversion error")
with patch.object(mixin, "web_execute") as mock_web_execute:
# Mock web_execute to return the converted result (simulating our fix)
mock_web_execute.return_value = {
"statusCode": 200,
"statusMessage": "OK",
"headers": {"content-type": "application/json"},
"content": '{"success": true}'
}
mock_deep_serialized.value = ExceptionRaisingList([["key", "value"]]) # type: ignore[list-item]
mock_remote_object.deep_serialized_value = mock_deep_serialized
result = await mixin.web_request("https://example.com/api")
with patch.object(mixin, "page") as mock_page:
mock_page.evaluate = AsyncMock(return_value = mock_remote_object)
# Verify the result is properly converted and subscriptable
assert result["statusCode"] == 200
assert result["statusMessage"] == "OK"
assert result["headers"]["content-type"] == "application/json"
assert result["content"] == '{"success": true}'
result = await mixin.web_execute("window.test")
@pytest.mark.asyncio
async def test_web_request_with_remoteobject_error_response(self) -> None:
"""Test web_request with RemoteObject error response."""
mixin = WebScrapingMixin()
# Should return the original RemoteObject when conversion fails
assert result is mock_remote_object
# Mock RemoteObject for error response
mock_remote_object = Mock()
mock_remote_object.deep_serialized_value = Mock()
mock_remote_object.deep_serialized_value.value = {
"statusCode": 404,
"statusMessage": "Not Found",
"headers": {"content-type": "text/html"},
"content": "<html>Not Found</html>"
}
with patch.object(mixin, "web_execute") as mock_web_execute:
# Mock web_execute to return the converted result (simulating our fix)
mock_web_execute.return_value = {
"statusCode": 404,
"statusMessage": "Not Found",
"headers": {"content-type": "text/html"},
"content": "<html>Not Found</html>"
}
# This should raise an exception due to invalid status code
with pytest.raises(Exception, match = "Invalid response"):
await mixin.web_request("https://example.com/api", valid_response_codes = [200])
@pytest.mark.asyncio
async def test_web_request_with_nested_remoteobject_structures(self) -> None:
"""Test web_request with complex nested RemoteObject structures."""
mixin = WebScrapingMixin()
# Mock RemoteObject with nested type/value structures
mock_remote_object = Mock()
mock_remote_object.deep_serialized_value = Mock()
mock_remote_object.deep_serialized_value.value = {
"statusCode": {"type": "number", "value": 200},
"statusMessage": {"type": "string", "value": "OK"},
"headers": {
"content-type": {"type": "string", "value": "application/json"}
},
"content": {"type": "string", "value": '{"data": "test"}'}
}
with patch.object(mixin, "web_execute") as mock_web_execute:
# Mock web_execute to return the converted result (simulating our fix)
mock_web_execute.return_value = {
"statusCode": 200,
"statusMessage": "OK",
"headers": {"content-type": "application/json"},
"content": '{"data": "test"}'
}
result = await mixin.web_request("https://example.com/api")
# Verify nested structures are properly converted
assert result["statusCode"] == 200
assert result["statusMessage"] == "OK"
assert result["headers"]["content-type"] == "application/json"
assert result["content"] == '{"data": "test"}'
class TestWebRequestRemoteObjectRegression:
"""Test web_request method to catch future RemoteObject regression issues."""
@pytest.mark.asyncio
async def test_web_request_without_remoteobject_conversion_fails(self) -> None:
"""Test that web_request fails without RemoteObject conversion (regression test)."""
mixin = WebScrapingMixin()
# Mock RemoteObject that would cause the original error
mock_remote_object = Mock()
mock_remote_object.deep_serialized_value = Mock()
mock_remote_object.deep_serialized_value.value = {
"statusCode": 200,
"statusMessage": "OK",
"headers": {"content-type": "application/json"},
"content": '{"success": true}'
}
# Mock web_execute to return the raw RemoteObject (simulating the bug)
with patch.object(mixin, "web_execute") as mock_web_execute:
mock_web_execute.return_value = mock_remote_object
# This should fail with the original error if our fix is removed
with pytest.raises(TypeError, match = "object is not subscriptable"):
await mixin.web_request("https://example.com/api")
@pytest.mark.asyncio
async def test_web_request_with_remoteobject_conversion_succeeds(self) -> None:
"""Test that web_request succeeds with RemoteObject conversion (our fix)."""
mixin = WebScrapingMixin()
# Mock RemoteObject
mock_remote_object = Mock()
mock_remote_object.deep_serialized_value = Mock()
mock_remote_object.deep_serialized_value.value = {
"statusCode": 200,
"statusMessage": "OK",
"headers": {"content-type": "application/json"},
"content": '{"success": true}'
}
# Mock web_execute to simulate our conversion logic
async def mock_web_execute_with_conversion(script:str) -> Any:
# Simulate the conversion that happens in our fix
return mock_remote_object.deep_serialized_value.value
with patch.object(mixin, "web_execute", side_effect = mock_web_execute_with_conversion):
result = await mixin.web_request("https://example.com/api")
# Verify the result works correctly
assert result["statusCode"] == 200
assert result["statusMessage"] == "OK"
assert result["headers"]["content-type"] == "application/json"
assert result["content"] == '{"success": true}'