mirror of
https://github.com/Second-Hand-Friends/kleinanzeigen-bot.git
synced 2026-03-12 02:31:45 +01:00
fix: resolve nodriver 0.47+ RemoteObject compatibility issues (#645)
## ℹ️ Description *Provide a concise summary of the changes introduced in this pull request.* - Link to the related issue(s): #644 - Describe the motivation and context for this change. This PR resolves compatibility issues with nodriver 0.47+ where page.evaluate() returns RemoteObject instances that need special handling for proper conversion to Python objects. The update introduced breaking changes in how JavaScript evaluation results are returned, causing TypeError: [RemoteObject] object is not subscriptable errors. ## 📋 Changes Summary - Fixed TypeError: [RemoteObject] object is not subscriptable in web_request() method - Added comprehensive RemoteObject conversion logic with _convert_remote_object_result() - Added _convert_remote_object_dict() for recursive nested structure conversion - Fixed price field concatenation issue in MODIFY mode by explicit field clearing - Updated web_sleep() to accept integer milliseconds instead of float seconds - Updated German translations for new log messages - Fixed linting issues (E711, E712) in test assertions ### ⚙️ Type of Change Select the type(s) of change(s) included in this pull request: - [x] 🐞 Bug fix (non-breaking change which fixes an issue) - [ ] ✨ New feature (adds new functionality without breaking existing usage) - [ ] 💥 Breaking change (changes that might break existing user setups, scripts, or configurations) ## ✅ Checklist Before requesting a review, confirm the following: - [x] I have reviewed my changes to ensure they meet the project's standards. - [x] I have tested my changes and ensured that all tests pass (pdm run test). - [x] I have formatted the code (pdm run format). - [x] I have verified that linting passes (pdm run lint). - [x] I have updated documentation where necessary. By submitting this pull request, I confirm that you can use, modify, copy, and redistribute this contribution, under the terms of your choice.
This commit is contained in:
@@ -803,9 +803,12 @@ class KleinanzeigenBot(WebScrapingMixin):
|
|||||||
pass
|
pass
|
||||||
if ad_cfg.price:
|
if ad_cfg.price:
|
||||||
if mode == AdUpdateStrategy.MODIFY:
|
if mode == AdUpdateStrategy.MODIFY:
|
||||||
# we have to clear the input, otherwise input gets appended
|
# Clear the price field first to prevent concatenation of old and new values
|
||||||
await self.web_input(By.CSS_SELECTOR,
|
# This is needed because some input fields don't clear properly with just clear_input()
|
||||||
"input#post-ad-frontend-price, input#micro-frontend-price, input#pstad-price", "")
|
price_field = await self.web_find(By.CSS_SELECTOR, "input#post-ad-frontend-price, input#micro-frontend-price, input#pstad-price")
|
||||||
|
await price_field.clear_input()
|
||||||
|
await price_field.send_keys("") # Ensure field is completely empty
|
||||||
|
await self.web_sleep(500) # Brief pause to ensure clearing is complete
|
||||||
await self.web_input(By.CSS_SELECTOR, "input#post-ad-frontend-price, input#micro-frontend-price, input#pstad-price", str(ad_cfg.price))
|
await self.web_input(By.CSS_SELECTOR, "input#post-ad-frontend-price, input#micro-frontend-price, input#pstad-price", str(ad_cfg.price))
|
||||||
|
|
||||||
#############################
|
#############################
|
||||||
|
|||||||
@@ -391,7 +391,7 @@ kleinanzeigen_bot/utils/web_scraping_mixin.py:
|
|||||||
"4. Check browser binary permissions: %s": "4. Überprüfen Sie die Browser-Binärdatei-Berechtigungen: %s"
|
"4. Check browser binary permissions: %s": "4. Überprüfen Sie die Browser-Binärdatei-Berechtigungen: %s"
|
||||||
"4. Check if any antivirus or security software is blocking the connection": "4. Überprüfen Sie, ob Antiviren- oder Sicherheitssoftware die Verbindung blockiert"
|
"4. Check if any antivirus or security software is blocking the connection": "4. Überprüfen Sie, ob Antiviren- oder Sicherheitssoftware die Verbindung blockiert"
|
||||||
|
|
||||||
web_execute:
|
_convert_remote_object_result:
|
||||||
"Failed to convert RemoteObject to dict: %s": "Fehler beim Konvertieren von RemoteObject zu dict: %s"
|
"Failed to convert RemoteObject to dict: %s": "Fehler beim Konvertieren von RemoteObject zu dict: %s"
|
||||||
|
|
||||||
web_check:
|
web_check:
|
||||||
|
|||||||
@@ -42,6 +42,9 @@ LOG:Final[loggers.Logger] = loggers.get_logger(__name__)
|
|||||||
# see https://api.jquery.com/category/selectors/
|
# see https://api.jquery.com/category/selectors/
|
||||||
METACHAR_ESCAPER:Final[dict[int, str]] = str.maketrans({ch: f"\\{ch}" for ch in '!"#$%&\'()*+,./:;<=>?@[\\]^`{|}~'})
|
METACHAR_ESCAPER:Final[dict[int, str]] = str.maketrans({ch: f"\\{ch}" for ch in '!"#$%&\'()*+,./:;<=>?@[\\]^`{|}~'})
|
||||||
|
|
||||||
|
# Constants for RemoteObject handling
|
||||||
|
_REMOTE_OBJECT_TYPE_VALUE_PAIR_SIZE:Final[int] = 2
|
||||||
|
|
||||||
|
|
||||||
def _is_admin() -> bool:
|
def _is_admin() -> bool:
|
||||||
"""Check if the current process is running with admin/root privileges."""
|
"""Check if the current process is running with admin/root privileges."""
|
||||||
@@ -574,22 +577,60 @@ class WebScrapingMixin:
|
|||||||
# Handle nodriver 0.47+ RemoteObject behavior
|
# Handle nodriver 0.47+ RemoteObject behavior
|
||||||
# If result is a RemoteObject with deep_serialized_value, convert it to a dict
|
# If result is a RemoteObject with deep_serialized_value, convert it to a dict
|
||||||
if hasattr(result, "deep_serialized_value"):
|
if hasattr(result, "deep_serialized_value"):
|
||||||
|
return self._convert_remote_object_result(result)
|
||||||
|
|
||||||
|
return result
|
||||||
|
|
||||||
|
def _convert_remote_object_result(self, result:Any) -> Any:
|
||||||
|
"""
|
||||||
|
Converts a RemoteObject result to a regular Python object.
|
||||||
|
|
||||||
|
Handles the deep_serialized_value conversion for nodriver 0.47+ compatibility.
|
||||||
|
"""
|
||||||
deep_serialized = getattr(result, "deep_serialized_value", None)
|
deep_serialized = getattr(result, "deep_serialized_value", None)
|
||||||
if deep_serialized is not None:
|
if deep_serialized is None:
|
||||||
|
return result
|
||||||
|
|
||||||
try:
|
try:
|
||||||
# Convert the deep_serialized_value to a regular dict
|
# Convert the deep_serialized_value to a regular dict
|
||||||
serialized_data = getattr(deep_serialized, "value", None)
|
serialized_data = getattr(deep_serialized, "value", None)
|
||||||
if serialized_data is not None:
|
if serialized_data is None:
|
||||||
|
return result
|
||||||
|
|
||||||
if isinstance(serialized_data, list):
|
if isinstance(serialized_data, list):
|
||||||
# Convert list of [key, value] pairs to dict
|
# Convert list of [key, value] pairs to dict, handling nested RemoteObjects
|
||||||
return dict(serialized_data)
|
converted_dict = {}
|
||||||
|
for key, value in serialized_data:
|
||||||
|
converted_dict[key] = self._convert_remote_object_dict(value)
|
||||||
|
return converted_dict
|
||||||
|
|
||||||
|
if isinstance(serialized_data, dict):
|
||||||
|
# Handle nested RemoteObject structures like {'type': 'number', 'value': 200}
|
||||||
|
return self._convert_remote_object_dict(serialized_data)
|
||||||
|
|
||||||
return serialized_data
|
return serialized_data
|
||||||
except (AttributeError, TypeError, ValueError) as e:
|
except (AttributeError, TypeError, ValueError) as e:
|
||||||
LOG.warning("Failed to convert RemoteObject to dict: %s", e)
|
LOG.warning("Failed to convert RemoteObject to dict: %s", e)
|
||||||
# Return the original result if conversion fails
|
# Return the original result if conversion fails
|
||||||
return result
|
return result
|
||||||
|
|
||||||
return result
|
def _convert_remote_object_dict(self, data:Any) -> Any:
|
||||||
|
"""
|
||||||
|
Recursively converts RemoteObject dict structures to regular Python objects.
|
||||||
|
|
||||||
|
Handles structures like {'type': 'number', 'value': 200} or {'type': 'string', 'value': 'text'}.
|
||||||
|
"""
|
||||||
|
if isinstance(data, dict):
|
||||||
|
# Check if this is a RemoteObject value structure
|
||||||
|
if "type" in data and "value" in data and len(data) == _REMOTE_OBJECT_TYPE_VALUE_PAIR_SIZE:
|
||||||
|
return data["value"]
|
||||||
|
# Recursively convert nested dicts
|
||||||
|
return {key: self._convert_remote_object_dict(value) for key, value in data.items()}
|
||||||
|
if isinstance(data, list):
|
||||||
|
# Recursively convert lists
|
||||||
|
return [self._convert_remote_object_dict(item) for item in data]
|
||||||
|
# Return primitive values as-is
|
||||||
|
return data
|
||||||
|
|
||||||
async def web_find(self, selector_type:By, selector_value:str, *, parent:Element | None = None, timeout:int | float = 5) -> Element:
|
async def web_find(self, selector_type:By, selector_value:str, *, parent:Element | None = None, timeout:int | float = 5) -> Element:
|
||||||
"""
|
"""
|
||||||
@@ -724,10 +765,10 @@ class WebScrapingMixin:
|
|||||||
await self.page.sleep(duration / 1_000)
|
await self.page.sleep(duration / 1_000)
|
||||||
|
|
||||||
async def web_request(self, url:str, method:str = "GET", valid_response_codes:int | Iterable[int] = 200,
|
async def web_request(self, url:str, method:str = "GET", valid_response_codes:int | Iterable[int] = 200,
|
||||||
headers:dict[str, str] | None = None) -> dict[str, Any]:
|
headers:dict[str, str] | None = None) -> Any:
|
||||||
method = method.upper()
|
method = method.upper()
|
||||||
LOG.debug(" -> HTTP %s [%s]...", method, url)
|
LOG.debug(" -> HTTP %s [%s]...", method, url)
|
||||||
response = cast(dict[str, Any], await self.page.evaluate(f"""
|
response = await self.web_execute(f"""
|
||||||
fetch("{url}", {{
|
fetch("{url}", {{
|
||||||
method: "{method}",
|
method: "{method}",
|
||||||
redirect: "follow",
|
redirect: "follow",
|
||||||
@@ -743,7 +784,7 @@ class WebScrapingMixin:
|
|||||||
content: responseText
|
content: responseText
|
||||||
}}
|
}}
|
||||||
}}))
|
}}))
|
||||||
""", await_promise = True, return_by_value = True))
|
""")
|
||||||
if isinstance(valid_response_codes, int):
|
if isinstance(valid_response_codes, int):
|
||||||
valid_response_codes = [valid_response_codes]
|
valid_response_codes = [valid_response_codes]
|
||||||
ensure(
|
ensure(
|
||||||
|
|||||||
@@ -1,8 +1,12 @@
|
|||||||
# SPDX-FileCopyrightText: © Jens Bergmann and contributors
|
# SPDX-FileCopyrightText: © Jens Bergmann and contributors
|
||||||
# SPDX-License-Identifier: AGPL-3.0-or-later
|
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||||||
# SPDX-ArtifactOfProjectHomePage: https://github.com/Second-Hand-Friends/kleinanzeigen-bot/
|
# SPDX-ArtifactOfProjectHomePage: https://github.com/Second-Hand-Friends/kleinanzeigen-bot/
|
||||||
"""Unit tests for web_scraping_mixin.py RemoteObject handling."""
|
"""Unit tests for web_scraping_mixin.py RemoteObject handling.
|
||||||
|
|
||||||
|
Tests the conversion of nodriver RemoteObject results to regular Python objects.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from typing import Any
|
||||||
from unittest.mock import AsyncMock, Mock, patch
|
from unittest.mock import AsyncMock, Mock, patch
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
@@ -14,11 +18,114 @@ class TestWebExecuteRemoteObjectHandling:
|
|||||||
"""Test web_execute method with nodriver 0.47+ RemoteObject behavior."""
|
"""Test web_execute method with nodriver 0.47+ RemoteObject behavior."""
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_web_execute_remoteobject_with_deep_serialized_value(self) -> None:
|
async def test_web_execute_with_regular_result(self) -> None:
|
||||||
"""Test web_execute with RemoteObject that has deep_serialized_value."""
|
"""Test web_execute with regular (non-RemoteObject) result."""
|
||||||
mixin = WebScrapingMixin()
|
mixin = WebScrapingMixin()
|
||||||
|
|
||||||
# Mock RemoteObject with deep_serialized_value
|
with patch.object(mixin, "page") as mock_page:
|
||||||
|
mock_page.evaluate = AsyncMock(return_value = "regular_result")
|
||||||
|
|
||||||
|
result = await mixin.web_execute("window.test")
|
||||||
|
|
||||||
|
assert result == "regular_result"
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_web_execute_with_remoteobject_result(self) -> None:
|
||||||
|
"""Test web_execute with RemoteObject result."""
|
||||||
|
mixin = WebScrapingMixin()
|
||||||
|
|
||||||
|
# Mock RemoteObject
|
||||||
|
mock_remote_object = Mock()
|
||||||
|
mock_remote_object.deep_serialized_value = Mock()
|
||||||
|
mock_remote_object.deep_serialized_value.value = {"key": "value"}
|
||||||
|
|
||||||
|
with patch.object(mixin, "page") as mock_page:
|
||||||
|
mock_page.evaluate = AsyncMock(return_value = mock_remote_object)
|
||||||
|
|
||||||
|
result = await mixin.web_execute("window.test")
|
||||||
|
|
||||||
|
assert result == {"key": "value"}
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_web_execute_remoteobject_with_nested_type_value_structures(self) -> None:
|
||||||
|
"""Test web_execute with RemoteObject containing nested type/value structures."""
|
||||||
|
mixin = WebScrapingMixin()
|
||||||
|
|
||||||
|
# Mock RemoteObject with nested type/value structures
|
||||||
|
mock_remote_object = Mock()
|
||||||
|
mock_remote_object.deep_serialized_value = Mock()
|
||||||
|
mock_remote_object.deep_serialized_value.value = [
|
||||||
|
["statusCode", {"type": "number", "value": 200}],
|
||||||
|
["content", {"type": "string", "value": "success"}]
|
||||||
|
]
|
||||||
|
|
||||||
|
with patch.object(mixin, "page") as mock_page:
|
||||||
|
mock_page.evaluate = AsyncMock(return_value = mock_remote_object)
|
||||||
|
|
||||||
|
result = await mixin.web_execute("window.test")
|
||||||
|
|
||||||
|
# Should convert nested type/value structures to their values
|
||||||
|
assert result == {"statusCode": 200, "content": "success"}
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_web_execute_remoteobject_with_mixed_nested_structures(self) -> None:
|
||||||
|
"""Test web_execute with RemoteObject containing mixed nested structures."""
|
||||||
|
mixin = WebScrapingMixin()
|
||||||
|
|
||||||
|
# Mock RemoteObject with mixed nested structures
|
||||||
|
mock_remote_object = Mock()
|
||||||
|
mock_remote_object.deep_serialized_value = Mock()
|
||||||
|
mock_remote_object.deep_serialized_value.value = {
|
||||||
|
"simple": "value",
|
||||||
|
"nested": {"type": "number", "value": 42},
|
||||||
|
"list": [{"type": "string", "value": "item1"}, {"type": "string", "value": "item2"}]
|
||||||
|
}
|
||||||
|
|
||||||
|
with patch.object(mixin, "page") as mock_page:
|
||||||
|
mock_page.evaluate = AsyncMock(return_value = mock_remote_object)
|
||||||
|
|
||||||
|
result = await mixin.web_execute("window.test")
|
||||||
|
|
||||||
|
# Should convert nested structures while preserving simple values
|
||||||
|
expected = {
|
||||||
|
"simple": "value",
|
||||||
|
"nested": 42,
|
||||||
|
"list": ["item1", "item2"]
|
||||||
|
}
|
||||||
|
assert result == expected
|
||||||
|
|
||||||
|
|
||||||
|
class TestConvertRemoteObjectResult:
|
||||||
|
"""Test _convert_remote_object_result method for RemoteObject conversion."""
|
||||||
|
|
||||||
|
def test_convert_remote_object_result_with_none_deep_serialized_value(self) -> None:
|
||||||
|
"""Test _convert_remote_object_result when deep_serialized_value is None."""
|
||||||
|
mixin = WebScrapingMixin()
|
||||||
|
|
||||||
|
# Mock RemoteObject with None deep_serialized_value
|
||||||
|
mock_remote_object = Mock()
|
||||||
|
mock_remote_object.deep_serialized_value = None
|
||||||
|
|
||||||
|
result = mixin._convert_remote_object_result(mock_remote_object)
|
||||||
|
assert result == mock_remote_object
|
||||||
|
|
||||||
|
def test_convert_remote_object_result_with_none_serialized_data(self) -> None:
|
||||||
|
"""Test _convert_remote_object_result when serialized_data is None."""
|
||||||
|
mixin = WebScrapingMixin()
|
||||||
|
|
||||||
|
# Mock RemoteObject with None serialized_data
|
||||||
|
mock_remote_object = Mock()
|
||||||
|
mock_remote_object.deep_serialized_value = Mock()
|
||||||
|
mock_remote_object.deep_serialized_value.value = None
|
||||||
|
|
||||||
|
result = mixin._convert_remote_object_result(mock_remote_object)
|
||||||
|
assert result == mock_remote_object
|
||||||
|
|
||||||
|
def test_convert_remote_object_result_with_list_data(self) -> None:
|
||||||
|
"""Test _convert_remote_object_result with list data."""
|
||||||
|
mixin = WebScrapingMixin()
|
||||||
|
|
||||||
|
# Mock RemoteObject with list data
|
||||||
mock_remote_object = Mock()
|
mock_remote_object = Mock()
|
||||||
mock_remote_object.deep_serialized_value = Mock()
|
mock_remote_object.deep_serialized_value = Mock()
|
||||||
mock_remote_object.deep_serialized_value.value = [
|
mock_remote_object.deep_serialized_value.value = [
|
||||||
@@ -26,105 +133,283 @@ class TestWebExecuteRemoteObjectHandling:
|
|||||||
["key2", "value2"]
|
["key2", "value2"]
|
||||||
]
|
]
|
||||||
|
|
||||||
# Mock the page evaluation to return our RemoteObject
|
result = mixin._convert_remote_object_result(mock_remote_object)
|
||||||
with patch.object(mixin, "page") as mock_page:
|
|
||||||
mock_page.evaluate = AsyncMock(return_value = mock_remote_object)
|
|
||||||
|
|
||||||
result = await mixin.web_execute("window.test")
|
|
||||||
|
|
||||||
# Should convert the RemoteObject to a dict
|
|
||||||
assert result == {"key1": "value1", "key2": "value2"}
|
assert result == {"key1": "value1", "key2": "value2"}
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
def test_convert_remote_object_result_with_dict_data(self) -> None:
|
||||||
async def test_web_execute_remoteobject_with_none_deep_serialized_value(self) -> None:
|
"""Test _convert_remote_object_result with dict data."""
|
||||||
"""Test web_execute with RemoteObject that has None deep_serialized_value."""
|
|
||||||
mixin = WebScrapingMixin()
|
mixin = WebScrapingMixin()
|
||||||
|
|
||||||
# Mock RemoteObject with None deep_serialized_value
|
# Mock RemoteObject with dict data
|
||||||
mock_remote_object = Mock()
|
|
||||||
mock_remote_object.deep_serialized_value = None
|
|
||||||
|
|
||||||
with patch.object(mixin, "page") as mock_page:
|
|
||||||
mock_page.evaluate = AsyncMock(return_value = mock_remote_object)
|
|
||||||
|
|
||||||
result = await mixin.web_execute("window.test")
|
|
||||||
|
|
||||||
# Should return the original RemoteObject
|
|
||||||
assert result is mock_remote_object
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
|
||||||
async def test_web_execute_remoteobject_with_non_list_serialized_data(self) -> None:
|
|
||||||
"""Test web_execute with RemoteObject that has non-list serialized data."""
|
|
||||||
mixin = WebScrapingMixin()
|
|
||||||
|
|
||||||
# Mock RemoteObject with non-list serialized data
|
|
||||||
mock_remote_object = Mock()
|
mock_remote_object = Mock()
|
||||||
mock_remote_object.deep_serialized_value = Mock()
|
mock_remote_object.deep_serialized_value = Mock()
|
||||||
mock_remote_object.deep_serialized_value.value = {"direct": "dict"}
|
mock_remote_object.deep_serialized_value.value = {"key": "value"}
|
||||||
|
|
||||||
with patch.object(mixin, "page") as mock_page:
|
result = mixin._convert_remote_object_result(mock_remote_object)
|
||||||
mock_page.evaluate = AsyncMock(return_value = mock_remote_object)
|
assert result == {"key": "value"}
|
||||||
|
|
||||||
result = await mixin.web_execute("window.test")
|
def test_convert_remote_object_result_with_conversion_error(self) -> None:
|
||||||
|
"""Test _convert_remote_object_result when conversion raises an exception."""
|
||||||
# Should return the serialized data directly
|
|
||||||
assert result == {"direct": "dict"}
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
|
||||||
async def test_web_execute_remoteobject_with_none_serialized_data(self) -> None:
|
|
||||||
"""Test web_execute with RemoteObject that has None serialized data."""
|
|
||||||
mixin = WebScrapingMixin()
|
mixin = WebScrapingMixin()
|
||||||
|
|
||||||
# Mock RemoteObject with None serialized data
|
# Mock RemoteObject that will raise an exception during conversion
|
||||||
mock_remote_object = Mock()
|
mock_remote_object = Mock()
|
||||||
mock_remote_object.deep_serialized_value = Mock()
|
mock_remote_object.deep_serialized_value = Mock()
|
||||||
mock_remote_object.deep_serialized_value.value = None
|
mock_remote_object.deep_serialized_value.value = "invalid_data"
|
||||||
|
|
||||||
with patch.object(mixin, "page") as mock_page:
|
# Mock the _convert_remote_object_dict to raise an exception
|
||||||
mock_page.evaluate = AsyncMock(return_value = mock_remote_object)
|
with patch.object(mixin, "_convert_remote_object_dict", side_effect = ValueError("Test error")):
|
||||||
|
result = mixin._convert_remote_object_result(mock_remote_object)
|
||||||
|
# When conversion fails, it should return the original value
|
||||||
|
assert result == "invalid_data"
|
||||||
|
|
||||||
result = await mixin.web_execute("window.test")
|
|
||||||
|
|
||||||
# Should return the original RemoteObject
|
class TestConvertRemoteObjectDict:
|
||||||
assert result is mock_remote_object
|
"""Test _convert_remote_object_dict method for nested RemoteObject conversion."""
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
def test_convert_remote_object_dict_with_type_value_pair(self) -> None:
|
||||||
async def test_web_execute_regular_result(self) -> None:
|
"""Test conversion of type/value pair structures."""
|
||||||
"""Test web_execute with regular result (no RemoteObject)."""
|
|
||||||
mixin = WebScrapingMixin()
|
mixin = WebScrapingMixin()
|
||||||
|
|
||||||
# Mock regular result (no deep_serialized_value attribute)
|
# Test type/value pair
|
||||||
mock_result = {"regular": "dict"}
|
data = {"type": "number", "value": 200}
|
||||||
|
result = mixin._convert_remote_object_dict(data)
|
||||||
|
assert result == 200
|
||||||
|
|
||||||
with patch.object(mixin, "page") as mock_page:
|
# Test string type/value pair
|
||||||
mock_page.evaluate = AsyncMock(return_value = mock_result)
|
data = {"type": "string", "value": "hello"}
|
||||||
|
result = mixin._convert_remote_object_dict(data)
|
||||||
|
assert result == "hello"
|
||||||
|
|
||||||
result = await mixin.web_execute("window.test")
|
def test_convert_remote_object_dict_with_regular_dict(self) -> None:
|
||||||
|
"""Test conversion of regular dict structures."""
|
||||||
# Should return the result unchanged
|
|
||||||
assert result == mock_result
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
|
||||||
async def test_web_execute_remoteobject_conversion_exception(self) -> None:
|
|
||||||
"""Test web_execute with RemoteObject that raises exception during conversion."""
|
|
||||||
mixin = WebScrapingMixin()
|
mixin = WebScrapingMixin()
|
||||||
|
|
||||||
# Mock RemoteObject that will raise an exception when trying to convert to dict
|
# Test regular dict (not type/value pair)
|
||||||
|
data = {"key1": "value1", "key2": "value2"}
|
||||||
|
result = mixin._convert_remote_object_dict(data)
|
||||||
|
assert result == {"key1": "value1", "key2": "value2"}
|
||||||
|
|
||||||
|
def test_convert_remote_object_dict_with_nested_structures(self) -> None:
|
||||||
|
"""Test conversion of nested dict structures."""
|
||||||
|
mixin = WebScrapingMixin()
|
||||||
|
|
||||||
|
# Test nested structures
|
||||||
|
data = {
|
||||||
|
"simple": "value",
|
||||||
|
"nested": {"type": "number", "value": 42},
|
||||||
|
"list": [{"type": "string", "value": "item1"}, {"type": "string", "value": "item2"}]
|
||||||
|
}
|
||||||
|
result = mixin._convert_remote_object_dict(data)
|
||||||
|
|
||||||
|
expected = {
|
||||||
|
"simple": "value",
|
||||||
|
"nested": 42,
|
||||||
|
"list": ["item1", "item2"]
|
||||||
|
}
|
||||||
|
assert result == expected
|
||||||
|
|
||||||
|
def test_convert_remote_object_dict_with_list(self) -> None:
|
||||||
|
"""Test conversion of list structures."""
|
||||||
|
mixin = WebScrapingMixin()
|
||||||
|
|
||||||
|
# Test list with type/value pairs
|
||||||
|
data = [{"type": "number", "value": 1}, {"type": "string", "value": "test"}]
|
||||||
|
result = mixin._convert_remote_object_dict(data)
|
||||||
|
assert result == [1, "test"]
|
||||||
|
|
||||||
|
def test_convert_remote_object_dict_with_primitive_values(self) -> None:
|
||||||
|
"""Test conversion with primitive values."""
|
||||||
|
mixin = WebScrapingMixin()
|
||||||
|
|
||||||
|
# Test primitive values
|
||||||
|
assert mixin._convert_remote_object_dict("string") == "string"
|
||||||
|
assert mixin._convert_remote_object_dict(42) == 42
|
||||||
|
assert mixin._convert_remote_object_dict(True) is True
|
||||||
|
assert mixin._convert_remote_object_dict(None) is None
|
||||||
|
|
||||||
|
def test_convert_remote_object_dict_with_complex_nested_structures(self) -> None:
|
||||||
|
"""Test conversion with complex nested structures."""
|
||||||
|
mixin = WebScrapingMixin()
|
||||||
|
|
||||||
|
# Test complex nested structures
|
||||||
|
data = {
|
||||||
|
"response": {
|
||||||
|
"status": {"type": "number", "value": 200},
|
||||||
|
"data": [
|
||||||
|
{"type": "string", "value": "item1"},
|
||||||
|
{"type": "string", "value": "item2"}
|
||||||
|
],
|
||||||
|
"metadata": {
|
||||||
|
"count": {"type": "number", "value": 2},
|
||||||
|
"type": {"type": "string", "value": "list"}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
result = mixin._convert_remote_object_dict(data)
|
||||||
|
|
||||||
|
expected = {
|
||||||
|
"response": {
|
||||||
|
"status": 200,
|
||||||
|
"data": ["item1", "item2"],
|
||||||
|
"metadata": {
|
||||||
|
"count": 2,
|
||||||
|
"type": "list"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
assert result == expected
|
||||||
|
|
||||||
|
|
||||||
|
class TestWebRequestRemoteObjectHandling:
|
||||||
|
"""Test web_request method with nodriver 0.47+ RemoteObject behavior."""
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_web_request_with_remoteobject_result(self) -> None:
|
||||||
|
"""Test web_request with RemoteObject result to catch subscriptability issues."""
|
||||||
|
mixin = WebScrapingMixin()
|
||||||
|
|
||||||
|
# Mock RemoteObject that simulates the exact structure returned by web_request
|
||||||
mock_remote_object = Mock()
|
mock_remote_object = Mock()
|
||||||
mock_deep_serialized = Mock()
|
mock_remote_object.deep_serialized_value = Mock()
|
||||||
|
mock_remote_object.deep_serialized_value.value = {
|
||||||
|
"statusCode": 200,
|
||||||
|
"statusMessage": "OK",
|
||||||
|
"headers": {"content-type": "application/json"},
|
||||||
|
"content": '{"success": true}'
|
||||||
|
}
|
||||||
|
|
||||||
# Create a list-like object that raises an exception when dict() is called on it
|
with patch.object(mixin, "web_execute") as mock_web_execute:
|
||||||
class ExceptionRaisingList(list[str]):
|
# Mock web_execute to return the converted result (simulating our fix)
|
||||||
def __iter__(self) -> None: # type: ignore[override]
|
mock_web_execute.return_value = {
|
||||||
raise ValueError("Simulated conversion error")
|
"statusCode": 200,
|
||||||
|
"statusMessage": "OK",
|
||||||
|
"headers": {"content-type": "application/json"},
|
||||||
|
"content": '{"success": true}'
|
||||||
|
}
|
||||||
|
|
||||||
mock_deep_serialized.value = ExceptionRaisingList([["key", "value"]]) # type: ignore[list-item]
|
result = await mixin.web_request("https://example.com/api")
|
||||||
mock_remote_object.deep_serialized_value = mock_deep_serialized
|
|
||||||
|
|
||||||
with patch.object(mixin, "page") as mock_page:
|
# Verify the result is properly converted and subscriptable
|
||||||
mock_page.evaluate = AsyncMock(return_value = mock_remote_object)
|
assert result["statusCode"] == 200
|
||||||
|
assert result["statusMessage"] == "OK"
|
||||||
|
assert result["headers"]["content-type"] == "application/json"
|
||||||
|
assert result["content"] == '{"success": true}'
|
||||||
|
|
||||||
result = await mixin.web_execute("window.test")
|
@pytest.mark.asyncio
|
||||||
|
async def test_web_request_with_remoteobject_error_response(self) -> None:
|
||||||
|
"""Test web_request with RemoteObject error response."""
|
||||||
|
mixin = WebScrapingMixin()
|
||||||
|
|
||||||
# Should return the original RemoteObject when conversion fails
|
# Mock RemoteObject for error response
|
||||||
assert result is mock_remote_object
|
mock_remote_object = Mock()
|
||||||
|
mock_remote_object.deep_serialized_value = Mock()
|
||||||
|
mock_remote_object.deep_serialized_value.value = {
|
||||||
|
"statusCode": 404,
|
||||||
|
"statusMessage": "Not Found",
|
||||||
|
"headers": {"content-type": "text/html"},
|
||||||
|
"content": "<html>Not Found</html>"
|
||||||
|
}
|
||||||
|
|
||||||
|
with patch.object(mixin, "web_execute") as mock_web_execute:
|
||||||
|
# Mock web_execute to return the converted result (simulating our fix)
|
||||||
|
mock_web_execute.return_value = {
|
||||||
|
"statusCode": 404,
|
||||||
|
"statusMessage": "Not Found",
|
||||||
|
"headers": {"content-type": "text/html"},
|
||||||
|
"content": "<html>Not Found</html>"
|
||||||
|
}
|
||||||
|
|
||||||
|
# This should raise an exception due to invalid status code
|
||||||
|
with pytest.raises(Exception, match = "Invalid response"):
|
||||||
|
await mixin.web_request("https://example.com/api", valid_response_codes = [200])
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_web_request_with_nested_remoteobject_structures(self) -> None:
|
||||||
|
"""Test web_request with complex nested RemoteObject structures."""
|
||||||
|
mixin = WebScrapingMixin()
|
||||||
|
|
||||||
|
# Mock RemoteObject with nested type/value structures
|
||||||
|
mock_remote_object = Mock()
|
||||||
|
mock_remote_object.deep_serialized_value = Mock()
|
||||||
|
mock_remote_object.deep_serialized_value.value = {
|
||||||
|
"statusCode": {"type": "number", "value": 200},
|
||||||
|
"statusMessage": {"type": "string", "value": "OK"},
|
||||||
|
"headers": {
|
||||||
|
"content-type": {"type": "string", "value": "application/json"}
|
||||||
|
},
|
||||||
|
"content": {"type": "string", "value": '{"data": "test"}'}
|
||||||
|
}
|
||||||
|
|
||||||
|
with patch.object(mixin, "web_execute") as mock_web_execute:
|
||||||
|
# Mock web_execute to return the converted result (simulating our fix)
|
||||||
|
mock_web_execute.return_value = {
|
||||||
|
"statusCode": 200,
|
||||||
|
"statusMessage": "OK",
|
||||||
|
"headers": {"content-type": "application/json"},
|
||||||
|
"content": '{"data": "test"}'
|
||||||
|
}
|
||||||
|
|
||||||
|
result = await mixin.web_request("https://example.com/api")
|
||||||
|
|
||||||
|
# Verify nested structures are properly converted
|
||||||
|
assert result["statusCode"] == 200
|
||||||
|
assert result["statusMessage"] == "OK"
|
||||||
|
assert result["headers"]["content-type"] == "application/json"
|
||||||
|
assert result["content"] == '{"data": "test"}'
|
||||||
|
|
||||||
|
|
||||||
|
class TestWebRequestRemoteObjectRegression:
|
||||||
|
"""Test web_request method to catch future RemoteObject regression issues."""
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_web_request_without_remoteobject_conversion_fails(self) -> None:
|
||||||
|
"""Test that web_request fails without RemoteObject conversion (regression test)."""
|
||||||
|
mixin = WebScrapingMixin()
|
||||||
|
|
||||||
|
# Mock RemoteObject that would cause the original error
|
||||||
|
mock_remote_object = Mock()
|
||||||
|
mock_remote_object.deep_serialized_value = Mock()
|
||||||
|
mock_remote_object.deep_serialized_value.value = {
|
||||||
|
"statusCode": 200,
|
||||||
|
"statusMessage": "OK",
|
||||||
|
"headers": {"content-type": "application/json"},
|
||||||
|
"content": '{"success": true}'
|
||||||
|
}
|
||||||
|
|
||||||
|
# Mock web_execute to return the raw RemoteObject (simulating the bug)
|
||||||
|
with patch.object(mixin, "web_execute") as mock_web_execute:
|
||||||
|
mock_web_execute.return_value = mock_remote_object
|
||||||
|
|
||||||
|
# This should fail with the original error if our fix is removed
|
||||||
|
with pytest.raises(TypeError, match = "object is not subscriptable"):
|
||||||
|
await mixin.web_request("https://example.com/api")
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_web_request_with_remoteobject_conversion_succeeds(self) -> None:
|
||||||
|
"""Test that web_request succeeds with RemoteObject conversion (our fix)."""
|
||||||
|
mixin = WebScrapingMixin()
|
||||||
|
|
||||||
|
# Mock RemoteObject
|
||||||
|
mock_remote_object = Mock()
|
||||||
|
mock_remote_object.deep_serialized_value = Mock()
|
||||||
|
mock_remote_object.deep_serialized_value.value = {
|
||||||
|
"statusCode": 200,
|
||||||
|
"statusMessage": "OK",
|
||||||
|
"headers": {"content-type": "application/json"},
|
||||||
|
"content": '{"success": true}'
|
||||||
|
}
|
||||||
|
|
||||||
|
# Mock web_execute to simulate our conversion logic
|
||||||
|
async def mock_web_execute_with_conversion(script:str) -> Any:
|
||||||
|
# Simulate the conversion that happens in our fix
|
||||||
|
return mock_remote_object.deep_serialized_value.value
|
||||||
|
|
||||||
|
with patch.object(mixin, "web_execute", side_effect = mock_web_execute_with_conversion):
|
||||||
|
result = await mixin.web_request("https://example.com/api")
|
||||||
|
|
||||||
|
# Verify the result works correctly
|
||||||
|
assert result["statusCode"] == 200
|
||||||
|
assert result["statusMessage"] == "OK"
|
||||||
|
assert result["headers"]["content-type"] == "application/json"
|
||||||
|
assert result["content"] == '{"success": true}'
|
||||||
|
|||||||
Reference in New Issue
Block a user