fix: resolve nodriver 0.47+ RemoteObject compatibility issues (#645)

## ℹ️ Description
*Provide a concise summary of the changes introduced in this pull
request.*

- Link to the related issue(s): #644
- Describe the motivation and context for this change.

This PR resolves compatibility issues with nodriver 0.47+ where
page.evaluate() returns RemoteObject instances that need special
handling for proper conversion to Python objects. The update introduced
breaking changes in how JavaScript evaluation results are returned,
causing TypeError: [RemoteObject] object is not subscriptable errors.

## 📋 Changes Summary

- Fixed TypeError: [RemoteObject] object is not subscriptable in
web_request() method
- Added comprehensive RemoteObject conversion logic with
_convert_remote_object_result()
- Added _convert_remote_object_dict() for recursive nested structure
conversion
- Fixed price field concatenation issue in MODIFY mode by explicit field
clearing
- Updated web_sleep() to accept integer milliseconds instead of float
seconds
- Updated German translations for new log messages
- Fixed linting issues (E711, E712) in test assertions

### ⚙️ Type of Change
Select the type(s) of change(s) included in this pull request:
- [x] 🐞 Bug fix (non-breaking change which fixes an issue)
- [ ]  New feature (adds new functionality without breaking existing
usage)
- [ ] 💥 Breaking change (changes that might break existing user setups,
scripts, or configurations)

##  Checklist
Before requesting a review, confirm the following:
- [x] I have reviewed my changes to ensure they meet the project's
standards.
- [x] I have tested my changes and ensured that all tests pass (pdm run
test).
- [x] I have formatted the code (pdm run format).
- [x] I have verified that linting passes (pdm run lint).
- [x] I have updated documentation where necessary.

By submitting this pull request, I confirm that you can use, modify,
copy, and redistribute this contribution, under the terms of your
choice.
This commit is contained in:
Jens
2025-10-18 19:38:51 +02:00
committed by GitHub
parent 34013cb869
commit 8aee313aba
4 changed files with 429 additions and 100 deletions

View File

@@ -803,9 +803,12 @@ class KleinanzeigenBot(WebScrapingMixin):
pass pass
if ad_cfg.price: if ad_cfg.price:
if mode == AdUpdateStrategy.MODIFY: if mode == AdUpdateStrategy.MODIFY:
# we have to clear the input, otherwise input gets appended # Clear the price field first to prevent concatenation of old and new values
await self.web_input(By.CSS_SELECTOR, # This is needed because some input fields don't clear properly with just clear_input()
"input#post-ad-frontend-price, input#micro-frontend-price, input#pstad-price", "") price_field = await self.web_find(By.CSS_SELECTOR, "input#post-ad-frontend-price, input#micro-frontend-price, input#pstad-price")
await price_field.clear_input()
await price_field.send_keys("") # Ensure field is completely empty
await self.web_sleep(500) # Brief pause to ensure clearing is complete
await self.web_input(By.CSS_SELECTOR, "input#post-ad-frontend-price, input#micro-frontend-price, input#pstad-price", str(ad_cfg.price)) await self.web_input(By.CSS_SELECTOR, "input#post-ad-frontend-price, input#micro-frontend-price, input#pstad-price", str(ad_cfg.price))
############################# #############################

View File

@@ -391,7 +391,7 @@ kleinanzeigen_bot/utils/web_scraping_mixin.py:
"4. Check browser binary permissions: %s": "4. Überprüfen Sie die Browser-Binärdatei-Berechtigungen: %s" "4. Check browser binary permissions: %s": "4. Überprüfen Sie die Browser-Binärdatei-Berechtigungen: %s"
"4. Check if any antivirus or security software is blocking the connection": "4. Überprüfen Sie, ob Antiviren- oder Sicherheitssoftware die Verbindung blockiert" "4. Check if any antivirus or security software is blocking the connection": "4. Überprüfen Sie, ob Antiviren- oder Sicherheitssoftware die Verbindung blockiert"
web_execute: _convert_remote_object_result:
"Failed to convert RemoteObject to dict: %s": "Fehler beim Konvertieren von RemoteObject zu dict: %s" "Failed to convert RemoteObject to dict: %s": "Fehler beim Konvertieren von RemoteObject zu dict: %s"
web_check: web_check:

View File

@@ -42,6 +42,9 @@ LOG:Final[loggers.Logger] = loggers.get_logger(__name__)
# see https://api.jquery.com/category/selectors/ # see https://api.jquery.com/category/selectors/
METACHAR_ESCAPER:Final[dict[int, str]] = str.maketrans({ch: f"\\{ch}" for ch in '!"#$%&\'()*+,./:;<=>?@[\\]^`{|}~'}) METACHAR_ESCAPER:Final[dict[int, str]] = str.maketrans({ch: f"\\{ch}" for ch in '!"#$%&\'()*+,./:;<=>?@[\\]^`{|}~'})
# Constants for RemoteObject handling
_REMOTE_OBJECT_TYPE_VALUE_PAIR_SIZE:Final[int] = 2
def _is_admin() -> bool: def _is_admin() -> bool:
"""Check if the current process is running with admin/root privileges.""" """Check if the current process is running with admin/root privileges."""
@@ -574,22 +577,60 @@ class WebScrapingMixin:
# Handle nodriver 0.47+ RemoteObject behavior # Handle nodriver 0.47+ RemoteObject behavior
# If result is a RemoteObject with deep_serialized_value, convert it to a dict # If result is a RemoteObject with deep_serialized_value, convert it to a dict
if hasattr(result, "deep_serialized_value"): if hasattr(result, "deep_serialized_value"):
return self._convert_remote_object_result(result)
return result
def _convert_remote_object_result(self, result:Any) -> Any:
"""
Converts a RemoteObject result to a regular Python object.
Handles the deep_serialized_value conversion for nodriver 0.47+ compatibility.
"""
deep_serialized = getattr(result, "deep_serialized_value", None) deep_serialized = getattr(result, "deep_serialized_value", None)
if deep_serialized is not None: if deep_serialized is None:
return result
try: try:
# Convert the deep_serialized_value to a regular dict # Convert the deep_serialized_value to a regular dict
serialized_data = getattr(deep_serialized, "value", None) serialized_data = getattr(deep_serialized, "value", None)
if serialized_data is not None: if serialized_data is None:
return result
if isinstance(serialized_data, list): if isinstance(serialized_data, list):
# Convert list of [key, value] pairs to dict # Convert list of [key, value] pairs to dict, handling nested RemoteObjects
return dict(serialized_data) converted_dict = {}
for key, value in serialized_data:
converted_dict[key] = self._convert_remote_object_dict(value)
return converted_dict
if isinstance(serialized_data, dict):
# Handle nested RemoteObject structures like {'type': 'number', 'value': 200}
return self._convert_remote_object_dict(serialized_data)
return serialized_data return serialized_data
except (AttributeError, TypeError, ValueError) as e: except (AttributeError, TypeError, ValueError) as e:
LOG.warning("Failed to convert RemoteObject to dict: %s", e) LOG.warning("Failed to convert RemoteObject to dict: %s", e)
# Return the original result if conversion fails # Return the original result if conversion fails
return result return result
return result def _convert_remote_object_dict(self, data:Any) -> Any:
"""
Recursively converts RemoteObject dict structures to regular Python objects.
Handles structures like {'type': 'number', 'value': 200} or {'type': 'string', 'value': 'text'}.
"""
if isinstance(data, dict):
# Check if this is a RemoteObject value structure
if "type" in data and "value" in data and len(data) == _REMOTE_OBJECT_TYPE_VALUE_PAIR_SIZE:
return data["value"]
# Recursively convert nested dicts
return {key: self._convert_remote_object_dict(value) for key, value in data.items()}
if isinstance(data, list):
# Recursively convert lists
return [self._convert_remote_object_dict(item) for item in data]
# Return primitive values as-is
return data
async def web_find(self, selector_type:By, selector_value:str, *, parent:Element | None = None, timeout:int | float = 5) -> Element: async def web_find(self, selector_type:By, selector_value:str, *, parent:Element | None = None, timeout:int | float = 5) -> Element:
""" """
@@ -724,10 +765,10 @@ class WebScrapingMixin:
await self.page.sleep(duration / 1_000) await self.page.sleep(duration / 1_000)
async def web_request(self, url:str, method:str = "GET", valid_response_codes:int | Iterable[int] = 200, async def web_request(self, url:str, method:str = "GET", valid_response_codes:int | Iterable[int] = 200,
headers:dict[str, str] | None = None) -> dict[str, Any]: headers:dict[str, str] | None = None) -> Any:
method = method.upper() method = method.upper()
LOG.debug(" -> HTTP %s [%s]...", method, url) LOG.debug(" -> HTTP %s [%s]...", method, url)
response = cast(dict[str, Any], await self.page.evaluate(f""" response = await self.web_execute(f"""
fetch("{url}", {{ fetch("{url}", {{
method: "{method}", method: "{method}",
redirect: "follow", redirect: "follow",
@@ -743,7 +784,7 @@ class WebScrapingMixin:
content: responseText content: responseText
}} }}
}})) }}))
""", await_promise = True, return_by_value = True)) """)
if isinstance(valid_response_codes, int): if isinstance(valid_response_codes, int):
valid_response_codes = [valid_response_codes] valid_response_codes = [valid_response_codes]
ensure( ensure(

View File

@@ -1,8 +1,12 @@
# SPDX-FileCopyrightText: © Jens Bergmann and contributors # SPDX-FileCopyrightText: © Jens Bergmann and contributors
# SPDX-License-Identifier: AGPL-3.0-or-later # SPDX-License-Identifier: AGPL-3.0-or-later
# SPDX-ArtifactOfProjectHomePage: https://github.com/Second-Hand-Friends/kleinanzeigen-bot/ # SPDX-ArtifactOfProjectHomePage: https://github.com/Second-Hand-Friends/kleinanzeigen-bot/
"""Unit tests for web_scraping_mixin.py RemoteObject handling.""" """Unit tests for web_scraping_mixin.py RemoteObject handling.
Tests the conversion of nodriver RemoteObject results to regular Python objects.
"""
from typing import Any
from unittest.mock import AsyncMock, Mock, patch from unittest.mock import AsyncMock, Mock, patch
import pytest import pytest
@@ -14,11 +18,114 @@ class TestWebExecuteRemoteObjectHandling:
"""Test web_execute method with nodriver 0.47+ RemoteObject behavior.""" """Test web_execute method with nodriver 0.47+ RemoteObject behavior."""
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_web_execute_remoteobject_with_deep_serialized_value(self) -> None: async def test_web_execute_with_regular_result(self) -> None:
"""Test web_execute with RemoteObject that has deep_serialized_value.""" """Test web_execute with regular (non-RemoteObject) result."""
mixin = WebScrapingMixin() mixin = WebScrapingMixin()
# Mock RemoteObject with deep_serialized_value with patch.object(mixin, "page") as mock_page:
mock_page.evaluate = AsyncMock(return_value = "regular_result")
result = await mixin.web_execute("window.test")
assert result == "regular_result"
@pytest.mark.asyncio
async def test_web_execute_with_remoteobject_result(self) -> None:
"""Test web_execute with RemoteObject result."""
mixin = WebScrapingMixin()
# Mock RemoteObject
mock_remote_object = Mock()
mock_remote_object.deep_serialized_value = Mock()
mock_remote_object.deep_serialized_value.value = {"key": "value"}
with patch.object(mixin, "page") as mock_page:
mock_page.evaluate = AsyncMock(return_value = mock_remote_object)
result = await mixin.web_execute("window.test")
assert result == {"key": "value"}
@pytest.mark.asyncio
async def test_web_execute_remoteobject_with_nested_type_value_structures(self) -> None:
"""Test web_execute with RemoteObject containing nested type/value structures."""
mixin = WebScrapingMixin()
# Mock RemoteObject with nested type/value structures
mock_remote_object = Mock()
mock_remote_object.deep_serialized_value = Mock()
mock_remote_object.deep_serialized_value.value = [
["statusCode", {"type": "number", "value": 200}],
["content", {"type": "string", "value": "success"}]
]
with patch.object(mixin, "page") as mock_page:
mock_page.evaluate = AsyncMock(return_value = mock_remote_object)
result = await mixin.web_execute("window.test")
# Should convert nested type/value structures to their values
assert result == {"statusCode": 200, "content": "success"}
@pytest.mark.asyncio
async def test_web_execute_remoteobject_with_mixed_nested_structures(self) -> None:
"""Test web_execute with RemoteObject containing mixed nested structures."""
mixin = WebScrapingMixin()
# Mock RemoteObject with mixed nested structures
mock_remote_object = Mock()
mock_remote_object.deep_serialized_value = Mock()
mock_remote_object.deep_serialized_value.value = {
"simple": "value",
"nested": {"type": "number", "value": 42},
"list": [{"type": "string", "value": "item1"}, {"type": "string", "value": "item2"}]
}
with patch.object(mixin, "page") as mock_page:
mock_page.evaluate = AsyncMock(return_value = mock_remote_object)
result = await mixin.web_execute("window.test")
# Should convert nested structures while preserving simple values
expected = {
"simple": "value",
"nested": 42,
"list": ["item1", "item2"]
}
assert result == expected
class TestConvertRemoteObjectResult:
"""Test _convert_remote_object_result method for RemoteObject conversion."""
def test_convert_remote_object_result_with_none_deep_serialized_value(self) -> None:
"""Test _convert_remote_object_result when deep_serialized_value is None."""
mixin = WebScrapingMixin()
# Mock RemoteObject with None deep_serialized_value
mock_remote_object = Mock()
mock_remote_object.deep_serialized_value = None
result = mixin._convert_remote_object_result(mock_remote_object)
assert result == mock_remote_object
def test_convert_remote_object_result_with_none_serialized_data(self) -> None:
"""Test _convert_remote_object_result when serialized_data is None."""
mixin = WebScrapingMixin()
# Mock RemoteObject with None serialized_data
mock_remote_object = Mock()
mock_remote_object.deep_serialized_value = Mock()
mock_remote_object.deep_serialized_value.value = None
result = mixin._convert_remote_object_result(mock_remote_object)
assert result == mock_remote_object
def test_convert_remote_object_result_with_list_data(self) -> None:
"""Test _convert_remote_object_result with list data."""
mixin = WebScrapingMixin()
# Mock RemoteObject with list data
mock_remote_object = Mock() mock_remote_object = Mock()
mock_remote_object.deep_serialized_value = Mock() mock_remote_object.deep_serialized_value = Mock()
mock_remote_object.deep_serialized_value.value = [ mock_remote_object.deep_serialized_value.value = [
@@ -26,105 +133,283 @@ class TestWebExecuteRemoteObjectHandling:
["key2", "value2"] ["key2", "value2"]
] ]
# Mock the page evaluation to return our RemoteObject result = mixin._convert_remote_object_result(mock_remote_object)
with patch.object(mixin, "page") as mock_page:
mock_page.evaluate = AsyncMock(return_value = mock_remote_object)
result = await mixin.web_execute("window.test")
# Should convert the RemoteObject to a dict
assert result == {"key1": "value1", "key2": "value2"} assert result == {"key1": "value1", "key2": "value2"}
@pytest.mark.asyncio def test_convert_remote_object_result_with_dict_data(self) -> None:
async def test_web_execute_remoteobject_with_none_deep_serialized_value(self) -> None: """Test _convert_remote_object_result with dict data."""
"""Test web_execute with RemoteObject that has None deep_serialized_value."""
mixin = WebScrapingMixin() mixin = WebScrapingMixin()
# Mock RemoteObject with None deep_serialized_value # Mock RemoteObject with dict data
mock_remote_object = Mock()
mock_remote_object.deep_serialized_value = None
with patch.object(mixin, "page") as mock_page:
mock_page.evaluate = AsyncMock(return_value = mock_remote_object)
result = await mixin.web_execute("window.test")
# Should return the original RemoteObject
assert result is mock_remote_object
@pytest.mark.asyncio
async def test_web_execute_remoteobject_with_non_list_serialized_data(self) -> None:
"""Test web_execute with RemoteObject that has non-list serialized data."""
mixin = WebScrapingMixin()
# Mock RemoteObject with non-list serialized data
mock_remote_object = Mock() mock_remote_object = Mock()
mock_remote_object.deep_serialized_value = Mock() mock_remote_object.deep_serialized_value = Mock()
mock_remote_object.deep_serialized_value.value = {"direct": "dict"} mock_remote_object.deep_serialized_value.value = {"key": "value"}
with patch.object(mixin, "page") as mock_page: result = mixin._convert_remote_object_result(mock_remote_object)
mock_page.evaluate = AsyncMock(return_value = mock_remote_object) assert result == {"key": "value"}
result = await mixin.web_execute("window.test") def test_convert_remote_object_result_with_conversion_error(self) -> None:
"""Test _convert_remote_object_result when conversion raises an exception."""
# Should return the serialized data directly
assert result == {"direct": "dict"}
@pytest.mark.asyncio
async def test_web_execute_remoteobject_with_none_serialized_data(self) -> None:
"""Test web_execute with RemoteObject that has None serialized data."""
mixin = WebScrapingMixin() mixin = WebScrapingMixin()
# Mock RemoteObject with None serialized data # Mock RemoteObject that will raise an exception during conversion
mock_remote_object = Mock() mock_remote_object = Mock()
mock_remote_object.deep_serialized_value = Mock() mock_remote_object.deep_serialized_value = Mock()
mock_remote_object.deep_serialized_value.value = None mock_remote_object.deep_serialized_value.value = "invalid_data"
with patch.object(mixin, "page") as mock_page: # Mock the _convert_remote_object_dict to raise an exception
mock_page.evaluate = AsyncMock(return_value = mock_remote_object) with patch.object(mixin, "_convert_remote_object_dict", side_effect = ValueError("Test error")):
result = mixin._convert_remote_object_result(mock_remote_object)
# When conversion fails, it should return the original value
assert result == "invalid_data"
result = await mixin.web_execute("window.test")
# Should return the original RemoteObject class TestConvertRemoteObjectDict:
assert result is mock_remote_object """Test _convert_remote_object_dict method for nested RemoteObject conversion."""
@pytest.mark.asyncio def test_convert_remote_object_dict_with_type_value_pair(self) -> None:
async def test_web_execute_regular_result(self) -> None: """Test conversion of type/value pair structures."""
"""Test web_execute with regular result (no RemoteObject)."""
mixin = WebScrapingMixin() mixin = WebScrapingMixin()
# Mock regular result (no deep_serialized_value attribute) # Test type/value pair
mock_result = {"regular": "dict"} data = {"type": "number", "value": 200}
result = mixin._convert_remote_object_dict(data)
assert result == 200
with patch.object(mixin, "page") as mock_page: # Test string type/value pair
mock_page.evaluate = AsyncMock(return_value = mock_result) data = {"type": "string", "value": "hello"}
result = mixin._convert_remote_object_dict(data)
assert result == "hello"
result = await mixin.web_execute("window.test") def test_convert_remote_object_dict_with_regular_dict(self) -> None:
"""Test conversion of regular dict structures."""
# Should return the result unchanged
assert result == mock_result
@pytest.mark.asyncio
async def test_web_execute_remoteobject_conversion_exception(self) -> None:
"""Test web_execute with RemoteObject that raises exception during conversion."""
mixin = WebScrapingMixin() mixin = WebScrapingMixin()
# Mock RemoteObject that will raise an exception when trying to convert to dict # Test regular dict (not type/value pair)
data = {"key1": "value1", "key2": "value2"}
result = mixin._convert_remote_object_dict(data)
assert result == {"key1": "value1", "key2": "value2"}
def test_convert_remote_object_dict_with_nested_structures(self) -> None:
"""Test conversion of nested dict structures."""
mixin = WebScrapingMixin()
# Test nested structures
data = {
"simple": "value",
"nested": {"type": "number", "value": 42},
"list": [{"type": "string", "value": "item1"}, {"type": "string", "value": "item2"}]
}
result = mixin._convert_remote_object_dict(data)
expected = {
"simple": "value",
"nested": 42,
"list": ["item1", "item2"]
}
assert result == expected
def test_convert_remote_object_dict_with_list(self) -> None:
"""Test conversion of list structures."""
mixin = WebScrapingMixin()
# Test list with type/value pairs
data = [{"type": "number", "value": 1}, {"type": "string", "value": "test"}]
result = mixin._convert_remote_object_dict(data)
assert result == [1, "test"]
def test_convert_remote_object_dict_with_primitive_values(self) -> None:
"""Test conversion with primitive values."""
mixin = WebScrapingMixin()
# Test primitive values
assert mixin._convert_remote_object_dict("string") == "string"
assert mixin._convert_remote_object_dict(42) == 42
assert mixin._convert_remote_object_dict(True) is True
assert mixin._convert_remote_object_dict(None) is None
def test_convert_remote_object_dict_with_complex_nested_structures(self) -> None:
"""Test conversion with complex nested structures."""
mixin = WebScrapingMixin()
# Test complex nested structures
data = {
"response": {
"status": {"type": "number", "value": 200},
"data": [
{"type": "string", "value": "item1"},
{"type": "string", "value": "item2"}
],
"metadata": {
"count": {"type": "number", "value": 2},
"type": {"type": "string", "value": "list"}
}
}
}
result = mixin._convert_remote_object_dict(data)
expected = {
"response": {
"status": 200,
"data": ["item1", "item2"],
"metadata": {
"count": 2,
"type": "list"
}
}
}
assert result == expected
class TestWebRequestRemoteObjectHandling:
"""Test web_request method with nodriver 0.47+ RemoteObject behavior."""
@pytest.mark.asyncio
async def test_web_request_with_remoteobject_result(self) -> None:
"""Test web_request with RemoteObject result to catch subscriptability issues."""
mixin = WebScrapingMixin()
# Mock RemoteObject that simulates the exact structure returned by web_request
mock_remote_object = Mock() mock_remote_object = Mock()
mock_deep_serialized = Mock() mock_remote_object.deep_serialized_value = Mock()
mock_remote_object.deep_serialized_value.value = {
"statusCode": 200,
"statusMessage": "OK",
"headers": {"content-type": "application/json"},
"content": '{"success": true}'
}
# Create a list-like object that raises an exception when dict() is called on it with patch.object(mixin, "web_execute") as mock_web_execute:
class ExceptionRaisingList(list[str]): # Mock web_execute to return the converted result (simulating our fix)
def __iter__(self) -> None: # type: ignore[override] mock_web_execute.return_value = {
raise ValueError("Simulated conversion error") "statusCode": 200,
"statusMessage": "OK",
"headers": {"content-type": "application/json"},
"content": '{"success": true}'
}
mock_deep_serialized.value = ExceptionRaisingList([["key", "value"]]) # type: ignore[list-item] result = await mixin.web_request("https://example.com/api")
mock_remote_object.deep_serialized_value = mock_deep_serialized
with patch.object(mixin, "page") as mock_page: # Verify the result is properly converted and subscriptable
mock_page.evaluate = AsyncMock(return_value = mock_remote_object) assert result["statusCode"] == 200
assert result["statusMessage"] == "OK"
assert result["headers"]["content-type"] == "application/json"
assert result["content"] == '{"success": true}'
result = await mixin.web_execute("window.test") @pytest.mark.asyncio
async def test_web_request_with_remoteobject_error_response(self) -> None:
"""Test web_request with RemoteObject error response."""
mixin = WebScrapingMixin()
# Should return the original RemoteObject when conversion fails # Mock RemoteObject for error response
assert result is mock_remote_object mock_remote_object = Mock()
mock_remote_object.deep_serialized_value = Mock()
mock_remote_object.deep_serialized_value.value = {
"statusCode": 404,
"statusMessage": "Not Found",
"headers": {"content-type": "text/html"},
"content": "<html>Not Found</html>"
}
with patch.object(mixin, "web_execute") as mock_web_execute:
# Mock web_execute to return the converted result (simulating our fix)
mock_web_execute.return_value = {
"statusCode": 404,
"statusMessage": "Not Found",
"headers": {"content-type": "text/html"},
"content": "<html>Not Found</html>"
}
# This should raise an exception due to invalid status code
with pytest.raises(Exception, match = "Invalid response"):
await mixin.web_request("https://example.com/api", valid_response_codes = [200])
@pytest.mark.asyncio
async def test_web_request_with_nested_remoteobject_structures(self) -> None:
"""Test web_request with complex nested RemoteObject structures."""
mixin = WebScrapingMixin()
# Mock RemoteObject with nested type/value structures
mock_remote_object = Mock()
mock_remote_object.deep_serialized_value = Mock()
mock_remote_object.deep_serialized_value.value = {
"statusCode": {"type": "number", "value": 200},
"statusMessage": {"type": "string", "value": "OK"},
"headers": {
"content-type": {"type": "string", "value": "application/json"}
},
"content": {"type": "string", "value": '{"data": "test"}'}
}
with patch.object(mixin, "web_execute") as mock_web_execute:
# Mock web_execute to return the converted result (simulating our fix)
mock_web_execute.return_value = {
"statusCode": 200,
"statusMessage": "OK",
"headers": {"content-type": "application/json"},
"content": '{"data": "test"}'
}
result = await mixin.web_request("https://example.com/api")
# Verify nested structures are properly converted
assert result["statusCode"] == 200
assert result["statusMessage"] == "OK"
assert result["headers"]["content-type"] == "application/json"
assert result["content"] == '{"data": "test"}'
class TestWebRequestRemoteObjectRegression:
"""Test web_request method to catch future RemoteObject regression issues."""
@pytest.mark.asyncio
async def test_web_request_without_remoteobject_conversion_fails(self) -> None:
"""Test that web_request fails without RemoteObject conversion (regression test)."""
mixin = WebScrapingMixin()
# Mock RemoteObject that would cause the original error
mock_remote_object = Mock()
mock_remote_object.deep_serialized_value = Mock()
mock_remote_object.deep_serialized_value.value = {
"statusCode": 200,
"statusMessage": "OK",
"headers": {"content-type": "application/json"},
"content": '{"success": true}'
}
# Mock web_execute to return the raw RemoteObject (simulating the bug)
with patch.object(mixin, "web_execute") as mock_web_execute:
mock_web_execute.return_value = mock_remote_object
# This should fail with the original error if our fix is removed
with pytest.raises(TypeError, match = "object is not subscriptable"):
await mixin.web_request("https://example.com/api")
@pytest.mark.asyncio
async def test_web_request_with_remoteobject_conversion_succeeds(self) -> None:
"""Test that web_request succeeds with RemoteObject conversion (our fix)."""
mixin = WebScrapingMixin()
# Mock RemoteObject
mock_remote_object = Mock()
mock_remote_object.deep_serialized_value = Mock()
mock_remote_object.deep_serialized_value.value = {
"statusCode": 200,
"statusMessage": "OK",
"headers": {"content-type": "application/json"},
"content": '{"success": true}'
}
# Mock web_execute to simulate our conversion logic
async def mock_web_execute_with_conversion(script:str) -> Any:
# Simulate the conversion that happens in our fix
return mock_remote_object.deep_serialized_value.value
with patch.object(mixin, "web_execute", side_effect = mock_web_execute_with_conversion):
result = await mixin.web_request("https://example.com/api")
# Verify the result works correctly
assert result["statusCode"] == 200
assert result["statusMessage"] == "OK"
assert result["headers"]["content-type"] == "application/json"
assert result["content"] == '{"success": true}'