From 8aee313aba7cabb05c22776325ba6ced887e7bf0 Mon Sep 17 00:00:00 2001 From: Jens <1742418+1cu@users.noreply.github.com> Date: Sat, 18 Oct 2025 19:38:51 +0200 Subject: [PATCH] fix: resolve nodriver 0.47+ RemoteObject compatibility issues (#645) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## ℹ️ Description *Provide a concise summary of the changes introduced in this pull request.* - Link to the related issue(s): #644 - Describe the motivation and context for this change. This PR resolves compatibility issues with nodriver 0.47+ where page.evaluate() returns RemoteObject instances that need special handling for proper conversion to Python objects. The update introduced breaking changes in how JavaScript evaluation results are returned, causing TypeError: [RemoteObject] object is not subscriptable errors. ## 📋 Changes Summary - Fixed TypeError: [RemoteObject] object is not subscriptable in web_request() method - Added comprehensive RemoteObject conversion logic with _convert_remote_object_result() - Added _convert_remote_object_dict() for recursive nested structure conversion - Fixed price field concatenation issue in MODIFY mode by explicit field clearing - Updated web_sleep() to accept integer milliseconds instead of float seconds - Updated German translations for new log messages - Fixed linting issues (E711, E712) in test assertions ### ⚙️ Type of Change Select the type(s) of change(s) included in this pull request: - [x] 🐞 Bug fix (non-breaking change which fixes an issue) - [ ] ✨ New feature (adds new functionality without breaking existing usage) - [ ] 💥 Breaking change (changes that might break existing user setups, scripts, or configurations) ## ✅ Checklist Before requesting a review, confirm the following: - [x] I have reviewed my changes to ensure they meet the project's standards. - [x] I have tested my changes and ensured that all tests pass (pdm run test). - [x] I have formatted the code (pdm run format). - [x] I have verified that linting passes (pdm run lint). - [x] I have updated documentation where necessary. By submitting this pull request, I confirm that you can use, modify, copy, and redistribute this contribution, under the terms of your choice. --- src/kleinanzeigen_bot/__init__.py | 9 +- .../resources/translations.de.yaml | 2 +- .../utils/web_scraping_mixin.py | 75 ++- .../test_web_scraping_mixin_remoteobject.py | 443 ++++++++++++++---- 4 files changed, 429 insertions(+), 100 deletions(-) diff --git a/src/kleinanzeigen_bot/__init__.py b/src/kleinanzeigen_bot/__init__.py index 772e183..2a06de6 100644 --- a/src/kleinanzeigen_bot/__init__.py +++ b/src/kleinanzeigen_bot/__init__.py @@ -803,9 +803,12 @@ class KleinanzeigenBot(WebScrapingMixin): pass if ad_cfg.price: if mode == AdUpdateStrategy.MODIFY: - # we have to clear the input, otherwise input gets appended - await self.web_input(By.CSS_SELECTOR, - "input#post-ad-frontend-price, input#micro-frontend-price, input#pstad-price", "") + # Clear the price field first to prevent concatenation of old and new values + # This is needed because some input fields don't clear properly with just clear_input() + price_field = await self.web_find(By.CSS_SELECTOR, "input#post-ad-frontend-price, input#micro-frontend-price, input#pstad-price") + await price_field.clear_input() + await price_field.send_keys("") # Ensure field is completely empty + await self.web_sleep(500) # Brief pause to ensure clearing is complete await self.web_input(By.CSS_SELECTOR, "input#post-ad-frontend-price, input#micro-frontend-price, input#pstad-price", str(ad_cfg.price)) ############################# diff --git a/src/kleinanzeigen_bot/resources/translations.de.yaml b/src/kleinanzeigen_bot/resources/translations.de.yaml index b6fea94..a29032a 100644 --- a/src/kleinanzeigen_bot/resources/translations.de.yaml +++ b/src/kleinanzeigen_bot/resources/translations.de.yaml @@ -391,7 +391,7 @@ kleinanzeigen_bot/utils/web_scraping_mixin.py: "4. Check browser binary permissions: %s": "4. Überprüfen Sie die Browser-Binärdatei-Berechtigungen: %s" "4. Check if any antivirus or security software is blocking the connection": "4. Überprüfen Sie, ob Antiviren- oder Sicherheitssoftware die Verbindung blockiert" - web_execute: + _convert_remote_object_result: "Failed to convert RemoteObject to dict: %s": "Fehler beim Konvertieren von RemoteObject zu dict: %s" web_check: diff --git a/src/kleinanzeigen_bot/utils/web_scraping_mixin.py b/src/kleinanzeigen_bot/utils/web_scraping_mixin.py index e9fa50d..58a5615 100644 --- a/src/kleinanzeigen_bot/utils/web_scraping_mixin.py +++ b/src/kleinanzeigen_bot/utils/web_scraping_mixin.py @@ -42,6 +42,9 @@ LOG:Final[loggers.Logger] = loggers.get_logger(__name__) # see https://api.jquery.com/category/selectors/ METACHAR_ESCAPER:Final[dict[int, str]] = str.maketrans({ch: f"\\{ch}" for ch in '!"#$%&\'()*+,./:;<=>?@[\\]^`{|}~'}) +# Constants for RemoteObject handling +_REMOTE_OBJECT_TYPE_VALUE_PAIR_SIZE:Final[int] = 2 + def _is_admin() -> bool: """Check if the current process is running with admin/root privileges.""" @@ -574,23 +577,61 @@ class WebScrapingMixin: # Handle nodriver 0.47+ RemoteObject behavior # If result is a RemoteObject with deep_serialized_value, convert it to a dict if hasattr(result, "deep_serialized_value"): - deep_serialized = getattr(result, "deep_serialized_value", None) - if deep_serialized is not None: - try: - # Convert the deep_serialized_value to a regular dict - serialized_data = getattr(deep_serialized, "value", None) - if serialized_data is not None: - if isinstance(serialized_data, list): - # Convert list of [key, value] pairs to dict - return dict(serialized_data) - return serialized_data - except (AttributeError, TypeError, ValueError) as e: - LOG.warning("Failed to convert RemoteObject to dict: %s", e) - # Return the original result if conversion fails - return result + return self._convert_remote_object_result(result) return result + def _convert_remote_object_result(self, result:Any) -> Any: + """ + Converts a RemoteObject result to a regular Python object. + + Handles the deep_serialized_value conversion for nodriver 0.47+ compatibility. + """ + deep_serialized = getattr(result, "deep_serialized_value", None) + if deep_serialized is None: + return result + + try: + # Convert the deep_serialized_value to a regular dict + serialized_data = getattr(deep_serialized, "value", None) + if serialized_data is None: + return result + + if isinstance(serialized_data, list): + # Convert list of [key, value] pairs to dict, handling nested RemoteObjects + converted_dict = {} + for key, value in serialized_data: + converted_dict[key] = self._convert_remote_object_dict(value) + return converted_dict + + if isinstance(serialized_data, dict): + # Handle nested RemoteObject structures like {'type': 'number', 'value': 200} + return self._convert_remote_object_dict(serialized_data) + + return serialized_data + except (AttributeError, TypeError, ValueError) as e: + LOG.warning("Failed to convert RemoteObject to dict: %s", e) + # Return the original result if conversion fails + return result + + def _convert_remote_object_dict(self, data:Any) -> Any: + """ + Recursively converts RemoteObject dict structures to regular Python objects. + + Handles structures like {'type': 'number', 'value': 200} or {'type': 'string', 'value': 'text'}. + """ + if isinstance(data, dict): + # Check if this is a RemoteObject value structure + if "type" in data and "value" in data and len(data) == _REMOTE_OBJECT_TYPE_VALUE_PAIR_SIZE: + return data["value"] + # Recursively convert nested dicts + return {key: self._convert_remote_object_dict(value) for key, value in data.items()} + if isinstance(data, list): + # Recursively convert lists + return [self._convert_remote_object_dict(item) for item in data] + # Return primitive values as-is + return data + async def web_find(self, selector_type:By, selector_value:str, *, parent:Element | None = None, timeout:int | float = 5) -> Element: """ Locates an HTML element by the given selector type and value. @@ -724,10 +765,10 @@ class WebScrapingMixin: await self.page.sleep(duration / 1_000) async def web_request(self, url:str, method:str = "GET", valid_response_codes:int | Iterable[int] = 200, - headers:dict[str, str] | None = None) -> dict[str, Any]: + headers:dict[str, str] | None = None) -> Any: method = method.upper() LOG.debug(" -> HTTP %s [%s]...", method, url) - response = cast(dict[str, Any], await self.page.evaluate(f""" + response = await self.web_execute(f""" fetch("{url}", {{ method: "{method}", redirect: "follow", @@ -743,7 +784,7 @@ class WebScrapingMixin: content: responseText }} }})) - """, await_promise = True, return_by_value = True)) + """) if isinstance(valid_response_codes, int): valid_response_codes = [valid_response_codes] ensure( diff --git a/tests/unit/test_web_scraping_mixin_remoteobject.py b/tests/unit/test_web_scraping_mixin_remoteobject.py index 44bba8a..dc70e92 100644 --- a/tests/unit/test_web_scraping_mixin_remoteobject.py +++ b/tests/unit/test_web_scraping_mixin_remoteobject.py @@ -1,8 +1,12 @@ # SPDX-FileCopyrightText: © Jens Bergmann and contributors # SPDX-License-Identifier: AGPL-3.0-or-later # SPDX-ArtifactOfProjectHomePage: https://github.com/Second-Hand-Friends/kleinanzeigen-bot/ -"""Unit tests for web_scraping_mixin.py RemoteObject handling.""" +"""Unit tests for web_scraping_mixin.py RemoteObject handling. +Tests the conversion of nodriver RemoteObject results to regular Python objects. +""" + +from typing import Any from unittest.mock import AsyncMock, Mock, patch import pytest @@ -14,11 +18,114 @@ class TestWebExecuteRemoteObjectHandling: """Test web_execute method with nodriver 0.47+ RemoteObject behavior.""" @pytest.mark.asyncio - async def test_web_execute_remoteobject_with_deep_serialized_value(self) -> None: - """Test web_execute with RemoteObject that has deep_serialized_value.""" + async def test_web_execute_with_regular_result(self) -> None: + """Test web_execute with regular (non-RemoteObject) result.""" mixin = WebScrapingMixin() - # Mock RemoteObject with deep_serialized_value + with patch.object(mixin, "page") as mock_page: + mock_page.evaluate = AsyncMock(return_value = "regular_result") + + result = await mixin.web_execute("window.test") + + assert result == "regular_result" + + @pytest.mark.asyncio + async def test_web_execute_with_remoteobject_result(self) -> None: + """Test web_execute with RemoteObject result.""" + mixin = WebScrapingMixin() + + # Mock RemoteObject + mock_remote_object = Mock() + mock_remote_object.deep_serialized_value = Mock() + mock_remote_object.deep_serialized_value.value = {"key": "value"} + + with patch.object(mixin, "page") as mock_page: + mock_page.evaluate = AsyncMock(return_value = mock_remote_object) + + result = await mixin.web_execute("window.test") + + assert result == {"key": "value"} + + @pytest.mark.asyncio + async def test_web_execute_remoteobject_with_nested_type_value_structures(self) -> None: + """Test web_execute with RemoteObject containing nested type/value structures.""" + mixin = WebScrapingMixin() + + # Mock RemoteObject with nested type/value structures + mock_remote_object = Mock() + mock_remote_object.deep_serialized_value = Mock() + mock_remote_object.deep_serialized_value.value = [ + ["statusCode", {"type": "number", "value": 200}], + ["content", {"type": "string", "value": "success"}] + ] + + with patch.object(mixin, "page") as mock_page: + mock_page.evaluate = AsyncMock(return_value = mock_remote_object) + + result = await mixin.web_execute("window.test") + + # Should convert nested type/value structures to their values + assert result == {"statusCode": 200, "content": "success"} + + @pytest.mark.asyncio + async def test_web_execute_remoteobject_with_mixed_nested_structures(self) -> None: + """Test web_execute with RemoteObject containing mixed nested structures.""" + mixin = WebScrapingMixin() + + # Mock RemoteObject with mixed nested structures + mock_remote_object = Mock() + mock_remote_object.deep_serialized_value = Mock() + mock_remote_object.deep_serialized_value.value = { + "simple": "value", + "nested": {"type": "number", "value": 42}, + "list": [{"type": "string", "value": "item1"}, {"type": "string", "value": "item2"}] + } + + with patch.object(mixin, "page") as mock_page: + mock_page.evaluate = AsyncMock(return_value = mock_remote_object) + + result = await mixin.web_execute("window.test") + + # Should convert nested structures while preserving simple values + expected = { + "simple": "value", + "nested": 42, + "list": ["item1", "item2"] + } + assert result == expected + + +class TestConvertRemoteObjectResult: + """Test _convert_remote_object_result method for RemoteObject conversion.""" + + def test_convert_remote_object_result_with_none_deep_serialized_value(self) -> None: + """Test _convert_remote_object_result when deep_serialized_value is None.""" + mixin = WebScrapingMixin() + + # Mock RemoteObject with None deep_serialized_value + mock_remote_object = Mock() + mock_remote_object.deep_serialized_value = None + + result = mixin._convert_remote_object_result(mock_remote_object) + assert result == mock_remote_object + + def test_convert_remote_object_result_with_none_serialized_data(self) -> None: + """Test _convert_remote_object_result when serialized_data is None.""" + mixin = WebScrapingMixin() + + # Mock RemoteObject with None serialized_data + mock_remote_object = Mock() + mock_remote_object.deep_serialized_value = Mock() + mock_remote_object.deep_serialized_value.value = None + + result = mixin._convert_remote_object_result(mock_remote_object) + assert result == mock_remote_object + + def test_convert_remote_object_result_with_list_data(self) -> None: + """Test _convert_remote_object_result with list data.""" + mixin = WebScrapingMixin() + + # Mock RemoteObject with list data mock_remote_object = Mock() mock_remote_object.deep_serialized_value = Mock() mock_remote_object.deep_serialized_value.value = [ @@ -26,105 +133,283 @@ class TestWebExecuteRemoteObjectHandling: ["key2", "value2"] ] - # Mock the page evaluation to return our RemoteObject - with patch.object(mixin, "page") as mock_page: - mock_page.evaluate = AsyncMock(return_value = mock_remote_object) + result = mixin._convert_remote_object_result(mock_remote_object) + assert result == {"key1": "value1", "key2": "value2"} - result = await mixin.web_execute("window.test") - - # Should convert the RemoteObject to a dict - assert result == {"key1": "value1", "key2": "value2"} - - @pytest.mark.asyncio - async def test_web_execute_remoteobject_with_none_deep_serialized_value(self) -> None: - """Test web_execute with RemoteObject that has None deep_serialized_value.""" + def test_convert_remote_object_result_with_dict_data(self) -> None: + """Test _convert_remote_object_result with dict data.""" mixin = WebScrapingMixin() - # Mock RemoteObject with None deep_serialized_value - mock_remote_object = Mock() - mock_remote_object.deep_serialized_value = None - - with patch.object(mixin, "page") as mock_page: - mock_page.evaluate = AsyncMock(return_value = mock_remote_object) - - result = await mixin.web_execute("window.test") - - # Should return the original RemoteObject - assert result is mock_remote_object - - @pytest.mark.asyncio - async def test_web_execute_remoteobject_with_non_list_serialized_data(self) -> None: - """Test web_execute with RemoteObject that has non-list serialized data.""" - mixin = WebScrapingMixin() - - # Mock RemoteObject with non-list serialized data + # Mock RemoteObject with dict data mock_remote_object = Mock() mock_remote_object.deep_serialized_value = Mock() - mock_remote_object.deep_serialized_value.value = {"direct": "dict"} + mock_remote_object.deep_serialized_value.value = {"key": "value"} - with patch.object(mixin, "page") as mock_page: - mock_page.evaluate = AsyncMock(return_value = mock_remote_object) + result = mixin._convert_remote_object_result(mock_remote_object) + assert result == {"key": "value"} - result = await mixin.web_execute("window.test") - - # Should return the serialized data directly - assert result == {"direct": "dict"} - - @pytest.mark.asyncio - async def test_web_execute_remoteobject_with_none_serialized_data(self) -> None: - """Test web_execute with RemoteObject that has None serialized data.""" + def test_convert_remote_object_result_with_conversion_error(self) -> None: + """Test _convert_remote_object_result when conversion raises an exception.""" mixin = WebScrapingMixin() - # Mock RemoteObject with None serialized data + # Mock RemoteObject that will raise an exception during conversion mock_remote_object = Mock() mock_remote_object.deep_serialized_value = Mock() - mock_remote_object.deep_serialized_value.value = None + mock_remote_object.deep_serialized_value.value = "invalid_data" - with patch.object(mixin, "page") as mock_page: - mock_page.evaluate = AsyncMock(return_value = mock_remote_object) + # Mock the _convert_remote_object_dict to raise an exception + with patch.object(mixin, "_convert_remote_object_dict", side_effect = ValueError("Test error")): + result = mixin._convert_remote_object_result(mock_remote_object) + # When conversion fails, it should return the original value + assert result == "invalid_data" - result = await mixin.web_execute("window.test") - # Should return the original RemoteObject - assert result is mock_remote_object +class TestConvertRemoteObjectDict: + """Test _convert_remote_object_dict method for nested RemoteObject conversion.""" - @pytest.mark.asyncio - async def test_web_execute_regular_result(self) -> None: - """Test web_execute with regular result (no RemoteObject).""" + def test_convert_remote_object_dict_with_type_value_pair(self) -> None: + """Test conversion of type/value pair structures.""" mixin = WebScrapingMixin() - # Mock regular result (no deep_serialized_value attribute) - mock_result = {"regular": "dict"} + # Test type/value pair + data = {"type": "number", "value": 200} + result = mixin._convert_remote_object_dict(data) + assert result == 200 - with patch.object(mixin, "page") as mock_page: - mock_page.evaluate = AsyncMock(return_value = mock_result) + # Test string type/value pair + data = {"type": "string", "value": "hello"} + result = mixin._convert_remote_object_dict(data) + assert result == "hello" - result = await mixin.web_execute("window.test") - - # Should return the result unchanged - assert result == mock_result - - @pytest.mark.asyncio - async def test_web_execute_remoteobject_conversion_exception(self) -> None: - """Test web_execute with RemoteObject that raises exception during conversion.""" + def test_convert_remote_object_dict_with_regular_dict(self) -> None: + """Test conversion of regular dict structures.""" mixin = WebScrapingMixin() - # Mock RemoteObject that will raise an exception when trying to convert to dict + # Test regular dict (not type/value pair) + data = {"key1": "value1", "key2": "value2"} + result = mixin._convert_remote_object_dict(data) + assert result == {"key1": "value1", "key2": "value2"} + + def test_convert_remote_object_dict_with_nested_structures(self) -> None: + """Test conversion of nested dict structures.""" + mixin = WebScrapingMixin() + + # Test nested structures + data = { + "simple": "value", + "nested": {"type": "number", "value": 42}, + "list": [{"type": "string", "value": "item1"}, {"type": "string", "value": "item2"}] + } + result = mixin._convert_remote_object_dict(data) + + expected = { + "simple": "value", + "nested": 42, + "list": ["item1", "item2"] + } + assert result == expected + + def test_convert_remote_object_dict_with_list(self) -> None: + """Test conversion of list structures.""" + mixin = WebScrapingMixin() + + # Test list with type/value pairs + data = [{"type": "number", "value": 1}, {"type": "string", "value": "test"}] + result = mixin._convert_remote_object_dict(data) + assert result == [1, "test"] + + def test_convert_remote_object_dict_with_primitive_values(self) -> None: + """Test conversion with primitive values.""" + mixin = WebScrapingMixin() + + # Test primitive values + assert mixin._convert_remote_object_dict("string") == "string" + assert mixin._convert_remote_object_dict(42) == 42 + assert mixin._convert_remote_object_dict(True) is True + assert mixin._convert_remote_object_dict(None) is None + + def test_convert_remote_object_dict_with_complex_nested_structures(self) -> None: + """Test conversion with complex nested structures.""" + mixin = WebScrapingMixin() + + # Test complex nested structures + data = { + "response": { + "status": {"type": "number", "value": 200}, + "data": [ + {"type": "string", "value": "item1"}, + {"type": "string", "value": "item2"} + ], + "metadata": { + "count": {"type": "number", "value": 2}, + "type": {"type": "string", "value": "list"} + } + } + } + result = mixin._convert_remote_object_dict(data) + + expected = { + "response": { + "status": 200, + "data": ["item1", "item2"], + "metadata": { + "count": 2, + "type": "list" + } + } + } + assert result == expected + + +class TestWebRequestRemoteObjectHandling: + """Test web_request method with nodriver 0.47+ RemoteObject behavior.""" + + @pytest.mark.asyncio + async def test_web_request_with_remoteobject_result(self) -> None: + """Test web_request with RemoteObject result to catch subscriptability issues.""" + mixin = WebScrapingMixin() + + # Mock RemoteObject that simulates the exact structure returned by web_request mock_remote_object = Mock() - mock_deep_serialized = Mock() + mock_remote_object.deep_serialized_value = Mock() + mock_remote_object.deep_serialized_value.value = { + "statusCode": 200, + "statusMessage": "OK", + "headers": {"content-type": "application/json"}, + "content": '{"success": true}' + } - # Create a list-like object that raises an exception when dict() is called on it - class ExceptionRaisingList(list[str]): - def __iter__(self) -> None: # type: ignore[override] - raise ValueError("Simulated conversion error") + with patch.object(mixin, "web_execute") as mock_web_execute: + # Mock web_execute to return the converted result (simulating our fix) + mock_web_execute.return_value = { + "statusCode": 200, + "statusMessage": "OK", + "headers": {"content-type": "application/json"}, + "content": '{"success": true}' + } - mock_deep_serialized.value = ExceptionRaisingList([["key", "value"]]) # type: ignore[list-item] - mock_remote_object.deep_serialized_value = mock_deep_serialized + result = await mixin.web_request("https://example.com/api") - with patch.object(mixin, "page") as mock_page: - mock_page.evaluate = AsyncMock(return_value = mock_remote_object) + # Verify the result is properly converted and subscriptable + assert result["statusCode"] == 200 + assert result["statusMessage"] == "OK" + assert result["headers"]["content-type"] == "application/json" + assert result["content"] == '{"success": true}' - result = await mixin.web_execute("window.test") + @pytest.mark.asyncio + async def test_web_request_with_remoteobject_error_response(self) -> None: + """Test web_request with RemoteObject error response.""" + mixin = WebScrapingMixin() - # Should return the original RemoteObject when conversion fails - assert result is mock_remote_object + # Mock RemoteObject for error response + mock_remote_object = Mock() + mock_remote_object.deep_serialized_value = Mock() + mock_remote_object.deep_serialized_value.value = { + "statusCode": 404, + "statusMessage": "Not Found", + "headers": {"content-type": "text/html"}, + "content": "Not Found" + } + + with patch.object(mixin, "web_execute") as mock_web_execute: + # Mock web_execute to return the converted result (simulating our fix) + mock_web_execute.return_value = { + "statusCode": 404, + "statusMessage": "Not Found", + "headers": {"content-type": "text/html"}, + "content": "Not Found" + } + + # This should raise an exception due to invalid status code + with pytest.raises(Exception, match = "Invalid response"): + await mixin.web_request("https://example.com/api", valid_response_codes = [200]) + + @pytest.mark.asyncio + async def test_web_request_with_nested_remoteobject_structures(self) -> None: + """Test web_request with complex nested RemoteObject structures.""" + mixin = WebScrapingMixin() + + # Mock RemoteObject with nested type/value structures + mock_remote_object = Mock() + mock_remote_object.deep_serialized_value = Mock() + mock_remote_object.deep_serialized_value.value = { + "statusCode": {"type": "number", "value": 200}, + "statusMessage": {"type": "string", "value": "OK"}, + "headers": { + "content-type": {"type": "string", "value": "application/json"} + }, + "content": {"type": "string", "value": '{"data": "test"}'} + } + + with patch.object(mixin, "web_execute") as mock_web_execute: + # Mock web_execute to return the converted result (simulating our fix) + mock_web_execute.return_value = { + "statusCode": 200, + "statusMessage": "OK", + "headers": {"content-type": "application/json"}, + "content": '{"data": "test"}' + } + + result = await mixin.web_request("https://example.com/api") + + # Verify nested structures are properly converted + assert result["statusCode"] == 200 + assert result["statusMessage"] == "OK" + assert result["headers"]["content-type"] == "application/json" + assert result["content"] == '{"data": "test"}' + + +class TestWebRequestRemoteObjectRegression: + """Test web_request method to catch future RemoteObject regression issues.""" + + @pytest.mark.asyncio + async def test_web_request_without_remoteobject_conversion_fails(self) -> None: + """Test that web_request fails without RemoteObject conversion (regression test).""" + mixin = WebScrapingMixin() + + # Mock RemoteObject that would cause the original error + mock_remote_object = Mock() + mock_remote_object.deep_serialized_value = Mock() + mock_remote_object.deep_serialized_value.value = { + "statusCode": 200, + "statusMessage": "OK", + "headers": {"content-type": "application/json"}, + "content": '{"success": true}' + } + + # Mock web_execute to return the raw RemoteObject (simulating the bug) + with patch.object(mixin, "web_execute") as mock_web_execute: + mock_web_execute.return_value = mock_remote_object + + # This should fail with the original error if our fix is removed + with pytest.raises(TypeError, match = "object is not subscriptable"): + await mixin.web_request("https://example.com/api") + + @pytest.mark.asyncio + async def test_web_request_with_remoteobject_conversion_succeeds(self) -> None: + """Test that web_request succeeds with RemoteObject conversion (our fix).""" + mixin = WebScrapingMixin() + + # Mock RemoteObject + mock_remote_object = Mock() + mock_remote_object.deep_serialized_value = Mock() + mock_remote_object.deep_serialized_value.value = { + "statusCode": 200, + "statusMessage": "OK", + "headers": {"content-type": "application/json"}, + "content": '{"success": true}' + } + + # Mock web_execute to simulate our conversion logic + async def mock_web_execute_with_conversion(script:str) -> Any: + # Simulate the conversion that happens in our fix + return mock_remote_object.deep_serialized_value.value + + with patch.object(mixin, "web_execute", side_effect = mock_web_execute_with_conversion): + result = await mixin.web_request("https://example.com/api") + + # Verify the result works correctly + assert result["statusCode"] == 200 + assert result["statusMessage"] == "OK" + assert result["headers"]["content-type"] == "application/json" + assert result["content"] == '{"success": true}'