mirror of
https://github.com/Second-Hand-Friends/kleinanzeigen-bot.git
synced 2026-03-12 02:31:45 +01:00
feat: Replace custom RemoteObject wrapper with direct NoDriver 0.47+ usage (#652)
## ℹ️ Description *Replace custom RemoteObject serialization wrapper with direct NoDriver 0.47+ RemoteObject API usage for better performance and maintainability.* - **Motivation**: The custom wrapper was unnecessary complexity when NoDriver 0.47+ provides direct RemoteObject API - **Context**: Upgrading from NoDriver 0.39 to 0.47 introduced RemoteObject, and we want to use it as intended - **Goal**: Future-proof implementation using the standard NoDriver patterns ## 📋 Changes Summary - Replace custom serialization wrapper with direct RemoteObject API usage - Implement proper RemoteObject detection and conversion in web_execute() - Add comprehensive _convert_remote_object_value() method for recursive conversion - Handle key/value list format from deep_serialized_value.value - Add type guards and proper type checking for RemoteObject instances - Maintain internal API stability while using RemoteObject as intended - Add 19 comprehensive test cases covering all conversion scenarios - Application tested and working with real ad download, update and publish ### ⚙️ Type of Change - [x] ✨ New feature (adds new functionality without breaking existing usage) - [x] 🐞 Bug fix (non-breaking change which fixes an issue) ## ✅ Checklist - [x] I have reviewed my changes to ensure they meet the project's standards. - [x] I have tested my changes and ensured that all tests pass (pdm run test). - [x] I have formatted the code (pdm run format). - [x] I have verified that linting passes (pdm run lint). - [x] I have updated documentation where necessary. By submitting this pull request, I confirm that you can use, modify, copy, and redistribute this contribution, under the terms of your choice.
This commit is contained in:
Jens
GitHub
@@ -1,25 +1,25 @@
|
||||
# SPDX-FileCopyrightText: © Jens Bergmann and contributors
|
||||
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
# SPDX-ArtifactOfProjectHomePage: https://github.com/Second-Hand-Friends/kleinanzeigen-bot/
|
||||
"""Unit tests for web_scraping_mixin.py RemoteObject handling.
|
||||
"""Unit tests for web_scraping_mixin.py JavaScript serialization handling.
|
||||
|
||||
Tests the conversion of nodriver RemoteObject results to regular Python objects.
|
||||
Tests the JSON serialization approach to ensure regular Python objects are returned.
|
||||
"""
|
||||
|
||||
from typing import Any
|
||||
from unittest.mock import AsyncMock, Mock, patch
|
||||
from unittest.mock import AsyncMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from kleinanzeigen_bot.utils.web_scraping_mixin import WebScrapingMixin
|
||||
|
||||
|
||||
class TestWebExecuteRemoteObjectHandling:
|
||||
"""Test web_execute method with nodriver 0.47+ RemoteObject behavior."""
|
||||
class TestWebExecuteJavaScriptSerialization:
|
||||
"""Test web_execute method with JSON serialization approach."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_web_execute_with_regular_result(self) -> None:
|
||||
"""Test web_execute with regular (non-RemoteObject) result."""
|
||||
"""Test web_execute with regular result."""
|
||||
mixin = WebScrapingMixin()
|
||||
|
||||
with patch.object(mixin, "page") as mock_page:
|
||||
@@ -30,386 +30,240 @@ class TestWebExecuteRemoteObjectHandling:
|
||||
assert result == "regular_result"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_web_execute_with_remoteobject_result(self) -> None:
|
||||
"""Test web_execute with RemoteObject result."""
|
||||
async def test_web_execute_with_dict_result(self) -> None:
|
||||
"""Test web_execute with dict result."""
|
||||
mixin = WebScrapingMixin()
|
||||
|
||||
# Mock RemoteObject
|
||||
mock_remote_object = Mock()
|
||||
mock_remote_object.deep_serialized_value = Mock()
|
||||
mock_remote_object.deep_serialized_value.value = {"key": "value"}
|
||||
|
||||
with patch.object(mixin, "page") as mock_page:
|
||||
mock_page.evaluate = AsyncMock(return_value = mock_remote_object)
|
||||
mock_page.evaluate = AsyncMock(return_value = {"key": "value"})
|
||||
|
||||
result = await mixin.web_execute("window.test")
|
||||
|
||||
assert result == {"key": "value"}
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_web_execute_remoteobject_with_nested_type_value_structures(self) -> None:
|
||||
"""Test web_execute with RemoteObject containing nested type/value structures."""
|
||||
async def test_web_execute_with_complex_dict_result(self) -> None:
|
||||
"""Test web_execute with complex dict result."""
|
||||
mixin = WebScrapingMixin()
|
||||
|
||||
# Mock RemoteObject with nested type/value structures
|
||||
mock_remote_object = Mock()
|
||||
mock_remote_object.deep_serialized_value = Mock()
|
||||
mock_remote_object.deep_serialized_value.value = [
|
||||
["statusCode", {"type": "number", "value": 200}],
|
||||
["content", {"type": "string", "value": "success"}]
|
||||
]
|
||||
|
||||
with patch.object(mixin, "page") as mock_page:
|
||||
mock_page.evaluate = AsyncMock(return_value = mock_remote_object)
|
||||
mock_page.evaluate = AsyncMock(return_value = {
|
||||
"statusCode": 200,
|
||||
"content": "success",
|
||||
"nested": {"key": "value"}
|
||||
})
|
||||
|
||||
result = await mixin.web_execute("window.test")
|
||||
|
||||
# Should convert nested type/value structures to their values
|
||||
assert result == {"statusCode": 200, "content": "success"}
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_web_execute_remoteobject_with_mixed_nested_structures(self) -> None:
|
||||
"""Test web_execute with RemoteObject containing mixed nested structures."""
|
||||
mixin = WebScrapingMixin()
|
||||
|
||||
# Mock RemoteObject with mixed nested structures
|
||||
mock_remote_object = Mock()
|
||||
mock_remote_object.deep_serialized_value = Mock()
|
||||
mock_remote_object.deep_serialized_value.value = {
|
||||
"simple": "value",
|
||||
"nested": {"type": "number", "value": 42},
|
||||
"list": [{"type": "string", "value": "item1"}, {"type": "string", "value": "item2"}]
|
||||
}
|
||||
|
||||
with patch.object(mixin, "page") as mock_page:
|
||||
mock_page.evaluate = AsyncMock(return_value = mock_remote_object)
|
||||
|
||||
result = await mixin.web_execute("window.test")
|
||||
|
||||
# Should convert nested structures while preserving simple values
|
||||
expected = {
|
||||
"simple": "value",
|
||||
"nested": 42,
|
||||
"list": ["item1", "item2"]
|
||||
"statusCode": 200,
|
||||
"content": "success",
|
||||
"nested": {"key": "value"}
|
||||
}
|
||||
assert result == expected
|
||||
|
||||
|
||||
class TestConvertRemoteObjectResult:
|
||||
"""Test _convert_remote_object_result method for RemoteObject conversion."""
|
||||
|
||||
def test_convert_remote_object_result_with_none_deep_serialized_value(self) -> None:
|
||||
"""Test _convert_remote_object_result when deep_serialized_value is None."""
|
||||
@pytest.mark.asyncio
|
||||
async def test_web_execute_with_remoteobject_conversion(self) -> None:
|
||||
"""Test web_execute with RemoteObject conversion."""
|
||||
mixin = WebScrapingMixin()
|
||||
|
||||
# Mock RemoteObject with None deep_serialized_value
|
||||
mock_remote_object = Mock()
|
||||
mock_remote_object.deep_serialized_value = None
|
||||
# Test the _convert_remote_object_value method directly
|
||||
test_data = [["key1", "value1"], ["key2", "value2"]]
|
||||
result = mixin._convert_remote_object_value(test_data)
|
||||
|
||||
result = mixin._convert_remote_object_result(mock_remote_object)
|
||||
assert result == mock_remote_object
|
||||
|
||||
def test_convert_remote_object_result_with_none_serialized_data(self) -> None:
|
||||
"""Test _convert_remote_object_result when serialized_data is None."""
|
||||
mixin = WebScrapingMixin()
|
||||
|
||||
# Mock RemoteObject with None serialized_data
|
||||
mock_remote_object = Mock()
|
||||
mock_remote_object.deep_serialized_value = Mock()
|
||||
mock_remote_object.deep_serialized_value.value = None
|
||||
|
||||
result = mixin._convert_remote_object_result(mock_remote_object)
|
||||
assert result == mock_remote_object
|
||||
|
||||
def test_convert_remote_object_result_with_list_data(self) -> None:
|
||||
"""Test _convert_remote_object_result with list data."""
|
||||
mixin = WebScrapingMixin()
|
||||
|
||||
# Mock RemoteObject with list data
|
||||
mock_remote_object = Mock()
|
||||
mock_remote_object.deep_serialized_value = Mock()
|
||||
mock_remote_object.deep_serialized_value.value = [
|
||||
["key1", "value1"],
|
||||
["key2", "value2"]
|
||||
]
|
||||
|
||||
result = mixin._convert_remote_object_result(mock_remote_object)
|
||||
# Should convert key/value list to dict
|
||||
assert result == {"key1": "value1", "key2": "value2"}
|
||||
|
||||
def test_convert_remote_object_result_with_dict_data(self) -> None:
|
||||
"""Test _convert_remote_object_result with dict data."""
|
||||
def test_convert_remote_object_value_key_value_list(self) -> None:
|
||||
"""Test _convert_remote_object_value with key/value list format."""
|
||||
mixin = WebScrapingMixin()
|
||||
|
||||
# Mock RemoteObject with dict data
|
||||
mock_remote_object = Mock()
|
||||
mock_remote_object.deep_serialized_value = Mock()
|
||||
mock_remote_object.deep_serialized_value.value = {"key": "value"}
|
||||
|
||||
result = mixin._convert_remote_object_result(mock_remote_object)
|
||||
assert result == {"key": "value"}
|
||||
|
||||
def test_convert_remote_object_result_with_conversion_error(self) -> None:
|
||||
"""Test _convert_remote_object_result when conversion raises an exception."""
|
||||
mixin = WebScrapingMixin()
|
||||
|
||||
# Mock RemoteObject that will raise an exception during conversion
|
||||
mock_remote_object = Mock()
|
||||
mock_remote_object.deep_serialized_value = Mock()
|
||||
mock_remote_object.deep_serialized_value.value = "invalid_data"
|
||||
|
||||
# Mock the _convert_remote_object_dict to raise an exception
|
||||
with patch.object(mixin, "_convert_remote_object_dict", side_effect = ValueError("Test error")):
|
||||
result = mixin._convert_remote_object_result(mock_remote_object)
|
||||
# When conversion fails, it should return the original value
|
||||
assert result == "invalid_data"
|
||||
|
||||
|
||||
class TestConvertRemoteObjectDict:
|
||||
"""Test _convert_remote_object_dict method for nested RemoteObject conversion."""
|
||||
|
||||
def test_convert_remote_object_dict_with_type_value_pair(self) -> None:
|
||||
"""Test conversion of type/value pair structures."""
|
||||
mixin = WebScrapingMixin()
|
||||
|
||||
# Test type/value pair
|
||||
data = {"type": "number", "value": 200}
|
||||
result = mixin._convert_remote_object_dict(data)
|
||||
assert result == 200
|
||||
|
||||
# Test string type/value pair
|
||||
data = {"type": "string", "value": "hello"}
|
||||
result = mixin._convert_remote_object_dict(data)
|
||||
assert result == "hello"
|
||||
|
||||
def test_convert_remote_object_dict_with_regular_dict(self) -> None:
|
||||
"""Test conversion of regular dict structures."""
|
||||
mixin = WebScrapingMixin()
|
||||
|
||||
# Test regular dict (not type/value pair)
|
||||
data = {"key1": "value1", "key2": "value2"}
|
||||
result = mixin._convert_remote_object_dict(data)
|
||||
# Test key/value list format
|
||||
test_data = [["key1", "value1"], ["key2", "value2"]]
|
||||
result = mixin._convert_remote_object_value(test_data)
|
||||
assert result == {"key1": "value1", "key2": "value2"}
|
||||
|
||||
def test_convert_remote_object_dict_with_nested_structures(self) -> None:
|
||||
"""Test conversion of nested dict structures."""
|
||||
def test_convert_remote_object_value_with_nested_type_value(self) -> None:
|
||||
"""Test _convert_remote_object_value with nested type/value structures."""
|
||||
mixin = WebScrapingMixin()
|
||||
|
||||
# Test nested structures
|
||||
data = {
|
||||
"simple": "value",
|
||||
"nested": {"type": "number", "value": 42},
|
||||
"list": [{"type": "string", "value": "item1"}, {"type": "string", "value": "item2"}]
|
||||
}
|
||||
result = mixin._convert_remote_object_dict(data)
|
||||
# Test with nested type/value structures
|
||||
test_data = [["key1", {"type": "string", "value": "nested_value"}]]
|
||||
result = mixin._convert_remote_object_value(test_data)
|
||||
assert result == {"key1": "nested_value"}
|
||||
|
||||
expected = {
|
||||
"simple": "value",
|
||||
"nested": 42,
|
||||
"list": ["item1", "item2"]
|
||||
}
|
||||
assert result == expected
|
||||
|
||||
def test_convert_remote_object_dict_with_list(self) -> None:
|
||||
"""Test conversion of list structures."""
|
||||
def test_convert_remote_object_value_regular_list(self) -> None:
|
||||
"""Test _convert_remote_object_value with regular list."""
|
||||
mixin = WebScrapingMixin()
|
||||
|
||||
# Test list with type/value pairs
|
||||
data = [{"type": "number", "value": 1}, {"type": "string", "value": "test"}]
|
||||
result = mixin._convert_remote_object_dict(data)
|
||||
assert result == [1, "test"]
|
||||
# Test regular list (not key/value format)
|
||||
test_data = ["item1", "item2", "item3"]
|
||||
result = mixin._convert_remote_object_value(test_data)
|
||||
assert result == ["item1", "item2", "item3"]
|
||||
|
||||
def test_convert_remote_object_dict_with_primitive_values(self) -> None:
|
||||
"""Test conversion with primitive values."""
|
||||
def test_convert_remote_object_value_nested_list(self) -> None:
|
||||
"""Test _convert_remote_object_value with nested list."""
|
||||
mixin = WebScrapingMixin()
|
||||
|
||||
# Test nested list that looks like key/value pairs (gets converted to dict)
|
||||
test_data = [["nested", "list"], ["another", "item"]]
|
||||
result = mixin._convert_remote_object_value(test_data)
|
||||
assert result == {"nested": "list", "another": "item"}
|
||||
|
||||
def test_convert_remote_object_value_type_value_dict(self) -> None:
|
||||
"""Test _convert_remote_object_value with type/value dict."""
|
||||
mixin = WebScrapingMixin()
|
||||
|
||||
# Test type/value dict
|
||||
test_data = {"type": "string", "value": "actual_value"}
|
||||
result = mixin._convert_remote_object_value(test_data)
|
||||
assert result == "actual_value"
|
||||
|
||||
def test_convert_remote_object_value_regular_dict(self) -> None:
|
||||
"""Test _convert_remote_object_value with regular dict."""
|
||||
mixin = WebScrapingMixin()
|
||||
|
||||
# Test regular dict
|
||||
test_data = {"key1": "value1", "key2": "value2"}
|
||||
result = mixin._convert_remote_object_value(test_data)
|
||||
assert result == {"key1": "value1", "key2": "value2"}
|
||||
|
||||
def test_convert_remote_object_value_nested_dict(self) -> None:
|
||||
"""Test _convert_remote_object_value with nested dict."""
|
||||
mixin = WebScrapingMixin()
|
||||
|
||||
# Test nested dict
|
||||
test_data = {"key1": {"nested": "value"}, "key2": "value2"}
|
||||
result = mixin._convert_remote_object_value(test_data)
|
||||
assert result == {"key1": {"nested": "value"}, "key2": "value2"}
|
||||
|
||||
def test_convert_remote_object_value_primitive(self) -> None:
|
||||
"""Test _convert_remote_object_value with primitive values."""
|
||||
mixin = WebScrapingMixin()
|
||||
|
||||
# Test primitive values
|
||||
assert mixin._convert_remote_object_dict("string") == "string"
|
||||
assert mixin._convert_remote_object_dict(42) == 42
|
||||
assert mixin._convert_remote_object_dict(True) is True
|
||||
assert mixin._convert_remote_object_dict(None) is None
|
||||
assert mixin._convert_remote_object_value("string") == "string"
|
||||
assert mixin._convert_remote_object_value(123) == 123
|
||||
assert mixin._convert_remote_object_value(True) is True
|
||||
assert mixin._convert_remote_object_value(None) is None
|
||||
|
||||
def test_convert_remote_object_dict_with_complex_nested_structures(self) -> None:
|
||||
"""Test conversion with complex nested structures."""
|
||||
def test_convert_remote_object_value_malformed_key_value_pair(self) -> None:
|
||||
"""Test _convert_remote_object_value with malformed key/value pairs."""
|
||||
mixin = WebScrapingMixin()
|
||||
|
||||
# Test complex nested structures
|
||||
data = {
|
||||
"response": {
|
||||
"status": {"type": "number", "value": 200},
|
||||
"data": [
|
||||
{"type": "string", "value": "item1"},
|
||||
{"type": "string", "value": "item2"}
|
||||
],
|
||||
"metadata": {
|
||||
"count": {"type": "number", "value": 2},
|
||||
"type": {"type": "string", "value": "list"}
|
||||
}
|
||||
}
|
||||
}
|
||||
result = mixin._convert_remote_object_dict(data)
|
||||
# Test with malformed key/value pairs (wrong length)
|
||||
test_data = [["key1", "value1"], ["key2"]] # Second item has wrong length
|
||||
result = mixin._convert_remote_object_value(test_data)
|
||||
# Should still convert the valid pairs and skip malformed ones
|
||||
assert result == {"key1": "value1"}
|
||||
|
||||
def test_convert_remote_object_value_empty_list(self) -> None:
|
||||
"""Test _convert_remote_object_value with empty list."""
|
||||
mixin = WebScrapingMixin()
|
||||
|
||||
# Test empty list
|
||||
test_data:list[Any] = []
|
||||
result = mixin._convert_remote_object_value(test_data)
|
||||
assert result == []
|
||||
|
||||
def test_convert_remote_object_value_complex_nested_structure(self) -> None:
|
||||
"""Test _convert_remote_object_value with complex nested structure."""
|
||||
mixin = WebScrapingMixin()
|
||||
|
||||
# Test complex nested structure
|
||||
test_data = [
|
||||
["key1", "value1"],
|
||||
["key2", {"type": "object", "value": {"nested": "value"}}],
|
||||
["key3", [["inner_key", "inner_value"]]]
|
||||
]
|
||||
result = mixin._convert_remote_object_value(test_data)
|
||||
expected = {
|
||||
"response": {
|
||||
"status": 200,
|
||||
"data": ["item1", "item2"],
|
||||
"metadata": {
|
||||
"count": 2,
|
||||
"type": "list"
|
||||
}
|
||||
}
|
||||
"key1": "value1",
|
||||
"key2": {"nested": "value"},
|
||||
"key3": {"inner_key": "inner_value"} # The inner list gets converted to dict too
|
||||
}
|
||||
assert result == expected
|
||||
|
||||
|
||||
class TestWebRequestRemoteObjectHandling:
|
||||
"""Test web_request method with nodriver 0.47+ RemoteObject behavior."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_web_request_with_remoteobject_result(self) -> None:
|
||||
"""Test web_request with RemoteObject result to catch subscriptability issues."""
|
||||
async def test_web_execute_remoteobject_exception_handling(self) -> None:
|
||||
"""Test web_execute with RemoteObject exception handling."""
|
||||
mixin = WebScrapingMixin()
|
||||
|
||||
# Mock RemoteObject that simulates the exact structure returned by web_request
|
||||
mock_remote_object = Mock()
|
||||
mock_remote_object.deep_serialized_value = Mock()
|
||||
mock_remote_object.deep_serialized_value.value = {
|
||||
"statusCode": 200,
|
||||
"statusMessage": "OK",
|
||||
"headers": {"content-type": "application/json"},
|
||||
"content": '{"success": true}'
|
||||
}
|
||||
# Create a mock RemoteObject that will raise an exception
|
||||
mock_remote_object = type("MockRemoteObject", (), {
|
||||
"__class__": type("MockClass", (), {"__name__": "RemoteObject"}),
|
||||
"value": None,
|
||||
"deep_serialized_value": None
|
||||
})()
|
||||
|
||||
with patch.object(mixin, "web_execute") as mock_web_execute:
|
||||
# Mock web_execute to return the converted result (simulating our fix)
|
||||
mock_web_execute.return_value = {
|
||||
"statusCode": 200,
|
||||
"statusMessage": "OK",
|
||||
"headers": {"content-type": "application/json"},
|
||||
"content": '{"success": true}'
|
||||
}
|
||||
with patch.object(mixin, "page") as mock_page:
|
||||
mock_page.evaluate = AsyncMock(return_value = mock_remote_object)
|
||||
|
||||
result = await mixin.web_request("https://example.com/api")
|
||||
# Mock the _convert_remote_object_value to raise an exception
|
||||
with patch.object(mixin, "_convert_remote_object_value", side_effect = Exception("Test exception")):
|
||||
result = await mixin.web_execute("window.test")
|
||||
|
||||
# Verify the result is properly converted and subscriptable
|
||||
assert result["statusCode"] == 200
|
||||
assert result["statusMessage"] == "OK"
|
||||
assert result["headers"]["content-type"] == "application/json"
|
||||
assert result["content"] == '{"success": true}'
|
||||
# Should return the original RemoteObject when exception occurs
|
||||
assert result == mock_remote_object
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_web_request_with_remoteobject_error_response(self) -> None:
|
||||
"""Test web_request with RemoteObject error response."""
|
||||
async def test_web_execute_remoteobject_with_value(self) -> None:
|
||||
"""Test web_execute with RemoteObject that has a value."""
|
||||
mixin = WebScrapingMixin()
|
||||
|
||||
# Mock RemoteObject for error response
|
||||
mock_remote_object = Mock()
|
||||
mock_remote_object.deep_serialized_value = Mock()
|
||||
mock_remote_object.deep_serialized_value.value = {
|
||||
"statusCode": 404,
|
||||
"statusMessage": "Not Found",
|
||||
"headers": {"content-type": "text/html"},
|
||||
"content": "<html>Not Found</html>"
|
||||
}
|
||||
# Create a mock RemoteObject with a value
|
||||
mock_remote_object = type("MockRemoteObject", (), {
|
||||
"__class__": type("MockClass", (), {"__name__": "RemoteObject"}),
|
||||
"value": "test_value",
|
||||
"deep_serialized_value": None
|
||||
})()
|
||||
|
||||
with patch.object(mixin, "web_execute") as mock_web_execute:
|
||||
# Mock web_execute to return the converted result (simulating our fix)
|
||||
mock_web_execute.return_value = {
|
||||
"statusCode": 404,
|
||||
"statusMessage": "Not Found",
|
||||
"headers": {"content-type": "text/html"},
|
||||
"content": "<html>Not Found</html>"
|
||||
}
|
||||
with patch.object(mixin, "page") as mock_page:
|
||||
mock_page.evaluate = AsyncMock(return_value = mock_remote_object)
|
||||
|
||||
# This should raise an exception due to invalid status code
|
||||
with pytest.raises(Exception, match = "Invalid response"):
|
||||
await mixin.web_request("https://example.com/api", valid_response_codes = [200])
|
||||
result = await mixin.web_execute("window.test")
|
||||
|
||||
# Should return the value directly
|
||||
assert result == "test_value"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_web_request_with_nested_remoteobject_structures(self) -> None:
|
||||
"""Test web_request with complex nested RemoteObject structures."""
|
||||
async def test_web_execute_remoteobject_with_deep_serialized_value(self) -> None:
|
||||
"""Test web_execute with RemoteObject that has deep_serialized_value."""
|
||||
mixin = WebScrapingMixin()
|
||||
|
||||
# Mock RemoteObject with nested type/value structures
|
||||
mock_remote_object = Mock()
|
||||
mock_remote_object.deep_serialized_value = Mock()
|
||||
mock_remote_object.deep_serialized_value.value = {
|
||||
"statusCode": {"type": "number", "value": 200},
|
||||
"statusMessage": {"type": "string", "value": "OK"},
|
||||
"headers": {
|
||||
"content-type": {"type": "string", "value": "application/json"}
|
||||
},
|
||||
"content": {"type": "string", "value": '{"data": "test"}'}
|
||||
}
|
||||
# Create a mock RemoteObject with deep_serialized_value
|
||||
mock_remote_object = type("MockRemoteObject", (), {
|
||||
"__class__": type("MockClass", (), {"__name__": "RemoteObject"}),
|
||||
"value": None,
|
||||
"deep_serialized_value": type("MockDeepSerialized", (), {
|
||||
"value": [["key1", "value1"], ["key2", "value2"]]
|
||||
})()
|
||||
})()
|
||||
|
||||
with patch.object(mixin, "web_execute") as mock_web_execute:
|
||||
# Mock web_execute to return the converted result (simulating our fix)
|
||||
mock_web_execute.return_value = {
|
||||
"statusCode": 200,
|
||||
"statusMessage": "OK",
|
||||
"headers": {"content-type": "application/json"},
|
||||
"content": '{"data": "test"}'
|
||||
}
|
||||
with patch.object(mixin, "page") as mock_page:
|
||||
mock_page.evaluate = AsyncMock(return_value = mock_remote_object)
|
||||
|
||||
result = await mixin.web_request("https://example.com/api")
|
||||
result = await mixin.web_execute("window.test")
|
||||
|
||||
# Verify nested structures are properly converted
|
||||
assert result["statusCode"] == 200
|
||||
assert result["statusMessage"] == "OK"
|
||||
assert result["headers"]["content-type"] == "application/json"
|
||||
assert result["content"] == '{"data": "test"}'
|
||||
|
||||
|
||||
class TestWebRequestRemoteObjectRegression:
|
||||
"""Test web_request method to catch future RemoteObject regression issues."""
|
||||
# Should convert the deep_serialized_value
|
||||
assert result == {"key1": "value1", "key2": "value2"}
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_web_request_without_remoteobject_conversion_fails(self) -> None:
|
||||
"""Test that web_request fails without RemoteObject conversion (regression test)."""
|
||||
async def test_web_execute_remoteobject_fallback(self) -> None:
|
||||
"""Test web_execute with RemoteObject fallback when no value or deep_serialized_value."""
|
||||
mixin = WebScrapingMixin()
|
||||
|
||||
# Mock RemoteObject that would cause the original error
|
||||
mock_remote_object = Mock()
|
||||
mock_remote_object.deep_serialized_value = Mock()
|
||||
mock_remote_object.deep_serialized_value.value = {
|
||||
"statusCode": 200,
|
||||
"statusMessage": "OK",
|
||||
"headers": {"content-type": "application/json"},
|
||||
"content": '{"success": true}'
|
||||
}
|
||||
# Create a mock RemoteObject with no value or deep_serialized_value
|
||||
mock_remote_object = type("MockRemoteObject", (), {
|
||||
"__class__": type("MockClass", (), {"__name__": "RemoteObject"}),
|
||||
"value": None,
|
||||
"deep_serialized_value": None
|
||||
})()
|
||||
|
||||
# Mock web_execute to return the raw RemoteObject (simulating the bug)
|
||||
with patch.object(mixin, "web_execute") as mock_web_execute:
|
||||
mock_web_execute.return_value = mock_remote_object
|
||||
with patch.object(mixin, "page") as mock_page:
|
||||
mock_page.evaluate = AsyncMock(return_value = mock_remote_object)
|
||||
|
||||
# This should fail with the original error if our fix is removed
|
||||
with pytest.raises(TypeError, match = "object is not subscriptable"):
|
||||
await mixin.web_request("https://example.com/api")
|
||||
result = await mixin.web_execute("window.test")
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_web_request_with_remoteobject_conversion_succeeds(self) -> None:
|
||||
"""Test that web_request succeeds with RemoteObject conversion (our fix)."""
|
||||
mixin = WebScrapingMixin()
|
||||
|
||||
# Mock RemoteObject
|
||||
mock_remote_object = Mock()
|
||||
mock_remote_object.deep_serialized_value = Mock()
|
||||
mock_remote_object.deep_serialized_value.value = {
|
||||
"statusCode": 200,
|
||||
"statusMessage": "OK",
|
||||
"headers": {"content-type": "application/json"},
|
||||
"content": '{"success": true}'
|
||||
}
|
||||
|
||||
# Mock web_execute to simulate our conversion logic
|
||||
async def mock_web_execute_with_conversion(script:str) -> Any:
|
||||
# Simulate the conversion that happens in our fix
|
||||
return mock_remote_object.deep_serialized_value.value
|
||||
|
||||
with patch.object(mixin, "web_execute", side_effect = mock_web_execute_with_conversion):
|
||||
result = await mixin.web_request("https://example.com/api")
|
||||
|
||||
# Verify the result works correctly
|
||||
assert result["statusCode"] == 200
|
||||
assert result["statusMessage"] == "OK"
|
||||
assert result["headers"]["content-type"] == "application/json"
|
||||
assert result["content"] == '{"success": true}'
|
||||
# Should return the original RemoteObject as fallback
|
||||
assert result == mock_remote_object
|
||||
|
||||
Reference in New Issue
Block a user