fix: resolve nodriver 0.47+ RemoteObject compatibility issues (#645)

## ℹ️ Description
*Provide a concise summary of the changes introduced in this pull
request.*

- Link to the related issue(s): #644
- Describe the motivation and context for this change.

This PR resolves compatibility issues with nodriver 0.47+ where
page.evaluate() returns RemoteObject instances that need special
handling for proper conversion to Python objects. The update introduced
breaking changes in how JavaScript evaluation results are returned,
causing TypeError: [RemoteObject] object is not subscriptable errors.

## 📋 Changes Summary

- Fixed TypeError: [RemoteObject] object is not subscriptable in
web_request() method
- Added comprehensive RemoteObject conversion logic with
_convert_remote_object_result()
- Added _convert_remote_object_dict() for recursive nested structure
conversion
- Fixed price field concatenation issue in MODIFY mode by explicit field
clearing
- Updated web_sleep() to accept integer milliseconds instead of float
seconds
- Updated German translations for new log messages
- Fixed linting issues (E711, E712) in test assertions

### ⚙️ Type of Change
Select the type(s) of change(s) included in this pull request:
- [x] 🐞 Bug fix (non-breaking change which fixes an issue)
- [ ]  New feature (adds new functionality without breaking existing
usage)
- [ ] 💥 Breaking change (changes that might break existing user setups,
scripts, or configurations)

##  Checklist
Before requesting a review, confirm the following:
- [x] I have reviewed my changes to ensure they meet the project's
standards.
- [x] I have tested my changes and ensured that all tests pass (pdm run
test).
- [x] I have formatted the code (pdm run format).
- [x] I have verified that linting passes (pdm run lint).
- [x] I have updated documentation where necessary.

By submitting this pull request, I confirm that you can use, modify,
copy, and redistribute this contribution, under the terms of your
choice.
This commit is contained in:
Jens
2025-10-18 19:38:51 +02:00
committed by GitHub
parent 34013cb869
commit 8aee313aba
4 changed files with 429 additions and 100 deletions

View File

@@ -803,9 +803,12 @@ class KleinanzeigenBot(WebScrapingMixin):
pass
if ad_cfg.price:
if mode == AdUpdateStrategy.MODIFY:
# we have to clear the input, otherwise input gets appended
await self.web_input(By.CSS_SELECTOR,
"input#post-ad-frontend-price, input#micro-frontend-price, input#pstad-price", "")
# Clear the price field first to prevent concatenation of old and new values
# This is needed because some input fields don't clear properly with just clear_input()
price_field = await self.web_find(By.CSS_SELECTOR, "input#post-ad-frontend-price, input#micro-frontend-price, input#pstad-price")
await price_field.clear_input()
await price_field.send_keys("") # Ensure field is completely empty
await self.web_sleep(500) # Brief pause to ensure clearing is complete
await self.web_input(By.CSS_SELECTOR, "input#post-ad-frontend-price, input#micro-frontend-price, input#pstad-price", str(ad_cfg.price))
#############################

View File

@@ -391,7 +391,7 @@ kleinanzeigen_bot/utils/web_scraping_mixin.py:
"4. Check browser binary permissions: %s": "4. Überprüfen Sie die Browser-Binärdatei-Berechtigungen: %s"
"4. Check if any antivirus or security software is blocking the connection": "4. Überprüfen Sie, ob Antiviren- oder Sicherheitssoftware die Verbindung blockiert"
web_execute:
_convert_remote_object_result:
"Failed to convert RemoteObject to dict: %s": "Fehler beim Konvertieren von RemoteObject zu dict: %s"
web_check:

View File

@@ -42,6 +42,9 @@ LOG:Final[loggers.Logger] = loggers.get_logger(__name__)
# see https://api.jquery.com/category/selectors/
METACHAR_ESCAPER:Final[dict[int, str]] = str.maketrans({ch: f"\\{ch}" for ch in '!"#$%&\'()*+,./:;<=>?@[\\]^`{|}~'})
# Constants for RemoteObject handling
_REMOTE_OBJECT_TYPE_VALUE_PAIR_SIZE:Final[int] = 2
def _is_admin() -> bool:
"""Check if the current process is running with admin/root privileges."""
@@ -574,23 +577,61 @@ class WebScrapingMixin:
# Handle nodriver 0.47+ RemoteObject behavior
# If result is a RemoteObject with deep_serialized_value, convert it to a dict
if hasattr(result, "deep_serialized_value"):
deep_serialized = getattr(result, "deep_serialized_value", None)
if deep_serialized is not None:
try:
# Convert the deep_serialized_value to a regular dict
serialized_data = getattr(deep_serialized, "value", None)
if serialized_data is not None:
if isinstance(serialized_data, list):
# Convert list of [key, value] pairs to dict
return dict(serialized_data)
return serialized_data
except (AttributeError, TypeError, ValueError) as e:
LOG.warning("Failed to convert RemoteObject to dict: %s", e)
# Return the original result if conversion fails
return result
return self._convert_remote_object_result(result)
return result
def _convert_remote_object_result(self, result:Any) -> Any:
"""
Converts a RemoteObject result to a regular Python object.
Handles the deep_serialized_value conversion for nodriver 0.47+ compatibility.
"""
deep_serialized = getattr(result, "deep_serialized_value", None)
if deep_serialized is None:
return result
try:
# Convert the deep_serialized_value to a regular dict
serialized_data = getattr(deep_serialized, "value", None)
if serialized_data is None:
return result
if isinstance(serialized_data, list):
# Convert list of [key, value] pairs to dict, handling nested RemoteObjects
converted_dict = {}
for key, value in serialized_data:
converted_dict[key] = self._convert_remote_object_dict(value)
return converted_dict
if isinstance(serialized_data, dict):
# Handle nested RemoteObject structures like {'type': 'number', 'value': 200}
return self._convert_remote_object_dict(serialized_data)
return serialized_data
except (AttributeError, TypeError, ValueError) as e:
LOG.warning("Failed to convert RemoteObject to dict: %s", e)
# Return the original result if conversion fails
return result
def _convert_remote_object_dict(self, data:Any) -> Any:
"""
Recursively converts RemoteObject dict structures to regular Python objects.
Handles structures like {'type': 'number', 'value': 200} or {'type': 'string', 'value': 'text'}.
"""
if isinstance(data, dict):
# Check if this is a RemoteObject value structure
if "type" in data and "value" in data and len(data) == _REMOTE_OBJECT_TYPE_VALUE_PAIR_SIZE:
return data["value"]
# Recursively convert nested dicts
return {key: self._convert_remote_object_dict(value) for key, value in data.items()}
if isinstance(data, list):
# Recursively convert lists
return [self._convert_remote_object_dict(item) for item in data]
# Return primitive values as-is
return data
async def web_find(self, selector_type:By, selector_value:str, *, parent:Element | None = None, timeout:int | float = 5) -> Element:
"""
Locates an HTML element by the given selector type and value.
@@ -724,10 +765,10 @@ class WebScrapingMixin:
await self.page.sleep(duration / 1_000)
async def web_request(self, url:str, method:str = "GET", valid_response_codes:int | Iterable[int] = 200,
headers:dict[str, str] | None = None) -> dict[str, Any]:
headers:dict[str, str] | None = None) -> Any:
method = method.upper()
LOG.debug(" -> HTTP %s [%s]...", method, url)
response = cast(dict[str, Any], await self.page.evaluate(f"""
response = await self.web_execute(f"""
fetch("{url}", {{
method: "{method}",
redirect: "follow",
@@ -743,7 +784,7 @@ class WebScrapingMixin:
content: responseText
}}
}}))
""", await_promise = True, return_by_value = True))
""")
if isinstance(valid_response_codes, int):
valid_response_codes = [valid_response_codes]
ensure(