mirror of
https://github.com/Second-Hand-Friends/kleinanzeigen-bot.git
synced 2026-03-12 10:31:50 +01:00
fix: resolve nodriver 0.47+ RemoteObject compatibility issues (#645)
## ℹ️ Description *Provide a concise summary of the changes introduced in this pull request.* - Link to the related issue(s): #644 - Describe the motivation and context for this change. This PR resolves compatibility issues with nodriver 0.47+ where page.evaluate() returns RemoteObject instances that need special handling for proper conversion to Python objects. The update introduced breaking changes in how JavaScript evaluation results are returned, causing TypeError: [RemoteObject] object is not subscriptable errors. ## 📋 Changes Summary - Fixed TypeError: [RemoteObject] object is not subscriptable in web_request() method - Added comprehensive RemoteObject conversion logic with _convert_remote_object_result() - Added _convert_remote_object_dict() for recursive nested structure conversion - Fixed price field concatenation issue in MODIFY mode by explicit field clearing - Updated web_sleep() to accept integer milliseconds instead of float seconds - Updated German translations for new log messages - Fixed linting issues (E711, E712) in test assertions ### ⚙️ Type of Change Select the type(s) of change(s) included in this pull request: - [x] 🐞 Bug fix (non-breaking change which fixes an issue) - [ ] ✨ New feature (adds new functionality without breaking existing usage) - [ ] 💥 Breaking change (changes that might break existing user setups, scripts, or configurations) ## ✅ Checklist Before requesting a review, confirm the following: - [x] I have reviewed my changes to ensure they meet the project's standards. - [x] I have tested my changes and ensured that all tests pass (pdm run test). - [x] I have formatted the code (pdm run format). - [x] I have verified that linting passes (pdm run lint). - [x] I have updated documentation where necessary. By submitting this pull request, I confirm that you can use, modify, copy, and redistribute this contribution, under the terms of your choice.
This commit is contained in:
@@ -803,9 +803,12 @@ class KleinanzeigenBot(WebScrapingMixin):
|
||||
pass
|
||||
if ad_cfg.price:
|
||||
if mode == AdUpdateStrategy.MODIFY:
|
||||
# we have to clear the input, otherwise input gets appended
|
||||
await self.web_input(By.CSS_SELECTOR,
|
||||
"input#post-ad-frontend-price, input#micro-frontend-price, input#pstad-price", "")
|
||||
# Clear the price field first to prevent concatenation of old and new values
|
||||
# This is needed because some input fields don't clear properly with just clear_input()
|
||||
price_field = await self.web_find(By.CSS_SELECTOR, "input#post-ad-frontend-price, input#micro-frontend-price, input#pstad-price")
|
||||
await price_field.clear_input()
|
||||
await price_field.send_keys("") # Ensure field is completely empty
|
||||
await self.web_sleep(500) # Brief pause to ensure clearing is complete
|
||||
await self.web_input(By.CSS_SELECTOR, "input#post-ad-frontend-price, input#micro-frontend-price, input#pstad-price", str(ad_cfg.price))
|
||||
|
||||
#############################
|
||||
|
||||
@@ -391,7 +391,7 @@ kleinanzeigen_bot/utils/web_scraping_mixin.py:
|
||||
"4. Check browser binary permissions: %s": "4. Überprüfen Sie die Browser-Binärdatei-Berechtigungen: %s"
|
||||
"4. Check if any antivirus or security software is blocking the connection": "4. Überprüfen Sie, ob Antiviren- oder Sicherheitssoftware die Verbindung blockiert"
|
||||
|
||||
web_execute:
|
||||
_convert_remote_object_result:
|
||||
"Failed to convert RemoteObject to dict: %s": "Fehler beim Konvertieren von RemoteObject zu dict: %s"
|
||||
|
||||
web_check:
|
||||
|
||||
@@ -42,6 +42,9 @@ LOG:Final[loggers.Logger] = loggers.get_logger(__name__)
|
||||
# see https://api.jquery.com/category/selectors/
|
||||
METACHAR_ESCAPER:Final[dict[int, str]] = str.maketrans({ch: f"\\{ch}" for ch in '!"#$%&\'()*+,./:;<=>?@[\\]^`{|}~'})
|
||||
|
||||
# Constants for RemoteObject handling
|
||||
_REMOTE_OBJECT_TYPE_VALUE_PAIR_SIZE:Final[int] = 2
|
||||
|
||||
|
||||
def _is_admin() -> bool:
|
||||
"""Check if the current process is running with admin/root privileges."""
|
||||
@@ -574,23 +577,61 @@ class WebScrapingMixin:
|
||||
# Handle nodriver 0.47+ RemoteObject behavior
|
||||
# If result is a RemoteObject with deep_serialized_value, convert it to a dict
|
||||
if hasattr(result, "deep_serialized_value"):
|
||||
deep_serialized = getattr(result, "deep_serialized_value", None)
|
||||
if deep_serialized is not None:
|
||||
try:
|
||||
# Convert the deep_serialized_value to a regular dict
|
||||
serialized_data = getattr(deep_serialized, "value", None)
|
||||
if serialized_data is not None:
|
||||
if isinstance(serialized_data, list):
|
||||
# Convert list of [key, value] pairs to dict
|
||||
return dict(serialized_data)
|
||||
return serialized_data
|
||||
except (AttributeError, TypeError, ValueError) as e:
|
||||
LOG.warning("Failed to convert RemoteObject to dict: %s", e)
|
||||
# Return the original result if conversion fails
|
||||
return result
|
||||
return self._convert_remote_object_result(result)
|
||||
|
||||
return result
|
||||
|
||||
def _convert_remote_object_result(self, result:Any) -> Any:
|
||||
"""
|
||||
Converts a RemoteObject result to a regular Python object.
|
||||
|
||||
Handles the deep_serialized_value conversion for nodriver 0.47+ compatibility.
|
||||
"""
|
||||
deep_serialized = getattr(result, "deep_serialized_value", None)
|
||||
if deep_serialized is None:
|
||||
return result
|
||||
|
||||
try:
|
||||
# Convert the deep_serialized_value to a regular dict
|
||||
serialized_data = getattr(deep_serialized, "value", None)
|
||||
if serialized_data is None:
|
||||
return result
|
||||
|
||||
if isinstance(serialized_data, list):
|
||||
# Convert list of [key, value] pairs to dict, handling nested RemoteObjects
|
||||
converted_dict = {}
|
||||
for key, value in serialized_data:
|
||||
converted_dict[key] = self._convert_remote_object_dict(value)
|
||||
return converted_dict
|
||||
|
||||
if isinstance(serialized_data, dict):
|
||||
# Handle nested RemoteObject structures like {'type': 'number', 'value': 200}
|
||||
return self._convert_remote_object_dict(serialized_data)
|
||||
|
||||
return serialized_data
|
||||
except (AttributeError, TypeError, ValueError) as e:
|
||||
LOG.warning("Failed to convert RemoteObject to dict: %s", e)
|
||||
# Return the original result if conversion fails
|
||||
return result
|
||||
|
||||
def _convert_remote_object_dict(self, data:Any) -> Any:
|
||||
"""
|
||||
Recursively converts RemoteObject dict structures to regular Python objects.
|
||||
|
||||
Handles structures like {'type': 'number', 'value': 200} or {'type': 'string', 'value': 'text'}.
|
||||
"""
|
||||
if isinstance(data, dict):
|
||||
# Check if this is a RemoteObject value structure
|
||||
if "type" in data and "value" in data and len(data) == _REMOTE_OBJECT_TYPE_VALUE_PAIR_SIZE:
|
||||
return data["value"]
|
||||
# Recursively convert nested dicts
|
||||
return {key: self._convert_remote_object_dict(value) for key, value in data.items()}
|
||||
if isinstance(data, list):
|
||||
# Recursively convert lists
|
||||
return [self._convert_remote_object_dict(item) for item in data]
|
||||
# Return primitive values as-is
|
||||
return data
|
||||
|
||||
async def web_find(self, selector_type:By, selector_value:str, *, parent:Element | None = None, timeout:int | float = 5) -> Element:
|
||||
"""
|
||||
Locates an HTML element by the given selector type and value.
|
||||
@@ -724,10 +765,10 @@ class WebScrapingMixin:
|
||||
await self.page.sleep(duration / 1_000)
|
||||
|
||||
async def web_request(self, url:str, method:str = "GET", valid_response_codes:int | Iterable[int] = 200,
|
||||
headers:dict[str, str] | None = None) -> dict[str, Any]:
|
||||
headers:dict[str, str] | None = None) -> Any:
|
||||
method = method.upper()
|
||||
LOG.debug(" -> HTTP %s [%s]...", method, url)
|
||||
response = cast(dict[str, Any], await self.page.evaluate(f"""
|
||||
response = await self.web_execute(f"""
|
||||
fetch("{url}", {{
|
||||
method: "{method}",
|
||||
redirect: "follow",
|
||||
@@ -743,7 +784,7 @@ class WebScrapingMixin:
|
||||
content: responseText
|
||||
}}
|
||||
}}))
|
||||
""", await_promise = True, return_by_value = True))
|
||||
""")
|
||||
if isinstance(valid_response_codes, int):
|
||||
valid_response_codes = [valid_response_codes]
|
||||
ensure(
|
||||
|
||||
Reference in New Issue
Block a user