mirror of
https://github.com/Second-Hand-Friends/kleinanzeigen-bot.git
synced 2026-03-12 10:31:50 +01:00
fix: resolve nodriver RemoteObject conversion bug (#651)
## ℹ️ Description *Fixes the nodriver 0.47.0 RemoteObject conversion bug that was causing KeyError and TypeError when accessing BelenConf dimensions.* - Link to the related issue(s): Issue #650 - The bot was crashing when downloading ads because nodriver 0.47.0 was returning JavaScript objects as lists of [key, value] pairs instead of proper Python dictionaries, causing BelenConf dimensions to be inaccessible. ## 📋 Changes Summary - **Fixed nodriver RemoteObject conversion bug** in `web_scraping_mixin.py`: - Added detection logic for list-of-pairs format in `web_execute` method - Enhanced `_convert_remote_object_dict` to recursively convert nested structures - Now properly converts JavaScript objects to Python dictionaries - **Bot functionality fully restored** - can now download ads with subcategories and special attributes ### ⚙️ Type of Change - [x] 🐞 Bug fix (non-breaking change which fixes an issue) - [ ] ✨ New feature (adds new functionality without breaking existing usage) - [ ] 💥 Breaking change (changes that might break existing user setups, scripts, or configurations) ## ✅ Checklist - [x] I have reviewed my changes to ensure they meet the project's standards. - [x] I have tested my changes and ensured that all tests pass (`pdm run test`). - [x] I have formatted the code (`pdm run format`). - [x] I have verified that linting passes (`pdm run lint`). - [x] I have updated documentation where necessary. By submitting this pull request, I confirm that you can use, modify, copy, and redistribute this contribution, under the terms of your choice.
This commit is contained in:
@@ -44,6 +44,7 @@ METACHAR_ESCAPER:Final[dict[int, str]] = str.maketrans({ch: f"\\{ch}" for ch in
|
|||||||
|
|
||||||
# Constants for RemoteObject handling
|
# Constants for RemoteObject handling
|
||||||
_REMOTE_OBJECT_TYPE_VALUE_PAIR_SIZE:Final[int] = 2
|
_REMOTE_OBJECT_TYPE_VALUE_PAIR_SIZE:Final[int] = 2
|
||||||
|
_KEY_VALUE_PAIR_SIZE:Final[int] = 2
|
||||||
|
|
||||||
|
|
||||||
def _is_admin() -> bool:
|
def _is_admin() -> bool:
|
||||||
@@ -579,6 +580,15 @@ class WebScrapingMixin:
|
|||||||
if hasattr(result, "deep_serialized_value"):
|
if hasattr(result, "deep_serialized_value"):
|
||||||
return self._convert_remote_object_result(result)
|
return self._convert_remote_object_result(result)
|
||||||
|
|
||||||
|
# Fix for nodriver 0.47+ bug: convert list-of-pairs back to dict
|
||||||
|
if isinstance(result, list) and all(isinstance(item, list) and len(item) == _KEY_VALUE_PAIR_SIZE for item in result):
|
||||||
|
# This looks like a list of [key, value] pairs that should be a dict
|
||||||
|
converted_dict = {}
|
||||||
|
for key, value in result:
|
||||||
|
# Recursively convert nested structures
|
||||||
|
converted_dict[key] = self._convert_remote_object_dict(value)
|
||||||
|
return converted_dict
|
||||||
|
|
||||||
return result
|
return result
|
||||||
|
|
||||||
def _convert_remote_object_result(self, result:Any) -> Any:
|
def _convert_remote_object_result(self, result:Any) -> Any:
|
||||||
@@ -601,7 +611,11 @@ class WebScrapingMixin:
|
|||||||
# Convert list of [key, value] pairs to dict, handling nested RemoteObjects
|
# Convert list of [key, value] pairs to dict, handling nested RemoteObjects
|
||||||
converted_dict = {}
|
converted_dict = {}
|
||||||
for key, value in serialized_data:
|
for key, value in serialized_data:
|
||||||
converted_dict[key] = self._convert_remote_object_dict(value)
|
# Handle the case where value is a RemoteObject with type/value structure
|
||||||
|
if isinstance(value, dict) and "type" in value and "value" in value:
|
||||||
|
converted_dict[key] = self._convert_remote_object_dict(value)
|
||||||
|
else:
|
||||||
|
converted_dict[key] = self._convert_remote_object_dict(value)
|
||||||
return converted_dict
|
return converted_dict
|
||||||
|
|
||||||
if isinstance(serialized_data, dict):
|
if isinstance(serialized_data, dict):
|
||||||
@@ -623,10 +637,24 @@ class WebScrapingMixin:
|
|||||||
if isinstance(data, dict):
|
if isinstance(data, dict):
|
||||||
# Check if this is a RemoteObject value structure
|
# Check if this is a RemoteObject value structure
|
||||||
if "type" in data and "value" in data and len(data) == _REMOTE_OBJECT_TYPE_VALUE_PAIR_SIZE:
|
if "type" in data and "value" in data and len(data) == _REMOTE_OBJECT_TYPE_VALUE_PAIR_SIZE:
|
||||||
return data["value"]
|
# Extract the actual value and recursively convert it
|
||||||
|
value = data["value"]
|
||||||
|
if isinstance(value, list) and all(isinstance(item, list) and len(item) == _KEY_VALUE_PAIR_SIZE for item in value):
|
||||||
|
# This is a list of [key, value] pairs that should be a dict
|
||||||
|
converted_dict = {}
|
||||||
|
for key, val in value:
|
||||||
|
converted_dict[key] = self._convert_remote_object_dict(val)
|
||||||
|
return converted_dict
|
||||||
|
return self._convert_remote_object_dict(value)
|
||||||
# Recursively convert nested dicts
|
# Recursively convert nested dicts
|
||||||
return {key: self._convert_remote_object_dict(value) for key, value in data.items()}
|
return {key: self._convert_remote_object_dict(value) for key, value in data.items()}
|
||||||
if isinstance(data, list):
|
if isinstance(data, list):
|
||||||
|
# Check if this is a list of [key, value] pairs that should be a dict
|
||||||
|
if all(isinstance(item, list) and len(item) == _KEY_VALUE_PAIR_SIZE for item in data):
|
||||||
|
converted_dict = {}
|
||||||
|
for key, value in data:
|
||||||
|
converted_dict[key] = self._convert_remote_object_dict(value)
|
||||||
|
return converted_dict
|
||||||
# Recursively convert lists
|
# Recursively convert lists
|
||||||
return [self._convert_remote_object_dict(item) for item in data]
|
return [self._convert_remote_object_dict(item) for item in data]
|
||||||
# Return primitive values as-is
|
# Return primitive values as-is
|
||||||
|
|||||||
Reference in New Issue
Block a user