Mirrors/kleinanzeigen-bot
1
0
Fork 0
mirror of https://github.com/Second-Hand-Friends/kleinanzeigen-bot.git synced 2026-03-12 02:31:45 +01:00
Code Issues Packages Projects Releases Wiki Activity

feat: upgrade nodriver from 0.39 to 0.47 (#635)

Browse Source 
## ℹ️ Description
Upgrade nodriver dependency from pinned version 0.39.0 to latest 0.47.0
to resolve browser startup issues and JavaScript evaluation problems
that affected versions 0.40-0.44.

- Link to the related issue(s): Resolves nodriver compatibility issues
- This upgrade addresses browser startup problems and window.BelenConf
evaluation failures that were blocking the use of newer nodriver
versions.

## 📋 Changes Summary

- Updated nodriver dependency from pinned 0.39.0 to >=0.47.0 in
pyproject.toml
- Fixed RemoteObject handling in web_execute method for nodriver 0.47
compatibility
- Added comprehensive BelenConf test fixture with real production data
structure
- Added integration test to validate window.BelenConf evaluation works
correctly
- Added German translation for new error message
- Replaced real user data with privacy-safe dummy data in test fixtures

### 🔧 Type Safety Improvements

**Added explicit `str()` conversions to resolve type inference issues:**

The comprehensive BelenConf test fixture contains deeply nested data
structures that caused pyright's type checker to infer complex
dictionary types throughout the codebase. To ensure type safety and
prevent runtime errors, I added explicit `str()` conversions in key
locations:

- **CSRF tokens**: `str(csrf_token)` - Ensures CSRF tokens are treated
as strings
- **Special attributes**: `str(special_attribute_value)` - Converts
special attribute values to strings
- **DOM attributes**: `str(special_attr_elem.attrs.id)` - Ensures
element IDs are strings
- **URL handling**: `str(current_img_url)` and `str(href_attributes)` -
Converts URLs and href attributes to strings
- **Price values**: `str(ad_cfg.price)` - Ensures price values are
strings

These conversions are defensive programming measures that ensure
backward compatibility and prevent type-related runtime errors, even if
the underlying data structures change in the future.

### ⚙️ Type of Change
- [x] ✨ New feature (adds new functionality without breaking existing
usage)
- [ ] 🐞 Bug fix (non-breaking change which fixes an issue)
- [ ] 💥 Breaking change (changes that might break existing user setups,
scripts, or configurations)

## ✅ Checklist
Before requesting a review, confirm the following:
- [x] I have reviewed my changes to ensure they meet the project's
standards.
- [x] I have tested my changes and ensured that all tests pass (`pdm run
test`).
- [x] I have formatted the code (`pdm run format`).
- [x] I have verified that linting passes (`pdm run lint`).
- [x] I have updated documentation where necessary.

By submitting this pull request, I confirm that you can use, modify,
copy, and redistribute this contribution, under the terms of your
choice.
This commit is contained in:
Jens
2025-10-12 21:22:46 +02:00
committed by GitHub
parent a2745c03b2
commit 36ca178574
12 changed files with 526 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
pdm.lock generated
View File

@@ -5,7 +5,7 @@
groups = ["default", "dev"]
strategy = ["inherit_metadata"]
lock_version = "4.5.0"
content_hash = "sha256:3bda32de316794f1c608898e17874857e2263ee1f3a5932440c630366cc40af2"
content_hash = "sha256:eecf66c65d3597b333bec94ec2b8efa5fba4f96d772c466ab6023550b8d7dd3a"
[[metadata.targets]]
requires_python = ">=3.10,<3.14"
@@ -772,7 +772,7 @@ files = [
[[package]]
name = "nodriver"
version = "0.39"
version = "0.47.0"
requires_python = ">=3.9"
summary = "[Docs here](https://ultrafunkamsterdam.github.io/nodriver)"
groups = ["default"]
@@ -782,8 +782,8 @@ dependencies = [
"websockets>=14",
]
files = [
{file = "nodriver-0.39-py3-none-any.whl", hash = "sha256:f245be52e6328393ece340a6dcbc8d5754fd7cf0838f0e1e40076944617178fc"},
{file = "nodriver-0.39.tar.gz", hash = "sha256:af84f76215877c74166f95c8e7615268e31f6118f4c7291d201f29003f2248ef"},
{file = "nodriver-0.47.0-py3-none-any.whl", hash = "sha256:2bdf69eac8fa33de09249fd513d6a70da95fc80809f9f99c4901cb3ae2e06219"},
{file = "nodriver-0.47.0.tar.gz", hash = "sha256:5fc31182a4db725ea56fc042269a29a13e55a2baf83dfdd730aa851dd5269608"},
]
[[package]]

6
pyproject.toml
View File

@@ -38,7 +38,7 @@ dependencies = [
"certifi",
"colorama",
"jaraco.text", # required by pkg_resources during runtime
"nodriver==0.39.0", # 0.40-0.44 have issues starting browsers and evaluating self.web_execute("window.BelenConf") fails
"nodriver>=0.47.0", # Updated from 0.39.0 - 0.40-0.44 had issues starting browsers and evaluating self.web_execute("window.BelenConf") fails
"pydantic>=2.0.0",
"ruamel.yaml",
"psutil",
@@ -366,6 +366,10 @@ data_file = ".temp/coverage.sqlite"
branch = true # track branch coverage
relative_files = true
[tool.coverage.report]
precision = 2
show_missing = true
skip_covered = false
#####################
# yamlfix

18
src/kleinanzeigen_bot/__init__.py
View File

@@ -690,13 +690,13 @@ class KleinanzeigenBot(WebScrapingMixin):
await self.web_request(
url = f"{self.root_url}/m-anzeigen-loeschen.json?ids={published_ad_id}",
method = "POST",
headers = {"x-csrf-token": csrf_token}
headers = {"x-csrf-token": str(csrf_token)}
)
elif ad_cfg.id:
await self.web_request(
url = f"{self.root_url}/m-anzeigen-loeschen.json?ids={ad_cfg.id}",
method = "POST",
headers = {"x-csrf-token": csrf_token},
headers = {"x-csrf-token": str(csrf_token)},
valid_response_codes = [200, 404]
)
@@ -1048,12 +1048,14 @@ class KleinanzeigenBot(WebScrapingMixin):
LOG.debug("Found %i special attributes", len(ad_cfg.special_attributes))
for special_attribute_key, special_attribute_value in ad_cfg.special_attributes.items():
# Ensure special_attribute_value is treated as a string
special_attribute_value_str = str(special_attribute_value)
if special_attribute_key == "condition_s":
await self.__set_condition(special_attribute_value)
await self.__set_condition(special_attribute_value_str)
continue
LOG.debug("Setting special attribute [%s] to [%s]...", special_attribute_key, special_attribute_value)
LOG.debug("Setting special attribute [%s] to [%s]...", special_attribute_key, special_attribute_value_str)
try:
# if the <select> element exists but is inside an invisible container, make the container visible
select_container_xpath = f"//div[@class='l-row' and descendant::select[@id='{special_attribute_key}']]"
@@ -1070,20 +1072,20 @@ class KleinanzeigenBot(WebScrapingMixin):
raise TimeoutError(f"Failed to set special attribute [{special_attribute_key}] (not found)") from ex
try:
elem_id = special_attr_elem.attrs.id
elem_id:str = str(special_attr_elem.attrs.id)
if special_attr_elem.local_name == "select":
LOG.debug("Attribute field '%s' seems to be a select...", special_attribute_key)
await self.web_select(By.ID, elem_id, special_attribute_value)
await self.web_select(By.ID, elem_id, special_attribute_value_str)
elif special_attr_elem.attrs.type == "checkbox":
LOG.debug("Attribute field '%s' seems to be a checkbox...", special_attribute_key)
await self.web_click(By.ID, elem_id)
else:
LOG.debug("Attribute field '%s' seems to be a text input...", special_attribute_key)
await self.web_input(By.ID, elem_id, special_attribute_value)
await self.web_input(By.ID, elem_id, special_attribute_value_str)
except TimeoutError as ex:
LOG.debug("Attribute field '%s' is not of kind radio button.", special_attribute_key)
raise TimeoutError(f"Failed to set special attribute [{special_attribute_key}]") from ex
LOG.debug("Successfully set attribute field [%s] to [%s]...", special_attribute_key, special_attribute_value)
LOG.debug("Successfully set attribute field [%s] to [%s]...", special_attribute_key, special_attribute_value_str)
async def __set_shipping(self, ad_cfg:Ad, mode:AdUpdateStrategy = AdUpdateStrategy.REPLACE) -> None:
if ad_cfg.shipping_type == "PICKUP":

14
src/kleinanzeigen_bot/extract.py
View File

@@ -85,7 +85,7 @@ class AdExtractor(WebScrapingMixin):
if current_img_url is None:
continue
with urllib_request.urlopen(current_img_url) as response: # noqa: S310 Audit URL open for permitted schemes.
with urllib_request.urlopen(str(current_img_url)) as response: # noqa: S310 Audit URL open for permitted schemes.
content_type = response.info().get_content_type()
file_ending = mimetypes.guess_extension(content_type)
img_path = f"{directory}/{img_fn_prefix}{img_nr}{file_ending}"
@@ -185,8 +185,8 @@ class AdExtractor(WebScrapingMixin):
# Extract references using the CORRECTED selector
try:
page_refs = [
(await self.web_find(By.CSS_SELECTOR, "div h3 a.text-onSurface", parent = li)).attrs["href"]
page_refs:list[str] = [
str((await self.web_find(By.CSS_SELECTOR, "div h3 a.text-onSurface", parent = li)).attrs["href"])
for li in list_items
]
refs.extend(page_refs)
@@ -405,13 +405,15 @@ class AdExtractor(WebScrapingMixin):
category_line = await self.web_find(By.ID, "vap-brdcrmb")
category_first_part = await self.web_find(By.CSS_SELECTOR, "a:nth-of-type(2)", parent = category_line)
category_second_part = await self.web_find(By.CSS_SELECTOR, "a:nth-of-type(3)", parent = category_line)
cat_num_first = category_first_part.attrs["href"].rsplit("/", maxsplit = 1)[-1][1:]
cat_num_second = category_second_part.attrs["href"].rsplit("/", maxsplit = 1)[-1][1:]
href_first:str = str(category_first_part.attrs["href"])
href_second:str = str(category_second_part.attrs["href"])
cat_num_first = href_first.rsplit("/", maxsplit = 1)[-1][1:]
cat_num_second = href_second.rsplit("/", maxsplit = 1)[-1][1:]
category:str = cat_num_first + "/" + cat_num_second
return category
async def _extract_special_attributes_from_ad_page(self, belen_conf:dict[str, Any]) -> dict[str, Any]:
async def _extract_special_attributes_from_ad_page(self, belen_conf:dict[str, Any]) -> dict[str, str]:
"""
Extracts the special attributes from an ad page.
If no items are available then special_attributes is empty

3
src/kleinanzeigen_bot/resources/translations.de.yaml
View File

@@ -391,6 +391,9 @@ kleinanzeigen_bot/utils/web_scraping_mixin.py:
"4. Check browser binary permissions: %s": "4. Überprüfen Sie die Browser-Binärdatei-Berechtigungen: %s"
"4. Check if any antivirus or security software is blocking the connection": "4. Überprüfen Sie, ob Antiviren- oder Sicherheitssoftware die Verbindung blockiert"
web_execute:
"Failed to convert RemoteObject to dict: %s": "Fehler beim Konvertieren von RemoteObject zu dict: %s"
web_check:
"Unsupported attribute: %s": "Nicht unterstütztes Attribut: %s"

18
src/kleinanzeigen_bot/utils/web_scraping_mixin.py
View File

@@ -562,6 +562,24 @@ class WebScrapingMixin:
LOG.debug("web_execute(`%s`) = `%s`", jscode, result)
self.__class__.web_execute._prev_jscode = jscode # type: ignore[attr-defined] # noqa: SLF001 Private member accessed
# Handle nodriver 0.47+ RemoteObject behavior
# If result is a RemoteObject with deep_serialized_value, convert it to a dict
if hasattr(result, "deep_serialized_value"):
deep_serialized = getattr(result, "deep_serialized_value", None)
if deep_serialized is not None:
try:
# Convert the deep_serialized_value to a regular dict
serialized_data = getattr(deep_serialized, "value", None)
if serialized_data is not None:
if isinstance(serialized_data, list):
# Convert list of [key, value] pairs to dict
return dict(serialized_data)
return serialized_data
except (AttributeError, TypeError, ValueError) as e:
LOG.warning("Failed to convert RemoteObject to dict: %s", e)
# Return the original result if conversion fails
return result
return result
async def web_find(self, selector_type:By, selector_value:str, *, parent:Element | None = None, timeout:int | float = 5) -> Element:

18
tests/conftest.py
View File

@@ -1,7 +1,9 @@
# SPDX-FileCopyrightText: © Jens Bergmann and contributors
# SPDX-License-Identifier: AGPL-3.0-or-later
# SPDX-ArtifactOfProjectHomePage: https://github.com/Second-Hand-Friends/kleinanzeigen-bot/
import json
import os
from pathlib import Path
from typing import Any, Final, cast
from unittest.mock import MagicMock
@@ -179,6 +181,22 @@ def mock_web_text_responses() -> list[str]:
]
@pytest.fixture
def belen_conf_sample() -> dict[str, Any]:
"""Provides sample BelenConf data for testing JavaScript evaluation.
This fixture loads the BelenConf sample data from the fixtures directory,
allowing tests to validate window.BelenConf evaluation without accessing
kleinanzeigen.de directly.
"""
fixtures_dir = Path(__file__).parent / "fixtures"
belen_conf_path = fixtures_dir / "belen_conf_sample.json"
with open(belen_conf_path, "r", encoding = "utf-8") as f:
data = json.load(f)
return cast(dict[str, Any], data)
@pytest.fixture(autouse = True)
def silence_nodriver_logs() -> None:
loggers.get_logger("nodriver").setLevel(loggers.WARNING)

128
tests/fixtures/belen_conf_sample.json vendored Normal file
View File

@@ -0,0 +1,128 @@
{
"jsBaseUrl": "https://static.kleinanzeigen.de/static/js",
"isBrowse": "false",
"isProd": true,
"initTime": 1704067200000,
"universalAnalyticsOpts": {
"account": "UA-24356365-9",
"domain": "kleinanzeigen.de",
"userId": "dummy_user_id_1234567890abcdef12",
"dimensions": {
"dimension1": "MyAds",
"dimension2": "",
"dimension3": "",
"dimension6": "",
"dimension7": "",
"dimension8": "",
"dimension9": "",
"dimension10": "",
"dimension11": "",
"dimension12": "",
"dimension13": "",
"dimension15": "de_DE",
"dimension20": "dummy_user_id_1234567890abcdefgh",
"dimension21": "dummy_encrypted_token_abcdef1234567890/1234567890abcdefgh+ijkl=lmnopqrstuvwxyz01234567==",
"dimension23": "true",
"dimension24": "private",
"dimension25": "0031_A|0042_A|0021_A|0030_A|0006_B|0028_A|0029_B|0007_C|0037_B|0026_B|0004_A|0005_A|0002_B|0036_B|0058_A|0003_B|0011_R|0022_B|0044_B|0012_B|0023_A|60_A|0008_B",
"dimension28": "distribution_test-c;yo_s-A;liberty-experimental-DEFAULT;liberty-experimental-2-DEFAULT;Lib_E;",
"dimension50": "(NULL)",
"dimension53": "",
"dimension90": "",
"dimension91": "",
"dimension94": "",
"dimension95": "",
"dimension96": "",
"dimension97": "",
"dimension121": "registered",
"dimension125": "distribution_test-c",
"dimension128": "yo_s-A",
"dimension130": "liberty-experimental-DEFAULT",
"dimension131": "liberty-experimental-2-DEFAULT",
"dimension135": "Lib_E",
"dimension136": "PRIVATE"
},
"extraDimensions": {
"dimension73": "1"
},
"sendPageView": true
},
"tnsPhoneVerificationBundleUrl": "https://www.kleinanzeigen.de/bffstatic/tns-phone-verification-web/tns-phone-verification-web-bundle.js",
"labs": {
"activeExperiments": {
"BLN-25381-ka-offboarding": "B",
"BLN-23248_BuyNow_SB": "B",
"BLN-22726_buyer_banner": "B",
"BLN-25958-greensunday": "A",
"EKTP-2111-page-extraction": "B",
"KARE-1015-Cont-Highlights": "B",
"FLPRO-130-churn-reason": "B",
"EKMO-100_reorder_postad": "B",
"BLN-27366_mortgage_sim": "A",
"KLUE-274-financing": "B",
"lws-aws-traffic": "B",
"SPEX-1052-ads-feedback": "B",
"BLN-24652_category_alert": "B",
"FLPRO-753-motors-fee": "B",
"BLN-21783_testingtime": "B",
"EBAYKAD-2252_group-assign": "A",
"liberty-experiment-style": "A",
"PRO-leads-feedback": "A",
"SPEX-1077-adfree-sub": "D",
"BLN-26740_enable_drafts": "B",
"ka-follower-network": "B",
"EKPAY-3287-counter-offer": "B",
"PLC-189_plc-migration": "A",
"EKMO-271_mweb": "A",
"audex-libertyjs-update": "A",
"performance-test-desktop": "B",
"BLN-26541-radius_feature": "A",
"EKPAY-3409-hermes-heavy": "A",
"SPEX-1077-adfree-sub-tech": "B",
"EKMO-243_MyAdsC2b_ABC": "C",
"Pro-Business-Hub": "A",
"fp_pla_desktop": "A",
"SPEX-1250_prebid_gpid": "B",
"prebid-update": "A",
"EKPAY-4088-negotiation": "B",
"desktop_payment_badge_SRP": "R",
"BLN-23401_buyNow_in_chat": "B",
"BLN-18532_highlight": "B",
"cmp-equal-choice": "B",
"BLN-27207_checkout_page": "B",
"I2I-homepage-trendsetter": "A",
"ignite_web_better_session": "C",
"EBAYKAD-3536_floor_ai": "B",
"ignite_improve_session": "C",
"EKPAY-3214-NudgeBanner": "A",
"BLN-24684-enc-brndg-data": "A",
"BLN-25794-watchlist-feed": "B",
"PRPL-252_ces_postad": "A",
"BLN-25659-car-financing": "B",
"EKPAY-3370_klarna_hide": "A",
"AUDEX-519_pb_ortb_cfg": "B",
"BLN-26398_stepstone_link": "B",
"BLN-25450_Initial_message": "A",
"cmp-leg-int": "B",
"audex-awr-update": "A",
"BLN-25216-new-user-badges": "B",
"KAD-333_dominant_category": "B",
"EKPAY-4460-kyc-entrypoint": "A",
"BLN-27350_plc_rollback": "B",
"BLN-25556_INIT_MSG_V2": "B",
"KARE-1294_private_label": "B",
"SPEX-1529_adnami-script": "A",
"DESKTOP-promo-switch": "A",
"EKPAY-3478-buyer-dispute": "A",
"FLPRO-693-ad-duplication": "B",
"BLN-27554_lds_kaos_test": "B",
"BLN-26961": "C",
"BIPHONE-9700_buy_now": "B",
"EKPAY-3336-interstial_grp": "A",
"BLN-27261_smava_provider": "A",
"10149_desktop_offboarding": "B",
"SPEX-1504-confiant": "A",
"PLC-104_plc-login": "B"
}
}
}

66
tests/integration/test_web_scraping_mixin_integration.py
View File

@@ -37,3 +37,69 @@ async def atest_init() -> None:
@pytest.mark.itest
def test_init() -> None:
nodriver.loop().run_until_complete(atest_init())
async def atest_belen_conf_evaluation() -> None:
"""Test that window.BelenConf can be evaluated correctly with nodriver."""
web_scraping_mixin = WebScrapingMixin()
if platform.system() == "Linux":
# required for Ubuntu 24.04 or newer
cast(list[str], web_scraping_mixin.browser_config.arguments).append("--no-sandbox")
browser_path = web_scraping_mixin.get_compatible_browser()
ensure(browser_path is not None, "Browser not auto-detected")
web_scraping_mixin.close_browser_session()
try:
await web_scraping_mixin.create_browser_session()
# Navigate to a simple page that can execute JavaScript
html_content = (
"data:text/html,<html><body><script>"
"window.BelenConf = {test: 'data', universalAnalyticsOpts: "
"{dimensions: {dimension92: 'test', dimension108: 'art_s:test'}}};"
"</script></body></html>"
)
await web_scraping_mixin.web_open(html_content)
await web_scraping_mixin.web_sleep(1000, 2000) # Wait for page to load
# Test JavaScript evaluation - this is the critical test for nodriver 0.40-0.44 issues
belen_conf = await web_scraping_mixin.web_execute("window.BelenConf")
# Verify the evaluation worked
assert belen_conf is not None, "window.BelenConf evaluation returned None"
# In nodriver 0.47+, JavaScript objects are returned as RemoteObject instances
# We need to check if it's either a dict (old behavior) or RemoteObject (new behavior)
is_dict = isinstance(belen_conf, dict)
is_remote_object = hasattr(belen_conf, "deep_serialized_value") and belen_conf.deep_serialized_value is not None
assert is_dict or is_remote_object, f"window.BelenConf should be a dict or RemoteObject, got {type(belen_conf)}"
if is_dict:
# Old behavior - direct dict access
assert "test" in belen_conf, "window.BelenConf should contain test data"
assert "universalAnalyticsOpts" in belen_conf, "window.BelenConf should contain universalAnalyticsOpts"
else:
# New behavior - RemoteObject with deep_serialized_value
assert hasattr(belen_conf, "deep_serialized_value"), "RemoteObject should have deep_serialized_value"
assert belen_conf.deep_serialized_value is not None, "deep_serialized_value should not be None"
if is_dict:
print(f"[OK] BelenConf evaluation successful: {list(belen_conf.keys())}")
else:
print("[OK] BelenConf evaluation successful: RemoteObject with deep_serialized_value")
finally:
web_scraping_mixin.close_browser_session()
@pytest.mark.flaky(reruns = 4, reruns_delay = 5)
@pytest.mark.itest
def test_belen_conf_evaluation() -> None:
"""Test that window.BelenConf JavaScript evaluation works correctly.
This test specifically validates the issue that affected nodriver 0.40-0.44
where window.BelenConf evaluation would fail.
"""
nodriver.loop().run_until_complete(atest_belen_conf_evaluation())

25
tests/unit/test_extract.py
View File

@@ -587,6 +587,31 @@ class TestAdExtractorCategory:
mock_web_find.assert_any_call(By.CSS_SELECTOR, "a:nth-of-type(2)", parent = category_line)
mock_web_find.assert_any_call(By.CSS_SELECTOR, "a:nth-of-type(3)", parent = category_line)
@pytest.mark.asyncio
# pylint: disable=protected-access
async def test_extract_category_with_non_string_href(self, extractor:AdExtractor) -> None:
"""Test category extraction with non-string href attributes to cover str() conversion."""
category_line = MagicMock()
first_part = MagicMock()
# Use non-string href to test str() conversion
first_part.attrs = {"href": 12345} # This will need str() conversion
second_part = MagicMock()
second_part.attrs = {"href": 67890} # This will need str() conversion
with patch.object(extractor, "web_find", new_callable = AsyncMock) as mock_web_find:
mock_web_find.side_effect = [
category_line,
first_part,
second_part
]
result = await extractor._extract_category_from_ad_page()
assert result == "2345/7890" # After str() conversion and slicing
mock_web_find.assert_any_call(By.ID, "vap-brdcrmb")
mock_web_find.assert_any_call(By.CSS_SELECTOR, "a:nth-of-type(2)", parent = category_line)
mock_web_find.assert_any_call(By.CSS_SELECTOR, "a:nth-of-type(3)", parent = category_line)
@pytest.mark.asyncio
# pylint: disable=protected-access
async def test_extract_special_attributes_empty(self, extractor:AdExtractor) -> None:

109
tests/unit/test_init.py
View File

@@ -6,7 +6,7 @@ from collections.abc import Generator
from contextlib import redirect_stdout
from datetime import timedelta
from pathlib import Path
from typing import Any
from typing import Any, cast
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
@@ -895,7 +895,7 @@ class TestKleinanzeigenBotAdDeletion:
# Create config with ID for deletion by ID
ad_cfg = Ad.model_validate(minimal_ad_config | {
id: "12345"
"id": "12345" # Fixed: use proper dict key syntax
})
published_ads = [
@@ -911,6 +911,38 @@ class TestKleinanzeigenBotAdDeletion:
result = await test_bot.delete_ad(ad_cfg, published_ads, delete_old_ads_by_title = False)
assert result is True
@pytest.mark.asyncio
async def test_delete_ad_by_id_with_non_string_csrf_token(self, test_bot:KleinanzeigenBot, minimal_ad_config:dict[str, Any]) -> None:
"""Test deleting an ad by ID with non-string CSRF token to cover str() conversion."""
test_bot.page = MagicMock()
test_bot.page.evaluate = AsyncMock(return_value = {"statusCode": 200, "content": "{}"})
test_bot.page.sleep = AsyncMock()
# Create config with ID for deletion by ID
ad_cfg = Ad.model_validate(minimal_ad_config | {
"id": "12345"
})
published_ads = [
{"title": "Different Title", "id": "12345"},
{"title": "Other Title", "id": "11111"}
]
with patch.object(test_bot, "web_open", new_callable = AsyncMock), \
patch.object(test_bot, "web_find", new_callable = AsyncMock) as mock_find, \
patch.object(test_bot, "web_click", new_callable = AsyncMock), \
patch.object(test_bot, "web_check", new_callable = AsyncMock, return_value = True), \
patch.object(test_bot, "web_request", new_callable = AsyncMock) as mock_request:
# Mock non-string CSRF token to test str() conversion
mock_find.return_value.attrs = {"content": 12345} # Non-string token
result = await test_bot.delete_ad(ad_cfg, published_ads, delete_old_ads_by_title = False)
assert result is True
# Verify that str() was called on the CSRF token
mock_request.assert_called_once()
call_args = mock_request.call_args
assert call_args[1]["headers"]["x-csrf-token"] == "12345" # Should be converted to string
class TestKleinanzeigenBotAdRepublication:
"""Tests for ad republication functionality."""
@@ -1067,6 +1099,79 @@ class TestKleinanzeigenBotShippingOptions:
# Verify the file was created in the temporary directory
assert ad_file.exists()
@pytest.mark.asyncio
async def test_special_attributes_with_non_string_values(self, test_bot:KleinanzeigenBot, base_ad_config:dict[str, Any]) -> None:
"""Test that special attributes with non-string values are converted to strings."""
# Create ad config with string special attributes first (to pass validation)
ad_cfg = Ad.model_validate(base_ad_config | {
"special_attributes": {
"art_s": "12345", # String value initially
"condition_s": "67890", # String value initially
"color_s": "red" # String value
},
"updated_on": "2024-01-01T00:00:00",
"created_on": "2024-01-01T00:00:00"
})
# Now modify the special attributes to non-string values to test str() conversion
# This simulates the scenario where the values come from external sources as non-strings
# We need to cast to Any to bypass type checking for this test
special_attrs = cast(Any, ad_cfg.special_attributes)
special_attrs["art_s"] = 12345 # Non-string value
special_attrs["condition_s"] = 67890 # Non-string value
# Mock special attribute elements
art_s_elem = MagicMock()
art_s_attrs = MagicMock()
art_s_attrs.id = "art_s"
art_s_attrs.name = "art_s"
art_s_elem.attrs = art_s_attrs
art_s_elem.local_name = "select"
condition_s_elem = MagicMock()
condition_s_attrs = MagicMock()
condition_s_attrs.id = "condition_s"
condition_s_attrs.name = "condition_s"
condition_s_elem.attrs = condition_s_attrs
condition_s_elem.local_name = "select"
color_s_elem = MagicMock()
color_s_attrs = MagicMock()
color_s_attrs.id = "color_s"
color_s_attrs.name = "color_s"
color_s_elem.attrs = color_s_attrs
color_s_elem.local_name = "select"
# Mock the necessary web interaction methods
with patch.object(test_bot, "web_find", new_callable = AsyncMock) as mock_find, \
patch.object(test_bot, "web_select", new_callable = AsyncMock) as mock_select, \
patch.object(test_bot, "web_check", new_callable = AsyncMock, return_value = True), \
patch.object(test_bot, "_KleinanzeigenBot__set_condition", new_callable = AsyncMock) as mock_set_condition:
# Mock web_find to simulate element detection
async def mock_find_side_effect(selector_type:By, selector_value:str, **_:Any) -> Element | None:
# Handle XPath queries for special attributes
if selector_type == By.XPATH and "contains(@name" in selector_value:
if "art_s" in selector_value:
return art_s_elem
if "condition_s" in selector_value:
return condition_s_elem
if "color_s" in selector_value:
return color_s_elem
return None
mock_find.side_effect = mock_find_side_effect
# Test the __set_special_attributes method directly
await getattr(test_bot, "_KleinanzeigenBot__set_special_attributes")(ad_cfg)
# Verify that web_select was called with string values (str() conversion)
mock_select.assert_any_call(By.ID, "art_s", "12345") # Converted to string
mock_select.assert_any_call(By.ID, "color_s", "red") # Already string
# Verify that __set_condition was called with string value
mock_set_condition.assert_called_once_with("67890") # Converted to string
class TestKleinanzeigenBotUrlConstruction:
"""Tests for URL construction functionality."""

134
tests/unit/test_web_scraping_mixin_remoteobject.py Normal file
View File

@@ -0,0 +1,134 @@
# SPDX-FileCopyrightText: © Jens Bergmann and contributors
# SPDX-License-Identifier: AGPL-3.0-or-later
# SPDX-ArtifactOfProjectHomePage: https://github.com/Second-Hand-Friends/kleinanzeigen-bot/
"""Unit tests for web_scraping_mixin.py RemoteObject handling.
Copyright (c) 2024, kleinanzeigen-bot contributors.
All rights reserved.
"""
from unittest.mock import AsyncMock, Mock, patch
import pytest
from kleinanzeigen_bot.utils.web_scraping_mixin import WebScrapingMixin
class TestWebExecuteRemoteObjectHandling:
"""Test web_execute method with nodriver 0.47+ RemoteObject behavior."""
@pytest.mark.asyncio
async def test_web_execute_remoteobject_with_deep_serialized_value(self) -> None:
"""Test web_execute with RemoteObject that has deep_serialized_value."""
mixin = WebScrapingMixin()
# Mock RemoteObject with deep_serialized_value
mock_remote_object = Mock()
mock_remote_object.deep_serialized_value = Mock()
mock_remote_object.deep_serialized_value.value = [
["key1", "value1"],
["key2", "value2"]
]
# Mock the page evaluation to return our RemoteObject
with patch.object(mixin, "page") as mock_page:
mock_page.evaluate = AsyncMock(return_value = mock_remote_object)
result = await mixin.web_execute("window.test")
# Should convert the RemoteObject to a dict
assert result == {"key1": "value1", "key2": "value2"}
@pytest.mark.asyncio
async def test_web_execute_remoteobject_with_none_deep_serialized_value(self) -> None:
"""Test web_execute with RemoteObject that has None deep_serialized_value."""
mixin = WebScrapingMixin()
# Mock RemoteObject with None deep_serialized_value
mock_remote_object = Mock()
mock_remote_object.deep_serialized_value = None
with patch.object(mixin, "page") as mock_page:
mock_page.evaluate = AsyncMock(return_value = mock_remote_object)
result = await mixin.web_execute("window.test")
# Should return the original RemoteObject
assert result is mock_remote_object
@pytest.mark.asyncio
async def test_web_execute_remoteobject_with_non_list_serialized_data(self) -> None:
"""Test web_execute with RemoteObject that has non-list serialized data."""
mixin = WebScrapingMixin()
# Mock RemoteObject with non-list serialized data
mock_remote_object = Mock()
mock_remote_object.deep_serialized_value = Mock()
mock_remote_object.deep_serialized_value.value = {"direct": "dict"}
with patch.object(mixin, "page") as mock_page:
mock_page.evaluate = AsyncMock(return_value = mock_remote_object)
result = await mixin.web_execute("window.test")
# Should return the serialized data directly
assert result == {"direct": "dict"}
@pytest.mark.asyncio
async def test_web_execute_remoteobject_with_none_serialized_data(self) -> None:
"""Test web_execute with RemoteObject that has None serialized data."""
mixin = WebScrapingMixin()
# Mock RemoteObject with None serialized data
mock_remote_object = Mock()
mock_remote_object.deep_serialized_value = Mock()
mock_remote_object.deep_serialized_value.value = None
with patch.object(mixin, "page") as mock_page:
mock_page.evaluate = AsyncMock(return_value = mock_remote_object)
result = await mixin.web_execute("window.test")
# Should return the original RemoteObject
assert result is mock_remote_object
@pytest.mark.asyncio
async def test_web_execute_regular_result(self) -> None:
"""Test web_execute with regular result (no RemoteObject)."""
mixin = WebScrapingMixin()
# Mock regular result (no deep_serialized_value attribute)
mock_result = {"regular": "dict"}
with patch.object(mixin, "page") as mock_page:
mock_page.evaluate = AsyncMock(return_value = mock_result)
result = await mixin.web_execute("window.test")
# Should return the result unchanged
assert result == mock_result
@pytest.mark.asyncio
async def test_web_execute_remoteobject_conversion_exception(self) -> None:
"""Test web_execute with RemoteObject that raises exception during conversion."""
mixin = WebScrapingMixin()
# Mock RemoteObject that will raise an exception when trying to convert to dict
mock_remote_object = Mock()
mock_deep_serialized = Mock()
# Create a list-like object that raises an exception when dict() is called on it
class ExceptionRaisingList(list[str]):
def __iter__(self) -> None: # type: ignore[override]
raise ValueError("Simulated conversion error")
mock_deep_serialized.value = ExceptionRaisingList([["key", "value"]]) # type: ignore[list-item]
mock_remote_object.deep_serialized_value = mock_deep_serialized
with patch.object(mixin, "page") as mock_page:
mock_page.evaluate = AsyncMock(return_value = mock_remote_object)
result = await mixin.web_execute("window.test")
# Should return the original RemoteObject when conversion fails
assert result is mock_remote_object