Mirrors/kleinanzeigen-bot
1
0
Fork 0
mirror of https://github.com/Second-Hand-Friends/kleinanzeigen-bot.git synced 2026-03-12 10:31:50 +01:00
Code Issues Packages Projects Releases Wiki Activity

enh: allow per-ad overriding of global description affixes (#416)

Browse Source
This commit is contained in:
Jens Bergmann
2025-02-11 23:39:26 +01:00
committed by GitHub
parent a67112d936
commit 4051620aed
9 changed files with 559 additions and 66 deletions
Download Patch File Download Diff File Expand all files Collapse all files

144
tests/unit/test_extract.py
View File

@@ -301,56 +301,110 @@ class TestAdExtractorNavigation:
class TestAdExtractorContent:
"""Tests for content extraction functionality."""
# pylint: disable=protected-access
@pytest.fixture
def extractor(self) -> AdExtractor:
browser_mock = MagicMock(spec = Browser)
config_mock = {
"ad_defaults": {
"description": {
"prefix": "Test Prefix",
"suffix": "Test Suffix"
}
}
}
return AdExtractor(browser_mock, config_mock)
def extractor_with_config(self) -> AdExtractor:
"""Create extractor with specific config for testing prefix/suffix handling."""
browser_mock = MagicMock(spec=Browser)
return AdExtractor(browser_mock, {}) # Empty config, will be overridden in tests
@pytest.mark.asyncio
# pylint: disable=protected-access
async def test_extract_title_and_description(self, extractor: AdExtractor) -> None:
"""Test basic extraction of title and description."""
page_mock = AsyncMock()
async def test_extract_description_with_affixes(
self,
test_extractor: AdExtractor,
description_test_cases: list[tuple[dict[str, Any], str, str]]
) -> None:
"""Test extraction of description with various prefix/suffix configurations."""
# Mock the page
page_mock = MagicMock()
page_mock.url = "https://www.kleinanzeigen.de/s-anzeige/test/12345"
test_extractor.page = page_mock
category_mock = AsyncMock()
category_mock.attrs = {'href': '/s-kategorie/c123'}
for config, raw_description, _ in description_test_cases: # Changed to _ since we don't use expected_description
test_extractor.config = config
with patch.object(extractor, 'page', page_mock), \
patch.object(extractor, 'web_text', new_callable = AsyncMock) as mock_web_text, \
patch.object(extractor, 'web_find', new_callable = AsyncMock, return_value = category_mock), \
patch.object(extractor, '_extract_category_from_ad_page', new_callable = AsyncMock, return_value = "17/23"), \
patch.object(extractor, '_extract_special_attributes_from_ad_page', new_callable = AsyncMock, return_value = {}), \
patch.object(extractor, '_extract_pricing_info_from_ad_page', new_callable = AsyncMock, return_value = (None, "NOT_APPLICABLE")), \
patch.object(extractor, '_extract_shipping_info_from_ad_page', new_callable = AsyncMock, return_value = ("NOT_APPLICABLE", None, None)), \
patch.object(extractor, '_extract_sell_directly_from_ad_page', new_callable = AsyncMock, return_value = False), \
patch.object(extractor, '_download_images_from_ad_page', new_callable = AsyncMock, return_value = []), \
patch.object(extractor, '_extract_contact_from_ad_page', new_callable = AsyncMock, return_value = {}):
mock_web_text.side_effect = [
"Test Title",
"Test Prefix Original Description Test Suffix",
"03.02.2025"
]
info = await extractor._extract_ad_page_info("/some/dir", 12345)
assert isinstance(info, dict)
assert info["title"] == "Test Title"
assert info["description"].strip() == "Original Description"
assert info["created_on"] == "2025-02-03T00:00:00"
with patch.multiple(test_extractor,
web_text=AsyncMock(side_effect=[
"Test Title", # Title
raw_description, # Raw description (without affixes)
"03.02.2025" # Creation date
]),
_extract_category_from_ad_page=AsyncMock(return_value="160"),
_extract_special_attributes_from_ad_page=AsyncMock(return_value={}),
_extract_pricing_info_from_ad_page=AsyncMock(return_value=(None, "NOT_APPLICABLE")),
_extract_shipping_info_from_ad_page=AsyncMock(return_value=("NOT_APPLICABLE", None, None)),
_extract_sell_directly_from_ad_page=AsyncMock(return_value=False),
_download_images_from_ad_page=AsyncMock(return_value=[]),
_extract_contact_from_ad_page=AsyncMock(return_value={})
):
info = await test_extractor._extract_ad_page_info("/some/dir", 12345)
assert info["description"] == raw_description
@pytest.mark.asyncio
# pylint: disable=protected-access
async def test_extract_sell_directly(self, extractor: AdExtractor) -> None:
async def test_extract_description_with_affixes_timeout(
self,
test_extractor: AdExtractor
) -> None:
"""Test handling of timeout when extracting description."""
# Mock the page
page_mock = MagicMock()
page_mock.url = "https://www.kleinanzeigen.de/s-anzeige/test/12345"
test_extractor.page = page_mock
with patch.multiple(test_extractor,
web_text=AsyncMock(side_effect=[
"Test Title", # Title succeeds
TimeoutError("Timeout"), # Description times out
"03.02.2025" # Date succeeds
]),
_extract_category_from_ad_page=AsyncMock(return_value="160"),
_extract_special_attributes_from_ad_page=AsyncMock(return_value={}),
_extract_pricing_info_from_ad_page=AsyncMock(return_value=(None, "NOT_APPLICABLE")),
_extract_shipping_info_from_ad_page=AsyncMock(return_value=("NOT_APPLICABLE", None, None)),
_extract_sell_directly_from_ad_page=AsyncMock(return_value=False),
_download_images_from_ad_page=AsyncMock(return_value=[]),
_extract_contact_from_ad_page=AsyncMock(return_value={})
):
try:
info = await test_extractor._extract_ad_page_info("/some/dir", 12345)
assert info["description"] == ""
except TimeoutError:
# This is also acceptable - depends on how we want to handle timeouts
pass
@pytest.mark.asyncio
async def test_extract_description_with_affixes_no_affixes(
self,
test_extractor: AdExtractor
) -> None:
"""Test extraction of description without any affixes in config."""
# Mock the page
page_mock = MagicMock()
page_mock.url = "https://www.kleinanzeigen.de/s-anzeige/test/12345"
test_extractor.page = page_mock
test_extractor.config = {"ad_defaults": {}} # Empty config
raw_description = "Original Description"
with patch.multiple(test_extractor,
web_text=AsyncMock(side_effect=[
"Test Title", # Title
raw_description, # Description without affixes
"03.02.2025" # Creation date
]),
_extract_category_from_ad_page=AsyncMock(return_value="160"),
_extract_special_attributes_from_ad_page=AsyncMock(return_value={}),
_extract_pricing_info_from_ad_page=AsyncMock(return_value=(None, "NOT_APPLICABLE")),
_extract_shipping_info_from_ad_page=AsyncMock(return_value=("NOT_APPLICABLE", None, None)),
_extract_sell_directly_from_ad_page=AsyncMock(return_value=False),
_download_images_from_ad_page=AsyncMock(return_value=[]),
_extract_contact_from_ad_page=AsyncMock(return_value={})
):
info = await test_extractor._extract_ad_page_info("/some/dir", 12345)
assert info["description"] == raw_description
@pytest.mark.asyncio
async def test_extract_sell_directly(self, test_extractor: AdExtractor) -> None:
"""Test extraction of sell directly option."""
test_cases = [
("Direkt kaufen", True),
@@ -358,12 +412,12 @@ class TestAdExtractorContent:
]
for text, expected in test_cases:
with patch.object(extractor, 'web_text', new_callable = AsyncMock, return_value = text):
result = await extractor._extract_sell_directly_from_ad_page()
with patch.object(test_extractor, 'web_text', new_callable=AsyncMock, return_value=text):
result = await test_extractor._extract_sell_directly_from_ad_page()
assert result is expected
with patch.object(extractor, 'web_text', new_callable = AsyncMock, side_effect = TimeoutError):
result = await extractor._extract_sell_directly_from_ad_page()
with patch.object(test_extractor, 'web_text', new_callable=AsyncMock, side_effect=TimeoutError):
result = await test_extractor._extract_sell_directly_from_ad_page()
assert result is None