Mirrors/kleinanzeigen-bot
1
0
Fork 0
mirror of https://github.com/Second-Hand-Friends/kleinanzeigen-bot.git synced 2026-03-12 02:31:45 +01:00
Code Issues Packages Projects Releases Wiki Activity

refact: apply consistent formatting

Browse Source
This commit is contained in:
sebthom
2025-04-27 23:54:22 +02:00
parent fe33a0e461
commit ef923a8337
21 changed files with 1020 additions and 709 deletions
Download Patch File Download Diff File Expand all files Collapse all files

50
tests/conftest.py
View File

@@ -1,8 +1,6 @@
"""
SPDX-FileCopyrightText: © Jens Bergmann and contributors
SPDX-License-Identifier: AGPL-3.0-or-later
SPDX-ArtifactOfProjectHomePage: https://github.com/Second-Hand-Friends/kleinanzeigen-bot/
"""
# SPDX-FileCopyrightText: © Jens Bergmann and contributors
# SPDX-License-Identifier: AGPL-3.0-or-later
# SPDX-ArtifactOfProjectHomePage: https://github.com/Second-Hand-Friends/kleinanzeigen-bot/
import os
from typing import Any, Final
from unittest.mock import MagicMock
@@ -21,7 +19,7 @@ LOG.setLevel(loggers.DEBUG)
@pytest.fixture
def test_data_dir(tmp_path: str) -> str:
def test_data_dir(tmp_path:str) -> str:
"""Provides a temporary directory for test data.
This fixture uses pytest's built-in tmp_path fixture to create a temporary
@@ -41,33 +39,33 @@ def sample_config() -> dict[str, Any]:
- Publishing settings
"""
return {
'login': {
'username': 'testuser',
'password': 'testpass'
"login": {
"username": "testuser",
"password": "testpass"
},
'browser': {
'arguments': [],
'binary_location': None,
'extensions': [],
'use_private_window': True,
'user_data_dir': None,
'profile_name': None
"browser": {
"arguments": [],
"binary_location": None,
"extensions": [],
"use_private_window": True,
"user_data_dir": None,
"profile_name": None
},
'ad_defaults': {
'description': {
'prefix': 'Test Prefix',
'suffix': 'Test Suffix'
"ad_defaults": {
"description": {
"prefix": "Test Prefix",
"suffix": "Test Suffix"
}
},
'publishing': {
'delete_old_ads': 'BEFORE_PUBLISH',
'delete_old_ads_by_title': False
"publishing": {
"delete_old_ads": "BEFORE_PUBLISH",
"delete_old_ads_by_title": False
}
}
@pytest.fixture
def test_bot(sample_config: dict[str, Any]) -> KleinanzeigenBot:
def test_bot(sample_config:dict[str, Any]) -> KleinanzeigenBot:
"""Provides a fresh KleinanzeigenBot instance for all test classes.
Dependencies:
@@ -89,7 +87,7 @@ def browser_mock() -> MagicMock:
@pytest.fixture
def log_file_path(test_data_dir: str) -> str:
def log_file_path(test_data_dir:str) -> str:
"""Provides a temporary path for log files.
Dependencies:
@@ -99,7 +97,7 @@ def log_file_path(test_data_dir: str) -> str:
@pytest.fixture
def test_extractor(browser_mock: MagicMock, sample_config: dict[str, Any]) -> AdExtractor:
def test_extractor(browser_mock:MagicMock, sample_config:dict[str, Any]) -> AdExtractor:
"""Provides a fresh AdExtractor instance for testing.
Dependencies:

12
tests/unit/test_ads.py
View File

@@ -122,9 +122,9 @@ def test_calculate_content_hash_with_none_values() -> None:
)
])
def test_get_description_affixes(
config: dict[str, Any],
prefix: bool,
expected: str
config:dict[str, Any],
prefix:bool,
expected:str
) -> None:
"""Test get_description_affixes function with various inputs."""
result = ads.get_description_affixes(config, prefix = prefix)
@@ -157,7 +157,7 @@ def test_get_description_affixes(
""
)
])
def test_get_description_affixes_edge_cases(config: dict[str, Any], prefix: bool, expected: str) -> None:
def test_get_description_affixes_edge_cases(config:dict[str, Any], prefix:bool, expected:str) -> None:
"""Test edge cases for description affix handling."""
assert ads.get_description_affixes(config, prefix = prefix) == expected
@@ -170,7 +170,7 @@ def test_get_description_affixes_edge_cases(config: dict[str, Any], prefix: bool
(3.14, ""), # Test with a float
(set(), ""), # Test with an empty set
])
def test_get_description_affixes_edge_cases_non_dict(config: Any, expected: str) -> None:
def test_get_description_affixes_edge_cases_non_dict(config:Any, expected:str) -> None:
"""Test get_description_affixes function with non-dict inputs."""
result = ads.get_description_affixes(config, prefix=True)
result = ads.get_description_affixes(config, prefix = True)
assert result == expected

6
tests/unit/test_bot.py
View File

@@ -12,21 +12,21 @@ class TestKleinanzeigenBot:
def bot(self) -> KleinanzeigenBot:
return KleinanzeigenBot()
def test_parse_args_help(self, bot: KleinanzeigenBot) -> None:
def test_parse_args_help(self, bot:KleinanzeigenBot) -> None:
"""Test parsing of help command"""
bot.parse_args(["app", "help"])
assert bot.command == "help"
assert bot.ads_selector == "due"
assert not bot.keep_old_ads
def test_parse_args_publish(self, bot: KleinanzeigenBot) -> None:
def test_parse_args_publish(self, bot:KleinanzeigenBot) -> None:
"""Test parsing of publish command with options"""
bot.parse_args(["app", "publish", "--ads=all", "--keep-old"])
assert bot.command == "publish"
assert bot.ads_selector == "all"
assert bot.keep_old_ads
def test_get_version(self, bot: KleinanzeigenBot) -> None:
def test_get_version(self, bot:KleinanzeigenBot) -> None:
"""Test version retrieval"""
version = bot.get_version()
assert isinstance(version, str)

286
tests/unit/test_extract.py
View File

@@ -12,31 +12,31 @@ from kleinanzeigen_bot.utils.web_scraping_mixin import Browser, By, Element
class _DimensionsDict(TypedDict):
dimension108: str
dimension108:str
class _UniversalAnalyticsOptsDict(TypedDict):
dimensions: _DimensionsDict
dimensions:_DimensionsDict
class _BelenConfDict(TypedDict):
universalAnalyticsOpts: _UniversalAnalyticsOptsDict
universalAnalyticsOpts:_UniversalAnalyticsOptsDict
class _SpecialAttributesDict(TypedDict, total = False):
art_s: str
condition_s: str
art_s:str
condition_s:str
class _TestCaseDict(TypedDict): # noqa: PYI049 Private TypedDict `...` is never used
belen_conf: _BelenConfDict
expected: _SpecialAttributesDict
belen_conf:_BelenConfDict
expected:_SpecialAttributesDict
class TestAdExtractorBasics:
"""Basic synchronous tests for AdExtractor."""
def test_constructor(self, browser_mock: MagicMock, sample_config: dict[str, Any]) -> None:
def test_constructor(self, browser_mock:MagicMock, sample_config:dict[str, Any]) -> None:
"""Test the constructor of AdExtractor"""
extractor = AdExtractor(browser_mock, sample_config)
assert extractor.browser == browser_mock
@@ -48,7 +48,7 @@ class TestAdExtractorBasics:
("https://www.kleinanzeigen.de/s-anzeige/invalid-id/abc", -1),
("https://www.kleinanzeigen.de/invalid-url", -1),
])
def test_extract_ad_id_from_ad_url(self, test_extractor: AdExtractor, url: str, expected_id: int) -> None:
def test_extract_ad_id_from_ad_url(self, test_extractor:AdExtractor, url:str, expected_id:int) -> None:
"""Test extraction of ad ID from different URL formats."""
assert test_extractor.extract_ad_id_from_ad_url(url) == expected_id
@@ -66,19 +66,19 @@ class TestAdExtractorPricing:
@pytest.mark.asyncio
# pylint: disable=protected-access
async def test_extract_pricing_info(
self, test_extractor: AdExtractor, price_text: str, expected_price: int | None, expected_type: str
self, test_extractor:AdExtractor, price_text:str, expected_price:int | None, expected_type:str
) -> None:
"""Test price extraction with different formats"""
with patch.object(test_extractor, 'web_text', new_callable = AsyncMock, return_value = price_text):
with patch.object(test_extractor, "web_text", new_callable = AsyncMock, return_value = price_text):
price, price_type = await test_extractor._extract_pricing_info_from_ad_page()
assert price == expected_price
assert price_type == expected_type
@pytest.mark.asyncio
# pylint: disable=protected-access
async def test_extract_pricing_info_timeout(self, test_extractor: AdExtractor) -> None:
async def test_extract_pricing_info_timeout(self, test_extractor:AdExtractor) -> None:
"""Test price extraction when element is not found"""
with patch.object(test_extractor, 'web_text', new_callable = AsyncMock, side_effect = TimeoutError):
with patch.object(test_extractor, "web_text", new_callable = AsyncMock, side_effect = TimeoutError):
price, price_type = await test_extractor._extract_pricing_info_from_ad_page()
assert price is None
assert price_type == "NOT_APPLICABLE"
@@ -95,15 +95,15 @@ class TestAdExtractorShipping:
@pytest.mark.asyncio
# pylint: disable=protected-access
async def test_extract_shipping_info(
self, test_extractor: AdExtractor, shipping_text: str, expected_type: str, expected_cost: float | None
self, test_extractor:AdExtractor, shipping_text:str, expected_type:str, expected_cost:float | None
) -> None:
"""Test shipping info extraction with different text formats."""
with patch.object(test_extractor, 'page', MagicMock()), \
patch.object(test_extractor, 'web_text', new_callable = AsyncMock, return_value = shipping_text), \
patch.object(test_extractor, 'web_request', new_callable = AsyncMock) as mock_web_request:
with patch.object(test_extractor, "page", MagicMock()), \
patch.object(test_extractor, "web_text", new_callable = AsyncMock, return_value = shipping_text), \
patch.object(test_extractor, "web_request", new_callable = AsyncMock) as mock_web_request:
if expected_cost:
shipping_response: dict[str, Any] = {
shipping_response:dict[str, Any] = {
"data": {
"shippingOptionsResponse": {
"options": [
@@ -125,7 +125,7 @@ class TestAdExtractorShipping:
@pytest.mark.asyncio
# pylint: disable=protected-access
async def test_extract_shipping_info_with_options(self, test_extractor: AdExtractor) -> None:
async def test_extract_shipping_info_with_options(self, test_extractor:AdExtractor) -> None:
"""Test shipping info extraction with shipping options."""
shipping_response = {
"content": json.dumps({
@@ -139,9 +139,9 @@ class TestAdExtractorShipping:
})
}
with patch.object(test_extractor, 'page', MagicMock()), \
patch.object(test_extractor, 'web_text', new_callable = AsyncMock, return_value = "+ Versand ab 5,49 €"), \
patch.object(test_extractor, 'web_request', new_callable = AsyncMock, return_value = shipping_response):
with patch.object(test_extractor, "page", MagicMock()), \
patch.object(test_extractor, "web_text", new_callable = AsyncMock, return_value = "+ Versand ab 5,49 €"), \
patch.object(test_extractor, "web_request", new_callable = AsyncMock, return_value = shipping_response):
shipping_type, costs, options = await test_extractor._extract_shipping_info_from_ad_page()
@@ -154,21 +154,21 @@ class TestAdExtractorNavigation:
"""Tests for navigation related functionality."""
@pytest.mark.asyncio
async def test_navigate_to_ad_page_with_url(self, test_extractor: AdExtractor) -> None:
async def test_navigate_to_ad_page_with_url(self, test_extractor:AdExtractor) -> None:
"""Test navigation to ad page using a URL."""
page_mock = AsyncMock()
page_mock.url = "https://www.kleinanzeigen.de/s-anzeige/test/12345"
with patch.object(test_extractor, 'page', page_mock), \
patch.object(test_extractor, 'web_open', new_callable = AsyncMock) as mock_web_open, \
patch.object(test_extractor, 'web_find', new_callable = AsyncMock, side_effect = TimeoutError):
with patch.object(test_extractor, "page", page_mock), \
patch.object(test_extractor, "web_open", new_callable = AsyncMock) as mock_web_open, \
patch.object(test_extractor, "web_find", new_callable = AsyncMock, side_effect = TimeoutError):
result = await test_extractor.naviagte_to_ad_page("https://www.kleinanzeigen.de/s-anzeige/test/12345")
assert result is True
mock_web_open.assert_called_with("https://www.kleinanzeigen.de/s-anzeige/test/12345")
@pytest.mark.asyncio
async def test_navigate_to_ad_page_with_id(self, test_extractor: AdExtractor) -> None:
async def test_navigate_to_ad_page_with_id(self, test_extractor:AdExtractor) -> None:
"""Test navigation to ad page using an ID."""
page_mock = AsyncMock()
page_mock.url = "https://www.kleinanzeigen.de/s-anzeige/test/12345"
@@ -186,7 +186,7 @@ class TestAdExtractorNavigation:
popup_close_mock.click = AsyncMock()
popup_close_mock.apply = AsyncMock(return_value = True)
def find_mock(selector_type: By, selector_value: str, **_: Any) -> Element | None:
def find_mock(selector_type:By, selector_value:str, **_:Any) -> Element | None:
if selector_type == By.ID and selector_value == "site-search-query":
return input_mock
if selector_type == By.ID and selector_value == "site-search-submit":
@@ -195,20 +195,20 @@ class TestAdExtractorNavigation:
return popup_close_mock
return None
with patch.object(test_extractor, 'page', page_mock), \
patch.object(test_extractor, 'web_open', new_callable = AsyncMock) as mock_web_open, \
patch.object(test_extractor, 'web_input', new_callable = AsyncMock), \
patch.object(test_extractor, 'web_check', new_callable = AsyncMock, return_value = True), \
patch.object(test_extractor, 'web_find', new_callable = AsyncMock, side_effect = find_mock):
with patch.object(test_extractor, "page", page_mock), \
patch.object(test_extractor, "web_open", new_callable = AsyncMock) as mock_web_open, \
patch.object(test_extractor, "web_input", new_callable = AsyncMock), \
patch.object(test_extractor, "web_check", new_callable = AsyncMock, return_value = True), \
patch.object(test_extractor, "web_find", new_callable = AsyncMock, side_effect = find_mock):
result = await test_extractor.naviagte_to_ad_page(12345)
assert result is True
mock_web_open.assert_called_with('https://www.kleinanzeigen.de/')
mock_web_open.assert_called_with("https://www.kleinanzeigen.de/")
submit_button_mock.click.assert_awaited_once()
popup_close_mock.click.assert_awaited_once()
@pytest.mark.asyncio
async def test_navigate_to_ad_page_with_popup(self, test_extractor: AdExtractor) -> None:
async def test_navigate_to_ad_page_with_popup(self, test_extractor:AdExtractor) -> None:
"""Test navigation to ad page with popup handling."""
page_mock = AsyncMock()
page_mock.url = "https://www.kleinanzeigen.de/s-anzeige/test/12345"
@@ -218,18 +218,18 @@ class TestAdExtractorNavigation:
input_mock.send_keys = AsyncMock()
input_mock.apply = AsyncMock(return_value = True)
with patch.object(test_extractor, 'page', page_mock), \
patch.object(test_extractor, 'web_open', new_callable = AsyncMock), \
patch.object(test_extractor, 'web_find', new_callable = AsyncMock, return_value = input_mock), \
patch.object(test_extractor, 'web_click', new_callable = AsyncMock) as mock_web_click, \
patch.object(test_extractor, 'web_check', new_callable = AsyncMock, return_value = True):
with patch.object(test_extractor, "page", page_mock), \
patch.object(test_extractor, "web_open", new_callable = AsyncMock), \
patch.object(test_extractor, "web_find", new_callable = AsyncMock, return_value = input_mock), \
patch.object(test_extractor, "web_click", new_callable = AsyncMock) as mock_web_click, \
patch.object(test_extractor, "web_check", new_callable = AsyncMock, return_value = True):
result = await test_extractor.naviagte_to_ad_page(12345)
assert result is True
mock_web_click.assert_called_with(By.CLASS_NAME, 'mfp-close')
mock_web_click.assert_called_with(By.CLASS_NAME, "mfp-close")
@pytest.mark.asyncio
async def test_navigate_to_ad_page_invalid_id(self, test_extractor: AdExtractor) -> None:
async def test_navigate_to_ad_page_invalid_id(self, test_extractor:AdExtractor) -> None:
"""Test navigation to ad page with invalid ID."""
page_mock = AsyncMock()
page_mock.url = "https://www.kleinanzeigen.de/s-suchen.html?k0"
@@ -240,22 +240,22 @@ class TestAdExtractorNavigation:
input_mock.apply = AsyncMock(return_value = True)
input_mock.attrs = {}
with patch.object(test_extractor, 'page', page_mock), \
patch.object(test_extractor, 'web_open', new_callable = AsyncMock), \
patch.object(test_extractor, 'web_find', new_callable = AsyncMock, return_value = input_mock):
with patch.object(test_extractor, "page", page_mock), \
patch.object(test_extractor, "web_open", new_callable = AsyncMock), \
patch.object(test_extractor, "web_find", new_callable = AsyncMock, return_value = input_mock):
result = await test_extractor.naviagte_to_ad_page(99999)
assert result is False
@pytest.mark.asyncio
async def test_extract_own_ads_urls(self, test_extractor: AdExtractor) -> None:
async def test_extract_own_ads_urls(self, test_extractor:AdExtractor) -> None:
"""Test extraction of own ads URLs - basic test."""
with patch.object(test_extractor, 'web_open', new_callable=AsyncMock), \
patch.object(test_extractor, 'web_sleep', new_callable=AsyncMock), \
patch.object(test_extractor, 'web_find', new_callable=AsyncMock) as mock_web_find, \
patch.object(test_extractor, 'web_find_all', new_callable=AsyncMock) as mock_web_find_all, \
patch.object(test_extractor, 'web_scroll_page_down', new_callable=AsyncMock), \
patch.object(test_extractor, 'web_execute', new_callable=AsyncMock):
with patch.object(test_extractor, "web_open", new_callable = AsyncMock), \
patch.object(test_extractor, "web_sleep", new_callable = AsyncMock), \
patch.object(test_extractor, "web_find", new_callable = AsyncMock) as mock_web_find, \
patch.object(test_extractor, "web_find_all", new_callable = AsyncMock) as mock_web_find_all, \
patch.object(test_extractor, "web_scroll_page_down", new_callable = AsyncMock), \
patch.object(test_extractor, "web_execute", new_callable = AsyncMock):
# --- Setup mock objects for DOM elements ---
# Mocks needed for the actual execution flow
@@ -263,7 +263,7 @@ class TestAdExtractorNavigation:
pagination_section_mock = MagicMock()
cardbox_mock = MagicMock() # Represents the <li> element
link_mock = MagicMock() # Represents the <a> element
link_mock.attrs = {'href': '/s-anzeige/test/12345'} # Configure the desired output
link_mock.attrs = {"href": "/s-anzeige/test/12345"} # Configure the desired output
# Mocks for elements potentially checked but maybe not strictly needed for output
# (depending on how robust the mocking is)
@@ -287,7 +287,7 @@ class TestAdExtractorNavigation:
# 2. Find all '.cardbox' elements (inside loop)
mock_web_find_all.side_effect = [
[], # Call 1: find 'button[aria-label="Nächste"]' -> No next button = single page
[cardbox_mock] # Call 2: find .cardbox -> One ad item
[cardbox_mock] # Call 2: find .cardbox -> One ad item
# Add more mocks here if pagination navigation calls web_find_all
]
@@ -295,20 +295,20 @@ class TestAdExtractorNavigation:
refs = await test_extractor.extract_own_ads_urls()
# --- Assertions ---
assert refs == ['/s-anzeige/test/12345'] # Now it should match
assert refs == ["/s-anzeige/test/12345"] # Now it should match
# Optional: Verify calls were made as expected
mock_web_find.assert_has_calls([
call(By.ID, 'my-manageitems-adlist'),
call(By.CSS_SELECTOR, '.Pagination', timeout=10),
call(By.ID, 'my-manageitems-adlist'),
call(By.CSS_SELECTOR, 'div.manageitems-item-ad h3 a.text-onSurface', parent=cardbox_mock),
], any_order=False) # Check order if important
call(By.ID, "my-manageitems-adlist"),
call(By.CSS_SELECTOR, ".Pagination", timeout = 10),
call(By.ID, "my-manageitems-adlist"),
call(By.CSS_SELECTOR, "div.manageitems-item-ad h3 a.text-onSurface", parent = cardbox_mock),
], any_order = False) # Check order if important
mock_web_find_all.assert_has_calls([
call(By.CSS_SELECTOR, 'button[aria-label="Nächste"]', parent=pagination_section_mock),
call(By.CLASS_NAME, 'cardbox', parent=ad_list_container_mock),
], any_order=False)
call(By.CSS_SELECTOR, 'button[aria-label="Nächste"]', parent = pagination_section_mock),
call(By.CLASS_NAME, "cardbox", parent = ad_list_container_mock),
], any_order = False)
class TestAdExtractorContent:
@@ -318,14 +318,14 @@ class TestAdExtractorContent:
@pytest.fixture
def extractor_with_config(self) -> AdExtractor:
"""Create extractor with specific config for testing prefix/suffix handling."""
browser_mock = MagicMock(spec=Browser)
browser_mock = MagicMock(spec = Browser)
return AdExtractor(browser_mock, {}) # Empty config, will be overridden in tests
@pytest.mark.asyncio
async def test_extract_description_with_affixes(
self,
test_extractor: AdExtractor,
description_test_cases: list[tuple[dict[str, Any], str, str]]
test_extractor:AdExtractor,
description_test_cases:list[tuple[dict[str, Any], str, str]]
) -> None:
"""Test extraction of description with various prefix/suffix configurations."""
# Mock the page
@@ -337,18 +337,18 @@ class TestAdExtractorContent:
test_extractor.config = config
with patch.multiple(test_extractor,
web_text=AsyncMock(side_effect=[
web_text = AsyncMock(side_effect = [
"Test Title", # Title
raw_description, # Raw description (without affixes)
"03.02.2025" # Creation date
]),
_extract_category_from_ad_page=AsyncMock(return_value="160"),
_extract_special_attributes_from_ad_page=AsyncMock(return_value={}),
_extract_pricing_info_from_ad_page=AsyncMock(return_value=(None, "NOT_APPLICABLE")),
_extract_shipping_info_from_ad_page=AsyncMock(return_value=("NOT_APPLICABLE", None, None)),
_extract_sell_directly_from_ad_page=AsyncMock(return_value=False),
_download_images_from_ad_page=AsyncMock(return_value=[]),
_extract_contact_from_ad_page=AsyncMock(return_value={})
_extract_category_from_ad_page = AsyncMock(return_value = "160"),
_extract_special_attributes_from_ad_page = AsyncMock(return_value = {}),
_extract_pricing_info_from_ad_page = AsyncMock(return_value = (None, "NOT_APPLICABLE")),
_extract_shipping_info_from_ad_page = AsyncMock(return_value = ("NOT_APPLICABLE", None, None)),
_extract_sell_directly_from_ad_page = AsyncMock(return_value = False),
_download_images_from_ad_page = AsyncMock(return_value = []),
_extract_contact_from_ad_page = AsyncMock(return_value = {})
):
info = await test_extractor._extract_ad_page_info("/some/dir", 12345)
assert info["description"] == raw_description
@@ -356,7 +356,7 @@ class TestAdExtractorContent:
@pytest.mark.asyncio
async def test_extract_description_with_affixes_timeout(
self,
test_extractor: AdExtractor
test_extractor:AdExtractor
) -> None:
"""Test handling of timeout when extracting description."""
# Mock the page
@@ -365,18 +365,18 @@ class TestAdExtractorContent:
test_extractor.page = page_mock
with patch.multiple(test_extractor,
web_text=AsyncMock(side_effect=[
web_text = AsyncMock(side_effect = [
"Test Title", # Title succeeds
TimeoutError("Timeout"), # Description times out
"03.02.2025" # Date succeeds
]),
_extract_category_from_ad_page=AsyncMock(return_value="160"),
_extract_special_attributes_from_ad_page=AsyncMock(return_value={}),
_extract_pricing_info_from_ad_page=AsyncMock(return_value=(None, "NOT_APPLICABLE")),
_extract_shipping_info_from_ad_page=AsyncMock(return_value=("NOT_APPLICABLE", None, None)),
_extract_sell_directly_from_ad_page=AsyncMock(return_value=False),
_download_images_from_ad_page=AsyncMock(return_value=[]),
_extract_contact_from_ad_page=AsyncMock(return_value={})
_extract_category_from_ad_page = AsyncMock(return_value = "160"),
_extract_special_attributes_from_ad_page = AsyncMock(return_value = {}),
_extract_pricing_info_from_ad_page = AsyncMock(return_value = (None, "NOT_APPLICABLE")),
_extract_shipping_info_from_ad_page = AsyncMock(return_value = ("NOT_APPLICABLE", None, None)),
_extract_sell_directly_from_ad_page = AsyncMock(return_value = False),
_download_images_from_ad_page = AsyncMock(return_value = []),
_extract_contact_from_ad_page = AsyncMock(return_value = {})
):
try:
info = await test_extractor._extract_ad_page_info("/some/dir", 12345)
@@ -388,7 +388,7 @@ class TestAdExtractorContent:
@pytest.mark.asyncio
async def test_extract_description_with_affixes_no_affixes(
self,
test_extractor: AdExtractor
test_extractor:AdExtractor
) -> None:
"""Test extraction of description without any affixes in config."""
# Mock the page
@@ -399,24 +399,24 @@ class TestAdExtractorContent:
raw_description = "Original Description"
with patch.multiple(test_extractor,
web_text=AsyncMock(side_effect=[
web_text = AsyncMock(side_effect = [
"Test Title", # Title
raw_description, # Description without affixes
"03.02.2025" # Creation date
]),
_extract_category_from_ad_page=AsyncMock(return_value="160"),
_extract_special_attributes_from_ad_page=AsyncMock(return_value={}),
_extract_pricing_info_from_ad_page=AsyncMock(return_value=(None, "NOT_APPLICABLE")),
_extract_shipping_info_from_ad_page=AsyncMock(return_value=("NOT_APPLICABLE", None, None)),
_extract_sell_directly_from_ad_page=AsyncMock(return_value=False),
_download_images_from_ad_page=AsyncMock(return_value=[]),
_extract_contact_from_ad_page=AsyncMock(return_value={})
_extract_category_from_ad_page = AsyncMock(return_value = "160"),
_extract_special_attributes_from_ad_page = AsyncMock(return_value = {}),
_extract_pricing_info_from_ad_page = AsyncMock(return_value = (None, "NOT_APPLICABLE")),
_extract_shipping_info_from_ad_page = AsyncMock(return_value = ("NOT_APPLICABLE", None, None)),
_extract_sell_directly_from_ad_page = AsyncMock(return_value = False),
_download_images_from_ad_page = AsyncMock(return_value = []),
_extract_contact_from_ad_page = AsyncMock(return_value = {})
):
info = await test_extractor._extract_ad_page_info("/some/dir", 12345)
assert info["description"] == raw_description
@pytest.mark.asyncio
async def test_extract_sell_directly(self, test_extractor: AdExtractor) -> None:
async def test_extract_sell_directly(self, test_extractor:AdExtractor) -> None:
"""Test extraction of sell directly option."""
test_cases = [
("Direkt kaufen", True),
@@ -424,11 +424,11 @@ class TestAdExtractorContent:
]
for text, expected in test_cases:
with patch.object(test_extractor, 'web_text', new_callable=AsyncMock, return_value=text):
with patch.object(test_extractor, "web_text", new_callable = AsyncMock, return_value = text):
result = await test_extractor._extract_sell_directly_from_ad_page()
assert result is expected
with patch.object(test_extractor, 'web_text', new_callable=AsyncMock, side_effect=TimeoutError):
with patch.object(test_extractor, "web_text", new_callable = AsyncMock, side_effect = TimeoutError):
result = await test_extractor._extract_sell_directly_from_ad_page()
assert result is None
@@ -451,15 +451,15 @@ class TestAdExtractorCategory:
@pytest.mark.asyncio
# pylint: disable=protected-access
async def test_extract_category(self, extractor: AdExtractor) -> None:
async def test_extract_category(self, extractor:AdExtractor) -> None:
"""Test category extraction from breadcrumb."""
category_line = MagicMock()
first_part = MagicMock()
first_part.attrs = {'href': '/s-familie-kind-baby/c17'}
first_part.attrs = {"href": "/s-familie-kind-baby/c17"}
second_part = MagicMock()
second_part.attrs = {'href': '/s-spielzeug/c23'}
second_part.attrs = {"href": "/s-spielzeug/c23"}
with patch.object(extractor, 'web_find', new_callable = AsyncMock) as mock_web_find:
with patch.object(extractor, "web_find", new_callable = AsyncMock) as mock_web_find:
mock_web_find.side_effect = [
category_line,
first_part,
@@ -469,15 +469,15 @@ class TestAdExtractorCategory:
result = await extractor._extract_category_from_ad_page()
assert result == "17/23"
mock_web_find.assert_any_call(By.ID, 'vap-brdcrmb')
mock_web_find.assert_any_call(By.CSS_SELECTOR, 'a:nth-of-type(2)', parent = category_line)
mock_web_find.assert_any_call(By.CSS_SELECTOR, 'a:nth-of-type(3)', parent = category_line)
mock_web_find.assert_any_call(By.ID, "vap-brdcrmb")
mock_web_find.assert_any_call(By.CSS_SELECTOR, "a:nth-of-type(2)", parent = category_line)
mock_web_find.assert_any_call(By.CSS_SELECTOR, "a:nth-of-type(3)", parent = category_line)
@pytest.mark.asyncio
# pylint: disable=protected-access
async def test_extract_special_attributes_empty(self, extractor: AdExtractor) -> None:
async def test_extract_special_attributes_empty(self, extractor:AdExtractor) -> None:
"""Test extraction of special attributes when empty."""
with patch.object(extractor, 'web_execute', new_callable = AsyncMock) as mock_web_execute:
with patch.object(extractor, "web_execute", new_callable = AsyncMock) as mock_web_execute:
mock_web_execute.return_value = {
"universalAnalyticsOpts": {
"dimensions": {
@@ -507,11 +507,11 @@ class TestAdExtractorContact:
@pytest.mark.asyncio
# pylint: disable=protected-access
async def test_extract_contact_info(self, extractor: AdExtractor) -> None:
async def test_extract_contact_info(self, extractor:AdExtractor) -> None:
"""Test extraction of contact information."""
with patch.object(extractor, 'page', MagicMock()), \
patch.object(extractor, 'web_text', new_callable = AsyncMock) as mock_web_text, \
patch.object(extractor, 'web_find', new_callable = AsyncMock) as mock_web_find:
with patch.object(extractor, "page", MagicMock()), \
patch.object(extractor, "web_text", new_callable = AsyncMock) as mock_web_text, \
patch.object(extractor, "web_find", new_callable = AsyncMock) as mock_web_find:
mock_web_text.side_effect = [
"12345 Berlin - Mitte",
@@ -535,22 +535,22 @@ class TestAdExtractorContact:
@pytest.mark.asyncio
# pylint: disable=protected-access
async def test_extract_contact_info_timeout(self, extractor: AdExtractor) -> None:
async def test_extract_contact_info_timeout(self, extractor:AdExtractor) -> None:
"""Test contact info extraction when elements are not found."""
with patch.object(extractor, 'page', MagicMock()), \
patch.object(extractor, 'web_text', new_callable = AsyncMock, side_effect = TimeoutError()), \
patch.object(extractor, 'web_find', new_callable = AsyncMock, side_effect = TimeoutError()), \
with patch.object(extractor, "page", MagicMock()), \
patch.object(extractor, "web_text", new_callable = AsyncMock, side_effect = TimeoutError()), \
patch.object(extractor, "web_find", new_callable = AsyncMock, side_effect = TimeoutError()), \
pytest.raises(TimeoutError):
await extractor._extract_contact_from_ad_page()
await extractor._extract_contact_from_ad_page()
@pytest.mark.asyncio
# pylint: disable=protected-access
async def test_extract_contact_info_with_phone(self, extractor: AdExtractor) -> None:
async def test_extract_contact_info_with_phone(self, extractor:AdExtractor) -> None:
"""Test extraction of contact information including phone number."""
with patch.object(extractor, 'page', MagicMock()), \
patch.object(extractor, 'web_text', new_callable = AsyncMock) as mock_web_text, \
patch.object(extractor, 'web_find', new_callable = AsyncMock) as mock_web_find:
with patch.object(extractor, "page", MagicMock()), \
patch.object(extractor, "web_text", new_callable = AsyncMock) as mock_web_text, \
patch.object(extractor, "web_find", new_callable = AsyncMock) as mock_web_find:
mock_web_text.side_effect = [
"12345 Berlin - Mitte",
@@ -588,19 +588,19 @@ class TestAdExtractorDownload:
return AdExtractor(browser_mock, config_mock)
@pytest.mark.asyncio
async def test_download_ad_existing_directory(self, extractor: AdExtractor) -> None:
async def test_download_ad_existing_directory(self, extractor:AdExtractor) -> None:
"""Test downloading an ad when the directory already exists."""
with patch('os.path.exists') as mock_exists, \
patch('os.path.isdir') as mock_isdir, \
patch('os.makedirs') as mock_makedirs, \
patch('os.mkdir') as mock_mkdir, \
patch('shutil.rmtree') as mock_rmtree, \
patch('kleinanzeigen_bot.extract.dicts.save_dict', autospec = True) as mock_save_dict, \
patch.object(extractor, '_extract_ad_page_info', new_callable = AsyncMock) as mock_extract:
with patch("os.path.exists") as mock_exists, \
patch("os.path.isdir") as mock_isdir, \
patch("os.makedirs") as mock_makedirs, \
patch("os.mkdir") as mock_mkdir, \
patch("shutil.rmtree") as mock_rmtree, \
patch("kleinanzeigen_bot.extract.dicts.save_dict", autospec = True) as mock_save_dict, \
patch.object(extractor, "_extract_ad_page_info", new_callable = AsyncMock) as mock_extract:
base_dir = 'downloaded-ads'
ad_dir = os.path.join(base_dir, 'ad_12345')
yaml_path = os.path.join(ad_dir, 'ad_12345.yaml')
base_dir = "downloaded-ads"
ad_dir = os.path.join(base_dir, "ad_12345")
yaml_path = os.path.join(ad_dir, "ad_12345.yaml")
# Configure mocks for directory checks
existing_paths = {base_dir, ad_dir}
@@ -632,32 +632,32 @@ class TestAdExtractorDownload:
# Workaround for hard-coded path in download_ad
actual_call = mock_save_dict.call_args
assert actual_call is not None
actual_path = actual_call[0][0].replace('/', os.path.sep)
actual_path = actual_call[0][0].replace("/", os.path.sep)
assert actual_path == yaml_path
assert actual_call[0][1] == mock_extract.return_value
@pytest.mark.asyncio
# pylint: disable=protected-access
async def test_download_images_no_images(self, extractor: AdExtractor) -> None:
async def test_download_images_no_images(self, extractor:AdExtractor) -> None:
"""Test image download when no images are found."""
with patch.object(extractor, 'web_find', new_callable = AsyncMock, side_effect = TimeoutError):
with patch.object(extractor, "web_find", new_callable = AsyncMock, side_effect = TimeoutError):
image_paths = await extractor._download_images_from_ad_page("/some/dir", 12345)
assert len(image_paths) == 0
@pytest.mark.asyncio
async def test_download_ad(self, extractor: AdExtractor) -> None:
async def test_download_ad(self, extractor:AdExtractor) -> None:
"""Test downloading an entire ad."""
with patch('os.path.exists') as mock_exists, \
patch('os.path.isdir') as mock_isdir, \
patch('os.makedirs') as mock_makedirs, \
patch('os.mkdir') as mock_mkdir, \
patch('shutil.rmtree') as mock_rmtree, \
patch('kleinanzeigen_bot.extract.dicts.save_dict', autospec = True) as mock_save_dict, \
patch.object(extractor, '_extract_ad_page_info', new_callable = AsyncMock) as mock_extract:
with patch("os.path.exists") as mock_exists, \
patch("os.path.isdir") as mock_isdir, \
patch("os.makedirs") as mock_makedirs, \
patch("os.mkdir") as mock_mkdir, \
patch("shutil.rmtree") as mock_rmtree, \
patch("kleinanzeigen_bot.extract.dicts.save_dict", autospec = True) as mock_save_dict, \
patch.object(extractor, "_extract_ad_page_info", new_callable = AsyncMock) as mock_extract:
base_dir = 'downloaded-ads'
ad_dir = os.path.join(base_dir, 'ad_12345')
yaml_path = os.path.join(ad_dir, 'ad_12345.yaml')
base_dir = "downloaded-ads"
ad_dir = os.path.join(base_dir, "ad_12345")
yaml_path = os.path.join(ad_dir, "ad_12345.yaml")
# Configure mocks for directory checks
mock_exists.return_value = False
@@ -690,6 +690,6 @@ class TestAdExtractorDownload:
# Get the actual call arguments
actual_call = mock_save_dict.call_args
assert actual_call is not None
actual_path = actual_call[0][0].replace('/', os.path.sep)
actual_path = actual_call[0][0].replace("/", os.path.sep)
assert actual_path == yaml_path
assert actual_call[0][1] == mock_extract.return_value

4
tests/unit/test_i18n.py
View File

@@ -13,7 +13,7 @@ from kleinanzeigen_bot.utils import i18n
("fr_CA", ("fr", "CA", "UTF-8")), # Test with language + region, no encoding
("pt_BR.iso8859-1", ("pt", "BR", "ISO8859-1")), # Test with language + region + encoding
])
def test_detect_locale(monkeypatch: MonkeyPatch, lang: str | None, expected: i18n.Locale) -> None:
def test_detect_locale(monkeypatch:MonkeyPatch, lang:str | None, expected:i18n.Locale) -> None:
"""
Pytest test case to verify detect_system_language() behavior under various LANG values.
"""
@@ -49,7 +49,7 @@ def test_pluralize(
noun:str,
count:int,
prefix_with_count:bool,
expected: str
expected:str
) -> None:
i18n.set_current_locale(i18n.Locale(lang, "US", "UTF_8"))

492
tests/unit/test_init.py
View File

File diff suppressed because it is too large Load Diff

94
tests/unit/test_translations.py
View File

@@ -26,12 +26,12 @@ from ruamel.yaml import YAML
from kleinanzeigen_bot import resources
# Messages that are intentionally not translated (internal/debug messages)
EXCLUDED_MESSAGES: dict[str, set[str]] = {
EXCLUDED_MESSAGES:dict[str, set[str]] = {
"kleinanzeigen_bot/__init__.py": {"############################################"}
}
# Special modules that are known to be needed even if not in messages_by_file
KNOWN_NEEDED_MODULES = {'getopt.py'}
KNOWN_NEEDED_MODULES = {"getopt.py"}
# Type aliases for better readability
ModulePath = str
@@ -45,12 +45,12 @@ MissingDict = dict[FunctionName, dict[Message, set[Message]]]
@dataclass
class MessageLocation:
"""Represents the location of a message in the codebase."""
module: str
function: str
message: str
module:str
function:str
message:str
def _get_function_name(node: ast.AST) -> str:
def _get_function_name(node:ast.AST) -> str:
"""
Get the name of the function containing this AST node.
This matches i18n.py's behavior which only uses the function name for translation lookups.
@@ -63,14 +63,14 @@ def _get_function_name(node: ast.AST) -> str:
The function name or "module" for module-level code
"""
def find_parent_context(n: ast.AST) -> tuple[str | None, str | None]:
def find_parent_context(n:ast.AST) -> tuple[str | None, str | None]:
"""Find the containing class and function names."""
class_name = None
function_name = None
current = n
while hasattr(current, '_parent'):
current = getattr(current, '_parent')
while hasattr(current, "_parent"):
current = getattr(current, "_parent")
if isinstance(current, ast.ClassDef) and not class_name:
class_name = current.name
elif isinstance(current, ast.FunctionDef) or isinstance(current, ast.AsyncFunctionDef) and not function_name:
@@ -84,7 +84,7 @@ def _get_function_name(node: ast.AST) -> str:
return "module" # For module-level code
def _extract_log_messages(file_path: str, exclude_debug:bool = False) -> MessageDict:
def _extract_log_messages(file_path:str, exclude_debug:bool = False) -> MessageDict:
"""
Extract all translatable messages from a Python file with their function context.
@@ -94,27 +94,27 @@ def _extract_log_messages(file_path: str, exclude_debug:bool = False) -> Message
Returns:
Dictionary mapping function names to their messages
"""
with open(file_path, 'r', encoding = 'utf-8') as file:
with open(file_path, "r", encoding = "utf-8") as file:
tree = ast.parse(file.read(), filename = file_path)
# Add parent references for context tracking
for parent in ast.walk(tree):
for child in ast.iter_child_nodes(parent):
setattr(child, '_parent', parent)
setattr(child, "_parent", parent)
messages: MessageDict = defaultdict(lambda: defaultdict(set))
messages:MessageDict = defaultdict(lambda: defaultdict(set))
def add_message(function: str, msg: str) -> None:
def add_message(function:str, msg:str) -> None:
"""Add a message to the messages dictionary."""
if function not in messages:
messages[function] = defaultdict(set)
if msg not in messages[function]:
messages[function][msg] = {msg}
def extract_string_value(node: ast.AST) -> str | None:
def extract_string_value(node:ast.AST) -> str | None:
"""Safely extract string value from an AST node."""
if isinstance(node, ast.Constant):
value = getattr(node, 'value', None)
value = getattr(node, "value", None)
return value if isinstance(value, str) else None
return None
@@ -127,24 +127,24 @@ def _extract_log_messages(file_path: str, exclude_debug:bool = False) -> Message
# Extract messages from various call types
if (isinstance(node.func, ast.Attribute) and
isinstance(node.func.value, ast.Name) and
node.func.value.id in {'LOG', 'logger', 'logging'} and
node.func.attr in {None if exclude_debug else 'debug', 'info', 'warning', 'error', 'exception', 'critical'}):
node.func.value.id in {"LOG", "logger", "logging"} and
node.func.attr in {None if exclude_debug else "debug", "info", "warning", "error", "exception", "critical"}):
if node.args:
msg = extract_string_value(node.args[0])
if msg:
add_message(function_name, msg)
# Handle gettext calls
elif ((isinstance(node.func, ast.Name) and node.func.id == '_') or
(isinstance(node.func, ast.Attribute) and node.func.attr == 'gettext')):
elif ((isinstance(node.func, ast.Name) and node.func.id == "_") or
(isinstance(node.func, ast.Attribute) and node.func.attr == "gettext")):
if node.args:
msg = extract_string_value(node.args[0])
if msg:
add_message(function_name, msg)
# Handle other translatable function calls
elif isinstance(node.func, ast.Name) and node.func.id in {'ainput', 'pluralize', 'ensure'}:
arg_index = 0 if node.func.id == 'ainput' else 1
elif isinstance(node.func, ast.Name) and node.func.id in {"ainput", "pluralize", "ensure"}:
arg_index = 0 if node.func.id == "ainput" else 1
if len(node.args) > arg_index:
msg = extract_string_value(node.args[arg_index])
if msg:
@@ -162,10 +162,10 @@ def _get_all_log_messages(exclude_debug:bool = False) -> dict[str, MessageDict]:
Returns:
Dictionary mapping module paths to their function messages
"""
src_dir = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), 'src', 'kleinanzeigen_bot')
src_dir = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), "src", "kleinanzeigen_bot")
print(f"\nScanning for messages in directory: {src_dir}")
messages_by_file: dict[str, MessageDict] = {
messages_by_file:dict[str, MessageDict] = {
# Special case for getopt.py which is imported
"getopt.py": {
"do_longs": {
@@ -187,15 +187,15 @@ def _get_all_log_messages(exclude_debug:bool = False) -> dict[str, MessageDict]:
for root, _, filenames in os.walk(src_dir):
for filename in filenames:
if filename.endswith('.py'):
if filename.endswith(".py"):
file_path = os.path.join(root, filename)
relative_path = os.path.relpath(file_path, src_dir)
if relative_path.startswith('resources/'):
if relative_path.startswith("resources/"):
continue
messages = _extract_log_messages(file_path, exclude_debug)
if messages:
module_path = os.path.join('kleinanzeigen_bot', relative_path)
module_path = module_path.replace(os.sep, '/')
module_path = os.path.join("kleinanzeigen_bot", relative_path)
module_path = module_path.replace(os.sep, "/")
messages_by_file[module_path] = messages
return messages_by_file
@@ -217,7 +217,7 @@ def _get_available_languages() -> list[str]:
return sorted(languages)
def _get_translations_for_language(lang: str) -> TranslationDict:
def _get_translations_for_language(lang:str) -> TranslationDict:
"""
Get translations for a specific language from its YAML file.
@@ -227,7 +227,7 @@ def _get_translations_for_language(lang: str) -> TranslationDict:
Returns:
Dictionary containing all translations for the language
"""
yaml = YAML(typ = 'safe')
yaml = YAML(typ = "safe")
translation_file = f"translations.{lang}.yaml"
print(f"Loading translations from {translation_file}")
content = files(resources).joinpath(translation_file).read_text()
@@ -235,10 +235,10 @@ def _get_translations_for_language(lang: str) -> TranslationDict:
return translations
def _find_translation(translations: TranslationDict,
module: str,
function: str,
message: str) -> bool:
def _find_translation(translations:TranslationDict,
module:str,
function:str,
message:str) -> bool:
"""
Check if a translation exists for a given message in the exact location where i18n.py will look.
This matches the lookup logic in i18n.py which uses dicts.safe_get().
@@ -253,11 +253,11 @@ def _find_translation(translations: TranslationDict,
True if translation exists in the correct location, False otherwise
"""
# Special case for getopt.py
if module == 'getopt.py':
if module == "getopt.py":
return bool(translations.get(module, {}).get(function, {}).get(message))
# Add kleinanzeigen_bot/ prefix if not present
module_path = f'kleinanzeigen_bot/{module}' if not module.startswith('kleinanzeigen_bot/') else module
module_path = f'kleinanzeigen_bot/{module}' if not module.startswith("kleinanzeigen_bot/") else module
# Check if module exists in translations
module_trans = translations.get(module_path, {})
@@ -277,10 +277,10 @@ def _find_translation(translations: TranslationDict,
return has_translation
def _message_exists_in_code(code_messages: dict[str, MessageDict],
module: str,
function: str,
message: str) -> bool:
def _message_exists_in_code(code_messages:dict[str, MessageDict],
module:str,
function:str,
message:str) -> bool:
"""
Check if a message exists in the code at the given location.
This is the reverse of _find_translation - it checks if a translation's message
@@ -296,11 +296,11 @@ def _message_exists_in_code(code_messages: dict[str, MessageDict],
True if message exists in the code, False otherwise
"""
# Special case for getopt.py
if module == 'getopt.py':
if module == "getopt.py":
return bool(code_messages.get(module, {}).get(function, {}).get(message))
# Remove kleinanzeigen_bot/ prefix if present for code message lookup
module_path = module[len('kleinanzeigen_bot/'):] if module.startswith('kleinanzeigen_bot/') else module
module_path = module[len("kleinanzeigen_bot/"):] if module.startswith("kleinanzeigen_bot/") else module
module_path = f'kleinanzeigen_bot/{module_path}'
# Check if module exists in code messages
@@ -318,7 +318,7 @@ def _message_exists_in_code(code_messages: dict[str, MessageDict],
@pytest.mark.parametrize("lang", _get_available_languages())
def test_all_log_messages_have_translations(lang: str) -> None:
def test_all_log_messages_have_translations(lang:str) -> None:
"""
Test that all translatable messages in the code have translations for each language.
@@ -345,7 +345,7 @@ def test_all_log_messages_have_translations(lang: str) -> None:
def make_inner_dict() -> defaultdict[str, set[str]]:
return defaultdict(set)
by_module: defaultdict[str, defaultdict[str, set[str]]] = defaultdict(make_inner_dict)
by_module:defaultdict[str, defaultdict[str, set[str]]] = defaultdict(make_inner_dict)
for loc in missing_translations:
assert isinstance(loc.module, str), "Module must be a string"
@@ -364,7 +364,7 @@ def test_all_log_messages_have_translations(lang: str) -> None:
@pytest.mark.parametrize("lang", _get_available_languages())
def test_no_obsolete_translations(lang: str) -> None:
def test_no_obsolete_translations(lang:str) -> None:
"""
Test that all translations in each language YAML file are actually used in the code.
@@ -376,7 +376,7 @@ def test_no_obsolete_translations(lang: str) -> None:
"""
messages_by_file = _get_all_log_messages(exclude_debug = False)
translations = _get_translations_for_language(lang)
obsolete_items: list[tuple[str, str, str]] = []
obsolete_items:list[tuple[str, str, str]] = []
for module, module_trans in translations.items():
if not isinstance(module_trans, dict):
@@ -402,7 +402,7 @@ def test_no_obsolete_translations(lang: str) -> None:
obsolete_str = f"\nObsolete translations found for language [{lang}]:\n"
# Group by module and function for better readability
by_module: defaultdict[str, defaultdict[str, list[str]]] = defaultdict(lambda: defaultdict(list))
by_module:defaultdict[str, defaultdict[str, list[str]]] = defaultdict(lambda: defaultdict(list))
for module, function, message in obsolete_items:
by_module[module][function].append(message)