From 0faa022e4d5509e26417406915cc4557bd028cdf Mon Sep 17 00:00:00 2001 From: Heavenfighter <33938595+Heavenfighter@users.noreply.github.com> Date: Wed, 14 May 2025 11:24:16 +0200 Subject: [PATCH] fix: Unable to download single ad (#509) --- src/kleinanzeigen_bot/__init__.py | 6 ++-- src/kleinanzeigen_bot/extract.py | 14 +++------ .../resources/translations.de.yaml | 2 +- tests/unit/test_extract.py | 29 +++++-------------- 4 files changed, 15 insertions(+), 36 deletions(-) diff --git a/src/kleinanzeigen_bot/__init__.py b/src/kleinanzeigen_bot/__init__.py index b2a7ed1..08192a1 100644 --- a/src/kleinanzeigen_bot/__init__.py +++ b/src/kleinanzeigen_bot/__init__.py @@ -1061,7 +1061,7 @@ class KleinanzeigenBot(WebScrapingMixin): # call download function for each ad page for add_url in own_ad_urls: ad_id = ad_extractor.extract_ad_id_from_ad_url(add_url) - if await ad_extractor.naviagte_to_ad_page(add_url): + if await ad_extractor.navigate_to_ad_page(add_url): await ad_extractor.download_ad(ad_id) success_count += 1 LOG.info("%d of %d ads were downloaded from your profile.", success_count, len(own_ad_urls)) @@ -1085,7 +1085,7 @@ class KleinanzeigenBot(WebScrapingMixin): LOG.info("The ad with id %d has already been saved.", ad_id) continue - if await ad_extractor.naviagte_to_ad_page(ad_url): + if await ad_extractor.navigate_to_ad_page(ad_url): await ad_extractor.download_ad(ad_id) new_count += 1 LOG.info("%s were downloaded from your profile.", pluralize("new ad", new_count)) @@ -1096,7 +1096,7 @@ class KleinanzeigenBot(WebScrapingMixin): LOG.info(" | ".join([str(ad_id) for ad_id in ids])) for ad_id in ids: # call download routine for every id - exists = await ad_extractor.naviagte_to_ad_page(ad_id) + exists = await ad_extractor.navigate_to_ad_page(ad_id) if exists: await ad_extractor.download_ad(ad_id) LOG.info("Downloaded ad with id %d", ad_id) diff --git a/src/kleinanzeigen_bot/extract.py b/src/kleinanzeigen_bot/extract.py index 52b7bf9..9bfa268 100644 --- a/src/kleinanzeigen_bot/extract.py +++ b/src/kleinanzeigen_bot/extract.py @@ -8,7 +8,7 @@ from typing import Any, Final from .ads import calculate_content_hash, get_description_affixes from .utils import dicts, i18n, loggers, misc, reflect -from .utils.web_scraping_mixin import Browser, By, Element, Is, WebScrapingMixin +from .utils.web_scraping_mixin import Browser, By, Element, WebScrapingMixin __all__ = [ "AdExtractor", @@ -239,20 +239,14 @@ class AdExtractor(WebScrapingMixin): return refs - async def naviagte_to_ad_page(self, id_or_url:int | str) -> bool: + async def navigate_to_ad_page(self, id_or_url: int | str) -> bool: """ Navigates to an ad page specified with an ad ID; or alternatively by a given URL. :return: whether the navigation to the ad page was successful """ if reflect.is_integer(id_or_url): - # navigate to start page, otherwise page can be None! - await self.web_open("https://www.kleinanzeigen.de/") - # enter the ad ID into the search bar - await self.web_input(By.ID, "site-search-query", id_or_url) - # navigate to ad page and wait - await self.web_check(By.ID, "site-search-submit", Is.CLICKABLE) - submit_button = await self.web_find(By.ID, "site-search-submit") - await submit_button.click() + # navigate to search page + await self.web_open("https://www.kleinanzeigen.de/s-suchanfrage.html?keywords={0}".format(id_or_url)) else: await self.web_open(str(id_or_url)) # navigate to URL directly given await self.web_sleep() diff --git a/src/kleinanzeigen_bot/resources/translations.de.yaml b/src/kleinanzeigen_bot/resources/translations.de.yaml index ac37b26..645c448 100644 --- a/src/kleinanzeigen_bot/resources/translations.de.yaml +++ b/src/kleinanzeigen_bot/resources/translations.de.yaml @@ -184,7 +184,7 @@ kleinanzeigen_bot/extract.py: "Found %s ad items on page %s.": "%s Anzeigen-Elemente auf Seite %s gefunden." "Successfully extracted %s refs from page %s.": "%s Referenzen von Seite %s erfolgreich extrahiert." - naviagte_to_ad_page: + navigate_to_ad_page: "There is no ad under the given ID.": "Es gibt keine Anzeige unter der angegebenen ID." "A popup appeared!": "Ein Popup ist erschienen!" diff --git a/tests/unit/test_extract.py b/tests/unit/test_extract.py index 272dc72..c9fbc0b 100644 --- a/tests/unit/test_extract.py +++ b/tests/unit/test_extract.py @@ -266,48 +266,33 @@ class TestAdExtractorNavigation: patch.object(test_extractor, "web_open", new_callable = AsyncMock) as mock_web_open, \ patch.object(test_extractor, "web_find", new_callable = AsyncMock, side_effect = TimeoutError): - result = await test_extractor.naviagte_to_ad_page("https://www.kleinanzeigen.de/s-anzeige/test/12345") + result = await test_extractor.navigate_to_ad_page("https://www.kleinanzeigen.de/s-anzeige/test/12345") assert result is True mock_web_open.assert_called_with("https://www.kleinanzeigen.de/s-anzeige/test/12345") @pytest.mark.asyncio async def test_navigate_to_ad_page_with_id(self, test_extractor:AdExtractor) -> None: """Test navigation to ad page using an ID.""" + ad_id = 12345 page_mock = AsyncMock() - page_mock.url = "https://www.kleinanzeigen.de/s-anzeige/test/12345" - - submit_button_mock = AsyncMock() - submit_button_mock.click = AsyncMock() - submit_button_mock.apply = AsyncMock(return_value = True) - - input_mock = AsyncMock() - input_mock.clear_input = AsyncMock() - input_mock.send_keys = AsyncMock() - input_mock.apply = AsyncMock(return_value = True) + page_mock.url = "https://www.kleinanzeigen.de/s-anzeige/test/{0}".format(ad_id) popup_close_mock = AsyncMock() popup_close_mock.click = AsyncMock() popup_close_mock.apply = AsyncMock(return_value = True) def find_mock(selector_type:By, selector_value:str, **_:Any) -> Element | None: - if selector_type == By.ID and selector_value == "site-search-query": - return input_mock - if selector_type == By.ID and selector_value == "site-search-submit": - return submit_button_mock if selector_type == By.CLASS_NAME and selector_value == "mfp-close": return popup_close_mock return None with patch.object(test_extractor, "page", page_mock), \ patch.object(test_extractor, "web_open", new_callable = AsyncMock) as mock_web_open, \ - patch.object(test_extractor, "web_input", new_callable = AsyncMock), \ - patch.object(test_extractor, "web_check", new_callable = AsyncMock, return_value = True), \ patch.object(test_extractor, "web_find", new_callable = AsyncMock, side_effect = find_mock): - result = await test_extractor.naviagte_to_ad_page(12345) + result = await test_extractor.navigate_to_ad_page(ad_id) assert result is True - mock_web_open.assert_called_with("https://www.kleinanzeigen.de/") - submit_button_mock.click.assert_awaited_once() + mock_web_open.assert_called_with("https://www.kleinanzeigen.de/s-suchanfrage.html?keywords={0}".format(ad_id)) popup_close_mock.click.assert_awaited_once() @pytest.mark.asyncio @@ -327,7 +312,7 @@ class TestAdExtractorNavigation: patch.object(test_extractor, "web_click", new_callable = AsyncMock) as mock_web_click, \ patch.object(test_extractor, "web_check", new_callable = AsyncMock, return_value = True): - result = await test_extractor.naviagte_to_ad_page(12345) + result = await test_extractor.navigate_to_ad_page(12345) assert result is True mock_web_click.assert_called_with(By.CLASS_NAME, "mfp-close") @@ -347,7 +332,7 @@ class TestAdExtractorNavigation: patch.object(test_extractor, "web_open", new_callable = AsyncMock), \ patch.object(test_extractor, "web_find", new_callable = AsyncMock, return_value = input_mock): - result = await test_extractor.naviagte_to_ad_page(99999) + result = await test_extractor.navigate_to_ad_page(99999) assert result is False @pytest.mark.asyncio