diff --git a/README.md b/README.md index d10a496..0be93ee 100644 --- a/README.md +++ b/README.md @@ -186,6 +186,7 @@ Usage: kleinanzeigen-bot COMMAND [OPTIONS] Commands: publish - (re-)publishes ads verify - verifies the configuration files + update - updates published ads delete - deletes ads download - downloads one or multiple ads -- @@ -206,6 +207,10 @@ Options: * all: downloads all ads from your profile * new: downloads ads from your profile that are not locally saved yet * : provide one or several ads by ID to download, like e.g. "--ads=1,2,3" + --ads=changed| (update) - specifies which ads to update (DEFAULT: changed) + Possible values: + * changed: only update ads that have been modified since last publication + * : provide one or several ads by ID to update, like e.g. "--ads=1,2,3" --force - alias for '--ads=all' --keep-old - don't delete old ads on republication --config= - path to the config YAML or JSON file (DEFAULT: ./config.yaml) diff --git a/pyproject.toml b/pyproject.toml index 5bd05ff..4a4abe2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -268,7 +268,7 @@ min-file-size = 256 # https://pylint.pycqa.org/en/latest/user_guide/checkers/features.html#design-checker-messages max-args = 6 # max. number of args for function / method (R0913) # max-attributes = 15 # TODO max. number of instance attrs for a class (R0902) -max-branches = 40 # max. number of branch for function / method body (R0912) +max-branches = 45 # max. number of branch for function / method body (R0912) max-locals = 30 # max. number of local vars for function / method body (R0914) max-returns = 15 # max. number of return / yield for function / method body (R0911) max-statements = 150 # max. number of statements in function / method body (R0915) diff --git a/src/kleinanzeigen_bot/__init__.py b/src/kleinanzeigen_bot/__init__.py index de32e90..b54e63f 100644 --- a/src/kleinanzeigen_bot/__init__.py +++ b/src/kleinanzeigen_bot/__init__.py @@ -1,7 +1,7 @@ # SPDX-FileCopyrightText: © Sebastian Thomschke and contributors # SPDX-License-Identifier: AGPL-3.0-or-later # SPDX-ArtifactOfProjectHomePage: https://github.com/Second-Hand-Friends/kleinanzeigen-bot/ -import atexit, json, os, re, signal, sys, textwrap # isort: skip +import atexit, enum, json, os, re, signal, sys, textwrap # isort: skip import getopt # pylint: disable=deprecated-module import urllib.parse as urllib_parse from gettext import gettext as _ @@ -30,6 +30,11 @@ LOG.setLevel(loggers.INFO) colorama.just_fix_windows_console() +class AdUpdateStrategy(enum.Enum): + REPLACE = enum.auto() + MODIFY = enum.auto() + + class KleinanzeigenBot(WebScrapingMixin): def __init__(self) -> None: @@ -107,6 +112,25 @@ class KleinanzeigenBot(WebScrapingMixin): LOG.info("############################################") LOG.info("DONE: No new/outdated ads found.") LOG.info("############################################") + case "update": + self.configure_file_logging() + self.load_config() + + if not (self.ads_selector in {"all", "changed"} or + any(selector in self.ads_selector.split(",") for selector in + ("all", "changed")) or + re.compile(r"\d+[,\d+]*").search(self.ads_selector)): + LOG.warning('You provided no ads selector. Defaulting to "changed".') + self.ads_selector = "changed" + + if ads := self.load_ads(): + await self.create_browser_session() + await self.login() + await self.update_ads(ads) + else: + LOG.info("############################################") + LOG.info("DONE: No changed ads found.") + LOG.info("############################################") case "delete": self.configure_file_logging() self.load_config() @@ -151,6 +175,7 @@ class KleinanzeigenBot(WebScrapingMixin): publish - (Wieder-)Veröffentlicht Anzeigen verify - Überprüft die Konfigurationsdateien delete - Löscht Anzeigen + update - Aktualisiert bestehende Anzeigen download - Lädt eine oder mehrere Anzeigen herunter update-content-hash - Berechnet den content_hash aller Anzeigen anhand der aktuellen ad_defaults neu; nach Änderungen an den config.yaml/ad_defaults verhindert es, dass alle Anzeigen als @@ -174,6 +199,10 @@ class KleinanzeigenBot(WebScrapingMixin): * all: Lädt alle Anzeigen aus Ihrem Profil herunter * new: Lädt Anzeigen aus Ihrem Profil herunter, die lokal noch nicht gespeichert sind * : Gibt eine oder mehrere Anzeigen-IDs zum Herunterladen an, z. B. "--ads=1,2,3" + --ads=changed| (update) - Gibt an, welche Anzeigen aktualisiert werden sollen (STANDARD: changed) + Mögliche Werte: + * changed: Aktualisiert nur Anzeigen, die seit der letzten Veröffentlichung geändert wurden + * : Gibt eine oder mehrere Anzeigen-IDs zum Aktualisieren an, z. B. "--ads=1,2,3" --force - Alias für '--ads=all' --keep-old - Verhindert das Löschen alter Anzeigen bei erneuter Veröffentlichung --config= - Pfad zur YAML- oder JSON-Konfigurationsdatei (STANDARD: ./config.yaml) @@ -189,6 +218,7 @@ class KleinanzeigenBot(WebScrapingMixin): publish - (re-)publishes ads verify - verifies the configuration files delete - deletes ads + update - updates published ads download - downloads one or multiple ads update-content-hash – recalculates each ad’s content_hash based on the current ad_defaults; use this after changing config.yaml/ad_defaults to avoid every ad being marked "changed" and republished @@ -210,6 +240,10 @@ class KleinanzeigenBot(WebScrapingMixin): * all: downloads all ads from your profile * new: downloads ads from your profile that are not locally saved yet * : provide one or several ads by ID to download, like e.g. "--ads=1,2,3" + --ads=changed| (update) - specifies which ads to update (DEFAULT: changed) + Possible values: + * changed: only update ads that have been modified since last publication + * : provide one or several ads by ID to update, like e.g. "--ads=1,2,3" --force - alias for '--ads=all' --keep-old - don't delete old ads on republication --config= - path to the config YAML or JSON file (DEFAULT: ./config.yaml) @@ -340,7 +374,7 @@ class KleinanzeigenBot(WebScrapingMixin): def load_ads(self, *, ignore_inactive:bool = True, exclude_ads_with_id:bool = True) -> list[tuple[str, Ad, dict[str, Any]]]: """ - Load and validate all ad config files, optionally filtering out inactive or already‐published ads. + Load and validate all ad config files, optionally filtering out inactive or already-published ads. Args: ignore_inactive (bool): @@ -640,7 +674,7 @@ class KleinanzeigenBot(WebScrapingMixin): count += 1 - await self.publish_ad(ad_file, ad_cfg, ad_cfg_orig, published_ads) + await self.publish_ad(ad_file, ad_cfg, ad_cfg_orig, published_ads, AdUpdateStrategy.REPLACE) await self.web_await(lambda: self.web_check(By.ID, "checking-done", Is.DISPLAYED), timeout = 5 * 60) if self.config.publishing.delete_old_ads == "AFTER_PUBLISH" and not self.keep_old_ads: @@ -650,24 +684,29 @@ class KleinanzeigenBot(WebScrapingMixin): LOG.info("DONE: (Re-)published %s", pluralize("ad", count)) LOG.info("############################################") - async def publish_ad(self, ad_file:str, ad_cfg:Ad, ad_cfg_orig:dict[str, Any], published_ads:list[dict[str, Any]]) -> None: + async def publish_ad(self, ad_file:str, ad_cfg:Ad, ad_cfg_orig:dict[str, Any], published_ads:list[dict[str, Any]], + mode:AdUpdateStrategy = AdUpdateStrategy.REPLACE) -> None: """ @param ad_cfg: the effective ad config (i.e. with default values applied etc.) @param ad_cfg_orig: the ad config as present in the YAML file @param published_ads: json list of published ads + @param mode: the mode of ad editing, either publishing a new or updating an existing ad """ - if self.config.publishing.delete_old_ads == "BEFORE_PUBLISH" and not self.keep_old_ads: - await self.delete_ad(ad_cfg, published_ads, delete_old_ads_by_title = self.config.publishing.delete_old_ads_by_title) + if mode == AdUpdateStrategy.REPLACE: + if self.config.publishing.delete_old_ads == "BEFORE_PUBLISH" and not self.keep_old_ads: + await self.delete_ad(ad_cfg, published_ads, delete_old_ads_by_title = self.config.publishing.delete_old_ads_by_title) - LOG.info("Publishing ad '%s'...", ad_cfg.title) + LOG.info("Publishing ad '%s'...", ad_cfg.title) + await self.web_open(f"{self.root_url}/p-anzeige-aufgeben-schritt2.html") + else: + LOG.info("Updating ad '%s'...", ad_cfg.title) + await self.web_open(f"{self.root_url}/p-anzeige-bearbeiten.html?adId={ad_cfg.id}") if loggers.is_debug(LOG): LOG.debug(" -> effective ad meta:") YAML().dump(ad_cfg.model_dump(), sys.stdout) - await self.web_open(f"{self.root_url}/p-anzeige-aufgeben-schritt2.html") - if ad_cfg.type == "WANTED": await self.web_click(By.ID, "adType2") @@ -698,7 +737,7 @@ class KleinanzeigenBot(WebScrapingMixin): except TimeoutError: LOG.warning("Failed to set shipping attribute for type '%s'!", ad_cfg.shipping_type) else: - await self.__set_shipping(ad_cfg) + await self.__set_shipping(ad_cfg, mode) ############################# # set price @@ -710,6 +749,10 @@ class KleinanzeigenBot(WebScrapingMixin): except TimeoutError: pass if ad_cfg.price: + if mode == AdUpdateStrategy.MODIFY: + # we have to clear the input, otherwise input gets appended + await self.web_input(By.CSS_SELECTOR, + "input#post-ad-frontend-price, input#micro-frontend-price, input#pstad-price", "") await self.web_input(By.CSS_SELECTOR, "input#post-ad-frontend-price, input#micro-frontend-price, input#pstad-price", str(ad_cfg.price)) ############################# @@ -742,6 +785,7 @@ class KleinanzeigenBot(WebScrapingMixin): try: await self.web_sleep(1) # Wait for city dropdown to populate options = await self.web_find_all(By.CSS_SELECTOR, "#pstad-citychsr option") + for option in options: if option.text == ad_cfg.contact.location: await self.web_select(By.ID, "pstad-citychsr", option.attrs.value) @@ -782,6 +826,16 @@ class KleinanzeigenBot(WebScrapingMixin): pass await self.web_input(By.ID, "postad-phonenumber", ad_cfg.contact.phone) + if mode == AdUpdateStrategy.MODIFY: + ############################# + # delete previous images because we don't know which have changed + ############################# + img_items = await self.web_find_all(By.CSS_SELECTOR, + "ul#j-pictureupload-thumbnails > li.ui-sortable-handle") + for element in img_items: + btn = await self.web_find(By.CSS_SELECTOR, "button.pictureupload-thumbnails-remove", parent=element) + await btn.click() + ############################# # upload images ############################# @@ -846,10 +900,50 @@ class KleinanzeigenBot(WebScrapingMixin): if not ad_cfg.created_on and not ad_cfg.id: ad_cfg_orig["created_on"] = ad_cfg_orig["updated_on"] - LOG.info(" -> SUCCESS: ad published with ID %s", ad_id) + if mode == AdUpdateStrategy.REPLACE: + LOG.info(" -> SUCCESS: ad published with ID %s", ad_id) + else: + LOG.info(" -> SUCCESS: ad updated with ID %s", ad_id) dicts.save_dict(ad_file, ad_cfg_orig) + async def update_ads(self, ad_cfgs:list[tuple[str, Ad, dict[str, Any]]]) -> None: + """ + Updates a list of ads. + The list gets filtered, so that only already published ads will be updated. + Calls publish_ad in MODIFY mode. + + Args: + ad_cfgs: List of ad configurations + + Returns: + None + """ + count = 0 + + published_ads = json.loads( + (await self.web_request(f"{self.root_url}/m-meine-anzeigen-verwalten.json?sort=DEFAULT"))["content"])["ads"] + + for (ad_file, ad_cfg, ad_cfg_orig) in ad_cfgs: + ad = next((ad for ad in published_ads if ad["id"] == ad_cfg.id), None) + + if not ad: + continue + + LOG.info("Processing %s/%s: '%s' from [%s]...", count + 1, len(ad_cfgs), ad_cfg.title, ad_file) + if ad["state"] == "paused": + LOG.info("Skipping because ad is reserved") + continue + + count += 1 + + await self.publish_ad(ad_file, ad_cfg, ad_cfg_orig, published_ads, AdUpdateStrategy.MODIFY) + await self.web_await(lambda: self.web_check(By.ID, "checking-done", Is.DISPLAYED), timeout = 5 * 60) + + LOG.info("############################################") + LOG.info("DONE: updated %s", pluralize("ad", count)) + LOG.info("############################################") + async def __set_condition(self, condition_value:str) -> None: condition_mapping = { "new_with_tag": "Neu mit Etikett", @@ -863,7 +957,7 @@ class KleinanzeigenBot(WebScrapingMixin): try: # Open condition dialog - await self.web_click(By.XPATH, '//*[contains(@id, "j-post-listing-frontend-conditions")]//button[contains(., "Bitte wählen")]') + await self.web_click(By.XPATH, '//*[@id="j-post-listing-frontend-conditions"]//button[contains(@class, "SelectionButton")]') except TimeoutError: LOG.debug("Unable to open condition dialog and select condition [%s]", condition_value, exc_info = True) return @@ -945,7 +1039,7 @@ class KleinanzeigenBot(WebScrapingMixin): raise TimeoutError(f"Failed to set special attribute [{special_attribute_key}]") from ex LOG.debug("Successfully set attribute field [%s] to [%s]...", special_attribute_key, special_attribute_value) - async def __set_shipping(self, ad_cfg:Ad) -> None: + async def __set_shipping(self, ad_cfg:Ad, mode:AdUpdateStrategy = AdUpdateStrategy.REPLACE) -> None: if ad_cfg.shipping_type == "PICKUP": try: await self.web_click(By.XPATH, @@ -954,6 +1048,14 @@ class KleinanzeigenBot(WebScrapingMixin): LOG.debug(ex, exc_info = True) elif ad_cfg.shipping_options: await self.web_click(By.XPATH, '//*[contains(@class, "SubSection")]//button[contains(@class, "SelectionButton")]') + + if mode == AdUpdateStrategy.MODIFY: + try: + # when "Andere Versandmethoden" is not available, go back and start over new + await self.web_find(By.XPATH, '//*[contains(@class, "CarrierSelectionModal")]//button[contains(., "Andere Versandmethoden")]', timeout=2) + except TimeoutError: + await self.web_click(By.XPATH, '//dialog//button[contains(., "Zurück")]') + await self.web_click(By.XPATH, '//*[contains(@class, "CarrierSelectionModal")]//button[contains(., "Andere Versandmethoden")]') await self.__set_shipping_options(ad_cfg) else: @@ -967,8 +1069,22 @@ class KleinanzeigenBot(WebScrapingMixin): # no options. only costs. Set custom shipping cost if ad_cfg.shipping_costs is not None: await self.web_click(By.XPATH, '//*[contains(@class, "SubSection")]//button[contains(@class, "SelectionButton")]') - await self.web_click(By.XPATH, '//*[contains(@class, "CarrierSelectionModal")]//button[contains(., "Andere Versandmethoden")]') - await self.web_click(By.XPATH, '//*[contains(@id, "INDIVIDUAL") and contains(@data-testid, "Individueller Versand")]') + + try: + # when "Andere Versandmethoden" is not available, then we are already on the individual page + await self.web_click(By.XPATH, + '//*[contains(@class, "CarrierSelectionModal")]//button[contains(., "Andere Versandmethoden")]') + except TimeoutError: + pass + + try: + # only click on "Individueller Versand" when "IndividualShippingInput" is not available, otherwise its already checked + # (important for mode = UPDATE) + await self.web_find(By.XPATH, + '//*[contains(@class, "IndividualPriceSection")]//div[contains(@class, "IndividualShippingInput")]', timeout=2) + except TimeoutError: + await self.web_click(By.XPATH, '//*[contains(@id, "INDIVIDUAL") and contains(@data-testid, "Individueller Versand")]') + await self.web_input(By.CSS_SELECTOR, '.IndividualShippingInput input[type="text"]', str.replace(str(ad_cfg.shipping_costs), ".", ",")) await self.web_click(By.XPATH, '//dialog//button[contains(., "Fertig")]') except TimeoutError as ex: diff --git a/src/kleinanzeigen_bot/resources/translations.de.yaml b/src/kleinanzeigen_bot/resources/translations.de.yaml index 8aa444f..20db202 100644 --- a/src/kleinanzeigen_bot/resources/translations.de.yaml +++ b/src/kleinanzeigen_bot/resources/translations.de.yaml @@ -86,13 +86,21 @@ kleinanzeigen_bot/__init__.py: publish_ad: "Publishing ad '%s'...": "Veröffentliche Anzeige '%s'..." + "Updating ad '%s'...": "Aktualisiere Anzeige '%s'..." "Failed to set shipping attribute for type '%s'!": "Fehler beim setzen des Versandattributs für den Typ '%s'!" "# Payment form detected! Please proceed with payment.": "# Bestellformular gefunden! Bitte mit der Bezahlung fortfahren." " -> SUCCESS: ad published with ID %s": " -> ERFOLG: Anzeige mit ID %s veröffentlicht" + " -> SUCCESS: ad updated with ID %s": " -> ERFOLG: Anzeige mit ID %s aktualisiert" " -> effective ad meta:": " -> effektive Anzeigen-Metadaten:" "Could not set city from location": "Stadt konnte nicht aus dem Standort gesetzt werden" "Press a key to continue...": "Eine Taste drücken, um fortzufahren..." + update_ads: + "Processing %s/%s: '%s' from [%s]...": "Verarbeite %s/%s: '%s' von [%s]..." + "Skipping because ad is reserved": "Überspringen, da Anzeige reserviert ist" + "DONE: updated %s": "FERTIG: %s aktualisiert" + "ad": "Anzeige" + __set_condition: "Unable to close condition dialog!": "Kann den Dialog für Artikelzustand nicht schließen!" "Unable to open condition dialog and select condition [%s]": "Zustandsdialog konnte nicht geöffnet und Zustand [%s] nicht ausgewählt werden" @@ -139,8 +147,10 @@ kleinanzeigen_bot/__init__.py: "DONE: No active ads found.": "FERTIG: Keine aktiven Anzeigen gefunden." "You provided no ads selector. Defaulting to \"due\".": "Es wurden keine Anzeigen-Selektor angegeben. Es wird \"due\" verwendet." "DONE: No new/outdated ads found.": "FERTIG: Keine neuen/veralteten Anzeigen gefunden." - "DONE: No ads to delete found.": "FERTIG: Keine zu löschnenden Anzeigen gefunden." + "DONE: No ads to delete found.": "FERTIG: Keine zu löschenden Anzeigen gefunden." + "DONE: No changed ads found.": "FERTIG: Keine geänderten Anzeigen gefunden." "You provided no ads selector. Defaulting to \"new\".": "Es wurden keine Anzeigen-Selektor angegeben. Es wird \"new\" verwendet." + "You provided no ads selector. Defaulting to \"changed\".": "Es wurden keine Anzeigen-Selektor angegeben. Es wird \"changed\" verwendet." "Unknown command: %s": "Unbekannter Befehl: %s" fill_login_data_and_send: