feat: add browser profile XDG support and documentation (#777)

This commit is contained in:
Jens
2026-01-23 22:45:22 +01:00
committed by GitHub
parent dc0d9404bf
commit eda1b4d0ec
15 changed files with 841 additions and 687 deletions

View File

@@ -28,7 +28,7 @@ from .utils.web_scraping_mixin import By, Element, Is, WebScrapingMixin
# W0406: possibly a bug, see https://github.com/PyCQA/pylint/issues/3933
LOG: Final[loggers.Logger] = loggers.get_logger(__name__)
LOG:Final[loggers.Logger] = loggers.get_logger(__name__)
LOG.setLevel(loggers.INFO)
colorama.just_fix_windows_console()
@@ -39,7 +39,7 @@ class AdUpdateStrategy(enum.Enum):
MODIFY = enum.auto()
def _repost_cycle_ready(ad_cfg: Ad, ad_file_relative: str) -> bool:
def _repost_cycle_ready(ad_cfg:Ad, ad_file_relative:str) -> bool:
"""
Check if the repost cycle delay has been satisfied.
@@ -72,7 +72,7 @@ def _repost_cycle_ready(ad_cfg: Ad, ad_file_relative: str) -> bool:
return True
def _day_delay_elapsed(ad_cfg: Ad, ad_file_relative: str) -> bool:
def _day_delay_elapsed(ad_cfg:Ad, ad_file_relative:str) -> bool:
"""
Check if the day delay has elapsed since the ad was last published.
@@ -100,7 +100,7 @@ def _day_delay_elapsed(ad_cfg: Ad, ad_file_relative: str) -> bool:
return True
def apply_auto_price_reduction(ad_cfg: Ad, _ad_cfg_orig: dict[str, Any], ad_file_relative: str) -> None:
def apply_auto_price_reduction(ad_cfg:Ad, _ad_cfg_orig:dict[str, Any], ad_file_relative:str) -> None:
"""
Apply automatic price reduction to an ad based on repost count and configuration.
@@ -132,7 +132,7 @@ def apply_auto_price_reduction(ad_cfg: Ad, _ad_cfg_orig: dict[str, Any], ad_file
applied_cycles = ad_cfg.price_reduction_count or 0
next_cycle = applied_cycles + 1
effective_price = calculate_auto_price(base_price=base_price, auto_price_reduction=ad_cfg.auto_price_reduction, target_reduction_cycle=next_cycle)
effective_price = calculate_auto_price(base_price = base_price, auto_price_reduction = ad_cfg.auto_price_reduction, target_reduction_cycle = next_cycle)
if effective_price is None:
return
@@ -149,7 +149,7 @@ def apply_auto_price_reduction(ad_cfg: Ad, _ad_cfg_orig: dict[str, Any], ad_file
# Note: price_reduction_count is persisted to ad_cfg_orig only after successful publish
class KleinanzeigenBot(WebScrapingMixin):
class KleinanzeigenBot(WebScrapingMixin): # noqa: PLR0904
def __init__(self) -> None:
# workaround for https://github.com/Second-Hand-Friends/kleinanzeigen-bot/issues/295
# see https://github.com/pyinstaller/pyinstaller/issues/7229#issuecomment-1309383026
@@ -159,17 +159,17 @@ class KleinanzeigenBot(WebScrapingMixin):
self.root_url = "https://www.kleinanzeigen.de"
self.config: Config
self.config:Config
self.config_file_path = abspath("config.yaml")
self.config_explicitly_provided = False
self.installation_mode: xdg_paths.InstallationMode | None = None
self.installation_mode:xdg_paths.InstallationMode | None = None
self.categories: dict[str, str] = {}
self.categories:dict[str, str] = {}
self.file_log: loggers.LogFileHandle | None = None
self.file_log:loggers.LogFileHandle | None = None
log_file_basename = is_frozen() and os.path.splitext(os.path.basename(sys.executable))[0] or self.__module__
self.log_file_path: str | None = abspath(f"{log_file_basename}.log")
self.log_file_path:str | None = abspath(f"{log_file_basename}.log")
self.log_file_basename = log_file_basename
self.log_file_explicitly_provided = False
@@ -245,7 +245,7 @@ class KleinanzeigenBot(WebScrapingMixin):
LOG.info(_("Installation mode: %s"), mode_display)
LOG.info(_("Config file: %s"), self.config_file_path)
async def run(self, args: list[str]) -> None:
async def run(self, args:list[str]) -> None:
self.parse_args(args)
self.finalize_installation_mode()
try:
@@ -277,7 +277,7 @@ class KleinanzeigenBot(WebScrapingMixin):
self.configure_file_logging()
self.load_config()
checker = UpdateChecker(self.config, self.installation_mode_or_portable)
checker.check_for_updates(skip_interval_check=True)
checker.check_for_updates(skip_interval_check = True)
case "update-content-hash":
self.configure_file_logging()
self.load_config()
@@ -285,7 +285,7 @@ class KleinanzeigenBot(WebScrapingMixin):
checker = UpdateChecker(self.config, self.installation_mode_or_portable)
checker.check_for_updates()
self.ads_selector = "all"
if ads := self.load_ads(exclude_ads_with_id=False):
if ads := self.load_ads(exclude_ads_with_id = False):
self.update_content_hashes(ads)
else:
LOG.info("############################################")
@@ -503,7 +503,7 @@ class KleinanzeigenBot(WebScrapingMixin):
)
)
def parse_args(self, args: list[str]) -> None:
def parse_args(self, args:list[str]) -> None:
try:
options, arguments = getopt.gnu_getopt(args[1:], "hv", ["ads=", "config=", "force", "help", "keep-old", "logfile=", "lang=", "verbose"])
except getopt.error as ex:
@@ -571,7 +571,7 @@ class KleinanzeigenBot(WebScrapingMixin):
default_config.login.password = "changeme" # noqa: S105 placeholder for default config, not a real password
dicts.save_dict(
self.config_file_path,
default_config.model_dump(exclude_none=True, exclude={"ad_defaults": {"description"}}),
default_config.model_dump(exclude_none = True, exclude = {"ad_defaults": {"description"}}),
header=(
"# yaml-language-server: $schema="
"https://raw.githubusercontent.com/Second-Hand-Friends/kleinanzeigen-bot"
@@ -585,7 +585,7 @@ class KleinanzeigenBot(WebScrapingMixin):
self.create_default_config()
config_yaml = dicts.load_dict_if_exists(self.config_file_path, _("config"))
self.config = Config.model_validate(config_yaml, strict=True, context=self.config_file_path)
self.config = Config.model_validate(config_yaml, strict = True, context = self.config_file_path)
# load built-in category mappings
self.categories = dicts.load_dict_from_module(resources, "categories.yaml", "categories")
@@ -598,13 +598,13 @@ class KleinanzeigenBot(WebScrapingMixin):
# populate browser_config object used by WebScrapingMixin
self.browser_config.arguments = self.config.browser.arguments
self.browser_config.binary_location = self.config.browser.binary_location
self.browser_config.extensions = [abspath(item, relative_to=self.config_file_path) for item in self.config.browser.extensions]
self.browser_config.extensions = [abspath(item, relative_to = self.config_file_path) for item in self.config.browser.extensions]
self.browser_config.use_private_window = self.config.browser.use_private_window
if self.config.browser.user_data_dir:
self.browser_config.user_data_dir = abspath(self.config.browser.user_data_dir, relative_to=self.config_file_path)
self.browser_config.user_data_dir = abspath(self.config.browser.user_data_dir, relative_to = self.config_file_path)
self.browser_config.profile_name = self.config.browser.profile_name
def __check_ad_republication(self, ad_cfg: Ad, ad_file_relative: str) -> bool:
def __check_ad_republication(self, ad_cfg:Ad, ad_file_relative:str) -> bool:
"""
Check if an ad needs to be republished based on republication interval.
Note: This method does not check for content changes. Use __check_ad_changed for that.
@@ -635,7 +635,7 @@ class KleinanzeigenBot(WebScrapingMixin):
return True
def __check_ad_changed(self, ad_cfg: Ad, ad_cfg_orig: dict[str, Any], ad_file_relative: str) -> bool:
def __check_ad_changed(self, ad_cfg:Ad, ad_cfg_orig:dict[str, Any], ad_file_relative:str) -> bool:
"""
Check if an ad has been changed since last publication.
@@ -662,7 +662,7 @@ class KleinanzeigenBot(WebScrapingMixin):
return False
def load_ads(self, *, ignore_inactive: bool = True, exclude_ads_with_id: bool = True) -> list[tuple[str, Ad, dict[str, Any]]]:
def load_ads(self, *, ignore_inactive:bool = True, exclude_ads_with_id:bool = True) -> list[tuple[str, Ad, dict[str, Any]]]:
"""
Load and validate all ad config files, optionally filtering out inactive or already-published ads.
@@ -678,12 +678,12 @@ class KleinanzeigenBot(WebScrapingMixin):
"""
LOG.info("Searching for ad config files...")
ad_files: dict[str, str] = {}
ad_files:dict[str, str] = {}
data_root_dir = os.path.dirname(self.config_file_path)
for file_pattern in self.config.ad_files:
for ad_file in glob.glob(file_pattern, root_dir=data_root_dir, flags=glob.GLOBSTAR | glob.BRACE | glob.EXTGLOB):
for ad_file in glob.glob(file_pattern, root_dir = data_root_dir, flags = glob.GLOBSTAR | glob.BRACE | glob.EXTGLOB):
if not str(ad_file).endswith("ad_fields.yaml"):
ad_files[abspath(ad_file, relative_to=data_root_dir)] = ad_file
ad_files[abspath(ad_file, relative_to = data_root_dir)] = ad_file
LOG.info(" -> found %s", pluralize("ad config file", ad_files))
if not ad_files:
return []
@@ -700,8 +700,8 @@ class KleinanzeigenBot(WebScrapingMixin):
ads = []
for ad_file, ad_file_relative in sorted(ad_files.items()):
ad_cfg_orig: dict[str, Any] = dicts.load_dict(ad_file, "ad")
ad_cfg: Ad = self.load_ad(ad_cfg_orig)
ad_cfg_orig:dict[str, Any] = dicts.load_dict(ad_file, "ad")
ad_cfg:Ad = self.load_ad(ad_cfg_orig)
if ignore_inactive and not ad_cfg.active:
LOG.info(" -> SKIPPED: inactive ad [%s]", ad_file_relative)
@@ -738,8 +738,8 @@ class KleinanzeigenBot(WebScrapingMixin):
if not should_include:
continue
ensure(self.__get_description(ad_cfg, with_affixes=False), f"-> property [description] not specified @ [{ad_file}]")
self.__get_description(ad_cfg, with_affixes=True) # validates complete description
ensure(self.__get_description(ad_cfg, with_affixes = False), f"-> property [description] not specified @ [{ad_file}]")
self.__get_description(ad_cfg, with_affixes = True) # validates complete description
if ad_cfg.category:
resolved_category_id = self.categories.get(ad_cfg.category)
@@ -758,13 +758,13 @@ class KleinanzeigenBot(WebScrapingMixin):
ad_dir = os.path.dirname(ad_file)
for image_pattern in ad_cfg.images:
pattern_images = set()
for image_file in glob.glob(image_pattern, root_dir=ad_dir, flags=glob.GLOBSTAR | glob.BRACE | glob.EXTGLOB):
for image_file in glob.glob(image_pattern, root_dir = ad_dir, flags = glob.GLOBSTAR | glob.BRACE | glob.EXTGLOB):
_, image_file_ext = os.path.splitext(image_file)
ensure(image_file_ext.lower() in {".gif", ".jpg", ".jpeg", ".png"}, f"Unsupported image file type [{image_file}]")
if os.path.isabs(image_file):
pattern_images.add(image_file)
else:
pattern_images.add(abspath(image_file, relative_to=ad_file))
pattern_images.add(abspath(image_file, relative_to = ad_file))
images.extend(sorted(pattern_images))
ensure(images or not ad_cfg.images, f"No images found for given file patterns {ad_cfg.images} at {ad_dir}")
ad_cfg.images = list(dict.fromkeys(images))
@@ -774,13 +774,13 @@ class KleinanzeigenBot(WebScrapingMixin):
LOG.info("Loaded %s", pluralize("ad", ads))
return ads
def load_ad(self, ad_cfg_orig: dict[str, Any]) -> Ad:
def load_ad(self, ad_cfg_orig:dict[str, Any]) -> Ad:
return AdPartial.model_validate(ad_cfg_orig).to_ad(self.config.ad_defaults)
async def check_and_wait_for_captcha(self, *, is_login_page: bool = True) -> None:
async def check_and_wait_for_captcha(self, *, is_login_page:bool = True) -> None:
try:
captcha_timeout = self._timeout("captcha_detection")
await self.web_find(By.CSS_SELECTOR, "iframe[name^='a-'][src^='https://www.google.com/recaptcha/api2/anchor?']", timeout=captcha_timeout)
await self.web_find(By.CSS_SELECTOR, "iframe[name^='a-'][src^='https://www.google.com/recaptcha/api2/anchor?']", timeout = captcha_timeout)
if not is_login_page and self.config.captcha.auto_restart:
LOG.warning("Captcha recognized - auto-restart enabled, abort run...")
@@ -833,14 +833,14 @@ class KleinanzeigenBot(WebScrapingMixin):
await self.web_input(By.ID, "login-password", "")
await self.web_input(By.ID, "login-password", self.config.login.password)
await self.check_and_wait_for_captcha(is_login_page=True)
await self.check_and_wait_for_captcha(is_login_page = True)
await self.web_click(By.CSS_SELECTOR, "form#login-form button[type='submit']")
async def handle_after_login_logic(self) -> None:
try:
sms_timeout = self._timeout("sms_verification")
await self.web_find(By.TEXT, "Wir haben dir gerade einen 6-stelligen Code für die Telefonnummer", timeout=sms_timeout)
await self.web_find(By.TEXT, "Wir haben dir gerade einen 6-stelligen Code für die Telefonnummer", timeout = sms_timeout)
LOG.warning("############################################")
LOG.warning("# Device verification message detected. Please follow the instruction displayed in the Browser.")
LOG.warning("############################################")
@@ -852,10 +852,10 @@ class KleinanzeigenBot(WebScrapingMixin):
try:
LOG.info("Handling GDPR disclaimer...")
gdpr_timeout = self._timeout("gdpr_prompt")
await self.web_find(By.ID, "gdpr-banner-accept", timeout=gdpr_timeout)
await self.web_find(By.ID, "gdpr-banner-accept", timeout = gdpr_timeout)
await self.web_click(By.ID, "gdpr-banner-cmp-button")
await self.web_click(
By.XPATH, "//div[@id='ConsentManagementPage']//*//button//*[contains(., 'Alle ablehnen und fortfahren')]", timeout=gdpr_timeout
By.XPATH, "//div[@id='ConsentManagementPage']//*//button//*[contains(., 'Alle ablehnen und fortfahren')]", timeout = gdpr_timeout
)
except TimeoutError:
# GDPR banner not shown within timeout.
@@ -873,7 +873,7 @@ class KleinanzeigenBot(WebScrapingMixin):
# Try to find the standard element first
try:
user_info = await self.web_text(By.CLASS_NAME, "mr-medium", timeout=login_check_timeout)
user_info = await self.web_text(By.CLASS_NAME, "mr-medium", timeout = login_check_timeout)
if username in user_info.lower():
LOG.debug(_("Login detected via .mr-medium element"))
return True
@@ -882,7 +882,7 @@ class KleinanzeigenBot(WebScrapingMixin):
# If standard element not found or didn't contain username, try the alternative
try:
user_info = await self.web_text(By.ID, "user-email", timeout=login_check_timeout)
user_info = await self.web_text(By.ID, "user-email", timeout = login_check_timeout)
if username in user_info.lower():
LOG.debug(_("Login detected via #user-email element"))
return True
@@ -892,7 +892,7 @@ class KleinanzeigenBot(WebScrapingMixin):
LOG.debug(_("No login detected - neither .mr-medium nor #user-email found with username"))
return False
async def delete_ads(self, ad_cfgs: list[tuple[str, Ad, dict[str, Any]]]) -> None:
async def delete_ads(self, ad_cfgs:list[tuple[str, Ad, dict[str, Any]]]) -> None:
count = 0
published_ads = json.loads((await self.web_request(f"{self.root_url}/m-meine-anzeigen-verwalten.json?sort=DEFAULT"))["content"])["ads"]
@@ -900,14 +900,14 @@ class KleinanzeigenBot(WebScrapingMixin):
for ad_file, ad_cfg, _ad_cfg_orig in ad_cfgs:
count += 1
LOG.info("Processing %s/%s: '%s' from [%s]...", count, len(ad_cfgs), ad_cfg.title, ad_file)
await self.delete_ad(ad_cfg, published_ads, delete_old_ads_by_title=self.config.publishing.delete_old_ads_by_title)
await self.delete_ad(ad_cfg, published_ads, delete_old_ads_by_title = self.config.publishing.delete_old_ads_by_title)
await self.web_sleep()
LOG.info("############################################")
LOG.info("DONE: Deleted %s", pluralize("ad", count))
LOG.info("############################################")
async def delete_ad(self, ad_cfg: Ad, published_ads: list[dict[str, Any]], *, delete_old_ads_by_title: bool) -> bool:
async def delete_ad(self, ad_cfg:Ad, published_ads:list[dict[str, Any]], *, delete_old_ads_by_title:bool) -> bool:
LOG.info("Deleting ad '%s' if already present...", ad_cfg.title)
await self.web_open(f"{self.root_url}/m-meine-anzeigen.html")
@@ -922,21 +922,21 @@ class KleinanzeigenBot(WebScrapingMixin):
if ad_cfg.id == published_ad_id or ad_cfg.title == published_ad_title:
LOG.info(" -> deleting %s '%s'...", published_ad_id, published_ad_title)
await self.web_request(
url=f"{self.root_url}/m-anzeigen-loeschen.json?ids={published_ad_id}", method="POST", headers={"x-csrf-token": str(csrf_token)}
url = f"{self.root_url}/m-anzeigen-loeschen.json?ids={published_ad_id}", method = "POST", headers = {"x-csrf-token": str(csrf_token)}
)
elif ad_cfg.id:
await self.web_request(
url=f"{self.root_url}/m-anzeigen-loeschen.json?ids={ad_cfg.id}",
method="POST",
headers={"x-csrf-token": str(csrf_token)},
valid_response_codes=[200, 404],
url = f"{self.root_url}/m-anzeigen-loeschen.json?ids={ad_cfg.id}",
method = "POST",
headers = {"x-csrf-token": str(csrf_token)},
valid_response_codes = [200, 404],
)
await self.web_sleep()
ad_cfg.id = None
return True
async def extend_ads(self, ad_cfgs: list[tuple[str, Ad, dict[str, Any]]]) -> None:
async def extend_ads(self, ad_cfgs:list[tuple[str, Ad, dict[str, Any]]]) -> None:
"""Extends ads that are close to expiry."""
# Fetch currently published ads from API
published_ads = json.loads((await self.web_request(f"{self.root_url}/m-meine-anzeigen-verwalten.json?sort=DEFAULT"))["content"])["ads"]
@@ -986,7 +986,7 @@ class KleinanzeigenBot(WebScrapingMixin):
# Process extensions
success_count = 0
for idx, (ad_file, ad_cfg, ad_cfg_orig, _published_ad) in enumerate(ads_to_extend, start=1):
for idx, (ad_file, ad_cfg, ad_cfg_orig, _published_ad) in enumerate(ads_to_extend, start = 1):
LOG.info(_("Processing %s/%s: '%s' from [%s]..."), idx, len(ads_to_extend), ad_cfg.title, ad_file)
if await self.extend_ad(ad_file, ad_cfg, ad_cfg_orig):
success_count += 1
@@ -996,7 +996,7 @@ class KleinanzeigenBot(WebScrapingMixin):
LOG.info(_("DONE: Extended %s"), pluralize("ad", success_count))
LOG.info("############################################")
async def extend_ad(self, ad_file: str, ad_cfg: Ad, ad_cfg_orig: dict[str, Any]) -> bool:
async def extend_ad(self, ad_file:str, ad_cfg:Ad, ad_cfg_orig:dict[str, Any]) -> bool:
"""Extends a single ad listing."""
LOG.info(_("Extending ad '%s' (ID: %s)..."), ad_cfg.title, ad_cfg.id)
@@ -1021,14 +1021,14 @@ class KleinanzeigenBot(WebScrapingMixin):
# Simply close the dialog with the X button (aria-label="Schließen")
try:
dialog_close_timeout = self._timeout("quick_dom")
await self.web_click(By.CSS_SELECTOR, 'button[aria-label="Schließen"]', timeout=dialog_close_timeout)
await self.web_click(By.CSS_SELECTOR, 'button[aria-label="Schließen"]', timeout = dialog_close_timeout)
LOG.debug(" -> Closed confirmation dialog")
except TimeoutError:
LOG.warning(_(" -> No confirmation dialog found, extension may have completed directly"))
# Update metadata in YAML file
# Update updated_on to track when ad was extended
ad_cfg_orig["updated_on"] = misc.now().isoformat(timespec="seconds")
ad_cfg_orig["updated_on"] = misc.now().isoformat(timespec = "seconds")
dicts.save_dict(ad_file, ad_cfg_orig)
LOG.info(_(" -> SUCCESS: ad extended with ID %s"), ad_cfg.id)
@@ -1045,7 +1045,7 @@ class KleinanzeigenBot(WebScrapingMixin):
# Check for success messages
return await self.web_check(By.ID, "checking-done", Is.DISPLAYED) or await self.web_check(By.ID, "not-completed", Is.DISPLAYED)
async def publish_ads(self, ad_cfgs: list[tuple[str, Ad, dict[str, Any]]]) -> None:
async def publish_ads(self, ad_cfgs:list[tuple[str, Ad, dict[str, Any]]]) -> None:
count = 0
failed_count = 0
max_retries = 3
@@ -1082,12 +1082,12 @@ class KleinanzeigenBot(WebScrapingMixin):
if success:
try:
publish_timeout = self._timeout("publishing_result")
await self.web_await(self.__check_publishing_result, timeout=publish_timeout)
await self.web_await(self.__check_publishing_result, timeout = publish_timeout)
except TimeoutError:
LOG.warning(_(" -> Could not confirm publishing for '%s', but ad may be online"), ad_cfg.title)
if success and self.config.publishing.delete_old_ads == "AFTER_PUBLISH" and not self.keep_old_ads:
await self.delete_ad(ad_cfg, published_ads, delete_old_ads_by_title=False)
await self.delete_ad(ad_cfg, published_ads, delete_old_ads_by_title = False)
LOG.info("############################################")
if failed_count > 0:
@@ -1097,7 +1097,7 @@ class KleinanzeigenBot(WebScrapingMixin):
LOG.info("############################################")
async def publish_ad(
self, ad_file: str, ad_cfg: Ad, ad_cfg_orig: dict[str, Any], published_ads: list[dict[str, Any]], mode: AdUpdateStrategy = AdUpdateStrategy.REPLACE
self, ad_file:str, ad_cfg:Ad, ad_cfg_orig:dict[str, Any], published_ads:list[dict[str, Any]], mode:AdUpdateStrategy = AdUpdateStrategy.REPLACE
) -> None:
"""
@param ad_cfg: the effective ad config (i.e. with default values applied etc.)
@@ -1108,7 +1108,7 @@ class KleinanzeigenBot(WebScrapingMixin):
if mode == AdUpdateStrategy.REPLACE:
if self.config.publishing.delete_old_ads == "BEFORE_PUBLISH" and not self.keep_old_ads:
await self.delete_ad(ad_cfg, published_ads, delete_old_ads_by_title=self.config.publishing.delete_old_ads_by_title)
await self.delete_ad(ad_cfg, published_ads, delete_old_ads_by_title = self.config.publishing.delete_old_ads_by_title)
# Apply auto price reduction only for REPLACE operations (actual reposts)
# This ensures price reductions only happen on republish, not on UPDATE
@@ -1197,12 +1197,12 @@ class KleinanzeigenBot(WebScrapingMixin):
elif not await self.web_check(By.ID, "radio-buy-now-no", Is.SELECTED):
await self.web_click(By.ID, "radio-buy-now-no")
except TimeoutError as ex:
LOG.debug(ex, exc_info=True)
LOG.debug(ex, exc_info = True)
#############################
# set description
#############################
description = self.__get_description(ad_cfg, with_affixes=True)
description = self.__get_description(ad_cfg, with_affixes = True)
await self.web_execute("document.querySelector('#pstad-descrptn').value = `" + description.replace("`", "'") + "`")
await self.__set_contact_fields(ad_cfg.contact)
@@ -1213,7 +1213,7 @@ class KleinanzeigenBot(WebScrapingMixin):
#############################
img_items = await self.web_find_all(By.CSS_SELECTOR, "ul#j-pictureupload-thumbnails > li:not(.is-placeholder)")
for element in img_items:
btn = await self.web_find(By.CSS_SELECTOR, "button.pictureupload-thumbnails-remove", parent=element)
btn = await self.web_find(By.CSS_SELECTOR, "button.pictureupload-thumbnails-remove", parent = element)
await btn.click()
#############################
@@ -1224,7 +1224,7 @@ class KleinanzeigenBot(WebScrapingMixin):
#############################
# wait for captcha
#############################
await self.check_and_wait_for_captcha(is_login_page=False)
await self.check_and_wait_for_captcha(is_login_page = False)
#############################
# submit
@@ -1250,7 +1250,7 @@ class KleinanzeigenBot(WebScrapingMixin):
#############################
try:
short_timeout = self._timeout("quick_dom")
await self.web_find(By.ID, "myftr-shppngcrt-frm", timeout=short_timeout)
await self.web_find(By.ID, "myftr-shppngcrt-frm", timeout = short_timeout)
LOG.warning("############################################")
LOG.warning("# Payment form detected! Please proceed with payment.")
@@ -1262,7 +1262,7 @@ class KleinanzeigenBot(WebScrapingMixin):
pass
confirmation_timeout = self._timeout("publishing_confirmation")
await self.web_await(lambda: "p-anzeige-aufgeben-bestaetigung.html?adId=" in self.page.url, timeout=confirmation_timeout)
await self.web_await(lambda: "p-anzeige-aufgeben-bestaetigung.html?adId=" in self.page.url, timeout = confirmation_timeout)
# extract the ad id from the URL's query parameter
current_url_query_params = urllib_parse.parse_qs(urllib_parse.urlparse(self.page.url).query)
@@ -1272,7 +1272,7 @@ class KleinanzeigenBot(WebScrapingMixin):
# Update content hash after successful publication
# Calculate hash on original config to ensure consistent comparison on restart
ad_cfg_orig["content_hash"] = AdPartial.model_validate(ad_cfg_orig).update_content_hash().content_hash
ad_cfg_orig["updated_on"] = misc.now().isoformat(timespec="seconds")
ad_cfg_orig["updated_on"] = misc.now().isoformat(timespec = "seconds")
if not ad_cfg.created_on and not ad_cfg.id:
ad_cfg_orig["created_on"] = ad_cfg_orig["updated_on"]
@@ -1299,7 +1299,7 @@ class KleinanzeigenBot(WebScrapingMixin):
dicts.save_dict(ad_file, ad_cfg_orig)
async def __set_contact_fields(self, contact: Contact) -> None:
async def __set_contact_fields(self, contact:Contact) -> None:
#############################
# set contact zipcode
#############################
@@ -1384,7 +1384,7 @@ class KleinanzeigenBot(WebScrapingMixin):
)
)
async def update_ads(self, ad_cfgs: list[tuple[str, Ad, dict[str, Any]]]) -> None:
async def update_ads(self, ad_cfgs:list[tuple[str, Ad, dict[str, Any]]]) -> None:
"""
Updates a list of ads.
The list gets filtered, so that only already published ads will be updated.
@@ -1415,25 +1415,25 @@ class KleinanzeigenBot(WebScrapingMixin):
await self.publish_ad(ad_file, ad_cfg, ad_cfg_orig, published_ads, AdUpdateStrategy.MODIFY)
publish_timeout = self._timeout("publishing_result")
await self.web_await(self.__check_publishing_result, timeout=publish_timeout)
await self.web_await(self.__check_publishing_result, timeout = publish_timeout)
LOG.info("############################################")
LOG.info("DONE: updated %s", pluralize("ad", count))
LOG.info("############################################")
async def __set_condition(self, condition_value: str) -> None:
async def __set_condition(self, condition_value:str) -> None:
try:
# Open condition dialog
await self.web_click(By.XPATH, '//*[@id="j-post-listing-frontend-conditions"]//button[@aria-haspopup="true"]')
except TimeoutError:
LOG.debug("Unable to open condition dialog and select condition [%s]", condition_value, exc_info=True)
LOG.debug("Unable to open condition dialog and select condition [%s]", condition_value, exc_info = True)
return
try:
# Click radio button
await self.web_click(By.ID, f"radio-button-{condition_value}")
except TimeoutError:
LOG.debug("Unable to select condition [%s]", condition_value, exc_info=True)
LOG.debug("Unable to select condition [%s]", condition_value, exc_info = True)
try:
# Click accept button
@@ -1441,7 +1441,7 @@ class KleinanzeigenBot(WebScrapingMixin):
except TimeoutError as ex:
raise TimeoutError(_("Unable to close condition dialog!")) from ex
async def __set_category(self, category: str | None, ad_file: str) -> None:
async def __set_category(self, category:str | None, ad_file:str) -> None:
# click on something to trigger automatic category detection
await self.web_click(By.ID, "pstad-descrptn")
@@ -1464,7 +1464,7 @@ class KleinanzeigenBot(WebScrapingMixin):
else:
ensure(is_category_auto_selected, f"No category specified in [{ad_file}] and automatic category detection failed")
async def __set_special_attributes(self, ad_cfg: Ad) -> None:
async def __set_special_attributes(self, ad_cfg:Ad) -> None:
if not ad_cfg.special_attributes:
return
@@ -1499,7 +1499,7 @@ class KleinanzeigenBot(WebScrapingMixin):
raise TimeoutError(_("Failed to set attribute '%s'") % special_attribute_key) from ex
try:
elem_id: str = str(special_attr_elem.attrs.id)
elem_id:str = str(special_attr_elem.attrs.id)
if special_attr_elem.local_name == "select":
LOG.debug(_("Attribute field '%s' seems to be a select..."), special_attribute_key)
await self.web_select(By.ID, elem_id, special_attribute_value_str)
@@ -1517,26 +1517,26 @@ class KleinanzeigenBot(WebScrapingMixin):
raise TimeoutError(_("Failed to set attribute '%s'") % special_attribute_key) from ex
LOG.debug("Successfully set attribute field [%s] to [%s]...", special_attribute_key, special_attribute_value_str)
async def __set_shipping(self, ad_cfg: Ad, mode: AdUpdateStrategy = AdUpdateStrategy.REPLACE) -> None:
async def __set_shipping(self, ad_cfg:Ad, mode:AdUpdateStrategy = AdUpdateStrategy.REPLACE) -> None:
short_timeout = self._timeout("quick_dom")
if ad_cfg.shipping_type == "PICKUP":
try:
await self.web_click(By.ID, "radio-pickup")
except TimeoutError as ex:
LOG.debug(ex, exc_info=True)
LOG.debug(ex, exc_info = True)
elif ad_cfg.shipping_options:
await self.web_click(By.XPATH, '//button//span[contains(., "Versandmethoden auswählen")]')
if mode == AdUpdateStrategy.MODIFY:
try:
# when "Andere Versandmethoden" is not available, go back and start over new
await self.web_find(By.XPATH, '//dialog//button[contains(., "Andere Versandmethoden")]', timeout=short_timeout)
await self.web_find(By.XPATH, '//dialog//button[contains(., "Andere Versandmethoden")]', timeout = short_timeout)
except TimeoutError:
await self.web_click(By.XPATH, '//dialog//button[contains(., "Zurück")]')
# in some categories we need to go another dialog back
try:
await self.web_find(By.XPATH, '//dialog//button[contains(., "Andere Versandmethoden")]', timeout=short_timeout)
await self.web_find(By.XPATH, '//dialog//button[contains(., "Andere Versandmethoden")]', timeout = short_timeout)
except TimeoutError:
await self.web_click(By.XPATH, '//dialog//button[contains(., "Zurück")]')
@@ -1562,7 +1562,7 @@ class KleinanzeigenBot(WebScrapingMixin):
try:
# only click on "Individueller Versand" when "IndividualShippingInput" is not available, otherwise its already checked
# (important for mode = UPDATE)
await self.web_find(By.XPATH, '//input[contains(@placeholder, "Versandkosten (optional)")]', timeout=short_timeout)
await self.web_find(By.XPATH, '//input[contains(@placeholder, "Versandkosten (optional)")]', timeout = short_timeout)
except TimeoutError:
# Input not visible yet; click the individual shipping option.
await self.web_click(By.XPATH, '//*[contains(@id, "INDIVIDUAL") and contains(@data-testid, "Individueller Versand")]')
@@ -1573,10 +1573,10 @@ class KleinanzeigenBot(WebScrapingMixin):
)
await self.web_click(By.XPATH, '//dialog//button[contains(., "Fertig")]')
except TimeoutError as ex:
LOG.debug(ex, exc_info=True)
LOG.debug(ex, exc_info = True)
raise TimeoutError(_("Unable to close shipping dialog!")) from ex
async def __set_shipping_options(self, ad_cfg: Ad, mode: AdUpdateStrategy = AdUpdateStrategy.REPLACE) -> None:
async def __set_shipping_options(self, ad_cfg:Ad, mode:AdUpdateStrategy = AdUpdateStrategy.REPLACE) -> None:
if not ad_cfg.shipping_options:
return
@@ -1596,7 +1596,7 @@ class KleinanzeigenBot(WebScrapingMixin):
except KeyError as ex:
raise KeyError(f"Unknown shipping option(s), please refer to the documentation/README: {ad_cfg.shipping_options}") from ex
shipping_sizes, shipping_selector, shipping_packages = zip(*mapped_shipping_options, strict=False)
shipping_sizes, shipping_selector, shipping_packages = zip(*mapped_shipping_options, strict = False)
try:
(shipping_size,) = set(shipping_sizes)
@@ -1652,19 +1652,19 @@ class KleinanzeigenBot(WebScrapingMixin):
for shipping_package in to_be_clicked_shipping_packages:
await self.web_click(By.XPATH, f'//dialog//input[contains(@data-testid, "{shipping_package}")]')
except TimeoutError as ex:
LOG.debug(ex, exc_info=True)
LOG.debug(ex, exc_info = True)
try:
# Click apply button
await self.web_click(By.XPATH, '//dialog//button[contains(., "Fertig")]')
except TimeoutError as ex:
raise TimeoutError(_("Unable to close shipping dialog!")) from ex
async def __upload_images(self, ad_cfg: Ad) -> None:
async def __upload_images(self, ad_cfg:Ad) -> None:
if not ad_cfg.images:
return
LOG.info(" -> found %s", pluralize("image", ad_cfg.images))
image_upload: Element = await self.web_find(By.CSS_SELECTOR, "input[type=file]")
image_upload:Element = await self.web_find(By.CSS_SELECTOR, "input[type=file]")
for image in ad_cfg.images:
LOG.info(" -> uploading image [%s]", image)
@@ -1680,7 +1680,7 @@ class KleinanzeigenBot(WebScrapingMixin):
thumbnails = await self.web_find_all(
By.CSS_SELECTOR,
"ul#j-pictureupload-thumbnails > li:not(.is-placeholder)",
timeout=self._timeout("quick_dom"), # Fast timeout for polling
timeout = self._timeout("quick_dom"), # Fast timeout for polling
)
current_count = len(thumbnails)
if current_count < expected_count:
@@ -1691,12 +1691,12 @@ class KleinanzeigenBot(WebScrapingMixin):
return False
try:
await self.web_await(check_thumbnails_uploaded, timeout=self._timeout("image_upload"), timeout_error_message=_("Image upload timeout exceeded"))
await self.web_await(check_thumbnails_uploaded, timeout = self._timeout("image_upload"), timeout_error_message = _("Image upload timeout exceeded"))
except TimeoutError as ex:
# Get current count for better error message
try:
thumbnails = await self.web_find_all(
By.CSS_SELECTOR, "ul#j-pictureupload-thumbnails > li:not(.is-placeholder)", timeout=self._timeout("quick_dom")
By.CSS_SELECTOR, "ul#j-pictureupload-thumbnails > li:not(.is-placeholder)", timeout = self._timeout("quick_dom")
)
current_count = len(thumbnails)
except TimeoutError:
@@ -1738,7 +1738,7 @@ class KleinanzeigenBot(WebScrapingMixin):
elif self.ads_selector == "new": # download only unsaved ads
# check which ads already saved
saved_ad_ids = []
ads = self.load_ads(ignore_inactive=False, exclude_ads_with_id=False) # do not skip because of existing IDs
ads = self.load_ads(ignore_inactive = False, exclude_ads_with_id = False) # do not skip because of existing IDs
for ad in ads:
saved_ad_id = ad[1].id
if saved_ad_id is None:
@@ -1775,7 +1775,7 @@ class KleinanzeigenBot(WebScrapingMixin):
else:
LOG.error("The page with the id %d does not exist!", ad_id)
def __get_description(self, ad_cfg: Ad, *, with_affixes: bool) -> str:
def __get_description(self, ad_cfg:Ad, *, with_affixes:bool) -> str:
"""Get the ad description optionally with prefix and suffix applied.
Precedence (highest to lowest):
@@ -1827,7 +1827,7 @@ class KleinanzeigenBot(WebScrapingMixin):
return final_description
def update_content_hashes(self, ads: list[tuple[str, Ad, dict[str, Any]]]) -> None:
def update_content_hashes(self, ads:list[tuple[str, Ad, dict[str, Any]]]) -> None:
count = 0
for ad_file, ad_cfg, ad_cfg_orig in ads:
@@ -1848,7 +1848,7 @@ class KleinanzeigenBot(WebScrapingMixin):
#############################
def main(args: list[str]) -> None:
def main(args:list[str]) -> None:
if "version" not in args:
print(
textwrap.dedent(rf"""
@@ -1861,7 +1861,7 @@ def main(args: list[str]) -> None:
https://github.com/Second-Hand-Friends/kleinanzeigen-bot
Version: {__version__}
""")[1:],
flush=True,
flush = True,
) # [1:] removes the first empty blank line
loggers.configure_console_logging()