mirror of
https://github.com/Second-Hand-Friends/kleinanzeigen-bot.git
synced 2026-03-12 10:31:50 +01:00
feat: add browser profile XDG support and documentation (#777)
This commit is contained in:
@@ -28,7 +28,7 @@ from .utils.web_scraping_mixin import By, Element, Is, WebScrapingMixin
|
||||
|
||||
# W0406: possibly a bug, see https://github.com/PyCQA/pylint/issues/3933
|
||||
|
||||
LOG: Final[loggers.Logger] = loggers.get_logger(__name__)
|
||||
LOG:Final[loggers.Logger] = loggers.get_logger(__name__)
|
||||
LOG.setLevel(loggers.INFO)
|
||||
|
||||
colorama.just_fix_windows_console()
|
||||
@@ -39,7 +39,7 @@ class AdUpdateStrategy(enum.Enum):
|
||||
MODIFY = enum.auto()
|
||||
|
||||
|
||||
def _repost_cycle_ready(ad_cfg: Ad, ad_file_relative: str) -> bool:
|
||||
def _repost_cycle_ready(ad_cfg:Ad, ad_file_relative:str) -> bool:
|
||||
"""
|
||||
Check if the repost cycle delay has been satisfied.
|
||||
|
||||
@@ -72,7 +72,7 @@ def _repost_cycle_ready(ad_cfg: Ad, ad_file_relative: str) -> bool:
|
||||
return True
|
||||
|
||||
|
||||
def _day_delay_elapsed(ad_cfg: Ad, ad_file_relative: str) -> bool:
|
||||
def _day_delay_elapsed(ad_cfg:Ad, ad_file_relative:str) -> bool:
|
||||
"""
|
||||
Check if the day delay has elapsed since the ad was last published.
|
||||
|
||||
@@ -100,7 +100,7 @@ def _day_delay_elapsed(ad_cfg: Ad, ad_file_relative: str) -> bool:
|
||||
return True
|
||||
|
||||
|
||||
def apply_auto_price_reduction(ad_cfg: Ad, _ad_cfg_orig: dict[str, Any], ad_file_relative: str) -> None:
|
||||
def apply_auto_price_reduction(ad_cfg:Ad, _ad_cfg_orig:dict[str, Any], ad_file_relative:str) -> None:
|
||||
"""
|
||||
Apply automatic price reduction to an ad based on repost count and configuration.
|
||||
|
||||
@@ -132,7 +132,7 @@ def apply_auto_price_reduction(ad_cfg: Ad, _ad_cfg_orig: dict[str, Any], ad_file
|
||||
applied_cycles = ad_cfg.price_reduction_count or 0
|
||||
next_cycle = applied_cycles + 1
|
||||
|
||||
effective_price = calculate_auto_price(base_price=base_price, auto_price_reduction=ad_cfg.auto_price_reduction, target_reduction_cycle=next_cycle)
|
||||
effective_price = calculate_auto_price(base_price = base_price, auto_price_reduction = ad_cfg.auto_price_reduction, target_reduction_cycle = next_cycle)
|
||||
|
||||
if effective_price is None:
|
||||
return
|
||||
@@ -149,7 +149,7 @@ def apply_auto_price_reduction(ad_cfg: Ad, _ad_cfg_orig: dict[str, Any], ad_file
|
||||
# Note: price_reduction_count is persisted to ad_cfg_orig only after successful publish
|
||||
|
||||
|
||||
class KleinanzeigenBot(WebScrapingMixin):
|
||||
class KleinanzeigenBot(WebScrapingMixin): # noqa: PLR0904
|
||||
def __init__(self) -> None:
|
||||
# workaround for https://github.com/Second-Hand-Friends/kleinanzeigen-bot/issues/295
|
||||
# see https://github.com/pyinstaller/pyinstaller/issues/7229#issuecomment-1309383026
|
||||
@@ -159,17 +159,17 @@ class KleinanzeigenBot(WebScrapingMixin):
|
||||
|
||||
self.root_url = "https://www.kleinanzeigen.de"
|
||||
|
||||
self.config: Config
|
||||
self.config:Config
|
||||
self.config_file_path = abspath("config.yaml")
|
||||
self.config_explicitly_provided = False
|
||||
|
||||
self.installation_mode: xdg_paths.InstallationMode | None = None
|
||||
self.installation_mode:xdg_paths.InstallationMode | None = None
|
||||
|
||||
self.categories: dict[str, str] = {}
|
||||
self.categories:dict[str, str] = {}
|
||||
|
||||
self.file_log: loggers.LogFileHandle | None = None
|
||||
self.file_log:loggers.LogFileHandle | None = None
|
||||
log_file_basename = is_frozen() and os.path.splitext(os.path.basename(sys.executable))[0] or self.__module__
|
||||
self.log_file_path: str | None = abspath(f"{log_file_basename}.log")
|
||||
self.log_file_path:str | None = abspath(f"{log_file_basename}.log")
|
||||
self.log_file_basename = log_file_basename
|
||||
self.log_file_explicitly_provided = False
|
||||
|
||||
@@ -245,7 +245,7 @@ class KleinanzeigenBot(WebScrapingMixin):
|
||||
LOG.info(_("Installation mode: %s"), mode_display)
|
||||
LOG.info(_("Config file: %s"), self.config_file_path)
|
||||
|
||||
async def run(self, args: list[str]) -> None:
|
||||
async def run(self, args:list[str]) -> None:
|
||||
self.parse_args(args)
|
||||
self.finalize_installation_mode()
|
||||
try:
|
||||
@@ -277,7 +277,7 @@ class KleinanzeigenBot(WebScrapingMixin):
|
||||
self.configure_file_logging()
|
||||
self.load_config()
|
||||
checker = UpdateChecker(self.config, self.installation_mode_or_portable)
|
||||
checker.check_for_updates(skip_interval_check=True)
|
||||
checker.check_for_updates(skip_interval_check = True)
|
||||
case "update-content-hash":
|
||||
self.configure_file_logging()
|
||||
self.load_config()
|
||||
@@ -285,7 +285,7 @@ class KleinanzeigenBot(WebScrapingMixin):
|
||||
checker = UpdateChecker(self.config, self.installation_mode_or_portable)
|
||||
checker.check_for_updates()
|
||||
self.ads_selector = "all"
|
||||
if ads := self.load_ads(exclude_ads_with_id=False):
|
||||
if ads := self.load_ads(exclude_ads_with_id = False):
|
||||
self.update_content_hashes(ads)
|
||||
else:
|
||||
LOG.info("############################################")
|
||||
@@ -503,7 +503,7 @@ class KleinanzeigenBot(WebScrapingMixin):
|
||||
)
|
||||
)
|
||||
|
||||
def parse_args(self, args: list[str]) -> None:
|
||||
def parse_args(self, args:list[str]) -> None:
|
||||
try:
|
||||
options, arguments = getopt.gnu_getopt(args[1:], "hv", ["ads=", "config=", "force", "help", "keep-old", "logfile=", "lang=", "verbose"])
|
||||
except getopt.error as ex:
|
||||
@@ -571,7 +571,7 @@ class KleinanzeigenBot(WebScrapingMixin):
|
||||
default_config.login.password = "changeme" # noqa: S105 placeholder for default config, not a real password
|
||||
dicts.save_dict(
|
||||
self.config_file_path,
|
||||
default_config.model_dump(exclude_none=True, exclude={"ad_defaults": {"description"}}),
|
||||
default_config.model_dump(exclude_none = True, exclude = {"ad_defaults": {"description"}}),
|
||||
header=(
|
||||
"# yaml-language-server: $schema="
|
||||
"https://raw.githubusercontent.com/Second-Hand-Friends/kleinanzeigen-bot"
|
||||
@@ -585,7 +585,7 @@ class KleinanzeigenBot(WebScrapingMixin):
|
||||
self.create_default_config()
|
||||
|
||||
config_yaml = dicts.load_dict_if_exists(self.config_file_path, _("config"))
|
||||
self.config = Config.model_validate(config_yaml, strict=True, context=self.config_file_path)
|
||||
self.config = Config.model_validate(config_yaml, strict = True, context = self.config_file_path)
|
||||
|
||||
# load built-in category mappings
|
||||
self.categories = dicts.load_dict_from_module(resources, "categories.yaml", "categories")
|
||||
@@ -598,13 +598,13 @@ class KleinanzeigenBot(WebScrapingMixin):
|
||||
# populate browser_config object used by WebScrapingMixin
|
||||
self.browser_config.arguments = self.config.browser.arguments
|
||||
self.browser_config.binary_location = self.config.browser.binary_location
|
||||
self.browser_config.extensions = [abspath(item, relative_to=self.config_file_path) for item in self.config.browser.extensions]
|
||||
self.browser_config.extensions = [abspath(item, relative_to = self.config_file_path) for item in self.config.browser.extensions]
|
||||
self.browser_config.use_private_window = self.config.browser.use_private_window
|
||||
if self.config.browser.user_data_dir:
|
||||
self.browser_config.user_data_dir = abspath(self.config.browser.user_data_dir, relative_to=self.config_file_path)
|
||||
self.browser_config.user_data_dir = abspath(self.config.browser.user_data_dir, relative_to = self.config_file_path)
|
||||
self.browser_config.profile_name = self.config.browser.profile_name
|
||||
|
||||
def __check_ad_republication(self, ad_cfg: Ad, ad_file_relative: str) -> bool:
|
||||
def __check_ad_republication(self, ad_cfg:Ad, ad_file_relative:str) -> bool:
|
||||
"""
|
||||
Check if an ad needs to be republished based on republication interval.
|
||||
Note: This method does not check for content changes. Use __check_ad_changed for that.
|
||||
@@ -635,7 +635,7 @@ class KleinanzeigenBot(WebScrapingMixin):
|
||||
|
||||
return True
|
||||
|
||||
def __check_ad_changed(self, ad_cfg: Ad, ad_cfg_orig: dict[str, Any], ad_file_relative: str) -> bool:
|
||||
def __check_ad_changed(self, ad_cfg:Ad, ad_cfg_orig:dict[str, Any], ad_file_relative:str) -> bool:
|
||||
"""
|
||||
Check if an ad has been changed since last publication.
|
||||
|
||||
@@ -662,7 +662,7 @@ class KleinanzeigenBot(WebScrapingMixin):
|
||||
|
||||
return False
|
||||
|
||||
def load_ads(self, *, ignore_inactive: bool = True, exclude_ads_with_id: bool = True) -> list[tuple[str, Ad, dict[str, Any]]]:
|
||||
def load_ads(self, *, ignore_inactive:bool = True, exclude_ads_with_id:bool = True) -> list[tuple[str, Ad, dict[str, Any]]]:
|
||||
"""
|
||||
Load and validate all ad config files, optionally filtering out inactive or already-published ads.
|
||||
|
||||
@@ -678,12 +678,12 @@ class KleinanzeigenBot(WebScrapingMixin):
|
||||
"""
|
||||
LOG.info("Searching for ad config files...")
|
||||
|
||||
ad_files: dict[str, str] = {}
|
||||
ad_files:dict[str, str] = {}
|
||||
data_root_dir = os.path.dirname(self.config_file_path)
|
||||
for file_pattern in self.config.ad_files:
|
||||
for ad_file in glob.glob(file_pattern, root_dir=data_root_dir, flags=glob.GLOBSTAR | glob.BRACE | glob.EXTGLOB):
|
||||
for ad_file in glob.glob(file_pattern, root_dir = data_root_dir, flags = glob.GLOBSTAR | glob.BRACE | glob.EXTGLOB):
|
||||
if not str(ad_file).endswith("ad_fields.yaml"):
|
||||
ad_files[abspath(ad_file, relative_to=data_root_dir)] = ad_file
|
||||
ad_files[abspath(ad_file, relative_to = data_root_dir)] = ad_file
|
||||
LOG.info(" -> found %s", pluralize("ad config file", ad_files))
|
||||
if not ad_files:
|
||||
return []
|
||||
@@ -700,8 +700,8 @@ class KleinanzeigenBot(WebScrapingMixin):
|
||||
|
||||
ads = []
|
||||
for ad_file, ad_file_relative in sorted(ad_files.items()):
|
||||
ad_cfg_orig: dict[str, Any] = dicts.load_dict(ad_file, "ad")
|
||||
ad_cfg: Ad = self.load_ad(ad_cfg_orig)
|
||||
ad_cfg_orig:dict[str, Any] = dicts.load_dict(ad_file, "ad")
|
||||
ad_cfg:Ad = self.load_ad(ad_cfg_orig)
|
||||
|
||||
if ignore_inactive and not ad_cfg.active:
|
||||
LOG.info(" -> SKIPPED: inactive ad [%s]", ad_file_relative)
|
||||
@@ -738,8 +738,8 @@ class KleinanzeigenBot(WebScrapingMixin):
|
||||
if not should_include:
|
||||
continue
|
||||
|
||||
ensure(self.__get_description(ad_cfg, with_affixes=False), f"-> property [description] not specified @ [{ad_file}]")
|
||||
self.__get_description(ad_cfg, with_affixes=True) # validates complete description
|
||||
ensure(self.__get_description(ad_cfg, with_affixes = False), f"-> property [description] not specified @ [{ad_file}]")
|
||||
self.__get_description(ad_cfg, with_affixes = True) # validates complete description
|
||||
|
||||
if ad_cfg.category:
|
||||
resolved_category_id = self.categories.get(ad_cfg.category)
|
||||
@@ -758,13 +758,13 @@ class KleinanzeigenBot(WebScrapingMixin):
|
||||
ad_dir = os.path.dirname(ad_file)
|
||||
for image_pattern in ad_cfg.images:
|
||||
pattern_images = set()
|
||||
for image_file in glob.glob(image_pattern, root_dir=ad_dir, flags=glob.GLOBSTAR | glob.BRACE | glob.EXTGLOB):
|
||||
for image_file in glob.glob(image_pattern, root_dir = ad_dir, flags = glob.GLOBSTAR | glob.BRACE | glob.EXTGLOB):
|
||||
_, image_file_ext = os.path.splitext(image_file)
|
||||
ensure(image_file_ext.lower() in {".gif", ".jpg", ".jpeg", ".png"}, f"Unsupported image file type [{image_file}]")
|
||||
if os.path.isabs(image_file):
|
||||
pattern_images.add(image_file)
|
||||
else:
|
||||
pattern_images.add(abspath(image_file, relative_to=ad_file))
|
||||
pattern_images.add(abspath(image_file, relative_to = ad_file))
|
||||
images.extend(sorted(pattern_images))
|
||||
ensure(images or not ad_cfg.images, f"No images found for given file patterns {ad_cfg.images} at {ad_dir}")
|
||||
ad_cfg.images = list(dict.fromkeys(images))
|
||||
@@ -774,13 +774,13 @@ class KleinanzeigenBot(WebScrapingMixin):
|
||||
LOG.info("Loaded %s", pluralize("ad", ads))
|
||||
return ads
|
||||
|
||||
def load_ad(self, ad_cfg_orig: dict[str, Any]) -> Ad:
|
||||
def load_ad(self, ad_cfg_orig:dict[str, Any]) -> Ad:
|
||||
return AdPartial.model_validate(ad_cfg_orig).to_ad(self.config.ad_defaults)
|
||||
|
||||
async def check_and_wait_for_captcha(self, *, is_login_page: bool = True) -> None:
|
||||
async def check_and_wait_for_captcha(self, *, is_login_page:bool = True) -> None:
|
||||
try:
|
||||
captcha_timeout = self._timeout("captcha_detection")
|
||||
await self.web_find(By.CSS_SELECTOR, "iframe[name^='a-'][src^='https://www.google.com/recaptcha/api2/anchor?']", timeout=captcha_timeout)
|
||||
await self.web_find(By.CSS_SELECTOR, "iframe[name^='a-'][src^='https://www.google.com/recaptcha/api2/anchor?']", timeout = captcha_timeout)
|
||||
|
||||
if not is_login_page and self.config.captcha.auto_restart:
|
||||
LOG.warning("Captcha recognized - auto-restart enabled, abort run...")
|
||||
@@ -833,14 +833,14 @@ class KleinanzeigenBot(WebScrapingMixin):
|
||||
await self.web_input(By.ID, "login-password", "")
|
||||
await self.web_input(By.ID, "login-password", self.config.login.password)
|
||||
|
||||
await self.check_and_wait_for_captcha(is_login_page=True)
|
||||
await self.check_and_wait_for_captcha(is_login_page = True)
|
||||
|
||||
await self.web_click(By.CSS_SELECTOR, "form#login-form button[type='submit']")
|
||||
|
||||
async def handle_after_login_logic(self) -> None:
|
||||
try:
|
||||
sms_timeout = self._timeout("sms_verification")
|
||||
await self.web_find(By.TEXT, "Wir haben dir gerade einen 6-stelligen Code für die Telefonnummer", timeout=sms_timeout)
|
||||
await self.web_find(By.TEXT, "Wir haben dir gerade einen 6-stelligen Code für die Telefonnummer", timeout = sms_timeout)
|
||||
LOG.warning("############################################")
|
||||
LOG.warning("# Device verification message detected. Please follow the instruction displayed in the Browser.")
|
||||
LOG.warning("############################################")
|
||||
@@ -852,10 +852,10 @@ class KleinanzeigenBot(WebScrapingMixin):
|
||||
try:
|
||||
LOG.info("Handling GDPR disclaimer...")
|
||||
gdpr_timeout = self._timeout("gdpr_prompt")
|
||||
await self.web_find(By.ID, "gdpr-banner-accept", timeout=gdpr_timeout)
|
||||
await self.web_find(By.ID, "gdpr-banner-accept", timeout = gdpr_timeout)
|
||||
await self.web_click(By.ID, "gdpr-banner-cmp-button")
|
||||
await self.web_click(
|
||||
By.XPATH, "//div[@id='ConsentManagementPage']//*//button//*[contains(., 'Alle ablehnen und fortfahren')]", timeout=gdpr_timeout
|
||||
By.XPATH, "//div[@id='ConsentManagementPage']//*//button//*[contains(., 'Alle ablehnen und fortfahren')]", timeout = gdpr_timeout
|
||||
)
|
||||
except TimeoutError:
|
||||
# GDPR banner not shown within timeout.
|
||||
@@ -873,7 +873,7 @@ class KleinanzeigenBot(WebScrapingMixin):
|
||||
|
||||
# Try to find the standard element first
|
||||
try:
|
||||
user_info = await self.web_text(By.CLASS_NAME, "mr-medium", timeout=login_check_timeout)
|
||||
user_info = await self.web_text(By.CLASS_NAME, "mr-medium", timeout = login_check_timeout)
|
||||
if username in user_info.lower():
|
||||
LOG.debug(_("Login detected via .mr-medium element"))
|
||||
return True
|
||||
@@ -882,7 +882,7 @@ class KleinanzeigenBot(WebScrapingMixin):
|
||||
|
||||
# If standard element not found or didn't contain username, try the alternative
|
||||
try:
|
||||
user_info = await self.web_text(By.ID, "user-email", timeout=login_check_timeout)
|
||||
user_info = await self.web_text(By.ID, "user-email", timeout = login_check_timeout)
|
||||
if username in user_info.lower():
|
||||
LOG.debug(_("Login detected via #user-email element"))
|
||||
return True
|
||||
@@ -892,7 +892,7 @@ class KleinanzeigenBot(WebScrapingMixin):
|
||||
LOG.debug(_("No login detected - neither .mr-medium nor #user-email found with username"))
|
||||
return False
|
||||
|
||||
async def delete_ads(self, ad_cfgs: list[tuple[str, Ad, dict[str, Any]]]) -> None:
|
||||
async def delete_ads(self, ad_cfgs:list[tuple[str, Ad, dict[str, Any]]]) -> None:
|
||||
count = 0
|
||||
|
||||
published_ads = json.loads((await self.web_request(f"{self.root_url}/m-meine-anzeigen-verwalten.json?sort=DEFAULT"))["content"])["ads"]
|
||||
@@ -900,14 +900,14 @@ class KleinanzeigenBot(WebScrapingMixin):
|
||||
for ad_file, ad_cfg, _ad_cfg_orig in ad_cfgs:
|
||||
count += 1
|
||||
LOG.info("Processing %s/%s: '%s' from [%s]...", count, len(ad_cfgs), ad_cfg.title, ad_file)
|
||||
await self.delete_ad(ad_cfg, published_ads, delete_old_ads_by_title=self.config.publishing.delete_old_ads_by_title)
|
||||
await self.delete_ad(ad_cfg, published_ads, delete_old_ads_by_title = self.config.publishing.delete_old_ads_by_title)
|
||||
await self.web_sleep()
|
||||
|
||||
LOG.info("############################################")
|
||||
LOG.info("DONE: Deleted %s", pluralize("ad", count))
|
||||
LOG.info("############################################")
|
||||
|
||||
async def delete_ad(self, ad_cfg: Ad, published_ads: list[dict[str, Any]], *, delete_old_ads_by_title: bool) -> bool:
|
||||
async def delete_ad(self, ad_cfg:Ad, published_ads:list[dict[str, Any]], *, delete_old_ads_by_title:bool) -> bool:
|
||||
LOG.info("Deleting ad '%s' if already present...", ad_cfg.title)
|
||||
|
||||
await self.web_open(f"{self.root_url}/m-meine-anzeigen.html")
|
||||
@@ -922,21 +922,21 @@ class KleinanzeigenBot(WebScrapingMixin):
|
||||
if ad_cfg.id == published_ad_id or ad_cfg.title == published_ad_title:
|
||||
LOG.info(" -> deleting %s '%s'...", published_ad_id, published_ad_title)
|
||||
await self.web_request(
|
||||
url=f"{self.root_url}/m-anzeigen-loeschen.json?ids={published_ad_id}", method="POST", headers={"x-csrf-token": str(csrf_token)}
|
||||
url = f"{self.root_url}/m-anzeigen-loeschen.json?ids={published_ad_id}", method = "POST", headers = {"x-csrf-token": str(csrf_token)}
|
||||
)
|
||||
elif ad_cfg.id:
|
||||
await self.web_request(
|
||||
url=f"{self.root_url}/m-anzeigen-loeschen.json?ids={ad_cfg.id}",
|
||||
method="POST",
|
||||
headers={"x-csrf-token": str(csrf_token)},
|
||||
valid_response_codes=[200, 404],
|
||||
url = f"{self.root_url}/m-anzeigen-loeschen.json?ids={ad_cfg.id}",
|
||||
method = "POST",
|
||||
headers = {"x-csrf-token": str(csrf_token)},
|
||||
valid_response_codes = [200, 404],
|
||||
)
|
||||
|
||||
await self.web_sleep()
|
||||
ad_cfg.id = None
|
||||
return True
|
||||
|
||||
async def extend_ads(self, ad_cfgs: list[tuple[str, Ad, dict[str, Any]]]) -> None:
|
||||
async def extend_ads(self, ad_cfgs:list[tuple[str, Ad, dict[str, Any]]]) -> None:
|
||||
"""Extends ads that are close to expiry."""
|
||||
# Fetch currently published ads from API
|
||||
published_ads = json.loads((await self.web_request(f"{self.root_url}/m-meine-anzeigen-verwalten.json?sort=DEFAULT"))["content"])["ads"]
|
||||
@@ -986,7 +986,7 @@ class KleinanzeigenBot(WebScrapingMixin):
|
||||
|
||||
# Process extensions
|
||||
success_count = 0
|
||||
for idx, (ad_file, ad_cfg, ad_cfg_orig, _published_ad) in enumerate(ads_to_extend, start=1):
|
||||
for idx, (ad_file, ad_cfg, ad_cfg_orig, _published_ad) in enumerate(ads_to_extend, start = 1):
|
||||
LOG.info(_("Processing %s/%s: '%s' from [%s]..."), idx, len(ads_to_extend), ad_cfg.title, ad_file)
|
||||
if await self.extend_ad(ad_file, ad_cfg, ad_cfg_orig):
|
||||
success_count += 1
|
||||
@@ -996,7 +996,7 @@ class KleinanzeigenBot(WebScrapingMixin):
|
||||
LOG.info(_("DONE: Extended %s"), pluralize("ad", success_count))
|
||||
LOG.info("############################################")
|
||||
|
||||
async def extend_ad(self, ad_file: str, ad_cfg: Ad, ad_cfg_orig: dict[str, Any]) -> bool:
|
||||
async def extend_ad(self, ad_file:str, ad_cfg:Ad, ad_cfg_orig:dict[str, Any]) -> bool:
|
||||
"""Extends a single ad listing."""
|
||||
LOG.info(_("Extending ad '%s' (ID: %s)..."), ad_cfg.title, ad_cfg.id)
|
||||
|
||||
@@ -1021,14 +1021,14 @@ class KleinanzeigenBot(WebScrapingMixin):
|
||||
# Simply close the dialog with the X button (aria-label="Schließen")
|
||||
try:
|
||||
dialog_close_timeout = self._timeout("quick_dom")
|
||||
await self.web_click(By.CSS_SELECTOR, 'button[aria-label="Schließen"]', timeout=dialog_close_timeout)
|
||||
await self.web_click(By.CSS_SELECTOR, 'button[aria-label="Schließen"]', timeout = dialog_close_timeout)
|
||||
LOG.debug(" -> Closed confirmation dialog")
|
||||
except TimeoutError:
|
||||
LOG.warning(_(" -> No confirmation dialog found, extension may have completed directly"))
|
||||
|
||||
# Update metadata in YAML file
|
||||
# Update updated_on to track when ad was extended
|
||||
ad_cfg_orig["updated_on"] = misc.now().isoformat(timespec="seconds")
|
||||
ad_cfg_orig["updated_on"] = misc.now().isoformat(timespec = "seconds")
|
||||
dicts.save_dict(ad_file, ad_cfg_orig)
|
||||
|
||||
LOG.info(_(" -> SUCCESS: ad extended with ID %s"), ad_cfg.id)
|
||||
@@ -1045,7 +1045,7 @@ class KleinanzeigenBot(WebScrapingMixin):
|
||||
# Check for success messages
|
||||
return await self.web_check(By.ID, "checking-done", Is.DISPLAYED) or await self.web_check(By.ID, "not-completed", Is.DISPLAYED)
|
||||
|
||||
async def publish_ads(self, ad_cfgs: list[tuple[str, Ad, dict[str, Any]]]) -> None:
|
||||
async def publish_ads(self, ad_cfgs:list[tuple[str, Ad, dict[str, Any]]]) -> None:
|
||||
count = 0
|
||||
failed_count = 0
|
||||
max_retries = 3
|
||||
@@ -1082,12 +1082,12 @@ class KleinanzeigenBot(WebScrapingMixin):
|
||||
if success:
|
||||
try:
|
||||
publish_timeout = self._timeout("publishing_result")
|
||||
await self.web_await(self.__check_publishing_result, timeout=publish_timeout)
|
||||
await self.web_await(self.__check_publishing_result, timeout = publish_timeout)
|
||||
except TimeoutError:
|
||||
LOG.warning(_(" -> Could not confirm publishing for '%s', but ad may be online"), ad_cfg.title)
|
||||
|
||||
if success and self.config.publishing.delete_old_ads == "AFTER_PUBLISH" and not self.keep_old_ads:
|
||||
await self.delete_ad(ad_cfg, published_ads, delete_old_ads_by_title=False)
|
||||
await self.delete_ad(ad_cfg, published_ads, delete_old_ads_by_title = False)
|
||||
|
||||
LOG.info("############################################")
|
||||
if failed_count > 0:
|
||||
@@ -1097,7 +1097,7 @@ class KleinanzeigenBot(WebScrapingMixin):
|
||||
LOG.info("############################################")
|
||||
|
||||
async def publish_ad(
|
||||
self, ad_file: str, ad_cfg: Ad, ad_cfg_orig: dict[str, Any], published_ads: list[dict[str, Any]], mode: AdUpdateStrategy = AdUpdateStrategy.REPLACE
|
||||
self, ad_file:str, ad_cfg:Ad, ad_cfg_orig:dict[str, Any], published_ads:list[dict[str, Any]], mode:AdUpdateStrategy = AdUpdateStrategy.REPLACE
|
||||
) -> None:
|
||||
"""
|
||||
@param ad_cfg: the effective ad config (i.e. with default values applied etc.)
|
||||
@@ -1108,7 +1108,7 @@ class KleinanzeigenBot(WebScrapingMixin):
|
||||
|
||||
if mode == AdUpdateStrategy.REPLACE:
|
||||
if self.config.publishing.delete_old_ads == "BEFORE_PUBLISH" and not self.keep_old_ads:
|
||||
await self.delete_ad(ad_cfg, published_ads, delete_old_ads_by_title=self.config.publishing.delete_old_ads_by_title)
|
||||
await self.delete_ad(ad_cfg, published_ads, delete_old_ads_by_title = self.config.publishing.delete_old_ads_by_title)
|
||||
|
||||
# Apply auto price reduction only for REPLACE operations (actual reposts)
|
||||
# This ensures price reductions only happen on republish, not on UPDATE
|
||||
@@ -1197,12 +1197,12 @@ class KleinanzeigenBot(WebScrapingMixin):
|
||||
elif not await self.web_check(By.ID, "radio-buy-now-no", Is.SELECTED):
|
||||
await self.web_click(By.ID, "radio-buy-now-no")
|
||||
except TimeoutError as ex:
|
||||
LOG.debug(ex, exc_info=True)
|
||||
LOG.debug(ex, exc_info = True)
|
||||
|
||||
#############################
|
||||
# set description
|
||||
#############################
|
||||
description = self.__get_description(ad_cfg, with_affixes=True)
|
||||
description = self.__get_description(ad_cfg, with_affixes = True)
|
||||
await self.web_execute("document.querySelector('#pstad-descrptn').value = `" + description.replace("`", "'") + "`")
|
||||
|
||||
await self.__set_contact_fields(ad_cfg.contact)
|
||||
@@ -1213,7 +1213,7 @@ class KleinanzeigenBot(WebScrapingMixin):
|
||||
#############################
|
||||
img_items = await self.web_find_all(By.CSS_SELECTOR, "ul#j-pictureupload-thumbnails > li:not(.is-placeholder)")
|
||||
for element in img_items:
|
||||
btn = await self.web_find(By.CSS_SELECTOR, "button.pictureupload-thumbnails-remove", parent=element)
|
||||
btn = await self.web_find(By.CSS_SELECTOR, "button.pictureupload-thumbnails-remove", parent = element)
|
||||
await btn.click()
|
||||
|
||||
#############################
|
||||
@@ -1224,7 +1224,7 @@ class KleinanzeigenBot(WebScrapingMixin):
|
||||
#############################
|
||||
# wait for captcha
|
||||
#############################
|
||||
await self.check_and_wait_for_captcha(is_login_page=False)
|
||||
await self.check_and_wait_for_captcha(is_login_page = False)
|
||||
|
||||
#############################
|
||||
# submit
|
||||
@@ -1250,7 +1250,7 @@ class KleinanzeigenBot(WebScrapingMixin):
|
||||
#############################
|
||||
try:
|
||||
short_timeout = self._timeout("quick_dom")
|
||||
await self.web_find(By.ID, "myftr-shppngcrt-frm", timeout=short_timeout)
|
||||
await self.web_find(By.ID, "myftr-shppngcrt-frm", timeout = short_timeout)
|
||||
|
||||
LOG.warning("############################################")
|
||||
LOG.warning("# Payment form detected! Please proceed with payment.")
|
||||
@@ -1262,7 +1262,7 @@ class KleinanzeigenBot(WebScrapingMixin):
|
||||
pass
|
||||
|
||||
confirmation_timeout = self._timeout("publishing_confirmation")
|
||||
await self.web_await(lambda: "p-anzeige-aufgeben-bestaetigung.html?adId=" in self.page.url, timeout=confirmation_timeout)
|
||||
await self.web_await(lambda: "p-anzeige-aufgeben-bestaetigung.html?adId=" in self.page.url, timeout = confirmation_timeout)
|
||||
|
||||
# extract the ad id from the URL's query parameter
|
||||
current_url_query_params = urllib_parse.parse_qs(urllib_parse.urlparse(self.page.url).query)
|
||||
@@ -1272,7 +1272,7 @@ class KleinanzeigenBot(WebScrapingMixin):
|
||||
# Update content hash after successful publication
|
||||
# Calculate hash on original config to ensure consistent comparison on restart
|
||||
ad_cfg_orig["content_hash"] = AdPartial.model_validate(ad_cfg_orig).update_content_hash().content_hash
|
||||
ad_cfg_orig["updated_on"] = misc.now().isoformat(timespec="seconds")
|
||||
ad_cfg_orig["updated_on"] = misc.now().isoformat(timespec = "seconds")
|
||||
if not ad_cfg.created_on and not ad_cfg.id:
|
||||
ad_cfg_orig["created_on"] = ad_cfg_orig["updated_on"]
|
||||
|
||||
@@ -1299,7 +1299,7 @@ class KleinanzeigenBot(WebScrapingMixin):
|
||||
|
||||
dicts.save_dict(ad_file, ad_cfg_orig)
|
||||
|
||||
async def __set_contact_fields(self, contact: Contact) -> None:
|
||||
async def __set_contact_fields(self, contact:Contact) -> None:
|
||||
#############################
|
||||
# set contact zipcode
|
||||
#############################
|
||||
@@ -1384,7 +1384,7 @@ class KleinanzeigenBot(WebScrapingMixin):
|
||||
)
|
||||
)
|
||||
|
||||
async def update_ads(self, ad_cfgs: list[tuple[str, Ad, dict[str, Any]]]) -> None:
|
||||
async def update_ads(self, ad_cfgs:list[tuple[str, Ad, dict[str, Any]]]) -> None:
|
||||
"""
|
||||
Updates a list of ads.
|
||||
The list gets filtered, so that only already published ads will be updated.
|
||||
@@ -1415,25 +1415,25 @@ class KleinanzeigenBot(WebScrapingMixin):
|
||||
|
||||
await self.publish_ad(ad_file, ad_cfg, ad_cfg_orig, published_ads, AdUpdateStrategy.MODIFY)
|
||||
publish_timeout = self._timeout("publishing_result")
|
||||
await self.web_await(self.__check_publishing_result, timeout=publish_timeout)
|
||||
await self.web_await(self.__check_publishing_result, timeout = publish_timeout)
|
||||
|
||||
LOG.info("############################################")
|
||||
LOG.info("DONE: updated %s", pluralize("ad", count))
|
||||
LOG.info("############################################")
|
||||
|
||||
async def __set_condition(self, condition_value: str) -> None:
|
||||
async def __set_condition(self, condition_value:str) -> None:
|
||||
try:
|
||||
# Open condition dialog
|
||||
await self.web_click(By.XPATH, '//*[@id="j-post-listing-frontend-conditions"]//button[@aria-haspopup="true"]')
|
||||
except TimeoutError:
|
||||
LOG.debug("Unable to open condition dialog and select condition [%s]", condition_value, exc_info=True)
|
||||
LOG.debug("Unable to open condition dialog and select condition [%s]", condition_value, exc_info = True)
|
||||
return
|
||||
|
||||
try:
|
||||
# Click radio button
|
||||
await self.web_click(By.ID, f"radio-button-{condition_value}")
|
||||
except TimeoutError:
|
||||
LOG.debug("Unable to select condition [%s]", condition_value, exc_info=True)
|
||||
LOG.debug("Unable to select condition [%s]", condition_value, exc_info = True)
|
||||
|
||||
try:
|
||||
# Click accept button
|
||||
@@ -1441,7 +1441,7 @@ class KleinanzeigenBot(WebScrapingMixin):
|
||||
except TimeoutError as ex:
|
||||
raise TimeoutError(_("Unable to close condition dialog!")) from ex
|
||||
|
||||
async def __set_category(self, category: str | None, ad_file: str) -> None:
|
||||
async def __set_category(self, category:str | None, ad_file:str) -> None:
|
||||
# click on something to trigger automatic category detection
|
||||
await self.web_click(By.ID, "pstad-descrptn")
|
||||
|
||||
@@ -1464,7 +1464,7 @@ class KleinanzeigenBot(WebScrapingMixin):
|
||||
else:
|
||||
ensure(is_category_auto_selected, f"No category specified in [{ad_file}] and automatic category detection failed")
|
||||
|
||||
async def __set_special_attributes(self, ad_cfg: Ad) -> None:
|
||||
async def __set_special_attributes(self, ad_cfg:Ad) -> None:
|
||||
if not ad_cfg.special_attributes:
|
||||
return
|
||||
|
||||
@@ -1499,7 +1499,7 @@ class KleinanzeigenBot(WebScrapingMixin):
|
||||
raise TimeoutError(_("Failed to set attribute '%s'") % special_attribute_key) from ex
|
||||
|
||||
try:
|
||||
elem_id: str = str(special_attr_elem.attrs.id)
|
||||
elem_id:str = str(special_attr_elem.attrs.id)
|
||||
if special_attr_elem.local_name == "select":
|
||||
LOG.debug(_("Attribute field '%s' seems to be a select..."), special_attribute_key)
|
||||
await self.web_select(By.ID, elem_id, special_attribute_value_str)
|
||||
@@ -1517,26 +1517,26 @@ class KleinanzeigenBot(WebScrapingMixin):
|
||||
raise TimeoutError(_("Failed to set attribute '%s'") % special_attribute_key) from ex
|
||||
LOG.debug("Successfully set attribute field [%s] to [%s]...", special_attribute_key, special_attribute_value_str)
|
||||
|
||||
async def __set_shipping(self, ad_cfg: Ad, mode: AdUpdateStrategy = AdUpdateStrategy.REPLACE) -> None:
|
||||
async def __set_shipping(self, ad_cfg:Ad, mode:AdUpdateStrategy = AdUpdateStrategy.REPLACE) -> None:
|
||||
short_timeout = self._timeout("quick_dom")
|
||||
if ad_cfg.shipping_type == "PICKUP":
|
||||
try:
|
||||
await self.web_click(By.ID, "radio-pickup")
|
||||
except TimeoutError as ex:
|
||||
LOG.debug(ex, exc_info=True)
|
||||
LOG.debug(ex, exc_info = True)
|
||||
elif ad_cfg.shipping_options:
|
||||
await self.web_click(By.XPATH, '//button//span[contains(., "Versandmethoden auswählen")]')
|
||||
|
||||
if mode == AdUpdateStrategy.MODIFY:
|
||||
try:
|
||||
# when "Andere Versandmethoden" is not available, go back and start over new
|
||||
await self.web_find(By.XPATH, '//dialog//button[contains(., "Andere Versandmethoden")]', timeout=short_timeout)
|
||||
await self.web_find(By.XPATH, '//dialog//button[contains(., "Andere Versandmethoden")]', timeout = short_timeout)
|
||||
except TimeoutError:
|
||||
await self.web_click(By.XPATH, '//dialog//button[contains(., "Zurück")]')
|
||||
|
||||
# in some categories we need to go another dialog back
|
||||
try:
|
||||
await self.web_find(By.XPATH, '//dialog//button[contains(., "Andere Versandmethoden")]', timeout=short_timeout)
|
||||
await self.web_find(By.XPATH, '//dialog//button[contains(., "Andere Versandmethoden")]', timeout = short_timeout)
|
||||
except TimeoutError:
|
||||
await self.web_click(By.XPATH, '//dialog//button[contains(., "Zurück")]')
|
||||
|
||||
@@ -1562,7 +1562,7 @@ class KleinanzeigenBot(WebScrapingMixin):
|
||||
try:
|
||||
# only click on "Individueller Versand" when "IndividualShippingInput" is not available, otherwise its already checked
|
||||
# (important for mode = UPDATE)
|
||||
await self.web_find(By.XPATH, '//input[contains(@placeholder, "Versandkosten (optional)")]', timeout=short_timeout)
|
||||
await self.web_find(By.XPATH, '//input[contains(@placeholder, "Versandkosten (optional)")]', timeout = short_timeout)
|
||||
except TimeoutError:
|
||||
# Input not visible yet; click the individual shipping option.
|
||||
await self.web_click(By.XPATH, '//*[contains(@id, "INDIVIDUAL") and contains(@data-testid, "Individueller Versand")]')
|
||||
@@ -1573,10 +1573,10 @@ class KleinanzeigenBot(WebScrapingMixin):
|
||||
)
|
||||
await self.web_click(By.XPATH, '//dialog//button[contains(., "Fertig")]')
|
||||
except TimeoutError as ex:
|
||||
LOG.debug(ex, exc_info=True)
|
||||
LOG.debug(ex, exc_info = True)
|
||||
raise TimeoutError(_("Unable to close shipping dialog!")) from ex
|
||||
|
||||
async def __set_shipping_options(self, ad_cfg: Ad, mode: AdUpdateStrategy = AdUpdateStrategy.REPLACE) -> None:
|
||||
async def __set_shipping_options(self, ad_cfg:Ad, mode:AdUpdateStrategy = AdUpdateStrategy.REPLACE) -> None:
|
||||
if not ad_cfg.shipping_options:
|
||||
return
|
||||
|
||||
@@ -1596,7 +1596,7 @@ class KleinanzeigenBot(WebScrapingMixin):
|
||||
except KeyError as ex:
|
||||
raise KeyError(f"Unknown shipping option(s), please refer to the documentation/README: {ad_cfg.shipping_options}") from ex
|
||||
|
||||
shipping_sizes, shipping_selector, shipping_packages = zip(*mapped_shipping_options, strict=False)
|
||||
shipping_sizes, shipping_selector, shipping_packages = zip(*mapped_shipping_options, strict = False)
|
||||
|
||||
try:
|
||||
(shipping_size,) = set(shipping_sizes)
|
||||
@@ -1652,19 +1652,19 @@ class KleinanzeigenBot(WebScrapingMixin):
|
||||
for shipping_package in to_be_clicked_shipping_packages:
|
||||
await self.web_click(By.XPATH, f'//dialog//input[contains(@data-testid, "{shipping_package}")]')
|
||||
except TimeoutError as ex:
|
||||
LOG.debug(ex, exc_info=True)
|
||||
LOG.debug(ex, exc_info = True)
|
||||
try:
|
||||
# Click apply button
|
||||
await self.web_click(By.XPATH, '//dialog//button[contains(., "Fertig")]')
|
||||
except TimeoutError as ex:
|
||||
raise TimeoutError(_("Unable to close shipping dialog!")) from ex
|
||||
|
||||
async def __upload_images(self, ad_cfg: Ad) -> None:
|
||||
async def __upload_images(self, ad_cfg:Ad) -> None:
|
||||
if not ad_cfg.images:
|
||||
return
|
||||
|
||||
LOG.info(" -> found %s", pluralize("image", ad_cfg.images))
|
||||
image_upload: Element = await self.web_find(By.CSS_SELECTOR, "input[type=file]")
|
||||
image_upload:Element = await self.web_find(By.CSS_SELECTOR, "input[type=file]")
|
||||
|
||||
for image in ad_cfg.images:
|
||||
LOG.info(" -> uploading image [%s]", image)
|
||||
@@ -1680,7 +1680,7 @@ class KleinanzeigenBot(WebScrapingMixin):
|
||||
thumbnails = await self.web_find_all(
|
||||
By.CSS_SELECTOR,
|
||||
"ul#j-pictureupload-thumbnails > li:not(.is-placeholder)",
|
||||
timeout=self._timeout("quick_dom"), # Fast timeout for polling
|
||||
timeout = self._timeout("quick_dom"), # Fast timeout for polling
|
||||
)
|
||||
current_count = len(thumbnails)
|
||||
if current_count < expected_count:
|
||||
@@ -1691,12 +1691,12 @@ class KleinanzeigenBot(WebScrapingMixin):
|
||||
return False
|
||||
|
||||
try:
|
||||
await self.web_await(check_thumbnails_uploaded, timeout=self._timeout("image_upload"), timeout_error_message=_("Image upload timeout exceeded"))
|
||||
await self.web_await(check_thumbnails_uploaded, timeout = self._timeout("image_upload"), timeout_error_message = _("Image upload timeout exceeded"))
|
||||
except TimeoutError as ex:
|
||||
# Get current count for better error message
|
||||
try:
|
||||
thumbnails = await self.web_find_all(
|
||||
By.CSS_SELECTOR, "ul#j-pictureupload-thumbnails > li:not(.is-placeholder)", timeout=self._timeout("quick_dom")
|
||||
By.CSS_SELECTOR, "ul#j-pictureupload-thumbnails > li:not(.is-placeholder)", timeout = self._timeout("quick_dom")
|
||||
)
|
||||
current_count = len(thumbnails)
|
||||
except TimeoutError:
|
||||
@@ -1738,7 +1738,7 @@ class KleinanzeigenBot(WebScrapingMixin):
|
||||
elif self.ads_selector == "new": # download only unsaved ads
|
||||
# check which ads already saved
|
||||
saved_ad_ids = []
|
||||
ads = self.load_ads(ignore_inactive=False, exclude_ads_with_id=False) # do not skip because of existing IDs
|
||||
ads = self.load_ads(ignore_inactive = False, exclude_ads_with_id = False) # do not skip because of existing IDs
|
||||
for ad in ads:
|
||||
saved_ad_id = ad[1].id
|
||||
if saved_ad_id is None:
|
||||
@@ -1775,7 +1775,7 @@ class KleinanzeigenBot(WebScrapingMixin):
|
||||
else:
|
||||
LOG.error("The page with the id %d does not exist!", ad_id)
|
||||
|
||||
def __get_description(self, ad_cfg: Ad, *, with_affixes: bool) -> str:
|
||||
def __get_description(self, ad_cfg:Ad, *, with_affixes:bool) -> str:
|
||||
"""Get the ad description optionally with prefix and suffix applied.
|
||||
|
||||
Precedence (highest to lowest):
|
||||
@@ -1827,7 +1827,7 @@ class KleinanzeigenBot(WebScrapingMixin):
|
||||
|
||||
return final_description
|
||||
|
||||
def update_content_hashes(self, ads: list[tuple[str, Ad, dict[str, Any]]]) -> None:
|
||||
def update_content_hashes(self, ads:list[tuple[str, Ad, dict[str, Any]]]) -> None:
|
||||
count = 0
|
||||
|
||||
for ad_file, ad_cfg, ad_cfg_orig in ads:
|
||||
@@ -1848,7 +1848,7 @@ class KleinanzeigenBot(WebScrapingMixin):
|
||||
#############################
|
||||
|
||||
|
||||
def main(args: list[str]) -> None:
|
||||
def main(args:list[str]) -> None:
|
||||
if "version" not in args:
|
||||
print(
|
||||
textwrap.dedent(rf"""
|
||||
@@ -1861,7 +1861,7 @@ def main(args: list[str]) -> None:
|
||||
https://github.com/Second-Hand-Friends/kleinanzeigen-bot
|
||||
Version: {__version__}
|
||||
""")[1:],
|
||||
flush=True,
|
||||
flush = True,
|
||||
) # [1:] removes the first empty blank line
|
||||
|
||||
loggers.configure_console_logging()
|
||||
|
||||
Reference in New Issue
Block a user