feat: add browser profile XDG support and documentation (#777)

This commit is contained in:
Jens
2026-01-23 22:45:22 +01:00
committed by GitHub
parent dc0d9404bf
commit eda1b4d0ec
15 changed files with 841 additions and 687 deletions

View File

@@ -172,10 +172,11 @@ jobs:
set -eux set -eux
case "${{ matrix.os }}" in case "${{ matrix.os }}" in
ubuntu-*) ubuntu-*)
sudo apt-get install --no-install-recommends -y xvfb sudo apt-get install --no-install-recommends -y xvfb
xvfb-run pdm run itest:cov -vv # Run tests INSIDE xvfb context
;; xvfb-run bash -c 'pdm run itest:cov -vv'
;;
*) pdm run itest:cov -vv *) pdm run itest:cov -vv
;; ;;
esac esac

View File

@@ -248,6 +248,25 @@ Limitation of `download`: It's only possible to extract the cheapest given shipp
All configuration files can be in YAML or JSON format. All configuration files can be in YAML or JSON format.
### Installation modes (portable vs. system-wide)
On first run, the app may ask which installation mode to use. In non-interactive environments (CI/headless), it defaults to portable mode and will not prompt; `--config` and `--logfile` override only their specific paths, and do not change other mode-dependent paths or the chosen installation mode behavior.
1. **Portable mode (recommended for most users, especially on Windows):**
- Stores config, logs, downloads, and state in the current directory
- No admin permissions required
- Easy backup/migration; works from USB drives
2. **System-wide mode (advanced users / multi-user setups):**
- Stores files in OS-standard locations
- Cleaner directory structure; better separation from working directory
- Requires proper permissions for user data directories
**OS notes (brief):**
- **Windows:** System-wide uses AppData (Roaming/Local); portable keeps everything beside the `.exe`.
- **Linux:** System-wide follows XDG Base Directory spec; portable stays in the current working directory.
- **macOS:** System-wide uses `~/Library/Application Support/kleinanzeigen-bot` (and related dirs); portable stays in the current directory.
### <a name="main-config"></a>1) Main configuration ### <a name="main-config"></a>1) Main configuration
When executing the app it by default looks for a `config.yaml` file in the current directory. If it does not exist it will be created automatically. When executing the app it by default looks for a `config.yaml` file in the current directory. If it does not exist it will be created automatically.

View File

@@ -111,7 +111,8 @@ lint = { composite = ["lint:ruff", "lint:mypy", "lint:pyright"] }
# Run unit tests only (exclude smoke and itest) # Run unit tests only (exclude smoke and itest)
utest = "python -m pytest --capture=tee-sys -m \"not itest and not smoke\"" utest = "python -m pytest --capture=tee-sys -m \"not itest and not smoke\""
# Run integration tests only (exclude smoke) # Run integration tests only (exclude smoke)
itest = "python -m pytest --capture=tee-sys -m \"itest and not smoke\"" # Uses -n 0 to disable xdist parallelization - browser tests are flaky with parallel workers
itest = "python -m pytest --capture=tee-sys -m \"itest and not smoke\" -n 0"
# Run smoke tests only # Run smoke tests only
smoke = "python -m pytest --capture=tee-sys -m smoke" smoke = "python -m pytest --capture=tee-sys -m smoke"
# Run all tests in order: unit, integration, smoke # Run all tests in order: unit, integration, smoke
@@ -126,7 +127,7 @@ test = { composite = ["utest", "itest", "smoke"] }
"coverage:prepare" = { shell = "python scripts/coverage_helper.py prepare" } "coverage:prepare" = { shell = "python scripts/coverage_helper.py prepare" }
"test:cov" = { composite = ["coverage:prepare", "utest:cov", "itest:cov", "smoke:cov", "coverage:combine"] } "test:cov" = { composite = ["coverage:prepare", "utest:cov", "itest:cov", "smoke:cov", "coverage:combine"] }
"utest:cov" = { shell = "python scripts/coverage_helper.py run .temp/.coverage-unit.sqlite .temp/coverage-unit.xml \"not itest and not smoke\"" } "utest:cov" = { shell = "python scripts/coverage_helper.py run .temp/.coverage-unit.sqlite .temp/coverage-unit.xml \"not itest and not smoke\"" }
"itest:cov" = { shell = "python scripts/coverage_helper.py run .temp/.coverage-itest.sqlite .temp/coverage-integration.xml \"itest and not smoke\"" } "itest:cov" = { shell = "python scripts/coverage_helper.py run .temp/.coverage-itest.sqlite .temp/coverage-integration.xml \"itest and not smoke\" -n 0" }
"smoke:cov" = { shell = "python scripts/coverage_helper.py run .temp/.coverage-smoke.sqlite .temp/coverage-smoke.xml smoke" } "smoke:cov" = { shell = "python scripts/coverage_helper.py run .temp/.coverage-smoke.sqlite .temp/coverage-smoke.xml smoke" }
"coverage:combine" = { shell = "python scripts/coverage_helper.py combine .temp/.coverage-unit.sqlite .temp/.coverage-itest.sqlite .temp/.coverage-smoke.sqlite" } "coverage:combine" = { shell = "python scripts/coverage_helper.py combine .temp/.coverage-unit.sqlite .temp/.coverage-itest.sqlite .temp/.coverage-smoke.sqlite" }
# Run all tests with coverage in a single invocation # Run all tests with coverage in a single invocation

View File

@@ -185,7 +185,7 @@
"BrowserConfig": { "BrowserConfig": {
"properties": { "properties": {
"arguments": { "arguments": {
"description": "See https://peter.sh/experiments/chromium-command-line-switches/", "description": "See https://peter.sh/experiments/chromium-command-line-switches/. Browser profile path is auto-configured based on installation mode (portable/XDG).",
"items": { "items": {
"type": "string" "type": "string"
}, },
@@ -227,8 +227,8 @@
"type": "null" "type": "null"
} }
], ],
"default": ".temp/browser-profile", "default": null,
"description": "See https://github.com/chromium/chromium/blob/main/docs/user_data_dir.md", "description": "See https://github.com/chromium/chromium/blob/main/docs/user_data_dir.md. If not specified, defaults to XDG cache directory in XDG mode or .temp/browser-profile in portable mode.",
"title": "User Data Dir" "title": "User Data Dir"
}, },
"profile_name": { "profile_name": {

View File

@@ -28,7 +28,7 @@ from .utils.web_scraping_mixin import By, Element, Is, WebScrapingMixin
# W0406: possibly a bug, see https://github.com/PyCQA/pylint/issues/3933 # W0406: possibly a bug, see https://github.com/PyCQA/pylint/issues/3933
LOG: Final[loggers.Logger] = loggers.get_logger(__name__) LOG:Final[loggers.Logger] = loggers.get_logger(__name__)
LOG.setLevel(loggers.INFO) LOG.setLevel(loggers.INFO)
colorama.just_fix_windows_console() colorama.just_fix_windows_console()
@@ -39,7 +39,7 @@ class AdUpdateStrategy(enum.Enum):
MODIFY = enum.auto() MODIFY = enum.auto()
def _repost_cycle_ready(ad_cfg: Ad, ad_file_relative: str) -> bool: def _repost_cycle_ready(ad_cfg:Ad, ad_file_relative:str) -> bool:
""" """
Check if the repost cycle delay has been satisfied. Check if the repost cycle delay has been satisfied.
@@ -72,7 +72,7 @@ def _repost_cycle_ready(ad_cfg: Ad, ad_file_relative: str) -> bool:
return True return True
def _day_delay_elapsed(ad_cfg: Ad, ad_file_relative: str) -> bool: def _day_delay_elapsed(ad_cfg:Ad, ad_file_relative:str) -> bool:
""" """
Check if the day delay has elapsed since the ad was last published. Check if the day delay has elapsed since the ad was last published.
@@ -100,7 +100,7 @@ def _day_delay_elapsed(ad_cfg: Ad, ad_file_relative: str) -> bool:
return True return True
def apply_auto_price_reduction(ad_cfg: Ad, _ad_cfg_orig: dict[str, Any], ad_file_relative: str) -> None: def apply_auto_price_reduction(ad_cfg:Ad, _ad_cfg_orig:dict[str, Any], ad_file_relative:str) -> None:
""" """
Apply automatic price reduction to an ad based on repost count and configuration. Apply automatic price reduction to an ad based on repost count and configuration.
@@ -132,7 +132,7 @@ def apply_auto_price_reduction(ad_cfg: Ad, _ad_cfg_orig: dict[str, Any], ad_file
applied_cycles = ad_cfg.price_reduction_count or 0 applied_cycles = ad_cfg.price_reduction_count or 0
next_cycle = applied_cycles + 1 next_cycle = applied_cycles + 1
effective_price = calculate_auto_price(base_price=base_price, auto_price_reduction=ad_cfg.auto_price_reduction, target_reduction_cycle=next_cycle) effective_price = calculate_auto_price(base_price = base_price, auto_price_reduction = ad_cfg.auto_price_reduction, target_reduction_cycle = next_cycle)
if effective_price is None: if effective_price is None:
return return
@@ -149,7 +149,7 @@ def apply_auto_price_reduction(ad_cfg: Ad, _ad_cfg_orig: dict[str, Any], ad_file
# Note: price_reduction_count is persisted to ad_cfg_orig only after successful publish # Note: price_reduction_count is persisted to ad_cfg_orig only after successful publish
class KleinanzeigenBot(WebScrapingMixin): class KleinanzeigenBot(WebScrapingMixin): # noqa: PLR0904
def __init__(self) -> None: def __init__(self) -> None:
# workaround for https://github.com/Second-Hand-Friends/kleinanzeigen-bot/issues/295 # workaround for https://github.com/Second-Hand-Friends/kleinanzeigen-bot/issues/295
# see https://github.com/pyinstaller/pyinstaller/issues/7229#issuecomment-1309383026 # see https://github.com/pyinstaller/pyinstaller/issues/7229#issuecomment-1309383026
@@ -159,17 +159,17 @@ class KleinanzeigenBot(WebScrapingMixin):
self.root_url = "https://www.kleinanzeigen.de" self.root_url = "https://www.kleinanzeigen.de"
self.config: Config self.config:Config
self.config_file_path = abspath("config.yaml") self.config_file_path = abspath("config.yaml")
self.config_explicitly_provided = False self.config_explicitly_provided = False
self.installation_mode: xdg_paths.InstallationMode | None = None self.installation_mode:xdg_paths.InstallationMode | None = None
self.categories: dict[str, str] = {} self.categories:dict[str, str] = {}
self.file_log: loggers.LogFileHandle | None = None self.file_log:loggers.LogFileHandle | None = None
log_file_basename = is_frozen() and os.path.splitext(os.path.basename(sys.executable))[0] or self.__module__ log_file_basename = is_frozen() and os.path.splitext(os.path.basename(sys.executable))[0] or self.__module__
self.log_file_path: str | None = abspath(f"{log_file_basename}.log") self.log_file_path:str | None = abspath(f"{log_file_basename}.log")
self.log_file_basename = log_file_basename self.log_file_basename = log_file_basename
self.log_file_explicitly_provided = False self.log_file_explicitly_provided = False
@@ -245,7 +245,7 @@ class KleinanzeigenBot(WebScrapingMixin):
LOG.info(_("Installation mode: %s"), mode_display) LOG.info(_("Installation mode: %s"), mode_display)
LOG.info(_("Config file: %s"), self.config_file_path) LOG.info(_("Config file: %s"), self.config_file_path)
async def run(self, args: list[str]) -> None: async def run(self, args:list[str]) -> None:
self.parse_args(args) self.parse_args(args)
self.finalize_installation_mode() self.finalize_installation_mode()
try: try:
@@ -277,7 +277,7 @@ class KleinanzeigenBot(WebScrapingMixin):
self.configure_file_logging() self.configure_file_logging()
self.load_config() self.load_config()
checker = UpdateChecker(self.config, self.installation_mode_or_portable) checker = UpdateChecker(self.config, self.installation_mode_or_portable)
checker.check_for_updates(skip_interval_check=True) checker.check_for_updates(skip_interval_check = True)
case "update-content-hash": case "update-content-hash":
self.configure_file_logging() self.configure_file_logging()
self.load_config() self.load_config()
@@ -285,7 +285,7 @@ class KleinanzeigenBot(WebScrapingMixin):
checker = UpdateChecker(self.config, self.installation_mode_or_portable) checker = UpdateChecker(self.config, self.installation_mode_or_portable)
checker.check_for_updates() checker.check_for_updates()
self.ads_selector = "all" self.ads_selector = "all"
if ads := self.load_ads(exclude_ads_with_id=False): if ads := self.load_ads(exclude_ads_with_id = False):
self.update_content_hashes(ads) self.update_content_hashes(ads)
else: else:
LOG.info("############################################") LOG.info("############################################")
@@ -503,7 +503,7 @@ class KleinanzeigenBot(WebScrapingMixin):
) )
) )
def parse_args(self, args: list[str]) -> None: def parse_args(self, args:list[str]) -> None:
try: try:
options, arguments = getopt.gnu_getopt(args[1:], "hv", ["ads=", "config=", "force", "help", "keep-old", "logfile=", "lang=", "verbose"]) options, arguments = getopt.gnu_getopt(args[1:], "hv", ["ads=", "config=", "force", "help", "keep-old", "logfile=", "lang=", "verbose"])
except getopt.error as ex: except getopt.error as ex:
@@ -571,7 +571,7 @@ class KleinanzeigenBot(WebScrapingMixin):
default_config.login.password = "changeme" # noqa: S105 placeholder for default config, not a real password default_config.login.password = "changeme" # noqa: S105 placeholder for default config, not a real password
dicts.save_dict( dicts.save_dict(
self.config_file_path, self.config_file_path,
default_config.model_dump(exclude_none=True, exclude={"ad_defaults": {"description"}}), default_config.model_dump(exclude_none = True, exclude = {"ad_defaults": {"description"}}),
header=( header=(
"# yaml-language-server: $schema=" "# yaml-language-server: $schema="
"https://raw.githubusercontent.com/Second-Hand-Friends/kleinanzeigen-bot" "https://raw.githubusercontent.com/Second-Hand-Friends/kleinanzeigen-bot"
@@ -585,7 +585,7 @@ class KleinanzeigenBot(WebScrapingMixin):
self.create_default_config() self.create_default_config()
config_yaml = dicts.load_dict_if_exists(self.config_file_path, _("config")) config_yaml = dicts.load_dict_if_exists(self.config_file_path, _("config"))
self.config = Config.model_validate(config_yaml, strict=True, context=self.config_file_path) self.config = Config.model_validate(config_yaml, strict = True, context = self.config_file_path)
# load built-in category mappings # load built-in category mappings
self.categories = dicts.load_dict_from_module(resources, "categories.yaml", "categories") self.categories = dicts.load_dict_from_module(resources, "categories.yaml", "categories")
@@ -598,13 +598,13 @@ class KleinanzeigenBot(WebScrapingMixin):
# populate browser_config object used by WebScrapingMixin # populate browser_config object used by WebScrapingMixin
self.browser_config.arguments = self.config.browser.arguments self.browser_config.arguments = self.config.browser.arguments
self.browser_config.binary_location = self.config.browser.binary_location self.browser_config.binary_location = self.config.browser.binary_location
self.browser_config.extensions = [abspath(item, relative_to=self.config_file_path) for item in self.config.browser.extensions] self.browser_config.extensions = [abspath(item, relative_to = self.config_file_path) for item in self.config.browser.extensions]
self.browser_config.use_private_window = self.config.browser.use_private_window self.browser_config.use_private_window = self.config.browser.use_private_window
if self.config.browser.user_data_dir: if self.config.browser.user_data_dir:
self.browser_config.user_data_dir = abspath(self.config.browser.user_data_dir, relative_to=self.config_file_path) self.browser_config.user_data_dir = abspath(self.config.browser.user_data_dir, relative_to = self.config_file_path)
self.browser_config.profile_name = self.config.browser.profile_name self.browser_config.profile_name = self.config.browser.profile_name
def __check_ad_republication(self, ad_cfg: Ad, ad_file_relative: str) -> bool: def __check_ad_republication(self, ad_cfg:Ad, ad_file_relative:str) -> bool:
""" """
Check if an ad needs to be republished based on republication interval. Check if an ad needs to be republished based on republication interval.
Note: This method does not check for content changes. Use __check_ad_changed for that. Note: This method does not check for content changes. Use __check_ad_changed for that.
@@ -635,7 +635,7 @@ class KleinanzeigenBot(WebScrapingMixin):
return True return True
def __check_ad_changed(self, ad_cfg: Ad, ad_cfg_orig: dict[str, Any], ad_file_relative: str) -> bool: def __check_ad_changed(self, ad_cfg:Ad, ad_cfg_orig:dict[str, Any], ad_file_relative:str) -> bool:
""" """
Check if an ad has been changed since last publication. Check if an ad has been changed since last publication.
@@ -662,7 +662,7 @@ class KleinanzeigenBot(WebScrapingMixin):
return False return False
def load_ads(self, *, ignore_inactive: bool = True, exclude_ads_with_id: bool = True) -> list[tuple[str, Ad, dict[str, Any]]]: def load_ads(self, *, ignore_inactive:bool = True, exclude_ads_with_id:bool = True) -> list[tuple[str, Ad, dict[str, Any]]]:
""" """
Load and validate all ad config files, optionally filtering out inactive or already-published ads. Load and validate all ad config files, optionally filtering out inactive or already-published ads.
@@ -678,12 +678,12 @@ class KleinanzeigenBot(WebScrapingMixin):
""" """
LOG.info("Searching for ad config files...") LOG.info("Searching for ad config files...")
ad_files: dict[str, str] = {} ad_files:dict[str, str] = {}
data_root_dir = os.path.dirname(self.config_file_path) data_root_dir = os.path.dirname(self.config_file_path)
for file_pattern in self.config.ad_files: for file_pattern in self.config.ad_files:
for ad_file in glob.glob(file_pattern, root_dir=data_root_dir, flags=glob.GLOBSTAR | glob.BRACE | glob.EXTGLOB): for ad_file in glob.glob(file_pattern, root_dir = data_root_dir, flags = glob.GLOBSTAR | glob.BRACE | glob.EXTGLOB):
if not str(ad_file).endswith("ad_fields.yaml"): if not str(ad_file).endswith("ad_fields.yaml"):
ad_files[abspath(ad_file, relative_to=data_root_dir)] = ad_file ad_files[abspath(ad_file, relative_to = data_root_dir)] = ad_file
LOG.info(" -> found %s", pluralize("ad config file", ad_files)) LOG.info(" -> found %s", pluralize("ad config file", ad_files))
if not ad_files: if not ad_files:
return [] return []
@@ -700,8 +700,8 @@ class KleinanzeigenBot(WebScrapingMixin):
ads = [] ads = []
for ad_file, ad_file_relative in sorted(ad_files.items()): for ad_file, ad_file_relative in sorted(ad_files.items()):
ad_cfg_orig: dict[str, Any] = dicts.load_dict(ad_file, "ad") ad_cfg_orig:dict[str, Any] = dicts.load_dict(ad_file, "ad")
ad_cfg: Ad = self.load_ad(ad_cfg_orig) ad_cfg:Ad = self.load_ad(ad_cfg_orig)
if ignore_inactive and not ad_cfg.active: if ignore_inactive and not ad_cfg.active:
LOG.info(" -> SKIPPED: inactive ad [%s]", ad_file_relative) LOG.info(" -> SKIPPED: inactive ad [%s]", ad_file_relative)
@@ -738,8 +738,8 @@ class KleinanzeigenBot(WebScrapingMixin):
if not should_include: if not should_include:
continue continue
ensure(self.__get_description(ad_cfg, with_affixes=False), f"-> property [description] not specified @ [{ad_file}]") ensure(self.__get_description(ad_cfg, with_affixes = False), f"-> property [description] not specified @ [{ad_file}]")
self.__get_description(ad_cfg, with_affixes=True) # validates complete description self.__get_description(ad_cfg, with_affixes = True) # validates complete description
if ad_cfg.category: if ad_cfg.category:
resolved_category_id = self.categories.get(ad_cfg.category) resolved_category_id = self.categories.get(ad_cfg.category)
@@ -758,13 +758,13 @@ class KleinanzeigenBot(WebScrapingMixin):
ad_dir = os.path.dirname(ad_file) ad_dir = os.path.dirname(ad_file)
for image_pattern in ad_cfg.images: for image_pattern in ad_cfg.images:
pattern_images = set() pattern_images = set()
for image_file in glob.glob(image_pattern, root_dir=ad_dir, flags=glob.GLOBSTAR | glob.BRACE | glob.EXTGLOB): for image_file in glob.glob(image_pattern, root_dir = ad_dir, flags = glob.GLOBSTAR | glob.BRACE | glob.EXTGLOB):
_, image_file_ext = os.path.splitext(image_file) _, image_file_ext = os.path.splitext(image_file)
ensure(image_file_ext.lower() in {".gif", ".jpg", ".jpeg", ".png"}, f"Unsupported image file type [{image_file}]") ensure(image_file_ext.lower() in {".gif", ".jpg", ".jpeg", ".png"}, f"Unsupported image file type [{image_file}]")
if os.path.isabs(image_file): if os.path.isabs(image_file):
pattern_images.add(image_file) pattern_images.add(image_file)
else: else:
pattern_images.add(abspath(image_file, relative_to=ad_file)) pattern_images.add(abspath(image_file, relative_to = ad_file))
images.extend(sorted(pattern_images)) images.extend(sorted(pattern_images))
ensure(images or not ad_cfg.images, f"No images found for given file patterns {ad_cfg.images} at {ad_dir}") ensure(images or not ad_cfg.images, f"No images found for given file patterns {ad_cfg.images} at {ad_dir}")
ad_cfg.images = list(dict.fromkeys(images)) ad_cfg.images = list(dict.fromkeys(images))
@@ -774,13 +774,13 @@ class KleinanzeigenBot(WebScrapingMixin):
LOG.info("Loaded %s", pluralize("ad", ads)) LOG.info("Loaded %s", pluralize("ad", ads))
return ads return ads
def load_ad(self, ad_cfg_orig: dict[str, Any]) -> Ad: def load_ad(self, ad_cfg_orig:dict[str, Any]) -> Ad:
return AdPartial.model_validate(ad_cfg_orig).to_ad(self.config.ad_defaults) return AdPartial.model_validate(ad_cfg_orig).to_ad(self.config.ad_defaults)
async def check_and_wait_for_captcha(self, *, is_login_page: bool = True) -> None: async def check_and_wait_for_captcha(self, *, is_login_page:bool = True) -> None:
try: try:
captcha_timeout = self._timeout("captcha_detection") captcha_timeout = self._timeout("captcha_detection")
await self.web_find(By.CSS_SELECTOR, "iframe[name^='a-'][src^='https://www.google.com/recaptcha/api2/anchor?']", timeout=captcha_timeout) await self.web_find(By.CSS_SELECTOR, "iframe[name^='a-'][src^='https://www.google.com/recaptcha/api2/anchor?']", timeout = captcha_timeout)
if not is_login_page and self.config.captcha.auto_restart: if not is_login_page and self.config.captcha.auto_restart:
LOG.warning("Captcha recognized - auto-restart enabled, abort run...") LOG.warning("Captcha recognized - auto-restart enabled, abort run...")
@@ -833,14 +833,14 @@ class KleinanzeigenBot(WebScrapingMixin):
await self.web_input(By.ID, "login-password", "") await self.web_input(By.ID, "login-password", "")
await self.web_input(By.ID, "login-password", self.config.login.password) await self.web_input(By.ID, "login-password", self.config.login.password)
await self.check_and_wait_for_captcha(is_login_page=True) await self.check_and_wait_for_captcha(is_login_page = True)
await self.web_click(By.CSS_SELECTOR, "form#login-form button[type='submit']") await self.web_click(By.CSS_SELECTOR, "form#login-form button[type='submit']")
async def handle_after_login_logic(self) -> None: async def handle_after_login_logic(self) -> None:
try: try:
sms_timeout = self._timeout("sms_verification") sms_timeout = self._timeout("sms_verification")
await self.web_find(By.TEXT, "Wir haben dir gerade einen 6-stelligen Code für die Telefonnummer", timeout=sms_timeout) await self.web_find(By.TEXT, "Wir haben dir gerade einen 6-stelligen Code für die Telefonnummer", timeout = sms_timeout)
LOG.warning("############################################") LOG.warning("############################################")
LOG.warning("# Device verification message detected. Please follow the instruction displayed in the Browser.") LOG.warning("# Device verification message detected. Please follow the instruction displayed in the Browser.")
LOG.warning("############################################") LOG.warning("############################################")
@@ -852,10 +852,10 @@ class KleinanzeigenBot(WebScrapingMixin):
try: try:
LOG.info("Handling GDPR disclaimer...") LOG.info("Handling GDPR disclaimer...")
gdpr_timeout = self._timeout("gdpr_prompt") gdpr_timeout = self._timeout("gdpr_prompt")
await self.web_find(By.ID, "gdpr-banner-accept", timeout=gdpr_timeout) await self.web_find(By.ID, "gdpr-banner-accept", timeout = gdpr_timeout)
await self.web_click(By.ID, "gdpr-banner-cmp-button") await self.web_click(By.ID, "gdpr-banner-cmp-button")
await self.web_click( await self.web_click(
By.XPATH, "//div[@id='ConsentManagementPage']//*//button//*[contains(., 'Alle ablehnen und fortfahren')]", timeout=gdpr_timeout By.XPATH, "//div[@id='ConsentManagementPage']//*//button//*[contains(., 'Alle ablehnen und fortfahren')]", timeout = gdpr_timeout
) )
except TimeoutError: except TimeoutError:
# GDPR banner not shown within timeout. # GDPR banner not shown within timeout.
@@ -873,7 +873,7 @@ class KleinanzeigenBot(WebScrapingMixin):
# Try to find the standard element first # Try to find the standard element first
try: try:
user_info = await self.web_text(By.CLASS_NAME, "mr-medium", timeout=login_check_timeout) user_info = await self.web_text(By.CLASS_NAME, "mr-medium", timeout = login_check_timeout)
if username in user_info.lower(): if username in user_info.lower():
LOG.debug(_("Login detected via .mr-medium element")) LOG.debug(_("Login detected via .mr-medium element"))
return True return True
@@ -882,7 +882,7 @@ class KleinanzeigenBot(WebScrapingMixin):
# If standard element not found or didn't contain username, try the alternative # If standard element not found or didn't contain username, try the alternative
try: try:
user_info = await self.web_text(By.ID, "user-email", timeout=login_check_timeout) user_info = await self.web_text(By.ID, "user-email", timeout = login_check_timeout)
if username in user_info.lower(): if username in user_info.lower():
LOG.debug(_("Login detected via #user-email element")) LOG.debug(_("Login detected via #user-email element"))
return True return True
@@ -892,7 +892,7 @@ class KleinanzeigenBot(WebScrapingMixin):
LOG.debug(_("No login detected - neither .mr-medium nor #user-email found with username")) LOG.debug(_("No login detected - neither .mr-medium nor #user-email found with username"))
return False return False
async def delete_ads(self, ad_cfgs: list[tuple[str, Ad, dict[str, Any]]]) -> None: async def delete_ads(self, ad_cfgs:list[tuple[str, Ad, dict[str, Any]]]) -> None:
count = 0 count = 0
published_ads = json.loads((await self.web_request(f"{self.root_url}/m-meine-anzeigen-verwalten.json?sort=DEFAULT"))["content"])["ads"] published_ads = json.loads((await self.web_request(f"{self.root_url}/m-meine-anzeigen-verwalten.json?sort=DEFAULT"))["content"])["ads"]
@@ -900,14 +900,14 @@ class KleinanzeigenBot(WebScrapingMixin):
for ad_file, ad_cfg, _ad_cfg_orig in ad_cfgs: for ad_file, ad_cfg, _ad_cfg_orig in ad_cfgs:
count += 1 count += 1
LOG.info("Processing %s/%s: '%s' from [%s]...", count, len(ad_cfgs), ad_cfg.title, ad_file) LOG.info("Processing %s/%s: '%s' from [%s]...", count, len(ad_cfgs), ad_cfg.title, ad_file)
await self.delete_ad(ad_cfg, published_ads, delete_old_ads_by_title=self.config.publishing.delete_old_ads_by_title) await self.delete_ad(ad_cfg, published_ads, delete_old_ads_by_title = self.config.publishing.delete_old_ads_by_title)
await self.web_sleep() await self.web_sleep()
LOG.info("############################################") LOG.info("############################################")
LOG.info("DONE: Deleted %s", pluralize("ad", count)) LOG.info("DONE: Deleted %s", pluralize("ad", count))
LOG.info("############################################") LOG.info("############################################")
async def delete_ad(self, ad_cfg: Ad, published_ads: list[dict[str, Any]], *, delete_old_ads_by_title: bool) -> bool: async def delete_ad(self, ad_cfg:Ad, published_ads:list[dict[str, Any]], *, delete_old_ads_by_title:bool) -> bool:
LOG.info("Deleting ad '%s' if already present...", ad_cfg.title) LOG.info("Deleting ad '%s' if already present...", ad_cfg.title)
await self.web_open(f"{self.root_url}/m-meine-anzeigen.html") await self.web_open(f"{self.root_url}/m-meine-anzeigen.html")
@@ -922,21 +922,21 @@ class KleinanzeigenBot(WebScrapingMixin):
if ad_cfg.id == published_ad_id or ad_cfg.title == published_ad_title: if ad_cfg.id == published_ad_id or ad_cfg.title == published_ad_title:
LOG.info(" -> deleting %s '%s'...", published_ad_id, published_ad_title) LOG.info(" -> deleting %s '%s'...", published_ad_id, published_ad_title)
await self.web_request( await self.web_request(
url=f"{self.root_url}/m-anzeigen-loeschen.json?ids={published_ad_id}", method="POST", headers={"x-csrf-token": str(csrf_token)} url = f"{self.root_url}/m-anzeigen-loeschen.json?ids={published_ad_id}", method = "POST", headers = {"x-csrf-token": str(csrf_token)}
) )
elif ad_cfg.id: elif ad_cfg.id:
await self.web_request( await self.web_request(
url=f"{self.root_url}/m-anzeigen-loeschen.json?ids={ad_cfg.id}", url = f"{self.root_url}/m-anzeigen-loeschen.json?ids={ad_cfg.id}",
method="POST", method = "POST",
headers={"x-csrf-token": str(csrf_token)}, headers = {"x-csrf-token": str(csrf_token)},
valid_response_codes=[200, 404], valid_response_codes = [200, 404],
) )
await self.web_sleep() await self.web_sleep()
ad_cfg.id = None ad_cfg.id = None
return True return True
async def extend_ads(self, ad_cfgs: list[tuple[str, Ad, dict[str, Any]]]) -> None: async def extend_ads(self, ad_cfgs:list[tuple[str, Ad, dict[str, Any]]]) -> None:
"""Extends ads that are close to expiry.""" """Extends ads that are close to expiry."""
# Fetch currently published ads from API # Fetch currently published ads from API
published_ads = json.loads((await self.web_request(f"{self.root_url}/m-meine-anzeigen-verwalten.json?sort=DEFAULT"))["content"])["ads"] published_ads = json.loads((await self.web_request(f"{self.root_url}/m-meine-anzeigen-verwalten.json?sort=DEFAULT"))["content"])["ads"]
@@ -986,7 +986,7 @@ class KleinanzeigenBot(WebScrapingMixin):
# Process extensions # Process extensions
success_count = 0 success_count = 0
for idx, (ad_file, ad_cfg, ad_cfg_orig, _published_ad) in enumerate(ads_to_extend, start=1): for idx, (ad_file, ad_cfg, ad_cfg_orig, _published_ad) in enumerate(ads_to_extend, start = 1):
LOG.info(_("Processing %s/%s: '%s' from [%s]..."), idx, len(ads_to_extend), ad_cfg.title, ad_file) LOG.info(_("Processing %s/%s: '%s' from [%s]..."), idx, len(ads_to_extend), ad_cfg.title, ad_file)
if await self.extend_ad(ad_file, ad_cfg, ad_cfg_orig): if await self.extend_ad(ad_file, ad_cfg, ad_cfg_orig):
success_count += 1 success_count += 1
@@ -996,7 +996,7 @@ class KleinanzeigenBot(WebScrapingMixin):
LOG.info(_("DONE: Extended %s"), pluralize("ad", success_count)) LOG.info(_("DONE: Extended %s"), pluralize("ad", success_count))
LOG.info("############################################") LOG.info("############################################")
async def extend_ad(self, ad_file: str, ad_cfg: Ad, ad_cfg_orig: dict[str, Any]) -> bool: async def extend_ad(self, ad_file:str, ad_cfg:Ad, ad_cfg_orig:dict[str, Any]) -> bool:
"""Extends a single ad listing.""" """Extends a single ad listing."""
LOG.info(_("Extending ad '%s' (ID: %s)..."), ad_cfg.title, ad_cfg.id) LOG.info(_("Extending ad '%s' (ID: %s)..."), ad_cfg.title, ad_cfg.id)
@@ -1021,14 +1021,14 @@ class KleinanzeigenBot(WebScrapingMixin):
# Simply close the dialog with the X button (aria-label="Schließen") # Simply close the dialog with the X button (aria-label="Schließen")
try: try:
dialog_close_timeout = self._timeout("quick_dom") dialog_close_timeout = self._timeout("quick_dom")
await self.web_click(By.CSS_SELECTOR, 'button[aria-label="Schließen"]', timeout=dialog_close_timeout) await self.web_click(By.CSS_SELECTOR, 'button[aria-label="Schließen"]', timeout = dialog_close_timeout)
LOG.debug(" -> Closed confirmation dialog") LOG.debug(" -> Closed confirmation dialog")
except TimeoutError: except TimeoutError:
LOG.warning(_(" -> No confirmation dialog found, extension may have completed directly")) LOG.warning(_(" -> No confirmation dialog found, extension may have completed directly"))
# Update metadata in YAML file # Update metadata in YAML file
# Update updated_on to track when ad was extended # Update updated_on to track when ad was extended
ad_cfg_orig["updated_on"] = misc.now().isoformat(timespec="seconds") ad_cfg_orig["updated_on"] = misc.now().isoformat(timespec = "seconds")
dicts.save_dict(ad_file, ad_cfg_orig) dicts.save_dict(ad_file, ad_cfg_orig)
LOG.info(_(" -> SUCCESS: ad extended with ID %s"), ad_cfg.id) LOG.info(_(" -> SUCCESS: ad extended with ID %s"), ad_cfg.id)
@@ -1045,7 +1045,7 @@ class KleinanzeigenBot(WebScrapingMixin):
# Check for success messages # Check for success messages
return await self.web_check(By.ID, "checking-done", Is.DISPLAYED) or await self.web_check(By.ID, "not-completed", Is.DISPLAYED) return await self.web_check(By.ID, "checking-done", Is.DISPLAYED) or await self.web_check(By.ID, "not-completed", Is.DISPLAYED)
async def publish_ads(self, ad_cfgs: list[tuple[str, Ad, dict[str, Any]]]) -> None: async def publish_ads(self, ad_cfgs:list[tuple[str, Ad, dict[str, Any]]]) -> None:
count = 0 count = 0
failed_count = 0 failed_count = 0
max_retries = 3 max_retries = 3
@@ -1082,12 +1082,12 @@ class KleinanzeigenBot(WebScrapingMixin):
if success: if success:
try: try:
publish_timeout = self._timeout("publishing_result") publish_timeout = self._timeout("publishing_result")
await self.web_await(self.__check_publishing_result, timeout=publish_timeout) await self.web_await(self.__check_publishing_result, timeout = publish_timeout)
except TimeoutError: except TimeoutError:
LOG.warning(_(" -> Could not confirm publishing for '%s', but ad may be online"), ad_cfg.title) LOG.warning(_(" -> Could not confirm publishing for '%s', but ad may be online"), ad_cfg.title)
if success and self.config.publishing.delete_old_ads == "AFTER_PUBLISH" and not self.keep_old_ads: if success and self.config.publishing.delete_old_ads == "AFTER_PUBLISH" and not self.keep_old_ads:
await self.delete_ad(ad_cfg, published_ads, delete_old_ads_by_title=False) await self.delete_ad(ad_cfg, published_ads, delete_old_ads_by_title = False)
LOG.info("############################################") LOG.info("############################################")
if failed_count > 0: if failed_count > 0:
@@ -1097,7 +1097,7 @@ class KleinanzeigenBot(WebScrapingMixin):
LOG.info("############################################") LOG.info("############################################")
async def publish_ad( async def publish_ad(
self, ad_file: str, ad_cfg: Ad, ad_cfg_orig: dict[str, Any], published_ads: list[dict[str, Any]], mode: AdUpdateStrategy = AdUpdateStrategy.REPLACE self, ad_file:str, ad_cfg:Ad, ad_cfg_orig:dict[str, Any], published_ads:list[dict[str, Any]], mode:AdUpdateStrategy = AdUpdateStrategy.REPLACE
) -> None: ) -> None:
""" """
@param ad_cfg: the effective ad config (i.e. with default values applied etc.) @param ad_cfg: the effective ad config (i.e. with default values applied etc.)
@@ -1108,7 +1108,7 @@ class KleinanzeigenBot(WebScrapingMixin):
if mode == AdUpdateStrategy.REPLACE: if mode == AdUpdateStrategy.REPLACE:
if self.config.publishing.delete_old_ads == "BEFORE_PUBLISH" and not self.keep_old_ads: if self.config.publishing.delete_old_ads == "BEFORE_PUBLISH" and not self.keep_old_ads:
await self.delete_ad(ad_cfg, published_ads, delete_old_ads_by_title=self.config.publishing.delete_old_ads_by_title) await self.delete_ad(ad_cfg, published_ads, delete_old_ads_by_title = self.config.publishing.delete_old_ads_by_title)
# Apply auto price reduction only for REPLACE operations (actual reposts) # Apply auto price reduction only for REPLACE operations (actual reposts)
# This ensures price reductions only happen on republish, not on UPDATE # This ensures price reductions only happen on republish, not on UPDATE
@@ -1197,12 +1197,12 @@ class KleinanzeigenBot(WebScrapingMixin):
elif not await self.web_check(By.ID, "radio-buy-now-no", Is.SELECTED): elif not await self.web_check(By.ID, "radio-buy-now-no", Is.SELECTED):
await self.web_click(By.ID, "radio-buy-now-no") await self.web_click(By.ID, "radio-buy-now-no")
except TimeoutError as ex: except TimeoutError as ex:
LOG.debug(ex, exc_info=True) LOG.debug(ex, exc_info = True)
############################# #############################
# set description # set description
############################# #############################
description = self.__get_description(ad_cfg, with_affixes=True) description = self.__get_description(ad_cfg, with_affixes = True)
await self.web_execute("document.querySelector('#pstad-descrptn').value = `" + description.replace("`", "'") + "`") await self.web_execute("document.querySelector('#pstad-descrptn').value = `" + description.replace("`", "'") + "`")
await self.__set_contact_fields(ad_cfg.contact) await self.__set_contact_fields(ad_cfg.contact)
@@ -1213,7 +1213,7 @@ class KleinanzeigenBot(WebScrapingMixin):
############################# #############################
img_items = await self.web_find_all(By.CSS_SELECTOR, "ul#j-pictureupload-thumbnails > li:not(.is-placeholder)") img_items = await self.web_find_all(By.CSS_SELECTOR, "ul#j-pictureupload-thumbnails > li:not(.is-placeholder)")
for element in img_items: for element in img_items:
btn = await self.web_find(By.CSS_SELECTOR, "button.pictureupload-thumbnails-remove", parent=element) btn = await self.web_find(By.CSS_SELECTOR, "button.pictureupload-thumbnails-remove", parent = element)
await btn.click() await btn.click()
############################# #############################
@@ -1224,7 +1224,7 @@ class KleinanzeigenBot(WebScrapingMixin):
############################# #############################
# wait for captcha # wait for captcha
############################# #############################
await self.check_and_wait_for_captcha(is_login_page=False) await self.check_and_wait_for_captcha(is_login_page = False)
############################# #############################
# submit # submit
@@ -1250,7 +1250,7 @@ class KleinanzeigenBot(WebScrapingMixin):
############################# #############################
try: try:
short_timeout = self._timeout("quick_dom") short_timeout = self._timeout("quick_dom")
await self.web_find(By.ID, "myftr-shppngcrt-frm", timeout=short_timeout) await self.web_find(By.ID, "myftr-shppngcrt-frm", timeout = short_timeout)
LOG.warning("############################################") LOG.warning("############################################")
LOG.warning("# Payment form detected! Please proceed with payment.") LOG.warning("# Payment form detected! Please proceed with payment.")
@@ -1262,7 +1262,7 @@ class KleinanzeigenBot(WebScrapingMixin):
pass pass
confirmation_timeout = self._timeout("publishing_confirmation") confirmation_timeout = self._timeout("publishing_confirmation")
await self.web_await(lambda: "p-anzeige-aufgeben-bestaetigung.html?adId=" in self.page.url, timeout=confirmation_timeout) await self.web_await(lambda: "p-anzeige-aufgeben-bestaetigung.html?adId=" in self.page.url, timeout = confirmation_timeout)
# extract the ad id from the URL's query parameter # extract the ad id from the URL's query parameter
current_url_query_params = urllib_parse.parse_qs(urllib_parse.urlparse(self.page.url).query) current_url_query_params = urllib_parse.parse_qs(urllib_parse.urlparse(self.page.url).query)
@@ -1272,7 +1272,7 @@ class KleinanzeigenBot(WebScrapingMixin):
# Update content hash after successful publication # Update content hash after successful publication
# Calculate hash on original config to ensure consistent comparison on restart # Calculate hash on original config to ensure consistent comparison on restart
ad_cfg_orig["content_hash"] = AdPartial.model_validate(ad_cfg_orig).update_content_hash().content_hash ad_cfg_orig["content_hash"] = AdPartial.model_validate(ad_cfg_orig).update_content_hash().content_hash
ad_cfg_orig["updated_on"] = misc.now().isoformat(timespec="seconds") ad_cfg_orig["updated_on"] = misc.now().isoformat(timespec = "seconds")
if not ad_cfg.created_on and not ad_cfg.id: if not ad_cfg.created_on and not ad_cfg.id:
ad_cfg_orig["created_on"] = ad_cfg_orig["updated_on"] ad_cfg_orig["created_on"] = ad_cfg_orig["updated_on"]
@@ -1299,7 +1299,7 @@ class KleinanzeigenBot(WebScrapingMixin):
dicts.save_dict(ad_file, ad_cfg_orig) dicts.save_dict(ad_file, ad_cfg_orig)
async def __set_contact_fields(self, contact: Contact) -> None: async def __set_contact_fields(self, contact:Contact) -> None:
############################# #############################
# set contact zipcode # set contact zipcode
############################# #############################
@@ -1384,7 +1384,7 @@ class KleinanzeigenBot(WebScrapingMixin):
) )
) )
async def update_ads(self, ad_cfgs: list[tuple[str, Ad, dict[str, Any]]]) -> None: async def update_ads(self, ad_cfgs:list[tuple[str, Ad, dict[str, Any]]]) -> None:
""" """
Updates a list of ads. Updates a list of ads.
The list gets filtered, so that only already published ads will be updated. The list gets filtered, so that only already published ads will be updated.
@@ -1415,25 +1415,25 @@ class KleinanzeigenBot(WebScrapingMixin):
await self.publish_ad(ad_file, ad_cfg, ad_cfg_orig, published_ads, AdUpdateStrategy.MODIFY) await self.publish_ad(ad_file, ad_cfg, ad_cfg_orig, published_ads, AdUpdateStrategy.MODIFY)
publish_timeout = self._timeout("publishing_result") publish_timeout = self._timeout("publishing_result")
await self.web_await(self.__check_publishing_result, timeout=publish_timeout) await self.web_await(self.__check_publishing_result, timeout = publish_timeout)
LOG.info("############################################") LOG.info("############################################")
LOG.info("DONE: updated %s", pluralize("ad", count)) LOG.info("DONE: updated %s", pluralize("ad", count))
LOG.info("############################################") LOG.info("############################################")
async def __set_condition(self, condition_value: str) -> None: async def __set_condition(self, condition_value:str) -> None:
try: try:
# Open condition dialog # Open condition dialog
await self.web_click(By.XPATH, '//*[@id="j-post-listing-frontend-conditions"]//button[@aria-haspopup="true"]') await self.web_click(By.XPATH, '//*[@id="j-post-listing-frontend-conditions"]//button[@aria-haspopup="true"]')
except TimeoutError: except TimeoutError:
LOG.debug("Unable to open condition dialog and select condition [%s]", condition_value, exc_info=True) LOG.debug("Unable to open condition dialog and select condition [%s]", condition_value, exc_info = True)
return return
try: try:
# Click radio button # Click radio button
await self.web_click(By.ID, f"radio-button-{condition_value}") await self.web_click(By.ID, f"radio-button-{condition_value}")
except TimeoutError: except TimeoutError:
LOG.debug("Unable to select condition [%s]", condition_value, exc_info=True) LOG.debug("Unable to select condition [%s]", condition_value, exc_info = True)
try: try:
# Click accept button # Click accept button
@@ -1441,7 +1441,7 @@ class KleinanzeigenBot(WebScrapingMixin):
except TimeoutError as ex: except TimeoutError as ex:
raise TimeoutError(_("Unable to close condition dialog!")) from ex raise TimeoutError(_("Unable to close condition dialog!")) from ex
async def __set_category(self, category: str | None, ad_file: str) -> None: async def __set_category(self, category:str | None, ad_file:str) -> None:
# click on something to trigger automatic category detection # click on something to trigger automatic category detection
await self.web_click(By.ID, "pstad-descrptn") await self.web_click(By.ID, "pstad-descrptn")
@@ -1464,7 +1464,7 @@ class KleinanzeigenBot(WebScrapingMixin):
else: else:
ensure(is_category_auto_selected, f"No category specified in [{ad_file}] and automatic category detection failed") ensure(is_category_auto_selected, f"No category specified in [{ad_file}] and automatic category detection failed")
async def __set_special_attributes(self, ad_cfg: Ad) -> None: async def __set_special_attributes(self, ad_cfg:Ad) -> None:
if not ad_cfg.special_attributes: if not ad_cfg.special_attributes:
return return
@@ -1499,7 +1499,7 @@ class KleinanzeigenBot(WebScrapingMixin):
raise TimeoutError(_("Failed to set attribute '%s'") % special_attribute_key) from ex raise TimeoutError(_("Failed to set attribute '%s'") % special_attribute_key) from ex
try: try:
elem_id: str = str(special_attr_elem.attrs.id) elem_id:str = str(special_attr_elem.attrs.id)
if special_attr_elem.local_name == "select": if special_attr_elem.local_name == "select":
LOG.debug(_("Attribute field '%s' seems to be a select..."), special_attribute_key) LOG.debug(_("Attribute field '%s' seems to be a select..."), special_attribute_key)
await self.web_select(By.ID, elem_id, special_attribute_value_str) await self.web_select(By.ID, elem_id, special_attribute_value_str)
@@ -1517,26 +1517,26 @@ class KleinanzeigenBot(WebScrapingMixin):
raise TimeoutError(_("Failed to set attribute '%s'") % special_attribute_key) from ex raise TimeoutError(_("Failed to set attribute '%s'") % special_attribute_key) from ex
LOG.debug("Successfully set attribute field [%s] to [%s]...", special_attribute_key, special_attribute_value_str) LOG.debug("Successfully set attribute field [%s] to [%s]...", special_attribute_key, special_attribute_value_str)
async def __set_shipping(self, ad_cfg: Ad, mode: AdUpdateStrategy = AdUpdateStrategy.REPLACE) -> None: async def __set_shipping(self, ad_cfg:Ad, mode:AdUpdateStrategy = AdUpdateStrategy.REPLACE) -> None:
short_timeout = self._timeout("quick_dom") short_timeout = self._timeout("quick_dom")
if ad_cfg.shipping_type == "PICKUP": if ad_cfg.shipping_type == "PICKUP":
try: try:
await self.web_click(By.ID, "radio-pickup") await self.web_click(By.ID, "radio-pickup")
except TimeoutError as ex: except TimeoutError as ex:
LOG.debug(ex, exc_info=True) LOG.debug(ex, exc_info = True)
elif ad_cfg.shipping_options: elif ad_cfg.shipping_options:
await self.web_click(By.XPATH, '//button//span[contains(., "Versandmethoden auswählen")]') await self.web_click(By.XPATH, '//button//span[contains(., "Versandmethoden auswählen")]')
if mode == AdUpdateStrategy.MODIFY: if mode == AdUpdateStrategy.MODIFY:
try: try:
# when "Andere Versandmethoden" is not available, go back and start over new # when "Andere Versandmethoden" is not available, go back and start over new
await self.web_find(By.XPATH, '//dialog//button[contains(., "Andere Versandmethoden")]', timeout=short_timeout) await self.web_find(By.XPATH, '//dialog//button[contains(., "Andere Versandmethoden")]', timeout = short_timeout)
except TimeoutError: except TimeoutError:
await self.web_click(By.XPATH, '//dialog//button[contains(., "Zurück")]') await self.web_click(By.XPATH, '//dialog//button[contains(., "Zurück")]')
# in some categories we need to go another dialog back # in some categories we need to go another dialog back
try: try:
await self.web_find(By.XPATH, '//dialog//button[contains(., "Andere Versandmethoden")]', timeout=short_timeout) await self.web_find(By.XPATH, '//dialog//button[contains(., "Andere Versandmethoden")]', timeout = short_timeout)
except TimeoutError: except TimeoutError:
await self.web_click(By.XPATH, '//dialog//button[contains(., "Zurück")]') await self.web_click(By.XPATH, '//dialog//button[contains(., "Zurück")]')
@@ -1562,7 +1562,7 @@ class KleinanzeigenBot(WebScrapingMixin):
try: try:
# only click on "Individueller Versand" when "IndividualShippingInput" is not available, otherwise its already checked # only click on "Individueller Versand" when "IndividualShippingInput" is not available, otherwise its already checked
# (important for mode = UPDATE) # (important for mode = UPDATE)
await self.web_find(By.XPATH, '//input[contains(@placeholder, "Versandkosten (optional)")]', timeout=short_timeout) await self.web_find(By.XPATH, '//input[contains(@placeholder, "Versandkosten (optional)")]', timeout = short_timeout)
except TimeoutError: except TimeoutError:
# Input not visible yet; click the individual shipping option. # Input not visible yet; click the individual shipping option.
await self.web_click(By.XPATH, '//*[contains(@id, "INDIVIDUAL") and contains(@data-testid, "Individueller Versand")]') await self.web_click(By.XPATH, '//*[contains(@id, "INDIVIDUAL") and contains(@data-testid, "Individueller Versand")]')
@@ -1573,10 +1573,10 @@ class KleinanzeigenBot(WebScrapingMixin):
) )
await self.web_click(By.XPATH, '//dialog//button[contains(., "Fertig")]') await self.web_click(By.XPATH, '//dialog//button[contains(., "Fertig")]')
except TimeoutError as ex: except TimeoutError as ex:
LOG.debug(ex, exc_info=True) LOG.debug(ex, exc_info = True)
raise TimeoutError(_("Unable to close shipping dialog!")) from ex raise TimeoutError(_("Unable to close shipping dialog!")) from ex
async def __set_shipping_options(self, ad_cfg: Ad, mode: AdUpdateStrategy = AdUpdateStrategy.REPLACE) -> None: async def __set_shipping_options(self, ad_cfg:Ad, mode:AdUpdateStrategy = AdUpdateStrategy.REPLACE) -> None:
if not ad_cfg.shipping_options: if not ad_cfg.shipping_options:
return return
@@ -1596,7 +1596,7 @@ class KleinanzeigenBot(WebScrapingMixin):
except KeyError as ex: except KeyError as ex:
raise KeyError(f"Unknown shipping option(s), please refer to the documentation/README: {ad_cfg.shipping_options}") from ex raise KeyError(f"Unknown shipping option(s), please refer to the documentation/README: {ad_cfg.shipping_options}") from ex
shipping_sizes, shipping_selector, shipping_packages = zip(*mapped_shipping_options, strict=False) shipping_sizes, shipping_selector, shipping_packages = zip(*mapped_shipping_options, strict = False)
try: try:
(shipping_size,) = set(shipping_sizes) (shipping_size,) = set(shipping_sizes)
@@ -1652,19 +1652,19 @@ class KleinanzeigenBot(WebScrapingMixin):
for shipping_package in to_be_clicked_shipping_packages: for shipping_package in to_be_clicked_shipping_packages:
await self.web_click(By.XPATH, f'//dialog//input[contains(@data-testid, "{shipping_package}")]') await self.web_click(By.XPATH, f'//dialog//input[contains(@data-testid, "{shipping_package}")]')
except TimeoutError as ex: except TimeoutError as ex:
LOG.debug(ex, exc_info=True) LOG.debug(ex, exc_info = True)
try: try:
# Click apply button # Click apply button
await self.web_click(By.XPATH, '//dialog//button[contains(., "Fertig")]') await self.web_click(By.XPATH, '//dialog//button[contains(., "Fertig")]')
except TimeoutError as ex: except TimeoutError as ex:
raise TimeoutError(_("Unable to close shipping dialog!")) from ex raise TimeoutError(_("Unable to close shipping dialog!")) from ex
async def __upload_images(self, ad_cfg: Ad) -> None: async def __upload_images(self, ad_cfg:Ad) -> None:
if not ad_cfg.images: if not ad_cfg.images:
return return
LOG.info(" -> found %s", pluralize("image", ad_cfg.images)) LOG.info(" -> found %s", pluralize("image", ad_cfg.images))
image_upload: Element = await self.web_find(By.CSS_SELECTOR, "input[type=file]") image_upload:Element = await self.web_find(By.CSS_SELECTOR, "input[type=file]")
for image in ad_cfg.images: for image in ad_cfg.images:
LOG.info(" -> uploading image [%s]", image) LOG.info(" -> uploading image [%s]", image)
@@ -1680,7 +1680,7 @@ class KleinanzeigenBot(WebScrapingMixin):
thumbnails = await self.web_find_all( thumbnails = await self.web_find_all(
By.CSS_SELECTOR, By.CSS_SELECTOR,
"ul#j-pictureupload-thumbnails > li:not(.is-placeholder)", "ul#j-pictureupload-thumbnails > li:not(.is-placeholder)",
timeout=self._timeout("quick_dom"), # Fast timeout for polling timeout = self._timeout("quick_dom"), # Fast timeout for polling
) )
current_count = len(thumbnails) current_count = len(thumbnails)
if current_count < expected_count: if current_count < expected_count:
@@ -1691,12 +1691,12 @@ class KleinanzeigenBot(WebScrapingMixin):
return False return False
try: try:
await self.web_await(check_thumbnails_uploaded, timeout=self._timeout("image_upload"), timeout_error_message=_("Image upload timeout exceeded")) await self.web_await(check_thumbnails_uploaded, timeout = self._timeout("image_upload"), timeout_error_message = _("Image upload timeout exceeded"))
except TimeoutError as ex: except TimeoutError as ex:
# Get current count for better error message # Get current count for better error message
try: try:
thumbnails = await self.web_find_all( thumbnails = await self.web_find_all(
By.CSS_SELECTOR, "ul#j-pictureupload-thumbnails > li:not(.is-placeholder)", timeout=self._timeout("quick_dom") By.CSS_SELECTOR, "ul#j-pictureupload-thumbnails > li:not(.is-placeholder)", timeout = self._timeout("quick_dom")
) )
current_count = len(thumbnails) current_count = len(thumbnails)
except TimeoutError: except TimeoutError:
@@ -1738,7 +1738,7 @@ class KleinanzeigenBot(WebScrapingMixin):
elif self.ads_selector == "new": # download only unsaved ads elif self.ads_selector == "new": # download only unsaved ads
# check which ads already saved # check which ads already saved
saved_ad_ids = [] saved_ad_ids = []
ads = self.load_ads(ignore_inactive=False, exclude_ads_with_id=False) # do not skip because of existing IDs ads = self.load_ads(ignore_inactive = False, exclude_ads_with_id = False) # do not skip because of existing IDs
for ad in ads: for ad in ads:
saved_ad_id = ad[1].id saved_ad_id = ad[1].id
if saved_ad_id is None: if saved_ad_id is None:
@@ -1775,7 +1775,7 @@ class KleinanzeigenBot(WebScrapingMixin):
else: else:
LOG.error("The page with the id %d does not exist!", ad_id) LOG.error("The page with the id %d does not exist!", ad_id)
def __get_description(self, ad_cfg: Ad, *, with_affixes: bool) -> str: def __get_description(self, ad_cfg:Ad, *, with_affixes:bool) -> str:
"""Get the ad description optionally with prefix and suffix applied. """Get the ad description optionally with prefix and suffix applied.
Precedence (highest to lowest): Precedence (highest to lowest):
@@ -1827,7 +1827,7 @@ class KleinanzeigenBot(WebScrapingMixin):
return final_description return final_description
def update_content_hashes(self, ads: list[tuple[str, Ad, dict[str, Any]]]) -> None: def update_content_hashes(self, ads:list[tuple[str, Ad, dict[str, Any]]]) -> None:
count = 0 count = 0
for ad_file, ad_cfg, ad_cfg_orig in ads: for ad_file, ad_cfg, ad_cfg_orig in ads:
@@ -1848,7 +1848,7 @@ class KleinanzeigenBot(WebScrapingMixin):
############################# #############################
def main(args: list[str]) -> None: def main(args:list[str]) -> None:
if "version" not in args: if "version" not in args:
print( print(
textwrap.dedent(rf""" textwrap.dedent(rf"""
@@ -1861,7 +1861,7 @@ def main(args: list[str]) -> None:
https://github.com/Second-Hand-Friends/kleinanzeigen-bot https://github.com/Second-Hand-Friends/kleinanzeigen-bot
Version: {__version__} Version: {__version__}
""")[1:], """)[1:],
flush=True, flush = True,
) # [1:] removes the first empty blank line ) # [1:] removes the first empty blank line
loggers.configure_console_logging() loggers.configure_console_logging()

View File

@@ -15,22 +15,22 @@ from kleinanzeigen_bot.utils import dicts
from kleinanzeigen_bot.utils.misc import get_attr from kleinanzeigen_bot.utils.misc import get_attr
from kleinanzeigen_bot.utils.pydantics import ContextualModel from kleinanzeigen_bot.utils.pydantics import ContextualModel
_MAX_PERCENTAGE: Final[int] = 100 _MAX_PERCENTAGE:Final[int] = 100
class AutoPriceReductionConfig(ContextualModel): class AutoPriceReductionConfig(ContextualModel):
enabled: bool = Field(default=False, description="automatically lower the price of reposted ads") enabled:bool = Field(default = False, description = "automatically lower the price of reposted ads")
strategy: Literal["FIXED", "PERCENTAGE"] | None = Field( strategy:Literal["FIXED", "PERCENTAGE"] | None = Field(
default=None, description="PERCENTAGE reduces by a percentage of the previous price, FIXED reduces by a fixed amount" default = None, description = "PERCENTAGE reduces by a percentage of the previous price, FIXED reduces by a fixed amount"
) )
amount: float | None = Field( amount:float | None = Field(
default=None, gt=0, description="magnitude of the reduction; interpreted as percent for PERCENTAGE or currency units for FIXED" default = None, gt = 0, description = "magnitude of the reduction; interpreted as percent for PERCENTAGE or currency units for FIXED"
) )
min_price: float | None = Field(default=None, ge=0, description="required when enabled is true; minimum price floor (use 0 for no lower bound)") min_price:float | None = Field(default = None, ge = 0, description = "required when enabled is true; minimum price floor (use 0 for no lower bound)")
delay_reposts: int = Field(default=0, ge=0, description="number of reposts to wait before applying the first automatic price reduction") delay_reposts:int = Field(default = 0, ge = 0, description = "number of reposts to wait before applying the first automatic price reduction")
delay_days: int = Field(default=0, ge=0, description="number of days to wait after publication before applying automatic price reductions") delay_days:int = Field(default = 0, ge = 0, description = "number of days to wait after publication before applying automatic price reductions")
@model_validator(mode="after") @model_validator(mode = "after")
def _validate_config(self) -> "AutoPriceReductionConfig": def _validate_config(self) -> "AutoPriceReductionConfig":
if self.enabled: if self.enabled:
if self.strategy is None: if self.strategy is None:
@@ -45,38 +45,38 @@ class AutoPriceReductionConfig(ContextualModel):
class ContactDefaults(ContextualModel): class ContactDefaults(ContextualModel):
name: str | None = None name:str | None = None
street: str | None = None street:str | None = None
zipcode: int | str | None = None zipcode:int | str | None = None
location: str | None = Field( location:str | None = Field(
default=None, description="city or locality of the listing (can include multiple districts)", examples=["Sample Town - District One"] default = None, description = "city or locality of the listing (can include multiple districts)", examples = ["Sample Town - District One"]
) )
phone: str | None = None phone:str | None = None
@deprecated("Use description_prefix/description_suffix instead") @deprecated("Use description_prefix/description_suffix instead")
class DescriptionAffixes(ContextualModel): class DescriptionAffixes(ContextualModel):
prefix: str | None = None prefix:str | None = None
suffix: str | None = None suffix:str | None = None
class AdDefaults(ContextualModel): class AdDefaults(ContextualModel):
active: bool = True active:bool = True
type: Literal["OFFER", "WANTED"] = "OFFER" type:Literal["OFFER", "WANTED"] = "OFFER"
description: DescriptionAffixes | None = None description:DescriptionAffixes | None = None
description_prefix: str | None = Field(default=None, description="prefix for the ad description") description_prefix:str | None = Field(default = None, description = "prefix for the ad description")
description_suffix: str | None = Field(default=None, description=" suffix for the ad description") description_suffix:str | None = Field(default = None, description = "suffix for the ad description")
price_type: Literal["FIXED", "NEGOTIABLE", "GIVE_AWAY", "NOT_APPLICABLE"] = "NEGOTIABLE" price_type:Literal["FIXED", "NEGOTIABLE", "GIVE_AWAY", "NOT_APPLICABLE"] = "NEGOTIABLE"
auto_price_reduction: AutoPriceReductionConfig = Field(default_factory=AutoPriceReductionConfig, description="automatic price reduction configuration") auto_price_reduction:AutoPriceReductionConfig = Field(default_factory = AutoPriceReductionConfig, description = "automatic price reduction configuration")
shipping_type: Literal["PICKUP", "SHIPPING", "NOT_APPLICABLE"] = "SHIPPING" shipping_type:Literal["PICKUP", "SHIPPING", "NOT_APPLICABLE"] = "SHIPPING"
sell_directly: bool = Field(default=False, description="requires shipping_type SHIPPING to take effect") sell_directly:bool = Field(default = False, description = "requires shipping_type SHIPPING to take effect")
images: list[str] | None = Field(default=None) images:list[str] | None = Field(default = None)
contact: ContactDefaults = Field(default_factory=ContactDefaults) contact:ContactDefaults = Field(default_factory = ContactDefaults)
republication_interval: int = 7 republication_interval:int = 7
@model_validator(mode="before") @model_validator(mode = "before")
@classmethod @classmethod
def migrate_legacy_description(cls, values: dict[str, Any]) -> dict[str, Any]: def migrate_legacy_description(cls, values:dict[str, Any]) -> dict[str, Any]:
# Ensure flat prefix/suffix take precedence over deprecated nested "description" # Ensure flat prefix/suffix take precedence over deprecated nested "description"
description_prefix = values.get("description_prefix") description_prefix = values.get("description_prefix")
description_suffix = values.get("description_suffix") description_suffix = values.get("description_suffix")
@@ -91,71 +91,74 @@ class AdDefaults(ContextualModel):
class DownloadConfig(ContextualModel): class DownloadConfig(ContextualModel):
include_all_matching_shipping_options: bool = Field(default=False, description="if true, all shipping options matching the package size will be included") include_all_matching_shipping_options:bool = Field(
excluded_shipping_options: list[str] = Field(default_factory=list, description="list of shipping options to exclude, e.g. ['DHL_2', 'DHL_5']") default = False,
folder_name_max_length: int = Field(default=100, ge=10, le=255, description="maximum length for folder names when downloading ads (default: 100)") description = "if true, all shipping options matching the package size will be included",
rename_existing_folders: bool = Field(default=False, description="if true, rename existing folders without titles to include titles (default: false)") )
excluded_shipping_options:list[str] = Field(default_factory = list, description = "list of shipping options to exclude, e.g. ['DHL_2', 'DHL_5']")
folder_name_max_length:int = Field(default = 100, ge = 10, le = 255, description = "maximum length for folder names when downloading ads (default: 100)")
rename_existing_folders:bool = Field(default = False, description = "if true, rename existing folders without titles to include titles (default: false)")
class BrowserConfig(ContextualModel): class BrowserConfig(ContextualModel):
arguments: list[str] = Field( arguments:list[str] = Field(
default_factory=list, default_factory = list,
description=( description=(
"See https://peter.sh/experiments/chromium-command-line-switches/. " "See https://peter.sh/experiments/chromium-command-line-switches/. "
"Browser profile path is auto-configured based on installation mode (portable/XDG)." "Browser profile path is auto-configured based on installation mode (portable/XDG)."
), ),
) )
binary_location: str | None = Field(default=None, description="path to custom browser executable, if not specified will be looked up on PATH") binary_location:str | None = Field(default = None, description = "path to custom browser executable, if not specified will be looked up on PATH")
extensions: list[str] = Field(default_factory=list, description="a list of .crx extension files to be loaded") extensions:list[str] = Field(default_factory = list, description = "a list of .crx extension files to be loaded")
use_private_window: bool = True use_private_window:bool = True
user_data_dir: str | None = Field( user_data_dir:str | None = Field(
default=None, default = None,
description=( description=(
"See https://github.com/chromium/chromium/blob/main/docs/user_data_dir.md. " "See https://github.com/chromium/chromium/blob/main/docs/user_data_dir.md. "
"If not specified, defaults to XDG cache directory in XDG mode or .temp/browser-profile in portable mode." "If not specified, defaults to XDG cache directory in XDG mode or .temp/browser-profile in portable mode."
), ),
) )
profile_name: str | None = None profile_name:str | None = None
class LoginConfig(ContextualModel): class LoginConfig(ContextualModel):
username: str = Field(..., min_length=1) username:str = Field(..., min_length = 1)
password: str = Field(..., min_length=1) password:str = Field(..., min_length = 1)
class PublishingConfig(ContextualModel): class PublishingConfig(ContextualModel):
delete_old_ads: Literal["BEFORE_PUBLISH", "AFTER_PUBLISH", "NEVER"] | None = "AFTER_PUBLISH" delete_old_ads:Literal["BEFORE_PUBLISH", "AFTER_PUBLISH", "NEVER"] | None = "AFTER_PUBLISH"
delete_old_ads_by_title: bool = Field(default=True, description="only works if delete_old_ads is set to BEFORE_PUBLISH") delete_old_ads_by_title:bool = Field(default = True, description = "only works if delete_old_ads is set to BEFORE_PUBLISH")
class CaptchaConfig(ContextualModel): class CaptchaConfig(ContextualModel):
auto_restart: bool = False auto_restart:bool = False
restart_delay: str = "6h" restart_delay:str = "6h"
class TimeoutConfig(ContextualModel): class TimeoutConfig(ContextualModel):
multiplier: float = Field(default=1.0, ge=0.1, description="Global multiplier applied to all timeout values.") multiplier:float = Field(default = 1.0, ge = 0.1, description = "Global multiplier applied to all timeout values.")
default: float = Field(default=5.0, ge=0.0, description="Baseline timeout for DOM interactions.") default:float = Field(default = 5.0, ge = 0.0, description = "Baseline timeout for DOM interactions.")
page_load: float = Field(default=15.0, ge=1.0, description="Page load timeout for web_open.") page_load:float = Field(default = 15.0, ge = 1.0, description = "Page load timeout for web_open.")
captcha_detection: float = Field(default=2.0, ge=0.1, description="Timeout for captcha iframe detection.") captcha_detection:float = Field(default = 2.0, ge = 0.1, description = "Timeout for captcha iframe detection.")
sms_verification: float = Field(default=4.0, ge=0.1, description="Timeout for SMS verification prompts.") sms_verification:float = Field(default = 4.0, ge = 0.1, description = "Timeout for SMS verification prompts.")
gdpr_prompt: float = Field(default=10.0, ge=1.0, description="Timeout for GDPR/consent dialogs.") gdpr_prompt:float = Field(default = 10.0, ge = 1.0, description = "Timeout for GDPR/consent dialogs.")
login_detection: float = Field(default=10.0, ge=1.0, description="Timeout for detecting existing login session via DOM elements.") login_detection:float = Field(default = 10.0, ge = 1.0, description = "Timeout for detecting existing login session via DOM elements.")
publishing_result: float = Field(default=300.0, ge=10.0, description="Timeout for publishing result checks.") publishing_result:float = Field(default = 300.0, ge = 10.0, description = "Timeout for publishing result checks.")
publishing_confirmation: float = Field(default=20.0, ge=1.0, description="Timeout for publish confirmation redirect.") publishing_confirmation:float = Field(default = 20.0, ge = 1.0, description = "Timeout for publish confirmation redirect.")
image_upload: float = Field(default=30.0, ge=5.0, description="Timeout for image upload and server-side processing.") image_upload:float = Field(default = 30.0, ge = 5.0, description = "Timeout for image upload and server-side processing.")
pagination_initial: float = Field(default=10.0, ge=1.0, description="Timeout for initial pagination lookup.") pagination_initial:float = Field(default = 10.0, ge = 1.0, description = "Timeout for initial pagination lookup.")
pagination_follow_up: float = Field(default=5.0, ge=1.0, description="Timeout for subsequent pagination navigation.") pagination_follow_up:float = Field(default = 5.0, ge = 1.0, description = "Timeout for subsequent pagination navigation.")
quick_dom: float = Field(default=2.0, ge=0.1, description="Generic short timeout for transient UI.") quick_dom:float = Field(default = 2.0, ge = 0.1, description = "Generic short timeout for transient UI.")
update_check: float = Field(default=10.0, ge=1.0, description="Timeout for GitHub update checks.") update_check:float = Field(default = 10.0, ge = 1.0, description = "Timeout for GitHub update checks.")
chrome_remote_probe: float = Field(default=2.0, ge=0.1, description="Timeout for local remote-debugging probes.") chrome_remote_probe:float = Field(default = 2.0, ge = 0.1, description = "Timeout for local remote-debugging probes.")
chrome_remote_debugging: float = Field(default=5.0, ge=1.0, description="Timeout for remote debugging API calls.") chrome_remote_debugging:float = Field(default = 5.0, ge = 1.0, description = "Timeout for remote debugging API calls.")
chrome_binary_detection: float = Field(default=10.0, ge=1.0, description="Timeout for chrome --version subprocesses.") chrome_binary_detection:float = Field(default = 10.0, ge = 1.0, description = "Timeout for chrome --version subprocesses.")
retry_enabled: bool = Field(default=True, description="Enable built-in retry/backoff for DOM operations.") retry_enabled:bool = Field(default = True, description = "Enable built-in retry/backoff for DOM operations.")
retry_max_attempts: int = Field(default=2, ge=1, description="Max retry attempts when retry is enabled.") retry_max_attempts:int = Field(default = 2, ge = 1, description = "Max retry attempts when retry is enabled.")
retry_backoff_factor: float = Field(default=1.5, ge=1.0, description="Exponential factor applied per retry attempt.") retry_backoff_factor:float = Field(default = 1.5, ge = 1.0, description = "Exponential factor applied per retry attempt.")
def resolve(self, key: str = "default", override: float | None = None) -> float: def resolve(self, key:str = "default", override:float | None = None) -> float:
""" """
Return the base timeout (seconds) for the given key without applying modifiers. Return the base timeout (seconds) for the given key without applying modifiers.
""" """
@@ -171,7 +174,7 @@ class TimeoutConfig(ContextualModel):
return float(self.default) return float(self.default)
def effective(self, key: str = "default", override: float | None = None, *, attempt: int = 0) -> float: def effective(self, key:str = "default", override:float | None = None, *, attempt:int = 0) -> float:
""" """
Return the effective timeout (seconds) with multiplier/backoff applied. Return the effective timeout (seconds) with multiplier/backoff applied.
""" """
@@ -180,7 +183,7 @@ class TimeoutConfig(ContextualModel):
return base * self.multiplier * backoff return base * self.multiplier * backoff
def _validate_glob_pattern(v: str) -> str: def _validate_glob_pattern(v:str) -> str:
if not v.strip(): if not v.strip():
raise ValueError("must be a non-empty, non-blank glob pattern") raise ValueError("must be a non-empty, non-blank glob pattern")
return v return v
@@ -190,20 +193,20 @@ GlobPattern = Annotated[str, AfterValidator(_validate_glob_pattern)]
class Config(ContextualModel): class Config(ContextualModel):
ad_files: list[GlobPattern] = Field( ad_files:list[GlobPattern] = Field(
default_factory=lambda: ["./**/ad_*.{json,yml,yaml}"], default_factory = lambda: ["./**/ad_*.{json,yml,yaml}"],
min_items=1, min_items = 1,
description=""" description = """
glob (wildcard) patterns to select ad configuration files glob (wildcard) patterns to select ad configuration files
if relative paths are specified, then they are relative to this configuration file if relative paths are specified, then they are relative to this configuration file
""", """,
) # type: ignore[call-overload] ) # type: ignore[call-overload]
ad_defaults: AdDefaults = Field(default_factory=AdDefaults, description="Default values for ads, can be overwritten in each ad configuration file") ad_defaults:AdDefaults = Field(default_factory = AdDefaults, description = "Default values for ads, can be overwritten in each ad configuration file")
categories: dict[str, str] = Field( categories:dict[str, str] = Field(
default_factory=dict, default_factory = dict,
description=""" description = """
additional name to category ID mappings, see default list at additional name to category ID mappings, see default list at
https://github.com/Second-Hand-Friends/kleinanzeigen-bot/blob/main/src/kleinanzeigen_bot/resources/categories.yaml https://github.com/Second-Hand-Friends/kleinanzeigen-bot/blob/main/src/kleinanzeigen_bot/resources/categories.yaml
@@ -214,13 +217,13 @@ Example:
""", """,
) )
download: DownloadConfig = Field(default_factory=DownloadConfig) download:DownloadConfig = Field(default_factory = DownloadConfig)
publishing: PublishingConfig = Field(default_factory=PublishingConfig) publishing:PublishingConfig = Field(default_factory = PublishingConfig)
browser: BrowserConfig = Field(default_factory=BrowserConfig, description="Browser configuration") browser:BrowserConfig = Field(default_factory = BrowserConfig, description = "Browser configuration")
login: LoginConfig = Field(default_factory=LoginConfig.model_construct, description="Login credentials") login:LoginConfig = Field(default_factory = LoginConfig.model_construct, description = "Login credentials")
captcha: CaptchaConfig = Field(default_factory=CaptchaConfig) captcha:CaptchaConfig = Field(default_factory = CaptchaConfig)
update_check: UpdateCheckConfig = Field(default_factory=UpdateCheckConfig, description="Update check configuration") update_check:UpdateCheckConfig = Field(default_factory = UpdateCheckConfig, description = "Update check configuration")
timeouts: TimeoutConfig = Field(default_factory=TimeoutConfig, description="Centralized timeout configuration.") timeouts:TimeoutConfig = Field(default_factory = TimeoutConfig, description = "Centralized timeout configuration.")
def with_values(self, values: dict[str, Any]) -> Config: def with_values(self, values:dict[str, Any]) -> Config:
return Config.model_validate(dicts.apply_defaults(copy.deepcopy(values), defaults=self.model_dump())) return Config.model_validate(dicts.apply_defaults(copy.deepcopy(values), defaults = self.model_dump()))

View File

@@ -457,6 +457,9 @@ kleinanzeigen_bot/utils/web_scraping_mixin.py:
" -> Browser profile name: %s": " -> Browser-Profilname: %s" " -> Browser profile name: %s": " -> Browser-Profilname: %s"
" -> Browser user data dir: %s": " -> Browser-Benutzerdatenverzeichnis: %s" " -> Browser user data dir: %s": " -> Browser-Benutzerdatenverzeichnis: %s"
" -> Custom Browser argument: %s": " -> Benutzerdefiniertes Browser-Argument: %s" " -> Custom Browser argument: %s": " -> Benutzerdefiniertes Browser-Argument: %s"
"Ignoring empty --user-data-dir= argument; falling back to configured user_data_dir.": "Ignoriere leeres --user-data-dir= Argument; verwende konfiguriertes user_data_dir."
"Configured browser.user_data_dir (%s) does not match --user-data-dir argument (%s); using the argument value.": "Konfiguriertes browser.user_data_dir (%s) stimmt nicht mit --user-data-dir Argument (%s) überein; verwende Argument-Wert."
"Remote debugging detected, but browser configuration looks invalid: %s": "Remote-Debugging erkannt, aber Browser-Konfiguration scheint ungültig: %s"
" -> Setting chrome prefs [%s]...": " -> Setze Chrome-Einstellungen [%s]..." " -> Setting chrome prefs [%s]...": " -> Setze Chrome-Einstellungen [%s]..."
" -> Adding Browser extension: [%s]": " -> Füge Browser-Erweiterung hinzu: [%s]" " -> Adding Browser extension: [%s]": " -> Füge Browser-Erweiterung hinzu: [%s]"
"Failed to connect to browser. This error often occurs when:": "Fehler beim Verbinden mit dem Browser. Dieser Fehler tritt häufig auf, wenn:" "Failed to connect to browser. This error often occurs when:": "Fehler beim Verbinden mit dem Browser. Dieser Fehler tritt häufig auf, wenn:"
@@ -546,8 +549,8 @@ kleinanzeigen_bot/utils/web_scraping_mixin.py:
" -> Unexpected error during browser version validation, skipping: %s": " -> Unerwarteter Fehler bei Browser-Versionsvalidierung, wird übersprungen: %s" " -> Unexpected error during browser version validation, skipping: %s": " -> Unerwarteter Fehler bei Browser-Versionsvalidierung, wird übersprungen: %s"
_diagnose_chrome_version_issues: _diagnose_chrome_version_issues:
"(info) %s version from binary: %s %s (major: %d)": "(Info) %s-Version von Binärdatei: %s %s (Hauptversion: %d)" "(info) %s version from binary: %s (major: %d)": "(Info) %s-Version von Binärdatei: %s (Hauptversion: %d)"
"(info) %s version from remote debugging: %s %s (major: %d)": "(Info) %s-Version von Remote-Debugging: %s %s (Hauptversion: %d)" "(info) %s version from remote debugging: %s (major: %d)": "(Info) %s-Version von Remote-Debugging: %s (Hauptversion: %d)"
"(info) %s 136+ detected - security validation required": "(Info) %s 136+ erkannt - Sicherheitsvalidierung erforderlich" "(info) %s 136+ detected - security validation required": "(Info) %s 136+ erkannt - Sicherheitsvalidierung erforderlich"
"(info) %s pre-136 detected - no special security requirements": "(Info) %s vor 136 erkannt - keine besonderen Sicherheitsanforderungen" "(info) %s pre-136 detected - no special security requirements": "(Info) %s vor 136 erkannt - keine besonderen Sicherheitsanforderungen"
"(info) Remote %s 136+ detected - validating configuration": "(Info) Remote %s 136+ erkannt - validiere Konfiguration" "(info) Remote %s 136+ detected - validating configuration": "(Info) Remote %s 136+ erkannt - validiere Konfiguration"

View File

@@ -31,7 +31,7 @@ colorama.init()
class UpdateChecker: class UpdateChecker:
"""Checks for updates to the bot.""" """Checks for updates to the bot."""
def __init__(self, config: "Config", installation_mode: str | xdg_paths.InstallationMode = "portable") -> None: def __init__(self, config:"Config", installation_mode:str | xdg_paths.InstallationMode = "portable") -> None:
"""Initialize the update checker. """Initialize the update checker.
Args: Args:
@@ -55,7 +55,7 @@ class UpdateChecker:
"""Return the effective timeout for HTTP calls.""" """Return the effective timeout for HTTP calls."""
return self.config.timeouts.effective("update_check") return self.config.timeouts.effective("update_check")
def _get_commit_hash(self, version: str) -> str | None: def _get_commit_hash(self, version:str) -> str | None:
"""Extract the commit hash from a version string. """Extract the commit hash from a version string.
Args: Args:
@@ -68,7 +68,7 @@ class UpdateChecker:
return version.split("+")[1] return version.split("+")[1]
return None return None
def _resolve_commitish(self, commitish: str) -> tuple[str | None, datetime | None]: def _resolve_commitish(self, commitish:str) -> tuple[str | None, datetime | None]:
"""Resolve a commit-ish to a full commit hash and date. """Resolve a commit-ish to a full commit hash and date.
Args: Args:
@@ -80,7 +80,7 @@ class UpdateChecker:
try: try:
response = requests.get( response = requests.get(
f"https://api.github.com/repos/Second-Hand-Friends/kleinanzeigen-bot/commits/{commitish}", f"https://api.github.com/repos/Second-Hand-Friends/kleinanzeigen-bot/commits/{commitish}",
timeout=self._request_timeout(), timeout = self._request_timeout(),
) )
response.raise_for_status() response.raise_for_status()
data = response.json() data = response.json()
@@ -96,7 +96,7 @@ class UpdateChecker:
logger.warning(_("Could not resolve commit '%s': %s"), commitish, e) logger.warning(_("Could not resolve commit '%s': %s"), commitish, e)
return None, None return None, None
def _get_short_commit_hash(self, commit: str) -> str: def _get_short_commit_hash(self, commit:str) -> str:
"""Get the short version of a commit hash. """Get the short version of a commit hash.
Args: Args:
@@ -107,7 +107,7 @@ class UpdateChecker:
""" """
return commit[:7] return commit[:7]
def _commits_match(self, local_commit: str, release_commit: str) -> bool: def _commits_match(self, local_commit:str, release_commit:str) -> bool:
"""Determine whether two commits refer to the same hash. """Determine whether two commits refer to the same hash.
This accounts for short vs. full hashes (e.g. 7 chars vs. 40 chars). This accounts for short vs. full hashes (e.g. 7 chars vs. 40 chars).
@@ -120,7 +120,7 @@ class UpdateChecker:
return True return True
return len(release_commit) < len(local_commit) and local_commit.startswith(release_commit) return len(release_commit) < len(local_commit) and local_commit.startswith(release_commit)
def check_for_updates(self, *, skip_interval_check: bool = False) -> None: def check_for_updates(self, *, skip_interval_check:bool = False) -> None:
"""Check for updates to the bot. """Check for updates to the bot.
Args: Args:
@@ -147,7 +147,7 @@ class UpdateChecker:
try: try:
if self.config.update_check.channel == "latest": if self.config.update_check.channel == "latest":
# Use /releases/latest endpoint for stable releases # Use /releases/latest endpoint for stable releases
response = requests.get("https://api.github.com/repos/Second-Hand-Friends/kleinanzeigen-bot/releases/latest", timeout=self._request_timeout()) response = requests.get("https://api.github.com/repos/Second-Hand-Friends/kleinanzeigen-bot/releases/latest", timeout = self._request_timeout())
response.raise_for_status() response.raise_for_status()
release = response.json() release = response.json()
# Defensive: ensure it's not a prerelease # Defensive: ensure it's not a prerelease
@@ -156,7 +156,7 @@ class UpdateChecker:
return return
elif self.config.update_check.channel == "preview": elif self.config.update_check.channel == "preview":
# Use /releases endpoint and select the most recent prerelease # Use /releases endpoint and select the most recent prerelease
response = requests.get("https://api.github.com/repos/Second-Hand-Friends/kleinanzeigen-bot/releases", timeout=self._request_timeout()) response = requests.get("https://api.github.com/repos/Second-Hand-Friends/kleinanzeigen-bot/releases", timeout = self._request_timeout())
response.raise_for_status() response.raise_for_status()
releases = response.json() releases = response.json()
# Find the most recent prerelease # Find the most recent prerelease

View File

@@ -4,6 +4,7 @@
import asyncio, enum, inspect, json, os, platform, secrets, shutil, subprocess, urllib.request # isort: skip # noqa: S404 import asyncio, enum, inspect, json, os, platform, secrets, shutil, subprocess, urllib.request # isort: skip # noqa: S404
from collections.abc import Awaitable, Callable, Coroutine, Iterable from collections.abc import Awaitable, Callable, Coroutine, Iterable
from gettext import gettext as _ from gettext import gettext as _
from pathlib import Path
from typing import Any, Final, Optional, cast from typing import Any, Final, Optional, cast
try: try:
@@ -22,7 +23,7 @@ from nodriver.core.tab import Tab as Page
from kleinanzeigen_bot.model.config_model import Config as BotConfig from kleinanzeigen_bot.model.config_model import Config as BotConfig
from kleinanzeigen_bot.model.config_model import TimeoutConfig from kleinanzeigen_bot.model.config_model import TimeoutConfig
from . import files, loggers, net from . import files, loggers, net, xdg_paths
from .chrome_version_detector import ( from .chrome_version_detector import (
ChromeVersionInfo, ChromeVersionInfo,
detect_chrome_version_from_binary, detect_chrome_version_from_binary,
@@ -40,6 +41,28 @@ if TYPE_CHECKING:
_KEY_VALUE_PAIR_SIZE = 2 _KEY_VALUE_PAIR_SIZE = 2
def _resolve_user_data_dir_paths(arg_value:str, config_value:str) -> tuple[Any, Any]:
"""Resolve the argument and config user_data_dir paths for comparison."""
try:
return (
Path(arg_value).expanduser().resolve(),
Path(config_value).expanduser().resolve(),
)
except OSError as exc:
LOG.debug("Failed to resolve user_data_dir paths for comparison: %s", exc)
return None, None
def _has_non_empty_user_data_dir_arg(args:Iterable[str]) -> bool:
for arg in args:
if not arg.startswith("--user-data-dir="):
continue
raw = arg.split("=", maxsplit = 1)[1].strip().strip('"').strip("'")
if raw:
return True
return False
def _is_remote_object(obj:Any) -> TypeGuard["RemoteObject"]: def _is_remote_object(obj:Any) -> TypeGuard["RemoteObject"]:
"""Type guard to check if an object is a RemoteObject.""" """Type guard to check if an object is a RemoteObject."""
return hasattr(obj, "__class__") and "RemoteObject" in str(type(obj)) return hasattr(obj, "__class__") and "RemoteObject" in str(type(obj))
@@ -58,7 +81,7 @@ __all__ = [
LOG:Final[loggers.Logger] = loggers.get_logger(__name__) LOG:Final[loggers.Logger] = loggers.get_logger(__name__)
# see https://api.jquery.com/category/selectors/ # see https://api.jquery.com/category/selectors/
METACHAR_ESCAPER:Final[dict[int, str]] = str.maketrans({ch: f"\\{ch}" for ch in '!"#$%&\'()*+,./:;<=>?@[\\]^`{|}~'}) METACHAR_ESCAPER:Final[dict[int, str]] = str.maketrans({ch: f"\\{ch}" for ch in "!\"#$%&'()*+,./:;<=>?@[\\]^`{|}~"})
def _is_admin() -> bool: def _is_admin() -> bool:
@@ -90,7 +113,6 @@ class Is(enum.Enum):
class BrowserConfig: class BrowserConfig:
def __init__(self) -> None: def __init__(self) -> None:
self.arguments:Iterable[str] = [] self.arguments:Iterable[str] = []
self.binary_location:str | None = None self.binary_location:str | None = None
@@ -102,37 +124,27 @@ class BrowserConfig:
def _write_initial_prefs(prefs_file:str) -> None: def _write_initial_prefs(prefs_file:str) -> None:
with open(prefs_file, "w", encoding = "UTF-8") as fd: with open(prefs_file, "w", encoding = "UTF-8") as fd:
json.dump({ json.dump(
"credentials_enable_service": False, {
"enable_do_not_track": True, "credentials_enable_service": False,
"google": { "enable_do_not_track": True,
"services": { "google": {"services": {"consented_to_sync": False}},
"consented_to_sync": False "profile": {
} "default_content_setting_values": {
}, "popups": 0,
"profile": { "notifications": 2, # 1 = allow, 2 = block browser notifications
"default_content_setting_values": { },
"popups": 0, "password_manager_enabled": False,
"notifications": 2 # 1 = allow, 2 = block browser notifications
}, },
"password_manager_enabled": False "signin": {"allowed": False},
"translate_site_blacklist": ["www.kleinanzeigen.de"],
"devtools": {"preferences": {"currentDockState": '"bottom"'}},
}, },
"signin": { fd,
"allowed": False )
},
"translate_site_blacklist": [
"www.kleinanzeigen.de"
],
"devtools": {
"preferences": {
"currentDockState": '"bottom"'
}
}
}, fd)
class WebScrapingMixin: class WebScrapingMixin:
def __init__(self) -> None: def __init__(self) -> None:
self.browser_config:Final[BrowserConfig] = BrowserConfig() self.browser_config:Final[BrowserConfig] = BrowserConfig()
self.browser:Browser = None # pyright: ignore[reportAttributeAccessIssue] self.browser:Browser = None # pyright: ignore[reportAttributeAccessIssue]
@@ -140,6 +152,11 @@ class WebScrapingMixin:
self._default_timeout_config:TimeoutConfig | None = None self._default_timeout_config:TimeoutConfig | None = None
self.config:BotConfig = cast(BotConfig, None) self.config:BotConfig = cast(BotConfig, None)
@property
def _installation_mode(self) -> str:
"""Get installation mode with fallback to portable."""
return getattr(self, "installation_mode_or_portable", "portable")
def _get_timeout_config(self) -> TimeoutConfig: def _get_timeout_config(self) -> TimeoutConfig:
config = getattr(self, "config", None) config = getattr(self, "config", None)
timeouts:TimeoutConfig | None = None timeouts:TimeoutConfig | None = None
@@ -172,12 +189,7 @@ class WebScrapingMixin:
return 1 + cfg.retry_max_attempts return 1 + cfg.retry_max_attempts
async def _run_with_timeout_retries( async def _run_with_timeout_retries(
self, self, operation:Callable[[float], Awaitable[T]], *, description:str, key:str = "default", override:float | None = None
operation:Callable[[float], Awaitable[T]],
*,
description:str,
key:str = "default",
override:float | None = None
) -> T: ) -> T:
""" """
Execute an async callable with retry/backoff handling for TimeoutError. Execute an async callable with retry/backoff handling for TimeoutError.
@@ -191,13 +203,7 @@ class WebScrapingMixin:
except TimeoutError: except TimeoutError:
if attempt >= attempts - 1: if attempt >= attempts - 1:
raise raise
LOG.debug( LOG.debug("Retrying %s after TimeoutError (attempt %d/%d, timeout %.1fs)", description, attempt + 1, attempts, effective_timeout)
"Retrying %s after TimeoutError (attempt %d/%d, timeout %.1fs)",
description,
attempt + 1,
attempts,
effective_timeout
)
raise TimeoutError(f"{description} failed without executing operation") raise TimeoutError(f"{description} failed without executing operation")
@@ -210,8 +216,25 @@ class WebScrapingMixin:
self.browser_config.binary_location = self.get_compatible_browser() self.browser_config.binary_location = self.get_compatible_browser()
LOG.info(" -> Browser binary location: %s", self.browser_config.binary_location) LOG.info(" -> Browser binary location: %s", self.browser_config.binary_location)
has_remote_debugging = any(arg.startswith("--remote-debugging-port=") for arg in self.browser_config.arguments)
is_test_environment = bool(os.environ.get("PYTEST_CURRENT_TEST"))
if (
not (self.browser_config.user_data_dir and self.browser_config.user_data_dir.strip())
and not _has_non_empty_user_data_dir_arg(self.browser_config.arguments)
and not has_remote_debugging
and not is_test_environment
):
self.browser_config.user_data_dir = str(xdg_paths.get_browser_profile_path(self._installation_mode))
# Chrome version detection and validation # Chrome version detection and validation
await self._validate_chrome_version_configuration() if has_remote_debugging:
try:
await self._validate_chrome_version_configuration()
except AssertionError as exc:
LOG.warning(_("Remote debugging detected, but browser configuration looks invalid: %s"), exc)
else:
await self._validate_chrome_version_configuration()
######################################################## ########################################################
# check if an existing browser instance shall be used... # check if an existing browser instance shall be used...
@@ -229,10 +252,12 @@ class WebScrapingMixin:
# Enhanced port checking with retry logic # Enhanced port checking with retry logic
port_available = await self._check_port_with_retry(remote_host, remote_port) port_available = await self._check_port_with_retry(remote_host, remote_port)
ensure(port_available, ensure(
port_available,
f"Browser process not reachable at {remote_host}:{remote_port}. " f"Browser process not reachable at {remote_host}:{remote_port}. "
f"Start the browser with --remote-debugging-port={remote_port} or remove this port from your config.yaml. " f"Start the browser with --remote-debugging-port={remote_port} or remove this port from your config.yaml. "
f"Make sure the browser is running and the port is not blocked by firewall.") f"Make sure the browser is running and the port is not blocked by firewall.",
)
try: try:
cfg = NodriverConfig( cfg = NodriverConfig(
@@ -255,8 +280,7 @@ class WebScrapingMixin:
LOG.error("Troubleshooting steps:") LOG.error("Troubleshooting steps:")
LOG.error("1. Close all browser instances and try again") LOG.error("1. Close all browser instances and try again")
LOG.error("2. Remove the user_data_dir configuration temporarily") LOG.error("2. Remove the user_data_dir configuration temporarily")
LOG.error("3. Start browser manually with: %s --remote-debugging-port=%d", LOG.error("3. Start browser manually with: %s --remote-debugging-port=%d", self.browser_config.binary_location, remote_port)
self.browser_config.binary_location, remote_port)
LOG.error("4. Check if any antivirus or security software is blocking the connection") LOG.error("4. Check if any antivirus or security software is blocking the connection")
raise raise
@@ -274,13 +298,11 @@ class WebScrapingMixin:
"--disable-sync", "--disable-sync",
"--no-experiments", "--no-experiments",
"--disable-search-engine-choice-screen", "--disable-search-engine-choice-screen",
"--disable-features=MediaRouter", "--disable-features=MediaRouter",
"--use-mock-keychain", "--use-mock-keychain",
"--test-type", # https://stackoverflow.com/a/36746675/5116073 "--test-type", # https://stackoverflow.com/a/36746675/5116073
# https://chromium.googlesource.com/chromium/src/+/master/net/dns/README.md#request-remapping # https://chromium.googlesource.com/chromium/src/+/master/net/dns/README.md#request-remapping
'--host-resolver-rules="MAP connect.facebook.net 127.0.0.1, MAP securepubads.g.doubleclick.net 127.0.0.1, MAP www.googletagmanager.com 127.0.0.1"' '--host-resolver-rules="MAP connect.facebook.net 127.0.0.1, MAP securepubads.g.doubleclick.net 127.0.0.1, MAP www.googletagmanager.com 127.0.0.1"',
] ]
is_edge = "edge" in self.browser_config.binary_location.lower() is_edge = "edge" in self.browser_config.binary_location.lower()
@@ -295,10 +317,36 @@ class WebScrapingMixin:
LOG.info(" -> Browser profile name: %s", self.browser_config.profile_name) LOG.info(" -> Browser profile name: %s", self.browser_config.profile_name)
browser_args.append(f"--profile-directory={self.browser_config.profile_name}") browser_args.append(f"--profile-directory={self.browser_config.profile_name}")
user_data_dir_from_args:str | None = None
for browser_arg in self.browser_config.arguments: for browser_arg in self.browser_config.arguments:
LOG.info(" -> Custom Browser argument: %s", browser_arg) LOG.info(" -> Custom Browser argument: %s", browser_arg)
if browser_arg.startswith("--user-data-dir="):
raw = browser_arg.split("=", maxsplit = 1)[1].strip().strip('"').strip("'")
if not raw:
LOG.warning(_("Ignoring empty --user-data-dir= argument; falling back to configured user_data_dir."))
continue
user_data_dir_from_args = raw
continue
browser_args.append(browser_arg) browser_args.append(browser_arg)
effective_user_data_dir = user_data_dir_from_args or self.browser_config.user_data_dir
if user_data_dir_from_args and self.browser_config.user_data_dir:
arg_path, cfg_path = await asyncio.get_running_loop().run_in_executor(
None,
_resolve_user_data_dir_paths,
user_data_dir_from_args,
self.browser_config.user_data_dir,
)
if arg_path is None or cfg_path is None or arg_path != cfg_path:
LOG.warning(
_("Configured browser.user_data_dir (%s) does not match --user-data-dir argument (%s); using the argument value."),
self.browser_config.user_data_dir,
user_data_dir_from_args,
)
if not effective_user_data_dir and not is_test_environment:
effective_user_data_dir = str(xdg_paths.get_browser_profile_path(self._installation_mode))
self.browser_config.user_data_dir = effective_user_data_dir
if not loggers.is_debug(LOG): if not loggers.is_debug(LOG):
browser_args.append("--log-level=3") # INFO: 0, WARNING: 1, ERROR: 2, FATAL: 3 browser_args.append("--log-level=3") # INFO: 0, WARNING: 1, ERROR: 2, FATAL: 3
@@ -309,7 +357,7 @@ class WebScrapingMixin:
headless = False, headless = False,
browser_executable_path = self.browser_config.binary_location, browser_executable_path = self.browser_config.binary_location,
browser_args = browser_args, browser_args = browser_args,
user_data_dir = self.browser_config.user_data_dir user_data_dir = self.browser_config.user_data_dir,
) )
# already logged by nodriver: # already logged by nodriver:
@@ -371,8 +419,7 @@ class WebScrapingMixin:
return True return True
if attempt < max_retries - 1: if attempt < max_retries - 1:
LOG.debug("Port %s:%s not available, retrying in %.1f seconds (attempt %d/%d)", LOG.debug("Port %s:%s not available, retrying in %.1f seconds (attempt %d/%d)", host, port, retry_delay, attempt + 1, max_retries)
host, port, retry_delay, attempt + 1, max_retries)
await asyncio.sleep(retry_delay) await asyncio.sleep(retry_delay)
return False return False
@@ -522,12 +569,7 @@ class WebScrapingMixin:
browser_paths:list[str | None] = [] browser_paths:list[str | None] = []
match platform.system(): match platform.system():
case "Linux": case "Linux":
browser_paths = [ browser_paths = [shutil.which("chromium"), shutil.which("chromium-browser"), shutil.which("google-chrome"), shutil.which("microsoft-edge")]
shutil.which("chromium"),
shutil.which("chromium-browser"),
shutil.which("google-chrome"),
shutil.which("microsoft-edge")
]
case "Darwin": case "Darwin":
browser_paths = [ browser_paths = [
@@ -540,18 +582,15 @@ class WebScrapingMixin:
browser_paths = [ browser_paths = [
os.environ.get("PROGRAMFILES", "C:\\Program Files") + r"\Microsoft\Edge\Application\msedge.exe", os.environ.get("PROGRAMFILES", "C:\\Program Files") + r"\Microsoft\Edge\Application\msedge.exe",
os.environ.get("PROGRAMFILES(X86)", "C:\\Program Files (x86)") + r"\Microsoft\Edge\Application\msedge.exe", os.environ.get("PROGRAMFILES(X86)", "C:\\Program Files (x86)") + r"\Microsoft\Edge\Application\msedge.exe",
os.environ["PROGRAMFILES"] + r"\Chromium\Application\chrome.exe", os.environ["PROGRAMFILES"] + r"\Chromium\Application\chrome.exe",
os.environ["PROGRAMFILES(X86)"] + r"\Chromium\Application\chrome.exe", os.environ["PROGRAMFILES(X86)"] + r"\Chromium\Application\chrome.exe",
os.environ["LOCALAPPDATA"] + r"\Chromium\Application\chrome.exe", os.environ["LOCALAPPDATA"] + r"\Chromium\Application\chrome.exe",
os.environ["PROGRAMFILES"] + r"\Chrome\Application\chrome.exe", os.environ["PROGRAMFILES"] + r"\Chrome\Application\chrome.exe",
os.environ["PROGRAMFILES(X86)"] + r"\Chrome\Application\chrome.exe", os.environ["PROGRAMFILES(X86)"] + r"\Chrome\Application\chrome.exe",
os.environ["LOCALAPPDATA"] + r"\Chrome\Application\chrome.exe", os.environ["LOCALAPPDATA"] + r"\Chrome\Application\chrome.exe",
shutil.which("msedge.exe"), shutil.which("msedge.exe"),
shutil.which("chromium.exe"), shutil.which("chromium.exe"),
shutil.which("chrome.exe") shutil.which("chrome.exe"),
] ]
case _ as os_name: case _ as os_name:
@@ -563,8 +602,14 @@ class WebScrapingMixin:
raise AssertionError(_("Installed browser could not be detected")) raise AssertionError(_("Installed browser could not be detected"))
async def web_await(self, condition:Callable[[], T | Never | Coroutine[Any, Any, T | Never]], *, async def web_await(
timeout:int | float | None = None, timeout_error_message:str = "", apply_multiplier:bool = True) -> T: self,
condition:Callable[[], T | Never | Coroutine[Any, Any, T | Never]],
*,
timeout:int | float | None = None,
timeout_error_message:str = "",
apply_multiplier:bool = True,
) -> T:
""" """
Blocks/waits until the given condition is met. Blocks/waits until the given condition is met.
@@ -604,7 +649,9 @@ class WebScrapingMixin:
return elem.attrs.get("disabled") is not None return elem.attrs.get("disabled") is not None
async def is_displayed(elem:Element) -> bool: async def is_displayed(elem:Element) -> bool:
return cast(bool, await elem.apply(""" return cast(
bool,
await elem.apply("""
function (element) { function (element) {
var style = window.getComputedStyle(element); var style = window.getComputedStyle(element);
return style.display !== 'none' return style.display !== 'none'
@@ -613,7 +660,8 @@ class WebScrapingMixin:
&& element.offsetWidth > 0 && element.offsetWidth > 0
&& element.offsetHeight > 0 && element.offsetHeight > 0
} }
""")) """),
)
elem:Element = await self.web_find(selector_type, selector_value, timeout = timeout) elem:Element = await self.web_find(selector_type, selector_value, timeout = timeout)
@@ -627,7 +675,9 @@ class WebScrapingMixin:
case Is.READONLY: case Is.READONLY:
return elem.attrs.get("readonly") is not None return elem.attrs.get("readonly") is not None
case Is.SELECTED: case Is.SELECTED:
return cast(bool, await elem.apply(""" return cast(
bool,
await elem.apply("""
function (element) { function (element) {
if (element.tagName.toLowerCase() === 'input') { if (element.tagName.toLowerCase() === 'input') {
if (element.type === 'checkbox' || element.type === 'radio') { if (element.type === 'checkbox' || element.type === 'radio') {
@@ -636,7 +686,8 @@ class WebScrapingMixin:
} }
return false return false
} }
""")) """),
)
raise AssertionError(_("Unsupported attribute: %s") % attr) raise AssertionError(_("Unsupported attribute: %s") % attr)
async def web_click(self, selector_type:By, selector_value:str, *, timeout:int | float | None = None) -> Element: async def web_click(self, selector_type:By, selector_value:str, *, timeout:int | float | None = None) -> Element:
@@ -743,11 +794,8 @@ class WebScrapingMixin:
async def attempt(effective_timeout:float) -> Element: async def attempt(effective_timeout:float) -> Element:
return await self._web_find_once(selector_type, selector_value, effective_timeout, parent = parent) return await self._web_find_once(selector_type, selector_value, effective_timeout, parent = parent)
return await self._run_with_timeout_retries( return await self._run_with_timeout_retries( # noqa: E501
attempt, attempt, description = f"web_find({selector_type.name}, {selector_value})", key = "default", override = timeout
description = f"web_find({selector_type.name}, {selector_value})",
key = "default",
override = timeout
) )
async def web_find_all(self, selector_type:By, selector_value:str, *, parent:Element | None = None, timeout:int | float | None = None) -> list[Element]: async def web_find_all(self, selector_type:By, selector_value:str, *, parent:Element | None = None, timeout:int | float | None = None) -> list[Element]:
@@ -762,10 +810,7 @@ class WebScrapingMixin:
return await self._web_find_all_once(selector_type, selector_value, effective_timeout, parent = parent) return await self._web_find_all_once(selector_type, selector_value, effective_timeout, parent = parent)
return await self._run_with_timeout_retries( return await self._run_with_timeout_retries(
attempt, attempt, description = f"web_find_all({selector_type.name}, {selector_value})", key = "default", override = timeout
description = f"web_find_all({selector_type.name}, {selector_value})",
key = "default",
override = timeout
) )
async def _web_find_once(self, selector_type:By, selector_value:str, timeout:float, *, parent:Element | None = None) -> Element: async def _web_find_once(self, selector_type:By, selector_value:str, timeout:float, *, parent:Element | None = None) -> Element:
@@ -778,40 +823,46 @@ class WebScrapingMixin:
lambda: self.page.query_selector(f"#{escaped_id}", parent), lambda: self.page.query_selector(f"#{escaped_id}", parent),
timeout = timeout, timeout = timeout,
timeout_error_message = f"No HTML element found with ID '{selector_value}'{timeout_suffix}", timeout_error_message = f"No HTML element found with ID '{selector_value}'{timeout_suffix}",
apply_multiplier = False) apply_multiplier = False,
)
case By.CLASS_NAME: case By.CLASS_NAME:
escaped_classname = selector_value.translate(METACHAR_ESCAPER) escaped_classname = selector_value.translate(METACHAR_ESCAPER)
return await self.web_await( return await self.web_await(
lambda: self.page.query_selector(f".{escaped_classname}", parent), lambda: self.page.query_selector(f".{escaped_classname}", parent),
timeout = timeout, timeout = timeout,
timeout_error_message = f"No HTML element found with CSS class '{selector_value}'{timeout_suffix}", timeout_error_message = f"No HTML element found with CSS class '{selector_value}'{timeout_suffix}",
apply_multiplier = False) apply_multiplier = False,
)
case By.TAG_NAME: case By.TAG_NAME:
return await self.web_await( return await self.web_await(
lambda: self.page.query_selector(selector_value, parent), lambda: self.page.query_selector(selector_value, parent),
timeout = timeout, timeout = timeout,
timeout_error_message = f"No HTML element found of tag <{selector_value}>{timeout_suffix}", timeout_error_message = f"No HTML element found of tag <{selector_value}>{timeout_suffix}",
apply_multiplier = False) apply_multiplier = False,
)
case By.CSS_SELECTOR: case By.CSS_SELECTOR:
return await self.web_await( return await self.web_await(
lambda: self.page.query_selector(selector_value, parent), lambda: self.page.query_selector(selector_value, parent),
timeout = timeout, timeout = timeout,
timeout_error_message = f"No HTML element found using CSS selector '{selector_value}'{timeout_suffix}", timeout_error_message = f"No HTML element found using CSS selector '{selector_value}'{timeout_suffix}",
apply_multiplier = False) apply_multiplier = False,
)
case By.TEXT: case By.TEXT:
ensure(not parent, f"Specifying a parent element currently not supported with selector type: {selector_type}") ensure(not parent, f"Specifying a parent element currently not supported with selector type: {selector_type}")
return await self.web_await( return await self.web_await(
lambda: self.page.find_element_by_text(selector_value, best_match = True), lambda: self.page.find_element_by_text(selector_value, best_match = True),
timeout = timeout, timeout = timeout,
timeout_error_message = f"No HTML element found containing text '{selector_value}'{timeout_suffix}", timeout_error_message = f"No HTML element found containing text '{selector_value}'{timeout_suffix}",
apply_multiplier = False) apply_multiplier = False,
)
case By.XPATH: case By.XPATH:
ensure(not parent, f"Specifying a parent element currently not supported with selector type: {selector_type}") ensure(not parent, f"Specifying a parent element currently not supported with selector type: {selector_type}")
return await self.web_await( return await self.web_await(
lambda: self.page.find_element_by_text(selector_value, best_match = True), lambda: self.page.find_element_by_text(selector_value, best_match = True),
timeout = timeout, timeout = timeout,
timeout_error_message = f"No HTML element found using XPath '{selector_value}'{timeout_suffix}", timeout_error_message = f"No HTML element found using XPath '{selector_value}'{timeout_suffix}",
apply_multiplier = False) apply_multiplier = False,
)
raise AssertionError(_("Unsupported selector type: %s") % selector_type) raise AssertionError(_("Unsupported selector type: %s") % selector_type)
@@ -825,33 +876,38 @@ class WebScrapingMixin:
lambda: self.page.query_selector_all(f".{escaped_classname}", parent), lambda: self.page.query_selector_all(f".{escaped_classname}", parent),
timeout = timeout, timeout = timeout,
timeout_error_message = f"No HTML elements found with CSS class '{selector_value}'{timeout_suffix}", timeout_error_message = f"No HTML elements found with CSS class '{selector_value}'{timeout_suffix}",
apply_multiplier = False) apply_multiplier = False,
)
case By.CSS_SELECTOR: case By.CSS_SELECTOR:
return await self.web_await( return await self.web_await(
lambda: self.page.query_selector_all(selector_value, parent), lambda: self.page.query_selector_all(selector_value, parent),
timeout = timeout, timeout = timeout,
timeout_error_message = f"No HTML elements found using CSS selector '{selector_value}'{timeout_suffix}", timeout_error_message = f"No HTML elements found using CSS selector '{selector_value}'{timeout_suffix}",
apply_multiplier = False) apply_multiplier = False,
)
case By.TAG_NAME: case By.TAG_NAME:
return await self.web_await( return await self.web_await(
lambda: self.page.query_selector_all(selector_value, parent), lambda: self.page.query_selector_all(selector_value, parent),
timeout = timeout, timeout = timeout,
timeout_error_message = f"No HTML elements found of tag <{selector_value}>{timeout_suffix}", timeout_error_message = f"No HTML elements found of tag <{selector_value}>{timeout_suffix}",
apply_multiplier = False) apply_multiplier = False,
)
case By.TEXT: case By.TEXT:
ensure(not parent, f"Specifying a parent element currently not supported with selector type: {selector_type}") ensure(not parent, f"Specifying a parent element currently not supported with selector type: {selector_type}")
return await self.web_await( return await self.web_await(
lambda: self.page.find_elements_by_text(selector_value), lambda: self.page.find_elements_by_text(selector_value),
timeout = timeout, timeout = timeout,
timeout_error_message = f"No HTML elements found containing text '{selector_value}'{timeout_suffix}", timeout_error_message = f"No HTML elements found containing text '{selector_value}'{timeout_suffix}",
apply_multiplier = False) apply_multiplier = False,
)
case By.XPATH: case By.XPATH:
ensure(not parent, f"Specifying a parent element currently not supported with selector type: {selector_type}") ensure(not parent, f"Specifying a parent element currently not supported with selector type: {selector_type}")
return await self.web_await( return await self.web_await(
lambda: self.page.find_elements_by_text(selector_value), lambda: self.page.find_elements_by_text(selector_value),
timeout = timeout, timeout = timeout,
timeout_error_message = f"No HTML elements found using XPath '{selector_value}'{timeout_suffix}", timeout_error_message = f"No HTML elements found using XPath '{selector_value}'{timeout_suffix}",
apply_multiplier = False) apply_multiplier = False,
)
raise AssertionError(_("Unsupported selector type: %s") % selector_type) raise AssertionError(_("Unsupported selector type: %s") % selector_type)
@@ -885,11 +941,12 @@ class WebScrapingMixin:
lambda: self.web_execute("document.readyState == 'complete'"), lambda: self.web_execute("document.readyState == 'complete'"),
timeout = page_timeout, timeout = page_timeout,
timeout_error_message = f"Page did not finish loading within {page_timeout} seconds.", timeout_error_message = f"Page did not finish loading within {page_timeout} seconds.",
apply_multiplier = False apply_multiplier = False,
) )
async def web_text(self, selector_type:By, selector_value:str, *, parent:Element | None = None, timeout:int | float | None = None) -> str: async def web_text(self, selector_type:By, selector_value:str, *, parent:Element | None = None, timeout:int | float | None = None) -> str:
return str(await (await self.web_find(selector_type, selector_value, parent = parent, timeout = timeout)).apply(""" return str(
await (await self.web_find(selector_type, selector_value, parent = parent, timeout = timeout)).apply("""
function (elem) { function (elem) {
let sel = window.getSelection() let sel = window.getSelection()
sel.removeAllRanges() sel.removeAllRanges()
@@ -900,16 +957,19 @@ class WebScrapingMixin:
sel.removeAllRanges() sel.removeAllRanges()
return visibleText return visibleText
} }
""")) """)
)
async def web_sleep(self, min_ms:int = 1_000, max_ms:int = 2_500) -> None: async def web_sleep(self, min_ms:int = 1_000, max_ms:int = 2_500) -> None:
duration = max_ms <= min_ms and min_ms or secrets.randbelow(max_ms - min_ms) + min_ms duration = max_ms <= min_ms and min_ms or secrets.randbelow(max_ms - min_ms) + min_ms
LOG.log(loggers.INFO if duration > 1_500 else loggers.DEBUG, # noqa: PLR2004 Magic value used in comparison LOG.log(
" ... pausing for %d ms ...", duration) loggers.INFO if duration > 1_500 else loggers.DEBUG, # noqa: PLR2004 Magic value used in comparison
" ... pausing for %d ms ...",
duration,
)
await self.page.sleep(duration / 1_000) await self.page.sleep(duration / 1_000)
async def web_request(self, url:str, method:str = "GET", valid_response_codes:int | Iterable[int] = 200, async def web_request(self, url:str, method:str = "GET", valid_response_codes:int | Iterable[int] = 200, headers:dict[str, str] | None = None) -> Any:
headers:dict[str, str] | None = None) -> Any:
method = method.upper() method = method.upper()
LOG.debug(" -> HTTP %s [%s]...", method, url) LOG.debug(" -> HTTP %s [%s]...", method, url)
response = await self.web_execute(f""" response = await self.web_execute(f"""
@@ -933,9 +993,10 @@ class WebScrapingMixin:
valid_response_codes = [valid_response_codes] valid_response_codes = [valid_response_codes]
ensure( ensure(
response["statusCode"] in valid_response_codes, response["statusCode"] in valid_response_codes,
f'Invalid response "{response["statusCode"]} response["statusMessage"]" received for HTTP {method} to {url}' f'Invalid response "{response["statusCode"]} {response["statusMessage"]}" received for HTTP {method} to {url}',
) )
return response return response
# pylint: enable=dangerous-default-value # pylint: enable=dangerous-default-value
async def web_scroll_page_down(self, scroll_length:int = 10, scroll_speed:int = 10_000, *, scroll_back_top:bool = False) -> None: async def web_scroll_page_down(self, scroll_length:int = 10, scroll_speed:int = 10_000, *, scroll_back_top:bool = False) -> None:
@@ -968,8 +1029,9 @@ class WebScrapingMixin:
:raises UnexpectedTagNameException: if element is not a <select> element :raises UnexpectedTagNameException: if element is not a <select> element
""" """
await self.web_await( await self.web_await(
lambda: self.web_check(selector_type, selector_value, Is.CLICKABLE), timeout = timeout, lambda: self.web_check(selector_type, selector_value, Is.CLICKABLE),
timeout_error_message = f"No clickable HTML element with selector: {selector_type}='{selector_value}' found" timeout = timeout,
timeout_error_message = f"No clickable HTML element with selector: {selector_type}='{selector_value}' found",
) )
elem = await self.web_find(selector_type, selector_value, timeout = timeout) elem = await self.web_find(selector_type, selector_value, timeout = timeout)
@@ -1107,9 +1169,7 @@ class WebScrapingMixin:
if port_available: if port_available:
try: try:
version_info = detect_chrome_version_from_remote_debugging( version_info = detect_chrome_version_from_remote_debugging(
remote_host, remote_host, remote_port, timeout = self._effective_timeout("chrome_remote_debugging")
remote_port,
timeout = self._effective_timeout("chrome_remote_debugging")
) )
if version_info: if version_info:
LOG.debug(" -> Detected version from existing browser: %s", version_info) LOG.debug(" -> Detected version from existing browser: %s", version_info)
@@ -1125,10 +1185,7 @@ class WebScrapingMixin:
binary_path = self.browser_config.binary_location binary_path = self.browser_config.binary_location
if binary_path: if binary_path:
LOG.debug(" -> No remote browser detected, trying binary detection") LOG.debug(" -> No remote browser detected, trying binary detection")
version_info = detect_chrome_version_from_binary( version_info = detect_chrome_version_from_binary(binary_path, timeout = self._effective_timeout("chrome_binary_detection"))
binary_path,
timeout = self._effective_timeout("chrome_binary_detection")
)
# Validate if Chrome 136+ detected # Validate if Chrome 136+ detected
if version_info and version_info.is_chrome_136_plus: if version_info and version_info.is_chrome_136_plus:
@@ -1158,14 +1215,8 @@ class WebScrapingMixin:
AssertionError: If configuration is invalid AssertionError: If configuration is invalid
""" """
# Check if user-data-dir is specified in arguments or configuration # Check if user-data-dir is specified in arguments or configuration
has_user_data_dir_arg = any( has_user_data_dir_arg = any(arg.startswith("--user-data-dir=") for arg in self.browser_config.arguments)
arg.startswith("--user-data-dir=") has_user_data_dir_config = self.browser_config.user_data_dir is not None and bool(self.browser_config.user_data_dir.strip())
for arg in self.browser_config.arguments
)
has_user_data_dir_config = (
self.browser_config.user_data_dir is not None and
self.browser_config.user_data_dir.strip()
)
if not has_user_data_dir_arg and not has_user_data_dir_config: if not has_user_data_dir_arg and not has_user_data_dir_config:
error_message = ( error_message = (
@@ -1198,14 +1249,18 @@ class WebScrapingMixin:
remote_host = "127.0.0.1", remote_host = "127.0.0.1",
remote_port = remote_port if remote_port > 0 else None, remote_port = remote_port if remote_port > 0 else None,
remote_timeout = self._effective_timeout("chrome_remote_debugging"), remote_timeout = self._effective_timeout("chrome_remote_debugging"),
binary_timeout = self._effective_timeout("chrome_binary_detection") binary_timeout = self._effective_timeout("chrome_binary_detection"),
) )
# Report binary detection results # Report binary detection results
if diagnostic_info["binary_detection"]: if diagnostic_info["binary_detection"]:
binary_info = diagnostic_info["binary_detection"] binary_info = diagnostic_info["binary_detection"]
LOG.info("(info) %s version from binary: %s %s (major: %d)", LOG.info(
binary_info["browser_name"], binary_info["browser_name"], binary_info["version_string"], binary_info["major_version"]) "(info) %s version from binary: %s (major: %d)",
binary_info["browser_name"],
binary_info["version_string"],
binary_info["major_version"],
)
if binary_info["is_chrome_136_plus"]: if binary_info["is_chrome_136_plus"]:
LOG.info("(info) %s 136+ detected - security validation required", binary_info["browser_name"]) LOG.info("(info) %s 136+ detected - security validation required", binary_info["browser_name"])
@@ -1215,17 +1270,18 @@ class WebScrapingMixin:
# Report remote detection results # Report remote detection results
if diagnostic_info["remote_detection"]: if diagnostic_info["remote_detection"]:
remote_info = diagnostic_info["remote_detection"] remote_info = diagnostic_info["remote_detection"]
LOG.info("(info) %s version from remote debugging: %s %s (major: %d)", LOG.info(
remote_info["browser_name"], remote_info["browser_name"], remote_info["version_string"], remote_info["major_version"]) "(info) %s version from remote debugging: %s (major: %d)",
remote_info["browser_name"],
remote_info["version_string"],
remote_info["major_version"],
)
if remote_info["is_chrome_136_plus"]: if remote_info["is_chrome_136_plus"]:
LOG.info("(info) Remote %s 136+ detected - validating configuration", remote_info["browser_name"]) LOG.info("(info) Remote %s 136+ detected - validating configuration", remote_info["browser_name"])
# Validate configuration for Chrome/Edge 136+ # Validate configuration for Chrome/Edge 136+
is_valid, error_message = validate_chrome_136_configuration( is_valid, error_message = validate_chrome_136_configuration(list(self.browser_config.arguments), self.browser_config.user_data_dir)
list(self.browser_config.arguments),
self.browser_config.user_data_dir
)
if not is_valid: if not is_valid:
LOG.error("(fail) %s 136+ configuration validation failed: %s", remote_info["browser_name"], error_message) LOG.error("(fail) %s 136+ configuration validation failed: %s", remote_info["browser_name"], error_message)

View File

@@ -20,26 +20,26 @@ import platformdirs
from kleinanzeigen_bot.utils import loggers from kleinanzeigen_bot.utils import loggers
LOG: Final[loggers.Logger] = loggers.get_logger(__name__) LOG:Final[loggers.Logger] = loggers.get_logger(__name__)
APP_NAME: Final[str] = "kleinanzeigen-bot" APP_NAME:Final[str] = "kleinanzeigen-bot"
InstallationMode = Literal["portable", "xdg"] InstallationMode = Literal["portable", "xdg"]
PathCategory = Literal["config", "cache", "state"] PathCategory = Literal["config", "cache", "state"]
def _normalize_mode(mode: str | InstallationMode) -> InstallationMode: def _normalize_mode(mode:str | InstallationMode) -> InstallationMode:
"""Validate and normalize installation mode input.""" """Validate and normalize installation mode input."""
if mode in {"portable", "xdg"}: if mode in {"portable", "xdg"}:
return cast(InstallationMode, mode) return cast(InstallationMode, mode)
raise ValueError(f"Unsupported installation mode: {mode}") raise ValueError(f"Unsupported installation mode: {mode}")
def _ensure_directory(path: Path, description: str) -> None: def _ensure_directory(path:Path, description:str) -> None:
"""Create directory and verify it exists.""" """Create directory and verify it exists."""
LOG.debug("Creating directory: %s", path) LOG.debug("Creating directory: %s", path)
try: try:
path.mkdir(parents=True, exist_ok=True) path.mkdir(parents = True, exist_ok = True)
except OSError as exc: except OSError as exc:
LOG.error("Failed to create %s %s: %s", description, path, exc) LOG.error("Failed to create %s %s: %s", description, path, exc)
raise raise
@@ -47,7 +47,7 @@ def _ensure_directory(path: Path, description: str) -> None:
raise NotADirectoryError(str(path)) raise NotADirectoryError(str(path))
def get_xdg_base_dir(category: PathCategory) -> Path: def get_xdg_base_dir(category:PathCategory) -> Path:
"""Get XDG base directory for the given category. """Get XDG base directory for the given category.
Args: Args:
@@ -56,7 +56,7 @@ def get_xdg_base_dir(category: PathCategory) -> Path:
Returns: Returns:
Path to the XDG base directory for this app Path to the XDG base directory for this app
""" """
resolved: str | None = None resolved:str | None = None
match category: match category:
case "config": case "config":
resolved = platformdirs.user_config_dir(APP_NAME) resolved = platformdirs.user_config_dir(APP_NAME)
@@ -130,7 +130,7 @@ def prompt_installation_mode() -> InstallationMode:
return "portable" return "portable"
if choice == "1": if choice == "1":
mode: InstallationMode = "portable" mode:InstallationMode = "portable"
LOG.info("User selected installation mode: %s", mode) LOG.info("User selected installation mode: %s", mode)
return mode return mode
if choice == "2": if choice == "2":
@@ -140,7 +140,7 @@ def prompt_installation_mode() -> InstallationMode:
print(_("Invalid choice. Please enter 1 or 2.")) print(_("Invalid choice. Please enter 1 or 2."))
def get_config_file_path(mode: str | InstallationMode) -> Path: def get_config_file_path(mode:str | InstallationMode) -> Path:
"""Get config.yaml file path for the given mode. """Get config.yaml file path for the given mode.
Args: Args:
@@ -156,7 +156,7 @@ def get_config_file_path(mode: str | InstallationMode) -> Path:
return config_path return config_path
def get_ad_files_search_dir(mode: str | InstallationMode) -> Path: def get_ad_files_search_dir(mode:str | InstallationMode) -> Path:
"""Get directory to search for ad files. """Get directory to search for ad files.
Ad files are searched relative to the config file directory, Ad files are searched relative to the config file directory,
@@ -175,7 +175,7 @@ def get_ad_files_search_dir(mode: str | InstallationMode) -> Path:
return search_dir return search_dir
def get_downloaded_ads_path(mode: str | InstallationMode) -> Path: def get_downloaded_ads_path(mode:str | InstallationMode) -> Path:
"""Get downloaded ads directory path. """Get downloaded ads directory path.
Args: Args:
@@ -198,7 +198,7 @@ def get_downloaded_ads_path(mode: str | InstallationMode) -> Path:
return ads_path return ads_path
def get_browser_profile_path(mode: str | InstallationMode, config_override: str | None = None) -> Path: def get_browser_profile_path(mode:str | InstallationMode, config_override:str | None = None) -> Path:
"""Get browser profile directory path. """Get browser profile directory path.
Args: Args:
@@ -213,13 +213,13 @@ def get_browser_profile_path(mode: str | InstallationMode, config_override: str
""" """
mode = _normalize_mode(mode) mode = _normalize_mode(mode)
if config_override: if config_override:
profile_path = Path(config_override) profile_path = Path(config_override).expanduser().resolve()
LOG.debug("Resolving browser profile path for mode '%s' (config override): %s", mode, profile_path) LOG.debug("Resolving browser profile path for mode '%s' (config override): %s", mode, profile_path)
elif mode == "portable": elif mode == "portable":
profile_path = Path.cwd() / ".temp" / "browser-profile" profile_path = (Path.cwd() / ".temp" / "browser-profile").resolve()
LOG.debug("Resolving browser profile path for mode '%s': %s", mode, profile_path) LOG.debug("Resolving browser profile path for mode '%s': %s", mode, profile_path)
else: # xdg else: # xdg
profile_path = get_xdg_base_dir("cache") / "browser-profile" profile_path = (get_xdg_base_dir("cache") / "browser-profile").resolve()
LOG.debug("Resolving browser profile path for mode '%s': %s", mode, profile_path) LOG.debug("Resolving browser profile path for mode '%s': %s", mode, profile_path)
# Create directory if it doesn't exist # Create directory if it doesn't exist
@@ -228,7 +228,7 @@ def get_browser_profile_path(mode: str | InstallationMode, config_override: str
return profile_path return profile_path
def get_log_file_path(basename: str, mode: str | InstallationMode) -> Path: def get_log_file_path(basename:str, mode:str | InstallationMode) -> Path:
"""Get log file path. """Get log file path.
Args: Args:
@@ -249,7 +249,7 @@ def get_log_file_path(basename: str, mode: str | InstallationMode) -> Path:
return log_path return log_path
def get_update_check_state_path(mode: str | InstallationMode) -> Path: def get_update_check_state_path(mode:str | InstallationMode) -> Path:
"""Get update check state file path. """Get update check state file path.
Args: Args:

View File

@@ -33,7 +33,7 @@ async def atest_init() -> None:
web_scraping_mixin.close_browser_session() web_scraping_mixin.close_browser_session()
@pytest.mark.flaky(reruns = 4, reruns_delay = 5) @pytest.mark.flaky(reruns = 5, reruns_delay = 10)
@pytest.mark.itest @pytest.mark.itest
def test_init() -> None: def test_init() -> None:
nodriver.loop().run_until_complete(atest_init()) # type: ignore[attr-defined] nodriver.loop().run_until_complete(atest_init()) # type: ignore[attr-defined]

File diff suppressed because it is too large Load Diff

View File

@@ -641,6 +641,31 @@ class TestKleinanzeigenBotArgParsing:
test_bot.parse_args(["script.py", "help", "version"]) test_bot.parse_args(["script.py", "help", "version"])
assert exc_info.value.code == 2 assert exc_info.value.code == 2
def test_parse_args_explicit_flags(self, test_bot:KleinanzeigenBot, tmp_path:Path) -> None:
"""Test that explicit flags are set when --config and --logfile options are provided."""
config_path = tmp_path / "custom_config.yaml"
log_path = tmp_path / "custom.log"
# Test --config flag sets config_explicitly_provided
test_bot.parse_args(["script.py", "--config", str(config_path), "help"])
assert test_bot.config_explicitly_provided is True
assert str(config_path.absolute()) == test_bot.config_file_path
# Reset for next test
test_bot.config_explicitly_provided = False
# Test --logfile flag sets log_file_explicitly_provided
test_bot.parse_args(["script.py", "--logfile", str(log_path), "help"])
assert test_bot.log_file_explicitly_provided is True
assert str(log_path.absolute()) == test_bot.log_file_path
# Test both flags together
test_bot.config_explicitly_provided = False
test_bot.log_file_explicitly_provided = False
test_bot.parse_args(["script.py", "--config", str(config_path), "--logfile", str(log_path), "help"])
assert test_bot.config_explicitly_provided is True
assert test_bot.log_file_explicitly_provided is True
class TestKleinanzeigenBotCommands: class TestKleinanzeigenBotCommands:
"""Tests for command execution.""" """Tests for command execution."""
@@ -863,7 +888,7 @@ class TestKleinanzeigenBotAdDeletion:
async def test_delete_ad_by_title(self, test_bot:KleinanzeigenBot, minimal_ad_config:dict[str, Any]) -> None: async def test_delete_ad_by_title(self, test_bot:KleinanzeigenBot, minimal_ad_config:dict[str, Any]) -> None:
"""Test deleting an ad by title.""" """Test deleting an ad by title."""
test_bot.page = MagicMock() test_bot.page = MagicMock()
test_bot.page.evaluate = AsyncMock(return_value = {"statusCode": 200, "content": "{}"}) test_bot.page.evaluate = AsyncMock(return_value = {"statusCode": 200, "statusMessage": "OK", "content": "{}"})
test_bot.page.sleep = AsyncMock() test_bot.page.sleep = AsyncMock()
# Use minimal config since we only need title for deletion by title # Use minimal config since we only need title for deletion by title
@@ -891,7 +916,7 @@ class TestKleinanzeigenBotAdDeletion:
async def test_delete_ad_by_id(self, test_bot:KleinanzeigenBot, minimal_ad_config:dict[str, Any]) -> None: async def test_delete_ad_by_id(self, test_bot:KleinanzeigenBot, minimal_ad_config:dict[str, Any]) -> None:
"""Test deleting an ad by ID.""" """Test deleting an ad by ID."""
test_bot.page = MagicMock() test_bot.page = MagicMock()
test_bot.page.evaluate = AsyncMock(return_value = {"statusCode": 200, "content": "{}"}) test_bot.page.evaluate = AsyncMock(return_value = {"statusCode": 200, "statusMessage": "OK", "content": "{}"})
test_bot.page.sleep = AsyncMock() test_bot.page.sleep = AsyncMock()
# Create config with ID for deletion by ID # Create config with ID for deletion by ID
@@ -918,7 +943,7 @@ class TestKleinanzeigenBotAdDeletion:
async def test_delete_ad_by_id_with_non_string_csrf_token(self, test_bot:KleinanzeigenBot, minimal_ad_config:dict[str, Any]) -> None: async def test_delete_ad_by_id_with_non_string_csrf_token(self, test_bot:KleinanzeigenBot, minimal_ad_config:dict[str, Any]) -> None:
"""Test deleting an ad by ID with non-string CSRF token to cover str() conversion.""" """Test deleting an ad by ID with non-string CSRF token to cover str() conversion."""
test_bot.page = MagicMock() test_bot.page = MagicMock()
test_bot.page.evaluate = AsyncMock(return_value = {"statusCode": 200, "content": "{}"}) test_bot.page.evaluate = AsyncMock(return_value = {"statusCode": 200, "statusMessage": "OK", "content": "{}"})
test_bot.page.sleep = AsyncMock() test_bot.page.sleep = AsyncMock()
# Create config with ID for deletion by ID # Create config with ID for deletion by ID

View File

@@ -20,9 +20,7 @@ class TestWebScrapingMixinChromeVersionValidation:
return WebScrapingMixin() return WebScrapingMixin()
@patch("kleinanzeigen_bot.utils.web_scraping_mixin.detect_chrome_version_from_binary") @patch("kleinanzeigen_bot.utils.web_scraping_mixin.detect_chrome_version_from_binary")
async def test_validate_chrome_version_configuration_chrome_136_plus_valid( async def test_validate_chrome_version_configuration_chrome_136_plus_valid(self, mock_detect:Mock, scraper:WebScrapingMixin) -> None:
self, mock_detect:Mock, scraper:WebScrapingMixin
) -> None:
"""Test Chrome 136+ validation with valid configuration.""" """Test Chrome 136+ validation with valid configuration."""
# Setup mocks # Setup mocks
mock_detect.return_value = ChromeVersionInfo("136.0.6778.0", 136, "Chrome") mock_detect.return_value = ChromeVersionInfo("136.0.6778.0", 136, "Chrome")
@@ -88,9 +86,7 @@ class TestWebScrapingMixinChromeVersionValidation:
os.environ["PYTEST_CURRENT_TEST"] = original_env os.environ["PYTEST_CURRENT_TEST"] = original_env
@patch("kleinanzeigen_bot.utils.web_scraping_mixin.detect_chrome_version_from_binary") @patch("kleinanzeigen_bot.utils.web_scraping_mixin.detect_chrome_version_from_binary")
async def test_validate_chrome_version_configuration_chrome_pre_136( async def test_validate_chrome_version_configuration_chrome_pre_136(self, mock_detect:Mock, scraper:WebScrapingMixin) -> None:
self, mock_detect:Mock, scraper:WebScrapingMixin
) -> None:
"""Test Chrome pre-136 validation (no special requirements).""" """Test Chrome pre-136 validation (no special requirements)."""
# Setup mocks # Setup mocks
mock_detect.return_value = ChromeVersionInfo("120.0.6099.109", 120, "Chrome") mock_detect.return_value = ChromeVersionInfo("120.0.6099.109", 120, "Chrome")
@@ -121,11 +117,7 @@ class TestWebScrapingMixinChromeVersionValidation:
@patch("kleinanzeigen_bot.utils.chrome_version_detector.detect_chrome_version_from_binary") @patch("kleinanzeigen_bot.utils.chrome_version_detector.detect_chrome_version_from_binary")
@patch("kleinanzeigen_bot.utils.web_scraping_mixin.detect_chrome_version_from_remote_debugging") @patch("kleinanzeigen_bot.utils.web_scraping_mixin.detect_chrome_version_from_remote_debugging")
async def test_validate_chrome_version_logs_remote_detection( async def test_validate_chrome_version_logs_remote_detection(
self, self, mock_remote:Mock, mock_binary:Mock, scraper:WebScrapingMixin, caplog:pytest.LogCaptureFixture
mock_remote:Mock,
mock_binary:Mock,
scraper:WebScrapingMixin,
caplog:pytest.LogCaptureFixture
) -> None: ) -> None:
"""When a remote browser responds, the detected version should be logged.""" """When a remote browser responds, the detected version should be logged."""
mock_remote.return_value = ChromeVersionInfo("136.0.6778.0", 136, "Chrome") mock_remote.return_value = ChromeVersionInfo("136.0.6778.0", 136, "Chrome")
@@ -134,17 +126,14 @@ class TestWebScrapingMixinChromeVersionValidation:
scraper.browser_config.binary_location = "/path/to/chrome" scraper.browser_config.binary_location = "/path/to/chrome"
caplog.set_level("DEBUG") caplog.set_level("DEBUG")
with patch.dict(os.environ, {}, clear = True), \ with patch.dict(os.environ, {}, clear = True), patch.object(scraper, "_check_port_with_retry", return_value = True):
patch.object(scraper, "_check_port_with_retry", return_value = True):
await scraper._validate_chrome_version_configuration() await scraper._validate_chrome_version_configuration()
assert "Detected version from existing browser" in caplog.text assert "Detected version from existing browser" in caplog.text
mock_remote.assert_called_once() mock_remote.assert_called_once()
@patch("kleinanzeigen_bot.utils.chrome_version_detector.detect_chrome_version_from_binary") @patch("kleinanzeigen_bot.utils.chrome_version_detector.detect_chrome_version_from_binary")
async def test_validate_chrome_version_configuration_no_binary_location( async def test_validate_chrome_version_configuration_no_binary_location(self, mock_detect:Mock, scraper:WebScrapingMixin) -> None:
self, mock_detect:Mock, scraper:WebScrapingMixin
) -> None:
"""Test Chrome version validation when no binary location is set.""" """Test Chrome version validation when no binary location is set."""
# Configure scraper without binary location # Configure scraper without binary location
scraper.browser_config.binary_location = None scraper.browser_config.binary_location = None
@@ -204,15 +193,10 @@ class TestWebScrapingMixinChromeVersionDiagnostics:
"""Test Chrome version diagnostics with binary detection.""" """Test Chrome version diagnostics with binary detection."""
# Setup mocks # Setup mocks
mock_get_diagnostic.return_value = { mock_get_diagnostic.return_value = {
"binary_detection": { "binary_detection": {"version_string": "136.0.6778.0", "major_version": 136, "browser_name": "Chrome", "is_chrome_136_plus": True},
"version_string": "136.0.6778.0",
"major_version": 136,
"browser_name": "Chrome",
"is_chrome_136_plus": True
},
"remote_detection": None, "remote_detection": None,
"chrome_136_plus_detected": True, "chrome_136_plus_detected": True,
"recommendations": [] "recommendations": [],
} }
mock_validate.return_value = (True, "") mock_validate.return_value = (True, "")
@@ -230,7 +214,7 @@ class TestWebScrapingMixinChromeVersionDiagnostics:
scraper._diagnose_chrome_version_issues(9222) scraper._diagnose_chrome_version_issues(9222)
# Verify logs # Verify logs
assert "Chrome version from binary: Chrome 136.0.6778.0 (major: 136)" in caplog.text assert "Chrome version from binary: 136.0.6778.0 (major: 136)" in caplog.text
assert "Chrome 136+ detected - security validation required" in caplog.text assert "Chrome 136+ detected - security validation required" in caplog.text
# Verify mocks were called # Verify mocks were called
@@ -255,14 +239,9 @@ class TestWebScrapingMixinChromeVersionDiagnostics:
# Setup mocks # Setup mocks
mock_get_diagnostic.return_value = { mock_get_diagnostic.return_value = {
"binary_detection": None, "binary_detection": None,
"remote_detection": { "remote_detection": {"version_string": "136.0.6778.0", "major_version": 136, "browser_name": "Chrome", "is_chrome_136_plus": True},
"version_string": "136.0.6778.0",
"major_version": 136,
"browser_name": "Chrome",
"is_chrome_136_plus": True
},
"chrome_136_plus_detected": True, "chrome_136_plus_detected": True,
"recommendations": [] "recommendations": [],
} }
mock_validate.return_value = (False, "Chrome 136+ requires --user-data-dir") mock_validate.return_value = (False, "Chrome 136+ requires --user-data-dir")
@@ -280,32 +259,22 @@ class TestWebScrapingMixinChromeVersionDiagnostics:
scraper._diagnose_chrome_version_issues(9222) scraper._diagnose_chrome_version_issues(9222)
# Verify logs # Verify logs
assert "Chrome version from remote debugging: Chrome 136.0.6778.0 (major: 136)" in caplog.text assert "(info) Chrome version from remote debugging: 136.0.6778.0 (major: 136)" in caplog.text
assert "Remote Chrome 136+ detected - validating configuration" in caplog.text assert "Remote Chrome 136+ detected - validating configuration" in caplog.text
assert "Chrome 136+ configuration validation failed" in caplog.text assert "Chrome 136+ configuration validation failed" in caplog.text
# Verify validation was called # Verify validation was called
mock_validate.assert_called_once_with( mock_validate.assert_called_once_with(["--remote-debugging-port=9222"], None)
["--remote-debugging-port=9222"],
None
)
finally: finally:
# Restore environment # Restore environment
if original_env: if original_env:
os.environ["PYTEST_CURRENT_TEST"] = original_env os.environ["PYTEST_CURRENT_TEST"] = original_env
@patch("kleinanzeigen_bot.utils.web_scraping_mixin.get_chrome_version_diagnostic_info") @patch("kleinanzeigen_bot.utils.web_scraping_mixin.get_chrome_version_diagnostic_info")
def test_diagnose_chrome_version_issues_no_detection( def test_diagnose_chrome_version_issues_no_detection(self, mock_get_diagnostic:Mock, scraper:WebScrapingMixin, caplog:pytest.LogCaptureFixture) -> None:
self, mock_get_diagnostic:Mock, scraper:WebScrapingMixin, caplog:pytest.LogCaptureFixture
) -> None:
"""Test Chrome version diagnostics with no detection.""" """Test Chrome version diagnostics with no detection."""
# Setup mocks # Setup mocks
mock_get_diagnostic.return_value = { mock_get_diagnostic.return_value = {"binary_detection": None, "remote_detection": None, "chrome_136_plus_detected": False, "recommendations": []}
"binary_detection": None,
"remote_detection": None,
"chrome_136_plus_detected": False,
"recommendations": []
}
# Configure scraper # Configure scraper
scraper.browser_config.binary_location = "/path/to/chrome" scraper.browser_config.binary_location = "/path/to/chrome"
@@ -334,15 +303,10 @@ class TestWebScrapingMixinChromeVersionDiagnostics:
"""Test Chrome version diagnostics with Chrome 136+ recommendations.""" """Test Chrome version diagnostics with Chrome 136+ recommendations."""
# Setup mocks # Setup mocks
mock_get_diagnostic.return_value = { mock_get_diagnostic.return_value = {
"binary_detection": { "binary_detection": {"version_string": "136.0.6778.0", "major_version": 136, "browser_name": "Chrome", "is_chrome_136_plus": True},
"version_string": "136.0.6778.0",
"major_version": 136,
"browser_name": "Chrome",
"is_chrome_136_plus": True
},
"remote_detection": None, "remote_detection": None,
"chrome_136_plus_detected": True, "chrome_136_plus_detected": True,
"recommendations": [] "recommendations": [],
} }
# Configure scraper # Configure scraper
@@ -377,11 +341,11 @@ class TestWebScrapingMixinChromeVersionDiagnostics:
"version_string": "120.0.6099.109", "version_string": "120.0.6099.109",
"major_version": 120, "major_version": 120,
"browser_name": "Chrome", "browser_name": "Chrome",
"is_chrome_136_plus": False # This triggers the else branch (lines 832-849) "is_chrome_136_plus": False, # This triggers the else branch (lines 832-849)
}, },
"remote_detection": None, # Ensure this is None to avoid other branches "remote_detection": None, # Ensure this is None to avoid other branches
"chrome_136_plus_detected": False, # Ensure this is False to avoid recommendations "chrome_136_plus_detected": False, # Ensure this is False to avoid recommendations
"recommendations": [] "recommendations": [],
} }
# Configure scraper # Configure scraper
@@ -420,14 +384,9 @@ class TestWebScrapingMixinChromeVersionDiagnostics:
# Setup mocks # Setup mocks
mock_get_diagnostic.return_value = { mock_get_diagnostic.return_value = {
"binary_detection": None, "binary_detection": None,
"remote_detection": { "remote_detection": {"version_string": "136.0.6778.0", "major_version": 136, "browser_name": "Chrome", "is_chrome_136_plus": True},
"version_string": "136.0.6778.0",
"major_version": 136,
"browser_name": "Chrome",
"is_chrome_136_plus": True
},
"chrome_136_plus_detected": True, "chrome_136_plus_detected": True,
"recommendations": [] "recommendations": [],
} }
mock_validate.return_value = (True, "") # This triggers the else branch (line 846) mock_validate.return_value = (True, "") # This triggers the else branch (line 846)
@@ -451,7 +410,7 @@ class TestWebScrapingMixinChromeVersionDiagnostics:
# Verify validation was called with correct arguments # Verify validation was called with correct arguments
mock_validate.assert_called_once_with( mock_validate.assert_called_once_with(
["--remote-debugging-port=9222", "--user-data-dir=/tmp/chrome-debug"], # noqa: S108 ["--remote-debugging-port=9222", "--user-data-dir=/tmp/chrome-debug"], # noqa: S108
"/tmp/chrome-debug" # noqa: S108 "/tmp/chrome-debug", # noqa: S108
) )
finally: finally:
# Restore environment # Restore environment
@@ -469,9 +428,7 @@ class TestWebScrapingMixinIntegration:
@patch.object(WebScrapingMixin, "_validate_chrome_version_configuration") @patch.object(WebScrapingMixin, "_validate_chrome_version_configuration")
@patch.object(WebScrapingMixin, "get_compatible_browser") @patch.object(WebScrapingMixin, "get_compatible_browser")
async def test_create_browser_session_calls_chrome_validation( async def test_create_browser_session_calls_chrome_validation(self, mock_get_browser:Mock, mock_validate:Mock, scraper:WebScrapingMixin) -> None:
self, mock_get_browser:Mock, mock_validate:Mock, scraper:WebScrapingMixin
) -> None:
"""Test that create_browser_session calls Chrome version validation.""" """Test that create_browser_session calls Chrome version validation."""
# Setup mocks # Setup mocks
mock_get_browser.return_value = "/path/to/chrome" mock_get_browser.return_value = "/path/to/chrome"
@@ -493,9 +450,7 @@ class TestWebScrapingMixinIntegration:
@patch.object(WebScrapingMixin, "_diagnose_chrome_version_issues") @patch.object(WebScrapingMixin, "_diagnose_chrome_version_issues")
@patch.object(WebScrapingMixin, "get_compatible_browser") @patch.object(WebScrapingMixin, "get_compatible_browser")
def test_diagnose_browser_issues_calls_chrome_diagnostics( def test_diagnose_browser_issues_calls_chrome_diagnostics(self, mock_get_browser:Mock, mock_diagnose:Mock, scraper:WebScrapingMixin) -> None:
self, mock_get_browser:Mock, mock_diagnose:Mock, scraper:WebScrapingMixin
) -> None:
"""Test that diagnose_browser_issues calls Chrome version diagnostics.""" """Test that diagnose_browser_issues calls Chrome version diagnostics."""
# Setup mocks # Setup mocks
mock_get_browser.return_value = "/path/to/chrome" mock_get_browser.return_value = "/path/to/chrome"
@@ -521,9 +476,7 @@ class TestWebScrapingMixinIntegration:
# Mock Chrome version detection to return pre-136 version # Mock Chrome version detection to return pre-136 version
with patch("kleinanzeigen_bot.utils.web_scraping_mixin.detect_chrome_version_from_binary") as mock_detect: with patch("kleinanzeigen_bot.utils.web_scraping_mixin.detect_chrome_version_from_binary") as mock_detect:
mock_detect.return_value = ChromeVersionInfo( mock_detect.return_value = ChromeVersionInfo("120.0.6099.109", 120, "Chrome")
"120.0.6099.109", 120, "Chrome"
)
# Temporarily unset PYTEST_CURRENT_TEST to allow validation to run # Temporarily unset PYTEST_CURRENT_TEST to allow validation to run
original_env = os.environ.get("PYTEST_CURRENT_TEST") original_env = os.environ.get("PYTEST_CURRENT_TEST")
@@ -541,3 +494,68 @@ class TestWebScrapingMixinIntegration:
# Restore environment # Restore environment
if original_env: if original_env:
os.environ["PYTEST_CURRENT_TEST"] = original_env os.environ["PYTEST_CURRENT_TEST"] = original_env
@patch("kleinanzeigen_bot.utils.web_scraping_mixin.detect_chrome_version_from_binary")
async def test_validate_chrome_136_configuration_with_whitespace_user_data_dir(
self, mock_detect:Mock, scraper:WebScrapingMixin, caplog:pytest.LogCaptureFixture
) -> None:
"""Test Chrome 136+ validation correctly handles whitespace-only user_data_dir."""
# Setup mocks
mock_detect.return_value = ChromeVersionInfo("136.0.6778.0", 136, "Chrome")
# Configure scraper with whitespace-only user_data_dir
scraper.browser_config.binary_location = "/path/to/chrome"
scraper.browser_config.arguments = ["--remote-debugging-port=9222"]
scraper.browser_config.user_data_dir = " " # Only whitespace
# Temporarily unset PYTEST_CURRENT_TEST to allow validation to run
original_env = os.environ.get("PYTEST_CURRENT_TEST")
if "PYTEST_CURRENT_TEST" in os.environ:
del os.environ["PYTEST_CURRENT_TEST"]
try:
# Test validation should fail because whitespace-only is treated as empty
await scraper._validate_chrome_version_configuration()
# Verify detection was called
assert mock_detect.call_count == 1
# Verify error was logged
assert "Chrome 136+ configuration validation failed" in caplog.text
assert "Chrome 136+ requires --user-data-dir" in caplog.text
finally:
# Restore environment
if original_env:
os.environ["PYTEST_CURRENT_TEST"] = original_env
@patch("kleinanzeigen_bot.utils.web_scraping_mixin.detect_chrome_version_from_binary")
async def test_validate_chrome_136_configuration_with_valid_user_data_dir(
self, mock_detect:Mock, scraper:WebScrapingMixin, caplog:pytest.LogCaptureFixture
) -> None:
"""Test Chrome 136+ validation passes with valid user_data_dir."""
# Setup mocks
mock_detect.return_value = ChromeVersionInfo("136.0.6778.0", 136, "Chrome")
# Configure scraper with valid user_data_dir
scraper.browser_config.binary_location = "/path/to/chrome"
scraper.browser_config.arguments = ["--remote-debugging-port=9222"]
scraper.browser_config.user_data_dir = "/tmp/valid-profile" # noqa: S108
# Temporarily unset PYTEST_CURRENT_TEST to allow validation to run
original_env = os.environ.get("PYTEST_CURRENT_TEST")
if "PYTEST_CURRENT_TEST" in os.environ:
del os.environ["PYTEST_CURRENT_TEST"]
try:
# Test validation should pass
await scraper._validate_chrome_version_configuration()
# Verify detection was called
assert mock_detect.call_count == 1
# Verify success was logged
assert "Chrome 136+ configuration validation passed" in caplog.text
finally:
# Restore environment
if original_env:
os.environ["PYTEST_CURRENT_TEST"] = original_env

View File

@@ -17,7 +17,7 @@ pytestmark = pytest.mark.unit
class TestGetXdgBaseDir: class TestGetXdgBaseDir:
"""Tests for get_xdg_base_dir function.""" """Tests for get_xdg_base_dir function."""
def test_returns_state_dir(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: def test_returns_state_dir(self, tmp_path:Path, monkeypatch:pytest.MonkeyPatch) -> None:
"""Test resolving XDG state directory.""" """Test resolving XDG state directory."""
state_dir = tmp_path / "state" state_dir = tmp_path / "state"
monkeypatch.setattr("platformdirs.user_state_dir", lambda app_name, *args, **kwargs: str(state_dir / app_name)) monkeypatch.setattr("platformdirs.user_state_dir", lambda app_name, *args, **kwargs: str(state_dir / app_name))
@@ -28,21 +28,21 @@ class TestGetXdgBaseDir:
def test_raises_for_unknown_category(self) -> None: def test_raises_for_unknown_category(self) -> None:
"""Test invalid category handling.""" """Test invalid category handling."""
with pytest.raises(ValueError, match="Unsupported XDG category"): with pytest.raises(ValueError, match = "Unsupported XDG category"):
xdg_paths.get_xdg_base_dir("invalid") # type: ignore[arg-type] xdg_paths.get_xdg_base_dir("invalid") # type: ignore[arg-type]
def test_raises_when_base_dir_is_none(self, monkeypatch: pytest.MonkeyPatch) -> None: def test_raises_when_base_dir_is_none(self, monkeypatch:pytest.MonkeyPatch) -> None:
"""Test runtime error when platformdirs returns None.""" """Test runtime error when platformdirs returns None."""
monkeypatch.setattr("platformdirs.user_state_dir", lambda _app_name, *args, **kwargs: None) monkeypatch.setattr("platformdirs.user_state_dir", lambda _app_name, *args, **kwargs: None)
with pytest.raises(RuntimeError, match="Failed to resolve XDG base directory"): with pytest.raises(RuntimeError, match = "Failed to resolve XDG base directory"):
xdg_paths.get_xdg_base_dir("state") xdg_paths.get_xdg_base_dir("state")
class TestDetectInstallationMode: class TestDetectInstallationMode:
"""Tests for detect_installation_mode function.""" """Tests for detect_installation_mode function."""
def test_detects_portable_mode_when_config_exists_in_cwd(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: def test_detects_portable_mode_when_config_exists_in_cwd(self, tmp_path:Path, monkeypatch:pytest.MonkeyPatch) -> None:
"""Test that portable mode is detected when config.yaml exists in CWD.""" """Test that portable mode is detected when config.yaml exists in CWD."""
# Setup: Create config.yaml in CWD # Setup: Create config.yaml in CWD
monkeypatch.chdir(tmp_path) monkeypatch.chdir(tmp_path)
@@ -54,11 +54,11 @@ class TestDetectInstallationMode:
# Verify # Verify
assert mode == "portable" assert mode == "portable"
def test_detects_xdg_mode_when_config_exists_in_xdg_location(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: def test_detects_xdg_mode_when_config_exists_in_xdg_location(self, tmp_path:Path, monkeypatch:pytest.MonkeyPatch) -> None:
"""Test that XDG mode is detected when config exists in XDG location.""" """Test that XDG mode is detected when config exists in XDG location."""
# Setup: Create config in mock XDG directory # Setup: Create config in mock XDG directory
xdg_config = tmp_path / "config" / "kleinanzeigen-bot" xdg_config = tmp_path / "config" / "kleinanzeigen-bot"
xdg_config.mkdir(parents=True) xdg_config.mkdir(parents = True)
(xdg_config / "config.yaml").touch() (xdg_config / "config.yaml").touch()
# Mock platformdirs to return our test directory # Mock platformdirs to return our test directory
@@ -75,7 +75,7 @@ class TestDetectInstallationMode:
# Verify # Verify
assert mode == "xdg" assert mode == "xdg"
def test_returns_none_when_no_config_found(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: def test_returns_none_when_no_config_found(self, tmp_path:Path, monkeypatch:pytest.MonkeyPatch) -> None:
"""Test that None is returned when no config exists anywhere.""" """Test that None is returned when no config exists anywhere."""
# Setup: Empty directories # Setup: Empty directories
monkeypatch.chdir(tmp_path) monkeypatch.chdir(tmp_path)
@@ -95,7 +95,7 @@ class TestDetectInstallationMode:
class TestGetConfigFilePath: class TestGetConfigFilePath:
"""Tests for get_config_file_path function.""" """Tests for get_config_file_path function."""
def test_returns_cwd_path_in_portable_mode(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: def test_returns_cwd_path_in_portable_mode(self, tmp_path:Path, monkeypatch:pytest.MonkeyPatch) -> None:
"""Test that portable mode returns ./config.yaml.""" """Test that portable mode returns ./config.yaml."""
monkeypatch.chdir(tmp_path) monkeypatch.chdir(tmp_path)
@@ -103,7 +103,7 @@ class TestGetConfigFilePath:
assert path == tmp_path / "config.yaml" assert path == tmp_path / "config.yaml"
def test_returns_xdg_path_in_xdg_mode(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: def test_returns_xdg_path_in_xdg_mode(self, tmp_path:Path, monkeypatch:pytest.MonkeyPatch) -> None:
"""Test that XDG mode returns XDG config path.""" """Test that XDG mode returns XDG config path."""
xdg_config = tmp_path / "config" xdg_config = tmp_path / "config"
monkeypatch.setattr( monkeypatch.setattr(
@@ -120,7 +120,7 @@ class TestGetConfigFilePath:
class TestGetAdFilesSearchDir: class TestGetAdFilesSearchDir:
"""Tests for get_ad_files_search_dir function.""" """Tests for get_ad_files_search_dir function."""
def test_returns_cwd_in_portable_mode(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: def test_returns_cwd_in_portable_mode(self, tmp_path:Path, monkeypatch:pytest.MonkeyPatch) -> None:
"""Test that portable mode searches in CWD.""" """Test that portable mode searches in CWD."""
monkeypatch.chdir(tmp_path) monkeypatch.chdir(tmp_path)
@@ -128,7 +128,7 @@ class TestGetAdFilesSearchDir:
assert search_dir == tmp_path assert search_dir == tmp_path
def test_returns_xdg_config_dir_in_xdg_mode(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: def test_returns_xdg_config_dir_in_xdg_mode(self, tmp_path:Path, monkeypatch:pytest.MonkeyPatch) -> None:
"""Test that XDG mode searches in XDG config directory (same as config file).""" """Test that XDG mode searches in XDG config directory (same as config file)."""
xdg_config = tmp_path / "config" xdg_config = tmp_path / "config"
monkeypatch.setattr( monkeypatch.setattr(
@@ -146,7 +146,7 @@ class TestGetAdFilesSearchDir:
class TestGetDownloadedAdsPath: class TestGetDownloadedAdsPath:
"""Tests for get_downloaded_ads_path function.""" """Tests for get_downloaded_ads_path function."""
def test_returns_cwd_downloaded_ads_in_portable_mode(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: def test_returns_cwd_downloaded_ads_in_portable_mode(self, tmp_path:Path, monkeypatch:pytest.MonkeyPatch) -> None:
"""Test that portable mode uses ./downloaded-ads/.""" """Test that portable mode uses ./downloaded-ads/."""
monkeypatch.chdir(tmp_path) monkeypatch.chdir(tmp_path)
@@ -154,7 +154,7 @@ class TestGetDownloadedAdsPath:
assert ads_path == tmp_path / "downloaded-ads" assert ads_path == tmp_path / "downloaded-ads"
def test_creates_directory_if_not_exists(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: def test_creates_directory_if_not_exists(self, tmp_path:Path, monkeypatch:pytest.MonkeyPatch) -> None:
"""Test that directory is created if it doesn't exist.""" """Test that directory is created if it doesn't exist."""
monkeypatch.chdir(tmp_path) monkeypatch.chdir(tmp_path)
@@ -163,7 +163,7 @@ class TestGetDownloadedAdsPath:
assert ads_path.exists() assert ads_path.exists()
assert ads_path.is_dir() assert ads_path.is_dir()
def test_returns_xdg_downloaded_ads_in_xdg_mode(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: def test_returns_xdg_downloaded_ads_in_xdg_mode(self, tmp_path:Path, monkeypatch:pytest.MonkeyPatch) -> None:
"""Test that XDG mode uses XDG config/downloaded-ads/.""" """Test that XDG mode uses XDG config/downloaded-ads/."""
xdg_config = tmp_path / "config" xdg_config = tmp_path / "config"
monkeypatch.setattr( monkeypatch.setattr(
@@ -180,7 +180,7 @@ class TestGetDownloadedAdsPath:
class TestGetBrowserProfilePath: class TestGetBrowserProfilePath:
"""Tests for get_browser_profile_path function.""" """Tests for get_browser_profile_path function."""
def test_returns_cwd_temp_in_portable_mode(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: def test_returns_cwd_temp_in_portable_mode(self, tmp_path:Path, monkeypatch:pytest.MonkeyPatch) -> None:
"""Test that portable mode uses ./.temp/browser-profile.""" """Test that portable mode uses ./.temp/browser-profile."""
monkeypatch.chdir(tmp_path) monkeypatch.chdir(tmp_path)
@@ -188,7 +188,7 @@ class TestGetBrowserProfilePath:
assert profile_path == tmp_path / ".temp" / "browser-profile" assert profile_path == tmp_path / ".temp" / "browser-profile"
def test_returns_xdg_cache_in_xdg_mode(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: def test_returns_xdg_cache_in_xdg_mode(self, tmp_path:Path, monkeypatch:pytest.MonkeyPatch) -> None:
"""Test that XDG mode uses XDG cache directory.""" """Test that XDG mode uses XDG cache directory."""
xdg_cache = tmp_path / "cache" xdg_cache = tmp_path / "cache"
monkeypatch.setattr( monkeypatch.setattr(
@@ -201,7 +201,7 @@ class TestGetBrowserProfilePath:
assert "kleinanzeigen-bot" in str(profile_path) assert "kleinanzeigen-bot" in str(profile_path)
assert profile_path.name == "browser-profile" assert profile_path.name == "browser-profile"
def test_creates_directory_if_not_exists(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: def test_creates_directory_if_not_exists(self, tmp_path:Path, monkeypatch:pytest.MonkeyPatch) -> None:
"""Test that browser profile directory is created.""" """Test that browser profile directory is created."""
monkeypatch.chdir(tmp_path) monkeypatch.chdir(tmp_path)
@@ -214,7 +214,7 @@ class TestGetBrowserProfilePath:
class TestGetLogFilePath: class TestGetLogFilePath:
"""Tests for get_log_file_path function.""" """Tests for get_log_file_path function."""
def test_returns_cwd_log_in_portable_mode(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: def test_returns_cwd_log_in_portable_mode(self, tmp_path:Path, monkeypatch:pytest.MonkeyPatch) -> None:
"""Test that portable mode uses ./{basename}.log.""" """Test that portable mode uses ./{basename}.log."""
monkeypatch.chdir(tmp_path) monkeypatch.chdir(tmp_path)
@@ -222,7 +222,7 @@ class TestGetLogFilePath:
assert log_path == tmp_path / "test.log" assert log_path == tmp_path / "test.log"
def test_returns_xdg_state_log_in_xdg_mode(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: def test_returns_xdg_state_log_in_xdg_mode(self, tmp_path:Path, monkeypatch:pytest.MonkeyPatch) -> None:
"""Test that XDG mode uses XDG state directory.""" """Test that XDG mode uses XDG state directory."""
xdg_state = tmp_path / "state" xdg_state = tmp_path / "state"
monkeypatch.setattr( monkeypatch.setattr(
@@ -239,7 +239,7 @@ class TestGetLogFilePath:
class TestGetUpdateCheckStatePath: class TestGetUpdateCheckStatePath:
"""Tests for get_update_check_state_path function.""" """Tests for get_update_check_state_path function."""
def test_returns_cwd_temp_in_portable_mode(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: def test_returns_cwd_temp_in_portable_mode(self, tmp_path:Path, monkeypatch:pytest.MonkeyPatch) -> None:
"""Test that portable mode uses ./.temp/update_check_state.json.""" """Test that portable mode uses ./.temp/update_check_state.json."""
monkeypatch.chdir(tmp_path) monkeypatch.chdir(tmp_path)
@@ -247,7 +247,7 @@ class TestGetUpdateCheckStatePath:
assert state_path == tmp_path / ".temp" / "update_check_state.json" assert state_path == tmp_path / ".temp" / "update_check_state.json"
def test_returns_xdg_state_in_xdg_mode(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: def test_returns_xdg_state_in_xdg_mode(self, tmp_path:Path, monkeypatch:pytest.MonkeyPatch) -> None:
"""Test that XDG mode uses XDG state directory.""" """Test that XDG mode uses XDG state directory."""
xdg_state = tmp_path / "state" xdg_state = tmp_path / "state"
monkeypatch.setattr( monkeypatch.setattr(
@@ -264,12 +264,12 @@ class TestGetUpdateCheckStatePath:
class TestPromptInstallationMode: class TestPromptInstallationMode:
"""Tests for prompt_installation_mode function.""" """Tests for prompt_installation_mode function."""
@pytest.fixture(autouse=True) @pytest.fixture(autouse = True)
def _force_identity_translation(self, monkeypatch: pytest.MonkeyPatch) -> None: def _force_identity_translation(self, monkeypatch:pytest.MonkeyPatch) -> None:
"""Ensure prompt strings are stable regardless of locale.""" """Ensure prompt strings are stable regardless of locale."""
monkeypatch.setattr(xdg_paths, "_", lambda message: message) monkeypatch.setattr(xdg_paths, "_", lambda message: message)
def test_returns_portable_for_non_interactive_mode_no_stdin(self, monkeypatch: pytest.MonkeyPatch) -> None: def test_returns_portable_for_non_interactive_mode_no_stdin(self, monkeypatch:pytest.MonkeyPatch) -> None:
"""Test that non-interactive mode (no stdin) defaults to portable.""" """Test that non-interactive mode (no stdin) defaults to portable."""
# Mock sys.stdin to be None (simulates non-interactive environment) # Mock sys.stdin to be None (simulates non-interactive environment)
monkeypatch.setattr("sys.stdin", None) monkeypatch.setattr("sys.stdin", None)
@@ -278,7 +278,7 @@ class TestPromptInstallationMode:
assert mode == "portable" assert mode == "portable"
def test_returns_portable_for_non_interactive_mode_not_tty(self, monkeypatch: pytest.MonkeyPatch) -> None: def test_returns_portable_for_non_interactive_mode_not_tty(self, monkeypatch:pytest.MonkeyPatch) -> None:
"""Test that non-interactive mode (not a TTY) defaults to portable.""" """Test that non-interactive mode (not a TTY) defaults to portable."""
# Mock sys.stdin.isatty() to return False (simulates piped input or file redirect) # Mock sys.stdin.isatty() to return False (simulates piped input or file redirect)
mock_stdin = io.StringIO() mock_stdin = io.StringIO()
@@ -289,7 +289,7 @@ class TestPromptInstallationMode:
assert mode == "portable" assert mode == "portable"
def test_returns_portable_when_user_enters_1(self, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]) -> None: def test_returns_portable_when_user_enters_1(self, monkeypatch:pytest.MonkeyPatch, capsys:pytest.CaptureFixture[str]) -> None:
"""Test that user entering '1' selects portable mode.""" """Test that user entering '1' selects portable mode."""
# Mock sys.stdin to simulate interactive terminal # Mock sys.stdin to simulate interactive terminal
mock_stdin = io.StringIO() mock_stdin = io.StringIO()
@@ -307,7 +307,7 @@ class TestPromptInstallationMode:
assert "Choose installation type:" in captured.out assert "Choose installation type:" in captured.out
assert "[1] Portable" in captured.out assert "[1] Portable" in captured.out
def test_returns_xdg_when_user_enters_2(self, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]) -> None: def test_returns_xdg_when_user_enters_2(self, monkeypatch:pytest.MonkeyPatch, capsys:pytest.CaptureFixture[str]) -> None:
"""Test that user entering '2' selects XDG mode.""" """Test that user entering '2' selects XDG mode."""
# Mock sys.stdin to simulate interactive terminal # Mock sys.stdin to simulate interactive terminal
mock_stdin = io.StringIO() mock_stdin = io.StringIO()
@@ -325,7 +325,7 @@ class TestPromptInstallationMode:
assert "Choose installation type:" in captured.out assert "Choose installation type:" in captured.out
assert "[2] System-wide" in captured.out assert "[2] System-wide" in captured.out
def test_reprompts_on_invalid_input_then_accepts_valid(self, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]) -> None: def test_reprompts_on_invalid_input_then_accepts_valid(self, monkeypatch:pytest.MonkeyPatch, capsys:pytest.CaptureFixture[str]) -> None:
"""Test that invalid input causes re-prompt, then valid input is accepted.""" """Test that invalid input causes re-prompt, then valid input is accepted."""
# Mock sys.stdin to simulate interactive terminal # Mock sys.stdin to simulate interactive terminal
mock_stdin = io.StringIO() mock_stdin = io.StringIO()
@@ -343,7 +343,7 @@ class TestPromptInstallationMode:
captured = capsys.readouterr() captured = capsys.readouterr()
assert "Invalid choice" in captured.out assert "Invalid choice" in captured.out
def test_returns_portable_on_eof_error(self, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]) -> None: def test_returns_portable_on_eof_error(self, monkeypatch:pytest.MonkeyPatch, capsys:pytest.CaptureFixture[str]) -> None:
"""Test that EOFError (Ctrl+D) defaults to portable mode.""" """Test that EOFError (Ctrl+D) defaults to portable mode."""
# Mock sys.stdin to simulate interactive terminal # Mock sys.stdin to simulate interactive terminal
mock_stdin = io.StringIO() mock_stdin = io.StringIO()
@@ -351,7 +351,7 @@ class TestPromptInstallationMode:
monkeypatch.setattr("sys.stdin", mock_stdin) monkeypatch.setattr("sys.stdin", mock_stdin)
# Mock input raising EOFError # Mock input raising EOFError
def mock_input(_: str) -> str: def mock_input(_:str) -> str:
raise EOFError raise EOFError
monkeypatch.setattr("builtins.input", mock_input) monkeypatch.setattr("builtins.input", mock_input)
@@ -363,7 +363,7 @@ class TestPromptInstallationMode:
captured = capsys.readouterr() captured = capsys.readouterr()
assert captured.out.endswith("\n") assert captured.out.endswith("\n")
def test_returns_portable_on_keyboard_interrupt(self, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]) -> None: def test_returns_portable_on_keyboard_interrupt(self, monkeypatch:pytest.MonkeyPatch, capsys:pytest.CaptureFixture[str]) -> None:
"""Test that KeyboardInterrupt (Ctrl+C) defaults to portable mode.""" """Test that KeyboardInterrupt (Ctrl+C) defaults to portable mode."""
# Mock sys.stdin to simulate interactive terminal # Mock sys.stdin to simulate interactive terminal
mock_stdin = io.StringIO() mock_stdin = io.StringIO()
@@ -371,7 +371,7 @@ class TestPromptInstallationMode:
monkeypatch.setattr("sys.stdin", mock_stdin) monkeypatch.setattr("sys.stdin", mock_stdin)
# Mock input raising KeyboardInterrupt # Mock input raising KeyboardInterrupt
def mock_input(_: str) -> str: def mock_input(_:str) -> str:
raise KeyboardInterrupt raise KeyboardInterrupt
monkeypatch.setattr("builtins.input", mock_input) monkeypatch.setattr("builtins.input", mock_input)
@@ -387,18 +387,18 @@ class TestPromptInstallationMode:
class TestGetBrowserProfilePathWithOverride: class TestGetBrowserProfilePathWithOverride:
"""Tests for get_browser_profile_path config_override parameter.""" """Tests for get_browser_profile_path config_override parameter."""
def test_respects_config_override_in_portable_mode(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: def test_respects_config_override_in_portable_mode(self, tmp_path:Path, monkeypatch:pytest.MonkeyPatch) -> None:
"""Test that config_override takes precedence in portable mode.""" """Test that config_override takes precedence in portable mode."""
monkeypatch.chdir(tmp_path) monkeypatch.chdir(tmp_path)
custom_path = str(tmp_path / "custom" / "browser") custom_path = str(tmp_path / "custom" / "browser")
profile_path = xdg_paths.get_browser_profile_path("portable", config_override=custom_path) profile_path = xdg_paths.get_browser_profile_path("portable", config_override = custom_path)
assert profile_path == Path(custom_path) assert profile_path == Path(custom_path)
assert profile_path.exists() # Verify directory was created assert profile_path.exists() # Verify directory was created
assert profile_path.is_dir() assert profile_path.is_dir()
def test_respects_config_override_in_xdg_mode(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: def test_respects_config_override_in_xdg_mode(self, tmp_path:Path, monkeypatch:pytest.MonkeyPatch) -> None:
"""Test that config_override takes precedence in XDG mode.""" """Test that config_override takes precedence in XDG mode."""
xdg_cache = tmp_path / "cache" xdg_cache = tmp_path / "cache"
monkeypatch.setattr( monkeypatch.setattr(
@@ -407,7 +407,7 @@ class TestGetBrowserProfilePathWithOverride:
) )
custom_path = str(tmp_path / "custom" / "browser") custom_path = str(tmp_path / "custom" / "browser")
profile_path = xdg_paths.get_browser_profile_path("xdg", config_override=custom_path) profile_path = xdg_paths.get_browser_profile_path("xdg", config_override = custom_path)
assert profile_path == Path(custom_path) assert profile_path == Path(custom_path)
# Verify it didn't use XDG cache directory # Verify it didn't use XDG cache directory
@@ -419,7 +419,7 @@ class TestGetBrowserProfilePathWithOverride:
class TestUnicodeHandling: class TestUnicodeHandling:
"""Tests for Unicode path handling (NFD vs NFC normalization).""" """Tests for Unicode path handling (NFD vs NFC normalization)."""
def test_portable_mode_handles_unicode_in_cwd(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: def test_portable_mode_handles_unicode_in_cwd(self, tmp_path:Path, monkeypatch:pytest.MonkeyPatch) -> None:
"""Test that portable mode works with Unicode characters in CWD path. """Test that portable mode works with Unicode characters in CWD path.
This tests the edge case where the current directory contains Unicode This tests the edge case where the current directory contains Unicode
@@ -442,7 +442,7 @@ class TestUnicodeHandling:
assert config_path.name == "config.yaml" assert config_path.name == "config.yaml"
assert log_path.name == "test.log" assert log_path.name == "test.log"
def test_xdg_mode_handles_unicode_in_paths(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: def test_xdg_mode_handles_unicode_in_paths(self, tmp_path:Path, monkeypatch:pytest.MonkeyPatch) -> None:
"""Test that XDG mode handles Unicode in XDG directory paths. """Test that XDG mode handles Unicode in XDG directory paths.
This tests the edge case where XDG directories contain Unicode This tests the edge case where XDG directories contain Unicode
@@ -451,7 +451,7 @@ class TestUnicodeHandling:
""" """
# Create XDG directory with umlaut # Create XDG directory with umlaut
xdg_base = tmp_path / "Users" / "Müller" / ".config" xdg_base = tmp_path / "Users" / "Müller" / ".config"
xdg_base.mkdir(parents=True) xdg_base.mkdir(parents = True)
monkeypatch.setattr( monkeypatch.setattr(
"platformdirs.user_config_dir", "platformdirs.user_config_dir",
@@ -465,11 +465,11 @@ class TestUnicodeHandling:
assert "Müller" in str(config_path) or "Mu\u0308ller" in str(config_path) assert "Müller" in str(config_path) or "Mu\u0308ller" in str(config_path)
assert config_path.name == "config.yaml" assert config_path.name == "config.yaml"
def test_downloaded_ads_path_handles_unicode(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: def test_downloaded_ads_path_handles_unicode(self, tmp_path:Path, monkeypatch:pytest.MonkeyPatch) -> None:
"""Test that downloaded ads directory creation works with Unicode paths.""" """Test that downloaded ads directory creation works with Unicode paths."""
# Create XDG config directory with umlaut # Create XDG config directory with umlaut
xdg_config = tmp_path / "config" / "Müller" xdg_config = tmp_path / "config" / "Müller"
xdg_config.mkdir(parents=True) xdg_config.mkdir(parents = True)
monkeypatch.setattr( monkeypatch.setattr(
"platformdirs.user_config_dir", "platformdirs.user_config_dir",