refact: apply consistent formatting

This commit is contained in:
sebthom
2025-04-27 23:54:22 +02:00
parent fe33a0e461
commit ef923a8337
21 changed files with 1020 additions and 709 deletions

View File

@@ -20,8 +20,8 @@ Select the type(s) of change(s) included in this pull request:
Before requesting a review, confirm the following: Before requesting a review, confirm the following:
- [ ] I have reviewed my changes to ensure they meet the project's standards. - [ ] I have reviewed my changes to ensure they meet the project's standards.
- [ ] I have tested my changes and ensured that all tests pass (`pdm run test`). - [ ] I have tested my changes and ensured that all tests pass (`pdm run test`).
- [ ] I have formatted the code (`pdm run format`).
- [ ] I have verified that linting passes (`pdm run lint`). - [ ] I have verified that linting passes (`pdm run lint`).
- [ ] I have run security scans and addressed any identified issues (`pdm run audit`).
- [ ] I have updated documentation where necessary. - [ ] I have updated documentation where necessary.
By submitting this pull request, I confirm that you can use, modify, copy, and redistribute this contribution, under the terms of your choice. By submitting this pull request, I confirm that you can use, modify, copy, and redistribute this contribution, under the terms of your choice.

View File

@@ -82,7 +82,7 @@ app = "python -m kleinanzeigen_bot"
compile.cmd = "python -O -m PyInstaller pyinstaller.spec --clean" compile.cmd = "python -O -m PyInstaller pyinstaller.spec --clean"
compile.env = {PYTHONHASHSEED = "1", SOURCE_DATE_EPOCH = "0"} # https://pyinstaller.org/en/stable/advanced-topics.html#creating-a-reproducible-build compile.env = {PYTHONHASHSEED = "1", SOURCE_DATE_EPOCH = "0"} # https://pyinstaller.org/en/stable/advanced-topics.html#creating-a-reproducible-build
debug = "python -m pdb -m kleinanzeigen_bot" debug = "python -m pdb -m kleinanzeigen_bot"
format = "autopep8 --recursive --in-place src tests --verbose" format = {shell = "autopep8 --recursive --in-place scripts src tests --verbose && python scripts/post_autopep8.py scripts src tests" }
lint = {shell = "ruff check && mypy && basedpyright" } lint = {shell = "ruff check && mypy && basedpyright" }
fix = {shell = "ruff check --fix" } fix = {shell = "ruff check --fix" }
test = "python -m pytest --capture=tee-sys -v" test = "python -m pytest --capture=tee-sys -v"
@@ -113,7 +113,7 @@ aggressive = 3
# https://docs.astral.sh/ruff/configuration/ # https://docs.astral.sh/ruff/configuration/
##################### #####################
[tool.ruff] [tool.ruff]
include = ["pyproject.toml", "src/**/*.py", "tests/**/*.py"] include = ["pyproject.toml", "scripts/**/*.py", "src/**/*.py", "tests/**/*.py"]
line-length = 160 line-length = 160
indent-width = 4 indent-width = 4
target-version = "py310" target-version = "py310"
@@ -208,14 +208,10 @@ ignore = [
"TC006", # Add quotes to type expression in `typing.cast()` "TC006", # Add quotes to type expression in `typing.cast()`
] ]
[tool.ruff.format]
quote-style = "double"
indent-style = "space"
line-ending = "native"
docstring-code-format = false
skip-magic-trailing-comma = false
[tool.ruff.lint.per-file-ignores] [tool.ruff.lint.per-file-ignores]
"scripts/**/*.py" = [
"INP001", # File `...` is part of an implicit namespace package. Add an `__init__.py`.
]
"tests/**/*.py" = [ "tests/**/*.py" = [
"ARG", "ARG",
"B", "B",
@@ -247,7 +243,7 @@ max-statements = 150 # max. number of statements in function / method body (R091
# https://mypy.readthedocs.io/en/stable/config_file.html # https://mypy.readthedocs.io/en/stable/config_file.html
#mypy_path = "$MYPY_CONFIG_FILE_DIR/tests/stubs" #mypy_path = "$MYPY_CONFIG_FILE_DIR/tests/stubs"
python_version = "3.10" python_version = "3.10"
files = "src,tests" files = "scripts,src,tests"
strict = true strict = true
disallow_untyped_calls = false disallow_untyped_calls = false
disallow_untyped_defs = true disallow_untyped_defs = true
@@ -264,7 +260,7 @@ verbosity = 0
##################### #####################
[tool.basedpyright] [tool.basedpyright]
# https://docs.basedpyright.com/latest/configuration/config-files/ # https://docs.basedpyright.com/latest/configuration/config-files/
include = ["src", "tests"] include = ["scripts", "src", "tests"]
defineConstant = { DEBUG = false } defineConstant = { DEBUG = false }
pythonVersion = "3.10" pythonVersion = "3.10"
typeCheckingMode = "standard" typeCheckingMode = "standard"

317
scripts/post_autopep8.py Normal file
View File

@@ -0,0 +1,317 @@
# SPDX-FileCopyrightText: © Sebastian Thomschke and contributors
# SPDX-License-Identifier: AGPL-3.0-or-later
# SPDX-ArtifactOfProjectHomePage: https://github.com/Second-Hand-Friends/kleinanzeigen-bot/
import ast, logging, re, sys # isort: skip
from pathlib import Path
from typing import Final, List, Protocol, Tuple
from typing_extensions import override
# Configure basic logging
logging.basicConfig(level = logging.INFO, format = "%(levelname)s: %(message)s")
LOG:Final[logging.Logger] = logging.getLogger(__name__)
class FormatterRule(Protocol):
"""
A code processor that can modify source lines based on the AST.
"""
def apply(self, tree:ast.AST, lines:List[str], path:Path) -> List[str]:
...
class NoSpaceAfterColonInTypeAnnotationRule(FormatterRule):
"""
Removes whitespace between the colon (:) and the type annotation in variable and function parameter declarations.
This rule enforces `a:int` instead of `a: int`.
It is the opposite behavior of autopep8 rule E231.
Example:
# Before
def foo(a: int, b : str) -> None:
pass
# After
def foo(a:int, b:str) -> None:
pass
"""
@override
def apply(self, tree:ast.AST, lines:List[str], path:Path) -> List[str]:
ann_positions:List[Tuple[int, int]] = []
for node in ast.walk(tree):
if isinstance(node, ast.arg) and node.annotation is not None:
ann_positions.append((node.annotation.lineno - 1, node.annotation.col_offset))
elif isinstance(node, ast.AnnAssign) and node.annotation is not None:
ann = node.annotation
ann_positions.append((ann.lineno - 1, ann.col_offset))
if not ann_positions:
return lines
new_lines:List[str] = []
for idx, line in enumerate(lines):
if line.lstrip().startswith("#"):
new_lines.append(line)
continue
chars = list(line)
offsets = [col for (lin, col) in ann_positions if lin == idx]
for col in sorted(offsets, reverse = True):
prefix = "".join(chars[:col])
colon_idx = prefix.rfind(":")
if colon_idx == -1:
continue
j = colon_idx + 1
while j < len(chars) and chars[j].isspace():
del chars[j]
new_lines.append("".join(chars))
return new_lines
class EqualSignSpacingInDefaultsAndNamedArgsRule(FormatterRule):
"""
Ensures that the '=' sign in default values for function parameters and keyword arguments in function calls
is surrounded by exactly one space on each side.
This rule enforces `a:int = 3` instead of `a:int=3`, and `x = 42` instead of `x=42` or `x =42`.
It is the opposite behavior of autopep8 rule E251.
Example:
# Before
def foo(a:int=3, b :str= "bar"):
pass
foo(x=42,y = "hello")
# After
def foo(a:int = 3, b:str = "bar"):
pass
foo(x = 42, y = "hello")
"""
@override
def apply(self, tree:ast.AST, lines:List[str], path:Path) -> List[str]:
equals_positions:List[Tuple[int, int]] = []
for node in ast.walk(tree):
# --- Defaults in function definitions, async defs & lambdas ---
if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.Lambda)):
# positional defaults
equals_positions.extend(
(d.lineno - 1, d.col_offset)
for d in node.args.defaults
if d is not None
)
# keyword-only defaults (only on defs, not lambdas)
if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
equals_positions.extend(
(d.lineno - 1, d.col_offset)
for d in node.args.kw_defaults
if d is not None
)
# --- Keyword arguments in calls ---
if isinstance(node, ast.Call):
equals_positions.extend(
(kw.value.lineno - 1, kw.value.col_offset)
for kw in node.keywords
if kw.arg is not None
)
if not equals_positions:
return lines
new_lines:List[str] = []
for line_idx, line in enumerate(lines):
if line.lstrip().startswith("#"):
new_lines.append(line)
continue
chars = list(line)
equals_offsets = [col for (lineno, col) in equals_positions if lineno == line_idx]
for col in sorted(equals_offsets, reverse = True):
prefix = "".join(chars[:col])
equal_sign_idx = prefix.rfind("=")
if equal_sign_idx == -1:
continue
# remove spaces before '='
left_index = equal_sign_idx - 1
while left_index >= 0 and chars[left_index].isspace():
del chars[left_index]
equal_sign_idx -= 1
left_index -= 1
# remove spaces after '='
right_index = equal_sign_idx + 1
while right_index < len(chars) and chars[right_index].isspace():
del chars[right_index]
# insert single spaces
chars.insert(equal_sign_idx, " ")
chars.insert(equal_sign_idx + 2, " ")
new_lines.append("".join(chars))
return new_lines
class PreferDoubleQuotesRule(FormatterRule):
"""
Ensures string literals use double quotes unless the content contains a double quote.
Example:
# Before
foo = 'hello'
bar = 'a "quote" inside'
# After
foo = "hello"
bar = 'a "quote" inside' # kept as-is, because it contains a double quote
"""
@override
def apply(self, tree:ast.AST, lines:List[str], path:Path) -> List[str]:
new_lines = lines.copy()
# Track how much each line has shifted so far
line_shifts:dict[int, int] = dict.fromkeys(range(len(lines)), 0)
# Build a parent map for f-string detection
parent_map:dict[ast.AST, ast.AST] = {}
for parent in ast.walk(tree):
for child in ast.iter_child_nodes(parent):
parent_map[child] = parent
def is_in_fstring(node:ast.AST) -> bool:
p = parent_map.get(node)
while p:
if isinstance(p, ast.JoinedStr):
return True
p = parent_map.get(p)
return False
# Regex to locate a single- or triple-quoted literal:
# (?P<prefix>[rRbuUfF]*) optional string flags (r, b, u, f, etc.), case-insensitive
# (?P<quote>'{3}|') the opening delimiter: either three single-quotes (''') or one ('),
# but never two in a row (so we won't mis-interpret adjacent quotes)
# (?P<content>.*?) the literal's content, non-greedy up to the next same delimiter
# (?P=quote) the matching closing delimiter (same length as the opener)
literal_re = re.compile(
r"(?P<prefix>[rRbuUfF]*)(?P<quote>'{3}|')(?P<content>.*?)(?P=quote)",
re.DOTALL,
)
for node in ast.walk(tree):
# only handle simple string constants
if not (isinstance(node, ast.Constant) and isinstance(node.value, str)):
continue
# skip anything inside an f-string, at any depth
if is_in_fstring(node):
continue
starting_line_number = getattr(node, "lineno", None)
starting_col_offset = getattr(node, "col_offset", None)
if starting_line_number is None or starting_col_offset is None:
continue
start_line = starting_line_number - 1
shift = line_shifts[start_line]
raw = new_lines[start_line]
# apply shift so we match against current edited line
idx = starting_col_offset + shift
if idx >= len(raw) or raw[idx] not in ("'", "r", "u", "b", "f", "R", "U", "B", "F"):
continue
# match literal at that column
m = literal_re.match(raw[idx:])
if not m:
continue
prefix = m.group("prefix")
quote = m.group("quote") # either "'" or "'''"
content = m.group("content") # what's inside
# skip if content has a double-quote already
if '"' in content:
continue
# build new literal with the same prefix, but doublequote delimiter
delim = '"' * len(quote)
escaped = content.replace(delim, "\\" + delim)
new_literal = f"{prefix}{delim}{escaped}{delim}"
literal_len = m.end() # how many chars we're replacing
before = raw[:idx]
after = raw[idx + literal_len:]
new_lines[start_line] = before + new_literal + after
# record shift delta for any further edits on this line
line_shifts[start_line] += len(new_literal) - literal_len
return new_lines
FORMATTER_RULES:List[FormatterRule] = [
NoSpaceAfterColonInTypeAnnotationRule(),
EqualSignSpacingInDefaultsAndNamedArgsRule(),
PreferDoubleQuotesRule(),
]
def format_file(path:Path) -> None:
# Read without newline conversion
with path.open("r", encoding = "utf-8", newline = "") as rf:
original_text = rf.read()
# Initial parse
try:
tree = ast.parse(original_text)
except SyntaxError as e:
LOG.error(
"Syntax error parsing %s[%d:%d]: %r -> %s",
path, e.lineno, e.offset, (e.text or "").rstrip(), e.msg
)
return
lines = original_text.splitlines(keepends = True)
formatted_text = original_text
success = True
for rule in FORMATTER_RULES:
lines = rule.apply(tree, lines, path)
formatted_text = "".join(lines)
# Re-parse the updated text
try:
tree = ast.parse(formatted_text)
except SyntaxError as e:
LOG.error(
"Syntax error after %s at %s[%d:%d]: %r -> %s",
rule.__class__.__name__, path, e.lineno, e.offset, (e.text or "").rstrip(), e.msg
)
success = False
break
if success and formatted_text != original_text:
with path.open("w", encoding = "utf-8", newline = "") as wf:
wf.write(formatted_text)
LOG.info("Formatted [%s].", path)
if __name__ == "__main__":
if len(sys.argv) < 2: # noqa: PLR2004 Magic value used in comparison
script_path = Path(sys.argv[0])
print(f"Usage: python {script_path} <directory1> [<directory2> ...]")
sys.exit(1)
for dir_arg in sys.argv[1:]:
root = Path(dir_arg)
if not root.exists():
LOG.warning("Directory [%s] does not exist, skipping...", root)
continue
for py_file in root.rglob("*.py"):
format_file(py_file)

View File

@@ -83,11 +83,11 @@ class KleinanzeigenBot(WebScrapingMixin):
self.configure_file_logging() self.configure_file_logging()
self.load_config() self.load_config()
if not (self.ads_selector in {'all', 'new', 'due', 'changed'} or if not (self.ads_selector in {"all", "new", "due", "changed"} or
any(selector in self.ads_selector.split(',') for selector in ('all', 'new', 'due', 'changed')) or any(selector in self.ads_selector.split(",") for selector in ("all", "new", "due", "changed")) or
re.compile(r'\d+[,\d+]*').search(self.ads_selector)): re.compile(r"\d+[,\d+]*").search(self.ads_selector)):
LOG.warning('You provided no ads selector. Defaulting to "due".') LOG.warning('You provided no ads selector. Defaulting to "due".')
self.ads_selector = 'due' self.ads_selector = "due"
if ads := self.load_ads(): if ads := self.load_ads():
await self.create_browser_session() await self.create_browser_session()
@@ -111,9 +111,9 @@ class KleinanzeigenBot(WebScrapingMixin):
case "download": case "download":
self.configure_file_logging() self.configure_file_logging()
# ad IDs depends on selector # ad IDs depends on selector
if not (self.ads_selector in {'all', 'new'} or re.compile(r'\d+[,\d+]*').search(self.ads_selector)): if not (self.ads_selector in {"all", "new"} or re.compile(r"\d+[,\d+]*").search(self.ads_selector)):
LOG.warning('You provided no ads selector. Defaulting to "new".') LOG.warning('You provided no ads selector. Defaulting to "new".')
self.ads_selector = 'new' self.ads_selector = "new"
self.load_config() self.load_config()
await self.create_browser_session() await self.create_browser_session()
await self.login() await self.login()
@@ -327,7 +327,7 @@ class KleinanzeigenBot(WebScrapingMixin):
data_root_dir = os.path.dirname(self.config_file_path) data_root_dir = os.path.dirname(self.config_file_path)
for file_pattern in self.config["ad_files"]: for file_pattern in self.config["ad_files"]:
for ad_file in glob.glob(file_pattern, root_dir = data_root_dir, flags = glob.GLOBSTAR | glob.BRACE | glob.EXTGLOB): for ad_file in glob.glob(file_pattern, root_dir = data_root_dir, flags = glob.GLOBSTAR | glob.BRACE | glob.EXTGLOB):
if not str(ad_file).endswith('ad_fields.yaml'): if not str(ad_file).endswith("ad_fields.yaml"):
ad_files[abspath(ad_file, relative_to = data_root_dir)] = ad_file ad_files[abspath(ad_file, relative_to = data_root_dir)] = ad_file
LOG.info(" -> found %s", pluralize("ad config file", ad_files)) LOG.info(" -> found %s", pluralize("ad config file", ad_files))
if not ad_files: if not ad_files:
@@ -335,13 +335,13 @@ class KleinanzeigenBot(WebScrapingMixin):
ids = [] ids = []
use_specific_ads = False use_specific_ads = False
selectors = self.ads_selector.split(',') selectors = self.ads_selector.split(",")
if re.compile(r'\d+[,\d+]*').search(self.ads_selector): if re.compile(r"\d+[,\d+]*").search(self.ads_selector):
ids = [int(n) for n in self.ads_selector.split(',')] ids = [int(n) for n in self.ads_selector.split(",")]
use_specific_ads = True use_specific_ads = True
LOG.info('Start fetch task for the ad(s) with id(s):') LOG.info("Start fetch task for the ad(s) with id(s):")
LOG.info(' | '.join([str(id_) for id_ in ids])) LOG.info(" | ".join([str(id_) for id_ in ids]))
ad_fields = dicts.load_dict_from_module(resources, "ad_fields.yaml") ad_fields = dicts.load_dict_from_module(resources, "ad_fields.yaml")
ads = [] ads = []
@@ -548,7 +548,7 @@ class KleinanzeigenBot(WebScrapingMixin):
async def is_logged_in(self) -> bool: async def is_logged_in(self) -> bool:
try: try:
user_info = await self.web_text(By.CLASS_NAME, "mr-medium") user_info = await self.web_text(By.CLASS_NAME, "mr-medium")
if self.config['login']['username'].lower() in user_info.lower(): if self.config["login"]["username"].lower() in user_info.lower():
return True return True
except TimeoutError: except TimeoutError:
return False return False
@@ -657,7 +657,7 @@ class KleinanzeigenBot(WebScrapingMixin):
############################# #############################
# set category # set category
############################# #############################
await self.__set_category(ad_cfg['category'], ad_file) await self.__set_category(ad_cfg["category"], ad_file)
############################# #############################
# set special attributes # set special attributes
@@ -674,7 +674,7 @@ class KleinanzeigenBot(WebScrapingMixin):
try: try:
await self.web_select(By.XPATH, "//select[contains(@id, '.versand_s')]", shipping_value) await self.web_select(By.XPATH, "//select[contains(@id, '.versand_s')]", shipping_value)
except TimeoutError: except TimeoutError:
LOG.warning("Failed to set shipping attribute for type '%s'!", ad_cfg['shipping_type']) LOG.warning("Failed to set shipping attribute for type '%s'!", ad_cfg["shipping_type"])
else: else:
await self.__set_shipping(ad_cfg) await self.__set_shipping(ad_cfg)
@@ -698,9 +698,9 @@ class KleinanzeigenBot(WebScrapingMixin):
if ad_cfg["shipping_type"] == "SHIPPING": if ad_cfg["shipping_type"] == "SHIPPING":
if sell_directly and ad_cfg["shipping_options"] and price_type in {"FIXED", "NEGOTIABLE"}: if sell_directly and ad_cfg["shipping_options"] and price_type in {"FIXED", "NEGOTIABLE"}:
if not await self.web_check(By.ID, "radio-buy-now-yes", Is.SELECTED): if not await self.web_check(By.ID, "radio-buy-now-yes", Is.SELECTED):
await self.web_click(By.ID, 'radio-buy-now-yes') await self.web_click(By.ID, "radio-buy-now-yes")
elif not await self.web_check(By.ID, "radio-buy-now-no", Is.SELECTED): elif not await self.web_check(By.ID, "radio-buy-now-no", Is.SELECTED):
await self.web_click(By.ID, 'radio-buy-now-no') await self.web_click(By.ID, "radio-buy-now-no")
except TimeoutError as ex: except TimeoutError as ex:
LOG.debug(ex, exc_info = True) LOG.debug(ex, exc_info = True)
@@ -886,7 +886,7 @@ class KleinanzeigenBot(WebScrapingMixin):
async def __set_special_attributes(self, ad_cfg:dict[str, Any]) -> None: async def __set_special_attributes(self, ad_cfg:dict[str, Any]) -> None:
if ad_cfg["special_attributes"]: if ad_cfg["special_attributes"]:
LOG.debug('Found %i special attributes', len(ad_cfg["special_attributes"])) LOG.debug("Found %i special attributes", len(ad_cfg["special_attributes"]))
for special_attribute_key, special_attribute_value in ad_cfg["special_attributes"].items(): for special_attribute_key, special_attribute_value in ad_cfg["special_attributes"].items():
if special_attribute_key == "condition_s": if special_attribute_key == "condition_s":
@@ -911,10 +911,10 @@ class KleinanzeigenBot(WebScrapingMixin):
try: try:
elem_id = special_attr_elem.attrs.id elem_id = special_attr_elem.attrs.id
if special_attr_elem.local_name == 'select': if special_attr_elem.local_name == "select":
LOG.debug("Attribute field '%s' seems to be a select...", special_attribute_key) LOG.debug("Attribute field '%s' seems to be a select...", special_attribute_key)
await self.web_select(By.ID, elem_id, special_attribute_value) await self.web_select(By.ID, elem_id, special_attribute_value)
elif special_attr_elem.attrs.type == 'checkbox': elif special_attr_elem.attrs.type == "checkbox":
LOG.debug("Attribute field '%s' seems to be a checkbox...", special_attribute_key) LOG.debug("Attribute field '%s' seems to be a checkbox...", special_attribute_key)
await self.web_click(By.ID, elem_id) await self.web_click(By.ID, elem_id)
else: else:
@@ -1036,7 +1036,7 @@ class KleinanzeigenBot(WebScrapingMixin):
async def assert_free_ad_limit_not_reached(self) -> None: async def assert_free_ad_limit_not_reached(self) -> None:
try: try:
await self.web_find(By.XPATH, '/html/body/div[1]/form/fieldset[6]/div[1]/header', timeout = 2) await self.web_find(By.XPATH, "/html/body/div[1]/form/fieldset[6]/div[1]/header", timeout = 2)
raise AssertionError(f"Cannot publish more ads. The monthly limit of free ads of account {self.config['login']['username']} is reached.") raise AssertionError(f"Cannot publish more ads. The monthly limit of free ads of account {self.config['login']['username']} is reached.")
except TimeoutError: except TimeoutError:
pass pass
@@ -1050,13 +1050,13 @@ class KleinanzeigenBot(WebScrapingMixin):
ad_extractor = extract.AdExtractor(self.browser, self.config) ad_extractor = extract.AdExtractor(self.browser, self.config)
# use relevant download routine # use relevant download routine
if self.ads_selector in {'all', 'new'}: # explore ads overview for these two modes if self.ads_selector in {"all", "new"}: # explore ads overview for these two modes
LOG.info('Scanning your ad overview...') LOG.info("Scanning your ad overview...")
own_ad_urls = await ad_extractor.extract_own_ads_urls() own_ad_urls = await ad_extractor.extract_own_ads_urls()
LOG.info('%s found.', pluralize("ad", len(own_ad_urls))) LOG.info("%s found.", pluralize("ad", len(own_ad_urls)))
if self.ads_selector == 'all': # download all of your adds if self.ads_selector == "all": # download all of your adds
LOG.info('Starting download of all ads...') LOG.info("Starting download of all ads...")
success_count = 0 success_count = 0
# call download function for each ad page # call download function for each ad page
@@ -1067,12 +1067,12 @@ class KleinanzeigenBot(WebScrapingMixin):
success_count += 1 success_count += 1
LOG.info("%d of %d ads were downloaded from your profile.", success_count, len(own_ad_urls)) LOG.info("%d of %d ads were downloaded from your profile.", success_count, len(own_ad_urls))
elif self.ads_selector == 'new': # download only unsaved ads elif self.ads_selector == "new": # download only unsaved ads
# check which ads already saved # check which ads already saved
saved_ad_ids = [] saved_ad_ids = []
ads = self.load_ads(ignore_inactive = False, check_id = False) # do not skip because of existing IDs ads = self.load_ads(ignore_inactive = False, check_id = False) # do not skip because of existing IDs
for ad in ads: for ad in ads:
ad_id = int(ad[2]['id']) ad_id = int(ad[2]["id"])
saved_ad_ids.append(ad_id) saved_ad_ids.append(ad_id)
# determine ad IDs from links # determine ad IDs from links
@@ -1083,26 +1083,26 @@ class KleinanzeigenBot(WebScrapingMixin):
for ad_url, ad_id in ad_id_by_url.items(): for ad_url, ad_id in ad_id_by_url.items():
# check if ad with ID already saved # check if ad with ID already saved
if ad_id in saved_ad_ids: if ad_id in saved_ad_ids:
LOG.info('The ad with id %d has already been saved.', ad_id) LOG.info("The ad with id %d has already been saved.", ad_id)
continue continue
if await ad_extractor.naviagte_to_ad_page(ad_url): if await ad_extractor.naviagte_to_ad_page(ad_url):
await ad_extractor.download_ad(ad_id) await ad_extractor.download_ad(ad_id)
new_count += 1 new_count += 1
LOG.info('%s were downloaded from your profile.', pluralize("new ad", new_count)) LOG.info("%s were downloaded from your profile.", pluralize("new ad", new_count))
elif re.compile(r'\d+[,\d+]*').search(self.ads_selector): # download ad(s) with specific id(s) elif re.compile(r"\d+[,\d+]*").search(self.ads_selector): # download ad(s) with specific id(s)
ids = [int(n) for n in self.ads_selector.split(',')] ids = [int(n) for n in self.ads_selector.split(",")]
LOG.info('Starting download of ad(s) with the id(s):') LOG.info("Starting download of ad(s) with the id(s):")
LOG.info(' | '.join([str(ad_id) for ad_id in ids])) LOG.info(" | ".join([str(ad_id) for ad_id in ids]))
for ad_id in ids: # call download routine for every id for ad_id in ids: # call download routine for every id
exists = await ad_extractor.naviagte_to_ad_page(ad_id) exists = await ad_extractor.naviagte_to_ad_page(ad_id)
if exists: if exists:
await ad_extractor.download_ad(ad_id) await ad_extractor.download_ad(ad_id)
LOG.info('Downloaded ad with id %d', ad_id) LOG.info("Downloaded ad with id %d", ad_id)
else: else:
LOG.error('The page with the id %d does not exist!', ad_id) LOG.error("The page with the id %d does not exist!", ad_id)
def __get_description_with_affixes(self, ad_cfg:dict[str, Any]) -> str: def __get_description_with_affixes(self, ad_cfg:dict[str, Any]) -> str:
"""Get the complete description with prefix and suffix applied. """Get the complete description with prefix and suffix applied.

View File

@@ -36,22 +36,22 @@ class AdExtractor(WebScrapingMixin):
""" """
# create sub-directory for ad(s) to download (if necessary): # create sub-directory for ad(s) to download (if necessary):
relative_directory = 'downloaded-ads' relative_directory = "downloaded-ads"
# make sure configured base directory exists # make sure configured base directory exists
if not os.path.exists(relative_directory) or not os.path.isdir(relative_directory): if not os.path.exists(relative_directory) or not os.path.isdir(relative_directory):
os.mkdir(relative_directory) os.mkdir(relative_directory)
LOG.info('Created ads directory at ./%s.', relative_directory) LOG.info("Created ads directory at ./%s.", relative_directory)
new_base_dir = os.path.join(relative_directory, f'ad_{ad_id}') new_base_dir = os.path.join(relative_directory, f'ad_{ad_id}')
if os.path.exists(new_base_dir): if os.path.exists(new_base_dir):
LOG.info('Deleting current folder of ad %s...', ad_id) LOG.info("Deleting current folder of ad %s...", ad_id)
shutil.rmtree(new_base_dir) shutil.rmtree(new_base_dir)
os.mkdir(new_base_dir) os.mkdir(new_base_dir)
LOG.info('New directory for ad created at %s.', new_base_dir) LOG.info("New directory for ad created at %s.", new_base_dir)
# call extraction function # call extraction function
info = await self._extract_ad_page_info(new_base_dir, ad_id) info = await self._extract_ad_page_info(new_base_dir, ad_id)
ad_file_path = new_base_dir + '/' + f'ad_{ad_id}.yaml' ad_file_path = new_base_dir + "/" + f'ad_{ad_id}.yaml'
dicts.save_dict(ad_file_path, info) dicts.save_dict(ad_file_path, info)
async def _download_images_from_ad_page(self, directory:str, ad_id:int) -> list[str]: async def _download_images_from_ad_page(self, directory:str, ad_id:int) -> list[str]:
@@ -67,18 +67,18 @@ class AdExtractor(WebScrapingMixin):
img_paths = [] img_paths = []
try: try:
# download all images from box # download all images from box
image_box = await self.web_find(By.CLASS_NAME, 'galleryimage-large') image_box = await self.web_find(By.CLASS_NAME, "galleryimage-large")
n_images = len(await self.web_find_all(By.CSS_SELECTOR, '.galleryimage-element[data-ix]', parent = image_box)) n_images = len(await self.web_find_all(By.CSS_SELECTOR, ".galleryimage-element[data-ix]", parent = image_box))
LOG.info('Found %s.', i18n.pluralize("image", n_images)) LOG.info("Found %s.", i18n.pluralize("image", n_images))
img_element:Element = await self.web_find(By.CSS_SELECTOR, 'div:nth-child(1) > img', parent = image_box) img_element:Element = await self.web_find(By.CSS_SELECTOR, "div:nth-child(1) > img", parent = image_box)
img_fn_prefix = 'ad_' + str(ad_id) + '__img' img_fn_prefix = "ad_" + str(ad_id) + "__img"
img_nr = 1 img_nr = 1
dl_counter = 0 dl_counter = 0
while img_nr <= n_images: # scrolling + downloading while img_nr <= n_images: # scrolling + downloading
current_img_url = img_element.attrs['src'] # URL of the image current_img_url = img_element.attrs["src"] # URL of the image
if current_img_url is None: if current_img_url is None:
continue continue
@@ -86,26 +86,26 @@ class AdExtractor(WebScrapingMixin):
content_type = response.info().get_content_type() content_type = response.info().get_content_type()
file_ending = mimetypes.guess_extension(content_type) file_ending = mimetypes.guess_extension(content_type)
img_path = f"{directory}/{img_fn_prefix}{img_nr}{file_ending}" img_path = f"{directory}/{img_fn_prefix}{img_nr}{file_ending}"
with open(img_path, 'wb') as f: with open(img_path, "wb") as f:
shutil.copyfileobj(response, f) shutil.copyfileobj(response, f)
dl_counter += 1 dl_counter += 1
img_paths.append(img_path.rsplit('/', maxsplit = 1)[-1]) img_paths.append(img_path.rsplit("/", maxsplit = 1)[-1])
# navigate to next image (if exists) # navigate to next image (if exists)
if img_nr < n_images: if img_nr < n_images:
try: try:
# click next button, wait, and re-establish reference # click next button, wait, and re-establish reference
await (await self.web_find(By.CLASS_NAME, 'galleryimage--navigation--next')).click() await (await self.web_find(By.CLASS_NAME, "galleryimage--navigation--next")).click()
new_div = await self.web_find(By.CSS_SELECTOR, f'div.galleryimage-element:nth-child({img_nr + 1})') new_div = await self.web_find(By.CSS_SELECTOR, f'div.galleryimage-element:nth-child({img_nr + 1})')
img_element = await self.web_find(By.TAG_NAME, 'img', parent = new_div) img_element = await self.web_find(By.TAG_NAME, "img", parent = new_div)
except TimeoutError: except TimeoutError:
LOG.error('NEXT button in image gallery somehow missing, aborting image fetching.') LOG.error("NEXT button in image gallery somehow missing, aborting image fetching.")
break break
img_nr += 1 img_nr += 1
LOG.info('Downloaded %s.', i18n.pluralize("image", dl_counter)) LOG.info("Downloaded %s.", i18n.pluralize("image", dl_counter))
except TimeoutError: # some ads do not require images except TimeoutError: # some ads do not require images
LOG.warning('No image area found. Continuing without downloading images.') LOG.warning("No image area found. Continuing without downloading images.")
return img_paths return img_paths
@@ -116,13 +116,13 @@ class AdExtractor(WebScrapingMixin):
:param url: the URL to the ad page :param url: the URL to the ad page
:return: the ad ID, a (ten-digit) integer number :return: the ad ID, a (ten-digit) integer number
""" """
num_part = url.split('/')[-1] # suffix num_part = url.split("/")[-1] # suffix
id_part = num_part.split('-')[0] id_part = num_part.split("-")[0]
try: try:
path = url.split('?', 1)[0] # Remove query string if present path = url.split("?", 1)[0] # Remove query string if present
last_segment = path.rstrip('/').split('/')[-1] # Get last path component last_segment = path.rstrip("/").split("/")[-1] # Get last path component
id_part = last_segment.split('-')[0] # Extract part before first hyphen id_part = last_segment.split("-")[0] # Extract part before first hyphen
return int(id_part) return int(id_part)
except (IndexError, ValueError) as ex: except (IndexError, ValueError) as ex:
LOG.warning("Failed to extract ad ID from URL '%s': %s", url, ex) LOG.warning("Failed to extract ad ID from URL '%s': %s", url, ex)
@@ -135,41 +135,41 @@ class AdExtractor(WebScrapingMixin):
:return: the links to your ad pages :return: the links to your ad pages
""" """
# navigate to "your ads" page # navigate to "your ads" page
await self.web_open('https://www.kleinanzeigen.de/m-meine-anzeigen.html') await self.web_open("https://www.kleinanzeigen.de/m-meine-anzeigen.html")
await self.web_sleep(2000, 3000) # Consider replacing with explicit waits later await self.web_sleep(2000, 3000) # Consider replacing with explicit waits later
# Try to find the main ad list container first # Try to find the main ad list container first
try: try:
ad_list_container = await self.web_find(By.ID, 'my-manageitems-adlist') ad_list_container = await self.web_find(By.ID, "my-manageitems-adlist")
except TimeoutError: except TimeoutError:
LOG.warning('Ad list container #my-manageitems-adlist not found. Maybe no ads present?') LOG.warning("Ad list container #my-manageitems-adlist not found. Maybe no ads present?")
return [] return []
# --- Pagination handling --- # --- Pagination handling ---
multi_page = False multi_page = False
try: try:
# Correct selector: Use uppercase '.Pagination' # Correct selector: Use uppercase '.Pagination'
pagination_section = await self.web_find(By.CSS_SELECTOR, '.Pagination', timeout=10) # Increased timeout slightly pagination_section = await self.web_find(By.CSS_SELECTOR, ".Pagination", timeout = 10) # Increased timeout slightly
# Correct selector: Use 'aria-label' # Correct selector: Use 'aria-label'
# Also check if the button is actually present AND potentially enabled (though enabled check isn't strictly necessary here, only for clicking later) # Also check if the button is actually present AND potentially enabled (though enabled check isn't strictly necessary here, only for clicking later)
next_buttons = await self.web_find_all(By.CSS_SELECTOR, 'button[aria-label="Nächste"]', parent = pagination_section) next_buttons = await self.web_find_all(By.CSS_SELECTOR, 'button[aria-label="Nächste"]', parent = pagination_section)
if next_buttons: if next_buttons:
# Check if at least one 'Nächste' button is not disabled (optional but good practice) # Check if at least one 'Nächste' button is not disabled (optional but good practice)
enabled_next_buttons = [btn for btn in next_buttons if not btn.attrs.get('disabled')] enabled_next_buttons = [btn for btn in next_buttons if not btn.attrs.get("disabled")]
if enabled_next_buttons: if enabled_next_buttons:
multi_page = True multi_page = True
LOG.info('Multiple ad pages detected.') LOG.info("Multiple ad pages detected.")
else: else:
LOG.info('Next button found but is disabled. Assuming single effective page.') LOG.info("Next button found but is disabled. Assuming single effective page.")
else: else:
LOG.info('No "Naechste" button found within pagination. Assuming single page.') LOG.info('No "Naechste" button found within pagination. Assuming single page.')
except TimeoutError: except TimeoutError:
# This will now correctly trigger only if the '.Pagination' div itself is not found # This will now correctly trigger only if the '.Pagination' div itself is not found
LOG.info('No pagination controls found. Assuming single page.') LOG.info("No pagination controls found. Assuming single page.")
except Exception as e: except Exception as e:
LOG.exception("Error during pagination detection: %s", e) LOG.exception("Error during pagination detection: %s", e)
LOG.info('Assuming single page due to error during pagination check.') LOG.info("Assuming single page due to error during pagination check.")
# --- End Pagination Handling --- # --- End Pagination Handling ---
refs:list[str] = [] refs:list[str] = []
@@ -182,8 +182,8 @@ class AdExtractor(WebScrapingMixin):
# Re-find the ad list container on the current page/state # Re-find the ad list container on the current page/state
try: try:
ad_list_container = await self.web_find(By.ID, 'my-manageitems-adlist') ad_list_container = await self.web_find(By.ID, "my-manageitems-adlist")
list_items = await self.web_find_all(By.CLASS_NAME, 'cardbox', parent=ad_list_container) list_items = await self.web_find_all(By.CLASS_NAME, "cardbox", parent = ad_list_container)
LOG.info("Found %s ad items on page %s.", len(list_items), current_page) LOG.info("Found %s ad items on page %s.", len(list_items), current_page)
except TimeoutError: except TimeoutError:
LOG.warning("Could not find ad list container or items on page %s.", current_page) LOG.warning("Could not find ad list container or items on page %s.", current_page)
@@ -192,7 +192,7 @@ class AdExtractor(WebScrapingMixin):
# Extract references using the CORRECTED selector # Extract references using the CORRECTED selector
try: try:
page_refs = [ page_refs = [
(await self.web_find(By.CSS_SELECTOR, 'div.manageitems-item-ad h3 a.text-onSurface', parent=li)).attrs['href'] (await self.web_find(By.CSS_SELECTOR, "div.manageitems-item-ad h3 a.text-onSurface", parent = li)).attrs["href"]
for li in list_items for li in list_items
] ]
refs.extend(page_refs) refs.extend(page_refs)
@@ -207,12 +207,12 @@ class AdExtractor(WebScrapingMixin):
# --- Navigate to next page --- # --- Navigate to next page ---
try: try:
# Find the pagination section again (scope might have changed after scroll/wait) # Find the pagination section again (scope might have changed after scroll/wait)
pagination_section = await self.web_find(By.CSS_SELECTOR, '.Pagination', timeout=5) pagination_section = await self.web_find(By.CSS_SELECTOR, ".Pagination", timeout = 5)
# Find the "Next" button using the correct aria-label selector and ensure it's not disabled # Find the "Next" button using the correct aria-label selector and ensure it's not disabled
next_button_element = None next_button_element = None
possible_next_buttons = await self.web_find_all(By.CSS_SELECTOR, 'button[aria-label="Nächste"]', parent = pagination_section) possible_next_buttons = await self.web_find_all(By.CSS_SELECTOR, 'button[aria-label="Nächste"]', parent = pagination_section)
for btn in possible_next_buttons: for btn in possible_next_buttons:
if not btn.attrs.get('disabled'): # Check if the button is enabled if not btn.attrs.get("disabled"): # Check if the button is enabled
next_button_element = btn next_button_element = btn
break # Found an enabled next button break # Found an enabled next button
@@ -235,7 +235,7 @@ class AdExtractor(WebScrapingMixin):
# --- End Navigation --- # --- End Navigation ---
if not refs: if not refs:
LOG.warning('No ad URLs were extracted.') LOG.warning("No ad URLs were extracted.")
return refs return refs
@@ -246,27 +246,27 @@ class AdExtractor(WebScrapingMixin):
""" """
if reflect.is_integer(id_or_url): if reflect.is_integer(id_or_url):
# navigate to start page, otherwise page can be None! # navigate to start page, otherwise page can be None!
await self.web_open('https://www.kleinanzeigen.de/') await self.web_open("https://www.kleinanzeigen.de/")
# enter the ad ID into the search bar # enter the ad ID into the search bar
await self.web_input(By.ID, "site-search-query", id_or_url) await self.web_input(By.ID, "site-search-query", id_or_url)
# navigate to ad page and wait # navigate to ad page and wait
await self.web_check(By.ID, 'site-search-submit', Is.CLICKABLE) await self.web_check(By.ID, "site-search-submit", Is.CLICKABLE)
submit_button = await self.web_find(By.ID, 'site-search-submit') submit_button = await self.web_find(By.ID, "site-search-submit")
await submit_button.click() await submit_button.click()
else: else:
await self.web_open(str(id_or_url)) # navigate to URL directly given await self.web_open(str(id_or_url)) # navigate to URL directly given
await self.web_sleep() await self.web_sleep()
# handle the case that invalid ad ID given # handle the case that invalid ad ID given
if self.page.url.endswith('k0'): if self.page.url.endswith("k0"):
LOG.error('There is no ad under the given ID.') LOG.error("There is no ad under the given ID.")
return False return False
# close (warning) popup, if given # close (warning) popup, if given
try: try:
await self.web_find(By.ID, 'vap-ovrly-secure') await self.web_find(By.ID, "vap-ovrly-secure")
LOG.warning('A popup appeared!') LOG.warning("A popup appeared!")
await self.web_click(By.CLASS_NAME, 'mfp-close') await self.web_click(By.CLASS_NAME, "mfp-close")
await self.web_sleep() await self.web_sleep()
except TimeoutError: except TimeoutError:
pass pass
@@ -280,18 +280,18 @@ class AdExtractor(WebScrapingMixin):
:param ad_id: the ad ID, already extracted by a calling function :param ad_id: the ad ID, already extracted by a calling function
:return: a dictionary with the keys as given in an ad YAML, and their respective values :return: a dictionary with the keys as given in an ad YAML, and their respective values
""" """
info:dict[str, Any] = {'active': True} info:dict[str, Any] = {"active": True}
# extract basic info # extract basic info
info['type'] = 'OFFER' if 's-anzeige' in self.page.url else 'WANTED' info["type"] = "OFFER" if "s-anzeige" in self.page.url else "WANTED"
title:str = await self.web_text(By.ID, 'viewad-title') title:str = await self.web_text(By.ID, "viewad-title")
LOG.info('Extracting information from ad with title "%s"', title) LOG.info('Extracting information from ad with title "%s"', title)
info['category'] = await self._extract_category_from_ad_page() info["category"] = await self._extract_category_from_ad_page()
info['title'] = title info["title"] = title
# Get raw description text # Get raw description text
raw_description = (await self.web_text(By.ID, 'viewad-description-text')).strip() raw_description = (await self.web_text(By.ID, "viewad-description-text")).strip()
# Get prefix and suffix from config # Get prefix and suffix from config
prefix = get_description_affixes(self.config, prefix = True) prefix = get_description_affixes(self.config, prefix = True)
@@ -304,38 +304,38 @@ class AdExtractor(WebScrapingMixin):
if suffix and description_text.endswith(suffix.strip()): if suffix and description_text.endswith(suffix.strip()):
description_text = description_text[:-len(suffix.strip())] description_text = description_text[:-len(suffix.strip())]
info['description'] = description_text.strip() info["description"] = description_text.strip()
info['special_attributes'] = await self._extract_special_attributes_from_ad_page() info["special_attributes"] = await self._extract_special_attributes_from_ad_page()
if "art_s" in info['special_attributes']: if "art_s" in info["special_attributes"]:
# change e.g. category "161/172" to "161/172/lautsprecher_kopfhoerer" # change e.g. category "161/172" to "161/172/lautsprecher_kopfhoerer"
info['category'] = f"{info['category']}/{info['special_attributes']['art_s']}" info["category"] = f"{info['category']}/{info['special_attributes']['art_s']}"
del info['special_attributes']['art_s'] del info["special_attributes"]["art_s"]
if "schaden_s" in info['special_attributes']: if "schaden_s" in info["special_attributes"]:
# change f to 'nein' and 't' to 'ja' # change f to 'nein' and 't' to 'ja'
info['special_attributes']['schaden_s'] = info['special_attributes']['schaden_s'].translate(str.maketrans({'t': 'ja', 'f': 'nein'})) info["special_attributes"]["schaden_s"] = info["special_attributes"]["schaden_s"].translate(str.maketrans({"t": "ja", "f": "nein"}))
info['price'], info['price_type'] = await self._extract_pricing_info_from_ad_page() info["price"], info["price_type"] = await self._extract_pricing_info_from_ad_page()
info['shipping_type'], info['shipping_costs'], info['shipping_options'] = await self._extract_shipping_info_from_ad_page() info["shipping_type"], info["shipping_costs"], info["shipping_options"] = await self._extract_shipping_info_from_ad_page()
info['sell_directly'] = await self._extract_sell_directly_from_ad_page() info["sell_directly"] = await self._extract_sell_directly_from_ad_page()
info['images'] = await self._download_images_from_ad_page(directory, ad_id) info["images"] = await self._download_images_from_ad_page(directory, ad_id)
info['contact'] = await self._extract_contact_from_ad_page() info["contact"] = await self._extract_contact_from_ad_page()
info['id'] = ad_id info["id"] = ad_id
try: # try different locations known for creation date element try: # try different locations known for creation date element
creation_date = await self.web_text(By.XPATH, creation_date = await self.web_text(By.XPATH,
'/html/body/div[1]/div[2]/div/section[2]/section/section/article/div[3]/div[2]/div[2]/div[1]/span') "/html/body/div[1]/div[2]/div/section[2]/section/section/article/div[3]/div[2]/div[2]/div[1]/span")
except TimeoutError: except TimeoutError:
creation_date = await self.web_text(By.CSS_SELECTOR, '#viewad-extra-info > div:nth-child(1) > span:nth-child(2)') creation_date = await self.web_text(By.CSS_SELECTOR, "#viewad-extra-info > div:nth-child(1) > span:nth-child(2)")
# convert creation date to ISO format # convert creation date to ISO format
created_parts = creation_date.split('.') created_parts = creation_date.split(".")
creation_date = created_parts[2] + '-' + created_parts[1] + '-' + created_parts[0] + ' 00:00:00' creation_date = created_parts[2] + "-" + created_parts[1] + "-" + created_parts[0] + " 00:00:00"
creation_date = datetime.fromisoformat(creation_date).isoformat() creation_date = datetime.fromisoformat(creation_date).isoformat()
info['created_on'] = creation_date info["created_on"] = creation_date
info['updated_on'] = None # will be set later on info["updated_on"] = None # will be set later on
# Calculate the initial hash for the downloaded ad # Calculate the initial hash for the downloaded ad
info['content_hash'] = calculate_content_hash(info) info["content_hash"] = calculate_content_hash(info)
return info return info
@@ -346,12 +346,12 @@ class AdExtractor(WebScrapingMixin):
:return: a category string of form abc/def, where a-f are digits :return: a category string of form abc/def, where a-f are digits
""" """
category_line = await self.web_find(By.ID, 'vap-brdcrmb') category_line = await self.web_find(By.ID, "vap-brdcrmb")
category_first_part = await self.web_find(By.CSS_SELECTOR, 'a:nth-of-type(2)', parent = category_line) category_first_part = await self.web_find(By.CSS_SELECTOR, "a:nth-of-type(2)", parent = category_line)
category_second_part = await self.web_find(By.CSS_SELECTOR, 'a:nth-of-type(3)', parent = category_line) category_second_part = await self.web_find(By.CSS_SELECTOR, "a:nth-of-type(3)", parent = category_line)
cat_num_first = category_first_part.attrs['href'].split('/')[-1][1:] cat_num_first = category_first_part.attrs["href"].split("/")[-1][1:]
cat_num_second = category_second_part.attrs['href'].split('/')[-1][1:] cat_num_second = category_second_part.attrs["href"].split("/")[-1][1:]
category:str = cat_num_first + '/' + cat_num_second category:str = cat_num_first + "/" + cat_num_second
return category return category
@@ -368,7 +368,7 @@ class AdExtractor(WebScrapingMixin):
special_attributes_str = belen_conf["universalAnalyticsOpts"]["dimensions"]["dimension108"] special_attributes_str = belen_conf["universalAnalyticsOpts"]["dimensions"]["dimension108"]
special_attributes = dict(item.split(":") for item in special_attributes_str.split("|") if ":" in item) special_attributes = dict(item.split(":") for item in special_attributes_str.split("|") if ":" in item)
special_attributes = {k: v for k, v in special_attributes.items() if not k.endswith('.versand_s') and k != "versand_s"} special_attributes = {k: v for k, v in special_attributes.items() if not k.endswith(".versand_s") and k != "versand_s"}
return special_attributes return special_attributes
async def _extract_pricing_info_from_ad_page(self) -> tuple[float | None, str]: async def _extract_pricing_info_from_ad_page(self) -> tuple[float | None, str]:
@@ -378,24 +378,24 @@ class AdExtractor(WebScrapingMixin):
:return: the price of the offer (optional); and the pricing type :return: the price of the offer (optional); and the pricing type
""" """
try: try:
price_str:str = await self.web_text(By.ID, 'viewad-price') price_str:str = await self.web_text(By.ID, "viewad-price")
price:int | None = None price:int | None = None
match price_str.split()[-1]: match price_str.split()[-1]:
case '': case "":
price_type = 'FIXED' price_type = "FIXED"
# replace('.', '') is to remove the thousands separator before parsing as int # replace('.', '') is to remove the thousands separator before parsing as int
price = int(price_str.replace('.', '').split()[0]) price = int(price_str.replace(".", "").split()[0])
case 'VB': case "VB":
price_type = 'NEGOTIABLE' price_type = "NEGOTIABLE"
if price_str != "VB": # can be either 'X € VB', or just 'VB' if price_str != "VB": # can be either 'X € VB', or just 'VB'
price = int(price_str.replace('.', '').split()[0]) price = int(price_str.replace(".", "").split()[0])
case 'verschenken': case "verschenken":
price_type = 'GIVE_AWAY' price_type = "GIVE_AWAY"
case _: case _:
price_type = 'NOT_APPLICABLE' price_type = "NOT_APPLICABLE"
return price, price_type return price, price_type
except TimeoutError: # no 'commercial' ad, has no pricing box etc. except TimeoutError: # no 'commercial' ad, has no pricing box etc.
return None, 'NOT_APPLICABLE' return None, "NOT_APPLICABLE"
async def _extract_shipping_info_from_ad_page(self) -> tuple[str, float | None, list[str] | None]: async def _extract_shipping_info_from_ad_page(self) -> tuple[str, float | None, list[str] | None]:
""" """
@@ -403,17 +403,17 @@ class AdExtractor(WebScrapingMixin):
:return: the shipping type, and the shipping price (optional) :return: the shipping type, and the shipping price (optional)
""" """
ship_type, ship_costs, shipping_options = 'NOT_APPLICABLE', None, None ship_type, ship_costs, shipping_options = "NOT_APPLICABLE", None, None
try: try:
shipping_text = await self.web_text(By.CLASS_NAME, 'boxedarticle--details--shipping') shipping_text = await self.web_text(By.CLASS_NAME, "boxedarticle--details--shipping")
# e.g. '+ Versand ab 5,49 €' OR 'Nur Abholung' # e.g. '+ Versand ab 5,49 €' OR 'Nur Abholung'
if shipping_text == 'Nur Abholung': if shipping_text == "Nur Abholung":
ship_type = 'PICKUP' ship_type = "PICKUP"
elif shipping_text == 'Versand möglich': elif shipping_text == "Versand möglich":
ship_type = 'SHIPPING' ship_type = "SHIPPING"
elif '' in shipping_text: elif "" in shipping_text:
shipping_price_parts = shipping_text.split(' ') shipping_price_parts = shipping_text.split(" ")
ship_type = 'SHIPPING' ship_type = "SHIPPING"
ship_costs = float(misc.parse_decimal(shipping_price_parts[-2])) ship_costs = float(misc.parse_decimal(shipping_price_parts[-2]))
# reading shipping option from kleinanzeigen # reading shipping option from kleinanzeigen
@@ -425,7 +425,7 @@ class AdExtractor(WebScrapingMixin):
internal_shipping_opt = [x for x in shipping_costs if x["priceInEuroCent"] == ship_costs * 100] internal_shipping_opt = [x for x in shipping_costs if x["priceInEuroCent"] == ship_costs * 100]
if not internal_shipping_opt: if not internal_shipping_opt:
return 'NOT_APPLICABLE', ship_costs, shipping_options return "NOT_APPLICABLE", ship_costs, shipping_options
# map to internal shipping identifiers used by kleinanzeigen-bot # map to internal shipping identifiers used by kleinanzeigen-bot
shipping_option_mapping = { shipping_option_mapping = {
@@ -440,13 +440,13 @@ class AdExtractor(WebScrapingMixin):
"HERMES_004": "Hermes_L" "HERMES_004": "Hermes_L"
} }
shipping_option = shipping_option_mapping.get(internal_shipping_opt[0]['id']) shipping_option = shipping_option_mapping.get(internal_shipping_opt[0]["id"])
if not shipping_option: if not shipping_option:
return 'NOT_APPLICABLE', ship_costs, shipping_options return "NOT_APPLICABLE", ship_costs, shipping_options
shipping_options = [shipping_option] shipping_options = [shipping_option]
except TimeoutError: # no pricing box -> no shipping given except TimeoutError: # no pricing box -> no shipping given
ship_type = 'NOT_APPLICABLE' ship_type = "NOT_APPLICABLE"
return ship_type, ship_costs, shipping_options return ship_type, ship_costs, shipping_options
@@ -457,7 +457,7 @@ class AdExtractor(WebScrapingMixin):
:return: a boolean indicating whether the sell directly option is active (optional) :return: a boolean indicating whether the sell directly option is active (optional)
""" """
try: try:
buy_now_is_active:bool = 'Direkt kaufen' in (await self.web_text(By.ID, 'payment-buttons-sidebar')) buy_now_is_active:bool = "Direkt kaufen" in (await self.web_text(By.ID, "payment-buttons-sidebar"))
return buy_now_is_active return buy_now_is_active
except TimeoutError: except TimeoutError:
return None return None
@@ -469,34 +469,34 @@ class AdExtractor(WebScrapingMixin):
:return: a dictionary containing the address parts with their corresponding values :return: a dictionary containing the address parts with their corresponding values
""" """
contact:dict[str, (str | None)] = {} contact:dict[str, (str | None)] = {}
address_text = await self.web_text(By.ID, 'viewad-locality') address_text = await self.web_text(By.ID, "viewad-locality")
# format: e.g. (Beispiel Allee 42,) 12345 Bundesland - Stadt # format: e.g. (Beispiel Allee 42,) 12345 Bundesland - Stadt
try: try:
street = (await self.web_text(By.ID, 'street-address'))[:-1] # trailing comma street = (await self.web_text(By.ID, "street-address"))[:-1] # trailing comma
contact['street'] = street contact["street"] = street
except TimeoutError: except TimeoutError:
LOG.info('No street given in the contact.') LOG.info("No street given in the contact.")
(zipcode, location) = address_text.split(" ", 1) (zipcode, location) = address_text.split(" ", 1)
contact['zipcode'] = zipcode # e.g. 19372 contact["zipcode"] = zipcode # e.g. 19372
contact['location'] = location # e.g. Mecklenburg-Vorpommern - Steinbeck contact["location"] = location # e.g. Mecklenburg-Vorpommern - Steinbeck
contact_person_element:Element = await self.web_find(By.ID, 'viewad-contact') contact_person_element:Element = await self.web_find(By.ID, "viewad-contact")
name_element = await self.web_find(By.CLASS_NAME, 'iconlist-text', parent = contact_person_element) name_element = await self.web_find(By.CLASS_NAME, "iconlist-text", parent = contact_person_element)
try: try:
name = await self.web_text(By.TAG_NAME, 'a', parent = name_element) name = await self.web_text(By.TAG_NAME, "a", parent = name_element)
except TimeoutError: # edge case: name without link except TimeoutError: # edge case: name without link
name = await self.web_text(By.TAG_NAME, 'span', parent = name_element) name = await self.web_text(By.TAG_NAME, "span", parent = name_element)
contact['name'] = name contact["name"] = name
if 'street' not in contact: if "street" not in contact:
contact['street'] = None contact["street"] = None
try: # phone number is unusual for non-professional sellers today try: # phone number is unusual for non-professional sellers today
phone_element = await self.web_find(By.ID, 'viewad-contact-phone') phone_element = await self.web_find(By.ID, "viewad-contact-phone")
phone_number = await self.web_text(By.TAG_NAME, 'a', parent = phone_element) phone_number = await self.web_text(By.TAG_NAME, "a", parent = phone_element)
contact['phone'] = ''.join(phone_number.replace('-', ' ').split(' ')).replace('+49(0)', '0') contact["phone"] = "".join(phone_number.replace("-", " ").split(" ")).replace("+49(0)", "0")
except TimeoutError: except TimeoutError:
contact['phone'] = None # phone seems to be a deprecated feature (for non-professional users) contact["phone"] = None # phone seems to be a deprecated feature (for non-professional users)
# also see 'https://themen.kleinanzeigen.de/hilfe/deine-anzeigen/Telefon/ # also see 'https://themen.kleinanzeigen.de/hilfe/deine-anzeigen/Telefon/
return contact return contact

View File

@@ -96,7 +96,7 @@ def save_dict(filepath:str, content:dict[str, Any]) -> None:
yaml.indent(mapping = 2, sequence = 4, offset = 2) yaml.indent(mapping = 2, sequence = 4, offset = 2)
yaml.representer.add_representer(str, # use YAML | block style for multi-line strings yaml.representer.add_representer(str, # use YAML | block style for multi-line strings
lambda dumper, data: lambda dumper, data:
dumper.represent_scalar('tag:yaml.org,2002:str', data, style = '|' if '\n' in data else None) dumper.represent_scalar("tag:yaml.org,2002:str", data, style = "|" if "\n" in data else None)
) )
yaml.allow_duplicate_keys = False yaml.allow_duplicate_keys = False
yaml.explicit_start = False yaml.explicit_start = False

View File

@@ -3,7 +3,7 @@
# SPDX-ArtifactOfProjectHomePage: https://github.com/Second-Hand-Friends/kleinanzeigen-bot/ # SPDX-ArtifactOfProjectHomePage: https://github.com/Second-Hand-Friends/kleinanzeigen-bot/
import sys, traceback # isort: skip import sys, traceback # isort: skip
from types import FrameType, TracebackType from types import FrameType, TracebackType
from typing import Any, Final from typing import Final
from . import loggers from . import loggers

View File

@@ -42,7 +42,7 @@ class Locale(NamedTuple):
return f"{self.language}{region_part}{encoding_part}" return f"{self.language}{region_part}{encoding_part}"
@staticmethod @staticmethod
def of(locale_string: str) -> 'Locale': def of(locale_string:str) -> "Locale":
""" """
>>> Locale.of("en_US.UTF-8") >>> Locale.of("en_US.UTF-8")
Locale(language='en', region='US', encoding='UTF-8') Locale(language='en', region='US', encoding='UTF-8')
@@ -105,7 +105,7 @@ def translate(text:object, caller: inspect.FrameInfo | None) -> str:
if not _TRANSLATIONS: if not _TRANSLATIONS:
return text return text
module_name = caller.frame.f_globals.get('__name__') # pylint: disable=redefined-outer-name module_name = caller.frame.f_globals.get("__name__") # pylint: disable=redefined-outer-name
file_basename = os.path.splitext(os.path.basename(caller.filename))[0] file_basename = os.path.splitext(os.path.basename(caller.filename))[0]
if module_name and module_name.endswith(f".{file_basename}"): if module_name and module_name.endswith(f".{file_basename}"):
module_name = module_name[:-(len(file_basename) + 1)] module_name = module_name[:-(len(file_basename) + 1)]
@@ -124,9 +124,9 @@ gettext.gettext = lambda message: translate(_original_gettext(message), reflect.
for module_name, module in sys.modules.items(): for module_name, module in sys.modules.items():
if module is None or module_name in sys.builtin_module_names: if module is None or module_name in sys.builtin_module_names:
continue continue
if hasattr(module, '_') and module._ is _original_gettext: if hasattr(module, "_") and module._ is _original_gettext:
module._ = gettext.gettext # type: ignore[attr-defined] module._ = gettext.gettext # type: ignore[attr-defined]
if hasattr(module, 'gettext') and module.gettext is _original_gettext: if hasattr(module, "gettext") and module.gettext is _original_gettext:
module.gettext = gettext.gettext # type: ignore[attr-defined] module.gettext = gettext.gettext # type: ignore[attr-defined]
@@ -190,8 +190,8 @@ def pluralize(noun:str, count:int | Sized, *, prefix_with_count:bool = True) ->
# English # English
if len(noun) < 2: # noqa: PLR2004 Magic value used in comparison if len(noun) < 2: # noqa: PLR2004 Magic value used in comparison
return f"{prefix}{noun}s" return f"{prefix}{noun}s"
if noun.endswith(('s', 'sh', 'ch', 'x', 'z')): if noun.endswith(("s", "sh", "ch", "x", "z")):
return f"{prefix}{noun}es" return f"{prefix}{noun}es"
if noun.endswith('y') and noun[-2].lower() not in "aeiou": if noun.endswith("y") and noun[-2].lower() not in "aeiou":
return f"{prefix}{noun[:-1]}ies" return f"{prefix}{noun[:-1]}ies"
return f"{prefix}{noun}s" return f"{prefix}{noun}s"

View File

@@ -10,7 +10,7 @@ from typing import Any, TypeVar
from . import i18n from . import i18n
# https://mypy.readthedocs.io/en/stable/generics.html#generic-functions # https://mypy.readthedocs.io/en/stable/generics.html#generic-functions
T = TypeVar('T') T = TypeVar("T")
def ensure(condition:Any | bool | Callable[[], bool], error_message:str, timeout:float = 5, poll_requency:float = 0.5) -> None: def ensure(condition:Any | bool | Callable[[], bool], error_message:str, timeout:float = 5, poll_requency:float = 0.5) -> None:
@@ -152,18 +152,18 @@ def parse_duration(text:str) -> timedelta:
>>> parse_duration("invalid input") >>> parse_duration("invalid input")
datetime.timedelta(0) datetime.timedelta(0)
""" """
pattern = re.compile(r'(\d+)\s*([dhms])') pattern = re.compile(r"(\d+)\s*([dhms])")
parts = pattern.findall(text.lower()) parts = pattern.findall(text.lower())
kwargs:dict[str, int] = {} kwargs:dict[str, int] = {}
for value, unit in parts: for value, unit in parts:
if unit == 'd': if unit == "d":
kwargs['days'] = kwargs.get('days', 0) + int(value) kwargs["days"] = kwargs.get("days", 0) + int(value)
elif unit == 'h': elif unit == "h":
kwargs['hours'] = kwargs.get('hours', 0) + int(value) kwargs["hours"] = kwargs.get("hours", 0) + int(value)
elif unit == 'm': elif unit == "m":
kwargs['minutes'] = kwargs.get('minutes', 0) + int(value) kwargs["minutes"] = kwargs.get("minutes", 0) + int(value)
elif unit == 's': elif unit == "s":
kwargs['seconds'] = kwargs.get('seconds', 0) + int(value) kwargs["seconds"] = kwargs.get("seconds", 0) + int(value)
return timedelta(**kwargs) return timedelta(**kwargs)

View File

@@ -165,7 +165,7 @@ class WebScrapingMixin:
prefs_file = os.path.join(profile_dir, "Preferences") prefs_file = os.path.join(profile_dir, "Preferences")
if not os.path.exists(prefs_file): if not os.path.exists(prefs_file):
LOG.info(" -> Setting chrome prefs [%s]...", prefs_file) LOG.info(" -> Setting chrome prefs [%s]...", prefs_file)
with open(prefs_file, "w", encoding = 'UTF-8') as fd: with open(prefs_file, "w", encoding = "UTF-8") as fd:
json.dump({ json.dump({
"credentials_enable_service": False, "credentials_enable_service": False,
"enable_do_not_track": True, "enable_do_not_track": True,
@@ -234,16 +234,16 @@ class WebScrapingMixin:
case "Windows": case "Windows":
browser_paths = [ browser_paths = [
os.environ.get("PROGRAMFILES", "C:\\Program Files") + r'\Microsoft\Edge\Application\msedge.exe', os.environ.get("PROGRAMFILES", "C:\\Program Files") + r"\Microsoft\Edge\Application\msedge.exe",
os.environ.get("PROGRAMFILES(X86)", "C:\\Program Files (x86)") + r'\Microsoft\Edge\Application\msedge.exe', os.environ.get("PROGRAMFILES(X86)", "C:\\Program Files (x86)") + r"\Microsoft\Edge\Application\msedge.exe",
os.environ["PROGRAMFILES"] + r'\Chromium\Application\chrome.exe', os.environ["PROGRAMFILES"] + r"\Chromium\Application\chrome.exe",
os.environ["PROGRAMFILES(X86)"] + r'\Chromium\Application\chrome.exe', os.environ["PROGRAMFILES(X86)"] + r"\Chromium\Application\chrome.exe",
os.environ["LOCALAPPDATA"] + r'\Chromium\Application\chrome.exe', os.environ["LOCALAPPDATA"] + r"\Chromium\Application\chrome.exe",
os.environ["PROGRAMFILES"] + r'\Chrome\Application\chrome.exe', os.environ["PROGRAMFILES"] + r"\Chrome\Application\chrome.exe",
os.environ["PROGRAMFILES(X86)"] + r'\Chrome\Application\chrome.exe', os.environ["PROGRAMFILES(X86)"] + r"\Chrome\Application\chrome.exe",
os.environ["LOCALAPPDATA"] + r'\Chrome\Application\chrome.exe', os.environ["LOCALAPPDATA"] + r"\Chrome\Application\chrome.exe",
shutil.which("msedge.exe"), shutil.which("msedge.exe"),
shutil.which("chromium.exe"), shutil.which("chromium.exe"),
@@ -532,7 +532,7 @@ class WebScrapingMixin:
:param scroll_back_top: whether to scroll the page back to the top after scrolling to the bottom :param scroll_back_top: whether to scroll the page back to the top after scrolling to the bottom
""" """
current_y_pos = 0 current_y_pos = 0
bottom_y_pos: int = await self.web_execute('document.body.scrollHeight') # get bottom position bottom_y_pos:int = await self.web_execute("document.body.scrollHeight") # get bottom position
while current_y_pos < bottom_y_pos: # scroll in steps until bottom reached while current_y_pos < bottom_y_pos: # scroll in steps until bottom reached
current_y_pos += scroll_length current_y_pos += scroll_length
await self.web_execute(f'window.scrollTo(0, {current_y_pos})') # scroll one step await self.web_execute(f'window.scrollTo(0, {current_y_pos})') # scroll one step

View File

@@ -1,8 +1,6 @@
""" # SPDX-FileCopyrightText: © Jens Bergmann and contributors
SPDX-FileCopyrightText: © Jens Bergmann and contributors # SPDX-License-Identifier: AGPL-3.0-or-later
SPDX-License-Identifier: AGPL-3.0-or-later # SPDX-ArtifactOfProjectHomePage: https://github.com/Second-Hand-Friends/kleinanzeigen-bot/
SPDX-ArtifactOfProjectHomePage: https://github.com/Second-Hand-Friends/kleinanzeigen-bot/
"""
import os import os
from typing import Any, Final from typing import Any, Final
from unittest.mock import MagicMock from unittest.mock import MagicMock
@@ -41,27 +39,27 @@ def sample_config() -> dict[str, Any]:
- Publishing settings - Publishing settings
""" """
return { return {
'login': { "login": {
'username': 'testuser', "username": "testuser",
'password': 'testpass' "password": "testpass"
}, },
'browser': { "browser": {
'arguments': [], "arguments": [],
'binary_location': None, "binary_location": None,
'extensions': [], "extensions": [],
'use_private_window': True, "use_private_window": True,
'user_data_dir': None, "user_data_dir": None,
'profile_name': None "profile_name": None
}, },
'ad_defaults': { "ad_defaults": {
'description': { "description": {
'prefix': 'Test Prefix', "prefix": "Test Prefix",
'suffix': 'Test Suffix' "suffix": "Test Suffix"
} }
}, },
'publishing': { "publishing": {
'delete_old_ads': 'BEFORE_PUBLISH', "delete_old_ads": "BEFORE_PUBLISH",
'delete_old_ads_by_title': False "delete_old_ads_by_title": False
} }
} }

View File

@@ -69,7 +69,7 @@ class TestAdExtractorPricing:
self, test_extractor:AdExtractor, price_text:str, expected_price:int | None, expected_type:str self, test_extractor:AdExtractor, price_text:str, expected_price:int | None, expected_type:str
) -> None: ) -> None:
"""Test price extraction with different formats""" """Test price extraction with different formats"""
with patch.object(test_extractor, 'web_text', new_callable = AsyncMock, return_value = price_text): with patch.object(test_extractor, "web_text", new_callable = AsyncMock, return_value = price_text):
price, price_type = await test_extractor._extract_pricing_info_from_ad_page() price, price_type = await test_extractor._extract_pricing_info_from_ad_page()
assert price == expected_price assert price == expected_price
assert price_type == expected_type assert price_type == expected_type
@@ -78,7 +78,7 @@ class TestAdExtractorPricing:
# pylint: disable=protected-access # pylint: disable=protected-access
async def test_extract_pricing_info_timeout(self, test_extractor:AdExtractor) -> None: async def test_extract_pricing_info_timeout(self, test_extractor:AdExtractor) -> None:
"""Test price extraction when element is not found""" """Test price extraction when element is not found"""
with patch.object(test_extractor, 'web_text', new_callable = AsyncMock, side_effect = TimeoutError): with patch.object(test_extractor, "web_text", new_callable = AsyncMock, side_effect = TimeoutError):
price, price_type = await test_extractor._extract_pricing_info_from_ad_page() price, price_type = await test_extractor._extract_pricing_info_from_ad_page()
assert price is None assert price is None
assert price_type == "NOT_APPLICABLE" assert price_type == "NOT_APPLICABLE"
@@ -98,9 +98,9 @@ class TestAdExtractorShipping:
self, test_extractor:AdExtractor, shipping_text:str, expected_type:str, expected_cost:float | None self, test_extractor:AdExtractor, shipping_text:str, expected_type:str, expected_cost:float | None
) -> None: ) -> None:
"""Test shipping info extraction with different text formats.""" """Test shipping info extraction with different text formats."""
with patch.object(test_extractor, 'page', MagicMock()), \ with patch.object(test_extractor, "page", MagicMock()), \
patch.object(test_extractor, 'web_text', new_callable = AsyncMock, return_value = shipping_text), \ patch.object(test_extractor, "web_text", new_callable = AsyncMock, return_value = shipping_text), \
patch.object(test_extractor, 'web_request', new_callable = AsyncMock) as mock_web_request: patch.object(test_extractor, "web_request", new_callable = AsyncMock) as mock_web_request:
if expected_cost: if expected_cost:
shipping_response:dict[str, Any] = { shipping_response:dict[str, Any] = {
@@ -139,9 +139,9 @@ class TestAdExtractorShipping:
}) })
} }
with patch.object(test_extractor, 'page', MagicMock()), \ with patch.object(test_extractor, "page", MagicMock()), \
patch.object(test_extractor, 'web_text', new_callable = AsyncMock, return_value = "+ Versand ab 5,49 €"), \ patch.object(test_extractor, "web_text", new_callable = AsyncMock, return_value = "+ Versand ab 5,49 €"), \
patch.object(test_extractor, 'web_request', new_callable = AsyncMock, return_value = shipping_response): patch.object(test_extractor, "web_request", new_callable = AsyncMock, return_value = shipping_response):
shipping_type, costs, options = await test_extractor._extract_shipping_info_from_ad_page() shipping_type, costs, options = await test_extractor._extract_shipping_info_from_ad_page()
@@ -159,9 +159,9 @@ class TestAdExtractorNavigation:
page_mock = AsyncMock() page_mock = AsyncMock()
page_mock.url = "https://www.kleinanzeigen.de/s-anzeige/test/12345" page_mock.url = "https://www.kleinanzeigen.de/s-anzeige/test/12345"
with patch.object(test_extractor, 'page', page_mock), \ with patch.object(test_extractor, "page", page_mock), \
patch.object(test_extractor, 'web_open', new_callable = AsyncMock) as mock_web_open, \ patch.object(test_extractor, "web_open", new_callable = AsyncMock) as mock_web_open, \
patch.object(test_extractor, 'web_find', new_callable = AsyncMock, side_effect = TimeoutError): patch.object(test_extractor, "web_find", new_callable = AsyncMock, side_effect = TimeoutError):
result = await test_extractor.naviagte_to_ad_page("https://www.kleinanzeigen.de/s-anzeige/test/12345") result = await test_extractor.naviagte_to_ad_page("https://www.kleinanzeigen.de/s-anzeige/test/12345")
assert result is True assert result is True
@@ -195,15 +195,15 @@ class TestAdExtractorNavigation:
return popup_close_mock return popup_close_mock
return None return None
with patch.object(test_extractor, 'page', page_mock), \ with patch.object(test_extractor, "page", page_mock), \
patch.object(test_extractor, 'web_open', new_callable = AsyncMock) as mock_web_open, \ patch.object(test_extractor, "web_open", new_callable = AsyncMock) as mock_web_open, \
patch.object(test_extractor, 'web_input', new_callable = AsyncMock), \ patch.object(test_extractor, "web_input", new_callable = AsyncMock), \
patch.object(test_extractor, 'web_check', new_callable = AsyncMock, return_value = True), \ patch.object(test_extractor, "web_check", new_callable = AsyncMock, return_value = True), \
patch.object(test_extractor, 'web_find', new_callable = AsyncMock, side_effect = find_mock): patch.object(test_extractor, "web_find", new_callable = AsyncMock, side_effect = find_mock):
result = await test_extractor.naviagte_to_ad_page(12345) result = await test_extractor.naviagte_to_ad_page(12345)
assert result is True assert result is True
mock_web_open.assert_called_with('https://www.kleinanzeigen.de/') mock_web_open.assert_called_with("https://www.kleinanzeigen.de/")
submit_button_mock.click.assert_awaited_once() submit_button_mock.click.assert_awaited_once()
popup_close_mock.click.assert_awaited_once() popup_close_mock.click.assert_awaited_once()
@@ -218,15 +218,15 @@ class TestAdExtractorNavigation:
input_mock.send_keys = AsyncMock() input_mock.send_keys = AsyncMock()
input_mock.apply = AsyncMock(return_value = True) input_mock.apply = AsyncMock(return_value = True)
with patch.object(test_extractor, 'page', page_mock), \ with patch.object(test_extractor, "page", page_mock), \
patch.object(test_extractor, 'web_open', new_callable = AsyncMock), \ patch.object(test_extractor, "web_open", new_callable = AsyncMock), \
patch.object(test_extractor, 'web_find', new_callable = AsyncMock, return_value = input_mock), \ patch.object(test_extractor, "web_find", new_callable = AsyncMock, return_value = input_mock), \
patch.object(test_extractor, 'web_click', new_callable = AsyncMock) as mock_web_click, \ patch.object(test_extractor, "web_click", new_callable = AsyncMock) as mock_web_click, \
patch.object(test_extractor, 'web_check', new_callable = AsyncMock, return_value = True): patch.object(test_extractor, "web_check", new_callable = AsyncMock, return_value = True):
result = await test_extractor.naviagte_to_ad_page(12345) result = await test_extractor.naviagte_to_ad_page(12345)
assert result is True assert result is True
mock_web_click.assert_called_with(By.CLASS_NAME, 'mfp-close') mock_web_click.assert_called_with(By.CLASS_NAME, "mfp-close")
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_navigate_to_ad_page_invalid_id(self, test_extractor:AdExtractor) -> None: async def test_navigate_to_ad_page_invalid_id(self, test_extractor:AdExtractor) -> None:
@@ -240,9 +240,9 @@ class TestAdExtractorNavigation:
input_mock.apply = AsyncMock(return_value = True) input_mock.apply = AsyncMock(return_value = True)
input_mock.attrs = {} input_mock.attrs = {}
with patch.object(test_extractor, 'page', page_mock), \ with patch.object(test_extractor, "page", page_mock), \
patch.object(test_extractor, 'web_open', new_callable = AsyncMock), \ patch.object(test_extractor, "web_open", new_callable = AsyncMock), \
patch.object(test_extractor, 'web_find', new_callable = AsyncMock, return_value = input_mock): patch.object(test_extractor, "web_find", new_callable = AsyncMock, return_value = input_mock):
result = await test_extractor.naviagte_to_ad_page(99999) result = await test_extractor.naviagte_to_ad_page(99999)
assert result is False assert result is False
@@ -250,12 +250,12 @@ class TestAdExtractorNavigation:
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_extract_own_ads_urls(self, test_extractor:AdExtractor) -> None: async def test_extract_own_ads_urls(self, test_extractor:AdExtractor) -> None:
"""Test extraction of own ads URLs - basic test.""" """Test extraction of own ads URLs - basic test."""
with patch.object(test_extractor, 'web_open', new_callable=AsyncMock), \ with patch.object(test_extractor, "web_open", new_callable = AsyncMock), \
patch.object(test_extractor, 'web_sleep', new_callable=AsyncMock), \ patch.object(test_extractor, "web_sleep", new_callable = AsyncMock), \
patch.object(test_extractor, 'web_find', new_callable=AsyncMock) as mock_web_find, \ patch.object(test_extractor, "web_find", new_callable = AsyncMock) as mock_web_find, \
patch.object(test_extractor, 'web_find_all', new_callable=AsyncMock) as mock_web_find_all, \ patch.object(test_extractor, "web_find_all", new_callable = AsyncMock) as mock_web_find_all, \
patch.object(test_extractor, 'web_scroll_page_down', new_callable=AsyncMock), \ patch.object(test_extractor, "web_scroll_page_down", new_callable = AsyncMock), \
patch.object(test_extractor, 'web_execute', new_callable=AsyncMock): patch.object(test_extractor, "web_execute", new_callable = AsyncMock):
# --- Setup mock objects for DOM elements --- # --- Setup mock objects for DOM elements ---
# Mocks needed for the actual execution flow # Mocks needed for the actual execution flow
@@ -263,7 +263,7 @@ class TestAdExtractorNavigation:
pagination_section_mock = MagicMock() pagination_section_mock = MagicMock()
cardbox_mock = MagicMock() # Represents the <li> element cardbox_mock = MagicMock() # Represents the <li> element
link_mock = MagicMock() # Represents the <a> element link_mock = MagicMock() # Represents the <a> element
link_mock.attrs = {'href': '/s-anzeige/test/12345'} # Configure the desired output link_mock.attrs = {"href": "/s-anzeige/test/12345"} # Configure the desired output
# Mocks for elements potentially checked but maybe not strictly needed for output # Mocks for elements potentially checked but maybe not strictly needed for output
# (depending on how robust the mocking is) # (depending on how robust the mocking is)
@@ -295,19 +295,19 @@ class TestAdExtractorNavigation:
refs = await test_extractor.extract_own_ads_urls() refs = await test_extractor.extract_own_ads_urls()
# --- Assertions --- # --- Assertions ---
assert refs == ['/s-anzeige/test/12345'] # Now it should match assert refs == ["/s-anzeige/test/12345"] # Now it should match
# Optional: Verify calls were made as expected # Optional: Verify calls were made as expected
mock_web_find.assert_has_calls([ mock_web_find.assert_has_calls([
call(By.ID, 'my-manageitems-adlist'), call(By.ID, "my-manageitems-adlist"),
call(By.CSS_SELECTOR, '.Pagination', timeout=10), call(By.CSS_SELECTOR, ".Pagination", timeout = 10),
call(By.ID, 'my-manageitems-adlist'), call(By.ID, "my-manageitems-adlist"),
call(By.CSS_SELECTOR, 'div.manageitems-item-ad h3 a.text-onSurface', parent=cardbox_mock), call(By.CSS_SELECTOR, "div.manageitems-item-ad h3 a.text-onSurface", parent = cardbox_mock),
], any_order = False) # Check order if important ], any_order = False) # Check order if important
mock_web_find_all.assert_has_calls([ mock_web_find_all.assert_has_calls([
call(By.CSS_SELECTOR, 'button[aria-label="Nächste"]', parent = pagination_section_mock), call(By.CSS_SELECTOR, 'button[aria-label="Nächste"]', parent = pagination_section_mock),
call(By.CLASS_NAME, 'cardbox', parent=ad_list_container_mock), call(By.CLASS_NAME, "cardbox", parent = ad_list_container_mock),
], any_order = False) ], any_order = False)
@@ -424,11 +424,11 @@ class TestAdExtractorContent:
] ]
for text, expected in test_cases: for text, expected in test_cases:
with patch.object(test_extractor, 'web_text', new_callable=AsyncMock, return_value=text): with patch.object(test_extractor, "web_text", new_callable = AsyncMock, return_value = text):
result = await test_extractor._extract_sell_directly_from_ad_page() result = await test_extractor._extract_sell_directly_from_ad_page()
assert result is expected assert result is expected
with patch.object(test_extractor, 'web_text', new_callable=AsyncMock, side_effect=TimeoutError): with patch.object(test_extractor, "web_text", new_callable = AsyncMock, side_effect = TimeoutError):
result = await test_extractor._extract_sell_directly_from_ad_page() result = await test_extractor._extract_sell_directly_from_ad_page()
assert result is None assert result is None
@@ -455,11 +455,11 @@ class TestAdExtractorCategory:
"""Test category extraction from breadcrumb.""" """Test category extraction from breadcrumb."""
category_line = MagicMock() category_line = MagicMock()
first_part = MagicMock() first_part = MagicMock()
first_part.attrs = {'href': '/s-familie-kind-baby/c17'} first_part.attrs = {"href": "/s-familie-kind-baby/c17"}
second_part = MagicMock() second_part = MagicMock()
second_part.attrs = {'href': '/s-spielzeug/c23'} second_part.attrs = {"href": "/s-spielzeug/c23"}
with patch.object(extractor, 'web_find', new_callable = AsyncMock) as mock_web_find: with patch.object(extractor, "web_find", new_callable = AsyncMock) as mock_web_find:
mock_web_find.side_effect = [ mock_web_find.side_effect = [
category_line, category_line,
first_part, first_part,
@@ -469,15 +469,15 @@ class TestAdExtractorCategory:
result = await extractor._extract_category_from_ad_page() result = await extractor._extract_category_from_ad_page()
assert result == "17/23" assert result == "17/23"
mock_web_find.assert_any_call(By.ID, 'vap-brdcrmb') mock_web_find.assert_any_call(By.ID, "vap-brdcrmb")
mock_web_find.assert_any_call(By.CSS_SELECTOR, 'a:nth-of-type(2)', parent = category_line) mock_web_find.assert_any_call(By.CSS_SELECTOR, "a:nth-of-type(2)", parent = category_line)
mock_web_find.assert_any_call(By.CSS_SELECTOR, 'a:nth-of-type(3)', parent = category_line) mock_web_find.assert_any_call(By.CSS_SELECTOR, "a:nth-of-type(3)", parent = category_line)
@pytest.mark.asyncio @pytest.mark.asyncio
# pylint: disable=protected-access # pylint: disable=protected-access
async def test_extract_special_attributes_empty(self, extractor:AdExtractor) -> None: async def test_extract_special_attributes_empty(self, extractor:AdExtractor) -> None:
"""Test extraction of special attributes when empty.""" """Test extraction of special attributes when empty."""
with patch.object(extractor, 'web_execute', new_callable = AsyncMock) as mock_web_execute: with patch.object(extractor, "web_execute", new_callable = AsyncMock) as mock_web_execute:
mock_web_execute.return_value = { mock_web_execute.return_value = {
"universalAnalyticsOpts": { "universalAnalyticsOpts": {
"dimensions": { "dimensions": {
@@ -509,9 +509,9 @@ class TestAdExtractorContact:
# pylint: disable=protected-access # pylint: disable=protected-access
async def test_extract_contact_info(self, extractor:AdExtractor) -> None: async def test_extract_contact_info(self, extractor:AdExtractor) -> None:
"""Test extraction of contact information.""" """Test extraction of contact information."""
with patch.object(extractor, 'page', MagicMock()), \ with patch.object(extractor, "page", MagicMock()), \
patch.object(extractor, 'web_text', new_callable = AsyncMock) as mock_web_text, \ patch.object(extractor, "web_text", new_callable = AsyncMock) as mock_web_text, \
patch.object(extractor, 'web_find', new_callable = AsyncMock) as mock_web_find: patch.object(extractor, "web_find", new_callable = AsyncMock) as mock_web_find:
mock_web_text.side_effect = [ mock_web_text.side_effect = [
"12345 Berlin - Mitte", "12345 Berlin - Mitte",
@@ -537,9 +537,9 @@ class TestAdExtractorContact:
# pylint: disable=protected-access # pylint: disable=protected-access
async def test_extract_contact_info_timeout(self, extractor:AdExtractor) -> None: async def test_extract_contact_info_timeout(self, extractor:AdExtractor) -> None:
"""Test contact info extraction when elements are not found.""" """Test contact info extraction when elements are not found."""
with patch.object(extractor, 'page', MagicMock()), \ with patch.object(extractor, "page", MagicMock()), \
patch.object(extractor, 'web_text', new_callable = AsyncMock, side_effect = TimeoutError()), \ patch.object(extractor, "web_text", new_callable = AsyncMock, side_effect = TimeoutError()), \
patch.object(extractor, 'web_find', new_callable = AsyncMock, side_effect = TimeoutError()), \ patch.object(extractor, "web_find", new_callable = AsyncMock, side_effect = TimeoutError()), \
pytest.raises(TimeoutError): pytest.raises(TimeoutError):
await extractor._extract_contact_from_ad_page() await extractor._extract_contact_from_ad_page()
@@ -548,9 +548,9 @@ class TestAdExtractorContact:
# pylint: disable=protected-access # pylint: disable=protected-access
async def test_extract_contact_info_with_phone(self, extractor:AdExtractor) -> None: async def test_extract_contact_info_with_phone(self, extractor:AdExtractor) -> None:
"""Test extraction of contact information including phone number.""" """Test extraction of contact information including phone number."""
with patch.object(extractor, 'page', MagicMock()), \ with patch.object(extractor, "page", MagicMock()), \
patch.object(extractor, 'web_text', new_callable = AsyncMock) as mock_web_text, \ patch.object(extractor, "web_text", new_callable = AsyncMock) as mock_web_text, \
patch.object(extractor, 'web_find', new_callable = AsyncMock) as mock_web_find: patch.object(extractor, "web_find", new_callable = AsyncMock) as mock_web_find:
mock_web_text.side_effect = [ mock_web_text.side_effect = [
"12345 Berlin - Mitte", "12345 Berlin - Mitte",
@@ -590,17 +590,17 @@ class TestAdExtractorDownload:
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_download_ad_existing_directory(self, extractor:AdExtractor) -> None: async def test_download_ad_existing_directory(self, extractor:AdExtractor) -> None:
"""Test downloading an ad when the directory already exists.""" """Test downloading an ad when the directory already exists."""
with patch('os.path.exists') as mock_exists, \ with patch("os.path.exists") as mock_exists, \
patch('os.path.isdir') as mock_isdir, \ patch("os.path.isdir") as mock_isdir, \
patch('os.makedirs') as mock_makedirs, \ patch("os.makedirs") as mock_makedirs, \
patch('os.mkdir') as mock_mkdir, \ patch("os.mkdir") as mock_mkdir, \
patch('shutil.rmtree') as mock_rmtree, \ patch("shutil.rmtree") as mock_rmtree, \
patch('kleinanzeigen_bot.extract.dicts.save_dict', autospec = True) as mock_save_dict, \ patch("kleinanzeigen_bot.extract.dicts.save_dict", autospec = True) as mock_save_dict, \
patch.object(extractor, '_extract_ad_page_info', new_callable = AsyncMock) as mock_extract: patch.object(extractor, "_extract_ad_page_info", new_callable = AsyncMock) as mock_extract:
base_dir = 'downloaded-ads' base_dir = "downloaded-ads"
ad_dir = os.path.join(base_dir, 'ad_12345') ad_dir = os.path.join(base_dir, "ad_12345")
yaml_path = os.path.join(ad_dir, 'ad_12345.yaml') yaml_path = os.path.join(ad_dir, "ad_12345.yaml")
# Configure mocks for directory checks # Configure mocks for directory checks
existing_paths = {base_dir, ad_dir} existing_paths = {base_dir, ad_dir}
@@ -632,7 +632,7 @@ class TestAdExtractorDownload:
# Workaround for hard-coded path in download_ad # Workaround for hard-coded path in download_ad
actual_call = mock_save_dict.call_args actual_call = mock_save_dict.call_args
assert actual_call is not None assert actual_call is not None
actual_path = actual_call[0][0].replace('/', os.path.sep) actual_path = actual_call[0][0].replace("/", os.path.sep)
assert actual_path == yaml_path assert actual_path == yaml_path
assert actual_call[0][1] == mock_extract.return_value assert actual_call[0][1] == mock_extract.return_value
@@ -640,24 +640,24 @@ class TestAdExtractorDownload:
# pylint: disable=protected-access # pylint: disable=protected-access
async def test_download_images_no_images(self, extractor:AdExtractor) -> None: async def test_download_images_no_images(self, extractor:AdExtractor) -> None:
"""Test image download when no images are found.""" """Test image download when no images are found."""
with patch.object(extractor, 'web_find', new_callable = AsyncMock, side_effect = TimeoutError): with patch.object(extractor, "web_find", new_callable = AsyncMock, side_effect = TimeoutError):
image_paths = await extractor._download_images_from_ad_page("/some/dir", 12345) image_paths = await extractor._download_images_from_ad_page("/some/dir", 12345)
assert len(image_paths) == 0 assert len(image_paths) == 0
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_download_ad(self, extractor:AdExtractor) -> None: async def test_download_ad(self, extractor:AdExtractor) -> None:
"""Test downloading an entire ad.""" """Test downloading an entire ad."""
with patch('os.path.exists') as mock_exists, \ with patch("os.path.exists") as mock_exists, \
patch('os.path.isdir') as mock_isdir, \ patch("os.path.isdir") as mock_isdir, \
patch('os.makedirs') as mock_makedirs, \ patch("os.makedirs") as mock_makedirs, \
patch('os.mkdir') as mock_mkdir, \ patch("os.mkdir") as mock_mkdir, \
patch('shutil.rmtree') as mock_rmtree, \ patch("shutil.rmtree") as mock_rmtree, \
patch('kleinanzeigen_bot.extract.dicts.save_dict', autospec = True) as mock_save_dict, \ patch("kleinanzeigen_bot.extract.dicts.save_dict", autospec = True) as mock_save_dict, \
patch.object(extractor, '_extract_ad_page_info', new_callable = AsyncMock) as mock_extract: patch.object(extractor, "_extract_ad_page_info", new_callable = AsyncMock) as mock_extract:
base_dir = 'downloaded-ads' base_dir = "downloaded-ads"
ad_dir = os.path.join(base_dir, 'ad_12345') ad_dir = os.path.join(base_dir, "ad_12345")
yaml_path = os.path.join(ad_dir, 'ad_12345.yaml') yaml_path = os.path.join(ad_dir, "ad_12345.yaml")
# Configure mocks for directory checks # Configure mocks for directory checks
mock_exists.return_value = False mock_exists.return_value = False
@@ -690,6 +690,6 @@ class TestAdExtractorDownload:
# Get the actual call arguments # Get the actual call arguments
actual_call = mock_save_dict.call_args actual_call = mock_save_dict.call_args
assert actual_call is not None assert actual_call is not None
actual_path = actual_call[0][0].replace('/', os.path.sep) actual_path = actual_call[0][0].replace("/", os.path.sep)
assert actual_path == yaml_path assert actual_path == yaml_path
assert actual_call[0][1] == mock_extract.return_value assert actual_call[0][1] == mock_extract.return_value

View File

@@ -135,10 +135,10 @@ def minimal_ad_config(base_ad_config: dict[str, Any]) -> dict[str, Any]:
def mock_config_setup(test_bot:KleinanzeigenBot) -> Generator[None]: def mock_config_setup(test_bot:KleinanzeigenBot) -> Generator[None]:
"""Provide a centralized mock configuration setup for tests. """Provide a centralized mock configuration setup for tests.
This fixture mocks load_config and other essential configuration-related methods.""" This fixture mocks load_config and other essential configuration-related methods."""
with patch.object(test_bot, 'load_config'), \ with patch.object(test_bot, "load_config"), \
patch.object(test_bot, 'create_browser_session', new_callable = AsyncMock), \ patch.object(test_bot, "create_browser_session", new_callable = AsyncMock), \
patch.object(test_bot, 'login', new_callable = AsyncMock), \ patch.object(test_bot, "login", new_callable = AsyncMock), \
patch.object(test_bot, 'web_request', new_callable = AsyncMock) as mock_request: patch.object(test_bot, "web_request", new_callable = AsyncMock) as mock_request:
# Mock the web request for published ads # Mock the web request for published ads
mock_request.return_value = {"content": '{"ads": []}'} mock_request.return_value = {"content": '{"ads": []}'}
yield yield
@@ -159,8 +159,8 @@ class TestKleinanzeigenBotInitialization:
def test_get_version_returns_correct_version(self, test_bot:KleinanzeigenBot) -> None: def test_get_version_returns_correct_version(self, test_bot:KleinanzeigenBot) -> None:
"""Verify version retrieval works correctly.""" """Verify version retrieval works correctly."""
with patch('kleinanzeigen_bot.__version__', '1.2.3'): with patch("kleinanzeigen_bot.__version__", "1.2.3"):
assert test_bot.get_version() == '1.2.3' assert test_bot.get_version() == "1.2.3"
class TestKleinanzeigenBotLogging: class TestKleinanzeigenBotLogging:
@@ -257,15 +257,15 @@ class TestKleinanzeigenBotConfiguration:
sample_config_with_categories = sample_config.copy() sample_config_with_categories = sample_config.copy()
sample_config_with_categories["categories"] = {} sample_config_with_categories["categories"] = {}
with patch('kleinanzeigen_bot.utils.dicts.load_dict_if_exists', return_value = None), \ with patch("kleinanzeigen_bot.utils.dicts.load_dict_if_exists", return_value = None), \
patch.object(LOG, 'warning') as mock_warning, \ patch.object(LOG, "warning") as mock_warning, \
patch('kleinanzeigen_bot.utils.dicts.save_dict') as mock_save, \ patch("kleinanzeigen_bot.utils.dicts.save_dict") as mock_save, \
patch('kleinanzeigen_bot.utils.dicts.load_dict_from_module') as mock_load_module: patch("kleinanzeigen_bot.utils.dicts.load_dict_from_module") as mock_load_module:
mock_load_module.side_effect = [ mock_load_module.side_effect = [
sample_config_with_categories, # config_defaults.yaml sample_config_with_categories, # config_defaults.yaml
{'cat1': 'id1'}, # categories.yaml {"cat1": "id1"}, # categories.yaml
{'cat2': 'id2'} # categories_old.yaml {"cat2": "id2"} # categories_old.yaml
] ]
test_bot.load_config() test_bot.load_config()
@@ -273,7 +273,7 @@ class TestKleinanzeigenBotConfiguration:
mock_save.assert_called_once_with(str(config_path), sample_config_with_categories) mock_save.assert_called_once_with(str(config_path), sample_config_with_categories)
# Verify categories were loaded # Verify categories were loaded
assert test_bot.categories == {'cat1': 'id1', 'cat2': 'id2'} assert test_bot.categories == {"cat1": "id1", "cat2": "id2"}
assert test_bot.config == sample_config_with_categories assert test_bot.config == sample_config_with_categories
def test_load_config_validates_required_fields(self, test_bot:KleinanzeigenBot, test_data_dir:str) -> None: def test_load_config_validates_required_fields(self, test_bot:KleinanzeigenBot, test_data_dir:str) -> None:
@@ -307,13 +307,13 @@ class TestKleinanzeigenBotAuthentication:
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_assert_free_ad_limit_not_reached_success(self, configured_bot:KleinanzeigenBot) -> None: async def test_assert_free_ad_limit_not_reached_success(self, configured_bot:KleinanzeigenBot) -> None:
"""Verify that free ad limit check succeeds when limit not reached.""" """Verify that free ad limit check succeeds when limit not reached."""
with patch.object(configured_bot, 'web_find', side_effect = TimeoutError): with patch.object(configured_bot, "web_find", side_effect = TimeoutError):
await configured_bot.assert_free_ad_limit_not_reached() await configured_bot.assert_free_ad_limit_not_reached()
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_assert_free_ad_limit_not_reached_limit_reached(self, configured_bot:KleinanzeigenBot) -> None: async def test_assert_free_ad_limit_not_reached_limit_reached(self, configured_bot:KleinanzeigenBot) -> None:
"""Verify that free ad limit check fails when limit is reached.""" """Verify that free ad limit check fails when limit is reached."""
with patch.object(configured_bot, 'web_find', return_value = AsyncMock()): with patch.object(configured_bot, "web_find", return_value = AsyncMock()):
with pytest.raises(AssertionError) as exc_info: with pytest.raises(AssertionError) as exc_info:
await configured_bot.assert_free_ad_limit_not_reached() await configured_bot.assert_free_ad_limit_not_reached()
assert "Cannot publish more ads" in str(exc_info.value) assert "Cannot publish more ads" in str(exc_info.value)
@@ -321,23 +321,23 @@ class TestKleinanzeigenBotAuthentication:
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_is_logged_in_returns_true_when_logged_in(self, configured_bot:KleinanzeigenBot) -> None: async def test_is_logged_in_returns_true_when_logged_in(self, configured_bot:KleinanzeigenBot) -> None:
"""Verify that login check returns true when logged in.""" """Verify that login check returns true when logged in."""
with patch.object(configured_bot, 'web_text', return_value = 'Welcome testuser'): with patch.object(configured_bot, "web_text", return_value = "Welcome testuser"):
assert await configured_bot.is_logged_in() is True assert await configured_bot.is_logged_in() is True
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_is_logged_in_returns_false_when_not_logged_in(self, configured_bot:KleinanzeigenBot) -> None: async def test_is_logged_in_returns_false_when_not_logged_in(self, configured_bot:KleinanzeigenBot) -> None:
"""Verify that login check returns false when not logged in.""" """Verify that login check returns false when not logged in."""
with patch.object(configured_bot, 'web_text', side_effect = TimeoutError): with patch.object(configured_bot, "web_text", side_effect = TimeoutError):
assert await configured_bot.is_logged_in() is False assert await configured_bot.is_logged_in() is False
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_login_flow_completes_successfully(self, configured_bot:KleinanzeigenBot) -> None: async def test_login_flow_completes_successfully(self, configured_bot:KleinanzeigenBot) -> None:
"""Verify that normal login flow completes successfully.""" """Verify that normal login flow completes successfully."""
with patch.object(configured_bot, 'web_open') as mock_open, \ with patch.object(configured_bot, "web_open") as mock_open, \
patch.object(configured_bot, 'is_logged_in', side_effect = [False, True]) as mock_logged_in, \ patch.object(configured_bot, "is_logged_in", side_effect = [False, True]) as mock_logged_in, \
patch.object(configured_bot, 'web_find', side_effect = TimeoutError), \ patch.object(configured_bot, "web_find", side_effect = TimeoutError), \
patch.object(configured_bot, 'web_input') as mock_input, \ patch.object(configured_bot, "web_input") as mock_input, \
patch.object(configured_bot, 'web_click') as mock_click: patch.object(configured_bot, "web_click") as mock_click:
await configured_bot.login() await configured_bot.login()
@@ -349,13 +349,13 @@ class TestKleinanzeigenBotAuthentication:
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_login_flow_handles_captcha(self, configured_bot:KleinanzeigenBot) -> None: async def test_login_flow_handles_captcha(self, configured_bot:KleinanzeigenBot) -> None:
"""Verify that login flow handles captcha correctly.""" """Verify that login flow handles captcha correctly."""
with patch.object(configured_bot, 'web_open'), \ with patch.object(configured_bot, "web_open"), \
patch.object(configured_bot, 'is_logged_in', return_value = False), \ patch.object(configured_bot, "is_logged_in", return_value = False), \
patch.object(configured_bot, 'web_find') as mock_find, \ patch.object(configured_bot, "web_find") as mock_find, \
patch.object(configured_bot, 'web_await') as mock_await, \ patch.object(configured_bot, "web_await") as mock_await, \
patch.object(configured_bot, 'web_input'), \ patch.object(configured_bot, "web_input"), \
patch.object(configured_bot, 'web_click'), \ patch.object(configured_bot, "web_click"), \
patch('kleinanzeigen_bot.ainput') as mock_ainput: patch("kleinanzeigen_bot.ainput") as mock_ainput:
mock_find.side_effect = [ mock_find.side_effect = [
AsyncMock(), # Captcha iframe AsyncMock(), # Captcha iframe
@@ -378,21 +378,21 @@ class TestKleinanzeigenBotLocalization:
def test_show_help_displays_german_text(self, test_bot:KleinanzeigenBot) -> None: def test_show_help_displays_german_text(self, test_bot:KleinanzeigenBot) -> None:
"""Verify that help text is displayed in German when language is German.""" """Verify that help text is displayed in German when language is German."""
with patch('kleinanzeigen_bot.get_current_locale') as mock_locale, \ with patch("kleinanzeigen_bot.get_current_locale") as mock_locale, \
patch('builtins.print') as mock_print: patch("builtins.print") as mock_print:
mock_locale.return_value.language = "de" mock_locale.return_value.language = "de"
test_bot.show_help() test_bot.show_help()
printed_text = ''.join(str(call.args[0]) for call in mock_print.call_args_list) printed_text = "".join(str(call.args[0]) for call in mock_print.call_args_list)
assert "Verwendung:" in printed_text assert "Verwendung:" in printed_text
assert "Befehle:" in printed_text assert "Befehle:" in printed_text
def test_show_help_displays_english_text(self, test_bot:KleinanzeigenBot) -> None: def test_show_help_displays_english_text(self, test_bot:KleinanzeigenBot) -> None:
"""Verify that help text is displayed in English when language is English.""" """Verify that help text is displayed in English when language is English."""
with patch('kleinanzeigen_bot.get_current_locale') as mock_locale, \ with patch("kleinanzeigen_bot.get_current_locale") as mock_locale, \
patch('builtins.print') as mock_print: patch("builtins.print") as mock_print:
mock_locale.return_value.language = "en" mock_locale.return_value.language = "en"
test_bot.show_help() test_bot.show_help()
printed_text = ''.join(str(call.args[0]) for call in mock_print.call_args_list) printed_text = "".join(str(call.args[0]) for call in mock_print.call_args_list)
assert "Usage:" in printed_text assert "Usage:" in printed_text
assert "Commands:" in printed_text assert "Commands:" in printed_text
@@ -421,7 +421,7 @@ class TestKleinanzeigenBotBasics:
"""Test closing browser session.""" """Test closing browser session."""
mock_close = MagicMock() mock_close = MagicMock()
test_bot.page = MagicMock() # Ensure page exists to trigger cleanup test_bot.page = MagicMock() # Ensure page exists to trigger cleanup
with patch.object(test_bot, 'close_browser_session', new = mock_close): with patch.object(test_bot, "close_browser_session", new = mock_close):
test_bot.close_browser_session() # Call directly instead of relying on __del__ test_bot.close_browser_session() # Call directly instead of relying on __del__
mock_close.assert_called_once() mock_close.assert_called_once()
@@ -441,7 +441,7 @@ class TestKleinanzeigenBotBasics:
# Reset log level to default # Reset log level to default
LOG.setLevel(loggers.INFO) LOG.setLevel(loggers.INFO)
assert not loggers.is_debug(LOG) assert not loggers.is_debug(LOG)
test_bot.parse_args(['script.py', '-v']) test_bot.parse_args(["script.py", "-v"])
assert loggers.is_debug(LOG) assert loggers.is_debug(LOG)
def test_get_config_file_path(self, test_bot:KleinanzeigenBot) -> None: def test_get_config_file_path(self, test_bot:KleinanzeigenBot) -> None:
@@ -472,64 +472,64 @@ class TestKleinanzeigenBotArgParsing:
def test_parse_args_help(self, test_bot:KleinanzeigenBot) -> None: def test_parse_args_help(self, test_bot:KleinanzeigenBot) -> None:
"""Test parsing help command.""" """Test parsing help command."""
test_bot.parse_args(['script.py', 'help']) test_bot.parse_args(["script.py", "help"])
assert test_bot.command == 'help' assert test_bot.command == "help"
def test_parse_args_version(self, test_bot:KleinanzeigenBot) -> None: def test_parse_args_version(self, test_bot:KleinanzeigenBot) -> None:
"""Test parsing version command.""" """Test parsing version command."""
test_bot.parse_args(['script.py', 'version']) test_bot.parse_args(["script.py", "version"])
assert test_bot.command == 'version' assert test_bot.command == "version"
def test_parse_args_verbose(self, test_bot:KleinanzeigenBot) -> None: def test_parse_args_verbose(self, test_bot:KleinanzeigenBot) -> None:
"""Test parsing verbose flag.""" """Test parsing verbose flag."""
test_bot.parse_args(['script.py', '-v', 'help']) test_bot.parse_args(["script.py", "-v", "help"])
assert loggers.is_debug(loggers.get_logger('kleinanzeigen_bot')) assert loggers.is_debug(loggers.get_logger("kleinanzeigen_bot"))
def test_parse_args_config_path(self, test_bot:KleinanzeigenBot) -> None: def test_parse_args_config_path(self, test_bot:KleinanzeigenBot) -> None:
"""Test parsing config path.""" """Test parsing config path."""
test_bot.parse_args(['script.py', '--config=test.yaml', 'help']) test_bot.parse_args(["script.py", "--config=test.yaml", "help"])
assert test_bot.config_file_path.endswith('test.yaml') assert test_bot.config_file_path.endswith("test.yaml")
def test_parse_args_logfile(self, test_bot:KleinanzeigenBot) -> None: def test_parse_args_logfile(self, test_bot:KleinanzeigenBot) -> None:
"""Test parsing log file path.""" """Test parsing log file path."""
test_bot.parse_args(['script.py', '--logfile=test.log', 'help']) test_bot.parse_args(["script.py", "--logfile=test.log", "help"])
assert test_bot.log_file_path is not None assert test_bot.log_file_path is not None
assert 'test.log' in test_bot.log_file_path assert "test.log" in test_bot.log_file_path
def test_parse_args_ads_selector(self, test_bot:KleinanzeigenBot) -> None: def test_parse_args_ads_selector(self, test_bot:KleinanzeigenBot) -> None:
"""Test parsing ads selector.""" """Test parsing ads selector."""
test_bot.parse_args(['script.py', '--ads=all', 'publish']) test_bot.parse_args(["script.py", "--ads=all", "publish"])
assert test_bot.ads_selector == 'all' assert test_bot.ads_selector == "all"
def test_parse_args_force(self, test_bot:KleinanzeigenBot) -> None: def test_parse_args_force(self, test_bot:KleinanzeigenBot) -> None:
"""Test parsing force flag.""" """Test parsing force flag."""
test_bot.parse_args(['script.py', '--force', 'publish']) test_bot.parse_args(["script.py", "--force", "publish"])
assert test_bot.ads_selector == 'all' assert test_bot.ads_selector == "all"
def test_parse_args_keep_old(self, test_bot:KleinanzeigenBot) -> None: def test_parse_args_keep_old(self, test_bot:KleinanzeigenBot) -> None:
"""Test parsing keep-old flag.""" """Test parsing keep-old flag."""
test_bot.parse_args(['script.py', '--keep-old', 'publish']) test_bot.parse_args(["script.py", "--keep-old", "publish"])
assert test_bot.keep_old_ads is True assert test_bot.keep_old_ads is True
def test_parse_args_logfile_empty(self, test_bot:KleinanzeigenBot) -> None: def test_parse_args_logfile_empty(self, test_bot:KleinanzeigenBot) -> None:
"""Test parsing empty log file path.""" """Test parsing empty log file path."""
test_bot.parse_args(['script.py', '--logfile=', 'help']) test_bot.parse_args(["script.py", "--logfile=", "help"])
assert test_bot.log_file_path is None assert test_bot.log_file_path is None
def test_parse_args_lang_option(self, test_bot:KleinanzeigenBot) -> None: def test_parse_args_lang_option(self, test_bot:KleinanzeigenBot) -> None:
"""Test parsing language option.""" """Test parsing language option."""
test_bot.parse_args(['script.py', '--lang=en', 'help']) test_bot.parse_args(["script.py", "--lang=en", "help"])
assert test_bot.command == 'help' assert test_bot.command == "help"
def test_parse_args_no_arguments(self, test_bot:KleinanzeigenBot) -> None: def test_parse_args_no_arguments(self, test_bot:KleinanzeigenBot) -> None:
"""Test parsing no arguments defaults to help.""" """Test parsing no arguments defaults to help."""
test_bot.parse_args(['script.py']) test_bot.parse_args(["script.py"])
assert test_bot.command == 'help' assert test_bot.command == "help"
def test_parse_args_multiple_commands(self, test_bot:KleinanzeigenBot) -> None: def test_parse_args_multiple_commands(self, test_bot:KleinanzeigenBot) -> None:
"""Test parsing multiple commands raises error.""" """Test parsing multiple commands raises error."""
with pytest.raises(SystemExit) as exc_info: with pytest.raises(SystemExit) as exc_info:
test_bot.parse_args(['script.py', 'help', 'version']) test_bot.parse_args(["script.py", "help", "version"])
assert exc_info.value.code == 2 assert exc_info.value.code == 2
@@ -539,22 +539,22 @@ class TestKleinanzeigenBotCommands:
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_run_version_command(self, test_bot:KleinanzeigenBot, capsys:Any) -> None: async def test_run_version_command(self, test_bot:KleinanzeigenBot, capsys:Any) -> None:
"""Test running version command.""" """Test running version command."""
await test_bot.run(['script.py', 'version']) await test_bot.run(["script.py", "version"])
captured = capsys.readouterr() captured = capsys.readouterr()
assert __version__ in captured.out assert __version__ in captured.out
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_run_help_command(self, test_bot:KleinanzeigenBot, capsys:Any) -> None: async def test_run_help_command(self, test_bot:KleinanzeigenBot, capsys:Any) -> None:
"""Test running help command.""" """Test running help command."""
await test_bot.run(['script.py', 'help']) await test_bot.run(["script.py", "help"])
captured = capsys.readouterr() captured = capsys.readouterr()
assert 'Usage:' in captured.out assert "Usage:" in captured.out
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_run_unknown_command(self, test_bot:KleinanzeigenBot) -> None: async def test_run_unknown_command(self, test_bot:KleinanzeigenBot) -> None:
"""Test running unknown command.""" """Test running unknown command."""
with pytest.raises(SystemExit) as exc_info: with pytest.raises(SystemExit) as exc_info:
await test_bot.run(['script.py', 'unknown']) await test_bot.run(["script.py", "unknown"])
assert exc_info.value.code == 2 assert exc_info.value.code == 2
@pytest.mark.asyncio @pytest.mark.asyncio
@@ -568,8 +568,8 @@ login:
password: test password: test
""") """)
test_bot.config_file_path = str(config_path) test_bot.config_file_path = str(config_path)
await test_bot.run(['script.py', 'verify']) await test_bot.run(["script.py", "verify"])
assert test_bot.config['login']['username'] == 'test' assert test_bot.config["login"]["username"] == "test"
class TestKleinanzeigenBotAdOperations: class TestKleinanzeigenBotAdOperations:
@@ -578,27 +578,27 @@ class TestKleinanzeigenBotAdOperations:
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_run_delete_command_no_ads(self, test_bot:KleinanzeigenBot, mock_config_setup:None) -> None: # pylint: disable=unused-argument async def test_run_delete_command_no_ads(self, test_bot:KleinanzeigenBot, mock_config_setup:None) -> None: # pylint: disable=unused-argument
"""Test running delete command with no ads.""" """Test running delete command with no ads."""
with patch.object(test_bot, 'load_ads', return_value = []): with patch.object(test_bot, "load_ads", return_value = []):
await test_bot.run(['script.py', 'delete']) await test_bot.run(["script.py", "delete"])
assert test_bot.command == 'delete' assert test_bot.command == "delete"
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_run_publish_command_no_ads(self, test_bot:KleinanzeigenBot, mock_config_setup:None) -> None: # pylint: disable=unused-argument async def test_run_publish_command_no_ads(self, test_bot:KleinanzeigenBot, mock_config_setup:None) -> None: # pylint: disable=unused-argument
"""Test running publish command with no ads.""" """Test running publish command with no ads."""
with patch.object(test_bot, 'load_ads', return_value = []): with patch.object(test_bot, "load_ads", return_value = []):
await test_bot.run(['script.py', 'publish']) await test_bot.run(["script.py", "publish"])
assert test_bot.command == 'publish' assert test_bot.command == "publish"
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_run_download_command_default_selector(self, test_bot:KleinanzeigenBot, mock_config_setup:None) -> None: # pylint: disable=unused-argument async def test_run_download_command_default_selector(self, test_bot:KleinanzeigenBot, mock_config_setup:None) -> None: # pylint: disable=unused-argument
"""Test running download command with default selector.""" """Test running download command with default selector."""
with patch.object(test_bot, 'download_ads', new_callable = AsyncMock): with patch.object(test_bot, "download_ads", new_callable = AsyncMock):
await test_bot.run(['script.py', 'download']) await test_bot.run(["script.py", "download"])
assert test_bot.ads_selector == 'new' assert test_bot.ads_selector == "new"
def test_load_ads_no_files(self, test_bot:KleinanzeigenBot) -> None: def test_load_ads_no_files(self, test_bot:KleinanzeigenBot) -> None:
"""Test loading ads with no files.""" """Test loading ads with no files."""
test_bot.config['ad_files'] = ['nonexistent/*.yaml'] test_bot.config["ad_files"] = ["nonexistent/*.yaml"]
ads = test_bot.load_ads() ads = test_bot.load_ads()
assert len(ads) == 0 assert len(ads) == 0
@@ -609,24 +609,24 @@ class TestKleinanzeigenBotAdManagement:
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_download_ads_with_specific_ids(self, test_bot:KleinanzeigenBot, mock_config_setup:None) -> None: # pylint: disable=unused-argument async def test_download_ads_with_specific_ids(self, test_bot:KleinanzeigenBot, mock_config_setup:None) -> None: # pylint: disable=unused-argument
"""Test downloading ads with specific IDs.""" """Test downloading ads with specific IDs."""
test_bot.ads_selector = '123,456' test_bot.ads_selector = "123,456"
with patch.object(test_bot, 'download_ads', new_callable = AsyncMock): with patch.object(test_bot, "download_ads", new_callable = AsyncMock):
await test_bot.run(['script.py', 'download', '--ads=123,456']) await test_bot.run(["script.py", "download", "--ads=123,456"])
assert test_bot.ads_selector == '123,456' assert test_bot.ads_selector == "123,456"
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_run_publish_invalid_selector(self, test_bot:KleinanzeigenBot, mock_config_setup:None) -> None: # pylint: disable=unused-argument async def test_run_publish_invalid_selector(self, test_bot:KleinanzeigenBot, mock_config_setup:None) -> None: # pylint: disable=unused-argument
"""Test running publish with invalid selector.""" """Test running publish with invalid selector."""
with patch.object(test_bot, 'load_ads', return_value = []): with patch.object(test_bot, "load_ads", return_value = []):
await test_bot.run(['script.py', 'publish', '--ads=invalid']) await test_bot.run(["script.py", "publish", "--ads=invalid"])
assert test_bot.ads_selector == 'due' assert test_bot.ads_selector == "due"
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_run_download_invalid_selector(self, test_bot:KleinanzeigenBot, mock_config_setup:None) -> None: # pylint: disable=unused-argument async def test_run_download_invalid_selector(self, test_bot:KleinanzeigenBot, mock_config_setup:None) -> None: # pylint: disable=unused-argument
"""Test running download with invalid selector.""" """Test running download with invalid selector."""
with patch.object(test_bot, 'download_ads', new_callable = AsyncMock): with patch.object(test_bot, "download_ads", new_callable = AsyncMock):
await test_bot.run(['script.py', 'download', '--ads=invalid']) await test_bot.run(["script.py", "download", "--ads=invalid"])
assert test_bot.ads_selector == 'new' assert test_bot.ads_selector == "new"
class TestKleinanzeigenBotAdConfiguration: class TestKleinanzeigenBotAdConfiguration:
@@ -645,8 +645,8 @@ categories:
""") """)
test_bot.config_file_path = str(config_path) test_bot.config_file_path = str(config_path)
test_bot.load_config() test_bot.load_config()
assert 'custom_cat' in test_bot.categories assert "custom_cat" in test_bot.categories
assert test_bot.categories['custom_cat'] == 'custom_id' assert test_bot.categories["custom_cat"] == "custom_id"
def test_load_ads_with_missing_title(self, test_bot:KleinanzeigenBot, tmp_path:Any, minimal_ad_config:dict[str, Any]) -> None: def test_load_ads_with_missing_title(self, test_bot:KleinanzeigenBot, tmp_path:Any, minimal_ad_config:dict[str, Any]) -> None:
"""Test loading ads with missing title.""" """Test loading ads with missing title."""
@@ -667,7 +667,7 @@ categories:
# Set config file path to tmp_path and use relative path for ad_files # Set config file path to tmp_path and use relative path for ad_files
test_bot.config_file_path = str(temp_path / "config.yaml") test_bot.config_file_path = str(temp_path / "config.yaml")
test_bot.config['ad_files'] = ["ads/*.yaml"] test_bot.config["ad_files"] = ["ads/*.yaml"]
with pytest.raises(AssertionError) as exc_info: with pytest.raises(AssertionError) as exc_info:
test_bot.load_ads() test_bot.load_ads()
assert "must be at least 10 characters long" in str(exc_info.value) assert "must be at least 10 characters long" in str(exc_info.value)
@@ -691,7 +691,7 @@ categories:
# Set config file path to tmp_path and use relative path for ad_files # Set config file path to tmp_path and use relative path for ad_files
test_bot.config_file_path = str(temp_path / "config.yaml") test_bot.config_file_path = str(temp_path / "config.yaml")
test_bot.config['ad_files'] = ["ads/*.yaml"] test_bot.config["ad_files"] = ["ads/*.yaml"]
with pytest.raises(AssertionError) as exc_info: with pytest.raises(AssertionError) as exc_info:
test_bot.load_ads() test_bot.load_ads()
assert "property [price_type] must be one of:" in str(exc_info.value) assert "property [price_type] must be one of:" in str(exc_info.value)
@@ -715,7 +715,7 @@ categories:
# Set config file path to tmp_path and use relative path for ad_files # Set config file path to tmp_path and use relative path for ad_files
test_bot.config_file_path = str(temp_path / "config.yaml") test_bot.config_file_path = str(temp_path / "config.yaml")
test_bot.config['ad_files'] = ["ads/*.yaml"] test_bot.config["ad_files"] = ["ads/*.yaml"]
with pytest.raises(AssertionError) as exc_info: with pytest.raises(AssertionError) as exc_info:
test_bot.load_ads() test_bot.load_ads()
assert "property [shipping_type] must be one of:" in str(exc_info.value) assert "property [shipping_type] must be one of:" in str(exc_info.value)
@@ -740,7 +740,7 @@ categories:
# Set config file path to tmp_path and use relative path for ad_files # Set config file path to tmp_path and use relative path for ad_files
test_bot.config_file_path = str(temp_path / "config.yaml") test_bot.config_file_path = str(temp_path / "config.yaml")
test_bot.config['ad_files'] = ["ads/*.yaml"] test_bot.config["ad_files"] = ["ads/*.yaml"]
with pytest.raises(AssertionError) as exc_info: with pytest.raises(AssertionError) as exc_info:
test_bot.load_ads() test_bot.load_ads()
assert "must not be specified for GIVE_AWAY ad" in str(exc_info.value) assert "must not be specified for GIVE_AWAY ad" in str(exc_info.value)
@@ -765,7 +765,7 @@ categories:
# Set config file path to tmp_path and use relative path for ad_files # Set config file path to tmp_path and use relative path for ad_files
test_bot.config_file_path = str(temp_path / "config.yaml") test_bot.config_file_path = str(temp_path / "config.yaml")
test_bot.config['ad_files'] = ["ads/*.yaml"] test_bot.config["ad_files"] = ["ads/*.yaml"]
with pytest.raises(AssertionError) as exc_info: with pytest.raises(AssertionError) as exc_info:
test_bot.load_ads() test_bot.load_ads()
assert "not specified" in str(exc_info.value) assert "not specified" in str(exc_info.value)
@@ -798,7 +798,7 @@ categories:
# Set config file path to tmp_path and use relative path for ad_files # Set config file path to tmp_path and use relative path for ad_files
test_bot.config_file_path = str(temp_path / "config.yaml") test_bot.config_file_path = str(temp_path / "config.yaml")
test_bot.config['ad_files'] = ["ads/*.yaml"] test_bot.config["ad_files"] = ["ads/*.yaml"]
with pytest.raises(AssertionError) as exc_info: with pytest.raises(AssertionError) as exc_info:
test_bot.load_ads() test_bot.load_ads()
assert "property [description] not specified" in str(exc_info.value) assert "property [description] not specified" in str(exc_info.value)
@@ -826,10 +826,10 @@ class TestKleinanzeigenBotAdDeletion:
{"title": "Other Title", "id": "11111"} {"title": "Other Title", "id": "11111"}
] ]
with patch.object(test_bot, 'web_open', new_callable = AsyncMock), \ with patch.object(test_bot, "web_open", new_callable = AsyncMock), \
patch.object(test_bot, 'web_find', new_callable = AsyncMock) as mock_find, \ patch.object(test_bot, "web_find", new_callable = AsyncMock) as mock_find, \
patch.object(test_bot, 'web_click', new_callable = AsyncMock), \ patch.object(test_bot, "web_click", new_callable = AsyncMock), \
patch.object(test_bot, 'web_check', new_callable = AsyncMock, return_value = True): patch.object(test_bot, "web_check", new_callable = AsyncMock, return_value = True):
mock_find.return_value.attrs = {"content": "some-token"} mock_find.return_value.attrs = {"content": "some-token"}
result = await test_bot.delete_ad(ad_cfg, published_ads, delete_old_ads_by_title = True) result = await test_bot.delete_ad(ad_cfg, published_ads, delete_old_ads_by_title = True)
assert result is True assert result is True
@@ -852,10 +852,10 @@ class TestKleinanzeigenBotAdDeletion:
{"title": "Other Title", "id": "11111"} {"title": "Other Title", "id": "11111"}
] ]
with patch.object(test_bot, 'web_open', new_callable = AsyncMock), \ with patch.object(test_bot, "web_open", new_callable = AsyncMock), \
patch.object(test_bot, 'web_find', new_callable = AsyncMock) as mock_find, \ patch.object(test_bot, "web_find", new_callable = AsyncMock) as mock_find, \
patch.object(test_bot, 'web_click', new_callable = AsyncMock), \ patch.object(test_bot, "web_click", new_callable = AsyncMock), \
patch.object(test_bot, 'web_check', new_callable = AsyncMock, return_value = True): patch.object(test_bot, "web_check", new_callable = AsyncMock, return_value = True):
mock_find.return_value.attrs = {"content": "some-token"} mock_find.return_value.attrs = {"content": "some-token"}
result = await test_bot.delete_ad(ad_cfg, published_ads, delete_old_ads_by_title = False) result = await test_bot.delete_ad(ad_cfg, published_ads, delete_old_ads_by_title = False)
assert result is True assert result is True
@@ -896,10 +896,10 @@ class TestKleinanzeigenBotAdRepublication:
# Set config file path and use relative path for ad_files # Set config file path and use relative path for ad_files
test_bot.config_file_path = str(temp_path / "config.yaml") test_bot.config_file_path = str(temp_path / "config.yaml")
test_bot.config['ad_files'] = ["ads/*.yaml"] test_bot.config["ad_files"] = ["ads/*.yaml"]
# Mock the loading of the original ad configuration # Mock the loading of the original ad configuration
with patch('kleinanzeigen_bot.utils.dicts.load_dict', side_effect = [ with patch("kleinanzeigen_bot.utils.dicts.load_dict", side_effect = [
ad_cfg, # First call returns the original ad config ad_cfg, # First call returns the original ad config
{} # Second call for ad_fields.yaml {} # Second call for ad_fields.yaml
]): ]):
@@ -925,9 +925,9 @@ class TestKleinanzeigenBotAdRepublication:
ad_cfg_orig["content_hash"] = current_hash ad_cfg_orig["content_hash"] = current_hash
# Mock the config to prevent actual file operations # Mock the config to prevent actual file operations
test_bot.config['ad_files'] = ['test.yaml'] test_bot.config["ad_files"] = ["test.yaml"]
with patch('kleinanzeigen_bot.utils.dicts.load_dict_if_exists', return_value = ad_cfg_orig), \ with patch("kleinanzeigen_bot.utils.dicts.load_dict_if_exists", return_value = ad_cfg_orig), \
patch('kleinanzeigen_bot.utils.dicts.load_dict', return_value = {}): # Mock ad_fields.yaml patch("kleinanzeigen_bot.utils.dicts.load_dict", return_value = {}): # Mock ad_fields.yaml
ads_to_publish = test_bot.load_ads() ads_to_publish = test_bot.load_ads()
assert len(ads_to_publish) == 0 # No ads should be marked for republication assert len(ads_to_publish) == 0 # No ads should be marked for republication
@@ -967,23 +967,23 @@ class TestKleinanzeigenBotShippingOptions:
# Mock web_execute to handle all JavaScript calls # Mock web_execute to handle all JavaScript calls
async def mock_web_execute(script:str) -> Any: async def mock_web_execute(script:str) -> Any:
if script == 'document.body.scrollHeight': if script == "document.body.scrollHeight":
return 0 # Return integer to prevent scrolling loop return 0 # Return integer to prevent scrolling loop
return None return None
# Mock the necessary web interaction methods # Mock the necessary web interaction methods
with patch.object(test_bot, 'web_execute', side_effect=mock_web_execute), \ with patch.object(test_bot, "web_execute", side_effect = mock_web_execute), \
patch.object(test_bot, 'web_click', new_callable=AsyncMock), \ patch.object(test_bot, "web_click", new_callable = AsyncMock), \
patch.object(test_bot, 'web_find', new_callable=AsyncMock) as mock_find, \ patch.object(test_bot, "web_find", new_callable = AsyncMock) as mock_find, \
patch.object(test_bot, 'web_select', new_callable = AsyncMock), \ patch.object(test_bot, "web_select", new_callable = AsyncMock), \
patch.object(test_bot, 'web_input', new_callable = AsyncMock), \ patch.object(test_bot, "web_input", new_callable = AsyncMock), \
patch.object(test_bot, 'web_open', new_callable = AsyncMock), \ patch.object(test_bot, "web_open", new_callable = AsyncMock), \
patch.object(test_bot, 'web_sleep', new_callable = AsyncMock), \ patch.object(test_bot, "web_sleep", new_callable = AsyncMock), \
patch.object(test_bot, 'web_check', new_callable = AsyncMock, return_value = True), \ patch.object(test_bot, "web_check", new_callable = AsyncMock, return_value = True), \
patch.object(test_bot, 'web_request', new_callable = AsyncMock), \ patch.object(test_bot, "web_request", new_callable = AsyncMock), \
patch.object(test_bot, 'web_find_all', new_callable = AsyncMock) as mock_find_all, \ patch.object(test_bot, "web_find_all", new_callable = AsyncMock) as mock_find_all, \
patch.object(test_bot, 'web_await', new_callable = AsyncMock), \ patch.object(test_bot, "web_await", new_callable = AsyncMock), \
patch('builtins.input', return_value=""): # Mock the input function patch("builtins.input", return_value = ""): # Mock the input function
# Mock the shipping options form elements # Mock the shipping options form elements
mock_find.side_effect = [ mock_find.side_effect = [
@@ -1000,7 +1000,7 @@ class TestKleinanzeigenBotShippingOptions:
mock_find_all.return_value = [] mock_find_all.return_value = []
# Mock web_check to return True for radio button checked state # Mock web_check to return True for radio button checked state
with patch.object(test_bot, 'web_check', new_callable = AsyncMock) as mock_check: with patch.object(test_bot, "web_check", new_callable = AsyncMock) as mock_check:
mock_check.return_value = True mock_check.return_value = True
# Test through the public interface by publishing an ad # Test through the public interface by publishing an ad
@@ -1075,7 +1075,7 @@ class TestKleinanzeigenBotDescriptionHandling:
"""Test that description works correctly when description is missing from main config.""" """Test that description works correctly when description is missing from main config."""
# Set up config without any description fields # Set up config without any description fields
test_bot.config = { test_bot.config = {
'ad_defaults': { "ad_defaults": {
# No description field at all # No description field at all
} }
} }
@@ -1093,9 +1093,9 @@ class TestKleinanzeigenBotDescriptionHandling:
def test_description_with_only_new_format_affixes(self, test_bot:KleinanzeigenBot) -> None: def test_description_with_only_new_format_affixes(self, test_bot:KleinanzeigenBot) -> None:
"""Test that description works with only new format affixes in config.""" """Test that description works with only new format affixes in config."""
test_bot.config = { test_bot.config = {
'ad_defaults': { "ad_defaults": {
'description_prefix': 'Prefix: ', "description_prefix": "Prefix: ",
'description_suffix': ' :Suffix' "description_suffix": " :Suffix"
} }
} }
@@ -1110,12 +1110,12 @@ class TestKleinanzeigenBotDescriptionHandling:
def test_description_with_mixed_config_formats(self, test_bot:KleinanzeigenBot) -> None: def test_description_with_mixed_config_formats(self, test_bot:KleinanzeigenBot) -> None:
"""Test that description works with both old and new format affixes in config.""" """Test that description works with both old and new format affixes in config."""
test_bot.config = { test_bot.config = {
'ad_defaults': { "ad_defaults": {
'description_prefix': 'New Prefix: ', "description_prefix": "New Prefix: ",
'description_suffix': ' :New Suffix', "description_suffix": " :New Suffix",
'description': { "description": {
'prefix': 'Old Prefix: ', "prefix": "Old Prefix: ",
'suffix': ' :Old Suffix' "suffix": " :Old Suffix"
} }
} }
} }
@@ -1131,9 +1131,9 @@ class TestKleinanzeigenBotDescriptionHandling:
def test_description_with_ad_level_affixes(self, test_bot:KleinanzeigenBot) -> None: def test_description_with_ad_level_affixes(self, test_bot:KleinanzeigenBot) -> None:
"""Test that ad-level affixes take precedence over config affixes.""" """Test that ad-level affixes take precedence over config affixes."""
test_bot.config = { test_bot.config = {
'ad_defaults': { "ad_defaults": {
'description_prefix': 'Config Prefix: ', "description_prefix": "Config Prefix: ",
'description_suffix': ' :Config Suffix' "description_suffix": " :Config Suffix"
} }
} }
@@ -1150,12 +1150,12 @@ class TestKleinanzeigenBotDescriptionHandling:
def test_description_with_none_values(self, test_bot:KleinanzeigenBot) -> None: def test_description_with_none_values(self, test_bot:KleinanzeigenBot) -> None:
"""Test that None values in affixes are handled correctly.""" """Test that None values in affixes are handled correctly."""
test_bot.config = { test_bot.config = {
'ad_defaults': { "ad_defaults": {
'description_prefix': None, "description_prefix": None,
'description_suffix': None, "description_suffix": None,
'description': { "description": {
'prefix': None, "prefix": None,
'suffix': None "suffix": None
} }
} }
} }
@@ -1171,7 +1171,7 @@ class TestKleinanzeigenBotDescriptionHandling:
def test_description_with_email_replacement(self, test_bot:KleinanzeigenBot) -> None: def test_description_with_email_replacement(self, test_bot:KleinanzeigenBot) -> None:
"""Test that @ symbols in description are replaced with (at).""" """Test that @ symbols in description are replaced with (at)."""
test_bot.config = { test_bot.config = {
'ad_defaults': {} "ad_defaults": {}
} }
ad_cfg = { ad_cfg = {
@@ -1228,10 +1228,10 @@ class TestKleinanzeigenBotChangedAds:
# Set config file path and use relative path for ad_files # Set config file path and use relative path for ad_files
test_bot.config_file_path = str(temp_path / "config.yaml") test_bot.config_file_path = str(temp_path / "config.yaml")
test_bot.config['ad_files'] = ["ads/*.yaml"] test_bot.config["ad_files"] = ["ads/*.yaml"]
# Mock the loading of the ad configuration # Mock the loading of the ad configuration
with patch('kleinanzeigen_bot.utils.dicts.load_dict', side_effect=[ with patch("kleinanzeigen_bot.utils.dicts.load_dict", side_effect = [
changed_ad, # First call returns the changed ad changed_ad, # First call returns the changed ad
{} # Second call for ad_fields.yaml {} # Second call for ad_fields.yaml
]): ]):
@@ -1280,10 +1280,10 @@ class TestKleinanzeigenBotChangedAds:
# Set config file path and use relative path for ad_files # Set config file path and use relative path for ad_files
test_bot.config_file_path = str(temp_path / "config.yaml") test_bot.config_file_path = str(temp_path / "config.yaml")
test_bot.config['ad_files'] = ["ads/*.yaml"] test_bot.config["ad_files"] = ["ads/*.yaml"]
# Mock the loading of the ad configuration # Mock the loading of the ad configuration
with patch('kleinanzeigen_bot.utils.dicts.load_dict', side_effect=[ with patch("kleinanzeigen_bot.utils.dicts.load_dict", side_effect = [
changed_ad, # First call returns the changed ad changed_ad, # First call returns the changed ad
{} # Second call for ad_fields.yaml {} # Second call for ad_fields.yaml
]): ]):

View File

@@ -31,7 +31,7 @@ EXCLUDED_MESSAGES: dict[str, set[str]] = {
} }
# Special modules that are known to be needed even if not in messages_by_file # Special modules that are known to be needed even if not in messages_by_file
KNOWN_NEEDED_MODULES = {'getopt.py'} KNOWN_NEEDED_MODULES = {"getopt.py"}
# Type aliases for better readability # Type aliases for better readability
ModulePath = str ModulePath = str
@@ -69,8 +69,8 @@ def _get_function_name(node: ast.AST) -> str:
function_name = None function_name = None
current = n current = n
while hasattr(current, '_parent'): while hasattr(current, "_parent"):
current = getattr(current, '_parent') current = getattr(current, "_parent")
if isinstance(current, ast.ClassDef) and not class_name: if isinstance(current, ast.ClassDef) and not class_name:
class_name = current.name class_name = current.name
elif isinstance(current, ast.FunctionDef) or isinstance(current, ast.AsyncFunctionDef) and not function_name: elif isinstance(current, ast.FunctionDef) or isinstance(current, ast.AsyncFunctionDef) and not function_name:
@@ -94,13 +94,13 @@ def _extract_log_messages(file_path: str, exclude_debug:bool = False) -> Message
Returns: Returns:
Dictionary mapping function names to their messages Dictionary mapping function names to their messages
""" """
with open(file_path, 'r', encoding = 'utf-8') as file: with open(file_path, "r", encoding = "utf-8") as file:
tree = ast.parse(file.read(), filename = file_path) tree = ast.parse(file.read(), filename = file_path)
# Add parent references for context tracking # Add parent references for context tracking
for parent in ast.walk(tree): for parent in ast.walk(tree):
for child in ast.iter_child_nodes(parent): for child in ast.iter_child_nodes(parent):
setattr(child, '_parent', parent) setattr(child, "_parent", parent)
messages:MessageDict = defaultdict(lambda: defaultdict(set)) messages:MessageDict = defaultdict(lambda: defaultdict(set))
@@ -114,7 +114,7 @@ def _extract_log_messages(file_path: str, exclude_debug:bool = False) -> Message
def extract_string_value(node:ast.AST) -> str | None: def extract_string_value(node:ast.AST) -> str | None:
"""Safely extract string value from an AST node.""" """Safely extract string value from an AST node."""
if isinstance(node, ast.Constant): if isinstance(node, ast.Constant):
value = getattr(node, 'value', None) value = getattr(node, "value", None)
return value if isinstance(value, str) else None return value if isinstance(value, str) else None
return None return None
@@ -127,24 +127,24 @@ def _extract_log_messages(file_path: str, exclude_debug:bool = False) -> Message
# Extract messages from various call types # Extract messages from various call types
if (isinstance(node.func, ast.Attribute) and if (isinstance(node.func, ast.Attribute) and
isinstance(node.func.value, ast.Name) and isinstance(node.func.value, ast.Name) and
node.func.value.id in {'LOG', 'logger', 'logging'} and node.func.value.id in {"LOG", "logger", "logging"} and
node.func.attr in {None if exclude_debug else 'debug', 'info', 'warning', 'error', 'exception', 'critical'}): node.func.attr in {None if exclude_debug else "debug", "info", "warning", "error", "exception", "critical"}):
if node.args: if node.args:
msg = extract_string_value(node.args[0]) msg = extract_string_value(node.args[0])
if msg: if msg:
add_message(function_name, msg) add_message(function_name, msg)
# Handle gettext calls # Handle gettext calls
elif ((isinstance(node.func, ast.Name) and node.func.id == '_') or elif ((isinstance(node.func, ast.Name) and node.func.id == "_") or
(isinstance(node.func, ast.Attribute) and node.func.attr == 'gettext')): (isinstance(node.func, ast.Attribute) and node.func.attr == "gettext")):
if node.args: if node.args:
msg = extract_string_value(node.args[0]) msg = extract_string_value(node.args[0])
if msg: if msg:
add_message(function_name, msg) add_message(function_name, msg)
# Handle other translatable function calls # Handle other translatable function calls
elif isinstance(node.func, ast.Name) and node.func.id in {'ainput', 'pluralize', 'ensure'}: elif isinstance(node.func, ast.Name) and node.func.id in {"ainput", "pluralize", "ensure"}:
arg_index = 0 if node.func.id == 'ainput' else 1 arg_index = 0 if node.func.id == "ainput" else 1
if len(node.args) > arg_index: if len(node.args) > arg_index:
msg = extract_string_value(node.args[arg_index]) msg = extract_string_value(node.args[arg_index])
if msg: if msg:
@@ -162,7 +162,7 @@ def _get_all_log_messages(exclude_debug:bool = False) -> dict[str, MessageDict]:
Returns: Returns:
Dictionary mapping module paths to their function messages Dictionary mapping module paths to their function messages
""" """
src_dir = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), 'src', 'kleinanzeigen_bot') src_dir = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), "src", "kleinanzeigen_bot")
print(f"\nScanning for messages in directory: {src_dir}") print(f"\nScanning for messages in directory: {src_dir}")
messages_by_file:dict[str, MessageDict] = { messages_by_file:dict[str, MessageDict] = {
@@ -187,15 +187,15 @@ def _get_all_log_messages(exclude_debug:bool = False) -> dict[str, MessageDict]:
for root, _, filenames in os.walk(src_dir): for root, _, filenames in os.walk(src_dir):
for filename in filenames: for filename in filenames:
if filename.endswith('.py'): if filename.endswith(".py"):
file_path = os.path.join(root, filename) file_path = os.path.join(root, filename)
relative_path = os.path.relpath(file_path, src_dir) relative_path = os.path.relpath(file_path, src_dir)
if relative_path.startswith('resources/'): if relative_path.startswith("resources/"):
continue continue
messages = _extract_log_messages(file_path, exclude_debug) messages = _extract_log_messages(file_path, exclude_debug)
if messages: if messages:
module_path = os.path.join('kleinanzeigen_bot', relative_path) module_path = os.path.join("kleinanzeigen_bot", relative_path)
module_path = module_path.replace(os.sep, '/') module_path = module_path.replace(os.sep, "/")
messages_by_file[module_path] = messages messages_by_file[module_path] = messages
return messages_by_file return messages_by_file
@@ -227,7 +227,7 @@ def _get_translations_for_language(lang: str) -> TranslationDict:
Returns: Returns:
Dictionary containing all translations for the language Dictionary containing all translations for the language
""" """
yaml = YAML(typ = 'safe') yaml = YAML(typ = "safe")
translation_file = f"translations.{lang}.yaml" translation_file = f"translations.{lang}.yaml"
print(f"Loading translations from {translation_file}") print(f"Loading translations from {translation_file}")
content = files(resources).joinpath(translation_file).read_text() content = files(resources).joinpath(translation_file).read_text()
@@ -253,11 +253,11 @@ def _find_translation(translations: TranslationDict,
True if translation exists in the correct location, False otherwise True if translation exists in the correct location, False otherwise
""" """
# Special case for getopt.py # Special case for getopt.py
if module == 'getopt.py': if module == "getopt.py":
return bool(translations.get(module, {}).get(function, {}).get(message)) return bool(translations.get(module, {}).get(function, {}).get(message))
# Add kleinanzeigen_bot/ prefix if not present # Add kleinanzeigen_bot/ prefix if not present
module_path = f'kleinanzeigen_bot/{module}' if not module.startswith('kleinanzeigen_bot/') else module module_path = f'kleinanzeigen_bot/{module}' if not module.startswith("kleinanzeigen_bot/") else module
# Check if module exists in translations # Check if module exists in translations
module_trans = translations.get(module_path, {}) module_trans = translations.get(module_path, {})
@@ -296,11 +296,11 @@ def _message_exists_in_code(code_messages: dict[str, MessageDict],
True if message exists in the code, False otherwise True if message exists in the code, False otherwise
""" """
# Special case for getopt.py # Special case for getopt.py
if module == 'getopt.py': if module == "getopt.py":
return bool(code_messages.get(module, {}).get(function, {}).get(message)) return bool(code_messages.get(module, {}).get(function, {}).get(message))
# Remove kleinanzeigen_bot/ prefix if present for code message lookup # Remove kleinanzeigen_bot/ prefix if present for code message lookup
module_path = module[len('kleinanzeigen_bot/'):] if module.startswith('kleinanzeigen_bot/') else module module_path = module[len("kleinanzeigen_bot/"):] if module.startswith("kleinanzeigen_bot/") else module
module_path = f'kleinanzeigen_bot/{module_path}' module_path = f'kleinanzeigen_bot/{module_path}'
# Check if module exists in code messages # Check if module exists in code messages