mirror of
https://github.com/Second-Hand-Friends/kleinanzeigen-bot.git
synced 2026-03-12 02:31:45 +01:00
refact: apply consistent formatting
This commit is contained in:
2
.github/PULL_REQUEST_TEMPLATE.md
vendored
2
.github/PULL_REQUEST_TEMPLATE.md
vendored
@@ -20,8 +20,8 @@ Select the type(s) of change(s) included in this pull request:
|
||||
Before requesting a review, confirm the following:
|
||||
- [ ] I have reviewed my changes to ensure they meet the project's standards.
|
||||
- [ ] I have tested my changes and ensured that all tests pass (`pdm run test`).
|
||||
- [ ] I have formatted the code (`pdm run format`).
|
||||
- [ ] I have verified that linting passes (`pdm run lint`).
|
||||
- [ ] I have run security scans and addressed any identified issues (`pdm run audit`).
|
||||
- [ ] I have updated documentation where necessary.
|
||||
|
||||
By submitting this pull request, I confirm that you can use, modify, copy, and redistribute this contribution, under the terms of your choice.
|
||||
|
||||
@@ -82,7 +82,7 @@ app = "python -m kleinanzeigen_bot"
|
||||
compile.cmd = "python -O -m PyInstaller pyinstaller.spec --clean"
|
||||
compile.env = {PYTHONHASHSEED = "1", SOURCE_DATE_EPOCH = "0"} # https://pyinstaller.org/en/stable/advanced-topics.html#creating-a-reproducible-build
|
||||
debug = "python -m pdb -m kleinanzeigen_bot"
|
||||
format = "autopep8 --recursive --in-place src tests --verbose"
|
||||
format = {shell = "autopep8 --recursive --in-place scripts src tests --verbose && python scripts/post_autopep8.py scripts src tests" }
|
||||
lint = {shell = "ruff check && mypy && basedpyright" }
|
||||
fix = {shell = "ruff check --fix" }
|
||||
test = "python -m pytest --capture=tee-sys -v"
|
||||
@@ -113,7 +113,7 @@ aggressive = 3
|
||||
# https://docs.astral.sh/ruff/configuration/
|
||||
#####################
|
||||
[tool.ruff]
|
||||
include = ["pyproject.toml", "src/**/*.py", "tests/**/*.py"]
|
||||
include = ["pyproject.toml", "scripts/**/*.py", "src/**/*.py", "tests/**/*.py"]
|
||||
line-length = 160
|
||||
indent-width = 4
|
||||
target-version = "py310"
|
||||
@@ -208,14 +208,10 @@ ignore = [
|
||||
"TC006", # Add quotes to type expression in `typing.cast()`
|
||||
]
|
||||
|
||||
[tool.ruff.format]
|
||||
quote-style = "double"
|
||||
indent-style = "space"
|
||||
line-ending = "native"
|
||||
docstring-code-format = false
|
||||
skip-magic-trailing-comma = false
|
||||
|
||||
[tool.ruff.lint.per-file-ignores]
|
||||
"scripts/**/*.py" = [
|
||||
"INP001", # File `...` is part of an implicit namespace package. Add an `__init__.py`.
|
||||
]
|
||||
"tests/**/*.py" = [
|
||||
"ARG",
|
||||
"B",
|
||||
@@ -247,7 +243,7 @@ max-statements = 150 # max. number of statements in function / method body (R091
|
||||
# https://mypy.readthedocs.io/en/stable/config_file.html
|
||||
#mypy_path = "$MYPY_CONFIG_FILE_DIR/tests/stubs"
|
||||
python_version = "3.10"
|
||||
files = "src,tests"
|
||||
files = "scripts,src,tests"
|
||||
strict = true
|
||||
disallow_untyped_calls = false
|
||||
disallow_untyped_defs = true
|
||||
@@ -264,7 +260,7 @@ verbosity = 0
|
||||
#####################
|
||||
[tool.basedpyright]
|
||||
# https://docs.basedpyright.com/latest/configuration/config-files/
|
||||
include = ["src", "tests"]
|
||||
include = ["scripts", "src", "tests"]
|
||||
defineConstant = { DEBUG = false }
|
||||
pythonVersion = "3.10"
|
||||
typeCheckingMode = "standard"
|
||||
|
||||
317
scripts/post_autopep8.py
Normal file
317
scripts/post_autopep8.py
Normal file
@@ -0,0 +1,317 @@
|
||||
# SPDX-FileCopyrightText: © Sebastian Thomschke and contributors
|
||||
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
# SPDX-ArtifactOfProjectHomePage: https://github.com/Second-Hand-Friends/kleinanzeigen-bot/
|
||||
import ast, logging, re, sys # isort: skip
|
||||
from pathlib import Path
|
||||
from typing import Final, List, Protocol, Tuple
|
||||
|
||||
from typing_extensions import override
|
||||
|
||||
# Configure basic logging
|
||||
logging.basicConfig(level = logging.INFO, format = "%(levelname)s: %(message)s")
|
||||
LOG:Final[logging.Logger] = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class FormatterRule(Protocol):
|
||||
"""
|
||||
A code processor that can modify source lines based on the AST.
|
||||
"""
|
||||
|
||||
def apply(self, tree:ast.AST, lines:List[str], path:Path) -> List[str]:
|
||||
...
|
||||
|
||||
|
||||
class NoSpaceAfterColonInTypeAnnotationRule(FormatterRule):
|
||||
"""
|
||||
Removes whitespace between the colon (:) and the type annotation in variable and function parameter declarations.
|
||||
|
||||
This rule enforces `a:int` instead of `a: int`.
|
||||
It is the opposite behavior of autopep8 rule E231.
|
||||
|
||||
Example:
|
||||
# Before
|
||||
def foo(a: int, b : str) -> None:
|
||||
pass
|
||||
|
||||
# After
|
||||
def foo(a:int, b:str) -> None:
|
||||
pass
|
||||
"""
|
||||
|
||||
@override
|
||||
def apply(self, tree:ast.AST, lines:List[str], path:Path) -> List[str]:
|
||||
ann_positions:List[Tuple[int, int]] = []
|
||||
for node in ast.walk(tree):
|
||||
if isinstance(node, ast.arg) and node.annotation is not None:
|
||||
ann_positions.append((node.annotation.lineno - 1, node.annotation.col_offset))
|
||||
elif isinstance(node, ast.AnnAssign) and node.annotation is not None:
|
||||
ann = node.annotation
|
||||
ann_positions.append((ann.lineno - 1, ann.col_offset))
|
||||
|
||||
if not ann_positions:
|
||||
return lines
|
||||
|
||||
new_lines:List[str] = []
|
||||
for idx, line in enumerate(lines):
|
||||
if line.lstrip().startswith("#"):
|
||||
new_lines.append(line)
|
||||
continue
|
||||
|
||||
chars = list(line)
|
||||
offsets = [col for (lin, col) in ann_positions if lin == idx]
|
||||
for col in sorted(offsets, reverse = True):
|
||||
prefix = "".join(chars[:col])
|
||||
colon_idx = prefix.rfind(":")
|
||||
if colon_idx == -1:
|
||||
continue
|
||||
j = colon_idx + 1
|
||||
while j < len(chars) and chars[j].isspace():
|
||||
del chars[j]
|
||||
new_lines.append("".join(chars))
|
||||
|
||||
return new_lines
|
||||
|
||||
|
||||
class EqualSignSpacingInDefaultsAndNamedArgsRule(FormatterRule):
|
||||
"""
|
||||
Ensures that the '=' sign in default values for function parameters and keyword arguments in function calls
|
||||
is surrounded by exactly one space on each side.
|
||||
|
||||
This rule enforces `a:int = 3` instead of `a:int=3`, and `x = 42` instead of `x=42` or `x =42`.
|
||||
It is the opposite behavior of autopep8 rule E251.
|
||||
|
||||
Example:
|
||||
# Before
|
||||
def foo(a:int=3, b :str= "bar"):
|
||||
pass
|
||||
|
||||
foo(x=42,y = "hello")
|
||||
|
||||
# After
|
||||
def foo(a:int = 3, b:str = "bar"):
|
||||
pass
|
||||
|
||||
foo(x = 42, y = "hello")
|
||||
"""
|
||||
|
||||
@override
|
||||
def apply(self, tree:ast.AST, lines:List[str], path:Path) -> List[str]:
|
||||
equals_positions:List[Tuple[int, int]] = []
|
||||
for node in ast.walk(tree):
|
||||
# --- Defaults in function definitions, async defs & lambdas ---
|
||||
if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.Lambda)):
|
||||
# positional defaults
|
||||
equals_positions.extend(
|
||||
(d.lineno - 1, d.col_offset)
|
||||
for d in node.args.defaults
|
||||
if d is not None
|
||||
)
|
||||
# keyword-only defaults (only on defs, not lambdas)
|
||||
if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
|
||||
equals_positions.extend(
|
||||
(d.lineno - 1, d.col_offset)
|
||||
for d in node.args.kw_defaults
|
||||
if d is not None
|
||||
)
|
||||
|
||||
# --- Keyword arguments in calls ---
|
||||
if isinstance(node, ast.Call):
|
||||
equals_positions.extend(
|
||||
(kw.value.lineno - 1, kw.value.col_offset)
|
||||
for kw in node.keywords
|
||||
if kw.arg is not None
|
||||
)
|
||||
|
||||
if not equals_positions:
|
||||
return lines
|
||||
|
||||
new_lines:List[str] = []
|
||||
for line_idx, line in enumerate(lines):
|
||||
if line.lstrip().startswith("#"):
|
||||
new_lines.append(line)
|
||||
continue
|
||||
|
||||
chars = list(line)
|
||||
equals_offsets = [col for (lineno, col) in equals_positions if lineno == line_idx]
|
||||
for col in sorted(equals_offsets, reverse = True):
|
||||
prefix = "".join(chars[:col])
|
||||
equal_sign_idx = prefix.rfind("=")
|
||||
if equal_sign_idx == -1:
|
||||
continue
|
||||
|
||||
# remove spaces before '='
|
||||
left_index = equal_sign_idx - 1
|
||||
while left_index >= 0 and chars[left_index].isspace():
|
||||
del chars[left_index]
|
||||
equal_sign_idx -= 1
|
||||
left_index -= 1
|
||||
|
||||
# remove spaces after '='
|
||||
right_index = equal_sign_idx + 1
|
||||
while right_index < len(chars) and chars[right_index].isspace():
|
||||
del chars[right_index]
|
||||
|
||||
# insert single spaces
|
||||
chars.insert(equal_sign_idx, " ")
|
||||
chars.insert(equal_sign_idx + 2, " ")
|
||||
new_lines.append("".join(chars))
|
||||
|
||||
return new_lines
|
||||
|
||||
|
||||
class PreferDoubleQuotesRule(FormatterRule):
|
||||
"""
|
||||
Ensures string literals use double quotes unless the content contains a double quote.
|
||||
|
||||
Example:
|
||||
# Before
|
||||
foo = 'hello'
|
||||
bar = 'a "quote" inside'
|
||||
|
||||
# After
|
||||
foo = "hello"
|
||||
bar = 'a "quote" inside' # kept as-is, because it contains a double quote
|
||||
"""
|
||||
|
||||
@override
|
||||
def apply(self, tree:ast.AST, lines:List[str], path:Path) -> List[str]:
|
||||
new_lines = lines.copy()
|
||||
|
||||
# Track how much each line has shifted so far
|
||||
line_shifts:dict[int, int] = dict.fromkeys(range(len(lines)), 0)
|
||||
|
||||
# Build a parent map for f-string detection
|
||||
parent_map:dict[ast.AST, ast.AST] = {}
|
||||
for parent in ast.walk(tree):
|
||||
for child in ast.iter_child_nodes(parent):
|
||||
parent_map[child] = parent
|
||||
|
||||
def is_in_fstring(node:ast.AST) -> bool:
|
||||
p = parent_map.get(node)
|
||||
while p:
|
||||
if isinstance(p, ast.JoinedStr):
|
||||
return True
|
||||
p = parent_map.get(p)
|
||||
return False
|
||||
|
||||
# Regex to locate a single- or triple-quoted literal:
|
||||
# (?P<prefix>[rRbuUfF]*) optional string flags (r, b, u, f, etc.), case-insensitive
|
||||
# (?P<quote>'{3}|') the opening delimiter: either three single-quotes (''') or one ('),
|
||||
# but never two in a row (so we won't mis-interpret adjacent quotes)
|
||||
# (?P<content>.*?) the literal's content, non-greedy up to the next same delimiter
|
||||
# (?P=quote) the matching closing delimiter (same length as the opener)
|
||||
literal_re = re.compile(
|
||||
r"(?P<prefix>[rRbuUfF]*)(?P<quote>'{3}|')(?P<content>.*?)(?P=quote)",
|
||||
re.DOTALL,
|
||||
)
|
||||
|
||||
for node in ast.walk(tree):
|
||||
# only handle simple string constants
|
||||
if not (isinstance(node, ast.Constant) and isinstance(node.value, str)):
|
||||
continue
|
||||
|
||||
# skip anything inside an f-string, at any depth
|
||||
if is_in_fstring(node):
|
||||
continue
|
||||
|
||||
starting_line_number = getattr(node, "lineno", None)
|
||||
starting_col_offset = getattr(node, "col_offset", None)
|
||||
if starting_line_number is None or starting_col_offset is None:
|
||||
continue
|
||||
|
||||
start_line = starting_line_number - 1
|
||||
shift = line_shifts[start_line]
|
||||
raw = new_lines[start_line]
|
||||
# apply shift so we match against current edited line
|
||||
idx = starting_col_offset + shift
|
||||
if idx >= len(raw) or raw[idx] not in ("'", "r", "u", "b", "f", "R", "U", "B", "F"):
|
||||
continue
|
||||
|
||||
# match literal at that column
|
||||
m = literal_re.match(raw[idx:])
|
||||
if not m:
|
||||
continue
|
||||
|
||||
prefix = m.group("prefix")
|
||||
quote = m.group("quote") # either "'" or "'''"
|
||||
content = m.group("content") # what's inside
|
||||
|
||||
# skip if content has a double-quote already
|
||||
if '"' in content:
|
||||
continue
|
||||
|
||||
# build new literal with the same prefix, but double‐quote delimiter
|
||||
delim = '"' * len(quote)
|
||||
escaped = content.replace(delim, "\\" + delim)
|
||||
new_literal = f"{prefix}{delim}{escaped}{delim}"
|
||||
|
||||
literal_len = m.end() # how many chars we're replacing
|
||||
before = raw[:idx]
|
||||
after = raw[idx + literal_len:]
|
||||
new_lines[start_line] = before + new_literal + after
|
||||
|
||||
# record shift delta for any further edits on this line
|
||||
line_shifts[start_line] += len(new_literal) - literal_len
|
||||
|
||||
return new_lines
|
||||
|
||||
|
||||
FORMATTER_RULES:List[FormatterRule] = [
|
||||
NoSpaceAfterColonInTypeAnnotationRule(),
|
||||
EqualSignSpacingInDefaultsAndNamedArgsRule(),
|
||||
PreferDoubleQuotesRule(),
|
||||
]
|
||||
|
||||
|
||||
def format_file(path:Path) -> None:
|
||||
# Read without newline conversion
|
||||
with path.open("r", encoding = "utf-8", newline = "") as rf:
|
||||
original_text = rf.read()
|
||||
|
||||
# Initial parse
|
||||
try:
|
||||
tree = ast.parse(original_text)
|
||||
except SyntaxError as e:
|
||||
LOG.error(
|
||||
"Syntax error parsing %s[%d:%d]: %r -> %s",
|
||||
path, e.lineno, e.offset, (e.text or "").rstrip(), e.msg
|
||||
)
|
||||
return
|
||||
|
||||
lines = original_text.splitlines(keepends = True)
|
||||
formatted_text = original_text
|
||||
success = True
|
||||
for rule in FORMATTER_RULES:
|
||||
lines = rule.apply(tree, lines, path)
|
||||
formatted_text = "".join(lines)
|
||||
|
||||
# Re-parse the updated text
|
||||
try:
|
||||
tree = ast.parse(formatted_text)
|
||||
except SyntaxError as e:
|
||||
LOG.error(
|
||||
"Syntax error after %s at %s[%d:%d]: %r -> %s",
|
||||
rule.__class__.__name__, path, e.lineno, e.offset, (e.text or "").rstrip(), e.msg
|
||||
)
|
||||
success = False
|
||||
break
|
||||
|
||||
if success and formatted_text != original_text:
|
||||
with path.open("w", encoding = "utf-8", newline = "") as wf:
|
||||
wf.write(formatted_text)
|
||||
LOG.info("Formatted [%s].", path)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
if len(sys.argv) < 2: # noqa: PLR2004 Magic value used in comparison
|
||||
script_path = Path(sys.argv[0])
|
||||
print(f"Usage: python {script_path} <directory1> [<directory2> ...]")
|
||||
sys.exit(1)
|
||||
|
||||
for dir_arg in sys.argv[1:]:
|
||||
root = Path(dir_arg)
|
||||
if not root.exists():
|
||||
LOG.warning("Directory [%s] does not exist, skipping...", root)
|
||||
continue
|
||||
for py_file in root.rglob("*.py"):
|
||||
format_file(py_file)
|
||||
@@ -83,11 +83,11 @@ class KleinanzeigenBot(WebScrapingMixin):
|
||||
self.configure_file_logging()
|
||||
self.load_config()
|
||||
|
||||
if not (self.ads_selector in {'all', 'new', 'due', 'changed'} or
|
||||
any(selector in self.ads_selector.split(',') for selector in ('all', 'new', 'due', 'changed')) or
|
||||
re.compile(r'\d+[,\d+]*').search(self.ads_selector)):
|
||||
if not (self.ads_selector in {"all", "new", "due", "changed"} or
|
||||
any(selector in self.ads_selector.split(",") for selector in ("all", "new", "due", "changed")) or
|
||||
re.compile(r"\d+[,\d+]*").search(self.ads_selector)):
|
||||
LOG.warning('You provided no ads selector. Defaulting to "due".')
|
||||
self.ads_selector = 'due'
|
||||
self.ads_selector = "due"
|
||||
|
||||
if ads := self.load_ads():
|
||||
await self.create_browser_session()
|
||||
@@ -111,9 +111,9 @@ class KleinanzeigenBot(WebScrapingMixin):
|
||||
case "download":
|
||||
self.configure_file_logging()
|
||||
# ad IDs depends on selector
|
||||
if not (self.ads_selector in {'all', 'new'} or re.compile(r'\d+[,\d+]*').search(self.ads_selector)):
|
||||
if not (self.ads_selector in {"all", "new"} or re.compile(r"\d+[,\d+]*").search(self.ads_selector)):
|
||||
LOG.warning('You provided no ads selector. Defaulting to "new".')
|
||||
self.ads_selector = 'new'
|
||||
self.ads_selector = "new"
|
||||
self.load_config()
|
||||
await self.create_browser_session()
|
||||
await self.login()
|
||||
@@ -327,7 +327,7 @@ class KleinanzeigenBot(WebScrapingMixin):
|
||||
data_root_dir = os.path.dirname(self.config_file_path)
|
||||
for file_pattern in self.config["ad_files"]:
|
||||
for ad_file in glob.glob(file_pattern, root_dir = data_root_dir, flags = glob.GLOBSTAR | glob.BRACE | glob.EXTGLOB):
|
||||
if not str(ad_file).endswith('ad_fields.yaml'):
|
||||
if not str(ad_file).endswith("ad_fields.yaml"):
|
||||
ad_files[abspath(ad_file, relative_to = data_root_dir)] = ad_file
|
||||
LOG.info(" -> found %s", pluralize("ad config file", ad_files))
|
||||
if not ad_files:
|
||||
@@ -335,13 +335,13 @@ class KleinanzeigenBot(WebScrapingMixin):
|
||||
|
||||
ids = []
|
||||
use_specific_ads = False
|
||||
selectors = self.ads_selector.split(',')
|
||||
selectors = self.ads_selector.split(",")
|
||||
|
||||
if re.compile(r'\d+[,\d+]*').search(self.ads_selector):
|
||||
ids = [int(n) for n in self.ads_selector.split(',')]
|
||||
if re.compile(r"\d+[,\d+]*").search(self.ads_selector):
|
||||
ids = [int(n) for n in self.ads_selector.split(",")]
|
||||
use_specific_ads = True
|
||||
LOG.info('Start fetch task for the ad(s) with id(s):')
|
||||
LOG.info(' | '.join([str(id_) for id_ in ids]))
|
||||
LOG.info("Start fetch task for the ad(s) with id(s):")
|
||||
LOG.info(" | ".join([str(id_) for id_ in ids]))
|
||||
|
||||
ad_fields = dicts.load_dict_from_module(resources, "ad_fields.yaml")
|
||||
ads = []
|
||||
@@ -548,7 +548,7 @@ class KleinanzeigenBot(WebScrapingMixin):
|
||||
async def is_logged_in(self) -> bool:
|
||||
try:
|
||||
user_info = await self.web_text(By.CLASS_NAME, "mr-medium")
|
||||
if self.config['login']['username'].lower() in user_info.lower():
|
||||
if self.config["login"]["username"].lower() in user_info.lower():
|
||||
return True
|
||||
except TimeoutError:
|
||||
return False
|
||||
@@ -657,7 +657,7 @@ class KleinanzeigenBot(WebScrapingMixin):
|
||||
#############################
|
||||
# set category
|
||||
#############################
|
||||
await self.__set_category(ad_cfg['category'], ad_file)
|
||||
await self.__set_category(ad_cfg["category"], ad_file)
|
||||
|
||||
#############################
|
||||
# set special attributes
|
||||
@@ -674,7 +674,7 @@ class KleinanzeigenBot(WebScrapingMixin):
|
||||
try:
|
||||
await self.web_select(By.XPATH, "//select[contains(@id, '.versand_s')]", shipping_value)
|
||||
except TimeoutError:
|
||||
LOG.warning("Failed to set shipping attribute for type '%s'!", ad_cfg['shipping_type'])
|
||||
LOG.warning("Failed to set shipping attribute for type '%s'!", ad_cfg["shipping_type"])
|
||||
else:
|
||||
await self.__set_shipping(ad_cfg)
|
||||
|
||||
@@ -698,9 +698,9 @@ class KleinanzeigenBot(WebScrapingMixin):
|
||||
if ad_cfg["shipping_type"] == "SHIPPING":
|
||||
if sell_directly and ad_cfg["shipping_options"] and price_type in {"FIXED", "NEGOTIABLE"}:
|
||||
if not await self.web_check(By.ID, "radio-buy-now-yes", Is.SELECTED):
|
||||
await self.web_click(By.ID, 'radio-buy-now-yes')
|
||||
await self.web_click(By.ID, "radio-buy-now-yes")
|
||||
elif not await self.web_check(By.ID, "radio-buy-now-no", Is.SELECTED):
|
||||
await self.web_click(By.ID, 'radio-buy-now-no')
|
||||
await self.web_click(By.ID, "radio-buy-now-no")
|
||||
except TimeoutError as ex:
|
||||
LOG.debug(ex, exc_info = True)
|
||||
|
||||
@@ -886,7 +886,7 @@ class KleinanzeigenBot(WebScrapingMixin):
|
||||
|
||||
async def __set_special_attributes(self, ad_cfg:dict[str, Any]) -> None:
|
||||
if ad_cfg["special_attributes"]:
|
||||
LOG.debug('Found %i special attributes', len(ad_cfg["special_attributes"]))
|
||||
LOG.debug("Found %i special attributes", len(ad_cfg["special_attributes"]))
|
||||
for special_attribute_key, special_attribute_value in ad_cfg["special_attributes"].items():
|
||||
|
||||
if special_attribute_key == "condition_s":
|
||||
@@ -911,10 +911,10 @@ class KleinanzeigenBot(WebScrapingMixin):
|
||||
|
||||
try:
|
||||
elem_id = special_attr_elem.attrs.id
|
||||
if special_attr_elem.local_name == 'select':
|
||||
if special_attr_elem.local_name == "select":
|
||||
LOG.debug("Attribute field '%s' seems to be a select...", special_attribute_key)
|
||||
await self.web_select(By.ID, elem_id, special_attribute_value)
|
||||
elif special_attr_elem.attrs.type == 'checkbox':
|
||||
elif special_attr_elem.attrs.type == "checkbox":
|
||||
LOG.debug("Attribute field '%s' seems to be a checkbox...", special_attribute_key)
|
||||
await self.web_click(By.ID, elem_id)
|
||||
else:
|
||||
@@ -1036,7 +1036,7 @@ class KleinanzeigenBot(WebScrapingMixin):
|
||||
|
||||
async def assert_free_ad_limit_not_reached(self) -> None:
|
||||
try:
|
||||
await self.web_find(By.XPATH, '/html/body/div[1]/form/fieldset[6]/div[1]/header', timeout = 2)
|
||||
await self.web_find(By.XPATH, "/html/body/div[1]/form/fieldset[6]/div[1]/header", timeout = 2)
|
||||
raise AssertionError(f"Cannot publish more ads. The monthly limit of free ads of account {self.config['login']['username']} is reached.")
|
||||
except TimeoutError:
|
||||
pass
|
||||
@@ -1050,13 +1050,13 @@ class KleinanzeigenBot(WebScrapingMixin):
|
||||
ad_extractor = extract.AdExtractor(self.browser, self.config)
|
||||
|
||||
# use relevant download routine
|
||||
if self.ads_selector in {'all', 'new'}: # explore ads overview for these two modes
|
||||
LOG.info('Scanning your ad overview...')
|
||||
if self.ads_selector in {"all", "new"}: # explore ads overview for these two modes
|
||||
LOG.info("Scanning your ad overview...")
|
||||
own_ad_urls = await ad_extractor.extract_own_ads_urls()
|
||||
LOG.info('%s found.', pluralize("ad", len(own_ad_urls)))
|
||||
LOG.info("%s found.", pluralize("ad", len(own_ad_urls)))
|
||||
|
||||
if self.ads_selector == 'all': # download all of your adds
|
||||
LOG.info('Starting download of all ads...')
|
||||
if self.ads_selector == "all": # download all of your adds
|
||||
LOG.info("Starting download of all ads...")
|
||||
|
||||
success_count = 0
|
||||
# call download function for each ad page
|
||||
@@ -1067,12 +1067,12 @@ class KleinanzeigenBot(WebScrapingMixin):
|
||||
success_count += 1
|
||||
LOG.info("%d of %d ads were downloaded from your profile.", success_count, len(own_ad_urls))
|
||||
|
||||
elif self.ads_selector == 'new': # download only unsaved ads
|
||||
elif self.ads_selector == "new": # download only unsaved ads
|
||||
# check which ads already saved
|
||||
saved_ad_ids = []
|
||||
ads = self.load_ads(ignore_inactive = False, check_id = False) # do not skip because of existing IDs
|
||||
for ad in ads:
|
||||
ad_id = int(ad[2]['id'])
|
||||
ad_id = int(ad[2]["id"])
|
||||
saved_ad_ids.append(ad_id)
|
||||
|
||||
# determine ad IDs from links
|
||||
@@ -1083,26 +1083,26 @@ class KleinanzeigenBot(WebScrapingMixin):
|
||||
for ad_url, ad_id in ad_id_by_url.items():
|
||||
# check if ad with ID already saved
|
||||
if ad_id in saved_ad_ids:
|
||||
LOG.info('The ad with id %d has already been saved.', ad_id)
|
||||
LOG.info("The ad with id %d has already been saved.", ad_id)
|
||||
continue
|
||||
|
||||
if await ad_extractor.naviagte_to_ad_page(ad_url):
|
||||
await ad_extractor.download_ad(ad_id)
|
||||
new_count += 1
|
||||
LOG.info('%s were downloaded from your profile.', pluralize("new ad", new_count))
|
||||
LOG.info("%s were downloaded from your profile.", pluralize("new ad", new_count))
|
||||
|
||||
elif re.compile(r'\d+[,\d+]*').search(self.ads_selector): # download ad(s) with specific id(s)
|
||||
ids = [int(n) for n in self.ads_selector.split(',')]
|
||||
LOG.info('Starting download of ad(s) with the id(s):')
|
||||
LOG.info(' | '.join([str(ad_id) for ad_id in ids]))
|
||||
elif re.compile(r"\d+[,\d+]*").search(self.ads_selector): # download ad(s) with specific id(s)
|
||||
ids = [int(n) for n in self.ads_selector.split(",")]
|
||||
LOG.info("Starting download of ad(s) with the id(s):")
|
||||
LOG.info(" | ".join([str(ad_id) for ad_id in ids]))
|
||||
|
||||
for ad_id in ids: # call download routine for every id
|
||||
exists = await ad_extractor.naviagte_to_ad_page(ad_id)
|
||||
if exists:
|
||||
await ad_extractor.download_ad(ad_id)
|
||||
LOG.info('Downloaded ad with id %d', ad_id)
|
||||
LOG.info("Downloaded ad with id %d", ad_id)
|
||||
else:
|
||||
LOG.error('The page with the id %d does not exist!', ad_id)
|
||||
LOG.error("The page with the id %d does not exist!", ad_id)
|
||||
|
||||
def __get_description_with_affixes(self, ad_cfg:dict[str, Any]) -> str:
|
||||
"""Get the complete description with prefix and suffix applied.
|
||||
|
||||
@@ -36,22 +36,22 @@ class AdExtractor(WebScrapingMixin):
|
||||
"""
|
||||
|
||||
# create sub-directory for ad(s) to download (if necessary):
|
||||
relative_directory = 'downloaded-ads'
|
||||
relative_directory = "downloaded-ads"
|
||||
# make sure configured base directory exists
|
||||
if not os.path.exists(relative_directory) or not os.path.isdir(relative_directory):
|
||||
os.mkdir(relative_directory)
|
||||
LOG.info('Created ads directory at ./%s.', relative_directory)
|
||||
LOG.info("Created ads directory at ./%s.", relative_directory)
|
||||
|
||||
new_base_dir = os.path.join(relative_directory, f'ad_{ad_id}')
|
||||
if os.path.exists(new_base_dir):
|
||||
LOG.info('Deleting current folder of ad %s...', ad_id)
|
||||
LOG.info("Deleting current folder of ad %s...", ad_id)
|
||||
shutil.rmtree(new_base_dir)
|
||||
os.mkdir(new_base_dir)
|
||||
LOG.info('New directory for ad created at %s.', new_base_dir)
|
||||
LOG.info("New directory for ad created at %s.", new_base_dir)
|
||||
|
||||
# call extraction function
|
||||
info = await self._extract_ad_page_info(new_base_dir, ad_id)
|
||||
ad_file_path = new_base_dir + '/' + f'ad_{ad_id}.yaml'
|
||||
ad_file_path = new_base_dir + "/" + f'ad_{ad_id}.yaml'
|
||||
dicts.save_dict(ad_file_path, info)
|
||||
|
||||
async def _download_images_from_ad_page(self, directory:str, ad_id:int) -> list[str]:
|
||||
@@ -67,18 +67,18 @@ class AdExtractor(WebScrapingMixin):
|
||||
img_paths = []
|
||||
try:
|
||||
# download all images from box
|
||||
image_box = await self.web_find(By.CLASS_NAME, 'galleryimage-large')
|
||||
image_box = await self.web_find(By.CLASS_NAME, "galleryimage-large")
|
||||
|
||||
n_images = len(await self.web_find_all(By.CSS_SELECTOR, '.galleryimage-element[data-ix]', parent = image_box))
|
||||
LOG.info('Found %s.', i18n.pluralize("image", n_images))
|
||||
n_images = len(await self.web_find_all(By.CSS_SELECTOR, ".galleryimage-element[data-ix]", parent = image_box))
|
||||
LOG.info("Found %s.", i18n.pluralize("image", n_images))
|
||||
|
||||
img_element:Element = await self.web_find(By.CSS_SELECTOR, 'div:nth-child(1) > img', parent = image_box)
|
||||
img_fn_prefix = 'ad_' + str(ad_id) + '__img'
|
||||
img_element:Element = await self.web_find(By.CSS_SELECTOR, "div:nth-child(1) > img", parent = image_box)
|
||||
img_fn_prefix = "ad_" + str(ad_id) + "__img"
|
||||
|
||||
img_nr = 1
|
||||
dl_counter = 0
|
||||
while img_nr <= n_images: # scrolling + downloading
|
||||
current_img_url = img_element.attrs['src'] # URL of the image
|
||||
current_img_url = img_element.attrs["src"] # URL of the image
|
||||
if current_img_url is None:
|
||||
continue
|
||||
|
||||
@@ -86,26 +86,26 @@ class AdExtractor(WebScrapingMixin):
|
||||
content_type = response.info().get_content_type()
|
||||
file_ending = mimetypes.guess_extension(content_type)
|
||||
img_path = f"{directory}/{img_fn_prefix}{img_nr}{file_ending}"
|
||||
with open(img_path, 'wb') as f:
|
||||
with open(img_path, "wb") as f:
|
||||
shutil.copyfileobj(response, f)
|
||||
dl_counter += 1
|
||||
img_paths.append(img_path.rsplit('/', maxsplit = 1)[-1])
|
||||
img_paths.append(img_path.rsplit("/", maxsplit = 1)[-1])
|
||||
|
||||
# navigate to next image (if exists)
|
||||
if img_nr < n_images:
|
||||
try:
|
||||
# click next button, wait, and re-establish reference
|
||||
await (await self.web_find(By.CLASS_NAME, 'galleryimage--navigation--next')).click()
|
||||
await (await self.web_find(By.CLASS_NAME, "galleryimage--navigation--next")).click()
|
||||
new_div = await self.web_find(By.CSS_SELECTOR, f'div.galleryimage-element:nth-child({img_nr + 1})')
|
||||
img_element = await self.web_find(By.TAG_NAME, 'img', parent = new_div)
|
||||
img_element = await self.web_find(By.TAG_NAME, "img", parent = new_div)
|
||||
except TimeoutError:
|
||||
LOG.error('NEXT button in image gallery somehow missing, aborting image fetching.')
|
||||
LOG.error("NEXT button in image gallery somehow missing, aborting image fetching.")
|
||||
break
|
||||
img_nr += 1
|
||||
LOG.info('Downloaded %s.', i18n.pluralize("image", dl_counter))
|
||||
LOG.info("Downloaded %s.", i18n.pluralize("image", dl_counter))
|
||||
|
||||
except TimeoutError: # some ads do not require images
|
||||
LOG.warning('No image area found. Continuing without downloading images.')
|
||||
LOG.warning("No image area found. Continuing without downloading images.")
|
||||
|
||||
return img_paths
|
||||
|
||||
@@ -116,13 +116,13 @@ class AdExtractor(WebScrapingMixin):
|
||||
:param url: the URL to the ad page
|
||||
:return: the ad ID, a (ten-digit) integer number
|
||||
"""
|
||||
num_part = url.split('/')[-1] # suffix
|
||||
id_part = num_part.split('-')[0]
|
||||
num_part = url.split("/")[-1] # suffix
|
||||
id_part = num_part.split("-")[0]
|
||||
|
||||
try:
|
||||
path = url.split('?', 1)[0] # Remove query string if present
|
||||
last_segment = path.rstrip('/').split('/')[-1] # Get last path component
|
||||
id_part = last_segment.split('-')[0] # Extract part before first hyphen
|
||||
path = url.split("?", 1)[0] # Remove query string if present
|
||||
last_segment = path.rstrip("/").split("/")[-1] # Get last path component
|
||||
id_part = last_segment.split("-")[0] # Extract part before first hyphen
|
||||
return int(id_part)
|
||||
except (IndexError, ValueError) as ex:
|
||||
LOG.warning("Failed to extract ad ID from URL '%s': %s", url, ex)
|
||||
@@ -135,41 +135,41 @@ class AdExtractor(WebScrapingMixin):
|
||||
:return: the links to your ad pages
|
||||
"""
|
||||
# navigate to "your ads" page
|
||||
await self.web_open('https://www.kleinanzeigen.de/m-meine-anzeigen.html')
|
||||
await self.web_open("https://www.kleinanzeigen.de/m-meine-anzeigen.html")
|
||||
await self.web_sleep(2000, 3000) # Consider replacing with explicit waits later
|
||||
|
||||
# Try to find the main ad list container first
|
||||
try:
|
||||
ad_list_container = await self.web_find(By.ID, 'my-manageitems-adlist')
|
||||
ad_list_container = await self.web_find(By.ID, "my-manageitems-adlist")
|
||||
except TimeoutError:
|
||||
LOG.warning('Ad list container #my-manageitems-adlist not found. Maybe no ads present?')
|
||||
LOG.warning("Ad list container #my-manageitems-adlist not found. Maybe no ads present?")
|
||||
return []
|
||||
|
||||
# --- Pagination handling ---
|
||||
multi_page = False
|
||||
try:
|
||||
# Correct selector: Use uppercase '.Pagination'
|
||||
pagination_section = await self.web_find(By.CSS_SELECTOR, '.Pagination', timeout=10) # Increased timeout slightly
|
||||
pagination_section = await self.web_find(By.CSS_SELECTOR, ".Pagination", timeout = 10) # Increased timeout slightly
|
||||
# Correct selector: Use 'aria-label'
|
||||
# Also check if the button is actually present AND potentially enabled (though enabled check isn't strictly necessary here, only for clicking later)
|
||||
next_buttons = await self.web_find_all(By.CSS_SELECTOR, 'button[aria-label="Nächste"]', parent = pagination_section)
|
||||
if next_buttons:
|
||||
# Check if at least one 'Nächste' button is not disabled (optional but good practice)
|
||||
enabled_next_buttons = [btn for btn in next_buttons if not btn.attrs.get('disabled')]
|
||||
enabled_next_buttons = [btn for btn in next_buttons if not btn.attrs.get("disabled")]
|
||||
if enabled_next_buttons:
|
||||
multi_page = True
|
||||
LOG.info('Multiple ad pages detected.')
|
||||
LOG.info("Multiple ad pages detected.")
|
||||
else:
|
||||
LOG.info('Next button found but is disabled. Assuming single effective page.')
|
||||
LOG.info("Next button found but is disabled. Assuming single effective page.")
|
||||
|
||||
else:
|
||||
LOG.info('No "Naechste" button found within pagination. Assuming single page.')
|
||||
except TimeoutError:
|
||||
# This will now correctly trigger only if the '.Pagination' div itself is not found
|
||||
LOG.info('No pagination controls found. Assuming single page.')
|
||||
LOG.info("No pagination controls found. Assuming single page.")
|
||||
except Exception as e:
|
||||
LOG.exception("Error during pagination detection: %s", e)
|
||||
LOG.info('Assuming single page due to error during pagination check.')
|
||||
LOG.info("Assuming single page due to error during pagination check.")
|
||||
# --- End Pagination Handling ---
|
||||
|
||||
refs:list[str] = []
|
||||
@@ -182,8 +182,8 @@ class AdExtractor(WebScrapingMixin):
|
||||
|
||||
# Re-find the ad list container on the current page/state
|
||||
try:
|
||||
ad_list_container = await self.web_find(By.ID, 'my-manageitems-adlist')
|
||||
list_items = await self.web_find_all(By.CLASS_NAME, 'cardbox', parent=ad_list_container)
|
||||
ad_list_container = await self.web_find(By.ID, "my-manageitems-adlist")
|
||||
list_items = await self.web_find_all(By.CLASS_NAME, "cardbox", parent = ad_list_container)
|
||||
LOG.info("Found %s ad items on page %s.", len(list_items), current_page)
|
||||
except TimeoutError:
|
||||
LOG.warning("Could not find ad list container or items on page %s.", current_page)
|
||||
@@ -192,7 +192,7 @@ class AdExtractor(WebScrapingMixin):
|
||||
# Extract references using the CORRECTED selector
|
||||
try:
|
||||
page_refs = [
|
||||
(await self.web_find(By.CSS_SELECTOR, 'div.manageitems-item-ad h3 a.text-onSurface', parent=li)).attrs['href']
|
||||
(await self.web_find(By.CSS_SELECTOR, "div.manageitems-item-ad h3 a.text-onSurface", parent = li)).attrs["href"]
|
||||
for li in list_items
|
||||
]
|
||||
refs.extend(page_refs)
|
||||
@@ -207,12 +207,12 @@ class AdExtractor(WebScrapingMixin):
|
||||
# --- Navigate to next page ---
|
||||
try:
|
||||
# Find the pagination section again (scope might have changed after scroll/wait)
|
||||
pagination_section = await self.web_find(By.CSS_SELECTOR, '.Pagination', timeout=5)
|
||||
pagination_section = await self.web_find(By.CSS_SELECTOR, ".Pagination", timeout = 5)
|
||||
# Find the "Next" button using the correct aria-label selector and ensure it's not disabled
|
||||
next_button_element = None
|
||||
possible_next_buttons = await self.web_find_all(By.CSS_SELECTOR, 'button[aria-label="Nächste"]', parent = pagination_section)
|
||||
for btn in possible_next_buttons:
|
||||
if not btn.attrs.get('disabled'): # Check if the button is enabled
|
||||
if not btn.attrs.get("disabled"): # Check if the button is enabled
|
||||
next_button_element = btn
|
||||
break # Found an enabled next button
|
||||
|
||||
@@ -235,7 +235,7 @@ class AdExtractor(WebScrapingMixin):
|
||||
# --- End Navigation ---
|
||||
|
||||
if not refs:
|
||||
LOG.warning('No ad URLs were extracted.')
|
||||
LOG.warning("No ad URLs were extracted.")
|
||||
|
||||
return refs
|
||||
|
||||
@@ -246,27 +246,27 @@ class AdExtractor(WebScrapingMixin):
|
||||
"""
|
||||
if reflect.is_integer(id_or_url):
|
||||
# navigate to start page, otherwise page can be None!
|
||||
await self.web_open('https://www.kleinanzeigen.de/')
|
||||
await self.web_open("https://www.kleinanzeigen.de/")
|
||||
# enter the ad ID into the search bar
|
||||
await self.web_input(By.ID, "site-search-query", id_or_url)
|
||||
# navigate to ad page and wait
|
||||
await self.web_check(By.ID, 'site-search-submit', Is.CLICKABLE)
|
||||
submit_button = await self.web_find(By.ID, 'site-search-submit')
|
||||
await self.web_check(By.ID, "site-search-submit", Is.CLICKABLE)
|
||||
submit_button = await self.web_find(By.ID, "site-search-submit")
|
||||
await submit_button.click()
|
||||
else:
|
||||
await self.web_open(str(id_or_url)) # navigate to URL directly given
|
||||
await self.web_sleep()
|
||||
|
||||
# handle the case that invalid ad ID given
|
||||
if self.page.url.endswith('k0'):
|
||||
LOG.error('There is no ad under the given ID.')
|
||||
if self.page.url.endswith("k0"):
|
||||
LOG.error("There is no ad under the given ID.")
|
||||
return False
|
||||
|
||||
# close (warning) popup, if given
|
||||
try:
|
||||
await self.web_find(By.ID, 'vap-ovrly-secure')
|
||||
LOG.warning('A popup appeared!')
|
||||
await self.web_click(By.CLASS_NAME, 'mfp-close')
|
||||
await self.web_find(By.ID, "vap-ovrly-secure")
|
||||
LOG.warning("A popup appeared!")
|
||||
await self.web_click(By.CLASS_NAME, "mfp-close")
|
||||
await self.web_sleep()
|
||||
except TimeoutError:
|
||||
pass
|
||||
@@ -280,18 +280,18 @@ class AdExtractor(WebScrapingMixin):
|
||||
:param ad_id: the ad ID, already extracted by a calling function
|
||||
:return: a dictionary with the keys as given in an ad YAML, and their respective values
|
||||
"""
|
||||
info:dict[str, Any] = {'active': True}
|
||||
info:dict[str, Any] = {"active": True}
|
||||
|
||||
# extract basic info
|
||||
info['type'] = 'OFFER' if 's-anzeige' in self.page.url else 'WANTED'
|
||||
title:str = await self.web_text(By.ID, 'viewad-title')
|
||||
info["type"] = "OFFER" if "s-anzeige" in self.page.url else "WANTED"
|
||||
title:str = await self.web_text(By.ID, "viewad-title")
|
||||
LOG.info('Extracting information from ad with title "%s"', title)
|
||||
|
||||
info['category'] = await self._extract_category_from_ad_page()
|
||||
info['title'] = title
|
||||
info["category"] = await self._extract_category_from_ad_page()
|
||||
info["title"] = title
|
||||
|
||||
# Get raw description text
|
||||
raw_description = (await self.web_text(By.ID, 'viewad-description-text')).strip()
|
||||
raw_description = (await self.web_text(By.ID, "viewad-description-text")).strip()
|
||||
|
||||
# Get prefix and suffix from config
|
||||
prefix = get_description_affixes(self.config, prefix = True)
|
||||
@@ -304,38 +304,38 @@ class AdExtractor(WebScrapingMixin):
|
||||
if suffix and description_text.endswith(suffix.strip()):
|
||||
description_text = description_text[:-len(suffix.strip())]
|
||||
|
||||
info['description'] = description_text.strip()
|
||||
info["description"] = description_text.strip()
|
||||
|
||||
info['special_attributes'] = await self._extract_special_attributes_from_ad_page()
|
||||
if "art_s" in info['special_attributes']:
|
||||
info["special_attributes"] = await self._extract_special_attributes_from_ad_page()
|
||||
if "art_s" in info["special_attributes"]:
|
||||
# change e.g. category "161/172" to "161/172/lautsprecher_kopfhoerer"
|
||||
info['category'] = f"{info['category']}/{info['special_attributes']['art_s']}"
|
||||
del info['special_attributes']['art_s']
|
||||
if "schaden_s" in info['special_attributes']:
|
||||
info["category"] = f"{info['category']}/{info['special_attributes']['art_s']}"
|
||||
del info["special_attributes"]["art_s"]
|
||||
if "schaden_s" in info["special_attributes"]:
|
||||
# change f to 'nein' and 't' to 'ja'
|
||||
info['special_attributes']['schaden_s'] = info['special_attributes']['schaden_s'].translate(str.maketrans({'t': 'ja', 'f': 'nein'}))
|
||||
info['price'], info['price_type'] = await self._extract_pricing_info_from_ad_page()
|
||||
info['shipping_type'], info['shipping_costs'], info['shipping_options'] = await self._extract_shipping_info_from_ad_page()
|
||||
info['sell_directly'] = await self._extract_sell_directly_from_ad_page()
|
||||
info['images'] = await self._download_images_from_ad_page(directory, ad_id)
|
||||
info['contact'] = await self._extract_contact_from_ad_page()
|
||||
info['id'] = ad_id
|
||||
info["special_attributes"]["schaden_s"] = info["special_attributes"]["schaden_s"].translate(str.maketrans({"t": "ja", "f": "nein"}))
|
||||
info["price"], info["price_type"] = await self._extract_pricing_info_from_ad_page()
|
||||
info["shipping_type"], info["shipping_costs"], info["shipping_options"] = await self._extract_shipping_info_from_ad_page()
|
||||
info["sell_directly"] = await self._extract_sell_directly_from_ad_page()
|
||||
info["images"] = await self._download_images_from_ad_page(directory, ad_id)
|
||||
info["contact"] = await self._extract_contact_from_ad_page()
|
||||
info["id"] = ad_id
|
||||
|
||||
try: # try different locations known for creation date element
|
||||
creation_date = await self.web_text(By.XPATH,
|
||||
'/html/body/div[1]/div[2]/div/section[2]/section/section/article/div[3]/div[2]/div[2]/div[1]/span')
|
||||
"/html/body/div[1]/div[2]/div/section[2]/section/section/article/div[3]/div[2]/div[2]/div[1]/span")
|
||||
except TimeoutError:
|
||||
creation_date = await self.web_text(By.CSS_SELECTOR, '#viewad-extra-info > div:nth-child(1) > span:nth-child(2)')
|
||||
creation_date = await self.web_text(By.CSS_SELECTOR, "#viewad-extra-info > div:nth-child(1) > span:nth-child(2)")
|
||||
|
||||
# convert creation date to ISO format
|
||||
created_parts = creation_date.split('.')
|
||||
creation_date = created_parts[2] + '-' + created_parts[1] + '-' + created_parts[0] + ' 00:00:00'
|
||||
created_parts = creation_date.split(".")
|
||||
creation_date = created_parts[2] + "-" + created_parts[1] + "-" + created_parts[0] + " 00:00:00"
|
||||
creation_date = datetime.fromisoformat(creation_date).isoformat()
|
||||
info['created_on'] = creation_date
|
||||
info['updated_on'] = None # will be set later on
|
||||
info["created_on"] = creation_date
|
||||
info["updated_on"] = None # will be set later on
|
||||
|
||||
# Calculate the initial hash for the downloaded ad
|
||||
info['content_hash'] = calculate_content_hash(info)
|
||||
info["content_hash"] = calculate_content_hash(info)
|
||||
|
||||
return info
|
||||
|
||||
@@ -346,12 +346,12 @@ class AdExtractor(WebScrapingMixin):
|
||||
|
||||
:return: a category string of form abc/def, where a-f are digits
|
||||
"""
|
||||
category_line = await self.web_find(By.ID, 'vap-brdcrmb')
|
||||
category_first_part = await self.web_find(By.CSS_SELECTOR, 'a:nth-of-type(2)', parent = category_line)
|
||||
category_second_part = await self.web_find(By.CSS_SELECTOR, 'a:nth-of-type(3)', parent = category_line)
|
||||
cat_num_first = category_first_part.attrs['href'].split('/')[-1][1:]
|
||||
cat_num_second = category_second_part.attrs['href'].split('/')[-1][1:]
|
||||
category:str = cat_num_first + '/' + cat_num_second
|
||||
category_line = await self.web_find(By.ID, "vap-brdcrmb")
|
||||
category_first_part = await self.web_find(By.CSS_SELECTOR, "a:nth-of-type(2)", parent = category_line)
|
||||
category_second_part = await self.web_find(By.CSS_SELECTOR, "a:nth-of-type(3)", parent = category_line)
|
||||
cat_num_first = category_first_part.attrs["href"].split("/")[-1][1:]
|
||||
cat_num_second = category_second_part.attrs["href"].split("/")[-1][1:]
|
||||
category:str = cat_num_first + "/" + cat_num_second
|
||||
|
||||
return category
|
||||
|
||||
@@ -368,7 +368,7 @@ class AdExtractor(WebScrapingMixin):
|
||||
special_attributes_str = belen_conf["universalAnalyticsOpts"]["dimensions"]["dimension108"]
|
||||
|
||||
special_attributes = dict(item.split(":") for item in special_attributes_str.split("|") if ":" in item)
|
||||
special_attributes = {k: v for k, v in special_attributes.items() if not k.endswith('.versand_s') and k != "versand_s"}
|
||||
special_attributes = {k: v for k, v in special_attributes.items() if not k.endswith(".versand_s") and k != "versand_s"}
|
||||
return special_attributes
|
||||
|
||||
async def _extract_pricing_info_from_ad_page(self) -> tuple[float | None, str]:
|
||||
@@ -378,24 +378,24 @@ class AdExtractor(WebScrapingMixin):
|
||||
:return: the price of the offer (optional); and the pricing type
|
||||
"""
|
||||
try:
|
||||
price_str:str = await self.web_text(By.ID, 'viewad-price')
|
||||
price_str:str = await self.web_text(By.ID, "viewad-price")
|
||||
price:int | None = None
|
||||
match price_str.split()[-1]:
|
||||
case '€':
|
||||
price_type = 'FIXED'
|
||||
case "€":
|
||||
price_type = "FIXED"
|
||||
# replace('.', '') is to remove the thousands separator before parsing as int
|
||||
price = int(price_str.replace('.', '').split()[0])
|
||||
case 'VB':
|
||||
price_type = 'NEGOTIABLE'
|
||||
price = int(price_str.replace(".", "").split()[0])
|
||||
case "VB":
|
||||
price_type = "NEGOTIABLE"
|
||||
if price_str != "VB": # can be either 'X € VB', or just 'VB'
|
||||
price = int(price_str.replace('.', '').split()[0])
|
||||
case 'verschenken':
|
||||
price_type = 'GIVE_AWAY'
|
||||
price = int(price_str.replace(".", "").split()[0])
|
||||
case "verschenken":
|
||||
price_type = "GIVE_AWAY"
|
||||
case _:
|
||||
price_type = 'NOT_APPLICABLE'
|
||||
price_type = "NOT_APPLICABLE"
|
||||
return price, price_type
|
||||
except TimeoutError: # no 'commercial' ad, has no pricing box etc.
|
||||
return None, 'NOT_APPLICABLE'
|
||||
return None, "NOT_APPLICABLE"
|
||||
|
||||
async def _extract_shipping_info_from_ad_page(self) -> tuple[str, float | None, list[str] | None]:
|
||||
"""
|
||||
@@ -403,17 +403,17 @@ class AdExtractor(WebScrapingMixin):
|
||||
|
||||
:return: the shipping type, and the shipping price (optional)
|
||||
"""
|
||||
ship_type, ship_costs, shipping_options = 'NOT_APPLICABLE', None, None
|
||||
ship_type, ship_costs, shipping_options = "NOT_APPLICABLE", None, None
|
||||
try:
|
||||
shipping_text = await self.web_text(By.CLASS_NAME, 'boxedarticle--details--shipping')
|
||||
shipping_text = await self.web_text(By.CLASS_NAME, "boxedarticle--details--shipping")
|
||||
# e.g. '+ Versand ab 5,49 €' OR 'Nur Abholung'
|
||||
if shipping_text == 'Nur Abholung':
|
||||
ship_type = 'PICKUP'
|
||||
elif shipping_text == 'Versand möglich':
|
||||
ship_type = 'SHIPPING'
|
||||
elif '€' in shipping_text:
|
||||
shipping_price_parts = shipping_text.split(' ')
|
||||
ship_type = 'SHIPPING'
|
||||
if shipping_text == "Nur Abholung":
|
||||
ship_type = "PICKUP"
|
||||
elif shipping_text == "Versand möglich":
|
||||
ship_type = "SHIPPING"
|
||||
elif "€" in shipping_text:
|
||||
shipping_price_parts = shipping_text.split(" ")
|
||||
ship_type = "SHIPPING"
|
||||
ship_costs = float(misc.parse_decimal(shipping_price_parts[-2]))
|
||||
|
||||
# reading shipping option from kleinanzeigen
|
||||
@@ -425,7 +425,7 @@ class AdExtractor(WebScrapingMixin):
|
||||
internal_shipping_opt = [x for x in shipping_costs if x["priceInEuroCent"] == ship_costs * 100]
|
||||
|
||||
if not internal_shipping_opt:
|
||||
return 'NOT_APPLICABLE', ship_costs, shipping_options
|
||||
return "NOT_APPLICABLE", ship_costs, shipping_options
|
||||
|
||||
# map to internal shipping identifiers used by kleinanzeigen-bot
|
||||
shipping_option_mapping = {
|
||||
@@ -440,13 +440,13 @@ class AdExtractor(WebScrapingMixin):
|
||||
"HERMES_004": "Hermes_L"
|
||||
}
|
||||
|
||||
shipping_option = shipping_option_mapping.get(internal_shipping_opt[0]['id'])
|
||||
shipping_option = shipping_option_mapping.get(internal_shipping_opt[0]["id"])
|
||||
if not shipping_option:
|
||||
return 'NOT_APPLICABLE', ship_costs, shipping_options
|
||||
return "NOT_APPLICABLE", ship_costs, shipping_options
|
||||
|
||||
shipping_options = [shipping_option]
|
||||
except TimeoutError: # no pricing box -> no shipping given
|
||||
ship_type = 'NOT_APPLICABLE'
|
||||
ship_type = "NOT_APPLICABLE"
|
||||
|
||||
return ship_type, ship_costs, shipping_options
|
||||
|
||||
@@ -457,7 +457,7 @@ class AdExtractor(WebScrapingMixin):
|
||||
:return: a boolean indicating whether the sell directly option is active (optional)
|
||||
"""
|
||||
try:
|
||||
buy_now_is_active:bool = 'Direkt kaufen' in (await self.web_text(By.ID, 'payment-buttons-sidebar'))
|
||||
buy_now_is_active:bool = "Direkt kaufen" in (await self.web_text(By.ID, "payment-buttons-sidebar"))
|
||||
return buy_now_is_active
|
||||
except TimeoutError:
|
||||
return None
|
||||
@@ -469,34 +469,34 @@ class AdExtractor(WebScrapingMixin):
|
||||
:return: a dictionary containing the address parts with their corresponding values
|
||||
"""
|
||||
contact:dict[str, (str | None)] = {}
|
||||
address_text = await self.web_text(By.ID, 'viewad-locality')
|
||||
address_text = await self.web_text(By.ID, "viewad-locality")
|
||||
# format: e.g. (Beispiel Allee 42,) 12345 Bundesland - Stadt
|
||||
try:
|
||||
street = (await self.web_text(By.ID, 'street-address'))[:-1] # trailing comma
|
||||
contact['street'] = street
|
||||
street = (await self.web_text(By.ID, "street-address"))[:-1] # trailing comma
|
||||
contact["street"] = street
|
||||
except TimeoutError:
|
||||
LOG.info('No street given in the contact.')
|
||||
LOG.info("No street given in the contact.")
|
||||
|
||||
(zipcode, location) = address_text.split(" ", 1)
|
||||
contact['zipcode'] = zipcode # e.g. 19372
|
||||
contact['location'] = location # e.g. Mecklenburg-Vorpommern - Steinbeck
|
||||
contact["zipcode"] = zipcode # e.g. 19372
|
||||
contact["location"] = location # e.g. Mecklenburg-Vorpommern - Steinbeck
|
||||
|
||||
contact_person_element:Element = await self.web_find(By.ID, 'viewad-contact')
|
||||
name_element = await self.web_find(By.CLASS_NAME, 'iconlist-text', parent = contact_person_element)
|
||||
contact_person_element:Element = await self.web_find(By.ID, "viewad-contact")
|
||||
name_element = await self.web_find(By.CLASS_NAME, "iconlist-text", parent = contact_person_element)
|
||||
try:
|
||||
name = await self.web_text(By.TAG_NAME, 'a', parent = name_element)
|
||||
name = await self.web_text(By.TAG_NAME, "a", parent = name_element)
|
||||
except TimeoutError: # edge case: name without link
|
||||
name = await self.web_text(By.TAG_NAME, 'span', parent = name_element)
|
||||
contact['name'] = name
|
||||
name = await self.web_text(By.TAG_NAME, "span", parent = name_element)
|
||||
contact["name"] = name
|
||||
|
||||
if 'street' not in contact:
|
||||
contact['street'] = None
|
||||
if "street" not in contact:
|
||||
contact["street"] = None
|
||||
try: # phone number is unusual for non-professional sellers today
|
||||
phone_element = await self.web_find(By.ID, 'viewad-contact-phone')
|
||||
phone_number = await self.web_text(By.TAG_NAME, 'a', parent = phone_element)
|
||||
contact['phone'] = ''.join(phone_number.replace('-', ' ').split(' ')).replace('+49(0)', '0')
|
||||
phone_element = await self.web_find(By.ID, "viewad-contact-phone")
|
||||
phone_number = await self.web_text(By.TAG_NAME, "a", parent = phone_element)
|
||||
contact["phone"] = "".join(phone_number.replace("-", " ").split(" ")).replace("+49(0)", "0")
|
||||
except TimeoutError:
|
||||
contact['phone'] = None # phone seems to be a deprecated feature (for non-professional users)
|
||||
contact["phone"] = None # phone seems to be a deprecated feature (for non-professional users)
|
||||
# also see 'https://themen.kleinanzeigen.de/hilfe/deine-anzeigen/Telefon/
|
||||
|
||||
return contact
|
||||
|
||||
@@ -96,7 +96,7 @@ def save_dict(filepath:str, content:dict[str, Any]) -> None:
|
||||
yaml.indent(mapping = 2, sequence = 4, offset = 2)
|
||||
yaml.representer.add_representer(str, # use YAML | block style for multi-line strings
|
||||
lambda dumper, data:
|
||||
dumper.represent_scalar('tag:yaml.org,2002:str', data, style = '|' if '\n' in data else None)
|
||||
dumper.represent_scalar("tag:yaml.org,2002:str", data, style = "|" if "\n" in data else None)
|
||||
)
|
||||
yaml.allow_duplicate_keys = False
|
||||
yaml.explicit_start = False
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
# SPDX-ArtifactOfProjectHomePage: https://github.com/Second-Hand-Friends/kleinanzeigen-bot/
|
||||
import sys, traceback # isort: skip
|
||||
from types import FrameType, TracebackType
|
||||
from typing import Any, Final
|
||||
from typing import Final
|
||||
|
||||
from . import loggers
|
||||
|
||||
|
||||
@@ -42,7 +42,7 @@ class Locale(NamedTuple):
|
||||
return f"{self.language}{region_part}{encoding_part}"
|
||||
|
||||
@staticmethod
|
||||
def of(locale_string: str) -> 'Locale':
|
||||
def of(locale_string:str) -> "Locale":
|
||||
"""
|
||||
>>> Locale.of("en_US.UTF-8")
|
||||
Locale(language='en', region='US', encoding='UTF-8')
|
||||
@@ -105,7 +105,7 @@ def translate(text:object, caller: inspect.FrameInfo | None) -> str:
|
||||
if not _TRANSLATIONS:
|
||||
return text
|
||||
|
||||
module_name = caller.frame.f_globals.get('__name__') # pylint: disable=redefined-outer-name
|
||||
module_name = caller.frame.f_globals.get("__name__") # pylint: disable=redefined-outer-name
|
||||
file_basename = os.path.splitext(os.path.basename(caller.filename))[0]
|
||||
if module_name and module_name.endswith(f".{file_basename}"):
|
||||
module_name = module_name[:-(len(file_basename) + 1)]
|
||||
@@ -124,9 +124,9 @@ gettext.gettext = lambda message: translate(_original_gettext(message), reflect.
|
||||
for module_name, module in sys.modules.items():
|
||||
if module is None or module_name in sys.builtin_module_names:
|
||||
continue
|
||||
if hasattr(module, '_') and module._ is _original_gettext:
|
||||
if hasattr(module, "_") and module._ is _original_gettext:
|
||||
module._ = gettext.gettext # type: ignore[attr-defined]
|
||||
if hasattr(module, 'gettext') and module.gettext is _original_gettext:
|
||||
if hasattr(module, "gettext") and module.gettext is _original_gettext:
|
||||
module.gettext = gettext.gettext # type: ignore[attr-defined]
|
||||
|
||||
|
||||
@@ -190,8 +190,8 @@ def pluralize(noun:str, count:int | Sized, *, prefix_with_count:bool = True) ->
|
||||
# English
|
||||
if len(noun) < 2: # noqa: PLR2004 Magic value used in comparison
|
||||
return f"{prefix}{noun}s"
|
||||
if noun.endswith(('s', 'sh', 'ch', 'x', 'z')):
|
||||
if noun.endswith(("s", "sh", "ch", "x", "z")):
|
||||
return f"{prefix}{noun}es"
|
||||
if noun.endswith('y') and noun[-2].lower() not in "aeiou":
|
||||
if noun.endswith("y") and noun[-2].lower() not in "aeiou":
|
||||
return f"{prefix}{noun[:-1]}ies"
|
||||
return f"{prefix}{noun}s"
|
||||
|
||||
@@ -10,7 +10,7 @@ from typing import Any, TypeVar
|
||||
from . import i18n
|
||||
|
||||
# https://mypy.readthedocs.io/en/stable/generics.html#generic-functions
|
||||
T = TypeVar('T')
|
||||
T = TypeVar("T")
|
||||
|
||||
|
||||
def ensure(condition:Any | bool | Callable[[], bool], error_message:str, timeout:float = 5, poll_requency:float = 0.5) -> None:
|
||||
@@ -152,18 +152,18 @@ def parse_duration(text:str) -> timedelta:
|
||||
>>> parse_duration("invalid input")
|
||||
datetime.timedelta(0)
|
||||
"""
|
||||
pattern = re.compile(r'(\d+)\s*([dhms])')
|
||||
pattern = re.compile(r"(\d+)\s*([dhms])")
|
||||
parts = pattern.findall(text.lower())
|
||||
kwargs:dict[str, int] = {}
|
||||
for value, unit in parts:
|
||||
if unit == 'd':
|
||||
kwargs['days'] = kwargs.get('days', 0) + int(value)
|
||||
elif unit == 'h':
|
||||
kwargs['hours'] = kwargs.get('hours', 0) + int(value)
|
||||
elif unit == 'm':
|
||||
kwargs['minutes'] = kwargs.get('minutes', 0) + int(value)
|
||||
elif unit == 's':
|
||||
kwargs['seconds'] = kwargs.get('seconds', 0) + int(value)
|
||||
if unit == "d":
|
||||
kwargs["days"] = kwargs.get("days", 0) + int(value)
|
||||
elif unit == "h":
|
||||
kwargs["hours"] = kwargs.get("hours", 0) + int(value)
|
||||
elif unit == "m":
|
||||
kwargs["minutes"] = kwargs.get("minutes", 0) + int(value)
|
||||
elif unit == "s":
|
||||
kwargs["seconds"] = kwargs.get("seconds", 0) + int(value)
|
||||
return timedelta(**kwargs)
|
||||
|
||||
|
||||
|
||||
@@ -165,7 +165,7 @@ class WebScrapingMixin:
|
||||
prefs_file = os.path.join(profile_dir, "Preferences")
|
||||
if not os.path.exists(prefs_file):
|
||||
LOG.info(" -> Setting chrome prefs [%s]...", prefs_file)
|
||||
with open(prefs_file, "w", encoding = 'UTF-8') as fd:
|
||||
with open(prefs_file, "w", encoding = "UTF-8") as fd:
|
||||
json.dump({
|
||||
"credentials_enable_service": False,
|
||||
"enable_do_not_track": True,
|
||||
@@ -234,16 +234,16 @@ class WebScrapingMixin:
|
||||
|
||||
case "Windows":
|
||||
browser_paths = [
|
||||
os.environ.get("PROGRAMFILES", "C:\\Program Files") + r'\Microsoft\Edge\Application\msedge.exe',
|
||||
os.environ.get("PROGRAMFILES(X86)", "C:\\Program Files (x86)") + r'\Microsoft\Edge\Application\msedge.exe',
|
||||
os.environ.get("PROGRAMFILES", "C:\\Program Files") + r"\Microsoft\Edge\Application\msedge.exe",
|
||||
os.environ.get("PROGRAMFILES(X86)", "C:\\Program Files (x86)") + r"\Microsoft\Edge\Application\msedge.exe",
|
||||
|
||||
os.environ["PROGRAMFILES"] + r'\Chromium\Application\chrome.exe',
|
||||
os.environ["PROGRAMFILES(X86)"] + r'\Chromium\Application\chrome.exe',
|
||||
os.environ["LOCALAPPDATA"] + r'\Chromium\Application\chrome.exe',
|
||||
os.environ["PROGRAMFILES"] + r"\Chromium\Application\chrome.exe",
|
||||
os.environ["PROGRAMFILES(X86)"] + r"\Chromium\Application\chrome.exe",
|
||||
os.environ["LOCALAPPDATA"] + r"\Chromium\Application\chrome.exe",
|
||||
|
||||
os.environ["PROGRAMFILES"] + r'\Chrome\Application\chrome.exe',
|
||||
os.environ["PROGRAMFILES(X86)"] + r'\Chrome\Application\chrome.exe',
|
||||
os.environ["LOCALAPPDATA"] + r'\Chrome\Application\chrome.exe',
|
||||
os.environ["PROGRAMFILES"] + r"\Chrome\Application\chrome.exe",
|
||||
os.environ["PROGRAMFILES(X86)"] + r"\Chrome\Application\chrome.exe",
|
||||
os.environ["LOCALAPPDATA"] + r"\Chrome\Application\chrome.exe",
|
||||
|
||||
shutil.which("msedge.exe"),
|
||||
shutil.which("chromium.exe"),
|
||||
@@ -532,7 +532,7 @@ class WebScrapingMixin:
|
||||
:param scroll_back_top: whether to scroll the page back to the top after scrolling to the bottom
|
||||
"""
|
||||
current_y_pos = 0
|
||||
bottom_y_pos: int = await self.web_execute('document.body.scrollHeight') # get bottom position
|
||||
bottom_y_pos:int = await self.web_execute("document.body.scrollHeight") # get bottom position
|
||||
while current_y_pos < bottom_y_pos: # scroll in steps until bottom reached
|
||||
current_y_pos += scroll_length
|
||||
await self.web_execute(f'window.scrollTo(0, {current_y_pos})') # scroll one step
|
||||
|
||||
@@ -1,8 +1,6 @@
|
||||
"""
|
||||
SPDX-FileCopyrightText: © Jens Bergmann and contributors
|
||||
SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
SPDX-ArtifactOfProjectHomePage: https://github.com/Second-Hand-Friends/kleinanzeigen-bot/
|
||||
"""
|
||||
# SPDX-FileCopyrightText: © Jens Bergmann and contributors
|
||||
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
# SPDX-ArtifactOfProjectHomePage: https://github.com/Second-Hand-Friends/kleinanzeigen-bot/
|
||||
import os
|
||||
from typing import Any, Final
|
||||
from unittest.mock import MagicMock
|
||||
@@ -41,27 +39,27 @@ def sample_config() -> dict[str, Any]:
|
||||
- Publishing settings
|
||||
"""
|
||||
return {
|
||||
'login': {
|
||||
'username': 'testuser',
|
||||
'password': 'testpass'
|
||||
"login": {
|
||||
"username": "testuser",
|
||||
"password": "testpass"
|
||||
},
|
||||
'browser': {
|
||||
'arguments': [],
|
||||
'binary_location': None,
|
||||
'extensions': [],
|
||||
'use_private_window': True,
|
||||
'user_data_dir': None,
|
||||
'profile_name': None
|
||||
"browser": {
|
||||
"arguments": [],
|
||||
"binary_location": None,
|
||||
"extensions": [],
|
||||
"use_private_window": True,
|
||||
"user_data_dir": None,
|
||||
"profile_name": None
|
||||
},
|
||||
'ad_defaults': {
|
||||
'description': {
|
||||
'prefix': 'Test Prefix',
|
||||
'suffix': 'Test Suffix'
|
||||
"ad_defaults": {
|
||||
"description": {
|
||||
"prefix": "Test Prefix",
|
||||
"suffix": "Test Suffix"
|
||||
}
|
||||
},
|
||||
'publishing': {
|
||||
'delete_old_ads': 'BEFORE_PUBLISH',
|
||||
'delete_old_ads_by_title': False
|
||||
"publishing": {
|
||||
"delete_old_ads": "BEFORE_PUBLISH",
|
||||
"delete_old_ads_by_title": False
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -69,7 +69,7 @@ class TestAdExtractorPricing:
|
||||
self, test_extractor:AdExtractor, price_text:str, expected_price:int | None, expected_type:str
|
||||
) -> None:
|
||||
"""Test price extraction with different formats"""
|
||||
with patch.object(test_extractor, 'web_text', new_callable = AsyncMock, return_value = price_text):
|
||||
with patch.object(test_extractor, "web_text", new_callable = AsyncMock, return_value = price_text):
|
||||
price, price_type = await test_extractor._extract_pricing_info_from_ad_page()
|
||||
assert price == expected_price
|
||||
assert price_type == expected_type
|
||||
@@ -78,7 +78,7 @@ class TestAdExtractorPricing:
|
||||
# pylint: disable=protected-access
|
||||
async def test_extract_pricing_info_timeout(self, test_extractor:AdExtractor) -> None:
|
||||
"""Test price extraction when element is not found"""
|
||||
with patch.object(test_extractor, 'web_text', new_callable = AsyncMock, side_effect = TimeoutError):
|
||||
with patch.object(test_extractor, "web_text", new_callable = AsyncMock, side_effect = TimeoutError):
|
||||
price, price_type = await test_extractor._extract_pricing_info_from_ad_page()
|
||||
assert price is None
|
||||
assert price_type == "NOT_APPLICABLE"
|
||||
@@ -98,9 +98,9 @@ class TestAdExtractorShipping:
|
||||
self, test_extractor:AdExtractor, shipping_text:str, expected_type:str, expected_cost:float | None
|
||||
) -> None:
|
||||
"""Test shipping info extraction with different text formats."""
|
||||
with patch.object(test_extractor, 'page', MagicMock()), \
|
||||
patch.object(test_extractor, 'web_text', new_callable = AsyncMock, return_value = shipping_text), \
|
||||
patch.object(test_extractor, 'web_request', new_callable = AsyncMock) as mock_web_request:
|
||||
with patch.object(test_extractor, "page", MagicMock()), \
|
||||
patch.object(test_extractor, "web_text", new_callable = AsyncMock, return_value = shipping_text), \
|
||||
patch.object(test_extractor, "web_request", new_callable = AsyncMock) as mock_web_request:
|
||||
|
||||
if expected_cost:
|
||||
shipping_response:dict[str, Any] = {
|
||||
@@ -139,9 +139,9 @@ class TestAdExtractorShipping:
|
||||
})
|
||||
}
|
||||
|
||||
with patch.object(test_extractor, 'page', MagicMock()), \
|
||||
patch.object(test_extractor, 'web_text', new_callable = AsyncMock, return_value = "+ Versand ab 5,49 €"), \
|
||||
patch.object(test_extractor, 'web_request', new_callable = AsyncMock, return_value = shipping_response):
|
||||
with patch.object(test_extractor, "page", MagicMock()), \
|
||||
patch.object(test_extractor, "web_text", new_callable = AsyncMock, return_value = "+ Versand ab 5,49 €"), \
|
||||
patch.object(test_extractor, "web_request", new_callable = AsyncMock, return_value = shipping_response):
|
||||
|
||||
shipping_type, costs, options = await test_extractor._extract_shipping_info_from_ad_page()
|
||||
|
||||
@@ -159,9 +159,9 @@ class TestAdExtractorNavigation:
|
||||
page_mock = AsyncMock()
|
||||
page_mock.url = "https://www.kleinanzeigen.de/s-anzeige/test/12345"
|
||||
|
||||
with patch.object(test_extractor, 'page', page_mock), \
|
||||
patch.object(test_extractor, 'web_open', new_callable = AsyncMock) as mock_web_open, \
|
||||
patch.object(test_extractor, 'web_find', new_callable = AsyncMock, side_effect = TimeoutError):
|
||||
with patch.object(test_extractor, "page", page_mock), \
|
||||
patch.object(test_extractor, "web_open", new_callable = AsyncMock) as mock_web_open, \
|
||||
patch.object(test_extractor, "web_find", new_callable = AsyncMock, side_effect = TimeoutError):
|
||||
|
||||
result = await test_extractor.naviagte_to_ad_page("https://www.kleinanzeigen.de/s-anzeige/test/12345")
|
||||
assert result is True
|
||||
@@ -195,15 +195,15 @@ class TestAdExtractorNavigation:
|
||||
return popup_close_mock
|
||||
return None
|
||||
|
||||
with patch.object(test_extractor, 'page', page_mock), \
|
||||
patch.object(test_extractor, 'web_open', new_callable = AsyncMock) as mock_web_open, \
|
||||
patch.object(test_extractor, 'web_input', new_callable = AsyncMock), \
|
||||
patch.object(test_extractor, 'web_check', new_callable = AsyncMock, return_value = True), \
|
||||
patch.object(test_extractor, 'web_find', new_callable = AsyncMock, side_effect = find_mock):
|
||||
with patch.object(test_extractor, "page", page_mock), \
|
||||
patch.object(test_extractor, "web_open", new_callable = AsyncMock) as mock_web_open, \
|
||||
patch.object(test_extractor, "web_input", new_callable = AsyncMock), \
|
||||
patch.object(test_extractor, "web_check", new_callable = AsyncMock, return_value = True), \
|
||||
patch.object(test_extractor, "web_find", new_callable = AsyncMock, side_effect = find_mock):
|
||||
|
||||
result = await test_extractor.naviagte_to_ad_page(12345)
|
||||
assert result is True
|
||||
mock_web_open.assert_called_with('https://www.kleinanzeigen.de/')
|
||||
mock_web_open.assert_called_with("https://www.kleinanzeigen.de/")
|
||||
submit_button_mock.click.assert_awaited_once()
|
||||
popup_close_mock.click.assert_awaited_once()
|
||||
|
||||
@@ -218,15 +218,15 @@ class TestAdExtractorNavigation:
|
||||
input_mock.send_keys = AsyncMock()
|
||||
input_mock.apply = AsyncMock(return_value = True)
|
||||
|
||||
with patch.object(test_extractor, 'page', page_mock), \
|
||||
patch.object(test_extractor, 'web_open', new_callable = AsyncMock), \
|
||||
patch.object(test_extractor, 'web_find', new_callable = AsyncMock, return_value = input_mock), \
|
||||
patch.object(test_extractor, 'web_click', new_callable = AsyncMock) as mock_web_click, \
|
||||
patch.object(test_extractor, 'web_check', new_callable = AsyncMock, return_value = True):
|
||||
with patch.object(test_extractor, "page", page_mock), \
|
||||
patch.object(test_extractor, "web_open", new_callable = AsyncMock), \
|
||||
patch.object(test_extractor, "web_find", new_callable = AsyncMock, return_value = input_mock), \
|
||||
patch.object(test_extractor, "web_click", new_callable = AsyncMock) as mock_web_click, \
|
||||
patch.object(test_extractor, "web_check", new_callable = AsyncMock, return_value = True):
|
||||
|
||||
result = await test_extractor.naviagte_to_ad_page(12345)
|
||||
assert result is True
|
||||
mock_web_click.assert_called_with(By.CLASS_NAME, 'mfp-close')
|
||||
mock_web_click.assert_called_with(By.CLASS_NAME, "mfp-close")
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_navigate_to_ad_page_invalid_id(self, test_extractor:AdExtractor) -> None:
|
||||
@@ -240,9 +240,9 @@ class TestAdExtractorNavigation:
|
||||
input_mock.apply = AsyncMock(return_value = True)
|
||||
input_mock.attrs = {}
|
||||
|
||||
with patch.object(test_extractor, 'page', page_mock), \
|
||||
patch.object(test_extractor, 'web_open', new_callable = AsyncMock), \
|
||||
patch.object(test_extractor, 'web_find', new_callable = AsyncMock, return_value = input_mock):
|
||||
with patch.object(test_extractor, "page", page_mock), \
|
||||
patch.object(test_extractor, "web_open", new_callable = AsyncMock), \
|
||||
patch.object(test_extractor, "web_find", new_callable = AsyncMock, return_value = input_mock):
|
||||
|
||||
result = await test_extractor.naviagte_to_ad_page(99999)
|
||||
assert result is False
|
||||
@@ -250,12 +250,12 @@ class TestAdExtractorNavigation:
|
||||
@pytest.mark.asyncio
|
||||
async def test_extract_own_ads_urls(self, test_extractor:AdExtractor) -> None:
|
||||
"""Test extraction of own ads URLs - basic test."""
|
||||
with patch.object(test_extractor, 'web_open', new_callable=AsyncMock), \
|
||||
patch.object(test_extractor, 'web_sleep', new_callable=AsyncMock), \
|
||||
patch.object(test_extractor, 'web_find', new_callable=AsyncMock) as mock_web_find, \
|
||||
patch.object(test_extractor, 'web_find_all', new_callable=AsyncMock) as mock_web_find_all, \
|
||||
patch.object(test_extractor, 'web_scroll_page_down', new_callable=AsyncMock), \
|
||||
patch.object(test_extractor, 'web_execute', new_callable=AsyncMock):
|
||||
with patch.object(test_extractor, "web_open", new_callable = AsyncMock), \
|
||||
patch.object(test_extractor, "web_sleep", new_callable = AsyncMock), \
|
||||
patch.object(test_extractor, "web_find", new_callable = AsyncMock) as mock_web_find, \
|
||||
patch.object(test_extractor, "web_find_all", new_callable = AsyncMock) as mock_web_find_all, \
|
||||
patch.object(test_extractor, "web_scroll_page_down", new_callable = AsyncMock), \
|
||||
patch.object(test_extractor, "web_execute", new_callable = AsyncMock):
|
||||
|
||||
# --- Setup mock objects for DOM elements ---
|
||||
# Mocks needed for the actual execution flow
|
||||
@@ -263,7 +263,7 @@ class TestAdExtractorNavigation:
|
||||
pagination_section_mock = MagicMock()
|
||||
cardbox_mock = MagicMock() # Represents the <li> element
|
||||
link_mock = MagicMock() # Represents the <a> element
|
||||
link_mock.attrs = {'href': '/s-anzeige/test/12345'} # Configure the desired output
|
||||
link_mock.attrs = {"href": "/s-anzeige/test/12345"} # Configure the desired output
|
||||
|
||||
# Mocks for elements potentially checked but maybe not strictly needed for output
|
||||
# (depending on how robust the mocking is)
|
||||
@@ -295,19 +295,19 @@ class TestAdExtractorNavigation:
|
||||
refs = await test_extractor.extract_own_ads_urls()
|
||||
|
||||
# --- Assertions ---
|
||||
assert refs == ['/s-anzeige/test/12345'] # Now it should match
|
||||
assert refs == ["/s-anzeige/test/12345"] # Now it should match
|
||||
|
||||
# Optional: Verify calls were made as expected
|
||||
mock_web_find.assert_has_calls([
|
||||
call(By.ID, 'my-manageitems-adlist'),
|
||||
call(By.CSS_SELECTOR, '.Pagination', timeout=10),
|
||||
call(By.ID, 'my-manageitems-adlist'),
|
||||
call(By.CSS_SELECTOR, 'div.manageitems-item-ad h3 a.text-onSurface', parent=cardbox_mock),
|
||||
call(By.ID, "my-manageitems-adlist"),
|
||||
call(By.CSS_SELECTOR, ".Pagination", timeout = 10),
|
||||
call(By.ID, "my-manageitems-adlist"),
|
||||
call(By.CSS_SELECTOR, "div.manageitems-item-ad h3 a.text-onSurface", parent = cardbox_mock),
|
||||
], any_order = False) # Check order if important
|
||||
|
||||
mock_web_find_all.assert_has_calls([
|
||||
call(By.CSS_SELECTOR, 'button[aria-label="Nächste"]', parent = pagination_section_mock),
|
||||
call(By.CLASS_NAME, 'cardbox', parent=ad_list_container_mock),
|
||||
call(By.CLASS_NAME, "cardbox", parent = ad_list_container_mock),
|
||||
], any_order = False)
|
||||
|
||||
|
||||
@@ -424,11 +424,11 @@ class TestAdExtractorContent:
|
||||
]
|
||||
|
||||
for text, expected in test_cases:
|
||||
with patch.object(test_extractor, 'web_text', new_callable=AsyncMock, return_value=text):
|
||||
with patch.object(test_extractor, "web_text", new_callable = AsyncMock, return_value = text):
|
||||
result = await test_extractor._extract_sell_directly_from_ad_page()
|
||||
assert result is expected
|
||||
|
||||
with patch.object(test_extractor, 'web_text', new_callable=AsyncMock, side_effect=TimeoutError):
|
||||
with patch.object(test_extractor, "web_text", new_callable = AsyncMock, side_effect = TimeoutError):
|
||||
result = await test_extractor._extract_sell_directly_from_ad_page()
|
||||
assert result is None
|
||||
|
||||
@@ -455,11 +455,11 @@ class TestAdExtractorCategory:
|
||||
"""Test category extraction from breadcrumb."""
|
||||
category_line = MagicMock()
|
||||
first_part = MagicMock()
|
||||
first_part.attrs = {'href': '/s-familie-kind-baby/c17'}
|
||||
first_part.attrs = {"href": "/s-familie-kind-baby/c17"}
|
||||
second_part = MagicMock()
|
||||
second_part.attrs = {'href': '/s-spielzeug/c23'}
|
||||
second_part.attrs = {"href": "/s-spielzeug/c23"}
|
||||
|
||||
with patch.object(extractor, 'web_find', new_callable = AsyncMock) as mock_web_find:
|
||||
with patch.object(extractor, "web_find", new_callable = AsyncMock) as mock_web_find:
|
||||
mock_web_find.side_effect = [
|
||||
category_line,
|
||||
first_part,
|
||||
@@ -469,15 +469,15 @@ class TestAdExtractorCategory:
|
||||
result = await extractor._extract_category_from_ad_page()
|
||||
assert result == "17/23"
|
||||
|
||||
mock_web_find.assert_any_call(By.ID, 'vap-brdcrmb')
|
||||
mock_web_find.assert_any_call(By.CSS_SELECTOR, 'a:nth-of-type(2)', parent = category_line)
|
||||
mock_web_find.assert_any_call(By.CSS_SELECTOR, 'a:nth-of-type(3)', parent = category_line)
|
||||
mock_web_find.assert_any_call(By.ID, "vap-brdcrmb")
|
||||
mock_web_find.assert_any_call(By.CSS_SELECTOR, "a:nth-of-type(2)", parent = category_line)
|
||||
mock_web_find.assert_any_call(By.CSS_SELECTOR, "a:nth-of-type(3)", parent = category_line)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
# pylint: disable=protected-access
|
||||
async def test_extract_special_attributes_empty(self, extractor:AdExtractor) -> None:
|
||||
"""Test extraction of special attributes when empty."""
|
||||
with patch.object(extractor, 'web_execute', new_callable = AsyncMock) as mock_web_execute:
|
||||
with patch.object(extractor, "web_execute", new_callable = AsyncMock) as mock_web_execute:
|
||||
mock_web_execute.return_value = {
|
||||
"universalAnalyticsOpts": {
|
||||
"dimensions": {
|
||||
@@ -509,9 +509,9 @@ class TestAdExtractorContact:
|
||||
# pylint: disable=protected-access
|
||||
async def test_extract_contact_info(self, extractor:AdExtractor) -> None:
|
||||
"""Test extraction of contact information."""
|
||||
with patch.object(extractor, 'page', MagicMock()), \
|
||||
patch.object(extractor, 'web_text', new_callable = AsyncMock) as mock_web_text, \
|
||||
patch.object(extractor, 'web_find', new_callable = AsyncMock) as mock_web_find:
|
||||
with patch.object(extractor, "page", MagicMock()), \
|
||||
patch.object(extractor, "web_text", new_callable = AsyncMock) as mock_web_text, \
|
||||
patch.object(extractor, "web_find", new_callable = AsyncMock) as mock_web_find:
|
||||
|
||||
mock_web_text.side_effect = [
|
||||
"12345 Berlin - Mitte",
|
||||
@@ -537,9 +537,9 @@ class TestAdExtractorContact:
|
||||
# pylint: disable=protected-access
|
||||
async def test_extract_contact_info_timeout(self, extractor:AdExtractor) -> None:
|
||||
"""Test contact info extraction when elements are not found."""
|
||||
with patch.object(extractor, 'page', MagicMock()), \
|
||||
patch.object(extractor, 'web_text', new_callable = AsyncMock, side_effect = TimeoutError()), \
|
||||
patch.object(extractor, 'web_find', new_callable = AsyncMock, side_effect = TimeoutError()), \
|
||||
with patch.object(extractor, "page", MagicMock()), \
|
||||
patch.object(extractor, "web_text", new_callable = AsyncMock, side_effect = TimeoutError()), \
|
||||
patch.object(extractor, "web_find", new_callable = AsyncMock, side_effect = TimeoutError()), \
|
||||
pytest.raises(TimeoutError):
|
||||
|
||||
await extractor._extract_contact_from_ad_page()
|
||||
@@ -548,9 +548,9 @@ class TestAdExtractorContact:
|
||||
# pylint: disable=protected-access
|
||||
async def test_extract_contact_info_with_phone(self, extractor:AdExtractor) -> None:
|
||||
"""Test extraction of contact information including phone number."""
|
||||
with patch.object(extractor, 'page', MagicMock()), \
|
||||
patch.object(extractor, 'web_text', new_callable = AsyncMock) as mock_web_text, \
|
||||
patch.object(extractor, 'web_find', new_callable = AsyncMock) as mock_web_find:
|
||||
with patch.object(extractor, "page", MagicMock()), \
|
||||
patch.object(extractor, "web_text", new_callable = AsyncMock) as mock_web_text, \
|
||||
patch.object(extractor, "web_find", new_callable = AsyncMock) as mock_web_find:
|
||||
|
||||
mock_web_text.side_effect = [
|
||||
"12345 Berlin - Mitte",
|
||||
@@ -590,17 +590,17 @@ class TestAdExtractorDownload:
|
||||
@pytest.mark.asyncio
|
||||
async def test_download_ad_existing_directory(self, extractor:AdExtractor) -> None:
|
||||
"""Test downloading an ad when the directory already exists."""
|
||||
with patch('os.path.exists') as mock_exists, \
|
||||
patch('os.path.isdir') as mock_isdir, \
|
||||
patch('os.makedirs') as mock_makedirs, \
|
||||
patch('os.mkdir') as mock_mkdir, \
|
||||
patch('shutil.rmtree') as mock_rmtree, \
|
||||
patch('kleinanzeigen_bot.extract.dicts.save_dict', autospec = True) as mock_save_dict, \
|
||||
patch.object(extractor, '_extract_ad_page_info', new_callable = AsyncMock) as mock_extract:
|
||||
with patch("os.path.exists") as mock_exists, \
|
||||
patch("os.path.isdir") as mock_isdir, \
|
||||
patch("os.makedirs") as mock_makedirs, \
|
||||
patch("os.mkdir") as mock_mkdir, \
|
||||
patch("shutil.rmtree") as mock_rmtree, \
|
||||
patch("kleinanzeigen_bot.extract.dicts.save_dict", autospec = True) as mock_save_dict, \
|
||||
patch.object(extractor, "_extract_ad_page_info", new_callable = AsyncMock) as mock_extract:
|
||||
|
||||
base_dir = 'downloaded-ads'
|
||||
ad_dir = os.path.join(base_dir, 'ad_12345')
|
||||
yaml_path = os.path.join(ad_dir, 'ad_12345.yaml')
|
||||
base_dir = "downloaded-ads"
|
||||
ad_dir = os.path.join(base_dir, "ad_12345")
|
||||
yaml_path = os.path.join(ad_dir, "ad_12345.yaml")
|
||||
|
||||
# Configure mocks for directory checks
|
||||
existing_paths = {base_dir, ad_dir}
|
||||
@@ -632,7 +632,7 @@ class TestAdExtractorDownload:
|
||||
# Workaround for hard-coded path in download_ad
|
||||
actual_call = mock_save_dict.call_args
|
||||
assert actual_call is not None
|
||||
actual_path = actual_call[0][0].replace('/', os.path.sep)
|
||||
actual_path = actual_call[0][0].replace("/", os.path.sep)
|
||||
assert actual_path == yaml_path
|
||||
assert actual_call[0][1] == mock_extract.return_value
|
||||
|
||||
@@ -640,24 +640,24 @@ class TestAdExtractorDownload:
|
||||
# pylint: disable=protected-access
|
||||
async def test_download_images_no_images(self, extractor:AdExtractor) -> None:
|
||||
"""Test image download when no images are found."""
|
||||
with patch.object(extractor, 'web_find', new_callable = AsyncMock, side_effect = TimeoutError):
|
||||
with patch.object(extractor, "web_find", new_callable = AsyncMock, side_effect = TimeoutError):
|
||||
image_paths = await extractor._download_images_from_ad_page("/some/dir", 12345)
|
||||
assert len(image_paths) == 0
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_download_ad(self, extractor:AdExtractor) -> None:
|
||||
"""Test downloading an entire ad."""
|
||||
with patch('os.path.exists') as mock_exists, \
|
||||
patch('os.path.isdir') as mock_isdir, \
|
||||
patch('os.makedirs') as mock_makedirs, \
|
||||
patch('os.mkdir') as mock_mkdir, \
|
||||
patch('shutil.rmtree') as mock_rmtree, \
|
||||
patch('kleinanzeigen_bot.extract.dicts.save_dict', autospec = True) as mock_save_dict, \
|
||||
patch.object(extractor, '_extract_ad_page_info', new_callable = AsyncMock) as mock_extract:
|
||||
with patch("os.path.exists") as mock_exists, \
|
||||
patch("os.path.isdir") as mock_isdir, \
|
||||
patch("os.makedirs") as mock_makedirs, \
|
||||
patch("os.mkdir") as mock_mkdir, \
|
||||
patch("shutil.rmtree") as mock_rmtree, \
|
||||
patch("kleinanzeigen_bot.extract.dicts.save_dict", autospec = True) as mock_save_dict, \
|
||||
patch.object(extractor, "_extract_ad_page_info", new_callable = AsyncMock) as mock_extract:
|
||||
|
||||
base_dir = 'downloaded-ads'
|
||||
ad_dir = os.path.join(base_dir, 'ad_12345')
|
||||
yaml_path = os.path.join(ad_dir, 'ad_12345.yaml')
|
||||
base_dir = "downloaded-ads"
|
||||
ad_dir = os.path.join(base_dir, "ad_12345")
|
||||
yaml_path = os.path.join(ad_dir, "ad_12345.yaml")
|
||||
|
||||
# Configure mocks for directory checks
|
||||
mock_exists.return_value = False
|
||||
@@ -690,6 +690,6 @@ class TestAdExtractorDownload:
|
||||
# Get the actual call arguments
|
||||
actual_call = mock_save_dict.call_args
|
||||
assert actual_call is not None
|
||||
actual_path = actual_call[0][0].replace('/', os.path.sep)
|
||||
actual_path = actual_call[0][0].replace("/", os.path.sep)
|
||||
assert actual_path == yaml_path
|
||||
assert actual_call[0][1] == mock_extract.return_value
|
||||
|
||||
@@ -135,10 +135,10 @@ def minimal_ad_config(base_ad_config: dict[str, Any]) -> dict[str, Any]:
|
||||
def mock_config_setup(test_bot:KleinanzeigenBot) -> Generator[None]:
|
||||
"""Provide a centralized mock configuration setup for tests.
|
||||
This fixture mocks load_config and other essential configuration-related methods."""
|
||||
with patch.object(test_bot, 'load_config'), \
|
||||
patch.object(test_bot, 'create_browser_session', new_callable = AsyncMock), \
|
||||
patch.object(test_bot, 'login', new_callable = AsyncMock), \
|
||||
patch.object(test_bot, 'web_request', new_callable = AsyncMock) as mock_request:
|
||||
with patch.object(test_bot, "load_config"), \
|
||||
patch.object(test_bot, "create_browser_session", new_callable = AsyncMock), \
|
||||
patch.object(test_bot, "login", new_callable = AsyncMock), \
|
||||
patch.object(test_bot, "web_request", new_callable = AsyncMock) as mock_request:
|
||||
# Mock the web request for published ads
|
||||
mock_request.return_value = {"content": '{"ads": []}'}
|
||||
yield
|
||||
@@ -159,8 +159,8 @@ class TestKleinanzeigenBotInitialization:
|
||||
|
||||
def test_get_version_returns_correct_version(self, test_bot:KleinanzeigenBot) -> None:
|
||||
"""Verify version retrieval works correctly."""
|
||||
with patch('kleinanzeigen_bot.__version__', '1.2.3'):
|
||||
assert test_bot.get_version() == '1.2.3'
|
||||
with patch("kleinanzeigen_bot.__version__", "1.2.3"):
|
||||
assert test_bot.get_version() == "1.2.3"
|
||||
|
||||
|
||||
class TestKleinanzeigenBotLogging:
|
||||
@@ -257,15 +257,15 @@ class TestKleinanzeigenBotConfiguration:
|
||||
sample_config_with_categories = sample_config.copy()
|
||||
sample_config_with_categories["categories"] = {}
|
||||
|
||||
with patch('kleinanzeigen_bot.utils.dicts.load_dict_if_exists', return_value = None), \
|
||||
patch.object(LOG, 'warning') as mock_warning, \
|
||||
patch('kleinanzeigen_bot.utils.dicts.save_dict') as mock_save, \
|
||||
patch('kleinanzeigen_bot.utils.dicts.load_dict_from_module') as mock_load_module:
|
||||
with patch("kleinanzeigen_bot.utils.dicts.load_dict_if_exists", return_value = None), \
|
||||
patch.object(LOG, "warning") as mock_warning, \
|
||||
patch("kleinanzeigen_bot.utils.dicts.save_dict") as mock_save, \
|
||||
patch("kleinanzeigen_bot.utils.dicts.load_dict_from_module") as mock_load_module:
|
||||
|
||||
mock_load_module.side_effect = [
|
||||
sample_config_with_categories, # config_defaults.yaml
|
||||
{'cat1': 'id1'}, # categories.yaml
|
||||
{'cat2': 'id2'} # categories_old.yaml
|
||||
{"cat1": "id1"}, # categories.yaml
|
||||
{"cat2": "id2"} # categories_old.yaml
|
||||
]
|
||||
|
||||
test_bot.load_config()
|
||||
@@ -273,7 +273,7 @@ class TestKleinanzeigenBotConfiguration:
|
||||
mock_save.assert_called_once_with(str(config_path), sample_config_with_categories)
|
||||
|
||||
# Verify categories were loaded
|
||||
assert test_bot.categories == {'cat1': 'id1', 'cat2': 'id2'}
|
||||
assert test_bot.categories == {"cat1": "id1", "cat2": "id2"}
|
||||
assert test_bot.config == sample_config_with_categories
|
||||
|
||||
def test_load_config_validates_required_fields(self, test_bot:KleinanzeigenBot, test_data_dir:str) -> None:
|
||||
@@ -307,13 +307,13 @@ class TestKleinanzeigenBotAuthentication:
|
||||
@pytest.mark.asyncio
|
||||
async def test_assert_free_ad_limit_not_reached_success(self, configured_bot:KleinanzeigenBot) -> None:
|
||||
"""Verify that free ad limit check succeeds when limit not reached."""
|
||||
with patch.object(configured_bot, 'web_find', side_effect = TimeoutError):
|
||||
with patch.object(configured_bot, "web_find", side_effect = TimeoutError):
|
||||
await configured_bot.assert_free_ad_limit_not_reached()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_assert_free_ad_limit_not_reached_limit_reached(self, configured_bot:KleinanzeigenBot) -> None:
|
||||
"""Verify that free ad limit check fails when limit is reached."""
|
||||
with patch.object(configured_bot, 'web_find', return_value = AsyncMock()):
|
||||
with patch.object(configured_bot, "web_find", return_value = AsyncMock()):
|
||||
with pytest.raises(AssertionError) as exc_info:
|
||||
await configured_bot.assert_free_ad_limit_not_reached()
|
||||
assert "Cannot publish more ads" in str(exc_info.value)
|
||||
@@ -321,23 +321,23 @@ class TestKleinanzeigenBotAuthentication:
|
||||
@pytest.mark.asyncio
|
||||
async def test_is_logged_in_returns_true_when_logged_in(self, configured_bot:KleinanzeigenBot) -> None:
|
||||
"""Verify that login check returns true when logged in."""
|
||||
with patch.object(configured_bot, 'web_text', return_value = 'Welcome testuser'):
|
||||
with patch.object(configured_bot, "web_text", return_value = "Welcome testuser"):
|
||||
assert await configured_bot.is_logged_in() is True
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_is_logged_in_returns_false_when_not_logged_in(self, configured_bot:KleinanzeigenBot) -> None:
|
||||
"""Verify that login check returns false when not logged in."""
|
||||
with patch.object(configured_bot, 'web_text', side_effect = TimeoutError):
|
||||
with patch.object(configured_bot, "web_text", side_effect = TimeoutError):
|
||||
assert await configured_bot.is_logged_in() is False
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_login_flow_completes_successfully(self, configured_bot:KleinanzeigenBot) -> None:
|
||||
"""Verify that normal login flow completes successfully."""
|
||||
with patch.object(configured_bot, 'web_open') as mock_open, \
|
||||
patch.object(configured_bot, 'is_logged_in', side_effect = [False, True]) as mock_logged_in, \
|
||||
patch.object(configured_bot, 'web_find', side_effect = TimeoutError), \
|
||||
patch.object(configured_bot, 'web_input') as mock_input, \
|
||||
patch.object(configured_bot, 'web_click') as mock_click:
|
||||
with patch.object(configured_bot, "web_open") as mock_open, \
|
||||
patch.object(configured_bot, "is_logged_in", side_effect = [False, True]) as mock_logged_in, \
|
||||
patch.object(configured_bot, "web_find", side_effect = TimeoutError), \
|
||||
patch.object(configured_bot, "web_input") as mock_input, \
|
||||
patch.object(configured_bot, "web_click") as mock_click:
|
||||
|
||||
await configured_bot.login()
|
||||
|
||||
@@ -349,13 +349,13 @@ class TestKleinanzeigenBotAuthentication:
|
||||
@pytest.mark.asyncio
|
||||
async def test_login_flow_handles_captcha(self, configured_bot:KleinanzeigenBot) -> None:
|
||||
"""Verify that login flow handles captcha correctly."""
|
||||
with patch.object(configured_bot, 'web_open'), \
|
||||
patch.object(configured_bot, 'is_logged_in', return_value = False), \
|
||||
patch.object(configured_bot, 'web_find') as mock_find, \
|
||||
patch.object(configured_bot, 'web_await') as mock_await, \
|
||||
patch.object(configured_bot, 'web_input'), \
|
||||
patch.object(configured_bot, 'web_click'), \
|
||||
patch('kleinanzeigen_bot.ainput') as mock_ainput:
|
||||
with patch.object(configured_bot, "web_open"), \
|
||||
patch.object(configured_bot, "is_logged_in", return_value = False), \
|
||||
patch.object(configured_bot, "web_find") as mock_find, \
|
||||
patch.object(configured_bot, "web_await") as mock_await, \
|
||||
patch.object(configured_bot, "web_input"), \
|
||||
patch.object(configured_bot, "web_click"), \
|
||||
patch("kleinanzeigen_bot.ainput") as mock_ainput:
|
||||
|
||||
mock_find.side_effect = [
|
||||
AsyncMock(), # Captcha iframe
|
||||
@@ -378,21 +378,21 @@ class TestKleinanzeigenBotLocalization:
|
||||
|
||||
def test_show_help_displays_german_text(self, test_bot:KleinanzeigenBot) -> None:
|
||||
"""Verify that help text is displayed in German when language is German."""
|
||||
with patch('kleinanzeigen_bot.get_current_locale') as mock_locale, \
|
||||
patch('builtins.print') as mock_print:
|
||||
with patch("kleinanzeigen_bot.get_current_locale") as mock_locale, \
|
||||
patch("builtins.print") as mock_print:
|
||||
mock_locale.return_value.language = "de"
|
||||
test_bot.show_help()
|
||||
printed_text = ''.join(str(call.args[0]) for call in mock_print.call_args_list)
|
||||
printed_text = "".join(str(call.args[0]) for call in mock_print.call_args_list)
|
||||
assert "Verwendung:" in printed_text
|
||||
assert "Befehle:" in printed_text
|
||||
|
||||
def test_show_help_displays_english_text(self, test_bot:KleinanzeigenBot) -> None:
|
||||
"""Verify that help text is displayed in English when language is English."""
|
||||
with patch('kleinanzeigen_bot.get_current_locale') as mock_locale, \
|
||||
patch('builtins.print') as mock_print:
|
||||
with patch("kleinanzeigen_bot.get_current_locale") as mock_locale, \
|
||||
patch("builtins.print") as mock_print:
|
||||
mock_locale.return_value.language = "en"
|
||||
test_bot.show_help()
|
||||
printed_text = ''.join(str(call.args[0]) for call in mock_print.call_args_list)
|
||||
printed_text = "".join(str(call.args[0]) for call in mock_print.call_args_list)
|
||||
assert "Usage:" in printed_text
|
||||
assert "Commands:" in printed_text
|
||||
|
||||
@@ -421,7 +421,7 @@ class TestKleinanzeigenBotBasics:
|
||||
"""Test closing browser session."""
|
||||
mock_close = MagicMock()
|
||||
test_bot.page = MagicMock() # Ensure page exists to trigger cleanup
|
||||
with patch.object(test_bot, 'close_browser_session', new = mock_close):
|
||||
with patch.object(test_bot, "close_browser_session", new = mock_close):
|
||||
test_bot.close_browser_session() # Call directly instead of relying on __del__
|
||||
mock_close.assert_called_once()
|
||||
|
||||
@@ -441,7 +441,7 @@ class TestKleinanzeigenBotBasics:
|
||||
# Reset log level to default
|
||||
LOG.setLevel(loggers.INFO)
|
||||
assert not loggers.is_debug(LOG)
|
||||
test_bot.parse_args(['script.py', '-v'])
|
||||
test_bot.parse_args(["script.py", "-v"])
|
||||
assert loggers.is_debug(LOG)
|
||||
|
||||
def test_get_config_file_path(self, test_bot:KleinanzeigenBot) -> None:
|
||||
@@ -472,64 +472,64 @@ class TestKleinanzeigenBotArgParsing:
|
||||
|
||||
def test_parse_args_help(self, test_bot:KleinanzeigenBot) -> None:
|
||||
"""Test parsing help command."""
|
||||
test_bot.parse_args(['script.py', 'help'])
|
||||
assert test_bot.command == 'help'
|
||||
test_bot.parse_args(["script.py", "help"])
|
||||
assert test_bot.command == "help"
|
||||
|
||||
def test_parse_args_version(self, test_bot:KleinanzeigenBot) -> None:
|
||||
"""Test parsing version command."""
|
||||
test_bot.parse_args(['script.py', 'version'])
|
||||
assert test_bot.command == 'version'
|
||||
test_bot.parse_args(["script.py", "version"])
|
||||
assert test_bot.command == "version"
|
||||
|
||||
def test_parse_args_verbose(self, test_bot:KleinanzeigenBot) -> None:
|
||||
"""Test parsing verbose flag."""
|
||||
test_bot.parse_args(['script.py', '-v', 'help'])
|
||||
assert loggers.is_debug(loggers.get_logger('kleinanzeigen_bot'))
|
||||
test_bot.parse_args(["script.py", "-v", "help"])
|
||||
assert loggers.is_debug(loggers.get_logger("kleinanzeigen_bot"))
|
||||
|
||||
def test_parse_args_config_path(self, test_bot:KleinanzeigenBot) -> None:
|
||||
"""Test parsing config path."""
|
||||
test_bot.parse_args(['script.py', '--config=test.yaml', 'help'])
|
||||
assert test_bot.config_file_path.endswith('test.yaml')
|
||||
test_bot.parse_args(["script.py", "--config=test.yaml", "help"])
|
||||
assert test_bot.config_file_path.endswith("test.yaml")
|
||||
|
||||
def test_parse_args_logfile(self, test_bot:KleinanzeigenBot) -> None:
|
||||
"""Test parsing log file path."""
|
||||
test_bot.parse_args(['script.py', '--logfile=test.log', 'help'])
|
||||
test_bot.parse_args(["script.py", "--logfile=test.log", "help"])
|
||||
assert test_bot.log_file_path is not None
|
||||
assert 'test.log' in test_bot.log_file_path
|
||||
assert "test.log" in test_bot.log_file_path
|
||||
|
||||
def test_parse_args_ads_selector(self, test_bot:KleinanzeigenBot) -> None:
|
||||
"""Test parsing ads selector."""
|
||||
test_bot.parse_args(['script.py', '--ads=all', 'publish'])
|
||||
assert test_bot.ads_selector == 'all'
|
||||
test_bot.parse_args(["script.py", "--ads=all", "publish"])
|
||||
assert test_bot.ads_selector == "all"
|
||||
|
||||
def test_parse_args_force(self, test_bot:KleinanzeigenBot) -> None:
|
||||
"""Test parsing force flag."""
|
||||
test_bot.parse_args(['script.py', '--force', 'publish'])
|
||||
assert test_bot.ads_selector == 'all'
|
||||
test_bot.parse_args(["script.py", "--force", "publish"])
|
||||
assert test_bot.ads_selector == "all"
|
||||
|
||||
def test_parse_args_keep_old(self, test_bot:KleinanzeigenBot) -> None:
|
||||
"""Test parsing keep-old flag."""
|
||||
test_bot.parse_args(['script.py', '--keep-old', 'publish'])
|
||||
test_bot.parse_args(["script.py", "--keep-old", "publish"])
|
||||
assert test_bot.keep_old_ads is True
|
||||
|
||||
def test_parse_args_logfile_empty(self, test_bot:KleinanzeigenBot) -> None:
|
||||
"""Test parsing empty log file path."""
|
||||
test_bot.parse_args(['script.py', '--logfile=', 'help'])
|
||||
test_bot.parse_args(["script.py", "--logfile=", "help"])
|
||||
assert test_bot.log_file_path is None
|
||||
|
||||
def test_parse_args_lang_option(self, test_bot:KleinanzeigenBot) -> None:
|
||||
"""Test parsing language option."""
|
||||
test_bot.parse_args(['script.py', '--lang=en', 'help'])
|
||||
assert test_bot.command == 'help'
|
||||
test_bot.parse_args(["script.py", "--lang=en", "help"])
|
||||
assert test_bot.command == "help"
|
||||
|
||||
def test_parse_args_no_arguments(self, test_bot:KleinanzeigenBot) -> None:
|
||||
"""Test parsing no arguments defaults to help."""
|
||||
test_bot.parse_args(['script.py'])
|
||||
assert test_bot.command == 'help'
|
||||
test_bot.parse_args(["script.py"])
|
||||
assert test_bot.command == "help"
|
||||
|
||||
def test_parse_args_multiple_commands(self, test_bot:KleinanzeigenBot) -> None:
|
||||
"""Test parsing multiple commands raises error."""
|
||||
with pytest.raises(SystemExit) as exc_info:
|
||||
test_bot.parse_args(['script.py', 'help', 'version'])
|
||||
test_bot.parse_args(["script.py", "help", "version"])
|
||||
assert exc_info.value.code == 2
|
||||
|
||||
|
||||
@@ -539,22 +539,22 @@ class TestKleinanzeigenBotCommands:
|
||||
@pytest.mark.asyncio
|
||||
async def test_run_version_command(self, test_bot:KleinanzeigenBot, capsys:Any) -> None:
|
||||
"""Test running version command."""
|
||||
await test_bot.run(['script.py', 'version'])
|
||||
await test_bot.run(["script.py", "version"])
|
||||
captured = capsys.readouterr()
|
||||
assert __version__ in captured.out
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_run_help_command(self, test_bot:KleinanzeigenBot, capsys:Any) -> None:
|
||||
"""Test running help command."""
|
||||
await test_bot.run(['script.py', 'help'])
|
||||
await test_bot.run(["script.py", "help"])
|
||||
captured = capsys.readouterr()
|
||||
assert 'Usage:' in captured.out
|
||||
assert "Usage:" in captured.out
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_run_unknown_command(self, test_bot:KleinanzeigenBot) -> None:
|
||||
"""Test running unknown command."""
|
||||
with pytest.raises(SystemExit) as exc_info:
|
||||
await test_bot.run(['script.py', 'unknown'])
|
||||
await test_bot.run(["script.py", "unknown"])
|
||||
assert exc_info.value.code == 2
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -568,8 +568,8 @@ login:
|
||||
password: test
|
||||
""")
|
||||
test_bot.config_file_path = str(config_path)
|
||||
await test_bot.run(['script.py', 'verify'])
|
||||
assert test_bot.config['login']['username'] == 'test'
|
||||
await test_bot.run(["script.py", "verify"])
|
||||
assert test_bot.config["login"]["username"] == "test"
|
||||
|
||||
|
||||
class TestKleinanzeigenBotAdOperations:
|
||||
@@ -578,27 +578,27 @@ class TestKleinanzeigenBotAdOperations:
|
||||
@pytest.mark.asyncio
|
||||
async def test_run_delete_command_no_ads(self, test_bot:KleinanzeigenBot, mock_config_setup:None) -> None: # pylint: disable=unused-argument
|
||||
"""Test running delete command with no ads."""
|
||||
with patch.object(test_bot, 'load_ads', return_value = []):
|
||||
await test_bot.run(['script.py', 'delete'])
|
||||
assert test_bot.command == 'delete'
|
||||
with patch.object(test_bot, "load_ads", return_value = []):
|
||||
await test_bot.run(["script.py", "delete"])
|
||||
assert test_bot.command == "delete"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_run_publish_command_no_ads(self, test_bot:KleinanzeigenBot, mock_config_setup:None) -> None: # pylint: disable=unused-argument
|
||||
"""Test running publish command with no ads."""
|
||||
with patch.object(test_bot, 'load_ads', return_value = []):
|
||||
await test_bot.run(['script.py', 'publish'])
|
||||
assert test_bot.command == 'publish'
|
||||
with patch.object(test_bot, "load_ads", return_value = []):
|
||||
await test_bot.run(["script.py", "publish"])
|
||||
assert test_bot.command == "publish"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_run_download_command_default_selector(self, test_bot:KleinanzeigenBot, mock_config_setup:None) -> None: # pylint: disable=unused-argument
|
||||
"""Test running download command with default selector."""
|
||||
with patch.object(test_bot, 'download_ads', new_callable = AsyncMock):
|
||||
await test_bot.run(['script.py', 'download'])
|
||||
assert test_bot.ads_selector == 'new'
|
||||
with patch.object(test_bot, "download_ads", new_callable = AsyncMock):
|
||||
await test_bot.run(["script.py", "download"])
|
||||
assert test_bot.ads_selector == "new"
|
||||
|
||||
def test_load_ads_no_files(self, test_bot:KleinanzeigenBot) -> None:
|
||||
"""Test loading ads with no files."""
|
||||
test_bot.config['ad_files'] = ['nonexistent/*.yaml']
|
||||
test_bot.config["ad_files"] = ["nonexistent/*.yaml"]
|
||||
ads = test_bot.load_ads()
|
||||
assert len(ads) == 0
|
||||
|
||||
@@ -609,24 +609,24 @@ class TestKleinanzeigenBotAdManagement:
|
||||
@pytest.mark.asyncio
|
||||
async def test_download_ads_with_specific_ids(self, test_bot:KleinanzeigenBot, mock_config_setup:None) -> None: # pylint: disable=unused-argument
|
||||
"""Test downloading ads with specific IDs."""
|
||||
test_bot.ads_selector = '123,456'
|
||||
with patch.object(test_bot, 'download_ads', new_callable = AsyncMock):
|
||||
await test_bot.run(['script.py', 'download', '--ads=123,456'])
|
||||
assert test_bot.ads_selector == '123,456'
|
||||
test_bot.ads_selector = "123,456"
|
||||
with patch.object(test_bot, "download_ads", new_callable = AsyncMock):
|
||||
await test_bot.run(["script.py", "download", "--ads=123,456"])
|
||||
assert test_bot.ads_selector == "123,456"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_run_publish_invalid_selector(self, test_bot:KleinanzeigenBot, mock_config_setup:None) -> None: # pylint: disable=unused-argument
|
||||
"""Test running publish with invalid selector."""
|
||||
with patch.object(test_bot, 'load_ads', return_value = []):
|
||||
await test_bot.run(['script.py', 'publish', '--ads=invalid'])
|
||||
assert test_bot.ads_selector == 'due'
|
||||
with patch.object(test_bot, "load_ads", return_value = []):
|
||||
await test_bot.run(["script.py", "publish", "--ads=invalid"])
|
||||
assert test_bot.ads_selector == "due"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_run_download_invalid_selector(self, test_bot:KleinanzeigenBot, mock_config_setup:None) -> None: # pylint: disable=unused-argument
|
||||
"""Test running download with invalid selector."""
|
||||
with patch.object(test_bot, 'download_ads', new_callable = AsyncMock):
|
||||
await test_bot.run(['script.py', 'download', '--ads=invalid'])
|
||||
assert test_bot.ads_selector == 'new'
|
||||
with patch.object(test_bot, "download_ads", new_callable = AsyncMock):
|
||||
await test_bot.run(["script.py", "download", "--ads=invalid"])
|
||||
assert test_bot.ads_selector == "new"
|
||||
|
||||
|
||||
class TestKleinanzeigenBotAdConfiguration:
|
||||
@@ -645,8 +645,8 @@ categories:
|
||||
""")
|
||||
test_bot.config_file_path = str(config_path)
|
||||
test_bot.load_config()
|
||||
assert 'custom_cat' in test_bot.categories
|
||||
assert test_bot.categories['custom_cat'] == 'custom_id'
|
||||
assert "custom_cat" in test_bot.categories
|
||||
assert test_bot.categories["custom_cat"] == "custom_id"
|
||||
|
||||
def test_load_ads_with_missing_title(self, test_bot:KleinanzeigenBot, tmp_path:Any, minimal_ad_config:dict[str, Any]) -> None:
|
||||
"""Test loading ads with missing title."""
|
||||
@@ -667,7 +667,7 @@ categories:
|
||||
|
||||
# Set config file path to tmp_path and use relative path for ad_files
|
||||
test_bot.config_file_path = str(temp_path / "config.yaml")
|
||||
test_bot.config['ad_files'] = ["ads/*.yaml"]
|
||||
test_bot.config["ad_files"] = ["ads/*.yaml"]
|
||||
with pytest.raises(AssertionError) as exc_info:
|
||||
test_bot.load_ads()
|
||||
assert "must be at least 10 characters long" in str(exc_info.value)
|
||||
@@ -691,7 +691,7 @@ categories:
|
||||
|
||||
# Set config file path to tmp_path and use relative path for ad_files
|
||||
test_bot.config_file_path = str(temp_path / "config.yaml")
|
||||
test_bot.config['ad_files'] = ["ads/*.yaml"]
|
||||
test_bot.config["ad_files"] = ["ads/*.yaml"]
|
||||
with pytest.raises(AssertionError) as exc_info:
|
||||
test_bot.load_ads()
|
||||
assert "property [price_type] must be one of:" in str(exc_info.value)
|
||||
@@ -715,7 +715,7 @@ categories:
|
||||
|
||||
# Set config file path to tmp_path and use relative path for ad_files
|
||||
test_bot.config_file_path = str(temp_path / "config.yaml")
|
||||
test_bot.config['ad_files'] = ["ads/*.yaml"]
|
||||
test_bot.config["ad_files"] = ["ads/*.yaml"]
|
||||
with pytest.raises(AssertionError) as exc_info:
|
||||
test_bot.load_ads()
|
||||
assert "property [shipping_type] must be one of:" in str(exc_info.value)
|
||||
@@ -740,7 +740,7 @@ categories:
|
||||
|
||||
# Set config file path to tmp_path and use relative path for ad_files
|
||||
test_bot.config_file_path = str(temp_path / "config.yaml")
|
||||
test_bot.config['ad_files'] = ["ads/*.yaml"]
|
||||
test_bot.config["ad_files"] = ["ads/*.yaml"]
|
||||
with pytest.raises(AssertionError) as exc_info:
|
||||
test_bot.load_ads()
|
||||
assert "must not be specified for GIVE_AWAY ad" in str(exc_info.value)
|
||||
@@ -765,7 +765,7 @@ categories:
|
||||
|
||||
# Set config file path to tmp_path and use relative path for ad_files
|
||||
test_bot.config_file_path = str(temp_path / "config.yaml")
|
||||
test_bot.config['ad_files'] = ["ads/*.yaml"]
|
||||
test_bot.config["ad_files"] = ["ads/*.yaml"]
|
||||
with pytest.raises(AssertionError) as exc_info:
|
||||
test_bot.load_ads()
|
||||
assert "not specified" in str(exc_info.value)
|
||||
@@ -798,7 +798,7 @@ categories:
|
||||
|
||||
# Set config file path to tmp_path and use relative path for ad_files
|
||||
test_bot.config_file_path = str(temp_path / "config.yaml")
|
||||
test_bot.config['ad_files'] = ["ads/*.yaml"]
|
||||
test_bot.config["ad_files"] = ["ads/*.yaml"]
|
||||
with pytest.raises(AssertionError) as exc_info:
|
||||
test_bot.load_ads()
|
||||
assert "property [description] not specified" in str(exc_info.value)
|
||||
@@ -826,10 +826,10 @@ class TestKleinanzeigenBotAdDeletion:
|
||||
{"title": "Other Title", "id": "11111"}
|
||||
]
|
||||
|
||||
with patch.object(test_bot, 'web_open', new_callable = AsyncMock), \
|
||||
patch.object(test_bot, 'web_find', new_callable = AsyncMock) as mock_find, \
|
||||
patch.object(test_bot, 'web_click', new_callable = AsyncMock), \
|
||||
patch.object(test_bot, 'web_check', new_callable = AsyncMock, return_value = True):
|
||||
with patch.object(test_bot, "web_open", new_callable = AsyncMock), \
|
||||
patch.object(test_bot, "web_find", new_callable = AsyncMock) as mock_find, \
|
||||
patch.object(test_bot, "web_click", new_callable = AsyncMock), \
|
||||
patch.object(test_bot, "web_check", new_callable = AsyncMock, return_value = True):
|
||||
mock_find.return_value.attrs = {"content": "some-token"}
|
||||
result = await test_bot.delete_ad(ad_cfg, published_ads, delete_old_ads_by_title = True)
|
||||
assert result is True
|
||||
@@ -852,10 +852,10 @@ class TestKleinanzeigenBotAdDeletion:
|
||||
{"title": "Other Title", "id": "11111"}
|
||||
]
|
||||
|
||||
with patch.object(test_bot, 'web_open', new_callable = AsyncMock), \
|
||||
patch.object(test_bot, 'web_find', new_callable = AsyncMock) as mock_find, \
|
||||
patch.object(test_bot, 'web_click', new_callable = AsyncMock), \
|
||||
patch.object(test_bot, 'web_check', new_callable = AsyncMock, return_value = True):
|
||||
with patch.object(test_bot, "web_open", new_callable = AsyncMock), \
|
||||
patch.object(test_bot, "web_find", new_callable = AsyncMock) as mock_find, \
|
||||
patch.object(test_bot, "web_click", new_callable = AsyncMock), \
|
||||
patch.object(test_bot, "web_check", new_callable = AsyncMock, return_value = True):
|
||||
mock_find.return_value.attrs = {"content": "some-token"}
|
||||
result = await test_bot.delete_ad(ad_cfg, published_ads, delete_old_ads_by_title = False)
|
||||
assert result is True
|
||||
@@ -896,10 +896,10 @@ class TestKleinanzeigenBotAdRepublication:
|
||||
|
||||
# Set config file path and use relative path for ad_files
|
||||
test_bot.config_file_path = str(temp_path / "config.yaml")
|
||||
test_bot.config['ad_files'] = ["ads/*.yaml"]
|
||||
test_bot.config["ad_files"] = ["ads/*.yaml"]
|
||||
|
||||
# Mock the loading of the original ad configuration
|
||||
with patch('kleinanzeigen_bot.utils.dicts.load_dict', side_effect = [
|
||||
with patch("kleinanzeigen_bot.utils.dicts.load_dict", side_effect = [
|
||||
ad_cfg, # First call returns the original ad config
|
||||
{} # Second call for ad_fields.yaml
|
||||
]):
|
||||
@@ -925,9 +925,9 @@ class TestKleinanzeigenBotAdRepublication:
|
||||
ad_cfg_orig["content_hash"] = current_hash
|
||||
|
||||
# Mock the config to prevent actual file operations
|
||||
test_bot.config['ad_files'] = ['test.yaml']
|
||||
with patch('kleinanzeigen_bot.utils.dicts.load_dict_if_exists', return_value = ad_cfg_orig), \
|
||||
patch('kleinanzeigen_bot.utils.dicts.load_dict', return_value = {}): # Mock ad_fields.yaml
|
||||
test_bot.config["ad_files"] = ["test.yaml"]
|
||||
with patch("kleinanzeigen_bot.utils.dicts.load_dict_if_exists", return_value = ad_cfg_orig), \
|
||||
patch("kleinanzeigen_bot.utils.dicts.load_dict", return_value = {}): # Mock ad_fields.yaml
|
||||
ads_to_publish = test_bot.load_ads()
|
||||
assert len(ads_to_publish) == 0 # No ads should be marked for republication
|
||||
|
||||
@@ -967,23 +967,23 @@ class TestKleinanzeigenBotShippingOptions:
|
||||
|
||||
# Mock web_execute to handle all JavaScript calls
|
||||
async def mock_web_execute(script:str) -> Any:
|
||||
if script == 'document.body.scrollHeight':
|
||||
if script == "document.body.scrollHeight":
|
||||
return 0 # Return integer to prevent scrolling loop
|
||||
return None
|
||||
|
||||
# Mock the necessary web interaction methods
|
||||
with patch.object(test_bot, 'web_execute', side_effect=mock_web_execute), \
|
||||
patch.object(test_bot, 'web_click', new_callable=AsyncMock), \
|
||||
patch.object(test_bot, 'web_find', new_callable=AsyncMock) as mock_find, \
|
||||
patch.object(test_bot, 'web_select', new_callable = AsyncMock), \
|
||||
patch.object(test_bot, 'web_input', new_callable = AsyncMock), \
|
||||
patch.object(test_bot, 'web_open', new_callable = AsyncMock), \
|
||||
patch.object(test_bot, 'web_sleep', new_callable = AsyncMock), \
|
||||
patch.object(test_bot, 'web_check', new_callable = AsyncMock, return_value = True), \
|
||||
patch.object(test_bot, 'web_request', new_callable = AsyncMock), \
|
||||
patch.object(test_bot, 'web_find_all', new_callable = AsyncMock) as mock_find_all, \
|
||||
patch.object(test_bot, 'web_await', new_callable = AsyncMock), \
|
||||
patch('builtins.input', return_value=""): # Mock the input function
|
||||
with patch.object(test_bot, "web_execute", side_effect = mock_web_execute), \
|
||||
patch.object(test_bot, "web_click", new_callable = AsyncMock), \
|
||||
patch.object(test_bot, "web_find", new_callable = AsyncMock) as mock_find, \
|
||||
patch.object(test_bot, "web_select", new_callable = AsyncMock), \
|
||||
patch.object(test_bot, "web_input", new_callable = AsyncMock), \
|
||||
patch.object(test_bot, "web_open", new_callable = AsyncMock), \
|
||||
patch.object(test_bot, "web_sleep", new_callable = AsyncMock), \
|
||||
patch.object(test_bot, "web_check", new_callable = AsyncMock, return_value = True), \
|
||||
patch.object(test_bot, "web_request", new_callable = AsyncMock), \
|
||||
patch.object(test_bot, "web_find_all", new_callable = AsyncMock) as mock_find_all, \
|
||||
patch.object(test_bot, "web_await", new_callable = AsyncMock), \
|
||||
patch("builtins.input", return_value = ""): # Mock the input function
|
||||
|
||||
# Mock the shipping options form elements
|
||||
mock_find.side_effect = [
|
||||
@@ -1000,7 +1000,7 @@ class TestKleinanzeigenBotShippingOptions:
|
||||
mock_find_all.return_value = []
|
||||
|
||||
# Mock web_check to return True for radio button checked state
|
||||
with patch.object(test_bot, 'web_check', new_callable = AsyncMock) as mock_check:
|
||||
with patch.object(test_bot, "web_check", new_callable = AsyncMock) as mock_check:
|
||||
mock_check.return_value = True
|
||||
|
||||
# Test through the public interface by publishing an ad
|
||||
@@ -1075,7 +1075,7 @@ class TestKleinanzeigenBotDescriptionHandling:
|
||||
"""Test that description works correctly when description is missing from main config."""
|
||||
# Set up config without any description fields
|
||||
test_bot.config = {
|
||||
'ad_defaults': {
|
||||
"ad_defaults": {
|
||||
# No description field at all
|
||||
}
|
||||
}
|
||||
@@ -1093,9 +1093,9 @@ class TestKleinanzeigenBotDescriptionHandling:
|
||||
def test_description_with_only_new_format_affixes(self, test_bot:KleinanzeigenBot) -> None:
|
||||
"""Test that description works with only new format affixes in config."""
|
||||
test_bot.config = {
|
||||
'ad_defaults': {
|
||||
'description_prefix': 'Prefix: ',
|
||||
'description_suffix': ' :Suffix'
|
||||
"ad_defaults": {
|
||||
"description_prefix": "Prefix: ",
|
||||
"description_suffix": " :Suffix"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1110,12 +1110,12 @@ class TestKleinanzeigenBotDescriptionHandling:
|
||||
def test_description_with_mixed_config_formats(self, test_bot:KleinanzeigenBot) -> None:
|
||||
"""Test that description works with both old and new format affixes in config."""
|
||||
test_bot.config = {
|
||||
'ad_defaults': {
|
||||
'description_prefix': 'New Prefix: ',
|
||||
'description_suffix': ' :New Suffix',
|
||||
'description': {
|
||||
'prefix': 'Old Prefix: ',
|
||||
'suffix': ' :Old Suffix'
|
||||
"ad_defaults": {
|
||||
"description_prefix": "New Prefix: ",
|
||||
"description_suffix": " :New Suffix",
|
||||
"description": {
|
||||
"prefix": "Old Prefix: ",
|
||||
"suffix": " :Old Suffix"
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1131,9 +1131,9 @@ class TestKleinanzeigenBotDescriptionHandling:
|
||||
def test_description_with_ad_level_affixes(self, test_bot:KleinanzeigenBot) -> None:
|
||||
"""Test that ad-level affixes take precedence over config affixes."""
|
||||
test_bot.config = {
|
||||
'ad_defaults': {
|
||||
'description_prefix': 'Config Prefix: ',
|
||||
'description_suffix': ' :Config Suffix'
|
||||
"ad_defaults": {
|
||||
"description_prefix": "Config Prefix: ",
|
||||
"description_suffix": " :Config Suffix"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1150,12 +1150,12 @@ class TestKleinanzeigenBotDescriptionHandling:
|
||||
def test_description_with_none_values(self, test_bot:KleinanzeigenBot) -> None:
|
||||
"""Test that None values in affixes are handled correctly."""
|
||||
test_bot.config = {
|
||||
'ad_defaults': {
|
||||
'description_prefix': None,
|
||||
'description_suffix': None,
|
||||
'description': {
|
||||
'prefix': None,
|
||||
'suffix': None
|
||||
"ad_defaults": {
|
||||
"description_prefix": None,
|
||||
"description_suffix": None,
|
||||
"description": {
|
||||
"prefix": None,
|
||||
"suffix": None
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1171,7 +1171,7 @@ class TestKleinanzeigenBotDescriptionHandling:
|
||||
def test_description_with_email_replacement(self, test_bot:KleinanzeigenBot) -> None:
|
||||
"""Test that @ symbols in description are replaced with (at)."""
|
||||
test_bot.config = {
|
||||
'ad_defaults': {}
|
||||
"ad_defaults": {}
|
||||
}
|
||||
|
||||
ad_cfg = {
|
||||
@@ -1228,10 +1228,10 @@ class TestKleinanzeigenBotChangedAds:
|
||||
|
||||
# Set config file path and use relative path for ad_files
|
||||
test_bot.config_file_path = str(temp_path / "config.yaml")
|
||||
test_bot.config['ad_files'] = ["ads/*.yaml"]
|
||||
test_bot.config["ad_files"] = ["ads/*.yaml"]
|
||||
|
||||
# Mock the loading of the ad configuration
|
||||
with patch('kleinanzeigen_bot.utils.dicts.load_dict', side_effect=[
|
||||
with patch("kleinanzeigen_bot.utils.dicts.load_dict", side_effect = [
|
||||
changed_ad, # First call returns the changed ad
|
||||
{} # Second call for ad_fields.yaml
|
||||
]):
|
||||
@@ -1280,10 +1280,10 @@ class TestKleinanzeigenBotChangedAds:
|
||||
|
||||
# Set config file path and use relative path for ad_files
|
||||
test_bot.config_file_path = str(temp_path / "config.yaml")
|
||||
test_bot.config['ad_files'] = ["ads/*.yaml"]
|
||||
test_bot.config["ad_files"] = ["ads/*.yaml"]
|
||||
|
||||
# Mock the loading of the ad configuration
|
||||
with patch('kleinanzeigen_bot.utils.dicts.load_dict', side_effect=[
|
||||
with patch("kleinanzeigen_bot.utils.dicts.load_dict", side_effect = [
|
||||
changed_ad, # First call returns the changed ad
|
||||
{} # Second call for ad_fields.yaml
|
||||
]):
|
||||
|
||||
@@ -31,7 +31,7 @@ EXCLUDED_MESSAGES: dict[str, set[str]] = {
|
||||
}
|
||||
|
||||
# Special modules that are known to be needed even if not in messages_by_file
|
||||
KNOWN_NEEDED_MODULES = {'getopt.py'}
|
||||
KNOWN_NEEDED_MODULES = {"getopt.py"}
|
||||
|
||||
# Type aliases for better readability
|
||||
ModulePath = str
|
||||
@@ -69,8 +69,8 @@ def _get_function_name(node: ast.AST) -> str:
|
||||
function_name = None
|
||||
current = n
|
||||
|
||||
while hasattr(current, '_parent'):
|
||||
current = getattr(current, '_parent')
|
||||
while hasattr(current, "_parent"):
|
||||
current = getattr(current, "_parent")
|
||||
if isinstance(current, ast.ClassDef) and not class_name:
|
||||
class_name = current.name
|
||||
elif isinstance(current, ast.FunctionDef) or isinstance(current, ast.AsyncFunctionDef) and not function_name:
|
||||
@@ -94,13 +94,13 @@ def _extract_log_messages(file_path: str, exclude_debug:bool = False) -> Message
|
||||
Returns:
|
||||
Dictionary mapping function names to their messages
|
||||
"""
|
||||
with open(file_path, 'r', encoding = 'utf-8') as file:
|
||||
with open(file_path, "r", encoding = "utf-8") as file:
|
||||
tree = ast.parse(file.read(), filename = file_path)
|
||||
|
||||
# Add parent references for context tracking
|
||||
for parent in ast.walk(tree):
|
||||
for child in ast.iter_child_nodes(parent):
|
||||
setattr(child, '_parent', parent)
|
||||
setattr(child, "_parent", parent)
|
||||
|
||||
messages:MessageDict = defaultdict(lambda: defaultdict(set))
|
||||
|
||||
@@ -114,7 +114,7 @@ def _extract_log_messages(file_path: str, exclude_debug:bool = False) -> Message
|
||||
def extract_string_value(node:ast.AST) -> str | None:
|
||||
"""Safely extract string value from an AST node."""
|
||||
if isinstance(node, ast.Constant):
|
||||
value = getattr(node, 'value', None)
|
||||
value = getattr(node, "value", None)
|
||||
return value if isinstance(value, str) else None
|
||||
return None
|
||||
|
||||
@@ -127,24 +127,24 @@ def _extract_log_messages(file_path: str, exclude_debug:bool = False) -> Message
|
||||
# Extract messages from various call types
|
||||
if (isinstance(node.func, ast.Attribute) and
|
||||
isinstance(node.func.value, ast.Name) and
|
||||
node.func.value.id in {'LOG', 'logger', 'logging'} and
|
||||
node.func.attr in {None if exclude_debug else 'debug', 'info', 'warning', 'error', 'exception', 'critical'}):
|
||||
node.func.value.id in {"LOG", "logger", "logging"} and
|
||||
node.func.attr in {None if exclude_debug else "debug", "info", "warning", "error", "exception", "critical"}):
|
||||
if node.args:
|
||||
msg = extract_string_value(node.args[0])
|
||||
if msg:
|
||||
add_message(function_name, msg)
|
||||
|
||||
# Handle gettext calls
|
||||
elif ((isinstance(node.func, ast.Name) and node.func.id == '_') or
|
||||
(isinstance(node.func, ast.Attribute) and node.func.attr == 'gettext')):
|
||||
elif ((isinstance(node.func, ast.Name) and node.func.id == "_") or
|
||||
(isinstance(node.func, ast.Attribute) and node.func.attr == "gettext")):
|
||||
if node.args:
|
||||
msg = extract_string_value(node.args[0])
|
||||
if msg:
|
||||
add_message(function_name, msg)
|
||||
|
||||
# Handle other translatable function calls
|
||||
elif isinstance(node.func, ast.Name) and node.func.id in {'ainput', 'pluralize', 'ensure'}:
|
||||
arg_index = 0 if node.func.id == 'ainput' else 1
|
||||
elif isinstance(node.func, ast.Name) and node.func.id in {"ainput", "pluralize", "ensure"}:
|
||||
arg_index = 0 if node.func.id == "ainput" else 1
|
||||
if len(node.args) > arg_index:
|
||||
msg = extract_string_value(node.args[arg_index])
|
||||
if msg:
|
||||
@@ -162,7 +162,7 @@ def _get_all_log_messages(exclude_debug:bool = False) -> dict[str, MessageDict]:
|
||||
Returns:
|
||||
Dictionary mapping module paths to their function messages
|
||||
"""
|
||||
src_dir = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), 'src', 'kleinanzeigen_bot')
|
||||
src_dir = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), "src", "kleinanzeigen_bot")
|
||||
print(f"\nScanning for messages in directory: {src_dir}")
|
||||
|
||||
messages_by_file:dict[str, MessageDict] = {
|
||||
@@ -187,15 +187,15 @@ def _get_all_log_messages(exclude_debug:bool = False) -> dict[str, MessageDict]:
|
||||
|
||||
for root, _, filenames in os.walk(src_dir):
|
||||
for filename in filenames:
|
||||
if filename.endswith('.py'):
|
||||
if filename.endswith(".py"):
|
||||
file_path = os.path.join(root, filename)
|
||||
relative_path = os.path.relpath(file_path, src_dir)
|
||||
if relative_path.startswith('resources/'):
|
||||
if relative_path.startswith("resources/"):
|
||||
continue
|
||||
messages = _extract_log_messages(file_path, exclude_debug)
|
||||
if messages:
|
||||
module_path = os.path.join('kleinanzeigen_bot', relative_path)
|
||||
module_path = module_path.replace(os.sep, '/')
|
||||
module_path = os.path.join("kleinanzeigen_bot", relative_path)
|
||||
module_path = module_path.replace(os.sep, "/")
|
||||
messages_by_file[module_path] = messages
|
||||
|
||||
return messages_by_file
|
||||
@@ -227,7 +227,7 @@ def _get_translations_for_language(lang: str) -> TranslationDict:
|
||||
Returns:
|
||||
Dictionary containing all translations for the language
|
||||
"""
|
||||
yaml = YAML(typ = 'safe')
|
||||
yaml = YAML(typ = "safe")
|
||||
translation_file = f"translations.{lang}.yaml"
|
||||
print(f"Loading translations from {translation_file}")
|
||||
content = files(resources).joinpath(translation_file).read_text()
|
||||
@@ -253,11 +253,11 @@ def _find_translation(translations: TranslationDict,
|
||||
True if translation exists in the correct location, False otherwise
|
||||
"""
|
||||
# Special case for getopt.py
|
||||
if module == 'getopt.py':
|
||||
if module == "getopt.py":
|
||||
return bool(translations.get(module, {}).get(function, {}).get(message))
|
||||
|
||||
# Add kleinanzeigen_bot/ prefix if not present
|
||||
module_path = f'kleinanzeigen_bot/{module}' if not module.startswith('kleinanzeigen_bot/') else module
|
||||
module_path = f'kleinanzeigen_bot/{module}' if not module.startswith("kleinanzeigen_bot/") else module
|
||||
|
||||
# Check if module exists in translations
|
||||
module_trans = translations.get(module_path, {})
|
||||
@@ -296,11 +296,11 @@ def _message_exists_in_code(code_messages: dict[str, MessageDict],
|
||||
True if message exists in the code, False otherwise
|
||||
"""
|
||||
# Special case for getopt.py
|
||||
if module == 'getopt.py':
|
||||
if module == "getopt.py":
|
||||
return bool(code_messages.get(module, {}).get(function, {}).get(message))
|
||||
|
||||
# Remove kleinanzeigen_bot/ prefix if present for code message lookup
|
||||
module_path = module[len('kleinanzeigen_bot/'):] if module.startswith('kleinanzeigen_bot/') else module
|
||||
module_path = module[len("kleinanzeigen_bot/"):] if module.startswith("kleinanzeigen_bot/") else module
|
||||
module_path = f'kleinanzeigen_bot/{module_path}'
|
||||
|
||||
# Check if module exists in code messages
|
||||
|
||||
Reference in New Issue
Block a user