mirror of
https://github.com/Second-Hand-Friends/kleinanzeigen-bot.git
synced 2026-07-09 20:51:05 +02:00
## ℹ️ Description Adds configurable human-like browser interaction behavior to make automation less mechanically deterministic while preserving the existing publish/update workflows. Related to #1120. ## 📋 Changes Summary - Added `humanization` configuration for action delays, typing jitter, thinking pauses, idle micro-actions, and randomized viewport sizes. - Applied humanized timing and interaction behavior to relevant browser automation paths. - Added validation for viewport size formats and min/max timing ranges, including German translations for user-facing validation errors. - Hardened fallback behavior for humanized click and typing flows so automation can continue when a lower-level interaction fails. - Added focused unit coverage for delay edge cases, config validation, and typing fallback behavior. - Regenerated committed config/schema artifacts for the new settings. ## 🧪 Validation - `pdm run pytest tests/unit/test_humanization.py tests/unit/test_web_scraping_mixin.py` - `pdm run ruff check src/kleinanzeigen_bot/utils/web_scraping_mixin.py src/kleinanzeigen_bot/model/config_model.py tests/unit/test_humanization.py` - Live DOM probes with the PR worktree code via `data/verify_dom_assumptions.py`: - `login-selectors` - `publish-selectors` - `price-type-controls` ### ⚙️ Type of Change - [ ] 🐞 Bug fix (non-breaking change which fixes an issue) - [x] ✨ New feature (adds new functionality without breaking existing usage) - [ ] 💥 Breaking change (changes that might break existing user setups, scripts, or configurations) ## ✅ Checklist Before requesting a review, confirm the following: - [x] I have reviewed my changes to ensure they meet the project's standards. - [x] I have tested my changes and ensured that all tests pass (`pdm run test`). - [x] I have formatted the code (`pdm run format`). - [x] I have verified that linting passes (`pdm run lint`). - [x] I have updated documentation where necessary. By submitting this pull request, I confirm that you can use, modify, copy, and redistribute this contribution, under the terms of your choice. <!-- This is an auto-generated comment: release notes by coderabbit.ai --> ## Summary by CodeRabbit * **New Features** * Added configurable “humanization” behavior for browser automation: optional mouse movement, per-keystroke typing jitter/delays, probabilistic thinking pauses, idle micro-actions, and optional launch viewport randomization. * Navigation timing now incorporates human-like action delays. * **Bug Fixes** * Improved resilience: humanized click/typing safely fall back to standard actions when needed. * Added stronger validation for viewport-size formatting and delay min/max range consistency. * **Tests** * Added comprehensive unit coverage for humanization routing, event dispatch, timing, fallbacks, idle/thinking behavior, viewport launch arguments, and validation. * **Documentation** * Updated default configuration, configuration schema, and a related translation message. <!-- end of auto-generated comment: release notes by coderabbit.ai --> --------- Co-authored-by: Jens <1742418+1cu@users.noreply.github.com>
413 lines
14 KiB
YAML
413 lines
14 KiB
YAML
# yaml-language-server: $schema=https://raw.githubusercontent.com/Second-Hand-Friends/kleinanzeigen-bot/main/schemas/config.schema.json
|
|
# glob (wildcard) patterns to select local ad configuration files. This only controls which files are loaded; it does not rename downloaded files. If relative paths are specified, they are relative to this configuration file
|
|
# Example usage:
|
|
# ad_files:
|
|
# - "./downloaded-ads/**/*.yaml"
|
|
# - "./**/ad_*.{json,yml,yaml}"
|
|
ad_files:
|
|
- ./**/ad_*.{json,yml,yaml}
|
|
|
|
# ################################################################################
|
|
# Default values for ads, can be overwritten in each ad configuration file
|
|
ad_defaults:
|
|
|
|
# whether the ad should be published (false = skip this ad)
|
|
active: true
|
|
|
|
# type of the ad listing
|
|
# Examples (choose one):
|
|
# • OFFER
|
|
# • WANTED
|
|
type: OFFER
|
|
|
|
# text to prepend to each ad (optional)
|
|
description_prefix: ''
|
|
|
|
# text to append to each ad (optional)
|
|
description_suffix: ''
|
|
|
|
# pricing strategy for the listing
|
|
# Examples (choose one):
|
|
# • FIXED
|
|
# • NEGOTIABLE
|
|
# • GIVE_AWAY
|
|
# • NOT_APPLICABLE
|
|
price_type: NEGOTIABLE
|
|
|
|
# automatic price reduction configuration for reposted ads
|
|
auto_price_reduction:
|
|
|
|
# automatically lower the price of reposted ads
|
|
enabled: false
|
|
|
|
# reduction strategy (required when enabled: true). PERCENTAGE = % of price, FIXED = absolute amount
|
|
# Examples (choose one):
|
|
# • PERCENTAGE
|
|
# • FIXED
|
|
strategy:
|
|
|
|
# reduction amount (required when enabled: true). For PERCENTAGE: use percent value (e.g., 10 = 10%%). For FIXED: use currency amount
|
|
# Examples (choose one):
|
|
# • 10.0
|
|
# • 5.0
|
|
# • 20.0
|
|
amount:
|
|
|
|
# minimum price floor (required when enabled: true). Use 0 for no minimum
|
|
# Examples (choose one):
|
|
# • 1.0
|
|
# • 5.0
|
|
# • 10.0
|
|
min_price:
|
|
|
|
# number of reposts to wait before applying the first automatic price reduction
|
|
delay_reposts: 0
|
|
|
|
# number of days to wait after publication before applying automatic price reductions
|
|
delay_days: 0
|
|
|
|
# also apply automatic price reduction during update runs (MODIFY mode). delay_days applies, delay_reposts is ignored
|
|
on_update: false
|
|
|
|
# shipping method for the item
|
|
# Examples (choose one):
|
|
# • PICKUP
|
|
# • SHIPPING
|
|
# • NOT_APPLICABLE
|
|
shipping_type: SHIPPING
|
|
|
|
# enable direct purchase option (requires shipping_type: SHIPPING, non-empty shipping_options, and price_type FIXED or NEGOTIABLE)
|
|
sell_directly: false
|
|
|
|
# default image glob patterns (optional). Leave empty for no default images
|
|
# Example usage:
|
|
# images:
|
|
# - "images/*.jpg"
|
|
# - "photos/*.{png,jpg}"
|
|
images: []
|
|
|
|
# default contact information for ads
|
|
contact:
|
|
|
|
# contact name displayed on the ad
|
|
name: ''
|
|
|
|
# street address for the listing
|
|
street: ''
|
|
|
|
# postal/ZIP code for the listing location
|
|
zipcode: ''
|
|
|
|
# city or locality of the listing (can include multiple districts)
|
|
# Example: Sample Town - District One
|
|
location: ''
|
|
|
|
# phone number for contact - only available for commercial accounts, personal accounts no longer support this
|
|
# Example: "01234 567890"
|
|
phone: ''
|
|
|
|
# number of days between automatic republication of ads
|
|
republication_interval: 7
|
|
|
|
# ################################################################################
|
|
# additional name to category ID mappings (optional). Leave as {} if not needed. See full list at: https://github.com/Second-Hand-Friends/kleinanzeigen-bot/blob/main/src/kleinanzeigen_bot/resources/categories.yaml To add: use format 'Category > Subcategory': 'ID'
|
|
# Examples (choose one):
|
|
# • "Elektronik > Notebooks": "161/278"
|
|
# • "Jobs > Praktika": "102/125"
|
|
categories: {}
|
|
|
|
# ################################################################################
|
|
download:
|
|
|
|
# directory where downloaded ads are written. The default literal 'downloaded-ads' uses workspace-specific resolution; custom relative paths are resolved against the config file location
|
|
# Examples (choose one):
|
|
# • "downloaded-ads"
|
|
# • "./ads"
|
|
dir: downloaded-ads
|
|
|
|
# if true, all shipping options matching the package size will be included
|
|
include_all_matching_shipping_options: false
|
|
|
|
# shipping options to exclude (optional). Leave as [] to include all. Add items like 'DHL_2' to exclude specific carriers
|
|
# Example usage:
|
|
# excluded_shipping_options:
|
|
# - "DHL_2"
|
|
# - "DHL_5"
|
|
# - "Hermes"
|
|
excluded_shipping_options: []
|
|
|
|
# maximum length for downloaded folder names (default: 100). does not limit downloaded file base names
|
|
folder_name_max_length: 100
|
|
|
|
# template for downloaded ad folder names. Default: "ad_{id}_{title}". Text outside {id} and {title} is copied literally. Allowed placeholders: {id}, {title}. Each placeholder may appear at most once. Template must include {id}; {title} is optional
|
|
# Examples (choose one):
|
|
# • "ad_{id}_{title}"
|
|
# • "ad_{id} {title}"
|
|
# • "{title} ({id})"
|
|
# • "{id}"
|
|
folder_name_template: ad_{id}_{title}
|
|
|
|
# template for the downloaded ad YAML stem and image prefix. Default: "ad_{id}". The bot writes the ad config as <base>.yaml and downloaded images as <base>__imgN.<ext>. Text outside {id} and {title} is copied literally. Supported placeholders: {id}, {title}. Each placeholder may appear at most once. Template must include {id}; {title} is optional. Long titles may be truncated to keep filename limits
|
|
# Examples (choose one):
|
|
# • "ad_{id}"
|
|
# • "ad_{id} {title}"
|
|
# • "{title} ({id})"
|
|
# • "{id}"
|
|
ad_file_name_template: ad_{id}
|
|
|
|
# if true, rename existing folders without titles to include titles (default: false)
|
|
rename_existing_folders: false
|
|
|
|
# if true, preserves local-only settings (auto_price_reduction, republication_interval, repost_count, price_reduction_count) when re-downloading an already saved ad. Useful for picking up live changes without losing local configuration.
|
|
preserve_local_settings: true
|
|
|
|
# ################################################################################
|
|
publishing:
|
|
|
|
# when to delete old versions of republished ads
|
|
# Examples (choose one):
|
|
# • BEFORE_PUBLISH
|
|
# • AFTER_PUBLISH
|
|
# • NEVER
|
|
delete_old_ads: AFTER_PUBLISH
|
|
|
|
# match old ads by title when deleting (only works with BEFORE_PUBLISH)
|
|
delete_old_ads_by_title: true
|
|
|
|
# local file and folder rename behavior after a successful publish changes the ad ID. When TEMPLATE_MATCH is enabled, the download.folder_name_template and download.ad_file_name_template are used to determine which paths qualify for renaming — only paths whose names match the template structure are updated.
|
|
local_path_renaming:
|
|
|
|
# rename local ad files/folders after a successful publish changes the ad ID. OFF keeps existing paths unchanged. TEMPLATE_MATCH only renames paths whose names match the structure defined by download.folder_name_template and download.ad_file_name_template. It replaces any numeric value inside the {id} slot and preserves all other text (including user-edited or previously truncated titles), so changing the download templates also controls which local paths are eligible for renaming.
|
|
# Examples (choose one):
|
|
# • OFF
|
|
# • TEMPLATE_MATCH
|
|
mode: OFF
|
|
|
|
# ################################################################################
|
|
# post-delete YAML cleanup configuration
|
|
deleting:
|
|
|
|
# what to do with the local ad YAML after a delete attempt (applies to both 200 and 404 responses)
|
|
# Examples (choose one):
|
|
# • NONE
|
|
# • RESET
|
|
# • DISABLE
|
|
after_delete: NONE
|
|
|
|
# ################################################################################
|
|
# Browser configuration
|
|
browser:
|
|
|
|
# additional Chromium command line switches (optional). Leave as [] for default behavior. See https://peter.sh/experiments/chromium-command-line-switches/ Common: --headless (no GUI), --disable-dev-shm-usage (Docker fix), --user-data-dir=/path
|
|
# Example usage:
|
|
# arguments:
|
|
# - "--headless"
|
|
# - "--disable-dev-shm-usage"
|
|
# - "--user-data-dir=/path/to/profile"
|
|
arguments: []
|
|
|
|
# path to custom browser executable (optional). Leave empty to use system default
|
|
binary_location: ''
|
|
|
|
# Chrome extensions to load (optional). Leave as [] for no extensions. Add .crx file paths relative to config file
|
|
# Example usage:
|
|
# extensions:
|
|
# - "extensions/adblock.crx"
|
|
# - "/absolute/path/to/extension.crx"
|
|
extensions: []
|
|
|
|
# open browser in private/incognito mode (recommended to avoid cookie conflicts)
|
|
use_private_window: true
|
|
|
|
# custom browser profile directory (optional). Leave empty for auto-configured default
|
|
user_data_dir: ''
|
|
|
|
# browser profile name (optional). Leave empty for default profile
|
|
# Example: "Profile 1"
|
|
profile_name: ''
|
|
|
|
# ################################################################################
|
|
# Login credentials
|
|
login:
|
|
|
|
# kleinanzeigen.de login email or username
|
|
username: changeme
|
|
|
|
# kleinanzeigen.de login password
|
|
password: changeme
|
|
|
|
# ################################################################################
|
|
captcha:
|
|
|
|
# if true, abort when captcha is detected and auto-retry after restart_delay (if false, wait for manual solving)
|
|
auto_restart: false
|
|
|
|
# duration to wait before retrying after captcha detection (e.g., 1h30m, 6h, 30m)
|
|
# Examples (choose one):
|
|
# • 6h
|
|
# • 1h30m
|
|
# • 30m
|
|
restart_delay: 6h
|
|
|
|
# ################################################################################
|
|
# Update check configuration
|
|
update_check:
|
|
|
|
# whether to check for updates on startup
|
|
enabled: true
|
|
|
|
# which release channel to check (latest = stable, preview = prereleases)
|
|
# Examples (choose one):
|
|
# • latest
|
|
# • preview
|
|
channel: latest
|
|
|
|
# how often to check for updates (e.g., 7d, 1d). If invalid, too short (<1d), or too long (>30d), uses defaults: 1d for 'preview' channel, 7d for 'latest' channel
|
|
# Examples (choose one):
|
|
# • 7d
|
|
# • 1d
|
|
# • 14d
|
|
interval: 7d
|
|
|
|
# ################################################################################
|
|
# Centralized timeout configuration.
|
|
timeouts:
|
|
|
|
# Global multiplier applied to all timeout values.
|
|
multiplier: 1.0
|
|
|
|
# Baseline timeout for DOM interactions.
|
|
default: 5.0
|
|
|
|
# Page load timeout for web_open.
|
|
page_load: 15.0
|
|
|
|
# Timeout for captcha iframe detection.
|
|
captcha_detection: 2.0
|
|
|
|
# Timeout for SMS verification prompts.
|
|
sms_verification: 5.0
|
|
|
|
# Timeout for email verification prompts.
|
|
email_verification: 5.0
|
|
|
|
# Timeout for detecting existing login session via DOM elements.
|
|
login_detection: 12.0
|
|
|
|
# Timeout for publishing result checks.
|
|
publishing_result: 300.0
|
|
|
|
# Timeout for publish confirmation redirect.
|
|
publishing_confirmation: 20.0
|
|
|
|
# Timeout for image upload and server-side processing.
|
|
image_upload: 30.0
|
|
|
|
# Timeout for initial pagination lookup.
|
|
pagination_initial: 10.0
|
|
|
|
# Timeout for subsequent pagination navigation.
|
|
pagination_follow_up: 5.0
|
|
|
|
# Generic short timeout for transient UI.
|
|
quick_dom: 2.0
|
|
|
|
# Timeout for GitHub update checks.
|
|
update_check: 10.0
|
|
|
|
# Timeout for local remote-debugging probes.
|
|
chrome_remote_probe: 2.0
|
|
|
|
# Timeout for remote debugging API calls.
|
|
chrome_remote_debugging: 5.0
|
|
|
|
# Timeout for chrome --version subprocesses.
|
|
chrome_binary_detection: 10.0
|
|
|
|
# Enable built-in retry/backoff for DOM operations.
|
|
retry_enabled: true
|
|
|
|
# Max retry attempts when retry is enabled.
|
|
retry_max_attempts: 2
|
|
|
|
# Exponential factor applied per retry attempt.
|
|
retry_backoff_factor: 1.5
|
|
|
|
# ################################################################################
|
|
# Human-like browser interaction settings to reduce automation detection.
|
|
humanization:
|
|
|
|
# master switch for all human-like interaction behavior
|
|
enabled: true
|
|
|
|
# click via real CDP mouse move + press/release instead of an instant element click (auto-falls back on failure)
|
|
mouse_movement: true
|
|
|
|
# type text character-by-character with variable per-keystroke delays instead of a single burst
|
|
typing_jitter: true
|
|
|
|
# minimum delay between individual keystrokes (ms)
|
|
typing_delay_min_ms: 40
|
|
|
|
# maximum delay between individual keystrokes (ms)
|
|
typing_delay_max_ms: 140
|
|
|
|
# minimum pause after interactions (ms); matches the historical web_sleep band
|
|
action_delay_min_ms: 1000
|
|
|
|
# maximum pause after interactions (ms); matches the historical web_sleep band
|
|
action_delay_max_ms: 2500
|
|
|
|
# probability of inserting a longer 'thinking' pause at boundaries
|
|
long_pause_probability: 0.1
|
|
|
|
# minimum duration of a 'thinking' pause (ms)
|
|
long_pause_min_ms: 1500
|
|
|
|
# maximum duration of a 'thinking' pause (ms)
|
|
long_pause_max_ms: 4000
|
|
|
|
# chance to run a random subset of idle micro-actions (scroll / mouse wiggle) at a page boundary
|
|
idle_action_probability: 0.3
|
|
|
|
# pick a random window size from viewport_sizes at launch (ignored if --window-size is set manually)
|
|
randomize_viewport: true
|
|
|
|
# whitelist of WxH desktop window sizes to randomly choose from when randomize_viewport is enabled
|
|
# Example usage:
|
|
# viewport_sizes:
|
|
# - "1920x1080"
|
|
# - "1366x768"
|
|
viewport_sizes:
|
|
- 1920x1080
|
|
- 1680x1050
|
|
- 1600x900
|
|
- 1536x864
|
|
- 1440x900
|
|
- 1366x768
|
|
|
|
# ################################################################################
|
|
# diagnostics capture configuration for troubleshooting
|
|
diagnostics:
|
|
|
|
# Enable diagnostics capture for specific operations.
|
|
capture_on:
|
|
|
|
# Capture screenshot and HTML when login state detection fails
|
|
login_detection: false
|
|
|
|
# Capture screenshot, HTML, and JSON on publish failures
|
|
publish: false
|
|
|
|
# If true, copy the entire bot log file when diagnostics are captured (may duplicate log content).
|
|
capture_log_copy: false
|
|
|
|
# If true, pause (interactive runs only) after capturing login detection diagnostics so that user can inspect the browser. Requires capture_on.login_detection to be enabled.
|
|
pause_on_login_detection_failure: false
|
|
|
|
# Optional output directory for diagnostics artifacts. If omitted, a safe default is used based on installation mode.
|
|
output_dir:
|
|
|
|
# If true, collect local timeout timing data and write it to diagnostics JSON for troubleshooting and tuning.
|
|
timing_collection: false
|