Mirrors/kleinanzeigen-bot
1
0
Fork 0
mirror of https://github.com/Second-Hand-Friends/kleinanzeigen-bot.git synced 2026-03-12 02:31:45 +01:00
Code Issues Packages Projects Releases Wiki Activity

refact: use ruff instead of autopep8,bandit,pylint for linting

Browse Source
This commit is contained in:
sebthom
2025-04-28 12:51:51 +02:00
parent f0b84ab335
commit 376ec76226
27 changed files with 437 additions and 605 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
.github/workflows/build.yml vendored
View File

@@ -134,12 +134,16 @@ jobs:
run: pdm show run: pdm show
- name: Security Audit - name: Check with ruff
run: pdm run audit run: pdm run ruff check
- name: Check code style - name: Check with mypy
run: pdm run lint run: pdm run mypy
- name: Check with pyright
run: pdm run pyright
- name: Run unit tests - name: Run unit tests

1
.gitignore vendored
View File

@@ -21,6 +21,7 @@ __pycache__
/.mypy_cache /.mypy_cache
/.pdm-build/ /.pdm-build/
/.pdm-python /.pdm-python
/.ruff_cache
# test coverage # test coverage
/.coverage /.coverage

254
pdm.lock generated
View File

@@ -5,7 +5,7 @@
groups = ["default", "dev"] groups = ["default", "dev"]
strategy = ["inherit_metadata"] strategy = ["inherit_metadata"]
lock_version = "4.5.0" lock_version = "4.5.0"
content_hash = "sha256:13c90956820a101f2b726823f3ecb63ce8c32b5923fee0a0b313c42df91d1e53" content_hash = "sha256:80e909dc69fd15a5392e613ee67a6bf196707436c64df6f8a4ba75c5ba8d974d"
[[metadata.targets]] [[metadata.targets]]
requires_python = ">=3.10,<3.14" requires_python = ">=3.10,<3.14"
@@ -20,20 +20,6 @@ files = [
{file = "altgraph-0.17.4.tar.gz", hash = "sha256:1b5afbb98f6c4dcadb2e2ae6ab9fa994bbb8c1d75f4fa96d340f9437ae454406"}, {file = "altgraph-0.17.4.tar.gz", hash = "sha256:1b5afbb98f6c4dcadb2e2ae6ab9fa994bbb8c1d75f4fa96d340f9437ae454406"},
] ]
[[package]]
name = "astroid"
version = "3.3.9"
requires_python = ">=3.9.0"
summary = "An abstract syntax tree for Python with inference support."
groups = ["dev"]
dependencies = [
"typing-extensions>=4.0.0; python_version < \"3.11\"",
]
files = [
{file = "astroid-3.3.9-py3-none-any.whl", hash = "sha256:d05bfd0acba96a7bd43e222828b7d9bc1e138aaeb0649707908d3702a9831248"},
{file = "astroid-3.3.9.tar.gz", hash = "sha256:622cc8e3048684aa42c820d9d218978021c3c3d174fb03a9f0d615921744f550"},
]
[[package]] [[package]]
name = "autocommand" name = "autocommand"
version = "2.2.2" version = "2.2.2"
@@ -72,23 +58,6 @@ files = [
{file = "backports_tarfile-1.2.0.tar.gz", hash = "sha256:d75e02c268746e1b8144c278978b6e98e85de6ad16f8e4b0844a154557eca991"}, {file = "backports_tarfile-1.2.0.tar.gz", hash = "sha256:d75e02c268746e1b8144c278978b6e98e85de6ad16f8e4b0844a154557eca991"},
] ]
[[package]]
name = "bandit"
version = "1.8.3"
requires_python = ">=3.9"
summary = "Security oriented static analyser for python code."
groups = ["dev"]
dependencies = [
"PyYAML>=5.3.1",
"colorama>=0.3.9; platform_system == \"Windows\"",
"rich",
"stevedore>=1.20.0",
]
files = [
{file = "bandit-1.8.3-py3-none-any.whl", hash = "sha256:28f04dc0d258e1dd0f99dee8eefa13d1cb5e3fde1a5ab0c523971f97b289bcd8"},
{file = "bandit-1.8.3.tar.gz", hash = "sha256:f5847beb654d309422985c36644649924e0ea4425c76dec2e89110b87506193a"},
]
[[package]] [[package]]
name = "bracex" name = "bracex"
version = "2.5.post1" version = "2.5.post1"
@@ -265,17 +234,6 @@ files = [
{file = "deprecated-1.2.18.tar.gz", hash = "sha256:422b6f6d859da6f2ef57857761bfb392480502a64c3028ca9bbe86085d72115d"}, {file = "deprecated-1.2.18.tar.gz", hash = "sha256:422b6f6d859da6f2ef57857761bfb392480502a64c3028ca9bbe86085d72115d"},
] ]
[[package]]
name = "dill"
version = "0.4.0"
requires_python = ">=3.8"
summary = "serialize all of Python"
groups = ["dev"]
files = [
{file = "dill-0.4.0-py3-none-any.whl", hash = "sha256:44f54bf6412c2c8464c14e8243eb163690a9800dbe2c367330883b19c7561049"},
{file = "dill-0.4.0.tar.gz", hash = "sha256:0633f1d2df477324f53a895b02c901fb961bdbf65a17122586ea7019292cbcf0"},
]
[[package]] [[package]]
name = "exceptiongroup" name = "exceptiongroup"
version = "1.2.2" version = "1.2.2"
@@ -299,17 +257,6 @@ files = [
{file = "iniconfig-2.1.0.tar.gz", hash = "sha256:3abbd2e30b36733fee78f9c7f7308f2d0050e88f0087fd25c2645f63c773e1c7"}, {file = "iniconfig-2.1.0.tar.gz", hash = "sha256:3abbd2e30b36733fee78f9c7f7308f2d0050e88f0087fd25c2645f63c773e1c7"},
] ]
[[package]]
name = "isort"
version = "6.0.1"
requires_python = ">=3.9.0"
summary = "A Python utility / library to sort Python imports."
groups = ["dev"]
files = [
{file = "isort-6.0.1-py3-none-any.whl", hash = "sha256:2dc5d7f65c9678d94c88dfc29161a320eec67328bc97aad576874cb4be1e9615"},
{file = "isort-6.0.1.tar.gz", hash = "sha256:1cb5df28dfbc742e490c5e41bad6da41b805b0a8be7bc93cd0fb2a8a890ac450"},
]
[[package]] [[package]]
name = "jaraco-context" name = "jaraco-context"
version = "6.0.1" version = "6.0.1"
@@ -370,42 +317,6 @@ files = [
{file = "macholib-1.16.3.tar.gz", hash = "sha256:07ae9e15e8e4cd9a788013d81f5908b3609aa76f9b1421bae9c4d7606ec86a30"}, {file = "macholib-1.16.3.tar.gz", hash = "sha256:07ae9e15e8e4cd9a788013d81f5908b3609aa76f9b1421bae9c4d7606ec86a30"},
] ]
[[package]]
name = "markdown-it-py"
version = "3.0.0"
requires_python = ">=3.8"
summary = "Python port of markdown-it. Markdown parsing, done right!"
groups = ["dev"]
dependencies = [
"mdurl~=0.1",
]
files = [
{file = "markdown-it-py-3.0.0.tar.gz", hash = "sha256:e3f60a94fa066dc52ec76661e37c851cb232d92f9886b15cb560aaada2df8feb"},
{file = "markdown_it_py-3.0.0-py3-none-any.whl", hash = "sha256:355216845c60bd96232cd8d8c40e8f9765cc86f46880e43a8fd22dc1a1a8cab1"},
]
[[package]]
name = "mccabe"
version = "0.7.0"
requires_python = ">=3.6"
summary = "McCabe checker, plugin for flake8"
groups = ["dev"]
files = [
{file = "mccabe-0.7.0-py2.py3-none-any.whl", hash = "sha256:6c2d30ab6be0e4a46919781807b4f0d834ebdd6c6e3dca0bda5a15f863427b6e"},
{file = "mccabe-0.7.0.tar.gz", hash = "sha256:348e0240c33b60bbdf4e523192ef919f28cb2c3d7d5c7794f74009290f236325"},
]
[[package]]
name = "mdurl"
version = "0.1.2"
requires_python = ">=3.7"
summary = "Markdown URL utilities"
groups = ["dev"]
files = [
{file = "mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8"},
{file = "mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba"},
]
[[package]] [[package]]
name = "more-itertools" name = "more-itertools"
version = "10.7.0" version = "10.7.0"
@@ -517,20 +428,6 @@ files = [
{file = "packaging-25.0.tar.gz", hash = "sha256:d443872c98d677bf60f6a1f2f8c1cb748e8fe762d2bf9d3148b5599295b0fc4f"}, {file = "packaging-25.0.tar.gz", hash = "sha256:d443872c98d677bf60f6a1f2f8c1cb748e8fe762d2bf9d3148b5599295b0fc4f"},
] ]
[[package]]
name = "pbr"
version = "6.1.1"
requires_python = ">=2.6"
summary = "Python Build Reasonableness"
groups = ["dev"]
dependencies = [
"setuptools",
]
files = [
{file = "pbr-6.1.1-py2.py3-none-any.whl", hash = "sha256:38d4daea5d9fa63b3f626131b9d34947fd0c8be9b05a29276870580050a25a76"},
{file = "pbr-6.1.1.tar.gz", hash = "sha256:93ea72ce6989eb2eed99d0f75721474f69ad88128afdef5ac377eb797c4bf76b"},
]
[[package]] [[package]]
name = "pefile" name = "pefile"
version = "2023.2.7" version = "2023.2.7"
@@ -593,17 +490,6 @@ files = [
{file = "pycodestyle-2.13.0.tar.gz", hash = "sha256:c8415bf09abe81d9c7f872502a6eee881fbe85d8763dd5b9924bb0a01d67efae"}, {file = "pycodestyle-2.13.0.tar.gz", hash = "sha256:c8415bf09abe81d9c7f872502a6eee881fbe85d8763dd5b9924bb0a01d67efae"},
] ]
[[package]]
name = "pygments"
version = "2.19.1"
requires_python = ">=3.8"
summary = "Pygments is a syntax highlighting package written in Python."
groups = ["dev"]
files = [
{file = "pygments-2.19.1-py3-none-any.whl", hash = "sha256:9ea1544ad55cecf4b8242fab6dd35a93bbce657034b0611ee383099054ab6d8c"},
{file = "pygments-2.19.1.tar.gz", hash = "sha256:61c16d2a8576dc0649d9f39e089b5f02bcd27fba10d8fb4dcc28173f7a45151f"},
]
[[package]] [[package]]
name = "pyinstaller" name = "pyinstaller"
version = "6.13.0" version = "6.13.0"
@@ -651,30 +537,6 @@ files = [
{file = "pyinstaller_hooks_contrib-2025.3.tar.gz", hash = "sha256:af129da5cd6219669fbda360e295cc822abac55b7647d03fec63a8fcf0a608cf"}, {file = "pyinstaller_hooks_contrib-2025.3.tar.gz", hash = "sha256:af129da5cd6219669fbda360e295cc822abac55b7647d03fec63a8fcf0a608cf"},
] ]
[[package]]
name = "pylint"
version = "3.3.6"
requires_python = ">=3.9.0"
summary = "python code static checker"
groups = ["dev"]
dependencies = [
"astroid<=3.4.0.dev0,>=3.3.8",
"colorama>=0.4.5; sys_platform == \"win32\"",
"dill>=0.2; python_version < \"3.11\"",
"dill>=0.3.6; python_version >= \"3.11\"",
"dill>=0.3.7; python_version >= \"3.12\"",
"isort!=5.13,<7,>=4.2.5",
"mccabe<0.8,>=0.6",
"platformdirs>=2.2",
"tomli>=1.1; python_version < \"3.11\"",
"tomlkit>=0.10.1",
"typing-extensions>=3.10; python_version < \"3.10\"",
]
files = [
{file = "pylint-3.3.6-py3-none-any.whl", hash = "sha256:8b7c2d3e86ae3f94fb27703d521dd0b9b6b378775991f504d7c3a6275aa0a6a6"},
{file = "pylint-3.3.6.tar.gz", hash = "sha256:b634a041aac33706d56a0d217e6587228c66427e20ec21a019bc4cdee48c040a"},
]
[[package]] [[package]]
name = "pyright" name = "pyright"
version = "1.1.400" version = "1.1.400"
@@ -766,68 +628,6 @@ files = [
{file = "pywin32_ctypes-0.2.3-py3-none-any.whl", hash = "sha256:8a1513379d709975552d202d942d9837758905c8d01eb82b8bcc30918929e7b8"}, {file = "pywin32_ctypes-0.2.3-py3-none-any.whl", hash = "sha256:8a1513379d709975552d202d942d9837758905c8d01eb82b8bcc30918929e7b8"},
] ]
[[package]]
name = "pyyaml"
version = "6.0.2"
requires_python = ">=3.8"
summary = "YAML parser and emitter for Python"
groups = ["dev"]
files = [
{file = "PyYAML-6.0.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0a9a2848a5b7feac301353437eb7d5957887edbf81d56e903999a75a3d743086"},
{file = "PyYAML-6.0.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:29717114e51c84ddfba879543fb232a6ed60086602313ca38cce623c1d62cfbf"},
{file = "PyYAML-6.0.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8824b5a04a04a047e72eea5cec3bc266db09e35de6bdfe34c9436ac5ee27d237"},
{file = "PyYAML-6.0.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7c36280e6fb8385e520936c3cb3b8042851904eba0e58d277dca80a5cfed590b"},
{file = "PyYAML-6.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ec031d5d2feb36d1d1a24380e4db6d43695f3748343d99434e6f5f9156aaa2ed"},
{file = "PyYAML-6.0.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:936d68689298c36b53b29f23c6dbb74de12b4ac12ca6cfe0e047bedceea56180"},
{file = "PyYAML-6.0.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:23502f431948090f597378482b4812b0caae32c22213aecf3b55325e049a6c68"},
{file = "PyYAML-6.0.2-cp310-cp310-win32.whl", hash = "sha256:2e99c6826ffa974fe6e27cdb5ed0021786b03fc98e5ee3c5bfe1fd5015f42b99"},
{file = "PyYAML-6.0.2-cp310-cp310-win_amd64.whl", hash = "sha256:a4d3091415f010369ae4ed1fc6b79def9416358877534caf6a0fdd2146c87a3e"},
{file = "PyYAML-6.0.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:cc1c1159b3d456576af7a3e4d1ba7e6924cb39de8f67111c735f6fc832082774"},
{file = "PyYAML-6.0.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1e2120ef853f59c7419231f3bf4e7021f1b936f6ebd222406c3b60212205d2ee"},
{file = "PyYAML-6.0.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5d225db5a45f21e78dd9358e58a98702a0302f2659a3c6cd320564b75b86f47c"},
{file = "PyYAML-6.0.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5ac9328ec4831237bec75defaf839f7d4564be1e6b25ac710bd1a96321cc8317"},
{file = "PyYAML-6.0.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ad2a3decf9aaba3d29c8f537ac4b243e36bef957511b4766cb0057d32b0be85"},
{file = "PyYAML-6.0.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:ff3824dc5261f50c9b0dfb3be22b4567a6f938ccce4587b38952d85fd9e9afe4"},
{file = "PyYAML-6.0.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:797b4f722ffa07cc8d62053e4cff1486fa6dc094105d13fea7b1de7d8bf71c9e"},
{file = "PyYAML-6.0.2-cp311-cp311-win32.whl", hash = "sha256:11d8f3dd2b9c1207dcaf2ee0bbbfd5991f571186ec9cc78427ba5bd32afae4b5"},
{file = "PyYAML-6.0.2-cp311-cp311-win_amd64.whl", hash = "sha256:e10ce637b18caea04431ce14fabcf5c64a1c61ec9c56b071a4b7ca131ca52d44"},
{file = "PyYAML-6.0.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:c70c95198c015b85feafc136515252a261a84561b7b1d51e3384e0655ddf25ab"},
{file = "PyYAML-6.0.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ce826d6ef20b1bc864f0a68340c8b3287705cae2f8b4b1d932177dcc76721725"},
{file = "PyYAML-6.0.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f71ea527786de97d1a0cc0eacd1defc0985dcf6b3f17bb77dcfc8c34bec4dc5"},
{file = "PyYAML-6.0.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9b22676e8097e9e22e36d6b7bda33190d0d400f345f23d4065d48f4ca7ae0425"},
{file = "PyYAML-6.0.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:80bab7bfc629882493af4aa31a4cfa43a4c57c83813253626916b8c7ada83476"},
{file = "PyYAML-6.0.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:0833f8694549e586547b576dcfaba4a6b55b9e96098b36cdc7ebefe667dfed48"},
{file = "PyYAML-6.0.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8b9c7197f7cb2738065c481a0461e50ad02f18c78cd75775628afb4d7137fb3b"},
{file = "PyYAML-6.0.2-cp312-cp312-win32.whl", hash = "sha256:ef6107725bd54b262d6dedcc2af448a266975032bc85ef0172c5f059da6325b4"},
{file = "PyYAML-6.0.2-cp312-cp312-win_amd64.whl", hash = "sha256:7e7401d0de89a9a855c839bc697c079a4af81cf878373abd7dc625847d25cbd8"},
{file = "PyYAML-6.0.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:efdca5630322a10774e8e98e1af481aad470dd62c3170801852d752aa7a783ba"},
{file = "PyYAML-6.0.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:50187695423ffe49e2deacb8cd10510bc361faac997de9efef88badc3bb9e2d1"},
{file = "PyYAML-6.0.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0ffe8360bab4910ef1b9e87fb812d8bc0a308b0d0eef8c8f44e0254ab3b07133"},
{file = "PyYAML-6.0.2-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:17e311b6c678207928d649faa7cb0d7b4c26a0ba73d41e99c4fff6b6c3276484"},
{file = "PyYAML-6.0.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:70b189594dbe54f75ab3a1acec5f1e3faa7e8cf2f1e08d9b561cb41b845f69d5"},
{file = "PyYAML-6.0.2-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:41e4e3953a79407c794916fa277a82531dd93aad34e29c2a514c2c0c5fe971cc"},
{file = "PyYAML-6.0.2-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:68ccc6023a3400877818152ad9a1033e3db8625d899c72eacb5a668902e4d652"},
{file = "PyYAML-6.0.2-cp313-cp313-win32.whl", hash = "sha256:bc2fa7c6b47d6bc618dd7fb02ef6fdedb1090ec036abab80d4681424b84c1183"},
{file = "PyYAML-6.0.2-cp313-cp313-win_amd64.whl", hash = "sha256:8388ee1976c416731879ac16da0aff3f63b286ffdd57cdeb95f3f2e085687563"},
{file = "pyyaml-6.0.2.tar.gz", hash = "sha256:d584d9ec91ad65861cc08d42e834324ef890a082e591037abe114850ff7bbc3e"},
]
[[package]]
name = "rich"
version = "14.0.0"
requires_python = ">=3.8.0"
summary = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal"
groups = ["dev"]
dependencies = [
"markdown-it-py>=2.2.0",
"pygments<3.0.0,>=2.13.0",
"typing-extensions<5.0,>=4.0.0; python_version < \"3.11\"",
]
files = [
{file = "rich-14.0.0-py3-none-any.whl", hash = "sha256:1c9491e1951aac09caffd42f448ee3d04e58923ffe14993f6e83068dc395d7e0"},
{file = "rich-14.0.0.tar.gz", hash = "sha256:82f1bc23a6a21ebca4ae0c45af9bdbc492ed20231dcb63f297d6d1021a9d5725"},
]
[[package]] [[package]]
name = "ruamel-yaml" name = "ruamel-yaml"
version = "0.18.10" version = "0.18.10"
@@ -889,6 +689,33 @@ files = [
{file = "ruamel.yaml.clib-0.2.12.tar.gz", hash = "sha256:6c8fbb13ec503f99a91901ab46e0b07ae7941cd527393187039aec586fdfd36f"}, {file = "ruamel.yaml.clib-0.2.12.tar.gz", hash = "sha256:6c8fbb13ec503f99a91901ab46e0b07ae7941cd527393187039aec586fdfd36f"},
] ]
[[package]]
name = "ruff"
version = "0.11.7"
requires_python = ">=3.7"
summary = "An extremely fast Python linter and code formatter, written in Rust."
groups = ["dev"]
files = [
{file = "ruff-0.11.7-py3-none-linux_armv6l.whl", hash = "sha256:d29e909d9a8d02f928d72ab7837b5cbc450a5bdf578ab9ebee3263d0a525091c"},
{file = "ruff-0.11.7-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:dd1fb86b168ae349fb01dd497d83537b2c5541fe0626e70c786427dd8363aaee"},
{file = "ruff-0.11.7-py3-none-macosx_11_0_arm64.whl", hash = "sha256:d3d7d2e140a6fbbc09033bce65bd7ea29d6a0adeb90b8430262fbacd58c38ada"},
{file = "ruff-0.11.7-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4809df77de390a1c2077d9b7945d82f44b95d19ceccf0c287c56e4dc9b91ca64"},
{file = "ruff-0.11.7-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f3a0c2e169e6b545f8e2dba185eabbd9db4f08880032e75aa0e285a6d3f48201"},
{file = "ruff-0.11.7-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:49b888200a320dd96a68e86736cf531d6afba03e4f6cf098401406a257fcf3d6"},
{file = "ruff-0.11.7-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:2b19cdb9cf7dae00d5ee2e7c013540cdc3b31c4f281f1dacb5a799d610e90db4"},
{file = "ruff-0.11.7-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:64e0ee994c9e326b43539d133a36a455dbaab477bc84fe7bfbd528abe2f05c1e"},
{file = "ruff-0.11.7-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:bad82052311479a5865f52c76ecee5d468a58ba44fb23ee15079f17dd4c8fd63"},
{file = "ruff-0.11.7-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7940665e74e7b65d427b82bffc1e46710ec7f30d58b4b2d5016e3f0321436502"},
{file = "ruff-0.11.7-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:169027e31c52c0e36c44ae9a9c7db35e505fee0b39f8d9fca7274a6305295a92"},
{file = "ruff-0.11.7-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:305b93f9798aee582e91e34437810439acb28b5fc1fee6b8205c78c806845a94"},
{file = "ruff-0.11.7-py3-none-musllinux_1_2_i686.whl", hash = "sha256:a681db041ef55550c371f9cd52a3cf17a0da4c75d6bd691092dfc38170ebc4b6"},
{file = "ruff-0.11.7-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:07f1496ad00a4a139f4de220b0c97da6d4c85e0e4aa9b2624167b7d4d44fd6b6"},
{file = "ruff-0.11.7-py3-none-win32.whl", hash = "sha256:f25dfb853ad217e6e5f1924ae8a5b3f6709051a13e9dad18690de6c8ff299e26"},
{file = "ruff-0.11.7-py3-none-win_amd64.whl", hash = "sha256:0a931d85959ceb77e92aea4bbedfded0a31534ce191252721128f77e5ae1f98a"},
{file = "ruff-0.11.7-py3-none-win_arm64.whl", hash = "sha256:778c1e5d6f9e91034142dfd06110534ca13220bfaad5c3735f6cb844654f6177"},
{file = "ruff-0.11.7.tar.gz", hash = "sha256:655089ad3224070736dc32844fde783454f8558e71f501cb207485fe4eee23d4"},
]
[[package]] [[package]]
name = "setuptools" name = "setuptools"
version = "79.0.1" version = "79.0.1"
@@ -900,20 +727,6 @@ files = [
{file = "setuptools-79.0.1.tar.gz", hash = "sha256:128ce7b8f33c3079fd1b067ecbb4051a66e8526e7b65f6cec075dfc650ddfa88"}, {file = "setuptools-79.0.1.tar.gz", hash = "sha256:128ce7b8f33c3079fd1b067ecbb4051a66e8526e7b65f6cec075dfc650ddfa88"},
] ]
[[package]]
name = "stevedore"
version = "5.4.1"
requires_python = ">=3.9"
summary = "Manage dynamic plugins for Python applications"
groups = ["dev"]
dependencies = [
"pbr>=2.0.0",
]
files = [
{file = "stevedore-5.4.1-py3-none-any.whl", hash = "sha256:d10a31c7b86cba16c1f6e8d15416955fc797052351a56af15e608ad20811fcfe"},
{file = "stevedore-5.4.1.tar.gz", hash = "sha256:3135b5ae50fe12816ef291baff420acb727fcd356106e3e9cbfa9e5985cd6f4b"},
]
[[package]] [[package]]
name = "tomli" name = "tomli"
version = "2.2.1" version = "2.2.1"
@@ -956,17 +769,6 @@ files = [
{file = "tomli-2.2.1.tar.gz", hash = "sha256:cd45e1dc79c835ce60f7404ec8119f2eb06d38b1deba146f07ced3bbc44505ff"}, {file = "tomli-2.2.1.tar.gz", hash = "sha256:cd45e1dc79c835ce60f7404ec8119f2eb06d38b1deba146f07ced3bbc44505ff"},
] ]
[[package]]
name = "tomlkit"
version = "0.13.2"
requires_python = ">=3.8"
summary = "Style preserving TOML library"
groups = ["dev"]
files = [
{file = "tomlkit-0.13.2-py3-none-any.whl", hash = "sha256:7a974427f6e119197f670fbbbeae7bef749a6c14e793db934baefc1b5f03efde"},
{file = "tomlkit-0.13.2.tar.gz", hash = "sha256:fff5fe59a87295b278abd31bec92c15d9bc4a06885ab12bcea52c71119392e79"},
]
[[package]] [[package]]
name = "typing-extensions" name = "typing-extensions"
version = "4.13.2" version = "4.13.2"

232
pyproject.toml
View File

@@ -53,13 +53,12 @@ dev = [
"pytest-cov>=6.0.0", "pytest-cov>=6.0.0",
# linting: # linting:
"autopep8", "autopep8",
"pylint", "ruff",
"mypy", "mypy",
"pyright", "pyright",
# security:
"bandit",
# packaging: # packaging:
"pyinstaller", "pyinstaller",
"platformdirs", # required by pyinstaller
] ]
[project.urls] [project.urls]
@@ -84,8 +83,8 @@ compile.cmd = "python -O -m PyInstaller pyinstaller.spec --clean"
compile.env = {PYTHONHASHSEED = "1", SOURCE_DATE_EPOCH = "0"} # https://pyinstaller.org/en/stable/advanced-topics.html#creating-a-reproducible-build compile.env = {PYTHONHASHSEED = "1", SOURCE_DATE_EPOCH = "0"} # https://pyinstaller.org/en/stable/advanced-topics.html#creating-a-reproducible-build
debug = "python -m pdb -m kleinanzeigen_bot" debug = "python -m pdb -m kleinanzeigen_bot"
format = "autopep8 --recursive --in-place src tests --verbose" format = "autopep8 --recursive --in-place src tests --verbose"
lint = {shell = "pylint -v src tests && autopep8 -v --exit-code --recursive --diff src tests && mypy" } lint = {shell = "ruff check && mypy && pyright" }
audit = "bandit -c pyproject.toml -r src" fix = {shell = "ruff check --fix" }
test = "python -m pytest --capture=tee-sys -v" test = "python -m pytest --capture=tee-sys -v"
utest = "python -m pytest --capture=tee-sys -v -m 'not itest'" utest = "python -m pytest --capture=tee-sys -v -m 'not itest'"
itest = "python -m pytest --capture=tee-sys -v -m 'itest'" itest = "python -m pytest --capture=tee-sys -v -m 'itest'"
@@ -109,11 +108,135 @@ aggressive = 3
##################### #####################
# bandit # ruff
# https://pypi.org/project/bandit/ # https://pypi.org/project/ruff/
# https://github.com/PyCQA/bandit # https://docs.astral.sh/ruff/configuration/
##################### #####################
[tool.bandit] [tool.ruff]
include = ["pyproject.toml", "src/**/*.py", "tests/**/*.py"]
line-length = 160
indent-width = 4
target-version = "py310"
[tool.ruff.lint]
# https://docs.astral.sh/ruff/rules/
select = [
"A", # flake8-builtins
"ARG", # flake8-unused-arguments
"ANN", # flake8-annotations
"ASYNC", # flake8-async
#"BLE", # flake8-blind-except
"B", # flake8-bugbear
"C4", # flake8-comprehensions
#"COM", # flake8-commas
#"CPY", # flake8-copyright
"DTZ", # flake8-datetimez
#"EM", # TODO flake8-errmsg
#"ERA", # eradicate commented-out code
"EXE", # flake8-executable
"FA", # flake8-future-annotations
"FBT", # flake8-boolean-trap
"FIX", # flake8-fixme
"G", # flake8-logging-format
"ICN", # flake8-import-conventions
"ISC", # flake8-implicit-str-concat
"INP", # flake8-no-pep420
"INT", # flake8-gettext
"LOG", # flake8-logging
"PIE", # flake8-pie
"PT", # flake8-pytest-style
#"PTH", # flake8-use-pathlib
"PYI", # flake8-pyi
"Q", # flake8-quotes
"RET", # flake8-return
"RSE", # flake8-raise
"S", # flake8-bandit
"SIM", # flake8-simplify
"SLF", # flake8-self
"SLOT", # flake8-slots
"T10", # flake8-debugger
#"T20", # flake8-print
"TC", # flake8-type-checking
"TD", # flake8-todo
"TID", # flake8-flake8-tidy-import
"YTT", # flake8-2020
"E", # pycodestyle-errors
"W", # pycodestyle-warnings
#"C90", # mccabe
"D", # pydocstyle
"F", # pyflakes
"FLY", # flynt
"I", # isort
"PERF", # perflint
"PGH", # pygrep-hooks
"PL", # pylint
]
ignore = [
"ANN401", # Dynamically typed expressions (typing.Any) are disallowed
"ASYNC210", # TODO Async functions should not call blocking HTTP methods
"ASYNC230", # TODO Async functions should not open files with blocking methods like `open`
"D1", # Missing docstring in ...
"D200", # One-line docstring should fit on one line
"D202", # No blank lines allowed after function docstring (found 1)
"D203", # 1 blank line required before class docstring
"D204", # 1 blank line required after class docstring
"D205", # 1 blank line required between summary line and description
"D209", # Multi-line docstring closing quotes should be on a separate line"
"D212", # Multi-line docstring summary should start at the first line
"D213", # Multi-line docstring summary should start at the second line
"D400", # First line should end with a period
"D401", # First line of docstring should be in imperative mood
"D402", # First line should not be the function's signature
"D404", # First word of the docstring should not be "This"
"D413", # Missing blank line after last section ("Returns")"
"D415", # First line should end with a period, question mark, or exclamation point
"D417", # Missing argument description in the docstring for
#"E124", # Don't change indention of multi-line statements
#"E128", # Don't change indention of multi-line statements
#"E231", # Don't add whitespace after colon (:) on type declaration
#"E251", # Don't remove whitespace around parameter '=' sign.
"E401", # Don't put imports on separate lines
"Q000", # TODO Single quotes found but double quotes preferred
"PERF203", # `try`-`except` within a loop incurs performance overhead
"RET504", # Unnecessary assignment to `...` before `return` statement
"PYI041", # Use `float` instead of `int | float`
"SIM102", # Use a single `if` statement instead of nested `if` statements
"SIM105", # Use `contextlib.suppress(TimeoutError)` instead of `try`-`except`-`pass`
"SIM114", # Combine `if` branches using logical `or` operator
"TC006", # Add quotes to type expression in `typing.cast()`
]
[tool.ruff.format]
quote-style = "double"
indent-style = "space"
line-ending = "native"
docstring-code-format = false
skip-magic-trailing-comma = false
[tool.ruff.lint.per-file-ignores]
"tests/**/*.py" = [
"ARG",
"B",
"FBT",
"INP",
"SLF",
"S101", # Use of `assert` detected
"PLR2004" # Magic value used in comparison
]
[tool.ruff.lint.pylint]
# https://pylint.pycqa.org/en/latest/user_guide/configuration/all-options.html#design-checker
# https://pylint.pycqa.org/en/latest/user_guide/checkers/features.html#design-checker-messages
#max-args = 6 # max. number of args for function / method (R0913)
#max-attributes = 15 # max. number of instance attrs for a class (R0902)
max-branches = 40 # max. number of branch for function / method body (R0912)
#max-locals = 30 # max. number of local vars for function / method body (R0914)
max-returns = 15 # max. number of return / yield for function / method body (R0911)
max-statements = 150 # max. number of statements in function / method body (R0915)
#max-public-methods = 30 # max. number of public methods for a class (R0904)
#max-positional-arguments = 6 # max. number of positional args for function / method (R0917)
##################### #####################
@@ -147,97 +270,6 @@ pythonVersion = "3.10"
typeCheckingMode = "standard" typeCheckingMode = "standard"
#####################
# pylint
# https://pypi.org/project/pylint/
# https://github.com/PyCQA/pylint
#####################
[tool.pylint.master]
extension-pkg-whitelist = "win32api"
ignore = "version.py"
jobs = 4
persistent = "no"
# https://pylint.pycqa.org/en/latest/user_guide/checkers/extensions.html
load-plugins = [
"pylint.extensions.bad_builtin",
#"pylint.extensions.broad_try_clause",
"pylint.extensions.check_elif",
"pylint.extensions.code_style",
"pylint.extensions.comparison_placement",
#"pylint.extensions.confusing_elif",
"pylint.extensions.consider_ternary_expression",
"pylint.extensions.consider_refactoring_into_while_condition",
"pylint.extensions.dict_init_mutate",
"pylint.extensions.docstyle",
#"pylint.extensions.docparams",
"pylint.extensions.dunder",
"pylint.extensions.empty_comment",
"pylint.extensions.eq_without_hash",
"pylint.extensions.for_any_all",
#"pylint.extensions.magic_value",
#"pylint.extensions.mccabe",
"pylint.extensions.set_membership",
"pylint.extensions.no_self_use",
"pylint.extensions.overlapping_exceptions",
"pylint.extensions.private_import",
"pylint.extensions.redefined_loop_name",
"pylint.extensions.redefined_variable_type",
"pylint.extensions.set_membership",
"pylint.extensions.typing",
#"pylint.extensions.while_used"
]
[tool.pylint.basic]
good-names = ["i", "j", "k", "v", "by", "ex", "fd", "_", "T"]
[tool.pylint.format]
# https://pylint.pycqa.org/en/latest/technical_reference/features.html#format-checker
# https://pylint.pycqa.org/en/latest/user_guide/checkers/features.html#format-checker-messages
max-line-length = 160 # maximum number of characters on a single line (C0301)
max-module-lines = 2000 # maximum number of lines in a module (C0302)
[tool.pylint.logging]
logging-modules = "logging"
[tool.pylint.messages_control]
# https://pylint.pycqa.org/en/latest/technical_reference/features.html#messages-control-options
disable = [
"broad-except",
"consider-using-assignment-expr",
"docstring-first-line-empty",
"global-statement",
"missing-docstring",
"multiple-imports",
"multiple-statements",
"no-self-use",
"no-member", # pylint cannot find async methods from super class
"too-few-public-methods"
]
[tool.pylint.tests]
# Configuration specific to test files
disable = [
"redefined-outer-name" # Allow pytest fixtures to be used as parameters
]
[tool.pylint.miscelaneous]
# https://pylint.pycqa.org/en/latest/user_guide/configuration/all-options.html#miscellaneous-checker
notes = [ "FIXME", "XXX", "TODO" ] # list of note tags to take in consideration
[tool.pylint.design]
# https://pylint.pycqa.org/en/latest/user_guide/configuration/all-options.html#design-checker
# https://pylint.pycqa.org/en/latest/user_guide/checkers/features.html#design-checker-messages
max-args = 6 # max. number of args for function / method (R0913)
max-attributes = 15 # max. number of instance attrs for a class (R0902)
max-branches = 40 # max. number of branch for function / method body (R0912)
max-locals = 30 # max. number of local vars for function / method body (R0914)
max-returns = 15 # max. number of return / yield for function / method body (R0911)
max-statements = 150 # max. number of statements in function / method body (R0915)
max-public-methods = 30 # max. number of public methods for a class (R0904)
max-positional-arguments = 6 # max. number of positional args for function / method (R0917)
##################### #####################
# pytest # pytest
# https://pypi.org/project/pytest/ # https://pypi.org/project/pytest/

51
src/kleinanzeigen_bot/__init__.py
View File

@@ -1,30 +1,26 @@
""" # SPDX-FileCopyrightText: © Sebastian Thomschke and contributors
SPDX-FileCopyrightText: © Sebastian Thomschke and contributors # SPDX-License-Identifier: AGPL-3.0-or-later
SPDX-License-Identifier: AGPL-3.0-or-later # SPDX-ArtifactOfProjectHomePage: https://github.com/Second-Hand-Friends/kleinanzeigen-bot/
SPDX-ArtifactOfProjectHomePage: https://github.com/Second-Hand-Friends/kleinanzeigen-bot/ import atexit, copy, json, os, re, signal, sys, textwrap # isort: skip
"""
import asyncio, atexit, copy, importlib.metadata, json, os, re, signal, shutil, sys, textwrap, time
import getopt # pylint: disable=deprecated-module import getopt # pylint: disable=deprecated-module
import urllib.parse as urllib_parse import urllib.parse as urllib_parse
import urllib.request as urllib_request
from collections.abc import Iterable from collections.abc import Iterable
from datetime import datetime
from gettext import gettext as _ from gettext import gettext as _
from typing import Any, Final from typing import Any, Final
import certifi, colorama, nodriver import certifi, colorama, nodriver # isort: skip
from ruamel.yaml import YAML from ruamel.yaml import YAML
from wcmatch import glob from wcmatch import glob
from . import extract, resources from . import extract, resources
from ._version import __version__ from ._version import __version__
from .ads import calculate_content_hash, get_description_affixes from .ads import MAX_DESCRIPTION_LENGTH, calculate_content_hash, get_description_affixes
from .utils import dicts, error_handlers, loggers, misc from .utils import dicts, error_handlers, loggers, misc
from .utils.exceptions import CaptchaEncountered from .utils.exceptions import CaptchaEncountered
from .utils.files import abspath from .utils.files import abspath
from .utils.i18n import Locale, get_current_locale, pluralize, set_current_locale from .utils.i18n import Locale, get_current_locale, pluralize, set_current_locale
from .utils.misc import ainput, ensure, is_frozen, parse_datetime, parse_decimal from .utils.misc import ainput, ensure, is_frozen, parse_datetime, parse_decimal
from .utils.web_scraping_mixin import By, Element, Is, Page, WebScrapingMixin from .utils.web_scraping_mixin import By, Element, Is, WebScrapingMixin
# W0406: possibly a bug, see https://github.com/PyCQA/pylint/issues/3933 # W0406: possibly a bug, see https://github.com/PyCQA/pylint/issues/3933
@@ -287,7 +283,7 @@ class KleinanzeigenBot(WebScrapingMixin):
return True return True
# Check republication interval # Check republication interval
ad_age = datetime.utcnow() - last_updated_on ad_age = misc.now() - last_updated_on
if ad_age.days <= ad_cfg["republication_interval"]: if ad_age.days <= ad_cfg["republication_interval"]:
LOG.info( LOG.info(
" -> SKIPPED: ad [%s] was last published %d days ago. republication is only required every %s days", " -> SKIPPED: ad [%s] was last published %d days ago. republication is only required every %s days",
@@ -394,12 +390,12 @@ class KleinanzeigenBot(WebScrapingMixin):
description = self.__get_description_with_affixes(ad_cfg) description = self.__get_description_with_affixes(ad_cfg)
# Validate total length # Validate total length
ensure(len(description) <= 4000, ensure(len(description) <= MAX_DESCRIPTION_LENGTH,
f"Length of ad description including prefix and suffix exceeds 4000 chars. " f"Length of ad description including prefix and suffix exceeds 4000 chars. "
f"Description length: {len(description)} chars. @ {ad_file}.") f"Description length: {len(description)} chars. @ {ad_file}.")
# pylint: disable=cell-var-from-loop
def assert_one_of(path:str, allowed:Iterable[str]) -> None: def assert_one_of(path:str, allowed:Iterable[str]) -> None:
# ruff: noqa: B023 function-uses-loop-variable
ensure(dicts.safe_get(ad_cfg, *path.split(".")) in allowed, f"-> property [{path}] must be one of: {allowed} @ [{ad_file}]") ensure(dicts.safe_get(ad_cfg, *path.split(".")) in allowed, f"-> property [{path}] must be one of: {allowed} @ [{ad_file}]")
def assert_min_len(path:str, minlen:int) -> None: def assert_min_len(path:str, minlen:int) -> None:
@@ -441,7 +437,7 @@ class KleinanzeigenBot(WebScrapingMixin):
ad_cfg["category"] = resolved_category_id ad_cfg["category"] = resolved_category_id
if ad_cfg["shipping_costs"]: if ad_cfg["shipping_costs"]:
ad_cfg["shipping_costs"] = str(round(misc.parse_decimal(ad_cfg["shipping_costs"]), 2)) ad_cfg["shipping_costs"] = str(round(parse_decimal(ad_cfg["shipping_costs"]), 2))
if ad_cfg["images"]: if ad_cfg["images"]:
images = [] images = []
@@ -564,17 +560,17 @@ class KleinanzeigenBot(WebScrapingMixin):
published_ads = json.loads( published_ads = json.loads(
(await self.web_request(f"{self.root_url}/m-meine-anzeigen-verwalten.json?sort=DEFAULT"))["content"])["ads"] (await self.web_request(f"{self.root_url}/m-meine-anzeigen-verwalten.json?sort=DEFAULT"))["content"])["ads"]
for (ad_file, ad_cfg, _) in ad_cfgs: for (ad_file, ad_cfg, _ad_cfg_orig) in ad_cfgs:
count += 1 count += 1
LOG.info("Processing %s/%s: '%s' from [%s]...", count, len(ad_cfgs), ad_cfg["title"], ad_file) LOG.info("Processing %s/%s: '%s' from [%s]...", count, len(ad_cfgs), ad_cfg["title"], ad_file)
await self.delete_ad(ad_cfg, self.config["publishing"]["delete_old_ads_by_title"], published_ads) await self.delete_ad(ad_cfg, published_ads, delete_old_ads_by_title = self.config["publishing"]["delete_old_ads_by_title"])
await self.web_sleep() await self.web_sleep()
LOG.info("############################################") LOG.info("############################################")
LOG.info("DONE: Deleted %s", pluralize("ad", count)) LOG.info("DONE: Deleted %s", pluralize("ad", count))
LOG.info("############################################") LOG.info("############################################")
async def delete_ad(self, ad_cfg: dict[str, Any], delete_old_ads_by_title: bool, published_ads: list[dict[str, Any]]) -> bool: async def delete_ad(self, ad_cfg: dict[str, Any], published_ads: list[dict[str, Any]], *, delete_old_ads_by_title: bool) -> bool:
LOG.info("Deleting ad '%s' if already present...", ad_cfg["title"]) LOG.info("Deleting ad '%s' if already present...", ad_cfg["title"])
await self.web_open(f"{self.root_url}/m-meine-anzeigen.html") await self.web_open(f"{self.root_url}/m-meine-anzeigen.html")
@@ -625,7 +621,7 @@ class KleinanzeigenBot(WebScrapingMixin):
await self.web_await(lambda: self.web_check(By.ID, "checking-done", Is.DISPLAYED), timeout = 5 * 60) await self.web_await(lambda: self.web_check(By.ID, "checking-done", Is.DISPLAYED), timeout = 5 * 60)
if self.config["publishing"]["delete_old_ads"] == "AFTER_PUBLISH" and not self.keep_old_ads: if self.config["publishing"]["delete_old_ads"] == "AFTER_PUBLISH" and not self.keep_old_ads:
await self.delete_ad(ad_cfg, False, published_ads) await self.delete_ad(ad_cfg, published_ads, delete_old_ads_by_title = False)
LOG.info("############################################") LOG.info("############################################")
LOG.info("DONE: (Re-)published %s", pluralize("ad", count)) LOG.info("DONE: (Re-)published %s", pluralize("ad", count))
@@ -640,7 +636,7 @@ class KleinanzeigenBot(WebScrapingMixin):
await self.assert_free_ad_limit_not_reached() await self.assert_free_ad_limit_not_reached()
if self.config["publishing"]["delete_old_ads"] == "BEFORE_PUBLISH" and not self.keep_old_ads: if self.config["publishing"]["delete_old_ads"] == "BEFORE_PUBLISH" and not self.keep_old_ads:
await self.delete_ad(ad_cfg, self.config["publishing"]["delete_old_ads_by_title"], published_ads) await self.delete_ad(ad_cfg, published_ads, delete_old_ads_by_title = self.config["publishing"]["delete_old_ads_by_title"])
LOG.info("Publishing ad '%s'...", ad_cfg["title"]) LOG.info("Publishing ad '%s'...", ad_cfg["title"])
@@ -828,7 +824,7 @@ class KleinanzeigenBot(WebScrapingMixin):
# Update content hash after successful publication # Update content hash after successful publication
# Calculate hash on original config to ensure consistent comparison on restart # Calculate hash on original config to ensure consistent comparison on restart
ad_cfg_orig["content_hash"] = calculate_content_hash(ad_cfg_orig) ad_cfg_orig["content_hash"] = calculate_content_hash(ad_cfg_orig)
ad_cfg_orig["updated_on"] = datetime.utcnow().isoformat() ad_cfg_orig["updated_on"] = misc.now().isoformat()
if not ad_cfg["created_on"] and not ad_cfg["id"]: if not ad_cfg["created_on"] and not ad_cfg["id"]:
ad_cfg_orig["created_on"] = ad_cfg_orig["updated_on"] ad_cfg_orig["created_on"] = ad_cfg_orig["updated_on"]
@@ -914,11 +910,11 @@ class KleinanzeigenBot(WebScrapingMixin):
raise TimeoutError(f"Failed to set special attribute [{special_attribute_key}] (not found)") from ex raise TimeoutError(f"Failed to set special attribute [{special_attribute_key}] (not found)") from ex
try: try:
elem_id = getattr(special_attr_elem.attrs, 'id') elem_id = special_attr_elem.attrs.id
if special_attr_elem.local_name == 'select': if special_attr_elem.local_name == 'select':
LOG.debug("Attribute field '%s' seems to be a select...", special_attribute_key) LOG.debug("Attribute field '%s' seems to be a select...", special_attribute_key)
await self.web_select(By.ID, elem_id, special_attribute_value) await self.web_select(By.ID, elem_id, special_attribute_value)
elif getattr(special_attr_elem.attrs, 'type') == 'checkbox': elif special_attr_elem.attrs.type == 'checkbox':
LOG.debug("Attribute field '%s' seems to be a checkbox...", special_attribute_key) LOG.debug("Attribute field '%s' seems to be a checkbox...", special_attribute_key)
await self.web_click(By.ID, elem_id) await self.web_click(By.ID, elem_id)
else: else:
@@ -950,7 +946,7 @@ class KleinanzeigenBot(WebScrapingMixin):
else: else:
try: try:
# no options. only costs. Set custom shipping cost # no options. only costs. Set custom shipping cost
if not ad_cfg["shipping_costs"] is None: if ad_cfg["shipping_costs"] is not None:
await self.web_click(By.XPATH, await self.web_click(By.XPATH,
'//*[contains(@class, "SubSection")]//*//button[contains(@class, "SelectionButton")]') '//*[contains(@class, "SubSection")]//*//button[contains(@class, "SelectionButton")]')
await self.web_click(By.XPATH, '//*[contains(@class, "CarrierSelectionModal")]//button[contains(text(),"Andere Versandmethoden")]') await self.web_click(By.XPATH, '//*[contains(@class, "CarrierSelectionModal")]//button[contains(text(),"Andere Versandmethoden")]')
@@ -984,7 +980,7 @@ class KleinanzeigenBot(WebScrapingMixin):
except KeyError as ex: except KeyError as ex:
raise KeyError(f"Unknown shipping option(s), please refer to the documentation/README: {ad_cfg['shipping_options']}") from ex raise KeyError(f"Unknown shipping option(s), please refer to the documentation/README: {ad_cfg['shipping_options']}") from ex
shipping_sizes, shipping_packages = zip(*mapped_shipping_options) shipping_sizes, shipping_packages = zip(*mapped_shipping_options, strict=False)
try: try:
shipping_size, = set(shipping_sizes) shipping_size, = set(shipping_sizes)
@@ -1159,8 +1155,9 @@ class KleinanzeigenBot(WebScrapingMixin):
final_description = final_description.replace("@", "(at)") final_description = final_description.replace("@", "(at)")
# Validate length # Validate length
ensure(len(final_description) <= 4000, ensure(len(final_description) <= MAX_DESCRIPTION_LENGTH,
f"Length of ad description including prefix and suffix exceeds 4000 chars. Description length: {len(final_description)} chars.") f"Length of ad description including prefix and suffix exceeds {MAX_DESCRIPTION_LENGTH} chars. "
f"Description length: {len(final_description)} chars.")
return final_description return final_description

11
src/kleinanzeigen_bot/__main__.py
View File

@@ -1,9 +1,7 @@
""" # SPDX-FileCopyrightText: © Sebastian Thomschke and contributors
SPDX-FileCopyrightText: © Sebastian Thomschke and contributors # SPDX-License-Identifier: AGPL-3.0-or-later
SPDX-License-Identifier: AGPL-3.0-or-later # SPDX-ArtifactOfProjectHomePage: https://github.com/Second-Hand-Friends/kleinanzeigen-bot/
SPDX-ArtifactOfProjectHomePage: https://github.com/Second-Hand-Friends/kleinanzeigen-bot/ import sys, time # isort: skip
"""
import sys, time
from gettext import gettext as _ from gettext import gettext as _
import kleinanzeigen_bot import kleinanzeigen_bot
@@ -17,7 +15,6 @@ while True:
try: try:
kleinanzeigen_bot.main(sys.argv) # runs & returns when finished kleinanzeigen_bot.main(sys.argv) # runs & returns when finished
sys.exit(0) # not using `break` to prevent process closing issues sys.exit(0) # not using `break` to prevent process closing issues
except CaptchaEncountered as ex: except CaptchaEncountered as ex:
delay = ex.restart_delay delay = ex.restart_delay
print(_("[INFO] Captcha detected. Sleeping %s before restart...") % format_timedelta(delay)) print(_("[INFO] Captcha detected. Sleeping %s before restart...") % format_timedelta(delay))

17
src/kleinanzeigen_bot/ads.py
View File

@@ -1,12 +1,13 @@
""" # SPDX-FileCopyrightText: © Jens Bergman and contributors
SPDX-FileCopyrightText: © Jens Bergman and contributors # SPDX-License-Identifier: AGPL-3.0-or-later
SPDX-License-Identifier: AGPL-3.0-or-later # SPDX-ArtifactOfProjectHomePage: https://github.com/Second-Hand-Friends/kleinanzeigen-bot/
SPDX-ArtifactOfProjectHomePage: https://github.com/Second-Hand-Friends/kleinanzeigen-bot/ import hashlib, json, os # isort: skip
""" from typing import Any, Final
import hashlib, json, os
from typing import Any
from .utils import dicts from .utils import dicts
MAX_DESCRIPTION_LENGTH:Final[int] = 4000
def calculate_content_hash(ad_cfg: dict[str, Any]) -> str: def calculate_content_hash(ad_cfg: dict[str, Any]) -> str:
"""Calculate a hash for user-modifiable fields of the ad.""" """Calculate a hash for user-modifiable fields of the ad."""
@@ -39,7 +40,7 @@ def calculate_content_hash(ad_cfg: dict[str, Any]) -> str:
return hashlib.sha256(content_str.encode()).hexdigest() return hashlib.sha256(content_str.encode()).hexdigest()
def get_description_affixes(config: dict[str, Any], prefix: bool = True) -> str: def get_description_affixes(config: dict[str, Any], *, prefix: bool = True) -> str:
"""Get prefix or suffix for description with proper precedence. """Get prefix or suffix for description with proper precedence.
This function handles both the new flattened format and legacy nested format: This function handles both the new flattened format and legacy nested format:

24
src/kleinanzeigen_bot/extract.py
View File

@@ -1,9 +1,7 @@
""" # SPDX-FileCopyrightText: © Sebastian Thomschke and contributors
SPDX-FileCopyrightText: © Sebastian Thomschke and contributors # SPDX-License-Identifier: AGPL-3.0-or-later
SPDX-License-Identifier: AGPL-3.0-or-later # SPDX-ArtifactOfProjectHomePage: https://github.com/Second-Hand-Friends/kleinanzeigen-bot/
SPDX-ArtifactOfProjectHomePage: https://github.com/Second-Hand-Friends/kleinanzeigen-bot/ import json, mimetypes, os, shutil # isort: skip
"""
import json, mimetypes, os, shutil
import urllib.request as urllib_request import urllib.request as urllib_request
from datetime import datetime from datetime import datetime
from typing import Any, Final from typing import Any, Final
@@ -24,7 +22,7 @@ class AdExtractor(WebScrapingMixin):
Wrapper class for ad extraction that uses an active bot´s browser session to extract specific elements from an ad page. Wrapper class for ad extraction that uses an active bot´s browser session to extract specific elements from an ad page.
""" """
def __init__(self, browser:Browser, config:dict[str, Any]): def __init__(self, browser:Browser, config:dict[str, Any]) -> None:
super().__init__() super().__init__()
self.browser = browser self.browser = browser
self.config = config self.config = config
@@ -84,7 +82,7 @@ class AdExtractor(WebScrapingMixin):
if current_img_url is None: if current_img_url is None:
continue continue
with urllib_request.urlopen(current_img_url) as response: # nosec B310 with urllib_request.urlopen(current_img_url) as response: # noqa: S310 Audit URL open for permitted schemes.
content_type = response.info().get_content_type() content_type = response.info().get_content_type()
file_ending = mimetypes.guess_extension(content_type) file_ending = mimetypes.guess_extension(content_type)
img_path = f"{directory}/{img_fn_prefix}{img_nr}{file_ending}" img_path = f"{directory}/{img_fn_prefix}{img_nr}{file_ending}"
@@ -170,7 +168,7 @@ class AdExtractor(WebScrapingMixin):
# This will now correctly trigger only if the '.Pagination' div itself is not found # This will now correctly trigger only if the '.Pagination' div itself is not found
LOG.info('No pagination controls found. Assuming single page.') LOG.info('No pagination controls found. Assuming single page.')
except Exception as e: except Exception as e:
LOG.error("Error during pagination detection: %s", e, exc_info=True) LOG.exception("Error during pagination detection: %s", e)
LOG.info('Assuming single page due to error during pagination check.') LOG.info('Assuming single page due to error during pagination check.')
# --- End Pagination Handling --- # --- End Pagination Handling ---
@@ -201,7 +199,7 @@ class AdExtractor(WebScrapingMixin):
LOG.info("Successfully extracted %s refs from page %s.", len(page_refs), current_page) LOG.info("Successfully extracted %s refs from page %s.", len(page_refs), current_page)
except Exception as e: except Exception as e:
# Log the error if extraction fails for some items, but try to continue # Log the error if extraction fails for some items, but try to continue
LOG.error("Error extracting refs on page %s: %s", current_page, e, exc_info=True) LOG.exception("Error extracting refs on page %s: %s", current_page, e)
if not multi_page: # only one iteration for single-page overview if not multi_page: # only one iteration for single-page overview
break break
@@ -232,7 +230,7 @@ class AdExtractor(WebScrapingMixin):
LOG.info("No pagination controls found after scrolling/waiting. Assuming last page.") LOG.info("No pagination controls found after scrolling/waiting. Assuming last page.")
break break
except Exception as e: except Exception as e:
LOG.error("Error during pagination navigation: %s", e, exc_info=True) LOG.exception("Error during pagination navigation: %s", e)
break break
# --- End Navigation --- # --- End Navigation ---
@@ -287,7 +285,7 @@ class AdExtractor(WebScrapingMixin):
# extract basic info # extract basic info
info['type'] = 'OFFER' if 's-anzeige' in self.page.url else 'WANTED' info['type'] = 'OFFER' if 's-anzeige' in self.page.url else 'WANTED'
title:str = await self.web_text(By.ID, 'viewad-title') title:str = await self.web_text(By.ID, 'viewad-title')
LOG.info('Extracting information from ad with title \"%s\"', title) LOG.info('Extracting information from ad with title "%s"', title)
info['category'] = await self._extract_category_from_ad_page() info['category'] = await self._extract_category_from_ad_page()
info['title'] = title info['title'] = title
@@ -389,7 +387,7 @@ class AdExtractor(WebScrapingMixin):
price = int(price_str.replace('.', '').split()[0]) price = int(price_str.replace('.', '').split()[0])
case 'VB': case 'VB':
price_type = 'NEGOTIABLE' price_type = 'NEGOTIABLE'
if not price_str == "VB": # can be either 'X € VB', or just 'VB' if price_str != "VB": # can be either 'X € VB', or just 'VB'
price = int(price_str.replace('.', '').split()[0]) price = int(price_str.replace('.', '').split()[0])
case 'verschenken': case 'verschenken':
price_type = 'GIVE_AWAY' price_type = 'GIVE_AWAY'

21
src/kleinanzeigen_bot/utils/dicts.py
View File

@@ -1,16 +1,15 @@
""" # SPDX-FileCopyrightText: © Sebastian Thomschke and contributors
SPDX-FileCopyrightText: © Sebastian Thomschke and contributors # SPDX-License-Identifier: AGPL-3.0-or-later
SPDX-License-Identifier: AGPL-3.0-or-later # SPDX-ArtifactOfProjectHomePage: https://github.com/Second-Hand-Friends/kleinanzeigen-bot/
SPDX-ArtifactOfProjectHomePage: https://github.com/Second-Hand-Friends/kleinanzeigen-bot/ import copy, json, os # isort: skip
"""
import copy, json, os
from collections.abc import Callable from collections.abc import Callable
from importlib.resources import read_text as get_resource_as_string
from gettext import gettext as _ from gettext import gettext as _
from importlib.resources import read_text as get_resource_as_string
from types import ModuleType from types import ModuleType
from typing import Any, Final from typing import Any, Final
from ruamel.yaml import YAML from ruamel.yaml import YAML
from . import files, loggers # pylint: disable=cyclic-import from . import files, loggers # pylint: disable=cyclic-import
LOG:Final[loggers.Logger] = loggers.get_logger(__name__) LOG:Final[loggers.Logger] = loggers.get_logger(__name__)
@@ -112,9 +111,9 @@ def safe_get(a_map:dict[Any, Any], *keys:str) -> Any:
'some_value' 'some_value'
""" """
if a_map: if a_map:
for key in keys: try:
try: for key in keys:
a_map = a_map[key] a_map = a_map[key]
except (KeyError, TypeError): except (KeyError, TypeError):
return None return None
return a_map return a_map

12
src/kleinanzeigen_bot/utils/error_handlers.py
View File

@@ -1,11 +1,9 @@
""" # SPDX-FileCopyrightText: © Sebastian Thomschke and contributors
SPDX-FileCopyrightText: © Sebastian Thomschke and contributors # SPDX-License-Identifier: AGPL-3.0-or-later
SPDX-License-Identifier: AGPL-3.0-or-later # SPDX-ArtifactOfProjectHomePage: https://github.com/Second-Hand-Friends/kleinanzeigen-bot/
SPDX-ArtifactOfProjectHomePage: https://github.com/Second-Hand-Friends/kleinanzeigen-bot/ import sys, traceback # isort: skip
"""
import sys, traceback
from types import FrameType, TracebackType from types import FrameType, TracebackType
from typing import Final from typing import Any, Final
from . import loggers from . import loggers

10
src/kleinanzeigen_bot/utils/exceptions.py
View File

@@ -1,8 +1,6 @@
""" # SPDX-FileCopyrightText: © Sebastian Thomschke and contributors
SPDX-FileCopyrightText: © Sebastian Thomschke and contributors # SPDX-License-Identifier: AGPL-3.0-or-later
SPDX-License-Identifier: AGPL-3.0-or-later # SPDX-ArtifactOfProjectHomePage: https://github.com/Second-Hand-Friends/kleinanzeigen-bot/
SPDX-ArtifactOfProjectHomePage: https://github.com/Second-Hand-Friends/kleinanzeigen-bot/
"""
from datetime import timedelta from datetime import timedelta
@@ -13,6 +11,6 @@ class KleinanzeigenBotError(RuntimeError):
class CaptchaEncountered(KleinanzeigenBotError): class CaptchaEncountered(KleinanzeigenBotError):
"""Raised when a Captcha was detected and auto-restart is enabled.""" """Raised when a Captcha was detected and auto-restart is enabled."""
def __init__(self, restart_delay: timedelta): def __init__(self, restart_delay: timedelta) -> None:
super().__init__() super().__init__()
self.restart_delay = restart_delay self.restart_delay = restart_delay

8
src/kleinanzeigen_bot/utils/files.py
View File

@@ -1,8 +1,6 @@
""" # SPDX-FileCopyrightText: © Sebastian Thomschke and contributors
SPDX-FileCopyrightText: © Sebastian Thomschke and contributors # SPDX-License-Identifier: AGPL-3.0-or-later
SPDX-License-Identifier: AGPL-3.0-or-later # SPDX-ArtifactOfProjectHomePage: https://github.com/Second-Hand-Friends/kleinanzeigen-bot/
SPDX-ArtifactOfProjectHomePage: https://github.com/Second-Hand-Friends/kleinanzeigen-bot/
"""
import os import os

27
src/kleinanzeigen_bot/utils/i18n.py
View File

@@ -1,13 +1,12 @@
""" # SPDX-FileCopyrightText: © Sebastian Thomschke and contributors
SPDX-FileCopyrightText: © Sebastian Thomschke and contributors # SPDX-License-Identifier: AGPL-3.0-or-later
SPDX-License-Identifier: AGPL-3.0-or-later # SPDX-ArtifactOfProjectHomePage: https://github.com/Second-Hand-Friends/kleinanzeigen-bot/
SPDX-ArtifactOfProjectHomePage: https://github.com/Second-Hand-Friends/kleinanzeigen-bot/ import ctypes, gettext, inspect, locale, logging, os, sys # isort: skip
"""
import ctypes, gettext, inspect, locale, logging, os, sys
from collections.abc import Sized from collections.abc import Sized
from typing import Any, Final, NamedTuple from typing import Any, Final, NamedTuple
from kleinanzeigen_bot import resources from kleinanzeigen_bot import resources
from . import dicts, reflect from . import dicts, reflect
__all__ = [ __all__ = [
@@ -96,7 +95,7 @@ def translate(text:object, caller: inspect.FrameInfo | None) -> str:
if not caller: if not caller:
return text return text
global _TRANSLATIONS global _TRANSLATIONS # noqa: PLW0603 Using the global statement to update `...` is discouraged
if _TRANSLATIONS is None: if _TRANSLATIONS is None:
try: try:
_TRANSLATIONS = dicts.load_dict_from_module(resources, f"translations.{_CURRENT_LOCALE[0]}.yaml") _TRANSLATIONS = dicts.load_dict_from_module(resources, f"translations.{_CURRENT_LOCALE[0]}.yaml")
@@ -125,10 +124,10 @@ gettext.gettext = lambda message: translate(_original_gettext(message), reflect.
for module_name, module in sys.modules.items(): for module_name, module in sys.modules.items():
if module is None or module_name in sys.builtin_module_names: if module is None or module_name in sys.builtin_module_names:
continue continue
if hasattr(module, '_') and getattr(module, '_') is _original_gettext: if hasattr(module, '_') and module._ is _original_gettext:
setattr(module, '_', gettext.gettext) module._ = gettext.gettext # type: ignore[attr-defined]
if hasattr(module, 'gettext') and getattr(module, 'gettext') is _original_gettext: if hasattr(module, 'gettext') and module.gettext is _original_gettext:
setattr(module, 'gettext', gettext.gettext) module.gettext = gettext.gettext # type: ignore[attr-defined]
def get_current_locale() -> Locale: def get_current_locale() -> Locale:
@@ -136,13 +135,13 @@ def get_current_locale() -> Locale:
def set_current_locale(new_locale:Locale) -> None: def set_current_locale(new_locale:Locale) -> None:
global _CURRENT_LOCALE, _TRANSLATIONS global _CURRENT_LOCALE, _TRANSLATIONS # noqa: PLW0603 Using the global statement to update `...` is discouraged
if new_locale.language != _CURRENT_LOCALE.language: if new_locale.language != _CURRENT_LOCALE.language:
_TRANSLATIONS = None _TRANSLATIONS = None
_CURRENT_LOCALE = new_locale _CURRENT_LOCALE = new_locale
def pluralize(noun:str, count:int | Sized, prefix_with_count:bool = True) -> str: def pluralize(noun:str, count:int | Sized, *, prefix_with_count:bool = True) -> str:
""" """
>>> set_current_locale(Locale("en")) # Setup for doctests >>> set_current_locale(Locale("en")) # Setup for doctests
>>> pluralize("field", 1) >>> pluralize("field", 1)
@@ -189,7 +188,7 @@ def pluralize(noun:str, count:int | Sized, prefix_with_count:bool = True) -> str
return f"{prefix}{noun}e" # Hund -> Hunde return f"{prefix}{noun}e" # Hund -> Hunde
# English # English
if len(noun) < 2: if len(noun) < 2: # noqa: PLR2004 Magic value used in comparison
return f"{prefix}{noun}s" return f"{prefix}{noun}s"
if noun.endswith(('s', 'sh', 'ch', 'x', 'z')): if noun.endswith(('s', 'sh', 'ch', 'x', 'z')):
return f"{prefix}{noun}es" return f"{prefix}{noun}es"

29
src/kleinanzeigen_bot/utils/loggers.py
View File

@@ -1,15 +1,14 @@
""" # SPDX-FileCopyrightText: © Sebastian Thomschke and contributors
SPDX-FileCopyrightText: © Sebastian Thomschke and contributors # SPDX-License-Identifier: AGPL-3.0-or-later
SPDX-License-Identifier: AGPL-3.0-or-later # SPDX-ArtifactOfProjectHomePage: https://github.com/Second-Hand-Friends/kleinanzeigen-bot/
SPDX-ArtifactOfProjectHomePage: https://github.com/Second-Hand-Friends/kleinanzeigen-bot/ import copy, logging, re, sys # isort: skip
"""
import copy, logging, re, sys
from gettext import gettext as _ from gettext import gettext as _
from logging import Logger, DEBUG, INFO, WARNING, ERROR, CRITICAL from logging import CRITICAL, DEBUG, ERROR, INFO, WARNING, Logger
from logging.handlers import RotatingFileHandler from logging.handlers import RotatingFileHandler
from typing import Any, Final # @UnusedImport from typing import Any, Final # @UnusedImport
import colorama import colorama
from . import i18n, reflect from . import i18n, reflect
__all__ = [ __all__ = [
@@ -27,6 +26,16 @@ __all__ = [
LOG_ROOT:Final[logging.Logger] = logging.getLogger() LOG_ROOT:Final[logging.Logger] = logging.getLogger()
class _MaxLevelFilter(logging.Filter):
def __init__(self, level: int) -> None:
super().__init__()
self.level = level
def filter(self, record: logging.LogRecord) -> bool:
return record.levelno <= self.level
def configure_console_logging() -> None: def configure_console_logging() -> None:
# if a StreamHandler already exists, do not append it again # if a StreamHandler already exists, do not append it again
if any(isinstance(h, logging.StreamHandler) for h in LOG_ROOT.handlers): if any(isinstance(h, logging.StreamHandler) for h in LOG_ROOT.handlers):
@@ -82,9 +91,7 @@ def configure_console_logging() -> None:
stdout_log = logging.StreamHandler(sys.stderr) stdout_log = logging.StreamHandler(sys.stderr)
stdout_log.setLevel(DEBUG) stdout_log.setLevel(DEBUG)
stdout_log.addFilter(type("", (logging.Filter,), { stdout_log.addFilter(_MaxLevelFilter(INFO))
"filter": lambda rec: rec.levelno <= INFO
}))
stdout_log.setFormatter(formatter) stdout_log.setFormatter(formatter)
LOG_ROOT.addHandler(stdout_log) LOG_ROOT.addHandler(stdout_log)
@@ -97,7 +104,7 @@ def configure_console_logging() -> None:
class LogFileHandle: class LogFileHandle:
"""Encapsulates a log file handler with close and status methods.""" """Encapsulates a log file handler with close and status methods."""
def __init__(self, file_path: str, handler: RotatingFileHandler, logger: logging.Logger): def __init__(self, file_path: str, handler: RotatingFileHandler, logger: logging.Logger) -> None:
self.file_path = file_path self.file_path = file_path
self._handler:RotatingFileHandler | None = handler self._handler:RotatingFileHandler | None = handler
self._logger = logger self._logger = logger

50
src/kleinanzeigen_bot/utils/misc.py
View File

@@ -1,11 +1,9 @@
""" # SPDX-FileCopyrightText: © Sebastian Thomschke and contributors
SPDX-FileCopyrightText: © Sebastian Thomschke and contributors # SPDX-License-Identifier: AGPL-3.0-or-later
SPDX-License-Identifier: AGPL-3.0-or-later # SPDX-ArtifactOfProjectHomePage: https://github.com/Second-Hand-Friends/kleinanzeigen-bot/
SPDX-ArtifactOfProjectHomePage: https://github.com/Second-Hand-Friends/kleinanzeigen-bot/ import asyncio, decimal, re, sys, time # isort: skip
"""
import asyncio, decimal, re, sys, time
from collections.abc import Callable from collections.abc import Callable
from datetime import datetime, timedelta from datetime import datetime, timedelta, timezone
from gettext import gettext as _ from gettext import gettext as _
from typing import Any, TypeVar from typing import Any, TypeVar
@@ -39,6 +37,10 @@ def ensure(condition:Any | bool | Callable[[], bool], error_message:str, timeout
time.sleep(poll_requency) time.sleep(poll_requency)
def now() -> datetime:
return datetime.now(timezone.utc)
def is_frozen() -> bool: def is_frozen() -> bool:
""" """
>>> is_frozen() >>> is_frozen()
@@ -81,12 +83,27 @@ def parse_decimal(number:float | int | str) -> decimal.Decimal:
raise decimal.DecimalException(f"Invalid number format: {number}") from ex raise decimal.DecimalException(f"Invalid number format: {number}") from ex
def parse_datetime(date:datetime | str | None) -> datetime | None: def parse_datetime(
date: datetime | str | None,
*,
add_timezone_if_missing: bool = True,
use_local_timezone: bool = True
) -> datetime | None:
""" """
>>> parse_datetime(datetime(2020, 1, 1, 0, 0)) Parses a datetime object or ISO-formatted string.
Args:
date: The input datetime object or ISO string.
add_timezone_if_missing: If True, add timezone info if missing.
use_local_timezone: If True, use local timezone; otherwise UTC if adding timezone.
Returns:
A timezone-aware or naive datetime object, depending on parameters.
>>> parse_datetime(datetime(2020, 1, 1, 0, 0), add_timezone_if_missing = False)
datetime.datetime(2020, 1, 1, 0, 0) datetime.datetime(2020, 1, 1, 0, 0)
>>> parse_datetime("2020-01-01T00:00:00") >>> parse_datetime("2020-01-01T00:00:00", add_timezone_if_missing = False)
datetime.datetime(2020, 1, 1, 0, 0) datetime.datetime(2020, 1, 1, 0, 0)
>>> parse_datetime(None) >>> parse_datetime(None)
@@ -94,9 +111,16 @@ def parse_datetime(date:datetime | str | None) -> datetime | None:
""" """
if date is None: if date is None:
return None return None
if isinstance(date, datetime):
return date dt = date if isinstance(date, datetime) else datetime.fromisoformat(date)
return datetime.fromisoformat(date)
if dt.tzinfo is None and add_timezone_if_missing:
dt = (
dt.astimezone() if use_local_timezone
else dt.replace(tzinfo = timezone.utc)
)
return dt
def parse_duration(text:str) -> timedelta: def parse_duration(text:str) -> timedelta:

8
src/kleinanzeigen_bot/utils/net.py
View File

@@ -1,8 +1,6 @@
""" # SPDX-FileCopyrightText: © Sebastian Thomschke and contributors
SPDX-FileCopyrightText: © Sebastian Thomschke and contributors # SPDX-License-Identifier: AGPL-3.0-or-later
SPDX-License-Identifier: AGPL-3.0-or-later # SPDX-ArtifactOfProjectHomePage: https://github.com/Second-Hand-Friends/kleinanzeigen-bot/
SPDX-ArtifactOfProjectHomePage: https://github.com/Second-Hand-Friends/kleinanzeigen-bot/
"""
import socket import socket

8
src/kleinanzeigen_bot/utils/reflect.py
View File

@@ -1,8 +1,6 @@
""" # SPDX-FileCopyrightText: © Sebastian Thomschke and contributors
SPDX-FileCopyrightText: © Sebastian Thomschke and contributors # SPDX-License-Identifier: AGPL-3.0-or-later
SPDX-License-Identifier: AGPL-3.0-or-later # SPDX-ArtifactOfProjectHomePage: https://github.com/Second-Hand-Friends/kleinanzeigen-bot/
SPDX-ArtifactOfProjectHomePage: https://github.com/Second-Hand-Friends/kleinanzeigen-bot/
"""
import inspect import inspect
from typing import Any from typing import Any

65
src/kleinanzeigen_bot/utils/web_scraping_mixin.py
View File

@@ -1,26 +1,24 @@
""" # SPDX-FileCopyrightText: © Sebastian Thomschke and contributors
SPDX-FileCopyrightText: © Sebastian Thomschke and contributors # SPDX-License-Identifier: AGPL-3.0-or-later
SPDX-License-Identifier: AGPL-3.0-or-later # SPDX-ArtifactOfProjectHomePage: https://github.com/Second-Hand-Friends/kleinanzeigen-bot/
SPDX-ArtifactOfProjectHomePage: https://github.com/Second-Hand-Friends/kleinanzeigen-bot/ import asyncio, enum, inspect, json, os, platform, secrets, shutil # isort: skip
"""
import asyncio, enum, inspect, json, os, platform, secrets, shutil, time
from collections.abc import Callable, Coroutine, Iterable from collections.abc import Callable, Coroutine, Iterable
from gettext import gettext as _ from gettext import gettext as _
from typing import cast, Any, Final from typing import Any, Final, cast
try: try:
from typing import Never # type: ignore[attr-defined,unused-ignore] # mypy from typing import Never # type: ignore[attr-defined,unused-ignore] # mypy
except ImportError: except ImportError:
from typing import NoReturn as Never # Python <3.11 from typing import NoReturn as Never # Python <3.11
import nodriver, psutil import nodriver, psutil # isort: skip
from nodriver.core.browser import Browser from nodriver.core.browser import Browser
from nodriver.core.config import Config from nodriver.core.config import Config
from nodriver.core.element import Element from nodriver.core.element import Element
from nodriver.core.tab import Tab as Page from nodriver.core.tab import Tab as Page
from . import loggers, net from . import loggers, net
from .misc import ensure, T from .misc import T, ensure
__all__ = [ __all__ = [
"Browser", "Browser",
@@ -70,8 +68,8 @@ class WebScrapingMixin:
def __init__(self) -> None: def __init__(self) -> None:
self.browser_config:Final[BrowserConfig] = BrowserConfig() self.browser_config:Final[BrowserConfig] = BrowserConfig()
self.browser:Browser = None # pyright: ignore self.browser:Browser = None # pyright: ignore[reportAttributeAccessIssue]
self.page:Page = None # pyright: ignore self.page:Page = None # pyright: ignore[reportAttributeAccessIssue]
async def create_browser_session(self) -> None: async def create_browser_session(self) -> None:
LOG.info("Creating Browser session...") LOG.info("Creating Browser session...")
@@ -96,7 +94,7 @@ class WebScrapingMixin:
if remote_port > 0: if remote_port > 0:
LOG.info("Using existing browser process at %s:%s", remote_host, remote_port) LOG.info("Using existing browser process at %s:%s", remote_host, remote_port)
ensure(net.is_port_open(remote_host, remote_port), ensure(net.is_port_open(remote_host, remote_port),
f"Browser process not reachable at {remote_host}:{remote_port}. " + f"Browser process not reachable at {remote_host}:{remote_port}. "
f"Start the browser with --remote-debugging-port={remote_port} or remove this port from your config.yaml") f"Start the browser with --remote-debugging-port={remote_port} or remove this port from your config.yaml")
cfg = Config( cfg = Config(
browser_executable_path = self.browser_config.binary_location # actually not necessary but nodriver fails without browser_executable_path = self.browser_config.binary_location # actually not necessary but nodriver fails without
@@ -208,14 +206,14 @@ class WebScrapingMixin:
def close_browser_session(self) -> None: def close_browser_session(self) -> None:
if self.browser: if self.browser:
LOG.debug("Closing Browser session...") LOG.debug("Closing Browser session...")
self.page = None # pyright: ignore self.page = None # pyright: ignore[reportAttributeAccessIssue]
browser_process = psutil.Process(self.browser._process_pid) # pylint: disable=protected-access browser_process = psutil.Process(self.browser._process_pid) # noqa: SLF001 Private member accessed
browser_children:list[psutil.Process] = browser_process.children() browser_children:list[psutil.Process] = browser_process.children()
self.browser.stop() self.browser.stop()
for p in browser_children: for p in browser_children:
if p.is_running(): if p.is_running():
p.kill() # terminate orphaned browser processes p.kill() # terminate orphaned browser processes
self.browser = None # pyright: ignore self.browser = None # pyright: ignore[reportAttributeAccessIssue]
def get_compatible_browser(self) -> str: def get_compatible_browser(self) -> str:
match platform.system(): match platform.system():
@@ -236,15 +234,15 @@ class WebScrapingMixin:
case "Windows": case "Windows":
browser_paths = [ browser_paths = [
os.environ.get("ProgramFiles", "C:\\Program Files") + r'\Microsoft\Edge\Application\msedge.exe', os.environ.get("PROGRAMFILES", "C:\\Program Files") + r'\Microsoft\Edge\Application\msedge.exe',
os.environ.get("ProgramFiles(x86)", "C:\\Program Files (x86)") + r'\Microsoft\Edge\Application\msedge.exe', os.environ.get("PROGRAMFILES(X86)", "C:\\Program Files (x86)") + r'\Microsoft\Edge\Application\msedge.exe',
os.environ["ProgramFiles"] + r'\Chromium\Application\chrome.exe', os.environ["PROGRAMFILES"] + r'\Chromium\Application\chrome.exe',
os.environ["ProgramFiles(x86)"] + r'\Chromium\Application\chrome.exe', os.environ["PROGRAMFILES(X86)"] + r'\Chromium\Application\chrome.exe',
os.environ["LOCALAPPDATA"] + r'\Chromium\Application\chrome.exe', os.environ["LOCALAPPDATA"] + r'\Chromium\Application\chrome.exe',
os.environ["ProgramFiles"] + r'\Chrome\Application\chrome.exe', os.environ["PROGRAMFILES"] + r'\Chrome\Application\chrome.exe',
os.environ["ProgramFiles(x86)"] + r'\Chrome\Application\chrome.exe', os.environ["PROGRAMFILES(X86)"] + r'\Chrome\Application\chrome.exe',
os.environ["LOCALAPPDATA"] + r'\Chrome\Application\chrome.exe', os.environ["LOCALAPPDATA"] + r'\Chrome\Application\chrome.exe',
shutil.which("msedge.exe"), shutil.which("msedge.exe"),
@@ -277,7 +275,7 @@ class WebScrapingMixin:
ex:Exception | None = None ex:Exception | None = None
try: try:
result_raw = condition() result_raw = condition()
result:T = (await result_raw) if inspect.isawaitable(result_raw) else result_raw result:T = cast(T, await result_raw if inspect.isawaitable(result_raw) else result_raw)
if result: if result:
return result return result
except Exception as ex1: except Exception as ex1:
@@ -359,11 +357,11 @@ class WebScrapingMixin:
_prev_jscode:str = getattr(self.__class__.web_execute, "_prev_jscode", "") _prev_jscode:str = getattr(self.__class__.web_execute, "_prev_jscode", "")
if not (jscode == _prev_jscode or (jscode.startswith("window.scrollTo") and _prev_jscode.startswith("window.scrollTo"))): if not (jscode == _prev_jscode or (jscode.startswith("window.scrollTo") and _prev_jscode.startswith("window.scrollTo"))):
LOG.debug("web_execute(`%s`) = `%s`", jscode, result) LOG.debug("web_execute(`%s`) = `%s`", jscode, result)
self.__class__.web_execute._prev_jscode = jscode # type: ignore[attr-defined] # pylint: disable=protected-access self.__class__.web_execute._prev_jscode = jscode # type: ignore[attr-defined] # noqa: SLF001 Private member accessed
return result return result
async def web_find(self, selector_type:By, selector_value:str, *, parent:Element = None, timeout:int | float = 5) -> Element: async def web_find(self, selector_type:By, selector_value:str, *, parent:Element | None = None, timeout:int | float = 5) -> Element:
""" """
Locates an HTML element by the given selector type and value. Locates an HTML element by the given selector type and value.
@@ -408,7 +406,7 @@ class WebScrapingMixin:
raise AssertionError(_("Unsupported selector type: %s") % selector_type) raise AssertionError(_("Unsupported selector type: %s") % selector_type)
async def web_find_all(self, selector_type:By, selector_value:str, *, parent:Element = None, timeout:int | float = 5) -> list[Element]: async def web_find_all(self, selector_type:By, selector_value:str, *, parent:Element | None = None, timeout:int | float = 5) -> list[Element]:
""" """
Locates an HTML element by ID. Locates an HTML element by ID.
@@ -460,7 +458,7 @@ class WebScrapingMixin:
await self.web_sleep() await self.web_sleep()
return input_field return input_field
async def web_open(self, url:str, *, timeout:int | float = 15000, reload_if_already_open:bool = False) -> None: async def web_open(self, url:str, *, timeout:int | float = 15_000, reload_if_already_open:bool = False) -> None:
""" """
:param url: url to open in browser :param url: url to open in browser
:param timeout: timespan in seconds within the page needs to be loaded :param timeout: timespan in seconds within the page needs to be loaded
@@ -475,7 +473,7 @@ class WebScrapingMixin:
await self.web_await(lambda: self.web_execute("document.readyState == 'complete'"), timeout = timeout, await self.web_await(lambda: self.web_execute("document.readyState == 'complete'"), timeout = timeout,
timeout_error_message = f"Page did not finish loading within {timeout} seconds.") timeout_error_message = f"Page did not finish loading within {timeout} seconds.")
async def web_text(self, selector_type:By, selector_value:str, *, parent:Element = None, timeout:int | float = 5) -> str: async def web_text(self, selector_type:By, selector_value:str, *, parent:Element | None = None, timeout:int | float = 5) -> str:
return str(await (await self.web_find(selector_type, selector_value, parent = parent, timeout = timeout)).apply(""" return str(await (await self.web_find(selector_type, selector_value, parent = parent, timeout = timeout)).apply("""
function (elem) { function (elem) {
let sel = window.getSelection() let sel = window.getSelection()
@@ -489,10 +487,11 @@ class WebScrapingMixin:
} }
""")) """))
async def web_sleep(self, min_ms:int = 1000, max_ms:int = 2500) -> None: async def web_sleep(self, min_ms:int = 1_000, max_ms:int = 2_500) -> None:
duration = max_ms <= min_ms and min_ms or secrets.randbelow(max_ms - min_ms) + min_ms duration = max_ms <= min_ms and min_ms or secrets.randbelow(max_ms - min_ms) + min_ms
LOG.log(loggers.INFO if duration > 1500 else loggers.DEBUG, " ... pausing for %d ms ...", duration) LOG.log(loggers.INFO if duration > 1_500 else loggers.DEBUG, # noqa: PLR2004 Magic value used in comparison
await self.page.sleep(duration / 1000) " ... pausing for %d ms ...", duration)
await self.page.sleep(duration / 1_000)
async def web_request(self, url:str, method:str = "GET", valid_response_codes:int | Iterable[int] = 200, async def web_request(self, url:str, method:str = "GET", valid_response_codes:int | Iterable[int] = 200,
headers:dict[str, str] | None = None) -> dict[str, Any]: headers:dict[str, str] | None = None) -> dict[str, Any]:
@@ -524,7 +523,7 @@ class WebScrapingMixin:
return response return response
# pylint: enable=dangerous-default-value # pylint: enable=dangerous-default-value
async def web_scroll_page_down(self, scroll_length: int = 10, scroll_speed: int = 10000, scroll_back_top: bool = False) -> None: async def web_scroll_page_down(self, scroll_length: int = 10, scroll_speed: int = 10_000, *, scroll_back_top: bool = False) -> None:
""" """
Smoothly scrolls the current web page down. Smoothly scrolls the current web page down.
@@ -537,13 +536,13 @@ class WebScrapingMixin:
while current_y_pos < bottom_y_pos: # scroll in steps until bottom reached while current_y_pos < bottom_y_pos: # scroll in steps until bottom reached
current_y_pos += scroll_length current_y_pos += scroll_length
await self.web_execute(f'window.scrollTo(0, {current_y_pos})') # scroll one step await self.web_execute(f'window.scrollTo(0, {current_y_pos})') # scroll one step
time.sleep(scroll_length / scroll_speed) await asyncio.sleep(scroll_length / scroll_speed)
if scroll_back_top: # scroll back to top in same style if scroll_back_top: # scroll back to top in same style
while current_y_pos > 0: while current_y_pos > 0:
current_y_pos -= scroll_length current_y_pos -= scroll_length
await self.web_execute(f'window.scrollTo(0, {current_y_pos})') await self.web_execute(f'window.scrollTo(0, {current_y_pos})')
time.sleep(scroll_length / scroll_speed / 2) # double speed await asyncio.sleep(scroll_length / scroll_speed / 2) # double speed
async def web_select(self, selector_type:By, selector_value:str, selected_value:Any, timeout:int | float = 5) -> Element: async def web_select(self, selector_type:By, selector_value:str, selected_value:Any, timeout:int | float = 5) -> Element:
""" """

2
tests/conftest.py
View File

@@ -10,8 +10,8 @@ from unittest.mock import MagicMock
import pytest import pytest
from kleinanzeigen_bot import KleinanzeigenBot from kleinanzeigen_bot import KleinanzeigenBot
from kleinanzeigen_bot.utils import loggers
from kleinanzeigen_bot.extract import AdExtractor from kleinanzeigen_bot.extract import AdExtractor
from kleinanzeigen_bot.utils import loggers
from kleinanzeigen_bot.utils.web_scraping_mixin import Browser from kleinanzeigen_bot.utils.web_scraping_mixin import Browser
loggers.configure_console_logging() loggers.configure_console_logging()

14
tests/integration/test_web_scraping_mixin.py
View File

@@ -1,12 +1,12 @@
""" # SPDX-FileCopyrightText: © Sebastian Thomschke and contributors
SPDX-FileCopyrightText: © Sebastian Thomschke and contributors # SPDX-License-Identifier: AGPL-3.0-or-later
SPDX-License-Identifier: AGPL-3.0-or-later # SPDX-ArtifactOfProjectHomePage: https://github.com/Second-Hand-Friends/kleinanzeigen-bot/
SPDX-ArtifactOfProjectHomePage: https://github.com/Second-Hand-Friends/kleinanzeigen-bot/ import os
""" import platform
import os, platform
from typing import cast from typing import cast
import nodriver, pytest import nodriver
import pytest
from kleinanzeigen_bot.utils import loggers from kleinanzeigen_bot.utils import loggers
from kleinanzeigen_bot.utils.misc import ensure from kleinanzeigen_bot.utils.misc import ensure

28
tests/unit/test_ads.py
View File

@@ -1,8 +1,6 @@
""" # SPDX-FileCopyrightText: © Sebastian Thomschke and contributors
SPDX-FileCopyrightText: © Sebastian Thomschke and contributors # SPDX-License-Identifier: AGPL-3.0-or-later
SPDX-License-Identifier: AGPL-3.0-or-later # SPDX-ArtifactOfProjectHomePage: https://github.com/Second-Hand-Friends/kleinanzeigen-bot/
SPDX-ArtifactOfProjectHomePage: https://github.com/Second-Hand-Friends/kleinanzeigen-bot/
"""
from typing import Any from typing import Any
import pytest import pytest
@@ -33,7 +31,7 @@ def test_calculate_content_hash_with_none_values() -> None:
assert len(hash_value) == 64 # SHA-256 hash is 64 characters long assert len(hash_value) == 64 # SHA-256 hash is 64 characters long
@pytest.mark.parametrize("config,prefix,expected", [ @pytest.mark.parametrize(("config", "prefix", "expected"), [
# Test new flattened format - prefix # Test new flattened format - prefix
( (
{"ad_defaults": {"description_prefix": "Hello"}}, {"ad_defaults": {"description_prefix": "Hello"}},
@@ -129,11 +127,11 @@ def test_get_description_affixes(
expected: str expected: str
) -> None: ) -> None:
"""Test get_description_affixes function with various inputs.""" """Test get_description_affixes function with various inputs."""
result = ads.get_description_affixes(config, prefix) result = ads.get_description_affixes(config, prefix = prefix)
assert result == expected assert result == expected
@pytest.mark.parametrize("config,prefix,expected", [ @pytest.mark.parametrize(("config", "prefix", "expected"), [
# Add test for malformed config # Add test for malformed config
( (
{}, # Empty config {}, # Empty config
@@ -161,16 +159,16 @@ def test_get_description_affixes(
]) ])
def test_get_description_affixes_edge_cases(config: dict[str, Any], prefix: bool, expected: str) -> None: def test_get_description_affixes_edge_cases(config: dict[str, Any], prefix: bool, expected: str) -> None:
"""Test edge cases for description affix handling.""" """Test edge cases for description affix handling."""
assert ads.get_description_affixes(config, prefix) == expected assert ads.get_description_affixes(config, prefix = prefix) == expected
@pytest.mark.parametrize("config,expected", [ @pytest.mark.parametrize(("config", "expected"), [
(None, ""), # Test with None (None, ""), # Test with None
([], ""), # Test with an empty list ([], ""), # Test with an empty list
("string", ""), # Test with a string ("string", ""), # Test with a string
(123, ""), # Test with an integer (123, ""), # Test with an integer
(3.14, ""), # Test with a float (3.14, ""), # Test with a float
(set(), ""), # Test with an empty set (set(), ""), # Test with an empty set
]) ])
def test_get_description_affixes_edge_cases_non_dict(config: Any, expected: str) -> None: def test_get_description_affixes_edge_cases_non_dict(config: Any, expected: str) -> None:
"""Test get_description_affixes function with non-dict inputs.""" """Test get_description_affixes function with non-dict inputs."""

11
tests/unit/test_bot.py
View File

@@ -1,9 +1,8 @@
""" # SPDX-FileCopyrightText: © Sebastian Thomschke and contributors
SPDX-FileCopyrightText: © Sebastian Thomschke and contributors # SPDX-License-Identifier: AGPL-3.0-or-later
SPDX-License-Identifier: AGPL-3.0-or-later # SPDX-ArtifactOfProjectHomePage: https://github.com/Second-Hand-Friends/kleinanzeigen-bot/
SPDX-ArtifactOfProjectHomePage: https://github.com/Second-Hand-Friends/kleinanzeigen-bot/ import gc, pytest # isort: skip
"""
import gc, pytest
from kleinanzeigen_bot import KleinanzeigenBot from kleinanzeigen_bot import KleinanzeigenBot

79
tests/unit/test_extract.py
View File

@@ -1,9 +1,7 @@
""" # SPDX-FileCopyrightText: © Sebastian Thomschke and contributors
SPDX-FileCopyrightText: © Jens Bergmann and contributors # SPDX-License-Identifier: AGPL-3.0-or-later
SPDX-License-Identifier: AGPL-3.0-or-later # SPDX-ArtifactOfProjectHomePage: https://github.com/Second-Hand-Friends/kleinanzeigen-bot/
SPDX-ArtifactOfProjectHomePage: https://github.com/Second-Hand-Friends/kleinanzeigen-bot/ import json, os # isort: skip
"""
import json, os
from typing import Any, TypedDict from typing import Any, TypedDict
from unittest.mock import AsyncMock, MagicMock, call, patch from unittest.mock import AsyncMock, MagicMock, call, patch
@@ -30,7 +28,7 @@ class _SpecialAttributesDict(TypedDict, total = False):
condition_s: str condition_s: str
class _TestCaseDict(TypedDict): class _TestCaseDict(TypedDict): # noqa: PYI049 Private TypedDict `...` is never used
belen_conf: _BelenConfDict belen_conf: _BelenConfDict
expected: _SpecialAttributesDict expected: _SpecialAttributesDict
@@ -44,15 +42,12 @@ class TestAdExtractorBasics:
assert extractor.browser == browser_mock assert extractor.browser == browser_mock
assert extractor.config == sample_config assert extractor.config == sample_config
@pytest.mark.parametrize( @pytest.mark.parametrize(("url", "expected_id"), [
"url,expected_id", ("https://www.kleinanzeigen.de/s-anzeige/test-title/12345678", 12345678),
[ ("https://www.kleinanzeigen.de/s-anzeige/another-test/98765432", 98765432),
("https://www.kleinanzeigen.de/s-anzeige/test-title/12345678", 12345678), ("https://www.kleinanzeigen.de/s-anzeige/invalid-id/abc", -1),
("https://www.kleinanzeigen.de/s-anzeige/another-test/98765432", 98765432), ("https://www.kleinanzeigen.de/invalid-url", -1),
("https://www.kleinanzeigen.de/s-anzeige/invalid-id/abc", -1), ])
("https://www.kleinanzeigen.de/invalid-url", -1),
],
)
def test_extract_ad_id_from_ad_url(self, test_extractor: AdExtractor, url: str, expected_id: int) -> None: def test_extract_ad_id_from_ad_url(self, test_extractor: AdExtractor, url: str, expected_id: int) -> None:
"""Test extraction of ad ID from different URL formats.""" """Test extraction of ad ID from different URL formats."""
assert test_extractor.extract_ad_id_from_ad_url(url) == expected_id assert test_extractor.extract_ad_id_from_ad_url(url) == expected_id
@@ -61,16 +56,13 @@ class TestAdExtractorBasics:
class TestAdExtractorPricing: class TestAdExtractorPricing:
"""Tests for pricing related functionality.""" """Tests for pricing related functionality."""
@pytest.mark.parametrize( @pytest.mark.parametrize(("price_text", "expected_price", "expected_type"), [
"price_text,expected_price,expected_type", ("50 €", 50, "FIXED"),
[ ("1.234 €", 1234, "FIXED"),
("50 €", 50, "FIXED"), ("50 € VB", 50, "NEGOTIABLE"),
("1.234 €", 1234, "FIXED"), ("VB", None, "NEGOTIABLE"),
("50 € VB", 50, "NEGOTIABLE"), ("Zu verschenken", None, "GIVE_AWAY"),
("VB", None, "NEGOTIABLE"), ])
("Zu verschenken", None, "GIVE_AWAY"),
],
)
@pytest.mark.asyncio @pytest.mark.asyncio
# pylint: disable=protected-access # pylint: disable=protected-access
async def test_extract_pricing_info( async def test_extract_pricing_info(
@@ -95,14 +87,11 @@ class TestAdExtractorPricing:
class TestAdExtractorShipping: class TestAdExtractorShipping:
"""Tests for shipping related functionality.""" """Tests for shipping related functionality."""
@pytest.mark.parametrize( @pytest.mark.parametrize(("shipping_text", "expected_type", "expected_cost"), [
"shipping_text,expected_type,expected_cost", ("+ Versand ab 2,99 €", "SHIPPING", 2.99),
[ ("Nur Abholung", "PICKUP", None),
("+ Versand ab 2,99 €", "SHIPPING", 2.99), ("Versand möglich", "SHIPPING", None),
("Nur Abholung", "PICKUP", None), ])
("Versand möglich", "SHIPPING", None),
],
)
@pytest.mark.asyncio @pytest.mark.asyncio
# pylint: disable=protected-access # pylint: disable=protected-access
async def test_extract_shipping_info( async def test_extract_shipping_info(
@@ -272,9 +261,9 @@ class TestAdExtractorNavigation:
# Mocks needed for the actual execution flow # Mocks needed for the actual execution flow
ad_list_container_mock = MagicMock() ad_list_container_mock = MagicMock()
pagination_section_mock = MagicMock() pagination_section_mock = MagicMock()
cardbox_mock = MagicMock() # Represents the <li> element cardbox_mock = MagicMock() # Represents the <li> element
link_mock = MagicMock() # Represents the <a> element link_mock = MagicMock() # Represents the <a> element
link_mock.attrs = {'href': '/s-anzeige/test/12345'} # Configure the desired output link_mock.attrs = {'href': '/s-anzeige/test/12345'} # Configure the desired output
# Mocks for elements potentially checked but maybe not strictly needed for output # Mocks for elements potentially checked but maybe not strictly needed for output
# (depending on how robust the mocking is) # (depending on how robust the mocking is)
@@ -287,18 +276,18 @@ class TestAdExtractorNavigation:
# 3. Find for ad list container (inside loop) # 3. Find for ad list container (inside loop)
# 4. Find for the link (inside list comprehension) # 4. Find for the link (inside list comprehension)
mock_web_find.side_effect = [ mock_web_find.side_effect = [
ad_list_container_mock, # Call 1: find #my-manageitems-adlist (before loop) ad_list_container_mock, # Call 1: find #my-manageitems-adlist (before loop)
pagination_section_mock, # Call 2: find .Pagination pagination_section_mock, # Call 2: find .Pagination
ad_list_container_mock, # Call 3: find #my-manageitems-adlist (inside loop) ad_list_container_mock, # Call 3: find #my-manageitems-adlist (inside loop)
link_mock # Call 4: find 'div.manageitems-item-ad h3 a.text-onSurface' link_mock # Call 4: find 'div.manageitems-item-ad h3 a.text-onSurface'
# Add more mocks here if the pagination navigation logic calls web_find again # Add more mocks here if the pagination navigation logic calls web_find again
] ]
# 1. Find all 'Nächste' buttons (pagination check) - Return empty list for single page test case # 1. Find all 'Nächste' buttons (pagination check) - Return empty list for single page test case
# 2. Find all '.cardbox' elements (inside loop) # 2. Find all '.cardbox' elements (inside loop)
mock_web_find_all.side_effect = [ mock_web_find_all.side_effect = [
[], # Call 1: find 'button[aria-label="Nächste"]' -> No next button = single page [], # Call 1: find 'button[aria-label="Nächste"]' -> No next button = single page
[cardbox_mock] # Call 2: find .cardbox -> One ad item [cardbox_mock] # Call 2: find .cardbox -> One ad item
# Add more mocks here if pagination navigation calls web_find_all # Add more mocks here if pagination navigation calls web_find_all
] ]
@@ -550,9 +539,9 @@ class TestAdExtractorContact:
"""Test contact info extraction when elements are not found.""" """Test contact info extraction when elements are not found."""
with patch.object(extractor, 'page', MagicMock()), \ with patch.object(extractor, 'page', MagicMock()), \
patch.object(extractor, 'web_text', new_callable = AsyncMock, side_effect = TimeoutError()), \ patch.object(extractor, 'web_text', new_callable = AsyncMock, side_effect = TimeoutError()), \
patch.object(extractor, 'web_find', new_callable = AsyncMock, side_effect = TimeoutError()): patch.object(extractor, 'web_find', new_callable = AsyncMock, side_effect = TimeoutError()), \
pytest.raises(TimeoutError):
with pytest.raises(TimeoutError):
await extractor._extract_contact_from_ad_page() await extractor._extract_contact_from_ad_page()
@pytest.mark.asyncio @pytest.mark.asyncio

15
tests/unit/test_i18n.py
View File

@@ -1,14 +1,13 @@
""" # SPDX-FileCopyrightText: © Sebastian Thomschke and contributors
SPDX-FileCopyrightText: © Sebastian Thomschke and contributors # SPDX-License-Identifier: AGPL-3.0-or-later
SPDX-License-Identifier: AGPL-3.0-or-later # SPDX-ArtifactOfProjectHomePage: https://github.com/Second-Hand-Friends/kleinanzeigen-bot/
SPDX-ArtifactOfProjectHomePage: https://github.com/Second-Hand-Friends/kleinanzeigen-bot/
"""
import pytest import pytest
from _pytest.monkeypatch import MonkeyPatch # pylint: disable=import-private-name from _pytest.monkeypatch import MonkeyPatch # pylint: disable=import-private-name
from kleinanzeigen_bot.utils import i18n from kleinanzeigen_bot.utils import i18n
@pytest.mark.parametrize("lang, expected", [ @pytest.mark.parametrize(("lang", "expected"), [
(None, ("en", "US", "UTF-8")), # Test with no LANG variable (should default to ("en", "US", "UTF-8")) (None, ("en", "US", "UTF-8")), # Test with no LANG variable (should default to ("en", "US", "UTF-8"))
("fr", ("fr", None, "UTF-8")), # Test with just a language code ("fr", ("fr", None, "UTF-8")), # Test with just a language code
("fr_CA", ("fr", "CA", "UTF-8")), # Test with language + region, no encoding ("fr_CA", ("fr", "CA", "UTF-8")), # Test with language + region, no encoding
@@ -29,7 +28,7 @@ def test_detect_locale(monkeypatch: MonkeyPatch, lang: str | None, expected: i18
assert result == expected, f"For LANG={lang}, expected {expected} but got {result}" assert result == expected, f"For LANG={lang}, expected {expected} but got {result}"
@pytest.mark.parametrize("lang, noun, count, prefix_with_count, expected", [ @pytest.mark.parametrize(("lang", "noun", "count", "prefix_with_count", "expected"), [
("en", "field", 1, True, "1 field"), ("en", "field", 1, True, "1 field"),
("en", "field", 2, True, "2 fields"), ("en", "field", 2, True, "2 fields"),
("en", "field", 2, False, "fields"), ("en", "field", 2, False, "fields"),
@@ -54,5 +53,5 @@ def test_pluralize(
) -> None: ) -> None:
i18n.set_current_locale(i18n.Locale(lang, "US", "UTF_8")) i18n.set_current_locale(i18n.Locale(lang, "US", "UTF_8"))
result = i18n.pluralize(noun, count, prefix_with_count) result = i18n.pluralize(noun, count, prefix_with_count = prefix_with_count)
assert result == expected, f"For LANG={lang}, expected {expected} but got {result}" assert result == expected, f"For LANG={lang}, expected {expected} but got {result}"

28
tests/unit/test_init.py
View File

@@ -1,11 +1,9 @@
""" # SPDX-FileCopyrightText: © Jens Bergmann and contributors
SPDX-FileCopyrightText: © Jens Bergmann and contributors # SPDX-License-Identifier: AGPL-3.0-or-later
SPDX-License-Identifier: AGPL-3.0-or-later # SPDX-ArtifactOfProjectHomePage: https://github.com/Second-Hand-Friends/kleinanzeigen-bot/
SPDX-ArtifactOfProjectHomePage: https://github.com/Second-Hand-Friends/kleinanzeigen-bot/ import copy, os, tempfile # isort: skip
"""
import copy, os, tempfile
from collections.abc import Generator from collections.abc import Generator
from datetime import datetime, timedelta from datetime import timedelta
from pathlib import Path from pathlib import Path
from typing import Any from typing import Any
from unittest.mock import AsyncMock, MagicMock, patch from unittest.mock import AsyncMock, MagicMock, patch
@@ -13,7 +11,7 @@ from unittest.mock import AsyncMock, MagicMock, patch
import pytest import pytest
from ruamel.yaml import YAML from ruamel.yaml import YAML
from kleinanzeigen_bot import LOG, KleinanzeigenBot from kleinanzeigen_bot import LOG, KleinanzeigenBot, misc
from kleinanzeigen_bot._version import __version__ from kleinanzeigen_bot._version import __version__
from kleinanzeigen_bot.ads import calculate_content_hash from kleinanzeigen_bot.ads import calculate_content_hash
from kleinanzeigen_bot.utils import loggers from kleinanzeigen_bot.utils import loggers
@@ -191,7 +189,7 @@ class TestKleinanzeigenBotLogging:
class TestKleinanzeigenBotCommandLine: class TestKleinanzeigenBotCommandLine:
"""Tests for command line argument parsing.""" """Tests for command line argument parsing."""
@pytest.mark.parametrize("args,expected_command,expected_selector,expected_keep_old", [ @pytest.mark.parametrize(("args", "expected_command", "expected_selector", "expected_keep_old"), [
(["publish", "--ads=all"], "publish", "all", False), (["publish", "--ads=all"], "publish", "all", False),
(["verify"], "verify", "due", False), (["verify"], "verify", "due", False),
(["download", "--ads=12345"], "download", "12345", False), (["download", "--ads=12345"], "download", "12345", False),
@@ -833,7 +831,7 @@ class TestKleinanzeigenBotAdDeletion:
patch.object(test_bot, 'web_click', new_callable = AsyncMock), \ patch.object(test_bot, 'web_click', new_callable = AsyncMock), \
patch.object(test_bot, 'web_check', new_callable = AsyncMock, return_value = True): patch.object(test_bot, 'web_check', new_callable = AsyncMock, return_value = True):
mock_find.return_value.attrs = {"content": "some-token"} mock_find.return_value.attrs = {"content": "some-token"}
result = await test_bot.delete_ad(ad_cfg, True, published_ads) result = await test_bot.delete_ad(ad_cfg, published_ads, delete_old_ads_by_title = True)
assert result is True assert result is True
@pytest.mark.asyncio @pytest.mark.asyncio
@@ -859,7 +857,7 @@ class TestKleinanzeigenBotAdDeletion:
patch.object(test_bot, 'web_click', new_callable = AsyncMock), \ patch.object(test_bot, 'web_click', new_callable = AsyncMock), \
patch.object(test_bot, 'web_check', new_callable = AsyncMock, return_value = True): patch.object(test_bot, 'web_check', new_callable = AsyncMock, return_value = True):
mock_find.return_value.attrs = {"content": "some-token"} mock_find.return_value.attrs = {"content": "some-token"}
result = await test_bot.delete_ad(ad_cfg, False, published_ads) result = await test_bot.delete_ad(ad_cfg, published_ads, delete_old_ads_by_title = False)
assert result is True assert result is True
@@ -910,7 +908,7 @@ class TestKleinanzeigenBotAdRepublication:
def test_check_ad_republication_no_changes(self, test_bot: KleinanzeigenBot, base_ad_config: dict[str, Any]) -> None: def test_check_ad_republication_no_changes(self, test_bot: KleinanzeigenBot, base_ad_config: dict[str, Any]) -> None:
"""Test that unchanged ads within interval are not marked for republication.""" """Test that unchanged ads within interval are not marked for republication."""
current_time = datetime.utcnow() current_time = misc.now()
three_days_ago = (current_time - timedelta(days = 3)).isoformat() three_days_ago = (current_time - timedelta(days = 3)).isoformat()
# Create ad config with timestamps for republication check # Create ad config with timestamps for republication check
@@ -1235,7 +1233,7 @@ class TestKleinanzeigenBotChangedAds:
# Mock the loading of the ad configuration # Mock the loading of the ad configuration
with patch('kleinanzeigen_bot.utils.dicts.load_dict', side_effect=[ with patch('kleinanzeigen_bot.utils.dicts.load_dict', side_effect=[
changed_ad, # First call returns the changed ad changed_ad, # First call returns the changed ad
{} # Second call for ad_fields.yaml {} # Second call for ad_fields.yaml
]): ]):
ads_to_publish = test_bot.load_ads() ads_to_publish = test_bot.load_ads()
@@ -1255,7 +1253,7 @@ class TestKleinanzeigenBotChangedAds:
} }
# Create a changed ad that is also due for republication # Create a changed ad that is also due for republication
current_time = datetime.utcnow() current_time = misc.now()
old_date = (current_time - timedelta(days=10)).isoformat() # Past republication interval old_date = (current_time - timedelta(days=10)).isoformat() # Past republication interval
changed_ad = create_ad_config( changed_ad = create_ad_config(
@@ -1287,7 +1285,7 @@ class TestKleinanzeigenBotChangedAds:
# Mock the loading of the ad configuration # Mock the loading of the ad configuration
with patch('kleinanzeigen_bot.utils.dicts.load_dict', side_effect=[ with patch('kleinanzeigen_bot.utils.dicts.load_dict', side_effect=[
changed_ad, # First call returns the changed ad changed_ad, # First call returns the changed ad
{} # Second call for ad_fields.yaml {} # Second call for ad_fields.yaml
]): ]):
ads_to_publish = test_bot.load_ads() ads_to_publish = test_bot.load_ads()

16
tests/unit/test_translations.py
View File

@@ -1,9 +1,9 @@
# SPDX-FileCopyrightText: © Jens Bergmann and contributors
# SPDX-License-Identifier: AGPL-3.0-or-later
# SPDX-ArtifactOfProjectHomePage: https://github.com/Second-Hand-Friends/kleinanzeigen-bot/
""" """
SPDX-FileCopyrightText: © Sebastian Thomschke and contributors
SPDX-License-Identifier: AGPL-3.0-or-later
SPDX-ArtifactOfProjectHomePage: https://github.com/Second-Hand-Friends/kleinanzeigen-bot/
This module contains tests for verifying the completeness and correctness of translations in the project. This module contains tests for verifying the completeness and correctness of translations in the project.
It ensures that: It ensures that:
1. All log messages in the code have corresponding translations 1. All log messages in the code have corresponding translations
2. All translations in the YAML files are actually used in the code 2. All translations in the YAML files are actually used in the code
@@ -15,7 +15,7 @@ The tests work by:
3. Comparing the extracted messages with translations 3. Comparing the extracted messages with translations
4. Verifying no unused translations exist 4. Verifying no unused translations exist
""" """
import ast, os import ast, os # isort: skip
from collections import defaultdict from collections import defaultdict
from dataclasses import dataclass from dataclasses import dataclass
from importlib.resources import files from importlib.resources import files
@@ -105,7 +105,7 @@ def _extract_log_messages(file_path: str, exclude_debug:bool = False) -> Message
messages: MessageDict = defaultdict(lambda: defaultdict(set)) messages: MessageDict = defaultdict(lambda: defaultdict(set))
def add_message(function: str, msg: str) -> None: def add_message(function: str, msg: str) -> None:
"""Helper to add a message to the messages dictionary.""" """Add a message to the messages dictionary."""
if function not in messages: if function not in messages:
messages[function] = defaultdict(set) messages[function] = defaultdict(set)
if msg not in messages[function]: if msg not in messages[function]:
@@ -128,7 +128,7 @@ def _extract_log_messages(file_path: str, exclude_debug:bool = False) -> Message
if (isinstance(node.func, ast.Attribute) and if (isinstance(node.func, ast.Attribute) and
isinstance(node.func.value, ast.Name) and isinstance(node.func.value, ast.Name) and
node.func.value.id in {'LOG', 'logger', 'logging'} and node.func.value.id in {'LOG', 'logger', 'logging'} and
node.func.attr in {None if exclude_debug else 'debug', 'info', 'warning', 'error', 'critical'}): node.func.attr in {None if exclude_debug else 'debug', 'info', 'warning', 'error', 'exception', 'critical'}):
if node.args: if node.args:
msg = extract_string_value(node.args[0]) msg = extract_string_value(node.args[0])
if msg: if msg:
@@ -390,7 +390,7 @@ def test_no_obsolete_translations(lang: str) -> None:
if not isinstance(function_trans, dict): if not isinstance(function_trans, dict):
continue continue
for original_message in function_trans.keys(): for original_message in function_trans:
# Check if this message exists in the code # Check if this message exists in the code
message_exists = _message_exists_in_code(messages_by_file, module, function, original_message) message_exists = _message_exists_in_code(messages_by_file, module, function, original_message)

9
tests/unit/test_utils_misc.py
View File

@@ -1,9 +1,8 @@
""" # SPDX-FileCopyrightText: © Sebastian Thomschke and contributors
SPDX-FileCopyrightText: © Sebastian Thomschke and contributors # SPDX-License-Identifier: AGPL-3.0-or-later
SPDX-License-Identifier: AGPL-3.0-or-later # SPDX-ArtifactOfProjectHomePage: https://github.com/Second-Hand-Friends/kleinanzeigen-bot/
SPDX-ArtifactOfProjectHomePage: https://github.com/Second-Hand-Friends/kleinanzeigen-bot/
"""
import pytest import pytest
from kleinanzeigen_bot.utils import misc from kleinanzeigen_bot.utils import misc