diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a851f5f..3791e39 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -134,12 +134,16 @@ jobs: run: pdm show - - name: Security Audit - run: pdm run audit + - name: Check with ruff + run: pdm run ruff check - - name: Check code style - run: pdm run lint + - name: Check with mypy + run: pdm run mypy + + + - name: Check with pyright + run: pdm run pyright - name: Run unit tests diff --git a/.gitignore b/.gitignore index ef06f00..55deff7 100644 --- a/.gitignore +++ b/.gitignore @@ -21,6 +21,7 @@ __pycache__ /.mypy_cache /.pdm-build/ /.pdm-python +/.ruff_cache # test coverage /.coverage diff --git a/pdm.lock b/pdm.lock index b0f6b88..a222f6a 100644 --- a/pdm.lock +++ b/pdm.lock @@ -5,7 +5,7 @@ groups = ["default", "dev"] strategy = ["inherit_metadata"] lock_version = "4.5.0" -content_hash = "sha256:13c90956820a101f2b726823f3ecb63ce8c32b5923fee0a0b313c42df91d1e53" +content_hash = "sha256:80e909dc69fd15a5392e613ee67a6bf196707436c64df6f8a4ba75c5ba8d974d" [[metadata.targets]] requires_python = ">=3.10,<3.14" @@ -20,20 +20,6 @@ files = [ {file = "altgraph-0.17.4.tar.gz", hash = "sha256:1b5afbb98f6c4dcadb2e2ae6ab9fa994bbb8c1d75f4fa96d340f9437ae454406"}, ] -[[package]] -name = "astroid" -version = "3.3.9" -requires_python = ">=3.9.0" -summary = "An abstract syntax tree for Python with inference support." -groups = ["dev"] -dependencies = [ - "typing-extensions>=4.0.0; python_version < \"3.11\"", -] -files = [ - {file = "astroid-3.3.9-py3-none-any.whl", hash = "sha256:d05bfd0acba96a7bd43e222828b7d9bc1e138aaeb0649707908d3702a9831248"}, - {file = "astroid-3.3.9.tar.gz", hash = "sha256:622cc8e3048684aa42c820d9d218978021c3c3d174fb03a9f0d615921744f550"}, -] - [[package]] name = "autocommand" version = "2.2.2" @@ -72,23 +58,6 @@ files = [ {file = "backports_tarfile-1.2.0.tar.gz", hash = "sha256:d75e02c268746e1b8144c278978b6e98e85de6ad16f8e4b0844a154557eca991"}, ] -[[package]] -name = "bandit" -version = "1.8.3" -requires_python = ">=3.9" -summary = "Security oriented static analyser for python code." -groups = ["dev"] -dependencies = [ - "PyYAML>=5.3.1", - "colorama>=0.3.9; platform_system == \"Windows\"", - "rich", - "stevedore>=1.20.0", -] -files = [ - {file = "bandit-1.8.3-py3-none-any.whl", hash = "sha256:28f04dc0d258e1dd0f99dee8eefa13d1cb5e3fde1a5ab0c523971f97b289bcd8"}, - {file = "bandit-1.8.3.tar.gz", hash = "sha256:f5847beb654d309422985c36644649924e0ea4425c76dec2e89110b87506193a"}, -] - [[package]] name = "bracex" version = "2.5.post1" @@ -265,17 +234,6 @@ files = [ {file = "deprecated-1.2.18.tar.gz", hash = "sha256:422b6f6d859da6f2ef57857761bfb392480502a64c3028ca9bbe86085d72115d"}, ] -[[package]] -name = "dill" -version = "0.4.0" -requires_python = ">=3.8" -summary = "serialize all of Python" -groups = ["dev"] -files = [ - {file = "dill-0.4.0-py3-none-any.whl", hash = "sha256:44f54bf6412c2c8464c14e8243eb163690a9800dbe2c367330883b19c7561049"}, - {file = "dill-0.4.0.tar.gz", hash = "sha256:0633f1d2df477324f53a895b02c901fb961bdbf65a17122586ea7019292cbcf0"}, -] - [[package]] name = "exceptiongroup" version = "1.2.2" @@ -299,17 +257,6 @@ files = [ {file = "iniconfig-2.1.0.tar.gz", hash = "sha256:3abbd2e30b36733fee78f9c7f7308f2d0050e88f0087fd25c2645f63c773e1c7"}, ] -[[package]] -name = "isort" -version = "6.0.1" -requires_python = ">=3.9.0" -summary = "A Python utility / library to sort Python imports." -groups = ["dev"] -files = [ - {file = "isort-6.0.1-py3-none-any.whl", hash = "sha256:2dc5d7f65c9678d94c88dfc29161a320eec67328bc97aad576874cb4be1e9615"}, - {file = "isort-6.0.1.tar.gz", hash = "sha256:1cb5df28dfbc742e490c5e41bad6da41b805b0a8be7bc93cd0fb2a8a890ac450"}, -] - [[package]] name = "jaraco-context" version = "6.0.1" @@ -370,42 +317,6 @@ files = [ {file = "macholib-1.16.3.tar.gz", hash = "sha256:07ae9e15e8e4cd9a788013d81f5908b3609aa76f9b1421bae9c4d7606ec86a30"}, ] -[[package]] -name = "markdown-it-py" -version = "3.0.0" -requires_python = ">=3.8" -summary = "Python port of markdown-it. Markdown parsing, done right!" -groups = ["dev"] -dependencies = [ - "mdurl~=0.1", -] -files = [ - {file = "markdown-it-py-3.0.0.tar.gz", hash = "sha256:e3f60a94fa066dc52ec76661e37c851cb232d92f9886b15cb560aaada2df8feb"}, - {file = "markdown_it_py-3.0.0-py3-none-any.whl", hash = "sha256:355216845c60bd96232cd8d8c40e8f9765cc86f46880e43a8fd22dc1a1a8cab1"}, -] - -[[package]] -name = "mccabe" -version = "0.7.0" -requires_python = ">=3.6" -summary = "McCabe checker, plugin for flake8" -groups = ["dev"] -files = [ - {file = "mccabe-0.7.0-py2.py3-none-any.whl", hash = "sha256:6c2d30ab6be0e4a46919781807b4f0d834ebdd6c6e3dca0bda5a15f863427b6e"}, - {file = "mccabe-0.7.0.tar.gz", hash = "sha256:348e0240c33b60bbdf4e523192ef919f28cb2c3d7d5c7794f74009290f236325"}, -] - -[[package]] -name = "mdurl" -version = "0.1.2" -requires_python = ">=3.7" -summary = "Markdown URL utilities" -groups = ["dev"] -files = [ - {file = "mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8"}, - {file = "mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba"}, -] - [[package]] name = "more-itertools" version = "10.7.0" @@ -517,20 +428,6 @@ files = [ {file = "packaging-25.0.tar.gz", hash = "sha256:d443872c98d677bf60f6a1f2f8c1cb748e8fe762d2bf9d3148b5599295b0fc4f"}, ] -[[package]] -name = "pbr" -version = "6.1.1" -requires_python = ">=2.6" -summary = "Python Build Reasonableness" -groups = ["dev"] -dependencies = [ - "setuptools", -] -files = [ - {file = "pbr-6.1.1-py2.py3-none-any.whl", hash = "sha256:38d4daea5d9fa63b3f626131b9d34947fd0c8be9b05a29276870580050a25a76"}, - {file = "pbr-6.1.1.tar.gz", hash = "sha256:93ea72ce6989eb2eed99d0f75721474f69ad88128afdef5ac377eb797c4bf76b"}, -] - [[package]] name = "pefile" version = "2023.2.7" @@ -593,17 +490,6 @@ files = [ {file = "pycodestyle-2.13.0.tar.gz", hash = "sha256:c8415bf09abe81d9c7f872502a6eee881fbe85d8763dd5b9924bb0a01d67efae"}, ] -[[package]] -name = "pygments" -version = "2.19.1" -requires_python = ">=3.8" -summary = "Pygments is a syntax highlighting package written in Python." -groups = ["dev"] -files = [ - {file = "pygments-2.19.1-py3-none-any.whl", hash = "sha256:9ea1544ad55cecf4b8242fab6dd35a93bbce657034b0611ee383099054ab6d8c"}, - {file = "pygments-2.19.1.tar.gz", hash = "sha256:61c16d2a8576dc0649d9f39e089b5f02bcd27fba10d8fb4dcc28173f7a45151f"}, -] - [[package]] name = "pyinstaller" version = "6.13.0" @@ -651,30 +537,6 @@ files = [ {file = "pyinstaller_hooks_contrib-2025.3.tar.gz", hash = "sha256:af129da5cd6219669fbda360e295cc822abac55b7647d03fec63a8fcf0a608cf"}, ] -[[package]] -name = "pylint" -version = "3.3.6" -requires_python = ">=3.9.0" -summary = "python code static checker" -groups = ["dev"] -dependencies = [ - "astroid<=3.4.0.dev0,>=3.3.8", - "colorama>=0.4.5; sys_platform == \"win32\"", - "dill>=0.2; python_version < \"3.11\"", - "dill>=0.3.6; python_version >= \"3.11\"", - "dill>=0.3.7; python_version >= \"3.12\"", - "isort!=5.13,<7,>=4.2.5", - "mccabe<0.8,>=0.6", - "platformdirs>=2.2", - "tomli>=1.1; python_version < \"3.11\"", - "tomlkit>=0.10.1", - "typing-extensions>=3.10; python_version < \"3.10\"", -] -files = [ - {file = "pylint-3.3.6-py3-none-any.whl", hash = "sha256:8b7c2d3e86ae3f94fb27703d521dd0b9b6b378775991f504d7c3a6275aa0a6a6"}, - {file = "pylint-3.3.6.tar.gz", hash = "sha256:b634a041aac33706d56a0d217e6587228c66427e20ec21a019bc4cdee48c040a"}, -] - [[package]] name = "pyright" version = "1.1.400" @@ -766,68 +628,6 @@ files = [ {file = "pywin32_ctypes-0.2.3-py3-none-any.whl", hash = "sha256:8a1513379d709975552d202d942d9837758905c8d01eb82b8bcc30918929e7b8"}, ] -[[package]] -name = "pyyaml" -version = "6.0.2" -requires_python = ">=3.8" -summary = "YAML parser and emitter for Python" -groups = ["dev"] -files = [ - {file = "PyYAML-6.0.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0a9a2848a5b7feac301353437eb7d5957887edbf81d56e903999a75a3d743086"}, - {file = "PyYAML-6.0.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:29717114e51c84ddfba879543fb232a6ed60086602313ca38cce623c1d62cfbf"}, - {file = "PyYAML-6.0.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8824b5a04a04a047e72eea5cec3bc266db09e35de6bdfe34c9436ac5ee27d237"}, - {file = "PyYAML-6.0.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7c36280e6fb8385e520936c3cb3b8042851904eba0e58d277dca80a5cfed590b"}, - {file = "PyYAML-6.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ec031d5d2feb36d1d1a24380e4db6d43695f3748343d99434e6f5f9156aaa2ed"}, - {file = "PyYAML-6.0.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:936d68689298c36b53b29f23c6dbb74de12b4ac12ca6cfe0e047bedceea56180"}, - {file = "PyYAML-6.0.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:23502f431948090f597378482b4812b0caae32c22213aecf3b55325e049a6c68"}, - {file = "PyYAML-6.0.2-cp310-cp310-win32.whl", hash = "sha256:2e99c6826ffa974fe6e27cdb5ed0021786b03fc98e5ee3c5bfe1fd5015f42b99"}, - {file = "PyYAML-6.0.2-cp310-cp310-win_amd64.whl", hash = "sha256:a4d3091415f010369ae4ed1fc6b79def9416358877534caf6a0fdd2146c87a3e"}, - {file = "PyYAML-6.0.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:cc1c1159b3d456576af7a3e4d1ba7e6924cb39de8f67111c735f6fc832082774"}, - {file = "PyYAML-6.0.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1e2120ef853f59c7419231f3bf4e7021f1b936f6ebd222406c3b60212205d2ee"}, - {file = "PyYAML-6.0.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5d225db5a45f21e78dd9358e58a98702a0302f2659a3c6cd320564b75b86f47c"}, - {file = "PyYAML-6.0.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5ac9328ec4831237bec75defaf839f7d4564be1e6b25ac710bd1a96321cc8317"}, - {file = "PyYAML-6.0.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ad2a3decf9aaba3d29c8f537ac4b243e36bef957511b4766cb0057d32b0be85"}, - {file = "PyYAML-6.0.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:ff3824dc5261f50c9b0dfb3be22b4567a6f938ccce4587b38952d85fd9e9afe4"}, - {file = "PyYAML-6.0.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:797b4f722ffa07cc8d62053e4cff1486fa6dc094105d13fea7b1de7d8bf71c9e"}, - {file = "PyYAML-6.0.2-cp311-cp311-win32.whl", hash = "sha256:11d8f3dd2b9c1207dcaf2ee0bbbfd5991f571186ec9cc78427ba5bd32afae4b5"}, - {file = "PyYAML-6.0.2-cp311-cp311-win_amd64.whl", hash = "sha256:e10ce637b18caea04431ce14fabcf5c64a1c61ec9c56b071a4b7ca131ca52d44"}, - {file = "PyYAML-6.0.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:c70c95198c015b85feafc136515252a261a84561b7b1d51e3384e0655ddf25ab"}, - {file = "PyYAML-6.0.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ce826d6ef20b1bc864f0a68340c8b3287705cae2f8b4b1d932177dcc76721725"}, - {file = "PyYAML-6.0.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f71ea527786de97d1a0cc0eacd1defc0985dcf6b3f17bb77dcfc8c34bec4dc5"}, - {file = "PyYAML-6.0.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9b22676e8097e9e22e36d6b7bda33190d0d400f345f23d4065d48f4ca7ae0425"}, - {file = "PyYAML-6.0.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:80bab7bfc629882493af4aa31a4cfa43a4c57c83813253626916b8c7ada83476"}, - {file = "PyYAML-6.0.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:0833f8694549e586547b576dcfaba4a6b55b9e96098b36cdc7ebefe667dfed48"}, - {file = "PyYAML-6.0.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8b9c7197f7cb2738065c481a0461e50ad02f18c78cd75775628afb4d7137fb3b"}, - {file = "PyYAML-6.0.2-cp312-cp312-win32.whl", hash = "sha256:ef6107725bd54b262d6dedcc2af448a266975032bc85ef0172c5f059da6325b4"}, - {file = "PyYAML-6.0.2-cp312-cp312-win_amd64.whl", hash = "sha256:7e7401d0de89a9a855c839bc697c079a4af81cf878373abd7dc625847d25cbd8"}, - {file = "PyYAML-6.0.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:efdca5630322a10774e8e98e1af481aad470dd62c3170801852d752aa7a783ba"}, - {file = "PyYAML-6.0.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:50187695423ffe49e2deacb8cd10510bc361faac997de9efef88badc3bb9e2d1"}, - {file = "PyYAML-6.0.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0ffe8360bab4910ef1b9e87fb812d8bc0a308b0d0eef8c8f44e0254ab3b07133"}, - {file = "PyYAML-6.0.2-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:17e311b6c678207928d649faa7cb0d7b4c26a0ba73d41e99c4fff6b6c3276484"}, - {file = "PyYAML-6.0.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:70b189594dbe54f75ab3a1acec5f1e3faa7e8cf2f1e08d9b561cb41b845f69d5"}, - {file = "PyYAML-6.0.2-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:41e4e3953a79407c794916fa277a82531dd93aad34e29c2a514c2c0c5fe971cc"}, - {file = "PyYAML-6.0.2-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:68ccc6023a3400877818152ad9a1033e3db8625d899c72eacb5a668902e4d652"}, - {file = "PyYAML-6.0.2-cp313-cp313-win32.whl", hash = "sha256:bc2fa7c6b47d6bc618dd7fb02ef6fdedb1090ec036abab80d4681424b84c1183"}, - {file = "PyYAML-6.0.2-cp313-cp313-win_amd64.whl", hash = "sha256:8388ee1976c416731879ac16da0aff3f63b286ffdd57cdeb95f3f2e085687563"}, - {file = "pyyaml-6.0.2.tar.gz", hash = "sha256:d584d9ec91ad65861cc08d42e834324ef890a082e591037abe114850ff7bbc3e"}, -] - -[[package]] -name = "rich" -version = "14.0.0" -requires_python = ">=3.8.0" -summary = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal" -groups = ["dev"] -dependencies = [ - "markdown-it-py>=2.2.0", - "pygments<3.0.0,>=2.13.0", - "typing-extensions<5.0,>=4.0.0; python_version < \"3.11\"", -] -files = [ - {file = "rich-14.0.0-py3-none-any.whl", hash = "sha256:1c9491e1951aac09caffd42f448ee3d04e58923ffe14993f6e83068dc395d7e0"}, - {file = "rich-14.0.0.tar.gz", hash = "sha256:82f1bc23a6a21ebca4ae0c45af9bdbc492ed20231dcb63f297d6d1021a9d5725"}, -] - [[package]] name = "ruamel-yaml" version = "0.18.10" @@ -889,6 +689,33 @@ files = [ {file = "ruamel.yaml.clib-0.2.12.tar.gz", hash = "sha256:6c8fbb13ec503f99a91901ab46e0b07ae7941cd527393187039aec586fdfd36f"}, ] +[[package]] +name = "ruff" +version = "0.11.7" +requires_python = ">=3.7" +summary = "An extremely fast Python linter and code formatter, written in Rust." +groups = ["dev"] +files = [ + {file = "ruff-0.11.7-py3-none-linux_armv6l.whl", hash = "sha256:d29e909d9a8d02f928d72ab7837b5cbc450a5bdf578ab9ebee3263d0a525091c"}, + {file = "ruff-0.11.7-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:dd1fb86b168ae349fb01dd497d83537b2c5541fe0626e70c786427dd8363aaee"}, + {file = "ruff-0.11.7-py3-none-macosx_11_0_arm64.whl", hash = "sha256:d3d7d2e140a6fbbc09033bce65bd7ea29d6a0adeb90b8430262fbacd58c38ada"}, + {file = "ruff-0.11.7-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4809df77de390a1c2077d9b7945d82f44b95d19ceccf0c287c56e4dc9b91ca64"}, + {file = "ruff-0.11.7-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f3a0c2e169e6b545f8e2dba185eabbd9db4f08880032e75aa0e285a6d3f48201"}, + {file = "ruff-0.11.7-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:49b888200a320dd96a68e86736cf531d6afba03e4f6cf098401406a257fcf3d6"}, + {file = "ruff-0.11.7-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:2b19cdb9cf7dae00d5ee2e7c013540cdc3b31c4f281f1dacb5a799d610e90db4"}, + {file = "ruff-0.11.7-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:64e0ee994c9e326b43539d133a36a455dbaab477bc84fe7bfbd528abe2f05c1e"}, + {file = "ruff-0.11.7-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:bad82052311479a5865f52c76ecee5d468a58ba44fb23ee15079f17dd4c8fd63"}, + {file = "ruff-0.11.7-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7940665e74e7b65d427b82bffc1e46710ec7f30d58b4b2d5016e3f0321436502"}, + {file = "ruff-0.11.7-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:169027e31c52c0e36c44ae9a9c7db35e505fee0b39f8d9fca7274a6305295a92"}, + {file = "ruff-0.11.7-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:305b93f9798aee582e91e34437810439acb28b5fc1fee6b8205c78c806845a94"}, + {file = "ruff-0.11.7-py3-none-musllinux_1_2_i686.whl", hash = "sha256:a681db041ef55550c371f9cd52a3cf17a0da4c75d6bd691092dfc38170ebc4b6"}, + {file = "ruff-0.11.7-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:07f1496ad00a4a139f4de220b0c97da6d4c85e0e4aa9b2624167b7d4d44fd6b6"}, + {file = "ruff-0.11.7-py3-none-win32.whl", hash = "sha256:f25dfb853ad217e6e5f1924ae8a5b3f6709051a13e9dad18690de6c8ff299e26"}, + {file = "ruff-0.11.7-py3-none-win_amd64.whl", hash = "sha256:0a931d85959ceb77e92aea4bbedfded0a31534ce191252721128f77e5ae1f98a"}, + {file = "ruff-0.11.7-py3-none-win_arm64.whl", hash = "sha256:778c1e5d6f9e91034142dfd06110534ca13220bfaad5c3735f6cb844654f6177"}, + {file = "ruff-0.11.7.tar.gz", hash = "sha256:655089ad3224070736dc32844fde783454f8558e71f501cb207485fe4eee23d4"}, +] + [[package]] name = "setuptools" version = "79.0.1" @@ -900,20 +727,6 @@ files = [ {file = "setuptools-79.0.1.tar.gz", hash = "sha256:128ce7b8f33c3079fd1b067ecbb4051a66e8526e7b65f6cec075dfc650ddfa88"}, ] -[[package]] -name = "stevedore" -version = "5.4.1" -requires_python = ">=3.9" -summary = "Manage dynamic plugins for Python applications" -groups = ["dev"] -dependencies = [ - "pbr>=2.0.0", -] -files = [ - {file = "stevedore-5.4.1-py3-none-any.whl", hash = "sha256:d10a31c7b86cba16c1f6e8d15416955fc797052351a56af15e608ad20811fcfe"}, - {file = "stevedore-5.4.1.tar.gz", hash = "sha256:3135b5ae50fe12816ef291baff420acb727fcd356106e3e9cbfa9e5985cd6f4b"}, -] - [[package]] name = "tomli" version = "2.2.1" @@ -956,17 +769,6 @@ files = [ {file = "tomli-2.2.1.tar.gz", hash = "sha256:cd45e1dc79c835ce60f7404ec8119f2eb06d38b1deba146f07ced3bbc44505ff"}, ] -[[package]] -name = "tomlkit" -version = "0.13.2" -requires_python = ">=3.8" -summary = "Style preserving TOML library" -groups = ["dev"] -files = [ - {file = "tomlkit-0.13.2-py3-none-any.whl", hash = "sha256:7a974427f6e119197f670fbbbeae7bef749a6c14e793db934baefc1b5f03efde"}, - {file = "tomlkit-0.13.2.tar.gz", hash = "sha256:fff5fe59a87295b278abd31bec92c15d9bc4a06885ab12bcea52c71119392e79"}, -] - [[package]] name = "typing-extensions" version = "4.13.2" diff --git a/pyproject.toml b/pyproject.toml index 8e5c1fc..f98194f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -53,13 +53,12 @@ dev = [ "pytest-cov>=6.0.0", # linting: "autopep8", - "pylint", + "ruff", "mypy", "pyright", - # security: - "bandit", # packaging: "pyinstaller", + "platformdirs", # required by pyinstaller ] [project.urls] @@ -84,8 +83,8 @@ compile.cmd = "python -O -m PyInstaller pyinstaller.spec --clean" compile.env = {PYTHONHASHSEED = "1", SOURCE_DATE_EPOCH = "0"} # https://pyinstaller.org/en/stable/advanced-topics.html#creating-a-reproducible-build debug = "python -m pdb -m kleinanzeigen_bot" format = "autopep8 --recursive --in-place src tests --verbose" -lint = {shell = "pylint -v src tests && autopep8 -v --exit-code --recursive --diff src tests && mypy" } -audit = "bandit -c pyproject.toml -r src" +lint = {shell = "ruff check && mypy && pyright" } +fix = {shell = "ruff check --fix" } test = "python -m pytest --capture=tee-sys -v" utest = "python -m pytest --capture=tee-sys -v -m 'not itest'" itest = "python -m pytest --capture=tee-sys -v -m 'itest'" @@ -109,11 +108,135 @@ aggressive = 3 ##################### -# bandit -# https://pypi.org/project/bandit/ -# https://github.com/PyCQA/bandit +# ruff +# https://pypi.org/project/ruff/ +# https://docs.astral.sh/ruff/configuration/ ##################### -[tool.bandit] +[tool.ruff] +include = ["pyproject.toml", "src/**/*.py", "tests/**/*.py"] +line-length = 160 +indent-width = 4 +target-version = "py310" + +[tool.ruff.lint] +# https://docs.astral.sh/ruff/rules/ +select = [ + "A", # flake8-builtins + "ARG", # flake8-unused-arguments + "ANN", # flake8-annotations + "ASYNC", # flake8-async + #"BLE", # flake8-blind-except + "B", # flake8-bugbear + "C4", # flake8-comprehensions + #"COM", # flake8-commas + #"CPY", # flake8-copyright + "DTZ", # flake8-datetimez + #"EM", # TODO flake8-errmsg + #"ERA", # eradicate commented-out code + "EXE", # flake8-executable + "FA", # flake8-future-annotations + "FBT", # flake8-boolean-trap + "FIX", # flake8-fixme + "G", # flake8-logging-format + "ICN", # flake8-import-conventions + "ISC", # flake8-implicit-str-concat + "INP", # flake8-no-pep420 + "INT", # flake8-gettext + "LOG", # flake8-logging + "PIE", # flake8-pie + "PT", # flake8-pytest-style + #"PTH", # flake8-use-pathlib + "PYI", # flake8-pyi + "Q", # flake8-quotes + "RET", # flake8-return + "RSE", # flake8-raise + "S", # flake8-bandit + "SIM", # flake8-simplify + "SLF", # flake8-self + "SLOT", # flake8-slots + "T10", # flake8-debugger + #"T20", # flake8-print + "TC", # flake8-type-checking + "TD", # flake8-todo + "TID", # flake8-flake8-tidy-import + "YTT", # flake8-2020 + + "E", # pycodestyle-errors + "W", # pycodestyle-warnings + + #"C90", # mccabe + "D", # pydocstyle + "F", # pyflakes + "FLY", # flynt + "I", # isort + "PERF", # perflint + "PGH", # pygrep-hooks + "PL", # pylint +] +ignore = [ + "ANN401", # Dynamically typed expressions (typing.Any) are disallowed + "ASYNC210", # TODO Async functions should not call blocking HTTP methods + "ASYNC230", # TODO Async functions should not open files with blocking methods like `open` + "D1", # Missing docstring in ... + "D200", # One-line docstring should fit on one line + "D202", # No blank lines allowed after function docstring (found 1) + "D203", # 1 blank line required before class docstring + "D204", # 1 blank line required after class docstring + "D205", # 1 blank line required between summary line and description + "D209", # Multi-line docstring closing quotes should be on a separate line" + "D212", # Multi-line docstring summary should start at the first line + "D213", # Multi-line docstring summary should start at the second line + "D400", # First line should end with a period + "D401", # First line of docstring should be in imperative mood + "D402", # First line should not be the function's signature + "D404", # First word of the docstring should not be "This" + "D413", # Missing blank line after last section ("Returns")" + "D415", # First line should end with a period, question mark, or exclamation point + "D417", # Missing argument description in the docstring for + #"E124", # Don't change indention of multi-line statements + #"E128", # Don't change indention of multi-line statements + #"E231", # Don't add whitespace after colon (:) on type declaration + #"E251", # Don't remove whitespace around parameter '=' sign. + "E401", # Don't put imports on separate lines + "Q000", # TODO Single quotes found but double quotes preferred + "PERF203", # `try`-`except` within a loop incurs performance overhead + "RET504", # Unnecessary assignment to `...` before `return` statement + "PYI041", # Use `float` instead of `int | float` + "SIM102", # Use a single `if` statement instead of nested `if` statements + "SIM105", # Use `contextlib.suppress(TimeoutError)` instead of `try`-`except`-`pass` + "SIM114", # Combine `if` branches using logical `or` operator + "TC006", # Add quotes to type expression in `typing.cast()` +] + +[tool.ruff.format] +quote-style = "double" +indent-style = "space" +line-ending = "native" +docstring-code-format = false +skip-magic-trailing-comma = false + +[tool.ruff.lint.per-file-ignores] +"tests/**/*.py" = [ + "ARG", + "B", + "FBT", + "INP", + "SLF", + "S101", # Use of `assert` detected + "PLR2004" # Magic value used in comparison +] + +[tool.ruff.lint.pylint] +# https://pylint.pycqa.org/en/latest/user_guide/configuration/all-options.html#design-checker +# https://pylint.pycqa.org/en/latest/user_guide/checkers/features.html#design-checker-messages +#max-args = 6 # max. number of args for function / method (R0913) +#max-attributes = 15 # max. number of instance attrs for a class (R0902) +max-branches = 40 # max. number of branch for function / method body (R0912) +#max-locals = 30 # max. number of local vars for function / method body (R0914) +max-returns = 15 # max. number of return / yield for function / method body (R0911) +max-statements = 150 # max. number of statements in function / method body (R0915) +#max-public-methods = 30 # max. number of public methods for a class (R0904) +#max-positional-arguments = 6 # max. number of positional args for function / method (R0917) ##################### @@ -147,97 +270,6 @@ pythonVersion = "3.10" typeCheckingMode = "standard" -##################### -# pylint -# https://pypi.org/project/pylint/ -# https://github.com/PyCQA/pylint -##################### -[tool.pylint.master] -extension-pkg-whitelist = "win32api" -ignore = "version.py" -jobs = 4 -persistent = "no" - -# https://pylint.pycqa.org/en/latest/user_guide/checkers/extensions.html -load-plugins = [ - "pylint.extensions.bad_builtin", - #"pylint.extensions.broad_try_clause", - "pylint.extensions.check_elif", - "pylint.extensions.code_style", - "pylint.extensions.comparison_placement", - #"pylint.extensions.confusing_elif", - "pylint.extensions.consider_ternary_expression", - "pylint.extensions.consider_refactoring_into_while_condition", - "pylint.extensions.dict_init_mutate", - "pylint.extensions.docstyle", - #"pylint.extensions.docparams", - "pylint.extensions.dunder", - "pylint.extensions.empty_comment", - "pylint.extensions.eq_without_hash", - "pylint.extensions.for_any_all", - #"pylint.extensions.magic_value", - #"pylint.extensions.mccabe", - "pylint.extensions.set_membership", - "pylint.extensions.no_self_use", - "pylint.extensions.overlapping_exceptions", - "pylint.extensions.private_import", - "pylint.extensions.redefined_loop_name", - "pylint.extensions.redefined_variable_type", - "pylint.extensions.set_membership", - "pylint.extensions.typing", - #"pylint.extensions.while_used" -] - -[tool.pylint.basic] -good-names = ["i", "j", "k", "v", "by", "ex", "fd", "_", "T"] - -[tool.pylint.format] -# https://pylint.pycqa.org/en/latest/technical_reference/features.html#format-checker -# https://pylint.pycqa.org/en/latest/user_guide/checkers/features.html#format-checker-messages -max-line-length = 160 # maximum number of characters on a single line (C0301) -max-module-lines = 2000 # maximum number of lines in a module (C0302) - -[tool.pylint.logging] -logging-modules = "logging" - -[tool.pylint.messages_control] -# https://pylint.pycqa.org/en/latest/technical_reference/features.html#messages-control-options -disable = [ - "broad-except", - "consider-using-assignment-expr", - "docstring-first-line-empty", - "global-statement", - "missing-docstring", - "multiple-imports", - "multiple-statements", - "no-self-use", - "no-member", # pylint cannot find async methods from super class - "too-few-public-methods" -] - -[tool.pylint.tests] -# Configuration specific to test files -disable = [ - "redefined-outer-name" # Allow pytest fixtures to be used as parameters -] - -[tool.pylint.miscelaneous] -# https://pylint.pycqa.org/en/latest/user_guide/configuration/all-options.html#miscellaneous-checker -notes = [ "FIXME", "XXX", "TODO" ] # list of note tags to take in consideration - -[tool.pylint.design] -# https://pylint.pycqa.org/en/latest/user_guide/configuration/all-options.html#design-checker -# https://pylint.pycqa.org/en/latest/user_guide/checkers/features.html#design-checker-messages -max-args = 6 # max. number of args for function / method (R0913) -max-attributes = 15 # max. number of instance attrs for a class (R0902) -max-branches = 40 # max. number of branch for function / method body (R0912) -max-locals = 30 # max. number of local vars for function / method body (R0914) -max-returns = 15 # max. number of return / yield for function / method body (R0911) -max-statements = 150 # max. number of statements in function / method body (R0915) -max-public-methods = 30 # max. number of public methods for a class (R0904) -max-positional-arguments = 6 # max. number of positional args for function / method (R0917) - - ##################### # pytest # https://pypi.org/project/pytest/ diff --git a/src/kleinanzeigen_bot/__init__.py b/src/kleinanzeigen_bot/__init__.py index 553affb..e13e83a 100644 --- a/src/kleinanzeigen_bot/__init__.py +++ b/src/kleinanzeigen_bot/__init__.py @@ -1,30 +1,26 @@ -""" -SPDX-FileCopyrightText: © Sebastian Thomschke and contributors -SPDX-License-Identifier: AGPL-3.0-or-later -SPDX-ArtifactOfProjectHomePage: https://github.com/Second-Hand-Friends/kleinanzeigen-bot/ -""" -import asyncio, atexit, copy, importlib.metadata, json, os, re, signal, shutil, sys, textwrap, time +# SPDX-FileCopyrightText: © Sebastian Thomschke and contributors +# SPDX-License-Identifier: AGPL-3.0-or-later +# SPDX-ArtifactOfProjectHomePage: https://github.com/Second-Hand-Friends/kleinanzeigen-bot/ +import atexit, copy, json, os, re, signal, sys, textwrap # isort: skip import getopt # pylint: disable=deprecated-module import urllib.parse as urllib_parse -import urllib.request as urllib_request from collections.abc import Iterable -from datetime import datetime from gettext import gettext as _ from typing import Any, Final -import certifi, colorama, nodriver +import certifi, colorama, nodriver # isort: skip from ruamel.yaml import YAML from wcmatch import glob from . import extract, resources from ._version import __version__ -from .ads import calculate_content_hash, get_description_affixes +from .ads import MAX_DESCRIPTION_LENGTH, calculate_content_hash, get_description_affixes from .utils import dicts, error_handlers, loggers, misc from .utils.exceptions import CaptchaEncountered from .utils.files import abspath from .utils.i18n import Locale, get_current_locale, pluralize, set_current_locale from .utils.misc import ainput, ensure, is_frozen, parse_datetime, parse_decimal -from .utils.web_scraping_mixin import By, Element, Is, Page, WebScrapingMixin +from .utils.web_scraping_mixin import By, Element, Is, WebScrapingMixin # W0406: possibly a bug, see https://github.com/PyCQA/pylint/issues/3933 @@ -287,7 +283,7 @@ class KleinanzeigenBot(WebScrapingMixin): return True # Check republication interval - ad_age = datetime.utcnow() - last_updated_on + ad_age = misc.now() - last_updated_on if ad_age.days <= ad_cfg["republication_interval"]: LOG.info( " -> SKIPPED: ad [%s] was last published %d days ago. republication is only required every %s days", @@ -394,12 +390,12 @@ class KleinanzeigenBot(WebScrapingMixin): description = self.__get_description_with_affixes(ad_cfg) # Validate total length - ensure(len(description) <= 4000, + ensure(len(description) <= MAX_DESCRIPTION_LENGTH, f"Length of ad description including prefix and suffix exceeds 4000 chars. " f"Description length: {len(description)} chars. @ {ad_file}.") - # pylint: disable=cell-var-from-loop def assert_one_of(path:str, allowed:Iterable[str]) -> None: + # ruff: noqa: B023 function-uses-loop-variable ensure(dicts.safe_get(ad_cfg, *path.split(".")) in allowed, f"-> property [{path}] must be one of: {allowed} @ [{ad_file}]") def assert_min_len(path:str, minlen:int) -> None: @@ -441,7 +437,7 @@ class KleinanzeigenBot(WebScrapingMixin): ad_cfg["category"] = resolved_category_id if ad_cfg["shipping_costs"]: - ad_cfg["shipping_costs"] = str(round(misc.parse_decimal(ad_cfg["shipping_costs"]), 2)) + ad_cfg["shipping_costs"] = str(round(parse_decimal(ad_cfg["shipping_costs"]), 2)) if ad_cfg["images"]: images = [] @@ -564,17 +560,17 @@ class KleinanzeigenBot(WebScrapingMixin): published_ads = json.loads( (await self.web_request(f"{self.root_url}/m-meine-anzeigen-verwalten.json?sort=DEFAULT"))["content"])["ads"] - for (ad_file, ad_cfg, _) in ad_cfgs: + for (ad_file, ad_cfg, _ad_cfg_orig) in ad_cfgs: count += 1 LOG.info("Processing %s/%s: '%s' from [%s]...", count, len(ad_cfgs), ad_cfg["title"], ad_file) - await self.delete_ad(ad_cfg, self.config["publishing"]["delete_old_ads_by_title"], published_ads) + await self.delete_ad(ad_cfg, published_ads, delete_old_ads_by_title = self.config["publishing"]["delete_old_ads_by_title"]) await self.web_sleep() LOG.info("############################################") LOG.info("DONE: Deleted %s", pluralize("ad", count)) LOG.info("############################################") - async def delete_ad(self, ad_cfg: dict[str, Any], delete_old_ads_by_title: bool, published_ads: list[dict[str, Any]]) -> bool: + async def delete_ad(self, ad_cfg: dict[str, Any], published_ads: list[dict[str, Any]], *, delete_old_ads_by_title: bool) -> bool: LOG.info("Deleting ad '%s' if already present...", ad_cfg["title"]) await self.web_open(f"{self.root_url}/m-meine-anzeigen.html") @@ -625,7 +621,7 @@ class KleinanzeigenBot(WebScrapingMixin): await self.web_await(lambda: self.web_check(By.ID, "checking-done", Is.DISPLAYED), timeout = 5 * 60) if self.config["publishing"]["delete_old_ads"] == "AFTER_PUBLISH" and not self.keep_old_ads: - await self.delete_ad(ad_cfg, False, published_ads) + await self.delete_ad(ad_cfg, published_ads, delete_old_ads_by_title = False) LOG.info("############################################") LOG.info("DONE: (Re-)published %s", pluralize("ad", count)) @@ -640,7 +636,7 @@ class KleinanzeigenBot(WebScrapingMixin): await self.assert_free_ad_limit_not_reached() if self.config["publishing"]["delete_old_ads"] == "BEFORE_PUBLISH" and not self.keep_old_ads: - await self.delete_ad(ad_cfg, self.config["publishing"]["delete_old_ads_by_title"], published_ads) + await self.delete_ad(ad_cfg, published_ads, delete_old_ads_by_title = self.config["publishing"]["delete_old_ads_by_title"]) LOG.info("Publishing ad '%s'...", ad_cfg["title"]) @@ -828,7 +824,7 @@ class KleinanzeigenBot(WebScrapingMixin): # Update content hash after successful publication # Calculate hash on original config to ensure consistent comparison on restart ad_cfg_orig["content_hash"] = calculate_content_hash(ad_cfg_orig) - ad_cfg_orig["updated_on"] = datetime.utcnow().isoformat() + ad_cfg_orig["updated_on"] = misc.now().isoformat() if not ad_cfg["created_on"] and not ad_cfg["id"]: ad_cfg_orig["created_on"] = ad_cfg_orig["updated_on"] @@ -914,11 +910,11 @@ class KleinanzeigenBot(WebScrapingMixin): raise TimeoutError(f"Failed to set special attribute [{special_attribute_key}] (not found)") from ex try: - elem_id = getattr(special_attr_elem.attrs, 'id') + elem_id = special_attr_elem.attrs.id if special_attr_elem.local_name == 'select': LOG.debug("Attribute field '%s' seems to be a select...", special_attribute_key) await self.web_select(By.ID, elem_id, special_attribute_value) - elif getattr(special_attr_elem.attrs, 'type') == 'checkbox': + elif special_attr_elem.attrs.type == 'checkbox': LOG.debug("Attribute field '%s' seems to be a checkbox...", special_attribute_key) await self.web_click(By.ID, elem_id) else: @@ -950,7 +946,7 @@ class KleinanzeigenBot(WebScrapingMixin): else: try: # no options. only costs. Set custom shipping cost - if not ad_cfg["shipping_costs"] is None: + if ad_cfg["shipping_costs"] is not None: await self.web_click(By.XPATH, '//*[contains(@class, "SubSection")]//*//button[contains(@class, "SelectionButton")]') await self.web_click(By.XPATH, '//*[contains(@class, "CarrierSelectionModal")]//button[contains(text(),"Andere Versandmethoden")]') @@ -984,7 +980,7 @@ class KleinanzeigenBot(WebScrapingMixin): except KeyError as ex: raise KeyError(f"Unknown shipping option(s), please refer to the documentation/README: {ad_cfg['shipping_options']}") from ex - shipping_sizes, shipping_packages = zip(*mapped_shipping_options) + shipping_sizes, shipping_packages = zip(*mapped_shipping_options, strict=False) try: shipping_size, = set(shipping_sizes) @@ -1159,8 +1155,9 @@ class KleinanzeigenBot(WebScrapingMixin): final_description = final_description.replace("@", "(at)") # Validate length - ensure(len(final_description) <= 4000, - f"Length of ad description including prefix and suffix exceeds 4000 chars. Description length: {len(final_description)} chars.") + ensure(len(final_description) <= MAX_DESCRIPTION_LENGTH, + f"Length of ad description including prefix and suffix exceeds {MAX_DESCRIPTION_LENGTH} chars. " + f"Description length: {len(final_description)} chars.") return final_description diff --git a/src/kleinanzeigen_bot/__main__.py b/src/kleinanzeigen_bot/__main__.py index e970750..94b427b 100644 --- a/src/kleinanzeigen_bot/__main__.py +++ b/src/kleinanzeigen_bot/__main__.py @@ -1,9 +1,7 @@ -""" -SPDX-FileCopyrightText: © Sebastian Thomschke and contributors -SPDX-License-Identifier: AGPL-3.0-or-later -SPDX-ArtifactOfProjectHomePage: https://github.com/Second-Hand-Friends/kleinanzeigen-bot/ -""" -import sys, time +# SPDX-FileCopyrightText: © Sebastian Thomschke and contributors +# SPDX-License-Identifier: AGPL-3.0-or-later +# SPDX-ArtifactOfProjectHomePage: https://github.com/Second-Hand-Friends/kleinanzeigen-bot/ +import sys, time # isort: skip from gettext import gettext as _ import kleinanzeigen_bot @@ -17,7 +15,6 @@ while True: try: kleinanzeigen_bot.main(sys.argv) # runs & returns when finished sys.exit(0) # not using `break` to prevent process closing issues - except CaptchaEncountered as ex: delay = ex.restart_delay print(_("[INFO] Captcha detected. Sleeping %s before restart...") % format_timedelta(delay)) diff --git a/src/kleinanzeigen_bot/ads.py b/src/kleinanzeigen_bot/ads.py index 5458a65..c4aefef 100644 --- a/src/kleinanzeigen_bot/ads.py +++ b/src/kleinanzeigen_bot/ads.py @@ -1,12 +1,13 @@ -""" -SPDX-FileCopyrightText: © Jens Bergman and contributors -SPDX-License-Identifier: AGPL-3.0-or-later -SPDX-ArtifactOfProjectHomePage: https://github.com/Second-Hand-Friends/kleinanzeigen-bot/ -""" -import hashlib, json, os -from typing import Any +# SPDX-FileCopyrightText: © Jens Bergman and contributors +# SPDX-License-Identifier: AGPL-3.0-or-later +# SPDX-ArtifactOfProjectHomePage: https://github.com/Second-Hand-Friends/kleinanzeigen-bot/ +import hashlib, json, os # isort: skip +from typing import Any, Final + from .utils import dicts +MAX_DESCRIPTION_LENGTH:Final[int] = 4000 + def calculate_content_hash(ad_cfg: dict[str, Any]) -> str: """Calculate a hash for user-modifiable fields of the ad.""" @@ -39,7 +40,7 @@ def calculate_content_hash(ad_cfg: dict[str, Any]) -> str: return hashlib.sha256(content_str.encode()).hexdigest() -def get_description_affixes(config: dict[str, Any], prefix: bool = True) -> str: +def get_description_affixes(config: dict[str, Any], *, prefix: bool = True) -> str: """Get prefix or suffix for description with proper precedence. This function handles both the new flattened format and legacy nested format: diff --git a/src/kleinanzeigen_bot/extract.py b/src/kleinanzeigen_bot/extract.py index c1908c7..ece4940 100644 --- a/src/kleinanzeigen_bot/extract.py +++ b/src/kleinanzeigen_bot/extract.py @@ -1,9 +1,7 @@ -""" -SPDX-FileCopyrightText: © Sebastian Thomschke and contributors -SPDX-License-Identifier: AGPL-3.0-or-later -SPDX-ArtifactOfProjectHomePage: https://github.com/Second-Hand-Friends/kleinanzeigen-bot/ -""" -import json, mimetypes, os, shutil +# SPDX-FileCopyrightText: © Sebastian Thomschke and contributors +# SPDX-License-Identifier: AGPL-3.0-or-later +# SPDX-ArtifactOfProjectHomePage: https://github.com/Second-Hand-Friends/kleinanzeigen-bot/ +import json, mimetypes, os, shutil # isort: skip import urllib.request as urllib_request from datetime import datetime from typing import Any, Final @@ -24,7 +22,7 @@ class AdExtractor(WebScrapingMixin): Wrapper class for ad extraction that uses an active bot´s browser session to extract specific elements from an ad page. """ - def __init__(self, browser:Browser, config:dict[str, Any]): + def __init__(self, browser:Browser, config:dict[str, Any]) -> None: super().__init__() self.browser = browser self.config = config @@ -84,7 +82,7 @@ class AdExtractor(WebScrapingMixin): if current_img_url is None: continue - with urllib_request.urlopen(current_img_url) as response: # nosec B310 + with urllib_request.urlopen(current_img_url) as response: # noqa: S310 Audit URL open for permitted schemes. content_type = response.info().get_content_type() file_ending = mimetypes.guess_extension(content_type) img_path = f"{directory}/{img_fn_prefix}{img_nr}{file_ending}" @@ -170,7 +168,7 @@ class AdExtractor(WebScrapingMixin): # This will now correctly trigger only if the '.Pagination' div itself is not found LOG.info('No pagination controls found. Assuming single page.') except Exception as e: - LOG.error("Error during pagination detection: %s", e, exc_info=True) + LOG.exception("Error during pagination detection: %s", e) LOG.info('Assuming single page due to error during pagination check.') # --- End Pagination Handling --- @@ -201,7 +199,7 @@ class AdExtractor(WebScrapingMixin): LOG.info("Successfully extracted %s refs from page %s.", len(page_refs), current_page) except Exception as e: # Log the error if extraction fails for some items, but try to continue - LOG.error("Error extracting refs on page %s: %s", current_page, e, exc_info=True) + LOG.exception("Error extracting refs on page %s: %s", current_page, e) if not multi_page: # only one iteration for single-page overview break @@ -232,7 +230,7 @@ class AdExtractor(WebScrapingMixin): LOG.info("No pagination controls found after scrolling/waiting. Assuming last page.") break except Exception as e: - LOG.error("Error during pagination navigation: %s", e, exc_info=True) + LOG.exception("Error during pagination navigation: %s", e) break # --- End Navigation --- @@ -287,7 +285,7 @@ class AdExtractor(WebScrapingMixin): # extract basic info info['type'] = 'OFFER' if 's-anzeige' in self.page.url else 'WANTED' title:str = await self.web_text(By.ID, 'viewad-title') - LOG.info('Extracting information from ad with title \"%s\"', title) + LOG.info('Extracting information from ad with title "%s"', title) info['category'] = await self._extract_category_from_ad_page() info['title'] = title @@ -389,7 +387,7 @@ class AdExtractor(WebScrapingMixin): price = int(price_str.replace('.', '').split()[0]) case 'VB': price_type = 'NEGOTIABLE' - if not price_str == "VB": # can be either 'X € VB', or just 'VB' + if price_str != "VB": # can be either 'X € VB', or just 'VB' price = int(price_str.replace('.', '').split()[0]) case 'verschenken': price_type = 'GIVE_AWAY' diff --git a/src/kleinanzeigen_bot/utils/dicts.py b/src/kleinanzeigen_bot/utils/dicts.py index 73ea914..5d0f2f4 100644 --- a/src/kleinanzeigen_bot/utils/dicts.py +++ b/src/kleinanzeigen_bot/utils/dicts.py @@ -1,16 +1,15 @@ -""" -SPDX-FileCopyrightText: © Sebastian Thomschke and contributors -SPDX-License-Identifier: AGPL-3.0-or-later -SPDX-ArtifactOfProjectHomePage: https://github.com/Second-Hand-Friends/kleinanzeigen-bot/ -""" -import copy, json, os +# SPDX-FileCopyrightText: © Sebastian Thomschke and contributors +# SPDX-License-Identifier: AGPL-3.0-or-later +# SPDX-ArtifactOfProjectHomePage: https://github.com/Second-Hand-Friends/kleinanzeigen-bot/ +import copy, json, os # isort: skip from collections.abc import Callable -from importlib.resources import read_text as get_resource_as_string from gettext import gettext as _ +from importlib.resources import read_text as get_resource_as_string from types import ModuleType from typing import Any, Final from ruamel.yaml import YAML + from . import files, loggers # pylint: disable=cyclic-import LOG:Final[loggers.Logger] = loggers.get_logger(__name__) @@ -112,9 +111,9 @@ def safe_get(a_map:dict[Any, Any], *keys:str) -> Any: 'some_value' """ if a_map: - for key in keys: - try: + try: + for key in keys: a_map = a_map[key] - except (KeyError, TypeError): - return None + except (KeyError, TypeError): + return None return a_map diff --git a/src/kleinanzeigen_bot/utils/error_handlers.py b/src/kleinanzeigen_bot/utils/error_handlers.py index 87c9e85..aa85b51 100644 --- a/src/kleinanzeigen_bot/utils/error_handlers.py +++ b/src/kleinanzeigen_bot/utils/error_handlers.py @@ -1,11 +1,9 @@ -""" -SPDX-FileCopyrightText: © Sebastian Thomschke and contributors -SPDX-License-Identifier: AGPL-3.0-or-later -SPDX-ArtifactOfProjectHomePage: https://github.com/Second-Hand-Friends/kleinanzeigen-bot/ -""" -import sys, traceback +# SPDX-FileCopyrightText: © Sebastian Thomschke and contributors +# SPDX-License-Identifier: AGPL-3.0-or-later +# SPDX-ArtifactOfProjectHomePage: https://github.com/Second-Hand-Friends/kleinanzeigen-bot/ +import sys, traceback # isort: skip from types import FrameType, TracebackType -from typing import Final +from typing import Any, Final from . import loggers diff --git a/src/kleinanzeigen_bot/utils/exceptions.py b/src/kleinanzeigen_bot/utils/exceptions.py index 1fb09de..34b9ab3 100644 --- a/src/kleinanzeigen_bot/utils/exceptions.py +++ b/src/kleinanzeigen_bot/utils/exceptions.py @@ -1,8 +1,6 @@ -""" -SPDX-FileCopyrightText: © Sebastian Thomschke and contributors -SPDX-License-Identifier: AGPL-3.0-or-later -SPDX-ArtifactOfProjectHomePage: https://github.com/Second-Hand-Friends/kleinanzeigen-bot/ -""" +# SPDX-FileCopyrightText: © Sebastian Thomschke and contributors +# SPDX-License-Identifier: AGPL-3.0-or-later +# SPDX-ArtifactOfProjectHomePage: https://github.com/Second-Hand-Friends/kleinanzeigen-bot/ from datetime import timedelta @@ -13,6 +11,6 @@ class KleinanzeigenBotError(RuntimeError): class CaptchaEncountered(KleinanzeigenBotError): """Raised when a Captcha was detected and auto-restart is enabled.""" - def __init__(self, restart_delay: timedelta): + def __init__(self, restart_delay: timedelta) -> None: super().__init__() self.restart_delay = restart_delay diff --git a/src/kleinanzeigen_bot/utils/files.py b/src/kleinanzeigen_bot/utils/files.py index b9840f4..f3138c9 100644 --- a/src/kleinanzeigen_bot/utils/files.py +++ b/src/kleinanzeigen_bot/utils/files.py @@ -1,8 +1,6 @@ -""" -SPDX-FileCopyrightText: © Sebastian Thomschke and contributors -SPDX-License-Identifier: AGPL-3.0-or-later -SPDX-ArtifactOfProjectHomePage: https://github.com/Second-Hand-Friends/kleinanzeigen-bot/ -""" +# SPDX-FileCopyrightText: © Sebastian Thomschke and contributors +# SPDX-License-Identifier: AGPL-3.0-or-later +# SPDX-ArtifactOfProjectHomePage: https://github.com/Second-Hand-Friends/kleinanzeigen-bot/ import os diff --git a/src/kleinanzeigen_bot/utils/i18n.py b/src/kleinanzeigen_bot/utils/i18n.py index 57dd7dc..c1b1a06 100644 --- a/src/kleinanzeigen_bot/utils/i18n.py +++ b/src/kleinanzeigen_bot/utils/i18n.py @@ -1,13 +1,12 @@ -""" -SPDX-FileCopyrightText: © Sebastian Thomschke and contributors -SPDX-License-Identifier: AGPL-3.0-or-later -SPDX-ArtifactOfProjectHomePage: https://github.com/Second-Hand-Friends/kleinanzeigen-bot/ -""" -import ctypes, gettext, inspect, locale, logging, os, sys +# SPDX-FileCopyrightText: © Sebastian Thomschke and contributors +# SPDX-License-Identifier: AGPL-3.0-or-later +# SPDX-ArtifactOfProjectHomePage: https://github.com/Second-Hand-Friends/kleinanzeigen-bot/ +import ctypes, gettext, inspect, locale, logging, os, sys # isort: skip from collections.abc import Sized from typing import Any, Final, NamedTuple from kleinanzeigen_bot import resources + from . import dicts, reflect __all__ = [ @@ -96,7 +95,7 @@ def translate(text:object, caller: inspect.FrameInfo | None) -> str: if not caller: return text - global _TRANSLATIONS + global _TRANSLATIONS # noqa: PLW0603 Using the global statement to update `...` is discouraged if _TRANSLATIONS is None: try: _TRANSLATIONS = dicts.load_dict_from_module(resources, f"translations.{_CURRENT_LOCALE[0]}.yaml") @@ -125,10 +124,10 @@ gettext.gettext = lambda message: translate(_original_gettext(message), reflect. for module_name, module in sys.modules.items(): if module is None or module_name in sys.builtin_module_names: continue - if hasattr(module, '_') and getattr(module, '_') is _original_gettext: - setattr(module, '_', gettext.gettext) - if hasattr(module, 'gettext') and getattr(module, 'gettext') is _original_gettext: - setattr(module, 'gettext', gettext.gettext) + if hasattr(module, '_') and module._ is _original_gettext: + module._ = gettext.gettext # type: ignore[attr-defined] + if hasattr(module, 'gettext') and module.gettext is _original_gettext: + module.gettext = gettext.gettext # type: ignore[attr-defined] def get_current_locale() -> Locale: @@ -136,13 +135,13 @@ def get_current_locale() -> Locale: def set_current_locale(new_locale:Locale) -> None: - global _CURRENT_LOCALE, _TRANSLATIONS + global _CURRENT_LOCALE, _TRANSLATIONS # noqa: PLW0603 Using the global statement to update `...` is discouraged if new_locale.language != _CURRENT_LOCALE.language: _TRANSLATIONS = None _CURRENT_LOCALE = new_locale -def pluralize(noun:str, count:int | Sized, prefix_with_count:bool = True) -> str: +def pluralize(noun:str, count:int | Sized, *, prefix_with_count:bool = True) -> str: """ >>> set_current_locale(Locale("en")) # Setup for doctests >>> pluralize("field", 1) @@ -189,7 +188,7 @@ def pluralize(noun:str, count:int | Sized, prefix_with_count:bool = True) -> str return f"{prefix}{noun}e" # Hund -> Hunde # English - if len(noun) < 2: + if len(noun) < 2: # noqa: PLR2004 Magic value used in comparison return f"{prefix}{noun}s" if noun.endswith(('s', 'sh', 'ch', 'x', 'z')): return f"{prefix}{noun}es" diff --git a/src/kleinanzeigen_bot/utils/loggers.py b/src/kleinanzeigen_bot/utils/loggers.py index 6ac4a48..7113957 100644 --- a/src/kleinanzeigen_bot/utils/loggers.py +++ b/src/kleinanzeigen_bot/utils/loggers.py @@ -1,15 +1,14 @@ -""" -SPDX-FileCopyrightText: © Sebastian Thomschke and contributors -SPDX-License-Identifier: AGPL-3.0-or-later -SPDX-ArtifactOfProjectHomePage: https://github.com/Second-Hand-Friends/kleinanzeigen-bot/ -""" -import copy, logging, re, sys +# SPDX-FileCopyrightText: © Sebastian Thomschke and contributors +# SPDX-License-Identifier: AGPL-3.0-or-later +# SPDX-ArtifactOfProjectHomePage: https://github.com/Second-Hand-Friends/kleinanzeigen-bot/ +import copy, logging, re, sys # isort: skip from gettext import gettext as _ -from logging import Logger, DEBUG, INFO, WARNING, ERROR, CRITICAL +from logging import CRITICAL, DEBUG, ERROR, INFO, WARNING, Logger from logging.handlers import RotatingFileHandler from typing import Any, Final # @UnusedImport import colorama + from . import i18n, reflect __all__ = [ @@ -27,6 +26,16 @@ __all__ = [ LOG_ROOT:Final[logging.Logger] = logging.getLogger() +class _MaxLevelFilter(logging.Filter): + + def __init__(self, level: int) -> None: + super().__init__() + self.level = level + + def filter(self, record: logging.LogRecord) -> bool: + return record.levelno <= self.level + + def configure_console_logging() -> None: # if a StreamHandler already exists, do not append it again if any(isinstance(h, logging.StreamHandler) for h in LOG_ROOT.handlers): @@ -82,9 +91,7 @@ def configure_console_logging() -> None: stdout_log = logging.StreamHandler(sys.stderr) stdout_log.setLevel(DEBUG) - stdout_log.addFilter(type("", (logging.Filter,), { - "filter": lambda rec: rec.levelno <= INFO - })) + stdout_log.addFilter(_MaxLevelFilter(INFO)) stdout_log.setFormatter(formatter) LOG_ROOT.addHandler(stdout_log) @@ -97,7 +104,7 @@ def configure_console_logging() -> None: class LogFileHandle: """Encapsulates a log file handler with close and status methods.""" - def __init__(self, file_path: str, handler: RotatingFileHandler, logger: logging.Logger): + def __init__(self, file_path: str, handler: RotatingFileHandler, logger: logging.Logger) -> None: self.file_path = file_path self._handler:RotatingFileHandler | None = handler self._logger = logger diff --git a/src/kleinanzeigen_bot/utils/misc.py b/src/kleinanzeigen_bot/utils/misc.py index 0393208..f7d2e27 100644 --- a/src/kleinanzeigen_bot/utils/misc.py +++ b/src/kleinanzeigen_bot/utils/misc.py @@ -1,11 +1,9 @@ -""" -SPDX-FileCopyrightText: © Sebastian Thomschke and contributors -SPDX-License-Identifier: AGPL-3.0-or-later -SPDX-ArtifactOfProjectHomePage: https://github.com/Second-Hand-Friends/kleinanzeigen-bot/ -""" -import asyncio, decimal, re, sys, time +# SPDX-FileCopyrightText: © Sebastian Thomschke and contributors +# SPDX-License-Identifier: AGPL-3.0-or-later +# SPDX-ArtifactOfProjectHomePage: https://github.com/Second-Hand-Friends/kleinanzeigen-bot/ +import asyncio, decimal, re, sys, time # isort: skip from collections.abc import Callable -from datetime import datetime, timedelta +from datetime import datetime, timedelta, timezone from gettext import gettext as _ from typing import Any, TypeVar @@ -39,6 +37,10 @@ def ensure(condition:Any | bool | Callable[[], bool], error_message:str, timeout time.sleep(poll_requency) +def now() -> datetime: + return datetime.now(timezone.utc) + + def is_frozen() -> bool: """ >>> is_frozen() @@ -81,12 +83,27 @@ def parse_decimal(number:float | int | str) -> decimal.Decimal: raise decimal.DecimalException(f"Invalid number format: {number}") from ex -def parse_datetime(date:datetime | str | None) -> datetime | None: +def parse_datetime( + date: datetime | str | None, + *, + add_timezone_if_missing: bool = True, + use_local_timezone: bool = True +) -> datetime | None: """ - >>> parse_datetime(datetime(2020, 1, 1, 0, 0)) + Parses a datetime object or ISO-formatted string. + + Args: + date: The input datetime object or ISO string. + add_timezone_if_missing: If True, add timezone info if missing. + use_local_timezone: If True, use local timezone; otherwise UTC if adding timezone. + + Returns: + A timezone-aware or naive datetime object, depending on parameters. + + >>> parse_datetime(datetime(2020, 1, 1, 0, 0), add_timezone_if_missing = False) datetime.datetime(2020, 1, 1, 0, 0) - >>> parse_datetime("2020-01-01T00:00:00") + >>> parse_datetime("2020-01-01T00:00:00", add_timezone_if_missing = False) datetime.datetime(2020, 1, 1, 0, 0) >>> parse_datetime(None) @@ -94,9 +111,16 @@ def parse_datetime(date:datetime | str | None) -> datetime | None: """ if date is None: return None - if isinstance(date, datetime): - return date - return datetime.fromisoformat(date) + + dt = date if isinstance(date, datetime) else datetime.fromisoformat(date) + + if dt.tzinfo is None and add_timezone_if_missing: + dt = ( + dt.astimezone() if use_local_timezone + else dt.replace(tzinfo = timezone.utc) + ) + + return dt def parse_duration(text:str) -> timedelta: diff --git a/src/kleinanzeigen_bot/utils/net.py b/src/kleinanzeigen_bot/utils/net.py index 9a9435d..3d70a01 100644 --- a/src/kleinanzeigen_bot/utils/net.py +++ b/src/kleinanzeigen_bot/utils/net.py @@ -1,8 +1,6 @@ -""" -SPDX-FileCopyrightText: © Sebastian Thomschke and contributors -SPDX-License-Identifier: AGPL-3.0-or-later -SPDX-ArtifactOfProjectHomePage: https://github.com/Second-Hand-Friends/kleinanzeigen-bot/ -""" +# SPDX-FileCopyrightText: © Sebastian Thomschke and contributors +# SPDX-License-Identifier: AGPL-3.0-or-later +# SPDX-ArtifactOfProjectHomePage: https://github.com/Second-Hand-Friends/kleinanzeigen-bot/ import socket diff --git a/src/kleinanzeigen_bot/utils/reflect.py b/src/kleinanzeigen_bot/utils/reflect.py index 694af77..aec388e 100644 --- a/src/kleinanzeigen_bot/utils/reflect.py +++ b/src/kleinanzeigen_bot/utils/reflect.py @@ -1,8 +1,6 @@ -""" -SPDX-FileCopyrightText: © Sebastian Thomschke and contributors -SPDX-License-Identifier: AGPL-3.0-or-later -SPDX-ArtifactOfProjectHomePage: https://github.com/Second-Hand-Friends/kleinanzeigen-bot/ -""" +# SPDX-FileCopyrightText: © Sebastian Thomschke and contributors +# SPDX-License-Identifier: AGPL-3.0-or-later +# SPDX-ArtifactOfProjectHomePage: https://github.com/Second-Hand-Friends/kleinanzeigen-bot/ import inspect from typing import Any diff --git a/src/kleinanzeigen_bot/utils/web_scraping_mixin.py b/src/kleinanzeigen_bot/utils/web_scraping_mixin.py index d84fe39..501252a 100644 --- a/src/kleinanzeigen_bot/utils/web_scraping_mixin.py +++ b/src/kleinanzeigen_bot/utils/web_scraping_mixin.py @@ -1,26 +1,24 @@ -""" -SPDX-FileCopyrightText: © Sebastian Thomschke and contributors -SPDX-License-Identifier: AGPL-3.0-or-later -SPDX-ArtifactOfProjectHomePage: https://github.com/Second-Hand-Friends/kleinanzeigen-bot/ -""" -import asyncio, enum, inspect, json, os, platform, secrets, shutil, time +# SPDX-FileCopyrightText: © Sebastian Thomschke and contributors +# SPDX-License-Identifier: AGPL-3.0-or-later +# SPDX-ArtifactOfProjectHomePage: https://github.com/Second-Hand-Friends/kleinanzeigen-bot/ +import asyncio, enum, inspect, json, os, platform, secrets, shutil # isort: skip from collections.abc import Callable, Coroutine, Iterable from gettext import gettext as _ -from typing import cast, Any, Final +from typing import Any, Final, cast try: from typing import Never # type: ignore[attr-defined,unused-ignore] # mypy except ImportError: from typing import NoReturn as Never # Python <3.11 -import nodriver, psutil +import nodriver, psutil # isort: skip from nodriver.core.browser import Browser from nodriver.core.config import Config from nodriver.core.element import Element from nodriver.core.tab import Tab as Page from . import loggers, net -from .misc import ensure, T +from .misc import T, ensure __all__ = [ "Browser", @@ -70,8 +68,8 @@ class WebScrapingMixin: def __init__(self) -> None: self.browser_config:Final[BrowserConfig] = BrowserConfig() - self.browser:Browser = None # pyright: ignore - self.page:Page = None # pyright: ignore + self.browser:Browser = None # pyright: ignore[reportAttributeAccessIssue] + self.page:Page = None # pyright: ignore[reportAttributeAccessIssue] async def create_browser_session(self) -> None: LOG.info("Creating Browser session...") @@ -96,7 +94,7 @@ class WebScrapingMixin: if remote_port > 0: LOG.info("Using existing browser process at %s:%s", remote_host, remote_port) ensure(net.is_port_open(remote_host, remote_port), - f"Browser process not reachable at {remote_host}:{remote_port}. " + + f"Browser process not reachable at {remote_host}:{remote_port}. " f"Start the browser with --remote-debugging-port={remote_port} or remove this port from your config.yaml") cfg = Config( browser_executable_path = self.browser_config.binary_location # actually not necessary but nodriver fails without @@ -208,14 +206,14 @@ class WebScrapingMixin: def close_browser_session(self) -> None: if self.browser: LOG.debug("Closing Browser session...") - self.page = None # pyright: ignore - browser_process = psutil.Process(self.browser._process_pid) # pylint: disable=protected-access + self.page = None # pyright: ignore[reportAttributeAccessIssue] + browser_process = psutil.Process(self.browser._process_pid) # noqa: SLF001 Private member accessed browser_children:list[psutil.Process] = browser_process.children() self.browser.stop() for p in browser_children: if p.is_running(): p.kill() # terminate orphaned browser processes - self.browser = None # pyright: ignore + self.browser = None # pyright: ignore[reportAttributeAccessIssue] def get_compatible_browser(self) -> str: match platform.system(): @@ -236,15 +234,15 @@ class WebScrapingMixin: case "Windows": browser_paths = [ - os.environ.get("ProgramFiles", "C:\\Program Files") + r'\Microsoft\Edge\Application\msedge.exe', - os.environ.get("ProgramFiles(x86)", "C:\\Program Files (x86)") + r'\Microsoft\Edge\Application\msedge.exe', + os.environ.get("PROGRAMFILES", "C:\\Program Files") + r'\Microsoft\Edge\Application\msedge.exe', + os.environ.get("PROGRAMFILES(X86)", "C:\\Program Files (x86)") + r'\Microsoft\Edge\Application\msedge.exe', - os.environ["ProgramFiles"] + r'\Chromium\Application\chrome.exe', - os.environ["ProgramFiles(x86)"] + r'\Chromium\Application\chrome.exe', + os.environ["PROGRAMFILES"] + r'\Chromium\Application\chrome.exe', + os.environ["PROGRAMFILES(X86)"] + r'\Chromium\Application\chrome.exe', os.environ["LOCALAPPDATA"] + r'\Chromium\Application\chrome.exe', - os.environ["ProgramFiles"] + r'\Chrome\Application\chrome.exe', - os.environ["ProgramFiles(x86)"] + r'\Chrome\Application\chrome.exe', + os.environ["PROGRAMFILES"] + r'\Chrome\Application\chrome.exe', + os.environ["PROGRAMFILES(X86)"] + r'\Chrome\Application\chrome.exe', os.environ["LOCALAPPDATA"] + r'\Chrome\Application\chrome.exe', shutil.which("msedge.exe"), @@ -277,7 +275,7 @@ class WebScrapingMixin: ex:Exception | None = None try: result_raw = condition() - result:T = (await result_raw) if inspect.isawaitable(result_raw) else result_raw + result:T = cast(T, await result_raw if inspect.isawaitable(result_raw) else result_raw) if result: return result except Exception as ex1: @@ -359,11 +357,11 @@ class WebScrapingMixin: _prev_jscode:str = getattr(self.__class__.web_execute, "_prev_jscode", "") if not (jscode == _prev_jscode or (jscode.startswith("window.scrollTo") and _prev_jscode.startswith("window.scrollTo"))): LOG.debug("web_execute(`%s`) = `%s`", jscode, result) - self.__class__.web_execute._prev_jscode = jscode # type: ignore[attr-defined] # pylint: disable=protected-access + self.__class__.web_execute._prev_jscode = jscode # type: ignore[attr-defined] # noqa: SLF001 Private member accessed return result - async def web_find(self, selector_type:By, selector_value:str, *, parent:Element = None, timeout:int | float = 5) -> Element: + async def web_find(self, selector_type:By, selector_value:str, *, parent:Element | None = None, timeout:int | float = 5) -> Element: """ Locates an HTML element by the given selector type and value. @@ -408,7 +406,7 @@ class WebScrapingMixin: raise AssertionError(_("Unsupported selector type: %s") % selector_type) - async def web_find_all(self, selector_type:By, selector_value:str, *, parent:Element = None, timeout:int | float = 5) -> list[Element]: + async def web_find_all(self, selector_type:By, selector_value:str, *, parent:Element | None = None, timeout:int | float = 5) -> list[Element]: """ Locates an HTML element by ID. @@ -460,7 +458,7 @@ class WebScrapingMixin: await self.web_sleep() return input_field - async def web_open(self, url:str, *, timeout:int | float = 15000, reload_if_already_open:bool = False) -> None: + async def web_open(self, url:str, *, timeout:int | float = 15_000, reload_if_already_open:bool = False) -> None: """ :param url: url to open in browser :param timeout: timespan in seconds within the page needs to be loaded @@ -475,7 +473,7 @@ class WebScrapingMixin: await self.web_await(lambda: self.web_execute("document.readyState == 'complete'"), timeout = timeout, timeout_error_message = f"Page did not finish loading within {timeout} seconds.") - async def web_text(self, selector_type:By, selector_value:str, *, parent:Element = None, timeout:int | float = 5) -> str: + async def web_text(self, selector_type:By, selector_value:str, *, parent:Element | None = None, timeout:int | float = 5) -> str: return str(await (await self.web_find(selector_type, selector_value, parent = parent, timeout = timeout)).apply(""" function (elem) { let sel = window.getSelection() @@ -489,10 +487,11 @@ class WebScrapingMixin: } """)) - async def web_sleep(self, min_ms:int = 1000, max_ms:int = 2500) -> None: + async def web_sleep(self, min_ms:int = 1_000, max_ms:int = 2_500) -> None: duration = max_ms <= min_ms and min_ms or secrets.randbelow(max_ms - min_ms) + min_ms - LOG.log(loggers.INFO if duration > 1500 else loggers.DEBUG, " ... pausing for %d ms ...", duration) - await self.page.sleep(duration / 1000) + LOG.log(loggers.INFO if duration > 1_500 else loggers.DEBUG, # noqa: PLR2004 Magic value used in comparison + " ... pausing for %d ms ...", duration) + await self.page.sleep(duration / 1_000) async def web_request(self, url:str, method:str = "GET", valid_response_codes:int | Iterable[int] = 200, headers:dict[str, str] | None = None) -> dict[str, Any]: @@ -524,7 +523,7 @@ class WebScrapingMixin: return response # pylint: enable=dangerous-default-value - async def web_scroll_page_down(self, scroll_length: int = 10, scroll_speed: int = 10000, scroll_back_top: bool = False) -> None: + async def web_scroll_page_down(self, scroll_length: int = 10, scroll_speed: int = 10_000, *, scroll_back_top: bool = False) -> None: """ Smoothly scrolls the current web page down. @@ -537,13 +536,13 @@ class WebScrapingMixin: while current_y_pos < bottom_y_pos: # scroll in steps until bottom reached current_y_pos += scroll_length await self.web_execute(f'window.scrollTo(0, {current_y_pos})') # scroll one step - time.sleep(scroll_length / scroll_speed) + await asyncio.sleep(scroll_length / scroll_speed) if scroll_back_top: # scroll back to top in same style while current_y_pos > 0: current_y_pos -= scroll_length await self.web_execute(f'window.scrollTo(0, {current_y_pos})') - time.sleep(scroll_length / scroll_speed / 2) # double speed + await asyncio.sleep(scroll_length / scroll_speed / 2) # double speed async def web_select(self, selector_type:By, selector_value:str, selected_value:Any, timeout:int | float = 5) -> Element: """ diff --git a/tests/conftest.py b/tests/conftest.py index ca9b4ac..818c385 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -10,8 +10,8 @@ from unittest.mock import MagicMock import pytest from kleinanzeigen_bot import KleinanzeigenBot -from kleinanzeigen_bot.utils import loggers from kleinanzeigen_bot.extract import AdExtractor +from kleinanzeigen_bot.utils import loggers from kleinanzeigen_bot.utils.web_scraping_mixin import Browser loggers.configure_console_logging() diff --git a/tests/integration/test_web_scraping_mixin.py b/tests/integration/test_web_scraping_mixin.py index d0ae00c..d7562f9 100644 --- a/tests/integration/test_web_scraping_mixin.py +++ b/tests/integration/test_web_scraping_mixin.py @@ -1,12 +1,12 @@ -""" -SPDX-FileCopyrightText: © Sebastian Thomschke and contributors -SPDX-License-Identifier: AGPL-3.0-or-later -SPDX-ArtifactOfProjectHomePage: https://github.com/Second-Hand-Friends/kleinanzeigen-bot/ -""" -import os, platform +# SPDX-FileCopyrightText: © Sebastian Thomschke and contributors +# SPDX-License-Identifier: AGPL-3.0-or-later +# SPDX-ArtifactOfProjectHomePage: https://github.com/Second-Hand-Friends/kleinanzeigen-bot/ +import os +import platform from typing import cast -import nodriver, pytest +import nodriver +import pytest from kleinanzeigen_bot.utils import loggers from kleinanzeigen_bot.utils.misc import ensure diff --git a/tests/unit/test_ads.py b/tests/unit/test_ads.py index 5ce7979..a64e93e 100644 --- a/tests/unit/test_ads.py +++ b/tests/unit/test_ads.py @@ -1,8 +1,6 @@ -""" -SPDX-FileCopyrightText: © Sebastian Thomschke and contributors -SPDX-License-Identifier: AGPL-3.0-or-later -SPDX-ArtifactOfProjectHomePage: https://github.com/Second-Hand-Friends/kleinanzeigen-bot/ -""" +# SPDX-FileCopyrightText: © Sebastian Thomschke and contributors +# SPDX-License-Identifier: AGPL-3.0-or-later +# SPDX-ArtifactOfProjectHomePage: https://github.com/Second-Hand-Friends/kleinanzeigen-bot/ from typing import Any import pytest @@ -33,7 +31,7 @@ def test_calculate_content_hash_with_none_values() -> None: assert len(hash_value) == 64 # SHA-256 hash is 64 characters long -@pytest.mark.parametrize("config,prefix,expected", [ +@pytest.mark.parametrize(("config", "prefix", "expected"), [ # Test new flattened format - prefix ( {"ad_defaults": {"description_prefix": "Hello"}}, @@ -129,11 +127,11 @@ def test_get_description_affixes( expected: str ) -> None: """Test get_description_affixes function with various inputs.""" - result = ads.get_description_affixes(config, prefix) + result = ads.get_description_affixes(config, prefix = prefix) assert result == expected -@pytest.mark.parametrize("config,prefix,expected", [ +@pytest.mark.parametrize(("config", "prefix", "expected"), [ # Add test for malformed config ( {}, # Empty config @@ -161,16 +159,16 @@ def test_get_description_affixes( ]) def test_get_description_affixes_edge_cases(config: dict[str, Any], prefix: bool, expected: str) -> None: """Test edge cases for description affix handling.""" - assert ads.get_description_affixes(config, prefix) == expected + assert ads.get_description_affixes(config, prefix = prefix) == expected -@pytest.mark.parametrize("config,expected", [ - (None, ""), # Test with None - ([], ""), # Test with an empty list +@pytest.mark.parametrize(("config", "expected"), [ + (None, ""), # Test with None + ([], ""), # Test with an empty list ("string", ""), # Test with a string - (123, ""), # Test with an integer - (3.14, ""), # Test with a float - (set(), ""), # Test with an empty set + (123, ""), # Test with an integer + (3.14, ""), # Test with a float + (set(), ""), # Test with an empty set ]) def test_get_description_affixes_edge_cases_non_dict(config: Any, expected: str) -> None: """Test get_description_affixes function with non-dict inputs.""" diff --git a/tests/unit/test_bot.py b/tests/unit/test_bot.py index cee09fc..3fd287f 100644 --- a/tests/unit/test_bot.py +++ b/tests/unit/test_bot.py @@ -1,9 +1,8 @@ -""" -SPDX-FileCopyrightText: © Sebastian Thomschke and contributors -SPDX-License-Identifier: AGPL-3.0-or-later -SPDX-ArtifactOfProjectHomePage: https://github.com/Second-Hand-Friends/kleinanzeigen-bot/ -""" -import gc, pytest +# SPDX-FileCopyrightText: © Sebastian Thomschke and contributors +# SPDX-License-Identifier: AGPL-3.0-or-later +# SPDX-ArtifactOfProjectHomePage: https://github.com/Second-Hand-Friends/kleinanzeigen-bot/ +import gc, pytest # isort: skip + from kleinanzeigen_bot import KleinanzeigenBot diff --git a/tests/unit/test_extract.py b/tests/unit/test_extract.py index a9c3632..58b805e 100644 --- a/tests/unit/test_extract.py +++ b/tests/unit/test_extract.py @@ -1,9 +1,7 @@ -""" -SPDX-FileCopyrightText: © Jens Bergmann and contributors -SPDX-License-Identifier: AGPL-3.0-or-later -SPDX-ArtifactOfProjectHomePage: https://github.com/Second-Hand-Friends/kleinanzeigen-bot/ -""" -import json, os +# SPDX-FileCopyrightText: © Sebastian Thomschke and contributors +# SPDX-License-Identifier: AGPL-3.0-or-later +# SPDX-ArtifactOfProjectHomePage: https://github.com/Second-Hand-Friends/kleinanzeigen-bot/ +import json, os # isort: skip from typing import Any, TypedDict from unittest.mock import AsyncMock, MagicMock, call, patch @@ -30,7 +28,7 @@ class _SpecialAttributesDict(TypedDict, total = False): condition_s: str -class _TestCaseDict(TypedDict): +class _TestCaseDict(TypedDict): # noqa: PYI049 Private TypedDict `...` is never used belen_conf: _BelenConfDict expected: _SpecialAttributesDict @@ -44,15 +42,12 @@ class TestAdExtractorBasics: assert extractor.browser == browser_mock assert extractor.config == sample_config - @pytest.mark.parametrize( - "url,expected_id", - [ - ("https://www.kleinanzeigen.de/s-anzeige/test-title/12345678", 12345678), - ("https://www.kleinanzeigen.de/s-anzeige/another-test/98765432", 98765432), - ("https://www.kleinanzeigen.de/s-anzeige/invalid-id/abc", -1), - ("https://www.kleinanzeigen.de/invalid-url", -1), - ], - ) + @pytest.mark.parametrize(("url", "expected_id"), [ + ("https://www.kleinanzeigen.de/s-anzeige/test-title/12345678", 12345678), + ("https://www.kleinanzeigen.de/s-anzeige/another-test/98765432", 98765432), + ("https://www.kleinanzeigen.de/s-anzeige/invalid-id/abc", -1), + ("https://www.kleinanzeigen.de/invalid-url", -1), + ]) def test_extract_ad_id_from_ad_url(self, test_extractor: AdExtractor, url: str, expected_id: int) -> None: """Test extraction of ad ID from different URL formats.""" assert test_extractor.extract_ad_id_from_ad_url(url) == expected_id @@ -61,16 +56,13 @@ class TestAdExtractorBasics: class TestAdExtractorPricing: """Tests for pricing related functionality.""" - @pytest.mark.parametrize( - "price_text,expected_price,expected_type", - [ - ("50 €", 50, "FIXED"), - ("1.234 €", 1234, "FIXED"), - ("50 € VB", 50, "NEGOTIABLE"), - ("VB", None, "NEGOTIABLE"), - ("Zu verschenken", None, "GIVE_AWAY"), - ], - ) + @pytest.mark.parametrize(("price_text", "expected_price", "expected_type"), [ + ("50 €", 50, "FIXED"), + ("1.234 €", 1234, "FIXED"), + ("50 € VB", 50, "NEGOTIABLE"), + ("VB", None, "NEGOTIABLE"), + ("Zu verschenken", None, "GIVE_AWAY"), + ]) @pytest.mark.asyncio # pylint: disable=protected-access async def test_extract_pricing_info( @@ -95,14 +87,11 @@ class TestAdExtractorPricing: class TestAdExtractorShipping: """Tests for shipping related functionality.""" - @pytest.mark.parametrize( - "shipping_text,expected_type,expected_cost", - [ - ("+ Versand ab 2,99 €", "SHIPPING", 2.99), - ("Nur Abholung", "PICKUP", None), - ("Versand möglich", "SHIPPING", None), - ], - ) + @pytest.mark.parametrize(("shipping_text", "expected_type", "expected_cost"), [ + ("+ Versand ab 2,99 €", "SHIPPING", 2.99), + ("Nur Abholung", "PICKUP", None), + ("Versand möglich", "SHIPPING", None), + ]) @pytest.mark.asyncio # pylint: disable=protected-access async def test_extract_shipping_info( @@ -272,9 +261,9 @@ class TestAdExtractorNavigation: # Mocks needed for the actual execution flow ad_list_container_mock = MagicMock() pagination_section_mock = MagicMock() - cardbox_mock = MagicMock() # Represents the
  • element - link_mock = MagicMock() # Represents the element - link_mock.attrs = {'href': '/s-anzeige/test/12345'} # Configure the desired output + cardbox_mock = MagicMock() # Represents the
  • element + link_mock = MagicMock() # Represents the element + link_mock.attrs = {'href': '/s-anzeige/test/12345'} # Configure the desired output # Mocks for elements potentially checked but maybe not strictly needed for output # (depending on how robust the mocking is) @@ -287,18 +276,18 @@ class TestAdExtractorNavigation: # 3. Find for ad list container (inside loop) # 4. Find for the link (inside list comprehension) mock_web_find.side_effect = [ - ad_list_container_mock, # Call 1: find #my-manageitems-adlist (before loop) - pagination_section_mock, # Call 2: find .Pagination - ad_list_container_mock, # Call 3: find #my-manageitems-adlist (inside loop) - link_mock # Call 4: find 'div.manageitems-item-ad h3 a.text-onSurface' + ad_list_container_mock, # Call 1: find #my-manageitems-adlist (before loop) + pagination_section_mock, # Call 2: find .Pagination + ad_list_container_mock, # Call 3: find #my-manageitems-adlist (inside loop) + link_mock # Call 4: find 'div.manageitems-item-ad h3 a.text-onSurface' # Add more mocks here if the pagination navigation logic calls web_find again ] # 1. Find all 'Nächste' buttons (pagination check) - Return empty list for single page test case # 2. Find all '.cardbox' elements (inside loop) mock_web_find_all.side_effect = [ - [], # Call 1: find 'button[aria-label="Nächste"]' -> No next button = single page - [cardbox_mock] # Call 2: find .cardbox -> One ad item + [], # Call 1: find 'button[aria-label="Nächste"]' -> No next button = single page + [cardbox_mock] # Call 2: find .cardbox -> One ad item # Add more mocks here if pagination navigation calls web_find_all ] @@ -550,9 +539,9 @@ class TestAdExtractorContact: """Test contact info extraction when elements are not found.""" with patch.object(extractor, 'page', MagicMock()), \ patch.object(extractor, 'web_text', new_callable = AsyncMock, side_effect = TimeoutError()), \ - patch.object(extractor, 'web_find', new_callable = AsyncMock, side_effect = TimeoutError()): + patch.object(extractor, 'web_find', new_callable = AsyncMock, side_effect = TimeoutError()), \ + pytest.raises(TimeoutError): - with pytest.raises(TimeoutError): await extractor._extract_contact_from_ad_page() @pytest.mark.asyncio diff --git a/tests/unit/test_i18n.py b/tests/unit/test_i18n.py index 11f86f1..77fe72c 100644 --- a/tests/unit/test_i18n.py +++ b/tests/unit/test_i18n.py @@ -1,14 +1,13 @@ -""" -SPDX-FileCopyrightText: © Sebastian Thomschke and contributors -SPDX-License-Identifier: AGPL-3.0-or-later -SPDX-ArtifactOfProjectHomePage: https://github.com/Second-Hand-Friends/kleinanzeigen-bot/ -""" +# SPDX-FileCopyrightText: © Sebastian Thomschke and contributors +# SPDX-License-Identifier: AGPL-3.0-or-later +# SPDX-ArtifactOfProjectHomePage: https://github.com/Second-Hand-Friends/kleinanzeigen-bot/ import pytest from _pytest.monkeypatch import MonkeyPatch # pylint: disable=import-private-name + from kleinanzeigen_bot.utils import i18n -@pytest.mark.parametrize("lang, expected", [ +@pytest.mark.parametrize(("lang", "expected"), [ (None, ("en", "US", "UTF-8")), # Test with no LANG variable (should default to ("en", "US", "UTF-8")) ("fr", ("fr", None, "UTF-8")), # Test with just a language code ("fr_CA", ("fr", "CA", "UTF-8")), # Test with language + region, no encoding @@ -29,7 +28,7 @@ def test_detect_locale(monkeypatch: MonkeyPatch, lang: str | None, expected: i18 assert result == expected, f"For LANG={lang}, expected {expected} but got {result}" -@pytest.mark.parametrize("lang, noun, count, prefix_with_count, expected", [ +@pytest.mark.parametrize(("lang", "noun", "count", "prefix_with_count", "expected"), [ ("en", "field", 1, True, "1 field"), ("en", "field", 2, True, "2 fields"), ("en", "field", 2, False, "fields"), @@ -54,5 +53,5 @@ def test_pluralize( ) -> None: i18n.set_current_locale(i18n.Locale(lang, "US", "UTF_8")) - result = i18n.pluralize(noun, count, prefix_with_count) + result = i18n.pluralize(noun, count, prefix_with_count = prefix_with_count) assert result == expected, f"For LANG={lang}, expected {expected} but got {result}" diff --git a/tests/unit/test_init.py b/tests/unit/test_init.py index a760895..bc384fd 100644 --- a/tests/unit/test_init.py +++ b/tests/unit/test_init.py @@ -1,11 +1,9 @@ -""" -SPDX-FileCopyrightText: © Jens Bergmann and contributors -SPDX-License-Identifier: AGPL-3.0-or-later -SPDX-ArtifactOfProjectHomePage: https://github.com/Second-Hand-Friends/kleinanzeigen-bot/ -""" -import copy, os, tempfile +# SPDX-FileCopyrightText: © Jens Bergmann and contributors +# SPDX-License-Identifier: AGPL-3.0-or-later +# SPDX-ArtifactOfProjectHomePage: https://github.com/Second-Hand-Friends/kleinanzeigen-bot/ +import copy, os, tempfile # isort: skip from collections.abc import Generator -from datetime import datetime, timedelta +from datetime import timedelta from pathlib import Path from typing import Any from unittest.mock import AsyncMock, MagicMock, patch @@ -13,7 +11,7 @@ from unittest.mock import AsyncMock, MagicMock, patch import pytest from ruamel.yaml import YAML -from kleinanzeigen_bot import LOG, KleinanzeigenBot +from kleinanzeigen_bot import LOG, KleinanzeigenBot, misc from kleinanzeigen_bot._version import __version__ from kleinanzeigen_bot.ads import calculate_content_hash from kleinanzeigen_bot.utils import loggers @@ -191,7 +189,7 @@ class TestKleinanzeigenBotLogging: class TestKleinanzeigenBotCommandLine: """Tests for command line argument parsing.""" - @pytest.mark.parametrize("args,expected_command,expected_selector,expected_keep_old", [ + @pytest.mark.parametrize(("args", "expected_command", "expected_selector", "expected_keep_old"), [ (["publish", "--ads=all"], "publish", "all", False), (["verify"], "verify", "due", False), (["download", "--ads=12345"], "download", "12345", False), @@ -833,7 +831,7 @@ class TestKleinanzeigenBotAdDeletion: patch.object(test_bot, 'web_click', new_callable = AsyncMock), \ patch.object(test_bot, 'web_check', new_callable = AsyncMock, return_value = True): mock_find.return_value.attrs = {"content": "some-token"} - result = await test_bot.delete_ad(ad_cfg, True, published_ads) + result = await test_bot.delete_ad(ad_cfg, published_ads, delete_old_ads_by_title = True) assert result is True @pytest.mark.asyncio @@ -859,7 +857,7 @@ class TestKleinanzeigenBotAdDeletion: patch.object(test_bot, 'web_click', new_callable = AsyncMock), \ patch.object(test_bot, 'web_check', new_callable = AsyncMock, return_value = True): mock_find.return_value.attrs = {"content": "some-token"} - result = await test_bot.delete_ad(ad_cfg, False, published_ads) + result = await test_bot.delete_ad(ad_cfg, published_ads, delete_old_ads_by_title = False) assert result is True @@ -910,7 +908,7 @@ class TestKleinanzeigenBotAdRepublication: def test_check_ad_republication_no_changes(self, test_bot: KleinanzeigenBot, base_ad_config: dict[str, Any]) -> None: """Test that unchanged ads within interval are not marked for republication.""" - current_time = datetime.utcnow() + current_time = misc.now() three_days_ago = (current_time - timedelta(days = 3)).isoformat() # Create ad config with timestamps for republication check @@ -1235,7 +1233,7 @@ class TestKleinanzeigenBotChangedAds: # Mock the loading of the ad configuration with patch('kleinanzeigen_bot.utils.dicts.load_dict', side_effect=[ changed_ad, # First call returns the changed ad - {} # Second call for ad_fields.yaml + {} # Second call for ad_fields.yaml ]): ads_to_publish = test_bot.load_ads() @@ -1255,7 +1253,7 @@ class TestKleinanzeigenBotChangedAds: } # Create a changed ad that is also due for republication - current_time = datetime.utcnow() + current_time = misc.now() old_date = (current_time - timedelta(days=10)).isoformat() # Past republication interval changed_ad = create_ad_config( @@ -1287,7 +1285,7 @@ class TestKleinanzeigenBotChangedAds: # Mock the loading of the ad configuration with patch('kleinanzeigen_bot.utils.dicts.load_dict', side_effect=[ changed_ad, # First call returns the changed ad - {} # Second call for ad_fields.yaml + {} # Second call for ad_fields.yaml ]): ads_to_publish = test_bot.load_ads() diff --git a/tests/unit/test_translations.py b/tests/unit/test_translations.py index 4141eec..edc8c80 100644 --- a/tests/unit/test_translations.py +++ b/tests/unit/test_translations.py @@ -1,9 +1,9 @@ +# SPDX-FileCopyrightText: © Jens Bergmann and contributors +# SPDX-License-Identifier: AGPL-3.0-or-later +# SPDX-ArtifactOfProjectHomePage: https://github.com/Second-Hand-Friends/kleinanzeigen-bot/ """ -SPDX-FileCopyrightText: © Sebastian Thomschke and contributors -SPDX-License-Identifier: AGPL-3.0-or-later -SPDX-ArtifactOfProjectHomePage: https://github.com/Second-Hand-Friends/kleinanzeigen-bot/ - This module contains tests for verifying the completeness and correctness of translations in the project. + It ensures that: 1. All log messages in the code have corresponding translations 2. All translations in the YAML files are actually used in the code @@ -15,7 +15,7 @@ The tests work by: 3. Comparing the extracted messages with translations 4. Verifying no unused translations exist """ -import ast, os +import ast, os # isort: skip from collections import defaultdict from dataclasses import dataclass from importlib.resources import files @@ -105,7 +105,7 @@ def _extract_log_messages(file_path: str, exclude_debug:bool = False) -> Message messages: MessageDict = defaultdict(lambda: defaultdict(set)) def add_message(function: str, msg: str) -> None: - """Helper to add a message to the messages dictionary.""" + """Add a message to the messages dictionary.""" if function not in messages: messages[function] = defaultdict(set) if msg not in messages[function]: @@ -128,7 +128,7 @@ def _extract_log_messages(file_path: str, exclude_debug:bool = False) -> Message if (isinstance(node.func, ast.Attribute) and isinstance(node.func.value, ast.Name) and node.func.value.id in {'LOG', 'logger', 'logging'} and - node.func.attr in {None if exclude_debug else 'debug', 'info', 'warning', 'error', 'critical'}): + node.func.attr in {None if exclude_debug else 'debug', 'info', 'warning', 'error', 'exception', 'critical'}): if node.args: msg = extract_string_value(node.args[0]) if msg: @@ -390,7 +390,7 @@ def test_no_obsolete_translations(lang: str) -> None: if not isinstance(function_trans, dict): continue - for original_message in function_trans.keys(): + for original_message in function_trans: # Check if this message exists in the code message_exists = _message_exists_in_code(messages_by_file, module, function, original_message) diff --git a/tests/unit/test_utils_misc.py b/tests/unit/test_utils_misc.py index 96c15eb..2d17b12 100644 --- a/tests/unit/test_utils_misc.py +++ b/tests/unit/test_utils_misc.py @@ -1,9 +1,8 @@ -""" -SPDX-FileCopyrightText: © Sebastian Thomschke and contributors -SPDX-License-Identifier: AGPL-3.0-or-later -SPDX-ArtifactOfProjectHomePage: https://github.com/Second-Hand-Friends/kleinanzeigen-bot/ -""" +# SPDX-FileCopyrightText: © Sebastian Thomschke and contributors +# SPDX-License-Identifier: AGPL-3.0-or-later +# SPDX-ArtifactOfProjectHomePage: https://github.com/Second-Hand-Friends/kleinanzeigen-bot/ import pytest + from kleinanzeigen_bot.utils import misc