Mirrors/kleinanzeigen-bot
1
0
Fork 0
mirror of https://github.com/Second-Hand-Friends/kleinanzeigen-bot.git synced 2026-03-12 02:31:45 +01:00
Code Issues Packages Projects Releases Wiki Activity

Fixes #17 #18 Use pdm + pyinstaller

Browse Source
This commit is contained in:
sebthom
2022-01-30 07:31:13 +01:00
parent 8e445f08c6
commit 1e1cffeab7
15 changed files with 1354 additions and 254 deletions
Download Patch File Download Diff File Expand all files Collapse all files

77
.github/workflows/build.yml vendored
View File

@@ -1,3 +1,6 @@
# Copyright (C) 2022 Sebastian Thomschke and contributors
# SPDX-License-Identifier: AGPL-3.0-or-later
#
# https://docs.github.com/en/free-pro-team@latest/actions/reference/workflow-syntax-for-github-actions
name: Build
@@ -41,28 +44,30 @@ jobs:
python --version
pip install .[dev]
python -m pip install --upgrade pip
- name: python setup.py check
run: |
# https://docs.python.org/3/distutils/apiref.html#module-distutils.command.check
python setup.py check --metadata --strict --verbose
pip install pdm
pip install pdm-packer
- name: bandit
# https://github.com/python/mypy/issues/11829
pip install -t __pypackages__/3.10/lib git+git://github.com/python/mypy.git@9b3147701f054bf8ef42bd96e33153b05976a5e1
# https://github.com/pdm-project/pdm/issues/728#issuecomment-1021771200
pip install -t __pypackages__/3.10/lib selenium
pdm install
- name: Scanning for security issues using bandit
run: |
set -eux
bandit -c pyproject.toml --exclude '*/.eggs/*' -r .
pdm run bandit
- name: pylint
run: |
set -eux
pip install pylint
pylint kleinanzeigen_bot
pdm run pylint
- name: pytest
run: |
set -eux
python -m pytest
pdm run pytest
- name: run kleinanzeigen_bot
run: |
@@ -74,9 +79,9 @@ jobs:
set -eux
python -m kleinanzeigen_bot help
python -m kleinanzeigen_bot version
python -m kleinanzeigen_bot verify
pdm run app help
pdm run app version
pdm run app verify
- name: "Build docker image"
if: startsWith(matrix.os, 'linux')
@@ -87,10 +92,44 @@ jobs:
docker run --rm kleinanzeigen-bot/kleinanzeigen-bot help
- name: py2exe
if: startsWith(matrix.os, 'windows')
- name: "Install: binutils (strip)"
if: startsWith(matrix.os, 'ubuntu')
run: sudo apt-get --no-install-recommends install -y binutils
- name: "Install: UPX"
run: |
python setup.py py2exe
set -eu
case "${{ matrix.target }}" in
macos-*)
brew install upx
;;
ubuntu-*)
mkdir /opt/upx
upx_download_url=$(curl -fsSL https://api.github.com/repos/upx/upx/releases/latest | grep browser_download_url | grep amd64_linux.tar.xz | cut "-d\"" -f4)
echo "Downloading [$upx_download_url]..."
curl -fL $upx_download_url | tar Jxv -C /opt/upx --strip-components=1
echo "/opt/upx" >> $GITHUB_PATH
;;
windows-*)
upx_download_url=$(curl -fsSL https://api.github.com/repos/upx/upx/releases/latest | grep browser_download_url | grep win64.zip | cut "-d\"" -f4)
echo "Downloading [$upx_download_url]..."
curl -fL -o /tmp/upx.zip $upx_download_url
echo "Extracting upx.zip..."
mkdir /tmp/upx
7z e /tmp/upx.zip -o/tmp/upx *.exe -r
echo "$(cygpath -wa /tmp/upx)" >> $GITHUB_PATH
;;
esac
- name: pyinstaller
run: |
set -eux
pdm run pyinstaller
ls -l dist
- name: run kleinanzeigen_bot.exe

2
.gitignore vendored
View File

@@ -11,11 +11,13 @@ _LOCAL/
kleinanzeigen_bot/version.py
# python
/__pypackages__
__pycache__
/build
/dist
/.eggs
/*.egg-info
/.pdm.toml
# Eclipse
/.project

21
README.md
View File

@@ -68,12 +68,17 @@ It is a spiritual successor to [AnzeigenOrg/ebayKleinanzeigen](https://github.co
cd kleinanzeigen-bot
```
1. Install the Python dependencies using:
```
pip install .
```bash
pip install pdm
# temporary workaround for https://github.com/pdm-project/pdm/issues/728#issuecomment-1021771200
pip install -t __pypackages__/3.10/lib selenium
pdm install
```
1. Run the app:
```
python -m kleinanzeigen_bot --help
pdm run app --help
```
### Installation using Docker
@@ -243,14 +248,14 @@ updated_on: # set automatically
> Please read [CONTRIBUTING.md](CONTRIBUTING.md) before contributing code. Thank you!
- Installing dev dependencies: `pip install .[dev]`
- Running unit tests: `python -m pytest` or `pytest`
- Running linter: `python -m pylint kleinanzeigen_bot` or `pylint kleinanzeigen_bot`
- Running unit tests: `pdm run pytest`
- Running linter: `pdm run pylint`
- Displaying effective version:`python setup.py --version`
- Creating Windows executable: `python setup.py py2exe`
- Creating Windows executable: `pdm run pyinstaller`
- Application bootstrap works like this:
```python
python -m kleinanzeigen_bot
pdm run app
|-> executes 'python -m kleinanzeigen_bot'
|-> executes 'kleinanzeigen_bot/__main__.py'
|-> executes main() function of 'kleinanzeigen_bot/__init__.py'
|-> executes KleinanzeigenBot().run()

38
docker/image/Dockerfile
View File

@@ -6,7 +6,7 @@
######################
# runtime image base
######################
FROM python:3-slim as runtime-base-image
FROM debian:stable-slim as runtime-base-image
LABEL maintainer="Sebastian Thomschke"
@@ -36,25 +36,47 @@ RUN set -eu \
FROM python:3-slim AS build-image
RUN apt-get update \
&& apt-get install --no-install-recommends -y git \
&& python -m pip install --upgrade pip
# install required libraries
&& apt-get install --no-install-recommends -y \
binutils `# required by pyinstaller` \
git `# required by pdm to generate app version` \
curl xz-utils `# required to install upx` \
# install upx
# upx is currently not supported on Linux, see https://github.com/pyinstaller/pyinstaller/discussions/6275
#&& mkdir /opt/upx \
#&& upx_download_url=$(curl -fsSL https://api.github.com/repos/upx/upx/releases/latest | grep browser_download_url | grep amd64_linux.tar.xz | cut "-d\"" -f4) \
#&& echo "Downloading [$upx_download_url]..." \
#&& curl -fL $upx_download_url | tar Jxv -C /opt/upx --strip-components=1 \
# upgrade pip
&& python -m pip install --upgrade pip \
# install pdm
&& pip install pdm
ENV PATH="/opt/upx:${PATH}"
COPY kleinanzeigen_bot /opt/app/kleinanzeigen_bot
COPY .git /opt/app/.git
COPY *.py *.txt *.toml /opt/app/
COPY README.md pdm.lock pyinstaller.spec pyproject.toml /opt/app/
RUN cd /opt/app \
&& ls -la . \
&& pip install --user . \
# generates version.py
&& python setup.py --version
# https://github.com/python/mypy/issues/11829
&& pip install -t __pypackages__/3.10/lib git+git://github.com/python/mypy.git@9b3147701f054bf8ef42bd96e33153b05976a5e1 \
# https://github.com/pdm-project/pdm/issues/728#issuecomment-1021771200
&& pip install -t __pypackages__/3.10/lib selenium \
&& pdm install \
&& ls -la kleinanzeigen_bot \
&& pdm run pyinstaller \
&& ls -l dist
RUN /opt/app/dist/kleinanzeigen-bot --help
######################
# final image
######################
FROM runtime-base-image
COPY --from=build-image /root/.local /root/.local
COPY --from=build-image /opt/app/dist/kleinanzeigen-bot /opt/kleinanzeigen-bot
ARG BUILD_DATE
ARG GIT_COMMIT_HASH

2
docker/image/run.sh
View File

@@ -21,4 +21,4 @@ fi
cd /mnt/data
python -m kleinanzeigen_bot --config $CONFIG_FILE "$@"
/opt/kleinanzeigen-bot --config $CONFIG_FILE "$@"

16
kleinanzeigen_bot/__init__.py
View File

@@ -4,6 +4,7 @@ SPDX-License-Identifier: AGPL-3.0-or-later
"""
import atexit, copy, getopt, glob, json, logging, os, signal, sys, textwrap, time, urllib
from datetime import datetime
import importlib.metadata
from logging.handlers import RotatingFileHandler
from typing import Any, Dict, Final, Iterable
@@ -20,11 +21,6 @@ LOG_ROOT:Final[logging.Logger] = logging.getLogger()
LOG:Final[logging.Logger] = logging.getLogger("kleinanzeigen_bot")
LOG.setLevel(logging.INFO)
try:
from .version import version as VERSION
except ModuleNotFoundError:
VERSION = "unknown"
class KleinanzeigenBot(SeleniumMixin):
@@ -50,7 +46,9 @@ class KleinanzeigenBot(SeleniumMixin):
def __del__(self):
if self.file_log:
LOG_ROOT.removeHandler(self.file_log)
super().__del__()
def get_version(self) -> str:
return importlib.metadata.version(__package__)
def run(self, args:Iterable[str]) -> None:
self.parse_args(args)
@@ -58,7 +56,7 @@ class KleinanzeigenBot(SeleniumMixin):
case "help":
self.show_help()
case "version":
print(VERSION)
print(self.get_version())
case "verify":
self.configure_file_logging()
self.load_config()
@@ -85,6 +83,8 @@ class KleinanzeigenBot(SeleniumMixin):
def show_help(self) -> None:
if is_frozen():
exe = sys.argv[0]
elif os.getenv("PDM_PROJECT_ROOT", ""):
exe = "pdm run app"
else:
exe = "python -m kleinanzeigen_bot"
@@ -143,6 +143,8 @@ class KleinanzeigenBot(SeleniumMixin):
self.file_log.setFormatter(logging.Formatter('%(asctime)s [%(levelname)s] %(message)s'))
LOG_ROOT.addHandler(self.file_log)
LOG.info("App version: %s", self.get_version())
def load_ads(self, exclude_inactive = True, exclude_undue = True) -> Iterable[Dict[str, Any]]:
LOG.info("Searching for ad files...")

30
kleinanzeigen_bot/selenium_mixin.py
View File

@@ -2,9 +2,8 @@
Copyright (C) 2022 Sebastian Thomschke and contributors
SPDX-License-Identifier: AGPL-3.0-or-later
"""
import logging, os, shutil, sys, tempfile
import logging, os, shutil, sys
from typing import Any, Callable, Dict, Final, Iterable, Tuple
from importlib.resources import read_text as get_resource_as_string
from selenium import webdriver
from selenium.common.exceptions import NoSuchElementException, TimeoutException
@@ -22,7 +21,7 @@ from webdriver_manager.chrome import ChromeDriverManager
from webdriver_manager.microsoft import EdgeChromiumDriverManager
from webdriver_manager.utils import ChromeType
from .utils import ensure, is_frozen, pause
from .utils import ensure, pause
LOG:Final[logging.Logger] = logging.getLogger("kleinanzeigen_bot.selenium_mixin")
@@ -34,10 +33,6 @@ class SeleniumMixin:
self.browser_binary_location:str = None
self.webdriver:WebDriver = None
def __del__(self):
if getattr(self, 'cacertfile', None):
os.remove(self.cacertfile)
def create_webdriver_session(self) -> None:
LOG.info("Creating WebDriver session...")
@@ -68,27 +63,6 @@ class SeleniumMixin:
LOG.info(" -> Chrome binary location: %s", self.browser_binary_location)
return browser_options
# if run via py2exe fix resource lookup
if is_frozen():
import pathlib # pylint: disable=import-outside-toplevel
if not os.getenv("REQUESTS_CA_BUNDLE", None) or not os.path.exists(os.getenv("REQUESTS_CA_BUNDLE", None)):
with tempfile.NamedTemporaryFile(delete = False) as tmp:
LOG.debug("Writing cacert file to [%s]...", tmp.name)
tmp.write(get_resource_as_string("certifi", "cacert.pem").encode('utf-8'))
self.cacertfile = tmp.name
os.environ['REQUESTS_CA_BUNDLE'] = self.cacertfile
read_text_orig = pathlib.Path.read_text
def read_text_new(self, encoding = None, errors = None):
path = str(self)
if "selenium_stealth" in path:
return get_resource_as_string("selenium_stealth", self.name)
return read_text_orig(self, encoding, errors)
pathlib.Path.read_text = read_text_new
# check if a chrome driver is present already
if shutil.which(DEFAULT_CHROMEDRIVER_PATH):
self.webdriver = webdriver.Chrome(options = init_browser_options(webdriver.ChromeOptions()))

1062
pdm.lock generated Normal file
View File

File diff suppressed because it is too large Load Diff

79
pyinstaller.spec Normal file
View File

@@ -0,0 +1,79 @@
# -*- mode: python ; coding: utf-8 -*-
"""
Copyright (C) 2022 Sebastian Thomschke and contributors
SPDX-License-Identifier: AGPL-3.0-or-later
PyInstaller config file, see https://pyinstaller.readthedocs.io/en/stable/spec-files.html
"""
from PyInstaller.utils.hooks import copy_metadata, collect_data_files
datas = [
* copy_metadata('kleinanzeigen_bot'), # required to get version info
* collect_data_files("kleinanzeigen_bot"), # embeds *.yaml files
* collect_data_files("selenium_stealth"), # embeds *.js files
]
block_cipher = None
analysis = Analysis(
['kleinanzeigen_bot/__main__.py'],
pathex = [],
binaries = [],
datas = datas,
hiddenimports = ['pkg_resources'],
hookspath = [],
hooksconfig = {},
runtime_hooks = [],
excludes = [
"_aix_support",
"_osx_support",
"argparse",
"backports",
"bz2",
"cryptography.hazmat",
"distutils",
"doctest",
"ftplib",
"lzma",
"pep517",
"pdb",
"pip",
"pydoc",
"pydoc_data",
"optparse",
"setuptools",
"six",
"statistics",
"test",
"unittest",
"xml.sax"
],
win_no_prefer_redirects = False,
win_private_assemblies = False,
cipher = block_cipher,
noarchive = False
)
pyz = PYZ(analysis.pure, analysis.zipped_data, cipher = block_cipher)
import shutil
exe = EXE(pyz,
analysis.scripts,
analysis.binaries,
analysis.zipfiles,
analysis.datas,
[],
name = 'kleinanzeigen-bot',
debug = False,
bootloader_ignore_signals = False,
strip = shutil.which("strip") is not None,
upx = shutil.which("upx") is not None,
upx_exclude = [],
runtime_tmpdir = None,
console = True,
disable_windowed_traceback = False,
target_arch = None,
codesign_identity = None,
entitlements_file = None
)

76
pyproject.toml
View File

@@ -1,10 +1,81 @@
# https://pip.pypa.io/en/stable/reference/build-system/pyproject-toml/
[build-system]
requires = ["pdm-pep517"]
build-backend = "pdm.pep517.api"
[project]
name = "kleinanzeigen-bot"
dynamic = ["version"]
description = "Command line tool to publish ads on ebay-kleinanzeigen.de"
readme = "README.md"
authors = [
{name = "sebthom", email = "sebthom@users.noreply.github.com"},
]
license = {text = "AGPL-3.0-or-later"}
classifiers = [ # https://pypi.org/classifiers/
"Development Status :: 4 - Beta",
"Environment :: Console",
"Operating System :: OS Independent",
"Intended Audience :: End Users/Desktop",
"Topic :: Office/Business",
"License :: OSI Approved :: GNU Affero General Public License v3 or later (AGPLv3+)",
"Programming Language :: Python :: 3.10"
]
requires-python = ">=3.10,<3.11" # <3.11 to get newer versions of pyinstaller
dependencies = [
"coloredlogs~=15.0",
"inflect~=5.3",
"ruamel.yaml~=0.17",
"pywin32==303; sys_platform == 'win32'",
"selenium~=4.1",
"selenium_stealth~=1.0",
"webdriver_manager~=3.5"]
[project.urls]
homepage = "https://github.com/kleinanzeigen-bot/kleinanzeigen-bot"
repository = "https://github.com/kleinanzeigen-bot/kleinanzeigen-bot"
documentation = "https://github.com/kleinanzeigen-bot/kleinanzeigen-bot/README.md"
[project.optional-dependencies]
[project.scripts]
# https://www.python.org/dev/peps/pep-0621/#entry-points
# https://blogs.thebitx.com/index.php/2021/09/02/pybites-how-to-package-and-deploy-cli-applications-with-python-pypa-setuptools-build/
kleinanzeigen_bot = 'kleinanzeigen_bot:main'
#####################
# pdm https://github.com/pdm-project/pdm/
#####################
[tool.pdm]
version = {use_scm = true}
[tool.pdm.dev-dependencies]
dev = [
"bandit~=1.7",
"pytest~=6.2",
"pyinstaller~=4.8",
"psutil",
"pylint~=2.12",
"mypy~=0.931",
]
[tool.pdm.scripts]
app = "python -m kleinanzeigen_bot"
bandit = "bandit -c pyproject.toml -r kleinanzeigen_bot"
pyinstaller = "pyinstaller pyinstaller.spec --clean"
pylint = "pylint kleinanzeigen_bot"
pytest = "python -m pytest -v"
#####################
# bandit https://github.com/PyCQA/bandit
#####################
[tool.bandit]
exclude = ["*/.eggs/*"] # broken :-( https://github.com/PyCQA/bandit/issues/657
#####################
@@ -46,9 +117,10 @@ max-locals = 30
max-returns = 10
max-statements = 70
#####################
# pytest
#####################
[tool.pytest.ini_options]
# https://docs.pytest.org/en/stable/reference.html#confval-addopts
addopts = "-p no:cacheprovider --doctest-modules --ignore=kleinanzeigen_bot/__main__.py"
addopts = "-p no:cacheprovider --doctest-modules --ignore=__pypackages__ --ignore=kleinanzeigen_bot/__main__.py"

6
requirements-dev.txt
View File

@@ -1,6 +0,0 @@
bandit~=1.7.1
py2exe~=0.11.0.1; sys_platform == 'win32'
pylint~=2.12.2
pytest~=6.2.5
setuptools~=60.5.0
wheel~=0.37.1

7
requirements.txt
View File

@@ -1,7 +0,0 @@
coloredlogs~=15.0.1
inflect~=5.3.0
ruamel.yaml~=0.17.20
pywin32==303; sys_platform == 'win32'
selenium~=4.1.0
selenium_stealth~=1.0.6
webdriver_manager~=3.5.2

161
setup.py
View File

@@ -1,161 +0,0 @@
#!/usr/bin/env python3
"""
Copyright (C) 2022 Sebastian Thomschke and contributors
SPDX-License-Identifier: AGPL-3.0-or-later
"""
import sys, warnings
import setuptools
warnings.filterwarnings("ignore", message = "setup_requires is deprecated", category = setuptools.SetuptoolsDeprecationWarning)
warnings.filterwarnings("ignore", message = "setuptools.installer is deprecated", category = setuptools.SetuptoolsDeprecationWarning)
setup_args = {}
if "py2exe" in sys.argv:
import importlib.resources, glob, os, py2exe, zipfile
# py2exe config https://www.py2exe.org/index.cgi/ListOfOptions
setup_args["options"] = {
"py2exe": {
"bundle_files": 1, # 1 = include the python runtime
"compressed": True,
"optimize": 2,
"includes": [
"kleinanzeigen_bot"
],
"excludes": [
"_aix_support",
"_osx_support",
"argparse",
"backports",
"bz2",
"cryptography.hazmat",
"distutils",
"doctest",
"ftplib",
"lzma",
"pep517",
"pip",
"pydoc",
"pydoc_data",
"optparse",
"pyexpat",
"six",
"statistics",
"test",
"unittest",
"xml.sax"
]
}
}
setup_args["console"] = [{
"script": "kleinanzeigen_bot/__main__.py",
"dest_base": "kleinanzeigen-bot",
}]
setup_args["zipfile"] = None
#
# embedding required DLLs directly into the exe
#
# http://www.py2exe.org/index.cgi/OverridingCriteraForIncludingDlls
bundle_dlls = ("libcrypto", "libffi", "libssl")
orig_determine_dll_type = py2exe.dllfinder.DllFinder.determine_dll_type
def determine_dll_type(self, dll_filepath):
basename = os.path.basename(dll_filepath)
if basename.startswith(bundle_dlls):
return "EXT"
return orig_determine_dll_type(self, dll_filepath)
py2exe.dllfinder.DllFinder.determine_dll_type = determine_dll_type
#
# embedding required resource files directly into the exe
#
files_to_embed = [
("kleinanzeigen_bot/resources", "kleinanzeigen_bot/resources/*.yaml"),
("certifi", importlib.resources.path("certifi", "cacert.pem")),
("selenium_stealth", os.path.dirname(importlib.resources.path("selenium_stealth.js", "util.js")))
]
orig_copy_files = py2exe.runtime.Runtime.copy_files
def embed_files(self, destdir):
orig_copy_files(self, destdir)
libpath = os.path.join(destdir, "kleinanzeigen-bot.exe")
with zipfile.ZipFile(libpath, "a", zipfile.ZIP_DEFLATED if self.options.compress else zipfile.ZIP_STORED) as arc:
for target, source in files_to_embed:
if os.path.isdir(source):
for file in os.listdir(source):
if self.options.verbose:
print(f"Embedding file {source}\\{file} in {libpath}")
arc.write(os.path.join(source, file), target + "/" + file)
elif isinstance(source, str):
for file in glob.glob(source, root_dir = os.getcwd(), recursive = True):
if self.options.verbose:
print(f"Embedding file {file} in {libpath}")
arc.write(file, target + "/" + os.path.basename(file))
else:
if self.options.verbose:
print(f"Embedding file {source} in {libpath}")
arc.write(source, target + "/" + os.path.basename(source))
os.remove(os.path.join(destdir, "cacert.pem")) # file was embedded
py2exe.runtime.Runtime.copy_files = embed_files
#
# use best zip compression level 9
#
from zipfile import ZipFile
class ZipFileExt(ZipFile):
def __init__(self, file, mode = "r", compression = zipfile.ZIP_STORED):
super().__init__(file, mode, compression, compresslevel = 9)
py2exe.runtime.zipfile.ZipFile = ZipFileExt
def load_requirements(filepath:str):
with open(filepath, encoding = "utf-8") as fd:
return [nonempty for line in fd if (nonempty := line.strip())]
setuptools.setup(
name = "kleinanzeigen-bot",
use_scm_version = {
"write_to": "kleinanzeigen_bot/version.py",
},
packages = setuptools.find_packages(),
package_data = {"kleinanzeigen_bot": ["resources/*.yaml"]},
# https://docs.python.org/3/distutils/setupscript.html#additional-meta-data
author = "The kleinanzeigen-bot authors",
author_email = "n/a",
url = "https://github.com/kleinanzeigen-bot/kleinanzeigen-bot",
description = "Command line tool to publish ads on ebay-kleinanzeigen.de",
license = "GNU AGPL 3.0+",
classifiers = [ # https://pypi.org/classifiers/
"Development Status :: 4 - Beta",
"Environment :: Console",
"Operating System :: OS Independent",
"Intended Audience :: End Users/Desktop",
"Topic :: Office/Business",
"License :: OSI Approved :: GNU Affero General Public License v3 or later (AGPLv3+)",
"Programming Language :: Python :: 3.10",
],
python_requires = ">=3.10",
install_requires = load_requirements("requirements.txt"),
extras_require = {
"dev": load_requirements("requirements-dev.txt")
},
setup_requires = [
"setuptools_scm"
],
** setup_args
)

4
tests/__init__.py Normal file
View File

@@ -0,0 +1,4 @@
"""
Copyright (C) 2022 Sebastian Thomschke and contributors
SPDX-License-Identifier: AGPL-3.0-or-later
"""

13
tests/test_utils.py Normal file
View File

@@ -0,0 +1,13 @@
"""
Copyright (C) 2022 Sebastian Thomschke and contributors
SPDX-License-Identifier: AGPL-3.0-or-later
"""
import time
from kleinanzeigen_bot import utils
def test_pause():
start = time.time()
utils.pause(100, 100)
elapsed = 1000 * (time.time() - start)
assert 99 < elapsed < 110