Mirrors/kleinanzeigen-bot
1
0
Fork 0
mirror of https://github.com/Second-Hand-Friends/kleinanzeigen-bot.git synced 2026-03-12 02:31:45 +01:00
Code Issues Packages Projects Releases Wiki Activity

upgrade webdriver-manager and improve browser version detection

Browse Source
This commit is contained in:
sebthom
2022-03-22 14:12:52 +01:00
parent 8d6486f53e
commit 1de13bf43b
5 changed files with 109 additions and 74 deletions
Download Patch File Download Diff File Expand all files Collapse all files

126
kleinanzeigen_bot/selenium_mixin.py
View File

@@ -2,7 +2,7 @@
Copyright (C) 2022 Sebastian Thomschke and contributors Copyright (C) 2022 Sebastian Thomschke and contributors
SPDX-License-Identifier: AGPL-3.0-or-later SPDX-License-Identifier: AGPL-3.0-or-later
""" """
import logging, os, shutil, sys import logging, os, shutil
from collections.abc import Callable, Iterable from collections.abc import Callable, Iterable
from typing import Any, Final from typing import Any, Final
@@ -21,7 +21,7 @@ import selenium_stealth
import webdriver_manager.utils as ChromeDriverManagerUtils import webdriver_manager.utils as ChromeDriverManagerUtils
from webdriver_manager.chrome import ChromeDriverManager from webdriver_manager.chrome import ChromeDriverManager
from webdriver_manager.microsoft import EdgeChromiumDriverManager from webdriver_manager.microsoft import EdgeChromiumDriverManager
from webdriver_manager.utils import ChromeType from webdriver_manager.utils import ChromeType, OSType
from .utils import ensure, pause, T from .utils import ensure, pause, T
@@ -92,26 +92,30 @@ class SeleniumMixin:
LOG.info(" -> Chrome binary location: %s", self.browser_config.binary_location) LOG.info(" -> Chrome binary location: %s", self.browser_config.binary_location)
return browser_options return browser_options
def create_webdriver_session(self) -> None: def create_webdriver_session(self, *, use_preinstalled_webdriver:bool = True) -> None:
LOG.info("Creating WebDriver session...") LOG.info("Creating WebDriver session...")
if not LOG.isEnabledFor(logging.DEBUG): if not LOG.isEnabledFor(logging.DEBUG):
os.environ['WDM_LOG_LEVEL'] = '0' # silence the web driver manager os.environ['WDM_LOG_LEVEL'] = '0' # silence the web driver manager
# check if a chrome driver is present already # check if a chrome driver is present already
if shutil.which(DEFAULT_CHROMEDRIVER_PATH): if use_preinstalled_webdriver and shutil.which(DEFAULT_CHROMEDRIVER_PATH):
LOG.info("Using pre-installed Chrome Driver [%s]", shutil.which(DEFAULT_CHROMEDRIVER_PATH))
self.webdriver = webdriver.Chrome(options = self._init_browser_options(webdriver.ChromeOptions())) self.webdriver = webdriver.Chrome(options = self._init_browser_options(webdriver.ChromeOptions()))
elif shutil.which(DEFAULT_EDGEDRIVER_PATH): elif use_preinstalled_webdriver and shutil.which(DEFAULT_EDGEDRIVER_PATH):
LOG.info("Using pre-installed Edge Driver [%s]", shutil.which(DEFAULT_EDGEDRIVER_PATH))
self.webdriver = webdriver.ChromiumEdge(options = self._init_browser_options(webdriver.EdgeOptions())) self.webdriver = webdriver.ChromiumEdge(options = self._init_browser_options(webdriver.EdgeOptions()))
else: else:
# determine browser major version # determine browser major version
if self.browser_config.binary_location: if self.browser_config.binary_location:
chrome_type, chrome_version = self.get_browser_version(self.browser_config.binary_location) chrome_type, chrome_version = self.get_browser_version(self.browser_config.binary_location)
else: else:
browser_info = self.get_browser_version_from_os() browser_info = self.find_compatible_browser()
if browser_info is None: if browser_info is None:
raise AssertionError("No supported browser found!") raise AssertionError("No supported browser found!")
chrome_type, chrome_version = browser_info chrome_path, chrome_type, chrome_version = browser_info
self.browser_config.binary_location = chrome_path
LOG.info("Using Browser: %s %s [%s]", chrome_type.upper(), chrome_version, self.browser_config.binary_location)
chrome_major_version = chrome_version.split(".", 1)[0] chrome_major_version = chrome_version.split(".", 1)[0]
# download and install matching chrome driver # download and install matching chrome driver
@@ -143,50 +147,88 @@ class SeleniumMixin:
LOG.info("New WebDriver session is: %s %s", self.webdriver.session_id, self.webdriver.command_executor._url) # pylint: disable=protected-access LOG.info("New WebDriver session is: %s %s", self.webdriver.session_id, self.webdriver.command_executor._url) # pylint: disable=protected-access
def get_browser_version(self, executable_path: str) -> tuple[ChromeType, str]: def get_browser_version(self, executable_path: str) -> tuple[ChromeType, str]:
if sys.platform == "win32": match ChromeDriverManagerUtils.os_name():
import win32api # pylint: disable=import-outside-toplevel,import-error case OSType.WIN:
# pylint: disable=no-member import win32api # pylint: disable=import-outside-toplevel,import-error
lang, codepage = win32api.GetFileVersionInfo(executable_path, "\\VarFileInfo\\Translation")[0] # pylint: disable=no-member
product_name = win32api.GetFileVersionInfo(executable_path, f"\\StringFileInfo\\{lang:04X}{codepage:04X}\\ProductName") lang, codepage = win32api.GetFileVersionInfo(executable_path, "\\VarFileInfo\\Translation")[0]
product_version = win32api.GetFileVersionInfo(executable_path, f"\\StringFileInfo\\{lang:04X}{codepage:04X}\\ProductVersion") product_name = win32api.GetFileVersionInfo(executable_path, f"\\StringFileInfo\\{lang:04X}{codepage:04X}\\ProductName")
# pylint: enable=no-member product_version = win32api.GetFileVersionInfo(executable_path, f"\\StringFileInfo\\{lang:04X}{codepage:04X}\\ProductVersion")
match product_name: # pylint: enable=no-member
case "Chromium": match product_name:
return (ChromeType.CHROMIUM, product_version) case "Chromium":
case "Microsoft Edge": return (ChromeType.CHROMIUM, product_version)
return (ChromeType.MSEDGE, product_version) case "Microsoft Edge":
case _: # "Google Chrome" return (ChromeType.MSEDGE, product_version)
return (ChromeType.GOOGLE, product_version) case _: # "Google Chrome"
return (ChromeType.GOOGLE, product_version)
if sys.platform.startswith("linux"): case OSType.LINUX:
cmd = ChromeDriverManagerUtils.linux_browser_apps_to_cmd(executable_path) version_cmd = ChromeDriverManagerUtils.linux_browser_apps_to_cmd(f'"{executable_path}"')
else:
cmd = executable_path + " --version" case _:
version_cmd = f'"{executable_path}" --version'
version = ChromeDriverManagerUtils.read_version_from_cmd(cmd, r'\d+\.\d+\.\d+')
filename = os.path.basename(executable_path).lower() filename = os.path.basename(executable_path).lower()
if "chromium" in filename: if "chromium" in filename:
return (ChromeType.CHROMIUM, version) return (
ChromeType.CHROMIUM,
ChromeDriverManagerUtils.read_version_from_cmd(version_cmd, ChromeDriverManagerUtils.PATTERN[ChromeType.CHROMIUM])
)
if "edge" in filename: if "edge" in filename:
return (ChromeType.MSEDGE, version) return (
return (ChromeType.GOOGLE, version) ChromeType.MSEDGE,
ChromeDriverManagerUtils.read_version_from_cmd(version_cmd, ChromeDriverManagerUtils.PATTERN[ChromeType.MSEDGE])
)
return (
ChromeType.GOOGLE,
ChromeDriverManagerUtils.read_version_from_cmd(version_cmd, ChromeDriverManagerUtils.PATTERN[ChromeType.GOOGLE])
)
def get_browser_version_from_os(self) -> tuple[ChromeType, str] | None: def find_compatible_browser(self) -> tuple[str, ChromeType, str] | None:
version = ChromeDriverManagerUtils.get_browser_version_from_os(ChromeType.CHROMIUM) match ChromeDriverManagerUtils.os_name():
if version != "UNKNOWN": case OSType.LINUX:
return (ChromeType.CHROMIUM, version) browser_paths = [
LOG.debug("Chromium not found") shutil.which("chromium"),
shutil.which("chromium-browser"),
shutil.which("google-chome"),
shutil.which("microsoft-edge")
]
version = ChromeDriverManagerUtils.get_browser_version_from_os(ChromeType.GOOGLE) case OSType.MAC:
if version != "UNKNOWN": browser_paths = [
return (ChromeType.GOOGLE, version) "/Applications/Chromium.app/Contents/MacOS/Chromium",
LOG.debug("Google Chrome not found") "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome",
"/Applications/Microsoft Edge.app/Contents/MacOS/Microsoft Edge",
]
version = ChromeDriverManagerUtils.get_browser_version_from_os(ChromeType.MSEDGE) case OSType.WIN:
if version != "UNKNOWN": browser_paths = [
return (ChromeType.MSEDGE, version) os.environ.get("ProgramFiles", "C:\\Program Files") + r'\Microsoft\Edge\Application\msedge.exe',
LOG.debug("Microsoft Edge not found") os.environ.get("ProgramFiles(x86)", "C:\\Program Files (x86)") + r'\Microsoft\Edge\Application\msedge.exe',
os.environ["ProgramFiles"] + r'\Chromium\Application\chrome.exe',
os.environ["ProgramFiles(x86)"] + r'\Chromium\Application\chrome.exe',
os.environ["LOCALAPPDATA"] + r'\Chromium\Application\chrome.exe',
os.environ["ProgramFiles"] + r'\Chrome\Application\chrome.exe',
os.environ["ProgramFiles(x86)"] + r'\Chrome\Application\chrome.exe',
os.environ["LOCALAPPDATA"] + r'\Chrome\Application\chrome.exe',
shutil.which("msedge.exe"),
shutil.which("chromium.exe"),
shutil.which("chrome.exe")
]
case _ as os_name:
LOG.warning("Installed browser for OS [%s] could not be detected", os_name)
return None
for browser_path in browser_paths:
if browser_path and os.path.isfile(browser_path):
return (browser_path, *self.get_browser_version(browser_path))
LOG.warning("Installed browser could not be detected")
return None return None
def web_await(self, condition: Callable[[WebDriver], T], timeout:float = 5, exception_on_timeout: Callable[[], Exception] | None = None) -> T: def web_await(self, condition: Callable[[WebDriver], T], timeout:float = 5, exception_on_timeout: Callable[[], Exception] | None = None) -> T:

34
pdm.lock generated
View File

@@ -87,20 +87,6 @@ dependencies = [
"humanfriendly>=9.1", "humanfriendly>=9.1",
] ]
[[package]]
name = "configparser"
version = "5.2.0"
requires_python = ">=3.6"
summary = "Updated configparser from Python 3.8 for Python 2.6+."
[[package]]
name = "crayons"
version = "0.4.0"
summary = "TextUI colors for Python."
dependencies = [
"colorama",
]
[[package]] [[package]]
name = "cryptography" name = "cryptography"
version = "36.0.2" version = "36.0.2"
@@ -528,12 +514,10 @@ dependencies = [
[[package]] [[package]]
name = "webdriver-manager" name = "webdriver-manager"
version = "3.5.3" version = "3.5.4"
requires_python = ">=3.6" requires_python = ">=3.6"
summary = "Library provides the way to automatically manage drivers for different browsers" summary = "Library provides the way to automatically manage drivers for different browsers"
dependencies = [ dependencies = [
"configparser",
"crayons",
"requests", "requests",
] ]
@@ -554,7 +538,7 @@ dependencies = [
[metadata] [metadata]
lock_version = "3.1" lock_version = "3.1"
content_hash = "sha256:2f987c3e7f97cf592c0366dfa6f3e7c95d77ad82b950872fc53577fd5a40267e" content_hash = "sha256:741030b205d43ca3ee40df89c6085d5e582f37e5b5100442e0608e5be4e728cc"
[metadata.files] [metadata.files]
"altgraph 0.17.2" = [ "altgraph 0.17.2" = [
@@ -653,14 +637,6 @@ content_hash = "sha256:2f987c3e7f97cf592c0366dfa6f3e7c95d77ad82b950872fc53577fd5
{file = "coloredlogs-15.0.1-py2.py3-none-any.whl", hash = "sha256:612ee75c546f53e92e70049c9dbfcc18c935a2b9a53b66085ce9ef6a6e5c0934"}, {file = "coloredlogs-15.0.1-py2.py3-none-any.whl", hash = "sha256:612ee75c546f53e92e70049c9dbfcc18c935a2b9a53b66085ce9ef6a6e5c0934"},
{file = "coloredlogs-15.0.1.tar.gz", hash = "sha256:7c991aa71a4577af2f82600d8f8f3a89f936baeaf9b50a9c197da014e5bf16b0"}, {file = "coloredlogs-15.0.1.tar.gz", hash = "sha256:7c991aa71a4577af2f82600d8f8f3a89f936baeaf9b50a9c197da014e5bf16b0"},
] ]
"configparser 5.2.0" = [
{file = "configparser-5.2.0-py3-none-any.whl", hash = "sha256:e8b39238fb6f0153a069aa253d349467c3c4737934f253ef6abac5fe0eca1e5d"},
{file = "configparser-5.2.0.tar.gz", hash = "sha256:1b35798fdf1713f1c3139016cfcbc461f09edbf099d1fb658d4b7479fcaa3daa"},
]
"crayons 0.4.0" = [
{file = "crayons-0.4.0-py2.py3-none-any.whl", hash = "sha256:e73ad105c78935d71fe454dd4b85c5c437ba199294e7ffd3341842bc683654b1"},
{file = "crayons-0.4.0.tar.gz", hash = "sha256:bd33b7547800f2cfbd26b38431f9e64b487a7de74a947b0fafc89b45a601813f"},
]
"cryptography 36.0.2" = [ "cryptography 36.0.2" = [
{file = "cryptography-36.0.2-cp36-abi3-macosx_10_10_universal2.whl", hash = "sha256:4e2dddd38a5ba733be6a025a1475a9f45e4e41139d1321f412c6b360b19070b6"}, {file = "cryptography-36.0.2-cp36-abi3-macosx_10_10_universal2.whl", hash = "sha256:4e2dddd38a5ba733be6a025a1475a9f45e4e41139d1321f412c6b360b19070b6"},
{file = "cryptography-36.0.2-cp36-abi3-macosx_10_10_x86_64.whl", hash = "sha256:4881d09298cd0b669bb15b9cfe6166f16fc1277b4ed0d04a22f3d6430cb30f1d"}, {file = "cryptography-36.0.2-cp36-abi3-macosx_10_10_x86_64.whl", hash = "sha256:4881d09298cd0b669bb15b9cfe6166f16fc1277b4ed0d04a22f3d6430cb30f1d"},
@@ -1051,9 +1027,9 @@ content_hash = "sha256:2f987c3e7f97cf592c0366dfa6f3e7c95d77ad82b950872fc53577fd5
{file = "urllib3-1.26.9-py2.py3-none-any.whl", hash = "sha256:44ece4d53fb1706f667c9bd1c648f5469a2ec925fcf3a776667042d645472c14"}, {file = "urllib3-1.26.9-py2.py3-none-any.whl", hash = "sha256:44ece4d53fb1706f667c9bd1c648f5469a2ec925fcf3a776667042d645472c14"},
{file = "urllib3-1.26.9.tar.gz", hash = "sha256:aabaf16477806a5e1dd19aa41f8c2b7950dd3c746362d7e3223dbe6de6ac448e"}, {file = "urllib3-1.26.9.tar.gz", hash = "sha256:aabaf16477806a5e1dd19aa41f8c2b7950dd3c746362d7e3223dbe6de6ac448e"},
] ]
"webdriver-manager 3.5.3" = [ "webdriver-manager 3.5.4" = [
{file = "webdriver_manager-3.5.3-py2.py3-none-any.whl", hash = "sha256:ef4320064bec22df5b2d7ea0509d08a99d3cecd1ddabd190f26f871dfa52a362"}, {file = "webdriver_manager-3.5.4-py2.py3-none-any.whl", hash = "sha256:b5b91b5df83181e002263fe27296967a5b19cb1ebe8e4a63ee83538394037df4"},
{file = "webdriver_manager-3.5.3.tar.gz", hash = "sha256:33c86736c51839abdcdeed1bbfd1f0d55277411bae4f232fdc9ace0c518e04f4"}, {file = "webdriver_manager-3.5.4.tar.gz", hash = "sha256:2eb7c2fe38ec5b06e2090164923e4dfb7c3ac4e7140333a3de9c7956f5047858"},
] ]
"wrapt 1.13.3" = [ "wrapt 1.13.3" = [
{file = "wrapt-1.13.3-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:e05e60ff3b2b0342153be4d1b597bbcfd8330890056b9619f4ad6b8d5c96a81a"}, {file = "wrapt-1.13.3-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:e05e60ff3b2b0342153be4d1b597bbcfd8330890056b9619f4ad6b8d5c96a81a"},

2
pyproject.toml
View File

@@ -33,7 +33,7 @@ dependencies = [
"pywin32==303; sys_platform == 'win32'", "pywin32==303; sys_platform == 'win32'",
"selenium~=4.1", "selenium~=4.1",
"selenium_stealth~=1.0", "selenium_stealth~=1.0",
"webdriver_manager==3.5.3" # version 3.5.4 breaks browser detection on Windows/MacOs "webdriver_manager~=3.5"
] ]
[project.urls] [project.urls]

9
tests/__init__.py
View File

@@ -2,3 +2,12 @@
Copyright (C) 2022 Sebastian Thomschke and contributors Copyright (C) 2022 Sebastian Thomschke and contributors
SPDX-License-Identifier: AGPL-3.0-or-later SPDX-License-Identifier: AGPL-3.0-or-later
""" """
import logging
from typing import Final
from kleinanzeigen_bot import utils
utils.configure_console_logging()
LOG:Final[logging.Logger] = logging.getLogger("kleinanzeigen_bot")
LOG.setLevel(logging.DEBUG)

12
tests/test_selenium_mixin.py
View File

@@ -12,11 +12,19 @@ from kleinanzeigen_bot import utils
def test_webdriver_auto_init(): def test_webdriver_auto_init():
selenium_mixin = SeleniumMixin() selenium_mixin = SeleniumMixin()
chrome_type, chrome_version = selenium_mixin.get_browser_version_from_os() browser_info = selenium_mixin.find_compatible_browser()
utils.ensure(browser_info is not None, "Chrome type not auto-detected")
chrome_path, chrome_type, chrome_version = browser_info
utils.ensure(chrome_path is not None, "Chrome type not auto-detected")
utils.ensure(chrome_type is not None, "Chrome type not auto-detected") utils.ensure(chrome_type is not None, "Chrome type not auto-detected")
utils.ensure(chrome_version is not None, "Chrome version not auto-detected") utils.ensure(chrome_version is not None, "Chrome version not auto-detected")
utils.ensure(selenium_mixin.webdriver is None, "Web driver must not be set before create_webdriver_session()") utils.ensure(selenium_mixin.webdriver is None, "Web driver must not be set before create_webdriver_session()")
selenium_mixin.create_webdriver_session() selenium_mixin.create_webdriver_session(use_preinstalled_webdriver = True)
utils.ensure(selenium_mixin.webdriver is not None, "Web driver must be set after create_webdriver_session()") utils.ensure(selenium_mixin.webdriver is not None, "Web driver must be set after create_webdriver_session()")
selenium_mixin.webdriver.quit() selenium_mixin.webdriver.quit()
selenium_mixin.webdriver = None
selenium_mixin.create_webdriver_session(use_preinstalled_webdriver = False)
selenium_mixin.webdriver.quit()