upgrade webdriver-manager and improve browser version detection

2026-03-12 10:31:50 +01:00 · 2022-03-22 14:12:52 +01:00
parent 8d6486f53e
commit 1de13bf43b
5 changed files with 109 additions and 74 deletions
--- a/kleinanzeigen_bot/selenium_mixin.py
+++ b/kleinanzeigen_bot/selenium_mixin.py
@@ -2,7 +2,7 @@
 Copyright (C) 2022 Sebastian Thomschke and contributors
 SPDX-License-Identifier: AGPL-3.0-or-later
 """
-import logging, os, shutil, sys
+import logging, os, shutil
 from collections.abc import Callable, Iterable
 from typing import Any, Final

@@ -21,7 +21,7 @@ import selenium_stealth
 import webdriver_manager.utils as ChromeDriverManagerUtils
 from webdriver_manager.chrome import ChromeDriverManager
 from webdriver_manager.microsoft import EdgeChromiumDriverManager
-from webdriver_manager.utils import ChromeType
+from webdriver_manager.utils import ChromeType, OSType

 from .utils import ensure, pause, T

@@ -92,26 +92,30 @@ class SeleniumMixin:
            LOG.info(" -> Chrome binary location: %s", self.browser_config.binary_location)
        return browser_options

-    def create_webdriver_session(self) -> None:
+    def create_webdriver_session(self, *, use_preinstalled_webdriver:bool = True) -> None:
        LOG.info("Creating WebDriver session...")

        if not LOG.isEnabledFor(logging.DEBUG):
            os.environ['WDM_LOG_LEVEL'] = '0'  # silence the web driver manager

        # check if a chrome driver is present already
-        if shutil.which(DEFAULT_CHROMEDRIVER_PATH):
+        if use_preinstalled_webdriver and shutil.which(DEFAULT_CHROMEDRIVER_PATH):
+            LOG.info("Using pre-installed Chrome Driver [%s]", shutil.which(DEFAULT_CHROMEDRIVER_PATH))
            self.webdriver = webdriver.Chrome(options = self._init_browser_options(webdriver.ChromeOptions()))
-        elif shutil.which(DEFAULT_EDGEDRIVER_PATH):
+        elif use_preinstalled_webdriver and shutil.which(DEFAULT_EDGEDRIVER_PATH):
+            LOG.info("Using pre-installed Edge Driver [%s]", shutil.which(DEFAULT_EDGEDRIVER_PATH))
            self.webdriver = webdriver.ChromiumEdge(options = self._init_browser_options(webdriver.EdgeOptions()))
        else:
            # determine browser major version
            if self.browser_config.binary_location:
                chrome_type, chrome_version = self.get_browser_version(self.browser_config.binary_location)
            else:
-                browser_info = self.get_browser_version_from_os()
+                browser_info = self.find_compatible_browser()
                if browser_info is None:
                    raise AssertionError("No supported browser found!")
-                chrome_type, chrome_version = browser_info
+                chrome_path, chrome_type, chrome_version = browser_info
+                self.browser_config.binary_location = chrome_path
+            LOG.info("Using Browser: %s %s [%s]", chrome_type.upper(), chrome_version, self.browser_config.binary_location)
            chrome_major_version = chrome_version.split(".", 1)[0]

            # download and install matching chrome driver
@@ -143,50 +147,88 @@ class SeleniumMixin:
        LOG.info("New WebDriver session is: %s %s", self.webdriver.session_id, self.webdriver.command_executor._url)  # pylint: disable=protected-access

    def get_browser_version(self, executable_path: str) -> tuple[ChromeType, str]:
-        if sys.platform == "win32":
-            import win32api  # pylint: disable=import-outside-toplevel,import-error
-            # pylint: disable=no-member
-            lang, codepage = win32api.GetFileVersionInfo(executable_path, "\\VarFileInfo\\Translation")[0]
-            product_name = win32api.GetFileVersionInfo(executable_path, f"\\StringFileInfo\\{lang:04X}{codepage:04X}\\ProductName")
-            product_version = win32api.GetFileVersionInfo(executable_path, f"\\StringFileInfo\\{lang:04X}{codepage:04X}\\ProductVersion")
-            # pylint: enable=no-member
-            match product_name:
-                case "Chromium":
-                    return (ChromeType.CHROMIUM, product_version)
-                case "Microsoft Edge":
-                    return (ChromeType.MSEDGE, product_version)
-                case _:  # "Google Chrome"
-                    return (ChromeType.GOOGLE, product_version)
+        match ChromeDriverManagerUtils.os_name():
+            case OSType.WIN:
+                import win32api  # pylint: disable=import-outside-toplevel,import-error
+                # pylint: disable=no-member
+                lang, codepage = win32api.GetFileVersionInfo(executable_path, "\\VarFileInfo\\Translation")[0]
+                product_name = win32api.GetFileVersionInfo(executable_path, f"\\StringFileInfo\\{lang:04X}{codepage:04X}\\ProductName")
+                product_version = win32api.GetFileVersionInfo(executable_path, f"\\StringFileInfo\\{lang:04X}{codepage:04X}\\ProductVersion")
+                # pylint: enable=no-member
+                match product_name:
+                    case "Chromium":
+                        return (ChromeType.CHROMIUM, product_version)
+                    case "Microsoft Edge":
+                        return (ChromeType.MSEDGE, product_version)
+                    case _:  # "Google Chrome"
+                        return (ChromeType.GOOGLE, product_version)

-        if sys.platform.startswith("linux"):
-            cmd = ChromeDriverManagerUtils.linux_browser_apps_to_cmd(executable_path)
-        else:
-            cmd = executable_path + " --version"
+            case OSType.LINUX:
+                version_cmd = ChromeDriverManagerUtils.linux_browser_apps_to_cmd(f'"{executable_path}"')
+
+            case _:
+                version_cmd = f'"{executable_path}" --version'

-        version = ChromeDriverManagerUtils.read_version_from_cmd(cmd, r'\d+\.\d+\.\d+')
        filename = os.path.basename(executable_path).lower()
        if "chromium" in filename:
-            return (ChromeType.CHROMIUM, version)
+            return (
+                ChromeType.CHROMIUM,
+                ChromeDriverManagerUtils.read_version_from_cmd(version_cmd, ChromeDriverManagerUtils.PATTERN[ChromeType.CHROMIUM])
+            )
        if "edge" in filename:
-            return (ChromeType.MSEDGE, version)
-        return (ChromeType.GOOGLE, version)
+            return (
+                ChromeType.MSEDGE,
+                ChromeDriverManagerUtils.read_version_from_cmd(version_cmd, ChromeDriverManagerUtils.PATTERN[ChromeType.MSEDGE])
+            )
+        return (
+            ChromeType.GOOGLE,
+            ChromeDriverManagerUtils.read_version_from_cmd(version_cmd, ChromeDriverManagerUtils.PATTERN[ChromeType.GOOGLE])
+        )

-    def get_browser_version_from_os(self) -> tuple[ChromeType, str] | None:
-        version = ChromeDriverManagerUtils.get_browser_version_from_os(ChromeType.CHROMIUM)
-        if version != "UNKNOWN":
-            return (ChromeType.CHROMIUM, version)
-        LOG.debug("Chromium not found")
+    def find_compatible_browser(self) -> tuple[str, ChromeType, str] | None:
+        match ChromeDriverManagerUtils.os_name():
+            case OSType.LINUX:
+                browser_paths = [
+                    shutil.which("chromium"),
+                    shutil.which("chromium-browser"),
+                    shutil.which("google-chome"),
+                    shutil.which("microsoft-edge")
+                ]

-        version = ChromeDriverManagerUtils.get_browser_version_from_os(ChromeType.GOOGLE)
-        if version != "UNKNOWN":
-            return (ChromeType.GOOGLE, version)
-        LOG.debug("Google Chrome not found")
+            case OSType.MAC:
+                browser_paths = [
+                    "/Applications/Chromium.app/Contents/MacOS/Chromium",
+                    "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome",
+                    "/Applications/Microsoft Edge.app/Contents/MacOS/Microsoft Edge",
+                ]

-        version = ChromeDriverManagerUtils.get_browser_version_from_os(ChromeType.MSEDGE)
-        if version != "UNKNOWN":
-            return (ChromeType.MSEDGE, version)
-        LOG.debug("Microsoft Edge not found")
+            case OSType.WIN:
+                browser_paths = [
+                    os.environ.get("ProgramFiles", "C:\\Program Files") + r'\Microsoft\Edge\Application\msedge.exe',
+                    os.environ.get("ProgramFiles(x86)", "C:\\Program Files (x86)") + r'\Microsoft\Edge\Application\msedge.exe',

+                    os.environ["ProgramFiles"] + r'\Chromium\Application\chrome.exe',
+                    os.environ["ProgramFiles(x86)"] + r'\Chromium\Application\chrome.exe',
+                    os.environ["LOCALAPPDATA"] + r'\Chromium\Application\chrome.exe',
+
+                    os.environ["ProgramFiles"] + r'\Chrome\Application\chrome.exe',
+                    os.environ["ProgramFiles(x86)"] + r'\Chrome\Application\chrome.exe',
+                    os.environ["LOCALAPPDATA"] + r'\Chrome\Application\chrome.exe',
+
+                    shutil.which("msedge.exe"),
+                    shutil.which("chromium.exe"),
+                    shutil.which("chrome.exe")
+                ]
+
+            case _ as os_name:
+                LOG.warning("Installed browser for OS [%s] could not be detected", os_name)
+                return None
+
+        for browser_path in browser_paths:
+            if browser_path and os.path.isfile(browser_path):
+                return (browser_path, *self.get_browser_version(browser_path))
+
+        LOG.warning("Installed browser could not be detected")
        return None

    def web_await(self, condition: Callable[[WebDriver], T], timeout:float = 5, exception_on_timeout: Callable[[], Exception] | None = None) -> T:
--- a/pdm.lock
+++ b/pdm.lock
@@ -87,20 +87,6 @@ dependencies = [
    "humanfriendly>=9.1",
 ]

-[[package]]
-name = "configparser"
-version = "5.2.0"
-requires_python = ">=3.6"
-summary = "Updated configparser from Python 3.8 for Python 2.6+."
-
-[[package]]
-name = "crayons"
-version = "0.4.0"
-summary = "TextUI colors for Python."
-dependencies = [
-    "colorama",
-]
-
 [[package]]
 name = "cryptography"
 version = "36.0.2"
@@ -528,12 +514,10 @@ dependencies = [

 [[package]]
 name = "webdriver-manager"
-version = "3.5.3"
+version = "3.5.4"
 requires_python = ">=3.6"
 summary = "Library provides the way to automatically manage drivers for different browsers"
 dependencies = [
-    "configparser",
-    "crayons",
    "requests",
 ]

@@ -554,7 +538,7 @@ dependencies = [

 [metadata]
 lock_version = "3.1"
-content_hash = "sha256:2f987c3e7f97cf592c0366dfa6f3e7c95d77ad82b950872fc53577fd5a40267e"
+content_hash = "sha256:741030b205d43ca3ee40df89c6085d5e582f37e5b5100442e0608e5be4e728cc"

 [metadata.files]
 "altgraph 0.17.2" = [
@@ -653,14 +637,6 @@ content_hash = "sha256:2f987c3e7f97cf592c0366dfa6f3e7c95d77ad82b950872fc53577fd5
    {file = "coloredlogs-15.0.1-py2.py3-none-any.whl", hash = "sha256:612ee75c546f53e92e70049c9dbfcc18c935a2b9a53b66085ce9ef6a6e5c0934"},
    {file = "coloredlogs-15.0.1.tar.gz", hash = "sha256:7c991aa71a4577af2f82600d8f8f3a89f936baeaf9b50a9c197da014e5bf16b0"},
 ]
-"configparser 5.2.0" = [
-    {file = "configparser-5.2.0-py3-none-any.whl", hash = "sha256:e8b39238fb6f0153a069aa253d349467c3c4737934f253ef6abac5fe0eca1e5d"},
-    {file = "configparser-5.2.0.tar.gz", hash = "sha256:1b35798fdf1713f1c3139016cfcbc461f09edbf099d1fb658d4b7479fcaa3daa"},
-]
-"crayons 0.4.0" = [
-    {file = "crayons-0.4.0-py2.py3-none-any.whl", hash = "sha256:e73ad105c78935d71fe454dd4b85c5c437ba199294e7ffd3341842bc683654b1"},
-    {file = "crayons-0.4.0.tar.gz", hash = "sha256:bd33b7547800f2cfbd26b38431f9e64b487a7de74a947b0fafc89b45a601813f"},
-]
 "cryptography 36.0.2" = [
    {file = "cryptography-36.0.2-cp36-abi3-macosx_10_10_universal2.whl", hash = "sha256:4e2dddd38a5ba733be6a025a1475a9f45e4e41139d1321f412c6b360b19070b6"},
    {file = "cryptography-36.0.2-cp36-abi3-macosx_10_10_x86_64.whl", hash = "sha256:4881d09298cd0b669bb15b9cfe6166f16fc1277b4ed0d04a22f3d6430cb30f1d"},
@@ -1051,9 +1027,9 @@ content_hash = "sha256:2f987c3e7f97cf592c0366dfa6f3e7c95d77ad82b950872fc53577fd5
    {file = "urllib3-1.26.9-py2.py3-none-any.whl", hash = "sha256:44ece4d53fb1706f667c9bd1c648f5469a2ec925fcf3a776667042d645472c14"},
    {file = "urllib3-1.26.9.tar.gz", hash = "sha256:aabaf16477806a5e1dd19aa41f8c2b7950dd3c746362d7e3223dbe6de6ac448e"},
 ]
-"webdriver-manager 3.5.3" = [
-    {file = "webdriver_manager-3.5.3-py2.py3-none-any.whl", hash = "sha256:ef4320064bec22df5b2d7ea0509d08a99d3cecd1ddabd190f26f871dfa52a362"},
-    {file = "webdriver_manager-3.5.3.tar.gz", hash = "sha256:33c86736c51839abdcdeed1bbfd1f0d55277411bae4f232fdc9ace0c518e04f4"},
+"webdriver-manager 3.5.4" = [
+    {file = "webdriver_manager-3.5.4-py2.py3-none-any.whl", hash = "sha256:b5b91b5df83181e002263fe27296967a5b19cb1ebe8e4a63ee83538394037df4"},
+    {file = "webdriver_manager-3.5.4.tar.gz", hash = "sha256:2eb7c2fe38ec5b06e2090164923e4dfb7c3ac4e7140333a3de9c7956f5047858"},
 ]
 "wrapt 1.13.3" = [
    {file = "wrapt-1.13.3-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:e05e60ff3b2b0342153be4d1b597bbcfd8330890056b9619f4ad6b8d5c96a81a"},
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -33,7 +33,7 @@ dependencies = [
    "pywin32==303; sys_platform == 'win32'",
    "selenium~=4.1",
    "selenium_stealth~=1.0",
-    "webdriver_manager==3.5.3" # version 3.5.4 breaks browser detection on Windows/MacOs
+    "webdriver_manager~=3.5"
 ]

 [project.urls]
--- a/tests/init.py
+++ b/tests/init.py
@@ -2,3 +2,12 @@
 Copyright (C) 2022 Sebastian Thomschke and contributors
 SPDX-License-Identifier: AGPL-3.0-or-later
 """
+import logging
+from typing import Final
+
+from kleinanzeigen_bot import utils
+
+utils.configure_console_logging()
+
+LOG:Final[logging.Logger] = logging.getLogger("kleinanzeigen_bot")
+LOG.setLevel(logging.DEBUG)
--- a/tests/test_selenium_mixin.py
+++ b/tests/test_selenium_mixin.py
@@ -12,11 +12,19 @@ from kleinanzeigen_bot import utils
 def test_webdriver_auto_init():
    selenium_mixin = SeleniumMixin()

-    chrome_type, chrome_version = selenium_mixin.get_browser_version_from_os()
+    browser_info = selenium_mixin.find_compatible_browser()
+    utils.ensure(browser_info is not None, "Chrome type not auto-detected")
+
+    chrome_path, chrome_type, chrome_version = browser_info
+    utils.ensure(chrome_path is not None, "Chrome type not auto-detected")
    utils.ensure(chrome_type is not None, "Chrome type not auto-detected")
    utils.ensure(chrome_version is not None, "Chrome version not auto-detected")

    utils.ensure(selenium_mixin.webdriver is None, "Web driver must not be set before create_webdriver_session()")
-    selenium_mixin.create_webdriver_session()
+    selenium_mixin.create_webdriver_session(use_preinstalled_webdriver = True)
    utils.ensure(selenium_mixin.webdriver is not None, "Web driver must be set after create_webdriver_session()")
    selenium_mixin.webdriver.quit()
+
+    selenium_mixin.webdriver = None
+    selenium_mixin.create_webdriver_session(use_preinstalled_webdriver = False)
+    selenium_mixin.webdriver.quit()