mirror of
https://github.com/Second-Hand-Friends/kleinanzeigen-bot.git
synced 2026-03-12 10:31:50 +01:00
initial import
This commit is contained in:
sebthom
261
kleinanzeigen_bot/selenium_mixin.py
Normal file
261
kleinanzeigen_bot/selenium_mixin.py
Normal file
@@ -0,0 +1,261 @@
|
||||
"""
|
||||
Copyright (C) 2022 Sebastian Thomschke and contributors
|
||||
SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
"""
|
||||
import logging, os, shutil, sys, tempfile
|
||||
from typing import Any, Callable, Dict, Final, Iterable, Tuple
|
||||
from importlib.resources import read_text as get_resource_as_string
|
||||
|
||||
from selenium import webdriver
|
||||
from selenium.common.exceptions import NoSuchElementException, TimeoutException
|
||||
from selenium.webdriver.common.by import By
|
||||
from selenium.webdriver.chrome.service import Service as ChromeService, DEFAULT_EXECUTEABLE_PATH as DEFAULT_CHROMEDRIVER_PATH
|
||||
from selenium.webdriver.chromium.webdriver import ChromiumDriver
|
||||
from selenium.webdriver.edge.service import Service as EdgeService, DEFAULT_EXECUTEABLE_PATH as DEFAULT_EDGEDRIVER_PATH
|
||||
from selenium.webdriver.remote.webdriver import WebDriver
|
||||
from selenium.webdriver.remote.webelement import WebElement
|
||||
from selenium.webdriver.support import expected_conditions as EC
|
||||
from selenium.webdriver.support.ui import Select, WebDriverWait
|
||||
import selenium_stealth
|
||||
import webdriver_manager.utils as ChromeDriverManagerUtils
|
||||
from webdriver_manager.chrome import ChromeDriverManager
|
||||
from webdriver_manager.microsoft import EdgeChromiumDriverManager
|
||||
from webdriver_manager.utils import ChromeType
|
||||
|
||||
from .utils import ensure, is_frozen, pause
|
||||
|
||||
LOG:Final[logging.Logger] = logging.getLogger("kleinanzeigen_bot.selenium_mixin")
|
||||
|
||||
|
||||
class SeleniumMixin:
|
||||
|
||||
def __init__(self):
|
||||
self.browser_arguments:Iterable[str] = []
|
||||
self.browser_binary_location:str = None
|
||||
self.webdriver:WebDriver = None
|
||||
|
||||
def __del__(self):
|
||||
if getattr(self, 'cacertfile', None):
|
||||
os.remove(self.cacertfile)
|
||||
|
||||
def create_webdriver_session(self) -> None:
|
||||
LOG.info("Creating WebDriver session...")
|
||||
|
||||
def init_browser_options(browser_options):
|
||||
browser_options.add_argument("--disable-crash-reporter")
|
||||
browser_options.add_argument("--no-first-run")
|
||||
browser_options.add_argument("--no-service-autorun")
|
||||
for chrome_option in self.browser_arguments:
|
||||
LOG.info(" -> Custom chrome argument: %s", chrome_option)
|
||||
browser_options.add_argument(chrome_option)
|
||||
|
||||
browser_options.add_experimental_option('excludeSwitches', ['enable-automation'])
|
||||
browser_options.add_experimental_option('useAutomationExtension', False)
|
||||
browser_options.add_experimental_option("prefs", {
|
||||
"credentials_enable_service": False,
|
||||
"profile.password_manager_enabled": False,
|
||||
"devtools.preferences.currentDockState": "\"bottom\""
|
||||
})
|
||||
|
||||
if self.browser_binary_location:
|
||||
browser_options.binary_location = self.browser_binary_location
|
||||
LOG.info(" -> Chrome binary location: %s", self.browser_binary_location)
|
||||
|
||||
return browser_options
|
||||
|
||||
# if run via py2exe fix resource lookup
|
||||
if is_frozen():
|
||||
import pathlib # pylint: disable=import-outside-toplevel
|
||||
|
||||
if not os.getenv("REQUESTS_CA_BUNDLE", None) or not os.path.exists(os.getenv("REQUESTS_CA_BUNDLE", None)):
|
||||
with tempfile.NamedTemporaryFile(delete = False) as tmp:
|
||||
LOG.debug("Writing cacert file to [%s]...", tmp.name)
|
||||
tmp.write(get_resource_as_string("certifi", "cacert.pem").encode('utf-8'))
|
||||
self.cacertfile = tmp.name
|
||||
os.environ['REQUESTS_CA_BUNDLE'] = self.cacertfile
|
||||
|
||||
read_text_orig = pathlib.Path.read_text
|
||||
|
||||
def read_text_new(self, encoding = None, errors = None):
|
||||
path = str(self)
|
||||
if "selenium_stealth" in path:
|
||||
return get_resource_as_string("selenium_stealth", self.name)
|
||||
return read_text_orig(self, encoding, errors)
|
||||
|
||||
pathlib.Path.read_text = read_text_new
|
||||
|
||||
# check if a chrome driver is present already
|
||||
if shutil.which(DEFAULT_CHROMEDRIVER_PATH):
|
||||
self.webdriver = webdriver.Chrome(options = init_browser_options(webdriver.ChromeOptions()))
|
||||
elif shutil.which(DEFAULT_EDGEDRIVER_PATH):
|
||||
self.webdriver = webdriver.ChromiumEdge(options = init_browser_options(webdriver.EdgeOptions()))
|
||||
else:
|
||||
# determine browser major version
|
||||
if self.browser_binary_location:
|
||||
chrome_type, chrome_version = self.get_browser_version(self.browser_binary_location)
|
||||
else:
|
||||
chrome_type, chrome_version = self.get_browser_version_from_os()
|
||||
chrome_major_version = chrome_version.split(".", 1)[0]
|
||||
|
||||
# download and install matching chrome driver
|
||||
if chrome_type == ChromeType.MSEDGE:
|
||||
webdriver_mgr = EdgeChromiumDriverManager(cache_valid_range = 14)
|
||||
webdriver_mgr.driver.browser_version = chrome_major_version
|
||||
webdriver_path = webdriver_mgr.install()
|
||||
self.webdriver = webdriver.ChromiumEdge(service = EdgeService(webdriver_path), options = init_browser_options(webdriver.EdgeOptions()))
|
||||
else:
|
||||
webdriver_mgr = ChromeDriverManager(chrome_type = chrome_type, cache_valid_range = 14)
|
||||
webdriver_mgr.driver.browser_version = chrome_major_version
|
||||
webdriver_path = webdriver_mgr.install()
|
||||
self.webdriver = webdriver.Chrome(service = ChromeService(webdriver_path), options = init_browser_options(webdriver.ChromeOptions()))
|
||||
|
||||
# workaround to support Edge, see https://github.com/diprajpatra/selenium-stealth/pull/25
|
||||
selenium_stealth.Driver = ChromiumDriver
|
||||
|
||||
selenium_stealth.stealth(self.webdriver, # https://github.com/diprajpatra/selenium-stealth#args
|
||||
languages = ("de-DE", "de", "en-US", "en"),
|
||||
vendor = "Google Inc.",
|
||||
platform = "Win32",
|
||||
webgl_vendor = "Intel Inc.",
|
||||
renderer = "Intel Iris OpenGL Engine",
|
||||
fix_hairline = True,
|
||||
)
|
||||
|
||||
LOG.info("New WebDriver session is: %s %s", self.webdriver.session_id, self.webdriver.command_executor._url) # pylint: disable=protected-access
|
||||
|
||||
def get_browser_version(self, executable_path: str) -> Tuple[ChromeType, str]:
|
||||
if sys.platform == "win32":
|
||||
import win32api # pylint: disable=import-outside-toplevel,import-error
|
||||
# pylint: disable=no-member
|
||||
lang, codepage = win32api.GetFileVersionInfo(executable_path, "\\VarFileInfo\\Translation")[0]
|
||||
product_name = win32api.GetFileVersionInfo(executable_path, f"\\StringFileInfo\\{lang:04X}{codepage:04X}\\ProductName")
|
||||
product_version = win32api.GetFileVersionInfo(executable_path, f"\\StringFileInfo\\{lang:04X}{codepage:04X}\\ProductVersion")
|
||||
# pylint: enable=no-member
|
||||
match product_name:
|
||||
case "Chromium":
|
||||
return (ChromeType.CHROMIUM, product_version)
|
||||
case "Microsoft Edge":
|
||||
return (ChromeType.MSEDGE, product_version)
|
||||
case _: # "Google Chrome"
|
||||
return (ChromeType.GOOGLE, product_version)
|
||||
|
||||
if sys.platform.startswith("linux"):
|
||||
cmd = ChromeDriverManagerUtils.linux_browser_apps_to_cmd(executable_path)
|
||||
else:
|
||||
cmd = executable_path + " --version"
|
||||
|
||||
version = ChromeDriverManagerUtils.read_version_from_cmd(cmd, r'\d+\.\d+\.\d+')
|
||||
filename = os.path.basename(executable_path).lower()
|
||||
if "chromium" in filename:
|
||||
return (ChromeType.CHROMIUM, version)
|
||||
if "edge" in filename:
|
||||
return (ChromeType.MSEDGE, version)
|
||||
return (ChromeType.GOOGLE, version)
|
||||
|
||||
def get_browser_version_from_os(self) -> Tuple[ChromeType, str]:
|
||||
version = ChromeDriverManagerUtils.get_browser_version_from_os(ChromeType.CHROMIUM)
|
||||
if version != "UNKNOWN":
|
||||
return (ChromeType.CHROMIUM, version)
|
||||
LOG.debug("Chromium not found")
|
||||
|
||||
version = ChromeDriverManagerUtils.get_browser_version_from_os(ChromeType.GOOGLE)
|
||||
if version != "UNKNOWN":
|
||||
return (ChromeType.GOOGLE, version)
|
||||
LOG.debug("Google Chrome not found")
|
||||
|
||||
version = ChromeDriverManagerUtils.get_browser_version_from_os(ChromeType.MSEDGE)
|
||||
if version != "UNKNOWN":
|
||||
return (ChromeType.MSEDGE, version)
|
||||
LOG.debug("Microsoft Edge not found")
|
||||
|
||||
return (None, None)
|
||||
|
||||
def web_await(self, condition: Callable[[WebDriver], WebElement], timeout:int = 5) -> WebElement:
|
||||
"""
|
||||
:param timeout: timeout in seconds
|
||||
:raises NoSuchElementException: if element could not be found within time
|
||||
"""
|
||||
try:
|
||||
return WebDriverWait(self.webdriver, timeout).until(condition)
|
||||
except TimeoutException as ex:
|
||||
raise NoSuchElementException from ex
|
||||
|
||||
def web_click(self, selector_type:By, selector_value:str, timeout:int = 5) -> WebElement:
|
||||
"""
|
||||
:param timeout: timeout in seconds
|
||||
:raises NoSuchElementException: if element could not be found within time
|
||||
"""
|
||||
elem = self.web_await(EC.element_to_be_clickable((selector_type, selector_value)), timeout)
|
||||
elem.click()
|
||||
pause()
|
||||
return elem
|
||||
|
||||
def web_execute(self, javascript:str) -> Any:
|
||||
"""
|
||||
:return: The command's JSON response
|
||||
"""
|
||||
return self.webdriver.execute_script(javascript)
|
||||
|
||||
def web_find(self, selector_type:By, selector_value:str, timeout:int = 5) -> WebElement:
|
||||
"""
|
||||
:param timeout: timeout in seconds
|
||||
:raises NoSuchElementException: if element could not be found within time
|
||||
"""
|
||||
return self.web_await(EC.presence_of_element_located((selector_type, selector_value)), timeout)
|
||||
|
||||
def web_input(self, selector_type:By, selector_value:str, text:str, timeout:int = 5) -> WebElement:
|
||||
"""
|
||||
:param timeout: timeout in seconds
|
||||
:raises NoSuchElementException: if element could not be found within time
|
||||
"""
|
||||
input_field = self.web_find(selector_type, selector_value, timeout)
|
||||
input_field.clear()
|
||||
input_field.send_keys(text)
|
||||
pause()
|
||||
|
||||
def web_open(self, url, timeout = 10, reload_if_already_open = False) -> None:
|
||||
LOG.debug(" -> Opening [%s]...", url)
|
||||
if not reload_if_already_open and url == self.webdriver.current_url:
|
||||
LOG.debug(" => skipping, [%s] is already open", url)
|
||||
return
|
||||
self.webdriver.get(url)
|
||||
WebDriverWait(self.webdriver, timeout).until(lambda _: self.web_execute("return document.readyState") == "complete")
|
||||
|
||||
# pylint: disable=dangerous-default-value
|
||||
def web_request(self, url:str, method:str = "GET", valid_response_codes:Iterable[int] = [200], headers:Dict[str, str] = None) -> Dict[str, Any]:
|
||||
method = method.upper()
|
||||
LOG.debug(" -> HTTP %s [%s]...", method, url)
|
||||
response = self.webdriver.execute_async_script(f"""
|
||||
var callback = arguments[arguments.length - 1];
|
||||
fetch("{url}", {{
|
||||
method: "{method}",
|
||||
redirect: "follow",
|
||||
headers: {headers or {}}
|
||||
}})
|
||||
.then(response => response.text().then(responseText => {{
|
||||
headers = {{}};
|
||||
response.headers.forEach((v, k) => headers[k] = v);
|
||||
callback({{
|
||||
"statusCode": response.status,
|
||||
"statusMessage": response.statusText,
|
||||
"headers": headers,
|
||||
"content": responseText
|
||||
}})
|
||||
}}))
|
||||
""")
|
||||
ensure(
|
||||
response["statusCode"] in valid_response_codes,
|
||||
f'Invalid response "{response["statusCode"]} response["statusMessage"]" received for HTTP {method} to {url}'
|
||||
)
|
||||
return response
|
||||
# pylint: enable=dangerous-default-value
|
||||
|
||||
def web_select(self, selector_type:By, selector_value:str, selected_value:Any, timeout:int = 5) -> WebElement:
|
||||
"""
|
||||
:param timeout: timeout in seconds
|
||||
:raises NoSuchElementException: if element could not be found within time
|
||||
"""
|
||||
elem = self.web_await(EC.element_to_be_clickable((selector_type, selector_value)), timeout)
|
||||
Select(elem).select_by_value(selected_value)
|
||||
pause()
|
||||
return elem
|
||||
Reference in New Issue
Block a user