fix: improve Chrome version detection to reuse existing browsers (#615)

This commit is contained in:
Jens Bergmann
2025-08-20 12:51:13 +02:00
committed by GitHub
parent 21cdabb469
commit 37a36988c3
6 changed files with 383 additions and 124 deletions

View File

@@ -426,31 +426,27 @@ kleinanzeigen_bot/utils/web_scraping_mixin.py:
"(fail) Browser binary is not executable": "(Fehler) Browser-Binärdatei ist nicht ausführbar"
"(fail) No compatible browser found": "(Fehler) Kein kompatibler Browser gefunden"
"(fail) User data directory permissions issue": "(Fehler) Benutzerdatenverzeichnis-Berechtigungsproblem"
"(fail) Remote debugging port is not open": "(Fehler) Remote-Debugging-Port ist nicht offen"
"(fail) Running as root - this can cause browser connection issues": "(Fehler) Läuft als Root - dies kann Browser-Verbindungsprobleme verursachen"
"(info) User data directory does not exist (will be created): %s": "(Info) Benutzerdatenverzeichnis existiert nicht (wird erstellt): %s"
"(info) Remote debugging port configured: %d": "(Info) Remote-Debugging-Port konfiguriert: %d"
"(info) Remote debugging port is not open": "(Info) Remote-Debugging-Port ist nicht offen"
"(info) No browser processes currently running": "(Info) Derzeit keine Browser-Prozesse aktiv"
"(fail) Running as root - this can cause browser issues": "(Fehler) Läuft als Root - dies kann Browser-Probleme verursachen"
"(info) Found %d browser processes running": "(Info) %d Browser-Prozesse aktiv gefunden"
"(info) Windows detected - check Windows Defender and antivirus software": "(Info) Windows erkannt - überprüfen Sie Windows Defender und Antivirensoftware"
"(info) macOS detected - check Gatekeeper and security settings": "(Info) macOS erkannt - überprüfen Sie Gatekeeper und Sicherheitseinstellungen"
"(info) Linux detected - check if running as root (not recommended)": "(Info) Linux erkannt - überprüfen Sie, ob als Root ausgeführt wird (nicht empfohlen)"
" - PID %d: %s": " - PID %d: %s"
" Make sure browser is started with: --remote-debugging-port=%d": " Stellen Sie sicher, dass der Browser gestartet wird mit: --remote-debugging-port=%d"
" - PID %d: %s (remote debugging enabled)": " - PID %d: %s (Remote-Debugging aktiviert)"
" - PID %d: %s (remote debugging NOT enabled)": " - PID %d: %s (Remote-Debugging NICHT aktiviert)"
"(ok) Remote debugging API accessible - Browser: %s": "(ok) Remote-Debugging-API zugänglich - Browser: %s"
"(fail) Remote debugging port is open but API not accessible: %s": "(Fehler) Remote-Debugging-Port ist offen, aber API nicht zugänglich: %s"
" This might indicate a browser update issue or configuration problem": " Dies könnte auf ein Browser-Update-Problem oder Konfigurationsproblem hinweisen"
_validate_chrome_136_configuration:
" -> %s 136+ configuration validation failed: %s": " -> %s 136+ Konfigurationsvalidierung fehlgeschlagen: %s"
" -> %s 136+ configuration validation passed": " -> %s 136+ Konfigurationsvalidierung bestanden"
_validate_chrome_version_configuration:
" -> %s 136+ detected: %s": " -> %s 136+ erkannt: %s"
" -> %s 136+ configuration validation passed": " -> %s 136+ Konfigurationsvalidierung bestanden"
" -> %s 136+ configuration validation failed: %s": " -> %s 136+ Konfigurationsvalidierung fehlgeschlagen: %s"
" -> %s version detected: %s (pre-136, no special validation required)": " -> %s-Version erkannt: %s (vor 136, keine besondere Validierung erforderlich)"
" -> Please update your configuration to include --user-data-dir for remote debugging": " -> Bitte aktualisieren Sie Ihre Konfiguration, um --user-data-dir für Remote-Debugging einzuschließen"
" -> Skipping browser version validation in test environment": " -> Browser-Versionsvalidierung in Testumgebung wird übersprungen"
" -> Browser version detection failed, skipping validation: %s": " -> Browser-Versionserkennung fehlgeschlagen, Validierung wird übersprungen: %s"
" -> Unexpected error during browser version validation, skipping: %s": " -> Unerwarteter Fehler bei Browser-Versionsvalidierung, wird übersprungen: %s"

View File

@@ -4,6 +4,7 @@
import json
import re
import subprocess # noqa: S404
import urllib.error
import urllib.request
from typing import Any, Final
@@ -59,6 +60,24 @@ def parse_version_string(version_string:str) -> int:
return int(match.group(1))
def _normalize_browser_name(browser_name:str) -> str:
"""
Normalize browser name for consistent detection.
Args:
browser_name: Raw browser name from detection
Returns:
Normalized browser name
"""
browser_name_lower = browser_name.lower()
if "edge" in browser_name_lower or "edg" in browser_name_lower:
return "Edge"
if "chromium" in browser_name_lower:
return "Chromium"
return "Chrome"
def detect_chrome_version_from_binary(binary_path:str) -> ChromeVersionInfo | None:
"""
Detect Chrome version by running the browser binary.
@@ -90,11 +109,7 @@ def detect_chrome_version_from_binary(binary_path:str) -> ChromeVersionInfo | No
version_string = version_match.group(1) if version_match else output
# Determine browser name from binary path
browser_name = "Chrome"
if "edge" in binary_path.lower():
browser_name = "Edge"
elif "chromium" in binary_path.lower():
browser_name = "Chromium"
browser_name = _normalize_browser_name(binary_path)
return ChromeVersionInfo(version_string, major_version, browser_name)
@@ -125,7 +140,7 @@ def detect_chrome_version_from_remote_debugging(host:str = "127.0.0.1", port:int
# Extract version information
user_agent = version_data.get("User-Agent", "")
browser_name = version_data.get("Browser", "Unknown")
browser_name = _normalize_browser_name(version_data.get("Browser", "Unknown"))
# Parse version from User-Agent string
# Example: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.6778.0 Safari/537.36"
@@ -139,6 +154,12 @@ def detect_chrome_version_from_remote_debugging(host:str = "127.0.0.1", port:int
return ChromeVersionInfo(version_string, major_version, browser_name)
except urllib.error.URLError as e:
LOG.debug("Remote debugging API not accessible: %s", e)
return None
except json.JSONDecodeError as e:
LOG.debug("Invalid JSON response from remote debugging API: %s", e)
return None
except Exception as e:
LOG.debug("Failed to detect browser version from remote debugging: %s", str(e))
return None
@@ -148,7 +169,7 @@ def validate_chrome_136_configuration(browser_arguments:list[str], user_data_dir
"""
Validate configuration for Chrome/Edge 136+ security requirements.
Chrome/Edge 136+ requires --user-data-dir to be specified when using --remote-debugging-port.
Chrome/Edge 136+ requires --user-data-dir to be specified for security reasons.
Args:
browser_arguments: List of browser arguments
@@ -157,15 +178,6 @@ def validate_chrome_136_configuration(browser_arguments:list[str], user_data_dir
Returns:
Tuple of (is_valid, error_message)
"""
# Check if remote debugging is enabled
has_remote_debugging = any(
arg.startswith("--remote-debugging-port=")
for arg in browser_arguments
)
if not has_remote_debugging:
return True, "" # No remote debugging, no validation needed
# Check if user-data-dir is specified in arguments
has_user_data_dir_arg = any(
arg.startswith("--user-data-dir=")
@@ -177,7 +189,7 @@ def validate_chrome_136_configuration(browser_arguments:list[str], user_data_dir
if not has_user_data_dir_arg and not has_user_data_dir_config:
return False, (
"Chrome/Edge 136+ requires --user-data-dir to be specified when using --remote-debugging-port. "
"Chrome/Edge 136+ requires --user-data-dir to be specified. "
"Add --user-data-dir=/path/to/directory to your browser arguments and "
'user_data_dir: "/path/to/directory" to your configuration.'
)

View File

@@ -19,7 +19,9 @@ from nodriver.core.tab import Tab as Page
from . import loggers, net
from .chrome_version_detector import (
ChromeVersionInfo,
detect_chrome_version_from_binary,
detect_chrome_version_from_remote_debugging,
get_chrome_version_diagnostic_info,
validate_chrome_136_configuration,
)
@@ -343,14 +345,43 @@ class WebScrapingMixin:
LOG.warning("(fail) Remote debugging port is open but API not accessible: %s", str(e))
LOG.info(" This might indicate a browser update issue or configuration problem")
else:
LOG.error("(fail) Remote debugging port is not open")
LOG.info(" Make sure browser is started with: --remote-debugging-port=%d", remote_port)
LOG.info("(info) Remote debugging port is not open")
# Check for running browser processes
browser_processes = []
target_browser_name = ""
# Get the target browser name for comparison
if self.browser_config.binary_location:
target_browser_name = os.path.basename(self.browser_config.binary_location).lower()
else:
try:
target_browser_path = self.get_compatible_browser()
target_browser_name = os.path.basename(target_browser_path).lower()
except (AssertionError, TypeError):
target_browser_name = ""
for proc in psutil.process_iter(["pid", "name", "cmdline"]):
try:
if proc.info["name"] and any(browser in proc.info["name"].lower() for browser in ["chrome", "chromium", "edge"]):
proc_name = proc.info["name"] or ""
cmdline = proc.info["cmdline"] or []
# Check if this is a browser process relevant to our diagnostics
is_relevant_browser = False
# Is this the target browser?
is_target_browser = target_browser_name and target_browser_name in proc_name.lower()
# Does it have remote debugging?
has_remote_debugging = cmdline and any(arg.startswith("--remote-debugging-port=") for arg in cmdline)
# Detect target browser processes for diagnostics
if is_target_browser:
is_relevant_browser = True
# Add debugging status to the process info for better diagnostics
proc.info["has_remote_debugging"] = has_remote_debugging
if is_relevant_browser:
browser_processes.append(proc.info)
except (psutil.NoSuchProcess, psutil.AccessDenied):
pass
@@ -358,19 +389,17 @@ class WebScrapingMixin:
if browser_processes:
LOG.info("(info) Found %d browser processes running", len(browser_processes))
for proc in browser_processes[:3]: # Show first 3
LOG.info(" - PID %d: %s", proc["pid"], proc["name"])
has_debugging = proc.get("has_remote_debugging", False)
if has_debugging:
LOG.info(" - PID %d: %s (remote debugging enabled)", proc["pid"], proc["name"])
else:
LOG.warning(" - PID %d: %s (remote debugging NOT enabled)", proc["pid"], proc["name"])
else:
LOG.info("(info) No browser processes currently running")
# Platform-specific checks
if platform.system() == "Windows":
LOG.info("(info) Windows detected - check Windows Defender and antivirus software")
elif platform.system() == "Darwin":
LOG.info("(info) macOS detected - check Gatekeeper and security settings")
elif platform.system() == "Linux":
LOG.info("(info) Linux detected - check if running as root (not recommended)")
if platform.system() == "Linux":
if _is_admin():
LOG.error("(fail) Running as root - this can cause browser connection issues")
LOG.error("(fail) Running as root - this can cause browser issues")
# Chrome version detection and validation
self._diagnose_chrome_version_issues(remote_port)
@@ -760,24 +789,45 @@ class WebScrapingMixin:
return
try:
# Detect Chrome version from binary
binary_path = self.browser_config.binary_location
version_info = detect_chrome_version_from_binary(binary_path) if binary_path else None
# Get remote debugging configuration
remote_host = "127.0.0.1"
remote_port = 0
for arg in self.browser_config.arguments:
if arg.startswith("--remote-debugging-host="):
remote_host = arg.split("=", maxsplit = 1)[1]
if arg.startswith("--remote-debugging-port="):
remote_port = int(arg.split("=", maxsplit = 1)[1])
version_info = None
# First, try to detect version from existing browser with remote debugging
if remote_port > 0:
LOG.debug(" -> Checking for existing browser with remote debugging at %s:%s", remote_host, remote_port)
# Reuse the same port checking logic as in create_browser_session
port_available = await self._check_port_with_retry(remote_host, remote_port)
if port_available:
try:
version_info = detect_chrome_version_from_remote_debugging(remote_host, remote_port)
if version_info:
LOG.debug(" -> Detected version from existing browser: %s", version_info)
else:
LOG.debug(" -> Port is open but remote debugging API not accessible")
except Exception as e:
LOG.debug(" -> Failed to detect version from existing browser: %s", e)
else:
LOG.debug(" -> No existing browser found at %s:%s", remote_host, remote_port)
# Only fall back to binary detection if no remote browser is running
if not version_info:
binary_path = self.browser_config.binary_location
if binary_path:
LOG.debug(" -> No remote browser detected, trying binary detection")
version_info = detect_chrome_version_from_binary(binary_path)
# Validate if Chrome 136+ detected
if version_info and version_info.is_chrome_136_plus:
LOG.info(" -> %s 136+ detected: %s", version_info.browser_name, version_info)
# Validate configuration for Chrome/Edge 136+
is_valid, error_message = validate_chrome_136_configuration(
list(self.browser_config.arguments),
self.browser_config.user_data_dir
)
if not is_valid:
LOG.error(" -> %s 136+ configuration validation failed: %s", version_info.browser_name, error_message)
LOG.error(" -> Please update your configuration to include --user-data-dir for remote debugging")
raise AssertionError(error_message)
LOG.info(" -> %s 136+ configuration validation passed", version_info.browser_name)
await self._validate_chrome_136_configuration(version_info)
elif version_info:
LOG.info(" -> %s version detected: %s (pre-136, no special validation required)", version_info.browser_name, version_info)
else:
@@ -789,6 +839,39 @@ class WebScrapingMixin:
LOG.warning(" -> Unexpected error during browser version validation, skipping: %s", e)
# Continue without validation rather than failing
async def _validate_chrome_136_configuration(self, version_info:ChromeVersionInfo) -> None:
"""
Validate Chrome 136+ configuration.
Chrome/Edge 136+ requires --user-data-dir to be specified for security reasons.
Args:
version_info: Chrome version information
Raises:
AssertionError: If configuration is invalid
"""
# Check if user-data-dir is specified in arguments or configuration
has_user_data_dir_arg = any(
arg.startswith("--user-data-dir=")
for arg in self.browser_config.arguments
)
has_user_data_dir_config = (
self.browser_config.user_data_dir is not None and
self.browser_config.user_data_dir.strip()
)
if not has_user_data_dir_arg and not has_user_data_dir_config:
error_message = (
f"{version_info.browser_name} 136+ requires --user-data-dir to be specified. "
"Add --user-data-dir=/path/to/directory to browser arguments and "
'user_data_dir: "/path/to/directory" to your configuration.'
)
LOG.error(" -> %s 136+ configuration validation failed: %s", version_info.browser_name, error_message)
raise AssertionError(error_message)
LOG.info(" -> %s 136+ configuration validation passed", version_info.browser_name)
def _diagnose_chrome_version_issues(self, remote_port:int) -> None:
"""
Diagnose Chrome version issues and provide specific recommendations.

View File

@@ -191,7 +191,7 @@ class TestDetectChromeVersionFromRemoteDebugging:
assert version_info is not None
assert version_info.version_string == "136.0.6778.0"
assert version_info.major_version == 136
assert version_info.browser_name == "Chrome/136.0.6778.0"
assert version_info.browser_name == "Chrome"
mock_urlopen.assert_called_once_with("http://127.0.0.1:9222/json/version", timeout = 5)
@patch("urllib.request.urlopen")
@@ -208,7 +208,7 @@ class TestDetectChromeVersionFromRemoteDebugging:
assert version_info is not None
assert version_info.major_version == 136
assert version_info.browser_name == "Edg/136.0.6778.0"
assert version_info.browser_name == "Edge"
@patch("urllib.request.urlopen")
def test_detect_chrome_version_from_remote_debugging_no_chrome_in_user_agent(self, mock_urlopen:Mock) -> None:
@@ -247,9 +247,10 @@ class TestValidateChrome136Configuration:
def test_validate_chrome_136_configuration_no_remote_debugging(self) -> None:
"""Test validation when no remote debugging is configured."""
# Chrome 136+ requires --user-data-dir regardless of remote debugging
is_valid, error_message = validate_chrome_136_configuration([], None)
assert is_valid is True
assert not error_message
assert is_valid is False
assert "Chrome/Edge 136+ requires --user-data-dir" in error_message
def test_validate_chrome_136_configuration_with_user_data_dir_arg(self) -> None:
"""Test validation with --user-data-dir in arguments."""
@@ -387,3 +388,17 @@ class TestGetChromeVersionDiagnosticInfo:
assert diagnostic_info["chrome_136_plus_detected"] is False
assert diagnostic_info["configuration_valid"] is True
assert diagnostic_info["recommendations"] == []
@patch("urllib.request.urlopen")
def test_detect_chrome_version_from_remote_debugging_json_decode_error(
self, mock_urlopen:Mock
) -> None:
"""Test detect_chrome_version_from_remote_debugging handles JSONDecodeError gracefully."""
# Mock urlopen to return invalid JSON
mock_response = Mock()
mock_response.read.return_value = b"invalid json content"
mock_urlopen.return_value = mock_response
# Should return None when JSON decode fails
result = detect_chrome_version_from_remote_debugging("127.0.0.1", 9222)
assert result is None

View File

@@ -900,14 +900,6 @@ class TestWebScrapingBrowserConfiguration:
assert "browser connection diagnostics" in log_output or "browser-verbindungsdiagnose" in log_output
assert "end diagnostics" in log_output or "ende der diagnose" in log_output
# Check for platform-specific information
if platform.system() == "Windows":
assert "windows detected" in log_output or "windows erkannt" in log_output
elif platform.system() == "Darwin":
assert "macos detected" in log_output or "macos erkannt" in log_output
elif platform.system() == "Linux":
assert "linux detected" in log_output or "linux erkannt" in log_output
class TestWebScrapingDiagnostics:
"""Test the diagnose_browser_issues method."""
@@ -1039,8 +1031,7 @@ class TestWebScrapingDiagnostics:
scraper_with_config.diagnose_browser_issues()
assert "(info) Remote debugging port configured: 9222" in caplog.text
assert "(fail) Remote debugging port is not open" in caplog.text
assert "Make sure browser is started with: --remote-debugging-port=9222" in caplog.text
assert "(info) Remote debugging port is not open" in caplog.text
def test_diagnose_browser_issues_remote_debugging_port_not_configured(
self, scraper_with_config:WebScrapingMixin, caplog:pytest.LogCaptureFixture) -> None:
@@ -1052,21 +1043,24 @@ class TestWebScrapingDiagnostics:
assert "Remote debugging port" not in caplog.text
def test_diagnose_browser_issues_browser_processes_found(self, scraper_with_config:WebScrapingMixin, caplog:pytest.LogCaptureFixture) -> None:
"""Test diagnostic when browser processes are found."""
"""Test diagnostic when browser processes are found.
Updated to test target browser detection with debugging status.
"""
mock_processes = [
Mock(info = {"pid": 1234, "name": "chrome"}),
Mock(info = {"pid": 5678, "name": "chromium"}),
Mock(info = {"pid": 9012, "name": "edge"}),
Mock(info = {"pid": 3456, "name": "chrome"})
Mock(info = {"pid": 1234, "name": "chrome", "cmdline": ["/usr/bin/chrome"]}),
Mock(info = {"pid": 5678, "name": "chromium", "cmdline": ["/usr/bin/chromium"]}),
Mock(info = {"pid": 9012, "name": "edge", "cmdline": ["/usr/bin/edge"]}),
Mock(info = {"pid": 3456, "name": "chrome", "cmdline": ["/usr/bin/chrome", "--remote-debugging-port=9222"]})
]
with patch("psutil.process_iter", return_value = mock_processes):
with patch("psutil.process_iter", return_value = mock_processes), \
patch.object(scraper_with_config, "get_compatible_browser", return_value = "/usr/bin/chrome"):
scraper_with_config.diagnose_browser_issues()
assert "(info) Found 4 browser processes running" in caplog.text
assert " - PID 1234: chrome" in caplog.text
assert " - PID 5678: chromium" in caplog.text
assert " - PID 9012: edge" in caplog.text
# Should find 2 chrome processes (target browser), one with debugging, one without
assert "(info) Found 2 browser processes running" in caplog.text
assert " - PID 1234: chrome (remote debugging NOT enabled)" in caplog.text
assert " - PID 3456: chrome (remote debugging enabled)" in caplog.text
def test_diagnose_browser_issues_no_browser_processes(self, scraper_with_config:WebScrapingMixin, caplog:pytest.LogCaptureFixture) -> None:
"""Test diagnostic when no browser processes are found."""
@@ -1075,38 +1069,55 @@ class TestWebScrapingDiagnostics:
assert "(info) No browser processes currently running" in caplog.text
def test_diagnose_browser_issues_windows_platform(self, scraper_with_config:WebScrapingMixin, caplog:pytest.LogCaptureFixture) -> None:
"""Test diagnostic on Windows platform."""
with patch("platform.system", return_value = "Windows"), \
patch.object(scraper_with_config, "get_compatible_browser", return_value = "/usr/bin/chrome"):
scraper_with_config.diagnose_browser_issues()
assert "(info) Windows detected - check Windows Defender and antivirus software" in caplog.text
def test_diagnose_browser_issues_macos_platform_no_user_data_dir(self, scraper_with_config:WebScrapingMixin, caplog:pytest.LogCaptureFixture) -> None:
"""Test diagnostic on macOS platform without user data directory."""
with patch("platform.system", return_value = "Darwin"), \
patch.object(scraper_with_config, "get_compatible_browser", return_value = "/usr/bin/chrome"):
scraper_with_config.browser_config.arguments = ["--remote-debugging-port=9222"]
scraper_with_config.browser_config.user_data_dir = None
scraper_with_config.diagnose_browser_issues()
assert "(info) macOS detected - check Gatekeeper and security settings" in caplog.text
@patch("kleinanzeigen_bot.utils.web_scraping_mixin.get_chrome_version_diagnostic_info")
def test_diagnose_browser_issues_macos_platform_with_user_data_dir(
self, scraper_with_config:WebScrapingMixin, caplog:pytest.LogCaptureFixture, tmp_path:Path
self, mock_get_diagnostic:Mock, scraper_with_config:WebScrapingMixin, caplog:pytest.LogCaptureFixture, tmp_path:Path
) -> None:
"""Test diagnostic on macOS platform with user data directory."""
test_dir = str(tmp_path / "chrome-profile")
with patch("platform.system", return_value = "Darwin"), \
patch("os.path.exists", return_value = True), \
patch("os.access", return_value = True), \
patch.object(scraper_with_config, "get_compatible_browser", return_value = "/usr/bin/chrome"):
scraper_with_config.browser_config.arguments = ["--remote-debugging-port=9222"]
scraper_with_config.browser_config.user_data_dir = test_dir
scraper_with_config.diagnose_browser_issues()
assert "(info) macOS detected - check Gatekeeper and security settings" in caplog.text
# Setup mock for Chrome 136+ detection with valid configuration
mock_get_diagnostic.return_value = {
"binary_detection": None,
"remote_detection": {
"version_string": "136.0.6778.0",
"major_version": 136,
"browser_name": "Chrome",
"is_chrome_136_plus": True
},
"chrome_136_plus_detected": True,
"recommendations": []
}
# Temporarily unset PYTEST_CURRENT_TEST to allow diagnostics to run
original_env = os.environ.get("PYTEST_CURRENT_TEST")
if "PYTEST_CURRENT_TEST" in os.environ:
del os.environ["PYTEST_CURRENT_TEST"]
try:
with patch("platform.system", return_value = "Darwin"), \
patch("os.path.exists", return_value = True), \
patch("os.access", return_value = True), \
patch("kleinanzeigen_bot.utils.net.is_port_open", return_value = True), \
patch("urllib.request.urlopen") as mock_urlopen, \
patch.object(scraper_with_config, "get_compatible_browser", return_value = "/usr/bin/chrome"):
# Mock Chrome 136+ detection from remote debugging
mock_response = Mock()
mock_response.read.return_value = b'{"Browser": "Chrome/136.0.6778.0"}'
mock_urlopen.return_value = mock_response
scraper_with_config.browser_config.arguments = ["--remote-debugging-port=9222"]
scraper_with_config.browser_config.user_data_dir = test_dir
scraper_with_config.diagnose_browser_issues()
# Should validate Chrome 136+ configuration and pass
assert "(info) Remote Chrome 136+ detected - validating configuration" in caplog.text
assert "(ok) Chrome 136+ configuration validation passed" in caplog.text
finally:
# Restore environment variable
if original_env is not None:
os.environ["PYTEST_CURRENT_TEST"] = original_env
def test_diagnose_browser_issues_linux_platform_not_root(self, scraper_with_config:WebScrapingMixin, caplog:pytest.LogCaptureFixture) -> None:
"""Test diagnostic on Linux platform when not running as root."""
@@ -1115,7 +1126,8 @@ class TestWebScrapingDiagnostics:
patch("kleinanzeigen_bot.utils.web_scraping_mixin._is_admin", return_value = False):
scraper_with_config.diagnose_browser_issues()
assert "(info) Linux detected - check if running as root (not recommended)" in caplog.text
# Linux platform detection was removed - no specific message expected
assert "Linux detected" not in caplog.text
# Should not show error about running as root
assert "(fail) Running as root" not in caplog.text
@@ -1126,8 +1138,9 @@ class TestWebScrapingDiagnostics:
patch("kleinanzeigen_bot.utils.web_scraping_mixin._is_admin", return_value = True):
scraper_with_config.diagnose_browser_issues()
assert "(info) Linux detected - check if running as root (not recommended)" in caplog.text
assert "(fail) Running as root - this can cause browser connection issues" in caplog.text
# Linux platform detection was removed - no specific message expected
assert "Linux detected" not in caplog.text
assert "(fail) Running as root - this can cause browser issues" in caplog.text
def test_diagnose_browser_issues_unknown_platform(self, scraper_with_config:WebScrapingMixin, caplog:pytest.LogCaptureFixture) -> None:
"""Test diagnostic on unknown platform."""
@@ -1148,6 +1161,54 @@ class TestWebScrapingDiagnostics:
scraper_with_config.browser_config.arguments = ["--remote-debugging-port=9222"]
scraper_with_config.diagnose_browser_issues()
@patch("kleinanzeigen_bot.utils.web_scraping_mixin.get_chrome_version_diagnostic_info")
def test_diagnose_browser_issues_chrome_136_plus_misconfigured(
self, mock_get_diagnostic:Mock, scraper_with_config:WebScrapingMixin, caplog:pytest.LogCaptureFixture
) -> None:
"""Test diagnostic when Chrome 136+ is detected but user data directory is not configured."""
# Setup mock for Chrome 136+ detection with invalid configuration
mock_get_diagnostic.return_value = {
"binary_detection": None,
"remote_detection": {
"version_string": "136.0.6778.0",
"major_version": 136,
"browser_name": "Chrome",
"is_chrome_136_plus": True
},
"chrome_136_plus_detected": True,
"recommendations": []
}
# Temporarily unset PYTEST_CURRENT_TEST to allow diagnostics to run
original_env = os.environ.get("PYTEST_CURRENT_TEST")
if "PYTEST_CURRENT_TEST" in os.environ:
del os.environ["PYTEST_CURRENT_TEST"]
try:
with patch("kleinanzeigen_bot.utils.net.is_port_open", return_value = True), \
patch("urllib.request.urlopen") as mock_urlopen, \
patch.object(scraper_with_config, "get_compatible_browser", return_value = "/usr/bin/chrome"):
# Mock Chrome 136+ detection from remote debugging
mock_response = Mock()
mock_response.read.return_value = b'{"Browser": "Chrome/136.0.6778.0"}'
mock_urlopen.return_value = mock_response
# Configure remote debugging but NO user data directory
scraper_with_config.browser_config.arguments = ["--remote-debugging-port=9222"]
scraper_with_config.browser_config.user_data_dir = None
scraper_with_config.diagnose_browser_issues()
# Should detect Chrome 136+ and show configuration error
assert "(info) Remote Chrome 136+ detected - validating configuration" in caplog.text
assert "(fail) Chrome 136+ configuration validation failed" in caplog.text
assert "Chrome/Edge 136+ requires --user-data-dir to be specified" in caplog.text
assert "Solution: Add --user-data-dir=/path/to/directory to browser arguments" in caplog.text
finally:
# Restore environment variable
if original_env is not None:
os.environ["PYTEST_CURRENT_TEST"] = original_env
def test_diagnose_browser_issues_complete_diagnostic_flow(
self, scraper_with_config:WebScrapingMixin, caplog:pytest.LogCaptureFixture, tmp_path:Path
) -> None:
@@ -1181,7 +1242,8 @@ class TestWebScrapingDiagnostics:
assert "(ok) Remote debugging port is open" in caplog.text
assert "(ok) Remote debugging API accessible - Browser: Chrome/120.0.0.0" in caplog.text
assert "(info) No browser processes currently running" in caplog.text
assert "(info) Linux detected - check if running as root (not recommended)" in caplog.text
# Linux platform detection was removed - no specific message expected
assert "Linux detected" not in caplog.text
assert "=== End Diagnostics ===" in caplog.text
def test_diagnose_browser_issues_remote_debugging_host_configured(
@@ -1348,7 +1410,103 @@ class TestWebScrapingDiagnostics:
patch.object(scraper_with_config, "get_compatible_browser", return_value = "/usr/bin/chrome"):
scraper_with_config.diagnose_browser_issues()
assert "(fail) Running as root - this can cause browser connection issues" in caplog.text
assert "(fail) Running as root - this can cause browser issues" in caplog.text
def test_is_admin_on_windows_system(self) -> None:
"""Test _is_admin function on Windows system."""
# Create a mock os module without geteuid
mock_os = Mock()
# Remove geteuid attribute to simulate Windows
del mock_os.geteuid
with patch("kleinanzeigen_bot.utils.web_scraping_mixin.os", mock_os):
assert _is_admin() is False
def test_diagnose_browser_issues_psutil_exceptions(self, web_scraper:WebScrapingMixin) -> None:
"""Test diagnose_browser_issues handles psutil exceptions gracefully."""
# Mock psutil.process_iter to return a list that will cause exceptions when accessing proc.info
mock_process1 = Mock()
mock_process1.info = {"name": "chrome"}
mock_process2 = Mock()
mock_process2.info = {"name": "edge"}
mock_processes = [mock_process1, mock_process2]
with patch("os.path.exists", return_value = True), \
patch("os.access", return_value = True), \
patch("psutil.process_iter", return_value = mock_processes), \
patch("platform.system", return_value = "Linux"), \
patch("kleinanzeigen_bot.utils.web_scraping_mixin._is_admin", return_value = False), \
patch("kleinanzeigen_bot.utils.web_scraping_mixin.WebScrapingMixin._diagnose_chrome_version_issues"), \
patch("kleinanzeigen_bot.utils.web_scraping_mixin.net.is_port_open", return_value = False), \
patch.object(web_scraper, "get_compatible_browser", return_value = "/usr/bin/chrome"), \
patch.object(mock_process1, "info", side_effect = psutil.NoSuchProcess(pid = 123)), \
patch.object(mock_process2, "info", side_effect = psutil.AccessDenied(pid = 456)):
# Should not raise any exceptions
web_scraper.diagnose_browser_issues()
@pytest.mark.asyncio
async def test_validate_chrome_version_configuration_port_open_but_api_inaccessible(
self, web_scraper:WebScrapingMixin
) -> None:
"""Test _validate_chrome_version_configuration when port is open but API is inaccessible."""
# Configure remote debugging
web_scraper.browser_config.arguments = ["--remote-debugging-port=9222"]
web_scraper.browser_config.binary_location = "/usr/bin/chrome"
with patch.dict("os.environ", {}, clear = True), \
patch("kleinanzeigen_bot.utils.web_scraping_mixin.WebScrapingMixin._check_port_with_retry", return_value = True), \
patch("kleinanzeigen_bot.utils.web_scraping_mixin.detect_chrome_version_from_remote_debugging", return_value = None), \
patch("kleinanzeigen_bot.utils.web_scraping_mixin.detect_chrome_version_from_binary", return_value = None), \
patch("kleinanzeigen_bot.utils.web_scraping_mixin.LOG") as mock_log:
# Should not raise any exceptions and should log the appropriate debug message
await web_scraper._validate_chrome_version_configuration()
# Verify the debug message was logged
mock_log.debug.assert_any_call(" -> Port is open but remote debugging API not accessible")
@pytest.mark.asyncio
async def test_validate_chrome_version_configuration_remote_detection_exception(
self, web_scraper:WebScrapingMixin
) -> None:
"""Test _validate_chrome_version_configuration when remote detection raises exception."""
# Configure remote debugging
web_scraper.browser_config.arguments = ["--remote-debugging-port=9222"]
web_scraper.browser_config.binary_location = "/usr/bin/chrome"
with patch.dict("os.environ", {}, clear = True), \
patch("kleinanzeigen_bot.utils.web_scraping_mixin.WebScrapingMixin._check_port_with_retry", return_value = True), \
patch("kleinanzeigen_bot.utils.web_scraping_mixin.detect_chrome_version_from_remote_debugging", side_effect = Exception("Test exception")), \
patch("kleinanzeigen_bot.utils.web_scraping_mixin.detect_chrome_version_from_binary", return_value = None), \
patch("kleinanzeigen_bot.utils.web_scraping_mixin.LOG") as mock_log:
# Should not raise any exceptions and should log the appropriate debug message
await web_scraper._validate_chrome_version_configuration()
# Verify the debug message was logged
# Check that the debug method was called with the expected message
debug_calls = [call for call in mock_log.debug.call_args_list if "Failed to detect version from existing browser" in str(call)]
assert len(debug_calls) > 0, "Expected debug message not found"
@pytest.mark.asyncio
async def test_validate_chrome_version_configuration_no_existing_browser(
self, web_scraper:WebScrapingMixin
) -> None:
"""Test _validate_chrome_version_configuration when no existing browser is found."""
# Configure remote debugging
web_scraper.browser_config.arguments = ["--remote-debugging-port=9222"]
web_scraper.browser_config.binary_location = "/usr/bin/chrome"
with patch.dict("os.environ", {}, clear = True), \
patch("kleinanzeigen_bot.utils.web_scraping_mixin.WebScrapingMixin._check_port_with_retry", return_value = False), \
patch("kleinanzeigen_bot.utils.web_scraping_mixin.detect_chrome_version_from_binary", return_value = None), \
patch("kleinanzeigen_bot.utils.web_scraping_mixin.LOG") as mock_log:
# Should not raise any exceptions and should log the appropriate debug message
await web_scraper._validate_chrome_version_configuration()
# Verify the debug message was logged
mock_log.debug.assert_any_call(" -> No existing browser found at %s:%s", "127.0.0.1", 9222)
class TestWebScrapingMixinPortRetry:

View File

@@ -20,14 +20,12 @@ class TestWebScrapingMixinChromeVersionValidation:
return WebScrapingMixin()
@patch("kleinanzeigen_bot.utils.web_scraping_mixin.detect_chrome_version_from_binary")
@patch("kleinanzeigen_bot.utils.web_scraping_mixin.validate_chrome_136_configuration")
async def test_validate_chrome_version_configuration_chrome_136_plus_valid(
self, mock_validate:Mock, mock_detect:Mock, scraper:WebScrapingMixin
self, mock_detect:Mock, scraper:WebScrapingMixin
) -> None:
"""Test Chrome 136+ validation with valid configuration."""
# Setup mocks
mock_detect.return_value = ChromeVersionInfo("136.0.6778.0", 136, "Chrome")
mock_validate.return_value = (True, "")
# Configure scraper
scraper.browser_config.binary_location = "/path/to/chrome"
@@ -43,26 +41,23 @@ class TestWebScrapingMixinChromeVersionValidation:
# Test validation
await scraper._validate_chrome_version_configuration()
# Verify mocks were called correctly
# Verify detection was called correctly
mock_detect.assert_called_once_with("/path/to/chrome")
mock_validate.assert_called_once_with(
["--remote-debugging-port=9222", "--user-data-dir=/tmp/chrome-debug"], # noqa: S108
"/tmp/chrome-debug" # noqa: S108
)
# Verify validation passed (no exception raised)
# The validation is now done internally in _validate_chrome_136_configuration
finally:
# Restore environment
if original_env:
os.environ["PYTEST_CURRENT_TEST"] = original_env
@patch("kleinanzeigen_bot.utils.web_scraping_mixin.detect_chrome_version_from_binary")
@patch("kleinanzeigen_bot.utils.web_scraping_mixin.validate_chrome_136_configuration")
async def test_validate_chrome_version_configuration_chrome_136_plus_invalid(
self, mock_validate:Mock, mock_detect:Mock, scraper:WebScrapingMixin, caplog:pytest.LogCaptureFixture
self, mock_detect:Mock, scraper:WebScrapingMixin, caplog:pytest.LogCaptureFixture
) -> None:
"""Test Chrome 136+ validation with invalid configuration."""
# Setup mocks
mock_detect.return_value = ChromeVersionInfo("136.0.6778.0", 136, "Chrome")
mock_validate.return_value = (False, "Chrome 136+ requires --user-data-dir")
# Configure scraper
scraper.browser_config.binary_location = "/path/to/chrome"