mirror of
https://github.com/Second-Hand-Friends/kleinanzeigen-bot.git
synced 2026-03-12 10:31:50 +01:00
feat: extend translations and add translation unit test (#427)
This commit is contained in:
Jens Bergmann
GitHub
383
tests/unit/test_translations.py
Normal file
383
tests/unit/test_translations.py
Normal file
@@ -0,0 +1,383 @@
|
||||
"""
|
||||
SPDX-FileCopyrightText: © Sebastian Thomschke and contributors
|
||||
SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
SPDX-ArtifactOfProjectHomePage: https://github.com/Second-Hand-Friends/kleinanzeigen-bot/
|
||||
|
||||
This module contains tests for verifying the completeness and correctness of translations in the project.
|
||||
It ensures that:
|
||||
1. All log messages in the code have corresponding translations
|
||||
2. All translations in the YAML files are actually used in the code
|
||||
3. No obsolete translations exist in the YAML files
|
||||
|
||||
The tests work by:
|
||||
1. Extracting all translatable messages from Python source files
|
||||
2. Loading translations from YAML files
|
||||
3. Comparing the extracted messages with translations
|
||||
4. Verifying no unused translations exist
|
||||
"""
|
||||
import ast, os
|
||||
from dataclasses import dataclass
|
||||
from importlib.resources import files
|
||||
from collections import defaultdict
|
||||
|
||||
from ruamel.yaml import YAML
|
||||
import pytest
|
||||
|
||||
from kleinanzeigen_bot import resources
|
||||
|
||||
# Messages that are intentionally not translated (internal/debug messages)
|
||||
EXCLUDED_MESSAGES: dict[str, set[str]] = {
|
||||
"kleinanzeigen_bot/__init__.py": {"############################################"}
|
||||
}
|
||||
|
||||
# Type aliases for better readability
|
||||
ModulePath = str
|
||||
FunctionName = str
|
||||
Message = str
|
||||
TranslationDict = dict[ModulePath, dict[FunctionName, dict[Message, str]]]
|
||||
MessageDict = dict[FunctionName, dict[Message, set[Message]]]
|
||||
MissingDict = dict[FunctionName, dict[Message, set[Message]]]
|
||||
|
||||
|
||||
@dataclass
|
||||
class MessageLocation:
|
||||
"""Represents the location of a message in the codebase."""
|
||||
module: str
|
||||
function: str
|
||||
message: str
|
||||
|
||||
|
||||
def _get_function_name(node: ast.AST) -> str:
|
||||
"""
|
||||
Get the name of the function containing this AST node.
|
||||
This matches i18n.py's behavior which only uses the function name for translation lookups.
|
||||
For module-level code, returns "module" to match i18n.py's convention.
|
||||
|
||||
Args:
|
||||
node: The AST node to analyze
|
||||
|
||||
Returns:
|
||||
The function name or "module" for module-level code
|
||||
"""
|
||||
def find_parent_context(n: ast.AST) -> tuple[str | None, str | None]:
|
||||
"""Find the containing class and function names."""
|
||||
class_name = None
|
||||
function_name = None
|
||||
current = n
|
||||
|
||||
while hasattr(current, '_parent'):
|
||||
current = getattr(current, '_parent')
|
||||
if isinstance(current, ast.ClassDef) and not class_name:
|
||||
class_name = current.name
|
||||
elif isinstance(current, ast.FunctionDef) or isinstance(current, ast.AsyncFunctionDef) and not function_name:
|
||||
function_name = current.name
|
||||
break # We only need the immediate function name
|
||||
return class_name, function_name
|
||||
|
||||
_, function_name = find_parent_context(node)
|
||||
if function_name:
|
||||
return function_name
|
||||
return "module" # For module-level code
|
||||
|
||||
|
||||
def _extract_log_messages(file_path: str) -> MessageDict:
|
||||
"""
|
||||
Extract all translatable messages from a Python file with their function context.
|
||||
|
||||
Args:
|
||||
file_path: Path to the Python file to analyze
|
||||
|
||||
Returns:
|
||||
Dictionary mapping function names to their messages
|
||||
"""
|
||||
with open(file_path, 'r', encoding='utf-8') as file:
|
||||
tree = ast.parse(file.read(), filename=file_path)
|
||||
|
||||
# Add parent references for context tracking
|
||||
for parent in ast.walk(tree):
|
||||
for child in ast.iter_child_nodes(parent):
|
||||
setattr(child, '_parent', parent)
|
||||
|
||||
messages: MessageDict = defaultdict(lambda: defaultdict(set))
|
||||
|
||||
def add_message(function: str, msg: str) -> None:
|
||||
"""Helper to add a message to the messages dictionary."""
|
||||
if function not in messages:
|
||||
messages[function] = defaultdict(set)
|
||||
if msg not in messages[function]:
|
||||
messages[function][msg] = {msg}
|
||||
|
||||
def extract_string_value(node: ast.AST) -> str | None:
|
||||
"""Safely extract string value from an AST node."""
|
||||
if isinstance(node, ast.Constant):
|
||||
value = getattr(node, 'value', None)
|
||||
return value if isinstance(value, str) else None
|
||||
return None
|
||||
|
||||
for node in ast.walk(tree):
|
||||
if not isinstance(node, ast.Call):
|
||||
continue
|
||||
|
||||
function_name = _get_function_name(node)
|
||||
|
||||
# Extract messages from various call types
|
||||
if (isinstance(node.func, ast.Attribute) and
|
||||
isinstance(node.func.value, ast.Name) and
|
||||
node.func.value.id in {'LOG', 'logger', 'logging'} and
|
||||
node.func.attr in {'debug', 'info', 'warning', 'error', 'critical'}):
|
||||
if node.args:
|
||||
msg = extract_string_value(node.args[0])
|
||||
if msg:
|
||||
add_message(function_name, msg)
|
||||
|
||||
# Handle gettext calls
|
||||
elif ((isinstance(node.func, ast.Name) and node.func.id == '_') or
|
||||
(isinstance(node.func, ast.Attribute) and node.func.attr == 'gettext')):
|
||||
if node.args:
|
||||
msg = extract_string_value(node.args[0])
|
||||
if msg:
|
||||
add_message(function_name, msg)
|
||||
|
||||
# Handle other translatable function calls
|
||||
elif isinstance(node.func, ast.Name) and node.func.id in {'ainput', 'pluralize', 'ensure'}:
|
||||
arg_index = 0 if node.func.id == 'ainput' else 1
|
||||
if len(node.args) > arg_index:
|
||||
msg = extract_string_value(node.args[arg_index])
|
||||
if msg:
|
||||
add_message(function_name, msg)
|
||||
|
||||
print(f"Messages: {messages}")
|
||||
|
||||
return messages
|
||||
|
||||
|
||||
def _get_all_log_messages() -> dict[str, MessageDict]:
|
||||
"""
|
||||
Get all translatable messages from all Python files in the project.
|
||||
|
||||
Returns:
|
||||
Dictionary mapping module paths to their function messages
|
||||
"""
|
||||
src_dir = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), 'src', 'kleinanzeigen_bot')
|
||||
print(f"\nScanning for messages in directory: {src_dir}")
|
||||
|
||||
messages_by_file: dict[str, MessageDict] = {
|
||||
# Special case for getopt.py which is imported
|
||||
"getopt.py": {
|
||||
"do_longs": {
|
||||
"option --%s requires argument": {"option --%s requires argument"},
|
||||
"option --%s must not have an argument": {"option --%s must not have an argument"}
|
||||
},
|
||||
"long_has_args": {
|
||||
"option --%s not recognized": {"option --%s not recognized"},
|
||||
"option --%s not a unique prefix": {"option --%s not a unique prefix"}
|
||||
},
|
||||
"do_shorts": {
|
||||
"option -%s requires argument": {"option -%s requires argument"}
|
||||
},
|
||||
"short_has_arg": {
|
||||
"option -%s not recognized": {"option -%s not recognized"}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for root, _, filenames in os.walk(src_dir):
|
||||
for filename in filenames:
|
||||
if filename.endswith('.py'):
|
||||
file_path = os.path.join(root, filename)
|
||||
relative_path = os.path.relpath(file_path, src_dir)
|
||||
if relative_path.startswith('resources/'):
|
||||
continue
|
||||
messages = _extract_log_messages(file_path)
|
||||
if messages:
|
||||
module_path = os.path.join('kleinanzeigen_bot', relative_path)
|
||||
module_path = module_path.replace(os.sep, '/')
|
||||
messages_by_file[module_path] = messages
|
||||
|
||||
return messages_by_file
|
||||
|
||||
|
||||
def _get_available_languages() -> list[str]:
|
||||
"""
|
||||
Get list of available translation languages from translation files.
|
||||
|
||||
Returns:
|
||||
List of language codes (e.g. ['de', 'en'])
|
||||
"""
|
||||
languages = []
|
||||
resources_path = files(resources)
|
||||
for file in resources_path.iterdir():
|
||||
if file.name.startswith("translations.") and file.name.endswith(".yaml"):
|
||||
lang = file.name[13:-5] # Remove "translations." and ".yaml"
|
||||
languages.append(lang)
|
||||
return sorted(languages)
|
||||
|
||||
|
||||
def _get_translations_for_language(lang: str) -> TranslationDict:
|
||||
"""
|
||||
Get translations for a specific language from its YAML file.
|
||||
|
||||
Args:
|
||||
lang: Language code (e.g. 'de')
|
||||
|
||||
Returns:
|
||||
Dictionary containing all translations for the language
|
||||
"""
|
||||
yaml = YAML(typ='safe')
|
||||
translation_file = f"translations.{lang}.yaml"
|
||||
print(f"Loading translations from {translation_file}")
|
||||
content = files(resources).joinpath(translation_file).read_text()
|
||||
translations = yaml.load(content) or {}
|
||||
return translations
|
||||
|
||||
|
||||
def _find_translation(translations: TranslationDict,
|
||||
module: str,
|
||||
function: str,
|
||||
message: str) -> bool:
|
||||
"""
|
||||
Check if a translation exists for a given message in the exact location where i18n.py will look.
|
||||
This matches the lookup logic in i18n.py which uses dicts.safe_get().
|
||||
|
||||
Args:
|
||||
translations: Dictionary of all translations
|
||||
module: Module path
|
||||
function: Function name
|
||||
message: Message to find translation for
|
||||
|
||||
Returns:
|
||||
True if translation exists in the correct location, False otherwise
|
||||
"""
|
||||
# Special case for getopt.py
|
||||
if module == 'getopt.py':
|
||||
return bool(translations.get(module, {}).get(function, {}).get(message))
|
||||
|
||||
# Add kleinanzeigen_bot/ prefix if not present
|
||||
module_path = f'kleinanzeigen_bot/{module}' if not module.startswith('kleinanzeigen_bot/') else module
|
||||
|
||||
# Check if module exists in translations
|
||||
module_trans = translations.get(module_path, {})
|
||||
if not isinstance(module_trans, dict):
|
||||
print(f"Module {module_path} translations is not a dictionary")
|
||||
return False
|
||||
|
||||
# Check if function exists in module translations
|
||||
function_trans = module_trans.get(function, {})
|
||||
if not isinstance(function_trans, dict):
|
||||
print(f"Function {function} translations in module {module_path} is not a dictionary")
|
||||
return False
|
||||
|
||||
# Check if message exists in function translations
|
||||
has_translation = message in function_trans
|
||||
|
||||
return has_translation
|
||||
|
||||
|
||||
@pytest.mark.parametrize("lang", _get_available_languages())
|
||||
def test_all_log_messages_have_translations(lang: str) -> None:
|
||||
"""
|
||||
Test that all translatable messages in the code have translations for each language.
|
||||
|
||||
This test ensures that no untranslated messages exist in the codebase.
|
||||
"""
|
||||
messages_by_file = _get_all_log_messages()
|
||||
translations = _get_translations_for_language(lang)
|
||||
|
||||
missing_translations = []
|
||||
|
||||
for module, functions in messages_by_file.items():
|
||||
excluded = EXCLUDED_MESSAGES.get(module, set())
|
||||
for function, messages in functions.items():
|
||||
for message in messages:
|
||||
# Skip excluded messages
|
||||
if message in excluded:
|
||||
continue
|
||||
if not _find_translation(translations, module, function, message):
|
||||
missing_translations.append(MessageLocation(module, function, message))
|
||||
|
||||
if missing_translations:
|
||||
missing_str = f"\nPlease add the following missing translations for language [{lang}]:\n"
|
||||
|
||||
def make_inner_dict() -> defaultdict[str, set[str]]:
|
||||
return defaultdict(set)
|
||||
|
||||
by_module: defaultdict[str, defaultdict[str, set[str]]] = defaultdict(make_inner_dict)
|
||||
|
||||
for loc in missing_translations:
|
||||
assert isinstance(loc.module, str), "Module must be a string"
|
||||
assert isinstance(loc.function, str), "Function must be a string"
|
||||
assert isinstance(loc.message, str), "Message must be a string"
|
||||
by_module[loc.module][loc.function].add(loc.message)
|
||||
|
||||
# There is a type error here, but it's not a problem
|
||||
for module, functions in sorted(by_module.items()): # type: ignore[assignment]
|
||||
missing_str += f" {module}:\n"
|
||||
for function, messages in sorted(functions.items()):
|
||||
missing_str += f" {function}:\n"
|
||||
for message in sorted(messages):
|
||||
missing_str += f' "{message}"\n'
|
||||
raise AssertionError(missing_str)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("lang", _get_available_languages())
|
||||
def test_no_obsolete_translations(lang: str) -> None:
|
||||
"""
|
||||
Test that all translations in each language YAML file are actually used in the code.
|
||||
|
||||
This test ensures there are no obsolete translations that should be removed.
|
||||
"""
|
||||
messages_by_file = _get_all_log_messages()
|
||||
translations = _get_translations_for_language(lang)
|
||||
|
||||
obsolete_items: list[tuple[str, str, str]] = []
|
||||
|
||||
for module, module_trans in translations.items():
|
||||
# Add kleinanzeigen_bot/ prefix if not present
|
||||
module_with_prefix = f'kleinanzeigen_bot/{module}' if not module.startswith('kleinanzeigen_bot/') else module
|
||||
|
||||
# Remove .py extension for comparison if present
|
||||
module_no_ext = module_with_prefix[:-3] if module_with_prefix.endswith('.py') else module_with_prefix
|
||||
|
||||
if module_no_ext not in messages_by_file:
|
||||
# Skip obsolete module check since we know these modules are needed
|
||||
continue
|
||||
|
||||
code_messages = messages_by_file[module_no_ext]
|
||||
for function, function_trans in module_trans.items():
|
||||
if not isinstance(function_trans, dict):
|
||||
continue
|
||||
|
||||
if function not in code_messages and function != 'module':
|
||||
# Skip obsolete function check since we know these functions are needed
|
||||
continue
|
||||
|
||||
for trans_message in function_trans:
|
||||
if function == 'module' or trans_message in code_messages.get(function, {}):
|
||||
continue
|
||||
obsolete_items.append((module, function, trans_message))
|
||||
|
||||
if obsolete_items:
|
||||
obsolete_str = f"\nPlease remove the following obsolete translations for language [{lang}]:\n"
|
||||
by_module: defaultdict[str, defaultdict[str, set[str]]] = defaultdict(lambda: defaultdict(set))
|
||||
for module, function, message in obsolete_items:
|
||||
if module not in by_module:
|
||||
by_module[module] = defaultdict(set)
|
||||
if function not in by_module[module]:
|
||||
by_module[module][function] = set()
|
||||
by_module[module][function].add(message)
|
||||
|
||||
for module, functions in sorted(by_module.items()):
|
||||
obsolete_str += f" {module}:\n"
|
||||
for function, messages in sorted(functions.items()):
|
||||
if function:
|
||||
obsolete_str += f" {function}:\n"
|
||||
for message in sorted(messages):
|
||||
obsolete_str += f' "{message}"\n'
|
||||
raise AssertionError(obsolete_str)
|
||||
|
||||
|
||||
def test_translation_files_exist() -> None:
|
||||
"""Test that at least one translation file exists."""
|
||||
languages = _get_available_languages()
|
||||
if not languages:
|
||||
raise AssertionError("No translation files found! Expected at least one translations.*.yaml file.")
|
||||
Reference in New Issue
Block a user