diff --git a/.github/workflows/translation-lint.yml b/.github/workflows/translation-lint.yml index 458fe354d..0a5d0931f 100644 --- a/.github/workflows/translation-lint.yml +++ b/.github/workflows/translation-lint.yml @@ -27,5 +27,5 @@ jobs: shell: bash run: | for i in src/duckstation-qt/translations/*.ts; do - python scripts/verify_translation_placeholders.py "$i" + python scripts/translation/validate_ts.py "$i" done diff --git a/scripts/translation/apply_ts.py b/scripts/translation/apply_ts.py new file mode 100644 index 000000000..96ba1d704 --- /dev/null +++ b/scripts/translation/apply_ts.py @@ -0,0 +1,163 @@ +#!/usr/bin/env python3 +"""Safely apply completed JSONL translation batches to a Qt TS catalog.""" + +from __future__ import annotations + +import argparse +import os +import tempfile +from pathlib import Path + +if __package__ in (None, ""): + import sys + + sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from translation.ts_utils import ( # noqa: E402 + SCHEMA, + MessageIdentity, + catalog_fingerprint, + load_jsonl, + parse_catalog, + placeholders_match, + replace_translation_node, + scan_raw_messages, + validate_translation, +) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("catalog", type=Path, help="Qt Linguist .ts catalog") + parser.add_argument("batches", nargs="+", type=Path, help="completed JSONL batch files") + parser.add_argument("--write", action="store_true", help="atomically update the catalog; default is dry-run") + return parser.parse_args() + + +def record_identity(record: dict[str, object]) -> MessageIdentity: + return MessageIdentity( + context=str(record.get("context", "")), + source=str(record.get("source", "")), + comment=str(record.get("comment", "")), + extracomment=str(record.get("extracomment", "")), + numerus=bool(record.get("numerus", False)), + occurrence=int(record.get("occurrence", 0)), + ) + + +def resolve_target(record: dict[str, object]) -> tuple[str | None, list[str] | None] | None: + singular = record.get("target_translation") + plurals = record.get("target_plural_translations") + accept_current = record.get("accept_current") is True + if accept_current: + if singular is not None or plurals is not None: + raise ValueError(f"{record.get('id')}: accept_current cannot be combined with target fields") + current_plurals = record.get("current_plural_translations") + if current_plurals: + plurals = current_plurals + else: + singular = record.get("current_translation") + + if singular is None and plurals is None: + return None + if singular is not None and plurals is not None: + raise ValueError(f"{record.get('id')}: set either singular or plural target, not both") + if singular is not None: + if not isinstance(singular, str) or not singular.strip(): + raise ValueError(f"{record.get('id')}: target_translation must be a nonempty string") + return singular, None + if ( + not isinstance(plurals, list) + or not plurals + or any(not isinstance(value, str) or not value.strip() for value in plurals) + ): + raise ValueError(f"{record.get('id')}: target_plural_translations must contain nonempty strings") + return None, plurals + + +def atomic_write(path: Path, text: str) -> None: + mode = path.stat().st_mode + temporary_name: str | None = None + try: + with tempfile.NamedTemporaryFile("w", encoding="utf-8", newline="", dir=path.parent, delete=False) as stream: + temporary_name = stream.name + stream.write(text) + stream.flush() + os.fsync(stream.fileno()) + os.chmod(temporary_name, mode) + os.replace(temporary_name, path) + finally: + if temporary_name and os.path.exists(temporary_name): + os.unlink(temporary_name) + + +def main() -> int: + args = parse_args() + metadata, records = load_jsonl(args.batches) + if len(metadata) != len(args.batches): + raise SystemExit("each batch must contain exactly one metadata record") + for item in metadata: + if item.get("schema") != SCHEMA: + raise SystemExit(f"unsupported batch schema: {item.get('schema')!r}") + + _, messages = parse_catalog(args.catalog) + fingerprint = catalog_fingerprint(messages) + expected_fingerprints = {str(item.get("catalog_fingerprint", "")) for item in metadata} + if expected_fingerprints != {fingerprint}: + raise SystemExit("catalog source identity has changed since extraction; export fresh batches") + + with args.catalog.open("r", encoding="utf-8", newline="") as stream: + raw_text = stream.read() + spans = scan_raw_messages(raw_text) + seen: set[str] = set() + replacements: list[tuple[int, int, str]] = [] + skipped = 0 + for record in records: + identifier = str(record.get("id", "")) + identity = record_identity(record) + if identifier != identity.identifier: + raise SystemExit(f"{identifier or ''}: record identity does not match its id") + if identifier in seen: + raise SystemExit(f"duplicate record id across batches: {identifier}") + seen.add(identifier) + target = resolve_target(record) + if target is None: + skipped += 1 + continue + span = spans.get(identifier) + if span is None: + raise SystemExit(f"message no longer exists in catalog: {identifier}") + if span.translation_type != "unfinished": + raise SystemExit(f"refusing to overwrite {span.translation_type} translation: {identifier}") + singular, plurals = target + if span.numerus != (plurals is not None): + raise SystemExit(f"singular/plural target does not match catalog message: {identifier}") + values = plurals if plurals is not None else [singular or ""] + for value in values: + problems = validate_translation( + identity.source, + value, + allow_missing_placeholders=plurals is not None, + ) + if problems: + raise SystemExit(f"{identifier}: {'; '.join(problems)}") + if plurals is not None and not any(placeholders_match(identity.source, value) for value in plurals): + raise SystemExit(f"{identifier}: no plural form preserves the complete source placeholder set") + replacement_block = replace_translation_node(span.block, singular, plurals) + replacements.append((span.start, span.end, replacement_block)) + + updated_text = raw_text + for start, end, replacement in sorted(replacements, reverse=True): + updated_text = updated_text[:start] + replacement + updated_text[end:] + if args.write and replacements: + atomic_write(args.catalog, updated_text) + + action = "Applied" if args.write else "Would apply" + print(f"{action} {len(replacements)} translation(s); skipped {skipped} record(s) without targets.") + if not args.write: + print("Dry-run only; pass --write to update the catalog.") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/translation/extract_ts.py b/scripts/translation/extract_ts.py new file mode 100644 index 000000000..7a8457a0f --- /dev/null +++ b/scripts/translation/extract_ts.py @@ -0,0 +1,179 @@ +#!/usr/bin/env python3 +"""Export active unfinished Qt TS messages to editable JSONL batches.""" + +from __future__ import annotations + +import argparse +import collections +import difflib +from pathlib import Path + +if __package__ in (None, ""): + import sys + + sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from translation.ts_utils import ( # noqa: E402 + SCHEMA, + SKIPPED_TYPES, + CatalogMessage, + catalog_fingerprint, + message_to_batch_record, + parse_catalog, + write_jsonl, +) + + +def translated_text(message: CatalogMessage) -> str | list[str] | None: + if message.plural_translations: + return message.plural_translations if any(value.strip() for value in message.plural_translations) else None + return message.translation if message.translation and message.translation.strip() else None + + +def build_suggestions( + target: CatalogMessage, + messages: list[CatalogMessage], + limit: int, + minimum_similarity: float, +) -> list[dict[str, object]]: + if limit <= 0: + return [] + candidates: list[tuple[float, CatalogMessage]] = [] + for candidate in messages: + text = translated_text(candidate) + if text is None or candidate.identifier == target.identifier: + continue + exact = candidate.identity.source == target.identity.source + if not exact and candidate.identity.context != target.identity.context: + continue + similarity = ( + 1.0 + if exact + else difflib.SequenceMatcher(None, target.identity.source, candidate.identity.source).ratio() + ) + if similarity >= minimum_similarity: + candidates.append((similarity, candidate)) + candidates.sort( + key=lambda pair: ( + pair[0], + pair[1].translation_type == "finished", + pair[1].identity.source == target.identity.source, + ), + reverse=True, + ) + output: list[dict[str, object]] = [] + seen: set[tuple[str, str]] = set() + for similarity, candidate in candidates: + text = translated_text(candidate) + signature = (candidate.identity.source, repr(text)) + if signature in seen: + continue + seen.add(signature) + output.append( + { + "source": candidate.identity.source, + "translation": text, + "context": candidate.identity.context, + "type": candidate.translation_type, + "similarity": round(similarity, 4), + } + ) + if len(output) == limit: + break + return output + + +def split_balanced(records: list[dict[str, object]], batch_count: int) -> list[list[dict[str, object]]]: + if batch_count <= 1 or len(records) <= 1: + return [records] + batch_count = min(batch_count, len(records)) + by_context: dict[str, list[dict[str, object]]] = collections.defaultdict(list) + for record in records: + by_context[str(record["context"])].append(record) + + target_size = max(1, (len(records) + batch_count - 1) // batch_count) + chunks: list[list[dict[str, object]]] = [] + for context_records in by_context.values(): + if len(context_records) > target_size * 3 // 2: + chunks.extend( + context_records[offset : offset + target_size] + for offset in range(0, len(context_records), target_size) + ) + else: + chunks.append(context_records) + + batches: list[list[dict[str, object]]] = [[] for _ in range(batch_count)] + for chunk in sorted(chunks, key=len, reverse=True): + destination = min(batches, key=len) + destination.extend(chunk) + return batches + + +def make_metadata(catalog: Path, fingerprint: str, batch_index: int, batch_count: int) -> dict[str, object]: + return { + "record_type": "metadata", + "schema": SCHEMA, + "catalog": str(catalog), + "catalog_fingerprint": fingerprint, + "batch_index": batch_index, + "batch_count": batch_count, + } + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("catalog", type=Path, help="Qt Linguist .ts catalog") + output = parser.add_mutually_exclusive_group(required=True) + output.add_argument("--output", type=Path, help="single JSONL output file") + output.add_argument("--batch-dir", type=Path, help="directory for numbered JSONL batches") + parser.add_argument("--batches", type=int, default=1, help="number of balanced batches (with --batch-dir)") + parser.add_argument("--context", action="append", default=[], help="include only this context; repeatable") + parser.add_argument("--suggestions", type=int, default=2, help="translation-memory suggestions per message") + parser.add_argument("--similarity", type=float, default=0.72, help="minimum fuzzy suggestion similarity") + return parser.parse_args() + + +def main() -> int: + args = parse_args() + if args.output and args.batches != 1: + raise SystemExit("--batches requires --batch-dir") + if args.batches < 1: + raise SystemExit("--batches must be at least 1") + if not 0.0 <= args.similarity <= 1.0: + raise SystemExit("--similarity must be between 0 and 1") + + _, messages = parse_catalog(args.catalog) + contexts = set(args.context) + targets = [ + message + for message in messages + if message.translation_type == "unfinished" + and message.translation_type not in SKIPPED_TYPES + and (not contexts or message.identity.context in contexts) + ] + records = [] + for message in targets: + record = message_to_batch_record(message) + record["suggestions"] = build_suggestions(message, messages, args.suggestions, args.similarity) + records.append(record) + + fingerprint = catalog_fingerprint(messages) + if args.output: + write_jsonl(args.output, make_metadata(args.catalog, fingerprint, 1, 1), records) + destinations = [args.output] + else: + batches = split_balanced(records, args.batches) + destinations = [] + for index, batch in enumerate(batches, 1): + destination = args.batch_dir / f"batch-{index:03d}.jsonl" + write_jsonl(destination, make_metadata(args.catalog, fingerprint, index, len(batches)), batch) + destinations.append(destination) + + print(f"Exported {len(records)} active unfinished messages to {len(destinations)} file(s).") + for destination in destinations: + print(destination) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/translation/tests/test_translation_tools.py b/scripts/translation/tests/test_translation_tools.py new file mode 100644 index 000000000..f73c76ba0 --- /dev/null +++ b/scripts/translation/tests/test_translation_tools.py @@ -0,0 +1,261 @@ +#!/usr/bin/env python3 + +from __future__ import annotations + +import json +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + +SCRIPTS_DIR = Path(__file__).resolve().parents[2] +REPO_ROOT = SCRIPTS_DIR.parent +sys.path.insert(0, str(SCRIPTS_DIR)) + +from translation.ts_utils import ( # noqa: E402 + TRANSLATION_RE, + catalog_fingerprint, + extract_placeholders, + extract_rich_tags, + parse_catalog, + placeholders_are_subset, + placeholders_match, + replace_translation_node, + validate_translation, +) + + +FIXTURE = """ + + + + Alpha + + + Hello %1 {0} ${title} %.1f <strong>world</strong> + + + + Register + レジスタ + + + %n file(s) + File count + + + + + + New wording + + + + Old wording + 古い文言 + + + Hello + こんにちは + + + + Beta + + Register + + + + Removed + 削除済み + + + +""" + + +class TranslationToolTests(unittest.TestCase): + def run_tool(self, script: str, *arguments: object, expect: int = 0) -> subprocess.CompletedProcess[str]: + result = subprocess.run( + [sys.executable, str(SCRIPTS_DIR / "translation" / script), *(str(value) for value in arguments)], + cwd=REPO_ROOT, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + check=False, + ) + self.assertEqual(expect, result.returncode, result.stdout) + return result + + def test_placeholder_recognition_avoids_percent_and_escaped_brace_false_positives(self) -> None: + text = "At 100% speed, 5% of users: %1 %n {} {0:08X} %.1f %s ${title} {{}} %%" + placeholders = extract_placeholders(text) + self.assertEqual(1, placeholders["qt:%1"]) + self.assertEqual(1, placeholders["qt:%n"]) + self.assertEqual(1, placeholders["fmt:{}"]) + self.assertEqual(1, placeholders["fmt:{0:08X}"]) + self.assertEqual(1, placeholders["printf:%.1f"]) + self.assertEqual(1, placeholders["printf:%s"]) + self.assertEqual(1, placeholders["template:${title}"]) + self.assertEqual(7, sum(placeholders.values())) + self.assertTrue(placeholders_match("{} {}", "{1} {0}")) + self.assertFalse(placeholders_match("{} {}", "{0} {0}")) + self.assertTrue(placeholders_are_subset("{} of %n", "%n")) + self.assertFalse(placeholders_are_subset("{} of %n", "%n %1")) + self.assertEqual({"qt:%1": 1}, dict(extract_placeholders("%1x"))) + + def test_rich_text_ignores_angle_bracket_labels(self) -> None: + self.assertFalse(extract_rich_tags("")) + self.assertEqual({"strong": 1, "/strong": 1}, dict(extract_rich_tags("Text"))) + self.assertTrue(validate_translation("Text", "テキスト")) + + def test_fingerprint_ignores_translation_changes(self) -> None: + with tempfile.TemporaryDirectory() as directory: + first = Path(directory) / "first.ts" + second = Path(directory) / "second.ts" + first.write_text(FIXTURE, encoding="utf-8") + second.write_text(FIXTURE.replace("レジスタ", "登録"), encoding="utf-8") + _, first_messages = parse_catalog(first) + _, second_messages = parse_catalog(second) + self.assertEqual(catalog_fingerprint(first_messages), catalog_fingerprint(second_messages)) + + def test_translation_node_replacement_preserves_message_text(self) -> None: + block = ( + " \n" + " A & B\n" + ' \n' + " " + ) + replaced = replace_translation_node(block, "A と B", None) + self.assertIn("A & B", replaced) + self.assertIn("A と B", replaced) + self.assertNotIn("unfinished", replaced) + + def test_end_to_end_extract_apply_validate(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + catalog = root / "catalog.ts" + batch = root / "batch.jsonl" + catalog.write_text(FIXTURE, encoding="utf-8") + original = catalog.read_bytes() + + self.run_tool("extract_ts.py", catalog, "--output", batch, "--suggestions", 3) + lines = [json.loads(line) for line in batch.read_text(encoding="utf-8").splitlines()] + records = [line for line in lines if line["record_type"] == "message"] + self.assertEqual(5, len(records)) + self.assertNotIn("Old wording", {record["source"] for record in records}) + new_wording = next(record for record in records if record["source"] == "New wording") + self.assertTrue(any(item["type"] == "vanished" for item in new_wording["suggestions"])) + + translations: dict[tuple[str, str], str | list[str]] = { + ( + "Alpha", + "Hello %1 {0} ${title} %.1f world", + ): "こんにちは %1 {0} ${title} %.1f 世界", + ("Alpha", "Register"): "登録", + ("Alpha", "%n file(s)"): ["%n ファイル"], + ("Alpha", "New wording"): "新しい文言", + ("Beta", "Register"): "登録", + } + for record in records: + target = translations[(record["context"], record["source"])] + if isinstance(target, list): + record["target_plural_translations"] = target + else: + record["target_translation"] = target + batch.write_text( + "\n".join(json.dumps(line, ensure_ascii=False) for line in lines) + "\n", + encoding="utf-8", + ) + + self.run_tool("apply_ts.py", catalog, batch) + self.assertEqual(original, catalog.read_bytes(), "dry-run modified the catalog") + self.run_tool("apply_ts.py", catalog, batch, "--write") + updated = catalog.read_text(encoding="utf-8") + + def normalize_translations(text: str) -> str: + return TRANSLATION_RE.sub("", text) + + self.assertEqual(normalize_translations(FIXTURE), normalize_translations(updated)) + self.assertIn('古い文言', updated) + self.assertIn('削除済み', updated) + self.assertNotIn('type="unfinished"', updated) + self.run_tool("validate_ts.py", catalog, "--require-complete") + + def test_accept_current_and_balanced_batches(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + catalog = root / "catalog.ts" + batches = root / "batches" + catalog.write_text(FIXTURE, encoding="utf-8") + self.run_tool("extract_ts.py", catalog, "--batch-dir", batches, "--batches", 3) + files = sorted(batches.glob("*.jsonl")) + self.assertEqual(3, len(files)) + found = False + for path in files: + lines = [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines()] + for line in lines: + if line.get("context") == "Alpha" and line.get("source") == "Register": + line["accept_current"] = True + found = True + path.write_text( + "\n".join(json.dumps(line, ensure_ascii=False) for line in lines) + "\n", + encoding="utf-8", + ) + self.assertTrue(found) + result = self.run_tool("apply_ts.py", catalog, *files) + self.assertIn("Would apply 1 translation", result.stdout) + + def test_batches_apply_sequentially_without_invalidating_fingerprint(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + catalog = root / "catalog.ts" + batches = root / "batches" + catalog.write_text(FIXTURE, encoding="utf-8") + self.run_tool("extract_ts.py", catalog, "--batch-dir", batches, "--batches", 2) + files = sorted(batches.glob("*.jsonl")) + self.assertEqual(2, len(files)) + for path in files: + lines = [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines()] + record = next(line for line in lines if line["record_type"] == "message") + if record["numerus"]: + record["target_plural_translations"] = [record["source"]] + else: + record["target_translation"] = record["source"] + path.write_text( + "\n".join(json.dumps(line, ensure_ascii=False) for line in lines) + "\n", + encoding="utf-8", + ) + self.run_tool("apply_ts.py", catalog, files[0], "--write") + self.run_tool("apply_ts.py", catalog, files[1], "--write") + + def test_stale_catalog_is_rejected(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + catalog = root / "catalog.ts" + batch = root / "batch.jsonl" + catalog.write_text(FIXTURE, encoding="utf-8") + self.run_tool("extract_ts.py", catalog, "--output", batch) + catalog.write_text(FIXTURE.replace("New wording", "Changed source"), encoding="utf-8") + result = self.run_tool("apply_ts.py", catalog, batch, expect=1) + self.assertIn("source identity has changed", result.stdout) + + def test_validation_reports_catalog_and_source_lines(self) -> None: + with tempfile.TemporaryDirectory() as directory: + catalog = Path(directory) / "catalog.ts" + broken = FIXTURE.replace( + '', + "プレースホルダーなし", + 1, + ) + catalog.write_text(broken, encoding="utf-8") + translation_offset = broken.index("プレースホルダーなし") + expected_line = broken.count("\n", 0, translation_offset) + 1 + result = self.run_tool("validate_ts.py", catalog, "--placeholders-only", expect=1) + self.assertIn(f"{catalog}:{expected_line}:", result.stdout) + self.assertIn("[source: alpha.cpp:1]", result.stdout) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/translation/ts_utils.py b/scripts/translation/ts_utils.py new file mode 100644 index 000000000..941148bea --- /dev/null +++ b/scripts/translation/ts_utils.py @@ -0,0 +1,355 @@ +#!/usr/bin/env python3 +"""Shared Qt TS catalog parsing, identity, and validation helpers.""" + +from __future__ import annotations + +import bisect +import collections +import hashlib +import json +import re +import xml.etree.ElementTree as ET +from dataclasses import dataclass +from pathlib import Path +from typing import Iterable, Iterator, Sequence + + +SKIPPED_TYPES = frozenset({"vanished", "obsolete"}) +SCHEMA = "duckstation-qt-translation-batch-v1" + +QT_PLACEHOLDER_RE = re.compile(r"%(?:L?\d+|Ln|n)") +PRINTF_PLACEHOLDER_RE = re.compile(r"%(?!%)(?:(?:[-+0#]+\d*|\d+)?(?:\.\d+)?[diuoxXfFeEgGaAcsp])") +FMT_PLACEHOLDER_RE = re.compile(r"(?]*>", + re.IGNORECASE, +) + +CONTEXT_RE = re.compile(r".*?", re.DOTALL) +MESSAGE_RE = re.compile(r"]*)?>.*?", re.DOTALL) +TRANSLATION_RE = re.compile( + r"[^>]*)>.*?|[^>]*)/>", re.DOTALL +) + + +@dataclass(frozen=True) +class MessageIdentity: + context: str + source: str + comment: str + extracomment: str + numerus: bool + occurrence: int + + def as_payload(self) -> dict[str, object]: + return { + "context": self.context, + "source": self.source, + "comment": self.comment, + "extracomment": self.extracomment, + "numerus": self.numerus, + "occurrence": self.occurrence, + } + + @property + def identifier(self) -> str: + encoded = json.dumps(self.as_payload(), ensure_ascii=False, sort_keys=True, separators=(",", ":")) + return hashlib.sha256(encoded.encode("utf-8")).hexdigest()[:20] + + +@dataclass +class CatalogMessage: + identity: MessageIdentity + translation_type: str + translation: str | None + plural_translations: list[str] + locations: list[dict[str, str]] + catalog_line: int | None = None + + @property + def identifier(self) -> str: + return self.identity.identifier + + +@dataclass(frozen=True) +class RawMessageSpan: + identifier: str + start: int + end: int + block: str + translation_type: str + numerus: bool + catalog_line: int + + +def element_text(element: ET.Element | None) -> str: + return "" if element is None else "".join(element.itertext()) + + +def identity_base(context: str, message: ET.Element) -> tuple[str, str, str, str, bool]: + return ( + context, + message.findtext("source") or "", + message.findtext("comment") or "", + message.findtext("extracomment") or "", + message.get("numerus") == "yes", + ) + + +def iter_catalog_messages(root: ET.Element) -> Iterator[CatalogMessage]: + for context_element in root.findall("context"): + context = context_element.findtext("name") or "" + occurrences: collections.Counter[tuple[str, str, str, str, bool]] = collections.Counter() + for message in context_element.findall("message"): + base = identity_base(context, message) + occurrence = occurrences[base] + occurrences[base] += 1 + identity = MessageIdentity(*base, occurrence) + translation_element = message.find("translation") + translation_type = "missing" if translation_element is None else translation_element.get("type", "finished") + plural_elements = [] if translation_element is None else translation_element.findall("numerusform") + plural_translations = [element_text(form) for form in plural_elements] + translation = None if translation_element is None or plural_elements else element_text(translation_element) + locations = [dict(location.attrib) for location in message.findall("location")] + yield CatalogMessage( + identity=identity, + translation_type=translation_type, + translation=translation, + plural_translations=plural_translations, + locations=locations, + ) + + +def parse_catalog(path: Path | str) -> tuple[ET.ElementTree, list[CatalogMessage]]: + tree = ET.parse(path) + messages = list(iter_catalog_messages(tree.getroot())) + with Path(path).open("r", encoding="utf-8", newline="") as stream: + spans = scan_raw_messages(stream.read()) + for message in messages: + span = spans.get(message.identifier) + if span is not None: + message.catalog_line = span.catalog_line + return tree, messages + + +def catalog_fingerprint(messages: Iterable[CatalogMessage]) -> str: + identities = [message.identity.as_payload() for message in messages] + encoded = json.dumps(identities, ensure_ascii=False, sort_keys=True, separators=(",", ":")) + return hashlib.sha256(encoded.encode("utf-8")).hexdigest() + + +def extract_placeholders(text: str) -> collections.Counter[str]: + tokens: list[str] = [] + occupied: list[tuple[int, int]] = [] + for kind, regex in ( + ("template", TEMPLATE_PLACEHOLDER_RE), + ("fmt", FMT_PLACEHOLDER_RE), + ("qt", QT_PLACEHOLDER_RE), + ("printf", PRINTF_PLACEHOLDER_RE), + ): + for match in regex.finditer(text): + if any(match.start() < end and match.end() > start for start, end in occupied): + continue + occupied.append(match.span()) + tokens.append(f"{kind}:{match.group(0)}") + return collections.Counter(tokens) + + +def placeholders_match(source: str, translation: str) -> bool: + source_tokens = extract_placeholders(source) + translated_tokens = extract_placeholders(translation) + if source_tokens == translated_tokens: + return True + + source_fmt = collections.Counter( + token.removeprefix("fmt:") for token in source_tokens.elements() if token.startswith("fmt:") + ) + translated_fmt = collections.Counter( + token.removeprefix("fmt:") for token in translated_tokens.elements() if token.startswith("fmt:") + ) + source_non_fmt = collections.Counter( + token for token in source_tokens.elements() if not token.startswith("fmt:") + ) + translated_non_fmt = collections.Counter( + token for token in translated_tokens.elements() if not token.startswith("fmt:") + ) + if source_non_fmt != translated_non_fmt: + return False + + # Qt's fmt usage permits translators to number anonymous plain placeholders + # when reordering arguments, e.g. "{} {}" -> "{1} {0}". + if set(source_fmt) == {"{}"}: + expected = collections.Counter(f"{{{index}}}" for index in range(source_fmt["{}"])) + return translated_fmt == expected + return False + + +def placeholders_are_subset(source: str, translation: str) -> bool: + """Allow a plural form to omit redundant arguments without adding any.""" + if placeholders_match(source, translation): + return True + source_tokens = extract_placeholders(source) + translated_tokens = extract_placeholders(translation) + if translated_tokens <= source_tokens: + return True + + source_fmt = collections.Counter( + token.removeprefix("fmt:") for token in source_tokens.elements() if token.startswith("fmt:") + ) + translated_fmt = collections.Counter( + token.removeprefix("fmt:") for token in translated_tokens.elements() if token.startswith("fmt:") + ) + source_non_fmt = collections.Counter( + token for token in source_tokens.elements() if not token.startswith("fmt:") + ) + translated_non_fmt = collections.Counter( + token for token in translated_tokens.elements() if not token.startswith("fmt:") + ) + if not translated_non_fmt <= source_non_fmt or set(source_fmt) != {"{}"}: + return False + allowed_numbered = {f"{{{index}}}" for index in range(source_fmt["{}"])} + return set(translated_fmt) <= allowed_numbered and all(count == 1 for count in translated_fmt.values()) + + +def extract_rich_tags(text: str) -> collections.Counter[str]: + tags: list[str] = [] + for match in RICH_TAG_RE.finditer(text): + closing = "/" if match.group(1) else "" + tags.append(f"{closing}{match.group(2).lower()}") + return collections.Counter(tags) + + +def validate_translation(source: str, translation: str, allow_missing_placeholders: bool = False) -> list[str]: + problems: list[str] = [] + source_placeholders = extract_placeholders(source) + translated_placeholders = extract_placeholders(translation) + placeholders_valid = ( + placeholders_are_subset(source, translation) + if allow_missing_placeholders + else placeholders_match(source, translation) + ) + if not placeholders_valid: + problems.append( + f"placeholder mismatch: source={dict(source_placeholders)} translation={dict(translated_placeholders)}" + ) + required_tags = extract_rich_tags(source) + translated_tags = extract_rich_tags(translation) + missing_tags = required_tags - translated_tags + if missing_tags: + problems.append(f"missing rich-text tags: {dict(missing_tags)}") + return problems + + +def extra_rich_tags(source: str, translation: str) -> collections.Counter[str]: + return extract_rich_tags(translation) - extract_rich_tags(source) + + +def message_to_batch_record(message: CatalogMessage) -> dict[str, object]: + return { + "record_type": "message", + "id": message.identifier, + **message.identity.as_payload(), + "locations": message.locations, + "current_type": message.translation_type, + "current_translation": message.translation, + "current_plural_translations": message.plural_translations, + "accept_current": False, + "target_translation": None, + "target_plural_translations": None, + "suggestions": [], + } + + +def load_jsonl(paths: Sequence[Path | str]) -> tuple[list[dict[str, object]], list[dict[str, object]]]: + metadata: list[dict[str, object]] = [] + records: list[dict[str, object]] = [] + for path_value in paths: + path = Path(path_value) + with path.open("r", encoding="utf-8") as stream: + for line_number, line in enumerate(stream, 1): + if not line.strip(): + continue + try: + item = json.loads(line) + except json.JSONDecodeError as error: + raise ValueError(f"{path}:{line_number}: invalid JSON: {error}") from error + if item.get("record_type") == "metadata": + metadata.append(item) + elif item.get("record_type") == "message": + records.append(item) + else: + raise ValueError(f"{path}:{line_number}: unknown record_type") + return metadata, records + + +def write_jsonl(path: Path, metadata: dict[str, object], records: Sequence[dict[str, object]]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("w", encoding="utf-8", newline="\n") as stream: + stream.write(json.dumps(metadata, ensure_ascii=False, sort_keys=True) + "\n") + for record in records: + stream.write(json.dumps(record, ensure_ascii=False, sort_keys=True) + "\n") + + +def scan_raw_messages(text: str) -> dict[str, RawMessageSpan]: + spans: dict[str, RawMessageSpan] = {} + newline_offsets = [index for index, character in enumerate(text) if character == "\n"] + for context_match in CONTEXT_RE.finditer(text): + context_block = context_match.group(0) + context_name_match = re.search(r"(.*?)", context_block, re.DOTALL) + if context_name_match is None: + continue + context_name = element_text(ET.fromstring(f"{context_name_match.group(1)}")) + occurrences: collections.Counter[tuple[str, str, str, str, bool]] = collections.Counter() + for message_match in MESSAGE_RE.finditer(context_block): + block = message_match.group(0) + message_element = ET.fromstring(block) + base = identity_base(context_name, message_element) + occurrence = occurrences[base] + occurrences[base] += 1 + identity = MessageIdentity(*base, occurrence) + translation = message_element.find("translation") + translation_type = "missing" if translation is None else translation.get("type", "finished") + absolute_start = context_match.start() + message_match.start() + absolute_end = context_match.start() + message_match.end() + translation_match = TRANSLATION_RE.search(block) + diagnostic_offset = absolute_start + (translation_match.start() if translation_match else 0) + span = RawMessageSpan( + identifier=identity.identifier, + start=absolute_start, + end=absolute_end, + block=block, + translation_type=translation_type, + numerus=message_element.get("numerus") == "yes", + catalog_line=bisect.bisect_left(newline_offsets, diagnostic_offset) + 1, + ) + if span.identifier in spans: + raise ValueError(f"duplicate stable message id: {span.identifier}") + spans[span.identifier] = span + return spans + + +def xml_escape_text(text: str) -> str: + return text.replace("&", "&").replace("<", "<").replace(">", ">") + + +def remove_unfinished_attribute(attributes: str) -> str: + attributes = re.sub(r"\s+type=(?:\"unfinished\"|'unfinished')", "", attributes) + return attributes.rstrip() + + +def replace_translation_node(block: str, singular: str | None, plurals: Sequence[str] | None) -> str: + match = TRANSLATION_RE.search(block) + if match is None: + raise ValueError("message has no translation element") + attributes = remove_unfinished_attribute(match.group("attrs") or match.group("self_attrs") or "") + line_start = block.rfind("\n", 0, match.start()) + 1 + indent = block[line_start:match.start()] + if plurals is not None: + forms = "\n".join(f"{indent} {xml_escape_text(value)}" for value in plurals) + replacement = f"\n{forms}\n{indent}" + else: + if singular is None: + raise ValueError("missing singular target translation") + replacement = f"{xml_escape_text(singular)}" + return block[: match.start()] + replacement + block[match.end() :] diff --git a/scripts/translation/validate_ts.py b/scripts/translation/validate_ts.py new file mode 100644 index 000000000..fe6a20257 --- /dev/null +++ b/scripts/translation/validate_ts.py @@ -0,0 +1,142 @@ +#!/usr/bin/env python3 +"""Validate Qt TS structure, completeness, placeholders, plurals, and rich text.""" + +from __future__ import annotations + +import argparse +import collections +import xml.etree.ElementTree as ET +from pathlib import Path + +if __package__ in (None, ""): + import sys + + sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from translation.ts_utils import ( # noqa: E402 + SKIPPED_TYPES, + CatalogMessage, + extra_rich_tags, + load_jsonl, + parse_catalog, + placeholders_match, + validate_translation, +) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("catalog", type=Path, help="Qt Linguist .ts catalog") + parser.add_argument("--require-complete", action="store_true", help="fail on unfinished or empty active messages") + parser.add_argument("--records", nargs="+", type=Path, help="validate only IDs contained in JSONL batches") + parser.add_argument( + "--placeholders-only", + action="store_true", + help="check placeholders only, allowing incomplete translations", + ) + parser.add_argument("--strict-extra-tags", action="store_true", help="treat added rich-text tags as errors") + return parser.parse_args() + + +def values_for(message: CatalogMessage) -> list[str]: + return message.plural_translations if message.plural_translations else [message.translation or ""] + + +def diagnostic_label(catalog: Path, message: CatalogMessage) -> str: + line = message.catalog_line or 1 + label = f"{catalog}:{line}: {message.identity.context}/{message.identifier}" + source_locations = [] + for location in message.locations: + filename = location.get("filename", "") + source_line = location.get("line", "") + if filename and source_line: + source_locations.append(f"{filename}:{source_line}") + elif filename: + source_locations.append(filename) + if source_locations: + label += f" [source: {', '.join(source_locations)}]" + return label + + +def main() -> int: + args = parse_args() + try: + _, messages = parse_catalog(args.catalog) + except ET.ParseError as error: + line, column = error.position + print(f"ERROR: {args.catalog}:{line}:{column + 1}: XML parse error: {error}") + return 1 + except (OSError, ValueError) as error: + print(f"ERROR: {args.catalog}:1: unable to read catalog: {error}") + return 1 + + selected_ids: set[str] | None = None + if args.records: + _, records = load_jsonl(args.records) + selected_ids = {str(record.get("id", "")) for record in records} + + errors: list[str] = [] + warnings: list[str] = [] + checked = 0 + type_counts: collections.Counter[str] = collections.Counter() + for message in messages: + type_counts[message.translation_type] += 1 + if message.translation_type in SKIPPED_TYPES: + continue + if selected_ids is not None and message.identifier not in selected_ids: + continue + checked += 1 + label = diagnostic_label(args.catalog, message) + if not args.placeholders_only: + if message.identity.numerus and not message.plural_translations: + errors.append(f"{label}: numerus message has no forms") + if not message.identity.numerus and message.plural_translations: + errors.append(f"{label}: singular message unexpectedly has plural forms") + if args.require_complete and message.translation_type == "unfinished": + errors.append(f"{label}: active translation is unfinished") + if args.require_complete and not any(value.strip() for value in values_for(message)): + errors.append(f"{label}: active translation is empty") + + for value in values_for(message): + if not value.strip() and not args.require_complete: + continue + problems = validate_translation( + message.identity.source, + value, + allow_missing_placeholders=message.identity.numerus, + ) + for problem in problems: + if args.placeholders_only and not problem.startswith("placeholder mismatch"): + continue + errors.append(f"{label}: {problem}") + if not args.placeholders_only: + extras = extra_rich_tags(message.identity.source, value) + if extras: + output = errors if args.strict_extra_tags else warnings + output.append(f"{label}: additional rich-text tags: {dict(extras)}") + if ( + message.identity.numerus + and any(value.strip() for value in values_for(message)) + and not any(placeholders_match(message.identity.source, value) for value in values_for(message)) + ): + errors.append(f"{label}: no plural form preserves the complete source placeholder set") + + if selected_ids is not None: + found = {message.identifier for message in messages} + for missing in sorted(selected_ids - found): + errors.append(f"{args.catalog}:1: {missing}: batch record not found in catalog") + + print(f"Checked {checked} active message(s). Translation types: {dict(type_counts)}") + for warning in warnings: + print(f"WARNING: {warning}") + for error in errors: + print(f"ERROR: {error}") + if errors: + print(f"Validation failed with {len(errors)} error(s) and {len(warnings)} warning(s).") + return 1 + print(f"Validation passed with {len(warnings)} warning(s).") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/verify_translation_placeholders.py b/scripts/verify_translation_placeholders.py deleted file mode 100644 index d8e95677f..000000000 --- a/scripts/verify_translation_placeholders.py +++ /dev/null @@ -1,181 +0,0 @@ - -#!/usr/bin/env python3 -""" -Qt Translation Placeholder Verifier - -This script verifies that placeholders in Qt translation files are consistent -between source and translation strings. - -Placeholder rules: -- {} placeholders: translation must have same total count as source -- {n} placeholders: translation must use same numbers as source (can repeat) -""" - -import xml.etree.ElementTree as ET -import re -import sys -from typing import List, Set, Tuple, Dict -from pathlib import Path - - -def extract_placeholders(text: str) -> Tuple[int, Set[int]]: - """ - Extract placeholder information from a string. - - Returns: - Tuple of (unnamed_count, set_of_numbered_placeholders) - """ - # Find all placeholders - placeholders = re.findall(r'\{(\d*)\}', text) - - unnamed_count = 0 - numbered_set = set() - - for placeholder in placeholders: - if placeholder == '': - unnamed_count += 1 - else: - numbered_set.add(int(placeholder)) - - return unnamed_count, numbered_set - - -def verify_placeholders(source: str, translation: str) -> Tuple[bool, str]: - """ - Verify that placeholders in source and translation are consistent. - - Returns: - Tuple of (is_valid, error_message) - """ - if not translation.strip(): - return True, "" # Empty translations are valid - - source_unnamed, source_numbered = extract_placeholders(source) - trans_unnamed, trans_numbered = extract_placeholders(translation) - - # Check if mixing numbered and unnumbered placeholders - if source_unnamed > 0 and len(source_numbered) > 0: - return False, "Source mixes numbered and unnumbered placeholders" - - if trans_unnamed > 0 and len(trans_numbered) > 0: - return False, "Translation mixes numbered and unnumbered placeholders" - - # If source uses unnumbered placeholders - if source_unnamed > 0: - # Translation can use either unnumbered or numbered placeholders - if trans_unnamed > 0: - # Both use unnumbered - simple count match - if source_unnamed != trans_unnamed: - return False, f"Placeholder count mismatch: source has {source_unnamed}, translation has {trans_unnamed}" - elif len(trans_numbered) > 0: - # Source uses unnumbered, translation uses numbered - # Check that numbered placeholders are 0-based and consecutive up to source count - expected_numbers = set(range(source_unnamed)) - if trans_numbered != expected_numbers: - if max(trans_numbered) >= source_unnamed: - return False, f"Numbered placeholders exceed source count: translation uses {{{max(trans_numbered)}}}, but source only has {source_unnamed} placeholders" - elif min(trans_numbered) != 0: - return False, f"Numbered placeholders must start from 0: found {{{min(trans_numbered)}}}" - else: - missing = expected_numbers - trans_numbered - return False, f"Missing numbered placeholders: {{{','.join(map(str, sorted(missing)))}}}" - - # If source uses numbered placeholders - elif len(source_numbered) > 0: - if trans_unnamed > 0: - return False, "Source uses numbered {n} but translation uses unnumbered {}" - if source_numbered != trans_numbered: - missing = source_numbered - trans_numbered - extra = trans_numbered - source_numbered - error_parts = [] - if missing: - error_parts.append(f"missing {{{','.join(map(str, sorted(missing)))}}}") - if extra: - error_parts.append(f"extra {{{','.join(map(str, sorted(extra)))}}}") - return False, f"Numbered placeholder mismatch: {', '.join(error_parts)}" - - # If translation has placeholders but source doesn't - elif trans_unnamed > 0 or len(trans_numbered) > 0: - return False, "Translation has placeholders but source doesn't" - - return True, "" - - -def verify_translation_file(file_path: str) -> List[Dict]: - """ - Verify all translations in a Qt translation file. - - Returns: - List of error dictionaries with details about invalid translations - """ - try: - tree = ET.parse(file_path) - root = tree.getroot() - except ET.ParseError as e: - return [{"error": f"XML parsing error: {e}", "line": None}] - except FileNotFoundError: - return [{"error": f"File not found: {file_path}", "line": None}] - - errors = [] - - # Find all message elements - for message in root.findall('.//message'): - source_elem = message.find('source') - translation_elem = message.find('translation') - location_elem = message.find('location') - - if source_elem is None or translation_elem is None: - continue - - source_text = source_elem.text or "" - translation_text = translation_elem.text or "" - - is_valid, error_msg = verify_placeholders(source_text, translation_text) - - if not is_valid: - error_info = { - "source": source_text, - "translation": translation_text, - "error": error_msg, - "line": location_elem.get('line') if location_elem is not None else None, - "filename": location_elem.get('filename') if location_elem is not None else None - } - errors.append(error_info) - - return errors - - -def main(): - if len(sys.argv) != 2: - print(f"Usage: {sys.argv[0]} ") - sys.exit(1) - - file_path = sys.argv[1] - - if not Path(file_path).exists(): - print(f"Error: File '{file_path}' not found") - sys.exit(1) - - print(f"Verifying placeholders in: {file_path}") - - errors = verify_translation_file(file_path) - - if not errors: - print("All placeholders are valid.") - sys.exit(0) - - print(f"Found {len(errors)} placeholder errors:") - - for i, error in enumerate(errors, 1): - print(f"\nError {i}:") - if error.get('filename') and error.get('line'): - print(f" Location: {error['filename']}:{error['line']}") - print(f" Source: {error['source']}") - print(f" Translation: {error['translation']}") - print(f" Issue: {error['error']}") - - sys.exit(1) - - -if __name__ == "__main__": - main() \ No newline at end of file