mirror of
https://github.com/stenzek/duckstation.git
synced 2026-07-11 01:24:11 +02:00
Scripts: Extend translation validation/linters
This commit is contained in:
Stenzek
@@ -0,0 +1,163 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Safely apply completed JSONL translation batches to a Qt TS catalog."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
if __package__ in (None, ""):
|
||||
import sys
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
|
||||
from translation.ts_utils import ( # noqa: E402
|
||||
SCHEMA,
|
||||
MessageIdentity,
|
||||
catalog_fingerprint,
|
||||
load_jsonl,
|
||||
parse_catalog,
|
||||
placeholders_match,
|
||||
replace_translation_node,
|
||||
scan_raw_messages,
|
||||
validate_translation,
|
||||
)
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("catalog", type=Path, help="Qt Linguist .ts catalog")
|
||||
parser.add_argument("batches", nargs="+", type=Path, help="completed JSONL batch files")
|
||||
parser.add_argument("--write", action="store_true", help="atomically update the catalog; default is dry-run")
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def record_identity(record: dict[str, object]) -> MessageIdentity:
|
||||
return MessageIdentity(
|
||||
context=str(record.get("context", "")),
|
||||
source=str(record.get("source", "")),
|
||||
comment=str(record.get("comment", "")),
|
||||
extracomment=str(record.get("extracomment", "")),
|
||||
numerus=bool(record.get("numerus", False)),
|
||||
occurrence=int(record.get("occurrence", 0)),
|
||||
)
|
||||
|
||||
|
||||
def resolve_target(record: dict[str, object]) -> tuple[str | None, list[str] | None] | None:
|
||||
singular = record.get("target_translation")
|
||||
plurals = record.get("target_plural_translations")
|
||||
accept_current = record.get("accept_current") is True
|
||||
if accept_current:
|
||||
if singular is not None or plurals is not None:
|
||||
raise ValueError(f"{record.get('id')}: accept_current cannot be combined with target fields")
|
||||
current_plurals = record.get("current_plural_translations")
|
||||
if current_plurals:
|
||||
plurals = current_plurals
|
||||
else:
|
||||
singular = record.get("current_translation")
|
||||
|
||||
if singular is None and plurals is None:
|
||||
return None
|
||||
if singular is not None and plurals is not None:
|
||||
raise ValueError(f"{record.get('id')}: set either singular or plural target, not both")
|
||||
if singular is not None:
|
||||
if not isinstance(singular, str) or not singular.strip():
|
||||
raise ValueError(f"{record.get('id')}: target_translation must be a nonempty string")
|
||||
return singular, None
|
||||
if (
|
||||
not isinstance(plurals, list)
|
||||
or not plurals
|
||||
or any(not isinstance(value, str) or not value.strip() for value in plurals)
|
||||
):
|
||||
raise ValueError(f"{record.get('id')}: target_plural_translations must contain nonempty strings")
|
||||
return None, plurals
|
||||
|
||||
|
||||
def atomic_write(path: Path, text: str) -> None:
|
||||
mode = path.stat().st_mode
|
||||
temporary_name: str | None = None
|
||||
try:
|
||||
with tempfile.NamedTemporaryFile("w", encoding="utf-8", newline="", dir=path.parent, delete=False) as stream:
|
||||
temporary_name = stream.name
|
||||
stream.write(text)
|
||||
stream.flush()
|
||||
os.fsync(stream.fileno())
|
||||
os.chmod(temporary_name, mode)
|
||||
os.replace(temporary_name, path)
|
||||
finally:
|
||||
if temporary_name and os.path.exists(temporary_name):
|
||||
os.unlink(temporary_name)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = parse_args()
|
||||
metadata, records = load_jsonl(args.batches)
|
||||
if len(metadata) != len(args.batches):
|
||||
raise SystemExit("each batch must contain exactly one metadata record")
|
||||
for item in metadata:
|
||||
if item.get("schema") != SCHEMA:
|
||||
raise SystemExit(f"unsupported batch schema: {item.get('schema')!r}")
|
||||
|
||||
_, messages = parse_catalog(args.catalog)
|
||||
fingerprint = catalog_fingerprint(messages)
|
||||
expected_fingerprints = {str(item.get("catalog_fingerprint", "")) for item in metadata}
|
||||
if expected_fingerprints != {fingerprint}:
|
||||
raise SystemExit("catalog source identity has changed since extraction; export fresh batches")
|
||||
|
||||
with args.catalog.open("r", encoding="utf-8", newline="") as stream:
|
||||
raw_text = stream.read()
|
||||
spans = scan_raw_messages(raw_text)
|
||||
seen: set[str] = set()
|
||||
replacements: list[tuple[int, int, str]] = []
|
||||
skipped = 0
|
||||
for record in records:
|
||||
identifier = str(record.get("id", ""))
|
||||
identity = record_identity(record)
|
||||
if identifier != identity.identifier:
|
||||
raise SystemExit(f"{identifier or '<missing id>'}: record identity does not match its id")
|
||||
if identifier in seen:
|
||||
raise SystemExit(f"duplicate record id across batches: {identifier}")
|
||||
seen.add(identifier)
|
||||
target = resolve_target(record)
|
||||
if target is None:
|
||||
skipped += 1
|
||||
continue
|
||||
span = spans.get(identifier)
|
||||
if span is None:
|
||||
raise SystemExit(f"message no longer exists in catalog: {identifier}")
|
||||
if span.translation_type != "unfinished":
|
||||
raise SystemExit(f"refusing to overwrite {span.translation_type} translation: {identifier}")
|
||||
singular, plurals = target
|
||||
if span.numerus != (plurals is not None):
|
||||
raise SystemExit(f"singular/plural target does not match catalog message: {identifier}")
|
||||
values = plurals if plurals is not None else [singular or ""]
|
||||
for value in values:
|
||||
problems = validate_translation(
|
||||
identity.source,
|
||||
value,
|
||||
allow_missing_placeholders=plurals is not None,
|
||||
)
|
||||
if problems:
|
||||
raise SystemExit(f"{identifier}: {'; '.join(problems)}")
|
||||
if plurals is not None and not any(placeholders_match(identity.source, value) for value in plurals):
|
||||
raise SystemExit(f"{identifier}: no plural form preserves the complete source placeholder set")
|
||||
replacement_block = replace_translation_node(span.block, singular, plurals)
|
||||
replacements.append((span.start, span.end, replacement_block))
|
||||
|
||||
updated_text = raw_text
|
||||
for start, end, replacement in sorted(replacements, reverse=True):
|
||||
updated_text = updated_text[:start] + replacement + updated_text[end:]
|
||||
if args.write and replacements:
|
||||
atomic_write(args.catalog, updated_text)
|
||||
|
||||
action = "Applied" if args.write else "Would apply"
|
||||
print(f"{action} {len(replacements)} translation(s); skipped {skipped} record(s) without targets.")
|
||||
if not args.write:
|
||||
print("Dry-run only; pass --write to update the catalog.")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,179 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Export active unfinished Qt TS messages to editable JSONL batches."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import collections
|
||||
import difflib
|
||||
from pathlib import Path
|
||||
|
||||
if __package__ in (None, ""):
|
||||
import sys
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
|
||||
from translation.ts_utils import ( # noqa: E402
|
||||
SCHEMA,
|
||||
SKIPPED_TYPES,
|
||||
CatalogMessage,
|
||||
catalog_fingerprint,
|
||||
message_to_batch_record,
|
||||
parse_catalog,
|
||||
write_jsonl,
|
||||
)
|
||||
|
||||
|
||||
def translated_text(message: CatalogMessage) -> str | list[str] | None:
|
||||
if message.plural_translations:
|
||||
return message.plural_translations if any(value.strip() for value in message.plural_translations) else None
|
||||
return message.translation if message.translation and message.translation.strip() else None
|
||||
|
||||
|
||||
def build_suggestions(
|
||||
target: CatalogMessage,
|
||||
messages: list[CatalogMessage],
|
||||
limit: int,
|
||||
minimum_similarity: float,
|
||||
) -> list[dict[str, object]]:
|
||||
if limit <= 0:
|
||||
return []
|
||||
candidates: list[tuple[float, CatalogMessage]] = []
|
||||
for candidate in messages:
|
||||
text = translated_text(candidate)
|
||||
if text is None or candidate.identifier == target.identifier:
|
||||
continue
|
||||
exact = candidate.identity.source == target.identity.source
|
||||
if not exact and candidate.identity.context != target.identity.context:
|
||||
continue
|
||||
similarity = (
|
||||
1.0
|
||||
if exact
|
||||
else difflib.SequenceMatcher(None, target.identity.source, candidate.identity.source).ratio()
|
||||
)
|
||||
if similarity >= minimum_similarity:
|
||||
candidates.append((similarity, candidate))
|
||||
candidates.sort(
|
||||
key=lambda pair: (
|
||||
pair[0],
|
||||
pair[1].translation_type == "finished",
|
||||
pair[1].identity.source == target.identity.source,
|
||||
),
|
||||
reverse=True,
|
||||
)
|
||||
output: list[dict[str, object]] = []
|
||||
seen: set[tuple[str, str]] = set()
|
||||
for similarity, candidate in candidates:
|
||||
text = translated_text(candidate)
|
||||
signature = (candidate.identity.source, repr(text))
|
||||
if signature in seen:
|
||||
continue
|
||||
seen.add(signature)
|
||||
output.append(
|
||||
{
|
||||
"source": candidate.identity.source,
|
||||
"translation": text,
|
||||
"context": candidate.identity.context,
|
||||
"type": candidate.translation_type,
|
||||
"similarity": round(similarity, 4),
|
||||
}
|
||||
)
|
||||
if len(output) == limit:
|
||||
break
|
||||
return output
|
||||
|
||||
|
||||
def split_balanced(records: list[dict[str, object]], batch_count: int) -> list[list[dict[str, object]]]:
|
||||
if batch_count <= 1 or len(records) <= 1:
|
||||
return [records]
|
||||
batch_count = min(batch_count, len(records))
|
||||
by_context: dict[str, list[dict[str, object]]] = collections.defaultdict(list)
|
||||
for record in records:
|
||||
by_context[str(record["context"])].append(record)
|
||||
|
||||
target_size = max(1, (len(records) + batch_count - 1) // batch_count)
|
||||
chunks: list[list[dict[str, object]]] = []
|
||||
for context_records in by_context.values():
|
||||
if len(context_records) > target_size * 3 // 2:
|
||||
chunks.extend(
|
||||
context_records[offset : offset + target_size]
|
||||
for offset in range(0, len(context_records), target_size)
|
||||
)
|
||||
else:
|
||||
chunks.append(context_records)
|
||||
|
||||
batches: list[list[dict[str, object]]] = [[] for _ in range(batch_count)]
|
||||
for chunk in sorted(chunks, key=len, reverse=True):
|
||||
destination = min(batches, key=len)
|
||||
destination.extend(chunk)
|
||||
return batches
|
||||
|
||||
|
||||
def make_metadata(catalog: Path, fingerprint: str, batch_index: int, batch_count: int) -> dict[str, object]:
|
||||
return {
|
||||
"record_type": "metadata",
|
||||
"schema": SCHEMA,
|
||||
"catalog": str(catalog),
|
||||
"catalog_fingerprint": fingerprint,
|
||||
"batch_index": batch_index,
|
||||
"batch_count": batch_count,
|
||||
}
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("catalog", type=Path, help="Qt Linguist .ts catalog")
|
||||
output = parser.add_mutually_exclusive_group(required=True)
|
||||
output.add_argument("--output", type=Path, help="single JSONL output file")
|
||||
output.add_argument("--batch-dir", type=Path, help="directory for numbered JSONL batches")
|
||||
parser.add_argument("--batches", type=int, default=1, help="number of balanced batches (with --batch-dir)")
|
||||
parser.add_argument("--context", action="append", default=[], help="include only this context; repeatable")
|
||||
parser.add_argument("--suggestions", type=int, default=2, help="translation-memory suggestions per message")
|
||||
parser.add_argument("--similarity", type=float, default=0.72, help="minimum fuzzy suggestion similarity")
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = parse_args()
|
||||
if args.output and args.batches != 1:
|
||||
raise SystemExit("--batches requires --batch-dir")
|
||||
if args.batches < 1:
|
||||
raise SystemExit("--batches must be at least 1")
|
||||
if not 0.0 <= args.similarity <= 1.0:
|
||||
raise SystemExit("--similarity must be between 0 and 1")
|
||||
|
||||
_, messages = parse_catalog(args.catalog)
|
||||
contexts = set(args.context)
|
||||
targets = [
|
||||
message
|
||||
for message in messages
|
||||
if message.translation_type == "unfinished"
|
||||
and message.translation_type not in SKIPPED_TYPES
|
||||
and (not contexts or message.identity.context in contexts)
|
||||
]
|
||||
records = []
|
||||
for message in targets:
|
||||
record = message_to_batch_record(message)
|
||||
record["suggestions"] = build_suggestions(message, messages, args.suggestions, args.similarity)
|
||||
records.append(record)
|
||||
|
||||
fingerprint = catalog_fingerprint(messages)
|
||||
if args.output:
|
||||
write_jsonl(args.output, make_metadata(args.catalog, fingerprint, 1, 1), records)
|
||||
destinations = [args.output]
|
||||
else:
|
||||
batches = split_balanced(records, args.batches)
|
||||
destinations = []
|
||||
for index, batch in enumerate(batches, 1):
|
||||
destination = args.batch_dir / f"batch-{index:03d}.jsonl"
|
||||
write_jsonl(destination, make_metadata(args.catalog, fingerprint, index, len(batches)), batch)
|
||||
destinations.append(destination)
|
||||
|
||||
print(f"Exported {len(records)} active unfinished messages to {len(destinations)} file(s).")
|
||||
for destination in destinations:
|
||||
print(destination)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,261 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
SCRIPTS_DIR = Path(__file__).resolve().parents[2]
|
||||
REPO_ROOT = SCRIPTS_DIR.parent
|
||||
sys.path.insert(0, str(SCRIPTS_DIR))
|
||||
|
||||
from translation.ts_utils import ( # noqa: E402
|
||||
TRANSLATION_RE,
|
||||
catalog_fingerprint,
|
||||
extract_placeholders,
|
||||
extract_rich_tags,
|
||||
parse_catalog,
|
||||
placeholders_are_subset,
|
||||
placeholders_match,
|
||||
replace_translation_node,
|
||||
validate_translation,
|
||||
)
|
||||
|
||||
|
||||
FIXTURE = """<?xml version="1.0" encoding="utf-8"?>
|
||||
<!DOCTYPE TS>
|
||||
<TS version="2.1" language="ja">
|
||||
<context>
|
||||
<name>Alpha</name>
|
||||
<message>
|
||||
<location filename="alpha.cpp" line="1"/>
|
||||
<source>Hello %1 {0} ${title} %.1f <strong>world</strong></source>
|
||||
<translation type="unfinished"></translation>
|
||||
</message>
|
||||
<message>
|
||||
<source>Register</source>
|
||||
<translation type="unfinished">レジスタ</translation>
|
||||
</message>
|
||||
<message numerus="yes">
|
||||
<source>%n file(s)</source>
|
||||
<comment>File count</comment>
|
||||
<translation type="unfinished">
|
||||
<numerusform></numerusform>
|
||||
</translation>
|
||||
</message>
|
||||
<message>
|
||||
<source>New wording</source>
|
||||
<translation type="unfinished"></translation>
|
||||
</message>
|
||||
<message>
|
||||
<source>Old wording</source>
|
||||
<translation type="vanished">古い文言</translation>
|
||||
</message>
|
||||
<message>
|
||||
<source>Hello</source>
|
||||
<translation>こんにちは</translation>
|
||||
</message>
|
||||
</context>
|
||||
<context>
|
||||
<name>Beta</name>
|
||||
<message>
|
||||
<source>Register</source>
|
||||
<translation type="unfinished"></translation>
|
||||
</message>
|
||||
<message>
|
||||
<source>Removed</source>
|
||||
<translation type="obsolete">削除済み</translation>
|
||||
</message>
|
||||
</context>
|
||||
</TS>
|
||||
"""
|
||||
|
||||
|
||||
class TranslationToolTests(unittest.TestCase):
|
||||
def run_tool(self, script: str, *arguments: object, expect: int = 0) -> subprocess.CompletedProcess[str]:
|
||||
result = subprocess.run(
|
||||
[sys.executable, str(SCRIPTS_DIR / "translation" / script), *(str(value) for value in arguments)],
|
||||
cwd=REPO_ROOT,
|
||||
text=True,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT,
|
||||
check=False,
|
||||
)
|
||||
self.assertEqual(expect, result.returncode, result.stdout)
|
||||
return result
|
||||
|
||||
def test_placeholder_recognition_avoids_percent_and_escaped_brace_false_positives(self) -> None:
|
||||
text = "At 100% speed, 5% of users: %1 %n {} {0:08X} %.1f %s ${title} {{}} %%"
|
||||
placeholders = extract_placeholders(text)
|
||||
self.assertEqual(1, placeholders["qt:%1"])
|
||||
self.assertEqual(1, placeholders["qt:%n"])
|
||||
self.assertEqual(1, placeholders["fmt:{}"])
|
||||
self.assertEqual(1, placeholders["fmt:{0:08X}"])
|
||||
self.assertEqual(1, placeholders["printf:%.1f"])
|
||||
self.assertEqual(1, placeholders["printf:%s"])
|
||||
self.assertEqual(1, placeholders["template:${title}"])
|
||||
self.assertEqual(7, sum(placeholders.values()))
|
||||
self.assertTrue(placeholders_match("{} {}", "{1} {0}"))
|
||||
self.assertFalse(placeholders_match("{} {}", "{0} {0}"))
|
||||
self.assertTrue(placeholders_are_subset("{} of %n", "%n"))
|
||||
self.assertFalse(placeholders_are_subset("{} of %n", "%n %1"))
|
||||
self.assertEqual({"qt:%1": 1}, dict(extract_placeholders("%1x")))
|
||||
|
||||
def test_rich_text_ignores_angle_bracket_labels(self) -> None:
|
||||
self.assertFalse(extract_rich_tags("<Parent Directory>"))
|
||||
self.assertEqual({"strong": 1, "/strong": 1}, dict(extract_rich_tags("<strong>Text</strong>")))
|
||||
self.assertTrue(validate_translation("<strong>Text</strong>", "テキスト"))
|
||||
|
||||
def test_fingerprint_ignores_translation_changes(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
first = Path(directory) / "first.ts"
|
||||
second = Path(directory) / "second.ts"
|
||||
first.write_text(FIXTURE, encoding="utf-8")
|
||||
second.write_text(FIXTURE.replace("レジスタ", "登録"), encoding="utf-8")
|
||||
_, first_messages = parse_catalog(first)
|
||||
_, second_messages = parse_catalog(second)
|
||||
self.assertEqual(catalog_fingerprint(first_messages), catalog_fingerprint(second_messages))
|
||||
|
||||
def test_translation_node_replacement_preserves_message_text(self) -> None:
|
||||
block = (
|
||||
" <message>\n"
|
||||
" <source>A & B</source>\n"
|
||||
' <translation type="unfinished"></translation>\n'
|
||||
" </message>"
|
||||
)
|
||||
replaced = replace_translation_node(block, "A と B", None)
|
||||
self.assertIn("<source>A & B</source>", replaced)
|
||||
self.assertIn("<translation>A と B</translation>", replaced)
|
||||
self.assertNotIn("unfinished", replaced)
|
||||
|
||||
def test_end_to_end_extract_apply_validate(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
root = Path(directory)
|
||||
catalog = root / "catalog.ts"
|
||||
batch = root / "batch.jsonl"
|
||||
catalog.write_text(FIXTURE, encoding="utf-8")
|
||||
original = catalog.read_bytes()
|
||||
|
||||
self.run_tool("extract_ts.py", catalog, "--output", batch, "--suggestions", 3)
|
||||
lines = [json.loads(line) for line in batch.read_text(encoding="utf-8").splitlines()]
|
||||
records = [line for line in lines if line["record_type"] == "message"]
|
||||
self.assertEqual(5, len(records))
|
||||
self.assertNotIn("Old wording", {record["source"] for record in records})
|
||||
new_wording = next(record for record in records if record["source"] == "New wording")
|
||||
self.assertTrue(any(item["type"] == "vanished" for item in new_wording["suggestions"]))
|
||||
|
||||
translations: dict[tuple[str, str], str | list[str]] = {
|
||||
(
|
||||
"Alpha",
|
||||
"Hello %1 {0} ${title} %.1f <strong>world</strong>",
|
||||
): "こんにちは %1 {0} ${title} %.1f <strong>世界</strong>",
|
||||
("Alpha", "Register"): "登録",
|
||||
("Alpha", "%n file(s)"): ["%n ファイル"],
|
||||
("Alpha", "New wording"): "新しい文言",
|
||||
("Beta", "Register"): "登録",
|
||||
}
|
||||
for record in records:
|
||||
target = translations[(record["context"], record["source"])]
|
||||
if isinstance(target, list):
|
||||
record["target_plural_translations"] = target
|
||||
else:
|
||||
record["target_translation"] = target
|
||||
batch.write_text(
|
||||
"\n".join(json.dumps(line, ensure_ascii=False) for line in lines) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
self.run_tool("apply_ts.py", catalog, batch)
|
||||
self.assertEqual(original, catalog.read_bytes(), "dry-run modified the catalog")
|
||||
self.run_tool("apply_ts.py", catalog, batch, "--write")
|
||||
updated = catalog.read_text(encoding="utf-8")
|
||||
|
||||
def normalize_translations(text: str) -> str:
|
||||
return TRANSLATION_RE.sub("<translation/>", text)
|
||||
|
||||
self.assertEqual(normalize_translations(FIXTURE), normalize_translations(updated))
|
||||
self.assertIn('<translation type="vanished">古い文言</translation>', updated)
|
||||
self.assertIn('<translation type="obsolete">削除済み</translation>', updated)
|
||||
self.assertNotIn('type="unfinished"', updated)
|
||||
self.run_tool("validate_ts.py", catalog, "--require-complete")
|
||||
|
||||
def test_accept_current_and_balanced_batches(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
root = Path(directory)
|
||||
catalog = root / "catalog.ts"
|
||||
batches = root / "batches"
|
||||
catalog.write_text(FIXTURE, encoding="utf-8")
|
||||
self.run_tool("extract_ts.py", catalog, "--batch-dir", batches, "--batches", 3)
|
||||
files = sorted(batches.glob("*.jsonl"))
|
||||
self.assertEqual(3, len(files))
|
||||
found = False
|
||||
for path in files:
|
||||
lines = [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines()]
|
||||
for line in lines:
|
||||
if line.get("context") == "Alpha" and line.get("source") == "Register":
|
||||
line["accept_current"] = True
|
||||
found = True
|
||||
path.write_text(
|
||||
"\n".join(json.dumps(line, ensure_ascii=False) for line in lines) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
self.assertTrue(found)
|
||||
result = self.run_tool("apply_ts.py", catalog, *files)
|
||||
self.assertIn("Would apply 1 translation", result.stdout)
|
||||
|
||||
def test_batches_apply_sequentially_without_invalidating_fingerprint(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
root = Path(directory)
|
||||
catalog = root / "catalog.ts"
|
||||
batches = root / "batches"
|
||||
catalog.write_text(FIXTURE, encoding="utf-8")
|
||||
self.run_tool("extract_ts.py", catalog, "--batch-dir", batches, "--batches", 2)
|
||||
files = sorted(batches.glob("*.jsonl"))
|
||||
self.assertEqual(2, len(files))
|
||||
for path in files:
|
||||
lines = [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines()]
|
||||
record = next(line for line in lines if line["record_type"] == "message")
|
||||
if record["numerus"]:
|
||||
record["target_plural_translations"] = [record["source"]]
|
||||
else:
|
||||
record["target_translation"] = record["source"]
|
||||
path.write_text(
|
||||
"\n".join(json.dumps(line, ensure_ascii=False) for line in lines) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
self.run_tool("apply_ts.py", catalog, files[0], "--write")
|
||||
self.run_tool("apply_ts.py", catalog, files[1], "--write")
|
||||
|
||||
def test_stale_catalog_is_rejected(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
root = Path(directory)
|
||||
catalog = root / "catalog.ts"
|
||||
batch = root / "batch.jsonl"
|
||||
catalog.write_text(FIXTURE, encoding="utf-8")
|
||||
self.run_tool("extract_ts.py", catalog, "--output", batch)
|
||||
catalog.write_text(FIXTURE.replace("New wording", "Changed source"), encoding="utf-8")
|
||||
result = self.run_tool("apply_ts.py", catalog, batch, expect=1)
|
||||
self.assertIn("source identity has changed", result.stdout)
|
||||
|
||||
def test_validation_reports_catalog_and_source_lines(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
catalog = Path(directory) / "catalog.ts"
|
||||
broken = FIXTURE.replace(
|
||||
'<translation type="unfinished"></translation>',
|
||||
"<translation>プレースホルダーなし</translation>",
|
||||
1,
|
||||
)
|
||||
catalog.write_text(broken, encoding="utf-8")
|
||||
translation_offset = broken.index("<translation>プレースホルダーなし</translation>")
|
||||
expected_line = broken.count("\n", 0, translation_offset) + 1
|
||||
result = self.run_tool("validate_ts.py", catalog, "--placeholders-only", expect=1)
|
||||
self.assertIn(f"{catalog}:{expected_line}:", result.stdout)
|
||||
self.assertIn("[source: alpha.cpp:1]", result.stdout)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,355 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Shared Qt TS catalog parsing, identity, and validation helpers."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import bisect
|
||||
import collections
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
import xml.etree.ElementTree as ET
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Iterable, Iterator, Sequence
|
||||
|
||||
|
||||
SKIPPED_TYPES = frozenset({"vanished", "obsolete"})
|
||||
SCHEMA = "duckstation-qt-translation-batch-v1"
|
||||
|
||||
QT_PLACEHOLDER_RE = re.compile(r"%(?:L?\d+|Ln|n)")
|
||||
PRINTF_PLACEHOLDER_RE = re.compile(r"%(?!%)(?:(?:[-+0#]+\d*|\d+)?(?:\.\d+)?[diuoxXfFeEgGaAcsp])")
|
||||
FMT_PLACEHOLDER_RE = re.compile(r"(?<!\{)\{(?:\d+)?(?::[^{}]+)?\}(?!\})")
|
||||
TEMPLATE_PLACEHOLDER_RE = re.compile(r"\$\{[^{}]+\}")
|
||||
RICH_TAG_RE = re.compile(
|
||||
r"<\s*(/?)\s*(html|head|body|p|span|strong|b|i|u|br|hr|h[1-6]|a|table|tr|td|ul|ol|li)\b[^>]*>",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
CONTEXT_RE = re.compile(r"<context>.*?</context>", re.DOTALL)
|
||||
MESSAGE_RE = re.compile(r"<message(?:\s[^>]*)?>.*?</message>", re.DOTALL)
|
||||
TRANSLATION_RE = re.compile(
|
||||
r"<translation(?P<attrs>[^>]*)>.*?</translation>|<translation(?P<self_attrs>[^>]*)/>", re.DOTALL
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class MessageIdentity:
|
||||
context: str
|
||||
source: str
|
||||
comment: str
|
||||
extracomment: str
|
||||
numerus: bool
|
||||
occurrence: int
|
||||
|
||||
def as_payload(self) -> dict[str, object]:
|
||||
return {
|
||||
"context": self.context,
|
||||
"source": self.source,
|
||||
"comment": self.comment,
|
||||
"extracomment": self.extracomment,
|
||||
"numerus": self.numerus,
|
||||
"occurrence": self.occurrence,
|
||||
}
|
||||
|
||||
@property
|
||||
def identifier(self) -> str:
|
||||
encoded = json.dumps(self.as_payload(), ensure_ascii=False, sort_keys=True, separators=(",", ":"))
|
||||
return hashlib.sha256(encoded.encode("utf-8")).hexdigest()[:20]
|
||||
|
||||
|
||||
@dataclass
|
||||
class CatalogMessage:
|
||||
identity: MessageIdentity
|
||||
translation_type: str
|
||||
translation: str | None
|
||||
plural_translations: list[str]
|
||||
locations: list[dict[str, str]]
|
||||
catalog_line: int | None = None
|
||||
|
||||
@property
|
||||
def identifier(self) -> str:
|
||||
return self.identity.identifier
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class RawMessageSpan:
|
||||
identifier: str
|
||||
start: int
|
||||
end: int
|
||||
block: str
|
||||
translation_type: str
|
||||
numerus: bool
|
||||
catalog_line: int
|
||||
|
||||
|
||||
def element_text(element: ET.Element | None) -> str:
|
||||
return "" if element is None else "".join(element.itertext())
|
||||
|
||||
|
||||
def identity_base(context: str, message: ET.Element) -> tuple[str, str, str, str, bool]:
|
||||
return (
|
||||
context,
|
||||
message.findtext("source") or "",
|
||||
message.findtext("comment") or "",
|
||||
message.findtext("extracomment") or "",
|
||||
message.get("numerus") == "yes",
|
||||
)
|
||||
|
||||
|
||||
def iter_catalog_messages(root: ET.Element) -> Iterator[CatalogMessage]:
|
||||
for context_element in root.findall("context"):
|
||||
context = context_element.findtext("name") or ""
|
||||
occurrences: collections.Counter[tuple[str, str, str, str, bool]] = collections.Counter()
|
||||
for message in context_element.findall("message"):
|
||||
base = identity_base(context, message)
|
||||
occurrence = occurrences[base]
|
||||
occurrences[base] += 1
|
||||
identity = MessageIdentity(*base, occurrence)
|
||||
translation_element = message.find("translation")
|
||||
translation_type = "missing" if translation_element is None else translation_element.get("type", "finished")
|
||||
plural_elements = [] if translation_element is None else translation_element.findall("numerusform")
|
||||
plural_translations = [element_text(form) for form in plural_elements]
|
||||
translation = None if translation_element is None or plural_elements else element_text(translation_element)
|
||||
locations = [dict(location.attrib) for location in message.findall("location")]
|
||||
yield CatalogMessage(
|
||||
identity=identity,
|
||||
translation_type=translation_type,
|
||||
translation=translation,
|
||||
plural_translations=plural_translations,
|
||||
locations=locations,
|
||||
)
|
||||
|
||||
|
||||
def parse_catalog(path: Path | str) -> tuple[ET.ElementTree, list[CatalogMessage]]:
|
||||
tree = ET.parse(path)
|
||||
messages = list(iter_catalog_messages(tree.getroot()))
|
||||
with Path(path).open("r", encoding="utf-8", newline="") as stream:
|
||||
spans = scan_raw_messages(stream.read())
|
||||
for message in messages:
|
||||
span = spans.get(message.identifier)
|
||||
if span is not None:
|
||||
message.catalog_line = span.catalog_line
|
||||
return tree, messages
|
||||
|
||||
|
||||
def catalog_fingerprint(messages: Iterable[CatalogMessage]) -> str:
|
||||
identities = [message.identity.as_payload() for message in messages]
|
||||
encoded = json.dumps(identities, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
|
||||
return hashlib.sha256(encoded.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def extract_placeholders(text: str) -> collections.Counter[str]:
|
||||
tokens: list[str] = []
|
||||
occupied: list[tuple[int, int]] = []
|
||||
for kind, regex in (
|
||||
("template", TEMPLATE_PLACEHOLDER_RE),
|
||||
("fmt", FMT_PLACEHOLDER_RE),
|
||||
("qt", QT_PLACEHOLDER_RE),
|
||||
("printf", PRINTF_PLACEHOLDER_RE),
|
||||
):
|
||||
for match in regex.finditer(text):
|
||||
if any(match.start() < end and match.end() > start for start, end in occupied):
|
||||
continue
|
||||
occupied.append(match.span())
|
||||
tokens.append(f"{kind}:{match.group(0)}")
|
||||
return collections.Counter(tokens)
|
||||
|
||||
|
||||
def placeholders_match(source: str, translation: str) -> bool:
|
||||
source_tokens = extract_placeholders(source)
|
||||
translated_tokens = extract_placeholders(translation)
|
||||
if source_tokens == translated_tokens:
|
||||
return True
|
||||
|
||||
source_fmt = collections.Counter(
|
||||
token.removeprefix("fmt:") for token in source_tokens.elements() if token.startswith("fmt:")
|
||||
)
|
||||
translated_fmt = collections.Counter(
|
||||
token.removeprefix("fmt:") for token in translated_tokens.elements() if token.startswith("fmt:")
|
||||
)
|
||||
source_non_fmt = collections.Counter(
|
||||
token for token in source_tokens.elements() if not token.startswith("fmt:")
|
||||
)
|
||||
translated_non_fmt = collections.Counter(
|
||||
token for token in translated_tokens.elements() if not token.startswith("fmt:")
|
||||
)
|
||||
if source_non_fmt != translated_non_fmt:
|
||||
return False
|
||||
|
||||
# Qt's fmt usage permits translators to number anonymous plain placeholders
|
||||
# when reordering arguments, e.g. "{} {}" -> "{1} {0}".
|
||||
if set(source_fmt) == {"{}"}:
|
||||
expected = collections.Counter(f"{{{index}}}" for index in range(source_fmt["{}"]))
|
||||
return translated_fmt == expected
|
||||
return False
|
||||
|
||||
|
||||
def placeholders_are_subset(source: str, translation: str) -> bool:
|
||||
"""Allow a plural form to omit redundant arguments without adding any."""
|
||||
if placeholders_match(source, translation):
|
||||
return True
|
||||
source_tokens = extract_placeholders(source)
|
||||
translated_tokens = extract_placeholders(translation)
|
||||
if translated_tokens <= source_tokens:
|
||||
return True
|
||||
|
||||
source_fmt = collections.Counter(
|
||||
token.removeprefix("fmt:") for token in source_tokens.elements() if token.startswith("fmt:")
|
||||
)
|
||||
translated_fmt = collections.Counter(
|
||||
token.removeprefix("fmt:") for token in translated_tokens.elements() if token.startswith("fmt:")
|
||||
)
|
||||
source_non_fmt = collections.Counter(
|
||||
token for token in source_tokens.elements() if not token.startswith("fmt:")
|
||||
)
|
||||
translated_non_fmt = collections.Counter(
|
||||
token for token in translated_tokens.elements() if not token.startswith("fmt:")
|
||||
)
|
||||
if not translated_non_fmt <= source_non_fmt or set(source_fmt) != {"{}"}:
|
||||
return False
|
||||
allowed_numbered = {f"{{{index}}}" for index in range(source_fmt["{}"])}
|
||||
return set(translated_fmt) <= allowed_numbered and all(count == 1 for count in translated_fmt.values())
|
||||
|
||||
|
||||
def extract_rich_tags(text: str) -> collections.Counter[str]:
|
||||
tags: list[str] = []
|
||||
for match in RICH_TAG_RE.finditer(text):
|
||||
closing = "/" if match.group(1) else ""
|
||||
tags.append(f"{closing}{match.group(2).lower()}")
|
||||
return collections.Counter(tags)
|
||||
|
||||
|
||||
def validate_translation(source: str, translation: str, allow_missing_placeholders: bool = False) -> list[str]:
|
||||
problems: list[str] = []
|
||||
source_placeholders = extract_placeholders(source)
|
||||
translated_placeholders = extract_placeholders(translation)
|
||||
placeholders_valid = (
|
||||
placeholders_are_subset(source, translation)
|
||||
if allow_missing_placeholders
|
||||
else placeholders_match(source, translation)
|
||||
)
|
||||
if not placeholders_valid:
|
||||
problems.append(
|
||||
f"placeholder mismatch: source={dict(source_placeholders)} translation={dict(translated_placeholders)}"
|
||||
)
|
||||
required_tags = extract_rich_tags(source)
|
||||
translated_tags = extract_rich_tags(translation)
|
||||
missing_tags = required_tags - translated_tags
|
||||
if missing_tags:
|
||||
problems.append(f"missing rich-text tags: {dict(missing_tags)}")
|
||||
return problems
|
||||
|
||||
|
||||
def extra_rich_tags(source: str, translation: str) -> collections.Counter[str]:
|
||||
return extract_rich_tags(translation) - extract_rich_tags(source)
|
||||
|
||||
|
||||
def message_to_batch_record(message: CatalogMessage) -> dict[str, object]:
|
||||
return {
|
||||
"record_type": "message",
|
||||
"id": message.identifier,
|
||||
**message.identity.as_payload(),
|
||||
"locations": message.locations,
|
||||
"current_type": message.translation_type,
|
||||
"current_translation": message.translation,
|
||||
"current_plural_translations": message.plural_translations,
|
||||
"accept_current": False,
|
||||
"target_translation": None,
|
||||
"target_plural_translations": None,
|
||||
"suggestions": [],
|
||||
}
|
||||
|
||||
|
||||
def load_jsonl(paths: Sequence[Path | str]) -> tuple[list[dict[str, object]], list[dict[str, object]]]:
|
||||
metadata: list[dict[str, object]] = []
|
||||
records: list[dict[str, object]] = []
|
||||
for path_value in paths:
|
||||
path = Path(path_value)
|
||||
with path.open("r", encoding="utf-8") as stream:
|
||||
for line_number, line in enumerate(stream, 1):
|
||||
if not line.strip():
|
||||
continue
|
||||
try:
|
||||
item = json.loads(line)
|
||||
except json.JSONDecodeError as error:
|
||||
raise ValueError(f"{path}:{line_number}: invalid JSON: {error}") from error
|
||||
if item.get("record_type") == "metadata":
|
||||
metadata.append(item)
|
||||
elif item.get("record_type") == "message":
|
||||
records.append(item)
|
||||
else:
|
||||
raise ValueError(f"{path}:{line_number}: unknown record_type")
|
||||
return metadata, records
|
||||
|
||||
|
||||
def write_jsonl(path: Path, metadata: dict[str, object], records: Sequence[dict[str, object]]) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with path.open("w", encoding="utf-8", newline="\n") as stream:
|
||||
stream.write(json.dumps(metadata, ensure_ascii=False, sort_keys=True) + "\n")
|
||||
for record in records:
|
||||
stream.write(json.dumps(record, ensure_ascii=False, sort_keys=True) + "\n")
|
||||
|
||||
|
||||
def scan_raw_messages(text: str) -> dict[str, RawMessageSpan]:
|
||||
spans: dict[str, RawMessageSpan] = {}
|
||||
newline_offsets = [index for index, character in enumerate(text) if character == "\n"]
|
||||
for context_match in CONTEXT_RE.finditer(text):
|
||||
context_block = context_match.group(0)
|
||||
context_name_match = re.search(r"<name>(.*?)</name>", context_block, re.DOTALL)
|
||||
if context_name_match is None:
|
||||
continue
|
||||
context_name = element_text(ET.fromstring(f"<name>{context_name_match.group(1)}</name>"))
|
||||
occurrences: collections.Counter[tuple[str, str, str, str, bool]] = collections.Counter()
|
||||
for message_match in MESSAGE_RE.finditer(context_block):
|
||||
block = message_match.group(0)
|
||||
message_element = ET.fromstring(block)
|
||||
base = identity_base(context_name, message_element)
|
||||
occurrence = occurrences[base]
|
||||
occurrences[base] += 1
|
||||
identity = MessageIdentity(*base, occurrence)
|
||||
translation = message_element.find("translation")
|
||||
translation_type = "missing" if translation is None else translation.get("type", "finished")
|
||||
absolute_start = context_match.start() + message_match.start()
|
||||
absolute_end = context_match.start() + message_match.end()
|
||||
translation_match = TRANSLATION_RE.search(block)
|
||||
diagnostic_offset = absolute_start + (translation_match.start() if translation_match else 0)
|
||||
span = RawMessageSpan(
|
||||
identifier=identity.identifier,
|
||||
start=absolute_start,
|
||||
end=absolute_end,
|
||||
block=block,
|
||||
translation_type=translation_type,
|
||||
numerus=message_element.get("numerus") == "yes",
|
||||
catalog_line=bisect.bisect_left(newline_offsets, diagnostic_offset) + 1,
|
||||
)
|
||||
if span.identifier in spans:
|
||||
raise ValueError(f"duplicate stable message id: {span.identifier}")
|
||||
spans[span.identifier] = span
|
||||
return spans
|
||||
|
||||
|
||||
def xml_escape_text(text: str) -> str:
|
||||
return text.replace("&", "&").replace("<", "<").replace(">", ">")
|
||||
|
||||
|
||||
def remove_unfinished_attribute(attributes: str) -> str:
|
||||
attributes = re.sub(r"\s+type=(?:\"unfinished\"|'unfinished')", "", attributes)
|
||||
return attributes.rstrip()
|
||||
|
||||
|
||||
def replace_translation_node(block: str, singular: str | None, plurals: Sequence[str] | None) -> str:
|
||||
match = TRANSLATION_RE.search(block)
|
||||
if match is None:
|
||||
raise ValueError("message has no translation element")
|
||||
attributes = remove_unfinished_attribute(match.group("attrs") or match.group("self_attrs") or "")
|
||||
line_start = block.rfind("\n", 0, match.start()) + 1
|
||||
indent = block[line_start:match.start()]
|
||||
if plurals is not None:
|
||||
forms = "\n".join(f"{indent} <numerusform>{xml_escape_text(value)}</numerusform>" for value in plurals)
|
||||
replacement = f"<translation{attributes}>\n{forms}\n{indent}</translation>"
|
||||
else:
|
||||
if singular is None:
|
||||
raise ValueError("missing singular target translation")
|
||||
replacement = f"<translation{attributes}>{xml_escape_text(singular)}</translation>"
|
||||
return block[: match.start()] + replacement + block[match.end() :]
|
||||
@@ -0,0 +1,142 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Validate Qt TS structure, completeness, placeholders, plurals, and rich text."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import collections
|
||||
import xml.etree.ElementTree as ET
|
||||
from pathlib import Path
|
||||
|
||||
if __package__ in (None, ""):
|
||||
import sys
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
|
||||
from translation.ts_utils import ( # noqa: E402
|
||||
SKIPPED_TYPES,
|
||||
CatalogMessage,
|
||||
extra_rich_tags,
|
||||
load_jsonl,
|
||||
parse_catalog,
|
||||
placeholders_match,
|
||||
validate_translation,
|
||||
)
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("catalog", type=Path, help="Qt Linguist .ts catalog")
|
||||
parser.add_argument("--require-complete", action="store_true", help="fail on unfinished or empty active messages")
|
||||
parser.add_argument("--records", nargs="+", type=Path, help="validate only IDs contained in JSONL batches")
|
||||
parser.add_argument(
|
||||
"--placeholders-only",
|
||||
action="store_true",
|
||||
help="check placeholders only, allowing incomplete translations",
|
||||
)
|
||||
parser.add_argument("--strict-extra-tags", action="store_true", help="treat added rich-text tags as errors")
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def values_for(message: CatalogMessage) -> list[str]:
|
||||
return message.plural_translations if message.plural_translations else [message.translation or ""]
|
||||
|
||||
|
||||
def diagnostic_label(catalog: Path, message: CatalogMessage) -> str:
|
||||
line = message.catalog_line or 1
|
||||
label = f"{catalog}:{line}: {message.identity.context}/{message.identifier}"
|
||||
source_locations = []
|
||||
for location in message.locations:
|
||||
filename = location.get("filename", "")
|
||||
source_line = location.get("line", "")
|
||||
if filename and source_line:
|
||||
source_locations.append(f"{filename}:{source_line}")
|
||||
elif filename:
|
||||
source_locations.append(filename)
|
||||
if source_locations:
|
||||
label += f" [source: {', '.join(source_locations)}]"
|
||||
return label
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = parse_args()
|
||||
try:
|
||||
_, messages = parse_catalog(args.catalog)
|
||||
except ET.ParseError as error:
|
||||
line, column = error.position
|
||||
print(f"ERROR: {args.catalog}:{line}:{column + 1}: XML parse error: {error}")
|
||||
return 1
|
||||
except (OSError, ValueError) as error:
|
||||
print(f"ERROR: {args.catalog}:1: unable to read catalog: {error}")
|
||||
return 1
|
||||
|
||||
selected_ids: set[str] | None = None
|
||||
if args.records:
|
||||
_, records = load_jsonl(args.records)
|
||||
selected_ids = {str(record.get("id", "")) for record in records}
|
||||
|
||||
errors: list[str] = []
|
||||
warnings: list[str] = []
|
||||
checked = 0
|
||||
type_counts: collections.Counter[str] = collections.Counter()
|
||||
for message in messages:
|
||||
type_counts[message.translation_type] += 1
|
||||
if message.translation_type in SKIPPED_TYPES:
|
||||
continue
|
||||
if selected_ids is not None and message.identifier not in selected_ids:
|
||||
continue
|
||||
checked += 1
|
||||
label = diagnostic_label(args.catalog, message)
|
||||
if not args.placeholders_only:
|
||||
if message.identity.numerus and not message.plural_translations:
|
||||
errors.append(f"{label}: numerus message has no <numerusform> forms")
|
||||
if not message.identity.numerus and message.plural_translations:
|
||||
errors.append(f"{label}: singular message unexpectedly has plural forms")
|
||||
if args.require_complete and message.translation_type == "unfinished":
|
||||
errors.append(f"{label}: active translation is unfinished")
|
||||
if args.require_complete and not any(value.strip() for value in values_for(message)):
|
||||
errors.append(f"{label}: active translation is empty")
|
||||
|
||||
for value in values_for(message):
|
||||
if not value.strip() and not args.require_complete:
|
||||
continue
|
||||
problems = validate_translation(
|
||||
message.identity.source,
|
||||
value,
|
||||
allow_missing_placeholders=message.identity.numerus,
|
||||
)
|
||||
for problem in problems:
|
||||
if args.placeholders_only and not problem.startswith("placeholder mismatch"):
|
||||
continue
|
||||
errors.append(f"{label}: {problem}")
|
||||
if not args.placeholders_only:
|
||||
extras = extra_rich_tags(message.identity.source, value)
|
||||
if extras:
|
||||
output = errors if args.strict_extra_tags else warnings
|
||||
output.append(f"{label}: additional rich-text tags: {dict(extras)}")
|
||||
if (
|
||||
message.identity.numerus
|
||||
and any(value.strip() for value in values_for(message))
|
||||
and not any(placeholders_match(message.identity.source, value) for value in values_for(message))
|
||||
):
|
||||
errors.append(f"{label}: no plural form preserves the complete source placeholder set")
|
||||
|
||||
if selected_ids is not None:
|
||||
found = {message.identifier for message in messages}
|
||||
for missing in sorted(selected_ids - found):
|
||||
errors.append(f"{args.catalog}:1: {missing}: batch record not found in catalog")
|
||||
|
||||
print(f"Checked {checked} active message(s). Translation types: {dict(type_counts)}")
|
||||
for warning in warnings:
|
||||
print(f"WARNING: {warning}")
|
||||
for error in errors:
|
||||
print(f"ERROR: {error}")
|
||||
if errors:
|
||||
print(f"Validation failed with {len(errors)} error(s) and {len(warnings)} warning(s).")
|
||||
return 1
|
||||
print(f"Validation passed with {len(warnings)} warning(s).")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -1,181 +0,0 @@
|
||||
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Qt Translation Placeholder Verifier
|
||||
|
||||
This script verifies that placeholders in Qt translation files are consistent
|
||||
between source and translation strings.
|
||||
|
||||
Placeholder rules:
|
||||
- {} placeholders: translation must have same total count as source
|
||||
- {n} placeholders: translation must use same numbers as source (can repeat)
|
||||
"""
|
||||
|
||||
import xml.etree.ElementTree as ET
|
||||
import re
|
||||
import sys
|
||||
from typing import List, Set, Tuple, Dict
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def extract_placeholders(text: str) -> Tuple[int, Set[int]]:
|
||||
"""
|
||||
Extract placeholder information from a string.
|
||||
|
||||
Returns:
|
||||
Tuple of (unnamed_count, set_of_numbered_placeholders)
|
||||
"""
|
||||
# Find all placeholders
|
||||
placeholders = re.findall(r'\{(\d*)\}', text)
|
||||
|
||||
unnamed_count = 0
|
||||
numbered_set = set()
|
||||
|
||||
for placeholder in placeholders:
|
||||
if placeholder == '':
|
||||
unnamed_count += 1
|
||||
else:
|
||||
numbered_set.add(int(placeholder))
|
||||
|
||||
return unnamed_count, numbered_set
|
||||
|
||||
|
||||
def verify_placeholders(source: str, translation: str) -> Tuple[bool, str]:
|
||||
"""
|
||||
Verify that placeholders in source and translation are consistent.
|
||||
|
||||
Returns:
|
||||
Tuple of (is_valid, error_message)
|
||||
"""
|
||||
if not translation.strip():
|
||||
return True, "" # Empty translations are valid
|
||||
|
||||
source_unnamed, source_numbered = extract_placeholders(source)
|
||||
trans_unnamed, trans_numbered = extract_placeholders(translation)
|
||||
|
||||
# Check if mixing numbered and unnumbered placeholders
|
||||
if source_unnamed > 0 and len(source_numbered) > 0:
|
||||
return False, "Source mixes numbered and unnumbered placeholders"
|
||||
|
||||
if trans_unnamed > 0 and len(trans_numbered) > 0:
|
||||
return False, "Translation mixes numbered and unnumbered placeholders"
|
||||
|
||||
# If source uses unnumbered placeholders
|
||||
if source_unnamed > 0:
|
||||
# Translation can use either unnumbered or numbered placeholders
|
||||
if trans_unnamed > 0:
|
||||
# Both use unnumbered - simple count match
|
||||
if source_unnamed != trans_unnamed:
|
||||
return False, f"Placeholder count mismatch: source has {source_unnamed}, translation has {trans_unnamed}"
|
||||
elif len(trans_numbered) > 0:
|
||||
# Source uses unnumbered, translation uses numbered
|
||||
# Check that numbered placeholders are 0-based and consecutive up to source count
|
||||
expected_numbers = set(range(source_unnamed))
|
||||
if trans_numbered != expected_numbers:
|
||||
if max(trans_numbered) >= source_unnamed:
|
||||
return False, f"Numbered placeholders exceed source count: translation uses {{{max(trans_numbered)}}}, but source only has {source_unnamed} placeholders"
|
||||
elif min(trans_numbered) != 0:
|
||||
return False, f"Numbered placeholders must start from 0: found {{{min(trans_numbered)}}}"
|
||||
else:
|
||||
missing = expected_numbers - trans_numbered
|
||||
return False, f"Missing numbered placeholders: {{{','.join(map(str, sorted(missing)))}}}"
|
||||
|
||||
# If source uses numbered placeholders
|
||||
elif len(source_numbered) > 0:
|
||||
if trans_unnamed > 0:
|
||||
return False, "Source uses numbered {n} but translation uses unnumbered {}"
|
||||
if source_numbered != trans_numbered:
|
||||
missing = source_numbered - trans_numbered
|
||||
extra = trans_numbered - source_numbered
|
||||
error_parts = []
|
||||
if missing:
|
||||
error_parts.append(f"missing {{{','.join(map(str, sorted(missing)))}}}")
|
||||
if extra:
|
||||
error_parts.append(f"extra {{{','.join(map(str, sorted(extra)))}}}")
|
||||
return False, f"Numbered placeholder mismatch: {', '.join(error_parts)}"
|
||||
|
||||
# If translation has placeholders but source doesn't
|
||||
elif trans_unnamed > 0 or len(trans_numbered) > 0:
|
||||
return False, "Translation has placeholders but source doesn't"
|
||||
|
||||
return True, ""
|
||||
|
||||
|
||||
def verify_translation_file(file_path: str) -> List[Dict]:
|
||||
"""
|
||||
Verify all translations in a Qt translation file.
|
||||
|
||||
Returns:
|
||||
List of error dictionaries with details about invalid translations
|
||||
"""
|
||||
try:
|
||||
tree = ET.parse(file_path)
|
||||
root = tree.getroot()
|
||||
except ET.ParseError as e:
|
||||
return [{"error": f"XML parsing error: {e}", "line": None}]
|
||||
except FileNotFoundError:
|
||||
return [{"error": f"File not found: {file_path}", "line": None}]
|
||||
|
||||
errors = []
|
||||
|
||||
# Find all message elements
|
||||
for message in root.findall('.//message'):
|
||||
source_elem = message.find('source')
|
||||
translation_elem = message.find('translation')
|
||||
location_elem = message.find('location')
|
||||
|
||||
if source_elem is None or translation_elem is None:
|
||||
continue
|
||||
|
||||
source_text = source_elem.text or ""
|
||||
translation_text = translation_elem.text or ""
|
||||
|
||||
is_valid, error_msg = verify_placeholders(source_text, translation_text)
|
||||
|
||||
if not is_valid:
|
||||
error_info = {
|
||||
"source": source_text,
|
||||
"translation": translation_text,
|
||||
"error": error_msg,
|
||||
"line": location_elem.get('line') if location_elem is not None else None,
|
||||
"filename": location_elem.get('filename') if location_elem is not None else None
|
||||
}
|
||||
errors.append(error_info)
|
||||
|
||||
return errors
|
||||
|
||||
|
||||
def main():
|
||||
if len(sys.argv) != 2:
|
||||
print(f"Usage: {sys.argv[0]} <translation_file.ts>")
|
||||
sys.exit(1)
|
||||
|
||||
file_path = sys.argv[1]
|
||||
|
||||
if not Path(file_path).exists():
|
||||
print(f"Error: File '{file_path}' not found")
|
||||
sys.exit(1)
|
||||
|
||||
print(f"Verifying placeholders in: {file_path}")
|
||||
|
||||
errors = verify_translation_file(file_path)
|
||||
|
||||
if not errors:
|
||||
print("All placeholders are valid.")
|
||||
sys.exit(0)
|
||||
|
||||
print(f"Found {len(errors)} placeholder errors:")
|
||||
|
||||
for i, error in enumerate(errors, 1):
|
||||
print(f"\nError {i}:")
|
||||
if error.get('filename') and error.get('line'):
|
||||
print(f" Location: {error['filename']}:{error['line']}")
|
||||
print(f" Source: {error['source']}")
|
||||
print(f" Translation: {error['translation']}")
|
||||
print(f" Issue: {error['error']}")
|
||||
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user