diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..adcd606 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,45 @@ +name: CI + +on: + push: + pull_request: + +permissions: + contents: read + +jobs: + test: + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, windows-latest] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + cache: pip + cache-dependency-path: | + unity-text-locator/requirements.txt + ainiee-translate/requirements.txt + - name: Install runtime dependencies + run: >- + python -m pip install + -r unity-text-locator/requirements.txt + -r ainiee-translate/requirements.txt + - name: Compile Python sources + run: python -m compileall -q unity-text-locator/scripts ainiee-translate/scripts + - name: Run regression tests + run: python -m unittest discover -s tests -v + - name: Smoke-test Unity command help + shell: python + run: | + import pathlib + import subprocess + import sys + + scripts = sorted(pathlib.Path("unity-text-locator/scripts").glob("*.py")) + for script in scripts: + subprocess.run([sys.executable, str(script), "--help"], check=True, stdout=subprocess.DEVNULL) + print(f"validated {len(scripts)} command entry points") diff --git a/NOTICE b/NOTICE index 722456f..67fbdae 100644 --- a/NOTICE +++ b/NOTICE @@ -25,6 +25,9 @@ The detailed vendored notice is kept at: Font assets ----------- -The `arialuni_sdf_u2019` TMP font asset is not committed to this repository. -If distributed, it should be attached separately as a GitHub Release asset and -its redistribution terms should be verified separately from this code license. +No third-party font binary is covered by this repository's AGPL-3.0 license. +`arialuni_sdf_u2019` must not be committed or published as a Release asset +unless the distributor has explicit redistribution rights for that derivative. +Noto-derived release packs may be distributed only with the applicable SIL +OFL 1.1 license, copyright/source records, modification disclosure, hashes, +and build metadata described in `docs/release-assets.md`. diff --git a/README.md b/README.md index e93abe2..3e76c95 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,8 @@ ```powershell $repo = "path\to\unity-text-locator" -$skills = Join-Path ($env:CODEX_HOME ?? (Join-Path $HOME ".codex")) "skills" +$codexHome = if ($env:CODEX_HOME) { $env:CODEX_HOME } else { Join-Path $HOME ".codex" } +$skills = Join-Path $codexHome "skills" Copy-Item (Join-Path $repo "unity-text-locator") $skills -Recurse -Force Copy-Item (Join-Path $repo "ainiee-translate") $skills -Recurse -Force ``` @@ -25,6 +26,12 @@ py -3.12 -m venv "$HOME\.venvs\ainiee-translate" & "$HOME\.venvs\ainiee-translate\Scripts\python.exe" -m pip install -r "$repo\ainiee-translate\requirements.txt" ``` +Unity 扫描、写回和字体检查还需要安装 Unity 运行依赖: + +```powershell +py -3.12 -m pip install -r "$repo\unity-text-locator\requirements.txt" +``` + ## 工作流 ```text @@ -59,7 +66,7 @@ py -3.12 -m venv "$HOME\.venvs\ainiee-translate" 仓库不捆绑或指定默认字体二进制。字体必须由使用者合法取得,并针对目标游戏验证: ```powershell -python unity-text-locator\scripts\inspect_tmp_font_bundle.py path\to\font.bundle --translation-csv path\to\game_translation.csv +python unity-text-locator\scripts\inspect_tmp_font_bundle.py path\to\font.bundle --translation-root path\to\translation-output ``` 检查结果应同时关注: diff --git a/ainiee-translate/SKILL.md b/ainiee-translate/SKILL.md index 0404b62..df02978 100644 --- a/ainiee-translate/SKILL.md +++ b/ainiee-translate/SKILL.md @@ -1,6 +1,6 @@ --- name: ainiee-translate -description: Use when the user wants to translate a novel/epub end-to-end on the coding agent itself (Claude Code or Codex; no AiNiee app), AiNiee-style — parse, build a locked glossary, translate chapter by chapter following rules (names preserved, title 3-way, term consistency), write back and export. Triggers on "用 agent 翻译这本书", "agent 版 aniee 翻译", "把这本 epub 翻译了" and similar. +description: Translate books, documents, AiNiee projects, or Unity localization caches end-to-end on the coding agent itself without the AiNiee app. Use for requests such as "用 agent 翻译这本书", "把 epub 翻译了", "翻译 Unity CSV/cache", or "AiNiee-style translation" that need a locked glossary, resumable batches, terminology consistency, validated cache writeback, and export. --- # ainiee-translate 技能指南 @@ -18,6 +18,8 @@ parse → 构建锁定词汇表 → 逐章翻译(agent 按规则) → 写回 - 进度状态驱动(`translation_status`):中断后重跑自动从首个未译段继续,天然可恢复。 - 每批写回前自动备份(时间戳):`cache.json.bak.YYYYMMDD_HHMMSS`。 +完整性门禁:写回前要求 `text_index` 唯一,写入使用锁和原子替换;批次存在 unmatched、验证存在 issue、或 Unity cache 缺行/状态未完成时默认退出非零。只有用户明确接受保留未译行时才使用 `--allow-partial`。 + 本技能自带管道脚本(`scripts/ainiee_translate/`),无需单独 `pip install` 本包;解析/导出模块也随技能打包。跨设备复制后,只需要用本目录的 `requirements.txt` 为运行脚本的 Python 安装依赖。 --- diff --git a/ainiee-translate/agents/openai.yaml b/ainiee-translate/agents/openai.yaml new file mode 100644 index 0000000..823a8cb --- /dev/null +++ b/ainiee-translate/agents/openai.yaml @@ -0,0 +1,4 @@ +interface: + display_name: "AiNiee Agent Translation" + short_description: "Translate books and Unity text caches safely." + default_prompt: "Use $ainiee-translate to translate this book or Unity localization cache with a locked glossary, resumable batches, validation, and safe export." diff --git a/ainiee-translate/requirements.txt b/ainiee-translate/requirements.txt index 165bca7..b3b2e3b 100644 --- a/ainiee-translate/requirements.txt +++ b/ainiee-translate/requirements.txt @@ -1,10 +1,10 @@ # Runtime dependencies for ainiee-translate. # Keep this in sync with upstream pyproject.toml dependencies. -msgspec -beautifulsoup4 -lxml -rich -openpyxl -polib -python-pptx -chardet +msgspec>=0.18,<1 +beautifulsoup4>=4.12,<5 +lxml>=4.9,<7 +rich>=13,<15 +openpyxl>=3.1,<4 +polib>=1.2,<2 +python-pptx>=1.0,<2 +chardet>=5,<6 diff --git a/ainiee-translate/scripts/ainiee_translate/batch.py b/ainiee-translate/scripts/ainiee_translate/batch.py index a9ccc89..b381e3e 100644 --- a/ainiee-translate/scripts/ainiee_translate/batch.py +++ b/ainiee-translate/scripts/ainiee_translate/batch.py @@ -66,9 +66,11 @@ def main(argv=None): applied = write_back(a.cache_path, translations) if applied < total: print(f"applied {applied} of {total} translation(s) ({total - applied} unmatched)") + return 1 else: print(f"applied {applied} translation(s)") + return 0 if __name__ == "__main__": - main() + raise SystemExit(main()) diff --git a/ainiee-translate/scripts/ainiee_translate/cache_io.py b/ainiee-translate/scripts/ainiee_translate/cache_io.py index 48d1f9c..336eeb7 100644 --- a/ainiee-translate/scripts/ainiee_translate/cache_io.py +++ b/ainiee-translate/scripts/ainiee_translate/cache_io.py @@ -2,7 +2,12 @@ iterate / mutate items by translation status. The CacheProject / CacheItem types are vendored from AiNiee under _vendor/ — no external AiNiee repo required.""" import json +import os +import tempfile +import time import types +from contextlib import contextmanager +from pathlib import Path import msgspec from . import helpers from ._vendor.ModuleFolders.Service.Cache.CacheProject import CacheProject, CacheProjectStatistics @@ -20,8 +25,46 @@ def save_cache(project, path: str) -> None: # Ensure stats_data is never null (its field type is non-Optional in msgspec eyes) if project.stats_data is None: project.stats_data = CacheProjectStatistics() - with open(path, "wb") as w: - w.write(msgspec.json.encode(project)) + target = Path(path) + target.parent.mkdir(parents=True, exist_ok=True) + payload = msgspec.json.encode(project) + fd, tmp_name = tempfile.mkstemp(prefix=f".{target.name}.", suffix=".tmp", dir=target.parent) + try: + with os.fdopen(fd, "wb") as w: + w.write(payload) + w.flush() + os.fsync(w.fileno()) + os.replace(tmp_name, target) + except BaseException: + try: + os.unlink(tmp_name) + except FileNotFoundError: + pass + raise + + +@contextmanager +def _cache_lock(cache_path: str, timeout: float = 10.0): + """Portable cooperative lock using atomic O_EXCL creation.""" + lock_path = f"{cache_path}.lock" + deadline = time.monotonic() + timeout + while True: + try: + fd = os.open(lock_path, os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o600) + break + except FileExistsError: + if time.monotonic() >= deadline: + raise TimeoutError(f"cache is locked: {cache_path}") + time.sleep(0.05) + try: + with os.fdopen(fd, "w", encoding="ascii") as handle: + handle.write(str(os.getpid())) + yield + finally: + try: + os.unlink(lock_path) + except FileNotFoundError: + pass def load_cache(path: str): @@ -79,11 +122,36 @@ def set_polish(project, text_index: int, polished_text: str) -> bool: def apply_writeback(cache_path: str, items: list[dict], setter, get_text) -> int: """Backup, load, apply `setter(project, text_index, get_text(item))` for each item, save. Shared by the translate (batch) and polish write paths.""" - helpers.backup_file(cache_path) - project = load_cache(cache_path) - applied = 0 - for it in items: - if setter(project, int(it["text_index"]), get_text(it)): - applied += 1 - save_cache(project, cache_path) - return applied + if not isinstance(items, list): + raise ValueError("writeback payload must be a JSON array") + indexes = [] + for position, item in enumerate(items, start=1): + if not isinstance(item, dict) or "text_index" not in item: + raise ValueError(f"writeback item {position} has no text_index") + raw = item["text_index"] + if isinstance(raw, bool): + raise ValueError(f"invalid text_index at item {position}: {raw!r}") + try: + index = int(raw) + except (TypeError, ValueError) as exc: + raise ValueError(f"invalid text_index at item {position}: {raw!r}") from exc + if index <= 0 or str(raw).strip() != str(index): + raise ValueError(f"invalid text_index at item {position}: {raw!r}") + indexes.append(index) + if len(indexes) != len(set(indexes)): + raise ValueError("duplicate text_index in writeback payload") + + with _cache_lock(cache_path): + project = load_cache(cache_path) + cache_indexes = [int(item.text_index) for item in iter_items(project)] + if len(cache_indexes) != len(set(cache_indexes)): + raise ValueError("cache contains duplicate text_index values") + applied = 0 + for item, index in zip(items, indexes): + if setter(project, index, get_text(item)): + applied += 1 + if applied != len(items): + return applied + helpers.backup_file(cache_path) + save_cache(project, cache_path) + return applied diff --git a/ainiee-translate/scripts/ainiee_translate/helpers.py b/ainiee-translate/scripts/ainiee_translate/helpers.py index 3c3f0c0..a637fee 100644 --- a/ainiee-translate/scripts/ainiee_translate/helpers.py +++ b/ainiee-translate/scripts/ainiee_translate/helpers.py @@ -2,15 +2,17 @@ ainiee-cache-fix skill (apostrophe normalization + Latin-only boundaries).""" import re import shutil -import time +from datetime import datetime, timezone +from uuid import uuid4 _APOS = {"'": "'", "ʼ": "'", "'": "'", "’": "'"} _CJK = re.compile(r"[一-鿿]") def backup_file(path: str) -> str: - """Timestamped copy alongside the original: path.bak.YYYYMMDD_HHMMSS.""" - dst = f"{path}.bak.{time.strftime('%Y%m%d_%H%M%S')}" + """Create a collision-resistant backup alongside *path*.""" + stamp = datetime.now(timezone.utc).strftime("%Y%m%d_%H%M%S_%fZ") + dst = f"{path}.bak.{stamp}.{uuid4().hex[:8]}" shutil.copy2(path, dst) return dst diff --git a/ainiee-translate/scripts/ainiee_translate/verify.py b/ainiee-translate/scripts/ainiee_translate/verify.py index c5c2147..7afa497 100644 --- a/ainiee-translate/scripts/ainiee_translate/verify.py +++ b/ainiee-translate/scripts/ainiee_translate/verify.py @@ -72,7 +72,8 @@ def main(argv=None): issues = check_items(_items_for_check(a.cache), locked) print(json.dumps(issues, ensure_ascii=False, indent=2)) print(f"\n{len(issues)} issue(s)") + return 1 if issues else 0 if __name__ == "__main__": - main() + raise SystemExit(main()) diff --git a/docs/install.md b/docs/install.md index 3fce033..be5131f 100644 --- a/docs/install.md +++ b/docs/install.md @@ -12,6 +12,13 @@ unity-text-locator/ 如果需要全自动 Unity 翻译,请同时安装两个 skill 目录。 如果只需要半自动的提取、校验、写回流程,只安装 `unity-text-locator` 即可。 +Unity 资源脚本依赖 Python 3.12 和 `UnityPy`: + +```powershell +$repo = "path\to\unity-text-locator" +py -3.12 -m pip install -r "$repo\unity-text-locator\requirements.txt" +``` + ## 安装到 Codex PowerShell: @@ -43,22 +50,19 @@ $env:AINIEE_PY = "$env:USERPROFILE\.venvs\ainiee-translate\Scripts\python.exe" `AINIEE_REPO` 是可选项,只在 vendored `ainiee-translate` reader 尚不支持某些格式时使用。 -## 字体 Release 资产 +## 字体 -安装 `unity-text-locator` skill 后,把 release 里的 `arialuni_sdf_u2019` 下载到 skill 的 `assets` 目录: +不要从本仓库下载或发布 `arialuni_sdf_u2019`;Arial Unicode MS 及其 SDF 衍生资产需要单独、明确的再分发授权。 -```powershell -$skillDir = "$env:USERPROFILE\.codex\skills\unity-text-locator" -New-Item -ItemType Directory "$skillDir\assets" -Force -Invoke-WebRequest ` - "https://github.com/timeance/unity-text-locator/releases/latest/download/arialuni_sdf_u2019" ` - -OutFile "$skillDir\assets\arialuni_sdf_u2019" -``` - -校验下载结果: +Mono/TMP 游戏优先使用自己合法取得的 Noto CJK/SC、TTF、OTF 或 TTC 文件作为 app-local 运行时 fallback: ```powershell -Get-FileHash "$skillDir\assets\arialuni_sdf_u2019" -Algorithm SHA256 +python unity-text-locator\scripts\install_tmp_chinese_font_fix.py "path\to\GameFolder" ` + --runtime-font-file "path\to\NotoSansCJKsc-Regular.otf" ` + --out-dir "path\to\GameFolder\_translation\unity-text-report\font-fix" ` + --dry-run ``` -不要把字体资产提交进 git;它只需要留在本机的 skill 目录里。 +检查 dry-run 报告后,移除 `--dry-run` 应用。安装器把字体复制到游戏 `_Data/ChineseFontFixer/Fonts/` 私有目录,与 DLL 和注入 JSON 一起备份、提交和失败回滚。报告会记录字体来源、目标、大小与 SHA-256。 + +静态 TMP font bundle 不是通用替代品。只有目标 Unity Editor 和 TMP package 版本精确匹配时才可作为候选,并且必须先做单字体、单界面 canary。发布 Noto 字体包的许可和 manifest 要求见 [release-assets.md](release-assets.md)。 diff --git a/docs/release-assets.md b/docs/release-assets.md index 9cc1454..fcd4f2f 100644 --- a/docs/release-assets.md +++ b/docs/release-assets.md @@ -1,46 +1,62 @@ -# Release 资产 +# 字体 Release 资产 -## arialuni_sdf_u2019 +字体不是通用的“修复文件”。运行时字体与静态 TMP FontAsset 应分别发布和验证。 -`arialuni_sdf_u2019` 是给 TMP 字体替换流程使用的本地资产。它不提交到 git,用户从 GitHub Release 下载后放进 `unity-text-locator` skill 的 `assets` 目录。 +## 允许发布的资产 -安装后的推荐路径: +仓库可以发布基于 Noto CJK/SC 制作的字体包,但发布者必须核对所用字体版本的 SIL Open Font License 1.1,并在压缩包中保留许可证、版权声明、来源和修改/子集化说明,同时遵守 Reserved Font Name 等适用条款。不得仅凭字体名称推断许可。 + +建议结构: ```text -%USERPROFILE%\.codex\skills\unity-text-locator\assets\arialuni_sdf_u2019 +noto-cjk-sc-font-pack-v1.zip + LICENSES/OFL-1.1.txt + font-manifest.json + runtime/NotoSansCJKsc-Regular.otf + tmp/unity-6000.0.x-tmp-x.y.z/font.bundle + SHA256SUMS ``` -准备本仓库时使用的本地参考资产信息: +运行时字体供 Mono 注入方案使用,通常比静态 TMP 包通用。静态包必须按精确 Unity Editor 与 TMP package 版本隔离;`NotoSansCJK_sdf_unity6000` 之类的名称只能表示候选构建版本,不能证明与所有 Unity 6 游戏兼容。 -```text -file: arialuni_sdf_u2019 -size: 30986431 bytes -sha256: 11B47CAE3262648DD9C8B8A29DC25D04309A18790E4130E94FD230791E55C037 -``` +## 禁止默认发布的资产 -release 结构: +`arialuni_sdf_u2019` 源自 Arial Unicode MS。它是专有字体的衍生资源,转换成 SDF、atlas 或 AssetBundle 不会自动产生再分发权。除非仓库维护者持有并公开记录涵盖该二进制资产的明确再分发授权,否则: -```text -Release vX.Y.Z - unity-text-locator source archive - arialuni_sdf_u2019 -``` +- 不提交到 git; +- 不附加到 GitHub Release; +- 不提供自动下载链接; +- 仅允许用户从其合法取得的本地字体自行生成或指定。 -下载到 skill 目录后的校验命令: +代码仓库的 AGPL-3.0 许可证不覆盖第三方字体。 -```powershell -Get-FileHash "$env:USERPROFILE\.codex\skills\unity-text-locator\assets\arialuni_sdf_u2019" -Algorithm SHA256 -``` +## `font-manifest.json` -哈希应匹配: +每个发布字体包至少记录: -```text -11B47CAE3262648DD9C8B8A29DC25D04309A18790E4130E94FD230791E55C037 +```json +{ + "schema_version": 1, + "font_family": "Noto Sans CJK SC", + "font_version": "verified upstream version", + "source_url": "official upstream URL", + "source_sha256": "...", + "license": "OFL-1.1", + "modified_or_subsetted": false, + "unity_editor_version": "6000.0.x", + "tmp_package_version": "x.y.z", + "character_set_sha256": "...", + "atlas": {"width": 4096, "height": 4096, "format": "...", "population_mode": "Static"}, + "files": [{"path": "...", "sha256": "...", "size": 0}] +} ``` -## 发布策略 +运行时字体包可将 Unity/TMP/atlas 字段设为 `null`;静态 TMP 包不得省略这些字段。 + +## 使用门槛 -- `arialuni_sdf_u2019` 不进入 git 历史。 -- release notes 说明它是单独附件,不属于源码许可证覆盖范围。 -- 用户下载后放进 `unity-text-locator/assets/`。 -- 对无法安全替换 TMP 资产的游戏,运行时字体 fallback 仍作为兼容方案保留。 +1. 先检测目标游戏 Unity 版本、Mono/IL2CPP、TMP 版本和实际缺字集合。 +2. Mono 注入可用时,优先使用 `--runtime-font-file` 安装 app-local Noto 字体。 +3. 静态 TMP 替换只允许精确版本候选,并先做单字体、单界面 canary。 +4. 校验 manifest 与每个文件的 SHA-256;哈希只证明身份,不证明授权或运行时兼容。 +5. 出现 atlas 碎片、材质异常、版本未知或加载错误时立即回滚。 diff --git a/tests/test_ainiee_integrity.py b/tests/test_ainiee_integrity.py new file mode 100644 index 0000000..47872d4 --- /dev/null +++ b/tests/test_ainiee_integrity.py @@ -0,0 +1,138 @@ +import csv +import importlib.util +import json +import os +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +AINIEE_SCRIPTS = ROOT / "ainiee-translate" / "scripts" +TO_CACHE = ROOT / "unity-text-locator" / "scripts" / "unity_csv_to_ainiee_cache.py" +FROM_CACHE = ROOT / "unity-text-locator" / "scripts" / "ainiee_cache_to_unity_translation.py" + + +HAS_MSGSPEC = importlib.util.find_spec("msgspec") is not None + + +def _load_script(name, path): + spec = importlib.util.spec_from_file_location(name, path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +class SourceFingerprintTests(unittest.TestCase): + def test_both_bridge_directions_use_the_same_fingerprint(self): + forward = _load_script("unity_to_ainiee", TO_CACHE) + reverse = _load_script("ainiee_to_unity", FROM_CACHE) + rows = ["一", "", "line\nwith newline", "emoji: 😀"] + self.assertEqual(forward.source_rows_sha256(rows), reverse.source_rows_sha256(rows)) + + def test_fingerprint_preserves_row_boundaries(self): + bridge = _load_script("ainiee_to_unity_boundaries", FROM_CACHE) + self.assertNotEqual( + bridge.source_rows_sha256(["ab", "c"]), + bridge.source_rows_sha256(["a", "bc"]), + ) + + +@unittest.skipUnless(HAS_MSGSPEC, "ainiee-translate runtime dependency msgspec is not installed") +class AinieeIntegrityTests(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.work = Path(self.temp.name) + self.source = self.work / "source.csv" + self.cache = self.work / "cache.json" + self.out = self.work / "translated.csv" + self._write_source(["一", "二"]) + self._run(TO_CACHE, "--source-csv", self.source, "--out-cache", self.cache) + + def tearDown(self): + self.temp.cleanup() + + def _write_source(self, rows): + with self.source.open("w", encoding="utf-8-sig", newline="") as handle: + writer = csv.writer(handle, lineterminator="\n") + writer.writerow(["original_flat"]) + writer.writerows([[row] for row in rows]) + + def _run(self, *args, check=True): + env = os.environ.copy() + existing = env.get("PYTHONPATH") + env["PYTHONPATH"] = os.pathsep.join( + [str(AINIEE_SCRIPTS), existing] if existing else [str(AINIEE_SCRIPTS)] + ) + return subprocess.run( + [sys.executable, *map(str, args)], cwd=ROOT, env=env, + capture_output=True, text=True, check=check, + ) + + def _batch_write(self, payload): + translations = self.work / "translations.json" + translations.write_text(json.dumps(payload, ensure_ascii=False), encoding="utf-8") + return self._run( + "-m", "ainiee_translate.batch", "write", self.cache, translations, + check=False, + ) + + def test_duplicate_writeback_indexes_are_rejected_without_mutation(self): + before = self.cache.read_bytes() + result = self._batch_write([ + {"text_index": 1, "translated_text": "甲"}, + {"text_index": 1, "translated_text": "乙"}, + ]) + self.assertNotEqual(result.returncode, 0) + self.assertIn("duplicate text_index", result.stderr) + self.assertEqual(before, self.cache.read_bytes()) + + def test_unmatched_writeback_is_nonzero_and_does_not_save_partial_changes(self): + before = self.cache.read_bytes() + result = self._batch_write([ + {"text_index": 1, "translated_text": "甲"}, + {"text_index": 99, "translated_text": "不存在"}, + ]) + self.assertEqual(result.returncode, 1) + self.assertIn("unmatched", result.stdout) + self.assertEqual(before, self.cache.read_bytes()) + + def test_bridge_requires_complete_translated_cache(self): + result = self._run( + FROM_CACHE, "--cache", self.cache, "--source-csv", self.source, + "--out-csv", self.out, check=False, + ) + self.assertNotEqual(result.returncode, 0) + self.assertIn("cache is incomplete", result.stderr) + self.assertFalse(self.out.exists()) + + def test_bridge_checks_source_content_hash(self): + self.assertEqual(self._batch_write([ + {"text_index": 1, "translated_text": "甲"}, + {"text_index": 2, "translated_text": "乙"}, + ]).returncode, 0) + self._write_source(["一", "已改变"]) + result = self._run( + FROM_CACHE, "--cache", self.cache, "--source-csv", self.source, + "--out-csv", self.out, check=False, + ) + self.assertNotEqual(result.returncode, 0) + self.assertIn("content hash does not match", result.stderr) + + def test_bridge_exports_complete_cache(self): + self.assertEqual(self._batch_write([ + {"text_index": 1, "translated_text": "甲"}, + {"text_index": 2, "translated_text": "乙"}, + ]).returncode, 0) + self._run( + FROM_CACHE, "--cache", self.cache, "--source-csv", self.source, + "--out-csv", self.out, + ) + with self.out.open("r", encoding="utf-8-sig", newline="") as handle: + self.assertEqual(list(csv.reader(handle)), [["zh_cn"], ["甲"], ["乙"]]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_binary_patch_paths.py b/tests/test_binary_patch_paths.py new file mode 100644 index 0000000..8d4fb0e --- /dev/null +++ b/tests/test_binary_patch_paths.py @@ -0,0 +1,162 @@ +import importlib.util +import gzip +import hashlib +import json +import os +import shutil +import subprocess +import tempfile +import unittest +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] + + +def load_script(name: str): + path = ROOT / "unity-text-locator" / "scripts" / name + spec = importlib.util.spec_from_file_location(name.removesuffix(".py"), path) + module = importlib.util.module_from_spec(spec) + assert spec.loader is not None + spec.loader.exec_module(module) + return module + + +APPLY = load_script("apply_binary_patch.py") +BUILD = load_script("build_binary_patch.py") + + +class CheckedRelativeTests(unittest.TestCase): + def test_accepts_nested_relative_path(self): + for module in (APPLY, BUILD): + self.assertEqual(module.checked_relative("Game_Data/resources.assets"), Path("Game_Data/resources.assets")) + + def test_rejects_windows_and_parent_escapes(self): + unsafe = ( + r"C:\outside\file", + r"C:outside\file", + r"\\server\share\file", + r"\rooted\file", + "/absolute/file", + "../outside/file", + "inside/../../outside", + "", + ) + for module in (APPLY, BUILD): + for value in unsafe: + with self.subTest(module=module.__name__, value=value): + with self.assertRaises(SystemExit): + module.checked_relative(value) + + +class ContainmentTests(unittest.TestCase): + def test_normal_child_is_returned(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + child = root / "nested" / "file.bin" + child.parent.mkdir() + child.write_bytes(b"data") + for module in (APPLY, BUILD): + self.assertEqual(module.contained_path(root, Path("nested/file.bin")), child.resolve()) + + def test_absolute_relative_object_cannot_override_root(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) / "root" + outside = Path(directory) / "outside.bin" + root.mkdir() + outside.write_bytes(b"data") + for module in (APPLY, BUILD): + with self.assertRaises(SystemExit): + module.contained_path(root, outside) + + @unittest.skipUnless(hasattr(os, "symlink"), "symlinks unavailable") + def test_symlink_escape_is_rejected(self): + with tempfile.TemporaryDirectory() as directory: + base = Path(directory) + root = base / "root" + outside = base / "outside" + root.mkdir() + outside.mkdir() + (outside / "payload.bin").write_bytes(b"data") + try: + (root / "linked").symlink_to(outside, target_is_directory=True) + except OSError as error: + self.skipTest(f"cannot create symlink: {error}") + for module in (APPLY, BUILD): + with self.assertRaises(SystemExit): + module.contained_path(root, Path("linked/payload.bin")) + + +@unittest.skipUnless(os.name == "nt" and shutil.which("powershell"), "Windows PowerShell unavailable") +class PowerShellInstallerTests(unittest.TestCase): + def run_installer(self, package: Path, game: Path) -> subprocess.CompletedProcess[str]: + template = ROOT / "unity-text-locator" / "assets" / "apply_cn_patch.ps1.txt" + def quote(value: Path) -> str: + return "'" + str(value).replace("'", "''") + "'" + command = ( + "$ErrorActionPreference = 'Stop'; " + f"$code = Get-Content -Raw -LiteralPath {quote(template)}; " + f"& ([ScriptBlock]::Create($code)) -PatchDir {quote(package)} -GameRoot {quote(game)}" + ) + return subprocess.run( + ["powershell", "-NoProfile", "-Command", command], + text=True, + capture_output=True, + check=False, + ) + + def write_manifest(self, package: Path, *, target: str, payload: str, current: bytes = b"old") -> None: + manifest = { + "format": "unity-cn-middle-span-v1", + "payload_compression": "gzip", + "files": [{ + "path": target, + "payload": payload, + "old_sha256": hashlib.sha256(current).hexdigest(), + "new_sha256": hashlib.sha256(b"new").hexdigest(), + "prefix_bytes": 0, + "suffix_bytes": 0, + }], + } + (package / "manifest.json").write_text(json.dumps(manifest), encoding="utf-8") + + def test_rejects_drive_qualified_target(self): + with tempfile.TemporaryDirectory() as directory: + base = Path(directory) + package, game = base / "package", base / "game" + package.mkdir() + game.mkdir() + self.write_manifest(package, target=r"C:outside.bin", payload="payload.bin.gz") + result = self.run_installer(package, game) + self.assertNotEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertIn("Unsafe patch path", result.stdout + result.stderr) + + def test_rejects_payload_parent_escape(self): + with tempfile.TemporaryDirectory() as directory: + base = Path(directory) + package, game = base / "package", base / "game" + package.mkdir() + game.mkdir() + (game / "target.bin").write_bytes(b"old") + (base / "payload.bin.gz").write_bytes(gzip.compress(b"new")) + self.write_manifest(package, target="target.bin", payload="../payload.bin.gz") + result = self.run_installer(package, game) + self.assertNotEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertIn("Unsafe patch path", result.stdout + result.stderr) + + def test_accepts_contained_target_and_payload(self): + with tempfile.TemporaryDirectory() as directory: + base = Path(directory) + package, game = base / "package", base / "game" + (package / "payloads").mkdir(parents=True) + (game / "data").mkdir(parents=True) + (game / "data" / "target.bin").write_bytes(b"old") + (package / "payloads" / "middle.bin.gz").write_bytes(gzip.compress(b"new")) + self.write_manifest(package, target="data/target.bin", payload="payloads/middle.bin.gz") + result = self.run_installer(package, game) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertEqual((game / "data" / "target.bin").read_bytes(), b"new") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_font_fix_installer.py b/tests/test_font_fix_installer.py new file mode 100644 index 0000000..7894edb --- /dev/null +++ b/tests/test_font_fix_installer.py @@ -0,0 +1,82 @@ +from __future__ import annotations + +import importlib.util +import tempfile +import unittest +from pathlib import Path + + +REPO_ROOT = Path(__file__).resolve().parents[1] +INSTALLER = REPO_ROOT / "unity-text-locator" / "scripts" / "install_tmp_chinese_font_fix.py" +FIXER_SOURCE = REPO_ROOT / "unity-text-locator" / "assets" / "ChineseFontFixer.cs" +SPEC = importlib.util.spec_from_file_location("font_fix_installer", INSTALLER) +assert SPEC and SPEC.loader +installer = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(installer) + + +class RuntimeFontTests(unittest.TestCase): + def test_runtime_font_destination_is_game_private(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + source = root / "NotoSansCJKsc-Regular.otf" + source.write_bytes(b"font") + destination = installer.runtime_font_destination(root / "Game_Data", source) + self.assertEqual(destination, root / "Game_Data" / "ChineseFontFixer" / "Fonts" / source.name) + + def test_runtime_font_rejects_non_font_extension(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + source = Path(temporary) / "payload.bin" + source.write_bytes(b"not-font") + with self.assertRaises(SystemExit): + installer.runtime_font_destination(Path(temporary) / "Game_Data", source) + + def test_runtime_font_rejects_empty_font(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + source = Path(temporary) / "empty.ttf" + source.touch() + with self.assertRaises(SystemExit): + installer.runtime_font_destination(Path(temporary) / "Game_Data", source) + + def test_commit_creates_nested_font_directory_and_backup(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) / "Game" + backup = Path(temporary) / "backup" + existing = root / "Game_Data" / "config.json" + existing.parent.mkdir(parents=True) + existing.write_bytes(b"old") + font = root / "Game_Data" / "ChineseFontFixer" / "Fonts" / "Noto.otf" + written, leftovers = installer.commit_outputs( + {existing: b"new", font: b"font"}, root, backup + ) + self.assertEqual(existing.read_bytes(), b"new") + self.assertEqual(font.read_bytes(), b"font") + self.assertEqual((backup / "Game_Data" / "config.json").read_bytes(), b"old") + self.assertIn("Game_Data/ChineseFontFixer/Fonts/Noto.otf", written) + self.assertEqual(leftovers, []) + + def test_commit_rolls_back_font_when_a_later_output_fails(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) / "Game" + backup = Path(temporary) / "backup" + font = root / "Game_Data" / "ChineseFontFixer" / "Fonts" / "Noto.otf" + blocked_parent = root / "blocked" + blocked_parent.parent.mkdir(parents=True) + blocked_parent.write_bytes(b"not-a-directory") + with self.assertRaises(OSError): + installer.commit_outputs( + {font: b"font", blocked_parent / "later.json": b"later"}, root, backup + ) + self.assertFalse(font.exists()) + + def test_runtime_does_not_preflight_new_font_with_has_character(self) -> None: + source = FIXER_SOURCE.read_text(encoding="utf-8") + create_legacy = source.split("private Font CreateLegacyFont()", 1)[1].split( + "private TMP_FontAsset CreateDynamicTMPFontAsset", 1 + )[0] + self.assertNotIn("HasCharacter", create_legacy) + self.assertLess(source.index("privateDirectory"), source.index("SpecialFolder.Fonts")) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_raw_candidate_gate.py b/tests/test_raw_candidate_gate.py new file mode 100644 index 0000000..a38ed16 --- /dev/null +++ b/tests/test_raw_candidate_gate.py @@ -0,0 +1,78 @@ +from __future__ import annotations + +import csv +import importlib.util +import json +import tempfile +import unittest +from pathlib import Path + + +ROOT = Path(__file__).parents[1] +SCRIPT = ROOT / "unity-text-locator" / "scripts" / "extract_unity_text.py" +SPEC = importlib.util.spec_from_file_location("extract_unity_text", SCRIPT) +assert SPEC and SPEC.loader +extractor = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(extractor) + + +class RawCandidateGateTests(unittest.TestCase): + def result(self, root: Path) -> dict[str, object]: + return { + "root": str(root), + "files_scanned": [ + {"path": "Game_Data/resources.assets", "sha256": "abc", "category": "resources.assets"} + ], + "occurrences": [ + { + "source_file": "Game_Data/resources.assets", + "offset_hex": "0x10", + "byte_length": 9, + "method": "raw-utf8-fallback", + "confidence": "medium", + "original_flat": "候補テキスト", + "notes": "raw candidate", + }, + { + "source_file": "Game_Data/resources.assets", + "offset_hex": "0x20", + "byte_length": 12, + "method": "unity-len-utf8", + "confidence": "high", + "original_flat": "表示テキスト", + "notes": "length-prefixed", + }, + ], + } + + def test_raw_candidates_are_audit_only_by_default(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) / "Game" + out = Path(temporary) / "text" + root.mkdir() + extractor.write_source_text_outputs(self.result(root), out) + + audit = out / "raw_utf8_candidates_audit.csv" + self.assertTrue(audit.is_file()) + with audit.open(encoding="utf-8-sig", newline="") as handle: + self.assertEqual([row["original_flat"] for row in csv.DictReader(handle)], ["候補テキスト"]) + + index = json.loads((out / "source_text_index.json").read_text(encoding="utf-8")) + source_csv = out / index[0]["source_csv"] + with source_csv.open(encoding="utf-8-sig", newline="") as handle: + rows = list(csv.DictReader(handle)) + self.assertEqual([row["original_flat"] for row in rows], ["表示テキスト"]) + + def test_explicit_opt_in_exposes_raw_candidates(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) / "Game" + out = Path(temporary) / "text" + root.mkdir() + extractor.write_source_text_outputs(self.result(root), out, include_raw_candidates=True) + index = json.loads((out / "source_text_index.json").read_text(encoding="utf-8")) + self.assertEqual(index[0]["rows"], 2) + self.assertFalse((out / "raw_utf8_candidates_audit.csv").exists()) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_unity_version_detection.py b/tests/test_unity_version_detection.py new file mode 100644 index 0000000..e44fb95 --- /dev/null +++ b/tests/test_unity_version_detection.py @@ -0,0 +1,42 @@ +from __future__ import annotations + +import importlib.util +import sys +import tempfile +import unittest +from pathlib import Path + + +SCRIPT = Path(__file__).parents[1] / "unity-text-locator" / "scripts" / "scan_unity_text.py" +SPEC = importlib.util.spec_from_file_location("scan_unity_text", SCRIPT) +assert SPEC and SPEC.loader +scanner = importlib.util.module_from_spec(SPEC) +sys.modules[SPEC.name] = scanner +SPEC.loader.exec_module(scanner) + + +class UnityVersionDetectionTests(unittest.TestCase): + def test_detects_legacy_and_unity_six_versions_with_evidence(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) / "Game" + data = root / "Game_Data" + data.mkdir(parents=True) + (data / "globalgamemanagers").write_bytes(b"header\x002019.4.40f1\x00tail") + (data / "resources.assets").write_bytes(b"header\x006000.0.35f1\x00tail") + + result = scanner.detect_unity_versions(root, [data]) + + self.assertEqual(result["unity_versions"], ["2019.4.40f1", "6000.0.35f1"]) + self.assertEqual(result["unity_version_status"], "detected") + self.assertIn("Game_Data/globalgamemanagers", result["unity_version_evidence"]["2019.4.40f1"]) + + def test_reports_unknown_without_evidence(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + result = scanner.detect_unity_versions(root, []) + self.assertEqual(result["unity_versions"], []) + self.assertEqual(result["unity_version_status"], "unknown") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_writeback_transaction.py b/tests/test_writeback_transaction.py new file mode 100644 index 0000000..8b2175e --- /dev/null +++ b/tests/test_writeback_transaction.py @@ -0,0 +1,112 @@ +from __future__ import annotations + +import importlib.util +import tempfile +import unittest +from unittest import mock +from pathlib import Path + +SCRIPT = Path(__file__).parents[1] / "unity-text-locator" / "scripts" / "writeback_unity_text.py" +SPEC = importlib.util.spec_from_file_location("writeback_unity_text", SCRIPT) +assert SPEC and SPEC.loader +writeback = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(writeback) + + +class WritebackTransactionTests(unittest.TestCase): + def test_resolve_under_root_rejects_parent_and_drive(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + for unsafe in ("../outside", r"C:\outside\file", r"\\server\share\file"): + with self.subTest(unsafe=unsafe), self.assertRaises(SystemExit): + writeback.resolve_under_root(root, unsafe) + + def test_commit_staged_files_creates_backup_and_replaces(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + tmp_path = Path(temporary) + real_root = tmp_path / "game" + stage_root = tmp_path / "stage" + backup_root = tmp_path / "backups" + relative = "Game_Data/resources.assets" + target = real_root / relative + staged = stage_root / relative + target.parent.mkdir(parents=True) + staged.parent.mkdir(parents=True) + target.write_bytes(b"original") + staged.write_bytes(b"translated") + + written = writeback.commit_staged_files( + real_root, + stage_root, + [relative], + {relative: writeback.file_sha256(target)}, + backup_root, + ) + + self.assertEqual(written, [relative]) + self.assertEqual(target.read_bytes(), b"translated") + self.assertEqual((backup_root / relative).read_bytes(), b"original") + + def test_commit_refuses_source_changed_after_staging(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + tmp_path = Path(temporary) + real_root = tmp_path / "game" + stage_root = tmp_path / "stage" + backup_root = tmp_path / "backups" + relative = "sharedassets0.assets" + target = real_root / relative + staged = stage_root / relative + target.parent.mkdir(parents=True) + staged.parent.mkdir(parents=True) + target.write_bytes(b"changed") + staged.write_bytes(b"translated") + + with self.assertRaisesRegex(RuntimeError, "source changed"): + writeback.commit_staged_files( + real_root, + stage_root, + [relative], + {relative: writeback.file_sha256(staged)}, + backup_root, + ) + + self.assertEqual(target.read_bytes(), b"changed") + self.assertFalse(backup_root.exists()) + + def test_commit_rolls_back_earlier_files_when_later_write_fails(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + tmp_path = Path(temporary) + real_root = tmp_path / "game" + stage_root = tmp_path / "stage" + backup_root = tmp_path / "backups" + relatives = ["a.assets", "b.assets"] + expected = {} + for relative in relatives: + target = real_root / relative + staged = stage_root / relative + target.parent.mkdir(parents=True, exist_ok=True) + staged.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes(("old-" + relative).encode()) + staged.write_bytes(("new-" + relative).encode()) + expected[relative] = writeback.file_sha256(target) + + real_atomic_copy = writeback.atomic_copy + second_staged = (stage_root / "b.assets").resolve() + + def fail_second_staged_copy(source: Path, target: Path) -> None: + if source.resolve() == second_staged: + raise OSError("simulated commit failure") + real_atomic_copy(source, target) + + with mock.patch.object(writeback, "atomic_copy", side_effect=fail_second_staged_copy): + with self.assertRaisesRegex(OSError, "simulated"): + writeback.commit_staged_files( + real_root, stage_root, relatives, expected, backup_root + ) + + self.assertEqual((real_root / "a.assets").read_bytes(), b"old-a.assets") + self.assertEqual((real_root / "b.assets").read_bytes(), b"old-b.assets") + + +if __name__ == "__main__": + unittest.main() diff --git a/unity-text-locator/SKILL.md b/unity-text-locator/SKILL.md index 96954f1..e38a138 100644 --- a/unity-text-locator/SKILL.md +++ b/unity-text-locator/SKILL.md @@ -1,6 +1,6 @@ --- name: unity-text-locator -description: Adaptive Unity Japanese-to-Chinese translation workflow. Use when Codex needs to inspect an unknown Unity game's text layout, extract project-named one-column text CSVs, accept matching one-column *_translation.csv files, validate and selectively apply safe rows, write approved rows back into Unity assets/Utage/BansheeGz BGDatabase/raw MonoBehaviour records, inspect TMP font-bundle identity/version/glyph coverage, diagnose tofu versus atlas corruption, validate Addressables or IL2CPP candidates, locate ES3 saves, or build verified binary-diff patch packages. +description: Safely localize unknown Unity games, especially Japanese-to-Chinese projects. Use to discover Unity version/layout/backend, extract occurrence-mapped one-column CSVs, bridge them through AiNiee-style translation, validate and transactionally write approved rows into serialized assets or verified middleware adapters, audit raw strings, diagnose TMP fonts, install a licensed app-local Noto fallback for supported Mono builds, inspect Addressables/IL2CPP risks, locate saves, or build SHA-verified patch packages. --- # Unity Translation Workflow @@ -24,6 +24,8 @@ Use three gates for every project: - Translator gate: expose only project-named one-column CSVs for row-mapped sources; keep broad `extracted_text.csv` as audit evidence, not as a writeback source. - Runtime gate: a clean CSV validation and a tool-level dry run are not enough for runtime-sensitive containers; prove the edited file loads through the same game path before distributing it. +Keep `raw-utf8-fallback` audit-only by default. Expose it with `--include-raw-candidates` and write it with `--allow-raw-fallback` only after an adapter-specific container review. Normal writes must complete in staging, pass exact object/file validation, and commit with automatic rollback. + Read [references/unity-text-map.md](references/unity-text-map.md) when locating candidate Unity text containers or middleware clues. Read [references/runtime-sensitive-adapters.md](references/runtime-sensitive-adapters.md) when handling BansheeGz databases, Addressables bundles/catalogs, ES3 saves, or runtime font compatibility. Read [references/workflow-modes.md](references/workflow-modes.md) before choosing semi-automatic or full-automatic translation. @@ -181,6 +183,8 @@ Use the sibling `ainiee-translate` skill to run the batch translation loop on th --out-csv "GameFolder/_translation/unity-text-report/text/GameTitle_text_translation.csv" ``` +This conversion is fail-closed: it verifies the source CSV fingerprint and rejects missing, duplicate, untranslated, or blank rows. `--allow-partial` is only for generating a review artifact and must not feed final writeback. + Then run `validate_translation_csv.py` exactly as in the semi-automatic path. When the user explicitly chooses to apply only structurally safe rows, filter row-addressable failures and then validate the filtered output again: @@ -283,11 +287,12 @@ For compatible Mono/TMP games with `ScriptingAssemblies.json` and `RuntimeInitia ```bash python scripts/install_tmp_chinese_font_fix.py "GameFolder" \ + --runtime-font-file "path/to/licensed/NotoSansCJKsc-Regular.otf" \ --out-dir "GameFolder/_translation/unity-text-report/font-fix" \ --dry-run ``` -The fixer compiles the DLL and validates injection files during preflight, then applies all planned writes with backups only when run without `--dry-run`. Its default runtime behavior chooses an installed CJK font file, adds a dynamic TMP fallback, and changes legacy UI fonts only for strings with missing CJK glyphs; it does not replace every TMP font. Use `--patch-embedded-fonts --font "Microsoft YaHei"` only when a reviewed embedded legacy `Font` replacement is needed. +The fixer compiles the DLL and validates injection files during preflight, then applies the DLL, JSON, and app-local runtime font as one rollback-capable change set. Prefer a legally redistributable Noto CJK/SC font; never publish `arialuni_sdf_u2019` without explicit redistribution rights. When no runtime font is supplied, the fixer may fall back to installed system CJK fonts. Use embedded replacement only after exact Unity/TMP compatibility review. After installation, launch the game and inspect `Player.log`. Require an initialization log and no `MissingMethodException`, `Unable to load font face`, or repeated missing-glyph failures. Some stripped Mono builds omit `Font.CreateDynamicFontFromOSFont` or `ParameterInfo.HasDefaultValue`; use the shipped file-path/reflection-compatible fixer rather than assuming those APIs exist. If the game directory permits content writes but rejects rename replacement, keep the backup, accept verified copy-overwrite only, and remove or report staged `.tmp` files. diff --git a/unity-text-locator/assets/ChineseFontFixer.cs b/unity-text-locator/assets/ChineseFontFixer.cs index 065a562..8a1814b 100644 --- a/unity-text-locator/assets/ChineseFontFixer.cs +++ b/unity-text-locator/assets/ChineseFontFixer.cs @@ -2,6 +2,7 @@ using System.Collections; using System.IO; using System.Reflection; +using System.Collections.Generic; using TMPro; using UnityEngine; using UnityEngine.UI; @@ -40,17 +41,33 @@ public sealed class ChineseFontFixerRuntime : MonoBehaviour private static string[] BuildFontPaths() { + List paths = new List(); + string privateDirectory = Path.Combine(Application.dataPath, "ChineseFontFixer", "Fonts"); + if (Directory.Exists(privateDirectory)) + { + string[] extensions = { "*.ttf", "*.otf", "*.ttc" }; + for (int i = 0; i < extensions.Length; i++) + { + try + { + paths.AddRange(Directory.GetFiles(privateDirectory, extensions[i])); + } + catch (Exception ex) + { + Debug.LogWarning("[ChineseFontFixer] App-local font discovery failed: " + ex.Message); + } + } + } string directory = Environment.GetFolderPath(Environment.SpecialFolder.Fonts); if (string.IsNullOrEmpty(directory)) { directory = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.Windows), "Fonts"); } - string[] paths = new string[FontFileNames.Length]; for (int i = 0; i < FontFileNames.Length; i++) { - paths[i] = Path.Combine(directory, FontFileNames[i]); + paths.Add(Path.Combine(directory, FontFileNames[i])); } - return paths; + return paths.ToArray(); } private IEnumerator Start() @@ -107,8 +124,11 @@ private Font CreateLegacyFont() try { Font candidate = new Font(FontPaths[i]); - if (candidate != null && candidate.HasCharacter('\u4e2d')) + // Unity may not populate a newly constructed file-backed Font's + // character table until the font engine/TMP asset uses it. + if (candidate != null) { + Debug.Log("[ChineseFontFixer] Using font file: " + FontPaths[i]); return candidate; } } @@ -117,7 +137,7 @@ private Font CreateLegacyFont() Debug.LogWarning("[ChineseFontFixer] Font path failed: " + FontPaths[i] + " " + ex.Message); } } - Debug.LogWarning("[ChineseFontFixer] No usable CJK system font file found."); + Debug.LogWarning("[ChineseFontFixer] No app-local or supported system font file found."); return null; } diff --git a/unity-text-locator/assets/apply_cn_patch.ps1.txt b/unity-text-locator/assets/apply_cn_patch.ps1.txt index 9fe3595..a1f47ad 100644 --- a/unity-text-locator/assets/apply_cn_patch.ps1.txt +++ b/unity-text-locator/assets/apply_cn_patch.ps1.txt @@ -7,6 +7,29 @@ $ErrorActionPreference = "Stop" $PatchDir = [IO.Path]::GetFullPath($PatchDir) $GameRoot = [IO.Path]::GetFullPath($GameRoot) +function Resolve-SafeChildPath([string]$Root, [string]$Relative, [bool]$MustExist = $true) { + if ([string]::IsNullOrWhiteSpace($Relative) -or + [IO.Path]::IsPathRooted($Relative) -or + -not [string]::IsNullOrEmpty([IO.Path]::GetPathRoot($Relative)) -or + (($Relative -replace "/", "\" -split "\\") -contains "..")) { + throw "Unsafe patch path: $Relative" + } + $rootFull = [IO.Path]::GetFullPath($Root).TrimEnd([IO.Path]::DirectorySeparatorChar, [IO.Path]::AltDirectorySeparatorChar) + $candidate = [IO.Path]::GetFullPath((Join-Path $rootFull ($Relative -replace "/", "\"))) + $prefix = $rootFull + [IO.Path]::DirectorySeparatorChar + if (-not $candidate.StartsWith($prefix, [StringComparison]::OrdinalIgnoreCase)) { + throw "Patch path escapes its root: $Relative" + } + if ($MustExist) { + $resolved = (Resolve-Path -LiteralPath $candidate -ErrorAction Stop).ProviderPath + if (-not ($resolved -eq $rootFull) -and -not $resolved.StartsWith($prefix, [StringComparison]::OrdinalIgnoreCase)) { + throw "Patch path escapes its root through a link: $Relative" + } + return $resolved + } + return $candidate +} + function Get-ByteHash([byte[]]$Bytes) { $sha = [Security.Cryptography.SHA256]::Create() try { @@ -42,13 +65,11 @@ $planned = @() $alreadyPatched = @() foreach ($entry in $manifest.files) { $relative = $entry.path -replace "/", "\" - if ([IO.Path]::IsPathRooted($relative) -or (($relative -split "\\") -contains "..")) { - throw "Unsafe patch path: $relative" - } - $target = Join-Path $GameRoot $relative + $target = Resolve-SafeChildPath $GameRoot $relative $false if (-not (Test-Path -LiteralPath $target -PathType Leaf)) { throw "Target file missing: $relative" } + $target = Resolve-SafeChildPath $GameRoot $relative $true [byte[]]$current = [IO.File]::ReadAllBytes($target) $currentHash = Get-ByteHash $current if ($currentHash -eq $entry.new_sha256) { @@ -58,7 +79,11 @@ foreach ($entry in $manifest.files) { if ($currentHash -ne $entry.old_sha256) { throw "Target does not match original or translated hash: $relative" } - [byte[]]$middle = Read-GZipPayload (Join-Path $PatchDir ($entry.payload -replace "/", "\")) + $payload = Resolve-SafeChildPath $PatchDir ([string]$entry.payload) $true + if (-not (Test-Path -LiteralPath $payload -PathType Leaf)) { + throw "Patch payload missing: $($entry.payload)" + } + [byte[]]$middle = Read-GZipPayload $payload $prefix = [int]$entry.prefix_bytes $suffix = [int]$entry.suffix_bytes [byte[]]$translated = New-Object byte[] ($prefix + $middle.Length + $suffix) diff --git a/unity-text-locator/references/ainiee-integration.md b/unity-text-locator/references/ainiee-integration.md index aa2ce87..5207d88 100644 --- a/unity-text-locator/references/ainiee-integration.md +++ b/unity-text-locator/references/ainiee-integration.md @@ -82,7 +82,7 @@ python unity-text-locator/scripts/validate_translation_csv.py \ ## Guardrails -- Do not let subagents write the same `cache.json` concurrently. +- Cache writes are atomic and cooperatively locked, but subagents should still produce separate batch JSON files and let one coordinator write `cache.json`. - Do not use AiNiee output directly for Unity writeback; always convert to `zh_cn` CSV and run Unity validation. -- Blank translations remain intentional skips. -- If the translated cache has fewer rows than the source CSV, conversion emits blank rows for missing indexes and validation/reporting should catch the gap before writeback. +- Conversion verifies the source CSV content hash, index continuity, translation status, and nonblank output. Any incomplete row fails by default. +- `--allow-partial` emits blanks only for an explicitly requested review artifact. Do not pass such output to writeback. diff --git a/unity-text-locator/references/unity-text-map.md b/unity-text-locator/references/unity-text-map.md index 3067a0c..fb32123 100644 --- a/unity-text-locator/references/unity-text-map.md +++ b/unity-text-locator/references/unity-text-map.md @@ -1,5 +1,13 @@ # Unity Text Location Map +## Contents + +- Common locations and middleware clues +- Discovery order and tool choice +- Per-source CSV/manifest contract +- Safe writeback and validation +- Font fallback and completion evidence + ## Common Locations | Location | What it may contain | First action | diff --git a/unity-text-locator/requirements.txt b/unity-text-locator/requirements.txt new file mode 100644 index 0000000..237fbe5 --- /dev/null +++ b/unity-text-locator/requirements.txt @@ -0,0 +1,2 @@ +# Unity asset inspection and serialized-file writeback. +UnityPy>=1.25,<2 diff --git a/unity-text-locator/scripts/ainiee_cache_to_unity_translation.py b/unity-text-locator/scripts/ainiee_cache_to_unity_translation.py index e442d8c..835354c 100644 --- a/unity-text-locator/scripts/ainiee_cache_to_unity_translation.py +++ b/unity-text-locator/scripts/ainiee_cache_to_unity_translation.py @@ -5,6 +5,7 @@ import argparse import csv +import hashlib import os import sys from pathlib import Path @@ -31,13 +32,22 @@ def add_ainiee_path(path: Path) -> None: sys.path.insert(0, str(path)) -def read_source_count(path: Path) -> int: +def read_source_rows(path: Path) -> list[str]: with path.open("r", encoding="utf-8-sig", newline="") as handle: reader = csv.DictReader(handle) fields = [field.lstrip("\ufeff") for field in (reader.fieldnames or [])] if fields != ["original_flat"]: raise SystemExit(f"{path} must contain exactly one original_flat column") - return sum(1 for _ in reader) + return [row.get("original_flat", "") for row in reader] + + +def source_rows_sha256(rows: list[str]) -> str: + digest = hashlib.sha256() + for row in rows: + encoded = row.encode("utf-8") + digest.update(len(encoded).to_bytes(8, "big")) + digest.update(encoded) + return digest.hexdigest() def main() -> int: @@ -50,6 +60,11 @@ def main() -> int: default=None, help="Optional Unity source CSV used to enforce output row count and order.", ) + parser.add_argument( + "--allow-partial", + action="store_true", + help="Allow missing, untranslated, or blank rows; unsafe for final writeback.", + ) parser.add_argument( "--ainiee-scripts", type=Path, @@ -63,16 +78,47 @@ def main() -> int: from ainiee_translate import cache_io project = cache_io.load_cache(str(args.cache)) - by_index = { - int(item.text_index): (item.translated_text or "") - for item in cache_io.iter_items(project) - } + items = list(cache_io.iter_items(project)) + indexes = [int(item.text_index) for item in items] + if any(index <= 0 for index in indexes) or len(indexes) != len(set(indexes)): + raise SystemExit("cache contains invalid or duplicate text_index values") + by_index = {int(item.text_index): item for item in items} if args.source_csv: - row_count = read_source_count(args.source_csv) - values = [by_index.get(index, "") for index in range(1, row_count + 1)] + source_rows = read_source_rows(args.source_csv) + row_count = len(source_rows) + expected_hash = (project.extra or {}).get("source_rows_sha256") + actual_hash = source_rows_sha256(source_rows) + if not expected_hash: + raise SystemExit("cache has no source_rows_sha256; recreate it from the Unity source CSV") + if expected_hash != actual_hash: + raise SystemExit("source CSV content hash does not match the cache") + source_mismatches = [ + index for index, source in enumerate(source_rows, start=1) + if index in by_index and (by_index[index].source_text or "") != source + ] + if source_mismatches: + raise SystemExit(f"cache source text differs at {len(source_mismatches)} row(s)") else: - values = [by_index[index] for index in sorted(by_index)] + row_count = max(indexes, default=0) + + expected_indexes = set(range(1, row_count + 1)) + missing = sorted(expected_indexes - set(by_index)) + extra = sorted(set(by_index) - expected_indexes) + incomplete = [] + values = [] + for index in range(1, row_count + 1): + item = by_index.get(index) + value = (item.translated_text or "") if item else "" + if item is None or item.translation_status not in (1, 2) or not value.strip(): + incomplete.append(index) + values.append(value) + if (missing or extra or incomplete) and not args.allow_partial: + raise SystemExit( + "cache is incomplete: " + f"{len(missing)} missing, {len(extra)} out-of-range, " + f"{len(incomplete)} untranslated/blank row(s); use --allow-partial only for review" + ) args.out_csv.parent.mkdir(parents=True, exist_ok=True) with args.out_csv.open("w", encoding="utf-8-sig", newline="") as handle: diff --git a/unity-text-locator/scripts/apply_binary_patch.py b/unity-text-locator/scripts/apply_binary_patch.py index 9808103..7df9d1e 100644 --- a/unity-text-locator/scripts/apply_binary_patch.py +++ b/unity-text-locator/scripts/apply_binary_patch.py @@ -10,7 +10,7 @@ import os import shutil from datetime import datetime -from pathlib import Path, PurePosixPath +from pathlib import Path, PurePosixPath, PureWindowsPath def sha256(data: bytes) -> str: @@ -18,12 +18,32 @@ def sha256(data: bytes) -> str: def checked_relative(value: str) -> Path: + if not value or "\x00" in value: + raise SystemExit(f"Unsafe patch path in manifest: {value}") normalized = PurePosixPath(value.replace("\\", "/")) - if normalized.is_absolute() or ".." in normalized.parts: + windows = PureWindowsPath(value) + if ( + normalized.is_absolute() + or windows.is_absolute() + or bool(windows.drive) + or ".." in normalized.parts + or not normalized.parts + ): raise SystemExit(f"Unsafe patch path in manifest: {value}") return Path(*normalized.parts) +def contained_path(root: Path, relative: Path, *, must_exist: bool = True) -> Path: + """Resolve a manifest path and require it to stay below root, including links.""" + root = root.resolve() + candidate = (root / relative).resolve(strict=must_exist) + try: + candidate.relative_to(root) + except ValueError: + raise SystemExit(f"Patch path escapes its root: {relative.as_posix()}") from None + return candidate + + def main() -> int: parser = argparse.ArgumentParser() parser.add_argument("--package", required=True, type=Path) @@ -40,7 +60,7 @@ def main() -> int: skipped_current: list[str] = [] for entry in manifest.get("files", []): relative = checked_relative(str(entry["path"])) - target = root / relative + target = contained_path(root, relative, must_exist=False) if not target.exists(): raise SystemExit(f"Target file missing: {relative.as_posix()}") current = target.read_bytes() @@ -52,7 +72,11 @@ def main() -> int: raise SystemExit(f"Target does not match original or translated hash: {relative.as_posix()}") prefix = int(entry["prefix_bytes"]) suffix = int(entry["suffix_bytes"]) - middle = gzip.decompress((package / checked_relative(str(entry["payload"]))).read_bytes()) + payload_relative = checked_relative(str(entry["payload"])) + payload = contained_path(package, payload_relative, must_exist=False) + if not payload.is_file(): + raise SystemExit(f"Patch payload missing: {payload_relative.as_posix()}") + middle = gzip.decompress(payload.read_bytes()) suffix_bytes = current[len(current) - suffix :] if suffix else b"" translated = current[:prefix] + middle + suffix_bytes if sha256(translated) != entry["new_sha256"]: diff --git a/unity-text-locator/scripts/build_binary_patch.py b/unity-text-locator/scripts/build_binary_patch.py index 1bb9bf8..c7c3163 100644 --- a/unity-text-locator/scripts/build_binary_patch.py +++ b/unity-text-locator/scripts/build_binary_patch.py @@ -9,7 +9,7 @@ import json import shutil import zipfile -from pathlib import Path, PurePosixPath +from pathlib import Path, PurePosixPath, PureWindowsPath def sha256(data: bytes) -> str: @@ -17,12 +17,32 @@ def sha256(data: bytes) -> str: def checked_relative(value: str) -> Path: + if not value or "\x00" in value: + raise SystemExit(f"Unsafe relative patch path: {value}") normalized = PurePosixPath(value.replace("\\", "/")) - if normalized.is_absolute() or ".." in normalized.parts: + windows = PureWindowsPath(value) + if ( + normalized.is_absolute() + or windows.is_absolute() + or bool(windows.drive) + or ".." in normalized.parts + or not normalized.parts + ): raise SystemExit(f"Unsafe relative patch path: {value}") return Path(*normalized.parts) +def contained_path(root: Path, relative: Path, *, must_exist: bool = True) -> Path: + """Resolve a user path and require it to stay below root, including links.""" + root = root.resolve() + candidate = (root / relative).resolve(strict=must_exist) + try: + candidate.relative_to(root) + except ValueError: + raise SystemExit(f"Patch path escapes its root: {relative.as_posix()}") from None + return candidate + + def split_middle(original: bytes, translated: bytes) -> tuple[int, int, bytes]: prefix = 0 prefix_limit = min(len(original), len(translated)) @@ -53,8 +73,8 @@ def main() -> int: entries: list[dict[str, object]] = [] for requested in args.files: relative = checked_relative(requested) - original_path = backup_root / relative - translated_path = game_root / relative + original_path = contained_path(backup_root, relative, must_exist=False) + translated_path = contained_path(game_root, relative, must_exist=False) if not original_path.exists() or not translated_path.exists(): raise SystemExit(f"Patch source missing for {relative.as_posix()}") original = original_path.read_bytes() diff --git a/unity-text-locator/scripts/extract_unity_text.py b/unity-text-locator/scripts/extract_unity_text.py index 34f6b21..4689f0d 100644 --- a/unity-text-locator/scripts/extract_unity_text.py +++ b/unity-text-locator/scripts/extract_unity_text.py @@ -462,7 +462,7 @@ def extract_records( } -def write_outputs(result: dict[str, object], out_dir: Path) -> None: +def write_outputs(result: dict[str, object], out_dir: Path, include_raw_candidates: bool = False) -> None: out_dir.mkdir(parents=True, exist_ok=True) unique_texts = result["unique_texts"] occurrences = result["occurrences"] @@ -478,7 +478,7 @@ def write_outputs(result: dict[str, object], out_dir: Path) -> None: manifest_path = out_dir / "extraction_manifest.json" manifest_path.write_text(json.dumps(result, ensure_ascii=False, indent=2), encoding="utf-8") - write_source_text_outputs(result, out_dir / "text") + write_source_text_outputs(result, out_dir / "text", include_raw_candidates=include_raw_candidates) category_counts = Counter(str(row["category"]) for row in occurrences if isinstance(row, dict)) method_counts = Counter(str(row["method"]) for row in occurrences if isinstance(row, dict)) @@ -512,7 +512,11 @@ def write_outputs(result: dict[str, object], out_dir: Path) -> None: (out_dir / "extraction-summary.md").write_text("\n".join(lines), encoding="utf-8") -def write_source_text_outputs(result: dict[str, object], text_dir: Path) -> None: +def write_source_text_outputs( + result: dict[str, object], + text_dir: Path, + include_raw_candidates: bool = False, +) -> None: occurrences = result["occurrences"] files_scanned = result["files_scanned"] assert isinstance(occurrences, list) @@ -523,9 +527,22 @@ def write_source_text_outputs(result: dict[str, object], text_dir: Path) -> None if isinstance(row, dict) and "path" in row } by_source: dict[str, list[dict[str, object]]] = defaultdict(list) + raw_audit: list[dict[str, object]] = [] for occurrence in occurrences: - if isinstance(occurrence, dict): - by_source[str(occurrence["source_file"])].append(occurrence) + if not isinstance(occurrence, dict): + continue + if occurrence.get("method") == "raw-utf8-fallback" and not include_raw_candidates: + raw_audit.append(occurrence) + continue + by_source[str(occurrence["source_file"])].append(occurrence) + if raw_audit: + text_dir.mkdir(parents=True, exist_ok=True) + audit_path = text_dir / "raw_utf8_candidates_audit.csv" + with audit_path.open("w", encoding="utf-8-sig", newline="") as handle: + fields = ["source_file", "offset_hex", "byte_length", "confidence", "original_flat", "notes"] + writer = csv.DictWriter(handle, fieldnames=fields, lineterminator="\n", extrasaction="ignore") + writer.writeheader() + writer.writerows(raw_audit) if not by_source: return @@ -582,10 +599,11 @@ def extract_to_outputs( out_dir: Path, max_file_mb: int = DEFAULT_MAX_FILE_MB, include_managed: bool = False, + include_raw_candidates: bool = False, ignored_dirs: set[str] | None = None, ) -> dict[str, object]: result = extract_records(root, max_file_mb=max_file_mb, include_managed=include_managed, ignored_dirs=ignored_dirs) - write_outputs(result, out_dir) + write_outputs(result, out_dir, include_raw_candidates=include_raw_candidates) return result @@ -595,6 +613,11 @@ def parse_args(argv: list[str]) -> argparse.Namespace: parser.add_argument("--out", default=None, help="Output report directory (default: /_translation/unity-text-report)") parser.add_argument("--max-file-mb", type=int, default=DEFAULT_MAX_FILE_MB, help="Maximum file size to scan") parser.add_argument("--include-managed", action="store_true", help="Also extract strings from Managed/Assembly-CSharp.dll") + parser.add_argument( + "--include-raw-candidates", + action="store_true", + help="Expose raw UTF-8 fallback rows to translators; otherwise keep them audit-only", + ) parser.add_argument("--ignore-dir", action="append", default=[], help="Directory name to skip; can be repeated") return parser.parse_args(argv) @@ -615,6 +638,7 @@ def main(argv: list[str]) -> int: out_dir=out_dir, max_file_mb=args.max_file_mb, include_managed=args.include_managed, + include_raw_candidates=args.include_raw_candidates, ignored_dirs=ignored_dirs, ) print(f"Wrote {out_dir / 'extracted_text.csv'}") diff --git a/unity-text-locator/scripts/install_tmp_chinese_font_fix.py b/unity-text-locator/scripts/install_tmp_chinese_font_fix.py index c7cfc0b..bb47794 100644 --- a/unity-text-locator/scripts/install_tmp_chinese_font_fix.py +++ b/unity-text-locator/scripts/install_tmp_chinese_font_fix.py @@ -4,6 +4,7 @@ from __future__ import annotations import argparse +import hashlib import json import os import shutil @@ -81,6 +82,20 @@ def installed_runtime_font_files() -> list[str]: return found +def sha256_bytes(payload: bytes) -> str: + return hashlib.sha256(payload).hexdigest() + + +def runtime_font_destination(data_dir: Path, source: Path) -> Path: + if not source.is_file(): + raise SystemExit(f"Runtime font file not found: {source}") + if source.stat().st_size == 0: + raise SystemExit("--runtime-font-file must not be empty") + if source.suffix.lower() not in {".ttf", ".otf", ".ttc"}: + raise SystemExit("--runtime-font-file must be a .ttf, .otf, or .ttc font") + return data_dir / "ChineseFontFixer" / "Fonts" / source.name + + def compile_dll(skill_dir: Path, managed: Path, csc: Path, output_dll: Path) -> None: source = skill_dir / "assets" / "ChineseFontFixer.cs" refs = [ @@ -210,6 +225,7 @@ def commit_outputs(outputs: dict[Path, bytes], root: Path, backup_root: Path) -> shutil.copy2(path, backup_path(path, root, backup_root)) try: for path, payload in outputs.items(): + path.parent.mkdir(parents=True, exist_ok=True) temporary = path.with_name(path.name + ".ChineseFontFixer.tmp") temporary.write_bytes(payload) try: @@ -241,6 +257,12 @@ def main() -> int: parser.add_argument("game_root", type=Path) parser.add_argument("--font", default="Microsoft YaHei", choices=sorted(FONT_CHOICES)) parser.add_argument("--font-file", type=Path, default=None) + parser.add_argument( + "--runtime-font-file", + type=Path, + default=None, + help="Copy a legally obtained TTF/OTF/TTC into the game-private runtime fallback directory", + ) parser.add_argument("--out-dir", type=Path, default=None) parser.add_argument("--font-name-pattern", action="append", default=None, help="Embedded Font asset name substring to replace") parser.add_argument("--patch-embedded-fonts", action="store_true", help="Also replace matching embedded legacy Font assets") @@ -261,6 +283,18 @@ def main() -> int: raise SystemExit("--font and --font-file select embedded replacement data only; use them with --patch-embedded-fonts") outputs: dict[Path, bytes] = dict(json_outputs) + runtime_font: dict | None = None + if args.runtime_font_file: + runtime_source = args.runtime_font_file.resolve() + runtime_payload = runtime_source.read_bytes() if runtime_source.is_file() else b"" + runtime_target = runtime_font_destination(data_dir, runtime_source) + outputs[runtime_target] = runtime_payload + runtime_font = { + "source": str(runtime_source), + "target": runtime_target.relative_to(root).as_posix(), + "size": len(runtime_payload), + "sha256": sha256_bytes(runtime_payload), + } patched_fonts: list[dict] = [] selected_font = args.font selected_font_file: str | None = None @@ -298,6 +332,7 @@ def main() -> int: "assembly_type": assembly_type, "runtime_fallback_fonts": list(FONT_CHOICES), "runtime_fallback_font_files_found": installed_runtime_font_files(), + "runtime_font_file": runtime_font, "patch_embedded_fonts": args.patch_embedded_fonts, "embedded_font_name": selected_font if args.patch_embedded_fonts else None, "embedded_font_file": selected_font_file, diff --git a/unity-text-locator/scripts/scan_unity_text.py b/unity-text-locator/scripts/scan_unity_text.py index 6445b81..9e20935 100644 --- a/unity-text-locator/scripts/scan_unity_text.py +++ b/unity-text-locator/scripts/scan_unity_text.py @@ -42,6 +42,7 @@ UNITY_ASSET_EXTS = {".assets", ".resS", ".resource"} BUNDLE_EXTS = {".bundle", ".unity3d", ".ab", ".assetbundle"} +UNITY_VERSION_RE = re.compile(rb"(? dict[str, object]: } +def detect_unity_versions(root: Path, data_dirs: list[Path]) -> dict[str, object]: + """Collect version strings from bounded Unity metadata headers.""" + evidence: dict[str, set[str]] = {} + candidates: list[Path] = [] + for data_dir in data_dirs or [root]: + candidates.extend(data_dir / name for name in ("globalgamemanagers", "resources.assets")) + candidates.extend(sorted(data_dir.glob("sharedassets*.assets"))[:4]) + for path in candidates: + if not path.is_file(): + continue + try: + with path.open("rb") as handle: + head = handle.read(4 * 1024 * 1024) + except OSError: + continue + versions = {match.group(1).decode("ascii") for match in UNITY_VERSION_RE.finditer(head)} + for version in versions: + evidence.setdefault(version, set()).add(rel(path, root)) + return { + "unity_versions": sorted(evidence), + "unity_version_evidence": {version: sorted(paths) for version, paths in sorted(evidence.items())}, + "unity_version_status": "detected" if evidence else "unknown", + } + + def detect_unity_markers(root: Path) -> dict[str, object]: data_dirs = find_data_dirs(root) markers: list[str] = [] @@ -291,6 +317,7 @@ def detect_unity_markers(root: Path) -> dict[str, object]: "data_dirs": [rel(p, root) for p in data_dirs], "exe_candidates": exe_candidates, "unity_markers": sorted(set(markers)), + **detect_unity_versions(root, data_dirs), **detect_backend(root, data_dirs), } @@ -400,6 +427,7 @@ def write_report(result: dict[str, object], out_dir: Path) -> None: lines.append(f"- Root: `{metadata.get('root')}`") lines.append(f"- Data dirs: {', '.join(f'`{p}`' for p in metadata.get('data_dirs', [])) or 'none detected'}") lines.append(f"- Backend: `{metadata.get('backend')}`") + lines.append(f"- Unity versions: `{', '.join(metadata.get('unity_versions', [])) or 'unknown'}`") lines.append(f"- Scanned files: {stats.get('scanned_files')}") lines.append(f"- Findings: {stats.get('findings')}") lines.append("") diff --git a/unity-text-locator/scripts/unity_csv_to_ainiee_cache.py b/unity-text-locator/scripts/unity_csv_to_ainiee_cache.py index de9155f..6d2f518 100644 --- a/unity-text-locator/scripts/unity_csv_to_ainiee_cache.py +++ b/unity-text-locator/scripts/unity_csv_to_ainiee_cache.py @@ -10,6 +10,7 @@ import argparse import csv +import hashlib import os import sys from datetime import datetime, timezone @@ -47,6 +48,15 @@ def read_source_csv(path: Path) -> list[str]: return [row.get("original_flat", "") for row in reader] +def source_rows_sha256(rows: list[str]) -> str: + digest = hashlib.sha256() + for row in rows: + encoded = row.encode("utf-8") + digest.update(len(encoded).to_bytes(8, "big")) + digest.update(encoded) + return digest.hexdigest() + + def main() -> int: parser = argparse.ArgumentParser(description="Unity source CSV -> AiNiee cache.json") parser.add_argument("--source-csv", required=True, type=Path) @@ -72,13 +82,14 @@ def main() -> int: ) rows = read_source_csv(args.source_csv) + source_hash = source_rows_sha256(rows) project_name = args.project_name or args.source_csv.stem cache_file = CacheFile( storage_path=args.source_csv.name, encoding="utf-8-sig", file_project_type=ProjectType.CSV, line_ending="\n", - extra={"source_format": "unity-one-column-csv"}, + extra={"source_format": "unity-one-column-csv", "source_rows_sha256": source_hash}, ) for index, source_text in enumerate(rows, start=1): cache_file.add_item( @@ -102,7 +113,7 @@ def main() -> int: stats_data=CacheProjectStatistics(total_line=len(rows)), detected_encoding="utf-8-sig", detected_line_ending="\n", - extra={"source_format": "unity-one-column-csv"}, + extra={"source_format": "unity-one-column-csv", "source_rows_sha256": source_hash}, ) project.add_file(cache_file) diff --git a/unity-text-locator/scripts/writeback_unity_text.py b/unity-text-locator/scripts/writeback_unity_text.py index 96775de..dc5baa3 100644 --- a/unity-text-locator/scripts/writeback_unity_text.py +++ b/unity-text-locator/scripts/writeback_unity_text.py @@ -7,12 +7,15 @@ import csv import hashlib import json +import os import re import shutil import struct +import tempfile +import uuid from collections import Counter, defaultdict from datetime import datetime -from pathlib import Path +from pathlib import Path, PurePosixPath, PureWindowsPath PLACEHOLDER_RE = re.compile(r"(<[^>]+>|\\[nrt]|\{[^{}]+\}|%[sdif]|\$[A-Za-z0-9_]+|\[[A-Za-z0-9_:-]+\])") @@ -164,13 +167,111 @@ def backup(path: Path, root: Path, backup_root: Path, backed_up: set[str]) -> No backed_up.add(relative) +def resolve_under_root(root: Path, relative: str) -> Path: + """Resolve an existing manifest path without allowing it to escape root.""" + normalized = PurePosixPath(relative.replace("\\", "/")) + windows = PureWindowsPath(relative) + if ( + not relative + or normalized.is_absolute() + or windows.is_absolute() + or bool(windows.drive) + or not normalized.parts + or ".." in normalized.parts + ): + raise SystemExit(f"unsafe source path in manifest: {relative}") + value = Path(*normalized.parts) + resolved_root = root.resolve() + resolved = (resolved_root / value).resolve() + try: + resolved.relative_to(resolved_root) + except ValueError as exc: + raise SystemExit(f"source path escapes game root: {relative}") from exc + return resolved + + +def file_sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def selected_source_files(selected: list[dict]) -> list[str]: + return sorted({str(item["source_file"]) for item in selected}) + + +def stage_source_files(real_root: Path, stage_root: Path, relatives: list[str]) -> dict[str, str]: + hashes: dict[str, str] = {} + for relative in relatives: + source = resolve_under_root(real_root, relative) + if not source.is_file(): + raise SystemExit(f"source file missing: {relative}") + destination = resolve_under_root(stage_root, relative) + destination.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(source, destination) + hashes[relative] = file_sha256(source) + return hashes + + +def atomic_copy(source: Path, target: Path) -> None: + temporary = target.with_name(f"{target.name}.writeback-{uuid.uuid4().hex}.tmp") + try: + shutil.copy2(source, temporary) + os.replace(temporary, target) + finally: + temporary.unlink(missing_ok=True) + + +def commit_staged_files( + real_root: Path, + stage_root: Path, + relatives: list[str], + expected_hashes: dict[str, str], + backup_root: Path, +) -> list[str]: + """Commit validated staged files and roll back every earlier write on failure.""" + targets: dict[str, Path] = {} + for relative in relatives: + target = resolve_under_root(real_root, relative) + if file_sha256(target) != expected_hashes[relative]: + raise RuntimeError(f"source changed while writeback was staged: {relative}") + staged = resolve_under_root(stage_root, relative) + if not staged.is_file(): + raise RuntimeError(f"staged output missing: {relative}") + targets[relative] = target + + for relative, target in targets.items(): + backup_path = backup_root / Path(relative) + backup_path.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(target, backup_path) + + written: list[str] = [] + try: + for relative, target in targets.items(): + atomic_copy(resolve_under_root(stage_root, relative), target) + written.append(relative) + except Exception: + rollback_failures: list[str] = [] + for relative in reversed(written): + try: + atomic_copy(backup_root / Path(relative), targets[relative]) + except Exception: + rollback_failures.append(relative) + if rollback_failures: + raise RuntimeError(f"writeback failed and rollback also failed: {rollback_failures}") + raise + return written + + def verify_source_hash(manifest: dict, selected: list[dict]) -> None: expected = manifest.get("source_sha256") source_file = manifest.get("source_file") if not expected or not source_file or not selected: return - path = Path(manifest["root"]) / Path(str(source_file)) - current = hashlib.sha256(path.read_bytes()).hexdigest() + path = resolve_under_root(Path(manifest["root"]), str(source_file)) + current = file_sha256(path) if current != expected: raise SystemExit(f"source file SHA256 changed since extraction: {source_file}") @@ -182,7 +283,13 @@ def find_object(objects, offset: int): return None -def patch_len_prefixed(manifest: dict, selected: list[dict], backup_root: Path, dry_run: bool) -> tuple[list[dict], list[dict], set[str]]: +def patch_len_prefixed( + manifest: dict, + selected: list[dict], + backup_root: Path, + dry_run: bool, + backed_up: set[str] | None = None, +) -> tuple[list[dict], list[dict], set[str]]: root = Path(manifest["root"]) by_file: dict[str, list[dict]] = defaultdict(list) for occurrence in selected: @@ -197,9 +304,10 @@ def patch_len_prefixed(manifest: dict, selected: list[dict], backup_root: Path, patched: list[dict] = [] skipped: list[dict] = [] - backed_up: set[str] = set() + if backed_up is None: + backed_up = set() for relative, occurrences in sorted(by_file.items()): - path = root / Path(relative) + path = resolve_under_root(root, relative) if not path.exists(): skipped.extend({"row": item["_row"], "source_file": relative, "reason": "file missing"} for item in occurrences) continue @@ -244,7 +352,15 @@ def patch_len_prefixed(manifest: dict, selected: list[dict], backup_root: Path, delta += len(segment) - (pad_end - relative_offset) object_changed = True file_changed = True - patched.append({"row": occurrence["_row"], "source_file": relative, "method": "unity-len-utf8", "written_text": new_text}) + patched.append( + { + "row": occurrence["_row"], + "source_file": relative, + "method": "unity-len-utf8", + "written_text": new_text, + "path_id": int(obj.path_id), + } + ) if object_changed and not dry_run: obj.set_raw_data(bytes(raw)) if file_changed and not dry_run: @@ -262,7 +378,7 @@ def patch_raw(manifest: dict, selected: list[dict], backup_root: Path, backed_up patched: list[dict] = [] skipped: list[dict] = [] for relative, occurrences in sorted(by_file.items()): - path = root / Path(relative) + path = resolve_under_root(root, relative) if not path.exists(): skipped.extend({"row": item["_row"], "source_file": relative, "reason": "file missing"} for item in occurrences) continue @@ -300,42 +416,123 @@ def patch_raw(manifest: dict, selected: list[dict], backup_root: Path, backed_up def validate_written(root: Path, patched: list[dict]) -> list[dict]: - expected: dict[str, Counter[str]] = defaultdict(Counter) + expected_raw: dict[str, Counter[str]] = defaultdict(Counter) + expected_objects: dict[str, dict[int, Counter[str]]] = defaultdict(lambda: defaultdict(Counter)) for item in patched: - expected[str(item["source_file"])][str(item["written_text"])] += 1 + relative = str(item["source_file"]) + text = str(item["written_text"]) + if item.get("method") == "unity-len-utf8" and "path_id" in item: + expected_objects[relative][int(item["path_id"])][text] += 1 + else: + expected_raw[relative][text] += 1 failures: list[dict] = [] - for relative, counter in expected.items(): - data = (root / Path(relative)).read_bytes() + for relative, counter in expected_raw.items(): + data = resolve_under_root(root, relative).read_bytes() for text, count in counter.items(): found = data.count(text.encode("utf-8")) if found < count: failures.append({"source_file": relative, "expected": flat(text), "expected_count": count, "found_count": found}) + if expected_objects: + try: + import UnityPy # type: ignore + except ImportError as exc: + raise SystemExit("UnityPy is required for exact staged writeback validation") from exc + for relative, by_path_id in expected_objects.items(): + env = UnityPy.load(str(resolve_under_root(root, relative))) + objects = {int(obj.path_id): obj for obj in env.objects} + for path_id, counter in by_path_id.items(): + obj = objects.get(path_id) + if obj is None: + failures.append({"source_file": relative, "path_id": path_id, "reason": "saved object missing"}) + continue + raw = obj.get_raw_data() + for text, count in counter.items(): + found = raw.count(text.encode("utf-8")) + if found < count: + failures.append( + { + "source_file": relative, + "path_id": path_id, + "expected": flat(text), + "expected_count": count, + "found_count": found, + } + ) return failures def main() -> int: parser = argparse.ArgumentParser() parser.add_argument("--manifest", required=True, type=Path) + parser.add_argument( + "--game-root", + type=Path, + default=None, + help="Rebase a relocated manifest to this game root; source hashes are still required", + ) parser.add_argument("--source-csv", required=True, type=Path) parser.add_argument("--translation-csv", required=True, type=Path) parser.add_argument("--out-dir", required=True, type=Path) parser.add_argument("--backup-dir", type=Path, default=None) parser.add_argument("--allow-newline-changes", action="store_true") + parser.add_argument( + "--allow-raw-fallback", + action="store_true", + help="Allow risky fixed-span raw UTF-8 writes after container-specific review", + ) parser.add_argument("--dry-run", action="store_true") args = parser.parse_args() manifest = json.loads(args.manifest.read_text(encoding="utf-8")) + if args.game_root: + manifest["root"] = str(args.game_root.resolve()) args.out_dir.mkdir(parents=True, exist_ok=True) selected, writeback_csv, translation_rows_used = prepare_plan( manifest, args.source_csv, args.translation_csv, args.out_dir, args.allow_newline_changes ) verify_source_hash(manifest, selected) - backup_root = args.backup_dir or (args.out_dir / "backups" / datetime.now().strftime("%Y%m%d-%H%M%S")) - len_patched, len_skipped, backed_up = patch_len_prefixed(manifest, selected, backup_root, args.dry_run) - raw_patched, raw_skipped = patch_raw(manifest, selected, backup_root, backed_up, args.dry_run) - patched = len_patched + raw_patched - skipped = len_skipped + raw_skipped - validation_failures = [] if args.dry_run else validate_written(Path(manifest["root"]), patched) + raw_selected = [item for item in selected if item.get("method") == "raw-utf8-fallback"] + if raw_selected and not args.allow_raw_fallback: + rows = sorted({int(item["_row"]) for item in raw_selected}) + raise SystemExit( + "raw-utf8-fallback rows are audit-only by default; review the container and rerun " + f"with --allow-raw-fallback to approve rows {rows[:20]}" + ) + + real_root = Path(manifest["root"]).resolve() + relatives = selected_source_files(selected) + backup_root = args.backup_dir or ( + args.out_dir / "backups" / datetime.now().strftime("%Y%m%d-%H%M%S-%f") + ) + files_written: list[str] = [] + if args.dry_run or not relatives: + backed_up: set[str] = set() + raw_patched, raw_skipped = patch_raw(manifest, selected, backup_root, backed_up, True) + len_patched, len_skipped, _ = patch_len_prefixed(manifest, selected, backup_root, True, backed_up) + patched = raw_patched + len_patched + skipped = raw_skipped + len_skipped + validation_failures: list[dict] = [] + else: + with tempfile.TemporaryDirectory(prefix=".writeback-stage-", dir=args.out_dir) as temporary: + stage_root = Path(temporary) / "game-root" + stage_root.mkdir(parents=True) + expected_hashes = stage_source_files(real_root, stage_root, relatives) + staged_manifest = dict(manifest) + staged_manifest["root"] = str(stage_root) + stage_backups = Path(temporary) / "internal-backups" + backed_up = set() + # Raw offsets are byte-stable; apply them before UnityPy serializes objects. + raw_patched, raw_skipped = patch_raw(staged_manifest, selected, stage_backups, backed_up, False) + len_patched, len_skipped, _ = patch_len_prefixed( + staged_manifest, selected, stage_backups, False, backed_up + ) + patched = raw_patched + len_patched + skipped = raw_skipped + len_skipped + validation_failures = validate_written(stage_root, patched) + if not skipped and not validation_failures: + files_written = commit_staged_files( + real_root, stage_root, relatives, expected_hashes, backup_root + ) result = { "dry_run": args.dry_run, "game_root": manifest.get("root"), @@ -347,7 +544,7 @@ def main() -> int: "patched_total": len(patched), "skipped": skipped, "validation_failures": validation_failures, - "files_written": sorted(backed_up), + "files_written": files_written, } (args.out_dir / "writeback_report.json").write_text(json.dumps(result, ensure_ascii=False, indent=2), encoding="utf-8") print(json.dumps(result, ensure_ascii=False, indent=2))