From 4d041e6116c2477dc93968089c31231117ecec11 Mon Sep 17 00:00:00 2001 From: insolor <2442833+insolor@users.noreply.github.com> Date: Sat, 11 Oct 2025 19:10:37 +0300 Subject: [PATCH 1/9] Implement create_mo_files.py --- automation/automation/create_mo_files.py | 60 ++++++++++++++++++++++++ automation/automation/process.py | 16 ++----- automation/automation/utils.py | 12 +++++ 3 files changed, 75 insertions(+), 13 deletions(-) create mode 100644 automation/automation/create_mo_files.py create mode 100644 automation/automation/utils.py diff --git a/automation/automation/create_mo_files.py b/automation/automation/create_mo_files.py new file mode 100644 index 00000000..f4094ea1 --- /dev/null +++ b/automation/automation/create_mo_files.py @@ -0,0 +1,60 @@ + +from pathlib import Path + +import typer +from df_translation_toolkit.convert import text_to_mo +from df_translation_toolkit.validation.validation_models import Diagnostics +from loguru import logger + +from automation.load_config import load_config +from automation.models import Context, LanguageInfo +from automation.utils import get_po_file_path + + +def process_resource(mo_directory: Path, language: LanguageInfo, resource: str, context: Context) -> None: + po_file_path = get_po_file_path( + working_directory=context.working_directory, + project_name=context.config.source.project, + resource_name="hardcoded_steam", + language_code=language.code, + ) + mo_file_path = mo_directory / f"{resource}.mo" + diagnostics = Diagnostics() + with po_file_path.open("rt", encoding="utf-8") as po_file, mo_file_path.open("wb") as mo_file: + text_to_mo.convert(po_file, mo_file, diagnostics) + + errors_file_path = mo_directory / f"{resource}_errors.txt" + if errors_file_path.exists(): + errors_file_path.unlink() + + if diagnostics.contains_problems(): + with errors_file_path.open("w", encoding="utf-8") as errors_file: + errors_file.write(str(diagnostics)) + + +@logger.catch(reraise=True) +def process_language(language: LanguageInfo, context: Context) -> None: + translation_build_directory = context.working_directory / "translation_build" + mo_directory = translation_build_directory / "mo" / language.name + mo_directory.mkdir(parents=True, exist_ok=True) + for resource in ("objects", "text_set"): + process_resource(mo_directory, language, resource, context) + + +def process_all(context: Context) -> None: + for language in context.config.languages: + process_language(language, context) + + +app = typer.Typer() + + +@app.command() +def main(working_directory: Path) -> None: + config = load_config(working_directory / "config.yaml") + context = Context(config=config, working_directory=working_directory) + process_all(context) + + +if __name__ == "__main__": + app() diff --git a/automation/automation/process.py b/automation/automation/process.py index d7453395..f02226d6 100644 --- a/automation/automation/process.py +++ b/automation/automation/process.py @@ -15,21 +15,11 @@ from automation.load_config import load_config from automation.models import Context, LanguageInfo +from automation.utils import get_po_file_path alternative_encodings.register_all() -def get_po_file_path(*, working_directory: Path, project_name: str, resource_name: str, language_code: str) -> Path: - return ( - working_directory - / "translations-backup" - / "translations" - / project_name - / resource_name - / f"{language_code}.po" - ) - - def load_po_file(file_path: Path) -> list[tuple[str, str]]: with file_path.open(encoding="utf-8") as file: return simple_read_po(file) @@ -96,7 +86,7 @@ def process_objects( @logger.catch(reraise=True) -def process(language: LanguageInfo, context: Context) -> None: +def process_language(language: LanguageInfo, context: Context) -> None: translation_build_directory = context.working_directory / "translation_build" csv_directory = translation_build_directory / "csv" / language.name csv_directory.mkdir(parents=True, exist_ok=True) @@ -129,7 +119,7 @@ def process(language: LanguageInfo, context: Context) -> None: def process_all(context: Context) -> None: for language in context.config.languages: - process(language, context) + process_language(language, context) app = typer.Typer() diff --git a/automation/automation/utils.py b/automation/automation/utils.py new file mode 100644 index 00000000..65a48f13 --- /dev/null +++ b/automation/automation/utils.py @@ -0,0 +1,12 @@ +from pathlib import Path + + +def get_po_file_path(*, working_directory: Path, project_name: str, resource_name: str, language_code: str) -> Path: + return ( + working_directory + / "translations-backup" + / "translations" + / project_name + / resource_name + / f"{language_code}.po" + ) From 234cb93891b3a6df2d56affedebefcdb76c0fdf3 Mon Sep 17 00:00:00 2001 From: insolor <2442833+insolor@users.noreply.github.com> Date: Sat, 11 Oct 2025 19:29:53 +0300 Subject: [PATCH 2/9] Add produce-mo job --- .github/workflows/build.yml | 36 ++++++++++++++++++++++++++++++++++++ automation/pyproject.toml | 1 + 2 files changed, 37 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 016a33fb..cc31e5d1 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -66,6 +66,42 @@ jobs: name: artifact_csv path: translation_build + produce-mo: + runs-on: ubuntu-latest + timeout-minutes: 60 + steps: + - uses: actions/checkout@v5 + + - name: Checkout and update the translation-backup submodule + run: git submodule update --init --remote + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: 3.12 + + - name: Install uv + uses: astral-sh/setup-uv@v6 + with: + version: latest + + - name: Install library and dependencies + run: | + cd automation + uv sync --no-group dev + pipx install poethepoet + + - name: Create mo files + run: | + cd automation + poe create-mo ../ + + - name: Upload mo artifact + uses: actions/upload-artifact@v4 + with: + name: artifact_mo + path: translation_build + push-changes: needs: produce-csv runs-on: ubuntu-latest diff --git a/automation/pyproject.toml b/automation/pyproject.toml index 4cfc20dc..ddad9953 100644 --- a/automation/pyproject.toml +++ b/automation/pyproject.toml @@ -42,3 +42,4 @@ ignore = [ [tool.poe.tasks] process.script = "automation.process:app" +create-mo.script = "automation.create_mo_files:app" From 204955d524993b8d989f4f0bcc431ea32f8947d4 Mon Sep 17 00:00:00 2001 From: insolor <2442833+insolor@users.noreply.github.com> Date: Sat, 11 Oct 2025 19:31:35 +0300 Subject: [PATCH 3/9] Add produce-mo as a requirement to push-changes job --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index cc31e5d1..12c70af9 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -103,7 +103,7 @@ jobs: path: translation_build push-changes: - needs: produce-csv + needs: [produce-csv, produce-mo] runs-on: ubuntu-latest timeout-minutes: 60 steps: From c120703ee89b42b0ccc79adad71aedd1bef8b7ae Mon Sep 17 00:00:00 2001 From: insolor <2442833+insolor@users.noreply.github.com> Date: Sat, 11 Oct 2025 19:54:28 +0300 Subject: [PATCH 4/9] Rename files --- .github/workflows/build.yml | 8 ++++---- .../automation/{process.py => produce_csv_files.py} | 0 .../{create_mo_files.py => produce_mo_files.py} | 0 automation/pyproject.toml | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) rename automation/automation/{process.py => produce_csv_files.py} (100%) rename automation/automation/{create_mo_files.py => produce_mo_files.py} (100%) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 12c70af9..8f34f37b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -55,10 +55,10 @@ jobs: uv sync --no-group dev pipx install poethepoet - - name: Process po files + - name: Produce csv files run: | cd automation - poe process ../ + poe produce-csv ../ - name: Upload csv artifact uses: actions/upload-artifact@v4 @@ -91,10 +91,10 @@ jobs: uv sync --no-group dev pipx install poethepoet - - name: Create mo files + - name: Produce mo files run: | cd automation - poe create-mo ../ + poe produce-mo ../ - name: Upload mo artifact uses: actions/upload-artifact@v4 diff --git a/automation/automation/process.py b/automation/automation/produce_csv_files.py similarity index 100% rename from automation/automation/process.py rename to automation/automation/produce_csv_files.py diff --git a/automation/automation/create_mo_files.py b/automation/automation/produce_mo_files.py similarity index 100% rename from automation/automation/create_mo_files.py rename to automation/automation/produce_mo_files.py diff --git a/automation/pyproject.toml b/automation/pyproject.toml index ddad9953..7ffba90e 100644 --- a/automation/pyproject.toml +++ b/automation/pyproject.toml @@ -41,5 +41,5 @@ ignore = [ ] [tool.poe.tasks] -process.script = "automation.process:app" -create-mo.script = "automation.create_mo_files:app" +produce-csv.script = "automation.produce_csv_files:app" +produce-mo.script = "automation.produce_mo_files:app" From 37bdf676a141168c04f65b980af3b9221d2b376f Mon Sep 17 00:00:00 2001 From: insolor <2442833+insolor@users.noreply.github.com> Date: Sat, 11 Oct 2025 20:02:50 +0300 Subject: [PATCH 5/9] Add a parameter for output directory --- .github/workflows/build.yml | 8 ++++---- automation/automation/models.py | 1 + automation/automation/produce_csv_files.py | 10 +++++----- automation/automation/produce_mo_files.py | 8 ++++---- 4 files changed, 14 insertions(+), 13 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 8f34f37b..679f7810 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -58,13 +58,13 @@ jobs: - name: Produce csv files run: | cd automation - poe produce-csv ../ + poe produce-csv ../ ../output - name: Upload csv artifact uses: actions/upload-artifact@v4 with: name: artifact_csv - path: translation_build + path: output produce-mo: runs-on: ubuntu-latest @@ -94,13 +94,13 @@ jobs: - name: Produce mo files run: | cd automation - poe produce-mo ../ + poe produce-mo ../ ../output - name: Upload mo artifact uses: actions/upload-artifact@v4 with: name: artifact_mo - path: translation_build + path: output push-changes: needs: [produce-csv, produce-mo] diff --git a/automation/automation/models.py b/automation/automation/models.py index 62b98340..e1e95b5b 100644 --- a/automation/automation/models.py +++ b/automation/automation/models.py @@ -21,3 +21,4 @@ class Config(BaseModel): class Context(BaseModel): config: Config working_directory: Path + destintion_directory: Path diff --git a/automation/automation/produce_csv_files.py b/automation/automation/produce_csv_files.py index f02226d6..a21fde4e 100644 --- a/automation/automation/produce_csv_files.py +++ b/automation/automation/produce_csv_files.py @@ -87,8 +87,7 @@ def process_objects( @logger.catch(reraise=True) def process_language(language: LanguageInfo, context: Context) -> None: - translation_build_directory = context.working_directory / "translation_build" - csv_directory = translation_build_directory / "csv" / language.name + csv_directory = context.destintion_directory / "csv" / language.name csv_directory.mkdir(parents=True, exist_ok=True) hardcoded_csv_file_path = csv_directory / "dfint_dictionary.csv" csv_hardcoded_data = process_hardcoded( @@ -101,7 +100,7 @@ def process_language(language: LanguageInfo, context: Context) -> None: exclude = {first for first, _ in csv_hardcoded_data} - csv_with_objects_directory = translation_build_directory / "csv_with_objects" / language.name + csv_with_objects_directory = context.destintion_directory / "csv_with_objects" / language.name csv_with_objects_directory.mkdir(parents=True, exist_ok=True) with_objects_csv_file_path = csv_with_objects_directory / "dfint_dictionary.csv" @@ -126,9 +125,10 @@ def process_all(context: Context) -> None: @app.command() -def main(working_directory: Path) -> None: +def main(working_directory: Path, destination_directory: Path | None = None) -> None: config = load_config(working_directory / "config.yaml") - context = Context(config=config, working_directory=working_directory) + destination_directory = destination_directory or working_directory / "translation_build" + context = Context(config=config, working_directory=working_directory, destintion_directory=destination_directory) process_all(context) diff --git a/automation/automation/produce_mo_files.py b/automation/automation/produce_mo_files.py index f4094ea1..7c729c19 100644 --- a/automation/automation/produce_mo_files.py +++ b/automation/automation/produce_mo_files.py @@ -34,8 +34,7 @@ def process_resource(mo_directory: Path, language: LanguageInfo, resource: str, @logger.catch(reraise=True) def process_language(language: LanguageInfo, context: Context) -> None: - translation_build_directory = context.working_directory / "translation_build" - mo_directory = translation_build_directory / "mo" / language.name + mo_directory = context.destintion_directory / "mo" / language.name mo_directory.mkdir(parents=True, exist_ok=True) for resource in ("objects", "text_set"): process_resource(mo_directory, language, resource, context) @@ -50,9 +49,10 @@ def process_all(context: Context) -> None: @app.command() -def main(working_directory: Path) -> None: +def main(working_directory: Path, destination_directory: Path | None = None) -> None: config = load_config(working_directory / "config.yaml") - context = Context(config=config, working_directory=working_directory) + destination_directory = destination_directory or working_directory / "translation_build" + context = Context(config=config, working_directory=working_directory, destintion_directory=destination_directory) process_all(context) From 61b0c72f25c67c22d00f5bbb5d445fdcb79b90c4 Mon Sep 17 00:00:00 2001 From: insolor <2442833+insolor@users.noreply.github.com> Date: Sat, 11 Oct 2025 20:07:00 +0300 Subject: [PATCH 6/9] Make destination directory parameter mandatory --- automation/automation/produce_csv_files.py | 3 +-- automation/automation/produce_mo_files.py | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/automation/automation/produce_csv_files.py b/automation/automation/produce_csv_files.py index a21fde4e..326fe6e0 100644 --- a/automation/automation/produce_csv_files.py +++ b/automation/automation/produce_csv_files.py @@ -125,9 +125,8 @@ def process_all(context: Context) -> None: @app.command() -def main(working_directory: Path, destination_directory: Path | None = None) -> None: +def main(working_directory: Path, destination_directory: Path) -> None: config = load_config(working_directory / "config.yaml") - destination_directory = destination_directory or working_directory / "translation_build" context = Context(config=config, working_directory=working_directory, destintion_directory=destination_directory) process_all(context) diff --git a/automation/automation/produce_mo_files.py b/automation/automation/produce_mo_files.py index 7c729c19..4d171439 100644 --- a/automation/automation/produce_mo_files.py +++ b/automation/automation/produce_mo_files.py @@ -49,9 +49,8 @@ def process_all(context: Context) -> None: @app.command() -def main(working_directory: Path, destination_directory: Path | None = None) -> None: +def main(working_directory: Path, destination_directory: Path) -> None: config = load_config(working_directory / "config.yaml") - destination_directory = destination_directory or working_directory / "translation_build" context = Context(config=config, working_directory=working_directory, destintion_directory=destination_directory) process_all(context) From eab5065dfc3972a931aaae00e9a134821be4c701 Mon Sep 17 00:00:00 2001 From: insolor <2442833+insolor@users.noreply.github.com> Date: Sat, 11 Oct 2025 20:12:04 +0300 Subject: [PATCH 7/9] Fix resource name parameter --- automation/automation/produce_mo_files.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/automation/automation/produce_mo_files.py b/automation/automation/produce_mo_files.py index 4d171439..57697f0b 100644 --- a/automation/automation/produce_mo_files.py +++ b/automation/automation/produce_mo_files.py @@ -15,7 +15,7 @@ def process_resource(mo_directory: Path, language: LanguageInfo, resource: str, po_file_path = get_po_file_path( working_directory=context.working_directory, project_name=context.config.source.project, - resource_name="hardcoded_steam", + resource_name=resource, language_code=language.code, ) mo_file_path = mo_directory / f"{resource}.mo" From 145718c7e89ba2b9c119bcdc4f9ed5cc1812fc51 Mon Sep 17 00:00:00 2001 From: insolor <2442833+insolor@users.noreply.github.com> Date: Sat, 11 Oct 2025 21:44:46 +0300 Subject: [PATCH 8/9] Update df-translation-toolkit --- automation/pyproject.toml | 2 +- automation/uv.lock | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/automation/pyproject.toml b/automation/pyproject.toml index 7ffba90e..5cd37396 100644 --- a/automation/pyproject.toml +++ b/automation/pyproject.toml @@ -10,7 +10,7 @@ dependencies = [ "loguru>=0.7.3,<0.8", "pydantic>=2,<3", "strictyaml>=1.7.3,<2", - "df-translation-toolkit>=0.11.2,<0.12", + "df-translation-toolkit>=0.12.0,<0.13", "alternative-encodings>=0.3.1,<0.4", "tqdm>=4.67.1,<5", ] diff --git a/automation/uv.lock b/automation/uv.lock index b7640039..fc3bf97f 100644 --- a/automation/uv.lock +++ b/automation/uv.lock @@ -45,7 +45,7 @@ dev = [ [package.metadata] requires-dist = [ { name = "alternative-encodings", specifier = ">=0.3.1,<0.4", index = "https://dfint.github.io/pypi-index/" }, - { name = "df-translation-toolkit", specifier = ">=0.11.2,<0.12", index = "https://dfint.github.io/pypi-index/" }, + { name = "df-translation-toolkit", specifier = ">=0.12.0,<0.13", index = "https://dfint.github.io/pypi-index/" }, { name = "loguru", specifier = ">=0.7.3,<0.8" }, { name = "pydantic", specifier = ">=2,<3" }, { name = "strictyaml", specifier = ">=1.7.3,<2" }, @@ -203,7 +203,7 @@ wheels = [ [[package]] name = "df-translation-toolkit" -version = "0.11.2" +version = "0.12.0" source = { registry = "https://dfint.github.io/pypi-index/" } dependencies = [ { name = "babel" }, @@ -215,7 +215,7 @@ dependencies = [ { name = "unidecode" }, ] wheels = [ - { url = "https://github.com/dfint/df-translation-toolkit/releases/download/0.11.2/df_translation_toolkit-0.11.2-py3-none-any.whl" }, + { url = "https://github.com/dfint/df-translation-toolkit/releases/download/0.12.0/df_translation_toolkit-0.12.0-py3-none-any.whl" }, ] [[package]] From 42aae6981069d29be3796c83b029b490fdccc83a Mon Sep 17 00:00:00 2001 From: insolor <2442833+insolor@users.noreply.github.com> Date: Sat, 11 Oct 2025 21:48:50 +0300 Subject: [PATCH 9/9] Switch convertion functions according to the resource type --- automation/automation/produce_mo_files.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/automation/automation/produce_mo_files.py b/automation/automation/produce_mo_files.py index 57697f0b..66861bad 100644 --- a/automation/automation/produce_mo_files.py +++ b/automation/automation/produce_mo_files.py @@ -2,7 +2,7 @@ from pathlib import Path import typer -from df_translation_toolkit.convert import text_to_mo +from df_translation_toolkit.convert import objects_po_to_mo, text_set_po_to_mo from df_translation_toolkit.validation.validation_models import Diagnostics from loguru import logger @@ -10,6 +10,11 @@ from automation.models import Context, LanguageInfo from automation.utils import get_po_file_path +conversion_functions = { + "objects": objects_po_to_mo.convert, + "text_set": text_set_po_to_mo.convert, +} + def process_resource(mo_directory: Path, language: LanguageInfo, resource: str, context: Context) -> None: po_file_path = get_po_file_path( @@ -21,7 +26,7 @@ def process_resource(mo_directory: Path, language: LanguageInfo, resource: str, mo_file_path = mo_directory / f"{resource}.mo" diagnostics = Diagnostics() with po_file_path.open("rt", encoding="utf-8") as po_file, mo_file_path.open("wb") as mo_file: - text_to_mo.convert(po_file, mo_file, diagnostics) + conversion_functions[resource](po_file, mo_file, diagnostics) errors_file_path = mo_directory / f"{resource}_errors.txt" if errors_file_path.exists():