From 4324f5058d7c5e9b41cd1baee93d6fd964dc3097 Mon Sep 17 00:00:00 2001 From: prandla Date: Tue, 22 Jul 2025 22:51:27 +0300 Subject: [PATCH] Fix translation of file types in CWS Previously, we tried to find shared-mime-info.mo and use that to translate the descriptions obtained from xdg.Mime into the right language. This was not the correct approach: first, the path to shared-mime-info.mo varied (on Ubuntu it was in locale-langpack instead of locale), and second, sometimes (e.g. on Fedora) it was not present at all. The right place to find these translations is in the mimetype xml files themselves. However, this caused other issues: first, pyxdg does not have proper support for using more than one language. I hacked around this by setting the (global) language before each call, and clearing pyxdg's own cache. And then adding my own cache on top that can cache multiple languages. The second problem was that on Debian/Ubuntu, the xml:lang tags that specify the language codes for each string use an incorrect format: they use POSIX-style locale names (like zh_CN) instead of XML-compliant BCP47 language codes (like zh-Hans-CN). pyxdg only knows how to handle these incorrect names, so I had to implement a hack to get it to recognize the correct names too. Also deleted the undocumented and now-unused shared_mime_info_prefix config option, and mentioned shared-mime-info as a dependency in the installation docs. --- cms/conf.py | 6 ----- cms/locale/locale.py | 12 ++++----- .../contest/templates/task_description.html | 6 ++--- cms/server/jinja2_toolbox.py | 4 +-- cmscommon/mimetypes.py | 25 +++++++++++++++++-- .../unit_tests/cmscommon/mimetypes_test.py | 4 +-- cmstestsuite/unit_tests/locale/locale_test.py | 18 ++++++------- docs/Installation.rst | 5 ++-- 8 files changed, 44 insertions(+), 36 deletions(-) diff --git a/cms/conf.py b/cms/conf.py index 5d656a2672..4d3468df77 100644 --- a/cms/conf.py +++ b/cms/conf.py @@ -140,12 +140,6 @@ def __init__(self): self.max_input_length = 5_000_000 # 5 MB self.stl_path = "/usr/share/cppreference/doc/html/" self.docs_path = None - # Prefix of 'shared-mime-info'[1] installation. It can be found - # out using `pkg-config --variable=prefix shared-mime-info`, but - # it's almost universally the same (i.e. '/usr') so it's hardly - # necessary to change it. - # [1] http://freedesktop.org/wiki/Software/shared-mime-info - self.shared_mime_info_prefix = "/usr" self.contest_admin_token = None # AdminWebServer. diff --git a/cms/locale/locale.py b/cms/locale/locale.py index 951cc67716..8ce09d51ed 100644 --- a/cms/locale/locale.py +++ b/cms/locale/locale.py @@ -33,7 +33,6 @@ import copy import logging import math -import os import babel.core import babel.dates @@ -43,8 +42,8 @@ import babel.units import importlib.resources -from cms import config from cmscommon.datetime import utc +from cmscommon.mimetypes import get_name_for_type from datetime import datetime, tzinfo, timedelta @@ -70,9 +69,6 @@ def __init__(self, lang_code, mofile=None): self.translation = babel.support.Translations(mofile, domain="cms") else: self.translation = babel.support.NullTranslations() - self.mimetype_translation = babel.support.Translations.load( - os.path.join(config.shared_mime_info_prefix, "share", "locale"), - [self.locale], "shared-mime-info") @property def identifier(self) -> str: @@ -266,7 +262,11 @@ def format_locale(self, code: str) -> str: return code def translate_mimetype(self, mimetype: str) -> str: - return self.mimetype_translation.gettext(mimetype) + lang_code = self.identifier + alt_lang_code = babel.core.get_locale_identifier( + (self.locale.language, self.locale.territory), sep="_" + ) + return get_name_for_type(mimetype, lang_code, alt_lang_code) DEFAULT_TRANSLATION = Translation("en") diff --git a/cms/server/contest/templates/task_description.html b/cms/server/contest/templates/task_description.html index ed4c0f1d3e..b560ab9bea 100644 --- a/cms/server/contest/templates/task_description.html +++ b/cms/server/contest/templates/task_description.html @@ -177,10 +177,8 @@

{% trans %}Attachments{% endtrans %}

{% for filename, attachment in task.attachments|dictsort(by="key") %} {% set mime_type = get_mimetype_for_file_name(filename) %} {% if mime_type is not none %} - {% set type_name = get_name_for_mimetype(mime_type) %} {% set type_icon = get_icon_for_mimetype(mime_type) %} {% else %} - {% set type_name = none %} {% set type_icon = none %} {% endif %} {% set file_size = handler.application.service.file_cacher.get_size(attachment.digest) %} @@ -195,8 +193,8 @@

{% trans %}Attachments{% endtrans %}

{{ filename }} {{ file_size|format_size }} - {% if type_name is not none %} - {{ translation.translate_mimetype(type_name) }} + {% if mime_type is not none %} + {{ translation.translate_mimetype(mime_type) }} {% endif %} diff --git a/cms/server/jinja2_toolbox.py b/cms/server/jinja2_toolbox.py index f13cda437f..4bfd9af245 100644 --- a/cms/server/jinja2_toolbox.py +++ b/cms/server/jinja2_toolbox.py @@ -42,8 +42,7 @@ from cmscommon.constants import \ SCORE_MODE_MAX, SCORE_MODE_MAX_SUBTASK, SCORE_MODE_MAX_TOKENED_LAST from cmscommon.datetime import make_datetime, make_timestamp, utc, local_tz -from cmscommon.mimetypes import get_type_for_file_name, get_name_for_type, \ - get_icon_for_type +from cmscommon.mimetypes import get_type_for_file_name, get_icon_for_type @contextfilter @@ -211,7 +210,6 @@ def instrument_cms_toolbox(env: Environment): env.globals["get_score_type"] = safe_get_score_type env.globals["get_mimetype_for_file_name"] = get_type_for_file_name - env.globals["get_name_for_mimetype"] = get_name_for_type env.globals["get_icon_for_mimetype"] = get_icon_for_type env.filters["to_language"] = get_language diff --git a/cmscommon/mimetypes.py b/cmscommon/mimetypes.py index 03131c8401..9eea2cc2df 100644 --- a/cmscommon/mimetypes.py +++ b/cmscommon/mimetypes.py @@ -16,10 +16,12 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . +import functools import os.path import xdg.BaseDirectory import xdg.Mime +import xdg.Locale __all__ = [ @@ -59,17 +61,36 @@ def get_icon_for_type(typename: str) -> str: return _icons[typename] return mimetype.media + "-x-generic" - -def get_name_for_type(typename: str) -> str: +# xdg.Mime is by default memoized, but since we need to change the language, we +# need to wipe the cache to load the correct language. So use our own caching +# on top of it. +@functools.cache +def get_name_for_type(typename: str, language: str, alt_language: str) -> str: """Get the natural language description of the MIME type. typename: a MIME type, e.g., "application/pdf". + language: the BCP47 code of the language for which to return the result. + alt_language: underscore-separated form of the language code, to work + around incorrect behavior in pyxdg. return: the human-readable description (also called comment) of the given MIME type, e.g., "PDF document". """ + # pyxdg expects the locale field to be provided as a posix-style locale + # name, e.g. zh_CN. It assumes this in both the provided language name, and + # in the xml:lang attribute of the mimetype xml files. Some distributions + # instead use BCP47 language codes, e.g. zh-Hans-CN, in the mimetype xml + # files (which is semantically more correct, as this is mandated by the xml + # spec). + # First parse the language from the posix format. + xdg.Locale.update(alt_language) + # Then, we make pyxdg think the BCP47 code is another variant of the + # current language name. + xdg.Locale.langs += [language] mimetype = xdg.Mime.lookup(typename).canonical() + # Force reloading the comment, because the language might have changed. + mimetype._comment = None return mimetype.get_comment() diff --git a/cmstestsuite/unit_tests/cmscommon/mimetypes_test.py b/cmstestsuite/unit_tests/cmscommon/mimetypes_test.py index cd9c27a787..b2d7380dda 100755 --- a/cmstestsuite/unit_tests/cmscommon/mimetypes_test.py +++ b/cmstestsuite/unit_tests/cmscommon/mimetypes_test.py @@ -42,11 +42,11 @@ def test_generic(self): class TestGetNameForType(unittest.TestCase): def test_basic(self): - self.assertEqual(get_name_for_type("application/pdf"), + self.assertEqual(get_name_for_type("application/pdf", "en", "en"), "PDF document") def test_alias(self): - self.assertEqual(get_name_for_type("text/x-octave"), + self.assertEqual(get_name_for_type("text/x-octave", "en", "en"), "MATLAB file") diff --git a/cmstestsuite/unit_tests/locale/locale_test.py b/cmstestsuite/unit_tests/locale/locale_test.py index e81d0fe458..e0c76539a2 100755 --- a/cmstestsuite/unit_tests/locale/locale_test.py +++ b/cmstestsuite/unit_tests/locale/locale_test.py @@ -53,6 +53,7 @@ ITALIAN = Translation("it") DANISH = Translation("da") CHINESE = Translation("zh_CN") +CHINESE_TRADITIONAL = Translation("zh_TW") class TestIdentifier(unittest.TestCase): @@ -591,20 +592,15 @@ def test_localized_decimal_and_thousands_separators(self): class TestTranslateMimetype(unittest.TestCase): - @unittest.skipIf(not os.path.isfile( - "/usr/share/locale/it/LC_MESSAGES/shared-mime-info.mo"), - reason="need Italian shared-mime-info translation") def test_translate_mimetype(self): - self.assertEqual(ENGLISH.translate_mimetype("PDF document"), + self.assertEqual(ENGLISH.translate_mimetype("application/pdf"), "PDF document") - self.assertEqual(ITALIAN.translate_mimetype("PDF document"), + self.assertEqual(ITALIAN.translate_mimetype("application/pdf"), "Documento PDF") - - def test_graceful_failure(self): - self.assertEqual(ENGLISH.translate_mimetype("Not a MIME type"), - "Not a MIME type") - self.assertEqual(ITALIAN.translate_mimetype("Not a MIME type"), - "Not a MIME type") + self.assertEqual(CHINESE.translate_mimetype("application/pdf"), + "PDF 文档") + self.assertEqual(CHINESE_TRADITIONAL.translate_mimetype("application/pdf"), + "PDF 文件") class TestFilterLanguageCodes(unittest.TestCase): diff --git a/docs/Installation.rst b/docs/Installation.rst index e5eba765db..f441462e2b 100644 --- a/docs/Installation.rst +++ b/docs/Installation.rst @@ -70,7 +70,7 @@ On Ubuntu 24.04, one will need to run the following script as root to satisfy al postgresql postgresql-client \ python3.12 python3.12-dev python3-pip python3-venv \ libpq-dev libcups2-dev libyaml-dev libffi-dev \ - cppreference-doc-en-html zip curl + shared-mime-info cppreference-doc-en-html zip curl # Isolate from upstream package repository echo 'deb [arch=amd64 signed-by=/etc/apt/keyrings/isolate.asc] http://www.ucw.cz/isolate/debian/ noble-isolate main' >/etc/apt/sources.list.d/isolate.list @@ -92,7 +92,8 @@ On Arch Linux, run the following commands as root to install almost all dependen .. sourcecode:: bash pacman -S base-devel jdk8-openjdk fpc postgresql postgresql-client \ - python python-pip postgresql-libs libcups libyaml + python python-pip postgresql-libs libcups libyaml \ + shared-mime-info # Install the following from AUR. # https://aur.archlinux.org/packages/cppreference/