From a741dbb9289febf05e9805b5e08b0c3d16d88448 Mon Sep 17 00:00:00 2001
From: prandla <prandla@r9.pm>
Date: Fri, 11 Jul 2025 00:58:42 +0300
Subject: [PATCH 1/2] make AWS more resilient towards missing ScoreTypes and
 Languages

---
 cms/server/admin/handlers/submission.py              | 12 ++++--------
 cms/server/admin/handlers/usertest.py                |  7 ++-----
 .../admin/templates/fragments/user_test_row.html     |  2 +-
 cms/server/admin/templates/macro/submission.html     |  2 +-
 cms/server/admin/templates/submission.html           | 12 ++++++++----
 cms/server/admin/templates/user_test.html            |  2 +-
 cms/server/jinja2_toolbox.py                         | 12 ++++++++++++
 7 files changed, 29 insertions(+), 20 deletions(-)

diff --git a/cms/server/admin/handlers/submission.py b/cms/server/admin/handlers/submission.py
index 2a062c7802..8e84b1e747 100644
--- a/cms/server/admin/handlers/submission.py
+++ b/cms/server/admin/handlers/submission.py
@@ -31,7 +31,7 @@
 import difflib
 
 from cms.db import Dataset, File, Submission
-from cms.grading.languagemanager import get_language
+from cms.server.jinja2_toolbox import safe_get_lang_filename
 from cmscommon.datetime import make_datetime
 from .base import BaseHandler, FileHandler, require_permission
 
@@ -80,10 +80,7 @@ def get(self, file_id):
         sub_file = self.safe_get_item(File, file_id)
         submission = sub_file.submission
 
-        real_filename = sub_file.filename
-        if submission.language is not None:
-            real_filename = real_filename.replace(
-                ".%l", get_language(submission.language).source_extension)
+        real_filename = safe_get_lang_filename(submission, sub_file.filename)
         digest = sub_file.digest
 
         self.sql_session.close()
@@ -130,10 +127,9 @@ def get(self, old_id, new_id):
         for fname in files_to_compare:
             if ".%l" in fname:
                 if sub_old.language == sub_new.language and sub_old.language is not None:
-                    ext = get_language(sub_old.language).source_extension
+                    real_fname = safe_get_lang_filename(sub_old, fname)
                 else:
-                    ext = ".txt"
-                real_fname = fname.replace(".%l", ext)
+                    real_fname = fname.replace(".%l", ".txt")
             else:
                 real_fname = fname
 
diff --git a/cms/server/admin/handlers/usertest.py b/cms/server/admin/handlers/usertest.py
index 9a6cdfdd54..d7d1caca25 100644
--- a/cms/server/admin/handlers/usertest.py
+++ b/cms/server/admin/handlers/usertest.py
@@ -21,7 +21,7 @@
 """
 
 from cms.db import Dataset, UserTestFile, UserTest
-from cms.grading.languagemanager import get_language
+from cms.server.jinja2_toolbox import safe_get_lang_filename
 
 from .base import BaseHandler, FileHandler, require_permission
 
@@ -60,10 +60,7 @@ def get(self, file_id):
         user_test_file = self.safe_get_item(UserTestFile, file_id)
         user_test = user_test_file.user_test
 
-        real_filename = user_test_file.filename
-        if user_test.language is not None:
-            real_filename = real_filename.replace(
-                ".%l", get_language(user_test.language).source_extension)
+        real_filename = safe_get_lang_filename(user_test, user_test_file.filename)
         digest = user_test_file.digest
 
         self.sql_session.close()
diff --git a/cms/server/admin/templates/fragments/user_test_row.html b/cms/server/admin/templates/fragments/user_test_row.html
index e3d24146a5..72fcca4532 100644
--- a/cms/server/admin/templates/fragments/user_test_row.html
+++ b/cms/server/admin/templates/fragments/user_test_row.html
@@ -52,7 +52,7 @@ <h3>Compilation output</h3>{# TODO: trim long outputs and add facility to see ra
   </td>
   <td>
     {% for filename, sub_file in ut.files|dictsort(by="key") %}
-      {% set real_filename = filename if ut.language is none else filename|replace(".%l", (ut.language|to_language).source_extension) %}
+      {% set real_filename = get_lang_filename(ut, filename) %}
     <a href="javascript:void(0);" onclick="utils.show_file('{{ real_filename }}','{{ url("user_test_file", sub_file.id) }}')">{{ real_filename }}</a><br/>
     {% endfor %}
   </td>
diff --git a/cms/server/admin/templates/macro/submission.html b/cms/server/admin/templates/macro/submission.html
index c6ee593137..668049d3ff 100644
--- a/cms/server/admin/templates/macro/submission.html
+++ b/cms/server/admin/templates/macro/submission.html
@@ -165,7 +165,7 @@ <h3>Compilation output</h3>{# TODO: trim long outputs and add facility to see ra
   </td>
   <td>
     {% for filename, sub_file in s.files|dictsort(by="key") %}
-      {% set real_filename = filename if s.language is none else filename|replace(".%l", (s.language|to_language).source_extension) %}
+      {% set real_filename = get_lang_filename(s, filename) %}
     <a href="javascript:void(0);" onclick="utils.show_file('{{ real_filename }}','{{ url("submission_file", sub_file.id) }}')">{{ real_filename }}</a><br/>
     {% endfor %}
   </td>
diff --git a/cms/server/admin/templates/submission.html b/cms/server/admin/templates/submission.html
index d00f72567c..a84aedce6a 100644
--- a/cms/server/admin/templates/submission.html
+++ b/cms/server/admin/templates/submission.html
@@ -48,7 +48,7 @@ <h2 id="title_details" class="toggling_on">Submission details</h2>
         <td>
           {% for filename in s.task.submission_format %}
             {% if filename in s.files %}
-              {% set real_filename = filename if s.language is none else filename|replace(".%l", (s.language|to_language).source_extension) %}
+              {% set real_filename = get_lang_filename(s, filename) %}
           <a href="javascript:void(0);" onclick="utils.show_file('{{ real_filename }}','{{ url("submission_file", s.files[filename].id) }}')">
             {{ real_filename }}
           </a>
@@ -234,10 +234,14 @@ <h2 id="title_evaluation_user" class="toggling_on">Evaluation (as seen by the us
 <div id="evaluation_user">
 
   <div class="score_details" id="evaluation_{{ s.id }}">
-    {% if s.tokened() %}
-      {{ st.get_html_details(sr.score_details, s.task.feedback_level)|safe }}
+    {% if st is defined %}
+      {% if s.tokened() %}
+        {{ st.get_html_details(sr.score_details, s.task.feedback_level)|safe }}
+      {% else %}
+        {{ st.get_html_details(sr.public_score_details, s.task.feedback_level)|safe }}
+      {% endif %}
     {% else %}
-      {{ st.get_html_details(sr.public_score_details, s.task.feedback_level)|safe }}
+      [Cannot get score type - see logs]
     {% endif %}
   </div>
 </div>
diff --git a/cms/server/admin/templates/user_test.html b/cms/server/admin/templates/user_test.html
index 8fb1f86ea0..2a7248291a 100644
--- a/cms/server/admin/templates/user_test.html
+++ b/cms/server/admin/templates/user_test.html
@@ -41,7 +41,7 @@ <h2 id="title_details" class="toggling_on">User test details</h2>
         <td>
           {% for filename in ut.task.submission_format %}
             {% if filename in ut.files %}
-              {% set real_filename = filename if ut.language is none else filename|replace(".%l", (ut.language|to_language).source_extension) %}
+              {% set real_filename = get_lang_filename(ut, filename) %}
           <a href="javascript:void(0);" onclick="utils.show_file('{{ real_filename }}','{{ url("user_test_file", ut.files[filename].id) }}')">
             {{ real_filename }}
           </a>
diff --git a/cms/server/jinja2_toolbox.py b/cms/server/jinja2_toolbox.py
index 4bfd9af245..ccacd3cdbd 100644
--- a/cms/server/jinja2_toolbox.py
+++ b/cms/server/jinja2_toolbox.py
@@ -35,7 +35,9 @@
     TOKEN_MODE_MIXED, FEEDBACK_LEVEL_FULL, FEEDBACK_LEVEL_RESTRICTED, \
     FEEDBACK_LEVEL_OI_RESTRICTED
 from cms.db import SubmissionResult, UserTestResult
+from cms.db.submission import Submission
 from cms.db.task import Dataset
+from cms.db.usertest import UserTest
 from cms.grading import format_status_text
 from cms.grading.languagemanager import get_language
 from cms.locale import Translation, DEFAULT_TRANSLATION
@@ -204,10 +206,20 @@ def safe_get_score_type(env: Environment, *, dataset: Dataset):
     except Exception as err:
         return env.undefined("ScoreType not found: %s" % err)
 
+def safe_get_lang_filename(submission: Submission | UserTest, filename: str) -> str:
+    if submission.language is None:
+        return filename
+    try:
+        lang = get_language(submission.language)
+        source_ext = lang.source_extension
+    except KeyError:
+        source_ext = ".txt"
+    return filename.replace(".%l", source_ext)
 
 def instrument_cms_toolbox(env: Environment):
     env.globals["get_task_type"] = safe_get_task_type
     env.globals["get_score_type"] = safe_get_score_type
+    env.globals["get_lang_filename"] = safe_get_lang_filename
 
     env.globals["get_mimetype_for_file_name"] = get_type_for_file_name
     env.globals["get_icon_for_mimetype"] = get_icon_for_type

From e5e6816e40ada106547db50f41695fefed65b115 Mon Sep 17 00:00:00 2001
From: prandla <prandla@r9.pm>
Date: Sat, 23 Aug 2025 19:46:12 +0300
Subject: [PATCH 2/2] Move safe_get_lang_filename to languagemanager.py, change
 fallback extension to .invalid_language

---
 cms/grading/languagemanager.py                | 24 ++++++++++++++++++-
 cms/server/admin/handlers/submission.py       |  6 ++---
 cms/server/admin/handlers/usertest.py         |  4 ++--
 .../templates/fragments/user_test_row.html    |  2 +-
 .../admin/templates/macro/submission.html     |  2 +-
 cms/server/admin/templates/submission.html    |  2 +-
 cms/server/admin/templates/user_test.html     |  2 +-
 cms/server/jinja2_toolbox.py                  | 14 +----------
 8 files changed, 33 insertions(+), 23 deletions(-)

diff --git a/cms/grading/languagemanager.py b/cms/grading/languagemanager.py
index 097dc0adc2..9bc8c917d5 100644
--- a/cms/grading/languagemanager.py
+++ b/cms/grading/languagemanager.py
@@ -18,10 +18,10 @@
 
 """Provide utilities to work with programming language classes."""
 
+import logging
 from cms import plugin_list
 from cms.grading.language import Language
 
-
 __all__ = [
     "LANGUAGES",
     "HEADER_EXTS", "SOURCE_EXTS", "OBJECT_EXTS",
@@ -29,6 +29,9 @@
 ]
 
 
+logger = logging.getLogger(__name__)
+
+
 LANGUAGES: list[Language] = list()
 _BY_NAME: dict[str, Language] = dict()
 HEADER_EXTS: set[str] = set()
@@ -50,6 +53,25 @@ def get_language(name: str) -> Language:
     return _BY_NAME[name]
 
 
+def safe_get_lang_filename(lang: str | None, filename: str) -> str:
+    """Get the filename of a file in a specific programming language,
+    avoiding errors if the language isn't recognized.
+
+    lang: name of the programming language
+    filename: filename template (containing .%l)
+    return: filename with the template replaced.
+    """
+    if lang is None:
+        return filename
+    try:
+        language = get_language(lang)
+        source_ext = language.source_extension
+    except KeyError:
+        logger.warning(f"Found invalid language {lang}!")
+        source_ext = ".invalid_language"
+    return filename.replace(".%l", source_ext)
+
+
 def filename_to_language(filename: str, available_languages: list[Language] | None=None) -> Language | None:
     """Return one of the languages inferred from the given filename.
 
diff --git a/cms/server/admin/handlers/submission.py b/cms/server/admin/handlers/submission.py
index 8e84b1e747..cd528cff1b 100644
--- a/cms/server/admin/handlers/submission.py
+++ b/cms/server/admin/handlers/submission.py
@@ -31,7 +31,7 @@
 import difflib
 
 from cms.db import Dataset, File, Submission
-from cms.server.jinja2_toolbox import safe_get_lang_filename
+from cms.grading.languagemanager import safe_get_lang_filename
 from cmscommon.datetime import make_datetime
 from .base import BaseHandler, FileHandler, require_permission
 
@@ -80,7 +80,7 @@ def get(self, file_id):
         sub_file = self.safe_get_item(File, file_id)
         submission = sub_file.submission
 
-        real_filename = safe_get_lang_filename(submission, sub_file.filename)
+        real_filename = safe_get_lang_filename(submission.language, sub_file.filename)
         digest = sub_file.digest
 
         self.sql_session.close()
@@ -127,7 +127,7 @@ def get(self, old_id, new_id):
         for fname in files_to_compare:
             if ".%l" in fname:
                 if sub_old.language == sub_new.language and sub_old.language is not None:
-                    real_fname = safe_get_lang_filename(sub_old, fname)
+                    real_fname = safe_get_lang_filename(sub_old.language, fname)
                 else:
                     real_fname = fname.replace(".%l", ".txt")
             else:
diff --git a/cms/server/admin/handlers/usertest.py b/cms/server/admin/handlers/usertest.py
index d7d1caca25..25328ed049 100644
--- a/cms/server/admin/handlers/usertest.py
+++ b/cms/server/admin/handlers/usertest.py
@@ -21,7 +21,7 @@
 """
 
 from cms.db import Dataset, UserTestFile, UserTest
-from cms.server.jinja2_toolbox import safe_get_lang_filename
+from cms.grading.languagemanager import safe_get_lang_filename
 
 from .base import BaseHandler, FileHandler, require_permission
 
@@ -60,7 +60,7 @@ def get(self, file_id):
         user_test_file = self.safe_get_item(UserTestFile, file_id)
         user_test = user_test_file.user_test
 
-        real_filename = safe_get_lang_filename(user_test, user_test_file.filename)
+        real_filename = safe_get_lang_filename(user_test.language, user_test_file.filename)
         digest = user_test_file.digest
 
         self.sql_session.close()
diff --git a/cms/server/admin/templates/fragments/user_test_row.html b/cms/server/admin/templates/fragments/user_test_row.html
index 72fcca4532..b4fac30e7f 100644
--- a/cms/server/admin/templates/fragments/user_test_row.html
+++ b/cms/server/admin/templates/fragments/user_test_row.html
@@ -52,7 +52,7 @@ <h3>Compilation output</h3>{# TODO: trim long outputs and add facility to see ra
   </td>
   <td>
     {% for filename, sub_file in ut.files|dictsort(by="key") %}
-      {% set real_filename = get_lang_filename(ut, filename) %}
+      {% set real_filename = get_lang_filename(ut.language, filename) %}
     <a href="javascript:void(0);" onclick="utils.show_file('{{ real_filename }}','{{ url("user_test_file", sub_file.id) }}')">{{ real_filename }}</a><br/>
     {% endfor %}
   </td>
diff --git a/cms/server/admin/templates/macro/submission.html b/cms/server/admin/templates/macro/submission.html
index 668049d3ff..0aec2850d9 100644
--- a/cms/server/admin/templates/macro/submission.html
+++ b/cms/server/admin/templates/macro/submission.html
@@ -165,7 +165,7 @@ <h3>Compilation output</h3>{# TODO: trim long outputs and add facility to see ra
   </td>
   <td>
     {% for filename, sub_file in s.files|dictsort(by="key") %}
-      {% set real_filename = get_lang_filename(s, filename) %}
+      {% set real_filename = get_lang_filename(s.language, filename) %}
     <a href="javascript:void(0);" onclick="utils.show_file('{{ real_filename }}','{{ url("submission_file", sub_file.id) }}')">{{ real_filename }}</a><br/>
     {% endfor %}
   </td>
diff --git a/cms/server/admin/templates/submission.html b/cms/server/admin/templates/submission.html
index a84aedce6a..be25b02fde 100644
--- a/cms/server/admin/templates/submission.html
+++ b/cms/server/admin/templates/submission.html
@@ -48,7 +48,7 @@ <h2 id="title_details" class="toggling_on">Submission details</h2>
         <td>
           {% for filename in s.task.submission_format %}
             {% if filename in s.files %}
-              {% set real_filename = get_lang_filename(s, filename) %}
+              {% set real_filename = get_lang_filename(s.language, filename) %}
           <a href="javascript:void(0);" onclick="utils.show_file('{{ real_filename }}','{{ url("submission_file", s.files[filename].id) }}')">
             {{ real_filename }}
           </a>
diff --git a/cms/server/admin/templates/user_test.html b/cms/server/admin/templates/user_test.html
index 2a7248291a..60d5213575 100644
--- a/cms/server/admin/templates/user_test.html
+++ b/cms/server/admin/templates/user_test.html
@@ -41,7 +41,7 @@ <h2 id="title_details" class="toggling_on">User test details</h2>
         <td>
           {% for filename in ut.task.submission_format %}
             {% if filename in ut.files %}
-              {% set real_filename = get_lang_filename(ut, filename) %}
+              {% set real_filename = get_lang_filename(ut.language, filename) %}
           <a href="javascript:void(0);" onclick="utils.show_file('{{ real_filename }}','{{ url("user_test_file", ut.files[filename].id) }}')">
             {{ real_filename }}
           </a>
diff --git a/cms/server/jinja2_toolbox.py b/cms/server/jinja2_toolbox.py
index ccacd3cdbd..0440b046bc 100644
--- a/cms/server/jinja2_toolbox.py
+++ b/cms/server/jinja2_toolbox.py
@@ -35,11 +35,9 @@
     TOKEN_MODE_MIXED, FEEDBACK_LEVEL_FULL, FEEDBACK_LEVEL_RESTRICTED, \
     FEEDBACK_LEVEL_OI_RESTRICTED
 from cms.db import SubmissionResult, UserTestResult
-from cms.db.submission import Submission
 from cms.db.task import Dataset
-from cms.db.usertest import UserTest
 from cms.grading import format_status_text
-from cms.grading.languagemanager import get_language
+from cms.grading.languagemanager import get_language, safe_get_lang_filename
 from cms.locale import Translation, DEFAULT_TRANSLATION
 from cmscommon.constants import \
     SCORE_MODE_MAX, SCORE_MODE_MAX_SUBTASK, SCORE_MODE_MAX_TOKENED_LAST
@@ -206,16 +204,6 @@ def safe_get_score_type(env: Environment, *, dataset: Dataset):
     except Exception as err:
         return env.undefined("ScoreType not found: %s" % err)
 
-def safe_get_lang_filename(submission: Submission | UserTest, filename: str) -> str:
-    if submission.language is None:
-        return filename
-    try:
-        lang = get_language(submission.language)
-        source_ext = lang.source_extension
-    except KeyError:
-        source_ext = ".txt"
-    return filename.replace(".%l", source_ext)
-
 def instrument_cms_toolbox(env: Environment):
     env.globals["get_task_type"] = safe_get_task_type
     env.globals["get_score_type"] = safe_get_score_type