diff --git a/cms/db/__init__.py b/cms/db/__init__.py index 1e7ee98152..18f87364c8 100644 --- a/cms/db/__init__.py +++ b/cms/db/__init__.py @@ -81,7 +81,7 @@ # Instantiate or import these objects. -version = 45 +version = 46 engine = create_engine(config.database, echo=config.database_debug, pool_timeout=60, pool_recycle=120) diff --git a/cms/db/submission.py b/cms/db/submission.py index 6dacba9b7f..10647e1d11 100644 --- a/cms/db/submission.py +++ b/cms/db/submission.py @@ -362,8 +362,11 @@ class SubmissionResult(Base): compilation_shard: int | None = Column( Integer, nullable=True) - compilation_sandbox: str | None = Column( - Unicode, + compilation_sandbox_paths: list[str] | None = Column( + ARRAY(Unicode), + nullable=True) + compilation_sandbox_digests: list[str] | None = Column( + ARRAY(String), nullable=True) # Evaluation outcome (can be None = yet to evaluate, "ok" = @@ -594,16 +597,22 @@ def invalidate_compilation(self): self.compilation_memory = None self.compilation_shard = None self.compilation_sandbox = None + self.compilation_sandbox_digests = [] self.executables = {} - def invalidate_evaluation(self): + def invalidate_evaluation(self, testcase_id: int | None = None): """Blank the evaluation outcomes and the score. + testcase_id: ID of testcase to invalidate, or None to invalidate all. + """ self.invalidate_score() self.evaluation_outcome = None self.evaluation_tries = 0 - self.evaluations = [] + if testcase_id: + self.evaluations = [e for e in self.evaluations if e.testcase_id != testcase_id] + else: + self.evaluations = [] def invalidate_score(self): """Blank the score. @@ -774,8 +783,11 @@ class Evaluation(Base): evaluation_shard: int | None = Column( Integer, nullable=True) - evaluation_sandbox: str | None = Column( - Unicode, + evaluation_sandbox_paths: list[str] | None = Column( + ARRAY(Unicode), + nullable=True) + evaluation_sandbox_digests: list[str] | None = Column( + ARRAY(String), nullable=True) @property diff --git a/cms/db/usertest.py b/cms/db/usertest.py index 35381a52c0..6c9e8a2a9a 100644 --- a/cms/db/usertest.py +++ b/cms/db/usertest.py @@ -312,8 +312,11 @@ class UserTestResult(Base): compilation_shard: int | None = Column( Integer, nullable=True) - compilation_sandbox: str | None = Column( - String, + compilation_sandbox_paths: list[str] | None = Column( + ARRAY(Unicode), + nullable=True) + compilation_sandbox_digests: list[str] | None = Column( + ARRAY(String), nullable=True) # Evaluation outcome (can be None = yet to evaluate, "ok" = @@ -352,8 +355,11 @@ class UserTestResult(Base): evaluation_shard: int | None = Column( Integer, nullable=True) - evaluation_sandbox: str | None = Column( - String, + evaluation_sandbox_paths: list[str] | None = Column( + ARRAY(Unicode), + nullable=True) + evaluation_sandbox_digests: list[str] | None = Column( + ARRAY(String), nullable=True) # These one-to-many relationships are the reversed directions of diff --git a/cms/grading/Job.py b/cms/grading/Job.py index 0e051ed3d6..3b08094d02 100644 --- a/cms/grading/Job.py +++ b/cms/grading/Job.py @@ -85,9 +85,11 @@ def __init__( task_type_parameters: object = None, language: str | None = None, multithreaded_sandbox: bool = False, + archive_sandbox: bool = False, shard: int | None = None, keep_sandbox: bool = False, sandboxes: list[str] | None = None, + sandbox_digests: dict[str, str] | None = None, info: str | None = None, success: bool | None = None, text: list[str] | None = None, @@ -104,12 +106,15 @@ def __init__( language: the language of the submission / user test. multithreaded_sandbox: whether the sandbox should allow multithreading. + archive_sandbox: whether the sandbox is to be archived. shard: the shard of the Worker completing this job. keep_sandbox: whether to forcefully keep the sandbox, even if other conditions (the config, the sandbox status) don't warrant it. sandboxes: the paths of the sandboxes used in the Worker during the execution of the job. + sandbox_digests: the digests of the sandbox archives used to + debug solutions. (map of sandbox path -> archive digest) info: a human readable description of the job. success: whether the job succeeded. text: description of the outcome of the job, @@ -125,6 +130,8 @@ def __init__( task_type = "" if sandboxes is None: sandboxes = [] + if sandbox_digests is None: + sandbox_digests = {} if info is None: info = "" if files is None: @@ -139,9 +146,11 @@ def __init__( self.task_type_parameters = task_type_parameters self.language = language self.multithreaded_sandbox = multithreaded_sandbox + self.archive_sandbox = archive_sandbox self.shard = shard self.keep_sandbox = keep_sandbox self.sandboxes = sandboxes + self.sandbox_digests = sandbox_digests self.info = info self.success = success @@ -161,9 +170,11 @@ def export_to_dict(self) -> dict: 'task_type_parameters': self.task_type_parameters, 'language': self.language, 'multithreaded_sandbox': self.multithreaded_sandbox, + 'archive_sandbox': self.archive_sandbox, 'shard': self.shard, 'keep_sandbox': self.keep_sandbox, 'sandboxes': self.sandboxes, + 'sandbox_digests': self.sandbox_digests, 'info': self.info, 'success': self.success, 'text': self.text, @@ -253,6 +264,26 @@ def from_operation( job = EvaluationJob.from_user_test(operation, object_, dataset) return job + def get_sandbox_digest_list(self) -> list[str] | None: + """ + Convert self.sandbox_digests into a list, where each index matches the + corresponding index in self.sandboxes. + """ + if not self.sandbox_digests: + return None + res: list[str | None] = [None] * len(self.sandboxes) + for k,v in self.sandbox_digests.items(): + if k in self.sandboxes: + index = self.sandboxes.index(k) + res[index] = v + else: + logger.warning("Have digest for unknown sandbox %s", k) + if None in res: + ind = res.index(None) + logger.warning("Sandbox %s was not archived", self.sandboxes[ind]) + return None + return res + class CompilationJob(Job): """Job representing a compilation. @@ -274,9 +305,11 @@ def __init__( shard: int | None = None, keep_sandbox: bool = False, sandboxes: list[str] | None = None, + sandbox_digests: dict[str, str] | None = None, info: str | None = None, language: str | None = None, multithreaded_sandbox: bool = False, + archive_sandbox: bool = False, files: dict[str, File] | None = None, managers: dict[str, Manager] | None = None, success: bool | None = None, @@ -296,9 +329,9 @@ def __init__( """ Job.__init__(self, operation, task_type, task_type_parameters, - language, multithreaded_sandbox, - shard, keep_sandbox, sandboxes, info, success, text, - files, managers, executables) + language, multithreaded_sandbox, archive_sandbox, + shard, keep_sandbox, sandboxes, sandbox_digests, info, success, + text, files, managers, executables) self.compilation_success = compilation_success self.plus = plus @@ -341,6 +374,7 @@ def from_submission( task_type_parameters=dataset.task_type_parameters, language=submission.language, multithreaded_sandbox=multithreaded, + archive_sandbox=operation.archive_sandbox, files=dict(submission.files), managers=dict(dataset.managers), info="compile submission %d" % (submission.id) @@ -367,7 +401,8 @@ def to_submission(self, sr: SubmissionResult): self.plus.get('execution_wall_clock_time') sr.compilation_memory = self.plus.get('execution_memory') sr.compilation_shard = self.shard - sr.compilation_sandbox = ":".join(self.sandboxes) + sr.compilation_sandbox_paths = self.sandboxes + sr.compilation_sandbox_digests = self.get_sandbox_digest_list() for executable in self.executables.values(): sr.executables.set(executable) @@ -431,6 +466,7 @@ def from_user_test( task_type_parameters=dataset.task_type_parameters, language=user_test.language, multithreaded_sandbox=multithreaded, + archive_sandbox=operation.archive_sandbox, files=dict(user_test.files), managers=managers, info="compile user test %d" % (user_test.id) @@ -457,7 +493,8 @@ def to_user_test(self, ur: UserTestResult): self.plus.get('execution_wall_clock_time') ur.compilation_memory = self.plus.get('execution_memory') ur.compilation_shard = self.shard - ur.compilation_sandbox = ":".join(self.sandboxes) + ur.compilation_sandbox_paths = self.sandboxes + ur.compilation_sandbox_digests = self.get_sandbox_digest_list() for executable in self.executables.values(): u_executable = UserTestExecutable( executable.filename, executable.digest) @@ -485,9 +522,11 @@ def __init__( shard: int | None = None, keep_sandbox: bool = False, sandboxes: list[str] | None = None, + sandbox_digests: dict[str, str] | None = None, info: str | None = None, language: str | None = None, multithreaded_sandbox: bool = False, + archive_sandbox: bool = False, files: dict[str, File] | None = None, managers: dict[str, Manager] | None = None, executables: dict[str, Executable] | None = None, @@ -526,9 +565,9 @@ def __init__( """ Job.__init__(self, operation, task_type, task_type_parameters, - language, multithreaded_sandbox, - shard, keep_sandbox, sandboxes, info, success, text, - files, managers, executables) + language, multithreaded_sandbox, archive_sandbox, + shard, keep_sandbox, sandboxes, sandbox_digests, info, success, + text, files, managers, executables) self.input = input self.output = output self.time_limit = time_limit @@ -592,6 +631,7 @@ def from_submission( task_type_parameters=dataset.task_type_parameters, language=submission.language, multithreaded_sandbox=multithreaded, + archive_sandbox=operation.archive_sandbox, files=dict(submission.files), managers=dict(dataset.managers), executables=dict(submission_result.executables), @@ -619,7 +659,8 @@ def to_submission(self, sr: SubmissionResult): 'execution_wall_clock_time'), execution_memory=self.plus.get('execution_memory'), evaluation_shard=self.shard, - evaluation_sandbox=":".join(self.sandboxes), + evaluation_sandbox_paths=self.sandboxes, + evaluation_sandbox_digests=self.get_sandbox_digest_list(), testcase=sr.dataset.testcases[self.operation.testcase_codename])] @staticmethod @@ -674,6 +715,7 @@ def from_user_test( task_type_parameters=dataset.task_type_parameters, language=user_test.language, multithreaded_sandbox=multithreaded, + archive_sandbox=operation.archive_sandbox, files=dict(user_test.files), managers=managers, executables=dict(user_test_result.executables), @@ -704,7 +746,8 @@ def to_user_test(self, ur: UserTestResult): self.plus.get('execution_wall_clock_time') ur.execution_memory = self.plus.get('execution_memory') ur.evaluation_shard = self.shard - ur.evaluation_sandbox = ":".join(self.sandboxes) + ur.evaluation_sandbox_paths = self.sandboxes + ur.evaluation_sandbox_digests = self.get_sandbox_digest_list() ur.output = self.user_output diff --git a/cms/grading/Sandbox.py b/cms/grading/Sandbox.py index 24aa619467..c22158f1d7 100644 --- a/cms/grading/Sandbox.py +++ b/cms/grading/Sandbox.py @@ -27,6 +27,7 @@ import stat import tempfile import time +import tarfile from abc import ABCMeta, abstractmethod from functools import wraps, partial import typing @@ -532,10 +533,32 @@ def cleanup(self, delete: bool = False): delete: if True, also delete get_root_path() and everything it contains. - """ pass + def archive(self) -> str | None: + """Archive the directory where the sandbox operated. + + Stores the archived sandbox in the file cacher and returns its digest. + Returns None if archiving failed. + + """ + logger.info("Archiving sandbox in %s.", self.get_root_path()) + + with tempfile.TemporaryFile(dir=self.temp_dir) as sandbox_archive: + # Archive the working directory + content_path = self.get_root_path() + try: + with tarfile.open(fileobj=sandbox_archive, mode='w:gz') as tar_file: + tar_file.add(content_path, os.path.basename(content_path)) + except Exception: + logger.warning("Failed to archive sandbox", exc_info=True) + return None + + # Put archive to FS + sandbox_archive.seek(0) + return self.file_cacher.put_file_from_fobj(sandbox_archive, "Sandbox %s" % self.get_root_path()) + class StupidSandbox(SandboxBase): """A stupid sandbox implementation. It has very few features and diff --git a/cms/grading/tasktypes/Batch.py b/cms/grading/tasktypes/Batch.py index 464aa3dc08..d23ab04e51 100644 --- a/cms/grading/tasktypes/Batch.py +++ b/cms/grading/tasktypes/Batch.py @@ -257,7 +257,7 @@ def _do_compile(self, job, file_cacher): Executable(executable_filename, digest) # Cleanup. - delete_sandbox(sandbox, job.success, job.keep_sandbox) + delete_sandbox(sandbox, job) def compile(self, job, file_cacher): """See TaskType.compile.""" @@ -380,7 +380,7 @@ def _evaluate_step(self, job, file_cacher, output_file_params, outcome, text, st job.plus = stats if sandbox is not None: - delete_sandbox(sandbox, job.success, job.keep_sandbox) + delete_sandbox(sandbox, job) def evaluate(self, job, file_cacher): """See TaskType.evaluate.""" diff --git a/cms/grading/tasktypes/Communication.py b/cms/grading/tasktypes/Communication.py index 441aef66eb..df5a8fdfc2 100644 --- a/cms/grading/tasktypes/Communication.py +++ b/cms/grading/tasktypes/Communication.py @@ -242,7 +242,7 @@ def compile(self, job, file_cacher): Executable(executable_filename, digest) # Cleanup. - delete_sandbox(sandbox, job.success, job.keep_sandbox) + delete_sandbox(sandbox, job) def evaluate(self, job, file_cacher): """See TaskType.evaluate.""" @@ -434,9 +434,9 @@ def evaluate(self, job, file_cacher): job.text = text job.plus = stats_user - delete_sandbox(sandbox_mgr, job.success, job.keep_sandbox) + delete_sandbox(sandbox_mgr, job) for s in sandbox_user: - delete_sandbox(s, job.success, job.keep_sandbox) + delete_sandbox(s, job) if job.success and not config.keep_sandbox and not job.keep_sandbox: for d in fifo_dir: rmtree(d) diff --git a/cms/grading/tasktypes/TwoSteps.py b/cms/grading/tasktypes/TwoSteps.py index 7e48e8e06e..da22b4278f 100644 --- a/cms/grading/tasktypes/TwoSteps.py +++ b/cms/grading/tasktypes/TwoSteps.py @@ -210,7 +210,7 @@ def compile(self, job, file_cacher): Executable(executable_filename, digest) # Cleanup - delete_sandbox(sandbox, job.success, job.keep_sandbox) + delete_sandbox(sandbox, job) def evaluate(self, job, file_cacher): """See TaskType.evaluate.""" @@ -346,5 +346,5 @@ def evaluate(self, job, file_cacher): job.text = text job.plus = stats - delete_sandbox(first_sandbox, job.success, job.keep_sandbox) - delete_sandbox(second_sandbox, job.success, job.keep_sandbox) + delete_sandbox(first_sandbox, job) + delete_sandbox(second_sandbox, job) diff --git a/cms/grading/tasktypes/util.py b/cms/grading/tasktypes/util.py index 1830b1816f..59f4dc1b07 100644 --- a/cms/grading/tasktypes/util.py +++ b/cms/grading/tasktypes/util.py @@ -69,21 +69,30 @@ def create_sandbox(file_cacher: FileCacher, name: str | None = None) -> Sandbox: return sandbox -def delete_sandbox(sandbox: Sandbox, success: bool = True, keep_sandbox: bool = False): +def delete_sandbox(sandbox: Sandbox, job: Job, success: bool | None = None): """Delete the sandbox, if the configuration and job was ok. sandbox: the sandbox to delete. - success: if the job succeeded (no system errors). - keep_sandbox: whether to keep the sandbox regardless of other - conditions. + job: the job currently running. + success: if the job succeeded (no system errors). If not provided, + job.success is used. """ + if success is None: + success = job.success + + # Archive the sandbox if required + if job.archive_sandbox: + sandbox_digest = sandbox.archive() + if sandbox_digest is not None: + job.sandbox_digests[sandbox.get_root_path()] = sandbox_digest + # If the job was not successful, we keep the sandbox around. if not success: logger.warning("Sandbox %s kept around because job did not succeed.", sandbox.get_root_path()) - delete = success and not config.keep_sandbox and not keep_sandbox + delete = success and not config.keep_sandbox and not job.keep_sandbox try: sandbox.cleanup(delete=delete) except OSError: @@ -270,7 +279,7 @@ def eval_output( sandbox, checker_digest, job.input, job.output, EVAL_USER_OUTPUT_FILENAME, extra_args) - delete_sandbox(sandbox, success, job.keep_sandbox) + delete_sandbox(sandbox, job, success) return success, outcome, text else: diff --git a/cms/server/admin/templates/submission.html b/cms/server/admin/templates/submission.html index 23105d9f18..d00f72567c 100644 --- a/cms/server/admin/templates/submission.html +++ b/cms/server/admin/templates/submission.html @@ -155,7 +155,34 @@