From 912253b4935355e1645991c9cf4edb172c07984d Mon Sep 17 00:00:00 2001 From: prandla Date: Fri, 25 Jul 2025 14:57:45 +0300 Subject: [PATCH 1/2] New config parser --- cms/__init__.py | 4 +- cms/conf.py | 352 ++++++++---------- cms/db/__init__.py | 2 +- cms/db/drop.py | 4 +- cms/db/filecacher.py | 10 +- cms/db/session.py | 8 +- cms/grading/Sandbox.py | 6 +- cms/grading/steps/compilation.py | 6 +- cms/grading/steps/evaluation.py | 4 +- cms/grading/steps/trusted.py | 6 +- cms/grading/tasktypes/Communication.py | 8 +- cms/grading/tasktypes/TwoSteps.py | 2 +- cms/grading/tasktypes/util.py | 2 +- cms/io/rpc.py | 5 +- cms/io/service.py | 16 +- cms/io/web_service.py | 6 +- cms/server/admin/authentication.py | 6 +- cms/server/admin/server.py | 12 +- cms/server/admin/templates/base.html | 2 +- .../templates/fragments/overload_warning.html | 4 +- cms/server/contest/authentication.py | 10 +- cms/server/contest/handlers/contest.py | 8 +- cms/server/contest/handlers/main.py | 16 +- cms/server/contest/handlers/tasksubmission.py | 2 +- cms/server/contest/handlers/taskusertest.py | 2 +- cms/server/contest/printing.py | 10 +- cms/server/contest/server.py | 21 +- cms/server/contest/submission/utils.py | 2 +- cms/server/contest/submission/workflow.py | 57 ++- cms/service/Checker.py | 2 +- cms/service/LogService.py | 4 +- cms/service/PrintingService.py | 18 +- cms/service/ProxyService.py | 4 +- cms/service/ResourceService.py | 2 +- cms/service/ScoringService.py | 2 +- cms/util.py | 10 +- cmscommon/archive.py | 4 +- cmscommon/conf_parser.py | 183 +++++++++ cmscontrib/PrometheusExporter.py | 4 +- cmscontrib/RWSHelper.py | 20 +- cmscontrib/TelegramBot.py | 8 +- cmsranking/Config.py | 156 ++------ cmsranking/RankingWebServer.py | 23 +- cmstestsuite/StressTest.py | 2 +- cmstestsuite/programstarter.py | 8 +- cmstestsuite/unit_tests/db/session_test.py | 2 +- .../grading/tasktypes/CommunicationTest.py | 12 +- .../grading/tasktypes/tasktypetestutils.py | 2 +- .../server/contest/authentication_test.py | 10 +- .../server/contest/printing_test.py | 8 +- .../server/contest/submission/utils_test.py | 2 +- .../contest/submission/workflow_test.py | 26 +- cmstestsuite/unit_tests/util_test.py | 17 +- cmstestsuite/web/CWSRequests.py | 4 +- config/cms.sample.toml | 312 ++++++++-------- config/cms_ranking.sample.toml | 15 + 56 files changed, 799 insertions(+), 654 deletions(-) create mode 100644 cmscommon/conf_parser.py diff --git a/cms/__init__.py b/cms/__init__.py index b81bb5c1ca..6204852f4a 100644 --- a/cms/__init__.py +++ b/cms/__init__.py @@ -37,7 +37,7 @@ # log # Nothing intended for external use, no need to advertise anything. # conf - "Address", "ServiceCoord", "ConfigError", "async_config", "config", + "Address", "ServiceCoord", "ConfigError", "config", # util "mkdir", "rmtree", "utf8_decoder", "get_safe_shard", "get_service_address", "get_service_shards", "contest_id_from_args", "default_argument_parser", @@ -74,7 +74,7 @@ FEEDBACK_LEVEL_OI_RESTRICTED = "oi_restricted" -from .conf import Address, ServiceCoord, ConfigError, async_config, config +from .conf import Address, ServiceCoord, ConfigError, config from .util import mkdir, rmtree, utf8_decoder, get_safe_shard, \ get_service_address, get_service_shards, contest_id_from_args, \ default_argument_parser diff --git a/cms/conf.py b/cms/conf.py index 4d3468df77..ec8baff4df 100644 --- a/cms/conf.py +++ b/cms/conf.py @@ -21,15 +21,16 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -import errno -import tomllib +import dataclasses import logging import os import sys import typing +from dataclasses import dataclass from cms.log import set_detailed_logs - +from cmscommon import conf_parser +from cmscommon.conf_parser import ConfigError logger = logging.getLogger(__name__) @@ -37,220 +38,183 @@ class Address(typing.NamedTuple): ip: str port: int - def __repr__(self): + + def __str__(self): return "%s:%d" % (self.ip, self.port) class ServiceCoord(typing.NamedTuple): """A compact representation for the name and the shard number of a service (thus identifying it). - """ + name: str shard: int - def __repr__(self): + + def __str__(self): return "%s,%d" % (self.name, self.shard) -class ConfigError(Exception): - """Exception for critical configuration errors.""" - pass +# Try to find CMS installation root from the venv in which we run +if sys.prefix == "/usr": + logger.critical("CMS must be run within a Python virtual environment") + sys.exit(1) +def default_path(name): + return os.path.join(sys.prefix, name) -class AsyncConfig: - """This class will contain the configuration for the - services. This needs to be populated at the initilization stage. - The *_services variables are dictionaries indexed by ServiceCoord - with values of type Address. +@dataclass() +class GlobalConfig: + temp_dir: str = "/tmp" + backdoor: bool = False + file_log_debug: bool = False + stream_log_detailed: bool = False + log_dir: str = default_path("log") + cache_dir: str = default_path("cache") + data_dir: str = default_path("lib") + run_dir: str = default_path("run") - Core services are the ones that are supposed to run whenever the - system is up. - Other services are not supposed to run when the system is up, or - anyway not constantly. +@dataclass() +class DatabaseConfig: + url: str + debug: bool = False + twophase_commit: bool = False - """ - core_services: dict[ServiceCoord, Address] = {} - other_services: dict[ServiceCoord, Address] = {} +@dataclass() +class WorkerConfig: + keep_sandbox: bool = False -async_config = AsyncConfig() +@dataclass() +class SandboxConfig: + sandbox_implementation: str = "isolate" + use_cgroups: bool = True + # Max size of each writable file during an evaluation step, in KiB. + max_file_size: int = 1024 * 1024 # 1 GiB + # Max processes, CPU time (s), memory (KiB) for compilation runs. + compilation_sandbox_max_processes: int = 1000 + compilation_sandbox_max_time_s: float = 10.0 + compilation_sandbox_max_memory_kib: int = 512 * 1024 # 512 MiB + # Max processes, CPU time (s), memory (KiB) for trusted runs. + trusted_sandbox_max_processes: int = 1000 + trusted_sandbox_max_time_s: float = 10.0 + trusted_sandbox_max_memory_kib: int = 4 * 1024 * 1024 # 4 GiB -class Config: - """This class will contain the configuration for CMS. This needs - to be populated at the initilization stage. This is loaded by - default with some sane data. See cms.sample.toml in the config - directory for information on the meaning of the fields. - """ - def __init__(self): - """Default values for configuration, plus decide if this - instance is running from the system path or from the source - directory. - - """ - self.async_config = async_config - - # System-wide - self.temp_dir = "/tmp" - self.backdoor = False - self.file_log_debug = False - self.stream_log_detailed = False - - # Database. - self.database = "postgresql+psycopg2://cmsuser@localhost/cms" - self.database_debug = False - self.twophase_commit = False - - # Worker. - self.keep_sandbox = True - self.use_cgroups = True - self.sandbox_implementation = 'isolate' - - # Sandbox. - # Max size of each writable file during an evaluation step, in KiB. - self.max_file_size = 1024 * 1024 # 1 GiB - # Max processes, CPU time (s), memory (KiB) for compilation runs. - self.compilation_sandbox_max_processes = 1000 - self.compilation_sandbox_max_time_s = 10.0 - self.compilation_sandbox_max_memory_kib = 512 * 1024 # 512 MiB - # Max processes, CPU time (s), memory (KiB) for trusted runs. - self.trusted_sandbox_max_processes = 1000 - self.trusted_sandbox_max_time_s = 10.0 - self.trusted_sandbox_max_memory_kib = 4 * 1024 * 1024 # 4 GiB - - # WebServers. - self.secret_key_default = "8e045a51e4b102ea803c06f92841a1fb" - self.secret_key = self.secret_key_default - self.tornado_debug = False - - # ContestWebServer. - self.contest_listen_address = [""] - self.contest_listen_port = [8888] - self.cookie_duration = 30 * 60 # 30 minutes - self.submit_local_copy = True - self.submit_local_copy_path = "%s/submissions/" - self.tests_local_copy = True - self.tests_local_copy_path = "%s/tests/" - self.is_proxy_used = None # (deprecated in favor of num_proxies_used) - self.num_proxies_used = None - self.max_submission_length = 100_000 # 100 KB - self.max_input_length = 5_000_000 # 5 MB - self.stl_path = "/usr/share/cppreference/doc/html/" - self.docs_path = None - self.contest_admin_token = None - - # AdminWebServer. - self.admin_listen_address = "" - self.admin_listen_port = 8889 - self.admin_cookie_duration = 10 * 60 * 60 # 10 hours - self.admin_num_proxies_used = None - - # ProxyService. - self.rankings = ["http://usern4me:passw0rd@localhost:8890/"] - self.https_certfile = None - - # PrintingService - self.max_print_length = 10_000_000 # 10 MB - self.printer = None - self.paper_size = "A4" - self.max_pages_per_job = 10 - self.max_jobs_per_user = 10 - self.pdf_printing_allowed = False - - # PrometheusExporter - self.prometheus_listen_address = "127.0.0.1" - self.prometheus_listen_port = 8811 - - # TelegramBot - self.telegram_bot_token = None - self.telegram_bot_chat_id = None - - # Try to find CMS installation root from the venv in which we run - self.base_dir = sys.prefix - if self.base_dir == '/usr': - logger.critical('CMS must be run within a Python virtual environment') - sys.exit(1) - self.log_dir = os.path.join(self.base_dir, 'log') - self.cache_dir = os.path.join(self.base_dir, 'cache') - self.data_dir = os.path.join(self.base_dir, 'lib') - self.run_dir = os.path.join(self.base_dir, 'run') - - # Default config file path can be overridden using environment - # variable 'CMS_CONFIG'. - default_config_file = os.path.join(self.base_dir, 'etc/cms.toml') - config_file = os.environ.get('CMS_CONFIG', default_config_file) - - if not self._load_config(config_file): - logging.critical(f'Cannot load configuration file {config_file}') - sys.exit(1) +@dataclass() +class WebServerConfig: + # This doesn't have a type hint, so @dataclass (and thus the config parser) + # ignore it. + DEFAULT_SECRET_KEY = "8e045a51e4b102ea803c06f92841a1fb" + secret_key: str = DEFAULT_SECRET_KEY + tornado_debug: bool = False + + +@dataclass() +class CWSConfig: + listen_address: tuple[str, ...] = ("127.0.0.1",) + listen_port: tuple[int, ...] = (8888,) + cookie_duration: int = 30 * 60 # 30 minutes + num_proxies_used: int = 0 + + submit_local_copy: bool = True + submit_local_copy_path: str = "%s/submissions/" + tests_local_copy: bool = True + tests_local_copy_path: str = "%s/tests/" + + max_submission_length: int = 100_000 # 100 KB + max_input_length: int = 5_000_000 # 5 MB + + stl_path: str = "/usr/share/cppreference/doc/html/" + docs_path: str | None = None + + contest_admin_token: str | None = None + + +@dataclass() +class AWSConfig: + listen_address: str = "127.0.0.1" + listen_port: int = 8889 + cookie_duration: int = 10 * 60 * 60 # 10 hours + num_proxies_used: int = 0 + + +@dataclass() +class ProxyServiceConfig: + rankings: tuple[str, ...] = () + https_certfile: str | None = None + + +@dataclass() +class PrintingServiceConfig: + max_print_length: int = 10_000_000 # 10 MB + printer: str | None = None + paper_size: str = "A4" + max_pages_per_job: int = 10 + max_jobs_per_user: int = 10 + pdf_printing_allowed: bool = False + + +@dataclass() +class PrometheusConfig: + listen_address: str = "127.0.0.1" + listen_port: int = 8811 + + +@dataclass() +class TelegramBotConfig: + bot_token: str + chat_id: str + + +field_helper = lambda T: dataclasses.field(default_factory=T) + +@dataclass(kw_only=True) +class Config: + # Ideally these would all look like + # global_: GlobalConfig = GlobalConfig() + # but dataclasses doesn't like it, because these are all mutable default + # values. We could make the individual config sections frozen, but then we + # can't easily patch them for unit tests. + global_: GlobalConfig = field_helper(GlobalConfig) + database: DatabaseConfig + worker: WorkerConfig = field_helper(WorkerConfig) + sandbox: SandboxConfig = field_helper(SandboxConfig) + web_server: WebServerConfig = field_helper(WebServerConfig) + contest_web_server: CWSConfig = field_helper(CWSConfig) + admin_web_server: AWSConfig = field_helper(AWSConfig) + proxy_service: ProxyServiceConfig = field_helper(ProxyServiceConfig) + printing: PrintingServiceConfig = field_helper(PrintingServiceConfig) + prometheus: PrometheusConfig = field_helper(PrometheusConfig) + telegram_bot: TelegramBotConfig | None = None + # This is the one that will be provided in the config file. + services_: dict[str, list[tuple[str, int]]] + # And this is the one we want to use inside CMS. + services: dict[ServiceCoord, Address] = dataclasses.field(init=False) + + def __post_init__(self): + self.services = {} + for service_name, instances in self.services_.items(): + for shard_number, shard in enumerate(instances): + coord = ServiceCoord(service_name, shard_number) + self.services[coord] = Address(*shard) # If the configuration says to print detailed log on stdout, # change the log configuration. - set_detailed_logs(self.stream_log_detailed) - - def _load_config(self, path: str) -> bool: - """Populate the Config class with everything that sits inside - the TOML file path (usually something like /etc/cms.toml). The - only pieces of data treated differently are the elements of - core_services and other_services that are sent to async - config. - - path: the path of the TOML config file. - returns: whether parsing was successful. - - """ - # Load config file. - try: - with open(path, 'rb') as f: - data = tomllib.load(f) - except FileNotFoundError: - logger.debug("Couldn't find config file %s (maybe you need to " - "convert it from cms.conf to cms.toml?).", path) - return False - except OSError as error: - logger.warning("I/O error while opening file %s: [%s] %s", - path, errno.errorcode[error.errno], - os.strerror(error.errno)) - return False - except ValueError as error: - logger.warning("Invalid syntax in file %s: %s", path, error) - return False - - if "is_proxy_used" in data: - logger.warning("The 'is_proxy_used' setting is deprecated, please " - "use 'num_proxies_used' instead.") - - # Put core and test services in async_config, ignoring those - # whose name begins with "_". - for service in data["core_services"]: - if service.startswith("_"): - continue - for shard_number, shard in \ - enumerate(data["core_services"][service]): - coord = ServiceCoord(service, shard_number) - self.async_config.core_services[coord] = Address(*shard) - del data["core_services"] - - for service in data["other_services"]: - if service.startswith("_"): - continue - for shard_number, shard in \ - enumerate(data["other_services"][service]): - coord = ServiceCoord(service, shard_number) - self.async_config.other_services[coord] = Address(*shard) - del data["other_services"] - - # Put everything else in self. - for key, value in data.items(): - if hasattr(self, key): - setattr(self, key, value) - else: - logger.warning("Unrecognized key %s in config!", key) - - return True - - -config = Config() + set_detailed_logs(self.global_.stream_log_detailed) + + +def make_config(): + # Default config file path can be overridden using environment + # variable 'CMS_CONFIG'. + default_config_file = default_path("etc/cms.toml") + config_file = os.environ.get("CMS_CONFIG", default_config_file) + + hint = " (maybe you need to convert it from cms.conf to cms.toml?)" + return conf_parser.parse_config(config_file, Config, hint) + + +config = make_config() diff --git a/cms/db/__init__.py b/cms/db/__init__.py index 18f87364c8..5533c72bb4 100644 --- a/cms/db/__init__.py +++ b/cms/db/__init__.py @@ -83,7 +83,7 @@ version = 46 -engine = create_engine(config.database, echo=config.database_debug, +engine = create_engine(config.database.url, echo=config.database.debug, pool_timeout=60, pool_recycle=120) metadata = MetaData(engine) diff --git a/cms/db/drop.py b/cms/db/drop.py index 9267f16331..29f78c90a7 100644 --- a/cms/db/drop.py +++ b/cms/db/drop.py @@ -62,7 +62,7 @@ def drop_db() -> bool: logger.error("Couldn't drop schema \"public\", probably you don't " "have the privileges. Please execute as database " "superuser: \"ALTER SCHEMA public OWNER TO %s;\" and " - "run again", make_url(config.database).username) + "run again", make_url(config.database.url).username) return False cursor.execute("CREATE SCHEMA public") @@ -73,7 +73,7 @@ def drop_db() -> bool: logger.error("Couldn't list large objects, probably you don't have " "the privileges. Please execute as database superuser: " "\"GRANT SELECT ON pg_largeobject TO %s;\" and run " - "again", make_url(config.database).username) + "again", make_url(config.database.url).username) return False rows = cursor.fetchall() for row in rows: diff --git a/cms/db/filecacher.py b/cms/db/filecacher.py index fee3ef1050..44d6689d2e 100644 --- a/cms/db/filecacher.py +++ b/cms/db/filecacher.py @@ -522,16 +522,16 @@ def __init__(self, service: "Service | None" = None, path: str | None = None, nu self.backend = FSBackend(path) # First we create the config directories. - self._create_directory_or_die(config.temp_dir) - self._create_directory_or_die(config.cache_dir) + self._create_directory_or_die(config.global_.temp_dir) + self._create_directory_or_die(config.global_.cache_dir) if not self.is_shared(): - self.file_dir = tempfile.mkdtemp(dir=config.temp_dir) + self.file_dir = tempfile.mkdtemp(dir=config.global_.temp_dir) # Delete this directory on exit since it has a random name and # won't be used again. atexit.register(lambda: rmtree(self.file_dir)) else: - self.file_dir = os.path.join(config.cache_dir, "fs-cache-shared") + self.file_dir = os.path.join(config.global_.cache_dir, "fs-cache-shared") self._create_directory_or_die(self.file_dir) # Temp dir must be a subdirectory of file_dir to avoid cross-filesystem @@ -895,7 +895,7 @@ def purge_cache(self): """ self.destroy_cache() - if not mkdir(config.cache_dir) or not mkdir(self.file_dir): + if not mkdir(config.global_.cache_dir) or not mkdir(self.file_dir): logger.error("Cannot create necessary directories.") raise RuntimeError("Cannot create necessary directories.") diff --git a/cms/db/session.py b/cms/db/session.py index 2389147e0c..39da3c1dc0 100644 --- a/cms/db/session.py +++ b/cms/db/session.py @@ -40,17 +40,15 @@ logger = logging.getLogger(__name__) +_session = sessionmaker(engine, twophase=config.database.twophase_commit) if typing.TYPE_CHECKING: # the type checker doesn't understand sessionmaker, so for type hints # define Session as the sqlalchemy class directly. Session = sqlalchemy.orm.Session else: - Session = sessionmaker(engine, twophase=config.twophase_commit) + Session = _session ScopedSession = scoped_session(Session) -# For two-phases transactions: -# Session = sessionmaker(db, twophase=True) - class SessionGen: """This allows us to create handy local sessions simply with: @@ -93,7 +91,7 @@ def custom_psycopg2_connection(**kwargs: dict[str, str]): configured to use psycopg2 as the DB-API driver. """ - database_url: URL = make_url(config.database) + database_url: URL = make_url(config.database.url) assert database_url.get_dialect().driver == "psycopg2" # For Unix-domain socket we don't have a port nor a host and that's fine. if database_url.port is None and database_url.host is not None: diff --git a/cms/grading/Sandbox.py b/cms/grading/Sandbox.py index c22158f1d7..06528ffa25 100644 --- a/cms/grading/Sandbox.py +++ b/cms/grading/Sandbox.py @@ -208,7 +208,7 @@ def __init__( """ self.file_cacher = file_cacher self.name = name if name is not None else "unnamed" - self.temp_dir = temp_dir if temp_dir is not None else config.temp_dir + self.temp_dir = temp_dir if temp_dir is not None else config.global_.temp_dir self.cmd_file: str = "commands.log" @@ -947,7 +947,7 @@ def __init__(self, file_cacher, name=None, temp_dir=None): # Default parameters for isolate self.box_id = box_id # -b - self.cgroup = config.use_cgroups # --cg + self.cgroup = config.sandbox.use_cgroups # --cg self.chdir = self._home_dest # -c self.dirs = [] # -d self.preserve_env = False # -e @@ -1520,4 +1520,4 @@ def cleanup(self, delete=False): Sandbox = { "stupid": StupidSandbox, "isolate": IsolateSandbox, - }[config.sandbox_implementation] + }[config.sandbox.sandbox_implementation] diff --git a/cms/grading/steps/compilation.py b/cms/grading/steps/compilation.py index dc96e3668d..3b56f2eaad 100644 --- a/cms/grading/steps/compilation.py +++ b/cms/grading/steps/compilation.py @@ -100,10 +100,10 @@ def compilation_step( # but it is a symlink to "/var/lib/ghc/package.conf.d" sandbox.maybe_add_mapped_directory("/var/lib/ghc") sandbox.preserve_env = True - sandbox.max_processes = config.compilation_sandbox_max_processes - sandbox.timeout = config.compilation_sandbox_max_time_s + sandbox.max_processes = config.sandbox.compilation_sandbox_max_processes + sandbox.timeout = config.sandbox.compilation_sandbox_max_time_s sandbox.wallclock_timeout = 2 * sandbox.timeout + 1 - sandbox.address_space = config.compilation_sandbox_max_memory_kib * 1024 + sandbox.address_space = config.sandbox.compilation_sandbox_max_memory_kib * 1024 # Run the compilation commands, copying stdout and stderr to stats. stats = generic_step(sandbox, commands, "compilation", collect_output=True) diff --git a/cms/grading/steps/evaluation.py b/cms/grading/steps/evaluation.py index f2c9359e7f..a58030d840 100644 --- a/cms/grading/steps/evaluation.py +++ b/cms/grading/steps/evaluation.py @@ -203,8 +203,8 @@ def evaluation_step_before_run( else: sandbox.address_space = None - # config.max_file_size is in KiB - sandbox.fsize = config.max_file_size * 1024 + # config.sandbox.max_file_size is in KiB + sandbox.fsize = config.sandbox.max_file_size * 1024 sandbox.stdin_file = stdin_redirect sandbox.stdout_file = stdout_redirect diff --git a/cms/grading/steps/trusted.py b/cms/grading/steps/trusted.py index 481bce7e0f..7c6513876e 100644 --- a/cms/grading/steps/trusted.py +++ b/cms/grading/steps/trusted.py @@ -150,10 +150,10 @@ def trusted_step( """ # Set sandbox parameters suitable for trusted commands. sandbox.preserve_env = True - sandbox.max_processes = config.trusted_sandbox_max_processes - sandbox.timeout = config.trusted_sandbox_max_time_s + sandbox.max_processes = config.sandbox.trusted_sandbox_max_processes + sandbox.timeout = config.sandbox.trusted_sandbox_max_time_s sandbox.wallclock_timeout = 2 * sandbox.timeout + 1 - sandbox.address_space = config.trusted_sandbox_max_memory_kib * 1024 + sandbox.address_space = config.sandbox.trusted_sandbox_max_memory_kib * 1024 # Run the trusted commands. stats = generic_step(sandbox, commands, "trusted") diff --git a/cms/grading/tasktypes/Communication.py b/cms/grading/tasktypes/Communication.py index df5a8fdfc2..60f53a993b 100644 --- a/cms/grading/tasktypes/Communication.py +++ b/cms/grading/tasktypes/Communication.py @@ -260,7 +260,7 @@ def evaluate(self, job, file_cacher): indices = range(self.num_processes) # Create FIFOs. - fifo_dir = [tempfile.mkdtemp(dir=config.temp_dir) for i in indices] + fifo_dir = [tempfile.mkdtemp(dir=config.global_.temp_dir) for i in indices] fifo_user_to_manager = [ os.path.join(fifo_dir[i], "u%d_to_m" % i) for i in indices] fifo_manager_to_user = [ @@ -316,12 +316,12 @@ def evaluate(self, job, file_cacher): # constraint on the total time can only be enforced after all user # programs terminated. manager_time_limit = max(self.num_processes * (job.time_limit + 1.0), - config.trusted_sandbox_max_time_s) + config.sandbox.trusted_sandbox_max_time_s) manager_ = evaluation_step_before_run( sandbox_mgr, manager_command, manager_time_limit, - config.trusted_sandbox_max_memory_kib * 1024, + config.sandbox.trusted_sandbox_max_memory_kib * 1024, dirs_map=dict((fifo_dir[i], (sandbox_fifo_dir[i], "rw")) for i in indices), writable_files=[self.OUTPUT_FILENAME], stdin_redirect=self.INPUT_FILENAME, @@ -437,6 +437,6 @@ def evaluate(self, job, file_cacher): delete_sandbox(sandbox_mgr, job) for s in sandbox_user: delete_sandbox(s, job) - if job.success and not config.keep_sandbox and not job.keep_sandbox: + if job.success and not config.worker.keep_sandbox and not job.keep_sandbox: for d in fifo_dir: rmtree(d) diff --git a/cms/grading/tasktypes/TwoSteps.py b/cms/grading/tasktypes/TwoSteps.py index da22b4278f..fbe0ee5ce4 100644 --- a/cms/grading/tasktypes/TwoSteps.py +++ b/cms/grading/tasktypes/TwoSteps.py @@ -225,7 +225,7 @@ def evaluate(self, job, file_cacher): job.sandboxes.append(first_sandbox.get_root_path()) job.sandboxes.append(second_sandbox.get_root_path()) - fifo_dir = tempfile.mkdtemp(dir=config.temp_dir) + fifo_dir = tempfile.mkdtemp(dir=config.global_.temp_dir) fifo = os.path.join(fifo_dir, "fifo") os.mkfifo(fifo) os.chmod(fifo_dir, 0o755) diff --git a/cms/grading/tasktypes/util.py b/cms/grading/tasktypes/util.py index 59f4dc1b07..609d7c5ac6 100644 --- a/cms/grading/tasktypes/util.py +++ b/cms/grading/tasktypes/util.py @@ -92,7 +92,7 @@ def delete_sandbox(sandbox: Sandbox, job: Job, success: bool | None = None): logger.warning("Sandbox %s kept around because job did not succeed.", sandbox.get_root_path()) - delete = success and not config.keep_sandbox and not job.keep_sandbox + delete = success and not config.worker.keep_sandbox and not job.keep_sandbox try: sandbox.cleanup(delete=delete) except OSError: diff --git a/cms/io/rpc.py b/cms/io/rpc.py index 3ec1b9dfe9..70e070fac9 100644 --- a/cms/io/rpc.py +++ b/cms/io/rpc.py @@ -154,7 +154,7 @@ def _repr_remote(self) -> str: remote address, for use in log messages and exceptions. """ - return "%s:%d" % (self.remote_address) + return str(self.remote_address) def initialize(self, sock: socket.socket, plus: object): """Activate the communication on the given socket. @@ -483,8 +483,7 @@ def __init__( def _repr_remote(self): """See RemoteServiceBase._repr_remote.""" - return "%s:%d (%r)" % (self.remote_address + - (self.remote_service_coord,)) + return f"{self.remote_address} ({self.remote_service_coord})" def finalize(self, reason=""): """See RemoteServiceBase.finalize.""" diff --git a/cms/io/service.py b/cms/io/service.py index cdafc37c1c..82a0edbd5f 100644 --- a/cms/io/service.py +++ b/cms/io/service.py @@ -96,7 +96,7 @@ def __init__(self, shard: int = 0): try: address = get_service_address(self._my_coord) except KeyError: - raise ConfigError("Unable to find address for service %r. " + raise ConfigError("Unable to find address for service %s. " "Is it specified in core_services in cms.toml?" % (self._my_coord,)) @@ -118,9 +118,9 @@ def initialize_logging(self): shell_handler.addFilter(filter_) # Determine location of log file, and make directories. - log_dir = os.path.join(config.log_dir, + log_dir = os.path.join(config.global_.log_dir, "%s-%d" % (self.name, self.shard)) - mkdir(config.log_dir) + mkdir(config.global_.log_dir) mkdir(log_dir) log_filename = time.strftime("%Y-%m-%d-%H-%M-%S.log") @@ -128,7 +128,7 @@ def initialize_logging(self): # Install a file handler. file_handler = FileHandler(os.path.join(log_dir, log_filename), mode='w', encoding='utf-8') - if config.file_log_debug: + if config.global_.file_log_debug: file_log_level = logging.DEBUG else: file_log_level = logging.INFO @@ -242,14 +242,14 @@ def exit(self): """Terminate the service at the next step. """ - logger.warning("%r received request to shut down.", self._my_coord) + logger.warning("%s received request to shut down.", self._my_coord) self.rpc_server.stop() def get_backdoor_path(self) -> str: """Return the path for a UNIX domain socket to use as backdoor. """ - return os.path.join(config.run_dir, "%s_%d" % (self.name, self.shard)) + return os.path.join(config.global_.run_dir, "%s_%d" % (self.name, self.shard)) @rpc_method def start_backdoor(self, backlog=50): @@ -316,7 +316,7 @@ def run(self) -> bool: else: raise - if config.backdoor: + if config.global_.backdoor: self.start_backdoor() logger.info("%s %d up and running!", *self._my_coord) @@ -326,7 +326,7 @@ def run(self) -> bool: logger.info("%s %d is shutting down", *self._my_coord) - if config.backdoor: + if config.global_.backdoor: self.stop_backdoor() self._disconnect_all() diff --git a/cms/io/web_service.py b/cms/io/web_service.py index a6032ee641..2d7390e657 100644 --- a/cms/io/web_service.py +++ b/cms/io/web_service.py @@ -65,7 +65,6 @@ def __init__( rpc_enabled = parameters.pop('rpc_enabled', False) rpc_auth = parameters.pop('rpc_auth', None) auth_middleware = parameters.pop('auth_middleware', None) - is_proxy_used = parameters.pop('is_proxy_used', None) num_proxies_used = parameters.pop('num_proxies_used', None) self.wsgi_app = tornado.wsgi.WSGIApplication(handlers, **parameters) @@ -101,10 +100,7 @@ def __init__( # were allowed to directlty communicate with the server they # could fake their IP and compromise the security of IP lock). if num_proxies_used is None: - if is_proxy_used: - num_proxies_used = 1 - else: - num_proxies_used = 0 + num_proxies_used = 0 if num_proxies_used > 0: self.wsgi_app = ProxyFix(self.wsgi_app, num_proxies_used) diff --git a/cms/server/admin/authentication.py b/cms/server/admin/authentication.py index 709a1c0d3e..23e2cb60f8 100644 --- a/cms/server/admin/authentication.py +++ b/cms/server/admin/authentication.py @@ -130,7 +130,7 @@ def wsgi_app(self, environ: dict, start_response: Callable): self._local.request = Request(environ) self._local.cookie = JSONSecureCookie.load_cookie( self._request, AWSAuthMiddleware.COOKIE, - hex_to_bin(config.secret_key)) + hex_to_bin(config.web_server.secret_key)) self._verify_cookie() def my_start_response(status, headers, exc_info=None): @@ -144,7 +144,7 @@ def my_start_response(status, headers, exc_info=None): response = Response(status=status, headers=headers) self._cookie.save_cookie( response, AWSAuthMiddleware.COOKIE, httponly=True, - max_age=config.admin_cookie_duration) + max_age=config.admin_web_server.cookie_duration) return start_response( status, response.headers.to_wsgi_list(), exc_info) @@ -170,6 +170,6 @@ def _verify_cookie(self): self.clear() return - if make_timestamp() - timestamp > config.admin_cookie_duration: + if make_timestamp() - timestamp > config.admin_web_server.cookie_duration: self.clear() return diff --git a/cms/server/admin/server.py b/cms/server/admin/server.py index 12d89c0c0f..d2af937bee 100644 --- a/cms/server/admin/server.py +++ b/cms/server/admin/server.py @@ -52,20 +52,20 @@ def __init__(self, shard: int): parameters = { "static_files": [("cms.server", "static"), ("cms.server.admin", "static")], - "cookie_secret": hex_to_bin(config.secret_key), - "debug": config.tornado_debug, - "num_proxies_used": config.admin_num_proxies_used, + "cookie_secret": hex_to_bin(config.web_server.secret_key), + "debug": config.web_server.tornado_debug, + "num_proxies_used": config.admin_web_server.num_proxies_used, "auth_middleware": AWSAuthMiddleware, "rpc_enabled": True, "rpc_auth": self.is_rpc_authorized, "xsrf_cookies": True, } super().__init__( - config.admin_listen_port, + config.admin_web_server.listen_port, HANDLERS, parameters, shard=shard, - listen_address=config.admin_listen_address) + listen_address=config.admin_web_server.listen_address) self.auth_handler: AWSAuthMiddleware self.jinja2_environment = AWS_ENVIRONMENT @@ -80,7 +80,7 @@ def __init__(self, shard: int): self.scoring_service = self.connect_to( ServiceCoord("ScoringService", 0)) - ranking_enabled = len(config.rankings) > 0 + ranking_enabled = len(config.proxy_service.rankings) > 0 self.proxy_service = self.connect_to( ServiceCoord("ProxyService", 0), must_be_present=ranking_enabled) diff --git a/cms/server/admin/templates/base.html b/cms/server/admin/templates/base.html index d48805bd4c..09036236bf 100644 --- a/cms/server/admin/templates/base.html +++ b/cms/server/admin/templates/base.html @@ -80,7 +80,7 @@

Administration

{{ contest.name }}

{% endif %} - {% if config.secret_key == config.secret_key_default %} + {% if config.web_server.secret_key == config.web_server.DEFAULT_SECRET_KEY %}
Change secret_key in cms.toml!
For example,
diff --git a/cms/server/admin/templates/fragments/overload_warning.html b/cms/server/admin/templates/fragments/overload_warning.html index ef9299b1ea..d760476ec8 100644 --- a/cms/server/admin/templates/fragments/overload_warning.html +++ b/cms/server/admin/templates/fragments/overload_warning.html @@ -8,8 +8,8 @@ warning early on. #} -{% if config.contest_listen_port|length > 0 %} - {% set ratio = (contest.participations|length / config.contest_listen_port|length)|round(2) %} +{% if config.contest_web_server.listen_port|length > 0 %} + {% set ratio = (contest.participations|length / config.contest_web_server.listen_port|length)|round(2) %} {% if ratio > 100 %}

At the moment, you have {{ ratio }} users for each ContestWebServer, on average.
diff --git a/cms/server/contest/authentication.py b/cms/server/contest/authentication.py index ccd89f2232..1e8779a049 100644 --- a/cms/server/contest/authentication.py +++ b/cms/server/contest/authentication.py @@ -123,8 +123,8 @@ def log_failed_attempt(msg, *args): return None, None if admin_token != "": - if (config.contest_admin_token is not None - and admin_token != config.contest_admin_token): + if (config.contest_web_server.contest_admin_token is not None + and admin_token != config.contest_web_server.contest_admin_token): log_failed_attempt("invalid admin token") return None, None @@ -375,9 +375,11 @@ def log_failed_attempt(msg, *args): *args) # Check if the cookie is expired. - if timestamp - last_update > timedelta(seconds=config.cookie_duration): + if timestamp - last_update > timedelta( + seconds=config.contest_web_server.cookie_duration + ): log_failed_attempt("cookie expired (lasts %d seconds)", - config.cookie_duration) + config.contest_web_server.cookie_duration) return None, None, False # Load participation from DB and make sure it exists. diff --git a/cms/server/contest/handlers/contest.py b/cms/server/contest/handlers/contest.py index f6deec3508..9624816666 100644 --- a/cms/server/contest/handlers/contest.py +++ b/cms/server/contest/handlers/contest.py @@ -181,7 +181,11 @@ def get_current_user(self) -> Participation | None: self.clear_cookie(cookie_name) elif self.refresh_cookie: self.set_secure_cookie( - cookie_name, cookie, expires_days=None, max_age=config.cookie_duration) + cookie_name, + cookie, + expires_days=None, + max_age=config.contest_web_server.cookie_duration, + ) self.impersonated_by_admin = impersonated return participation @@ -196,7 +200,7 @@ def render_params(self): ret["phase"] = self.contest.phase(self.timestamp) - ret["printing_enabled"] = (config.printer is not None) + ret["printing_enabled"] = (config.printing.printer is not None) ret["questions_enabled"] = self.contest.allow_questions ret["testing_enabled"] = self.contest.allow_user_tests diff --git a/cms/server/contest/handlers/main.py b/cms/server/contest/handlers/main.py index 4ce2edade7..42121eaf0b 100644 --- a/cms/server/contest/handlers/main.py +++ b/cms/server/contest/handlers/main.py @@ -246,7 +246,11 @@ def post(self): self.clear_cookie(cookie_name) else: self.set_secure_cookie( - cookie_name, cookie, expires_days=None, max_age=config.cookie_duration) + cookie_name, + cookie, + expires_days=None, + max_age=config.contest_web_server.cookie_duration, + ) if participation is None: self.redirect(error_page) @@ -337,13 +341,13 @@ def get(self): .all() ) - remaining_jobs = max(0, config.max_jobs_per_user - len(printjobs)) + remaining_jobs = max(0, config.printing.max_jobs_per_user - len(printjobs)) self.render("printing.html", printjobs=printjobs, remaining_jobs=remaining_jobs, - max_pages=config.max_pages_per_job, - pdf_printing_allowed=config.pdf_printing_allowed, + max_pages=config.printing.max_pages_per_job, + pdf_printing_allowed=config.printing.pdf_printing_allowed, **self.r_params) @tornado.web.authenticated @@ -379,10 +383,10 @@ def get(self): languages = [get_language(lang) for lang in contest.languages] language_docs = [] - if config.docs_path is not None: + if config.contest_web_server.docs_path is not None: for language in languages: ext = language.source_extensions[0][1:] # remove dot - path = os.path.join(config.docs_path, ext) + path = os.path.join(config.contest_web_server.docs_path, ext) if os.path.exists(path): language_docs.append((language.name, ext)) else: diff --git a/cms/server/contest/handlers/tasksubmission.py b/cms/server/contest/handlers/tasksubmission.py index 96f5921a35..609613c749 100644 --- a/cms/server/contest/handlers/tasksubmission.py +++ b/cms/server/contest/handlers/tasksubmission.py @@ -113,7 +113,7 @@ def post(self, task_name): # (nor it discloses information to the user), but it is # useful for automatic testing to obtain the submission id). query_args["submission_id"] = \ - encrypt_number(submission.id, config.secret_key) + encrypt_number(submission.id, config.web_server.secret_key) self.redirect(self.contest_url("tasks", task.name, "submissions", **query_args)) diff --git a/cms/server/contest/handlers/taskusertest.py b/cms/server/contest/handlers/taskusertest.py index 9971967ee8..7ac572cb21 100644 --- a/cms/server/contest/handlers/taskusertest.py +++ b/cms/server/contest/handlers/taskusertest.py @@ -156,7 +156,7 @@ def post(self, task_name): # (nor it discloses information to the user), but it is # useful for automatic testing to obtain the user test id). query_args["user_test_id"] = \ - encrypt_number(user_test.id, config.secret_key) + encrypt_number(user_test.id, config.web_server.secret_key) self.redirect(self.contest_url("testing", task_name=task.name, **query_args)) diff --git a/cms/server/contest/printing.py b/cms/server/contest/printing.py index e4b18c293c..0334365831 100644 --- a/cms/server/contest/printing.py +++ b/cms/server/contest/printing.py @@ -95,16 +95,16 @@ def accept_print_job( """ - if config.printer is None: + if config.printing.printer is None: raise PrintingDisabled() old_count = sql_session.query(func.count(PrintJob.id)) \ .filter(PrintJob.participation == participation).scalar() - if config.max_jobs_per_user <= old_count: + if config.printing.max_jobs_per_user <= old_count: raise UnacceptablePrintJob( N_("Too many print jobs!"), N_("You have reached the maximum limit of at most %d print jobs."), - config.max_jobs_per_user) + config.printing.max_jobs_per_user) if len(files) != 1 or "file" not in files or len(files["file"]) != 1: raise UnacceptablePrintJob( @@ -114,11 +114,11 @@ def accept_print_job( filename = files["file"][0].filename data = files["file"][0].body - if len(data) > config.max_print_length: + if len(data) > config.printing.max_print_length: raise UnacceptablePrintJob( N_("File too big!"), N_("Each file must be at most %d bytes long."), - config.max_print_length) + config.printing.max_print_length) try: digest = file_cacher.put_file_content( diff --git a/cms/server/contest/server.py b/cms/server/contest/server.py index ed410d1b38..172cad4267 100644 --- a/cms/server/contest/server.py +++ b/cms/server/contest/server.py @@ -66,20 +66,20 @@ def __init__(self, shard: int, contest_id: int | None = None): parameters = { "static_files": [("cms.server", "static"), ("cms.server.contest", "static")], - "cookie_secret": hex_to_bin(config.secret_key), - "debug": config.tornado_debug, - "is_proxy_used": config.is_proxy_used, - "num_proxies_used": config.num_proxies_used, + "cookie_secret": hex_to_bin(config.web_server.secret_key), + "debug": config.web_server.tornado_debug, + "is_proxy_used": None, + "num_proxies_used": config.contest_web_server.num_proxies_used, "xsrf_cookies": True, } try: - listen_address = config.contest_listen_address[shard] - listen_port = config.contest_listen_port[shard] + listen_address = config.contest_web_server.listen_address[shard] + listen_port = config.contest_web_server.listen_port[shard] except IndexError: raise ConfigError("Wrong shard number for %s, or missing " "address/port configuration. Please check " - "contest_listen_address and contest_listen_port " + "listen_address and listen_port " "in cms.toml." % __name__) self.contest_id = contest_id @@ -101,7 +101,8 @@ def __init__(self, shard: int, contest_id: int | None = None): listen_address=listen_address) self.wsgi_app = SharedDataMiddleware( - self.wsgi_app, {"/docs": config.docs_path or config.stl_path}, + self.wsgi_app, {"/docs": config.contest_web_server.docs_path + or config.contest_web_server.stl_path}, cache=True, cache_timeout=SECONDS_IN_A_YEAR, fallback_mimetype="application/octet-stream") @@ -122,12 +123,12 @@ def __init__(self, shard: int, contest_id: int | None = None): self.scoring_service = self.connect_to( ServiceCoord("ScoringService", 0)) - ranking_enabled = len(config.rankings) > 0 + ranking_enabled = len(config.proxy_service.rankings) > 0 self.proxy_service = self.connect_to( ServiceCoord("ProxyService", 0), must_be_present=ranking_enabled) - printing_enabled = config.printer is not None + printing_enabled = config.printing.printer is not None self.printing_service = self.connect_to( ServiceCoord("PrintingService", 0), must_be_present=printing_enabled) diff --git a/cms/server/contest/submission/utils.py b/cms/server/contest/submission/utils.py index 9246c1aab2..a233574924 100644 --- a/cms/server/contest/submission/utils.py +++ b/cms/server/contest/submission/utils.py @@ -146,7 +146,7 @@ def store_local_copy( """ try: - path = os.path.join(path.replace("%s", config.data_dir), + path = os.path.join(path.replace("%s", config.global_.data_dir), participation.user.username) if not os.path.exists(path): os.makedirs(path) diff --git a/cms/server/contest/submission/workflow.py b/cms/server/contest/submission/workflow.py index 0a72542d7f..9210fbd80f 100644 --- a/cms/server/contest/submission/workflow.py +++ b/cms/server/contest/submission/workflow.py @@ -166,7 +166,9 @@ def accept_submission( # the largest allowed. Since we don't yet know which files from the archive # are used and which are extraneous, this size limit applies to the entire # archive in total. - archive_size_limit = config.max_submission_length * len(required_codenames) + archive_size_limit = config.contest_web_server.max_submission_length * len( + required_codenames + ) # Honest users never need to submit more than required_codenames files, but # we are a bit lenient to allow .DS_Store or other hidden files that might # accidentally end up in an archive. @@ -180,7 +182,7 @@ def accept_submission( raise UnacceptableSubmission( N_("Submission too big!"), N_("Each source file must be at most %d bytes long."), - config.max_submission_length) + config.contest_web_server.max_submission_length) if e.too_many_files: raise UnacceptableSubmission( N_("Submission too big!"), @@ -214,19 +216,26 @@ def accept_submission( N_("Invalid submission format!"), N_("Please select the correct files.")) - if any(len(content) > config.max_submission_length - for content in files.values()): + if any( + len(content) > config.contest_web_server.max_submission_length + for content in files.values() + ): raise UnacceptableSubmission( N_("Submission too big!"), N_("Each source file must be at most %d bytes long."), - config.max_submission_length) + config.contest_web_server.max_submission_length) # All checks done, submission accepted. - if config.submit_local_copy: + if config.contest_web_server.submit_local_copy: try: - store_local_copy(config.submit_local_copy_path, participation, - task, timestamp, files) + store_local_copy( + config.contest_web_server.submit_local_copy_path, + participation, + task, + timestamp, + files, + ) except StorageFailed: logger.error("Submission local copy failed.", exc_info=True) @@ -374,7 +383,9 @@ def accept_user_test( required_codenames.add("input") # See accept_submission() for these variables. - archive_size_limit = config.max_submission_length * len(required_codenames) + archive_size_limit = config.contest_web_server.max_submission_length * len( + required_codenames + ) archive_max_files = 2 * len(required_codenames) try: received_files = extract_files_from_tornado( @@ -413,25 +424,35 @@ def accept_user_test( N_("Invalid test format!"), N_("Please select the correct files.")) - if any(len(content) > config.max_submission_length - for codename, content in files.items() - if codename != "input"): + if any( + len(content) > config.contest_web_server.max_submission_length + for codename, content in files.items() + if codename != "input" + ): raise UnacceptableUserTest( N_("Test too big!"), N_("Each source file must be at most %d bytes long."), - config.max_submission_length) - if "input" in files and len(files["input"]) > config.max_input_length: + config.contest_web_server.max_submission_length) + if ( + "input" in files + and len(files["input"]) > config.contest_web_server.max_input_length + ): raise UnacceptableUserTest( N_("Input too big!"), N_("The input file must be at most %d bytes long."), - config.max_input_length) + config.contest_web_server.max_input_length) # All checks done, submission accepted. - if config.tests_local_copy: + if config.contest_web_server.tests_local_copy: try: - store_local_copy(config.tests_local_copy_path, participation, task, - timestamp, files) + store_local_copy( + config.contest_web_server.tests_local_copy_path, + participation, + task, + timestamp, + files, + ) except StorageFailed: logger.error("Test local copy failed.", exc_info=True) diff --git a/cms/service/Checker.py b/cms/service/Checker.py index bb885fbd15..3c78ca010c 100644 --- a/cms/service/Checker.py +++ b/cms/service/Checker.py @@ -39,7 +39,7 @@ class Checker(Service): def __init__(self, shard): Service.__init__(self, shard) - for service in config.async_config.core_services: + for service in config.services: self.connect_to(service) self.add_timeout(self.check, None, 90.0, immediately=True) diff --git a/cms/service/LogService.py b/cms/service/LogService.py index 96c5cb250d..d0c1bb7fc4 100644 --- a/cms/service/LogService.py +++ b/cms/service/LogService.py @@ -47,8 +47,8 @@ def __init__(self, shard: int): Service.__init__(self, shard) # Determine location of log file, and make directories. - log_dir = os.path.join(config.log_dir, "cms") - if not mkdir(config.log_dir) or \ + log_dir = os.path.join(config.global_.log_dir, "cms") + if not mkdir(config.global_.log_dir) or \ not mkdir(log_dir): logger.error("Cannot create necessary directories.") self.exit() diff --git a/cms/service/PrintingService.py b/cms/service/PrintingService.py index 959b4de655..73ed924a7b 100644 --- a/cms/service/PrintingService.py +++ b/cms/service/PrintingService.py @@ -102,7 +102,7 @@ def execute(self, entry: QueueEntry[PrintingOperation]): logger.info("Print job %d was already sent to the printer.", printjob_id) - directory = tempfile.mkdtemp(dir=config.temp_dir) + directory = tempfile.mkdtemp(dir=config.global_.temp_dir) logger.info("Preparing print job in directory %s", directory) # Take the base name just to be sure. @@ -111,7 +111,7 @@ def execute(self, entry: QueueEntry[PrintingOperation]): with open(source, "wb") as file_: self.file_cacher.get_file_to_fobj(printjob.digest, file_) - if filename.endswith(".pdf") and config.pdf_printing_allowed: + if filename.endswith(".pdf") and config.printing.pdf_printing_allowed: source_pdf = source else: # Convert text to ps. @@ -120,11 +120,11 @@ def execute(self, entry: QueueEntry[PrintingOperation]): source, "--delegate=no", "--output=" + source_ps, - "--medium=%s" % config.paper_size.capitalize(), + "--medium=%s" % config.printing.paper_size.capitalize(), "--portrait", "--columns=1", "--rows=1", - "--pages=1-%d" % (config.max_pages_per_job), + "--pages=1-%d" % (config.printing.max_pages_per_job), "--header=", "--footer=", "--left-footer=", @@ -147,7 +147,7 @@ def execute(self, entry: QueueEntry[PrintingOperation]): # Convert ps to pdf source_pdf = os.path.join(directory, "source.pdf") cmd = ["ps2pdf", - "-sPAPERSIZE=%s" % config.paper_size.lower(), + "-sPAPERSIZE=%s" % config.printing.paper_size.lower(), source_ps] try: subprocess.check_call(cmd, cwd=directory) @@ -162,7 +162,7 @@ def execute(self, entry: QueueEntry[PrintingOperation]): logger.info("Preparing %d page(s) (plus the title page)", page_count) - if page_count > config.max_pages_per_job: + if page_count > config.printing.max_pages_per_job: logger.info("Too many pages.") printjob.done = True printjob.status = [N_("Print job has too many pages")] @@ -178,7 +178,7 @@ def execute(self, entry: QueueEntry[PrintingOperation]): .render(user=user, filename=filename, timestr=timestr, page_count=page_count, - paper_size=config.paper_size)) + paper_size=config.printing.paper_size)) cmd = ["pdflatex", "-interaction", "nonstopmode", @@ -197,7 +197,7 @@ def execute(self, entry: QueueEntry[PrintingOperation]): try: printer_connection = cups.Connection() printer_connection.printFile( - config.printer, result, + config.printing.printer, result, "Printout %d" % printjob_id, {}) except cups.IPPError as error: logger.error("Unable to print: `%s'.", error) @@ -225,7 +225,7 @@ def __init__(self, shard: int): self.add_executor(PrintingExecutor(self.file_cacher)) self.start_sweeper(61.0) - if config.printer is None: + if config.printing.printer is None: logger.info("Printing is disabled, so the PrintingService is " "idle.") return diff --git a/cms/service/ProxyService.py b/cms/service/ProxyService.py index 18f276bf2d..6927e64985 100644 --- a/cms/service/ProxyService.py +++ b/cms/service/ProxyService.py @@ -90,7 +90,7 @@ def safe_put_data(ranking: str, resource: str, data: dict, operation: str): res = requests.put(url, json.dumps(data), auth=(auth.username, auth.password), headers={'content-type': 'application/json'}, - verify=config.https_certfile) + verify=config.proxy_service.https_certfile) except requests.exceptions.RequestException as error: msg = "%s while %s: %s." % (type(error).__name__, operation, error) logger.warning(msg) @@ -273,7 +273,7 @@ def __init__(self, shard: int, contest_id: int): # Create one executor for each ranking. self.rankings = list() - for ranking in config.rankings: + for ranking in config.proxy_service.rankings: self.add_executor(ProxyExecutor(ranking)) # Enqueue the dispatch of some initial data to rankings. Needs diff --git a/cms/service/ResourceService.py b/cms/service/ResourceService.py index 6ec43d4b75..d893acf1ad 100644 --- a/cms/service/ResourceService.py +++ b/cms/service/ResourceService.py @@ -280,7 +280,7 @@ def _find_local_services(self) -> list[ServiceCoord]: """ logger.debug("ResourceService._find_local_services") - services = config.async_config.core_services + services = config.services local_machine = services[self._my_coord].ip local_services = [x for x in services diff --git a/cms/service/ScoringService.py b/cms/service/ScoringService.py index 67c9bf0c72..be67982155 100644 --- a/cms/service/ScoringService.py +++ b/cms/service/ScoringService.py @@ -136,7 +136,7 @@ def __init__(self, shard: int): super().__init__(shard) # Set up communication with ProxyService. - ranking_enabled = len(config.rankings) > 0 + ranking_enabled = len(config.proxy_service.rankings) > 0 self.proxy_service = self.connect_to( ServiceCoord("ProxyService", 0), must_be_present=ranking_enabled) diff --git a/cms/util.py b/cms/util.py index 5fab61609a..efdf2019cd 100644 --- a/cms/util.py +++ b/cms/util.py @@ -35,7 +35,7 @@ import gevent import gevent.socket -from cms import ServiceCoord, Address, ConfigError, async_config +from cms import ServiceCoord, Address, ConfigError, config import typing if typing.TYPE_CHECKING: @@ -144,7 +144,7 @@ def get_safe_shard(service: str, provided_shard: int | None) -> int: return computed_shard else: coord = ServiceCoord(service, provided_shard) - if coord not in async_config.core_services: + if coord not in config.services: logger.critical("The provided shard number for service %s " "cannot be found in the configuration, " "quitting.", service) @@ -160,10 +160,8 @@ def get_service_address(key: ServiceCoord) -> Address: returns: listening address of key. """ - if key in async_config.core_services: - return async_config.core_services[key] - elif key in async_config.other_services: - return async_config.other_services[key] + if key in config.services: + return config.services[key] else: raise KeyError("Service not found.") diff --git a/cmscommon/archive.py b/cmscommon/archive.py index 7fa6d47635..8e8c2908eb 100644 --- a/cmscommon/archive.py +++ b/cmscommon/archive.py @@ -110,7 +110,7 @@ def from_raw_data(raw_data: bytes) -> "Archive | None": archive or None, if raw_data doesn't represent an archive. """ - temp_file, temp_filename = tempfile.mkstemp(dir=config.temp_dir) + temp_file, temp_filename = tempfile.mkstemp(dir=config.global_.temp_dir) with open(temp_file, "wb") as temp_file: temp_file.write(raw_data) @@ -140,7 +140,7 @@ def unpack(self) -> str: return: the path of the temporary directory. """ - self.temp_dir = tempfile.mkdtemp(dir=config.temp_dir) + self.temp_dir = tempfile.mkdtemp(dir=config.global_.temp_dir) patoolib.extract_archive(self.path, outdir=self.temp_dir, interactive=False) return self.temp_dir diff --git a/cmscommon/conf_parser.py b/cmscommon/conf_parser.py new file mode 100644 index 0000000000..67b1ce447e --- /dev/null +++ b/cmscommon/conf_parser.py @@ -0,0 +1,183 @@ +import dataclasses +import logging +import re +import tomllib +import sys +import types +import typing + + +class ConfigError(Exception): + """Exception for critical configuration errors.""" + + pass + + +class ConfigTypeError(ConfigError): + def __init__(self, path: str, expected: str, got: object): + msg = f"Expected {path} to be {expected}, got {type(got).__name__}" + super().__init__(msg) + + +_T = typing.TypeVar("_T") + + +def parse_config( + config_file_path: str, config_class: type[_T], enoent_help: str = "" +) -> _T: + """ + Load a TOML config file into a config class, checking for type errors. + + config_class must be a dataclass. Each of its fields must have a type that + is either one of the basic TOML types (str, int, float, bool), another + dataclass satisfying the same rules, a list[T] with T satisfying these + rules, a dict[str, T] with T satisfying these rules, a tuple whose types + satisfy these rules, or an optional form (T | None) of any of the above. + + Dataclasses correspond to tables with specific keys in the TOML file. All + other values correspond directly to the TOML types. + + If a dataclass field's name ends with "_", the corresponding TOML table key + will not have the underscore. This is to allow table keys corresponding to + python keywords. + + config_file_path: Path to the config TOML file. + config_class: Dataclass to load the configuration into. + enoent_help: Extra help text for "file not found" error. + """ + try: + data = tomllib.load(open(config_file_path, "rb")) + return parse_config_obj(data, config_class, "") + except FileNotFoundError: + logging.critical( + f"Cannot find configuration file {config_file_path}{enoent_help}" + ) + sys.exit(1) + except (ConfigError, tomllib.TOMLDecodeError) as e: + # Don't show stacktrace for basic errors. + logging.critical(f"Cannot load configuration file {config_file_path}: {e}") + sys.exit(1) + except Exception: + logging.critical( + f"Cannot load configuration file {config_file_path}", exc_info=True + ) + sys.exit(1) + + +def format_key(key: str): + if re.fullmatch(r"[A-Za-z0-9_-]+", key): + return key + else: + # This should be a valid TOML key, assuming python escape sequences are + # compatible with toml ones. In any case, it's good enough for error + # messages. + return repr(key) + +def join_path(path: str, new_part: str): + if path != "": + return path + "." + new_part + else: + return new_part + +# tomllib return types: str, int, float, bool, datetime (not really relevant), +# list[X], dict[str, X] + +def parse_config_obj(data: object, obj_class: type[_T], path: str) -> _T: + if typing.get_origin(obj_class) in (typing.Union, types.UnionType): + # The only unions we support are " | None", for optionals. + args = typing.get_args(obj_class) + assert len(args) == 2 and args[1] is type(None) + # If we reached this function, then we are trying to parse `data` as a + # value of type `obj_class`. TOML has no `null`, so this means `data` + # must match the non-None side of the union. + obj_class = args[0] + + if dataclasses.is_dataclass(obj_class): + if not isinstance(data, dict): + raise ConfigTypeError(path, "a table", data) + kw_args = {} + for field in dataclasses.fields(obj_class): + if not field.init: + continue + # Some field names are suffixed with _ because they would otherwise + # conflict with python keywords. + config_name = field.name.removesuffix("_") + + is_required = ( + field.default is dataclasses.MISSING + and field.default_factory is dataclasses.MISSING + ) + field_path = join_path(path, format_key(config_name)) + if is_required and config_name not in data: + raise ConfigError(f"Key {field_path} is required") + + if config_name in data: + kw_args[field.name] = parse_config_obj( + data[config_name], typing.cast(type[_T], field.type), field_path + ) + del data[config_name] + + for k in data: + thispath = join_path(path, format_key(k)) + logging.warning(f"Unrecognized key {thispath} in config, ignoring.") + + return obj_class(**kw_args) + + elif typing.get_origin(obj_class) is dict: + args = typing.get_args(obj_class) + assert args[0] is str + value_type = args[1] + + if not isinstance(data, dict): + raise ConfigTypeError(path, "a table", data) + + result = {} + for k, v in data.items(): + result[k] = parse_config_obj(v, value_type, join_path(path, format_key(k))) + + # As far as I can tell, Python's type system doesn't support narrowing + # _T based on runtime checks of obj_class, so the simplest way to get + # this to type-check is to build result as a generic dict and tell the + # type checker that it has the right type. + return typing.cast(_T, result) + + elif typing.get_origin(obj_class) in (tuple, list): + args = typing.get_args(obj_class) + list_mode = True + if typing.get_origin(obj_class) is tuple: + # Tuples can be either immutable lists (tuple[T, ...]) or lists + # having elements of different types (anything else). + list_mode = len(args) == 2 and args[1] == Ellipsis + + if not isinstance(data, list): + raise ConfigTypeError(path, "a list", data) + + result = [] + if list_mode: + value_type = args[0] + for i, x in enumerate(data): + result.append(parse_config_obj(x, value_type, path + f"[{i}]")) + else: + if len(args) != len(data): + raise ConfigError( + f"Expected {path} to have {len(args)} elements, got {len(data)}" + ) + for i, (type_, val) in enumerate(zip(args, data)): + result.append(parse_config_obj(val, type_, path + f"[{i}]")) + + # typing.cast isn't enough to make the type checker happy here. + return obj_class(result) # type: ignore + + elif obj_class in (str, int, bool): + if not isinstance(data, obj_class): + raise ConfigTypeError(path, obj_class.__name__, data) + return data + + elif obj_class is float: + # Allow specifying floats as ints. + if not isinstance(data, int | float): + raise ConfigTypeError(path, "float", data) + return typing.cast(_T, float(data)) + + else: + raise AssertionError(f"Unsupported type found in configuration: {obj_class}") diff --git a/cmscontrib/PrometheusExporter.py b/cmscontrib/PrometheusExporter.py index add4657dc7..4c22773a39 100644 --- a/cmscontrib/PrometheusExporter.py +++ b/cmscontrib/PrometheusExporter.py @@ -302,12 +302,12 @@ def main(): parser.add_argument( "--host", help="IP address to bind to", - default=config.prometheus_listen_address, + default=config.prometheus.listen_address, ) parser.add_argument( "--port", help="Port to use", - default=config.prometheus_listen_port, + default=config.prometheus.listen_port, type=int, ) parser.add_argument( diff --git a/cmscontrib/RWSHelper.py b/cmscontrib/RWSHelper.py index 5d7cb83712..1fad44b37e 100644 --- a/cmscontrib/RWSHelper.py +++ b/cmscontrib/RWSHelper.py @@ -61,7 +61,9 @@ def get_url(shard: int, entity_type: str, entity_id: str): - return urljoin(config.rankings[shard], '%ss/%s' % (entity_type, entity_id)) + return urljoin( + config.proxy_service.rankings[shard], "%ss/%s" % (entity_type, entity_id) + ) def main(): @@ -72,9 +74,15 @@ def main(): # FIXME It would be nice to use '--rankings' with action='store' # and nargs='+' but it doesn't seem to work with subparsers... parser.add_argument( - '-r', '--ranking', dest='rankings', action='append', type=int, - choices=list(range(len(config.rankings))), metavar='shard', - help="select which RWS to connect to (omit for 'all')") + "-r", + "--ranking", + dest="rankings", + action="append", + type=int, + choices=list(range(len(config.proxy_service.rankings))), + metavar="shard", + help="select which RWS to connect to (omit for 'all')", + ) subparsers = parser.add_subparsers( title='available actions', metavar='action', help='what to ask the RWS to do with the entity') @@ -124,7 +132,7 @@ def main(): if args.rankings is not None: shards = args.rankings else: - shards = list(range(len(config.rankings))) + shards = list(range(len(config.proxy_service.rankings))) s = Session() had_error = False @@ -154,7 +162,7 @@ def main(): logger.info("Sending request") try: - res = s.send(req, verify=config.https_certfile) + res = s.send(req, verify=config.proxy_service.https_certfile) except RequestException: logger.error("Failed", exc_info=True) had_error = True diff --git a/cmscontrib/TelegramBot.py b/cmscontrib/TelegramBot.py index 18ae86f364..ccc8877fb6 100644 --- a/cmscontrib/TelegramBot.py +++ b/cmscontrib/TelegramBot.py @@ -129,10 +129,10 @@ def __init__(self, chat_id: str, token: str, contest_id: int | None) -> None: self.application.add_handler(CallbackQueryHandler(self.answer)) self.question_storage_dir = os.path.join( - config.data_dir, "telegram", "question" + config.global_.data_dir, "telegram", "question" ) self.announcement_storage_dir = os.path.join( - config.data_dir, "telegram", "announcement" + config.global_.data_dir, "telegram", "announcement" ) os.makedirs(self.question_storage_dir, exist_ok=True) os.makedirs(self.announcement_storage_dir, exist_ok=True) @@ -464,12 +464,12 @@ def main(): if contest_id == "ALL": contest_id = None - if config.telegram_bot_token is None or config.telegram_bot_chat_id is None: + if config.telegram_bot is None: raise ConfigError( "Need to configure the Telegram bot before starting it") bot = TelegramBot( - config.telegram_bot_chat_id, config.telegram_bot_token, contest_id + config.telegram_bot.chat_id, config.telegram_bot.bot_token, contest_id ) asyncio.run(bot.run()) diff --git a/cmsranking/Config.py b/cmsranking/Config.py index d65d58c4f6..27a37d3a20 100644 --- a/cmsranking/Config.py +++ b/cmsranking/Config.py @@ -16,135 +16,57 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -import atexit -import errno +from dataclasses import dataclass import logging import os import sys -import tomllib -import typing - -import importlib.resources +from cmscommon import conf_parser from cmsranking.Logger import add_file_handler logger = logging.getLogger(__name__) -CMS_RANKING_CONFIG_ENV_VAR = "CMS_RANKING_CONFIG" +# Try to find CMS installation root from the venv in which we run +if sys.prefix == "/usr": + logger.critical("CMS must be run within a Python virtual environment") + sys.exit(1) +def default_path(name): + return os.path.join(sys.prefix, name) -class Config: - """An object holding the current configuration. - - """ - def __init__(self): - """Fill this object with the default values for each key. - - """ - # Connection. - self.bind_address = '' - self.http_port: int | None = 8890 - self.https_port: int | None = None - self.https_certfile: str | None = None - self.https_keyfile: str | None = None - # TODO unused??? - self.timeout = 600 # 10 minutes (in seconds) - - # Authentication. - self.realm_name = 'Scoreboard' - self.username = 'usern4me' - self.password = 'passw0rd' - - # Buffers - self.buffer_size = 100 # Needs to be strictly positive. - - # Keep the static files context manager alive for the application lifetime - self._static_files_context = importlib.resources.path("cmsranking", "static") - self.web_dir = str(self._static_files_context.__enter__()) - # Register cleanup handler to properly exit the context manager - atexit.register(self._static_files_context.__exit__, None, None, None) - - # Try to find CMS installation root from the venv in which we run - self.base_dir = sys.prefix - if self.base_dir == '/usr': - logger.critical('CMS must be run within a Python virtual environment') - sys.exit(1) - self.log_dir = os.path.join(self.base_dir, 'log/ranking') - self.lib_dir = os.path.join(self.base_dir, 'lib/ranking') - - # Default config file path can be overridden using environment - # variable 'CMS_RANKING_CONFIG'. - default_config_file = os.path.join(self.base_dir, 'etc/cms_ranking.toml') - self.config_file = os.environ.get('CMS_RANKING_CONFIG', default_config_file) - - def get(self, key): - """Get the config value for the given key. - - """ - return getattr(self, key) - - def load(self, config_override: str | None = None): - """Load the configuration file. - - """ - - config_file = config_override if config_override is not None else self.config_file - if not self._load_config(config_file): - logging.critical(f'Cannot load configuration file {config_file}') - sys.exit(1) - - try: - os.makedirs(self.lib_dir) - except OSError: - pass # We assume the directory already exists... - - try: - os.makedirs(self.web_dir) - except OSError: - pass # We assume the directory already exists... - - try: - os.makedirs(self.log_dir) - except OSError: - pass # We assume the directory already exists... +@dataclass +class Config: + # Connection. + bind_address: str = "127.0.0.1" + http_port: int | None = 8890 + https_port: int | None = None + https_certfile: str | None = None + https_keyfile: str | None = None + + # Authentication. + realm_name: str = "Scoreboard" + username: str = "usern4me" + password: str = "passw0rd" + + # Buffers + buffer_size: int = 100 + + log_dir: str = default_path("log/ranking") + lib_dir: str = default_path("lib/ranking") + + def __post_init__(self): + os.makedirs(self.lib_dir, exist_ok=True) + os.makedirs(self.log_dir, exist_ok=True) add_file_handler(self.log_dir) - def _load_config(self, path: str) -> bool: - """Populate config parameters from the given file. - - Parse it as TOML and store in self all configuration properties - it defines. Log critical message and return False if anything - goes wrong or seems odd. - - path: the path of the TOML config file. - returns: whether parsing was successful. - - """ - # Load config file. - try: - with open(path, 'rb') as f: - data = tomllib.load(f) - except FileNotFoundError: - logger.debug("Couldn't find config file %s (maybe you need to " - "convert it from cms.ranking.conf to cms_ranking.toml?).", path) - return False - except OSError as error: - logger.warning("I/O error while opening file %s: [%s] %s", - path, errno.errorcode[error.errno], - os.strerror(error.errno)) - return False - except ValueError as error: - logger.warning("Invalid syntax in file %s: %s", path, error) - return False - - # Store every config property. - for key, value in data.items(): - if hasattr(self, key): - setattr(self, key, value) - else: - logger.warning("Unrecognized key %s in config!", key) - - return True + +def load_config(config_override: str | None = None) -> Config: + default_config_file = default_path("etc/cms_ranking.toml") + config_file = os.environ.get("CMS_RANKING_CONFIG", default_config_file) + if config_override is not None: + config_file = config_override + hint = " (maybe you need to convert it from cms.ranking.conf to cms_ranking.toml?)" + return conf_parser.parse_config(config_file, Config, hint) diff --git a/cmsranking/RankingWebServer.py b/cmsranking/RankingWebServer.py index a6c1ba24ad..bd55a0035f 100755 --- a/cmsranking/RankingWebServer.py +++ b/cmsranking/RankingWebServer.py @@ -17,7 +17,9 @@ # along with this program. If not, see . import argparse +import atexit import functools +import importlib.resources import json import logging import os @@ -40,7 +42,7 @@ # Needed for initialization. Do not remove. import cmsranking.Logger # noqa from cmscommon.eventsource import EventSource -from cmsranking.Config import Config +from cmsranking.Config import load_config from cmsranking.Contest import Contest from cmsranking.Entity import InvalidData from cmsranking.Scoring import ScoringStore @@ -533,8 +535,13 @@ def main() -> int: help="do not require confirmation on dropping data") args = parser.parse_args() - config = Config() - config.load(args.config) + config = load_config(args.config) + + # Keep the static files context manager alive for the application lifetime + static_files_context = importlib.resources.path("cmsranking", "static") + web_dir = str(static_files_context.__enter__()) + # Register cleanup handler to properly exit the context manager + atexit.register(static_files_context.__exit__, None, None, None) if args.drop: if args.yes: @@ -580,11 +587,11 @@ def main() -> int: stores["scoring"].init_store() toplevel_handler = RoutingHandler( - RootHandler(config.web_dir), + RootHandler(web_dir), DataWatcher(stores, config.buffer_size), ImageHandler( os.path.join(config.lib_dir, '%(name)s'), - os.path.join(config.web_dir, 'img', 'logo.png')), + os.path.join(web_dir, 'img', 'logo.png')), ScoreHandler(stores), HistoryHandler(stores)) @@ -610,12 +617,12 @@ def main() -> int: config.username, config.password, config.realm_name), '/faces': ImageHandler( os.path.join(config.lib_dir, 'faces', '%(name)s'), - os.path.join(config.web_dir, 'img', 'face.png')), + os.path.join(web_dir, 'img', 'face.png')), '/flags': ImageHandler( os.path.join(config.lib_dir, 'flags', '%(name)s'), - os.path.join(config.web_dir, 'img', 'flag.png')), + os.path.join(web_dir, 'img', 'flag.png')), '/sublist': SubListHandler(stores), - }), {'/': config.web_dir}) + }), {'/': web_dir}) servers: list[WSGIServer] = list() if config.http_port is not None: diff --git a/cmstestsuite/StressTest.py b/cmstestsuite/StressTest.py index 1655b3cf08..126464fe0e 100755 --- a/cmstestsuite/StressTest.py +++ b/cmstestsuite/StressTest.py @@ -361,7 +361,7 @@ def main(): else: base_url = "http://%s:%d/" % \ (get_service_address(ServiceCoord('ContestWebServer', 0))[0], - config.contest_listen_port[0]) + config.contest_web_server.listen_port[0]) metrics = DEFAULT_METRICS metrics["time_coeff"] = args.time_coeff diff --git a/cmstestsuite/programstarter.py b/cmstestsuite/programstarter.py index 7c6959308e..ceffab0aeb 100644 --- a/cmstestsuite/programstarter.py +++ b/cmstestsuite/programstarter.py @@ -55,7 +55,7 @@ class RemoteService: """ def __init__(self, cms_config, service_name, shard): - address, port = cms_config["core_services"][service_name][shard] + address, port = cms_config["services"][service_name][shard] self.service_name = service_name self.shard = shard @@ -227,9 +227,9 @@ def _check_service(self): # In case it is a server, we also check HTTP is serving. if self.service_name == "AdminWebServer": - port = self.cms_config["admin_listen_port"] + port = self.cms_config["admin_web_server"]["listen_port"] elif self.service_name == "ContestWebServer": - port = self.cms_config["contest_listen_port"][self.shard] + port = self.cms_config["contest_web_server"]["listen_port"][self.shard] else: return @@ -239,7 +239,7 @@ def _check_service(self): def _check_ranking_web_server(self): """Health checker for RWS.""" - url = urlsplit(self.cms_config["rankings"][0]) + url = urlsplit(self.cms_config["proxy_service"]["rankings"][0]) sock = socket.socket() sock.connect((url.hostname, url.port)) sock.close() diff --git a/cmstestsuite/unit_tests/db/session_test.py b/cmstestsuite/unit_tests/db/session_test.py index 9f4c7fc9f6..ce0a468687 100755 --- a/cmstestsuite/unit_tests/db/session_test.py +++ b/cmstestsuite/unit_tests/db/session_test.py @@ -27,7 +27,7 @@ def _patch_db(s): """Patch the db connection string in the configuration""" - return patch.object(config, "database", s) + return patch.object(config.database, "url", s) @patch("psycopg2.connect") diff --git a/cmstestsuite/unit_tests/grading/tasktypes/CommunicationTest.py b/cmstestsuite/unit_tests/grading/tasktypes/CommunicationTest.py index f840e68edb..bdd298dd0c 100755 --- a/cmstestsuite/unit_tests/grading/tasktypes/CommunicationTest.py +++ b/cmstestsuite/unit_tests/grading/tasktypes/CommunicationTest.py @@ -352,8 +352,8 @@ def _set_evaluation_step_return_values( self.evaluation_step_after_run.side_effect = \ lambda sandbox, *args, **kwargs: sandbox_to_return_value[sandbox] - @patch.object(config, "trusted_sandbox_max_time_s", 4321) - @patch.object(config, "trusted_sandbox_max_memory_kib", 1234 * 1024) + @patch.object(config.sandbox, "trusted_sandbox_max_time_s", 4321) + @patch.object(config.sandbox, "trusted_sandbox_max_memory_kib", 1234 * 1024) def test_single_process_success(self): tt, job = self.prepare( [1, "stub", "fifo_io"], @@ -405,7 +405,7 @@ def test_single_process_success(self): sandbox_mgr.cleanup.assert_called_once_with(delete=True) sandbox_usr.cleanup.assert_called_once_with(delete=True) - @patch.object(config, "trusted_sandbox_max_time_s", 1) + @patch.object(config.sandbox, "trusted_sandbox_max_time_s", 1) def test_single_process_success_long_time_limit(self): # If the time limit is longer than trusted step default time limit, # the manager run should use the task time limit. @@ -596,8 +596,8 @@ def test_single_process_std_io(self): stdout_redirect="/fifo0/u0_to_m", multiprocess=ANY)]) - @patch.object(config, "trusted_sandbox_max_time_s", 4321) - @patch.object(config, "trusted_sandbox_max_memory_kib", 1234 * 1024) + @patch.object(config.sandbox, "trusted_sandbox_max_time_s", 4321) + @patch.object(config.sandbox, "trusted_sandbox_max_memory_kib", 1234 * 1024) def test_many_processes_success(self): tt, job = self.prepare( [2, "stub", "fifo_io"], @@ -666,7 +666,7 @@ def test_many_processes_success(self): sandbox_usr0.cleanup.assert_called_once_with(delete=True) sandbox_usr1.cleanup.assert_called_once_with(delete=True) - @patch.object(config, "trusted_sandbox_max_time_s", 3) + @patch.object(config.sandbox, "trusted_sandbox_max_time_s", 3) def test_many_processes_success_long_time_limit(self): # If the time limit is longer than trusted step default time limit, # the manager run should use the task time limit. diff --git a/cmstestsuite/unit_tests/grading/tasktypes/tasktypetestutils.py b/cmstestsuite/unit_tests/grading/tasktypes/tasktypetestutils.py index 6aad6bbe6f..5d3982b5c3 100644 --- a/cmstestsuite/unit_tests/grading/tasktypes/tasktypetestutils.py +++ b/cmstestsuite/unit_tests/grading/tasktypes/tasktypetestutils.py @@ -96,7 +96,7 @@ def setUpMocks(self, tasktype: str): self.tasktype = tasktype # Ensure we don't retain all sandboxes so we can verify delete(). - patcher = patch.object(config, "keep_sandbox", False) + patcher = patch.object(config.worker, "keep_sandbox", False) self.addCleanup(patcher.stop) patcher.start() diff --git a/cmstestsuite/unit_tests/server/contest/authentication_test.py b/cmstestsuite/unit_tests/server/contest/authentication_test.py index d98eee6c50..82b6b7cf8a 100755 --- a/cmstestsuite/unit_tests/server/contest/authentication_test.py +++ b/cmstestsuite/unit_tests/server/contest/authentication_test.py @@ -50,7 +50,9 @@ def setUp(self): contest=self.contest, user=self.user) # Set up a temporary admin token - patcher = patch.object(config, "contest_admin_token", "admin-token") + patcher = patch.object( + config.contest_web_server, "contest_admin_token", "admin-token" + ) self.addCleanup(patcher.stop) patcher.start() @@ -201,7 +203,9 @@ def setUp(self): self.impersonated_user = self.add_user(username="otheruser") self.impersonated_participation = self.add_participation( contest=self.contest, user=self.impersonated_user) - with patch.object(config, "contest_admin_token", "admin-token"): + with patch.object( + config.contest_web_server, "contest_admin_token", "admin-token" + ): _, self.impersonated_cookie = validate_login( self.session, self.contest, self.timestamp, "otheruser", "", ipaddress.ip_address("10.0.0.2"), "admin-token") @@ -281,7 +285,7 @@ def assertFailure(self, **kwargs): self.assertIsNone(cookie) self.assertIs(impersonated, False) - @patch.object(config, "cookie_duration", 10) + @patch.object(config.contest_web_server, "cookie_duration", 10) def test_cookie_contains_timestamp(self): self.contest.ip_autologin = False self.contest.allow_password_authentication = True diff --git a/cmstestsuite/unit_tests/server/contest/printing_test.py b/cmstestsuite/unit_tests/server/contest/printing_test.py index ffe8a85ae6..f8d6c0033a 100755 --- a/cmstestsuite/unit_tests/server/contest/printing_test.py +++ b/cmstestsuite/unit_tests/server/contest/printing_test.py @@ -41,7 +41,7 @@ FILE_DIGEST = bytes_digest(FILE_CONTENT) -@patch.object(config, "printer", "not none") +@patch.object(config.printing, "printer", "not none") class TestAcceptPrintJob(DatabaseMixin, unittest.TestCase): def setUp(self): @@ -69,7 +69,7 @@ def test_success(self): self.timestamp)) def test_printing_not_allowed(self): - with patch.object(config, "printer", None): + with patch.object(config.printing, "printer", None): with self.assertRaises(PrintingDisabled): self.call({"file": [MockHTTPFile("myfile.pdf", FILE_CONTENT)]}) @@ -92,12 +92,12 @@ def test_storage_failure(self): with self.assertRaises(UnacceptablePrintJob): self.call({"file": [MockHTTPFile("myfile.pdf", FILE_CONTENT)]}) - @patch.object(config, "max_print_length", len(FILE_CONTENT) - 1) + @patch.object(config.printing, "max_print_length", len(FILE_CONTENT) - 1) def test_file_too_big(self): with self.assertRaises(UnacceptablePrintJob): self.call({"file": [MockHTTPFile("myfile.pdf", FILE_CONTENT)]}) - @patch.object(config, "max_jobs_per_user", 1) + @patch.object(config.printing, "max_jobs_per_user", 1) def test_too_many_print_jobs(self): self.call({"file": [MockHTTPFile("myfile.pdf", FILE_CONTENT)]}) with self.assertRaises(UnacceptablePrintJob): diff --git a/cmstestsuite/unit_tests/server/contest/submission/utils_test.py b/cmstestsuite/unit_tests/server/contest/submission/utils_test.py index a5e6fc3571..c4b840a138 100755 --- a/cmstestsuite/unit_tests/server/contest/submission/utils_test.py +++ b/cmstestsuite/unit_tests/server/contest/submission/utils_test.py @@ -230,7 +230,7 @@ def test_success_many_times(self): def test_success_with_data_dir(self): content = self.generate_content() - with patch.object(config, "data_dir", self.base_dir): + with patch.object(config.global_, "data_dir", self.base_dir): store_local_copy("%s/bar", self.participation, self.task, self.timestamp, {"foo.%l": content}) self.assertSomeFileContains(content, diff --git a/cmstestsuite/unit_tests/server/contest/submission/workflow_test.py b/cmstestsuite/unit_tests/server/contest/submission/workflow_test.py index 752025e22b..c109aee5b0 100755 --- a/cmstestsuite/unit_tests/server/contest/submission/workflow_test.py +++ b/cmstestsuite/unit_tests/server/contest/submission/workflow_test.py @@ -136,12 +136,15 @@ def setUp(self): self.fetch_file_digests_from_previous_submission.side_effect = \ lambda *args, **kwargs: self.digests - patcher = patch.object(config, "submit_local_copy", True) + patcher = patch.object(config.contest_web_server, "submit_local_copy", True) patcher.start() self.addCleanup(patcher.stop) patcher = patch.object( - config, "submit_local_copy_path", self.submit_local_copy_path) + config.contest_web_server, + "submit_local_copy_path", + self.submit_local_copy_path, + ) patcher.start() self.addCleanup(patcher.stop) @@ -342,13 +345,13 @@ def test_failure_due_to_files_too_large(self): self.files["foo.%l"] = FOO_CONTENT * 100 max_size = len(FOO_CONTENT) * 100 - 1 - with patch.object(config, "max_submission_length", max_size): + with patch.object(config.contest_web_server, "max_submission_length", max_size): with self.assertRaisesRegex(UnacceptableSubmission, "%d" % max_size): self.call() def test_success_without_store_local_copy(self): - with patch.object(config, "submit_local_copy", False): + with patch.object(config.contest_web_server, "submit_local_copy", False): submission = self.call() self.assertSubmissionIsValid( @@ -466,12 +469,15 @@ def setUp(self): self.fetch_file_digests_from_previous_submission.side_effect = \ lambda *args, **kwargs: self.digests - patcher = patch.object(config, "tests_local_copy", True) + patcher = patch.object(config.contest_web_server, "tests_local_copy", True) patcher.start() self.addCleanup(patcher.stop) patcher = patch.object( - config, "tests_local_copy_path", self.tests_local_copy_path) + config.contest_web_server, + "tests_local_copy_path", + self.tests_local_copy_path, + ) patcher.start() self.addCleanup(patcher.stop) @@ -681,7 +687,7 @@ def test_failure_due_to_files_too_large(self): self.files["foo.%l"] = FOO_CONTENT * 100 max_size = len(FOO_CONTENT) * 100 - 1 - with patch.object(config, "max_submission_length", max_size): + with patch.object(config.contest_web_server, "max_submission_length", max_size): with self.assertRaisesRegex(UnacceptableUserTest, "%d" % max_size): self.call() @@ -689,7 +695,7 @@ def test_failure_due_to_managers_too_large(self): self.files["spam.%l"] = SPAM_CONTENT * 100 max_size = len(SPAM_CONTENT) * 100 - 1 - with patch.object(config, "max_submission_length", max_size): + with patch.object(config.contest_web_server, "max_submission_length", max_size): with self.assertRaisesRegex(UnacceptableUserTest, "%d" % max_size): self.call() @@ -697,12 +703,12 @@ def test_failure_due_to_input_too_large(self): self.files["input"] = INPUT_CONTENT * 100 max_size = len(INPUT_CONTENT) * 100 - 1 - with patch.object(config, "max_input_length", max_size): + with patch.object(config.contest_web_server, "max_input_length", max_size): with self.assertRaisesRegex(UnacceptableUserTest, "%d" % max_size): self.call() def test_success_without_store_local_copy(self): - with patch.object(config, "tests_local_copy", False): + with patch.object(config.contest_web_server, "tests_local_copy", False): user_test = self.call() self.assertUserTestIsValid( diff --git a/cmstestsuite/unit_tests/util_test.py b/cmstestsuite/unit_tests/util_test.py index 24493824c5..1f485a66b4 100755 --- a/cmstestsuite/unit_tests/util_test.py +++ b/cmstestsuite/unit_tests/util_test.py @@ -30,23 +30,20 @@ get_safe_shard, get_service_address, get_service_shards, rmtree -class FakeAsyncConfig: - """Fake class for the configuration of service addresses.""" - core_services = { - ServiceCoord("Service", 0): Address("0.0.0.0", 0), - ServiceCoord("Service", 1): Address("0.0.0.1", 1), - } - other_services = {} +fake_async_config = { + ServiceCoord("Service", 0): Address("0.0.0.0", 0), + ServiceCoord("Service", 1): Address("0.0.0.1", 1), +} def _set_up_async_config(restore=False): """Fake the async config.""" if not restore: if not hasattr(_set_up_async_config, "original"): - _set_up_async_config.original = cms.util.async_config - cms.util.async_config = FakeAsyncConfig() + _set_up_async_config.original = cms.config.services + cms.config.services = fake_async_config else: - cms.util.async_config = _set_up_async_config.original + cms.config.services = _set_up_async_config.original def _set_up_ip_addresses(addresses=None, restore=False): diff --git a/cmstestsuite/web/CWSRequests.py b/cmstestsuite/web/CWSRequests.py index cd0c292235..6c68ee7a0e 100644 --- a/cmstestsuite/web/CWSRequests.py +++ b/cmstestsuite/web/CWSRequests.py @@ -162,7 +162,7 @@ def get_submission_id(self): return None try: submission_id = decrypt_number(query["submission_id"][0], - config.secret_key) + config.web_server.secret_key) except Exception: logger.warning("Unable to decrypt submission id from page: `%s'", self.redirected_to) @@ -229,7 +229,7 @@ def get_user_test_id(self): return None try: user_test_id = decrypt_number(query["user_test_id"][0], - config.secret_key) + config.web_server.secret_key) except Exception: logger.warning("Unable to decrypt user test id from page: `%s'", self.redirected_to) diff --git a/config/cms.sample.toml b/config/cms.sample.toml index 5108f54daa..505246c4a4 100644 --- a/config/cms.sample.toml +++ b/config/cms.sample.toml @@ -1,208 +1,224 @@ -############################# -# System-wide configuration # -############################# - -temp_dir = "/tmp" - +[global] # Whether to have a backdoor (see doc for the risks). backdoor = false +# Whether to print debug-level messages into the per-service log file. +file_log_debug = false +# Whether to print more detailed logs on stdout. +# The detailed log contains the thread name, file and function name, in +# addition to the operation, if present. +stream_log_detailed = false + +# Directories used by CMS. The default values are shown in the comments, +# with INSTALL_DIR representing the path of the virtual env that CMS was +# installed into. +# Short-lived temporary files. +temp_dir = "/tmp" +# Log files. +#log_dir = "INSTALL_DIR/log" +# Cached files. +#cache_dir = "INSTALL_DIR/cache" +# Miscellaneous files generated by CMS. +#data_dir = "INSTALL_DIR/lib" +# Run-time data (e.g. socket files). +#run_dir = "INSTALL_DIR/run" + +[services] +# Each service has some number of shards, defined in this table. For +# most services, it only makes sense to have one shard, but there should +# be one ResourceService per host, and it's possible to have muliple +# Workers and ContestWebServers. + +# For each shard of a service, the first element of the list specifies +# the hostname it will run on, and the second one its port for RPC +# communication with the rest of CMS. Each ResourceService will manage +# services with the same hostname. If you run services on different +# hosts, make sure they can connect to each other using the specified +# hostnames and ports. -############ -# Database # -############ +LogService = [["localhost", 29000]] +ResourceService = [["localhost", 28000]] +ScoringService = [["localhost", 28500]] +Checker = [["localhost", 22000]] +EvaluationService = [["localhost", 25000]] +Worker = [ + ["localhost", 26000], + ["localhost", 26001], + ["localhost", 26002], + ["localhost", 26003], + ["localhost", 26004], + ["localhost", 26005], + ["localhost", 26006], + ["localhost", 26007], + ["localhost", 26008], + ["localhost", 26009], + ["localhost", 26010], + ["localhost", 26011], + ["localhost", 26012], + ["localhost", 26013], + ["localhost", 26014], + ["localhost", 26015], +] +ContestWebServer = [["localhost", 21000]] +AdminWebServer = [["localhost", 21100]] +ProxyService = [["localhost", 28600]] +PrintingService = [["localhost", 25123]] +PrometheusExporter = [] +TelegramBot = [] + +[database] # Connection string for the database. -database = "postgresql+psycopg2://cmsuser:your_password_here@localhost:5432/cmsdb" +url = "postgresql+psycopg2://cmsuser:your_password_here@localhost:5432/cmsdb" # Whether SQLAlchemy prints DB queries on stdout. -database_debug = false +debug = false # Whether to use two-phase commit. twophase_commit = false -########## -# Worker # -########## -# Don't delete the sandbox directory under /tmp/ when they -# are not needed anymore. Warning: this can easily eat GB -# of space very soon. +[worker] +# Don't delete the sandbox directory under /tmp/ when they are not +# needed anymore. Warning: this can easily eat GB of space very soon. keep_sandbox = false -########### -# Sandbox # -########### - -# Do not allow contestants' solutions to write files bigger -# than this size (expressed in KB; defaults to 1 GB). -max_file_size = 1048576 -############### -# Web servers # -############### - -# This key is used to encode information that can be seen -# by the user, namely cookies and auto-incremented -# numbers. It should be changed for each -# contest. Particularly, you should not use this example -# for other than testing. It must be a 16 bytes long -# hexadecimal number. You can easily create a key calling: +[sandbox] +# Which sandbox implementation to use. Currently only isolate is +# supported. +sandbox_implementation = "isolate" +# Whether to use cgroups in isolate. +use_cgroups = true +# Do not allow contestants' solutions to write files bigger than this +# size (expressed in KB; defaults to 1 GB). +max_file_size = 1_048_576 + +# Max processes, CPU time (s), memory (KiB) for compilation runs. +compilation_sandbox_max_processes = 1000 +compilation_sandbox_max_time_s = 10.0 +compilation_sandbox_max_memory_kib = 524_288 # 512 MiB +# Max processes, CPU time (s), memory (KiB) for trusted (e.g. checker) +# runs. +trusted_sandbox_max_processes = 1000 +trusted_sandbox_max_time_s = 10.0 +trusted_sandbox_max_memory_kib = 4_194_304 # 4 GiB + + +[web_server] +# This key is used to encode information that can be seen by the user, +# namely cookies and auto-incremented numbers. It should be changed for +# each contest. Particularly, you should not use this example for other +# than testing. It must be a 16 bytes long hexadecimal number. You can +# easily create a key calling: # python -c 'from cmscommon import crypto; print(crypto.get_hex_random_key())' secret_key = "8e045a51e4b102ea803c06f92841a1fb" # Whether Tornado prints debug information on stdout. tornado_debug = false -#################### -# ContestWebServer # -#################### -# Listening HTTP addresses and ports for the CWSs listed below -# in core_services. By default only listens on localhost, meaning -# you need a separate reverse proxy to access it from the web. -# Set to empty string to allow connecting from anywhere. -contest_listen_address = ["127.0.0.1"] -contest_listen_port = [8888] +[contest_web_server] +# Listening HTTP addresses and ports for the CWSs listed above in +# [services]. By default only listens on localhost, meaning you need a +# separate reverse proxy to access it from the web. Set to empty string +# to allow connecting from anywhere. +listen_address = ["127.0.0.1"] +listen_port = [8888] -# Login cookie duration in seconds. The duration is refreshed -# on every manual request. +# Login cookie duration in seconds. The duration is refreshed on every +# manual request. cookie_duration = 10800 -# If CWSs write submissions to disk before storing them in -# the DB, and where to save them. %s = DATA_DIR. -submit_local_copy = true -submit_local_copy_path = "%s/submissions/" - -# The number of proxies that will be crossed before CWSs get -# the request. This is used to decide whether to assume that -# the real source IP address is the one listed in the request -# headers or not. For example, if you're using nginx as a load -# balancer, you will likely want to set this value to 1. +# The number of proxies that will be crossed before CWSs get the +# request. This is used to decide whether to assume that the real source +# IP address is the one listed in the request headers or not. For +# example, if you're using nginx as a load balancer, you will likely +# want to set this value to 1. num_proxies_used = 0 -# Maximum size of a submission in bytes. If you use a proxy -# and set these sizes to large values remember to change -# client_max_body_size in nginx.conf too. -max_submission_length = 100000 -max_input_length = 5000000 - -# Path to the documentation exposed by CWS. To show a documentation -# link add a folder for each language with index.html inside. For -# example for C++ add 'cpp/index.html', for Java 'java/index.html'. +# If CWSs write submissions to disk before storing them in the DB, and +# where to save them. %s = DATA_DIR. +submit_local_copy = true +submit_local_copy_path = "%s/submissions/" +# Same for user tests. +tests_local_copy = true +tests_local_copy_path = "%s/tests/" + +# Maximum size of a submission in bytes. If you use a proxy and set +# these sizes to large values remember to change client_max_body_size in +# nginx.conf too. +max_submission_length = 100_000 +# Maximum size of an input file for an user test. +max_input_length = 5_000_000 + +# Path to the documentation exposed by CWS. To show a documentation link +# add a folder for each language with index.html inside. For example for +# C++ add 'cpp/index.html', for Java 'java/index.html'. docs_path = "/usr/share/cms/docs" -# An authentication token that can be used by the administrator -# to impersonate an arbitrary user and bypass submit restrictions. -# contest_admin_token = "CHANGE-ME" +# An authentication token that can be used by the administrator to +# impersonate an arbitrary user and bypass submit restrictions. +#contest_admin_token = "CHANGE-ME" -################## -# AdminWebServer # -################## -# Listening HTTP address and port for the AWS. By default only -# listens on localhost, meaning you need a separate reverse proxy -# to access it from the web. Set to empty string to allow -# connecting from anywhere. -admin_listen_address = "127.0.0.1" -admin_listen_port = 8889 +[admin_web_server] +# Listening HTTP address and port for the AWS. By default only listens +# on localhost, meaning you need a separate reverse proxy to access it +# from the web. Set to empty string to allow connecting from anywhere. +listen_address = "127.0.0.1" +listen_port = 8889 # Login cookie duration for admins in seconds. # The duration is refreshed on every manual request. -admin_cookie_duration = 36000 +cookie_duration = 36000 # The number of proxies that will be crossed before AWS gets # the request. This is used to determine the request's real # source IP address. For example, if you're using nginx as # a proxy, you will likely want to set this value to 1. -admin_num_proxies_used = 0 +num_proxies_used = 0 -################ -# ProxyService # -################ -# List of URLs (with embedded username and password) of the -# RWSs where the scores are to be sent. Don't include the -# load balancing proxy (if any), just the backends. If any -# of them uses HTTPS specify a file with the certificates -# you trust. +[proxy_service] +# List of URLs (with embedded username and password) of the RWSs where +# the scores are to be sent. Don't include the load balancing proxy (if +# any), just the backends. rankings = ["http://usern4me:passw0rd@localhost:8890/"] +# If any ranking uses HTTPS, specify a file with the certificates you +# trust. (this string is passed as the "verify" option to requests.put, +# see the documentation of requests for more info.) #https_certfile = "..." -################### -# PrintingService # -################### +[printing] # Maximum size of a print job in bytes. -max_print_length = 10000000 +max_print_length = 10_000_000 -# Printer name (can be found out using 'lpstat -p'; -# if missing, printing is disabled) +# Printer name (can be found out using 'lpstat -p'; if missing, printing +# is disabled) #printer = "..." # Output paper size (probably A4 or Letter) paper_size = "A4" -# Maximum number of pages a user can print per print job -# (excluding the title page). Text files are cropped to this -# length. Too long pdf files are rejected. +# Maximum number of pages a user can print per print job (excluding the +# title page). Text files are cropped to this length. Too long pdf files +# are rejected. max_pages_per_job = 10 max_jobs_per_user = 10 pdf_printing_allowed = false -###################### -# PrometheusExporter # -###################### - -# Listening HTTP address and port for the exporter. If exposed -# this may leak private information, make sure to secure this endpoint. -prometheus_listen_address = "127.0.0.1" -prometheus_listen_port = 8811 -############### -# TelegramBot # -############### +[prometheus] +# Listening HTTP address and port for the exporter. If exposed this may +# leak private information, make sure to secure this endpoint. +listen_address = "127.0.0.1" +listen_port = 8811 -# Bot token and chat ID for the telegram bot. The Telegram bot will -# sync all questions with this chat, if present. -#telegram_bot_token = "..." -#telegram_bot_chat_id = "..." +# Bot token and chat ID for the telegram bot. The Telegram bot will sync +# all questions with this chat, if present. +#[telegram_bot] +#bot_token = "..." +#chat_id = "..." -######################### -# Service configuration # -######################### - -# TODO: delete this -other_services = {} - -[core_services] - -LogService = [["localhost", 29000]] -ResourceService = [["localhost", 28000]] -ScoringService = [["localhost", 28500]] -Checker = [["localhost", 22000]] -EvaluationService = [["localhost", 25000]] -Worker = [ - ["localhost", 26000], - ["localhost", 26001], - ["localhost", 26002], - ["localhost", 26003], - ["localhost", 26004], - ["localhost", 26005], - ["localhost", 26006], - ["localhost", 26007], - ["localhost", 26008], - ["localhost", 26009], - ["localhost", 26010], - ["localhost", 26011], - ["localhost", 26012], - ["localhost", 26013], - ["localhost", 26014], - ["localhost", 26015], -] -ContestWebServer = [["localhost", 21000]] -AdminWebServer = [["localhost", 21100]] -ProxyService = [["localhost", 28600]] -PrintingService = [["localhost", 25123]] -PrometheusExporter = [] -TelegramBot = [] diff --git a/config/cms_ranking.sample.toml b/config/cms_ranking.sample.toml index 1434ec3c7d..5d006e4890 100644 --- a/config/cms_ranking.sample.toml +++ b/config/cms_ranking.sample.toml @@ -7,6 +7,21 @@ bind_address = "127.0.0.1" # Listening port for RankingWebServer. http_port = 8890 +# Socket parameters for HTTPS. For certfile and keyfile, see +# . +#https_port = ... +#https_certfile = "..." +#https_keyfile = "..." + # Login information for adding and editing data. username = "usern4me" password = "passw0rd" +realm_name = "Scoreboard" + +# How many events to keep buffered for the server-sent events stream. +buffer_size = 100 + +# Log files. +#log_dir = "INSTALL_DIR/log/ranking" +# Data directory (the scoreboard data is stored here). +#lib_dir = "INSTALL_DIR/lib/ranking" From 839202dba83e62c2caba41e4ab155b651f65bfa0 Mon Sep 17 00:00:00 2001 From: prandla Date: Mon, 4 Aug 2025 19:58:36 +0300 Subject: [PATCH 2/2] Delete sandbox.use_cgroups option --- cms/conf.py | 1 - cms/grading/Sandbox.py | 20 ++++---------------- config/cms.sample.toml | 2 -- 3 files changed, 4 insertions(+), 19 deletions(-) diff --git a/cms/conf.py b/cms/conf.py index ec8baff4df..6e7495443f 100644 --- a/cms/conf.py +++ b/cms/conf.py @@ -91,7 +91,6 @@ class WorkerConfig: @dataclass() class SandboxConfig: sandbox_implementation: str = "isolate" - use_cgroups: bool = True # Max size of each writable file during an evaluation step, in KiB. max_file_size: int = 1024 * 1024 # 1 GiB # Max processes, CPU time (s), memory (KiB) for compilation runs. diff --git a/cms/grading/Sandbox.py b/cms/grading/Sandbox.py index 06528ffa25..da57b60eae 100644 --- a/cms/grading/Sandbox.py +++ b/cms/grading/Sandbox.py @@ -216,7 +216,6 @@ def __init__( # TODO: move all other common properties here. self.box_id: int = 0 self.fsize: int | None = None - self.cgroup: bool = False self.dirs: list[tuple[str, str, str | None]] = [] self.preserve_env: bool = False self.inherit_env: list[str] = [] @@ -947,7 +946,6 @@ def __init__(self, file_cacher, name=None, temp_dir=None): # Default parameters for isolate self.box_id = box_id # -b - self.cgroup = config.sandbox.use_cgroups # --cg self.chdir = self._home_dest # -c self.dirs = [] # -d self.preserve_env = False # -e @@ -1127,11 +1125,9 @@ def build_box_options(self) -> list[str]: return: the arguments list as strings. """ - res = list() + res = ["--cg"] if self.box_id is not None: res += ["--box-id=%d" % self.box_id] - if self.cgroup: - res += ["--cg"] if self.chdir is not None: res += ["--chdir=%s" % self.chdir] for src, dest, options in self.dirs: @@ -1155,10 +1151,7 @@ def build_box_options(self) -> list[str]: res += ["--stack=%d" % (self.stack_space // 1024)] if self.address_space is not None: # Isolate wants memory size as KiB. - if self.cgroup: - res += ["--cg-mem=%d" % (self.address_space // 1024)] - else: - res += ["--mem=%d" % (self.address_space // 1024)] + res += ["--cg-mem=%d" % (self.address_space // 1024)] if self.stdout_file is not None: res += ["--stdout=%s" % self.inner_absolute_path(self.stdout_file)] if self.max_processes is not None: @@ -1468,10 +1461,7 @@ def translate_box_exitcode(self, exitcode: int) -> bool: def initialize_isolate(self): """Initialize isolate's box.""" - init_cmd = ( - [self.box_exec] - + (["--cg"] if self.cgroup else []) - + ["--box-id=%d" % self.box_id, "--init"]) + init_cmd = [self.box_exec, "--box-id=%d" % self.box_id, "--cg", "--init"] try: subprocess.check_call(init_cmd) except subprocess.CalledProcessError as e: @@ -1487,9 +1477,7 @@ def cleanup(self, delete=False): # will be able to delete everything. If not, we leave the files as they # are to avoid masking possible problems the admin wanted to debug. - exe = [self.box_exec] \ - + (["--cg"] if self.cgroup else []) \ - + ["--box-id=%d" % self.box_id] + exe = [self.box_exec, "--box-id=%d" % self.box_id, "--cg"] if delete: # Ignore exit status as some files may be owned by our user diff --git a/config/cms.sample.toml b/config/cms.sample.toml index 505246c4a4..f609ffe089 100644 --- a/config/cms.sample.toml +++ b/config/cms.sample.toml @@ -87,8 +87,6 @@ keep_sandbox = false # Which sandbox implementation to use. Currently only isolate is # supported. sandbox_implementation = "isolate" -# Whether to use cgroups in isolate. -use_cgroups = true # Do not allow contestants' solutions to write files bigger than this # size (expressed in KB; defaults to 1 GB). max_file_size = 1_048_576