diff --git a/mig/install/MiGserver-template.conf b/mig/install/MiGserver-template.conf
index 8f934818a..5289a06f6 100644
--- a/mig/install/MiGserver-template.conf
+++ b/mig/install/MiGserver-template.conf
@@ -785,3 +785,7 @@ logo_right = /images/skin/__SKIN__/logo-right.png
# Optional data safety notice and popup on Files page
datasafety_link = __DATASAFETY_LINK__
datasafety_text = __DATASAFETY_TEXT__
+
+[TEMPLATES]
+base_packages = __TEMPLATES_BASE_PACKAGES__
+cache_dir = __TEMPLATES_CACHE_DIR__
diff --git a/mig/install/generateconfs.py b/mig/install/generateconfs.py
index bb7dde1b2..66d949b31 100755
--- a/mig/install/generateconfs.py
+++ b/mig/install/generateconfs.py
@@ -218,6 +218,8 @@ def main(argv, _generate_confs=generate_confs, _print=print):
'ca_smtp',
'datasafety_link',
'datasafety_text',
+ 'templates_cache_dir',
+ 'templates_base_package',
]
int_names = [
'cert_valid_days',
diff --git a/mig/lib/templates/__init__.py b/mig/lib/templates/__init__.py
new file mode 100644
index 000000000..69e871b73
--- /dev/null
+++ b/mig/lib/templates/__init__.py
@@ -0,0 +1,280 @@
+# -*- coding: utf-8 -*-
+#
+# --- BEGIN_HEADER ---
+#
+# templates/__init__ - main logic for template support
+# Copyright (C) 2003-2026 The MiG Project by the Science HPC Center at UCPH
+#
+# This file is part of MiG.
+#
+# MiG is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# MiG is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301,
+# USA.
+#
+# -- END_HEADER ---
+#
+
+"""
+Template support library code.
+"""
+
+import importlib
+import os
+from operator import itemgetter
+
+from jinja2 import (
+ Environment,
+ FileSystemBytecodeCache,
+ PackageLoader,
+ Template,
+)
+from jinja2 import meta as jinja2_meta
+from jinja2 import (
+ select_autoescape,
+)
+
+
+def _expand_base_packages(base_packages):
+ template_packages = []
+ for package_name in base_packages:
+ try:
+ package = importlib.import_module(package_name)
+ except (ImportError, ModuleNotFoundError):
+ raise UnknownTemplateError(package_name)
+ template_packages.extend(package.TEMPLATE_PACKAGES)
+ return template_packages
+
+
+def _strip_template_ext(template_name_with_ext):
+ return os.path.splitext(os.path.splitext(template_name_with_ext)[0])[0]
+
+
+class _NoopContext:
+ """
+ Adapter class to allow templates to be directly rendered.
+
+ Note that this is in contrast to further work making use of the
+ same provisions that allows the selection of translations.
+ """
+
+ def __init__(self, *args):
+ self._tmpl = None
+ self._tmpl_args = None
+
+ def extend(self, template, template_args):
+ self._tmpl = template
+ self._tmpl_args = template_args
+ return self
+
+ def render(self):
+ return self._tmpl.render(**self._tmpl_args)
+
+
+class TemplateStore:
+ """
+ An abstraction for interacting with an enable series of template packages.
+ """
+
+ def __init__(self, packages, cache_dir=None, extra_globals=None):
+ assert cache_dir is not None
+
+ self._packages = packages
+ self._cache_dir = cache_dir
+ self._template_globals = extra_globals
+ self._template_env_by_package = {}
+
+ @property
+ def cache_dir(self):
+ return self._cache_dir
+
+ @property
+ def context(self):
+ return self._template_globals
+
+ def _env_for_package(self, package_name):
+ """
+ Direct access to a jinja2 Environment for a package exposing templates.
+ """
+
+ if package_name not in self._packages:
+ raise UnknownTemplateError(package_name)
+
+ if package_name in self._template_env_by_package:
+ return self._template_env_by_package[package_name]
+
+ package_cache_key = "%s-%%s.jinja_cache" % (package_name,)
+ template_env = Environment(
+ loader=PackageLoader(package_name),
+ bytecode_cache=FileSystemBytecodeCache(self._cache_dir, package_cache_key),
+ autoescape=select_autoescape(),
+ )
+ self._template_env_by_package[package_name] = template_env
+ return template_env
+
+ def grab_template(
+ self,
+ template_name,
+ template_group,
+ output_format,
+ template_globals=None,
+ **kwargs
+ ):
+ """
+ Directly access an enabled template.
+ """
+
+ template_env = self._env_for_package(template_group)
+ template_fqname = "%s.%s.jinja" % (template_name, output_format)
+ try:
+ return template_env.get_template(
+ template_fqname, globals=template_globals
+ )
+ except FileNotFoundError:
+ raise UnknownTemplateError(template_group, template_name)
+
+ def list_templates(self):
+ """
+ Return a list of templates for all enabled packages.
+ """
+
+ template_and_group_pairs = []
+ for template_group in self._packages:
+ template_env = self._env_for_package(template_group)
+ pairs = (
+ (_strip_template_ext(template), template_group)
+ for template in template_env.list_templates()
+ )
+ template_and_group_pairs.extend(pairs)
+ template_and_group_pairs.sort(key=itemgetter(1, 0))
+ return template_and_group_pairs
+
+ def list_templates_groups(self):
+ """
+ Return the set of enabled packages that expose templates.
+ """
+
+ nonunique_template_groups = (
+ template_group for _, template_group in self.list_templates()
+ )
+ return set(nonunique_template_groups)
+
+ def prime_templates(self):
+ """
+ Precompile all templates across the enabled packages.
+ """
+
+ os.makedirs(self.cache_dir, exist_ok=True)
+
+ for template_group in self.list_templates_groups():
+ template_group_cache_dir = os.path.join(
+ self.cache_dir, template_group
+ )
+ os.makedirs(template_group_cache_dir, exist_ok=True)
+
+ primed_count = 0
+
+ for template_name, template_group in self.list_templates():
+ primed_count += 1
+ self.grab_template(template_name, template_group, "html")
+
+ return primed_count
+
+ def extract_variables(
+ self,
+ template_or_name,
+ template_group,
+ output_format=None,
+ template_globals=None,
+ ):
+ """
+ Return the expected variables for a given template.
+ """
+
+ template_env = self._env_for_package(template_group)
+ if isinstance(template_or_name, Template):
+ raise NotImplementedError()
+ else:
+ template = self.grab_template(
+ template_or_name,
+ template_group,
+ output_format,
+ globals=template_globals,
+ )
+ with open(template.filename) as f:
+ template_source = f.read()
+ ast = template_env.parse(template_source)
+ return jinja2_meta.find_undeclared_variables(ast)
+
+ @staticmethod
+ def from_configuration(configuration):
+ """
+ Create a TemplateStore instance for a specified configuration.
+ """
+
+ template_division = configuration.division(section_name="TEMPLATES")
+
+ return TemplateStore.from_names(
+ template_division.base_packages,
+ cache_dir=template_division.cache_dir,
+ context=_NoopContext(configuration),
+ )
+
+ @staticmethod
+ def from_names(template_packages, *, cache_dir=None, context=None):
+ """
+ Create a template store from a list of package names.
+ """
+
+ assert cache_dir is not None
+ if context is None:
+ context = _NoopContext()
+
+ packages = _expand_base_packages(template_packages)
+
+ return TemplateStore(
+ packages,
+ cache_dir=cache_dir,
+ extra_globals=context,
+ )
+
+
+def init_global_templates(runtime_configuration):
+ """
+ Make a TemplateStore available within the active request context.
+ """
+
+ store = runtime_configuration.context_get("templates")
+ if store:
+ return store
+ store = TemplateStore.from_configuration(runtime_configuration)
+ runtime_configuration.context_set("templates", store)
+ return store
+
+
+def render_html_template(
+ runtime_configuration, template_name, template_group, template_args
+):
+ """
+ Render a template available within the active request context.
+ """
+
+ store = init_global_templates(runtime_configuration)
+ template = store.grab_template(template_name, template_group, "html")
+ bound = store.context.extend(template, template_args)
+ return bound.render()
+
+
+class UnknownTemplateError(KeyError):
+ def __init__(self, template_group, template_name="*"):
+ super().__init__("%s.%s" % (template_group, template_name))
diff --git a/mig/lib/templates/__main__.py b/mig/lib/templates/__main__.py
new file mode 100644
index 000000000..5375bda95
--- /dev/null
+++ b/mig/lib/templates/__main__.py
@@ -0,0 +1,84 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+#
+# --- BEGIN_HEADER ---
+#
+# templates/__main__ - templates CLI
+# Copyright (C) 2003-2026 The MiG Project by the Science HPC Center at UCPH
+#
+# This file is part of MiG.
+#
+# MiG is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# MiG is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301,
+# USA.
+#
+# -- END_HEADER ---
+#
+
+"""
+Template support CLI code.
+"""
+
+import sys
+
+from mig.lib.templates import TemplateStore
+from mig.shared.conf import get_configuration_object
+
+
+def warn(message):
+ print(message, file=sys.stderr, flush=True)
+
+
+def main(args, _print=print):
+ configuration = get_configuration_object(
+ config_file=args.config_file, skip_log=True, disable_auth_log=True
+ )
+ template_store = TemplateStore.from_configuration(configuration)
+
+ command = args.command
+ if command == "cache":
+ templates_division = configuration.division(section_name="TEMPLATES")
+ _print(templates_division.cache_dir)
+ elif command == "show":
+ _print(template_store.list_templates())
+ elif command == "prime":
+ primed_count = template_store.prime_templates()
+ if primed_count == 0:
+ _print("No templates were specified.")
+ elif command == "vars":
+ for template_name, template_group in template_store.list_templates():
+ _print("<%s.%s>" % (template_group, template_name))
+ for var in template_store.extract_variables(
+ template_name, template_group, "html"
+ ):
+ _print(" {{%s}}" % (var,))
+ _print("%s.%s>" % (template_group, template_name))
+ else:
+ raise RuntimeError("unknown command: %s" % (command,))
+
+
+if __name__ == "__main__":
+ import argparse
+
+ parser = argparse.ArgumentParser()
+ parser.add_argument("-c", dest="config_file", required=True)
+ parser.add_argument("command")
+ args = parser.parse_args()
+
+ try:
+ main(args)
+ sys.exit(0)
+ except Exception as exc:
+ warn(str(exc))
+ sys.exit(1)
diff --git a/mig/shared/configuration.py b/mig/shared/configuration.py
index 4dca877ee..37529ee00 100644
--- a/mig/shared/configuration.py
+++ b/mig/shared/configuration.py
@@ -37,6 +37,7 @@
import base64
import copy
import datetime
+from enum import Enum
import functools
import inspect
import os
@@ -45,6 +46,7 @@
import socket
import sys
import time
+from types import SimpleNamespace
# Init future py2/3 compatibility helpers
@@ -61,7 +63,8 @@
# NOTE: protect migrid import from autopep8 reordering
try:
from mig.shared.base import force_native_str
- from mig.shared.defaults import CSRF_MINIMAL, CSRF_WARN, CSRF_MEDIUM, \
+ from mig.shared.defaults import MIG_BASE, \
+ CSRF_MINIMAL, CSRF_WARN, CSRF_MEDIUM, \
CSRF_FULL, POLICY_NONE, POLICY_WEAK, POLICY_MEDIUM, POLICY_HIGH, \
POLICY_MODERN, POLICY_CUSTOM, freeze_flavors, cert_field_order, \
default_css_filename, keyword_any, keyword_auto, keyword_all, \
@@ -84,6 +87,7 @@
'skip_log',
'verbose',
'logger',
+ '_divisions',
])
@@ -192,14 +196,14 @@ def expand_external_sources(logger, val):
return expanded
-def fix_missing(config_file, verbose=True):
- """Add missing configuration options - used by checkconf script"""
+class _SubstitutePlaceholders(Enum):
+ ADMIN_EMAIL = object()
+ FQDN = object()
+ MIGSERVER_HTTPURL = object()
+ MIGSERVER_ID = object()
- config = ConfigParser()
- config.read([config_file])
- fqdn = socket.getfqdn()
- user = os.environ['USER']
+def _generate_fix_missing_definitions():
global_section = {
'enable_server_dist': False,
'auto_add_cert_user': False,
@@ -210,9 +214,9 @@ def fix_missing(config_file, verbose=True):
'auto_add_user_with_peer': 'distinguished_name:.*',
'auto_add_filter_method': '',
'auto_add_filter_fields': '',
- 'server_fqdn': fqdn,
+ 'server_fqdn': _SubstitutePlaceholders.FQDN,
'support_email': '',
- 'admin_email': '%s@%s' % (user, fqdn),
+ 'admin_email': _SubstitutePlaceholders.ADMIN_EMAIL,
'admin_list': '',
'ca_fqdn': '',
'ca_smtp': '',
@@ -286,36 +290,36 @@ def fix_missing(config_file, verbose=True):
'trac_admin_path': '/usr/bin/trac-admin',
'trac_ini_path': '~/mig/server/trac.ini',
'trac_id_field': 'email',
- 'migserver_http_url': 'http://%%(server_fqdn)s',
+ 'migserver_http_url': _SubstitutePlaceholders.MIGSERVER_HTTPURL,
'migserver_https_url': '',
'myfiles_py_location': '',
- 'mig_server_id': '%s.0' % fqdn,
+ 'mig_server_id': _SubstitutePlaceholders.MIGSERVER_ID,
'empty_job_name': 'no_suitable_job-',
- 'smtp_server': fqdn,
+ 'smtp_server': _SubstitutePlaceholders.FQDN,
'smtp_sender': '',
'smtp_send_as_user': False,
'smtp_reply_to': '',
- 'user_sftp_address': fqdn,
+ 'user_sftp_address': _SubstitutePlaceholders.FQDN,
'user_sftp_port': 2222,
'user_sftp_key': '~/certs/combined.pem',
'user_sftp_key_pub': '~/certs/server.pub',
'user_sftp_key_md5': '',
'user_sftp_key_sha256': '',
- 'user_sftp_key_from_dns': '',
+ 'user_sftp_key_from_dns': False,
'user_sftp_auth': ['publickey', 'password'],
'user_sftp_alias': '',
'user_sftp_log': 'sftp.log',
- 'user_sftp_subsys_address': fqdn,
+ 'user_sftp_subsys_address': _SubstitutePlaceholders.FQDN,
'user_sftp_subsys_port': 22,
'user_sftp_subsys_log': 'sftp-subsys.log',
- 'user_davs_address': fqdn,
+ 'user_davs_address': _SubstitutePlaceholders.FQDN,
'user_davs_port': 4443,
'user_davs_key': '~/certs/combined.pem',
'user_davs_key_sha256': '',
'user_davs_auth': ['password'],
'user_davs_alias': '',
'user_davs_log': 'davs.log',
- 'user_ftps_address': fqdn,
+ 'user_ftps_address': _SubstitutePlaceholders.FQDN,
'user_ftps_ctrl_port': 8021,
'user_ftps_pasv_ports': list(range(8100, 8400)),
'user_ftps_key': '~/certs/combined.pem',
@@ -339,7 +343,7 @@ def fix_missing(config_file, verbose=True):
'user_imnotify_log': 'imnotify.log',
'user_chkuserroot_log': 'chkchroot.log',
'user_chksidroot_log': 'chkchroot.log',
- 'user_openid_address': fqdn,
+ 'user_openid_address': _SubstitutePlaceholders.FQDN,
'user_openid_port': 8443,
'user_openid_key': '~/certs/combined.pem',
'user_openid_auth': ['password'],
@@ -410,8 +414,10 @@ def fix_missing(config_file, verbose=True):
'user_limit': 1024**4,
'vgrid_limit': 1024**4}
accounting_section = {'update_interval': 3600}
+ templates_section = {'base_packages': keyword_auto,
+ 'cache_dir': keyword_auto}
- defaults = {
+ return {
'GLOBAL': global_section,
'SCHEDULER': scheduler_section,
'MONITOR': monitor_section,
@@ -420,19 +426,57 @@ def fix_missing(config_file, verbose=True):
'WORKFLOWS': workflows_section,
'QUOTA': quota_section,
'ACCOUNTING': accounting_section,
+ 'TEMPLATES': templates_section,
}
- for section in defaults:
+
+
+def _split_non_empty_list(value):
+ maybe_list = value.split(' ')
+ if len(maybe_list) == 1 and maybe_list[0] == '':
+ return []
+ return maybe_list
+
+
+_FIX_MISSING_DEFINITIONS = _generate_fix_missing_definitions()
+
+
+def fix_missing(config_file, verbose=False, fqdn=None, user=None, print=print):
+ """Add missing configuration options used by checkconf script"""
+
+ if fqdn is None:
+ fqdn = socket.getfqdn()
+ if user is None:
+ user = os.getenv('USER', 'mig')
+
+ _marker_substitutions = {
+ _SubstitutePlaceholders.ADMIN_EMAIL: "%s@%s" % (user, fqdn),
+ _SubstitutePlaceholders.FQDN: fqdn,
+ _SubstitutePlaceholders.MIGSERVER_HTTPURL: "http://%s" % (fqdn,),
+ _SubstitutePlaceholders.MIGSERVER_ID: '%s.0' % (fqdn,),
+ }
+ assert set(_marker_substitutions.keys()) == set(_SubstitutePlaceholders)
+
+ config = ConfigParser()
+ config.read([config_file])
+
+ modified = False
+ for (section, settings) in _FIX_MISSING_DEFINITIONS.items():
if not section in config.sections():
config.add_section(section)
- modified = False
- for (section, settings) in defaults.items():
for (option, value) in settings.items():
if not config.has_option(section, option):
+ if isinstance(value, _SubstitutePlaceholders):
+ value = _marker_substitutions[value]
+
+ # only string values can be set - coerce the value as is
+ # required so set the same thing as output when verbose
+ value_to_set = str(value)
+
if verbose:
print('setting %s->%s to %s' % (section, option,
- value))
- config.set(section, option, value)
+ value_to_set))
+ config.set(section, option, value_to_set)
modified = True
if modified:
backup_path = '%s.%d' % (config_file, time.time())
@@ -778,6 +822,9 @@ def __init__(self, config_file, verbose=False, skip_log=False,
self.logger_obj = None
self.logger = None
+ # structured
+ self._divisions = {}
+
configuration_properties = copy.deepcopy(_CONFIGURATION_PROPERTIES)
for k, v in configuration_properties.items():
@@ -804,11 +851,9 @@ def reload_config(self, verbose, skip_log=False, disable_auth_log=False,
_config_file = _config_file or self.config_file
assert _config_file is not None
- try:
- if self.logger:
- self.logger.info('reloading configuration and reopening log')
- except:
- pass
+ _logger = getattr(self, 'logger', None)
+ if _logger:
+ _logger.info('reloading configuration and reopening log')
try:
config_file_is_path = os.path.isfile(_config_file)
@@ -859,10 +904,12 @@ def reload_config(self, verbose, skip_log=False, disable_auth_log=False,
# reopen or initialize logger
- if self.logger_obj:
- self.logger_obj.reopen()
+ _logger_obj = getattr(self, 'logger_obj', None)
+ if _logger_obj:
+ _logger_obj.reopen()
else:
- self.logger_obj = Logger(self.loglevel, logfile=self.log_path)
+ _logger_obj = Logger(self.loglevel, logfile=self.log_path)
+ self.logger_obj = _logger_obj
logger = self.logger_obj.logger
self.logger = logger
@@ -1024,6 +1071,9 @@ def reload_config(self, verbose, skip_log=False, disable_auth_log=False,
pass
raise Exception('Failed to parse configuration: %s' % err)
+ # handle structured sections
+ self.apply_loaded_config_to_templates(config)
+
# Remaining options in order of importance - i.e. options needed for
# later parsing must be parsed and set first.
@@ -1670,7 +1720,7 @@ def reload_config(self, verbose, skip_log=False, disable_auth_log=False,
self.user_openid_providers = providers
if config.has_option('GLOBAL', 'user_mig_oidc_title'):
self.user_mig_oidc_title = config.get('GLOBAL',
- 'user_migc_oid_title')
+ 'user_mig_oid_title')
elif self.user_mig_oid_title:
# Fall back to oid title
self.user_mig_oidc_title = self.user_mig_oid_title
@@ -2755,6 +2805,39 @@ def reload_config(self, verbose, skip_log=False, disable_auth_log=False,
% keyword_all)
self.site_twofactor_mandatory_protos = [keyword_all]
+ def apply_loaded_config_to_templates(self, loaded_config):
+ template_division = self.division(section_name='TEMPLATES')
+ template_division_dict = template_division.__dict__
+
+ try:
+ candidates = dict(loaded_config['TEMPLATES'])
+ except KeyError:
+ candidates = {}
+
+ if 'base_packages' in candidates:
+ base_packages = _split_non_empty_list(candidates['base_packages'])
+ else:
+ base_packages = []
+ template_division_dict['base_packages'] = base_packages
+
+ if 'cache_dir' in candidates:
+ cache_dir = os.path.abspath(candidates['cache_dir'])
+ else:
+ mig_system_run = os.path.normpath(self.mig_system_run)
+ cache_dir = os.path.join(self.mig_system_run, 'templates')
+ template_division_dict['cache_dir'] = cache_dir
+
+ def division(self, section_name):
+ if section_name not in _FIX_MISSING_DEFINITIONS:
+ raise NotImplementedError()
+
+ try:
+ return self._divisions[section_name]
+ except KeyError:
+ new_division = SimpleNamespace(**_FIX_MISSING_DEFINITIONS[section_name])
+ self._divisions[section_name] = new_division
+ return new_division
+
def parse_peers(self, peerfile):
# read peer information from peerfile
diff --git a/mig/shared/init.py b/mig/shared/init.py
index 354df5825..d16fc776d 100644
--- a/mig/shared/init.py
+++ b/mig/shared/init.py
@@ -80,6 +80,14 @@ def find_entry(output_objects, kind):
return None
+def find_entry_index(output_objects, kind):
+ """Find entry in output_objects"""
+ for index, entry in enumerate(output_objects):
+ if kind == entry['object_type']:
+ return index
+ return -1
+
+
def start_download(configuration, path, output):
"""Helper to set the headers required to force a file download instead of
plain output delivery. Automatically detects mimetype of path and sets
@@ -117,12 +125,14 @@ def start_error(configuration, output_format, status_pair):
def initialize_main_variables(client_id, op_title=True, op_header=True,
- op_menu=True):
- """Script initialization is identical for most scripts in
+ op_menu=True, configuration=None):
+ """Script initialization is identical for most scripts in
shared/functionality. This function should be called in most cases.
"""
- configuration = get_configuration_object()
+ if configuration is None:
+ configuration = get_configuration_object()
+
logger = configuration.logger
output_objects = []
start_entry = make_start_entry()
diff --git a/mig/shared/install.py b/mig/shared/install.py
index 63c814314..ebecde73c 100644
--- a/mig/shared/install.py
+++ b/mig/shared/install.py
@@ -544,6 +544,8 @@ def generate_confs(
datasafety_link='',
datasafety_text='',
wwwserve_max_bytes=-1,
+ templates_cache_dir=keyword_auto,
+ templates_base_packages='',
_getpwnam=pwd.getpwnam,
_prepare=None,
_writefiles=None,
@@ -872,6 +874,8 @@ def _generate_confs_prepare(
datasafety_link,
datasafety_text,
wwwserve_max_bytes,
+ templates_cache_dir,
+ templates_base_packages,
):
"""Prepate conf generator run"""
user_dict = {}
@@ -1132,6 +1136,8 @@ def _generate_confs_prepare(
user_dict['__DATASAFETY_LINK__'] = datasafety_link
user_dict['__DATASAFETY_TEXT__'] = datasafety_text
user_dict['__WWWSERVE_MAX_BYTES__'] = "%d" % (wwwserve_max_bytes)
+ user_dict['__TEMPLATES_CACHE_DIR__'] = "%s" % (templates_cache_dir)
+ user_dict['__TEMPLATES_BASE_PACKAGES__'] = "%s" % (templates_base_packages)
user_dict['__MIG_USER__'] = "%s" % (options['user_uname'])
user_dict['__MIG_GROUP__'] = "%s" % (options['user_group'])
diff --git a/mig/shared/objecttypes.py b/mig/shared/objecttypes.py
index 521737c2c..bbf4bb45f 100644
--- a/mig/shared/objecttypes.py
+++ b/mig/shared/objecttypes.py
@@ -28,9 +28,13 @@
""" Defines valid objecttypes and provides a method to verify if an object is correct """
+from mig.lib.templates import init_global_templates
+
+
start = {'object_type': 'start', 'required': [], 'optional': ['headers'
]}
end = {'object_type': 'end', 'required': [], 'optional': []}
+template = {'object_type': 'template'}
timing_info = {'object_type': 'timing_info', 'required': [],
'optional': []}
title = {'object_type': 'title', 'required': ['text'],
@@ -320,6 +324,7 @@
valid_types_list = [
start,
end,
+ template,
timing_info,
title,
text,
@@ -417,6 +422,12 @@
table_pager,
]
+valid_template_type_attrs_and_types = {
+ 'template_name': str,
+ 'template_group': str,
+ 'template_args': dict,
+}
+
# valid_types_dict = {"title":title, "link":link, "header":header}
# autogenerate dict based on list. Dictionary access is prefered to allow
@@ -457,8 +468,8 @@ def get_object_type_info(object_type_list):
return out
-def validate(input_object):
- """ validate input_object """
+def validate(input_object, configuration=None):
+ """ validate presented objects against their definitions """
if not type(input_object) == type([]):
return (False, 'validate object must be a list' % ())
@@ -478,6 +489,31 @@ def validate(input_object):
this_object_type = obj['object_type']
valid_object_type = valid_types_dict[this_object_type]
+
+ if this_object_type == 'template':
+ # in contrast to other types here there are two components to
+ # validating a template object: the output object itself but
+ # in addition the arguments that will be provided to templates
+
+ # first, directly validate the template output object
+ missing_attrs = [attr for attr, attr_type
+ in valid_template_type_attrs_and_types.items()
+ if not isinstance(obj.get(attr, None), attr_type)]
+ if missing_attrs:
+ return (False,
+ 'template object is not a valid: %r' % obj)
+
+ # second, repurpose required keys stuff below given that
+ # templates know what they need in terms of data thus are
+ # self-documenting - use this fact to perform validation of
+ # what we will attempt to render
+ store = init_global_templates(configuration)
+ required = store.extract_variables(obj['template_name'], obj['template_group'], 'html')
+ valid_object_type = {
+ 'required': required
+ }
+ obj = obj['template_args']
+
if 'required' in valid_object_type:
for req in valid_object_type['required']:
if req not in obj:
diff --git a/mig/shared/output.py b/mig/shared/output.py
index 5f0f52fd8..cba0d832d 100644
--- a/mig/shared/output.py
+++ b/mig/shared/output.py
@@ -38,11 +38,13 @@
exit(1)
from past.builtins import basestring
+import inspect
import os
import sys
import time
import traceback
+from mig.lib.templates import render_html_template
from mig.shared import returnvalues
from mig.shared.bailout import bailout_title, crash_helper, \
filter_output_objects
@@ -52,7 +54,7 @@
from mig.shared.defaults import file_dest_sep, keyword_any, keyword_updating
from mig.shared.htmlgen import get_xgi_html_header, get_xgi_html_footer, \
vgrid_items, html_post_helper, tablesorter_pager
-from mig.shared.init import find_entry
+from mig.shared.init import find_entry, find_entry_index
from mig.shared.objecttypes import validate
from mig.shared.prettyprinttable import pprint_table
from mig.shared.pwcrypto import sorted_hash_algos
@@ -64,6 +66,17 @@
'yaml', 'xmlrpc', 'resource', 'json', 'file']
+def kwargs_for_functionality(main, configuration=None, environ=None):
+ parameters = inspect.signature(main).parameters
+
+ kwargs = dict()
+ if 'configuration' in parameters:
+ kwargs['configuration'] = configuration
+ if 'environ' in parameters:
+ kwargs['environ'] = environ
+ return kwargs
+
+
def reject_main(client_id, user_arguments_dict):
"""A simple main-function to use if functionality backend is disabled"""
output_objs = [bailout_title(None, 'Access Error'),
@@ -735,9 +748,28 @@ def html_format(configuration, ret_val, ret_msg, out_obj):
""" % (ret_val, ret_msg)
+
+ # The title entry controls what should be included in the start of a page.
+ # Meanwhile, the logic has writing a page _footer_ being conditional on
+ # status_line - which is always set (something that appears very likely a
+ # regression that crept in over time). Thus, in order to omit the usual
+ # page wrapping as is needed when returning a page fragment rendered from
+ # a template, when we omit the title also clear the status line which makes
+ # the preexisting conditional omit the footer and "hey presto" - raw html!
+ should_send_document = bool(find_entry(out_obj, 'title'))
+ if not should_send_document:
+ status_line = None
+
for i in out_obj:
if i['object_type'] == 'start':
pass
+ elif i['object_type'] == 'template':
+ lines.append(render_html_template(
+ configuration,
+ i['template_name'],
+ i['template_group'],
+ i['template_args'],
+ ))
elif i['object_type'] == 'error_text':
msg = "%(text)s" % i
if i.get('exc', False):
@@ -2678,9 +2710,17 @@ def xmlrpc_format(configuration, ret_val, ret_msg, out_obj):
def json_format(configuration, ret_val, ret_msg, out_obj):
"""Generate output in json format"""
+ start_entry_index = find_entry_index(out_obj, 'start')
+ if start_entry_index > -1:
+ # do not send the start entry in the output JSON
+ objects_to_dump = list(out_obj)
+ objects_to_dump.pop(start_entry_index)
+ else:
+ objects_to_dump = out_obj
+
# python >=2.6 includes native json module with loads/dumps methods
import json
- return json.dumps(out_obj)
+ return json.dumps(objects_to_dump)
def resource_format(configuration, ret_val, ret_msg, out_obj):
@@ -2785,7 +2825,7 @@ def format_output(
logger = configuration.logger
# logger.debug("format output to %s" % outputformat)
valid_formats = get_valid_outputformats()
- (val_ret, val_msg) = validate(out_obj)
+ (val_ret, val_msg) = validate(out_obj, configuration)
if not val_ret:
logger.error("%s formatting failed: %s (%s)" %
(outputformat, val_msg, val_ret))
diff --git a/mig/wsgi-bin/migwsgi.py b/mig/wsgi-bin/migwsgi.py
index cdbf26fc1..79e3c26fd 100755
--- a/mig/wsgi-bin/migwsgi.py
+++ b/mig/wsgi-bin/migwsgi.py
@@ -44,7 +44,8 @@
from mig.shared.defaults import download_block_size, default_fs_coding
from mig.shared.conf import get_configuration_object
from mig.shared.objecttypes import get_object_type_info
-from mig.shared.output import validate, format_output, dummy_main, reject_main
+from mig.shared.output import validate, format_output, \
+ kwargs_for_functionality, dummy_main, reject_main
from mig.shared.safeinput import valid_backend_name, html_escape, InputException
from mig.shared.scriptinput import fieldstorage_to_dict, FixedFieldStorage
@@ -124,8 +125,12 @@ def stub(configuration, client_id, import_path, backend, user_arguments_dict,
return (output_objects, returnvalues.INVALID_ARGUMENT)
try:
+ main_kwargs = kwargs_for_functionality(main,
+ configuration=configuration,
+ environ=environ)
(output_objects, (ret_code, ret_msg)) = main(client_id,
- user_arguments_dict)
+ user_arguments_dict,
+ **main_kwargs)
except Exception as err:
import traceback
_logger.error("%s script crashed:\n%s" % (_addr,
@@ -133,7 +138,7 @@ def stub(configuration, client_id, import_path, backend, user_arguments_dict,
crash_helper(configuration, backend, output_objects)
return (output_objects, returnvalues.ERROR)
- (val_ret, val_msg) = validate(output_objects)
+ (val_ret, val_msg) = validate(output_objects, configuration=configuration)
if not val_ret:
(ret_code, ret_msg) = returnvalues.OUTPUT_VALIDATION_ERROR
bailout_helper(configuration, backend, output_objects,
diff --git a/requirements.txt b/requirements.txt
index 32cb77203..dfc07595f 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -13,6 +13,10 @@ pyyaml
dnspython>=2.6.1
email-validator
+jinja2<3;python_version < "3"
+jinja2==3.0.*;python_version >= "3" and python_version < "3.7"
+jinja2;python_version >= "3.7"
+
# NOTE: additional optional dependencies depending on site conf are listed
# in recommended.txt and can be installed in the same manner by pointing
# pip there.
diff --git a/tests/data/MiGserver--empty_templates.conf b/tests/data/MiGserver--empty_templates.conf
new file mode 100644
index 000000000..4b2decb2b
--- /dev/null
+++ b/tests/data/MiGserver--empty_templates.conf
@@ -0,0 +1,789 @@
+# MiG server configuration file
+[GLOBAL]
+# Run server in test mode?
+# Server distribution is disabled per default.
+# Set to True to let a set og MiG servers migrate jobs (EXPERIMENTAL!).
+#enable_server_dist = False
+
+# Allow users and resources joining without admin action?
+# Security is then left to vgrids admission as entities will only have access
+# to the default_vgrid when created.
+# Auto create MiG users with valid certificate
+auto_add_cert_user = False
+# Auto create MiG users with authenticated OpenID 2.0 login
+auto_add_oid_user = False
+# Auto create MiG users with authenticated OpenID Connect login
+auto_add_oidc_user = False
+# Auto create dedicated MiG resources from valid users
+#auto_add_resource = False
+# Apply filters to handle illegal characters e.g. in names during auto add
+# User ID fields to filter: full_name, organization, ...
+# Leave filter fields empty or unset to disable all filters and let input
+# validation simply reject user sign up if names contain such characters.
+auto_add_filter_fields =
+# How to handle each illegal character in the configured filter fields. The
+# default is to skip each such character. Other valid options include hexlify
+# to encode each such character with the corresponding hex codepoint.
+auto_add_filter_method = skip
+# Optional limit on users who may sign up through autocreate without operator
+# interaction. Defaults to allow ANY distinguished name if unset but only for
+# auth methods explicitly enabled with auto_add_X_user. Space separated list of
+# user field and regexp-filter pattern pairs separated by colons.
+auto_add_user_permit = distinguished_name:.*
+# Optional limit on users who may sign up through autocreate without operator
+# interaction if a valid peer exists. Defaults to allow ANY distinguished name
+# if unset but only for auth methods explicitly enabled with auto_add_X_user.
+# Space separated list of user field and regexp-filter pattern pairs separated
+# by colons.
+auto_add_user_with_peer = distinguished_name:.*
+# Default account expiry unless set. Renew and web login extends by default.
+cert_valid_days = 365
+oid_valid_days = 365
+oidc_valid_days = 365
+generic_valid_days = 365
+
+# Fully qualified domain name of this MiG server
+server_fqdn =
+
+# The Email address for support requests
+support_email =
+# The Email addresses of the Administrators of this MiG server
+# (comma-separated list with a space following each comma)
+admin_email = mig
+
+# The Distinguished Name of the Administrators of this MiG server
+# (comma-separated list with optional leading and trailing spaces)
+admin_list =
+
+# Send out notification emails with From pointing to smtp_sender. Useful e.g.
+# to set it to a local no-reply address to avoid bounces from stale users and
+# invitation (auto-)replies ending up here.
+# If left empty the sender defaults to something like testuser@ .
+smtp_sender =
+
+# Optional client certificate authentication
+# FQDN of the Certificate Authority host managing/signing user certificates.
+# Leave empty to disable unless you want client certificate authentication and
+# have your own CA to handle that part.
+ca_fqdn =
+# Local user account used for certificate handling on the CA host. Defaults to
+# mig-ca if unset but only ever used if ca_fqdn is set.
+ca_user = mig-ca
+# SMTP server used in relation to the user certificate handling. Defaults to
+# localhost if unset but only ever used if ca_fqdn is set.
+ca_smtp = localhost
+
+# Base paths
+# TODO: tilde in paths is not expanded where configparser is used directly!
+state_path = /test/path/state
+certs_path = /test/path/certs
+mig_path = /test/path/mig
+
+# Code paths
+mig_server_home = %(mig_path)s/server/
+grid_stdin = %(mig_server_home)s/server.stdin
+im_notify_stdin = %(mig_server_home)s/notify.stdin
+
+# State paths
+jupyter_mount_files_dir = %(state_path)s/jupyter_mount_files/
+mrsl_files_dir = %(state_path)s/mrsl_files/
+re_files_dir = %(state_path)s/re_files/
+re_pending_dir = %(state_path)s/re_pending/
+log_dir = %(state_path)s/log/
+gridstat_files_dir = %(state_path)s/gridstat_files/
+re_home = %(state_path)s/re_home/
+resource_home = %(state_path)s/resource_home/
+vgrid_home = %(state_path)s/vgrid_home/
+vgrid_files_home = %(state_path)s/vgrid_files_home/
+vgrid_public_base = %(state_path)s/vgrid_public_base/
+vgrid_private_base = %(state_path)s/vgrid_private_base/
+resource_pending = %(state_path)s/resource_pending/
+user_pending = %(state_path)s/user_pending/
+user_home = %(state_path)s/user_home/
+user_settings = %(state_path)s/user_settings/
+user_db_home = %(state_path)s/user_db_home/
+user_cache = %(state_path)s/user_cache/
+server_home = %(state_path)s/server_home/
+webserver_home = %(state_path)s/webserver_home/
+sessid_to_mrsl_link_home = %(state_path)s/sessid_to_mrsl_link_home/
+sessid_to_jupyter_mount_link_home = %(state_path)s/sessid_to_jupyter_mount_link_home/
+mig_system_files = %(state_path)s/mig_system_files/
+mig_system_storage = %(state_path)s/mig_system_storage/
+mig_system_run = %(state_path)s/mig_system_run/
+wwwpublic = %(state_path)s/wwwpublic/
+freeze_home = %(state_path)s/freeze_home/
+freeze_tape = %(state_path)s/freeze_tape/
+sharelink_home = %(state_path)s/sharelink_home/
+seafile_mount = %(state_path)s/seafile_mount/
+openid_store = %(state_path)s/openid_store/
+sitestats_home = %(state_path)s/sitestats_home/
+events_home = %(state_path)s/events_home/
+twofactor_home = %(state_path)s/twofactor_home/
+gdp_home = %(state_path)s/gdp_home/
+workflows_home = %(state_path)s/workflows_home/
+workflows_db_home = %(state_path)s/workflows_db_home/
+notify_home = %(state_path)s/notify_home/
+quota_home = %(state_path)s/quota_home/
+accounting_home = %(state_path)s/accounting_home/
+
+# GDP data categories metadata and helpers json file
+gdp_data_categories = %(gdp_home)s/data_categories.json
+
+# GDP ID helper to scramble IDs in gdp.log
+# Supported values include safe_encrypt, safe_hash, simple_hash and false with
+# safe_hash being the default SHA256 hash, simple_hash the classic MD5 hash,
+# safe_encrypt the Fernet encrypt using CRYPTO_SALT and false leaving the IDs
+# untouched. The corresponding underlying algorithm names md5, sha256 and
+# fernet can also be used instead of the above.
+gdp_id_scramble = safe_hash
+# GDP path helper to scramble possibly sensitive path names in gdp.log
+# Supported values include safe_encrypt, safe_hash, simple_hash and false with
+# safe_hash being the SHA256 hash, simple_hash the classic MD5 hash,
+# safe_encrypt the default Fernet encrypt using CRYPTO_SALT and false leaving
+# the paths untouched. The corresponding underlying algorithm names md5, sha256
+# and fernet can also be used instead of the above.
+gdp_path_scramble = safe_encrypt
+
+# For write-restricted VGrid shared folders
+# The readonly dir MUST be a 'ro' (possibly bind) mounted version of the
+# writable dir for this to work. Write-restricted VGrid support will remain
+# disabled unless these are both set and adhere to those requirements.
+vgrid_files_readonly = %(state_path)s/vgrid_files_readonly/
+vgrid_files_writable = %(state_path)s/vgrid_files_writable/
+
+# VGrid state files
+vgrid_owners = owners
+vgrid_members = members
+vgrid_resources = resources
+vgrid_triggers = triggers
+vgrid_settings = settings
+vgrid_sharelinks = sharelinks
+vgrid_monitor = monitor
+
+# Optional shared ssh public key presented to resource owners
+public_key_file = ~/.ssh/id_rsa.pub
+
+# x.509 certificate and key used for interserver communication
+server_cert = %(certs_path)s/combined.pem
+server_key = %(certs_path)s/combined.pem
+ca_cert = %(certs_path)s/cacert.pem
+# URLs
+migserver_public_url =
+migserver_public_alias_url =
+migserver_http_url =
+migserver_https_url =
+migserver_https_mig_cert_url =
+migserver_https_ext_cert_url =
+migserver_https_mig_oid_url =
+migserver_https_ext_oid_url =
+migserver_https_mig_oidc_url =
+migserver_https_ext_oidc_url =
+migserver_https_sid_url =
+
+# unique id of the MiG server
+mig_server_id = %(server_fqdn)s.0
+empty_job_name = no_grid_jobs_in_grid_scheduler
+notify_protocols = email
+smtp_server = localhost
+gdp_email_notify = False
+
+# Optional space-separated prioritized list of efficient storage access
+# protocols to advertize to clients. Leave to AUTO to use the ones actually
+# enabled with the corresponding enable_SERVICE options. Default is AUTO and
+# other allowed values are one or more of sftp, ftps and davs.
+# NOTE: the sftpsubsys service is advertized as just sftp to fit the protocol.
+storage_protocols = AUTO
+
+# Optional limit which instructs the web backends to deliver at most N bytes of
+# output to the users in order to avoid excessive memory use when serving files
+# which are subject to in-memory buffering. Defaults to a value of -1 which
+# disables the limit completely while 0 or any positive integer is intepreted
+# as a total number of bytes to allow serving in a single request.
+#
+# For now the limit is only enforced on cat.py, which is used implicitly for
+# certain downloads depending on site settings as well as for explicit client
+# requests e.g. xmlrpc requests or user scripts. The limit may or may not go
+# away in the future if better streaming of such requests can be made.
+wwwserve_max_bytes = -1
+
+# Optional extra service interfaces with common structure
+# * user_X_address is the host address to listen on
+# * user_X_port is the host port to listen on
+# * user_X_key is the host RSA key used for TLS/SSL securing connections
+# * user_X_auth is the allowed user auth methods (e.g. publickey or password)
+# * user_X_alias is user field(s) from user DB to allow as alias username
+# * user_X_show_address is the host address to advertise on Setup page
+# * user_X_show_port is the host port to advertise on Setup page
+#
+# NOTE: either use only one of grid_sftp and sftp_subsys or set them up on
+# separate address+port combination.
+# grid_sftp settings - standalone python sftp service
+# empty address means listen on all interfaces
+user_sftp_address =
+user_sftp_port = 2222
+# file with concatenated private key and public certificate for sftp server
+user_sftp_key =
+# file with ssh public host key matching the private key from above
+user_sftp_key_pub =
+# Optional ssh key fingerprint(s) of the key above for clients to verify.
+# They can typically be extracted from the command:
+# ssh-keygen -l -f %(user_sftp_key_pub)s .
+user_sftp_key_md5 =
+user_sftp_key_sha256 =
+# Optional ssh host key fingerprint verification from SSHFP record in DNS.
+user_sftp_key_from_dns = False
+# space separated list of sftp user authentication methods
+# (default: publickey password)
+#user_sftp_auth = publickey password
+user_sftp_alias =
+# Tuned packet sizes - window size 16M and max packet size 512K (default)
+# Paramiko comes with default window size 2M and max packet size 32K
+#user_sftp_window_size = 16777216
+#user_sftp_max_packet_size = 524288
+# Number of concurrent sftp logins per-user. Useful if they get too taxing.
+# A negative value means the limit is disabled (default).
+user_sftp_max_sessions = -1
+# sftp_subsys settings - optimized openssh+subsys sftp service
+# empty address means listen on all interfaces
+user_sftp_subsys_address =
+user_sftp_subsys_port = 22
+# If active sftp is exposed on another address/port (e.g. with port forward for
+# firewall-friendly access) it can be set here for display on Setup page.
+# If both sftp and sftpsubsys are enabled the preferred one may be exposed here.
+#user_sftp_show_address =
+#user_sftp_show_port =
+# grid_webdavs settings
+# empty address means listen on all interfaces
+user_davs_address =
+user_davs_port = 4443
+# file with concatenated private key and public certificate for davs server
+user_davs_key =
+# Optional davs key fingerprint(s) of the key above for clients to verify.
+# They can typically be extracted from the command:
+# openssl x509 -noout -fingerprint -sha256 -in %(user_davs_key)s .
+user_davs_key_sha256 =
+# space separated list of davs user authentication methods (default: password)
+# priority from order and allowed values are password (basic auth) and digest
+# IMPORTANT: digest auth breaks 2GB+ uploads from win 7 (OverflowError)
+#user_davs_auth = password
+user_davs_alias =
+# If davs is exposed on another address/port (e.g. with port forward for
+# firewall-friendly access) it can be set here for display on Setup page.
+#user_davs_show_address =
+#user_davs_show_port =
+# grid_ftps settings
+# empty address means listen on all interfaces
+user_ftps_address =
+user_ftps_ctrl_port = 8021
+user_ftps_pasv_ports = 8100:8400
+# file with concatenated private key and public certificate for ftps server
+user_ftps_key =
+# Optional ftps key fingerprint(s) of the key above for clients to verify.
+# They can typically be extracted from the command:
+# openssl x509 -noout -fingerprint -sha256 -in %(user_ftps_key)s .
+user_ftps_key_sha256 =
+# space separated list of ftps user authentication methods (default: password)
+#user_ftps_auth = password
+user_ftps_alias =
+# If ftps is exposed on another address/port (e.g. with port forward for
+# firewall-friendly access) it can be set here for display on Setup page.
+#user_ftps_show_address =
+#user_ftps_show_ctrl_port =
+# file with diffie-hellman parameters for strong SSL/TLS, shared by IO daemons
+user_shared_dhparams =
+# seafile integration settings
+# where seafile web hub is located (defaults to /seafile on same address)
+user_seahub_url = /seafile
+# where seafile clients should connect (defaults to seafile on SID address)
+user_seafile_url = https:///seafile
+# space separated list of seafile user authentication methods (default: password)
+#user_seafile_auth = password
+user_seafile_alias =
+# if seafile instance runs locally rather than stand-alone (default: False)
+user_seafile_local_instance = False
+# if local read-only mount is available for user home integration (default: False)
+user_seafile_ro_access = True
+# Priority list of protocols allowed in Duplicati backups (sftp, ftps, davs)
+user_duplicati_protocols = AUTO
+# Cloud settings for remote access - more in individual service sections
+# space separated list of cloud user authentication methods
+# (default: publickey)
+#user_cloud_ssh_auth = publickey
+user_cloud_alias =
+# IM notify helper setup - keep any login here secret to avoid abuse
+user_imnotify_address =
+user_imnotify_port = 6667
+user_imnotify_channel =
+user_imnotify_username =
+user_imnotify_password =
+# grid_openid settings for optional OpenID provider from MiG user DB
+# empty address means listen on all interfaces
+# NOTE: by default we listen on private high port and optionally proxy in vhost
+user_openid_address =
+user_openid_port = 8443
+# If openid is exposed on another address/port (e.g. with port forward for
+# firewall-friendly access or vhost proxy) it can be set here for automatic
+# masquerading inside the openid daemon. For most setups it makes sense to
+# uncomment the next two and specify a mig_oid_provider URL
+# with in it rather than the actual backend on provided
+# address.
+# Generated apache conf automatically includes proxy to support that:
+# https:///openid/
+# ->
+# https://:8443/openid/
+#user_openid_show_address =
+#user_openid_show_port =
+# file with concatenated private key and public certificate for openid server
+user_openid_key =
+# space separated list of openid user authentication methods (default: password)
+#user_openid_auth = password
+user_openid_alias =
+# Should local OpenID enforce MiG-users.db account expire? (Default: True)
+#user_openid_enforce_expire = True
+
+# Optional internal/external OpenID 2.0 identity provider(s) - leave empty
+# to disable OpenID 2.0 access or enable the local openid service and point it
+# there to allow username/password logins for the web interfaces. Typically
+# with a value like
+# https://%(mig_oid_fqdn)s:%(mig_oid_port)d/openid/id/
+# or with the previously mentioned automatic proxy setup
+# https:///openid/id/
+# It is possible to have users login using the credentials they registered in
+# the local MiG-users.db
+user_mig_oid_title = MiG
+user_mig_oid_provider =
+# Optional OpenID provider alias to same account in dual-head setups.
+# When set account signup will result in a htaccess file with BOTH mig and alt
+# OpenID provider in allowed IDs, so that signup through one head doesn't
+# remove cert_redirect access through the other.
+#user_mig_oid_provider_alias =
+user_ext_oid_title = External
+user_ext_oid_provider =
+user_openid_providers =
+
+# Optional internal/external OpenID Connect identity provider(s) - leave empty
+# to disable openid connect access. Values should be the server meta URL here.
+#user_mig_oidc_title = MiG
+#user_mig_oidc_provider =
+user_ext_oidc_title = External
+user_ext_oidc_provider =
+user_openidconnect_providers =
+# OpenID Connect provider provides issuer and audience as part of the ID claim.
+# We use those fields and crypto signature to check authenticity of claims.
+#user_mig_oidc_issuer =
+#user_mig_oidc_audience = 1234abcd-12ab-34cd-ef56-ghijklmn7890
+user_ext_oidc_issuer =
+user_ext_oidc_audience =
+
+#user_mig_cert_title = MiG
+#user_ext_cert_title = Other
+
+logfile = mig.log
+loglevel = info
+peerfile = MiGpeers.conf
+sleep_period_for_empty_jobs = 120
+cputime_for_empty_jobs = 180
+min_seconds_between_live_update_requests = 60
+
+# Please note that order *does* matter for these lists!
+# First entry is default for resource creation cgi
+architectures = X86 AMD64 IA64 SPARC SPARC64 ITANIUM SUN4U SPARC-T1 SPARC-T2
+scriptlanguages = sh python
+jobtypes = batch interactive all
+lrmstypes = Native Native-execution-leader Batch Batch-execution-leader
+
+# Include any additional section confs files from a per-section conf folder
+include_sections = %(mig_server_home)s/MiGserver.d
+
+# Jupyter integration sections
+### E.g. ###
+# [JUPYTER_DAG]
+# service_name=dag
+# service_desc=This is an awesome service
+# service_hosts=https://192.168.1.10 https://hub002.com http://hub003.com
+###
+# During install.py the individual sections will be generated
+# in accordance with the jupyter_services parameter content
+# For each section a apache proxy balancer config is generated,
+# which will setup the target url location.
+# In the example provided, the system will generate a location called /dag
+# as defined by the 'service_name' in the /etc/httpd/conf.extras.d/MiG-jupyter-def.conf file
+
+
+# Cloud integration sections
+### E.g. ###
+# [CLOUD_MIST]
+# General cloud provider settings and limits
+# service_name=MIST
+# service_desc=This is an awesome service
+# service_provider_flavor = openstack
+# service_hosts = REST API URL
+# service_max_user_instances = 16
+# Semi-colon separated list of img=user login pairs when img and user differs
+# service_user_map = centos7=centos;centos8=centos;ubuntu-xenial=ubuntu
+# Cloud instance defaults
+# The general structure is a default option and an optional user override map
+# service_flavor_id = INSERT CLOUD FLAVOR ID
+# Semi-colon separated list of user=flavor pairs to override for some users
+# service_flavor_id_map =
+# service_network_id = INSERT CLOUD NETWORK ID
+# Semi-colon separated list of user=net pairs to override for some users
+# service_network_id_map =
+# service_key_id = INSERT DEFAULT KEY ID
+# Semi-colon separated list of user=keyid pairs to override for some users
+# service_key_id_map =
+# service_sec_group_id = INSERT CLOUD SEC GROUP ID
+# Semi-colon separated list of user=secgrp pairs to override for some users
+# service_sec_group_id_map
+# service_floating_network_id = INSERT CLOUD FLOATING NETWORK ID
+# Semi-colon separated list of user=floatnet pairs to override for some users
+# service_floating_network_id_map =
+# service_availability_zone = INSERT CLOUD AVAILABILITY ZONE
+# Semi-colon separated list of user=availzone pairs to override for some users
+# service_availability_zone_map =
+# Optional jump host so that instances are shielded fom direct ssh access
+# service_jumphost_address =
+# Semi-colon separated list of user=jumpaddr pairs to override for some users
+# service_jumphost_address_map =
+# service_jumphost_user = mist
+# Path to the ssh key used for managing user public keys on cloud jumphost
+# service_jumphost_key = ~/.ssh/cloud-jumphost-key
+# Semi-colon separated list of user=jumpuser pairs to override for some users
+# service_jumphost_user_map =
+# Helper to automatically add user pub keys on jumphost
+# The script and coding values are used like this under the hood:
+# ssh %(jumphost_user)s@%(jumphost_address)s %(jumphost_manage_keys_script)s add \
+# %(jumphost_manage_keys_coding)s %(encoded_client_id)s %(encoded_pub_keys)s
+# where coding is applied to client_id and pub_keys to yield encoded_X versions
+# service_jumphost_manage_keys_script = manage_mist_keys.py
+# service_jumphost_manage_keys_coding = base16
+###
+# During install.py the individual sections will be generated
+# in accordance with the cloud_services parameter content
+
+
+
+[SCHEDULER]
+# Scheduling algorithm to use
+# Currently supported: FIFO, FirstFit, BestFit, FairFit, Random and MaxThroughput
+algorithm = FairFit
+#
+# How long to keep jobs that can not be scheduled in the queue.
+# Jobs that stay 'expire_after' seconds in the queue can be expired by
+# the scheduler. Setting expire_after to 0 disables expiry.
+# 1 day: 86400 seconds
+# 7 days: 604800 seconds
+# 90 days: 7776000 seconds
+# 1 year: 31536000 seconds
+# 2 year: 63072000 seconds
+expire_after = 31536000
+
+job_retries = 2
+
+[MONITOR]
+sleep_secs = 120
+sleep_update_totals = 600
+slackperiod = 600
+
+[WORKFLOWS]
+# Workflow specific settings
+# Directory paths relative to an individual vgrid
+vgrid_patterns_home = .workflow_patterns_home/
+vgrid_recipes_home = .workflow_recipes_home/
+vgrid_tasks_home = .workflow_tasks_home/
+
+[SETTINGS]
+language = English
+submitui = fields textarea files
+
+[SCM]
+hg_path =
+hgweb_scripts =
+
+[TRACKER]
+trac_admin_path =
+# Note: We can't use mig_server_home from GLOBAL section here
+trac_ini_path =
+# IMPORTANT: Keep trac_id_field in sync with apache trac login section
+#trac_id_field = email
+
+[RESOURCES]
+default_mount_re = SSHFS-2.X-1
+
+[QUOTA]
+backend = lustre
+update_interval = 3600
+user_limit = 1099511627776
+vgrid_limit = 1099511627776
+
+[ACCOUNTING]
+update_interval = 3600
+
+[SITE]
+# Web site appearance
+# Whether to use Python 3 for all Python invocations
+prefer_python3 = False
+# Dynamic entry page to pick user default with fallback to site landing page
+autolaunch_page = /wsgi-bin/autolaunch.py
+# Entry page if not explictly provided or overriden by user
+landing_page = /wsgi-bin/home.py
+# Skin to style all pages with (taken from mig/images/skin/NAME)
+skin = migrid-basic
+# Which skin to style pages without theme with
+static_css = /images/skin/migrid-basic/core.css
+# Optional space separated list of extra javascripts to inject on user pages
+extra_userpage_scripts =
+# Optional space separated list of extra stylesheets to inject on user pages
+extra_userpage_styles =
+# Selectable base menus (simple, default or advanced to match X_menu options below)
+base_menu = default
+# Default sorted menu items to include
+#valid menu items are: home dashboard submitjob files jobs vgrids resources downloads runtimeenvs archives settings statistics docs people migadmin transfers sharelinks crontab seafile jupyter peers logout close
+default_menu = home files submitjob jobs vgrids resources runtimeenvs people settings downloads transfers sharelinks crontab docs logout
+#simple_menu = home files vgrids settings logout
+#advanced_menu = home files submitjob jobs vgrids resources runtimeenvs people settings downloads archives transfers sharelinks crontab docs logout
+# Additional sorted user selectable menu entries
+user_menu =
+# Selectable VGrid component links (default or advanced to match X_vgrid_links options below)
+collaboration_links = default advanced
+# VGrid component visibility and order - automatically tries auto detection if not set.
+default_vgrid_links = files web
+advanced_vgrid_links = files web scm tracker workflows monitor
+# VGrid label allows setting another name to use instead of VGrid
+vgrid_label = VGrid
+#script_deps = jquery.js jquery.contextmenu.js jquery.contextmenu.css jquery.form.js jquery.prettyprint.js jquery.tablesorter.js jquery.tablesorter.pager.js jquery-ui.js jquery-ui.css jquery-ui-theme.css jquery-ui-theme.custom.css jquery.calendar-widget.js jquery.calculator.js jquery.calculator.css jquery.countdown.js jquery.countdown.css jquery.epiclock.js jquery.epiclock.css jquery.jgcharts.js jquery.sparkline.js jquery.form.wizard.js
+#default_css = /images/default.css
+fav_icon = /images/skin/migrid-basic/favicon.ico
+title = Minimum intrusion Grid
+short_title = MiG
+# Optional external help url e.g. used as Help in the V3 user menu
+external_doc = https://sourceforge.net/p/migrid/wiki
+# Enable web-based site administration for admin_list users
+enable_migadmin = False
+# Further restrictions for admin_list users to view and act through migadmin
+# where the default is ANY for no extra restrictions and a space separated list
+# of the allowed values (cert, oid and oidc) can be given to limit to those.
+#migadmin_view_access = ANY
+#migadmin_act_access = ANY
+# Enable strict access control and logs for compliance with the General Data
+# Protection Regulation (GDPR) imposed by the EU. You probably want this if
+# and only if your users need to store sensitive/personal data. More info at
+# https://en.wikipedia.org/wiki/General_Data_Protection_Regulation
+enable_gdp = False
+# Enable user job execution on any associated compute resources
+enable_jobs = False
+# Enable execution and storage resources for vgrids
+enable_resources = False
+# Enable that workflows are available
+enable_workflows = False
+# Enable vgrid workflow triggers for file system events
+enable_events = False
+# Enable efficient I/O daemons - sftp, ftps and webdavs
+# Pure Python Paramiko-based sftp daemon
+enable_sftp = False
+# OpenSSH sftp daemon with just the Paramiko fs layer as subsys handler
+enable_sftp_subsys = False
+# Pure Python WsgiDAV-based webdav(s) daemon
+enable_davs = False
+# Allow sub-optimal but still relatively strong legacy TLS support in WebDAVS
+# NOTE: Python-2.7+ ssl supports TLSv1.2+ with strong ciphers and all popular
+# clients (including Windows 10+ native WebDAVS) also work with those.
+#enable_davs_legacy_tls = False
+# Pure Python pyftpdlib-based ftp(s) daemon
+enable_ftps = False
+# Allow sub-optimal but still relatively strong legacy TLS supports in FTPS
+# NOTE: Modern PyOpenSSL supports TLSv1.2+ with strong ciphers and all popular
+# clients also work with those.
+#enable_ftps_legacy_tls = False
+# Enable WSGI served web pages (faster than CGI) - requires apache wsgi module
+enable_wsgi = True
+# Enable system notify helper used e.g. to warn about failed user logins
+enable_notify = False
+# Enable IM notify helper - additionally requires configuration above
+enable_imnotify = False
+# Enable users to schedule tasks with a cron/at-like interface
+enable_crontab = True
+# Enable janitor service to handle recurring tasks like clean up and cache updates
+enable_janitor = False
+# Enable 2FA for web access and IO services with any TOTP authenticator client
+# IMPORTANT: Do NOT change this option manually here (requires Apache changes)!
+# use generateconfs.py --enable_twofactor=True|False
+enable_twofactor = True
+# Always require twofactor authentication for one or more protocols.
+twofactor_mandatory_protos =
+# Require logins to come from already active 2FA session IP address
+# if user has enabled 2FA for them.
+# IMPORTANT: Do NOT change this option manually here (requires Apache changes)!
+# use generateconfs.py --enable_twofactor_strict_address=True|False
+twofactor_strict_address = False
+# Which 2FA authenticator apps to mention on setup wizard. A space-separated
+# list of names from: bitwarden, freeotp, google, microfocus, microsoft, yubico
+twofactor_auth_apps =
+# Enable Peers system for site-local users to vouch for external users
+enable_peers = False
+# Whether external user requests must explicitly specify their sponsor (Peers)
+peers_mandatory = False
+# Explicit fields to request on external user sign up forms (full_name, email)
+peers_explicit_fields =
+# Short description of whom to point to as contact(s) in the Peers system
+peers_contact_hint = employed here and authorized to invite external users
+# Enable OpenID daemon for web access with user/pw from local user DB
+enable_openid = False
+# Allow sub-optimal but still relatively strong legacy TLS support in OpenID 2.0
+# NOTE: Python-2.7+ ssl supports TLSv1.2+ with strong ciphers and all popular
+# clients also work with those.
+#enable_openid_legacy_tls = False
+# Enable share links for easy external exchange of data with anyone
+enable_sharelinks = True
+# Enable storage quota
+enable_quota = False
+# Enable storage accounting
+enable_accounting = False
+# Enable background data transfers daemon - requires lftp and rsync
+enable_transfers = False
+# Explicit background transfer source addresses for use in pub key restrictions
+# It may be necessary to set it to match the FQDN of the default outgoing NIC
+transfers_from =
+# Custom per-user overall transfer log location for shared fs sites
+#transfer_log = transfer.log
+# Enable freeze archive handlers - support for write-once archiving of files
+# for e.g. the data associated with a research paper.
+enable_freeze = True
+# Which frozen archive flavors can be deleted (True for all, False or empty for
+# none and a space-separated list of flavors for individual control.
+permanent_freeze = no
+# The Distinguished Name of freeze administrators who can always delete their
+# archives no matter what permanent_freeze says, useful after testing.
+# (comma-separated list with optional leading and trailing spaces)
+#freeze_admins =
+# Delay before frozen archives are expected to hit tape (e.g. 5m, 4d or 2w).
+# Leave unset or empty if no tape archiving is available.
+freeze_to_tape =
+# Enable Jupyter integration - requires a remote Jupyter server configured to
+# allow our users to connect and then integrates mount of user home there
+enable_jupyter = False
+# Enable cloud integration - requires a remote OpenStack server configured to
+# allow our users to connect and then integrates mount of user home there
+enable_cloud = False
+# Enable Seafile synchronization service - requires local Seafile install
+enable_seafile = False
+# Enable Duplicati user computer backup integration
+enable_duplicati = False
+# Enable gravatar.com integration for user profile avatars
+enable_gravatars = False
+# Enable dynamic site status integration particularly in UI V3
+enable_sitestatus = True
+# Where to find json-formatted list of site events for dynamic site status
+# NOTE: either create this file or symlink to the included one.
+#status_events = /public/status-events.json
+# Include status events with system set to one of these (ANY disables filter)
+status_system_match = ANY
+# Enable legacy grid.dk features
+#enable_griddk = False
+# Whether to enforce automatic IO protocol access expiry after weeks of web
+# inactivity. When enabled users have to do a web log in once in a while to
+# preserve full SFTP, WebDAVS and FTPS service access.
+io_account_expire = False
+# User interfaces for users to select with first as default (allowed: V2, V3)
+user_interface = V3 V2
+# For gradual transition to new user interface set default here for new users
+#new_user_default_ui =
+# Security scanners to let scan e.g. for common logins without notify on errors
+security_scanners = UNSET
+# Cross Site Request Forgery protection level (MINIMAL, WARN, MEDIUM or FULL).
+# Where MINIMAL only requires a POST on changes, FULL additionally requires
+# CSRF tokens for all such operations, and MEDIUM likewise requires CSRF tokens
+# but with the exception that legacy user script and xmlrpc clients are allowed
+# access without. The default will likely change to FULL in the future when all
+# clients are ready. The transitional WARN mode basically enforces MINIMAL but
+# checks and logs all CSRF failures like FULL.
+csrf_protection = MEDIUM
+# Password strength policy (NONE, WEAK, MEDIUM, HIGH, MODERN:L or CUSTOM:L:C)
+# for all password-enabled services, e.g. sftp, webdavs, ftps and openid.
+# Where NONE is the legacy behavior of no explicit length or character class
+# checks - except safeinput min len and optionally any cracklib requirements
+# if enabled. The other plain names require increasing strength in terms of
+# length and number of different character classes included. MODERN:L leaves
+# the outdated focus on character classes behind and only requires longer
+# passwords of any L characters, and recommends multi-factor auth and cracklib
+# enforcement for added security. The CUSTOM:L:C version offers complete
+# control over the required length (L) and number of character classes (C).
+password_policy = MEDIUM
+# Since the password_policy is used both in password selection and during
+# actual log in, it may be necessary to allow old passwords ONLY for log in
+# until all passwords have been changed to fit a new policy.
+# The optional password_legacy_policy can be set to the old policy for that
+# purpose, and otherwise defaults to disabled.
+password_legacy_policy =
+# Optional additional guard against simple passwords with the cracklib library
+password_cracklib = False
+# Optional prefilter on users who may sign up as site users with sign up forms.
+# Used as a coarse filter to reject invalid user requests early by only
+# filtering on form values (organization, email). E.g. to avoid internal users
+# signing up as externals. Space separated list of user field and regexp-filter
+# pattern pairs separated by colons.
+signup_prefilter = email:.*
+# Optional prefilter on users who may potenially invite peers as site users.
+# Used as a coarse filter to reject clearly invalid user requests early by only
+# filtering on form values (peers_full_name and peers_email). Space separated
+# list of user field and regexp-filter pattern pairs separated by colons.
+peers_prefilter = peers_email:.*
+# Optional limit on users who may invite peers as site users. Space separated
+# list of user field and regexp-filter pattern pairs separated by colons.
+peers_permit = distinguished_name:.*
+# Optional html banner on Peers page to inform e.g. about access restrictions
+#peers_notice =
+# Optional limit on users who can create vgrids. Space separated list of user
+# field and regexp-filter pattern pairs separated by colons.
+vgrid_creators = distinguished_name:.*
+# Optional limit on users who can manage vgrids. Space separated list of user
+# field and regexp-filter pattern pairs separated by colons.
+vgrid_managers = distinguished_name:.*
+# Space separated list of methods to include on the signup page: default is
+# extcert only and order is used on the signup page
+signup_methods = extcert
+# Space separated list of methods to include on the login page: default is same
+# as signup_methods and order is used on login page and various other pages
+# presenting the users with one or more possible https urls.
+login_methods = extcert
+# Extra note displayed during sign up
+#signup_hint =
+# Digest authentication hex salt for scrambling saved digest credentials
+# IMPORTANT: digest credentials need to be saved again if this is changed
+# Can be a plain string, a path to a file or an environment value and the
+# content must be a string of e.g. 32 hex characters. If two FILE values are
+# given the value is read from the 2nd (cache) file if available and read from
+# first (persistent) file path and saved to cache path otherwise. Useful e.g.
+# with a tmpfs cache.
+#digest_salt = 084528A93A4E0A40905609A729394F5C
+#digest_salt = FILE::/path/to/digest-salt.hex
+#digest_salt = FILE::/path/to/persistent-digest-salt.hex$$/path/to/cached-digest-salt.hex
+#digest_salt = ENV::DIGEST_SALT
+digest_salt = DDDD12344321DDDD
+# Optional crypto helper salt used to protect data stored on disk or in logs
+# Can be a plain string, a path to a file or an environment value and the
+# content must be a string of e.g. 32 hex characters. If two FILE values are
+# given the value is read from the 2nd (cache) file if available and read from
+# first (persistent) file path and saved to cache path otherwise. Useful e.g.
+# with a tmpfs cache.
+#crypto_salt = 280845A93A4E0A40905609A7294F5C39
+#crypto_salt = FILE::/path/to/crypto-salt.hex
+#crypto_salt = FILE::/path/to/persistent-crypto-salt.hex$$/path/to/cached-crypto-salt.hex
+#crypto_salt = ENV::CRYPTO_SALT
+crypto_salt = CCCC12344321CCCC
+# Optional software catalogue from grid.dk
+#swrepo_url = /software-repository/
+# Use left logo from skin and default center text for top banner
+logo_left = /images/skin/migrid-basic/logo-left.png
+logo_center = MiG
+# Uncomment to also enable right logo from skin in top banner
+logo_right = /images/skin/migrid-basic/logo-right.png
+#support_text = Support & Questions
+#privacy_text =
+#credits_text = 2003-2023, The MiG Project
+#credits_image = /images/copyright.png
+# Optional data safety notice and popup on Files page
+datasafety_link =
+datasafety_text =
+
+[TEMPLATES]
diff --git a/tests/data/MiGserver--templates.conf b/tests/data/MiGserver--templates.conf
new file mode 100644
index 000000000..24809cf66
--- /dev/null
+++ b/tests/data/MiGserver--templates.conf
@@ -0,0 +1,791 @@
+# MiG server configuration file
+[GLOBAL]
+# Run server in test mode?
+# Server distribution is disabled per default.
+# Set to True to let a set og MiG servers migrate jobs (EXPERIMENTAL!).
+#enable_server_dist = False
+
+# Allow users and resources joining without admin action?
+# Security is then left to vgrids admission as entities will only have access
+# to the default_vgrid when created.
+# Auto create MiG users with valid certificate
+auto_add_cert_user = False
+# Auto create MiG users with authenticated OpenID 2.0 login
+auto_add_oid_user = False
+# Auto create MiG users with authenticated OpenID Connect login
+auto_add_oidc_user = False
+# Auto create dedicated MiG resources from valid users
+#auto_add_resource = False
+# Apply filters to handle illegal characters e.g. in names during auto add
+# User ID fields to filter: full_name, organization, ...
+# Leave filter fields empty or unset to disable all filters and let input
+# validation simply reject user sign up if names contain such characters.
+auto_add_filter_fields =
+# How to handle each illegal character in the configured filter fields. The
+# default is to skip each such character. Other valid options include hexlify
+# to encode each such character with the corresponding hex codepoint.
+auto_add_filter_method = skip
+# Optional limit on users who may sign up through autocreate without operator
+# interaction. Defaults to allow ANY distinguished name if unset but only for
+# auth methods explicitly enabled with auto_add_X_user. Space separated list of
+# user field and regexp-filter pattern pairs separated by colons.
+auto_add_user_permit = distinguished_name:.*
+# Optional limit on users who may sign up through autocreate without operator
+# interaction if a valid peer exists. Defaults to allow ANY distinguished name
+# if unset but only for auth methods explicitly enabled with auto_add_X_user.
+# Space separated list of user field and regexp-filter pattern pairs separated
+# by colons.
+auto_add_user_with_peer = distinguished_name:.*
+# Default account expiry unless set. Renew and web login extends by default.
+cert_valid_days = 365
+oid_valid_days = 365
+oidc_valid_days = 365
+generic_valid_days = 365
+
+# Fully qualified domain name of this MiG server
+server_fqdn =
+
+# The Email address for support requests
+support_email =
+# The Email addresses of the Administrators of this MiG server
+# (comma-separated list with a space following each comma)
+admin_email = mig
+
+# The Distinguished Name of the Administrators of this MiG server
+# (comma-separated list with optional leading and trailing spaces)
+admin_list =
+
+# Send out notification emails with From pointing to smtp_sender. Useful e.g.
+# to set it to a local no-reply address to avoid bounces from stale users and
+# invitation (auto-)replies ending up here.
+# If left empty the sender defaults to something like testuser@ .
+smtp_sender =
+
+# Optional client certificate authentication
+# FQDN of the Certificate Authority host managing/signing user certificates.
+# Leave empty to disable unless you want client certificate authentication and
+# have your own CA to handle that part.
+ca_fqdn =
+# Local user account used for certificate handling on the CA host. Defaults to
+# mig-ca if unset but only ever used if ca_fqdn is set.
+ca_user = mig-ca
+# SMTP server used in relation to the user certificate handling. Defaults to
+# localhost if unset but only ever used if ca_fqdn is set.
+ca_smtp = localhost
+
+# Base paths
+# TODO: tilde in paths is not expanded where configparser is used directly!
+state_path = /test/path/state
+certs_path = /test/path/certs
+mig_path = /test/path/mig
+
+# Code paths
+mig_server_home = %(mig_path)s/server/
+grid_stdin = %(mig_server_home)s/server.stdin
+im_notify_stdin = %(mig_server_home)s/notify.stdin
+
+# State paths
+jupyter_mount_files_dir = %(state_path)s/jupyter_mount_files/
+mrsl_files_dir = %(state_path)s/mrsl_files/
+re_files_dir = %(state_path)s/re_files/
+re_pending_dir = %(state_path)s/re_pending/
+log_dir = %(state_path)s/log/
+gridstat_files_dir = %(state_path)s/gridstat_files/
+re_home = %(state_path)s/re_home/
+resource_home = %(state_path)s/resource_home/
+vgrid_home = %(state_path)s/vgrid_home/
+vgrid_files_home = %(state_path)s/vgrid_files_home/
+vgrid_public_base = %(state_path)s/vgrid_public_base/
+vgrid_private_base = %(state_path)s/vgrid_private_base/
+resource_pending = %(state_path)s/resource_pending/
+user_pending = %(state_path)s/user_pending/
+user_home = %(state_path)s/user_home/
+user_settings = %(state_path)s/user_settings/
+user_db_home = %(state_path)s/user_db_home/
+user_cache = %(state_path)s/user_cache/
+server_home = %(state_path)s/server_home/
+webserver_home = %(state_path)s/webserver_home/
+sessid_to_mrsl_link_home = %(state_path)s/sessid_to_mrsl_link_home/
+sessid_to_jupyter_mount_link_home = %(state_path)s/sessid_to_jupyter_mount_link_home/
+mig_system_files = %(state_path)s/mig_system_files/
+mig_system_storage = %(state_path)s/mig_system_storage/
+mig_system_run = %(state_path)s/mig_system_run/
+wwwpublic = %(state_path)s/wwwpublic/
+freeze_home = %(state_path)s/freeze_home/
+freeze_tape = %(state_path)s/freeze_tape/
+sharelink_home = %(state_path)s/sharelink_home/
+seafile_mount = %(state_path)s/seafile_mount/
+openid_store = %(state_path)s/openid_store/
+sitestats_home = %(state_path)s/sitestats_home/
+events_home = %(state_path)s/events_home/
+twofactor_home = %(state_path)s/twofactor_home/
+gdp_home = %(state_path)s/gdp_home/
+workflows_home = %(state_path)s/workflows_home/
+workflows_db_home = %(state_path)s/workflows_db_home/
+notify_home = %(state_path)s/notify_home/
+quota_home = %(state_path)s/quota_home/
+accounting_home = %(state_path)s/accounting_home/
+
+# GDP data categories metadata and helpers json file
+gdp_data_categories = %(gdp_home)s/data_categories.json
+
+# GDP ID helper to scramble IDs in gdp.log
+# Supported values include safe_encrypt, safe_hash, simple_hash and false with
+# safe_hash being the default SHA256 hash, simple_hash the classic MD5 hash,
+# safe_encrypt the Fernet encrypt using CRYPTO_SALT and false leaving the IDs
+# untouched. The corresponding underlying algorithm names md5, sha256 and
+# fernet can also be used instead of the above.
+gdp_id_scramble = safe_hash
+# GDP path helper to scramble possibly sensitive path names in gdp.log
+# Supported values include safe_encrypt, safe_hash, simple_hash and false with
+# safe_hash being the SHA256 hash, simple_hash the classic MD5 hash,
+# safe_encrypt the default Fernet encrypt using CRYPTO_SALT and false leaving
+# the paths untouched. The corresponding underlying algorithm names md5, sha256
+# and fernet can also be used instead of the above.
+gdp_path_scramble = safe_encrypt
+
+# For write-restricted VGrid shared folders
+# The readonly dir MUST be a 'ro' (possibly bind) mounted version of the
+# writable dir for this to work. Write-restricted VGrid support will remain
+# disabled unless these are both set and adhere to those requirements.
+vgrid_files_readonly = %(state_path)s/vgrid_files_readonly/
+vgrid_files_writable = %(state_path)s/vgrid_files_writable/
+
+# VGrid state files
+vgrid_owners = owners
+vgrid_members = members
+vgrid_resources = resources
+vgrid_triggers = triggers
+vgrid_settings = settings
+vgrid_sharelinks = sharelinks
+vgrid_monitor = monitor
+
+# Optional shared ssh public key presented to resource owners
+public_key_file = ~/.ssh/id_rsa.pub
+
+# x.509 certificate and key used for interserver communication
+server_cert = %(certs_path)s/combined.pem
+server_key = %(certs_path)s/combined.pem
+ca_cert = %(certs_path)s/cacert.pem
+# URLs
+migserver_public_url =
+migserver_public_alias_url =
+migserver_http_url =
+migserver_https_url =
+migserver_https_mig_cert_url =
+migserver_https_ext_cert_url =
+migserver_https_mig_oid_url =
+migserver_https_ext_oid_url =
+migserver_https_mig_oidc_url =
+migserver_https_ext_oidc_url =
+migserver_https_sid_url =
+
+# unique id of the MiG server
+mig_server_id = %(server_fqdn)s.0
+empty_job_name = no_grid_jobs_in_grid_scheduler
+notify_protocols = email
+smtp_server = localhost
+gdp_email_notify = False
+
+# Optional space-separated prioritized list of efficient storage access
+# protocols to advertize to clients. Leave to AUTO to use the ones actually
+# enabled with the corresponding enable_SERVICE options. Default is AUTO and
+# other allowed values are one or more of sftp, ftps and davs.
+# NOTE: the sftpsubsys service is advertized as just sftp to fit the protocol.
+storage_protocols = AUTO
+
+# Optional limit which instructs the web backends to deliver at most N bytes of
+# output to the users in order to avoid excessive memory use when serving files
+# which are subject to in-memory buffering. Defaults to a value of -1 which
+# disables the limit completely while 0 or any positive integer is intepreted
+# as a total number of bytes to allow serving in a single request.
+#
+# For now the limit is only enforced on cat.py, which is used implicitly for
+# certain downloads depending on site settings as well as for explicit client
+# requests e.g. xmlrpc requests or user scripts. The limit may or may not go
+# away in the future if better streaming of such requests can be made.
+wwwserve_max_bytes = -1
+
+# Optional extra service interfaces with common structure
+# * user_X_address is the host address to listen on
+# * user_X_port is the host port to listen on
+# * user_X_key is the host RSA key used for TLS/SSL securing connections
+# * user_X_auth is the allowed user auth methods (e.g. publickey or password)
+# * user_X_alias is user field(s) from user DB to allow as alias username
+# * user_X_show_address is the host address to advertise on Setup page
+# * user_X_show_port is the host port to advertise on Setup page
+#
+# NOTE: either use only one of grid_sftp and sftp_subsys or set them up on
+# separate address+port combination.
+# grid_sftp settings - standalone python sftp service
+# empty address means listen on all interfaces
+user_sftp_address =
+user_sftp_port = 2222
+# file with concatenated private key and public certificate for sftp server
+user_sftp_key =
+# file with ssh public host key matching the private key from above
+user_sftp_key_pub =
+# Optional ssh key fingerprint(s) of the key above for clients to verify.
+# They can typically be extracted from the command:
+# ssh-keygen -l -f %(user_sftp_key_pub)s .
+user_sftp_key_md5 =
+user_sftp_key_sha256 =
+# Optional ssh host key fingerprint verification from SSHFP record in DNS.
+user_sftp_key_from_dns = False
+# space separated list of sftp user authentication methods
+# (default: publickey password)
+#user_sftp_auth = publickey password
+user_sftp_alias =
+# Tuned packet sizes - window size 16M and max packet size 512K (default)
+# Paramiko comes with default window size 2M and max packet size 32K
+#user_sftp_window_size = 16777216
+#user_sftp_max_packet_size = 524288
+# Number of concurrent sftp logins per-user. Useful if they get too taxing.
+# A negative value means the limit is disabled (default).
+user_sftp_max_sessions = -1
+# sftp_subsys settings - optimized openssh+subsys sftp service
+# empty address means listen on all interfaces
+user_sftp_subsys_address =
+user_sftp_subsys_port = 22
+# If active sftp is exposed on another address/port (e.g. with port forward for
+# firewall-friendly access) it can be set here for display on Setup page.
+# If both sftp and sftpsubsys are enabled the preferred one may be exposed here.
+#user_sftp_show_address =
+#user_sftp_show_port =
+# grid_webdavs settings
+# empty address means listen on all interfaces
+user_davs_address =
+user_davs_port = 4443
+# file with concatenated private key and public certificate for davs server
+user_davs_key =
+# Optional davs key fingerprint(s) of the key above for clients to verify.
+# They can typically be extracted from the command:
+# openssl x509 -noout -fingerprint -sha256 -in %(user_davs_key)s .
+user_davs_key_sha256 =
+# space separated list of davs user authentication methods (default: password)
+# priority from order and allowed values are password (basic auth) and digest
+# IMPORTANT: digest auth breaks 2GB+ uploads from win 7 (OverflowError)
+#user_davs_auth = password
+user_davs_alias =
+# If davs is exposed on another address/port (e.g. with port forward for
+# firewall-friendly access) it can be set here for display on Setup page.
+#user_davs_show_address =
+#user_davs_show_port =
+# grid_ftps settings
+# empty address means listen on all interfaces
+user_ftps_address =
+user_ftps_ctrl_port = 8021
+user_ftps_pasv_ports = 8100:8400
+# file with concatenated private key and public certificate for ftps server
+user_ftps_key =
+# Optional ftps key fingerprint(s) of the key above for clients to verify.
+# They can typically be extracted from the command:
+# openssl x509 -noout -fingerprint -sha256 -in %(user_ftps_key)s .
+user_ftps_key_sha256 =
+# space separated list of ftps user authentication methods (default: password)
+#user_ftps_auth = password
+user_ftps_alias =
+# If ftps is exposed on another address/port (e.g. with port forward for
+# firewall-friendly access) it can be set here for display on Setup page.
+#user_ftps_show_address =
+#user_ftps_show_ctrl_port =
+# file with diffie-hellman parameters for strong SSL/TLS, shared by IO daemons
+user_shared_dhparams =
+# seafile integration settings
+# where seafile web hub is located (defaults to /seafile on same address)
+user_seahub_url = /seafile
+# where seafile clients should connect (defaults to seafile on SID address)
+user_seafile_url = https:///seafile
+# space separated list of seafile user authentication methods (default: password)
+#user_seafile_auth = password
+user_seafile_alias =
+# if seafile instance runs locally rather than stand-alone (default: False)
+user_seafile_local_instance = False
+# if local read-only mount is available for user home integration (default: False)
+user_seafile_ro_access = True
+# Priority list of protocols allowed in Duplicati backups (sftp, ftps, davs)
+user_duplicati_protocols = AUTO
+# Cloud settings for remote access - more in individual service sections
+# space separated list of cloud user authentication methods
+# (default: publickey)
+#user_cloud_ssh_auth = publickey
+user_cloud_alias =
+# IM notify helper setup - keep any login here secret to avoid abuse
+user_imnotify_address =
+user_imnotify_port = 6667
+user_imnotify_channel =
+user_imnotify_username =
+user_imnotify_password =
+# grid_openid settings for optional OpenID provider from MiG user DB
+# empty address means listen on all interfaces
+# NOTE: by default we listen on private high port and optionally proxy in vhost
+user_openid_address =
+user_openid_port = 8443
+# If openid is exposed on another address/port (e.g. with port forward for
+# firewall-friendly access or vhost proxy) it can be set here for automatic
+# masquerading inside the openid daemon. For most setups it makes sense to
+# uncomment the next two and specify a mig_oid_provider URL
+# with in it rather than the actual backend on provided
+# address.
+# Generated apache conf automatically includes proxy to support that:
+# https:///openid/
+# ->
+# https://:8443/openid/
+#user_openid_show_address =
+#user_openid_show_port =
+# file with concatenated private key and public certificate for openid server
+user_openid_key =
+# space separated list of openid user authentication methods (default: password)
+#user_openid_auth = password
+user_openid_alias =
+# Should local OpenID enforce MiG-users.db account expire? (Default: True)
+#user_openid_enforce_expire = True
+
+# Optional internal/external OpenID 2.0 identity provider(s) - leave empty
+# to disable OpenID 2.0 access or enable the local openid service and point it
+# there to allow username/password logins for the web interfaces. Typically
+# with a value like
+# https://%(mig_oid_fqdn)s:%(mig_oid_port)d/openid/id/
+# or with the previously mentioned automatic proxy setup
+# https:///openid/id/
+# It is possible to have users login using the credentials they registered in
+# the local MiG-users.db
+user_mig_oid_title = MiG
+user_mig_oid_provider =
+# Optional OpenID provider alias to same account in dual-head setups.
+# When set account signup will result in a htaccess file with BOTH mig and alt
+# OpenID provider in allowed IDs, so that signup through one head doesn't
+# remove cert_redirect access through the other.
+#user_mig_oid_provider_alias =
+user_ext_oid_title = External
+user_ext_oid_provider =
+user_openid_providers =
+
+# Optional internal/external OpenID Connect identity provider(s) - leave empty
+# to disable openid connect access. Values should be the server meta URL here.
+#user_mig_oidc_title = MiG
+#user_mig_oidc_provider =
+user_ext_oidc_title = External
+user_ext_oidc_provider =
+user_openidconnect_providers =
+# OpenID Connect provider provides issuer and audience as part of the ID claim.
+# We use those fields and crypto signature to check authenticity of claims.
+#user_mig_oidc_issuer =
+#user_mig_oidc_audience = 1234abcd-12ab-34cd-ef56-ghijklmn7890
+user_ext_oidc_issuer =
+user_ext_oidc_audience =
+
+#user_mig_cert_title = MiG
+#user_ext_cert_title = Other
+
+logfile = mig.log
+loglevel = info
+peerfile = MiGpeers.conf
+sleep_period_for_empty_jobs = 120
+cputime_for_empty_jobs = 180
+min_seconds_between_live_update_requests = 60
+
+# Please note that order *does* matter for these lists!
+# First entry is default for resource creation cgi
+architectures = X86 AMD64 IA64 SPARC SPARC64 ITANIUM SUN4U SPARC-T1 SPARC-T2
+scriptlanguages = sh python
+jobtypes = batch interactive all
+lrmstypes = Native Native-execution-leader Batch Batch-execution-leader
+
+# Include any additional section confs files from a per-section conf folder
+include_sections = %(mig_server_home)s/MiGserver.d
+
+# Jupyter integration sections
+### E.g. ###
+# [JUPYTER_DAG]
+# service_name=dag
+# service_desc=This is an awesome service
+# service_hosts=https://192.168.1.10 https://hub002.com http://hub003.com
+###
+# During install.py the individual sections will be generated
+# in accordance with the jupyter_services parameter content
+# For each section a apache proxy balancer config is generated,
+# which will setup the target url location.
+# In the example provided, the system will generate a location called /dag
+# as defined by the 'service_name' in the /etc/httpd/conf.extras.d/MiG-jupyter-def.conf file
+
+
+# Cloud integration sections
+### E.g. ###
+# [CLOUD_MIST]
+# General cloud provider settings and limits
+# service_name=MIST
+# service_desc=This is an awesome service
+# service_provider_flavor = openstack
+# service_hosts = REST API URL
+# service_max_user_instances = 16
+# Semi-colon separated list of img=user login pairs when img and user differs
+# service_user_map = centos7=centos;centos8=centos;ubuntu-xenial=ubuntu
+# Cloud instance defaults
+# The general structure is a default option and an optional user override map
+# service_flavor_id = INSERT CLOUD FLAVOR ID
+# Semi-colon separated list of user=flavor pairs to override for some users
+# service_flavor_id_map =
+# service_network_id = INSERT CLOUD NETWORK ID
+# Semi-colon separated list of user=net pairs to override for some users
+# service_network_id_map =
+# service_key_id = INSERT DEFAULT KEY ID
+# Semi-colon separated list of user=keyid pairs to override for some users
+# service_key_id_map =
+# service_sec_group_id = INSERT CLOUD SEC GROUP ID
+# Semi-colon separated list of user=secgrp pairs to override for some users
+# service_sec_group_id_map
+# service_floating_network_id = INSERT CLOUD FLOATING NETWORK ID
+# Semi-colon separated list of user=floatnet pairs to override for some users
+# service_floating_network_id_map =
+# service_availability_zone = INSERT CLOUD AVAILABILITY ZONE
+# Semi-colon separated list of user=availzone pairs to override for some users
+# service_availability_zone_map =
+# Optional jump host so that instances are shielded fom direct ssh access
+# service_jumphost_address =
+# Semi-colon separated list of user=jumpaddr pairs to override for some users
+# service_jumphost_address_map =
+# service_jumphost_user = mist
+# Path to the ssh key used for managing user public keys on cloud jumphost
+# service_jumphost_key = ~/.ssh/cloud-jumphost-key
+# Semi-colon separated list of user=jumpuser pairs to override for some users
+# service_jumphost_user_map =
+# Helper to automatically add user pub keys on jumphost
+# The script and coding values are used like this under the hood:
+# ssh %(jumphost_user)s@%(jumphost_address)s %(jumphost_manage_keys_script)s add \
+# %(jumphost_manage_keys_coding)s %(encoded_client_id)s %(encoded_pub_keys)s
+# where coding is applied to client_id and pub_keys to yield encoded_X versions
+# service_jumphost_manage_keys_script = manage_mist_keys.py
+# service_jumphost_manage_keys_coding = base16
+###
+# During install.py the individual sections will be generated
+# in accordance with the cloud_services parameter content
+
+
+
+[SCHEDULER]
+# Scheduling algorithm to use
+# Currently supported: FIFO, FirstFit, BestFit, FairFit, Random and MaxThroughput
+algorithm = FairFit
+#
+# How long to keep jobs that can not be scheduled in the queue.
+# Jobs that stay 'expire_after' seconds in the queue can be expired by
+# the scheduler. Setting expire_after to 0 disables expiry.
+# 1 day: 86400 seconds
+# 7 days: 604800 seconds
+# 90 days: 7776000 seconds
+# 1 year: 31536000 seconds
+# 2 year: 63072000 seconds
+expire_after = 31536000
+
+job_retries = 2
+
+[MONITOR]
+sleep_secs = 120
+sleep_update_totals = 600
+slackperiod = 600
+
+[WORKFLOWS]
+# Workflow specific settings
+# Directory paths relative to an individual vgrid
+vgrid_patterns_home = .workflow_patterns_home/
+vgrid_recipes_home = .workflow_recipes_home/
+vgrid_tasks_home = .workflow_tasks_home/
+
+[SETTINGS]
+language = English
+submitui = fields textarea files
+
+[SCM]
+hg_path =
+hgweb_scripts =
+
+[TRACKER]
+trac_admin_path =
+# Note: We can't use mig_server_home from GLOBAL section here
+trac_ini_path =
+# IMPORTANT: Keep trac_id_field in sync with apache trac login section
+#trac_id_field = email
+
+[RESOURCES]
+default_mount_re = SSHFS-2.X-1
+
+[QUOTA]
+backend = lustre
+update_interval = 3600
+user_limit = 1099511627776
+vgrid_limit = 1099511627776
+
+[ACCOUNTING]
+update_interval = 3600
+
+[SITE]
+# Web site appearance
+# Whether to use Python 3 for all Python invocations
+prefer_python3 = False
+# Dynamic entry page to pick user default with fallback to site landing page
+autolaunch_page = /wsgi-bin/autolaunch.py
+# Entry page if not explictly provided or overriden by user
+landing_page = /wsgi-bin/home.py
+# Skin to style all pages with (taken from mig/images/skin/NAME)
+skin = migrid-basic
+# Which skin to style pages without theme with
+static_css = /images/skin/migrid-basic/core.css
+# Optional space separated list of extra javascripts to inject on user pages
+extra_userpage_scripts =
+# Optional space separated list of extra stylesheets to inject on user pages
+extra_userpage_styles =
+# Selectable base menus (simple, default or advanced to match X_menu options below)
+base_menu = default
+# Default sorted menu items to include
+#valid menu items are: home dashboard submitjob files jobs vgrids resources downloads runtimeenvs archives settings statistics docs people migadmin transfers sharelinks crontab seafile jupyter peers logout close
+default_menu = home files submitjob jobs vgrids resources runtimeenvs people settings downloads transfers sharelinks crontab docs logout
+#simple_menu = home files vgrids settings logout
+#advanced_menu = home files submitjob jobs vgrids resources runtimeenvs people settings downloads archives transfers sharelinks crontab docs logout
+# Additional sorted user selectable menu entries
+user_menu =
+# Selectable VGrid component links (default or advanced to match X_vgrid_links options below)
+collaboration_links = default advanced
+# VGrid component visibility and order - automatically tries auto detection if not set.
+default_vgrid_links = files web
+advanced_vgrid_links = files web scm tracker workflows monitor
+# VGrid label allows setting another name to use instead of VGrid
+vgrid_label = VGrid
+#script_deps = jquery.js jquery.contextmenu.js jquery.contextmenu.css jquery.form.js jquery.prettyprint.js jquery.tablesorter.js jquery.tablesorter.pager.js jquery-ui.js jquery-ui.css jquery-ui-theme.css jquery-ui-theme.custom.css jquery.calendar-widget.js jquery.calculator.js jquery.calculator.css jquery.countdown.js jquery.countdown.css jquery.epiclock.js jquery.epiclock.css jquery.jgcharts.js jquery.sparkline.js jquery.form.wizard.js
+#default_css = /images/default.css
+fav_icon = /images/skin/migrid-basic/favicon.ico
+title = Minimum intrusion Grid
+short_title = MiG
+# Optional external help url e.g. used as Help in the V3 user menu
+external_doc = https://sourceforge.net/p/migrid/wiki
+# Enable web-based site administration for admin_list users
+enable_migadmin = False
+# Further restrictions for admin_list users to view and act through migadmin
+# where the default is ANY for no extra restrictions and a space separated list
+# of the allowed values (cert, oid and oidc) can be given to limit to those.
+#migadmin_view_access = ANY
+#migadmin_act_access = ANY
+# Enable strict access control and logs for compliance with the General Data
+# Protection Regulation (GDPR) imposed by the EU. You probably want this if
+# and only if your users need to store sensitive/personal data. More info at
+# https://en.wikipedia.org/wiki/General_Data_Protection_Regulation
+enable_gdp = False
+# Enable user job execution on any associated compute resources
+enable_jobs = False
+# Enable execution and storage resources for vgrids
+enable_resources = False
+# Enable that workflows are available
+enable_workflows = False
+# Enable vgrid workflow triggers for file system events
+enable_events = False
+# Enable efficient I/O daemons - sftp, ftps and webdavs
+# Pure Python Paramiko-based sftp daemon
+enable_sftp = False
+# OpenSSH sftp daemon with just the Paramiko fs layer as subsys handler
+enable_sftp_subsys = False
+# Pure Python WsgiDAV-based webdav(s) daemon
+enable_davs = False
+# Allow sub-optimal but still relatively strong legacy TLS support in WebDAVS
+# NOTE: Python-2.7+ ssl supports TLSv1.2+ with strong ciphers and all popular
+# clients (including Windows 10+ native WebDAVS) also work with those.
+#enable_davs_legacy_tls = False
+# Pure Python pyftpdlib-based ftp(s) daemon
+enable_ftps = False
+# Allow sub-optimal but still relatively strong legacy TLS supports in FTPS
+# NOTE: Modern PyOpenSSL supports TLSv1.2+ with strong ciphers and all popular
+# clients also work with those.
+#enable_ftps_legacy_tls = False
+# Enable WSGI served web pages (faster than CGI) - requires apache wsgi module
+enable_wsgi = True
+# Enable system notify helper used e.g. to warn about failed user logins
+enable_notify = False
+# Enable IM notify helper - additionally requires configuration above
+enable_imnotify = False
+# Enable users to schedule tasks with a cron/at-like interface
+enable_crontab = True
+# Enable janitor service to handle recurring tasks like clean up and cache updates
+enable_janitor = False
+# Enable 2FA for web access and IO services with any TOTP authenticator client
+# IMPORTANT: Do NOT change this option manually here (requires Apache changes)!
+# use generateconfs.py --enable_twofactor=True|False
+enable_twofactor = True
+# Always require twofactor authentication for one or more protocols.
+twofactor_mandatory_protos =
+# Require logins to come from already active 2FA session IP address
+# if user has enabled 2FA for them.
+# IMPORTANT: Do NOT change this option manually here (requires Apache changes)!
+# use generateconfs.py --enable_twofactor_strict_address=True|False
+twofactor_strict_address = False
+# Which 2FA authenticator apps to mention on setup wizard. A space-separated
+# list of names from: bitwarden, freeotp, google, microfocus, microsoft, yubico
+twofactor_auth_apps =
+# Enable Peers system for site-local users to vouch for external users
+enable_peers = False
+# Whether external user requests must explicitly specify their sponsor (Peers)
+peers_mandatory = False
+# Explicit fields to request on external user sign up forms (full_name, email)
+peers_explicit_fields =
+# Short description of whom to point to as contact(s) in the Peers system
+peers_contact_hint = employed here and authorized to invite external users
+# Enable OpenID daemon for web access with user/pw from local user DB
+enable_openid = False
+# Allow sub-optimal but still relatively strong legacy TLS support in OpenID 2.0
+# NOTE: Python-2.7+ ssl supports TLSv1.2+ with strong ciphers and all popular
+# clients also work with those.
+#enable_openid_legacy_tls = False
+# Enable share links for easy external exchange of data with anyone
+enable_sharelinks = True
+# Enable storage quota
+enable_quota = False
+# Enable storage accounting
+enable_accounting = False
+# Enable background data transfers daemon - requires lftp and rsync
+enable_transfers = False
+# Explicit background transfer source addresses for use in pub key restrictions
+# It may be necessary to set it to match the FQDN of the default outgoing NIC
+transfers_from =
+# Custom per-user overall transfer log location for shared fs sites
+#transfer_log = transfer.log
+# Enable freeze archive handlers - support for write-once archiving of files
+# for e.g. the data associated with a research paper.
+enable_freeze = True
+# Which frozen archive flavors can be deleted (True for all, False or empty for
+# none and a space-separated list of flavors for individual control.
+permanent_freeze = no
+# The Distinguished Name of freeze administrators who can always delete their
+# archives no matter what permanent_freeze says, useful after testing.
+# (comma-separated list with optional leading and trailing spaces)
+#freeze_admins =
+# Delay before frozen archives are expected to hit tape (e.g. 5m, 4d or 2w).
+# Leave unset or empty if no tape archiving is available.
+freeze_to_tape =
+# Enable Jupyter integration - requires a remote Jupyter server configured to
+# allow our users to connect and then integrates mount of user home there
+enable_jupyter = False
+# Enable cloud integration - requires a remote OpenStack server configured to
+# allow our users to connect and then integrates mount of user home there
+enable_cloud = False
+# Enable Seafile synchronization service - requires local Seafile install
+enable_seafile = False
+# Enable Duplicati user computer backup integration
+enable_duplicati = False
+# Enable gravatar.com integration for user profile avatars
+enable_gravatars = False
+# Enable dynamic site status integration particularly in UI V3
+enable_sitestatus = True
+# Where to find json-formatted list of site events for dynamic site status
+# NOTE: either create this file or symlink to the included one.
+#status_events = /public/status-events.json
+# Include status events with system set to one of these (ANY disables filter)
+status_system_match = ANY
+# Enable legacy grid.dk features
+#enable_griddk = False
+# Whether to enforce automatic IO protocol access expiry after weeks of web
+# inactivity. When enabled users have to do a web log in once in a while to
+# preserve full SFTP, WebDAVS and FTPS service access.
+io_account_expire = False
+# User interfaces for users to select with first as default (allowed: V2, V3)
+user_interface = V3 V2
+# For gradual transition to new user interface set default here for new users
+#new_user_default_ui =
+# Security scanners to let scan e.g. for common logins without notify on errors
+security_scanners = UNSET
+# Cross Site Request Forgery protection level (MINIMAL, WARN, MEDIUM or FULL).
+# Where MINIMAL only requires a POST on changes, FULL additionally requires
+# CSRF tokens for all such operations, and MEDIUM likewise requires CSRF tokens
+# but with the exception that legacy user script and xmlrpc clients are allowed
+# access without. The default will likely change to FULL in the future when all
+# clients are ready. The transitional WARN mode basically enforces MINIMAL but
+# checks and logs all CSRF failures like FULL.
+csrf_protection = MEDIUM
+# Password strength policy (NONE, WEAK, MEDIUM, HIGH, MODERN:L or CUSTOM:L:C)
+# for all password-enabled services, e.g. sftp, webdavs, ftps and openid.
+# Where NONE is the legacy behavior of no explicit length or character class
+# checks - except safeinput min len and optionally any cracklib requirements
+# if enabled. The other plain names require increasing strength in terms of
+# length and number of different character classes included. MODERN:L leaves
+# the outdated focus on character classes behind and only requires longer
+# passwords of any L characters, and recommends multi-factor auth and cracklib
+# enforcement for added security. The CUSTOM:L:C version offers complete
+# control over the required length (L) and number of character classes (C).
+password_policy = MEDIUM
+# Since the password_policy is used both in password selection and during
+# actual log in, it may be necessary to allow old passwords ONLY for log in
+# until all passwords have been changed to fit a new policy.
+# The optional password_legacy_policy can be set to the old policy for that
+# purpose, and otherwise defaults to disabled.
+password_legacy_policy =
+# Optional additional guard against simple passwords with the cracklib library
+password_cracklib = False
+# Optional prefilter on users who may sign up as site users with sign up forms.
+# Used as a coarse filter to reject invalid user requests early by only
+# filtering on form values (organization, email). E.g. to avoid internal users
+# signing up as externals. Space separated list of user field and regexp-filter
+# pattern pairs separated by colons.
+signup_prefilter = email:.*
+# Optional prefilter on users who may potenially invite peers as site users.
+# Used as a coarse filter to reject clearly invalid user requests early by only
+# filtering on form values (peers_full_name and peers_email). Space separated
+# list of user field and regexp-filter pattern pairs separated by colons.
+peers_prefilter = peers_email:.*
+# Optional limit on users who may invite peers as site users. Space separated
+# list of user field and regexp-filter pattern pairs separated by colons.
+peers_permit = distinguished_name:.*
+# Optional html banner on Peers page to inform e.g. about access restrictions
+#peers_notice =
+# Optional limit on users who can create vgrids. Space separated list of user
+# field and regexp-filter pattern pairs separated by colons.
+vgrid_creators = distinguished_name:.*
+# Optional limit on users who can manage vgrids. Space separated list of user
+# field and regexp-filter pattern pairs separated by colons.
+vgrid_managers = distinguished_name:.*
+# Space separated list of methods to include on the signup page: default is
+# extcert only and order is used on the signup page
+signup_methods = extcert
+# Space separated list of methods to include on the login page: default is same
+# as signup_methods and order is used on login page and various other pages
+# presenting the users with one or more possible https urls.
+login_methods = extcert
+# Extra note displayed during sign up
+#signup_hint =
+# Digest authentication hex salt for scrambling saved digest credentials
+# IMPORTANT: digest credentials need to be saved again if this is changed
+# Can be a plain string, a path to a file or an environment value and the
+# content must be a string of e.g. 32 hex characters. If two FILE values are
+# given the value is read from the 2nd (cache) file if available and read from
+# first (persistent) file path and saved to cache path otherwise. Useful e.g.
+# with a tmpfs cache.
+#digest_salt = 084528A93A4E0A40905609A729394F5C
+#digest_salt = FILE::/path/to/digest-salt.hex
+#digest_salt = FILE::/path/to/persistent-digest-salt.hex$$/path/to/cached-digest-salt.hex
+#digest_salt = ENV::DIGEST_SALT
+digest_salt = DDDD12344321DDDD
+# Optional crypto helper salt used to protect data stored on disk or in logs
+# Can be a plain string, a path to a file or an environment value and the
+# content must be a string of e.g. 32 hex characters. If two FILE values are
+# given the value is read from the 2nd (cache) file if available and read from
+# first (persistent) file path and saved to cache path otherwise. Useful e.g.
+# with a tmpfs cache.
+#crypto_salt = 280845A93A4E0A40905609A7294F5C39
+#crypto_salt = FILE::/path/to/crypto-salt.hex
+#crypto_salt = FILE::/path/to/persistent-crypto-salt.hex$$/path/to/cached-crypto-salt.hex
+#crypto_salt = ENV::CRYPTO_SALT
+crypto_salt = CCCC12344321CCCC
+# Optional software catalogue from grid.dk
+#swrepo_url = /software-repository/
+# Use left logo from skin and default center text for top banner
+logo_left = /images/skin/migrid-basic/logo-left.png
+logo_center = MiG
+# Uncomment to also enable right logo from skin in top banner
+logo_right = /images/skin/migrid-basic/logo-right.png
+#support_text = Support & Questions
+#privacy_text =
+#credits_text = 2003-2023, The MiG Project
+#credits_image = /images/copyright.png
+# Optional data safety notice and popup on Files page
+datasafety_link =
+datasafety_text =
+
+[TEMPLATES]
+base_packages = testplugin
+cache_dir = ./envhelp/output/__test_template_cache
diff --git a/tests/data/testplugin/__init__.py b/tests/data/testplugin/__init__.py
new file mode 100644
index 000000000..0e76234df
--- /dev/null
+++ b/tests/data/testplugin/__init__.py
@@ -0,0 +1,59 @@
+# -*- coding: utf-8 -*-
+#
+# --- BEGIN_HEADER ---
+#
+# tests/data/testplugin - demonstration of a route plugin for use by tests
+# Copyright (C) 2003-2026 The MiG Project by the Science HPC Center at UCPH
+#
+# This file is part of MiG.
+#
+# MiG is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# MiG is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301,
+# USA.
+#
+# -- END_HEADER ---
+#
+
+
+"""
+Demonstration of a route plugin for use in tests.
+"""
+
+
+def tests_data__GET_testpluginendpoint(request_info, data):
+ """
+ Request handler: GET /testpluginendpoint
+ """
+
+ greeting = request_info._arg_string("greeting", "")
+
+ return {
+ "template_args": {
+ "greeting": greeting,
+ },
+ "template_name": "test_something",
+ }
+
+
+TEMPLATE_PACKAGES = [
+ "testplugin",
+ "testplugin.inner",
+]
+
+
+TEMPLATE_ROUTES = {
+ "GET /testpluginendpoint": {
+ "generate_args": tests_data__GET_testpluginendpoint,
+ }
+}
diff --git a/tests/data/testplugin/inner/templates/inner_template.html.jinja b/tests/data/testplugin/inner/templates/inner_template.html.jinja
new file mode 100644
index 000000000..8772116ce
--- /dev/null
+++ b/tests/data/testplugin/inner/templates/inner_template.html.jinja
@@ -0,0 +1 @@
+{{ inner_variable }}
diff --git a/tests/data/testplugin/templates/test_empty.html.jinja b/tests/data/testplugin/templates/test_empty.html.jinja
new file mode 100644
index 000000000..efdb63874
--- /dev/null
+++ b/tests/data/testplugin/templates/test_empty.html.jinja
@@ -0,0 +1 @@
+noarrrrgs
diff --git a/tests/data/testplugin/templates/test_other.html.jinja b/tests/data/testplugin/templates/test_other.html.jinja
new file mode 100644
index 000000000..f22ed0101
--- /dev/null
+++ b/tests/data/testplugin/templates/test_other.html.jinja
@@ -0,0 +1 @@
+{{ other }}
diff --git a/tests/data/testplugin/templates/test_something.html.jinja b/tests/data/testplugin/templates/test_something.html.jinja
new file mode 100644
index 000000000..96373154e
--- /dev/null
+++ b/tests/data/testplugin/templates/test_something.html.jinja
@@ -0,0 +1 @@
+
diff --git a/tests/fixture/confs-stdlocal/MiGserver.conf b/tests/fixture/confs-stdlocal/MiGserver.conf
index c2ed5dd01..e9be20de0 100644
--- a/tests/fixture/confs-stdlocal/MiGserver.conf
+++ b/tests/fixture/confs-stdlocal/MiGserver.conf
@@ -785,3 +785,7 @@ logo_right = /images/skin/migrid-basic/logo-right.png
# Optional data safety notice and popup on Files page
datasafety_link =
datasafety_text =
+
+[TEMPLATES]
+base_packages =
+cache_dir = AUTO
diff --git a/tests/snapshots/test_objects_with_type_template.html b/tests/snapshots/test_objects_with_type_template.html
new file mode 100644
index 000000000..8eaf5bde5
--- /dev/null
+++ b/tests/snapshots/test_objects_with_type_template.html
@@ -0,0 +1 @@
+
\ No newline at end of file
diff --git a/tests/snapshots/test_objects_with_type_template_without_args.html b/tests/snapshots/test_objects_with_type_template_without_args.html
new file mode 100644
index 000000000..d0b5a6c00
--- /dev/null
+++ b/tests/snapshots/test_objects_with_type_template_without_args.html
@@ -0,0 +1 @@
+noarrrrgs
\ No newline at end of file
diff --git a/tests/support/__init__.py b/tests/support/__init__.py
index 9b52ab8ad..f91f4e0a5 100644
--- a/tests/support/__init__.py
+++ b/tests/support/__init__.py
@@ -284,9 +284,9 @@ def assertDirEmpty(self, relative_path):
entries = os.listdir(absolute_path)
assert not entries, "directory is not empty"
- def assertDirNotEmpty(self, relative_path):
+ def assertDirNotEmpty(self, relative_path, allow_anypath=False):
"""Make sure the supplied path is a non-empty directory"""
- path_kind = self.assertPathExists(relative_path)
+ path_kind = self.assertPathExists(relative_path, allow_anypath=allow_anypath)
assert path_kind == "dir", "expected a directory but found %s" % (
path_kind, )
absolute_path = os.path.join(TEST_OUTPUT_DIR, relative_path)
@@ -320,13 +320,15 @@ def assertFileExists(self, relative_path):
path_kind, )
return os.path.join(TEST_OUTPUT_DIR, relative_path)
- def assertPathExists(self, relative_path):
+ def assertPathExists(self, relative_path, allow_anypath=False):
"""Make sure file in relative_path exists"""
- if os.path.isabs(relative_path):
- self.assertPathWithin(relative_path, start=TEST_OUTPUT_DIR)
+ if not os.path.isabs(relative_path):
+ absolute_path = os.path.join(TEST_OUTPUT_DIR, relative_path)
+ elif allow_anypath:
absolute_path = relative_path
else:
- absolute_path = os.path.join(TEST_OUTPUT_DIR, relative_path)
+ self.assertPathWithin(relative_path, start=TEST_OUTPUT_DIR)
+ absolute_path = relative_path
return MigTestCase._absolute_path_kind(absolute_path)
def assertLogs(self, name=None, level=None):
diff --git a/tests/support/snapshotsupp.py b/tests/support/snapshotsupp.py
index 355095814..2af2d7810 100644
--- a/tests/support/snapshotsupp.py
+++ b/tests/support/snapshotsupp.py
@@ -67,6 +67,43 @@ def _html_content_only(value):
return value[content_start_index:content_end_index].strip()
+def _html_fragment_only(value):
+ """For a given HTML input extract only the portion that corresponds to the
+ page content. This is somewhat convoluted due to having to work around an
+ inability to move the comment markers to enclose only the content.
+ """
+
+ content_length = len(value)
+
+ first_tag_index = value.find('<')
+
+ first_tag_name = ""
+ first_tag_index_close = first_tag_index + 1
+ while first_tag_index_close < content_length:
+ character = value[first_tag_index_close]
+ if character == '>':
+ break
+ first_tag_name += character
+ first_tag_index_close += 1
+
+ # remove any attributes from the first found tag
+ try:
+ first_tag_name = first_tag_name[0:first_tag_name.index(' ')]
+ except ValueError:
+ pass
+ assert first_tag_name, "fragment did not begin with a valid tag"
+ assert first_tag_index_close != content_length, "fragment is unclosed"
+
+ first_tag_closing = "%s>" % (first_tag_name,)
+ after_first_tag_closing_index = value.rfind(first_tag_closing) + len(first_tag_closing)
+
+ trailing_characters_if_any = value[after_first_tag_closing_index:content_length]
+
+ assert trailing_characters_if_any.rstrip() == '', "fragment did not end with its opening tag"
+
+ return value
+
+
def _delimited_lines(value):
"""Break a value by newlines into lines suitable for diffing."""
@@ -129,8 +166,10 @@ def _snapshotsupp_compare_snapshot(self, extension, actual_content):
'expected',
'actual'
)
+
+ display_snapshot_file = os.path.relpath(file_path, TEST_SNAPSHOTS_DIR)
raise AssertionError(
- "content did not match snapshot\n\n%s" % (''.join(udiff),))
+ "content did not match snapshot file: %s\n\n%s" % (display_snapshot_file, ''.join(udiff),))
def assertSnapshot(self, actual_content, extension=None):
"""Load a snapshot corresponding to the named test and check that what
@@ -141,11 +180,14 @@ def assertSnapshot(self, actual_content, extension=None):
self._snapshotsupp_compare_snapshot(extension, actual_content)
- def assertSnapshotOfHtmlContent(self, actual_content):
+ def assertSnapshotOfHtmlContent(self, actual_content, is_fragment=False):
"""Load a snapshot corresponding to the named test and check that what
it contains, which is the expectation, matches against the portion of
what was actually given that corresponds to the output HTML content.
"""
- actual_content = _html_content_only(actual_content)
+ if is_fragment:
+ actual_content = _html_fragment_only(actual_content)
+ else:
+ actual_content = _html_content_only(actual_content)
self._snapshotsupp_compare_snapshot('html', actual_content)
diff --git a/tests/test_mig_install_generateconfs.py b/tests/test_mig_install_generateconfs.py
index e8ed90241..8acd0eaca 100644
--- a/tests/test_mig_install_generateconfs.py
+++ b/tests/test_mig_install_generateconfs.py
@@ -33,7 +33,13 @@
import os
import sys
-from tests.support import MIG_BASE, MigTestCase, testmain, cleanpath
+from tests.support import MIG_BASE, TEST_OUTPUT_DIR, \
+ MigTestCase, testmain, cleanpath
+
+
+DUMMY_CACHE_DIR = os.path.join(
+ MIG_BASE, "envhelp", "output", "__test_template_cache"
+)
def _import_generateconfs():
@@ -99,6 +105,38 @@ def test_option_storage_protocols(self):
self.assertIn('storage_protocols', settings)
self.assertEqual(settings['storage_protocols'], 'proto1 proto2 proto3')
+ def test_option_templates_base_package(self):
+ expected_generated_dir = cleanpath('confs-stdlocal', self,
+ ensure_dir=True)
+ with open(os.path.join(expected_generated_dir, "instructions.txt"),
+ "w"):
+ pass
+ fake_generate_confs = create_fake_generate_confs(
+ dict(destination_dir=expected_generated_dir))
+ test_arguments = ['--templates_base_package', 'pkg1,pkg2,pkg3']
+
+ exit_code = main(
+ test_arguments, _generate_confs=fake_generate_confs, _print=noop)
+ settings = fake_generate_confs.settings
+ self.assertIn('templates_base_package', settings)
+ self.assertEqual(settings['templates_base_package'], 'pkg1,pkg2,pkg3')
+
+ def test_option_templates_cache_dir(self):
+ expected_generated_dir = cleanpath('confs-stdlocal', self,
+ ensure_dir=True)
+ with open(os.path.join(expected_generated_dir, "instructions.txt"),
+ "w"):
+ pass
+ fake_generate_confs = create_fake_generate_confs(
+ dict(destination_dir=expected_generated_dir))
+ test_arguments = ['--templates_cache_dir', DUMMY_CACHE_DIR]
+
+ exit_code = main(
+ test_arguments, _generate_confs=fake_generate_confs, _print=noop)
+ settings = fake_generate_confs.settings
+ self.assertIn('templates_cache_dir', settings)
+ self.assertEqual(settings['templates_cache_dir'], DUMMY_CACHE_DIR)
+
def test_option_wwwserve_max_bytes(self):
expected_generated_dir = cleanpath('confs-stdlocal', self,
ensure_dir=True)
diff --git a/tests/test_mig_lib_templates.py b/tests/test_mig_lib_templates.py
new file mode 100644
index 000000000..721684199
--- /dev/null
+++ b/tests/test_mig_lib_templates.py
@@ -0,0 +1,218 @@
+# -*- coding: utf-8 -*-
+#
+# --- BEGIN_HEADER ---
+#
+# test_mig_lib_templates - unit tests of core templates logic
+# Copyright (C) 2003-2026 The MiG Project by the Science HPC Center at UCPH
+#
+# This file is part of MiG.
+#
+# MiG is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# MiG is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301,
+# USA.
+#
+# --- END_HEADER ---
+#
+
+import importlib
+import os
+import shutil
+import sys
+from types import SimpleNamespace
+
+from jinja2 import Template
+
+from mig.lib.templates import (
+ TemplateStore,
+ UnknownTemplateError,
+ init_global_templates,
+)
+from mig.shared.conf import get_configuration_object
+from tests.support import (
+ MIG_BASE,
+ TEST_DATA_DIR,
+ TEST_OUTPUT_DIR,
+ MigTestCase,
+ testmain,
+)
+
+DUMMY_CACHE_DIR = os.path.join(
+ MIG_BASE, "envhelp", "output", "__test_template_cache"
+)
+TEST_BASE_PACKAGES = ["testplugin"]
+TEST_CACHE_DIR = os.path.join(TEST_OUTPUT_DIR, "__template_cache__")
+
+
+def noop(*args):
+ pass
+
+
+class TestMigSharedTemplates_instance(MigTestCase):
+
+ def before_each(self):
+ # make template cache directory
+ os.makedirs(TEST_CACHE_DIR)
+
+ # allow the dummy plugin to be loaded
+ sys.path.append(TEST_DATA_DIR)
+ self._register_check(lambda: sys.path.pop())
+
+ def _provide_configuration(self):
+ return "testconfig"
+
+ def test_creation_of_a_template_store(self):
+ store = TemplateStore.from_names(
+ TEST_BASE_PACKAGES, cache_dir=TEST_CACHE_DIR
+ )
+ self.assertIsInstance(store, TemplateStore)
+
+ def test_creation_of_a_template_store_should_expand_packages(self):
+ store = TemplateStore.from_names(
+ TEST_BASE_PACKAGES, cache_dir=TEST_CACHE_DIR
+ )
+ self.assertEqual(store._packages, ["testplugin", "testplugin.inner"])
+
+ def test_grab_template(self):
+ store = TemplateStore.from_names(
+ TEST_BASE_PACKAGES, cache_dir=TEST_CACHE_DIR
+ )
+ template = store.grab_template(
+ "inner_template", "testplugin.inner", "html"
+ )
+ self.assertIsInstance(template, Template)
+
+ def test_extract_variables(self):
+ store = TemplateStore.from_names(
+ TEST_BASE_PACKAGES, cache_dir=TEST_CACHE_DIR
+ )
+ template_vars = store.extract_variables(
+ "inner_template", "testplugin.inner", "html"
+ )
+ self.assertEqual(template_vars, set(["inner_variable"]))
+
+ def test_extract_variables_empty(self):
+ store = TemplateStore.from_names(
+ TEST_BASE_PACKAGES, cache_dir=TEST_CACHE_DIR
+ )
+ template_vars = store.extract_variables(
+ "test_empty", "testplugin", "html"
+ )
+ self.assertEqual(template_vars, set())
+
+ def test_extract_variables_umprimed(self):
+ store = TemplateStore(TEST_BASE_PACKAGES, cache_dir=TEST_CACHE_DIR)
+
+ with self.assertRaises(UnknownTemplateError) as raised:
+ store.extract_variables(
+ "inner_template", "testplugin.inner", "html"
+ )
+ theexception = raised.exception
+ self.assertEqual(str(theexception), "'testplugin.inner.*'")
+
+
+class TestMigSharedTemplates_instance_with_configuration(MigTestCase):
+
+ def test_default_template_cache_location(self):
+ test_conf_file = os.path.join(
+ TEST_DATA_DIR, "MiGserver--empty_templates.conf"
+ )
+ configuration = get_configuration_object(
+ test_conf_file, skip_log=True, disable_auth_log=True
+ )
+
+ store = init_global_templates(configuration)
+
+ self.assertEqual(
+ store.cache_dir, "/test/path/state/mig_system_run/templates"
+ )
+
+ def test_default_packages_is_empty(self):
+ test_conf_file = os.path.join(
+ TEST_DATA_DIR, "MiGserver--empty_templates.conf"
+ )
+ configuration = get_configuration_object(
+ test_conf_file, skip_log=True, disable_auth_log=True
+ )
+
+ store = init_global_templates(configuration)
+
+ self.assertEqual(store._packages, [])
+
+ def test_specified_base_packages(self):
+ test_conf_file = os.path.join(
+ TEST_DATA_DIR, "MiGserver--templates.conf"
+ )
+ configuration = get_configuration_object(
+ test_conf_file, skip_log=True, disable_auth_log=True
+ )
+
+ store = init_global_templates(configuration)
+
+ self.assertEqual(
+ store.list_templates(),
+ [
+ ("test_empty", "testplugin"),
+ ("test_other", "testplugin"),
+ ("test_something", "testplugin"),
+ ("inner_template", "testplugin.inner"),
+ ],
+ )
+
+ def test_specified_cache_dir(self):
+ test_conf_file = os.path.join(
+ TEST_DATA_DIR, "MiGserver--templates.conf"
+ )
+ configuration = get_configuration_object(
+ test_conf_file, skip_log=True, disable_auth_log=True
+ )
+
+ store = init_global_templates(configuration)
+
+ expected_cache_dir = DUMMY_CACHE_DIR
+ self.assertEqual(store.cache_dir, expected_cache_dir)
+
+
+class TestMigSharedTemplates_cli(MigTestCase):
+
+ TEMPLATES_CLI = importlib.import_module("mig.lib.templates.__main__")
+
+ def before_each(self):
+ self.test_conf_file = os.path.join(
+ TEST_DATA_DIR, "MiGserver--templates.conf"
+ )
+
+ # allow the dummy plugin to be loaded
+ sys.path.append(TEST_DATA_DIR)
+ self._register_check(lambda: sys.path.pop())
+
+ def after_each(self):
+ # clean up the configuration file specified cache directory
+ shutil.rmtree(DUMMY_CACHE_DIR, ignore_errors=True)
+
+ def test_command_cache(self):
+ args = SimpleNamespace(config_file=self.test_conf_file, command="cache")
+ last_printed_line = None
+
+ def _print(value):
+ nonlocal last_printed_line
+ last_printed_line = value
+
+ self.TEMPLATES_CLI.main(args, _print=_print)
+
+ expected_cache_dir = DUMMY_CACHE_DIR
+ self.assertEqual(last_printed_line, expected_cache_dir)
+
+
+if __name__ == "__main__":
+ testmain()
diff --git a/tests/test_mig_shared_conf.py b/tests/test_mig_shared_conf.py
index 19eba4c85..e6f7c4104 100644
--- a/tests/test_mig_shared_conf.py
+++ b/tests/test_mig_shared_conf.py
@@ -2,7 +2,7 @@
#
# --- BEGIN_HEADER ---
#
-# test_mig_shared_configuration - unit test of configuration
+# test_mig_shared_conf - unit test of conf
# Copyright (C) 2003-2026 The MiG Project by the Science HPC Center at UCPH
#
# This file is part of MiG.
@@ -27,28 +27,25 @@
"""Unit tests for shared conf"""
-import inspect
-import os
-import unittest
-
-from tests.support import MigTestCase, TEST_DATA_DIR, PY2, testmain
-from tests.support.fixturesupp import FixtureAssertMixin
-
-from mig.shared.conf import Configuration, \
- RuntimeConfiguration, \
- get_configuration_object
+from mig.shared.conf import (
+ Configuration,
+ RuntimeConfiguration,
+ get_configuration_object,
+)
+from tests.support import MigTestCase, testmain
class MigSharedConf(MigTestCase):
"""Coverage of module methods."""
def test_get_configuration_object_returns_runtime_configuration(self):
- configuration = get_configuration_object(skip_log=True,
- disable_auth_log=True)
+ configuration = get_configuration_object(
+ skip_log=True, disable_auth_log=True
+ )
self.assertIsInstance(configuration, RuntimeConfiguration)
static_configuration = configuration._configuration
self.assertIsInstance(static_configuration, Configuration)
-if __name__ == '__main__':
+if __name__ == "__main__":
testmain()
diff --git a/tests/test_mig_shared_configuration.py b/tests/test_mig_shared_configuration.py
index 56eb98b8f..0d0ea9960 100644
--- a/tests/test_mig_shared_configuration.py
+++ b/tests/test_mig_shared_configuration.py
@@ -3,7 +3,7 @@
# --- BEGIN_HEADER ---
#
# test_mig_shared_configuration - unit test of configuration
-# Copyright (C) 2003-2025 The MiG Project by the Science HPC Center at UCPH
+# Copyright (C) 2003-2026 The MiG Project by the Science HPC Center at UCPH
#
# This file is part of MiG.
#
@@ -31,12 +31,17 @@
import os
import unittest
-from tests.support import MigTestCase, TEST_DATA_DIR, PY2, testmain
+from tests.support import MigTestCase, testmain, \
+ MIG_BASE, TEST_OUTPUT_DIR, TEST_DATA_DIR
from tests.support.fixturesupp import FixtureAssertMixin
-from mig.shared.configuration import Configuration, \
+from mig.shared.configuration import Configuration, fix_missing, \
_CONFIGURATION_ARGUMENTS, _CONFIGURATION_PROPERTIES
+DUMMY_CACHE_DIR = os.path.join(
+ MIG_BASE, "envhelp", "output", "__test_template_cache"
+)
+
class MigSharedConfiguration__static_definitions(MigTestCase):
"""Coverage of the static definitions underlying Configuration objects."""
@@ -49,6 +54,66 @@ def test_consistent_parameters(self):
"configuration defaults do not match arguments")
+class MigSharedConfiguration__incomplete_configurations(MigTestCase):
+ """Coverage of loaded Configuration instances."""
+
+ SUBSTITUTED_PROPERTIES = [
+ 'server_fqdn',
+ 'admin_email',
+ 'migserver_http_url',
+ 'mig_server_id',
+ 'smtp_server',
+ 'user_sftp_address',
+ 'user_sftp_subsys_address',
+ 'user_davs_address',
+ 'user_ftps_address',
+ 'user_openid_address',
+ ]
+
+ def test_fix_missing_completes_an_empty_file(self):
+ conf_file = os.path.join(TEST_OUTPUT_DIR, "empty.conf")
+ open(conf_file, 'w').close()
+
+ def noop(*args):
+ pass
+
+ fix_missing(conf_file, print=noop)
+
+ # check it is now a valid configuration
+ try:
+ Configuration(conf_file, skip_log=True, disable_auth_log=True)
+ except Exception as exc:
+ self.assertFalse(True, 'should not be reached')
+
+ def test_fix_missing_performs_substitutions(self):
+ conf_file = os.path.join(TEST_OUTPUT_DIR, "empty.conf")
+ open(conf_file, 'w').close()
+
+ def noop(*args):
+ pass
+
+ fix_missing(conf_file, user='testuser', fqdn='testhost', print=noop)
+ fixed_configuration = Configuration(
+ conf_file, skip_log=True, disable_auth_log=True)
+
+ # check the substitutions were made correctly
+ only_substituted_properties = {attr: getattr(fixed_configuration, attr)
+ for attr in self.SUBSTITUTED_PROPERTIES}
+ admin_email = only_substituted_properties.pop('admin_email')
+ admin_email.endswith('@localhost')
+ self.assertEqual(only_substituted_properties, {
+ 'mig_server_id': 'testhost.0',
+ 'migserver_http_url': 'http://testhost',
+ 'server_fqdn': 'testhost',
+ 'smtp_server': 'testhost',
+ 'user_davs_address': 'testhost',
+ 'user_ftps_address': 'testhost',
+ 'user_openid_address': 'testhost',
+ 'user_sftp_address': 'testhost',
+ 'user_sftp_subsys_address': 'testhost',
+ })
+
+
class MigSharedConfiguration__loaded_configurations(MigTestCase):
"""Coverage of loaded Configuration instances."""
@@ -327,11 +392,36 @@ def test_argument_include_sections_multi_ignores_other_sections(self):
# TODO: rename file to valid section name we can check and enable next?
# self.assertEqual(configuration.multi, 'blabla')
+ def test_structured_templates_defaults(self):
+ test_conf_file = os.path.join(
+ TEST_DATA_DIR, 'MiGserver--empty_templates.conf')
+
+ configuration = Configuration(
+ test_conf_file, skip_log=True, disable_auth_log=True)
+
+ division = configuration.division(section_name='TEMPLATES')
+ self.assertEqual(division.__dict__, {
+ 'base_packages': [],
+ 'cache_dir': '/test/path/state/mig_system_run/templates',
+ })
+
+ def test_structured_templates_enabled(self):
+ test_conf_file = os.path.join(
+ TEST_DATA_DIR, 'MiGserver--templates.conf')
+
+ configuration = Configuration(
+ test_conf_file, skip_log=True, disable_auth_log=True)
+
+ division = configuration.division(section_name='TEMPLATES')
+ self.assertEqual(division.__dict__, {
+ 'base_packages': ['testplugin'],
+ 'cache_dir': DUMMY_CACHE_DIR,
+ })
+
class MigSharedConfiguration__new_instance(MigTestCase, FixtureAssertMixin):
"""Coverage of programatically created Configuration instances."""
- @unittest.skipIf(PY2, "Python 3 only")
def test_default_object(self):
prepared_fixture = self.prepareFixtureAssert(
'mig_shared_configuration--new', fixture_format='json')
diff --git a/tests/test_mig_wsgibin.py b/tests/test_mig_wsgibin.py
index 1d0f9ecdb..ff6b59c7f 100644
--- a/tests/test_mig_wsgibin.py
+++ b/tests/test_mig_wsgibin.py
@@ -28,22 +28,23 @@
"""Unit tests for the MiG WSGI glue"""
import codecs
+from configparser import ConfigParser
+from html.parser import HTMLParser
import importlib
import os
+import shutil
import stat
import sys
import unittest
-from configparser import ConfigParser
-from html.parser import HTMLParser
# Imports required for the unit test wrapping
-import mig.shared.returnvalues as returnvalues
from mig.shared.base import allow_script, brief_list, client_dir_id, \
client_id_dir, get_short_id, invisible_path
+import mig.shared.returnvalues as returnvalues
from mig.shared.compat import SimpleNamespace
# Imports required for the unit tests themselves
-from tests.support import MIG_BASE, MigTestCase, ensure_dirs_exist, \
- is_path_within, testmain
+from tests.support import MIG_BASE, TEST_DATA_DIR, TEST_OUTPUT_DIR, \
+ MigTestCase, testmain
from tests.support.snapshotsupp import SnapshotAssertMixin
from tests.support.wsgisupp import WsgiAssertMixin, prepare_wsgi
@@ -243,6 +244,9 @@ def test_objects_containing_only_title_matches_snapshot(self):
self.assertSnapshot(output, extension='html')
+TEST_TEMPLATE_CACHE_DIR = os.path.join(TEST_OUTPUT_DIR, '__template_cache__')
+
+
class MigWsgibin_output_objects(MigTestCase, WsgiAssertMixin,
SnapshotAssertMixin):
"""Unit tests for output_object related part of wsgi functions."""
@@ -250,6 +254,9 @@ class MigWsgibin_output_objects(MigTestCase, WsgiAssertMixin,
def _provide_configuration(self):
return 'testconfig'
+ def after_each(self):
+ shutil.rmtree(TEST_TEMPLATE_CACHE_DIR, ignore_errors=True)
+
def before_each(self):
self.fake_backend = FakeBackend()
self.fake_wsgi = prepare_wsgi(self.configuration, 'http://localhost/')
@@ -264,6 +271,28 @@ def before_each(self):
_set_os_environ=False,
)
+ def _arrange_use_of_testplugin(self):
+ templates_division = self.configuration.division('TEMPLATES')
+ # overwrite the template source and cache directory to known locations for
+ # the duration of the tests. this allows us to control which templates are
+ # available as well as ensures the template cache directory is cleaned out
+ # on test completion as part of the standard output directory cleanup
+ templates_division.__dict__.update({
+ 'base_packages': ['testplugin'],
+ 'cache_dir': TEST_TEMPLATE_CACHE_DIR,
+ })
+
+ # provision template directories
+ os.makedirs(TEST_TEMPLATE_CACHE_DIR)
+
+ # allow the dummy plugin to be loaded
+ sys.path.append(TEST_DATA_DIR)
+ self._register_check(lambda: sys.path.pop())
+
+ import testplugin
+ for package_name in testplugin.TEMPLATE_PACKAGES:
+ os.makedirs(os.path.join(TEST_TEMPLATE_CACHE_DIR, package_name))
+
def assertIsValidHtmlDocument(self, value):
parser = DocumentBasicsHtmlParser()
parser.feed(value)
@@ -289,7 +318,6 @@ def test_unknown_object_type_generates_valid_error_page(self):
def test_objects_with_type_text(self):
output_objects = [
- # workaround invalid HTML being generated with no title object
{
'object_type': 'title',
'text': 'TEST'
@@ -309,6 +337,48 @@ def test_objects_with_type_text(self):
output, _ = self.assertWsgiResponse(wsgi_result, self.fake_wsgi, 200)
self.assertSnapshotOfHtmlContent(output)
+ def test_objects_with_type_template(self):
+ output_objects = [
+ {
+ 'object_type': 'template',
+ 'template_name': 'test_something',
+ 'template_group': 'testplugin',
+ 'template_args': {
+ 'greeting': 'here!!'
+ }
+ }
+ ]
+ self.fake_backend.set_response(output_objects, returnvalues.OK)
+ self._arrange_use_of_testplugin()
+
+ wsgi_result = migwsgi.application(
+ *self.application_args,
+ **self.application_kwargs
+ )
+
+ output, _ = self.assertWsgiResponse(wsgi_result, self.fake_wsgi, 200)
+ self.assertSnapshotOfHtmlContent(output, is_fragment=True)
+
+ def test_objects_with_type_template_without_args(self):
+ output_objects = [
+ {
+ 'object_type': 'template',
+ 'template_name': 'test_empty',
+ 'template_group': 'testplugin',
+ 'template_args': {}
+ }
+ ]
+ self.fake_backend.set_response(output_objects, returnvalues.OK)
+ self._arrange_use_of_testplugin()
+
+ wsgi_result = migwsgi.application(
+ *self.application_args,
+ **self.application_kwargs
+ )
+
+ output, _ = self.assertWsgiResponse(wsgi_result, self.fake_wsgi, 200)
+ self.assertSnapshotOfHtmlContent(output, is_fragment=True)
+
class MigWsgibin_input_object(MigTestCase, WsgiAssertMixin,
SnapshotAssertMixin):