diff --git a/docs/data/transfer.md b/docs/data/transfer.md index 735560570..4be009ca6 100644 --- a/docs/data/transfer.md +++ b/docs/data/transfer.md @@ -292,6 +292,7 @@ When you no longer need the mounted remote directory, you **must** unmount your ``` diskutil umount ~/ulhpc ``` + !!! tip In some APT based systems (e.g. Debian and Ubuntu) the command to unmount the remote directory may be called `fusermount3`. diff --git a/mkdocs.yml b/mkdocs.yml index 381e586aa..f7d9eda1e 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -145,8 +145,8 @@ nav: - Utilities: 'software/swsets/tools.md' - Visualisation: 'software/swsets/vis.md' - Software Sets: - - 2019b: 'software/swsets/2019b.md' - - 2020a: 'software/swsets/2020b.md' + - 2019b: 'software/swsets/2019b.md' + - 2020a: 'software/swsets/2020b.md' ### END GENERATED CONTENT # - Software Set Versioning: 'software/swsets.md' - Compiling/building your own software: 'software/build.md' diff --git a/scripts/extract_module_metadata.py b/scripts/extract_module_metadata.py new file mode 100644 index 000000000..af4050e09 --- /dev/null +++ b/scripts/extract_module_metadata.py @@ -0,0 +1,307 @@ +import yaml +import argparse +import typing +import pandas as pd +import itertools +import functools +import pathlib +from easybuild.framework.easyconfig.parser import EasyConfigParser + +#----------------------------------------- +# Run only with Easybuild versions 5.3.x or above +# Add only software in "release" modules! Only one build number in release. + +def toolchain_to_maybe_string(toolchain): + if toolchain['name'] == "system": + return "system" + + return "{name}-{version}".format(name = toolchain['name'], version = toolchain['version']) + +def add_package_if_module_exists(package, installation_path, stack): + module_filename = package['Version'] + if package['Toolchain'] != "system": + module_filename += '-' + module_filename += package['Toolchain'] + module_filename += '.lua' + + module_path = installation_path / "modules" / "all" / package['Category'] / package['Name'] / module_filename + + if module_path.exists(): + stack.append(package) + +def extract_easyconfig_info(eb, installation_path, stack): + try: + parser = EasyConfigParser(filename = str(eb)) + ec = parser.get_config_dict(validate = False) + + package = { + 'Name': ec['name'], + 'Version': ec['version'], + 'Homepage': ec['homepage'], + 'Description': ec['description'], + 'Toolchain': toolchain_to_maybe_string(ec['toolchain']), + 'Category': ec.get('moduleclass'), + } + + add_package_if_module_exists(package, installation_path, stack) + + except Exception as excpt: + print(f"Failed: {eb}> {excpt}") + + return stack + +def get_installation_target_package_list(installation_path): + software_path = installation_path / "software" + + nested_report_ebs = (p.glob("*.eb") for p in software_path.glob("*/*/easybuild/reprod") if p.is_dir()) + report_ebs = itertools.chain.from_iterable(nested_report_ebs) + + stack = list() + for eb in report_ebs: + extract_easyconfig_info(eb, installation_path, stack) + + return stack + +def homepage_link(text, url): + if pd.notna(url) and url: + return f"[{text}]({url})" + return text + +def clean_software_description(software_table, collapse_descr = True): + software_table['Description'] = (software_table['Description'] + .fillna("") + .str.replace(r"\s+", " ", regex=True) + .str.strip()) + + if collapse_descr == True: + software_table['Description'] = software_table['Description'].apply( + lambda d: ( + "
" + "Show" + f"{d}" + "
" + ) + ) + +def generate_installation_target_package_table(stack, collapse_descr = True): + df = pd.DataFrame(columns=['Name', 'Version', 'Homepage', 'Description', 'Toolchain', 'Category']) + + if len(stack) > 0: + df = pd.concat([df, pd.DataFrame(stack)], ignore_index=True) + clean_software_description(df, collapse_descr) + + if len(df) > 0: + df['Software'] = df.apply( + lambda row: homepage_link(row['Name'], row['Homepage']), + axis=1, + ) + else: + df['Software'] = pd.Series() + + df = df[['Name', 'Version', 'Software', 'Toolchain', 'Category', 'Description']] + + return df + +def get_build_software_tables(base_path: pathlib.Path, build_number: str, releases: dict[str, dict[str, dict[str, list[str]]]]): + software_tables = dict() + + release = releases.get(build_number) + if release is None: + return software_tables + + for cluster in release.keys(): + for arch in release[cluster].keys(): + for release_id in release[cluster][arch]: + target_installation_path = base_path / cluster / arch / release_id / build_number + stack = get_installation_target_package_list(target_installation_path) + table = generate_installation_target_package_table(stack) + + # One build per (cluster, arch) in release! + software_tables[(cluster, arch, release_id)] = table + + return software_tables + +class ArchitectureBuilds(typing.NamedTuple): + architecture: str + toolchains: set[str] + + def __str__(self) -> str: + toolchain_list = ", ".join(self.toolchains) + return self.architecture + "(" + toolchain_list + ")" + + def __hash__(self) -> int: + return hash(self.architecture) + +class ClusterBuilds(typing.NamedTuple): + cluster: str + architecture_builds: set[ArchitectureBuilds] + + def __str__(self) -> str: + res = "" + res += "" + self.cluster + "" + res += "" + ", ".join(map(lambda s: str(s), self.architecture_builds)) + "" + res += "" + return res + + def __hash__(self) -> int: + return hash(self.cluster) + +class RelaseBuilds(typing.NamedTuple): + cluster_builds: set[ClusterBuilds] + + def __str__(self) -> str: + res = "
Show " + res += " ".join(map(lambda s: str(s), self.cluster_builds)) + res += "
" + return res + + def __hash__(self) -> int: + return hash("_".join(self.cluster_builds)) + +def merge_release_package_installations(tables): + df = pd.concat(tables, ignore_index=True) + + if len(df) == 0: + df = pd.DataFrame(columns=['Name', 'Version', 'Software', 'Category', 'Description', 'Built instances']) + return df + + descriptions = df.apply(lambda row: {row['Description']}, axis=1) + categories = df.apply(lambda row: {row['Category']}, axis=1) + df = df[['Name', 'Version', 'Software', 'Cluster', 'Architecture', 'Toolchain']] + df['Description'] = descriptions + df['Category'] = categories + + df = df.groupby( + ['Name', 'Version', 'Software', 'Cluster', 'Architecture'], + as_index=False, + ).aggregate( + { + 'Toolchain': (lambda s: set(s)), + 'Category': (lambda s: functools.reduce(lambda acc, v: acc | v, s, set())), + 'Description': (lambda s: functools.reduce(lambda acc, v: acc | v, s, set())), + } + ) + + architecture_builds = df.apply(lambda row: ArchitectureBuilds(architecture=row['Architecture'], toolchains=row['Toolchain']), axis=1) + df = df[['Name', 'Version', 'Software', 'Cluster', 'Category', 'Description']] + df['ArchitectureBuilds'] = architecture_builds + + df = df.groupby( + ['Name', 'Version', 'Software', 'Cluster'], + as_index=False, + ).aggregate( + { + 'ArchitectureBuilds': (lambda s: set(s)), + 'Category': (lambda s: functools.reduce(lambda acc, v: acc | v, s, set())), + 'Description': (lambda s: functools.reduce(lambda acc, v: acc | v, s, set())), + } + ) + + cluster_builds = df.apply(lambda row: ClusterBuilds(cluster=row['Cluster'], architecture_builds=row['ArchitectureBuilds']), axis=1) + df = df[['Name', 'Version', 'Software', 'Category', 'Description']] + df['ClusterBuilds'] = cluster_builds + + df = df.groupby( + ['Name', 'Version', 'Software'], + as_index=False, + ).aggregate( + { + 'ClusterBuilds': (lambda s: set(s)), + 'Category': (lambda s: functools.reduce(lambda acc, v: acc | v, s, set())), + 'Description': (lambda s: functools.reduce(lambda acc, v: acc | v, s, set())), + } + ) + + release_builds = df.apply(lambda row: RelaseBuilds(cluster_builds=row['ClusterBuilds']), axis=1) + df = df[['Name', 'Version', 'Software', 'Category', 'Description']] + df['Built instances'] = release_builds.apply(lambda r: str(r)) + + if functools.reduce(lambda acc, v: max(acc, v), df['Category'].apply(len), 1) != 1: + raise Exception(f"Ambigious category definitions in software set.") + if functools.reduce(lambda acc, v: max(acc, v), df['Description'].apply(len), 1) != 1: + raise Exception(f"Ambigious description definitions in software set.") + + df['Category'] = df['Category'].apply(lambda x: x.pop()) + df['Description'] = df['Description'].apply(lambda x: x.pop()) + + df = df.sort_values(by=['Name', 'Version']) + + return df + +def merge_build_tables_per_release(build_tables): + tables_listed_per_release = dict() + for release in build_tables.keys(): + cluster, arch, release_id = release + table = build_tables[release] + table['Cluster'] = pd.Series([cluster for i in range(len(table))]) + table['Architecture'] = pd.Series([arch for i in range(len(table))]) + + if tables_listed_per_release.get(release_id) is None: + tables_listed_per_release[release_id] = [table] + else: + tables_listed_per_release[release_id].append(table) + + return { release: merge_release_package_installations(tables) for release, tables in tables_listed_per_release.items() } + +def get_build_tables_per_release(base_path: pathlib.Path, build_number: str, releases: dict[str, dict[str, dict[str, list[str]]]]): + build_tables = get_build_software_tables(base_path, build_number, releases) + build_tables_per_release = merge_build_tables_per_release(build_tables) + + return build_tables_per_release + +def save_build_tables_per_release(build_tables_per_release, output_path: pathlib.Path): + for release, table in build_tables_per_release.items(): + output_file = output_path / (release + ".md") + with open(str(output_file), "w") as file: + file.write( + table[ + [ + 'Software', + 'Version', + 'Category', + 'Built instances', + 'Description', + ] + ].to_markdown(index = False)) + +def get_software_installation_config(config_path: pathlib.Path) -> dict: + with open(str(config_path), 'r') as config_file: + config = yaml.safe_load(config_file) + return config + +def get_releases(config: dict) -> dict[str, dict[str, dict[str, list[str]]]]: + return config['sofware'] + +def get_base_path(config: dict) -> pathlib.Path: + path: str = config['base_path'] + return pathlib.Path(path) + +def create_build_table_pages_per_release( + configuration: pathlib.Path, + build_number: str, + output_path: pathlib.Path +): + config = get_software_installation_config(configuration) + + releases = get_releases(config) + base_path = get_base_path(config) + + build_tables = get_build_tables_per_release(base_path, build_number, releases) + save_build_tables_per_release(build_tables, output_path) + +def main(): + parser = argparse.ArgumentParser( + prog = "extract_module_metadata", + description = "Collect the modules installed with the easybuild stack in a set of pages for easy search in the UL HPC documentation" + ) + + parser.add_argument("configuration", type=pathlib.Path, help="a YAML file with the configurations that will be processed") + parser.add_argument("release", type=str, help="the release in the configuration file for which the sfoftware tables will be generated") + parser.add_argument("output_path", type=pathlib.Path, help="path to the directory where the software list will be stored") + + args = parser.parse_args() + + create_build_table_pages_per_release(args.configuration, args.release, args.output_path) + +if __name__ == "__main__": + main() diff --git a/scripts/resif3_module2markdown.py b/scripts/resif3_module2markdown.py deleted file mode 100755 index 7682de1ce..000000000 --- a/scripts/resif3_module2markdown.py +++ /dev/null @@ -1,638 +0,0 @@ -#!/usr/bin/env python3 -# Time-stamp: -############################################################################### - -""" -Collect and analyse the current RESIF3 software set available on the ULHPC -platform -- see https://github.com/ULHPC/sw - -Render (i.e.) generate output markdown files reflecting the available software -and modules to be integrated into the current hpc-docs.uni.lu site. -""" - -import subprocess -import re -import pandas as pd -import click # prefered for CLI -import confuse -import sys -import os -import logging -import logging.config -import argparse -import pathlib -import pprint -import socket -import itertools -import yaml -from functools import reduce - -APPNAME = 'resif3_module2markdown' -__version__ = '1.0.0' - -CONTEXT_SETTINGS = dict(help_option_names=['-h', '--help']) -DEFAULT_SETTINGS = { - 'clusters': [ - 'iris', - 'aion' - ], - 'archs': [ - 'broadwell', - 'skylake', - 'gpu', - 'epyc' - ], - 'swsets_versions': [ - '2019b', - '2020b' - ], - 'resif_root_path': '/opt/apps/resif', - 'yamlfile': 'resif_modules.yaml', - 'output_dir': 'docs/software/swsets', - 'categories': { - 'bio': "Biology", - 'cae': "CFD/Finite element modelling", - 'chem': "Chemistry", - 'compiler': "Compilers", - 'data': "Data processing", - 'debugger': "Debugging", - 'devel': "Development", - 'geo': "Weather modelling", - 'lang': "Programming Languages", - 'lib': "Libraries", - 'math': "Mathematics", - 'mpi': "MPI", - 'numlib': "Numerical libraries", - 'perf': "Performance measurements", - 'phys': "Physics", - 'system': "System-level software", - 'toolchain': "Toolchains (software stacks)", - 'tools': "Utilities", - 'vis': "Visualisation" - } -} - -def dict_merge(dct, merge_dct): - """ - Deep Dictionary Merge - see https://gist.github.com/angstwad/bf22d1822c38a92ec0a9 - """ - dct = dct.copy() - - for k, v in merge_dct.items(): - if (k in dct and isinstance(dct[k], dict) and isinstance(merge_dct[k], dict)): - dct[k] = dict_merge(dct[k], merge_dct[k]) - else: - if (bool(dct) and k in dct.keys() and isinstance(dct[k], list)): - dct[k] = list(set(dct[k] + merge_dct[k])) - else: - dct[k] = merge_dct[k] - - return dct - -### -# GENERIC SETTINGS / LOGS Management -### -## settings management with confuse -class ConfigValueNotFound(Exception): - pass -confuse.NotFoundError = ConfigValueNotFound -settings = dict_merge(confuse.Configuration(APPNAME, __name__).get(), - DEFAULT_SETTINGS) - -## logging -FORMATTER = logging.Formatter("[%(name)s] %(asctime)s — %(levelname)s: %(message)s") - -def get_console_handler(): - console_handler = logging.StreamHandler(sys.stdout) - console_handler.setFormatter(FORMATTER) - return console_handler -# def get_file_handler(): -# file_handler = TimedRotatingFileHandler(LOG_FILE, when='midnight') -# file_handler.setFormatter(FORMATTER) -# return file_handler -def get_logger(logger_name): - logger = logging.getLogger(logger_name) - logger.setLevel(logging.INFO) - logger.addHandler(get_console_handler()) - # logger.addHandler(get_file_handler()) - # with this pattern, it's rarely necessary to propagate the error up to parent - logger.propagate = False - return logger - -log = get_logger(APPNAME) - -### -# UTILS HELPERS -### - -def dict_contains(dct, fkey, fvalue): - """ - Deep dictionary searching with filters - """ - for k, v in dct.items(): - if k == fkey: - # Simple check if the key is present (no value comparison) - if fvalue is None: - return True - # Check presence of the value in the corresponding list - if isinstance(v, list): - if v[0] in fvalue: - return True - # Check if the value is a key - if isinstance(v, dict): - if list(v.keys())[0] in fvalue: - return True - if v == fvalue: - return True - - # We continue to dig into the dictionary if we can continue to go down - if isinstance(v, dict): - if dict_contains(v, fkey, fvalue): - return True - - return False - -def create_output_path(path): - """ - Create and return the created path/folder object - """ - folder = pathlib.Path(path) - if not folder.exists(): - folder.mkdir(parents=True) - return folder - -def get_catlongname(cat): - """ - Return a long name (if known) for a given category. - """ - knowncats = settings['categories'] - if cat is None: return knowncats - if cat in knowncats.keys(): return knowncats[cat] - else: return cat.upper() - - - - - -#################### COLLECT ##################### - -def get_module_details_from_file(mfpath, filters): - """ - Get module information based on its LUA filepath - - Args: - mfpath (str): path to resif root directory - filters (dict): eventual filters to apply - - Return module dictionary with description, category, ... if succesfully retrieved - Return False is failed - """ - - try: - - if "/opt/apps/resif/" not in mfpath: - raise Exception('File provided does not come from /opt/apps/resif') - - path_splitted = mfpath.split('/') - path_splitted.pop(0) - - if path_splitted[-1].split('.')[-1] != 'lua': - raise Exception('File provided does not have the LUA extension') - - # Read provided file - f = open(mfpath, 'r') - raw = f.readlines() - f.close() - - # Get information from filepath - category_name = path_splitted[8].lower() - software_name = path_splitted[9] - cluster = path_splitted[3] - arch = path_splitted[5] - swset = path_splitted[4] - - # Parse the file to retrieve description and www page - # The description is split between many lines, get them in a list - desc = [] - version = "" - isDescLine = False - for line in raw: - line = line.strip() - if line.startswith("whatis([[Description:") or line.startswith("whatis([==[Description:"): isDescLine = True - if line.startswith("whatis([[Homepage:") or line.startswith("whatis([==[Homepage:"): - isDescLine = False # description ends before www whatis block - match_homepage = line - if isDescLine and (line != ''): desc.append(line) - if line.startswith("setenv(\"EBVERSION"): - version = re.search('setenv\("EBVERSION.*", "(.+)\"\)', line).group(1) - - desc[0] = desc[0].replace('whatis([[Description: ',"").replace('whatis([==[Description: ',"").replace('whatis([==[Description:',"") - desc[-1] = desc[-1].replace(']])','').replace(']==])','') - full_desc = (" ".join(filter(None, desc))) - - www = match_homepage.replace('whatis([[Homepage: ',"").replace('whatis([==[Homepage: ',"").replace(']])',"").replace(']==])',"") - - module_details = { - category_name: { - software_name: { - "www": www, - "desc": full_desc, - "versions": { - version: { - "swsets": { - swset: { - "clusters": [cluster], - "archs": [arch], - } - } - } - } - } - } - } - - # Excluding the result if the module does not correspond to filters - for filter_key, filter_value in filters.items(): - if not dict_contains(module_details, filter_key, filter_value): - return False - - return module_details - - except Exception as e: - print(e) - return False - - -def collect_softwares(paths, filters=None): - """ - Iterate and deepmerge over list of module filepath to generate a map - - Args: - paths (str): resif root path to analyse - filters (dict): eventual list of filters to apply ('archs','clusters' or 'swsets') - Default: None - - Return dict of collected software - """ - collected_softwares = {} - - for filepath in paths: - module_details = get_module_details_from_file(filepath.rstrip(), filters) - # If the module is found and corresponds to filters we gave, then we add it to the returned map - if module_details: - collected_softwares = dict_merge(collected_softwares, module_details) - - return collected_softwares - -#################### RENDER ##################### - -### -# Render markdown files from collected software list -## -def render_markdown_from_collect(collected_softwares, - output_path='docs/software/swsets', - filters=None): - """ - Write into markdown files software details taken out from the available - software modules analysed by the 'collect' action (invoking - collect_softwares(...)) aimed to be displayed in the mkdocs[-material] - website - This will typically generate the following file structure: - - / - ├── all_softwares.md list of all software ever built - ├── .md software list in RESIF swset - ├── .md list of all software belonging to category '' - └── / - . ├── .md short summary and available version for software - . └── [...] belonging to category - - Args: - collected_softwares (dict): dictionnary of all collected software (typically loaded - from a yaml file) - output_path (str): where to store the generated markdown files - filters (dict): eventual filters to apply - """ - output_folder = create_output_path(output_path) - all_softwares={} - category={} - softwares_swset={} - - for category_name, category_softwares in collected_softwares.items(): - category_folder = create_output_path(output_folder / category_name) - - for software_name, software_details in category_softwares.items(): - software_file = category_folder / (software_name + ".md") - - # Flattening software details so we can easily parse and retrieve data from the nested dict - # Using '|' as separator as other caracters such as '_', '-' or '.' can be used in a version label - software_details_flatten = pd.json_normalize(software_details, sep='|').to_dict(orient='records')[0] - - # Append website to software name in a markdown manner for later use (table indexing) - software_key = "[{1}]({0})".format(software_details['www'] if software_details['www'] else '#', software_name) - - # Retrieve generic information of a software (all_software.md) formatted to be usable in a DataFrame - software_without_details = { - software_key: [] - } - - # Getting versions, swsets, archs, clusters, long category name and description - available_versions = software_details["versions"].keys() - software_without_details[software_key].append(', '.join(available_versions)) # Versions - - software_without_details[software_key].append( - ', '.join( - pd.unique( - list(map( - lambda x: x.split('|')[3], - filter( - lambda key: bool(re.search('versions\|.+\|swsets\|.+', key)), - software_details_flatten.keys() - ) - )) - ).tolist() - ) - ) # Swets - - software_without_details[software_key].append( - ', '.join( - pd.unique( - list(itertools.chain.from_iterable(map( - lambda x: x[1], - filter( - lambda item: bool(re.search('versions\|.+\|swsets\|.+\|archs', item[0])), - software_details_flatten.items() - )))) - ).tolist() - ) - ) # Archs - software_without_details[software_key].append( - ', '.join( - pd.unique( - list(itertools.chain.from_iterable(map( - lambda x: x[1], - filter( - lambda item: bool(re.search('versions\|.+\|swsets\|.+\|clusters', item[0])), - software_details_flatten.items() - )))) - ).tolist() - ) - ) # Clusters - - software_without_details[software_key].append(get_catlongname(category_name)) #Category - software_without_details[software_key].append(software_details['desc']) #Description - - # Retrieve detailed information of a software (category/software.md) formatted to be usable in a DataFrame - detailed_software_array = [] - for v in available_versions: - - # Get all swsets available for the given version - available_swsets = pd.unique( - list(map( - lambda x: x.split('|')[3], - filter( - lambda key: bool(re.search('versions\|' + re.escape(v) + '\|swsets\|.+', key)), - software_details_flatten.keys() - ) - )) - ).tolist() - - for s in available_swsets: - - # Get clusters and archs for a given (version, swset) of the current software - keys = list(filter( - lambda key: bool(re.search('versions\|' + re.escape(v) + '|swsets\|' + re.escape(s), key)), - software_details_flatten.keys() - )) - cluster_key = list(filter(lambda key: 'clusters' in key ,keys))[0] - archs_key = list(filter(lambda key: 'archs' in key ,keys))[0] - - detailed_software_array.append([ - v, - s, - ', '.join(software_details_flatten[archs_key]), - ', '.join(software_details_flatten[cluster_key]) - ]) - - # Taking advantage of parsing all swset to add the current version of this software to the corresponding swset structure (swset.md) - if s not in softwares_swset: - softwares_swset[s] = {} - softwares_swset[s][software_key + ' ' + v] = [ - ', '.join(software_details_flatten[archs_key]), - ', '.join(software_details_flatten[cluster_key]), - get_catlongname(category_name), - software_details['desc'] - ] - - # Writing into category/software.md information we gathered - if filters['categories'] and category_name not in filters['categories']: continue - if filters['swsets']: continue - log.info(f'Generating markdown file { software_file }') - df = pd.DataFrame(detailed_software_array, columns=['Version','Swset','Architectures','Clusters']) - with software_file.open("w") as fd: - fd.write("### %s\n" % software_key) - fd.write("\n") - fd.write("* [Official website](%s)\n" % software_details['www']) - fd.write("* __Category__: %s (%s)\n" % (get_catlongname(category_name), category_name)) - fd.write(" - `module load %s/%s[/]`\n" % (category_name, software_name)) - fd.write("\n") - fd.write("Available versions of %s on ULHPC platforms:\n" % software_key) - fd.write("\n") - with software_file.open("a") as fd: - df.to_markdown(fd) - with software_file.open("a") as fd: - fd.write("\n\n") - fd.write("> %s\n" % software_details['desc']) - - - - # Make sure we keep generic information for all_softwares.md - all_softwares = {**all_softwares, **software_without_details} - - # Write into SWSET.md (example: 2020b.md) all included softwares information - for swset_label, swset_list_softwares in softwares_swset.items(): - if filters['swsets'] and (swset_label not in filters['swsets']): continue - swset_file = output_folder / (swset_label + ".md") - log.info(f'Generating markdown file { swset_file }') - df = pd.DataFrame.from_dict(swset_list_softwares, orient="index", columns=['Architectures','Clusters','Category','Description']) - df.index.name='Software' - df.sort_values(by=['Software'], inplace=True) - with (swset_file).open("w") as fd: - fd.write("Alphabetical list of available ULHPC software ") - fd.write("belonging to the '%s' software set.\n" % swset_label) - fd.write("To load a software of this set, use:\n") - fd.write("```bash\n") - fd.write("# Eventually: resif-load-swset-[...]\n") - fd.write("module load /[/]\n") - fd.write("```\n") - fd.write("\n") - with (swset_file).open("a") as fd: - df.to_markdown(fd) - - df = pd.DataFrame.from_dict(all_softwares, orient="index", - columns=['Versions', - 'Swsets', - 'Architectures', - 'Clusters', - 'Category', - 'Description']) - df.index.name='Software' - df.sort_values(by=['Software'], inplace=True) - # Write generic software information we gathered in all_software.md - #if (not filters): - log.info(f'Generating markdown file { output_folder }/all_software.md') - with (output_folder / "all_softwares.md").open("w") as fd: - df.to_markdown(fd) - - # restart focusing on software category - # Write into .md (Ex: bio.md) all associated softwares information - df.sort_values(by=['Category'], inplace=True) - for category_name in collected_softwares.keys(): - if filters['categories'] and (category_name not in filters['categories']): - continue - category_df = df[df['Category'] == get_catlongname(category_name)] - category_df = category_df.drop('Category', 1) - #pprint.pprint(category_df) - category_file = output_folder / (category_name + ".md") - if not category_df.empty: - log.info(f'Generating markdown file { category_file }') - with (category_file).open("w") as fd: - #fd.write("## %s (%s)\n" % (get_catlongname(category_name), category_name)) - #fd.write("\n") - fd.write("Alphabetical list of available ULHPC software ") - fd.write("belonging to the '%s' category.\n" % category_name) - fd.write("To load a software of this category, use: ") - fd.write("` module load %s/[/]`\n" % category_name) - fd.write("\n") - with (category_file).open("a") as fd: - category_df.sort_values(by=['Software']).to_markdown(fd) - - -############################################################################### -# Command line interface with click -@click.group(invoke_without_command=True, context_settings=CONTEXT_SETTINGS) -@click.pass_context -@click.option('-V', '--version', flag_value=True, help='Return the version of this script.') -@click.option('-v', '--verbose', count=True, help='Verbosity level') -@click.option('--debug', is_flag=True, default=False, help='Debug mode') -@click.option('--noop', '--dry-run', is_flag=True, default=False, help='Dry run mode') -def cli(ctx, version, verbose, debug, noop): - """ - Main command line interface - """ - if noop: settings['noop'] = True - if debug: - settings['debug'] = True - log.setLevel(logging.DEBUG) - if (verbose > 0): log.setLevel(logging.DEBUG) - if ctx.invoked_subcommand is None: - if version: - click.echo("This is " + os.path.basename(__file__) + " version " + __version__) - else: - click.echo(ctx.get_help()) - # pprint.pprint(settings) - -# sub command 'collect' -@cli.command() -@click.pass_context -@click.option('-a', '--arch', type=click.Choice(settings['archs'], case_sensitive=False), - help='Filter output by RESIF architecture') -@click.option('-c', '--cluster', type=click.Choice(settings['clusters'], case_sensitive=False), - help='Filter output by cluster') -@click.option('-s', '--swset', multiple=True, metavar='YYYY{a|b}', - default=settings['swsets_versions'], show_default=True, - help='Filter output by RESIF software set version (Ex: 2020b)') -@click.option('-p', '--resif-root-path', type=click.Path(exists=True), default=settings['resif_root_path'], - help='set RESIF root path. In particular, modules and software installed by RESIF' + - 'can be found under PATH////{software,modules}') -@click.option('-o', '--output', type=click.File('w'), metavar='YAMLFILE', - default=None, help="Set output file for the dict (Default output to STDOUT)") -def collect(ctx, - arch, # type: Union['broadwell','skylake','gpu','epyc'] - cluster, # type: Union['iris', 'aion'] - swset, # type: Array[str] - resif_root_path, # type: str (directory path) - output # type: Union[None, str] (writable file path) - ): - """ - Collect meta-data dict of the RESIF3 modules installed and - (eventually) export them as YAML - - /!\ IMPORTANT: you probably want to run this operation on the cluster - to access the resif directories - - Use 'make resif-collect' for that purpose - """ - log.info('Collect meta-data for the available RESIF modules') - log.debug(f'click context:\n { pprint.pformat(ctx.params) } ') - log.info(f'RESIF root path: { resif_root_path }') - if bool(re.match('.*(iris|aion)-cluster\.uni\.lux$', socket.getfqdn())): - default_search_paths = "%s/iris %s/aion" % (resif_root_path, resif_root_path) - else: - default_search_paths = "%s" % resif_root_path - log.info(f'default searched path: { default_search_paths }') - find_cmd = [ - "find", - default_search_paths, - " -type d", - "\( -name ebfiles_repo -o -name software \)", - "-prune", - "-false", - "-o", - "-type f \( -iname '*.lua' ! -iname '.*' \)" - ] - log.info(f'Find command to collect lua module files in production:\n { " ".join(find_cmd) }') - # Python 3.6 on iris/aion -- starting 3.7, it is recommended to use - # subprocess.run([ ... ], capture_output=True) - if sys.version_info < (3, 7): - lualist = subprocess.check_output(f'{ " ".join(find_cmd) }', shell=True).decode('utf-8').split() - else: - lualist = subprocess.run(f'{ " ".join(find_cmd) }', capture_output=True).stdout.decode('utf-8').split() - # log.debug(f'List of LUA files to analyse:\n { pprint.pformat(lualist) }') - - filters = {} - if arch is not None: filters['archs'] = arch - if cluster is not None: filters['clusters'] = cluster - if swset is not None: filters['swsets'] = swset - log.debug(f'Filters to apply: { pprint.pformat(filters) }') - result = collect_softwares(lualist, filters) - pprint.pprint(type(result)) - log.info(f'Resulting collected dict: \n{ pprint.pformat(result) }') - if output is not None: - yaml.dump(result, output, allow_unicode=True, default_flow_style=False) - output.close() - -# sub command 'render' -@cli.command(short_help='Generate markdown files summarizing available ULHPC modules') -@click.pass_context -@click.option('-i', '--input', type=click.File('r'), metavar='YAMLFILE', - default='data/%s' % settings['yamlfile'], show_default=True, - help="Set YAML input file for the dict storing resif module informations (generated by the 'collect' subcommand)") -@click.option('-o', '--output-dir', type=click.Path(exists=True), metavar='DIR', - default=settings['output_dir'], show_default=True, - help="Set output directory where to generate the markdown files") -@click.option('-s', '--swset', multiple=True, metavar='YYYY{a|b}', default=None, - help='generate only markdown for the specified software set (Ex: 2020b)') -@click.option('-c', '--category', multiple=True, metavar='NAME', - type=click.Choice(settings['categories'].keys()), - help='generate only markdown for the specified category (Ex: bio)') -def render(ctx, input, output_dir, swset, category): - """ - Generate/Render markdown files summarizing the available software modules - under / - """ - log.info('Render meta-data for the available RESIF modules') - log.debug(f'click context:\n { pprint.pformat(ctx.params) } ') - - log.info(f'Load input resif module information from file { input }') - resif_modules = yaml.load(input, Loader=yaml.SafeLoader) - # pprint.pprint(resif_modules) - filters = {} - if swset is not None: filters['swsets'] = swset - if category is not None: filters['categories'] = category - render_markdown_from_collect(resif_modules, output_dir, filters) - - -if __name__ == "__main__": - cli() diff --git a/scripts/software_installations.yaml b/scripts/software_installations.yaml new file mode 100644 index 000000000..580d2ad97 --- /dev/null +++ b/scripts/software_installations.yaml @@ -0,0 +1,25 @@ +sofware: + rhel810-20260107: + aion: + epyc: + - 2025a + - 2024a + - 2023b + iris: + broadwell: + - 2025a + - 2024a + - 2023b + skylake: + - 2025a + - 2024a + - 2023b + gpu: + - 2025a + - 2024a + - 2023b + hopper: + - 2025a + - 2024a + - 2023b +base_path: /opt/apps/easybuild/systems