From 787992b113654ad239e51cee75d59c835e7ecaa3 Mon Sep 17 00:00:00 2001 From: K Y Date: Wed, 21 Dec 2022 08:09:56 -0800 Subject: [PATCH] added --init-commands option to put a line into job script changed jupyter executable check from jupyter --version to which jupyter, since my jupyter compiled on GPU node failed on login node, with illegal instruction error. --- galyleo.sh | 38 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 36 insertions(+), 2 deletions(-) diff --git a/galyleo.sh b/galyleo.sh index 6eb8ebc..818210b 100755 --- a/galyleo.sh +++ b/galyleo.sh @@ -42,7 +42,7 @@ # future, may include a Makefile with a PREFIX option to install # correctly and set this variable. e.g., https://github.com/oyvinev/log.sh -declare -xr GALYLEO_INSTALL_DIR="${PWD}" +declare -xr GALYLEO_INSTALL_DIR="/home/kenneth/info/elba/galyleo" # Declare a global environment variable to set the galyleo cache # directory, which will hold all output files generated by galyleo. @@ -106,6 +106,7 @@ fi # -s | --sif # -B | --bind # | --nv +# | --init-commands # | --conda-init # | --conda-env # | --conda-yml @@ -152,6 +153,9 @@ function galyleo_launch() { # Declare input variables associated with environment modules. local env_modules='' + # Declare input variables associated with init script. + local init_commands='' + # Declare input variables associated with Singularity containers. local singularity_image_file='' local singularity_bind_mounts='' @@ -263,6 +267,10 @@ function galyleo_launch() { singularity_gpu_type='rocm' shift 1 ;; + --init-commands ) + init_commands="${2}" + shift 2 + ;; --conda-init ) conda_init="${2}" shift 2 @@ -326,6 +334,7 @@ function galyleo_launch() { slog output -m " -s | --sif : ${singularity_image_file}" slog output -m " -B | --bind : ${singularity_bind_mounts}" slog output -m " | --nv : ${singularity_gpu_type}" + slog output -m " | --init-commands : ${init_commands}" slog output -m " | --conda-init : ${conda_init}" slog output -m " | --conda-env : ${conda_env}" slog output -m " | --conda-yml : ${conda_yml}" @@ -398,6 +407,16 @@ function galyleo_launch() { done fi + # Check if init line specified by the user, if any, is + # available and can be run successfully. If not, then halt the launch. + if [[ -n "${init_commands}" ]]; then + sh -c "${init_commands}" + if [[ $? -ne 0 ]]; then + slog error -m "init-commands failed: ${init_commands}" + return 1 + fi + fi + # Check if the conda environment specified by the user, if any, can be # initialized and activated successfully. If not, then halt the launch. if [[ -n "${conda_env}" ]] && [[ -z "${conda_yml}" ]]; then @@ -446,7 +465,8 @@ function galyleo_launch() { fi else if [[ -z "${conda_yml}" ]]; then - jupyter "${jupyter_interface}" --version > /dev/null + #jupyter "${jupyter_interface}" --version > /dev/null + which jupyter if [[ "${?}" -ne 0 ]]; then slog error -m "No jupyter executable was found within the software environment." return 1 @@ -468,15 +488,18 @@ function galyleo_launch() { slog error -m "Unable to connect to the Satellite reverse proxy service: ${http_status_code}" return 1 fi + #: # don't run, just testing # Extract the reverse proxy connection token and export it as a # read-only environment variable. declare -xr REVERSE_PROXY_TOKEN="$(echo ${http_response} | awk 'NF>1{printf $((NF-1))}' -)" + #: # don't run, just testing # Generate an authentication token to be used for first-time # connections to the Jupyter notebook server and export it as a # read-only environment variable. declare -xr JUPYTER_TOKEN="$(openssl rand -hex 16)" + #: # don't run, just testing # Change present working directory to GALYLEO_CACHE_DIR. Generate and # store all Jupyter launch scripts and standard output files in the @@ -499,6 +522,7 @@ function galyleo_launch() { # Generate a Jupyter launch script. slog output -m 'Generating Jupyter launch script ...' + echo ${GALYLEO_SCHEDULER} if [[ ! -f "${job_name}.sh" ]]; then slog append -f "${job_name}.sh" -m '#!/usr/bin/env sh' @@ -624,6 +648,11 @@ function galyleo_launch() { done fi + # Run user init commands. + if [[ -n "${init_commands}" ]]; then + slog append -f "${job_name}.sh" -m "${init_commands}" + fi + # Activate a pre-installed conda environment specified by the user. if [[ -n "${conda_env}" ]] && [[ -z "${conda_yml}" ]]; then if [[ -n "${conda_init}" ]]; then @@ -741,6 +770,9 @@ function galyleo_launch() { # Submit Jupyter launch script to scheduler. if [[ "${GALYLEO_SCHEDULER}" == 'slurm' ]]; then job_id="$(sbatch ${job_name}.sh | grep -o '[[:digit:]]*')" + #cat ${job_name}.sh + #job_id=0 # don't run, just testing + #: # don't run, just testing if [[ "${?}" -ne 0 ]]; then slog error -m 'Failed job submission to Slurm.' http_response="$(curl -s https://manage.${GALYLEO_REVERSE_PROXY_FQDN}/destroytoken.cgi?token=${REVERSE_PROXY_TOKEN})" @@ -764,6 +796,7 @@ function galyleo_launch() { # Associate batch job id to the connection token from the reverse proxy service. http_response="$(curl -s https://manage.${GALYLEO_REVERSE_PROXY_FQDN}/linktoken.cgi?token=${REVERSE_PROXY_TOKEN}\&jobid=${job_id})" slog output -m "${http_response}" + #: # don't run, just testing # Always print to standard output the URL where the Jupyter notebook # server may be accessed by the user. @@ -974,6 +1007,7 @@ function galyleo_help() { slog output -m " -B | --bind : ${singularity_bind_mounts}" slog output -m " | --nv : ${singularity_gpu_type}" slog output -m " -e | --env-modules : ${env_modules}" + slog output -m " | --init-commands : ${init_commands}" slog output -m " | --conda-init : ${conda_init}" slog output -m " | --conda-env : ${conda_env}" slog output -m " | --conda-yml : ${conda_yml}"