From 9fc3ed5f464b6af89514589523ea660cffbdfd52 Mon Sep 17 00:00:00 2001 From: Janek Date: Thu, 9 Apr 2026 14:34:52 +0200 Subject: [PATCH 1/5] user dependent pixi paths --- configs/_cluster/entropy.yaml | 2 +- configs/_cluster/entropy_a100.yaml | 2 +- configs/_cluster/helios.yaml | 4 ++-- configs/_cluster/lem.yaml | 3 ++- 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/configs/_cluster/entropy.yaml b/configs/_cluster/entropy.yaml index f341770e..8b0da52d 100644 --- a/configs/_cluster/entropy.yaml +++ b/configs/_cluster/entropy.yaml @@ -19,7 +19,7 @@ infrastructure: - 'export HYDRA_FULL_ERROR=1' # export pixi variables - - 'export PIXI_HOME=/storage_nvme_4/nano/pixi_new' + - 'export PIXI_HOME=$PROJECT_HOME_PATH/$USER/pixi' - 'export PATH="$PIXI_HOME/bin:$PATH"' - 'export XDG_DATA_HOME="$PIXI_HOME/data"' - 'export XDG_CACHE_HOME="$PIXI_HOME/cache"' diff --git a/configs/_cluster/entropy_a100.yaml b/configs/_cluster/entropy_a100.yaml index 376ac76d..a0a33eaf 100644 --- a/configs/_cluster/entropy_a100.yaml +++ b/configs/_cluster/entropy_a100.yaml @@ -19,7 +19,7 @@ infrastructure: - 'export HYDRA_FULL_ERROR=1' # export pixi variables - - 'export PIXI_HOME=/storage_ssd_1/nano/pixi' + - 'export PIXI_HOME=$PROJECT_HOME_PATH/$USER/pixi' - 'export PATH="$PIXI_HOME/bin:$PATH"' - 'export XDG_DATA_HOME="$PIXI_HOME/data"' - 'export XDG_CACHE_HOME="$PIXI_HOME/cache"' diff --git a/configs/_cluster/helios.yaml b/configs/_cluster/helios.yaml index b4a77aae..13e5e542 100644 --- a/configs/_cluster/helios.yaml +++ b/configs/_cluster/helios.yaml @@ -13,14 +13,14 @@ infrastructure: script: - '${export_env_variables_placeholders:}' - 'module load ML-bundle/25.04' - - 'export PROJECT_HOME_PATH=$PLG_GROUPS_STORAGE/plggllmeffi/nano' + - 'export PROJECT_HOME_PATH=$PLG_GROUPS_STORAGE/plggllmefficont3/nano' - 'export HF_HOME=$PROJECT_HOME_PATH/hf_cache' # hydra errors - 'export HYDRA_FULL_ERROR=1' # export pixi variables - - 'export PIXI_HOME=$PROJECT_HOME_PATH/pixi' + - 'export PIXI_HOME=$PROJECT_HOME_PATH/$USER/pixi' - 'export PATH="$HOME/.pixi/bin:$PATH"' - 'export XDG_DATA_HOME="PROJECT_HOME_PATH/data"' - 'export XDG_CACHE_HOME="$PROJECT_HOME_PATH/cache"' diff --git a/configs/_cluster/lem.yaml b/configs/_cluster/lem.yaml index be42a658..e2422c21 100644 --- a/configs/_cluster/lem.yaml +++ b/configs/_cluster/lem.yaml @@ -19,7 +19,8 @@ infrastructure: - 'export HYDRA_FULL_ERROR=1' # export pixi variables - - 'export PIXI_HOME=/lustre/pd01/plgrid/plgllmefficont2/pixi' + - 'export PROJECT_HOME_PATH=/lustre/pd01/plgrid/plgllmefficont3/nano' + - 'export PIXI_HOME=$PROJECT_HOME_PATH/$USER/pixi' - 'export PATH="$PIXI_HOME/bin:$PATH"' - 'export XDG_DATA_HOME="$PIXI_HOME/data"' - 'export XDG_CACHE_HOME="$PIXI_HOME/cache"' From 5fc57dbb9607e98cdfd5ab8377ca13175e68b218 Mon Sep 17 00:00:00 2001 From: j321m <97523346+j321m@users.noreply.github.com> Date: Thu, 9 Apr 2026 14:57:34 +0200 Subject: [PATCH 2/5] fix --- configs/_cluster/lem.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/configs/_cluster/lem.yaml b/configs/_cluster/lem.yaml index e2422c21..3793884e 100644 --- a/configs/_cluster/lem.yaml +++ b/configs/_cluster/lem.yaml @@ -13,7 +13,7 @@ infrastructure: script: - '${export_env_variables_placeholders:}' - - 'export HF_HOME="/lustre/pd01/plgrid/plgllmefficont2/crewtool/hf"' + - 'export HF_HOME="/lustre/pd03/plgrid/plgllmefficont2/crewtool/hf"' # hydra errors - 'export HYDRA_FULL_ERROR=1' @@ -35,4 +35,4 @@ cluster_switch: train_path_c4: "/lustre/pd01/plgrid/plgllmefficont2/datasets/c4/train" eval_path_c4: "/lustre/pd01/plgrid/plgllmefficont2/datasets/c4/validation" train_path_fineweb: "/lustre/pd01/plgrid/plgllmefficont2/crewtool/proba2/fineweb-edu/train/train" - eval_path_fineweb: "/lustre/pd01/plgrid/plgllmefficont2/crewtool/proba2/fineweb-edu/train/train" \ No newline at end of file + eval_path_fineweb: "/lustre/pd01/plgrid/plgllmefficont2/crewtool/proba2/fineweb-edu/train/train" From adca52f5c801eb5aa56331d992fbd7593729f70b Mon Sep 17 00:00:00 2001 From: j321m <97523346+j321m@users.noreply.github.com> Date: Thu, 9 Apr 2026 14:58:18 +0200 Subject: [PATCH 3/5] fix --- configs/_cluster/lem.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/configs/_cluster/lem.yaml b/configs/_cluster/lem.yaml index 3793884e..e9d73ab3 100644 --- a/configs/_cluster/lem.yaml +++ b/configs/_cluster/lem.yaml @@ -13,13 +13,13 @@ infrastructure: script: - '${export_env_variables_placeholders:}' - - 'export HF_HOME="/lustre/pd03/plgrid/plgllmefficont2/crewtool/hf"' + - 'export PROJECT_HOME_PATH=/lustre/pd03/plgrid/plgllmefficont3/nano' + - 'export HF_HOME=$PROJECT_HOME_PATH/hf_cache' # hydra errors - 'export HYDRA_FULL_ERROR=1' # export pixi variables - - 'export PROJECT_HOME_PATH=/lustre/pd01/plgrid/plgllmefficont3/nano' - 'export PIXI_HOME=$PROJECT_HOME_PATH/$USER/pixi' - 'export PATH="$PIXI_HOME/bin:$PATH"' - 'export XDG_DATA_HOME="$PIXI_HOME/data"' From b415bf0914dfdb214ac081ce946e9ac29c08b54e Mon Sep 17 00:00:00 2001 From: j321m <97523346+j321m@users.noreply.github.com> Date: Thu, 9 Apr 2026 14:59:32 +0200 Subject: [PATCH 4/5] Update dataset paths in helios.yaml configurationfix helios dataset paths --- configs/_cluster/helios.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/configs/_cluster/helios.yaml b/configs/_cluster/helios.yaml index 13e5e542..bc89fe20 100644 --- a/configs/_cluster/helios.yaml +++ b/configs/_cluster/helios.yaml @@ -32,7 +32,7 @@ infrastructure: - 'cd -' cluster_switch: - train_path_c4: "/net/scratch/hscra/plgrid/plgmaciejpioro/c4/train" - eval_path_c4: "/net/scratch/hscra/plgrid/plgmaciejpioro/c4/validation" - train_path_fineweb: "/net/scratch/hscra/plgrid/plgmaciejpioro/fineweb-edu/train/train" - eval_path_fineweb: "/net/scratch/hscra/plgrid/plgmaciejpioro/fineweb-edu/train/train" + train_path_c4: "/net/storage/pr3/plgrid/plggllmeffi3/datasets/c4/train" + eval_path_c4: "/net/storage/pr3/plgrid/plggllmeffi3/datasets/c4/validation" + train_path_fineweb: "/net/storage/pr3/plgrid/plggllmeffi3/datasets/fineweb/train" + eval_path_fineweb: "/net/storage/pr3/plgrid/plggllmeffi3/datasets/fineweb/train" From e3cb6a1ac8091006587d6123eebd6a885e7b7c28 Mon Sep 17 00:00:00 2001 From: Janek Date: Mon, 13 Apr 2026 18:23:24 +0200 Subject: [PATCH 5/5] fix PATH --- configs/_cluster/entropy.yaml | 2 +- configs/_cluster/lem.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/configs/_cluster/entropy.yaml b/configs/_cluster/entropy.yaml index 8b0da52d..00c6ba85 100644 --- a/configs/_cluster/entropy.yaml +++ b/configs/_cluster/entropy.yaml @@ -20,7 +20,7 @@ infrastructure: # export pixi variables - 'export PIXI_HOME=$PROJECT_HOME_PATH/$USER/pixi' - - 'export PATH="$PIXI_HOME/bin:$PATH"' + - 'export PATH="$HOME/.pixi/bin:$PATH"' - 'export XDG_DATA_HOME="$PIXI_HOME/data"' - 'export XDG_CACHE_HOME="$PIXI_HOME/cache"' - 'export XDG_STATE_HOME="$PIXI_HOME/state"' diff --git a/configs/_cluster/lem.yaml b/configs/_cluster/lem.yaml index e9d73ab3..99bf7833 100644 --- a/configs/_cluster/lem.yaml +++ b/configs/_cluster/lem.yaml @@ -21,7 +21,7 @@ infrastructure: # export pixi variables - 'export PIXI_HOME=$PROJECT_HOME_PATH/$USER/pixi' - - 'export PATH="$PIXI_HOME/bin:$PATH"' + - 'export PATH="$HOME/.pixi/bin:$PATH"' - 'export XDG_DATA_HOME="$PIXI_HOME/data"' - 'export XDG_CACHE_HOME="$PIXI_HOME/cache"' - 'export XDG_STATE_HOME="$PIXI_HOME/state"'