diff --git a/configs/_cluster/entropy.yaml b/configs/_cluster/entropy.yaml index f341770e..00c6ba85 100644 --- a/configs/_cluster/entropy.yaml +++ b/configs/_cluster/entropy.yaml @@ -19,8 +19,8 @@ infrastructure: - 'export HYDRA_FULL_ERROR=1' # export pixi variables - - 'export PIXI_HOME=/storage_nvme_4/nano/pixi_new' - - 'export PATH="$PIXI_HOME/bin:$PATH"' + - 'export PIXI_HOME=$PROJECT_HOME_PATH/$USER/pixi' + - 'export PATH="$HOME/.pixi/bin:$PATH"' - 'export XDG_DATA_HOME="$PIXI_HOME/data"' - 'export XDG_CACHE_HOME="$PIXI_HOME/cache"' - 'export XDG_STATE_HOME="$PIXI_HOME/state"' diff --git a/configs/_cluster/entropy_a100.yaml b/configs/_cluster/entropy_a100.yaml index 376ac76d..a0a33eaf 100644 --- a/configs/_cluster/entropy_a100.yaml +++ b/configs/_cluster/entropy_a100.yaml @@ -19,7 +19,7 @@ infrastructure: - 'export HYDRA_FULL_ERROR=1' # export pixi variables - - 'export PIXI_HOME=/storage_ssd_1/nano/pixi' + - 'export PIXI_HOME=$PROJECT_HOME_PATH/$USER/pixi' - 'export PATH="$PIXI_HOME/bin:$PATH"' - 'export XDG_DATA_HOME="$PIXI_HOME/data"' - 'export XDG_CACHE_HOME="$PIXI_HOME/cache"' diff --git a/configs/_cluster/helios.yaml b/configs/_cluster/helios.yaml index b4a77aae..bc89fe20 100644 --- a/configs/_cluster/helios.yaml +++ b/configs/_cluster/helios.yaml @@ -13,14 +13,14 @@ infrastructure: script: - '${export_env_variables_placeholders:}' - 'module load ML-bundle/25.04' - - 'export PROJECT_HOME_PATH=$PLG_GROUPS_STORAGE/plggllmeffi/nano' + - 'export PROJECT_HOME_PATH=$PLG_GROUPS_STORAGE/plggllmefficont3/nano' - 'export HF_HOME=$PROJECT_HOME_PATH/hf_cache' # hydra errors - 'export HYDRA_FULL_ERROR=1' # export pixi variables - - 'export PIXI_HOME=$PROJECT_HOME_PATH/pixi' + - 'export PIXI_HOME=$PROJECT_HOME_PATH/$USER/pixi' - 'export PATH="$HOME/.pixi/bin:$PATH"' - 'export XDG_DATA_HOME="PROJECT_HOME_PATH/data"' - 'export XDG_CACHE_HOME="$PROJECT_HOME_PATH/cache"' @@ -32,7 +32,7 @@ infrastructure: - 'cd -' cluster_switch: - train_path_c4: "/net/scratch/hscra/plgrid/plgmaciejpioro/c4/train" - eval_path_c4: "/net/scratch/hscra/plgrid/plgmaciejpioro/c4/validation" - train_path_fineweb: "/net/scratch/hscra/plgrid/plgmaciejpioro/fineweb-edu/train/train" - eval_path_fineweb: "/net/scratch/hscra/plgrid/plgmaciejpioro/fineweb-edu/train/train" + train_path_c4: "/net/storage/pr3/plgrid/plggllmeffi3/datasets/c4/train" + eval_path_c4: "/net/storage/pr3/plgrid/plggllmeffi3/datasets/c4/validation" + train_path_fineweb: "/net/storage/pr3/plgrid/plggllmeffi3/datasets/fineweb/train" + eval_path_fineweb: "/net/storage/pr3/plgrid/plggllmeffi3/datasets/fineweb/train" diff --git a/configs/_cluster/lem.yaml b/configs/_cluster/lem.yaml index be42a658..99bf7833 100644 --- a/configs/_cluster/lem.yaml +++ b/configs/_cluster/lem.yaml @@ -13,14 +13,15 @@ infrastructure: script: - '${export_env_variables_placeholders:}' - - 'export HF_HOME="/lustre/pd01/plgrid/plgllmefficont2/crewtool/hf"' + - 'export PROJECT_HOME_PATH=/lustre/pd03/plgrid/plgllmefficont3/nano' + - 'export HF_HOME=$PROJECT_HOME_PATH/hf_cache' # hydra errors - 'export HYDRA_FULL_ERROR=1' # export pixi variables - - 'export PIXI_HOME=/lustre/pd01/plgrid/plgllmefficont2/pixi' - - 'export PATH="$PIXI_HOME/bin:$PATH"' + - 'export PIXI_HOME=$PROJECT_HOME_PATH/$USER/pixi' + - 'export PATH="$HOME/.pixi/bin:$PATH"' - 'export XDG_DATA_HOME="$PIXI_HOME/data"' - 'export XDG_CACHE_HOME="$PIXI_HOME/cache"' - 'export XDG_STATE_HOME="$PIXI_HOME/state"' @@ -34,4 +35,4 @@ cluster_switch: train_path_c4: "/lustre/pd01/plgrid/plgllmefficont2/datasets/c4/train" eval_path_c4: "/lustre/pd01/plgrid/plgllmefficont2/datasets/c4/validation" train_path_fineweb: "/lustre/pd01/plgrid/plgllmefficont2/crewtool/proba2/fineweb-edu/train/train" - eval_path_fineweb: "/lustre/pd01/plgrid/plgllmefficont2/crewtool/proba2/fineweb-edu/train/train" \ No newline at end of file + eval_path_fineweb: "/lustre/pd01/plgrid/plgllmefficont2/crewtool/proba2/fineweb-edu/train/train"