Tropical loss enhance pftmask cnpratio badvars

CNP_IO_updated9_cnpratio_reduced.txt

@@ -0,0 +1,75 @@
+# Dataset roots (any absolute paths)
+TRENDY1_PATH: /mnt/proj-shared/AI4BGC_7xw/TrainingData/Trendy_1_data_CNP
+TRENDY05_PATH: /mnt/proj-shared/AI4BGC_7xw/TrainingData/Trendy_05_data_CNP
+TVA4KM_PATH: /mnt/proj-shared/AI4BGC_7xw/TrainingData/TVA_4km_data_CNP
+
+# Optional extra roots (comma-separated)
+DATA_PATHS: /another/path1, /another/path2
+
+# Global fallback pattern if a dataset-specific one isn't set
+#FILE_PATTERN: enhanced_1_training_data_batch_*.pkl
+FILE_PATTERN: 1_training_data_batch_*.pkl
+
+# Per-dataset patterns (overrides FILE_PATTERN for that path only)
+TVA4KM_FILE_PATTERN: enhanced_monthly_training_data_batch_*.pkl
+
+
+LONGITUDE FILTERING - 2 longitudes:
+• 0, 358.75
+
+TIME SERIES VARIABLES (Climate Forcing) - 6 variables:
+• FLDS, PSRF, FSDS, QBOT, PRECTmms, TBOT 
+
+SURFACE PROPERTIES - 49 variables:
+• Latitude, Longitude, AREA, landfrac, LANDFRAC_PFT, PCT_NATVEG
+
+• OCCLUDED_P, SECONDARY_P, LABILE_P, APATITE_P
+
+• SOIL_COLOR, SOIL_ORDER
+
+• PCT_NAT_PFT_0, PCT_NAT_PFT_1, PCT_NAT_PFT_2, PCT_NAT_PFT_3, PCT_NAT_PFT_4, PCT_NAT_PFT_5, PCT_NAT_PFT_6, PCT_NAT_PFT_7, PCT_NAT_PFT_8
+• PCT_NAT_PFT_9, PCT_NAT_PFT_10, PCT_NAT_PFT_11, PCT_NAT_PFT_12, PCT_NAT_PFT_13, PCT_NAT_PFT_14, PCT_NAT_PFT_15, PCT_NAT_PFT_16
+
+• PCT_CLAY_0, PCT_CLAY_1, PCT_CLAY_2, PCT_CLAY_3, PCT_CLAY_4, PCT_CLAY_5, PCT_CLAY_6, PCT_CLAY_7, PCT_CLAY_8, PCT_CLAY_9
+• PCT_SAND_0, PCT_SAND_1, PCT_SAND_2, PCT_SAND_3, PCT_SAND_4, PCT_SAND_5, PCT_SAND_6, PCT_SAND_7, PCT_SAND_8, PCT_SAND_9
+
+PFT PARAMETERS (Plant Functional Type Characteristics) - 44 variables:
+
+• pft_deadwdcn, pft_frootcn, pft_leafcn, pft_lflitcn, pft_livewdcn, pft_c3psn, pft_croot_stem, pft_crop, pft_dleaf 
+• pft_dsladlai, pft_evergreen, pft_fcur, pft_flivewd, pft_flnr, pft_fr_fcel, pft_fr_flab, pft_fr_flig, pft_froot_leaf 
+• pft_grperc, pft_grpnow, pft_leaf_long, pft_lf_fcel, pft_lf_flab, pft_lf_flig, pft_rholnir, pft_rholvis, pft_rhosnir, pft_rhosvis 
+• pft_roota_par, pft_rootb_par, pft_rootprof_beta, pft_season_decid, pft_slatop, pft_smpsc, pft_smpso, pft_stem_leaf, pft_stress_decid 
+• pft_taulnir, pft_taulvis, pft_tausnir, pft_tausvis, pft_woody, pft_xl, pft_z0mr
+
+SCALAR VARIABLES (1D - 4 variables):
+• GPP, NPP, AR, HR
+
+1D PFT VARIABLES (41 variables):
+
+• deadcrootc, deadcrootc_storage
+• deadstemc, deadstemc_storage
+
+• leafc, leafc_storage 
+• frootc, frootc_storage
+
+• livestemc, livestemc_storage 
+• livecrootc, livecrootc_storage
+
+• cpool, npool, ppool
+
+• tlai, totvegc
+
+2D VARIABLES (layered - 25 variables):
+
+• cwdc_vr, cwdn_vr, cwdp_vr
+
+• litr2c_vr, litr3c_vr
+• litr2n_vr, litr3n_vr
+• litr2p_vr, litr3p_vr
+
+• soil1c_vr
+• soil2c_vr
+• soil3c_vr
+• soil4c_vr
+
+• labilep_vr , occlp_vr, primp_vr, secondp_vr, solutionp_vr

CNP_IO_updated9_dev_dw.txt

@@ -7,7 +7,8 @@ TVA4KM_PATH: /mnt/proj-shared/AI4BGC_7xw/TrainingData/TVA_4km_data_CNP
 DATA_PATHS: /another/path1, /another/path2
 
 # Global fallback pattern if a dataset-specific one isn't set
-FILE_PATTERN: enhanced_1_training_data_batch_*.pkl
+#FILE_PATTERN: enhanced_1_training_data_batch_*.pkl
+FILE_PATTERN: 1_training_data_batch_*.pkl
 
 # Per-dataset patterns (overrides FILE_PATTERN for that path only)
 TVA4KM_FILE_PATTERN: enhanced_monthly_training_data_batch_*.pkl
@@ -62,9 +63,9 @@ SCALAR VARIABLES (1D - 4 variables):
 
 • cwdc_vr, cwdn_vr, cwdp_vr
 
-• litr2c_vr, litr3c_vr
-• litr2n_vr, litr3n_vr
-• litr2p_vr, litr3p_vr
+• litr1c_vr, litr2c_vr, litr3c_vr
+• litr1n_vr, litr2n_vr, litr3n_vr
+• litr1p_vr, litr2p_vr, litr3p_vr
 
 • soil1c_vr, soil1n_vr, soil1p_vr
 • soil2c_vr, soil2n_vr, soil2p_vr

CNP_model_config_tokenization_default.txt

@@ -0,0 +1,58 @@
+# CNP Model default configuration
+# Mirrors ModelConfig defaults in config/training_config.py
+# Format: key = value. Lists can be comma-separated or Python lists.
+# Section headers (in brackets) are optional and ignored by the parser.
+
+[ENCODERS]
+# Time series tokenization (months per token)
+patch_size = 60
+
+# LSTM encoder
+lstm_hidden_size = 64
+
+# Fully connected layers
+fc_hidden_size = 32
+static_fc_size = 64
+
+# PFT parameters (default: no CNN)
+use_cnn_for_pft_param = false
+pft_param_cnn_channels = [32, 64]
+pft_param_cnn_kernel_size = 3
+pft_param_cnn_padding = 1
+pft_param_size = 44
+num_pfts = 17
+
+# Water encoder
+water_fc_size = 64
+
+# Scalar encoder
+scalar_fc_size = 64
+
+# 1D PFT encoder
+pft_1d_fc_size = 64
+
+[SOIL2D_CNN]
+conv_channels = [32, 64, 128]
+conv_kernel_size = 3
+conv_padding = 1
+
+[TRANSFORMER]
+# Feature fusion transformer
+embed_dim = 256
+num_tokens = 4
+token_dim = 64
+transformer_layers = 2
+transformer_heads = 4
+
+# Global dropout probability
+dropout_p = 0.1
+
+[OUTPUTS]
+# Output geometry (do not set scalar/vector/matrix sizes here)
+vector_length = 16
+matrix_rows = 1
+matrix_cols = 10
+
+# Usage:
+#   python train_cnp_model.py --model-config CNP_model_config_default.txt
+#   python scripts/run_inference_all.py --model-config CNP_model_config_default.txt

commands.txt

@@ -0,0 +1,163 @@
+02/07/2026:
+
+python train_cnp_model.py \
+  --variable-list CNP_IO_updated9_dev_dw.txt \
+  --epoch 100 \
+  --pft-zero-sparsity-weight 1.0 \
+  --pft-zero-threshold 1e-8 \
+  --tropical-only
+
+cd cnp_resuls/run_20260207_125838
+
+python ../../scripts/cnp_result_validationplot.py --stats-only &&
+python ../../scripts/generate_prediction_quality_report.py
+
+
+python train_cnp_model.py \
+  --variable-list CNP_IO_updated9_dev_dw.txt \
+  --epoch 100 \
+  --pft-zero-sparsity-weight 1.0 \
+  --pft-zero-sparsity-weights-json pft_zero_weights.json \
+  --pft1d-activation abs \
+  --pft1d-activation-overrides-json pft1d_activation_overrides.json \
+  --tropical-only
+
+python train_cnp_model.py \
+  --variable-list CNP_IO_updated9_dev_dw.txt \
+  --epoch 100 \
+  --tail-aware-vars cpool,deadstemc,deadcrootc,livestemc,livecrootc \
+  --tail-aware-loss log1p_mse \
+  --tail-aware-eps 1e-8 \
+  --tropical-only
+
+
+python train_cnp_model.py \
+  --variable-list CNP_IO_updated9_dev_dw.txt \
+  --epoch 100 \
+  --tail-aware-vars cpool,deadstemc,deadcrootc,livestemc,livecrootc \
+  --tail-aware-loss log1p_quantile \
+  --tail-aware-quantile-tau 0.9 \
+  --tail-aware-weights-json tail_weights.json \
+  --tropical-only
+
+python train_cnp_model.py \
+  --variable-list CNP_IO_updated9_dev_dw.txt \
+  --epoch 100 \
+  --pft1d-activation abs \
+  --tail-aware-vars cpool,deadstemc,deadcrootc,livestemc,livecrootc \
+  --tail-aware-loss log1p_mse \
+  --tail-aware-eps 1e-8
+
+
+python scripts/ai_predictions_to_restart.py \
+    --ai-predictions <ai_predictions.nc> \
+    --restart-file /path/to/original_restart.nc \
+    --output /path/to/original_restart.nc
+
+python scripts/extract_elm_restart_point.py \
+  --restart-file /mnt/proj-shared/AI4BGC_7xw/AI4BGC/cnp_results/run_20260207_161353/updated_restart_CNP_IO_updated9_dev_dw_20251201_TRENDY2024_default_ICB1850CNRDCTCBC_ad_spinup.elm.r.0021-01-01-00000.nc \
+  --lat -17.4246 \
+  --lon 303.75 \
+  --output-file /mnt/proj-shared/AI4BGC_7xw/AI4BGC/cnp_results/run_20260207_161353/single_point_file.nc>
+
+
+python train_cnp_model.py \
+  --variable-list CNP_IO_updated9_cnpratio_reduced.txt \
+  --epoch 100 \
+  --pft1d-activation abs \
+  --tail-aware-vars cpool,deadstemc,deadcrootc,livestemc,livecrootc,npool,ppool,labilep \
+  --tail-aware-loss log1p_mse \
+  --tail-aware-eps 1e-8 \
+  --tropical-only
+
+
+  python train_cnp_model.py \
+  --variable-list CNP_IO_updated9_dev_dw.txt \
+  --epoch 100 \
+  --tail-aware-vars cpool,deadstemc,deadcrootc,livestemc,livecrootc,npool,ppool,labilep,primp_vr,occld_vr,litr2c_vr,litr2p_vr,litr2n,litr3c_vr,litr3p_vr,litr3n_vr \
+  --tail-aware-loss log1p_mse \
+  --tail-aware-eps 1e-8 \
+  --tail-aware-weights-json list1_tail_weights.json \
+  --tropical-only
+
+
+python train_cnp_model.py \
+  --variable-list CNP_IO_updated9_dev_dw.txt \
+  --training-config-json config/training_config_unified.json \
+  --tail-aware-loss log1p_mse \
+  --tail-aware-eps 1e-8 \
+  --epoch 100 \
+  --tropical-only
+
+
+  python train_cnp_model.py \
+  --variable-list CNP_IO_updated9_cnpratio_reduced.txt \
+  --training-config-json config/training_config_unified.json \
+  --epoch 100 \
+  --tropical-only
+
+  python train_cnp_model.py \
+  --variable-list CNP_IO_updated9_dev_dw.txt \
+  --training-config-json config/training_config_experiment_cpool.json \
+  --pft-zero-sparsity-weight 1.0 \
+  --epoch 100 \
+  --tropical-only \
+  2>&1 | tee training_cpool_experiment.log &
+
+
+
+  python train_cnp_model.py --training-config-json config/training_config_experiment_cpool.json ...
+
+  python train_cnp_model.py \
+  --variable-list CNP_IO_updated9_dev_dw.txt \
+  --training-config-json config/training_config_experiment_2.json \
+  --pft-zero-sparsity-weight 1.0 \
+  --epoch 50 \
+  --tropical-only \
+  2>&1 | tee training_cpool_experiment.log &
+
+  python ../../scripts/run_inference_all.py > run_inference_all.log 2>&1 &
+  python ../../scripts/ai_predictions_to_netcdf.py  > ai_prediction_to_netcdf.log 2>&1 &
+
+  python ../../scripts/run_inference_all.py \
+  --use-training-config \
+  --data-paths "/mnt/proj-shared/AI4BGC_7xw/TrainingData/Trendy_1_data_CNP" \
+  --file-pattern "1_training_data_batch_*.pkl"
+
+
+    python train_cnp_model.py \
+  --variable-list CNP_IO_updated9_dev_dw.txt \
+  --training-config-json config/training_config_experiment_3.json \
+  --pft-zero-sparsity-weight 1.0 \
+  --epoch 50 \
+  2>&1 | tee training_cpool_experiment.log &
+
+
+   python train_cnp_model.py \
+  --variable-list CNP_IO_updated9_dev_dw.txt \
+  --training-config-json config/training_config_experiment_5.json \
+  --epoch 50 \
+  --tropical-only \
+  2>&1 | tee training_cpool_experiment4.log &
+
+
+python train_cnp_model.py \
+  --variable-list CNP_IO_updated9_dev_dw.txt \
+  --training-config config/training_config_experiment_2.json \
+  --model-config large_model_tokenization_config_v02.txt \
+  --epoch 80 \
+  --tropical-only
+
+
+python train_cnp_model.py \
+  --variable-list CNP_IO_updated9_dev_dw.txt \
+  --training-config config/training_config_experiment_2.json \
+  --pft-presence-threshold 2.0 \
+  --epoch 80 \
+  --tropical-only
+
+  python train_cnp_model.py \
+  --variable-list CNP_IO_updated9_dev_dw.txt \
+  --training-config config/training_config_experiment_2_plus.json \
+  --epoch 80 \
+  --tropical-only