Tropical badvars natveg nppool 2 pimprove

commands.txt

@@ -509,7 +509,7 @@ python scripts/extract_elm_restart_point.py \
   --restart-file cnp_results/run_20260311_204845_phase3_tworegions/updated_restart_phase3_tworegions_5P_bias_corrected_tropical.nc \
   --lat 0.0 \
   --lon 28.0 \
-  --output-file cnp_results/run_20260311_204845_phase3_tworegions/Africa_phase3_tworegions_restart.nc
+  --output-filcnp_results/run_20260313_224805_phase2_tropical_soilp_amazon_africae cnp_results/run_20260311_204845_phase3_tworegions/Africa_phase3_tworegions_restart.nc
 
 python scripts/generate_site_5p_restart_comparison.py   --lon 303.75   --lat -17.434553   --site-name amazon_303_17S
 python scripts/generate_site_5p_restart_comparison.py  --lon 292.5  --lat -15.55   --site-name amazon_292_15S
@@ -519,4 +519,82 @@ python scripts/generate_site_5p_restart_comparison.py   --lon 28   --lat 0   --s
 python scripts/generate_site_5p_restart_comparison.py   --lon 14   --lat 0   --site-name africa
 
 
+cd /mnt/proj-shared/AI4BGC_7xw/AI4BGC
+
+# Amazon-only bias/scale
+python scripts/apply_5p_bias_scale_correction.py \
+  --run-dir cnp_results/run_20260311_204845_phase3_tworegions\
+  --region-config-json config/training_config_amazon_5p_box.json \
+  --output-subdir soil_2d_predictions_5P_bias_corrected_amazon
+
+# Africa-only bias/scale
+python scripts/apply_5p_bias_scale_correction.py \
+  --run-dir cnp_results/run_20260311_204845_phase3_tworegions\
+  --region-config-json config/training_config_africa_5p_box.json \
+  --output-subdir soil_2d_predictions_5P_bias_corrected_africa
+
+python scripts/merge_5p_bias_corrected_amazon_africa.py \
+  --run-dir cnp_results/run_20260311_204845_phase3_tworegions
+
+
+python scripts/ai_predictions_to_netcdf.py \
+  --ai-predictions cnp_results/run_20260311_204845_phase3_tworegions/cnp_inference_entire_dataset/cnp_predictions/ \
+  --soil-2d-bias-corrected-subdir soil_2d_predictions_5P_bias_corrected_amazon_africa \
+  --variable-list CNP_IO_updated9_dev_dw.txt \
+  --output cnp_results/run_20260311_204845_phase3_tworegions/comparison_results/ai_predictions_5P_bias_corrected_amazon_africa.nc
+
+
+  python scripts/ai_predictions_to_restart.py \
+  --ai-predictions cnp_results/run_20260311_204845_phase3_tworegions/comparison_results/ai_predictions_5P_bias_corrected_amazon_africa.nc \
+  --restart-file cnp_results/run_20260311_174843_phase2_tropical/updated_restart_phase2_tropical_5P_raw.nc \
+  --output cnp_results/run_20260311_204845_phase3_tworegions/updated_restart_phase3_tworegions_5P_bias_corrected_amazon_africa.nc \
+  --variable-list CNP_IO_updated9_dev_dw.txt \
+  --variables-to-update labilep_vr,occlp_vr,solutionp_vr,secondp_vr,primp_vr \
+  "--tropical-lat-range=-30,30"
+
+  python scripts/generate_site_5p_restart_comparison.py \
+  --lon 28.0 \
+  --lat 0.0 \
+  --site-name africa_28_0_amazon_africa_nc \
+  --output-dir cnp_results/run_20260311_204845_phase3_tworegions/analysis/africa_28_0_amazon_africa_5p_restart_comparison
+
+
+  python scripts/generate_site_5p_restart_comparison.py \
+  --lon 303.75 \
+  --lat -17.43 \
+  --site-name amazon303_tworegion_biascorrection \
+  --output-dir cnp_results/run_20260311_204845_phase3_tworegions/analysis/amazon_5p_restart_comparison_twocorrections
+
+  python train_cnp_model.py --training-config-json config/training_config_phase2_tropical_soilp_only.json --variable-list CNP_IO_updated9_dev_dw.txt --tropical-lat-range -30,30 --output-dir-suffix phase2_tropical_soilp_amazon_africa
+
+
+# --- 05/28/2026: TESNORTHERA full-domain export-only CSV + normal-spinup restart ---
+# Uses the completed batched model inference predictions.pkl for the full north domain
+# (259,535 samples), so this does not rerun the expensive GPU inference.
+cd /mnt/proj-shared/AI4BGC_7xw/AI4BGC
 
+TESNORTH_RUN=/mnt/proj-shared/AI4BGC_7xw/AI4BGC/cnp_results/run_20260528_0946_tesnorthera_inference_batched_from_tesnorth10pc
+TESNORTH_INFER="$TESNORTH_RUN/cnp_inference_entire_domain"
+TESNORTH_TRAIN_RUN=/mnt/proj-shared/AI4BGC_7xw/AI4BGC/cnp_results/run_20260319_134216_tesnorth10pc
+TESNORTH_VAR_LIST=/mnt/proj-shared/AI4BGC_7xw/AI4BGC/CNP_IO_tesnorth10pct.txt
+TESNORTH_RESTART=/mnt/proj-shared/AI4BGC_7xw/AI4BGC/ELM_data/TESNORTHERA5_20adspin_restartfile/uELM_NORTHERA5_ERA5REF_I1850uELMCNPRDCTCBC.elm.r.0021-01-01-00000.nc
+
+python scripts/export_saved_predictions_to_csv.py \
+  --inference-dir "$TESNORTH_INFER" \
+  --config "$TESNORTH_TRAIN_RUN/cnp_config.json" \
+  --scalers-dir "$TESNORTH_TRAIN_RUN/cnp_predictions/scalers" \
+  --derive-np-from-c \
+  2>&1 | tee "$TESNORTH_RUN/export_saved_predictions_to_csv.log"
+
+python scripts/ai_predictions_to_netcdf.py \
+  --ai-predictions "$TESNORTH_INFER/cnp_predictions" \
+  --variable-list "$TESNORTH_VAR_LIST" \
+  --output "$TESNORTH_INFER/comparison_results/ai_predictions_for_restart.nc" \
+  2>&1 | tee "$TESNORTH_RUN/ai_predictions_to_netcdf.log"
+
+python -u scripts/ai_predictions_to_restart.py \
+  --ai-predictions "$TESNORTH_INFER/comparison_results/ai_predictions_for_restart.nc" \
+  --restart-file "$TESNORTH_RESTART" \
+  --output "$TESNORTH_RUN/updated_restart_normal_spinup_uELM_NORTHERA5_ERA5REF_I1850uELMCNPRDCTCBC.elm.r.0021-01-01-00000.nc" \
+  --variable-list "$TESNORTH_VAR_LIST" \
+  2>&1 | tee "$TESNORTH_RUN/ai_predictions_to_restart.kdtree.log"

config/finetune_phase2_africa_5p_only.json

@@ -0,0 +1,136 @@
+{
+  "_description": "Fine-tune Phase2 P-focused model on Africa region only, 5P-focused. Use: python scripts/run_finetuning_json.py --config-json config/finetune_phase2_africa_5p_only.json --output-dir-suffix phase2_africa_5p. Run from repo root. Improves Africa 5P without changing Amazon or other variables.",
+  "finetuning_paths": {
+    "variable_list_path": "../CNP_IO_updated9_dev_dw.txt",
+    "pretrained_model_path": "../cnp_results/run_20260305_153217_phase2_pvariable_focus/cnp_predictions/model.pth",
+    "dataset_path": "/mnt/proj-shared/AI4BGC_7xw/TrainingData/Trendy_1_data_CNP",
+    "file_pattern": "training_data_batch_*.pkl",
+    "output_dir": "../cnp_results",
+    "finetuned_model_filename": "finetuned_phase2_africa_5p.pth"
+  },
+  "training_hyperparameters": {
+    "num_epochs": 50,
+    "batch_size": 128,
+    "learning_rate": 1e-5,
+    "optimizer_type": "adamw",
+    "weight_decay": 0.01,
+    "use_scheduler": true,
+    "scheduler_type": "cosine",
+    "scheduler_step_size": 10,
+    "scheduler_gamma": 0.1,
+    "xsmrpool_loss_weight": 10,
+    "litter_c_loss_weight": 1,
+    "litter_n_loss_weight": 1,
+    "litter_p_loss_weight": 4,
+    "scalar_loss_weight": 0.06,
+    "vector_loss_weight": 0.4,
+    "matrix_loss_weight": 2.5
+  },
+  "reproducibility_config": {
+    "random_seed": 42,
+    "strict_determinism": false,
+    "train_split": 0.8,
+    "normalization": "individual",
+    "dropout_p": 0.12
+  },
+  "data_filtering_config": {
+    "tropical_only": false,
+    "tropical_lat_range": [-30.0, 30.0],
+    "longitudes_to_drop": [0, 358.75],
+    "natveg_only": true,
+    "natveg_filter_before_split": false,
+    "region_boxes": [
+      [-15, 15, 0, 30]
+    ]
+  },
+  "variable_weights": {
+    "pft1d_weights": {
+      "cpool": 3,
+      "npool": 4,
+      "ppool": 6,
+      "totvegc": 3,
+      "leafc": 3,
+      "leafn": 3,
+      "leafp": 4,
+      "leafc_storage": 2,
+      "leafn_storage": 2,
+      "leafp_storage": 3,
+      "deadstemc": 3,
+      "deadstemn": 3,
+      "deadstemp": 4,
+      "frootc": 2,
+      "frootn": 2,
+      "frootp": 3,
+      "frootc_storage": 1.5,
+      "frootn_storage": 1.5,
+      "frootp_storage": 2,
+      "tlai": 2
+    },
+    "soil2d_weights": {
+      "cwdc_vr": 0.1,
+      "cwdn_vr": 0.1,
+      "cwdp_vr": 0.1,
+      "primp_vr": 35,
+      "occlp_vr": 50,
+      "labilep_vr": 40,
+      "secondp_vr": 40,
+      "solutionp_vr": 40,
+      "litr2c_vr": 0.1,
+      "litr2p_vr": 0.1,
+      "litr2n_vr": 0.1,
+      "litr3c_vr": 0.1,
+      "litr3n_vr": 0.1,
+      "litr3p_vr": 0.1,
+      "soil1c_vr": 0.1,
+      "soil1n_vr": 0.1,
+      "soil1p_vr": 0.1,
+      "soil2c_vr": 0.1,
+      "soil2n_vr": 0.1,
+      "soil2p_vr": 0.1,
+      "soil3c_vr": 0.1,
+      "soil3n_vr": 0.1,
+      "soil3p_vr": 0.1,
+      "soil4c_vr": 0.1,
+      "soil4n_vr": 0.1,
+      "soil4p_vr": 0.1
+    },
+    "scalar_weights": {
+      "GPP": 0.08,
+      "NPP": 0.08,
+      "AR": 0.08,
+      "HR": 0.08
+    }
+  },
+  "tail_aware_config": {
+    "loss": "log1p_huber",
+    "epsilon": 1e-8,
+    "huber_delta": 0.7
+  },
+  "tail_aware_weights": {
+    "cpool": 3,
+    "deadstemc": 3,
+    "deadcrootc": 3,
+    "livestemc": 3,
+    "livecrootc": 3,
+    "npool": 4,
+    "ppool": 6,
+    "totvegc": 3,
+    "leafc": 3,
+    "leafn": 3,
+    "leafp": 4,
+    "labilep": 10,
+    "labilep_vr": 40,
+    "primp_vr": 35,
+    "occlp_vr": 50,
+    "secondp_vr": 40,
+    "solutionp_vr": 40
+  },
+  "pft_mask_config": {
+    "mask_absent_pfts": true,
+    "pft_presence_threshold": 0
+  },
+  "pft_zero_sparsity_config": {
+    "weight": 1,
+    "threshold": 1e-8
+  }
+}

config/training_config_africa_5p_box.json

@@ -0,0 +1,143 @@
+{
+  "_description": "Africa-only 5P region config: used to drive bias/scale correction on Africa box only. Data filtering keeps only Africa natveg cells for fitting, but model training can still be global. Primary use is with apply_5p_bias_scale_correction.py via --region-config-json. Africa box: lat [-15, 15], lon [0, 30].",
+  "training_hyperparameters": {
+    "num_epochs": 150,
+    "batch_size": 128,
+    "learning_rate": 5e-5,
+    "optimizer_type": "adamw",
+    "weight_decay": 0.01,
+    "use_scheduler": true,
+    "scheduler_type": "cosine",
+    "scheduler_step_size": 10,
+    "scheduler_gamma": 0.1,
+    "xsmrpool_loss_weight": 10,
+    "litter_c_loss_weight": 1,
+    "litter_n_loss_weight": 1,
+    "litter_p_loss_weight": 4,
+    "scalar_loss_weight": 0.06,
+    "vector_loss_weight": 0.4,
+    "matrix_loss_weight": 2.5
+  },
+  "reproducibility_config": {
+    "random_seed": 42,
+    "strict_determinism": false,
+    "train_split": 0.8,
+    "normalization": "individual",
+    "dropout_p": 0.12
+  },
+  "data_filtering_config": {
+    "tropical_only": false,
+    "tropical_lat_range": [-30.0, 30.0],
+    "longitudes_to_drop": [0, 358.75],
+    "natveg_only": true,
+    "natveg_filter_before_split": false,
+    "region_boxes": [
+      [-15.0, 15.0, 0.0, 30.0]
+    ]
+  },
+  "variable_weights": {
+    "pft1d_weights": {
+      "cpool": 3,
+      "npool": 4,
+      "ppool": 6,
+      "totvegc": 3,
+      "leafc": 3,
+      "leafn": 3,
+      "leafp": 4,
+      "leafc_storage": 2,
+      "leafn_storage": 2,
+      "leafp_storage": 3,
+      "deadstemc": 3,
+      "deadstemn": 3,
+      "deadstemp": 4,
+      "frootc": 2,
+      "frootn": 2,
+      "frootp": 3,
+      "frootc_storage": 1.5,
+      "frootn_storage": 1.5,
+      "frootp_storage": 2,
+      "tlai": 2
+    },
+    "soil2d_weights": {
+      "cwdc_vr": 0.1,
+      "cwdn_vr": 0.1,
+      "cwdp_vr": 0.1,
+      "primp_vr": 35,
+      "occlp_vr": 50,
+      "labilep_vr": 40,
+      "secondp_vr": 40,
+      "solutionp_vr": 40,
+      "litr2c_vr": 0.1,
+      "litr2p_vr": 0.1,
+      "litr2n_vr": 0.1,
+      "litr3c_vr": 0.1,
+      "litr3n_vr": 0.1,
+      "litr3p_vr": 0.1,
+      "soil1c_vr": 0.1,
+      "soil1n_vr": 0.1,
+      "soil1p_vr": 0.1,
+      "soil2c_vr": 0.1,
+      "soil2n_vr": 0.1,
+      "soil2p_vr": 0.1,
+      "soil3c_vr": 0.1,
+      "soil3n_vr": 0.1,
+      "soil3p_vr": 0.1,
+      "soil4c_vr": 0.1,
+      "soil4n_vr": 0.1,
+      "soil4p_vr": 0.1
+    },
+    "scalar_weights": {
+      "GPP": 0.08,
+      "NPP": 0.08,
+      "AR": 0.08,
+      "HR": 0.08
+    }
+  },
+  "tail_aware_config": {
+    "loss": "log1p_huber",
+    "epsilon": 1e-8,
+    "huber_delta": 0.7
+  },
+  "tail_aware_weights": {
+    "cpool": 3,
+    "deadstemc": 3,
+    "deadcrootc": 3,
+    "livestemc": 3,
+    "livecrootc": 3,
+    "npool": 4,
+    "ppool": 6,
+    "totvegc": 3,
+    "leafc": 3,
+    "leafn": 3,
+    "leafp": 4,
+    "labilep": 10,
+    "labilep_vr": 40,
+    "primp_vr": 35,
+    "occlp_vr": 50,
+    "secondp_vr": 40,
+    "solutionp_vr": 40
+  },
+  "pft_mask_config": {
+    "mask_absent_pfts": true,
+    "pft_presence_threshold": 0
+  },
+  "pft_zero_sparsity_config": {
+    "weight": 1,
+    "threshold": 1e-8
+  },
+  "pft_zero_sparsity_weights": {
+    "cpool": 2,
+    "deadstemc": 1,
+    "deadcrootc": 1,
+    "livestemc": 1,
+    "livecrootc": 1
+  },
+  "pft1d_activation_overrides": {
+    "cpool": "abs",
+    "deadstemc": "abs",
+    "deadcrootc": "abs",
+    "livestemc": "abs",
+    "livecrootc": "abs"
+  }
+}
+