Trendy 1 ai restart creation

CNP_IO_5P_tworegions.txt

@@ -0,0 +1,52 @@
+# Dataset roots (any absolute paths)
+TRENDY1_PATH: /mnt/proj-shared/AI4BGC_7xw/TrainingData/Trendy_1_data_CNP
+TRENDY05_PATH: /mnt/proj-shared/AI4BGC_7xw/TrainingData/Trendy_05_data_CNP
+TVA4KM_PATH: /mnt/proj-shared/AI4BGC_7xw/TrainingData/TVA_4km_data_CNP
+
+# Optional extra roots (comma-separated)
+DATA_PATHS: /another/path1, /another/path2
+
+# Global fallback pattern if a dataset-specific one isn't set
+#FILE_PATTERN: enhanced_1_training_data_batch_*.pkl
+#FILE_PATTERN: 1_training_data_batch_*.pkl
+FILE_PATTERN: training_data_batch_*.pkl
+
+# Per-dataset patterns (overrides FILE_PATTERN for that path only)
+TVA4KM_FILE_PATTERN: enhanced_monthly_training_data_batch_*.pkl
+
+
+#LONGITUDE FILTERING - 2 longitudes:
+#• 0, 358.75
+
+TIME SERIES VARIABLES (Climate Forcing) - 6 variables:
+• FLDS, PSRF, FSDS, QBOT, PRECTmms, TBOT 
+
+SURFACE PROPERTIES - 49 variables:
+• Latitude, Longitude, AREA, landfrac, LANDFRAC_PFT, PCT_NATVEG
+
+• OCCLUDED_P, SECONDARY_P, LABILE_P, APATITE_P
+
+• SOIL_COLOR, SOIL_ORDER
+
+• PCT_NAT_PFT_0, PCT_NAT_PFT_1, PCT_NAT_PFT_2, PCT_NAT_PFT_3, PCT_NAT_PFT_4, PCT_NAT_PFT_5, PCT_NAT_PFT_6, PCT_NAT_PFT_7, PCT_NAT_PFT_8
+• PCT_NAT_PFT_9, PCT_NAT_PFT_10, PCT_NAT_PFT_11, PCT_NAT_PFT_12, PCT_NAT_PFT_13, PCT_NAT_PFT_14, PCT_NAT_PFT_15, PCT_NAT_PFT_16
+
+• PCT_CLAY_0, PCT_CLAY_1, PCT_CLAY_2, PCT_CLAY_3, PCT_CLAY_4, PCT_CLAY_5, PCT_CLAY_6, PCT_CLAY_7, PCT_CLAY_8, PCT_CLAY_9
+• PCT_SAND_0, PCT_SAND_1, PCT_SAND_2, PCT_SAND_3, PCT_SAND_4, PCT_SAND_5, PCT_SAND_6, PCT_SAND_7, PCT_SAND_8, PCT_SAND_9
+
+PFT PARAMETERS (Plant Functional Type Characteristics) - 44 variables:
+
+• pft_deadwdcn, pft_frootcn, pft_leafcn, pft_lflitcn, pft_livewdcn, pft_c3psn, pft_croot_stem, pft_crop, pft_dleaf 
+• pft_dsladlai, pft_evergreen, pft_fcur, pft_flivewd, pft_flnr, pft_fr_fcel, pft_fr_flab, pft_fr_flig, pft_froot_leaf 
+• pft_grperc, pft_grpnow, pft_leaf_long, pft_lf_fcel, pft_lf_flab, pft_lf_flig, pft_rholnir, pft_rholvis, pft_rhosnir, pft_rhosvis 
+• pft_roota_par, pft_rootb_par, pft_rootprof_beta, pft_season_decid, pft_slatop, pft_smpsc, pft_smpso, pft_stem_leaf, pft_stress_decid 
+• pft_taulnir, pft_taulvis, pft_tausnir, pft_tausvis, pft_woody, pft_xl, pft_z0mr
+
+SCALAR VARIABLES (1D - 4 variables):
+• GPP
+
+1D PFT VARIABLES (41 variables):
+
+2D VARIABLES (layered - 25 variables):
+
+• labilep_vr , occlp_vr, primp_vr, secondp_vr, solutionp_vr

CNP_IO_updated9_dev_dw.txt

@@ -8,14 +8,15 @@ DATA_PATHS: /another/path1, /another/path2
 
 # Global fallback pattern if a dataset-specific one isn't set
 #FILE_PATTERN: enhanced_1_training_data_batch_*.pkl
-FILE_PATTERN: 1_training_data_batch_*.pkl
+#FILE_PATTERN: 1_training_data_batch_*.pkl
+FILE_PATTERN: training_data_batch_*.pkl
 
 # Per-dataset patterns (overrides FILE_PATTERN for that path only)
 TVA4KM_FILE_PATTERN: enhanced_monthly_training_data_batch_*.pkl
 
 
-LONGITUDE FILTERING - 2 longitudes:
-• 0, 358.75
+#LONGITUDE FILTERING - 2 longitudes:
+#• 0, 358.75
 
 TIME SERIES VARIABLES (Climate Forcing) - 6 variables:
 • FLDS, PSRF, FSDS, QBOT, PRECTmms, TBOT 
@@ -63,9 +64,9 @@ SCALAR VARIABLES (1D - 4 variables):
 
 • cwdc_vr, cwdn_vr, cwdp_vr
 
-• litr1c_vr, litr2c_vr, litr3c_vr
-• litr1n_vr, litr2n_vr, litr3n_vr
-• litr1p_vr, litr2p_vr, litr3p_vr
+• litr2c_vr, litr3c_vr
+• litr2n_vr, litr3n_vr
+• litr2p_vr, litr3p_vr
 
 • soil1c_vr, soil1n_vr, soil1p_vr
 • soil2c_vr, soil2n_vr, soil2p_vr

config/UNIFIED_CONFIG_README.md

@@ -10,6 +10,16 @@ Instead of managing multiple separate JSON files, you can now use a single unifi
 - Tail-aware weights
 - PFT zero sparsity weights
 - PFT1D activation overrides
+- **Data filtering** (tropical band, natveg-only) — see `data_filtering_config` below
+
+### Where to put options: config JSON vs CNP_IO.txt
+
+| Purpose | Use |
+|--------|-----|
+| **Variable lists**, **data paths**, **file patterns** | **CNP_IO.txt** (and optional paths in variable list) |
+| **Training filters** (tropical, natveg, longitude drop), **hyperparameters**, **loss weights**, **reproducibility** | **config/training_*.json** (unified or per-experiment) |
+
+Put **natveg filter**, **tropical selection**, and **longitude filtering** in the **training config JSON** under `data_filtering_config`, not in CNP_IO. That way one variable list can be reused with different filter choices, and the run’s saved config records exactly which filters were applied.
 
 ## JSON File Format
 
@@ -145,11 +155,16 @@ Settings for reproducibility and data handling:
 **Note**: CLI arguments (`--split-seed`, `--strict-determinism`, `--train-split`, `--normalization`, `--dropout-p`) take precedence over JSON values.
 
 ### `data_filtering_config`
-Settings for data filtering:
+Settings for which samples are included at training time. **Use the training config JSON here, not CNP_IO.txt**: CNP_IO defines variables and data paths; filtering (tropical, natveg) is a run choice and belongs in config so the same variable list can be reused with different filters.
+
 - `tropical_only`: Filter dataset to tropical latitudes (default: `false`)
 - `tropical_lat_range`: Latitude range as `[min, max]` or `"min,max"` string (default: `[-23.5, 23.5]`)
+- `tropical_lat_column`: Optional column name for latitude (default: auto-detect, e.g. `Latitude`)
+- `natveg_only`: If `true`, keep only gridcells with natural vegetation: `PCT_NATVEG > 0` and `PCT_NAT_PFT_0 < 100` (default: `false`). See `docs/EXCLUDED_SAMPLE_ANALYSIS.md` for impact (~33% of samples excluded on typical data).
+- `longitudes_to_drop`: List of longitudes (degrees) to exclude from training, e.g. `[0, 358.75]`. Samples whose longitude matches (within tolerance) are dropped. Overrides the same option in CNP_IO. Use 0–360° convention to match data.
+- `region_boxes`: Optional list of boxes to **keep**; only gridcells inside at least one box are used. Each box is `[lat_min, lat_max, lon_min, lon_max]` with longitude in **0–360°**. E.g. Amazon + Africa: `[[-30, 10, 270, 330], [-15, 15, 0, 30]]`. See `docs/PLAN_TWO_REGION_FIVE_P_TRAINING.md`.
 
-**Note**: CLI arguments (`--tropical-only`, `--tropical-lat-range`) take precedence over JSON values.
+**Note**: CLI arguments (`--tropical-only`, `--tropical-lat-range`, `--natveg-only`, `--longitudes-to-drop`) take precedence over JSON values. JSON `data_filtering_config` overrides values from the CNP_IO variable list (e.g. longitude filtering in CNP_IO).
 
 ### `variable_weights`
 Contains three subsections:
@@ -190,7 +205,7 @@ When both unified config and CLI arguments are specified:
 
 1. **Training hyperparameters**: CLI args (`--epochs`, `--batch-size`, `--learning-rate`, `--xsmrpool-loss-weight`, `--litter-*-loss-weight`) override unified config
 2. **Reproducibility config**: CLI args (`--split-seed`, `--strict-determinism`, `--train-split`, `--normalization`, `--dropout-p`) override unified config
-3. **Data filtering config**: CLI args (`--tropical-only`, `--tropical-lat-range`) override unified config
+3. **Data filtering config**: CLI args (`--tropical-only`, `--tropical-lat-range`, `--natveg-only`, `--longitudes-to-drop`) override unified config
 4. **PFT mask config**: CLI args (`--mask-absent-pfts` / `--no-mask-absent-pfts`, `--pft-presence-threshold`) override unified config
 5. **Tail-aware config**: CLI args (`--tail-aware-loss`, `--tail-aware-eps`) override unified config
 6. **Tail-aware weights**: Individual `--tail-aware-weights-json` overrides unified config
@@ -227,7 +242,8 @@ This allows you to:
   },
   "data_filtering_config": {
     "tropical_only": true,
-    "tropical_lat_range": [-23.5, 23.5]
+    "tropical_lat_range": [-23.5, 23.5],
+    "longitudes_to_drop": [0, 358.75]
   },
   "variable_weights": {
     "pft1d_weights": {

config/training_config.py

@@ -100,8 +100,13 @@ class DataConfig:
     tropical_only: bool = False
     tropical_lat_range: Tuple[float, float] = (-23.5, 23.5)
     tropical_lat_column: Optional[str] = None
-
-
+    # Natveg-only filtering: keep only gridcells with PCT_NATVEG > 0 and PCT_NAT_PFT_0 < 100
+    natveg_only: bool = False
+    # When True: filter to natveg before shuffle/split (legacy). When False: split on full data, then
+    # filter only training set to natveg, so test set is the same as no-filter run (natveg test ⊂ no-filter test).
+    # Default True preserves legacy behavior when key is missing.
+    natveg_filter_before_split: bool = True
+
     # File loading limits (for testing)
     max_files: Optional[int] = None  # Maximum number of files to load (None = all files)
 
@@ -112,6 +117,10 @@ class DataConfig:
     # Longitude filtering - list of longitude values to drop from dataset
     longitudes_to_drop: List[float] = field(default_factory=list)
 
+    # Region boxes: keep only gridcells inside any box. List of (lat_min, lat_max, lon_min, lon_max).
+    # Longitude in 0-360 convention. None = no region filter (use tropical_only / longitudes_to_drop instead).
+    region_boxes: Optional[List[Tuple[float, float, float, float]]] = None
+
 
 
 
@@ -167,6 +176,11 @@ class ModelConfig:
     # PFT1D activation control
     pft1d_activation: str = 'abs'  # 'abs', 'relu', 'softplus', 'linear'
     pft1d_activation_overrides: Dict[str, str] = field(default_factory=dict)
+
+    # Multihead/Multimode configuration
+    use_multihead_attention: bool = False  # Enable multi-head attention for mode-specific feature extraction
+    multihead_num_heads: int = 4  # Number of attention heads for multihead attention
+    use_mode_specific_heads: bool = False  # Enable separate feature extractors for each output mode
 
 
 @dataclass

config/training_config_experiment_2.json

@@ -23,7 +23,8 @@
   },
   "data_filtering_config": {
     "tropical_only": true,
-    "tropical_lat_range": [-23.5, 23.5]
+    "tropical_lat_range": [-23.5, 23.5],
+    "longitudes_to_drop": [0, 358.75]
   },
   "variable_weights": {
     "pft1d_weights": {

config/training_config_experiment_3_global.json

@@ -0,0 +1,139 @@
+{
+  "_description": "Experiment 3 global: full training config for global run (20,826 samples). Same structure as training_config_experiment_2.json. Tail-aware log1p_huber; primp_vr weight 11; PFT mask and zero sparsity as in experiment_3.",
+  "training_hyperparameters": {
+    "num_epochs": 100,
+    "batch_size": 128,
+    "learning_rate": 0.0001,
+    "optimizer_type": "adamw",
+    "weight_decay": 0.01,
+    "use_scheduler": true,
+    "scheduler_type": "cosine",
+    "scheduler_step_size": 10,
+    "scheduler_gamma": 0.1,
+    "xsmrpool_loss_weight": 10,
+    "litter_c_loss_weight": 1,
+    "litter_n_loss_weight": 1,
+    "litter_p_loss_weight": 1,
+    "scalar_loss_weight": 1,
+    "vector_loss_weight": 1,
+    "matrix_loss_weight": 1
+  },
+  "reproducibility_config": {
+    "random_seed": 42,
+    "strict_determinism": false,
+    "train_split": 0.8,
+    "normalization": "individual",
+    "dropout_p": 0.1
+  },
+  "data_filtering_config": {
+    "tropical_only": false,
+    "tropical_lat_range": null,
+    "longitudes_to_drop": [0, 358.75],
+    "natveg_only": true,
+    "natveg_filter_before_split": true
+  },
+  "variable_weights": {
+    "pft1d_weights": {
+      "cpool": 6,
+      "npool": 8,
+      "ppool": 8,
+      "totvegc": 6,
+      "leafc": 6,
+      "leafn": 6,
+      "leafp": 6,
+      "leafc_storage": 5,
+      "leafn_storage": 5,
+      "leafp_storage": 5,
+      "deadstemc": 6,
+      "deadstemn": 6,
+      "deadstemp": 5,
+      "frootc": 4,
+      "frootn": 4,
+      "frootp": 4,
+      "frootc_storage": 3,
+      "frootn_storage": 3,
+      "frootp_storage": 3,
+      "tlai": 4
+    },
+    "soil2d_weights": {
+      "primp_vr": 11,
+      "litr2c_vr": 8,
+      "litr2p_vr": 8,
+      "litr2n_vr": 8,
+      "soil1c_vr": 7,
+      "soil1n_vr": 7,
+      "soil1p_vr": 7,
+      "litr3n_vr": 5,
+      "labilep_vr": 5,
+      "secondp_vr": 2.5,
+      "solutinorganicp_vr": 5,
+      "litr3p_vr": 5
+    },
+    "scalar_weights": {
+      "GPP": 1.5,
+      "NPP": 1.5,
+      "AR": 1.2,
+      "HR": 1.2
+    }
+  },
+  "tail_aware_config": {
+    "loss": "log1p_huber",
+    "epsilon": 1e-08,
+    "huber_delta": 1
+  },
+  "tail_aware_weights": {
+    "cpool": 6,
+    "deadstemc": 6,
+    "deadcrootc": 5,
+    "livestemc": 5,
+    "livecrootc": 5,
+    "npool": 8,
+    "ppool": 8,
+    "totvegc": 6,
+    "leafc": 6,
+    "leafn": 6,
+    "leafp": 6,
+    "leafc_storage": 5,
+    "leafn_storage": 5,
+    "leafp_storage": 5,
+    "deadstemp": 5,
+    "deadstemn": 6,
+    "frootc": 4,
+    "frootn": 4,
+    "frootp": 4,
+    "labilep": 5,
+    "labilep_vr": 5,
+    "primp_vr": 11,
+    "litr2c_vr": 8,
+    "litr2p_vr": 8,
+    "litr2n_vr": 8,
+    "litr3c_vr": 5,
+    "litr3p_vr": 5,
+    "litr3n_vr": 5,
+    "soil1c_vr": 7,
+    "soil1n_vr": 7,
+    "soil1p_vr": 7
+  },
+  "pft_mask_config": {
+    "mask_absent_pfts": true,
+    "pft_presence_threshold": 0
+  },
+  "pft_zero_sparsity_config": {
+    "weight": 1,
+    "threshold": 1e-08
+  },
+  "pft_zero_sparsity_weights": {
+    "cpool": 2,
+    "deadstemc": 1,
+    "deadcrootc": 1,
+    "livestemc": 1,
+    "livecrootc": 1
+  },
+  "pft1d_activation_overrides": {
+    "cpool": "abs",
+    "deadstemc": "abs",
+    "deadcrootc": "abs",
+    "livestemc": "abs",
+    "livecrootc": "abs"
+  }
+}