Second version making changes to use photerr

examples/2.2i/config.yml

@@ -66,6 +66,10 @@ TXSourceSelector:
     r_i_cut: 2.0
     # may also need one for r_cpar_cut
 
+TXSourceTomography:
+    bands: riz #used for selection
+    source_zbin_edges: [0.3, 0.5, 0.7, 0.9, 2.0]
+
 TXRandomCat:
     chunk_rows: 100000
     density: 10 # gals per sq arcmin

examples/2.2i/pipeline.yml

@@ -27,6 +27,8 @@ stages:
     nprocess: 32
   - name: TXSourceSelector
     nprocess: 32
+  - name: TXSourceTomography
+    threads_per_process: 4
   - name: TXShearCalibration
     nodes: 1
     nprocess: 8

examples/2.2i/pipeline_1tract.yml

@@ -24,6 +24,7 @@ stages:
   - name: TXSourceNoiseMaps
   - name: TXLensNoiseMaps
   - name: TXSourceSelector
+  - name: TXSourceTomography
   - name: PZRailEstimateSource
   - name: PZRailEstimateLensFromSource
   - name: TXShearCalibration

examples/cosmodc2/Cluster_pipelines/config-1deg2-CL.yml

@@ -11,6 +11,10 @@ TXSourceSelectorMetadetect:
     shear_prefix: ''
 
 
+TXSourceTomography:
+    bands: riz #used for selection
+    source_zbin_edges: [0.1, 3.0]
+
 Inform_BPZ_lite:
     zmin: 0.0
     zmax: 3.0

examples/cosmodc2/Cluster_pipelines/config-20deg2-CL.yml

@@ -10,6 +10,10 @@ TXSourceSelectorMetadetect:
     true_z: false
     shear_prefix: ''
 
+TXSourceTomography:
+    bands: riz #used for selection
+    source_zbin_edges: [0.1, 3.0]
+
 BPZliteInformer:
     zmin: 0.0
     zmax: 3.0

examples/cosmodc2/Cluster_pipelines/pipeline-20deg2-CL-in2p3.yml

@@ -23,6 +23,8 @@ python_paths: []
 stages:
   - name: TXSourceSelectorMetadetect
     nprocess: 30
+  - name: TXSourceTomography
+    threads_per_process: 4
   - name: BPZliteInformer
     nprocess: 1
     aliases:

examples/cosmodc2/Cluster_pipelines/pipeline-20deg2-CL-laptop.yml

@@ -23,6 +23,8 @@ python_paths: []
 stages:
   - name: TXSourceSelectorMetadetect
     nprocess: 6
+  - name: TXSourceTomography
+    threads_per_process: 4
   - name: BPZliteInformer
     nprocess: 1
     aliases:

examples/cosmodc2/Cluster_pipelines/pipeline-20deg2-CL-nersc.yml

@@ -22,6 +22,8 @@ python_paths: []
 stages:
   - name: TXSourceSelectorMetadetect
     nprocess: 30
+  - name: TXSourceTomography
+    threads_per_process: 4
   - name: BPZliteInformer
     nprocess: 1
     aliases:

examples/cosmodc2/config-20deg2.yml

@@ -116,6 +116,10 @@ TXSourceSelectorMetadetect:
     true_z: false
     shear_prefix: ''
 
+TXSourceTomography:
+    bands: riz #used for selection
+    source_zbin_edges: [0.19285902, 0.40831394, 0.65503818, 0.94499109, 1.2947086, 1.72779632, 2.27855242, 3.]  # 7 bins
+
 TXDiagnosticQuantiles:
     nbins: 20
     psf_prefix: 00/mcal_psf_

examples/cosmodc2/config.yml

@@ -102,6 +102,10 @@ TXSourceSelectorMetadetect:
     shear_prefix: ''
 
 
+TXSourceTomography:
+    bands: riz #used for selection
+    source_zbin_edges: [0.19285902, 0.40831394, 0.65503818, 0.94499109, 1.2947086, 1.72779632, 2.27855242, 3.]  # 7 bins
+
 TXRandomCat:
     density: 10 # gals per sq arcmin
 

examples/cosmodc2/config_no_shape_noise.yml

@@ -98,6 +98,10 @@ TXSourceSelector:
     true_z: False
     shear_prefix: mcal_
 
+TXSourceTomography:
+    bands: riz #used for selection
+    source_zbin_edges: [0.19285902, 0.40831394, 0.65503818, 0.94499109, 1.2947086, 1.72779632, 2.27855242, 3. ] # 7 bins
+
 TXRandomCat:
     chunk_rows: 100000
     density: 10 # gals per sq arcmin

examples/cosmodc2/ingest-small-config.yml

@@ -107,6 +107,10 @@ TXSourceSelectorMetadetect:
     shear_prefix: ''
 
 
+TXSourceTomography:
+    bands: riz #used for selection
+    source_zbin_edges: [0.19285902, 0.40831394, 0.65503818, 0.94499109, 1.2947086, 1.72779632, 2.27855242, 3. ] # 7 bins
+
 TXRandomCat:
     chunk_rows: 100000
     density: 10 # gals per sq arcmin

examples/cosmodc2/pipeline-20deg2-laptop.yml

@@ -16,6 +16,8 @@ stages:
     nprocess: 12
   - name: TXSourceSelectorMetadetect
     nprocess: 12
+  - name: TXSourceTomography
+    threads_per_process: 4
   - name: TXLSSWeightsUnit
   - name: TXRandomForestLensSelector
     nprocess: 12

examples/cosmodc2/pipeline-20deg2-nersc.yml

@@ -16,6 +16,8 @@ stages:
     nprocess: 32
   - name: TXSourceSelectorMetadetect
     nprocess: 32
+  - name: TXSourceTomography
+    threads_per_process: 4
   - name: TXLSSWeightsUnit
   - name: TXRandomForestLensSelector
     nprocess: 32

examples/cosmodc2/pipeline-pz.yml

@@ -32,6 +32,8 @@ stages:
         output: spectroscopic_catalog
   - name: TXSourceSelectorMetacal
     nprocess: 32
+  - name: TXSourceTomography
+    threads_per_process: 4
   - name: TXShearCalibration
     nprocess: 8
   - name: PZPrepareEstimatorLens     # Prepare the p(z) estimator

examples/cosmodc2/pipeline.yml

@@ -32,6 +32,8 @@ stages:
         output: spectroscopic_catalog
   - name: TXSourceSelectorMetacal
     nprocess: 32
+  - name: TXSourceTomography
+    threads_per_process: 4
   - name: TXShearCalibration
     nprocess: 32
   - name: PZPrepareEstimatorLens     # Prepare the p(z) estimator

examples/cosmodc2/pipeline_redmagic.yml

@@ -17,6 +17,8 @@ stages:
   - name: TXSourceSelectorMetacal
     nprocess: 64
     nodes: 2
+  - name: TXSourceTomography
+    threads_per_process: 4
   - name: TXShearCalibration
     nprocess: 7
     nodes: 1

examples/cosmodc2/pipeline_redmagic_full.yml

@@ -22,6 +22,8 @@ stages:
   - name: TXSourceSelectorMetadetect
     nprocess: 64
     nodes: 2
+  - name: TXSourceTomography
+    threads_per_process: 4
   - name: TXShearCalibration
     nprocess: 7
     nodes: 1

examples/cosmodc2/pipeline_redmagic_no_shape_noise.yml

@@ -17,6 +17,8 @@ stages:
     - name: TXSourceSelector
       nprocess: 64
       nodes: 3
+    - name: TXSourceTomography
+      threads_per_process: 4
     - name: TXShearCalibration
       nprocess: 7
       nodes: 1

examples/cosmodc2/pipeline_redmagic_no_shape_noise_full.yml

@@ -21,6 +21,8 @@ stages:
     - name: TXSourceSelector
       nprocess: 64
       nodes: 2
+    - name: TXSourceTomography
+      threads_per_process: 4
     - name: TXSourceTrueNumberDensity
     - name: TXPhotozPlot
     - name: TXJackknifeCenters

examples/desy1/config.yml

@@ -66,6 +66,10 @@ TXTruthLensCatalogSplitter: {}
 TXStarCatalogSplitter: {} 
 TXPhotozPlot: {}
 
+TXSourceTomography:
+    bands: riz #used for selection
+    source_zbin_edges: [0.2, 0.43, 0.63, 0.90, 1.30]
+
 TXSourceMaps:
     sparse: True  # Generate sparse maps - faster if using small areas
 

examples/desy1/pipeline.yml

@@ -102,6 +102,7 @@ inputs:
     # For the self-calibration extension we are not using Random_cat_source for now
     # So it is set to Null, so the yaml intepreter returns a None value to python. 
     random_cats_source: Null
+    shear_tomography_classifier: none
     mask: /global/cfs/cdirs/lsst/groups/WL/projects/txpipe-sys-tests/des-y1/mask_desy1.h5
     #random_cats: /global/cfs/cdirs/lsst/groups/WL/projects/txpipe-sys-tests/des-y1/randoms_desy1_RM.hdf5
     #binned_random_catalog: /global/cfs/cdirs/lsst/groups/WL/projects/txpipe-sys-tests/des-y1/binned_randoms_desy1_RM.hdf5

examples/desy3/config.yml

@@ -82,6 +82,10 @@ TXTruthLensCatalogSplitter: {}
 TXStarCatalogSplitter: {} 
 TXPhotozPlots: {}
 
+TXSourceTomography:
+    bands: riz #used for selection
+    source_zbin_edges: [0.0, 0.2, 0.4, 0.6, 0.8]
+
 TXSourceMaps:
     sparse: True  # Generate sparse maps - faster if using small areas
 

examples/desy3/pipeline.yml

@@ -98,6 +98,7 @@ inputs:
     calibration_table: data/example/inputs/sample_cosmodc2_w10year_errors.dat
     fiducial_cosmology: data/fiducial_cosmology.yml
     random_cats_source: Null
+    shear_tomography_classifier: none
     flow: data/example/inputs/example_flow.pkl
 
 # if supported by the launcher, restart the pipeline where it left off

examples/dp0.2/config.yml

@@ -84,6 +84,10 @@ TXSimpleMask: {}
 
 TXTracerMetadata: {}
 
+TXSourceTomography:
+    bands: ugrizy
+    source_zbin_edges: [0.3, 0.5, 0.7, 0.9]
+
 TXRandomCat:
     method: spherical_projection
 TXJackknifeCenters:

examples/dp0.2/pipeline.yml

@@ -29,6 +29,8 @@ stages:
   - name: TXSourceSelectorHSC
     nprocess: 32
     threads_per_process: 2
+  - name: TXSourceTomography
+    threads_per_process: 4
   - name: TXPhotozSourceStack
     nprocess: 32
     threads_per_process: 2

examples/dp1/config.yml

@@ -69,10 +69,14 @@ TXSourceSelectorSimple:
     s2n_cut: 10.0
     source_zbin_edges: [0.5, 0.7, 0.9, 1.1, 2.0]
     shear_prefix: ''
+
+
+TXSourceTomography:
+    bands: ugrizy
+    source_zbin_edges: [0.5, 0.7, 0.9, 1.1, 2.0]
     spec_mag_column_format: "{band}_cModelMag"
     spec_redshift_column: redshift
 
-
 TXRandomForestLensSelector:
     verbose: true
     bands: ugrizy

examples/dp1/config_ww.yml

@@ -69,10 +69,14 @@ TXSourceSelectorSimple:
     s2n_cut: 10.0
     source_zbin_edges: [0.5, 0.7, 0.9, 1.1, 2.0]
     shear_prefix: ''
+
+
+TXSourceTomography:
+    bands: ugrizy
+    source_zbin_edges: [0.5, 0.7, 0.9, 1.1, 2.0]
     spec_mag_column_format: "{band}_cModelMag"
     spec_redshift_column: redshift
 
-
 TXRandomForestLensSelector:
     verbose: true
     bands: ugrizy

examples/dp1/pipeline-edfs.yml

@@ -23,6 +23,7 @@ stages:
   - name: TXSourceSelectorSimple
     nprocess: 1
     threads_per_process: 1
+  - name: TXSourceTomography
   - name: TXShearCalibration
   - name: TXLSSWeightsUnit
   - name: TXRandomForestLensSelector

examples/dp1/pipeline.yml

@@ -41,6 +41,7 @@ stages:
   - name: TXSourceSelectorSimple
     nprocess: 1
     threads_per_process: 1
+  - name: TXSourceTomography
   - name: TXShearCalibration
   - name: TXLSSWeightsUnit
   - name: TXRandomForestLensSelector

examples/dp1/pipeline_ww.yml

@@ -39,6 +39,7 @@ stages:
   - name: TXSourceSelectorSimple
     nprocess: 1
     threads_per_process: 1
+  - name: TXSourceTomography
   - name: TXShearCalibration
   - name: TXLSSDensityNullTests
   #- name: TXLSSWeightsUnit

examples/ext_cross_corr/config_1deg.yml

@@ -175,6 +175,10 @@ TXSourceSelectorMetadetect:
     shear_prefix: ""
     true_z: False
 
+TXSourceTomography:
+    bands: riz #used for selection
+    source_zbin_edges: [0.5, 0.7, 0.9, 1.1, 2.0]
+
 TXTruthLensSelector:
     # Mag cuts
     input_pz: False

examples/ext_cross_corr/pipeline_1deg.yml

@@ -20,6 +20,7 @@ stages:
     - name: PZRailSummarizeSource  # Run the DIR method on the lens sample to find n(z)
       classname: PZRailSummarize
     - name: TXSourceSelectorMetadetect  # select and split objects into source bins
+    - name: TXSourceTomography
     - name: NZDirInformerSource     # Prepare the DIR method inputs for the source sample
       classname: NZDirInformer
     - name: TXShearCalibration     # Calibrate and split the source sample tomographically

examples/gaussian_sims/config.yml

@@ -78,6 +78,10 @@ TXSourceSelectorMetadetect:
     true_z: True
     shear_prefix: ''
 
+TXSourceTomography:
+    bands: riz #used for selection
+    source_zbin_edges: [0.19285902, 0.40831394, 0.65503818, 0.94499109, 1.2947086, 1.72779632] # 5 bins
+
 TXRandomCat:
     chunk_rows: 10000000 
     density: 3 # gals per sq arcmin

examples/gaussian_sims/pipeline_gaussian_sims.yml

@@ -22,6 +22,8 @@ stages:
     - name: TXSourceSelectorMetadetect
       nprocess: 64
       nodes: 2
+    - name: TXSourceTomography
+      threads_per_process: 4
     - name: TXShearCalibration
       nprocess: 7
       nodes: 1

examples/hscy1/config.yml

@@ -79,6 +79,10 @@ TXTruthLensCatalogSplitter: {}
 TXStarCatalogSplitter: {} 
 TXPhotozPlots: {}
 
+TXSourceTomography:
+    bands: i #used for selection
+    source_zbin_edges: [0.3, 0.6, 0.9, 1.2, 1.5]
+
 TXSourceMaps:
     sparse: True  # Generate sparse maps - faster if using small areas
 

examples/hscy1/hsc_pipeline.yaml

@@ -52,6 +52,9 @@ inputs:
     # For the self-calibration extension we are not using Random_cat_source for now
     # So it is set to Null, so the yaml intepreter returns a None value to python. 
     random_cats_source: Null
+    # this is required by the source selector but not used because we
+    # use the photo-z values for selection instead of the random forest classifier.
+    shear_tomography_classifier: none
 
 # if supported by the launcher, restart the pipeline where it left off
 # if interrupted

examples/hscy3/config.yml

@@ -146,6 +146,10 @@ TXSourceSelectorMetacal:
     shear_prefix: mcal_
     use_diagonal_response: true
 
+TXSourceTomography:
+    bands: i    #used for selection
+    source_zbin_edges: [0.5, 1.5, 2.5, 3.5, 4.5]
+
 TXShearCalibration:
     use_true_shear: false
     chunk_rows: 100000

examples/hscy3/hsc_pipeline.yaml

@@ -116,6 +116,7 @@ inputs:
     star_catalog: /global/cfs/cdirs/lsst/groups/WL/users/yomori/repo/nulltests_txpipe/hscy1/star_catalog_hscy1_allfields.h5
     fiducial_cosmology: data/fiducial_cosmology.yml
     random_cats_source:
+    shear_tomography_classifier: none
     random_cats: /global/cfs/cdirs/lsst/groups/WL/projects/txpipe-sys-tests/hsc-y3/shear/random_cats.hdf5
 
     binned_random_catalog: /global/cfs/cdirs/lsst/groups/WL/projects/txpipe-sys-tests/hsc-y3/shear/random_cats.hdf5

examples/kids-1000/config.yml

@@ -81,6 +81,10 @@ TXTruthLensCatalogSplitter: {}
 TXStarCatalogSplitter: {} 
 TXPhotozPlots: {}
 
+TXSourceTomography:
+    bands: riz #used for selection
+    source_zbin_edges: [0.2, 0.43, 0.63, 0.90, 1.30]
+
 TXSourceMaps:
     sparse: True  # Generate sparse maps - faster if using small areas