DES-Y3 3x2pt

examples/desy3/pipeline.yml

@@ -1,78 +1,187 @@
 
 # Stages to run
 stages:
+  # INGESTION
+  - name: TXIngestDESY3Gold
+  - name: TXIngestDESY3Footprint
+  - name: TXIngestDESY3SpeczCat
+  - name: TXIngestDESY3Shear
+  - name: TXCustomMask
 
-    - name: TXSourceSelectorMetacal # select and split objects into source bins
-    - name: TXShearCalibration   # Calibrate and split the source sample tomographically
-    #- name: TXPhotozPlots
-    #- name: TXPhotozSourceStack 
-    #- name: TXTruthLensCatalogSplitter
-    #- name: TXStarCatalogSplitter  # Split the star catalog into separate bins (psf/non-psf)    
-    #- name: TXSourceMaps
-      #nprocess: 6
-      #threads_per_process: 1
-      #nodes: 1
-    #- name: TXLensMaps
-    #- name: TXAuxiliarySourceMaps  # make PSF and flag maps
-    #- name: TXAuxiliaryLensMaps    # make depth and bright object maps
-    #- name: TXSourceNoiseMaps    # noise maps for sources using random rotations
-    # stage below is not working because TXAuxiliaryLensMaps is broken
-    #- name: TXLensNoiseMaps      # noise maps for lenses using density splits
-    #- name: TXDensityMaps        # turn mask and ngal maps into overdensity maps
-    #- name: TXMapPlots           # make pictures of all the maps
-    #- name: TXTracerMetadata     # collate metadata
-    #- name: TXRandomCat
-    #- name: TXJackknifeCenters   # Split the area into jackknife regions
-    #- name: TXTwoPoint           # Compute real-space 2-point correlations
-    #  threads_per_process: 128
-    #- name: TXBlinding           # Blind the data following Muir et al
-        #threads_per_process: 2
-    #- name: TXTwoPointTheoryReal # compute theory using CCL to save in sacc file and plot later
-    #- name: TXTwoPointPlots      # Make plots of 2pt correlations
-    #- name: TXSourceDiagnosticPlots    # Make a suite of diagnostic plots
-      #nprocess: 8 
-      #nodes: 2
-    #- name: TXLensDiagnosticPlots    # Make a suite of diagnostic plots
-    #- name: TXGammaTFieldCenters # Compute and plot gamma_t around center points # ERROR: EXPOSURE FILE
-      ##threads_per_process: 2
-    #- name: TXGammaTStars  # Compute and plot gamma_t around bright stars
-      #threads_per_process: 2
-    #- name: TXGammaTRandoms      # Compute and plot gamma_t around randoms
-      #threads_per_process: 2
-    #- name: TXRoweStatistics     # Compute and plot Rowe statistics
-    #  threads_per_process: 128
-    #- name: TXFocalPlanePlot      # mean PSF ellipticity and mean residual on the focal plane 
-      #- name: TXGalaxyStarDensity  # Compute and plot the star-galaxy density cross-correlation
-      #- name: TXGalaxyStarShear    # Compute and plot the star-galaxy shear cross-correlation
-    #- name: TXPSFDiagnostics     # Compute and plots other PSF diagnostics
-    #- name: TXBrighterFatterPlot # Make plots tracking the brighter-fatter effect
-    #- name: TXConvergenceMaps    # Make convergence kappa maps from g1, g2 maps #ERROR: WLMASSMAP
-    #- name: TXConvergenceMapPlots # Plot the convergence map
-    #- name: TXMapCorrelations    # plot the correlations between systematics and data # ERROR: NEEDS CONVERGENCE MAP
-    #- name: TXApertureMass        # Compute aperture-mass statistics
-      ##threads_per_process: 2
-    # Disabled since not yet synchronised with new Treecorr MPI
-    # - name: TXSelfCalibrationIA   # Self-calibration intrinsic alignments of galaxies
-    #
-    # Disabling these as they takes too long for a quick test.
-    # The latter three need NaMaster
-
-      ##- name: TXRealGaussianCovariance   # Compute covariance of real-space correlations
-      ##- name: TXTwoPointFourier          # Compute power spectra C_ell
-      ##- name: TXFourierNamasterCovariance # Compute the C_ell covariance
-      ##- name: TXFourierTJPCovariance     # Compute the C_ell covariance with TJPCov 
+  # PER-OBJECT PHOTO-Z
+
+  # Train the DNF model on the spectroscopic sample
+  - name: PZPrepareEstimatorLens 
+    classname: DNFInformer 
+    aliases:
+        input: spectroscopic_catalog
+        model: lens_photoz_model
+
+  # Apply the trained DNF model to the photometric sample to get lens photo-z
+  - name: PZEstimatorLens 
+    classname: DNFEstimator
+    nprocess: 64
+    threads_per_process: 1
+    aliases:
+        model: lens_photoz_model
+        input: photometry_catalog
+        output: lens_photoz_pdfs
+
+  # Train the DNF model on the spectroscopic sample. This uses a restricted set of bands
+  # measured under metacalibration
+  - name: PZPrepareEstimatorSource
+    classname: DNFInformer 
+    aliases:
+        input: spectroscopic_catalog
+        model: source_photoz_model
+
+  # Apply the trained DNF model to the metacalibration photometric sample to get source photo-z
+  - name: PZEstimatorSource
+    nprocess: 64
+    threads_per_process: 1
+    classname: DNFEstimator
+    aliases:
+        model: source_photoz_model
+        input: shear_catalog
+        output: source_photoz_pdfs
+
+  # SELECTION & CALIBRATION
+
+  # DES-Y3 maglim selection
+  - name: TXCustomLensSelector
+    nprocess: 32
+
+  # Split the lens catalog into tomographic bins
+  - name: TXLensCatalogSplitter
+    nprocess: 8
+
+  # Compute correlations between survey properties and lens density
+  - name: TXLSSDensityNullTests
+    threads_per_process: 8
+
+  # Compute weights that mitigate correlations between survey properties and lens density
+  - name: TXLSSWeightsLinBinned
+    threads_per_process: 8
+
+  # Train a tomography model from the spectroscopic sample for the sources
+  - name: TXSourceTomography
+    threads_per_process: 8
+
+  # Compute metacalibration shear response and selection bias corrections
+  - name: TXSourceSelectorMetacal
+    nprocess: 32
+    threads_per_process: 4
+
+  # Generate calibrated tomographic source catalogs
+  - name: TXShearCalibration
+    nproces: 8
+    threads_per_process: 4
+
+  # PHOTO-Z SUMMARIZATION
+
+  # Compute the ensemble n(z) for the lens bins using point estimate histograms
+  - name: PZRailSummarizeLens
+    classname: PZRailPZSummarize
+    nprocess: 32
+    threads_per_process: 4
+    aliases:
+        tomography_catalog: lens_tomography_catalog_unweighted
+        photoz_pdfs: lens_photoz_pdfs
+        model: lens_photoz_model
+        photoz_stack: lens_photoz_stack
+        photoz_realizations: lens_photoz_realizations
+
+  # Compute the ensemble n(z) for the source bins using point estimate histograms
+  - name: PZRailSummarizeSource
+    classname: PZRailPZSummarize
+    nprocess: 32
+    threads_per_process: 4
+    aliases:
+        tomography_catalog: shear_tomography_catalog
+        photoz_pdfs: source_photoz_pdfs
+        model: source_photoz_model
+        photoz_stack: shear_photoz_stack
+        photoz_realizations: source_photoz_realizations
+
+  # Plot the source n(z) distribution
+  - name: TXPhotozPlotSource
+    classname: TXPhotozPlot
+    aliases:
+        photoz_stack: shear_photoz_stack
+        nz_plot: nz_source
+
+  # Plot the lens n(z) distribution
+  - name: TXPhotozPlotLens
+    classname: TXPhotozPlot
+    aliases:
+        photoz_stack: lens_photoz_stack
+        nz_plot: nz_lens
+
+  # Plot realizations of the source n(z) distribution
+  - name: PZRealizationsPlotSource
+    classname: PZRealizationsPlot
+    aliases:
+        photoz_realizations: source_photoz_realizations
+        photoz_realizations_plot: source_photoz_realizations_plot
+
+
+  # TWO-POINT ESTIMATION
+
+  # Generate random catalogs for the lens sample, for use in two-point estimation
+  - name: TXRandomCat
+    nprocess: 32
+    threads_per_process: 4
+
+  # Collate metadata for the samples
+  - name: TXTracerMetadata
+
+  # Choose jackknife centers for two-point estimation speed from randoms
+  - name: TXJackknifeCenters
+
+  # Measure the real-space two-point correlation functions
+  - name: TXTwoPoint
+    threads_per_process: 128
+
+  # Plot the correlations functions
+  - name: TXTwoPointPlots
+
+  # Pretend to blind the two-point functions
+  - name: TXNullBlinding
+  # - name: TXRealGaussianCovariance
+
+  # MAPPING
+
+  # Make PSF and similar maps for the sources
+  - name: TXAuxiliarySourceMaps
+
+  # Make maps of the lens counts
+  - name: TXLensMaps
+
+  # Make maps of the lens over-density
+  - name: TXDensityMaps
+
+  # Make maps of the shear field
+  - name: TXSourceMaps
+
+  # Plot all the maps
+  - name: TXMapPlots  
 
-#===========
 
 launcher:
     name: mini
     interval: 1.0
 
 site:
-    name: local 
+    name: nersc-interactive
     max_threads: 128
 
-modules: txpipe
+modules: >
+    txpipe
+    rail.estimation.algos.dnf
+    rail.estimation.algos.naive_stack
+    rail.estimation.algos.point_est_hist
+    rail.estimation.algos.uniform_binning 
+
 
 # where to find any modules that are not in this repo,
 # and any other code we need.
@@ -88,22 +197,14 @@ config: examples/desy3/config.yml
 
 # These are overall inputs to the whole pipeline, not generated within it
 inputs:
-    shear_catalog: //global/cfs/cdirs/lsst/groups/WL/projects/txpipe-sys-tests/des-y3/shear_catalog_desy3_unmasked_withfakez_v2.h5
-    photometry_catalog: 
-    #shear_tomography_catalog: /global/cfs/cdirs/desc-wl/projects/txpipe-sys-tests/des-y3/shear_tomography_desy3_unmasked_test.h5
-    lens_tomography_catalog: /global/cfs/cdirs/lsst/groups/WL/projects/txpipe-sys-tests/des-y1/lens_tomography_catalog_desy1_RM.h5
-    lens_photoz_stack: 
-    mask: /global/cfs/cdirs/lsst/groups/WL/projects/txpipe-sys-tests/des-y1/mask_desy1.h5
-    star_catalog: /global/cfs/cdirs/lsst/groups/WL/projects/txpipe-sys-tests/des-y3/DES_psf_y3_catalog.h5
-    calibration_table: data/example/inputs/sample_cosmodc2_w10year_errors.dat
+    des_photometry_catalog: data/desy3/inputs/DESY3_GOLD_2_2.1.h5
+    des_shear_catalog: data/desy3/inputs/DESY3_metacal_v03-004.h5
+    des_specz_catalog: data/desy3/inputs/y6gold22_trainnov20_matchedtoy3.fits
     fiducial_cosmology: data/fiducial_cosmology.yml
-    random_cats_source: Null
-    shear_tomography_classifier: none
-    flow: data/example/inputs/example_flow.pkl
 
 # if supported by the launcher, restart the pipeline where it left off
 # if interrupted
-resume: False
+resume: resume
 
 # where to put output logs for individual stages
 log_dir: data/desy3/logs

examples/lssweights/des-y1/config_binned.yml

@@ -11,15 +11,15 @@ global:
     sparse: True  # Generate sparse maps - faster if using small areas
 
 TXLSSDensityNullTests:
-    supreme_path_root: /global/cfs/cdirs/lsst/groups/WL/users/jelvinpo/txpipe-weights/des-y1/spmaps/
+    supreme_path_root: data/desy1//spmaps/
     outlier_fraction: 0.01  # Will remove this fraction of the maps is most extreme regions when computing 1d trends
     nbin: 10                # number of bins per SP map to compute 1d trends
     simple_cov: True        # If True will use diagonal shot noise covariance
     diag_blocks_only: True  # If True and simple_cov=False, will compute only diagonal blocks of covariance
     b0: [1.4, 1.6, 1.6, 1.93, 1.98]
 
 TXLSSWeightsLinBinned:
-    supreme_path_root: /global/cfs/cdirs/lsst/groups/WL/users/jelvinpo/txpipe-weights/des-y1/spmaps/
+    supreme_path_root: data/desy1//spmaps/
     outlier_fraction: 0.01  # Will remove this fraction of the maps is most extreme regions when computing 1d trends
     nbin: 10                # number of bins per SP map to compute 1d trends
     pvalue_threshold: 0.05  # max p-value for maps to be corrected (0 is no maps, 1 is all maps)
@@ -30,7 +30,7 @@ TXTwoPoint:
     do_pos_pos: True
     do_shear_shear: False
     do_shear_pos: False
-    flip_g2: True  # use true when using metacal shears
+    flip_g2: True  # use true when using metacal sh ears
     min_sep: 2.5
     max_sep: 250.0
     nbins: 20

examples/lssweights/des-y1/pipeline_binned.yml

@@ -23,7 +23,7 @@ stages:
 modules: txpipe 
 
 # Where to put outputs
-output_dir: examples/lssweights/des-y1/output_binned/
+output_dir: data/desy1/output
 
 # How to run the pipeline: mini, parsl, or cwl
 launcher:
@@ -32,20 +32,20 @@ launcher:
 
 # Where to run the pipeline: cori-interactive, cori-batch, or local
 site:
-    name: nersc-interactive
+    name: local
 
 # configuration settings
-config: examples/lssweights/des-y1//config_binned.yml
+config: examples/lssweights/des-y1/config_binned.yml
 
 inputs:
     # See README for paths to download these files
-    photometry_catalog: /global/cfs/cdirs/lsst/groups/WL/projects/txpipe-sys-tests/des-y1/photometry_catalog_desy1_RM.h5
-    lens_tomography_catalog_unweighted: /global/cfs/cdirs/lsst/groups/WL/users/jelvinpo/txpipe-weights/des-y1/lens_tomography_catalog_desy1_RM_unweighted.h5
-    random_cats: /global/cfs/cdirs/lsst/groups/WL/projects/txpipe-sys-tests/des-y1/randoms_desy1_RM.hdf5
-    binned_random_catalog: /global/cfs/cdirs/lsst/groups/WL/projects/txpipe-sys-tests/des-y1/binned_randoms_desy1_RM.hdf5
-    binned_random_catalog_sub: /global/cfs/cdirs/lsst/groups/WL/projects/txpipe-sys-tests/des-y1/binned_randoms_desy1_RM.hdf5
-    mask: /global/cfs/cdirs/lsst/groups/WL/users/jelvinpo/txpipe-weights/des-y1/mask_desy1_4096.h5
-    lens_photoz_stack: /global/cfs/cdirs/lsst/groups/WL/projects/txpipe-sys-tests/des-y1/lens_photoz_stack.hdf5
+    photometry_catalog: data/desy1/photometry_catalog_desy1_RM.h5
+    lens_tomography_catalog_unweighted: data/desy1/lens_tomography_catalog_desy1_RM_unweighted.h5
+    random_cats: data/desy1/randoms_desy1_RM.hdf5
+    binned_random_catalog: data/desy1/binned_randoms_desy1_RM.hdf5
+    binned_random_catalog_sub: data/desy1/binned_randoms_desy1_RM.hdf5
+    mask: data/desy1/mask_desy1_4096.h5
+    lens_photoz_stack: data/desy1/lens_photoz_stack.hdf5
     shear_catalog: none
     shear_tomography_catalog: none
     binned_shear_catalog: none
@@ -62,6 +62,6 @@ inputs:
 # if interrupted
 resume: False
 # where to put output logs for individual stages
-log_dir: examples/lssweights/des-y1/output_binned/logs/
+log_dir: lssweights_logs
 # where to put an overall parsl pipeline log
-pipeline_log: examples/lssweights/des-y1/output_binned/log.txt
+pipeline_log: ./log.txt

txpipe/base_stage.py

@@ -1,4 +1,4 @@
-from ceci import PipelineStage as PipelineStageBase
+from ceci import PipelineStage as PipelineStageBase, file_types
 from .data_types import HDFFile
 from textwrap import dedent
 from .utils.provenance import find_module_versions, git_diff, git_current_revision
@@ -183,13 +183,17 @@ def open_output(self, tag, wrapper=False, **kwargs):
         new_tag = self.get_aliased_tag(tag)
         path = self.get_output(new_tag)
 
-        # HDF files can be opened for parallel writing
+        # HDF files and directory outputs can be opened for parallel writing
         # under MPI.  This checks if:
         # - we have been told to open in parallel
         # - we are actually running under MPI
         # and adds the flags required if all these are true
         run_parallel = kwargs.pop("parallel", False) and self.is_mpi()
-        if run_parallel:
+        if run_parallel and issubclass(output_class, file_types.Directory):
+            kwargs["parallel"] = True
+            kwargs["comm"] = self.comm
+
+        elif run_parallel:
             if not output_class.supports_parallel_write:
                 raise ValueError(
                     f"Tried to open file for parallel output, but not"

txpipe/ingest/base.py

@@ -282,7 +282,7 @@ def process_maps(self, input_filepaths, input_labels, output_name):
         metadata = {
             "pixelization": "healpix",
             "nside": self.config["input_nside"],
-            "nest": False,  # currently TXPipe defaults to ring format
+            "nest": True,  # currently TXPipe defaults to ring format
         }
         print(f"Input nside {self.config['input_nside']}")