Updating noise levels

bin/cut_out_example.py

@@ -10,34 +10,41 @@
 
 
 def trim_file(input_file, output_file, group_name, thin):
+    if isinstance(group_name, str):
+        group_name = [group_name]
     file_in = h5py.File(input_file, 'r')
     file_out = h5py.File(output_file, 'w')
 
-    group_in = file_in[group_name]
-    group_out = file_out.create_group(group_name)
 
     file_in.copy('provenance', file_out)
 
-    ra = group_in['ra'][:]
-    print("read RA", ra.size)
-    dec = group_in['dec'][:]
-    print("read dec")
-    w  = ra > ra_min
-    w &= ra < ra_max
-    w &= dec > dec_min
-    w &= dec < dec_max
-    del ra, dec
-    w[np.arange(w.size) % thin > 0] = False
+    for group_name in group_name:
+        group_in = file_in[group_name]
+        group_out = file_out.create_group(group_name)
 
-    print('Output count:', w.sum())
+        ra = group_in['ra'][:]
+        print("read group", group_name, "size:", ra.size)
+        dec = group_in['dec'][:]
+        w  = ra > ra_min
+        w &= ra < ra_max
+        w &= dec > dec_min
+        w &= dec < dec_max
+        del ra, dec
 
-    for col in group_in.keys():
-        print("Copying", col)
-        d = group_in[col][w]
-        group_out.create_dataset(col, data=d)
+        if thin > 1:
+            w[np.arange(w.size) % thin > 0] = False
+
+        print('Output count:', w.sum())
+
+        for col in group_in.keys():
+            print("Copying", col)
+            d = group_in[col][w]
+            group_out.create_dataset(col, data=d)
 
 
 if __name__ == '__main__':
-    trim_file('photometry_catalog.hdf5', 'example_photometry_catalog.hdf5', 'photometry', 5)
-    trim_file('shear_catalog.hdf5', 'example_shear_catalog.hdf5', 'photometry', 5)
-    trim_file('star_catalog.hdf5', 'example_star_catalog.hdf5', 'stars', 5)
+    trim_file('photometry_catalog.hdf5', 'example_photometry_catalog.hdf5', 'photometry', 1)
+
+    shear_groups = ["shear/00", "shear/1p", "shear/1m", "shear/2p", "shear/2m"]
+    trim_file('shear_catalog.hdf5', 'example_shear_catalog.hdf5', shear_groups, 1)
+    # trim_file('star_catalog.hdf5', 'example_star_catalog.hdf5', 'stars', 5)

bin/get_des_y6_response.py

@@ -0,0 +1,63 @@
+from parallel_statistics import ParallelMean
+import h5py
+import os
+
+params = [
+    "gauss_g_1",
+    "gauss_g_2",
+    "gauss_s2n",
+    "pgauss_band_flux_g_nodered",
+    "pgauss_band_flux_r_nodered",
+    "pgauss_band_flux_i_nodered",
+    "pgauss_band_flux_z_nodered",
+    "pgauss_T",
+]
+
+nparam = len(params)
+variants = [
+    "noshear",
+    "1p",
+    "1m",
+    "2p",
+    "2m",
+]
+
+stats = {
+    v: ParallelMean(nparam) for v in variants
+}
+
+results = {}
+delta_gamma = 0.02
+chunk_size = 1_000_000
+
+cfs_dir = os.environ["CFS"]
+filename = os.path.join(cfs_dir, "des", "y6kp-cats", "final_version", "metadetect_notomo2024-11-07.hdf5")
+
+with h5py.File(filename) as f:
+    for v in variants:
+        stat = stats[v]
+        s = 0
+        e = 0
+        ntot = f[f"{v}/ra"].shape[0]
+        while e < ntot:
+            s = e
+            e = min(s + chunk_size, ntot)
+            # print(f"Processing {v} {s}:{e} / {ntot}")
+            data = {p: f[f"{v}/{p}"][s:e] for p in params}
+            flag = [f"{v}/mdet_flags"][s:e]
+            sel = flag == 0
+            for i, p in enumerate(params):
+                stat.add_data(i, data[p][sel])
+        _, mu = stat.collect()
+        print(f"Variant {v}:")
+        for i, p in enumerate(params):
+            print(f"  {p}: {mu[i]}")
+        print("")
+        results[v] = mu
+
+print("")
+for i, p in enumerate(params):
+    dp_dg1 = (results["1p"][i] - results["1m"][i]) / delta_gamma
+    dp_dg2 = (results["2p"][i] - results["2m"][i]) / delta_gamma
+    print(f"d({p}) / dg1: {dp_dg1}")
+    print(f"d({p}) / dg2: {dp_dg2}")

examples/cosmodc2/config.yml

@@ -2,18 +2,20 @@ global:
     pixelization: healpix
     chunk_rows: 1000000
     chunk_size: 200000
+    nside: 512
+    ell_max: 1536
 
 TXLSSWeightsUnit:
-    nside: 2048
     pixelization: healpix
 
 TXDiagnosticQuantiles:
-    psf_prefix: mcal_psf_
-    shear_prefix: mcal_
+    psf_prefix: psf_
+    shear_prefix: ""
     nbins: 20
 
 TXShearCalibration:
-    shear_prefix: mcal_
+    shear_prefix: ""
+
 
 TXCosmoDC2Mock:
     cat_name: cosmoDC2_v1.1.4_image
@@ -43,22 +45,18 @@ TXLensTrueNumberDensity:
 
 TXLensMaps:
     pixelization: healpix
-    nside: 2048
     sparse: true
 
 TXSourceMaps:
-    nside: 2048
     sparse: true
     pixelization: healpix
     true_shear: false
 
 TXExternalLensMaps:
-    nside: 2048
     sparse: true
     pixelization: healpix
 
 TXExternalLensNoiseMaps:
-    nside: 2048
     pixelization: healpix
 
 TXAuxiliarySourceMaps:
@@ -67,27 +65,18 @@ TXAuxiliarySourceMaps:
 
 TXAuxiliaryLensMaps:
     sparse: true
-    nside: 2048
     pixelization: healpix
     bright_obj_threshold: 22.0
 
 TXSimpleMask:
     depth_cut: 23.0
     bright_object_max: 10.0
 
-
-
-TXSourceSelectorMetacal:
-    input_pz: false
+TXSourceTomography:
     bands: riz #used for selection
-    T_cut: 0.5
-    s2n_cut: 10.0
-    max_rows: 1000
-    delta_gamma: 0.02
+    random_seed: 123213
     source_zbin_edges: [0.19285902, 0.40831394, 0.65503818, 0.94499109, 1.2947086, 1.72779632, 2.27855242, 3.]  # 7 bins
     # source_zbin_edges: [0.25588604, 0.55455363, 0.91863365, 1.38232001, 2.] # 4 bins
-    true_z: false
-    shear_prefix: mcal_
 
 TXSourceSelectorMetadetect:
     input_pz: false
@@ -96,10 +85,10 @@ TXSourceSelectorMetadetect:
     s2n_cut: 10.0
     max_rows: 1000
     delta_gamma: 0.02
-    source_zbin_edges: [0.19285902, 0.40831394, 0.65503818, 0.94499109, 1.2947086, 1.72779632, 2.27855242, 3.]  # 7 bins
-    # source_zbin_edges: [0.25588604, 0.55455363, 0.91863365, 1.38232001, 2.] # 4 bins
     true_z: false
     shear_prefix: ''
+    source_zbin_edges: [0.19285902, 0.40831394, 0.65503818, 0.94499109, 1.2947086, 1.72779632, 2.27855242, 3.]  # 7 bins
+    # source_zbin_edges: [0.25588604, 0.55455363, 0.91863365, 1.38232001, 2.] # 4 bins
 
 
 TXRandomCat:
@@ -109,15 +98,16 @@ TXJackknifeCenters:
     npatch: 40
 
 TXSourceDiagnosticPlots:
-    shear_prefix: mcal_
+    shear_prefix: "00/"
+    psf_prefix: "00/psf_"
 
 TXFourierGaussianCovariance:
     galaxy_bias: [1.404, 1.458, 1.693, 1.922, 2.133] # Tinker bias values
-    cache_dir: ./cache_nmt/cosmodc2/nside2048/
+    cache_dir: ./cache_nmt/cosmodc2/nside512/
 
 TXFourierTJPCovariance:
     galaxy_bias: [1.404, 1.458, 1.693, 1.922, 2.133] # Tinker bias values
-    cache_dir: ./cache_nmt/cosmodc2/nside2048/
+    cache_dir: ./cache_nmt/cosmodc2/nside512/
     IA: 0.
 
 TXRealGaussianCovariance:
@@ -131,11 +121,9 @@ TXTwoPointFourier:
     flip_g1: true
     flip_g2: true
     apodization_size: 0.0
-    cache_dir: ./cache_nmt/cosmodc2/nside2048/
+    cache_dir: ./cache_nmt/cosmodc2/nside512/
     true_shear: false
     n_ell: 30
-    ell_max: 6144 # nside * 3 , since Namaster computes that anyway. 
-    nside: 2048
     analytic_noise: true
 
 TXTwoPoint:
@@ -179,6 +167,21 @@ FlowCreator:
     n_samples: 1000000
     seed: 5763248
 
+LSSTErrorModel:
+    renameDict: {u: mag_u_lsst, g: mag_g_lsst, r: mag_r_lsst, i: mag_i_lsst, z: mag_z_lsst, "y": mag_y_lsst}
+    seed: 29
+
+InvRedshiftIncompleteness:
+    pivot_redshift: 1.0
+
+LineConfusion:
+    true_wavelen: 5007.0
+    wrong_wavelen: 3727.0
+    frac_wrong: 0.05
+
+QuantityCut:
+    cuts: {mag_i_lsst: 25.0}
+
 
 GridSelection:
     redshift_cut: 5.1