Leakage from auxiliary observables

.gitignore

@@ -164,3 +164,7 @@ notebooks/calibrate_comprehensive_cat.ipynb
 notebooks/demo_apply_hsp_masks.py
 notebooks/demo_apply_hsp_masks.ipynb
 notebooks/plot_footprints.ipynb
+notebooks/demo_comprehensive_to_minimal_cat.ipynb
+notebooks/demo_create_footprint_mask.ipynb
+notebooks/demo_check_footprint.ipynb
+notebooks/demo_comprehensive_to_minimal_cat.ipynb

notebooks/calibrate_comprehensive_cat.py

@@ -7,7 +7,7 @@
 #       format_version: '1.5'
 #       jupytext_version: 1.15.1
 #   kernelspec:
-#     display_name: sp_validation
+#     display_name: Python 3
 #     language: python
 #     name: python3
 # ---
@@ -260,6 +260,9 @@
 
 add_cols_data["e1_PSF"] = cat_gal["e1_PSF"]
 add_cols_data["e2_PSF"] = cat_gal["e2_PSF"]
+add_cols_data["fwhm_PSF"] = cat.get_col(
+    dat, "fwhm_PSF", mask_combined._mask, mask_metacal
+)
 
 # +
 # Add information to FITS header

notebooks/cosmo_val/cat_config.yaml

@@ -681,7 +681,7 @@ SP_v1.4.1_noleakage:
 
 SP_v1.4.5:
    pipeline: SP
-   subdir: UNIONS/v1.x/ShapePipe/v1.4.x/v1.4.5
+   subdir: v1.4.x/v1.4.5
    ls: dashed
    colour: blue
    marker: d
@@ -698,7 +698,7 @@ SP_v1.4.5:
       e2_col: e2
       e1_PSF_col: e1_PSF
       e2_PSF_col: e2_PSF
-      w_col: w_iv
+      w_col: w_des
       R: 1.0
       redshift_distr: CFIS/v1.0/nz/dndz_SP_A.txt
    star:
@@ -741,17 +741,17 @@ SP_v1.4.5_glass_mock:
       e2_col: e2
       e1_PSF_col: e1_PSF
       e2_PSF_col: e2_PSF
-      w: w
+      w_col: w
       R: 1.0
       redshift_distr: CFIS/v1.0/nz/dndz_SP_A.txt
    star:
-      path: unions_shapepipe_star_2024_v1.4.1.fits
+      path: unions_shapepipe_star_2024_v1.4.a.fits
       ra_col: RA
       dec_col: Dec
       e1_col: e1
       e2_col: e2
    psf:
-      path: unions_shapepipe_psf_2024_v1.4.1.fits
+      path: unions_shapepipe_psf_2024_v1.4.a.fits
       hdu: 1
       ra_col: RA
       dec_col: Dec
@@ -784,17 +784,17 @@ SP_test:
       e2_col: e2
       e1_PSF_col: e1_PSF
       e2_PSF_col: e2_PSF
-      w: w_iv
+      w_col: w_iv
       R: 1.0
       redshift_distr: CFIS/v1.0/nz/dndz_SP_A.txt
    star:
-      path: unions_shapepipe_star_2024_v1.4.1.fits
+      path: unions_shapepipe_star_2024_v1.4.a.fits
       ra_col: RA
       dec_col: Dec
       e1_col: e1
       e2_col: e2
    psf:
-      path: unions_shapepipe_psf_2024_v1.4.1.fits
+      path: unions_shapepipe_psf_2024_v1.4.a.fits
       hdu: 1
       ra_col: RA
       dec_col: Dec
@@ -810,7 +810,7 @@ SP_test:
 
 SP_v1.4.5_leak_corr:
    pipeline: SP
-   subdir: UNIONS/v1.x/ShapePipe/v1.4.x/v1.4.5
+   subdir: v1.4.x/v1.4.5
    ls: dashed
    colour: midnightblue
    marker: d
@@ -851,9 +851,52 @@ SP_v1.4.5_leak_corr:
       star_flag: "FLAG_STAR_HSM"
       square_size: True
 
+SP_v1.5.3:
+   pipeline: SP
+   subdir: CFIS/v1.0/ShapePipe/v1.5.x/v1.5.3
+   ls: dashed
+   colour: blue
+   marker: v
+   getdist_colour: 0.0, 0.5, 1.0
+   cov_th:
+      A: 2420.2651014497287 #deg^2
+      n_e: 7.040818382014773 #arcmin-2
+      n_psf: 0.752316232272063 #arcmin-2
+      sigma_e: 0.30961528707207325
+   shear:
+      path: unions_shapepipe_cut_struc_2024_v1.5.3.fits
+      covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt
+      e1_col: e1
+      e2_col: e2
+      e1_PSF_col: e1_PSF
+      e2_PSF_col: e2_PSF
+      w_col: w_iv
+      R: 1.0
+   star:
+      path: unions_shapepipe_star_2024_v1.5.a.fits
+      ra_col: RA
+      dec_col: Dec
+      e1_col: e1
+      e2_col: e2
+   psf:
+      path: unions_shapepipe_psf_2024_v1.5.a.fits
+      hdu: 1
+      ra_col: RA
+      dec_col: Dec
+      e1_PSF_col: E1_PSF_HSM
+      e2_PSF_col: E2_PSF_HSM
+      e1_star_col: E1_STAR_HSM
+      e2_star_col: E2_STAR_HSM
+      PSF_size: SIGMA_PSF_HSM
+      star_size: SIGMA_STAR_HSM
+      PSF_flag: "FLAG_PSF_HSM"
+      star_flag: "FLAG_STAR_HSM"
+      square_size: True
+
+
 SP_v1.5.4:
    pipeline: SP
-   subdir: CFIS/v1.0/ShapePipe/
+   subdir: CFIS/v1.0/ShapePipe/v1.5.x/v1.5.4
    ls: dashed
    colour: green
    marker: d
@@ -864,7 +907,7 @@ SP_v1.5.4:
       n_psf: 0.752316232272063 #arcmin-2
       sigma_e: 0.30961528707207325
    shear:
-      path: v1.5.x/v1.5.4/unions_shapepipe_cut_struc_2024_v1.5.4.fits
+      path: unions_shapepipe_cut_struc_2024_v1.5.4.fits
       covmat_file: ./covs/shapepipe_A/cov_shapepipe_A.txt
       e1_col: e1
       e2_col: e2
@@ -873,13 +916,13 @@ SP_v1.5.4:
       w_col: w_iv
       R: 1.0
    star:
-      path: unions_shapepipe_star_2024_v1.4.1.fits
+      path: unions_shapepipe_star_2024_v1.5.a.fits
       ra_col: RA
       dec_col: Dec
       e1_col: e1
       e2_col: e2
    psf:
-      path: v1.5.x/unions_shapepipe_psf_2024_v1.5.2.fits
+      path: unions_shapepipe_psf_2024_v1.5.a.fits
       hdu: 1
       ra_col: RA
       dec_col: Dec

notebooks/cosmo_val/run_cosmo_val.py

@@ -1,6 +1,10 @@
 # %%
 # %load_ext autoreload
 # %autoreload 2
+
+import matplotlib
+matplotlib.use("agg")
+
 import matplotlib.pyplot as plt
 import numpy as np
 import treecorr
@@ -30,14 +34,20 @@
 cv.plot_scale_dependent_leakage()
 
 # %%
-#cv.plot_objectwise_leakage()
+cv.plot_objectwise_leakage()
+
+# %%
+cv.plot_objectwise_leakage_aux()
 
 # %%
 cv.plot_ellipticity()
 
 # %%
 cv.plot_separation()
 
+# %%
+cv.calculate_additive_bias()
+
 # %%
 cv.plot_2pcf()
 

notebooks/demo_apply_hsp_masks.py

@@ -52,14 +52,15 @@
 
 # +
 # Set parameters
-obj._params["input_path"] = "unions_shapepipe_comprehensive_2024_v1.4.c.hdf5"
-obj._params["output_path"] = "unions_shapepipe_comprehensive_struc_2024_v1.4.c.hdf5"
-obj._params["mask_dir"] = f"{os.environ['HOME']}/v1.4.x/masks"
+ver_maj = "v1.5"
+obj._params["input_path"] = f"unions_shapepipe_comprehensive_2024_{ver_maj}.c.hdf5"
+obj._params["output_path"] = f"unions_shapepipe_comprehensive_struc_2024_{ver_maj}.c.hdf5"
+obj._params["mask_dir"] = f"{os.environ['HOME']}/{ver_maj}.x/masks"
 obj._params["nside"] = 131072
 obj._params["file_base"] = "mask_r_"
 obj._params["bits"] = bits
 
-obj._params["aux_mask_files"] = f"{obj._params['mask_dir']}/coverage.hsp"
+obj._params["aux_mask_files"] = f"{obj._params['mask_dir']}/coverage_{ver_maj}.x.hsp"
 obj._params["aux_mask_labels"] = "npoint3"
 obj._params["verbose"] = True
 # -