PureB

.github/workflows/ci.yml

@@ -273,6 +273,7 @@ jobs:
         ceci --dry-run examples/buzzard/pipeline.yml
         ceci --dry-run examples/cosmodc2/pipeline.yml
         ceci --dry-run examples/skysim/pipeline.yml
+        ceci --dry-run examples/desy3/pipeline_Bmodes.yml
 
 
   Test_Auto_Installer:

examples/desy1/pipeline_Bmodes.yml

@@ -0,0 +1,116 @@
+
+# Stages to run
+stages:
+
+    - name: TXSourceSelectorMetacal     # select and split objects into source bins
+    - name: TXShearCalibration          # Calibrate and split the source sample tomographically
+    #- name: TXTruthLensCatalogSplitter
+    #- name: TXStarCatalogSplitter  # Split the star catalog into separate bins (psf/non-psf)    
+    #- name: TXPhotozPlot 
+    - name: TXSourceMaps
+      #nprocess: 6
+      #threads_per_process: 1
+      #nodes: 1
+    #- name: TXLensMaps
+    #- name: TXAuxiliarySourceMaps  # make PSF and flag maps
+    #- name: TXAuxiliaryLensMaps    # make depth and bright object maps
+    #- name: TXSourceNoiseMaps      # noise maps for sources using random rotations
+    # stage below is not working because TXAuxiliaryLensMaps is broken
+    #- name: TXLensNoiseMaps      # noise maps for lenses using density splits
+    #- name: TXDensityMaps         # turn mask and ngal maps into overdensity maps
+    #- name: TXMapPlots            # make pictures of all the maps
+    #- name: TXTracerMetadata      # collate metadata
+    #- name: TXJackknifeCenters    # Split the area into jackknife regions
+    #- name: TXRandomCat
+    #- name: TXTwoPoint            # Compute real-space 2-point correlations
+    #  threads_per_process: 128
+    #- name: TXBlinding           # Blind the data following Muir et al
+        #threads_per_process: 2
+    #- name: TXTwoPointTheoryReal # compute theory using CCL to save in sacc file and plot later
+    # - name: TXTwoPointPlots      # Make plots of 2pt correlations
+    #- name: TXSourceHistogramPlots
+    #- name: TXSourceDiagnosticPlots    # Make a suite of diagnostic plots
+      #nprocess: 8 
+      #nodes: 2
+    #- name: TXLensDiagnosticPlots    # Make a suite of diagnostic plots
+    #- name: TXGammaTFieldCenters # Compute and plot gamma_t around center points # ERROR: EXPOSURE FILE
+      ##threads_per_process: 2
+    #- name: TXGammaTStars  # Compute and plot gamma_t around bright stars
+      #threads_per_process: 2
+    #- name: TXGammaTRandoms      # Compute and plot gamma_t around randoms
+      #threads_per_process: 2
+    #- name: TXRoweStatistics     # Compute and plot Rowe statistics
+    # threads_per_process: 128
+    #- name: TXFocalPlanePlot      # mean PSF ellipticity and mean residual on the focal plane 
+      #- name: TXGalaxyStarDensity  # Compute and plot the star-galaxy density cross-correlation
+      #- name: TXGalaxyStarShear    # Compute and plot the star-galaxy shear cross-correlation
+      #- name: TXPSFDiagnostics     # Compute and plots other PSF diagnostics
+      #- name: TXBrighterFatterPlot # Make plots tracking the brighter-fatter effect
+      #- name: TXConvergenceMaps    # Make convergence kappa maps from g1, g2 maps #ERROR: WLMASSMAP
+      #- name: TXConvergenceMapPlots # Plot the convergence map
+      #- name: TXMapCorrelations    # plot the correlations between systematics and data # ERROR: NEEDS CONVERGENCE MAP
+      #- name: TXApertureMass        # Compute aperture-mass statistics
+      ##threads_per_process: 2
+    # Disabled since not yet synchronised with new Treecorr MPI
+    # - name: TXSelfCalibrationIA   # Self-calibration intrinsic alignments of galaxies
+    #
+    # Disabling these as they takes too long for a quick test.
+    # The latter three need NaMaster
+
+      ##- name: TXRealGaussianCovariance   # Compute covariance of real-space correlations
+    - name: TXTwoPointFourier          # Compute power spectra C_ell
+      ##- name: TXFourierNamasterCovariance # Compute the C_ell covariance
+      ##- name: TXFourierTJPCovariance     # Compute the C_ell covariance with TJPCov 
+
+#===========
+
+launcher:
+    name: mini
+    interval: 1.0
+
+site:
+    name: local 
+    max_threads: 128
+
+modules: txpipe
+
+# where to find any modules that are not in this repo,
+# and any other code we need.
+python_paths:
+    - submodules/WLMassMap/python/desc/
+    - submodules/TJPCov
+
+# where to put outputs
+output_dir: data/desy1/outputs
+
+# configuration settings
+config: examples/desy1/config.yml
+
+# These are overall inputs to the whole pipeline, not generated within it
+inputs:
+    shear_catalog: /global/cfs/cdirs/lsst/groups/WL/projects/txpipe-sys-tests/des-y1/shear_catalog_desy1_masked.h5
+    # subsampled catalog for debugging:
+    #shear_catalog: /global/cscratch1/sd/jjeffers/TXPipe/data/desy1/inputs/small_DESY1_shear_catalog.h5               
+    photometry_catalog: /global/cfs/cdirs/lsst/groups/WL/projects/txpipe-sys-tests/des-y1/photometry_catalog_desy1_RM.h5
+    lens_tomography_catalog: /global/cfs/cdirs/lsst/groups/WL/projects/txpipe-sys-tests/des-y1/lens_tomography_catalog_desy1_RM_091423.h5
+    shear_photoz_stack: /global/cfs/cdirs/lsst/groups/WL/projects/txpipe-sys-tests/des-y1/shear_photoz_stack.hdf5
+    lens_photoz_stack: /global/cfs/cdirs/lsst/groups/WL/projects/txpipe-sys-tests/des-y1/lens_photoz_stack.hdf5
+    calibration_table: data/example/inputs/sample_cosmodc2_w10year_errors.dat
+    exposures: data/example/inputs/exposures.hdf5
+    star_catalog: /global/cfs/cdirs/lsst/groups/WL/projects/txpipe-sys-tests/des-y1/DES_psf_catalog.hdf5
+    fiducial_cosmology: data/fiducial_cosmology.yml
+    # For the self-calibration extension we are not using Random_cat_source for now
+    # So it is set to Null, so the yaml intepreter returns a None value to python. 
+    random_cats_source: Null
+    mask: /global/cfs/cdirs/lsst/groups/WL/projects/txpipe-sys-tests/des-y1/mask_desy1.h5
+    #random_cats: /global/cfs/cdirs/lsst/groups/WL/projects/txpipe-sys-tests/des-y1/randoms_desy1_RM.hdf5
+    #binned_random_catalog: /global/cfs/cdirs/lsst/groups/WL/projects/txpipe-sys-tests/des-y1/binned_randoms_desy1_RM.hdf5
+# if supported by the launcher, restart the pipeline where it left off
+# if interrupted
+resume: True
+
+# where to put output logs for individual stages
+log_dir: data/desy1/logs
+
+# where to put an overall parsl pipeline log
+pipeline_log: data/desy1/log.txt

examples/desy3/config.yml

@@ -60,7 +60,7 @@ TXSourceSelectorMetacal:
     delta_gamma: 0.02
     source_zbin_edges: [0.0, 0.2, 0.4, 0.6, 0.8]
     shear_prefix: mcal_
-    resp_mean_diag : True
+    use_diagonal_response : True
 
 TXSourceSelectorLensfit:
     bands: ri #used for selection

examples/desy3/config_Bmodes.yml

@@ -0,0 +1,219 @@
+# Values in this section are accessible to all the different stages.
+# They can be overridden by individual stages though.
+global:
+    # This is read by many stages that read complete
+    # catalog data, and tells them how many rows to read
+    # at once
+    chunk_rows: 100000
+    # These mapping options are also read by a range of stages
+    pixelization: healpix
+    nside: 512
+    sparse: True  # Generate sparse maps - faster if using small areas
+
+PZPrepareEstimatorLens:
+    name: PZPrepareEstimatorLens
+    classname: Inform_BPZ_lite
+    aliases:
+        input: spectroscopic_catalog
+        model: lens_photoz_model
+    zmin: 0.0
+    zmax: 3.0
+    nzbins: 301
+    columns_file: ./data/bpz_ugrizy.columns
+    data_path: ./data/example/rail-bpz-inputs
+    spectra_file: CWWSB4.list
+    prior_band: i
+    ref_band: i
+    # Not sure about this
+    prior_file: hdfn_gen
+    p_min: 0.005
+    gauss_kernel: 0.0
+    mag_err_min: 0.005
+    inform_options: {'save_train': False, 'load_model': False, 'modelfile': 'BPZpriormodel.out'}  
+    madau_reddening: no
+    bands: riz
+    zp_errors: [0.01, 0.01, 0.01, 0.01, 0.01, 0.01]
+    hdf5_groupname: photometry
+
+
+PZEstimatorLens:
+    name: PZEstimatorLens
+    classname: BPZ_lite
+    aliases:
+        model: lens_photoz_model
+        input: photometry_catalog
+        output: lens_photoz_pdfs
+    zmin: 0.0
+    zmax: 3.0
+    dz: 0.01
+    nzbins: 301
+    band_names: [mag_u, mag_g, mag_r, mag_i, mag_z, mag_y]
+    bands: [mag_u, mag_g, mag_r, mag_i, mag_z, mag_y]
+    err_bands: [mag_err_u, mag_err_g, mag_err_r, mag_err_i, mag_err_z, mag_err_y]
+    hdf5_groupname: photometry
+    nondetect_val: .inf
+    columns_file: ./data/bpz_ugrizy.columns
+    data_path: ./data/example/rail-bpz-inputs
+    spectra_file: CWWSB4.list
+    prior_band: mag_i
+    ref_band: mag_i
+    prior_file: hdfn_gen
+    p_min: 0.005
+    gauss_kernel: 0.0
+    zp_errors: [0.01, 0.01, 0.01, 0.01, 0.01, 0.01]
+    mag_err_min: 0.005
+    madau_reddening: false
+    mag_limits:
+        mag_u: 27.79
+        mag_g: 29.04
+        mag_r: 29.06
+        mag_i: 28.62
+        mag_z: 27.98
+        mag_y: 27.05    
+
+TXSourceSelectorMetacal:
+    input_pz: True
+    true_z: False
+    bands: riz #used for selection
+    T_cut: 0.5
+    s2n_cut: 10.0
+    max_rows: 10000000000
+    delta_gamma: 0.02
+    source_zbin_edges: [0.0, 0.2, 0.4, 0.6, 0.8]
+    shear_prefix: mcal_
+    use_diagonal_response : True
+
+TXMeanLensSelector:
+    # Mag cuts
+    lens_zbin_edges: [0.0,0.2,0.4]
+    cperp_cut: 0.2
+    r_cpar_cut: 13.5
+    r_lo_cut: 16.0
+    r_hi_cut: 21.6
+    i_lo_cut: 17.5
+    i_hi_cut: 21.9
+    r_i_cut: 2.0
+
+TXModeLensSelector:
+    # Mag cuts
+    lens_zbin_edges: [0.0,0.2,0.4]
+    cperp_cut: 0.2
+    r_cpar_cut: 13.5
+    r_lo_cut: 16.0
+    r_hi_cut: 21.6
+    i_lo_cut: 17.5
+    i_hi_cut: 21.9
+    r_i_cut: 2.0
+
+TXRandomCat:
+    density: 10 # gals per sq arcmin
+
+TXTwoPoint:
+    bin_slop: 0.1
+    delta_gamma: 0.02
+    do_pos_pos: True
+    do_shear_shear: True
+    do_shear_pos: True
+    flip_g2: True  # use true when using metacal shears
+    min_sep: 2.5
+    max_sep: 60.0
+    nbins: 1000
+    verbose: 0
+    subtract_mean_shear: True
+    var_method: "jackknife"
+
+TXSourceDiagnosticPlots:
+    psf_prefix: mcal_psf_
+    shear_prefix: mcal_
+    nbins: 20
+    g_min: -0.01
+    g_max: 0.01
+    psfT_min: 0.2
+    psfT_max: 0.28
+    T_min: 0.01
+    T_max: 2.1
+    s2n_min: 1.25
+    s2n_max: 300
+
+TXSourceMaps:
+    sparse: True  # Generate sparse maps - faster if using small areas
+
+TXLensMaps:
+    sparse: True  # Generate sparse maps - faster if using small areas
+    nside: 128
+
+TXDensityMaps:
+    nside: 128
+
+TXAuxiliaryLensMaps:
+    flag_exponent_max: 8
+    bright_obj_threshold: 22.0 # The magnitude threshold for a object to be counted as bright
+    depth_band : i
+    snr_threshold: 10.0  # The S/N value to generate maps for (e.g. 5 for 5-sigma depth)
+    snr_delta: 1.0  # The range threshold +/- delta is used for finding objects at the boundary
+
+TXJackknifeCenters:
+    npatch: 5
+
+TXSimpleMask:
+    depth_cut : 23.5
+    bright_object_max: 10.0
+
+PZRailSummarizeLens:
+    leafsize: 20
+    zmin: 0.0
+    zmax: 3.0
+    nzbins: 50
+    name: PZRailSummarizeLens
+    catalog_group: "photometry"
+    tomography_name: "lens"
+    aliases:
+        tomography_catalog: lens_tomography_catalog
+        photometry_catalog: photometry_catalog
+        model: lens_direct_calibration_model
+        photoz_stack: lens_photoz_stack
+    model: None
+
+FlowCreator:
+    n_samples: 1000000
+    seed: 5763248
+    aliases:
+        # This input was generated using get_example_flow in pzflow,
+        # not something specific.
+        output: ideal_specz_catalog
+        model: flow
+
+TXParqetToHDF:
+    hdf_group: photometry
+    aliases:
+        input: specz_catalog_pq
+        output: spectroscopic_catalog
+
+
+Inform_NZDirSource:
+    name: Inform_NZDirSource
+    usecols: [r, i, z]
+    hdf5_groupname: photometry
+    aliases:
+        input: spectroscopic_catalog
+        model: source_direct_calibration_model
+
+Inform_NZDirLens:
+    name: Inform_NZDirLens
+    usecols: [u, g, r, i, z, "y"]
+    hdf5_groupname: photometry
+    aliases:
+        input: spectroscopic_catalog
+        model: lens_direct_calibration_model
+
+TXPureB:
+    method    : 'namaster'                 # 'namaster' or 'hybrideb'
+    Nell      : 20                         # Number of ell bins
+    lmin      : 100                        # ell min 
+    lmax      : 2000                       # ell max
+    lspacing  : 'log'                      # ell spacing for binning 
+    bin_file  :                            # Load predfined bin edges instead        
+    theta_min : 2.5                        # (only for hybrideb) theta min in arcmin 
+    theta_max : 250                        # (only for hybrideb) theta max in arcmin 
+    Ntheta    : 1000                       # (only for hybrideb) Number of theta bins 
+    Nsims     : 3                         # (only for hybrideb) Number of Gaussian draws to use to estimate errors