From fae821b1d0c30f11ead4d55f97fed0421d86a590 Mon Sep 17 00:00:00 2001 From: eduardojsbarroso Date: Tue, 23 Sep 2025 04:11:22 -0700 Subject: [PATCH 1/7] Initial files --- .../gt_treecor_test/Run_CL_pipeline.ipynb | 663 ++++++++++++++++++ .../gt_treecor_test/config-1deg2-CL.yml | 84 +++ .../pipeline-1deg2-CL-nersc.yml | 59 ++ 3 files changed, 806 insertions(+) create mode 100644 notebooks/cluster_counts/gt_treecor_test/Run_CL_pipeline.ipynb create mode 100644 notebooks/cluster_counts/gt_treecor_test/config-1deg2-CL.yml create mode 100644 notebooks/cluster_counts/gt_treecor_test/pipeline-1deg2-CL-nersc.yml diff --git a/notebooks/cluster_counts/gt_treecor_test/Run_CL_pipeline.ipynb b/notebooks/cluster_counts/gt_treecor_test/Run_CL_pipeline.ipynb new file mode 100644 index 00000000..1c3afcae --- /dev/null +++ b/notebooks/cluster_counts/gt_treecor_test/Run_CL_pipeline.ipynb @@ -0,0 +1,663 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "709caa5a", + "metadata": { + "tags": [] + }, + "source": [ + "*This notebook was developed by Marina Ricci for the DESC CL_Cosmo_Pipeline team.*\n", + "# This notebook aims at presenting the cluster pipeline for \n", + "# stacked $\\Delta\\Sigma$ profiles and counts in richness/redshift bins.\n", + "\n", + "It is meant to be run independently. If you already produced the necessary outputs `ceci` will use them, and otherwise create them.\n", + "\n", + "___" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "2edf6e90", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "import os\n", + "from pprint import pprint\n", + "\n", + "import ceci\n", + "import h5py\n", + "\n", + "%matplotlib inline\n", + "import matplotlib.pyplot as plt\n", + "import numpy as np\n", + "import yaml\n", + "from IPython.display import Image\n", + "from astropy.table import Table\n", + "\n", + "import re\n", + "import sacc" + ] + }, + { + "cell_type": "markdown", + "id": "8fe12a8c", + "metadata": {}, + "source": [ + "Make sure to change your path in the next cell that leads to your TXPipe directory. See examples for IN2P3 and NERSC below." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "5eebecd8", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "# user specific paths -- IN2P3 example\n", + "my_txpipe_dir = \"/pbs/home/m/mricci/throng_mricci/desc/TXPipe\"\n", + "\n", + "# user specific paths -- NERSC example\n", + "# my_txpipe_dir = \"/pscratch/sd/a/avestruz/TXPipe\"\n", + "\n", + "os.chdir(my_txpipe_dir)\n", + "\n", + "import txpipe" + ] + }, + { + "cell_type": "markdown", + "id": "b670e283", + "metadata": {}, + "source": [ + "___" + ] + }, + { + "cell_type": "markdown", + "id": "6a8f1144", + "metadata": {}, + "source": [ + "# 1 - Launching the pipeline" + ] + }, + { + "cell_type": "markdown", + "id": "9f53e707", + "metadata": {}, + "source": [ + "## **Pipeline approach**\n", + "\n", + "Here we will use the 20deg2, but we can also use the 1deg2 files (just need to change 20deg2 to 1deg2 in the name of the files)\n", + "\n", + "\n", + "Let's have a look at the submission script for this pipeline:\n", + "- to work at CCin2p3 we can use: `examples/cosmodc2/Cluster_pipelines/cosmodc2/20deg2-in2p3.sub`:\n", + "- to work at NERSC we can use: `examples/cosmodc2/Cluster_pipelines/cosmodc2/20deg2-nersc.sub`:" + ] + }, + { + "cell_type": "markdown", + "id": "40e2c578", + "metadata": {}, + "source": [ + "## **Comands to run the pipeline**\n", + "This will launch a job of up to one hour (it should finish in 30 min) on a single CC-IN2P3 node to run a pipeline. After the first run, the output files are created and following runs take much less time.\n" + ] + }, + { + "cell_type": "markdown", + "id": "379a258b", + "metadata": {}, + "source": [ + "\n", + "\n", + "> ### In a terminal, **navigate to your TXPipe directory on IN2P3 and run**:\n", + ">```\n", + "sbatch examples/cosmodc2/Cluster_pipelines/20deg2-in2p3.sub\n", + "```" + ] + }, + { + "cell_type": "markdown", + "id": "1abf63da", + "metadata": {}, + "source": [ + "\n", + "> ### If you are **on NERSC, you will instead run**:\n", + ">```\n", + "sbatch examples/cosmodc2/Cluster_pipelines/20deg2-nersc.sub\n", + "```" + ] + }, + { + "cell_type": "markdown", + "id": "994aa495", + "metadata": {}, + "source": [ + "If you are at CCin2p3 you can look at the ouput of your submission in the file `slurm-xxx.out`, with xxx the number of your batch process. If you see *'Pipeline successful. Joy is sparked.'* : congratulations, it worked ! \n", + "\n", + "Once the pipeline is run you can go directly to 4) to look at the results." + ] + }, + { + "cell_type": "markdown", + "id": "b914ddff", + "metadata": {}, + "source": [ + "___" + ] + }, + { + "cell_type": "markdown", + "id": "8f68fc0d", + "metadata": {}, + "source": [ + "# 2 - Looking at the different pipeline files" + ] + }, + { + "cell_type": "markdown", + "id": "47d36ebf", + "metadata": { + "tags": [] + }, + "source": [ + "### **Let's look at the submission script :**" + ] + }, + { + "cell_type": "markdown", + "id": "4dec2e8c", + "metadata": { + "tags": [] + }, + "source": [ + "=> If we use the CCin2p3 example:" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "9a360058", + "metadata": {}, + "outputs": [], + "source": [ + "! cat examples/cosmodc2/Cluster_pipelines/20deg2-in2p3.sub" + ] + }, + { + "cell_type": "markdown", + "id": "6757a600", + "metadata": {}, + "source": [ + "=> If we use the NERSC example:" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "id": "6e0d6b74", + "metadata": {}, + "outputs": [], + "source": [ + "! cat examples/cosmodc2/Cluster_pipelines/20deg2-nersc.sub" + ] + }, + { + "cell_type": "markdown", + "id": "6333c236", + "metadata": {}, + "source": [ + "### **The submission script is specifying the pipeline yaml file. Let's look at it :**" + ] + }, + { + "cell_type": "markdown", + "id": "1d0c713f", + "metadata": {}, + "source": [ + "=> The only differences between NERSC and CCin2p3 are in the first block, that defines on witch machine your run. " + ] + }, + { + "cell_type": "code", + "execution_count": 112, + "id": "1458bf0c", + "metadata": {}, + "outputs": [], + "source": [ + "! cat examples/cosmodc2/Cluster_pipelines/pipeline-20deg2-CL-nersc.yml" + ] + }, + { + "cell_type": "markdown", + "id": "63473b4a", + "metadata": {}, + "source": [ + "___" + ] + }, + { + "cell_type": "markdown", + "id": "7ccef7e0", + "metadata": {}, + "source": [ + "# 3 - Producing and looking at the pipeline diagram" + ] + }, + { + "cell_type": "markdown", + "id": "cc256472", + "metadata": {}, + "source": [ + "Here we run the pipeline in \"dry-run\" mode, to check that it can run and to produce a pipeline diagram in png." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "82a24dd3", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "# Read the appropriate pipeline configuration, and ask for a flow-chart.\n", + "\n", + "pipeline_file = \"examples/cosmodc2/Cluster_pipelines/pipeline-20deg2-CL-in2p3.yml\"\n", + "# pipeline_file = \"examples/cosmodc2/Cluster_pipelines/pipeline-20deg2-CL-nersc.yml\"\n", + "flowchart_file = \"CL_pipeline.png\"\n", + "\n", + "\n", + "pipeline_config = ceci.Pipeline.build_config(pipeline_file, flow_chart=flowchart_file, dry_run=True)\n", + "\n", + "# Run the flow-chart pipeline\n", + "ceci.run_pipeline(pipeline_config);" + ] + }, + { + "cell_type": "markdown", + "id": "f3c19dcb", + "metadata": {}, + "source": [ + "Here we have 6 uncommented stages, and 5 uncommented inputs files.\n", + "\n", + "This translate in the pipeline charts as 6 red ellipses, and 5 yellow boxes. The blue boxes represents the ouput files." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9e60cace", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "Image(flowchart_file)" + ] + }, + { + "cell_type": "markdown", + "id": "b986818d", + "metadata": {}, + "source": [ + "___" + ] + }, + { + "cell_type": "markdown", + "id": "5dd025a3", + "metadata": {}, + "source": [ + "# 4 - Opening and looking at the outputs" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "1463b613", + "metadata": {}, + "outputs": [], + "source": [ + "#At the moment the ouput file is a pickle file\n", + "import pickle as pkl" + ] + }, + { + "cell_type": "markdown", + "id": "f65b5c1b", + "metadata": {}, + "source": [ + "### **Open the pipeline file to load correct input/output file names**" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "bbc4cb52", + "metadata": {}, + "outputs": [], + "source": [ + "with open(pipeline_file, \"r\") as file:\n", + " pipeline_content = yaml.safe_load(file)" + ] + }, + { + "cell_type": "markdown", + "id": "05c9f270", + "metadata": {}, + "source": [ + "### **Open the output**" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "5550ef72", + "metadata": {}, + "outputs": [], + "source": [ + "data = pkl.load(open(pipeline_content[\"output_dir\"] + \"/cluster_profiles.pkl\", \"rb\"))" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "e1dcff76", + "metadata": {}, + "outputs": [], + "source": [ + "data" + ] + }, + { + "cell_type": "markdown", + "id": "0fdf85f7", + "metadata": {}, + "source": [ + "The output is a dictionary containing a `CLMM` `ClusterEnsemble` object for each redshift/richness bin." + ] + }, + { + "cell_type": "markdown", + "id": "504fc48b", + "metadata": {}, + "source": [ + "### **Exploring the output**" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "84b36b0d", + "metadata": {}, + "outputs": [], + "source": [ + "example_bin = 'bin_zbin_0_richbin_0'" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "2d11ee3b", + "metadata": {}, + "outputs": [], + "source": [ + "#This is the info for this bin\n", + "data[example_bin]" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "e9926eca", + "metadata": {}, + "outputs": [], + "source": [ + "#This shows the table with all cluster in this bin and their corresponding profiles\n", + "data[example_bin]['clmm_cluster_ensemble'].data" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "d148accc", + "metadata": {}, + "outputs": [], + "source": [ + "#This shows the table with all cluster in this bin and their corresponding profiles\n", + "data[example_bin]['clmm_cluster_ensemble'].data.meta" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "id": "56a39522", + "metadata": {}, + "outputs": [], + "source": [ + "#This shows the ensemble stacked profiles\n", + "data[example_bin]['clmm_cluster_ensemble'].stacked_data.meta" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "1da10d8a", + "metadata": {}, + "outputs": [], + "source": [ + "#This shows the covarinace for the ensemble stacked profiles\n", + "data[example_bin]['clmm_cluster_ensemble'].cov" + ] + }, + { + "cell_type": "markdown", + "id": "3c32d5a1", + "metadata": {}, + "source": [ + "### **Plot the ouput**" + ] + }, + { + "cell_type": "code", + "execution_count": 111, + "id": "203de6d1", + "metadata": {}, + "outputs": [], + "source": [ + "fig, ax = plt.subplots()\n", + "\n", + "ax.semilogx(\n", + " data[example_bin]['clmm_cluster_ensemble'].stacked_data[\"radius\"],\n", + " data[example_bin]['clmm_cluster_ensemble'].stacked_data[\"tangential_comp\"],\n", + " \"bx-\",\n", + " label=\"tan\",)\n", + "\n", + "ax.semilogx(\n", + " data[example_bin]['clmm_cluster_ensemble'].stacked_data[\"radius\"],\n", + " data[example_bin]['clmm_cluster_ensemble'].stacked_data[\"cross_comp\"],\n", + " \"r.-\",\n", + " label=\"cross\",)\n", + "\n", + "ax.errorbar(\n", + " data[example_bin]['clmm_cluster_ensemble'].stacked_data[\"radius\"],\n", + " data[example_bin]['clmm_cluster_ensemble'].stacked_data[\"tangential_comp\"],\n", + " yerr=data[example_bin]['clmm_cluster_ensemble'].cov[\"tan_sc\"].diagonal() ** 0.5,\n", + " color=\"blue\",)\n", + "\n", + "\n", + "ax.errorbar(\n", + " data[example_bin]['clmm_cluster_ensemble'].stacked_data[\"radius\"],\n", + " data[example_bin]['clmm_cluster_ensemble'].stacked_data[\"cross_comp\"],\n", + " yerr=data[example_bin]['clmm_cluster_ensemble'].cov[\"cross_sc\"].diagonal() ** 0.5,\n", + " color=\"red\",)\n", + "\n", + "\n", + "\n", + "ax.set_xlabel('radius [Mpc]')\n", + "ax.set_ylabel('$\\\\Delta \\\\Sigma$')\n", + "\n", + "plt.legend()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "TXPipe-2023-Jul-12", + "language": "python", + "name": "txpipe-2023-jul-12" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.53.10.12" + }, + "nbdime-conflicts": { + "local_diff": [ + { + "diff": [ + { + "diff": [ + { + "key": 0, + "op": "addrange", + "valuelist": [ + "clmm" + ] + }, + { + "key": 0, + "length": 1, + "op": "removerange" + } + ], + "key": "display_name", + "op": "patch" + }, + { + "diff": [ + { + "key": 0, + "op": "addrange", + "valuelist": [ + "clmm" + ] + }, + { + "key": 0, + "length": 1, + "op": "removerange" + } + ], + "key": "name", + "op": "patch" + } + ], + "key": "kernelspec", + "op": "patch" + }, + { + "diff": [ + { + "diff": [ + { + "diff": [ + { + "key": 5, + "length": 1, + "op": "removerange" + } + ], + "key": 0, + "op": "patch" + } + ], + "key": "version", + "op": "patch" + } + ], + "key": "language_info", + "op": "patch" + } + ], + "remote_diff": [ + { + "diff": [ + { + "diff": [ + { + "key": 0, + "op": "addrange", + "valuelist": [ + "Python (firecrown2.0)" + ] + }, + { + "key": 0, + "length": 1, + "op": "removerange" + } + ], + "key": "display_name", + "op": "patch" + }, + { + "diff": [ + { + "key": 0, + "op": "addrange", + "valuelist": [ + "firecrown" + ] + }, + { + "key": 0, + "length": 1, + "op": "removerange" + } + ], + "key": "name", + "op": "patch" + } + ], + "key": "kernelspec", + "op": "patch" + }, + { + "diff": [ + { + "diff": [ + { + "key": 0, + "length": 1, + "op": "removerange" + } + ], + "key": "version", + "op": "patch" + } + ], + "key": "language_info", + "op": "patch" + } + ] + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/notebooks/cluster_counts/gt_treecor_test/config-1deg2-CL.yml b/notebooks/cluster_counts/gt_treecor_test/config-1deg2-CL.yml new file mode 100644 index 00000000..a8d00e88 --- /dev/null +++ b/notebooks/cluster_counts/gt_treecor_test/config-1deg2-CL.yml @@ -0,0 +1,84 @@ +TXSourceSelectorMetadetect: + input_pz: false + bands: riz #used for selection + T_cut: 0.5 + s2n_cut: 10.0 + max_rows: 1000 + delta_gamma: 0.02 + source_zbin_edges: [0.1, 3.0] + chunk_rows: 100000 + true_z: false + shear_prefix: '' + + +Inform_BPZ_lite: + zmin: 0.0 + zmax: 3.0 + nzbins: 301 + columns_file: ./data/bpz_riz.columns + data_path: ./data/example/rail-bpz-inputs + spectra_file: CWWSB4.list + prior_band: i + ref_band: i + # Not sure about this + prior_file: hdfn_gen + p_min: 0.005 + gauss_kernel: 0.0 + mag_err_min: 0.005 + inform_options: {save_train: false, load_model: false, modelfile: BPZpriormodel.out} + madau_reddening: no + bands: riz + zp_errors: [0.01, 0.01, 0.01] + hdf5_groupname: photometry + + + +BPZ_lite: + zmin: 0.0 + zmax: 3.0 + dz: 0.01 + nzbins: 301 + data_path: ./data/example/rail-bpz-inputs + bands: [mag_r, mag_i, mag_z] + err_bands: [mag_err_r, mag_err_i, mag_err_z] + hdf5_groupname: shear/00 + nondetect_val: .inf + columns_file: ./data/bpz_riz.columns + spectra_file: CWWSB4.list + ref_band: mag_i + prior_file: hdfn_gen + p_min: 0.005 + gauss_kernel: 0.0 + zp_errors: [0.01, 0.01, 0.01] + mag_err_min: 0.005 + madau_reddening: false + mag_limits: + mag_r: 29.06 + mag_i: 28.62 + mag_z: 27.98 + +CLClusterBinningRedshiftRichness: + zedge: [0.2, 0.4, 0.6, 0.8] + richedge: [5., 10., 100.] + +CLClusterShearCatalogs: + chunk_rows: 100_000 # rows to read at once from source cat + max_radius: 5 # Mpc + delta_z: 0.2 # redshift buffer + redshift_cut_criterion: ztrue + redshift_weight_criterion: ztrue + redshift_cut_criterion_pdf_fraction: 0.9 + subtract_mean_shear: false + coordinate_system: celestial #euclidean or celestial, when using shear_true we shoudl use celestial + use_true_shear: false + +CLClusterEnsembleProfiles: + #radial bin definition + r_min: 0.5 #in Mpc + r_max: 5. #in Mpc + nbins: 5 # number of bins + #type of profile + delta_sigma_profile: true + shear_profile: false + magnification_profile: false + diff --git a/notebooks/cluster_counts/gt_treecor_test/pipeline-1deg2-CL-nersc.yml b/notebooks/cluster_counts/gt_treecor_test/pipeline-1deg2-CL-nersc.yml new file mode 100644 index 00000000..f93c9c55 --- /dev/null +++ b/notebooks/cluster_counts/gt_treecor_test/pipeline-1deg2-CL-nersc.yml @@ -0,0 +1,59 @@ +#this step depends on where you run +#for CCin2p3 +# site: +# name: cc-parallel +# mpi_command: "mpirun -n" + +#for NERSC +site: + name: cori-batch + image: ghcr.io/lsstdesc/txpipe-dev + + +#all the following steps should not depend on where you run +launcher: + name: mini + interval: 3.0 + +modules: > + txpipe + rail.estimation.algos.bpz_lite + +python_paths: [] + +stages: +# - name: TXSourceSelectorMetadetect +# nprocess: 1 +# - name: BPZliteInformer +# nprocess: 1 +# - name: BPZ_lite +# nprocess: 1 + - name: CLClusterBinningRedshiftRichness + nprocess: 1 + - name: CLClusterShearCatalogs + nprocess: 1 + - name: CLClusterEnsembleProfiles + nprocess: 1 +# - name: CLClusterDataVector +# nprocess: 1 + + + +output_dir: ./data/cosmodc2/outputs-1deg2-CL +config: examples/cosmodc2/Cluster_pipelines/config-1deg2-CL.yml + +inputs: + # See README for paths to download these files + shear_catalog: ./data/example/inputs/metadetect_shear_catalog.hdf5 + #photometry_catalog: ./data/example/inputs/photometry_catalog.hdf5 + fiducial_cosmology: ./data/fiducial_cosmology.yml + #calibration_table: ./data/example/inputs/sample_cosmodc2_w10year_errors.dat + #spectroscopic_catalog: ./data/example/inputs/mock_spectroscopic_catalog.hdf5 + cluster_catalog: ./data/example/inputs/cluster_catalog.hdf5 + shear_tomography_catalog: ./data/example/outputs_metadetect/shear_tomography_catalog.hdf5 + source_photoz_pdfs: ./data/example/inputs/photoz_pdfs.hdf5 + #cluster_shear_catalogs: ./data/cosmodc2/outputs-1deg2-CL/cluster_shear_catalogs.hdf5 +resume: true +log_dir: ./data/cosmodc2/logs +pipeline_log: ./data/cosmodc2/log_1deg2.txt + From 3346ad1c4d02614b0dbeba1daef835ce9a6a3aeb Mon Sep 17 00:00:00 2001 From: eduardojsbarroso Date: Tue, 23 Sep 2025 05:25:13 -0700 Subject: [PATCH 2/7] ADDED INITIAL NOTEBOOK --- .../gt_treecor_test/Run_CL_pipeline.ipynb | 283 ++++++++++-------- .../pipeline-1deg2-CL-nersc.yml | 24 +- txpipe/extensions/cluster_counts/__init__.py | 4 +- 3 files changed, 171 insertions(+), 140 deletions(-) diff --git a/notebooks/cluster_counts/gt_treecor_test/Run_CL_pipeline.ipynb b/notebooks/cluster_counts/gt_treecor_test/Run_CL_pipeline.ipynb index 1c3afcae..a01bde7f 100644 --- a/notebooks/cluster_counts/gt_treecor_test/Run_CL_pipeline.ipynb +++ b/notebooks/cluster_counts/gt_treecor_test/Run_CL_pipeline.ipynb @@ -18,12 +18,21 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 1, "id": "2edf6e90", "metadata": { "tags": [] }, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/global/cfs/projectdirs/lsst/groups/CL/cl_pipeline_project/conda_envs/txpipe_clp/lib/python3.10/site-packages/ceci/__init__.py:12: UserWarning: pkg_resources is deprecated as an API. See https://setuptools.pypa.io/en/latest/pkg_resources.html. The pkg_resources package is slated for removal as early as 2025-11-30. Refrain from using this package or pin to Setuptools<81.\n", + " from pkg_resources import DistributionNotFound\n" + ] + } + ], "source": [ "import os\n", "from pprint import pprint\n", @@ -52,7 +61,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 2, "id": "5eebecd8", "metadata": { "tags": [] @@ -60,7 +69,7 @@ "outputs": [], "source": [ "# user specific paths -- IN2P3 example\n", - "my_txpipe_dir = \"/pbs/home/m/mricci/throng_mricci/desc/TXPipe\"\n", + "my_txpipe_dir = \"/global/homes/e/edujb/TXPipe/\"\n", "\n", "# user specific paths -- NERSC example\n", "# my_txpipe_dir = \"/pscratch/sd/a/avestruz/TXPipe\"\n", @@ -79,160 +88,186 @@ ] }, { - "cell_type": "markdown", - "id": "6a8f1144", - "metadata": {}, - "source": [ - "# 1 - Launching the pipeline" - ] - }, - { - "cell_type": "markdown", - "id": "9f53e707", + "cell_type": "code", + "execution_count": 3, + "id": "9d3d1e18-eec5-469c-9952-9d0868a2cc2e", "metadata": {}, + "outputs": [], "source": [ - "## **Pipeline approach**\n", + "step1 = txpipe.TXSourceSelectorMetadetect.make_stage(\n", + " # This file is the input metadetect shear catalog\n", + " shear_catalog=f\"{my_txpipe_dir}/data/example/inputs/metadetect_shear_catalog.hdf5\",\n", + " # This is an input training set for the tomographic selection\n", + " calibration_table=f\"{my_txpipe_dir}data/example/inputs/sample_cosmodc2_w10year_errors.dat\",\n", "\n", - "Here we will use the 20deg2, but we can also use the 1deg2 files (just need to change 20deg2 to 1deg2 in the name of the files)\n", + " # This contains all the options for this stage. You can override them here\n", + " # manually too.\n", + " config=f\"{my_txpipe_dir}/examples/metadetect/config.yml\",\n", "\n", - "\n", - "Let's have a look at the submission script for this pipeline:\n", - "- to work at CCin2p3 we can use: `examples/cosmodc2/Cluster_pipelines/cosmodc2/20deg2-in2p3.sub`:\n", - "- to work at NERSC we can use: `examples/cosmodc2/Cluster_pipelines/cosmodc2/20deg2-nersc.sub`:" + " # This is the output file for this stage\n", + " output_dir=f\"{my_txpipe_dir}/data/example/outputs_metadetect/\"\n", + ")" ] }, { - "cell_type": "markdown", - "id": "40e2c578", + "cell_type": "code", + "execution_count": 4, + "id": "c5e6e548-7d43-43fb-93d7-debaedd9882c", "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Process 0 running selection for rows 0-82,200\n" + ] + } + ], "source": [ - "## **Comands to run the pipeline**\n", - "This will launch a job of up to one hour (it should finish in 30 min) on a single CC-IN2P3 node to run a pipeline. After the first run, the output files are created and following runs take much less time.\n" + "step1.run()\n", + "step1.finalize()" ] }, { - "cell_type": "markdown", - "id": "379a258b", + "cell_type": "code", + "execution_count": 5, + "id": "073b332a-ace9-4e27-9c6e-e1c9807bd2e0", "metadata": {}, + "outputs": [ + { + "ename": "IndentationError", + "evalue": "unexpected indent (260147577.py, line 16)", + "output_type": "error", + "traceback": [ + "\u001b[0;36m Cell \u001b[0;32mIn[5], line 16\u001b[0;36m\u001b[0m\n\u001b[0;31m (\"binned_shear_catalog\", ShearCatalog),\u001b[0m\n\u001b[0m ^\u001b[0m\n\u001b[0;31mIndentationError\u001b[0m\u001b[0;31m:\u001b[0m unexpected indent\n" + ] + } + ], "source": [ + "step2 = txpipe.CLClusterBinningRedshiftRichness.make_stage(\n", + " # This file is the input metadetect shear catalog\n", + " cluster_catalog=f\"{my_txpipe_dir}data/example/inputs/cluster_catalog.hdf5\",\n", + " # This is an input training set for the tomographic selection\n", "\n", + " # This contains all the options for this stage. You can override them here\n", + " # manually too.\n", + " config=f\"{my_txpipe_dir}/notebooks/cluster_counts/gt_treecor_test/config-1deg2-CL.yml\",\n", "\n", - "> ### In a terminal, **navigate to your TXPipe directory on IN2P3 and run**:\n", - ">```\n", - "sbatch examples/cosmodc2/Cluster_pipelines/20deg2-in2p3.sub\n", - "```" - ] - }, - { - "cell_type": "markdown", - "id": "1abf63da", - "metadata": {}, - "source": [ + " # This is the output file for this stage\n", + " output_dir=f\"{my_txpipe_dir}/data/example/output_binning/\"\n", + ")\n", "\n", - "> ### If you are **on NERSC, you will instead run**:\n", - ">```\n", - "sbatch examples/cosmodc2/Cluster_pipelines/20deg2-nersc.sub\n", - "```" - ] - }, - { - "cell_type": "markdown", - "id": "994aa495", - "metadata": {}, - "source": [ - "If you are at CCin2p3 you can look at the ouput of your submission in the file `slurm-xxx.out`, with xxx the number of your batch process. If you see *'Pipeline successful. Joy is sparked.'* : congratulations, it worked ! \n", "\n", - "Once the pipeline is run you can go directly to 4) to look at the results." - ] - }, - { - "cell_type": "markdown", - "id": "b914ddff", - "metadata": {}, - "source": [ - "___" + "\n", + " (\"binned_shear_catalog\", ShearCatalog),\n", + " (\"binned_lens_catalog\", HDFFile),\n", + " (\"binned_random_catalog\", HDFFile),\n", + " (\"binned_random_catalog_sub\", HDFFile),\n", + " (\"shear_photoz_stack\", QPNOfZFile),\n", + " (\"lens_photoz_stack\", QPNOfZFile),\n", + " (\"patch_centers\", TextFile),\n", + " (\"tracer_metadata\", HDFFile)," ] }, { - "cell_type": "markdown", - "id": "8f68fc0d", + "cell_type": "code", + "execution_count": 6, + "id": "e4dc452e-f860-4b46-b141-ccb2243dc752", "metadata": {}, + "outputs": [ + { + "ename": "NameError", + "evalue": "name 'step2' is not defined", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn[6], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43mstep2\u001b[49m\u001b[38;5;241m.\u001b[39mrun()\n\u001b[1;32m 2\u001b[0m step2\u001b[38;5;241m.\u001b[39mfinalize()\n", + "\u001b[0;31mNameError\u001b[0m: name 'step2' is not defined" + ] + } + ], "source": [ - "# 2 - Looking at the different pipeline files" - ] - }, - { - "cell_type": "markdown", - "id": "47d36ebf", - "metadata": { - "tags": [] - }, - "source": [ - "### **Let's look at the submission script :**" - ] - }, - { - "cell_type": "markdown", - "id": "4dec2e8c", - "metadata": { - "tags": [] - }, - "source": [ - "=> If we use the CCin2p3 example:" + "step2.run()\n", + "step2.finalize()" ] }, { "cell_type": "code", - "execution_count": 21, - "id": "9a360058", + "execution_count": 7, + "id": "a7c432f8-da69-4f18-a1d9-2eaf5e55ce74", "metadata": {}, "outputs": [], "source": [ - "! cat examples/cosmodc2/Cluster_pipelines/20deg2-in2p3.sub" + "step3 = txpipe.extensions.cluster_counts.TXTwoPointRLens.make_stage(\n", + " # This file is the input metadetect shear catalog\n", + " binned_lens_catalog=f\"global/homes/e/edujb/TXPipe/cluster_catalog_tomography.hdf5\",\n", + " # This is an input training set for the tomographic selection\n", + " binned_shear_catalog=\"global/homes/e/edujb/TXPipe/shear_tomography_catalog.hdf5\",\n", + " binned_random_catalog=f\"global/homes/e/edujb/TXPipe/cluster_catalog_tomography.hdf5\",\n", + " patch_centers=f\"global/homes/e/edujb/TXPipe/cluster_catalog_tomography.hdf5\",\n", + " tracer_metadata=f\"global/homes/e/edujb/TXPipe/cluster_catalog_tomography.hdf5\",\n", + " config=f\"{my_txpipe_dir}notebooks/cluster_counts/gt_treecor_test/config-1deg2-CL.yml\",\n", + " output_dir=f\"{my_txpipe_dir}data/example/output_binning/\"\n", + ")" ] }, { - "cell_type": "markdown", - "id": "6757a600", + "cell_type": "code", + "execution_count": 8, + "id": "81801925-4343-4e7a-9866-9df35ef8f8f2", "metadata": {}, + "outputs": [ + { + "ename": "FileNotFoundError", + "evalue": "[Errno 2] Unable to synchronously open file (unable to open file: name = 'global/homes/e/edujb/TXPipe/shear_tomography_catalog.hdf5', errno = 2, error message = 'No such file or directory', flags = 0, o_flags = 0)", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mFileNotFoundError\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn[8], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43mstep3\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrun\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 2\u001b[0m step3\u001b[38;5;241m.\u001b[39mfinalize()\n", + "File \u001b[0;32m/global/u2/e/edujb/TXPipe/txpipe/twopoint.py:93\u001b[0m, in \u001b[0;36mTXTwoPoint.run\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 90\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21;01mtreecorr\u001b[39;00m\n\u001b[1;32m 92\u001b[0m \u001b[38;5;66;03m# Binning information\u001b[39;00m\n\u001b[0;32m---> 93\u001b[0m source_list, lens_list \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mread_nbin\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 94\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mconfig[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mnum_threads\u001b[39m\u001b[38;5;124m\"\u001b[39m] \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mint\u001b[39m(os\u001b[38;5;241m.\u001b[39menviron\u001b[38;5;241m.\u001b[39mget(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mOMP_NUM_THREADS\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;241m1\u001b[39m))\n\u001b[1;32m 96\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mrank \u001b[38;5;241m==\u001b[39m \u001b[38;5;241m0\u001b[39m:\n\u001b[1;32m 97\u001b[0m \u001b[38;5;66;03m# This is a workaround for the fact the the ceci config stuff doesn't\u001b[39;00m\n\u001b[1;32m 98\u001b[0m \u001b[38;5;66;03m# quite handle the get method properly.\u001b[39;00m\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 101\u001b[0m \u001b[38;5;66;03m# and friends, below. In this base class only the 2D metrics will be\u001b[39;00m\n\u001b[1;32m 102\u001b[0m \u001b[38;5;66;03m# available, but subclasses can specify to load a distance column too.\u001b[39;00m\n", + "File \u001b[0;32m/global/u2/e/edujb/TXPipe/txpipe/twopoint.py:176\u001b[0m, in \u001b[0;36mTXTwoPoint.read_nbin\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 171\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m 172\u001b[0m \u001b[38;5;124;03mDetermine the bins to use in this analysis, either from the input file\u001b[39;00m\n\u001b[1;32m 173\u001b[0m \u001b[38;5;124;03mor from the configuration.\u001b[39;00m\n\u001b[1;32m 174\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m 175\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mconfig[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124msource_bins\u001b[39m\u001b[38;5;124m\"\u001b[39m] \u001b[38;5;241m==\u001b[39m [\u001b[38;5;241m-\u001b[39m\u001b[38;5;241m1\u001b[39m] \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mconfig[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mlens_bins\u001b[39m\u001b[38;5;124m\"\u001b[39m] \u001b[38;5;241m==\u001b[39m [\u001b[38;5;241m-\u001b[39m\u001b[38;5;241m1\u001b[39m]:\n\u001b[0;32m--> 176\u001b[0m source_list, lens_list \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_read_nbin_from_tomography\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 177\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m 178\u001b[0m source_list, lens_list \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_read_nbin_from_config()\n", + "File \u001b[0;32m/global/u2/e/edujb/TXPipe/txpipe/twopoint.py:192\u001b[0m, in \u001b[0;36mTXTwoPoint._read_nbin_from_tomography\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 190\u001b[0m nbin_source \u001b[38;5;241m=\u001b[39m \u001b[38;5;241m0\u001b[39m\n\u001b[1;32m 191\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m--> 192\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mopen_input\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mbinned_shear_catalog\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m)\u001b[49m \u001b[38;5;28;01mas\u001b[39;00m f:\n\u001b[1;32m 193\u001b[0m nbin_source \u001b[38;5;241m=\u001b[39m f[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mshear\u001b[39m\u001b[38;5;124m\"\u001b[39m]\u001b[38;5;241m.\u001b[39mattrs[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mnbin_source\u001b[39m\u001b[38;5;124m\"\u001b[39m]\n\u001b[1;32m 195\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mget_input(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mbinned_lens_catalog\u001b[39m\u001b[38;5;124m\"\u001b[39m) \u001b[38;5;241m==\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mnone\u001b[39m\u001b[38;5;124m\"\u001b[39m:\n", + "File \u001b[0;32m/global/cfs/projectdirs/lsst/groups/CL/cl_pipeline_project/conda_envs/txpipe_clp/lib/python3.10/site-packages/ceci/stage.py:1090\u001b[0m, in \u001b[0;36mPipelineStage.open_input\u001b[0;34m(self, tag, wrapper, **kwargs)\u001b[0m\n\u001b[1;32m 1088\u001b[0m path \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mget_input(tag)\n\u001b[1;32m 1089\u001b[0m input_class \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mget_input_type(tag)\n\u001b[0;32m-> 1090\u001b[0m obj \u001b[38;5;241m=\u001b[39m \u001b[43minput_class\u001b[49m\u001b[43m(\u001b[49m\u001b[43mpath\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mr\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1092\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m wrapper: \u001b[38;5;66;03m# pragma: no cover\u001b[39;00m\n\u001b[1;32m 1093\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m obj\n", + "File \u001b[0;32m/global/u2/e/edujb/TXPipe/txpipe/data_types/base.py:45\u001b[0m, in \u001b[0;36mDataFile.__init__\u001b[0;34m(self, path, mode, extra_provenance, validate, **kwargs)\u001b[0m\n\u001b[1;32m 42\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m mode \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;129;01min\u001b[39;00m [\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mr\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mw\u001b[39m\u001b[38;5;124m\"\u001b[39m]:\n\u001b[1;32m 43\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mFile \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mmode\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m argument must be \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mr\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m or \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mw\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m not \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mmode\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m---> 45\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mfile \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mopen\u001b[49m\u001b[43m(\u001b[49m\u001b[43mpath\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmode\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 47\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m validate \u001b[38;5;129;01mand\u001b[39;00m mode \u001b[38;5;241m==\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mr\u001b[39m\u001b[38;5;124m\"\u001b[39m:\n\u001b[1;32m 48\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mvalidate()\n", + "File \u001b[0;32m/global/u2/e/edujb/TXPipe/txpipe/data_types/base.py:165\u001b[0m, in \u001b[0;36mHDFFile.open\u001b[0;34m(cls, path, mode, **kwargs)\u001b[0m\n\u001b[1;32m 163\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21;01mh5py\u001b[39;00m\n\u001b[1;32m 164\u001b[0m \u001b[38;5;66;03m# Return an open h5py File\u001b[39;00m\n\u001b[0;32m--> 165\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mh5py\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mFile\u001b[49m\u001b[43m(\u001b[49m\u001b[43mpath\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmode\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n", + "File \u001b[0;32m/global/cfs/projectdirs/lsst/groups/CL/cl_pipeline_project/conda_envs/txpipe_clp/lib/python3.10/site-packages/h5py/_hl/files.py:564\u001b[0m, in \u001b[0;36mFile.__init__\u001b[0;34m(self, name, mode, driver, libver, userblock_size, swmr, rdcc_nslots, rdcc_nbytes, rdcc_w0, track_order, fs_strategy, fs_persist, fs_threshold, fs_page_size, page_buf_size, min_meta_keep, min_raw_keep, locking, alignment_threshold, alignment_interval, meta_block_size, **kwds)\u001b[0m\n\u001b[1;32m 555\u001b[0m fapl \u001b[38;5;241m=\u001b[39m make_fapl(driver, libver, rdcc_nslots, rdcc_nbytes, rdcc_w0,\n\u001b[1;32m 556\u001b[0m locking, page_buf_size, min_meta_keep, min_raw_keep,\n\u001b[1;32m 557\u001b[0m alignment_threshold\u001b[38;5;241m=\u001b[39malignment_threshold,\n\u001b[1;32m 558\u001b[0m alignment_interval\u001b[38;5;241m=\u001b[39malignment_interval,\n\u001b[1;32m 559\u001b[0m meta_block_size\u001b[38;5;241m=\u001b[39mmeta_block_size,\n\u001b[1;32m 560\u001b[0m \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwds)\n\u001b[1;32m 561\u001b[0m fcpl \u001b[38;5;241m=\u001b[39m make_fcpl(track_order\u001b[38;5;241m=\u001b[39mtrack_order, fs_strategy\u001b[38;5;241m=\u001b[39mfs_strategy,\n\u001b[1;32m 562\u001b[0m fs_persist\u001b[38;5;241m=\u001b[39mfs_persist, fs_threshold\u001b[38;5;241m=\u001b[39mfs_threshold,\n\u001b[1;32m 563\u001b[0m fs_page_size\u001b[38;5;241m=\u001b[39mfs_page_size)\n\u001b[0;32m--> 564\u001b[0m fid \u001b[38;5;241m=\u001b[39m \u001b[43mmake_fid\u001b[49m\u001b[43m(\u001b[49m\u001b[43mname\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmode\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43muserblock_size\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mfapl\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mfcpl\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mswmr\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mswmr\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 566\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(libver, \u001b[38;5;28mtuple\u001b[39m):\n\u001b[1;32m 567\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_libver \u001b[38;5;241m=\u001b[39m libver\n", + "File \u001b[0;32m/global/cfs/projectdirs/lsst/groups/CL/cl_pipeline_project/conda_envs/txpipe_clp/lib/python3.10/site-packages/h5py/_hl/files.py:238\u001b[0m, in \u001b[0;36mmake_fid\u001b[0;34m(name, mode, userblock_size, fapl, fcpl, swmr)\u001b[0m\n\u001b[1;32m 236\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m swmr \u001b[38;5;129;01mand\u001b[39;00m swmr_support:\n\u001b[1;32m 237\u001b[0m flags \u001b[38;5;241m|\u001b[39m\u001b[38;5;241m=\u001b[39m h5f\u001b[38;5;241m.\u001b[39mACC_SWMR_READ\n\u001b[0;32m--> 238\u001b[0m fid \u001b[38;5;241m=\u001b[39m \u001b[43mh5f\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mopen\u001b[49m\u001b[43m(\u001b[49m\u001b[43mname\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mflags\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mfapl\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mfapl\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 239\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m mode \u001b[38;5;241m==\u001b[39m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mr+\u001b[39m\u001b[38;5;124m'\u001b[39m:\n\u001b[1;32m 240\u001b[0m fid \u001b[38;5;241m=\u001b[39m h5f\u001b[38;5;241m.\u001b[39mopen(name, h5f\u001b[38;5;241m.\u001b[39mACC_RDWR, fapl\u001b[38;5;241m=\u001b[39mfapl)\n", + "File \u001b[0;32mh5py/_objects.pyx:54\u001b[0m, in \u001b[0;36mh5py._objects.with_phil.wrapper\u001b[0;34m()\u001b[0m\n", + "File \u001b[0;32mh5py/_objects.pyx:55\u001b[0m, in \u001b[0;36mh5py._objects.with_phil.wrapper\u001b[0;34m()\u001b[0m\n", + "File \u001b[0;32mh5py/h5f.pyx:102\u001b[0m, in \u001b[0;36mh5py.h5f.open\u001b[0;34m()\u001b[0m\n", + "\u001b[0;31mFileNotFoundError\u001b[0m: [Errno 2] Unable to synchronously open file (unable to open file: name = 'global/homes/e/edujb/TXPipe/shear_tomography_catalog.hdf5', errno = 2, error message = 'No such file or directory', flags = 0, o_flags = 0)" + ] + } + ], "source": [ - "=> If we use the NERSC example:" + "step3.run()\n", + "step3.finalize()" ] }, { "cell_type": "code", - "execution_count": 22, - "id": "6e0d6b74", + "execution_count": null, + "id": "7f65c156-fe6d-47cd-8b0b-57baa4e7755f", "metadata": {}, "outputs": [], - "source": [ - "! cat examples/cosmodc2/Cluster_pipelines/20deg2-nersc.sub" - ] + "source": [] }, { - "cell_type": "markdown", - "id": "6333c236", + "cell_type": "code", + "execution_count": null, + "id": "76324c34-3dfa-4886-b78b-72589f9dc49b", "metadata": {}, - "source": [ - "### **The submission script is specifying the pipeline yaml file. Let's look at it :**" - ] + "outputs": [], + "source": [] }, { "cell_type": "markdown", - "id": "1d0c713f", - "metadata": {}, - "source": [ - "=> The only differences between NERSC and CCin2p3 are in the first block, that defines on witch machine your run. " - ] - }, - { - "cell_type": "code", - "execution_count": 112, - "id": "1458bf0c", + "id": "b914ddff", "metadata": {}, - "outputs": [], "source": [ - "! cat examples/cosmodc2/Cluster_pipelines/pipeline-20deg2-CL-nersc.yml" + "___" ] }, { @@ -261,7 +296,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "id": "82a24dd3", "metadata": { "tags": [] @@ -321,7 +356,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "id": "1463b613", "metadata": {}, "outputs": [], @@ -340,7 +375,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": null, "id": "bbc4cb52", "metadata": {}, "outputs": [], @@ -359,7 +394,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": null, "id": "5550ef72", "metadata": {}, "outputs": [], @@ -369,7 +404,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": null, "id": "e1dcff76", "metadata": {}, "outputs": [], @@ -395,7 +430,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": null, "id": "84b36b0d", "metadata": {}, "outputs": [], @@ -405,7 +440,7 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": null, "id": "2d11ee3b", "metadata": {}, "outputs": [], @@ -416,7 +451,7 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": null, "id": "e9926eca", "metadata": {}, "outputs": [], @@ -427,7 +462,7 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": null, "id": "d148accc", "metadata": {}, "outputs": [], @@ -438,7 +473,7 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": null, "id": "56a39522", "metadata": {}, "outputs": [], @@ -449,7 +484,7 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": null, "id": "1da10d8a", "metadata": {}, "outputs": [], @@ -468,7 +503,7 @@ }, { "cell_type": "code", - "execution_count": 111, + "execution_count": null, "id": "203de6d1", "metadata": {}, "outputs": [], @@ -511,9 +546,9 @@ ], "metadata": { "kernelspec": { - "display_name": "TXPipe-2023-Jul-12", + "display_name": "Txpipe_clp", "language": "python", - "name": "txpipe-2023-jul-12" + "name": "txpipe_clp" }, "language_info": { "codemirror_mode": { @@ -525,7 +560,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.53.10.12" + "version": "3.10.18" }, "nbdime-conflicts": { "local_diff": [ diff --git a/notebooks/cluster_counts/gt_treecor_test/pipeline-1deg2-CL-nersc.yml b/notebooks/cluster_counts/gt_treecor_test/pipeline-1deg2-CL-nersc.yml index f93c9c55..2eb7a1cc 100644 --- a/notebooks/cluster_counts/gt_treecor_test/pipeline-1deg2-CL-nersc.yml +++ b/notebooks/cluster_counts/gt_treecor_test/pipeline-1deg2-CL-nersc.yml @@ -22,29 +22,25 @@ modules: > python_paths: [] stages: -# - name: TXSourceSelectorMetadetect -# nprocess: 1 -# - name: BPZliteInformer -# nprocess: 1 -# - name: BPZ_lite -# nprocess: 1 - - name: CLClusterBinningRedshiftRichness + - name: TXSourceSelectorMetadetect + nprocess: 1 + - name: BPZliteInformer nprocess: 1 - - name: CLClusterShearCatalogs + - name: BPZ_lite + nprocess: 1 + - name: CLClusterBinningRedshiftRichness nprocess: 1 - - name: CLClusterEnsembleProfiles + - name: TXTwoPointRLens nprocess: 1 -# - name: CLClusterDataVector -# nprocess: 1 -output_dir: ./data/cosmodc2/outputs-1deg2-CL -config: examples/cosmodc2/Cluster_pipelines/config-1deg2-CL.yml +output_dir: notebooks/cluster_counts/gt_treecor_test/outputs-1deg2-CL +config: notebooks/cluster_counts/gt_treecor_test/config-1deg2-CL.yml inputs: # See README for paths to download these files - shear_catalog: ./data/example/inputs/metadetect_shear_catalog.hdf5 + #shear_catalog: ./data/example/inputs/metadetect_shear_catalog.hdf5 #photometry_catalog: ./data/example/inputs/photometry_catalog.hdf5 fiducial_cosmology: ./data/fiducial_cosmology.yml #calibration_table: ./data/example/inputs/sample_cosmodc2_w10year_errors.dat diff --git a/txpipe/extensions/cluster_counts/__init__.py b/txpipe/extensions/cluster_counts/__init__.py index 1a5365e6..b69ad61e 100644 --- a/txpipe/extensions/cluster_counts/__init__.py +++ b/txpipe/extensions/cluster_counts/__init__.py @@ -1,8 +1,8 @@ #from .ingest import * -#from .rlens import TXTwoPointRLens +from .rlens import TXTwoPointRLens #from .select import CLClusterShearCatalogs from .bin_cluster import CLClusterBinningRedshiftRichness from .sources_select_compute import * from .make_ensemble_profile import CLClusterEnsembleProfiles -from .convert_to_sacc import CLClusterSACC \ No newline at end of file +from .convert_to_sacc import CLClusterSACC From 664b2519a0e88c7acf65ff76dd367da00492089b Mon Sep 17 00:00:00 2001 From: eduardojsbarroso Date: Tue, 23 Sep 2025 07:12:13 -0700 Subject: [PATCH 3/7] Added Notebook --- .../gt_treecor_test/Run_CL_pipeline.ipynb | 297 ------------------ 1 file changed, 297 deletions(-) diff --git a/notebooks/cluster_counts/gt_treecor_test/Run_CL_pipeline.ipynb b/notebooks/cluster_counts/gt_treecor_test/Run_CL_pipeline.ipynb index a01bde7f..22d4e8f0 100644 --- a/notebooks/cluster_counts/gt_treecor_test/Run_CL_pipeline.ipynb +++ b/notebooks/cluster_counts/gt_treecor_test/Run_CL_pipeline.ipynb @@ -245,303 +245,6 @@ "step3.run()\n", "step3.finalize()" ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "7f65c156-fe6d-47cd-8b0b-57baa4e7755f", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "76324c34-3dfa-4886-b78b-72589f9dc49b", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "markdown", - "id": "b914ddff", - "metadata": {}, - "source": [ - "___" - ] - }, - { - "cell_type": "markdown", - "id": "63473b4a", - "metadata": {}, - "source": [ - "___" - ] - }, - { - "cell_type": "markdown", - "id": "7ccef7e0", - "metadata": {}, - "source": [ - "# 3 - Producing and looking at the pipeline diagram" - ] - }, - { - "cell_type": "markdown", - "id": "cc256472", - "metadata": {}, - "source": [ - "Here we run the pipeline in \"dry-run\" mode, to check that it can run and to produce a pipeline diagram in png." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "82a24dd3", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "# Read the appropriate pipeline configuration, and ask for a flow-chart.\n", - "\n", - "pipeline_file = \"examples/cosmodc2/Cluster_pipelines/pipeline-20deg2-CL-in2p3.yml\"\n", - "# pipeline_file = \"examples/cosmodc2/Cluster_pipelines/pipeline-20deg2-CL-nersc.yml\"\n", - "flowchart_file = \"CL_pipeline.png\"\n", - "\n", - "\n", - "pipeline_config = ceci.Pipeline.build_config(pipeline_file, flow_chart=flowchart_file, dry_run=True)\n", - "\n", - "# Run the flow-chart pipeline\n", - "ceci.run_pipeline(pipeline_config);" - ] - }, - { - "cell_type": "markdown", - "id": "f3c19dcb", - "metadata": {}, - "source": [ - "Here we have 6 uncommented stages, and 5 uncommented inputs files.\n", - "\n", - "This translate in the pipeline charts as 6 red ellipses, and 5 yellow boxes. The blue boxes represents the ouput files." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "9e60cace", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "Image(flowchart_file)" - ] - }, - { - "cell_type": "markdown", - "id": "b986818d", - "metadata": {}, - "source": [ - "___" - ] - }, - { - "cell_type": "markdown", - "id": "5dd025a3", - "metadata": {}, - "source": [ - "# 4 - Opening and looking at the outputs" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "1463b613", - "metadata": {}, - "outputs": [], - "source": [ - "#At the moment the ouput file is a pickle file\n", - "import pickle as pkl" - ] - }, - { - "cell_type": "markdown", - "id": "f65b5c1b", - "metadata": {}, - "source": [ - "### **Open the pipeline file to load correct input/output file names**" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "bbc4cb52", - "metadata": {}, - "outputs": [], - "source": [ - "with open(pipeline_file, \"r\") as file:\n", - " pipeline_content = yaml.safe_load(file)" - ] - }, - { - "cell_type": "markdown", - "id": "05c9f270", - "metadata": {}, - "source": [ - "### **Open the output**" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "5550ef72", - "metadata": {}, - "outputs": [], - "source": [ - "data = pkl.load(open(pipeline_content[\"output_dir\"] + \"/cluster_profiles.pkl\", \"rb\"))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "e1dcff76", - "metadata": {}, - "outputs": [], - "source": [ - "data" - ] - }, - { - "cell_type": "markdown", - "id": "0fdf85f7", - "metadata": {}, - "source": [ - "The output is a dictionary containing a `CLMM` `ClusterEnsemble` object for each redshift/richness bin." - ] - }, - { - "cell_type": "markdown", - "id": "504fc48b", - "metadata": {}, - "source": [ - "### **Exploring the output**" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "84b36b0d", - "metadata": {}, - "outputs": [], - "source": [ - "example_bin = 'bin_zbin_0_richbin_0'" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "2d11ee3b", - "metadata": {}, - "outputs": [], - "source": [ - "#This is the info for this bin\n", - "data[example_bin]" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "e9926eca", - "metadata": {}, - "outputs": [], - "source": [ - "#This shows the table with all cluster in this bin and their corresponding profiles\n", - "data[example_bin]['clmm_cluster_ensemble'].data" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "d148accc", - "metadata": {}, - "outputs": [], - "source": [ - "#This shows the table with all cluster in this bin and their corresponding profiles\n", - "data[example_bin]['clmm_cluster_ensemble'].data.meta" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "56a39522", - "metadata": {}, - "outputs": [], - "source": [ - "#This shows the ensemble stacked profiles\n", - "data[example_bin]['clmm_cluster_ensemble'].stacked_data.meta" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "1da10d8a", - "metadata": {}, - "outputs": [], - "source": [ - "#This shows the covarinace for the ensemble stacked profiles\n", - "data[example_bin]['clmm_cluster_ensemble'].cov" - ] - }, - { - "cell_type": "markdown", - "id": "3c32d5a1", - "metadata": {}, - "source": [ - "### **Plot the ouput**" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "203de6d1", - "metadata": {}, - "outputs": [], - "source": [ - "fig, ax = plt.subplots()\n", - "\n", - "ax.semilogx(\n", - " data[example_bin]['clmm_cluster_ensemble'].stacked_data[\"radius\"],\n", - " data[example_bin]['clmm_cluster_ensemble'].stacked_data[\"tangential_comp\"],\n", - " \"bx-\",\n", - " label=\"tan\",)\n", - "\n", - "ax.semilogx(\n", - " data[example_bin]['clmm_cluster_ensemble'].stacked_data[\"radius\"],\n", - " data[example_bin]['clmm_cluster_ensemble'].stacked_data[\"cross_comp\"],\n", - " \"r.-\",\n", - " label=\"cross\",)\n", - "\n", - "ax.errorbar(\n", - " data[example_bin]['clmm_cluster_ensemble'].stacked_data[\"radius\"],\n", - " data[example_bin]['clmm_cluster_ensemble'].stacked_data[\"tangential_comp\"],\n", - " yerr=data[example_bin]['clmm_cluster_ensemble'].cov[\"tan_sc\"].diagonal() ** 0.5,\n", - " color=\"blue\",)\n", - "\n", - "\n", - "ax.errorbar(\n", - " data[example_bin]['clmm_cluster_ensemble'].stacked_data[\"radius\"],\n", - " data[example_bin]['clmm_cluster_ensemble'].stacked_data[\"cross_comp\"],\n", - " yerr=data[example_bin]['clmm_cluster_ensemble'].cov[\"cross_sc\"].diagonal() ** 0.5,\n", - " color=\"red\",)\n", - "\n", - "\n", - "\n", - "ax.set_xlabel('radius [Mpc]')\n", - "ax.set_ylabel('$\\\\Delta \\\\Sigma$')\n", - "\n", - "plt.legend()" - ] } ], "metadata": { From a746c39a7faff80188649c085245ab466046005d Mon Sep 17 00:00:00 2001 From: Celine Combet Date: Wed, 24 Sep 2025 01:44:29 -0700 Subject: [PATCH 4/7] Demo of CLMM gt averaging versus TreeCorr approach, not using TXpipe and on ideal mock data --- .../gt_CLMM_versus_treecorr.ipynb | 842 ++++++++++++++++++ 1 file changed, 842 insertions(+) create mode 100644 notebooks/cluster_counts/gt_treecor_test/gt_CLMM_versus_treecorr.ipynb diff --git a/notebooks/cluster_counts/gt_treecor_test/gt_CLMM_versus_treecorr.ipynb b/notebooks/cluster_counts/gt_treecor_test/gt_CLMM_versus_treecorr.ipynb new file mode 100644 index 00000000..ab60b410 --- /dev/null +++ b/notebooks/cluster_counts/gt_treecor_test/gt_CLMM_versus_treecorr.ipynb @@ -0,0 +1,842 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "10a9ad67-1bc5-4d5d-995d-b8fed6e8d9cd", + "metadata": { + "tags": [] + }, + "source": [ + "# Compare TreeCorr to CLMM for stacked reduced tangential shear measurement" + ] + }, + { + "cell_type": "markdown", + "id": "9aae087b-4b7c-4c9b-aa64-8099ca1265bd", + "metadata": {}, + "source": [ + "" + ] + }, + { + "cell_type": "markdown", + "id": "a97719af-09d7-40f2-8a7d-0d2790b42764", + "metadata": {}, + "source": [ + "This notebook compares in a very ideal setup the computation of the stacked reduced shear profile either using CLMM or using TreeCorr to cross-correlate cluster positions with background source shears. The mock data is generated using CLMM.\n", + "\n", + "This provides a quick demo of what issue TXPpipe issue #409 aims at doing in spirit..." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "10b56798-1583-4ef3-a491-cfe0ce1ef74a", + "metadata": {}, + "outputs": [], + "source": [ + "import sys\n", + "import numpy as np\n", + "import pyccl as ccl\n", + "import matplotlib.pyplot as plt\n", + "import scipy\n", + "import clmm\n", + "from clmm import GalaxyCluster, ClusterEnsemble, GCData\n", + "from clmm import Cosmology\n", + "from clmm.support import mock_data as mock\n", + "\n", + "%matplotlib inline" + ] + }, + { + "cell_type": "markdown", + "id": "513df539-fe8c-4a3e-8659-599db2a0c73d", + "metadata": {}, + "source": [ + "For reproducibility:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "35e4438c-aa82-48e5-b23d-6a211136b64b", + "metadata": {}, + "outputs": [], + "source": [ + "np.random.seed(11)" + ] + }, + { + "cell_type": "markdown", + "id": "7b8270d7-a8b5-4def-a4d7-ff6a6e58acbb", + "metadata": {}, + "source": [ + "## Draw pairs of mass and redshift according to a mass function" + ] + }, + { + "cell_type": "markdown", + "id": "2d8af669-b22a-43ca-a1b3-9bf85bae65c9", + "metadata": {}, + "source": [ + "### Mass function - CCL implementation" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4aea0ec5-56b6-4a2e-bf09-3889357c0552", + "metadata": {}, + "outputs": [], + "source": [ + "cosmo = ccl.Cosmology(\n", + " Omega_c=0.265,\n", + " Omega_b=0.0448,\n", + " h=0.71,\n", + " sigma8=0.8,\n", + " n_s=0.96,\n", + " Neff=3.04,\n", + " m_nu=1.0e-05,\n", + " mass_split=\"single\",\n", + ")\n", + "hmd_200c = ccl.halos.MassDef(200, \"critical\")\n", + "\n", + "# For a different multiplicty function, the user must change this function below\n", + "def tinker08_ccl(logm, z):\n", + " mass = 10 ** (logm)\n", + " hmf_200c = ccl.halos.MassFuncTinker08(mass_def=hmd_200c)\n", + " nm = hmf_200c(cosmo, mass, 1.0 / (1 + z))\n", + " return nm # dn/dlog10M\n", + "\n", + "\n", + "# Computing the volume element\n", + "def dV_over_dOmega_dz(z):\n", + " a = 1.0 / (1.0 + z)\n", + " da = ccl.background.angular_diameter_distance(cosmo, a)\n", + " E = ccl.background.h_over_h0(cosmo, a)\n", + " return ((1.0 + z) ** 2) * (da**2) * ccl.physical_constants.CLIGHT_HMPC / cosmo[\"h\"] / E\n", + "\n", + "\n", + "def pdf_tinker08_ccl(logm, z):\n", + " return tinker08_ccl(logm, z) * dV_over_dOmega_dz(z)" + ] + }, + { + "cell_type": "markdown", + "id": "f2e14265-ee36-4669-9275-5758348acc4e", + "metadata": {}, + "source": [ + "### Acceptance-rejection method to sample (M,z) from the mass function" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3ec03552-6957-4aac-adfc-c4fc2744dd35", + "metadata": {}, + "outputs": [], + "source": [ + "def bivariate_draw(\n", + " pdf, N=1000, logm_min=14.0, logm_max=15.0, zmin=0.01, zmax=1, Ngrid_m=30, Ngrid_z=30\n", + "):\n", + " \"\"\"\n", + " Uses the rejection method for generating random numbers derived from an arbitrary\n", + " probability distribution.\n", + "\n", + " Parameters\n", + " ----------\n", + " pdf : func\n", + " 2d distribution function to sample\n", + " N : int\n", + " number of points to generate\n", + " log_min,logm_max : float\n", + " log10 mass range\n", + " zmin,zmax : float\n", + " redshift range\n", + "\n", + " Returns:\n", + " --------\n", + " ran_logm : list\n", + " accepted logm values\n", + " ran_z : list\n", + " accepted redshift values\n", + " acceptance : float\n", + " acceptance ratio of the method\n", + " \"\"\"\n", + "\n", + " # maximum value of the pdf over the mass and redshift space.\n", + " # the pdf is not monotonous in mass and redshift, so we need\n", + " # to find the maximum numerically. Here we scan the space\n", + " # with a regular grid and use the maximum value.\n", + " # Accuracy of the results depends on Ngrid_m and Ngrid_z\n", + " # This should probably be improved\n", + "\n", + " logM_arr = np.linspace(logm_min, logm_max, Ngrid_m)\n", + " z_arr = np.logspace(np.log10(zmin), np.log10(zmax), Ngrid_z)\n", + " p = []\n", + " for logM in logM_arr:\n", + " for z in z_arr:\n", + " p.append(pdf(logM, z))\n", + " pmax = np.max(p)\n", + " pmin = np.min(p)\n", + " # Counters\n", + " naccept = 0\n", + " ntrial = 0\n", + "\n", + " # Keeps drawing until N points are accepted\n", + " ran_logm = [] # output list of random numbers\n", + " ran_z = [] # output list of random numbers\n", + " while naccept < N:\n", + " # draw (logm,z) from uniform distribution\n", + " # draw p from uniform distribution\n", + " logm = np.random.uniform(logm_min, logm_max) # x'\n", + " z = np.random.uniform(zmin, zmax) # x'\n", + " p = np.random.uniform(0.0, pmax) # y'\n", + "\n", + " if p < pdf(logm, z):\n", + " # keep the point\n", + " # print(logm, z, p, pdf(logm, z))\n", + " ran_logm.append(logm)\n", + " ran_z.append(z)\n", + " naccept = naccept + 1\n", + " ntrial = ntrial + 1\n", + "\n", + " ran_logm = np.asarray(ran_logm)\n", + " ran_z = np.asarray(ran_z)\n", + "\n", + " acceptance = float(N / ntrial)\n", + " print(f\"acceptance = {acceptance}\")\n", + " return ran_logm, ran_z, acceptance" + ] + }, + { + "cell_type": "markdown", + "id": "cafac5cf-3ef1-4189-b88e-326b4df28829", + "metadata": {}, + "source": [ + "### Draw N=30 (M,z) pairs" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "84bd6be3-60b8-47a0-b86b-1659081d7c95", + "metadata": {}, + "outputs": [], + "source": [ + "# Define the numbers of pairs to draw and the mass and redshift ranges\n", + "N = 30\n", + "logm_min = 14.5\n", + "logm_max = 14.8\n", + "zmin = 0.2\n", + "zmax = 0.3\n", + "\n", + "pdf = pdf_tinker08_ccl\n", + "\n", + "# Normalisation factor for the pdf to be used later\n", + "norm = (\n", + " 1.0\n", + " / scipy.integrate.dblquad(\n", + " pdf, zmin, zmax, lambda x: logm_min, lambda x: logm_max, epsrel=1.0e-4\n", + " )[0]\n", + ")\n", + "\n", + "# Random draw\n", + "ran_logm, ran_z, acceptance = bivariate_draw(\n", + " pdf, N=N, logm_min=logm_min, logm_max=logm_max, zmin=zmin, zmax=zmax\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "1988a5a5-8a75-4a7b-b8f4-84c50cd45b1f", + "metadata": {}, + "source": [ + "## Generate a cluster catalog and associated source catalogs\n", + "- First, set the cluster masses and redshifts given the generated mock data.\n", + "- Then, instantiate a CCL concentration object to compute the concentration for each cluster from a mass-concentration realtion (Duffy et al. 2008). The actual concentration for each cluster is drawn from a lognormal distribution around the mean value\n", + "- Last, we randomly generate the cluster center coordinates over the full sky (from 0 to 360 deg for ra and from -90 to 90 deg to dec)" + ] + }, + { + "cell_type": "markdown", + "id": "66911b7b-b7cf-49a3-97da-0f93895cfafe", + "metadata": {}, + "source": [ + "### Cluster ensemble masses, redshifts and positions" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "35d49918-69ea-43cb-91f4-426373247407", + "metadata": {}, + "outputs": [], + "source": [ + "cluster_m = 10 ** ran_logm\n", + "cluster_z = ran_z\n", + "\n", + "# Concentration CCL object to compute the theoretical concentration\n", + "conc_obj = ccl.halos.ConcentrationDuffy08(mass_def=hmd_200c)\n", + "conc_list = []\n", + "for number in range(0, len(cluster_m)):\n", + " a = 1.0 / (1.0 + (cluster_z[number]))\n", + " # mean value of the concentration for that cluster\n", + " lnc_mean = np.log(conc_obj(cosmo, M=(cluster_m[number]), a=a))\n", + " # random draw of actual concentration from normal distribution around lnc_mean, with a 0.14 scatter\n", + " lnc = np.random.normal(lnc_mean, 0.14)\n", + " conc_list.append(np.exp(lnc))\n", + "\n", + "conc_list = np.array(conc_list)\n", + "\n", + "# randomly draw cluster positions over the full sky\n", + "ra = np.random.random(N) * 360 # from 0 to 360 deg\n", + "sindec = np.random.random(N) * 2 - 1\n", + "dec = np.arcsin(sindec) * 180 / np.pi # from -90 to 90 deg" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ed3c55f5-8541-45a1-a7bf-6269239b1cba", + "metadata": {}, + "outputs": [], + "source": [ + "len(cluster_m)\n", + "N" + ] + }, + { + "cell_type": "markdown", + "id": "6a7196b0-697c-4547-8305-7dbad76a3537", + "metadata": {}, + "source": [ + "### Background galaxy catalog generation\n", + "\n", + "For each cluster of the ensemble, we use `mock_data` to generate a background galaxy catalog and store the results in a `GalaxyCluster` object. Note that:\n", + "- The cluster density profiles follow the NFW parametrisation\n", + "- The source redshifts follow the Chang et al. distribution and have associated pdfs\n", + "- The shapes include shape noise and shape measurement errors\n", + "- Background galaxy catalogs are independent, even if the clusters are close (i.e., no common galaxy between two catalogs).\n", + "- For each cluster we then compute\n", + " - the tangential and cross $\\Delta\\Sigma$ for each background galaxy\n", + " - the weights `w_ls` to be used to compute the corresponding radial profiles (see `demo_compute_deltasigma_weights.ipynb` notebook for details)\n", + "\n", + "The cluster objects are then stored in `gclist`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "16bc0935-8663-413a-90a6-4b888d2d32ca", + "metadata": {}, + "outputs": [], + "source": [ + "import warnings\n", + "\n", + "warnings.simplefilter(\n", + " \"ignore\"\n", + ") # just to prevent warning print out when looping over the cluster ensemble below" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "76577e8d-4e94-48bd-afb3-483ac0d252be", + "metadata": {}, + "outputs": [], + "source": [ + "from astropy.table import Table, vstack\n", + "gclist = []\n", + "tables = []\n", + "# number of galaxies in each cluster field (alternatively, can use the galaxy density instead)\n", + "#n_gals = 10000\n", + "ngal_density = 10\n", + "cosmo_clmm = Cosmology(H0=71.0, Omega_dm0=0.265 - 0.0448, Omega_b0=0.0448, Omega_k0=0.0)\n", + "cosmo_clmm.set_be_cosmo(cosmo)\n", + "gal_args = dict(\n", + " cosmo=cosmo_clmm,\n", + " zsrc=\"chang13\",\n", + " delta_so=200,\n", + " massdef=\"critical\",\n", + " halo_profile_model=\"nfw\",\n", + " zsrc_max=2.0,\n", + " field_size=30.0,\n", + " shapenoise=0.04,\n", + " photoz_sigma_unscaled=0.02,\n", + "# ngals=n_gals,\n", + " ngal_density=ngal_density,\n", + " mean_e_err=0.1,\n", + ")\n", + "\n", + "for i in range(N):\n", + " # generate background galaxy catalog for cluster i\n", + " cl = clmm.GalaxyCluster(\n", + " f\"mock_cluster_{i:04}\",\n", + " ra[i],\n", + " dec[i],\n", + " cluster_z[i],\n", + " galcat=mock.generate_galaxy_catalog(\n", + " cluster_m[i],\n", + " cluster_z[i],\n", + " conc_list[i],\n", + " cluster_ra=ra[i],\n", + " cluster_dec=dec[i],\n", + " zsrc_min=0.35, # cluster zmax = 0.3 in this example\n", + " **gal_args,\n", + " ),\n", + " )\n", + "\n", + " # compute DeltaSigma for each background galaxy\n", + " cl.compute_tangential_and_cross_components(\n", + " shape_component1=\"e1\",\n", + " shape_component2=\"e2\",\n", + " tan_component=\"g_t\",\n", + " cross_component=\"g_x\",\n", + " cosmo=cosmo_clmm,\n", + " is_deltasigma=False,\n", + " use_pdz=True,\n", + " )\n", + "\n", + " # compute the weights to be used to bluid the DeltaSigma radial profiles\n", + " cl.compute_galaxy_weights(\n", + " use_pdz=True,\n", + " use_shape_noise=True,\n", + " shape_component1=\"e1\",\n", + " shape_component2=\"e2\",\n", + " use_shape_error=True,\n", + " shape_component1_err=\"e_err\",\n", + " shape_component2_err=\"e_err\",\n", + " weight_name=\"w_ls\",\n", + " cosmo=cosmo_clmm,\n", + " is_deltasigma=False,\n", + " add=True,\n", + " )\n", + "\n", + " # append the cluster in the list\n", + " gclist.append(cl)\n", + " tables.append(cl.galcat)\n", + " \n", + "galcat_tot = vstack(tables) " + ] + }, + { + "cell_type": "markdown", + "id": "bf115b3e-d68b-4ea1-bf8c-5372cba5c88f", + "metadata": {}, + "source": [ + "## Create ClusterEnsemble object and estimation of individual excess surface density profiles\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1e96711c-2628-4052-aa77-83c008c9ec8a", + "metadata": {}, + "outputs": [], + "source": [ + "#bins = np.logspace(np.log10(0.3), np.log10(5), 10)\n", + "bins = np.logspace(np.log10(0.5),np.log10(60), 12)\n", + "ensemble_id = 1\n", + "clusterensemble = ClusterEnsemble(ensemble_id)\n", + "for cluster in gclist:\n", + " clusterensemble.make_individual_radial_profile(\n", + " galaxycluster=cluster,\n", + " tan_component_in=\"g_t\",\n", + " cross_component_in=\"g_x\",\n", + " tan_component_out=\"g_t\",\n", + " cross_component_out=\"g_x\",\n", + " weights_in=\"w_ls\",\n", + " weights_out=\"W_l\",\n", + " bins=bins,\n", + " bin_units=\"arcmin\",\n", + "# bin_units=\"Mpc\",\n", + " cosmo=cosmo_clmm,\n", + " )" + ] + }, + { + "cell_type": "markdown", + "id": "a490213d-5051-4f33-acda-2be29e928e82", + "metadata": {}, + "source": [ + "The individual cluster data and profiles are stored at the `.data` table of the `ClusterEnsemble`:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9865b58c-a428-4e54-b942-2a671c0a771a", + "metadata": {}, + "outputs": [], + "source": [ + "clusterensemble.data[:3]" + ] + }, + { + "cell_type": "markdown", + "id": "798d4180-10c4-4e25-89a0-9c25d0b2192e", + "metadata": {}, + "source": [ + "The edges of the radial bins, their units, and the cosmology are stored on the metadata of this table:" + ] + }, + { + "cell_type": "markdown", + "id": "6666b394-f7df-4b13-8539-2da83dd5be68", + "metadata": { + "tags": [] + }, + "source": [ + "### Stacked profile of the cluster ensemble\n", + "The stacked radial profile of the ensemble is then obtained as " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "de7daf03-ef68-4060-b385-c89f734284d2", + "metadata": {}, + "outputs": [], + "source": [ + "clusterensemble.make_stacked_radial_profile(tan_component=\"g_t\", cross_component=\"g_x\", weights=\"W_l\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f2f4f65a-6331-4f64-bac8-3ebe5f4d8d4c", + "metadata": {}, + "outputs": [], + "source": [ + "clusterensemble.stacked_data" + ] + }, + { + "cell_type": "markdown", + "id": "7de99242-1c98-4df0-bf23-3e04ebfc258d", + "metadata": { + "tags": [] + }, + "source": [ + "### Jackknife covariance of the stack between radial bins" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b2d2b8d3-6e33-4085-92d7-23c1f5c03427", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "clusterensemble.compute_jackknife_covariance(\n", + " n_side=16, tan_component=\"g_t\", cross_component=\"g_x\"\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "d21e0a19-de24-4b3e-bbc9-527e3a58019a", + "metadata": { + "tags": [] + }, + "source": [ + "### Visualizing the stacked profiles\n", + "In the figure below, we plot:\n", + "- the individual g_t profiles of the clusters (light blue)\n", + "- the stacked signal (red symbols)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9f7f0afb-5a8a-4193-9a08-70817fd2129c", + "metadata": {}, + "outputs": [], + "source": [ + "moo = clmm.Modeling(massdef=\"critical\", delta_mdef=200, halo_profile_model=\"nfw\")\n", + "moo.set_cosmo(cosmo_clmm)\n", + "# Average values of mass and concentration of the ensemble to be used below\n", + "# to overplot the model on the stacked profile\n", + "moo.set_concentration(conc_list.mean())\n", + "moo.set_mass(cluster_m.mean())" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "05dea61d-f0aa-4d13-8b2e-c7fc915bd3a4", + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "from clmm.redshift.distributions import desc_srd\n", + "\n", + "r_stack, gt_stack, gx_stack = (clusterensemble.stacked_data[c] for c in (\"radius\", \"g_t\", \"g_x\"))\n", + "plt.rcParams[\"axes.linewidth\"] = 2\n", + "fig, axs = plt.subplots(1, 2, figsize=(17, 6))\n", + "\n", + "err_gt = clusterensemble.cov[\"tan_jk\"].diagonal() ** 0.5\n", + "err_gx = clusterensemble.cov[\"cross_jk\"].diagonal() ** 0.5\n", + "\n", + "axs[0].errorbar(\n", + " r_stack,\n", + " gt_stack,\n", + " err_gt,\n", + " markersize=5,\n", + " c=\"r\",\n", + " fmt=\"o\",\n", + " capsize=10,\n", + " elinewidth=1,\n", + " zorder=1000,\n", + " alpha=1,\n", + " label=\"Stack\",\n", + ")\n", + "axs[1].errorbar(\n", + " r_stack,\n", + " gx_stack,\n", + " err_gx,\n", + " markersize=5,\n", + " c=\"r\",\n", + " fmt=\"o\",\n", + " capsize=10,\n", + " elinewidth=1,\n", + " zorder=1000,\n", + " alpha=1,\n", + " label=\"Stack\",\n", + ")\n", + "\n", + "axs[0].set_xscale(\"log\")\n", + "axs[0].set_yscale(\"log\")\n", + "axs[1].set_xscale(\"log\")\n", + "\n", + "\n", + "for i in range(N):\n", + " axs[0].plot(\n", + " clusterensemble.data[\"radius\"][i],\n", + " clusterensemble.data[\"g_t\"][i],\n", + " color=\"cyan\",\n", + " label=\"Individual\",\n", + " alpha=1,\n", + " linewidth=1,\n", + " )\n", + " axs[1].plot(\n", + " clusterensemble.data[\"radius\"][i],\n", + " clusterensemble.data[\"g_x\"][i],\n", + " color=\"cyan\",\n", + " label=\"Individual\",\n", + " alpha=1,\n", + " linewidth=1,\n", + " )\n", + " if i == 0:\n", + " axs[0].legend(frameon=False, fontsize=15)\n", + " axs[1].legend(frameon=False, fontsize=15)\n", + "# axs[0].plot(np.average(clusterensemble.data['radius'], axis = 0), np.average(clusterensemble.data['gt'], weights = None, axis = 0)/1e13)\n", + "# axs[0].set_xlabel(\"R [Mpc]\", fontsize=20)\n", + "# axs[1].set_xlabel(\"R [Mpc]\", fontsize=20)\n", + "axs[0].set_xlabel(\"separation [arcmin]\", fontsize=20)\n", + "axs[1].set_xlabel(\"separation [arcmin]\", fontsize=20)\n", + "axs[0].tick_params(axis=\"both\", which=\"major\", labelsize=18)\n", + "axs[1].tick_params(axis=\"both\", which=\"major\", labelsize=18)\n", + "axs[0].set_ylabel(r\"$g_+$\", fontsize=20)\n", + "axs[1].set_ylabel(r\"$g_\\times$\", fontsize=20)\n", + "axs[0].set_title(r\"Tangential\", fontsize=20)\n", + "axs[1].set_title(r\"Cross\", fontsize=20)\n", + "\n", + "for ax in axs:\n", + " ax.minorticks_on()\n", + " ax.grid(lw=0.5)\n", + " ax.grid(which=\"minor\", lw=0.1)\n", + "fig.tight_layout()" + ] + }, + { + "cell_type": "markdown", + "id": "7d31ea5f-b7c3-4281-8c97-6d6bd9ccca35", + "metadata": { + "tags": [] + }, + "source": [ + "## Saving/Loading ClusterEnsemble\n", + "The `ClusterEnsemble` object also have an option for saving/loading:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "515f4ea6-e759-465c-93f6-f1c5ea9130a3", + "metadata": {}, + "outputs": [], + "source": [ + "clusterensemble.save(\"ce.pkl\")" + ] + }, + { + "cell_type": "markdown", + "id": "4d012995-d07d-4c1e-94b8-63bbc56cb457", + "metadata": {}, + "source": [ + "## Compute reduced shear profile with TreeCorr and compare to CLMM stacked profile" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9a4ea865-74cb-42ef-b4c0-50bf8ce2bf9d", + "metadata": {}, + "outputs": [], + "source": [ + "import treecorr\n", + "# Source catalog\n", + "cat_source = treecorr.Catalog(\n", + " ra = galcat_tot[\"ra\"],\n", + " dec = galcat_tot[\"dec\"],\n", + " g1 = galcat_tot[\"e1\"],\n", + " g2 = galcat_tot[\"e2\"],\n", + " ra_units = \"deg\",\n", + " dec_units = \"deg\",\n", + " w=galcat_tot[\"w_ls\"] \n", + ")\n", + "\n", + "# Cluster catalog\n", + "cat_cluster = treecorr.Catalog(\n", + " ra = clusterensemble[\"ra\"],\n", + " dec = clusterensemble[\"dec\"],\n", + " ra_units = \"deg\",\n", + " dec_units = \"deg\",\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9efc3109-4327-4768-9465-d72898791d3e", + "metadata": {}, + "outputs": [], + "source": [ + "ng = treecorr.NGCorrelation(nbins=11, min_sep=0.5, max_sep=60, sep_units=\"arcmin\",bin_type = \"Log\")#, bin_slop=0.1)\n", + "ng.process(cat_cluster,cat_source)\n", + "theta = ng.meanr # average angular separation in each bin\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a9feccc2-2178-45e7-9f1e-4bfb2d307179", + "metadata": {}, + "outputs": [], + "source": [ + "cosmo_clmm.be_cosmo" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "459bba0e-213e-4b86-89f2-7068cc391a32", + "metadata": {}, + "outputs": [], + "source": [ + "da = cosmo.angular_diameter_distance(1./(1.+np.mean(clusterensemble[\"z\"])))\n", + "da_clmm = cosmo_clmm.eval_da(np.mean(clusterensemble[\"z\"]))\n", + "da, da_clmm" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5c7b0ead-7733-467a-9c5c-bdf8f0278f20", + "metadata": {}, + "outputs": [], + "source": [ + "arcmin_to_Mpc = np.pi / (60. * 180) * da_clmm\n", + "\n", + "fig, ax = plt.subplots(nrows=1, ncols=1, figsize=(10,5))\n", + "# ax.errorbar(R, ng.xi, np.sqrt(ng.varxi), color='blue', marker='o', \n", + "# linestyle='', label='Treecorr - clusters positions x source shear')\n", + "ax.errorbar(theta, ng.xi, np.sqrt(ng.varxi), color='blue', marker='x', markersize=15,\n", + " linestyle='', label='Treecorr - cluster positions x source shears')\n", + "\n", + "\n", + "# axes[0].errorbar(1.1*R_highm_ran*cosmo.H0/100., ng_highm_ran.xi, np.sqrt(ng_highm_ran.varxi), color = 'grey', marker='o',\n", + "# linestyle='', label='shuffle RA ~ random cluster locations')\n", + "ax.errorbar(\n", + " r_stack,\n", + " gt_stack,\n", + " err_gt,\n", + " markersize=5,\n", + " c=\"r\",\n", + " fmt=\".\",\n", + " capsize=10,\n", + " elinewidth=1,\n", + " zorder=1000,\n", + " alpha=1,\n", + " label=\"CLMM - stacked reduced rangential shear\",\n", + ")\n", + "\n", + "ax.plot(\n", + " clusterensemble.data[\"radius\"][0],\n", + " moo.eval_reduced_tangential_shear(clusterensemble.data[\"radius\"][0]*arcmin_to_Mpc, cluster_z.mean(), \n", + " z_src=desc_srd, z_src_info='distribution'),\n", + " \"--k\",\n", + " linewidth=1,\n", + " label=\"CLMM prediction for stack 'mean' cluster\",\n", + " zorder=100,\n", + ")\n", + "\n", + "ax.set_xscale('log')\n", + "ax.set_yscale('log')\n", + "ax.set_ylim([-0.02,0.15])\n", + "ax.axhline(0, color='k', ls=':')\n", + "ax.set_xlabel('separation [arcmin]')\n", + "ax.set_ylabel('$\\widehat{g_t}$')\n", + "ax.legend()\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "992a44f3-0f57-44fd-bd78-62326016631c", + "metadata": {}, + "outputs": [], + "source": [ + "plt.plot(err_gt, np.sqrt(ng.varxi))\n", + "plt.plot([0, 0.007],[0, 0.007], 'k:')\n", + "plt.xlabel(\"Error from CLMM\")\n", + "plt.ylabel(\"Error from TreeCorr\")" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Txpipe_clp", + "language": "python", + "name": "txpipe_clp" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.18" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} From ea60a5ad8d8dd6c32e84809c95556ae45fd95554 Mon Sep 17 00:00:00 2001 From: eduardojsbarroso Date: Fri, 26 Sep 2025 10:06:51 +0200 Subject: [PATCH 5/7] Added new files --- .../gt_treecor_test/Run_CL_pipeline.ipynb | 249 +++-- .../cluster_counts/gt_treecor_test/debug.log | 0 txpipe/extensions/cluster_counts/rlens_new.py | 958 ++++++++++++++++++ 3 files changed, 1133 insertions(+), 74 deletions(-) create mode 100644 notebooks/cluster_counts/gt_treecor_test/debug.log create mode 100644 txpipe/extensions/cluster_counts/rlens_new.py diff --git a/notebooks/cluster_counts/gt_treecor_test/Run_CL_pipeline.ipynb b/notebooks/cluster_counts/gt_treecor_test/Run_CL_pipeline.ipynb index 22d4e8f0..56a1693f 100644 --- a/notebooks/cluster_counts/gt_treecor_test/Run_CL_pipeline.ipynb +++ b/notebooks/cluster_counts/gt_treecor_test/Run_CL_pipeline.ipynb @@ -18,21 +18,23 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 2, + "id": "adbec32f-9dc4-4169-b0e3-4b5ef28076eb", + "metadata": {}, + "outputs": [], + "source": [ + "%load_ext autoreload\n", + "%autoreload 2" + ] + }, + { + "cell_type": "code", + "execution_count": 3, "id": "2edf6e90", "metadata": { "tags": [] }, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/global/cfs/projectdirs/lsst/groups/CL/cl_pipeline_project/conda_envs/txpipe_clp/lib/python3.10/site-packages/ceci/__init__.py:12: UserWarning: pkg_resources is deprecated as an API. See https://setuptools.pypa.io/en/latest/pkg_resources.html. The pkg_resources package is slated for removal as early as 2025-11-30. Refrain from using this package or pin to Setuptools<81.\n", - " from pkg_resources import DistributionNotFound\n" - ] - } - ], + "outputs": [], "source": [ "import os\n", "from pprint import pprint\n", @@ -61,7 +63,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 5, "id": "5eebecd8", "metadata": { "tags": [] @@ -69,7 +71,7 @@ "outputs": [], "source": [ "# user specific paths -- IN2P3 example\n", - "my_txpipe_dir = \"/global/homes/e/edujb/TXPipe/\"\n", + "my_txpipe_dir = \"/sps/lsst/users/ebarroso/TXPipe/\"\n", "\n", "# user specific paths -- NERSC example\n", "# my_txpipe_dir = \"/pscratch/sd/a/avestruz/TXPipe\"\n", @@ -89,7 +91,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 4, "id": "9d3d1e18-eec5-469c-9952-9d0868a2cc2e", "metadata": {}, "outputs": [], @@ -111,7 +113,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 5, "id": "c5e6e548-7d43-43fb-93d7-debaedd9882c", "metadata": {}, "outputs": [ @@ -130,19 +132,10 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 6, "id": "073b332a-ace9-4e27-9c6e-e1c9807bd2e0", "metadata": {}, - "outputs": [ - { - "ename": "IndentationError", - "evalue": "unexpected indent (260147577.py, line 16)", - "output_type": "error", - "traceback": [ - "\u001b[0;36m Cell \u001b[0;32mIn[5], line 16\u001b[0;36m\u001b[0m\n\u001b[0;31m (\"binned_shear_catalog\", ShearCatalog),\u001b[0m\n\u001b[0m ^\u001b[0m\n\u001b[0;31mIndentationError\u001b[0m\u001b[0;31m:\u001b[0m unexpected indent\n" - ] - } - ], + "outputs": [], "source": [ "step2 = txpipe.CLClusterBinningRedshiftRichness.make_stage(\n", " # This file is the input metadetect shear catalog\n", @@ -156,37 +149,15 @@ " # This is the output file for this stage\n", " output_dir=f\"{my_txpipe_dir}/data/example/output_binning/\"\n", ")\n", - "\n", - "\n", - "\n", - " (\"binned_shear_catalog\", ShearCatalog),\n", - " (\"binned_lens_catalog\", HDFFile),\n", - " (\"binned_random_catalog\", HDFFile),\n", - " (\"binned_random_catalog_sub\", HDFFile),\n", - " (\"shear_photoz_stack\", QPNOfZFile),\n", - " (\"lens_photoz_stack\", QPNOfZFile),\n", - " (\"patch_centers\", TextFile),\n", - " (\"tracer_metadata\", HDFFile)," + "\n" ] }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 7, "id": "e4dc452e-f860-4b46-b141-ccb2243dc752", "metadata": {}, - "outputs": [ - { - "ename": "NameError", - "evalue": "name 'step2' is not defined", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)", - "Cell \u001b[0;32mIn[6], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43mstep2\u001b[49m\u001b[38;5;241m.\u001b[39mrun()\n\u001b[1;32m 2\u001b[0m step2\u001b[38;5;241m.\u001b[39mfinalize()\n", - "\u001b[0;31mNameError\u001b[0m: name 'step2' is not defined" - ] - } - ], + "outputs": [], "source": [ "step2.run()\n", "step2.finalize()" @@ -194,19 +165,19 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 8, "id": "a7c432f8-da69-4f18-a1d9-2eaf5e55ce74", "metadata": {}, "outputs": [], "source": [ - "step3 = txpipe.extensions.cluster_counts.TXTwoPointRLens.make_stage(\n", + "step3 = txpipe.extensions.cluster_counts.TXTwoPointCluster.make_stage(\n", " # This file is the input metadetect shear catalog\n", - " binned_lens_catalog=f\"global/homes/e/edujb/TXPipe/cluster_catalog_tomography.hdf5\",\n", + " binned_lens_catalog=f\"{my_txpipe_dir}cluster_catalog_tomography.hdf5\",\n", " # This is an input training set for the tomographic selection\n", - " binned_shear_catalog=\"global/homes/e/edujb/TXPipe/shear_tomography_catalog.hdf5\",\n", - " binned_random_catalog=f\"global/homes/e/edujb/TXPipe/cluster_catalog_tomography.hdf5\",\n", - " patch_centers=f\"global/homes/e/edujb/TXPipe/cluster_catalog_tomography.hdf5\",\n", - " tracer_metadata=f\"global/homes/e/edujb/TXPipe/cluster_catalog_tomography.hdf5\",\n", + " binned_shear_catalog=f\"{my_txpipe_dir}shear_tomography_catalog.hdf5\",\n", + " binned_random_catalog=f\"{my_txpipe_dir}cluster_catalog_tomography.hdf5\",\n", + " patch_centers=f\"{my_txpipe_dir}cluster_catalog_tomography.hdf5\",\n", + " tracer_metadata=f\"{my_txpipe_dir}cluster_catalog_tomography.hdf5\",\n", " config=f\"{my_txpipe_dir}notebooks/cluster_counts/gt_treecor_test/config-1deg2-CL.yml\",\n", " output_dir=f\"{my_txpipe_dir}data/example/output_binning/\"\n", ")" @@ -214,30 +185,28 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 9, "id": "81801925-4343-4e7a-9866-9df35ef8f8f2", "metadata": {}, "outputs": [ { - "ename": "FileNotFoundError", - "evalue": "[Errno 2] Unable to synchronously open file (unable to open file: name = 'global/homes/e/edujb/TXPipe/shear_tomography_catalog.hdf5', errno = 2, error message = 'No such file or directory', flags = 0, o_flags = 0)", + "ename": "KeyError", + "evalue": "\"Unable to synchronously open object (object 'shear' doesn't exist)\"", "output_type": "error", "traceback": [ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mFileNotFoundError\u001b[0m Traceback (most recent call last)", - "Cell \u001b[0;32mIn[8], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43mstep3\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrun\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 2\u001b[0m step3\u001b[38;5;241m.\u001b[39mfinalize()\n", - "File \u001b[0;32m/global/u2/e/edujb/TXPipe/txpipe/twopoint.py:93\u001b[0m, in \u001b[0;36mTXTwoPoint.run\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 90\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21;01mtreecorr\u001b[39;00m\n\u001b[1;32m 92\u001b[0m \u001b[38;5;66;03m# Binning information\u001b[39;00m\n\u001b[0;32m---> 93\u001b[0m source_list, lens_list \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mread_nbin\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 94\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mconfig[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mnum_threads\u001b[39m\u001b[38;5;124m\"\u001b[39m] \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mint\u001b[39m(os\u001b[38;5;241m.\u001b[39menviron\u001b[38;5;241m.\u001b[39mget(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mOMP_NUM_THREADS\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;241m1\u001b[39m))\n\u001b[1;32m 96\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mrank \u001b[38;5;241m==\u001b[39m \u001b[38;5;241m0\u001b[39m:\n\u001b[1;32m 97\u001b[0m \u001b[38;5;66;03m# This is a workaround for the fact the the ceci config stuff doesn't\u001b[39;00m\n\u001b[1;32m 98\u001b[0m \u001b[38;5;66;03m# quite handle the get method properly.\u001b[39;00m\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 101\u001b[0m \u001b[38;5;66;03m# and friends, below. In this base class only the 2D metrics will be\u001b[39;00m\n\u001b[1;32m 102\u001b[0m \u001b[38;5;66;03m# available, but subclasses can specify to load a distance column too.\u001b[39;00m\n", - "File \u001b[0;32m/global/u2/e/edujb/TXPipe/txpipe/twopoint.py:176\u001b[0m, in \u001b[0;36mTXTwoPoint.read_nbin\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 171\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m 172\u001b[0m \u001b[38;5;124;03mDetermine the bins to use in this analysis, either from the input file\u001b[39;00m\n\u001b[1;32m 173\u001b[0m \u001b[38;5;124;03mor from the configuration.\u001b[39;00m\n\u001b[1;32m 174\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m 175\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mconfig[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124msource_bins\u001b[39m\u001b[38;5;124m\"\u001b[39m] \u001b[38;5;241m==\u001b[39m [\u001b[38;5;241m-\u001b[39m\u001b[38;5;241m1\u001b[39m] \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mconfig[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mlens_bins\u001b[39m\u001b[38;5;124m\"\u001b[39m] \u001b[38;5;241m==\u001b[39m [\u001b[38;5;241m-\u001b[39m\u001b[38;5;241m1\u001b[39m]:\n\u001b[0;32m--> 176\u001b[0m source_list, lens_list \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_read_nbin_from_tomography\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 177\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m 178\u001b[0m source_list, lens_list \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_read_nbin_from_config()\n", - "File \u001b[0;32m/global/u2/e/edujb/TXPipe/txpipe/twopoint.py:192\u001b[0m, in \u001b[0;36mTXTwoPoint._read_nbin_from_tomography\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 190\u001b[0m nbin_source \u001b[38;5;241m=\u001b[39m \u001b[38;5;241m0\u001b[39m\n\u001b[1;32m 191\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m--> 192\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mopen_input\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mbinned_shear_catalog\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m)\u001b[49m \u001b[38;5;28;01mas\u001b[39;00m f:\n\u001b[1;32m 193\u001b[0m nbin_source \u001b[38;5;241m=\u001b[39m f[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mshear\u001b[39m\u001b[38;5;124m\"\u001b[39m]\u001b[38;5;241m.\u001b[39mattrs[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mnbin_source\u001b[39m\u001b[38;5;124m\"\u001b[39m]\n\u001b[1;32m 195\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mget_input(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mbinned_lens_catalog\u001b[39m\u001b[38;5;124m\"\u001b[39m) \u001b[38;5;241m==\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mnone\u001b[39m\u001b[38;5;124m\"\u001b[39m:\n", - "File \u001b[0;32m/global/cfs/projectdirs/lsst/groups/CL/cl_pipeline_project/conda_envs/txpipe_clp/lib/python3.10/site-packages/ceci/stage.py:1090\u001b[0m, in \u001b[0;36mPipelineStage.open_input\u001b[0;34m(self, tag, wrapper, **kwargs)\u001b[0m\n\u001b[1;32m 1088\u001b[0m path \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mget_input(tag)\n\u001b[1;32m 1089\u001b[0m input_class \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mget_input_type(tag)\n\u001b[0;32m-> 1090\u001b[0m obj \u001b[38;5;241m=\u001b[39m \u001b[43minput_class\u001b[49m\u001b[43m(\u001b[49m\u001b[43mpath\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mr\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1092\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m wrapper: \u001b[38;5;66;03m# pragma: no cover\u001b[39;00m\n\u001b[1;32m 1093\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m obj\n", - "File \u001b[0;32m/global/u2/e/edujb/TXPipe/txpipe/data_types/base.py:45\u001b[0m, in \u001b[0;36mDataFile.__init__\u001b[0;34m(self, path, mode, extra_provenance, validate, **kwargs)\u001b[0m\n\u001b[1;32m 42\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m mode \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;129;01min\u001b[39;00m [\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mr\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mw\u001b[39m\u001b[38;5;124m\"\u001b[39m]:\n\u001b[1;32m 43\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mFile \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mmode\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m argument must be \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mr\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m or \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mw\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m not \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mmode\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m---> 45\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mfile \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mopen\u001b[49m\u001b[43m(\u001b[49m\u001b[43mpath\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmode\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 47\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m validate \u001b[38;5;129;01mand\u001b[39;00m mode \u001b[38;5;241m==\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mr\u001b[39m\u001b[38;5;124m\"\u001b[39m:\n\u001b[1;32m 48\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mvalidate()\n", - "File \u001b[0;32m/global/u2/e/edujb/TXPipe/txpipe/data_types/base.py:165\u001b[0m, in \u001b[0;36mHDFFile.open\u001b[0;34m(cls, path, mode, **kwargs)\u001b[0m\n\u001b[1;32m 163\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21;01mh5py\u001b[39;00m\n\u001b[1;32m 164\u001b[0m \u001b[38;5;66;03m# Return an open h5py File\u001b[39;00m\n\u001b[0;32m--> 165\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mh5py\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mFile\u001b[49m\u001b[43m(\u001b[49m\u001b[43mpath\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmode\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n", - "File \u001b[0;32m/global/cfs/projectdirs/lsst/groups/CL/cl_pipeline_project/conda_envs/txpipe_clp/lib/python3.10/site-packages/h5py/_hl/files.py:564\u001b[0m, in \u001b[0;36mFile.__init__\u001b[0;34m(self, name, mode, driver, libver, userblock_size, swmr, rdcc_nslots, rdcc_nbytes, rdcc_w0, track_order, fs_strategy, fs_persist, fs_threshold, fs_page_size, page_buf_size, min_meta_keep, min_raw_keep, locking, alignment_threshold, alignment_interval, meta_block_size, **kwds)\u001b[0m\n\u001b[1;32m 555\u001b[0m fapl \u001b[38;5;241m=\u001b[39m make_fapl(driver, libver, rdcc_nslots, rdcc_nbytes, rdcc_w0,\n\u001b[1;32m 556\u001b[0m locking, page_buf_size, min_meta_keep, min_raw_keep,\n\u001b[1;32m 557\u001b[0m alignment_threshold\u001b[38;5;241m=\u001b[39malignment_threshold,\n\u001b[1;32m 558\u001b[0m alignment_interval\u001b[38;5;241m=\u001b[39malignment_interval,\n\u001b[1;32m 559\u001b[0m meta_block_size\u001b[38;5;241m=\u001b[39mmeta_block_size,\n\u001b[1;32m 560\u001b[0m \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwds)\n\u001b[1;32m 561\u001b[0m fcpl \u001b[38;5;241m=\u001b[39m make_fcpl(track_order\u001b[38;5;241m=\u001b[39mtrack_order, fs_strategy\u001b[38;5;241m=\u001b[39mfs_strategy,\n\u001b[1;32m 562\u001b[0m fs_persist\u001b[38;5;241m=\u001b[39mfs_persist, fs_threshold\u001b[38;5;241m=\u001b[39mfs_threshold,\n\u001b[1;32m 563\u001b[0m fs_page_size\u001b[38;5;241m=\u001b[39mfs_page_size)\n\u001b[0;32m--> 564\u001b[0m fid \u001b[38;5;241m=\u001b[39m \u001b[43mmake_fid\u001b[49m\u001b[43m(\u001b[49m\u001b[43mname\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmode\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43muserblock_size\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mfapl\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mfcpl\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mswmr\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mswmr\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 566\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(libver, \u001b[38;5;28mtuple\u001b[39m):\n\u001b[1;32m 567\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_libver \u001b[38;5;241m=\u001b[39m libver\n", - "File \u001b[0;32m/global/cfs/projectdirs/lsst/groups/CL/cl_pipeline_project/conda_envs/txpipe_clp/lib/python3.10/site-packages/h5py/_hl/files.py:238\u001b[0m, in \u001b[0;36mmake_fid\u001b[0;34m(name, mode, userblock_size, fapl, fcpl, swmr)\u001b[0m\n\u001b[1;32m 236\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m swmr \u001b[38;5;129;01mand\u001b[39;00m swmr_support:\n\u001b[1;32m 237\u001b[0m flags \u001b[38;5;241m|\u001b[39m\u001b[38;5;241m=\u001b[39m h5f\u001b[38;5;241m.\u001b[39mACC_SWMR_READ\n\u001b[0;32m--> 238\u001b[0m fid \u001b[38;5;241m=\u001b[39m \u001b[43mh5f\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mopen\u001b[49m\u001b[43m(\u001b[49m\u001b[43mname\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mflags\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mfapl\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mfapl\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 239\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m mode \u001b[38;5;241m==\u001b[39m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mr+\u001b[39m\u001b[38;5;124m'\u001b[39m:\n\u001b[1;32m 240\u001b[0m fid \u001b[38;5;241m=\u001b[39m h5f\u001b[38;5;241m.\u001b[39mopen(name, h5f\u001b[38;5;241m.\u001b[39mACC_RDWR, fapl\u001b[38;5;241m=\u001b[39mfapl)\n", + "\u001b[0;31mKeyError\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn[9], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43mstep3\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrun\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 2\u001b[0m step3\u001b[38;5;241m.\u001b[39mfinalize()\n", + "File \u001b[0;32m/sps/lsst/users/maguena/git_codes/TXPipe/txpipe/extensions/cluster_counts/rlens_new.py:102\u001b[0m, in \u001b[0;36mTXTwoPointCluster.run\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 99\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21;01mtreecorr\u001b[39;00m\n\u001b[1;32m 101\u001b[0m \u001b[38;5;66;03m# Binning information\u001b[39;00m\n\u001b[0;32m--> 102\u001b[0m source_list, lens_list \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mread_nbin\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 103\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mconfig[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mnum_threads\u001b[39m\u001b[38;5;124m\"\u001b[39m] \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mint\u001b[39m(os\u001b[38;5;241m.\u001b[39menviron\u001b[38;5;241m.\u001b[39mget(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mOMP_NUM_THREADS\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;241m1\u001b[39m))\n\u001b[1;32m 105\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mrank \u001b[38;5;241m==\u001b[39m \u001b[38;5;241m0\u001b[39m:\n\u001b[1;32m 106\u001b[0m \u001b[38;5;66;03m# This is a workaround for the fact the the ceci config stuff doesn't\u001b[39;00m\n\u001b[1;32m 107\u001b[0m \u001b[38;5;66;03m# quite handle the get method properly.\u001b[39;00m\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 110\u001b[0m \u001b[38;5;66;03m# and friends, below. In this base class only the 2D metrics will be\u001b[39;00m\n\u001b[1;32m 111\u001b[0m \u001b[38;5;66;03m# available, but subclasses can specify to load a distance column too.\u001b[39;00m\n", + "File \u001b[0;32m/sps/lsst/users/maguena/git_codes/TXPipe/txpipe/extensions/cluster_counts/rlens_new.py:185\u001b[0m, in \u001b[0;36mTXTwoPointCluster.read_nbin\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 180\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m 181\u001b[0m \u001b[38;5;124;03mDetermine the bins to use in this analysis, either from the input file\u001b[39;00m\n\u001b[1;32m 182\u001b[0m \u001b[38;5;124;03mor from the configuration.\u001b[39;00m\n\u001b[1;32m 183\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m 184\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mconfig[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124msource_bins\u001b[39m\u001b[38;5;124m\"\u001b[39m] \u001b[38;5;241m==\u001b[39m [\u001b[38;5;241m-\u001b[39m\u001b[38;5;241m1\u001b[39m] \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mconfig[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mlens_bins\u001b[39m\u001b[38;5;124m\"\u001b[39m] \u001b[38;5;241m==\u001b[39m [\u001b[38;5;241m-\u001b[39m\u001b[38;5;241m1\u001b[39m]:\n\u001b[0;32m--> 185\u001b[0m source_list, lens_list \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_read_nbin_from_tomography\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 186\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m 187\u001b[0m source_list, lens_list \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_read_nbin_from_config()\n", + "File \u001b[0;32m/sps/lsst/users/maguena/git_codes/TXPipe/txpipe/extensions/cluster_counts/rlens_new.py:202\u001b[0m, in \u001b[0;36mTXTwoPointCluster._read_nbin_from_tomography\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 200\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m 201\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mopen_input(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mbinned_shear_catalog\u001b[39m\u001b[38;5;124m\"\u001b[39m) \u001b[38;5;28;01mas\u001b[39;00m f:\n\u001b[0;32m--> 202\u001b[0m nbin_source \u001b[38;5;241m=\u001b[39m \u001b[43mf\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mshear\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m]\u001b[49m\u001b[38;5;241m.\u001b[39mattrs[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mnbin_source\u001b[39m\u001b[38;5;124m\"\u001b[39m]\n\u001b[1;32m 204\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mget_input(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mbinned_lens_catalog\u001b[39m\u001b[38;5;124m\"\u001b[39m) \u001b[38;5;241m==\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mnone\u001b[39m\u001b[38;5;124m\"\u001b[39m:\n\u001b[1;32m 205\u001b[0m nbin_lens \u001b[38;5;241m=\u001b[39m \u001b[38;5;241m0\u001b[39m\n", + "File \u001b[0;32mh5py/_objects.pyx:54\u001b[0m, in \u001b[0;36mh5py._objects.with_phil.wrapper\u001b[0;34m()\u001b[0m\n", + "File \u001b[0;32mh5py/_objects.pyx:55\u001b[0m, in \u001b[0;36mh5py._objects.with_phil.wrapper\u001b[0;34m()\u001b[0m\n", + "File \u001b[0;32m/sps/lsst/users/ebarroso/conda_envs/txpipe_clp/lib/python3.10/site-packages/h5py/_hl/group.py:360\u001b[0m, in \u001b[0;36mGroup.__getitem__\u001b[0;34m(self, name)\u001b[0m\n\u001b[1;32m 358\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mInvalid HDF5 object reference\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m 359\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(name, (\u001b[38;5;28mbytes\u001b[39m, \u001b[38;5;28mstr\u001b[39m)):\n\u001b[0;32m--> 360\u001b[0m oid \u001b[38;5;241m=\u001b[39m \u001b[43mh5o\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mopen\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mid\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_e\u001b[49m\u001b[43m(\u001b[49m\u001b[43mname\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mlapl\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_lapl\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 361\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m 362\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mTypeError\u001b[39;00m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mAccessing a group is done with bytes or str, \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 363\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mnot \u001b[39m\u001b[38;5;132;01m{}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;241m.\u001b[39mformat(\u001b[38;5;28mtype\u001b[39m(name)))\n", "File \u001b[0;32mh5py/_objects.pyx:54\u001b[0m, in \u001b[0;36mh5py._objects.with_phil.wrapper\u001b[0;34m()\u001b[0m\n", "File \u001b[0;32mh5py/_objects.pyx:55\u001b[0m, in \u001b[0;36mh5py._objects.with_phil.wrapper\u001b[0;34m()\u001b[0m\n", - "File \u001b[0;32mh5py/h5f.pyx:102\u001b[0m, in \u001b[0;36mh5py.h5f.open\u001b[0;34m()\u001b[0m\n", - "\u001b[0;31mFileNotFoundError\u001b[0m: [Errno 2] Unable to synchronously open file (unable to open file: name = 'global/homes/e/edujb/TXPipe/shear_tomography_catalog.hdf5', errno = 2, error message = 'No such file or directory', flags = 0, o_flags = 0)" + "File \u001b[0;32mh5py/h5o.pyx:257\u001b[0m, in \u001b[0;36mh5py.h5o.open\u001b[0;34m()\u001b[0m\n", + "\u001b[0;31mKeyError\u001b[0m: \"Unable to synchronously open object (object 'shear' doesn't exist)\"" ] } ], @@ -245,11 +214,143 @@ "step3.run()\n", "step3.finalize()" ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2c71fbfb-a177-4bea-8284-c724bf080c11", + "metadata": {}, + "outputs": [], + "source": [ + "step4 = txpipe.TXShearCalibration.make_stage(\n", + " # This file is the input metadetect shear catalog\n", + " shear_catalog=f\"{my_txpipe_dir}data/example/inputs/shear_catalog.hdf5\",\n", + " shear_tomography_catalog=f\"{my_txpipe_dir}shear_tomography_catalog.hdf5\",\n", + " # This is an input training set for the tomographic selection\n", + "\n", + " # This contains all the options for this stage. You can override them here\n", + " # manually too.\n", + " # This is the output file for this stage\n", + " output_dir=f\"{my_txpipe_dir}/data/example/output_binning_shear/\"\n", + ")\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b1b390bb-896d-4c0a-80c6-13b175bb9fbc", + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "step4.run()\n", + "step4.finalize()" + ] + }, + { + "cell_type": "markdown", + "id": "d0cc8655-91c0-49c2-af4e-310ca53655c0", + "metadata": {}, + "source": [ + "## Tests" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7b81f6ee-792b-492d-b3cc-57b9b5e1e865", + "metadata": {}, + "outputs": [], + "source": [ + "import h5py" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a5002638-c2b7-4d75-a60b-f9b8777992dc", + "metadata": {}, + "outputs": [], + "source": [ + "with h5py.File(f\"{my_txpipe_dir}shear_tomography_catalog.hdf5\", \"r\") as data:\n", + " print(data.keys())\n", + " for key in data.keys():\n", + " print(key, data[key].attrs.keys())" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "24013968-3ee4-4420-9b24-6d406af40a0a", + "metadata": {}, + "outputs": [], + "source": [ + "tb = h5py.File(f\"{my_txpipe_dir}/shear_tomography_catalog.hdf5\", \"r\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "359d3bb3-2590-4ab6-8063-dc13203e13f3", + "metadata": {}, + "outputs": [], + "source": [ + "for k in tb[\"counts\"].keys():\n", + " print(k, tb[\"counts\"][k])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "094c7154-a5fb-4a16-8c22-74e6a4eef79d", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b6cb2a7f-2ee4-46e9-a724-712f59e1334b", + "metadata": {}, + "outputs": [], + "source": [ + "tb[\"tomography\"][\"bin\"][:]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "66784225-4d0f-434a-b199-37ec822b1d71", + "metadata": {}, + "outputs": [], + "source": [ + "data = h5py.File(f\"{my_txpipe_dir}/data/example/inputs/metadetect_shear_catalog.hdf5\", \"r\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8e7abd32-844b-4613-9389-7d8ed280b8fe", + "metadata": {}, + "outputs": [], + "source": [ + "Table(data[\"shear\"][\"00\"])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "60f1e2d2-76ce-45cb-b28b-29a1d9226363", + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { "kernelspec": { - "display_name": "Txpipe_clp", + "display_name": "Python (txpipe_clp)", "language": "python", "name": "txpipe_clp" }, diff --git a/notebooks/cluster_counts/gt_treecor_test/debug.log b/notebooks/cluster_counts/gt_treecor_test/debug.log new file mode 100644 index 00000000..e69de29b diff --git a/txpipe/extensions/cluster_counts/rlens_new.py b/txpipe/extensions/cluster_counts/rlens_new.py new file mode 100644 index 00000000..25198d38 --- /dev/null +++ b/txpipe/extensions/cluster_counts/rlens_new.py @@ -0,0 +1,958 @@ +from ...base_stage import PipelineStage +from ...data_types import ( + HDFFile, + ShearCatalog, + SACCFile, + TextFile, + MapsFile, + QPNOfZFile, +) +from ...utils.patches import PatchMaker +import numpy as np +import collections +import sys +import os +import pathlib +from time import perf_counter +import gc +from ...utils import choose_pixelization + +# This creates a little mini-type, like a struct, +# for holding individual measurements +Measurement = collections.namedtuple("Measurement", ["corr_type", "object", "i", "j"]) + +SHEAR_SHEAR = 0 +SHEAR_POS = 1 +POS_POS = 2 + +#external cross correlations +POS_EXT = 3 +SHEAR_EXT = 4 + + + +from ...twopoint import TXTwoPoint +from ...data_types import ( + HDFFile, + ShearCatalog, + PNGFile, + TextFile, +) +import numpy as np + +class TXTwoPointCluster(PipelineStage): + """ + Make 2pt measurements using TreeCorr + + This stage make the full set of cosmic shear, galaxy-galaxy lensing, + and galaxy density measurements on the tomographic catalog using TreeCorr. + + Results are saved to a sacc file. + """ + name = "TXTwoPointClusterXXX" + inputs = [ + ("cluster_catalog_tomography", HDFFile), + ("shear_tomography_catalog", HDFFile), + ("shear_catalog", HDFFile), + ("binned_random_catalog", HDFFile), + ("patch_centers", TextFile), + ("tracer_metadata", HDFFile), + ] + outputs = [("rlens_measurement", TextFile)] + # Add values to the config file that are not previously defined + config_options = { + # TODO: Allow more fine-grained selection of 2pt subsets to compute + "calcs": [0, 1, 2], + "min_sep": 0.5, + "max_sep": 300.0, + "nbins": 9, + "bin_slop": 0.0, + "sep_units": "arcmin", + "flip_g1": False, + "flip_g2": True, + "cores_per_task": 20, + "verbose": 1, + "source_bins": [-1], + "lens_bins": [-1], + "reduce_randoms_size": 1.0, + "do_shear_shear": True, + "do_shear_pos": True, + "do_pos_pos": True, + "auto_only": False, + "var_method": "jackknife", + "use_randoms": True, + "low_mem": False, + "patch_dir": "./cache/patches", + "chunk_rows": 100_000, + "share_patch_files": False, + "metric": "Euclidean", + "gaussian_sims_factor": [1.], + "use_subsampled_randoms": True, #use subsampled randoms file for RR + } + + def run(self): + """ + Run the analysis for this stage. + """ + import sacc + import healpy + import treecorr + + # Binning information + source_list, lens_list = self.read_nbin() + self.config["num_threads"] = int(os.environ.get("OMP_NUM_THREADS", 1)) + + if self.rank == 0: + # This is a workaround for the fact the the ceci config stuff doesn't + # quite handle the get method properly. + # Which metrics are available, and how they are interpreted, depends on + # whether a distance is in the catalogs returned in get_shear_catalog + # and friends, below. In this base class only the 2D metrics will be + # available, but subclasses can specify to load a distance column too. + metric = self.config["metric"] if "metric" in self.config else "Euclidean" + print(f"Running TreeCorr with metric \"{metric}\"") + + # Calculate metadata like the area and related + # quantities + meta = self.read_metadata() + + # Choose which pairs of bins to calculate + calcs = self.select_calculations(source_list, lens_list) + sys.stdout.flush() + + # Split the catalogs into patch files + self.prepare_patches(calcs, meta) + + results = [] + for i, j, k in calcs: + result = self.call_treecorr(i, j, k) + results.append(result) + + if self.comm: + self.comm.Barrier() + + # Save the results + self.write_output(source_list, lens_list, meta, results) + + def select_calculations(self, source_list, lens_list): + calcs = [] + + # For shear-shear we omit pairs with j>i + if self.config["do_shear_shear"]: + print('DOING SHEAR-SHEAR') + k = SHEAR_SHEAR + for i in source_list: + for j in range(i + 1): + if j in source_list: + calcs.append((i, j, k)) + + # For shear-position we use all pairs + if self.config["do_shear_pos"]: + print('DOING SHEAR-POS') + k = SHEAR_POS + for i in source_list: + for j in lens_list: + calcs.append((i, j, k)) + + # For position-position we omit pairs with j>i + if self.config["do_pos_pos"]: + print('DOING POS-POS') + if not self.config["use_randoms"]: + raise ValueError( + "You need to have a random catalog to calculate position-position correlations" + ) + k = POS_POS + if self.config["auto_only"]: + for i in lens_list: + calcs.append((i, i, k)) + else: + for i in lens_list: + for j in range(i + 1): + if j in lens_list: + calcs.append((i, j, k)) + + if self.rank == 0: + print(f"Running {len(calcs)} calculations: {calcs}") + + return calcs + + def read_nbin(self): + """ + Determine the bins to use in this analysis, either from the input file + or from the configuration. + """ + if self.config["source_bins"] == [-1] and self.config["lens_bins"] == [-1]: + source_list, lens_list = self._read_nbin_from_tomography() + else: + source_list, lens_list = self._read_nbin_from_config() + + ns = len(source_list) + nl = len(lens_list) + if self.rank == 0: + print(f"Running with {ns} source bins and {nl} lens bins") + + return source_list, lens_list + + # These two functions can be combined into a single one. + def _read_nbin_from_tomography(self): + if self.get_input("shear_tomography_catalog") == "none": + nbin_source = 0 + else: + with self.open_input("shear_tomography_catalog") as f: + nbin_source = f["shear"].attrs["nbin_source"] + + if self.get_input("binned_lens_catalog") == "none": + nbin_lens = 0 + else: + with self.open_input("binned_lens_catalog") as f: + nbin_lens = f["lens"].attrs["nbin_lens"] + + source_list = list(range(nbin_source)) + lens_list = list(range(nbin_lens)) + + return source_list, lens_list + + def _read_nbin_from_config(self): + # TODO handle the case where the user only specefies + # bins for only sources or only lenses + source_list = self.config["source_bins"] + lens_list = self.config["lens_bins"] + + # catch bad input + tomo_source_list, tomo_lens_list = self._read_nbin_from_tomography() + tomo_nbin_source = len(tomo_source_list) + tomo_nbin_lens = len(tomo_lens_list) + + nbin_source = len(source_list) + nbin_lens = len(lens_list) + + if source_list == [-1]: + source_list = tomo_source_list + if lens_list == [-1]: + lens_list = tomo_lens_list + + # if more bins are input than exist, raise an error + if not nbin_source <= tomo_nbin_source: + raise ValueError( + f"Requested too many source bins in the config ({nbin_source}): max is {tomo_nbin_source}" + ) + if not nbin_lens <= tomo_nbin_lens: + raise ValueError( + f"Requested too many lens bins in the config ({nbin_lens}): max is {tomo_nbin_lens}" + ) + + # make sure the bin numbers actually exist + for i in source_list: + if i not in tomo_source_list: + raise ValueError( + f"Requested source bin {i} that is not in the input file" + ) + + for i in lens_list: + if i not in tomo_lens_list: + raise ValueError( + f"Requested lens bin {i} that is not in the input file" + ) + + return source_list, lens_list + + + def add_data_points(self, S, results): + import treecorr + import sacc + + XI = "combined" + XIP = sacc.standard_types.galaxy_shear_xi_plus + XIM = sacc.standard_types.galaxy_shear_xi_minus + GAMMAT = sacc.standard_types.galaxy_shearDensity_xi_t + GAMMAX = sacc.standard_types.galaxy_shearDensity_xi_x + WTHETA = sacc.standard_types.galaxy_density_xi + + comb = [] + for index, d in enumerate(results): + # First the tracers and generic tags + tracer1 = f"source_{d.i}" if d.corr_type in [XI, GAMMAT] else f"lens_{d.i}" + tracer2 = f"source_{d.j}" if d.corr_type in [XI] else f"lens_{d.j}" + + # This happens when there is an empty bin. We can't do a covariance + # here, or anything useful, really, so we just skip this bin. + if d.object is None: + continue + + theta = np.exp(d.object.meanlogr) + npair = d.object.npairs + weight = d.object.weight + # xip / xim is a special case because it has two observables. + # the other two are together below + if d.corr_type == XI: + xip = d.object.xip + xim = d.object.xim + xiperr = np.sqrt(d.object.varxip) + ximerr = np.sqrt(d.object.varxim) + n = len(xip) + # add all the data points to the sacc + for i in range(n): + S.add_data_point( + XIP, + (tracer1, tracer2), + xip[i], + theta=theta[i], + error=xiperr[i], + npair=npair[i], + weight=weight[i], + ) + for i in range(n): + S.add_data_point( + XIM, + (tracer1, tracer2), + xim[i], + theta=theta[i], + error=ximerr[i], + npair=npair[i], + weight=weight[i], + ) + else: + if self.config['gaussian_sims_factor'] != [1.]: + # only for gammat and wtheta, for the gaussian simulations we need to scale the measurements up to correct for + # the scaling of the density field when building the simulations. + if 'lens' in tracer2: + if 'lens' in tracer1: + scaling_factor = self.config['gaussian_sims_factor'][int(tracer1[-1])]*self.config['gaussian_sims_factor'][int(tracer2[-1])] + else: + scaling_factor = self.config['gaussian_sims_factor'][int(tracer2[-1])] + + d.object.xi *=scaling_factor + d.object.varxi *=(scaling_factor**2) + + xi = d.object.xi + err = np.sqrt(d.object.varxi) + n = len(xi) + for i in range(n): + S.add_data_point( + d.corr_type, + (tracer1, tracer2), + xi[i], + theta=theta[i], + error=err[i], + weight=weight[i], + ) + + # We build up the comb list to get the covariance of it later + # in the same order as our data points + comb.append(d.object) + + + + # Add the covariance. There are several different jackknife approaches + # available - see the treecorr docs + if treecorr.__version__.startswith("4.2."): + if self.rank == 0: + print("Using old TreeCorr - covariance may be slow. " + "Consider using 4.3 from github main branch.") + cov = treecorr.estimate_multi_cov(comb, self.config["var_method"]) + else: + if self.rank == 0: + print("Using new TreeCorr 4.3 or above") + cov = treecorr.estimate_multi_cov(comb, self.config["var_method"], comm=self.comm) + S.add_covariance(cov) + + def add_gamma_x_data_points(self, S, results): + import treecorr + import sacc + + XI = "combined" + GAMMAT = sacc.standard_types.galaxy_shearDensity_xi_t + GAMMAX = sacc.standard_types.galaxy_shearDensity_xi_x + + covs = [] + for index, d in enumerate(results): + tracer1 = ( + f"source_{d.i}" if d.corr_type in [XI, GAMMAT] else f"lens_{d.i}" + ) + tracer2 = f"source_{d.j}" if d.corr_type in [XI] else f"lens_{d.j}" + + if d.corr_type == GAMMAT: + theta = np.exp(d.object.meanlogr) + npair = d.object.npairs + weight = d.object.weight + xi_x = d.object.xi_im + covX = d.object.estimate_cov("shot") + # TreeCorr v5 returns the diagonal of the covariance matrix + # instead of a full but diagal (so almost all zero) format. + if treecorr.__version_info__[0] >= 5: + covX = np.diag(covX) + covs.append(covX) + err = np.sqrt(np.diag(covX)) + n = len(xi_x) + for i in range(n): + S.add_data_point( + GAMMAX, + (tracer1, tracer2), + xi_x[i], + theta=theta[i], + error=err[i], + weight=weight[i], + ) + S.add_covariance(covs) + + + def write_output(self, source_list, lens_list, meta, results): + import sacc + import treecorr + + XI = "combined" + XIP = sacc.standard_types.galaxy_shear_xi_plus + XIM = sacc.standard_types.galaxy_shear_xi_minus + GAMMAT = sacc.standard_types.galaxy_shearDensity_xi_t + GAMMAX = sacc.standard_types.galaxy_shearDensity_xi_x + WTHETA = sacc.standard_types.galaxy_density_xi + + S = sacc.Sacc() + S2 = sacc.Sacc() + + # We include the n(z) data in the output. + # So here we load it in and add it to the data + + # Load the tracer data N(z) from an input file and + # copy it to the output, for convenience + if self.config["do_shear_pos"] or self.config["do_shear_shear"]: + if source_list: + with self.open_input("shear_photoz_stack", wrapper=True) as f: + for i in source_list: + z, Nz = f.get_bin_n_of_z(i) + S.add_tracer("NZ", f"source_{i}", z, Nz) + if self.config["do_shear_pos"] == True: + S2.add_tracer("NZ", f"source_{i}", z, Nz) + else: + sys.exit("Requesting a measurement that requires source galaxies but no source_list provided") + + if self.config["do_pos_pos"] or self.config["do_shear_pos"]: + if lens_list: + with self.open_input("lens_photoz_stack", wrapper=True) as f: + # For both source and lens + for i in lens_list: + z, Nz = f.get_bin_n_of_z(i) + S.add_tracer("NZ", f"lens_{i}", z, Nz) + if self.config["do_shear_pos"] == True: + S2.add_tracer("NZ", f"lens_{i}", z, Nz) + else: + sys.exit("Requesting a measurement that requires lens galaxies but no lens_list provided") + + # Now build up the collection of data points, adding them all to + # the sacc data one by one. + self.add_data_points(S, results) + + # The other processes are only needed for the covariance estimation. + # They do a bunch of other stuff here that isn't actually needed, but + # it should all be very fast. After this point they are not needed + # at all so return + if self.rank != 0: + return + + + # Our data points may currently be in any order depending on which processes + # ran which calculations. Re-order them. + S.to_canonical_order() + + + self.write_metadata(S, meta) + + + # Finally, save the output to Sacc file + S.save_fits(self.get_output("twopoint_data_real_raw"), overwrite=True) + + + # Adding the gammaX calculation: + if self.config["do_shear_pos"] == True: + self.add_gamma_x_data_points(S2, results) + S2.to_canonical_order() + self.write_metadata(S2, meta) + # always write the file, even if it is empty + S2.save_fits(self.get_output("twopoint_gamma_x"), overwrite=True) + + def write_metadata(self, S, meta): + # We also save the associated metadata to the file + for k, v in meta.items(): + if np.isscalar(v): + S.metadata[k] = v + else: + for i, vi in enumerate(v): + S.metadata[f"{k}_{i}"] = vi + + # Add provenance metadata. In managed formats this is done + # automatically, but because the Sacc library is external + # we do it manually here. + provenance = self.gather_provenance() + provenance.update(SACCFile.generate_provenance()) + for key, value in provenance.items(): + if isinstance(value, str) and "\n" in value: + values = value.split("\n") + for i, v in enumerate(values): + S.metadata[f"provenance/{key}_{i}"] = v + else: + S.metadata[f"provenance/{key}"] = value + + def call_treecorr(self, i, j, k): + """ + This is a wrapper for interaction with treecorr. + """ + import sacc + import pickle + #TODO: fix up the caching code + if self.name == "TXTwoPoint" or self.name == "TXTwoPointPixel": + pickle_filename = self.get_output("twopoint_data_real_raw") + f".checkpoint-{i}-{j}-{k}.pkl" + #pickle_filename = f"treecorr-cache-{i}-{j}-{k}.pkl" + + if os.path.exists(pickle_filename): + print(f"{self.rank} WARNING USING THIS PICKLE FILE I FOUND: {pickle_filename}") + with open(pickle_filename, "rb") as f: + result = pickle.load(f) + return result + + if k == SHEAR_SHEAR: + xx = self.calculate_shear_shear(i, j) + xtype = "combined" + elif k == SHEAR_POS: + xx = self.calculate_shear_pos(i, j) + xtype = sacc.standard_types.galaxy_shearDensity_xi_t + elif k == POS_POS: + xx = self.calculate_pos_pos(i, j) + xtype = sacc.standard_types.galaxy_density_xi + else: + raise ValueError(f"Unknown correlation function {k}") + + # Force garbage collection here to make sure all the + # catalogs are definitely freed + gc.collect() + + # The measurement object collects the results and type info. + # we use it because the ordering will not be simple if we have + # parallelized, so it's good to keep explicit track. + result = Measurement(xtype, xx, i, j) + + sys.stdout.flush() + + if self.comm: + self.comm.Barrier() + + if self.name == "TXTwoPoint" or self.name == "TXTwoPointPixel": + if self.rank == 0: + print(f"Pickling result to {pickle_filename}") + with open(pickle_filename, "wb") as f: + pickle.dump(result, f) + + return result + + def prepare_patches(self, calcs, meta): + """ + For each catalog to be generated, have one process load the catalog + and write its patch files out to disc. These are then re-used later + by all the different processes. + + Parameters + ---------- + + calcs: list + A list of (bin1, bin2, bin_type) where bin1 and bin2 are indices + or bin labels and bin_type is one of the constants SHEAR_SHEAR, + SHEAR_POS, or POS_POS. + + meta: dict + A dict to which the number of patches (or zero, if no patches) will + be added for each catalog type, with keys "npatch_shear", "npatch_pos", + and "npatch_ran". + """ + # Make the full list of catalogs to run + cats = set() + + # Use shear-shear and pos-pos only here as they represent + # catalogs not pairs. + for i, j, k in calcs: + if k == SHEAR_SHEAR: + cats.add((i, SHEAR_SHEAR)) + cats.add((j, SHEAR_SHEAR)) + elif k == SHEAR_POS: + cats.add((i, SHEAR_SHEAR)) + cats.add((j, POS_POS)) + elif k == POS_POS: + cats.add((i, POS_POS)) + cats.add((j, POS_POS)) + cats = list(cats) + cats.sort(key=str) + + chunk_rows = self.config["chunk_rows"] + npatch_shear = 0 + npatch_pos = 0 + npatch_ran = 0 + + self.empty_patch_exists = {} + + # Parallelization is now done at the patch level + for (h, k) in cats: + ktxt = "shear" if k == SHEAR_SHEAR else "position" + print(f"Rank {self.rank} making patches for {ktxt} catalog bin {h}") + + # For shear we just have the one catalog. For position we may + # have randoms also. We explicitly delete catalogs after loading + # them to ensure we don't have two in memory at once. + if k == SHEAR_SHEAR: + cat = self.get_shear_catalog(h) + npatch_shear,contains_empty = PatchMaker.run(cat, chunk_rows, self.comm) + self.empty_patch_exists[cat.save_patch_dir] = contains_empty + del cat + else: + cat = self.get_lens_catalog(h) + npatch_pos,contains_empty = PatchMaker.run(cat, chunk_rows, self.comm) + self.empty_patch_exists[cat.save_patch_dir] = contains_empty + del cat + + ran_cat = self.get_random_catalog(h) + # support use_randoms = False + if ran_cat is None: + continue + npatch_ran,contains_empty = PatchMaker.run(ran_cat, chunk_rows, self.comm) + self.empty_patch_exists[ran_cat.save_patch_dir] = contains_empty + del ran_cat + + if self.config["use_subsampled_randoms"]: + ran_cat = self.get_subsampled_random_catalog(h) + npatch_ran,contains_empty = PatchMaker.run(ran_cat, chunk_rows, self.comm) + self.empty_patch_exists[ran_cat.save_patch_dir] = contains_empty + del ran_cat + + meta["npatch_shear"] = npatch_shear + meta["npatch_pos"] = npatch_pos + meta["npatch_ran"] = npatch_ran + # stop other processes progressing to the rest of the code and + # trying to load things we have not written yet + if self.comm is not None: + self.comm.Barrier() + + def get_patch_dir(self, input_tag, b): + """ + Select a patch directory for the file with the given input tag + and with a bin number/label. + + To ensure that if you change the catalog the patch dir will also + change, the directory path includes the unique ID of the input file. + + Parameters + ---------- + input_tag: str + One of the tags in the class's inputs attribute + b: any + An additional label used as the last component in the returned + directory + + Returns + ------- + str: a directory, which has been created if it did not exist already. + """ + # start from a user-specified base directory + patch_base = self.config["patch_dir"] + + # append the unique identifier for the parent catalog file + with self.open_input(input_tag, wrapper=True) as f: + p = f.read_provenance() + uuid = p["uuid"] + pth = pathlib.Path(f.path).resolve() + ctime = os.stat(pth).st_ctime + + # We expect the input files to be generated within a pipeline and so always + # have input files to have a unique ID. But if for some reason it doesn't + # have one we handle that too. + if uuid == "UNKNOWN": + ident = hash(f"{pth}{ctime}").to_bytes(8, "big", signed=True).hex() + name = f"{input_tag}_{ident}" + else: + name = f"{input_tag}_{uuid}" + + # Include a tag for the current stage name, so that + # if we are running several subclasses at the same time + # they don't interfere with each other. This is a waste of + # disc space, but hopefully we are not short of that. + if not self.config["share_patch_files"]: + name = self.instance_name + name + + # And finally append the bin name or number + patch_dir = pathlib.Path(patch_base) / name / str(b) + + # Make the directory and return it + pathlib.Path(patch_dir).mkdir(exist_ok=True, parents=True) + return patch_dir + + def get_shear_catalog(self, i): + import treecorr + + # Load and calibrate the appropriate bin data + cat = treecorr.Catalog( + self.get_input("shear_tomography_catalog"), + ext=f"/shear/bin_{i}", + g1_col="g1", + g2_col="g2", + ra_col="ra", + dec_col="dec", + w_col="weight", + ra_units="degree", + dec_units="degree", + patch_centers=self.get_input("patch_centers"), + save_patch_dir=self.get_patch_dir("shear_tomography_catalog", i), + flip_g1=self.config["flip_g1"], + flip_g2=self.config["flip_g2"], + ) + + return cat + + + def get_lens_catalog(self, i): + import treecorr + + # Load and calibrate the appropriate bin data + cat = treecorr.Catalog( + self.get_input("binned_lens_catalog"), + ext=f"/lens/bin_{i}", + ra_col="ra", + dec_col="dec", + w_col="weight", + ra_units="degree", + dec_units="degree", + patch_centers=self.get_input("patch_centers"), + save_patch_dir=self.get_patch_dir("binned_lens_catalog", i), + ) + + return cat + + def get_random_catalog(self, i): + import treecorr + + if not self.config["use_randoms"]: + return None + + rancat = treecorr.Catalog( + self.get_input("binned_random_catalog"), + ext=f"/randoms/bin_{i}", + ra_col="ra", + dec_col="dec", + ra_units="degree", + dec_units="degree", + patch_centers=self.get_input("patch_centers"), + save_patch_dir=self.get_patch_dir("binned_random_catalog", i), + ) + + return rancat + + def get_subsampled_random_catalog(self, i): + import treecorr + + if not self.config["use_randoms"]: + return None + + rancat = treecorr.Catalog( + self.get_input("binned_random_catalog_sub"), + ext=f"/randoms/bin_{i}", + ra_col="ra", + dec_col="dec", + ra_units="degree", + dec_units="degree", + patch_centers=self.get_input("patch_centers"), + save_patch_dir=self.get_patch_dir("binned_random_catalog_sub", i), + ) + + return rancat + + def touch_patches(self, cat): + # If any patches were empty for this cat + # run get_patches on rank 0 and bcast + # this will re-make patches but prevents processes conflicting + # in the gg.process + # If no patches are empty returns the cat, unaltered + if cat is None: + return cat + + if self.empty_patch_exists[cat.save_patch_dir]: + if self.rank==0: + cat.get_patches() + if self.comm is not None: + cat = self.comm.bcast(cat, root=0) + + return cat + + def calculate_shear_shear(self, i, j): + import treecorr + + cat_i = self.get_shear_catalog(i) + cat_i = self.touch_patches(cat_i) + n_i = cat_i.nobj + + if i == j: + cat_j = None + n_j = n_i + else: + cat_j = self.get_shear_catalog(j) + cat_j = self.touch_patches(cat_j) + n_j = cat_j.nobj + + + if self.rank == 0: + print( + f"Calculating shear-shear bin pair ({i},{j}): {n_i} x {n_j} objects using MPI" + ) + + if n_i == 0 or n_j == 0: + if self.rank == 0: + print("Empty catalog: returning None") + return None + + gg = treecorr.GGCorrelation(self.config) + t1 = perf_counter() + gg.process(cat_i, cat_j, low_mem=self.config["low_mem"], comm=self.comm) + t2 = perf_counter() + if self.rank == 0: + print(f"Processing took {t2 - t1:.1f} seconds") + + return gg + + def calculate_shear_pos(self, i, j): + import treecorr + + cat_i = self.get_shear_catalog(i) + cat_i = self.touch_patches(cat_i) + n_i = cat_i.nobj + + cat_j = self.get_lens_catalog(j) + cat_j = self.touch_patches(cat_j) + rancat_j = self.get_random_catalog(j) + rancat_j = self.touch_patches(rancat_j) + n_j = cat_j.nobj + n_rand_j = rancat_j.nobj if rancat_j is not None else 0 + + if self.rank == 0: + print( + f"Calculating shear-position bin pair ({i},{j}): {n_i} x {n_j} objects, {n_rand_j} randoms" + ) + + if n_i == 0 or n_j == 0: + if self.rank == 0: + print("Empty catalog: returning None") + return None + + ng = treecorr.NGCorrelation(self.config) + t1 = perf_counter() + ng.process(cat_j, cat_i, comm=self.comm, low_mem=self.config["low_mem"]) + + if rancat_j: + rg = treecorr.NGCorrelation(self.config) + rg.process(rancat_j, cat_i, comm=self.comm, low_mem=self.config["low_mem"]) + else: + rg = None + + ng.calculateXi(rg=rg) + t2 = perf_counter() + if self.rank == 0: + print(f"Processing took {t2 - t1:.1f} seconds") + + return ng + + def calculate_pos_pos(self, i, j): + import treecorr + + cat_i = self.get_lens_catalog(i) + cat_i = self.touch_patches(cat_i) + rancat_i = self.get_random_catalog(i) + rancat_i = self.touch_patches(rancat_i) + n_i = cat_i.nobj + n_rand_i = rancat_i.nobj if rancat_i is not None else 0 + + if i == j: + cat_j = None + rancat_j = rancat_i + n_j = n_i + n_rand_j = n_rand_i + else: + cat_j = self.get_lens_catalog(j) + cat_j = self.touch_patches(cat_j) + rancat_j = self.get_random_catalog(j) + rancat_j = self.touch_patches(rancat_j) + n_j = cat_j.nobj + n_rand_j = rancat_j.nobj + + if self.config['use_subsampled_randoms']: + rancat_sub_i = self.get_subsampled_random_catalog(i) + rancat_sub_i = self.touch_patches(rancat_sub_i) + n_rand_sub_i = rancat_sub_i.nobj if rancat_sub_i is not None else 0 + + if i == j: + rancat_sub_j = rancat_sub_i + n_rand_sub_j = n_rand_sub_i + else: + rancat_sub_j = self.get_subsampled_random_catalog(j) + rancat_sub_j = self.touch_patches(rancat_sub_j) + n_rand_sub_j = rancat_sub_j.nobj if rancat_sub_j is not None else 0 + + + if self.rank == 0: + print( + f"Calculating position-position bin pair ({i}, {j}): {n_i} x {n_j} objects, {n_rand_i} x {n_rand_j} randoms" + ) + if self.config["use_subsampled_randoms"]: + print(f"and for the rr term, {n_rand_sub_i} x {n_rand_sub_j} pairs") + + if n_i == 0 or n_j == 0: + if self.rank == 0: + print("Empty catalog: returning None") + return None + + t1 = perf_counter() + + nn = treecorr.NNCorrelation(self.config) + nn.process(cat_i, cat_j, comm=self.comm, low_mem=self.config["low_mem"]) + + nr = treecorr.NNCorrelation(self.config) + nr.process(cat_i, rancat_j, comm=self.comm, low_mem=self.config["low_mem"]) + + # The next calculation is faster if we explicitly tell TreeCorr + # that its two catalogs here are the same one. + if i == j: + rancat_j = None + rancat_sub_j = None + + rr = treecorr.NNCorrelation(self.config) + if self.config["use_subsampled_randoms"]: + rr.process(rancat_sub_i, rancat_sub_j, comm=self.comm, low_mem=self.config["low_mem"]) + else: + rr.process(rancat_i, rancat_j, comm=self.comm, low_mem=self.config["low_mem"]) + + if i == j: + rn = None + else: + rn = treecorr.NNCorrelation(self.config) + rn.process(rancat_i, cat_j, comm=self.comm, low_mem=self.config["low_mem"]) + + t2 = perf_counter() + nn.calculateXi(rr=rr, dr=nr, rd=rn) + if self.rank == 0: + print(f"Processing took {t2 - t1:.1f} seconds") + + return nn + + def read_metadata(self): + meta_data = self.open_input("tracer_metadata") + area = meta_data["tracers"].attrs["area"] + meta = {} + meta["area"] = area + try: + sigma_e = meta_data["tracers/sigma_e"][:] + N_eff = meta_data["tracers/N_eff"][:] + mean_e1 = meta_data["tracers/mean_e1"][:] + mean_e2 = meta_data["tracers/mean_e2"][:] + + meta["neff"] = N_eff + meta["area"] = area + meta["sigma_e"] = sigma_e + meta["mean_e1"] = mean_e1 + meta["mean_e2"] = mean_e2 + + except KeyError: #will happen for lens only runs + pass + + return meta From 0c96a1aed985831acf77f2d65eba1d93dc3c2871 Mon Sep 17 00:00:00 2001 From: eduardojsbarroso Date: Wed, 15 Oct 2025 05:51:15 -0700 Subject: [PATCH 6/7] Updated rlens. Notebook to test it. --- .../gt_treecor_test/Run_CL_pipeline.ipynb | 502 ----------- .../Run_CL_pipeline_treecor.ipynb | 831 ++++++++++++++++++ .../gt_treecor_test/config-1deg2-CL.yml | 3 + txpipe/__init__.py | 2 +- txpipe/extensions/cluster_counts/rlens.py | 71 +- 5 files changed, 898 insertions(+), 511 deletions(-) delete mode 100644 notebooks/cluster_counts/gt_treecor_test/Run_CL_pipeline.ipynb create mode 100644 notebooks/cluster_counts/gt_treecor_test/Run_CL_pipeline_treecor.ipynb diff --git a/notebooks/cluster_counts/gt_treecor_test/Run_CL_pipeline.ipynb b/notebooks/cluster_counts/gt_treecor_test/Run_CL_pipeline.ipynb deleted file mode 100644 index 56a1693f..00000000 --- a/notebooks/cluster_counts/gt_treecor_test/Run_CL_pipeline.ipynb +++ /dev/null @@ -1,502 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "709caa5a", - "metadata": { - "tags": [] - }, - "source": [ - "*This notebook was developed by Marina Ricci for the DESC CL_Cosmo_Pipeline team.*\n", - "# This notebook aims at presenting the cluster pipeline for \n", - "# stacked $\\Delta\\Sigma$ profiles and counts in richness/redshift bins.\n", - "\n", - "It is meant to be run independently. If you already produced the necessary outputs `ceci` will use them, and otherwise create them.\n", - "\n", - "___" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "id": "adbec32f-9dc4-4169-b0e3-4b5ef28076eb", - "metadata": {}, - "outputs": [], - "source": [ - "%load_ext autoreload\n", - "%autoreload 2" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "id": "2edf6e90", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "import os\n", - "from pprint import pprint\n", - "\n", - "import ceci\n", - "import h5py\n", - "\n", - "%matplotlib inline\n", - "import matplotlib.pyplot as plt\n", - "import numpy as np\n", - "import yaml\n", - "from IPython.display import Image\n", - "from astropy.table import Table\n", - "\n", - "import re\n", - "import sacc" - ] - }, - { - "cell_type": "markdown", - "id": "8fe12a8c", - "metadata": {}, - "source": [ - "Make sure to change your path in the next cell that leads to your TXPipe directory. See examples for IN2P3 and NERSC below." - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "id": "5eebecd8", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "# user specific paths -- IN2P3 example\n", - "my_txpipe_dir = \"/sps/lsst/users/ebarroso/TXPipe/\"\n", - "\n", - "# user specific paths -- NERSC example\n", - "# my_txpipe_dir = \"/pscratch/sd/a/avestruz/TXPipe\"\n", - "\n", - "os.chdir(my_txpipe_dir)\n", - "\n", - "import txpipe" - ] - }, - { - "cell_type": "markdown", - "id": "b670e283", - "metadata": {}, - "source": [ - "___" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "id": "9d3d1e18-eec5-469c-9952-9d0868a2cc2e", - "metadata": {}, - "outputs": [], - "source": [ - "step1 = txpipe.TXSourceSelectorMetadetect.make_stage(\n", - " # This file is the input metadetect shear catalog\n", - " shear_catalog=f\"{my_txpipe_dir}/data/example/inputs/metadetect_shear_catalog.hdf5\",\n", - " # This is an input training set for the tomographic selection\n", - " calibration_table=f\"{my_txpipe_dir}data/example/inputs/sample_cosmodc2_w10year_errors.dat\",\n", - "\n", - " # This contains all the options for this stage. You can override them here\n", - " # manually too.\n", - " config=f\"{my_txpipe_dir}/examples/metadetect/config.yml\",\n", - "\n", - " # This is the output file for this stage\n", - " output_dir=f\"{my_txpipe_dir}/data/example/outputs_metadetect/\"\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "id": "c5e6e548-7d43-43fb-93d7-debaedd9882c", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Process 0 running selection for rows 0-82,200\n" - ] - } - ], - "source": [ - "step1.run()\n", - "step1.finalize()" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "id": "073b332a-ace9-4e27-9c6e-e1c9807bd2e0", - "metadata": {}, - "outputs": [], - "source": [ - "step2 = txpipe.CLClusterBinningRedshiftRichness.make_stage(\n", - " # This file is the input metadetect shear catalog\n", - " cluster_catalog=f\"{my_txpipe_dir}data/example/inputs/cluster_catalog.hdf5\",\n", - " # This is an input training set for the tomographic selection\n", - "\n", - " # This contains all the options for this stage. You can override them here\n", - " # manually too.\n", - " config=f\"{my_txpipe_dir}/notebooks/cluster_counts/gt_treecor_test/config-1deg2-CL.yml\",\n", - "\n", - " # This is the output file for this stage\n", - " output_dir=f\"{my_txpipe_dir}/data/example/output_binning/\"\n", - ")\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "id": "e4dc452e-f860-4b46-b141-ccb2243dc752", - "metadata": {}, - "outputs": [], - "source": [ - "step2.run()\n", - "step2.finalize()" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "id": "a7c432f8-da69-4f18-a1d9-2eaf5e55ce74", - "metadata": {}, - "outputs": [], - "source": [ - "step3 = txpipe.extensions.cluster_counts.TXTwoPointCluster.make_stage(\n", - " # This file is the input metadetect shear catalog\n", - " binned_lens_catalog=f\"{my_txpipe_dir}cluster_catalog_tomography.hdf5\",\n", - " # This is an input training set for the tomographic selection\n", - " binned_shear_catalog=f\"{my_txpipe_dir}shear_tomography_catalog.hdf5\",\n", - " binned_random_catalog=f\"{my_txpipe_dir}cluster_catalog_tomography.hdf5\",\n", - " patch_centers=f\"{my_txpipe_dir}cluster_catalog_tomography.hdf5\",\n", - " tracer_metadata=f\"{my_txpipe_dir}cluster_catalog_tomography.hdf5\",\n", - " config=f\"{my_txpipe_dir}notebooks/cluster_counts/gt_treecor_test/config-1deg2-CL.yml\",\n", - " output_dir=f\"{my_txpipe_dir}data/example/output_binning/\"\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "id": "81801925-4343-4e7a-9866-9df35ef8f8f2", - "metadata": {}, - "outputs": [ - { - "ename": "KeyError", - "evalue": "\"Unable to synchronously open object (object 'shear' doesn't exist)\"", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mKeyError\u001b[0m Traceback (most recent call last)", - "Cell \u001b[0;32mIn[9], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43mstep3\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrun\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 2\u001b[0m step3\u001b[38;5;241m.\u001b[39mfinalize()\n", - "File \u001b[0;32m/sps/lsst/users/maguena/git_codes/TXPipe/txpipe/extensions/cluster_counts/rlens_new.py:102\u001b[0m, in \u001b[0;36mTXTwoPointCluster.run\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 99\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21;01mtreecorr\u001b[39;00m\n\u001b[1;32m 101\u001b[0m \u001b[38;5;66;03m# Binning information\u001b[39;00m\n\u001b[0;32m--> 102\u001b[0m source_list, lens_list \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mread_nbin\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 103\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mconfig[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mnum_threads\u001b[39m\u001b[38;5;124m\"\u001b[39m] \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mint\u001b[39m(os\u001b[38;5;241m.\u001b[39menviron\u001b[38;5;241m.\u001b[39mget(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mOMP_NUM_THREADS\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;241m1\u001b[39m))\n\u001b[1;32m 105\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mrank \u001b[38;5;241m==\u001b[39m \u001b[38;5;241m0\u001b[39m:\n\u001b[1;32m 106\u001b[0m \u001b[38;5;66;03m# This is a workaround for the fact the the ceci config stuff doesn't\u001b[39;00m\n\u001b[1;32m 107\u001b[0m \u001b[38;5;66;03m# quite handle the get method properly.\u001b[39;00m\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 110\u001b[0m \u001b[38;5;66;03m# and friends, below. In this base class only the 2D metrics will be\u001b[39;00m\n\u001b[1;32m 111\u001b[0m \u001b[38;5;66;03m# available, but subclasses can specify to load a distance column too.\u001b[39;00m\n", - "File \u001b[0;32m/sps/lsst/users/maguena/git_codes/TXPipe/txpipe/extensions/cluster_counts/rlens_new.py:185\u001b[0m, in \u001b[0;36mTXTwoPointCluster.read_nbin\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 180\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m 181\u001b[0m \u001b[38;5;124;03mDetermine the bins to use in this analysis, either from the input file\u001b[39;00m\n\u001b[1;32m 182\u001b[0m \u001b[38;5;124;03mor from the configuration.\u001b[39;00m\n\u001b[1;32m 183\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m 184\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mconfig[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124msource_bins\u001b[39m\u001b[38;5;124m\"\u001b[39m] \u001b[38;5;241m==\u001b[39m [\u001b[38;5;241m-\u001b[39m\u001b[38;5;241m1\u001b[39m] \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mconfig[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mlens_bins\u001b[39m\u001b[38;5;124m\"\u001b[39m] \u001b[38;5;241m==\u001b[39m [\u001b[38;5;241m-\u001b[39m\u001b[38;5;241m1\u001b[39m]:\n\u001b[0;32m--> 185\u001b[0m source_list, lens_list \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_read_nbin_from_tomography\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 186\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m 187\u001b[0m source_list, lens_list \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_read_nbin_from_config()\n", - "File \u001b[0;32m/sps/lsst/users/maguena/git_codes/TXPipe/txpipe/extensions/cluster_counts/rlens_new.py:202\u001b[0m, in \u001b[0;36mTXTwoPointCluster._read_nbin_from_tomography\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 200\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m 201\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mopen_input(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mbinned_shear_catalog\u001b[39m\u001b[38;5;124m\"\u001b[39m) \u001b[38;5;28;01mas\u001b[39;00m f:\n\u001b[0;32m--> 202\u001b[0m nbin_source \u001b[38;5;241m=\u001b[39m \u001b[43mf\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mshear\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m]\u001b[49m\u001b[38;5;241m.\u001b[39mattrs[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mnbin_source\u001b[39m\u001b[38;5;124m\"\u001b[39m]\n\u001b[1;32m 204\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mget_input(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mbinned_lens_catalog\u001b[39m\u001b[38;5;124m\"\u001b[39m) \u001b[38;5;241m==\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mnone\u001b[39m\u001b[38;5;124m\"\u001b[39m:\n\u001b[1;32m 205\u001b[0m nbin_lens \u001b[38;5;241m=\u001b[39m \u001b[38;5;241m0\u001b[39m\n", - "File \u001b[0;32mh5py/_objects.pyx:54\u001b[0m, in \u001b[0;36mh5py._objects.with_phil.wrapper\u001b[0;34m()\u001b[0m\n", - "File \u001b[0;32mh5py/_objects.pyx:55\u001b[0m, in \u001b[0;36mh5py._objects.with_phil.wrapper\u001b[0;34m()\u001b[0m\n", - "File \u001b[0;32m/sps/lsst/users/ebarroso/conda_envs/txpipe_clp/lib/python3.10/site-packages/h5py/_hl/group.py:360\u001b[0m, in \u001b[0;36mGroup.__getitem__\u001b[0;34m(self, name)\u001b[0m\n\u001b[1;32m 358\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mInvalid HDF5 object reference\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m 359\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(name, (\u001b[38;5;28mbytes\u001b[39m, \u001b[38;5;28mstr\u001b[39m)):\n\u001b[0;32m--> 360\u001b[0m oid \u001b[38;5;241m=\u001b[39m \u001b[43mh5o\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mopen\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mid\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_e\u001b[49m\u001b[43m(\u001b[49m\u001b[43mname\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mlapl\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_lapl\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 361\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m 362\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mTypeError\u001b[39;00m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mAccessing a group is done with bytes or str, \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 363\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mnot \u001b[39m\u001b[38;5;132;01m{}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;241m.\u001b[39mformat(\u001b[38;5;28mtype\u001b[39m(name)))\n", - "File \u001b[0;32mh5py/_objects.pyx:54\u001b[0m, in \u001b[0;36mh5py._objects.with_phil.wrapper\u001b[0;34m()\u001b[0m\n", - "File \u001b[0;32mh5py/_objects.pyx:55\u001b[0m, in \u001b[0;36mh5py._objects.with_phil.wrapper\u001b[0;34m()\u001b[0m\n", - "File \u001b[0;32mh5py/h5o.pyx:257\u001b[0m, in \u001b[0;36mh5py.h5o.open\u001b[0;34m()\u001b[0m\n", - "\u001b[0;31mKeyError\u001b[0m: \"Unable to synchronously open object (object 'shear' doesn't exist)\"" - ] - } - ], - "source": [ - "step3.run()\n", - "step3.finalize()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "2c71fbfb-a177-4bea-8284-c724bf080c11", - "metadata": {}, - "outputs": [], - "source": [ - "step4 = txpipe.TXShearCalibration.make_stage(\n", - " # This file is the input metadetect shear catalog\n", - " shear_catalog=f\"{my_txpipe_dir}data/example/inputs/shear_catalog.hdf5\",\n", - " shear_tomography_catalog=f\"{my_txpipe_dir}shear_tomography_catalog.hdf5\",\n", - " # This is an input training set for the tomographic selection\n", - "\n", - " # This contains all the options for this stage. You can override them here\n", - " # manually too.\n", - " # This is the output file for this stage\n", - " output_dir=f\"{my_txpipe_dir}/data/example/output_binning_shear/\"\n", - ")\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "b1b390bb-896d-4c0a-80c6-13b175bb9fbc", - "metadata": { - "scrolled": true - }, - "outputs": [], - "source": [ - "step4.run()\n", - "step4.finalize()" - ] - }, - { - "cell_type": "markdown", - "id": "d0cc8655-91c0-49c2-af4e-310ca53655c0", - "metadata": {}, - "source": [ - "## Tests" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "7b81f6ee-792b-492d-b3cc-57b9b5e1e865", - "metadata": {}, - "outputs": [], - "source": [ - "import h5py" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "a5002638-c2b7-4d75-a60b-f9b8777992dc", - "metadata": {}, - "outputs": [], - "source": [ - "with h5py.File(f\"{my_txpipe_dir}shear_tomography_catalog.hdf5\", \"r\") as data:\n", - " print(data.keys())\n", - " for key in data.keys():\n", - " print(key, data[key].attrs.keys())" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "24013968-3ee4-4420-9b24-6d406af40a0a", - "metadata": {}, - "outputs": [], - "source": [ - "tb = h5py.File(f\"{my_txpipe_dir}/shear_tomography_catalog.hdf5\", \"r\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "359d3bb3-2590-4ab6-8063-dc13203e13f3", - "metadata": {}, - "outputs": [], - "source": [ - "for k in tb[\"counts\"].keys():\n", - " print(k, tb[\"counts\"][k])" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "094c7154-a5fb-4a16-8c22-74e6a4eef79d", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "b6cb2a7f-2ee4-46e9-a724-712f59e1334b", - "metadata": {}, - "outputs": [], - "source": [ - "tb[\"tomography\"][\"bin\"][:]" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "66784225-4d0f-434a-b199-37ec822b1d71", - "metadata": {}, - "outputs": [], - "source": [ - "data = h5py.File(f\"{my_txpipe_dir}/data/example/inputs/metadetect_shear_catalog.hdf5\", \"r\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "8e7abd32-844b-4613-9389-7d8ed280b8fe", - "metadata": {}, - "outputs": [], - "source": [ - "Table(data[\"shear\"][\"00\"])" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "60f1e2d2-76ce-45cb-b28b-29a1d9226363", - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python (txpipe_clp)", - "language": "python", - "name": "txpipe_clp" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.18" - }, - "nbdime-conflicts": { - "local_diff": [ - { - "diff": [ - { - "diff": [ - { - "key": 0, - "op": "addrange", - "valuelist": [ - "clmm" - ] - }, - { - "key": 0, - "length": 1, - "op": "removerange" - } - ], - "key": "display_name", - "op": "patch" - }, - { - "diff": [ - { - "key": 0, - "op": "addrange", - "valuelist": [ - "clmm" - ] - }, - { - "key": 0, - "length": 1, - "op": "removerange" - } - ], - "key": "name", - "op": "patch" - } - ], - "key": "kernelspec", - "op": "patch" - }, - { - "diff": [ - { - "diff": [ - { - "diff": [ - { - "key": 5, - "length": 1, - "op": "removerange" - } - ], - "key": 0, - "op": "patch" - } - ], - "key": "version", - "op": "patch" - } - ], - "key": "language_info", - "op": "patch" - } - ], - "remote_diff": [ - { - "diff": [ - { - "diff": [ - { - "key": 0, - "op": "addrange", - "valuelist": [ - "Python (firecrown2.0)" - ] - }, - { - "key": 0, - "length": 1, - "op": "removerange" - } - ], - "key": "display_name", - "op": "patch" - }, - { - "diff": [ - { - "key": 0, - "op": "addrange", - "valuelist": [ - "firecrown" - ] - }, - { - "key": 0, - "length": 1, - "op": "removerange" - } - ], - "key": "name", - "op": "patch" - } - ], - "key": "kernelspec", - "op": "patch" - }, - { - "diff": [ - { - "diff": [ - { - "key": 0, - "length": 1, - "op": "removerange" - } - ], - "key": "version", - "op": "patch" - } - ], - "key": "language_info", - "op": "patch" - } - ] - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/notebooks/cluster_counts/gt_treecor_test/Run_CL_pipeline_treecor.ipynb b/notebooks/cluster_counts/gt_treecor_test/Run_CL_pipeline_treecor.ipynb new file mode 100644 index 00000000..21db5a7d --- /dev/null +++ b/notebooks/cluster_counts/gt_treecor_test/Run_CL_pipeline_treecor.ipynb @@ -0,0 +1,831 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "adbec32f-9dc4-4169-b0e3-4b5ef28076eb", + "metadata": {}, + "outputs": [], + "source": [ + "%load_ext autoreload\n", + "%autoreload 2" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "2edf6e90", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "import os\n", + "from pprint import pprint\n", + "\n", + "import ceci\n", + "import h5py\n", + "\n", + "%matplotlib inline\n", + "import matplotlib.pyplot as plt\n", + "import numpy as np\n", + "import yaml\n", + "from IPython.display import Image\n", + "from astropy.table import Table\n", + "\n", + "import re\n", + "import sacc" + ] + }, + { + "cell_type": "markdown", + "id": "8fe12a8c", + "metadata": {}, + "source": [ + "Make sure to change your path in the next cell that leads to your TXPipe directory. See examples for IN2P3 and NERSC below." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "5eebecd8", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "# user specific paths -- IN2P3 example\n", + "#my_txpipe_dir = \"/sps/lsst/users/ebarroso/TXPipe/\"\n", + "my_txpipe_dir = \"/global/homes/e/edujb/TXPipe/\"\n", + "import sys\n", + "sys.path.insert(0, my_txpipe_dir) \n", + "# user specific paths -- NERSC example\n", + "# my_txpipe_dir = \"/pscratch/sd/a/avestruz/TXPipe\"\n", + "\n", + "#os.chdir(my_txpipe_dir)\n", + "\n", + "import txpipe" + ] + }, + { + "cell_type": "markdown", + "id": "b670e283", + "metadata": {}, + "source": [ + "___" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "9d3d1e18-eec5-469c-9952-9d0868a2cc2e", + "metadata": {}, + "outputs": [], + "source": [ + "step1 = txpipe.TXSourceSelectorMetadetect.make_stage(\n", + " # This file is the input metadetect shear catalog\n", + " shear_catalog=f\"{my_txpipe_dir}/data/example/inputs/metadetect_shear_catalog.hdf5\",\n", + " # This is an input training set for the tomographic selection\n", + " calibration_table=f\"{my_txpipe_dir}data/example/inputs/sample_cosmodc2_w10year_errors.dat\",\n", + "\n", + " # This contains all the options for this stage. You can override them here\n", + " # manually too.\n", + " config=f\"{my_txpipe_dir}/examples/metadetect/config.yml\",\n", + "\n", + " # This is the output file for this stage\n", + " shear_tomography_catalog=f\"{my_txpipe_dir}/data/example/outputs_metadetect/shear_tomography_catalog.hdf5\"\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "c5e6e548-7d43-43fb-93d7-debaedd9882c", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Process 0 running selection for rows 0-82,200\n" + ] + } + ], + "source": [ + "step1.run()\n", + "step1.finalize()" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "073b332a-ace9-4e27-9c6e-e1c9807bd2e0", + "metadata": {}, + "outputs": [], + "source": [ + "step2 = txpipe.CLClusterBinningRedshiftRichness.make_stage(\n", + " # This file is the input metadetect shear catalog\n", + " cluster_catalog=f\"{my_txpipe_dir}data/example/inputs/cluster_catalog.hdf5\",\n", + " # This is an input training set for the tomographic selection\n", + "\n", + " # This contains all the options for this stage. You can override them here\n", + " # manually too.\n", + " config=f\"{my_txpipe_dir}/notebooks/cluster_counts/gt_treecor_test/config-1deg2-CL.yml\",\n", + "\n", + " # This is the output file for this stage\n", + " cluster_catalog_tomography=f\"{my_txpipe_dir}/data/example/output_binning/cluster_catalog_tomography.hdf5\"\n", + ")\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "e4dc452e-f860-4b46-b141-ccb2243dc752", + "metadata": {}, + "outputs": [], + "source": [ + "step2.run()\n", + "step2.finalize()" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "3e7eb73a-2bb5-4910-9596-5400ff185862", + "metadata": {}, + "outputs": [], + "source": [ + "step3 = txpipe.TXShearCalibration.make_stage(\n", + " # This file is the input metadetect shear catalog\n", + " shear_catalog=f\"{my_txpipe_dir}/data/example/inputs/metadetect_shear_catalog.hdf5\",\n", + " shear_tomography_catalog=f\"{my_txpipe_dir}/data/example/outputs_metadetect/shear_tomography_catalog.hdf5\",\n", + " binned_shear_catalog=f\"{my_txpipe_dir}data/example/output_shearcalib/binned_shear_catalog.hdf5\",\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "87c878d4-41aa-4658-8c9b-d3c5d969c7fa", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Copying extra columns: []\n", + "Copying magnitude: ['mag_i', 'mag_r', 'mag_z', 'mag_err_i', 'mag_err_r', 'mag_err_z']\n", + "Using calibration method: MetaDetectCalibrator\n", + "Reading these columns from the 'shear' group:\n", + " - 00/g1\n", + " - 00/g2\n", + " - 00/ra\n", + " - 00/dec\n", + " - 00/weight\n", + " - 00/mag_i\n", + " - 00/mag_r\n", + " - 00/mag_z\n", + " - 00/mag_err_i\n", + " - 00/mag_err_r\n", + " - 00/mag_err_z\n", + "Renaming these columns:\n", + " - 00/g1 -> g1\n", + " - 00/g2 -> g2\n", + " - 00/ra -> ra\n", + " - 00/dec -> dec\n", + " - 00/weight -> weight\n", + " - 00/mag_i -> mag_i\n", + " - 00/mag_r -> mag_r\n", + " - 00/mag_z -> mag_z\n", + " - 00/mag_err_i -> mag_err_i\n", + " - 00/mag_err_r -> mag_err_r\n", + " - 00/mag_err_z -> mag_err_z\n", + "Process 0 collating bins: [0, 1, 2, 3, all]\n", + "Rank 0 processing data 0 - 82,200\n" + ] + } + ], + "source": [ + "step3.run()\n", + "step3.finalize()" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "cc8203a2-38e5-4414-b1ac-1b7e878d8958", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Redshift bin 1 has 28 clusters.\n" + ] + } + ], + "source": [ + "path = \"/global/homes/e/edujb/TXPipe/cluster_catalog_tomography.hdf5\"\n", + "\n", + "with h5py.File(path, \"r\") as f:\n", + " zbin_groups = {}\n", + " \n", + " # Iterate through all subgroups under cluster_bin\n", + " for name in f[\"cluster_bin\"].keys():\n", + " if name.startswith(\"bin_zbin_\"):\n", + " # Extract redshift bin index\n", + " zbin = name.split(\"_richbin_\")[0].replace(\"bin_zbin_\", \"\")\n", + " if zbin not in zbin_groups:\n", + " zbin_groups[zbin] = []\n", + " zbin_groups[zbin].append(f[\"cluster_bin\"][name])\n", + " \n", + " # Now combine data for each zbin\n", + " clusters_by_zbin = {}\n", + " for zbin, groups in zbin_groups.items():\n", + " combined = {}\n", + " for g in groups:\n", + " for key in g.keys():\n", + " if key not in combined:\n", + " combined[key] = np.array(g[key])\n", + " else:\n", + " combined[key] = np.concatenate([combined[key], np.array(g[key])])\n", + " clusters_by_zbin[zbin] = combined\n", + "\n", + "# Example: Access zbin_1 data\n", + "ra_z1 = clusters_by_zbin[\"1\"][\"ra\"]\n", + "dec_z1 = clusters_by_zbin[\"1\"][\"dec\"]\n", + "print(f\"Redshift bin 1 has {len(ra_z1)} clusters.\")" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "16b401f6-d74e-4813-b87b-85c0802c97cb", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "cluster_bin\n", + "cluster_bin/bin_zbin_0_richbin_0\n", + "cluster_bin/bin_zbin_0_richbin_0/cluster_id\n", + "cluster_bin/bin_zbin_0_richbin_0/dec\n", + "cluster_bin/bin_zbin_0_richbin_0/ra\n", + "cluster_bin/bin_zbin_0_richbin_0/redshift\n", + "cluster_bin/bin_zbin_0_richbin_0/redshift_err\n", + "cluster_bin/bin_zbin_0_richbin_0/richness\n", + "cluster_bin/bin_zbin_0_richbin_0/richness_err\n", + "cluster_bin/bin_zbin_0_richbin_0/scaleval\n", + "cluster_bin/bin_zbin_0_richbin_1\n", + "cluster_bin/bin_zbin_0_richbin_1/cluster_id\n", + "cluster_bin/bin_zbin_0_richbin_1/dec\n", + "cluster_bin/bin_zbin_0_richbin_1/ra\n", + "cluster_bin/bin_zbin_0_richbin_1/redshift\n", + "cluster_bin/bin_zbin_0_richbin_1/redshift_err\n", + "cluster_bin/bin_zbin_0_richbin_1/richness\n", + "cluster_bin/bin_zbin_0_richbin_1/richness_err\n", + "cluster_bin/bin_zbin_0_richbin_1/scaleval\n", + "cluster_bin/bin_zbin_1_richbin_0\n", + "cluster_bin/bin_zbin_1_richbin_0/cluster_id\n", + "cluster_bin/bin_zbin_1_richbin_0/dec\n", + "cluster_bin/bin_zbin_1_richbin_0/ra\n", + "cluster_bin/bin_zbin_1_richbin_0/redshift\n", + "cluster_bin/bin_zbin_1_richbin_0/redshift_err\n", + "cluster_bin/bin_zbin_1_richbin_0/richness\n", + "cluster_bin/bin_zbin_1_richbin_0/richness_err\n", + "cluster_bin/bin_zbin_1_richbin_0/scaleval\n", + "cluster_bin/bin_zbin_1_richbin_1\n", + "cluster_bin/bin_zbin_1_richbin_1/cluster_id\n", + "cluster_bin/bin_zbin_1_richbin_1/dec\n", + "cluster_bin/bin_zbin_1_richbin_1/ra\n", + "cluster_bin/bin_zbin_1_richbin_1/redshift\n", + "cluster_bin/bin_zbin_1_richbin_1/redshift_err\n", + "cluster_bin/bin_zbin_1_richbin_1/richness\n", + "cluster_bin/bin_zbin_1_richbin_1/richness_err\n", + "cluster_bin/bin_zbin_1_richbin_1/scaleval\n", + "cluster_bin/bin_zbin_2_richbin_0\n", + "cluster_bin/bin_zbin_2_richbin_0/cluster_id\n", + "cluster_bin/bin_zbin_2_richbin_0/dec\n", + "cluster_bin/bin_zbin_2_richbin_0/ra\n", + "cluster_bin/bin_zbin_2_richbin_0/redshift\n", + "cluster_bin/bin_zbin_2_richbin_0/redshift_err\n", + "cluster_bin/bin_zbin_2_richbin_0/richness\n", + "cluster_bin/bin_zbin_2_richbin_0/richness_err\n", + "cluster_bin/bin_zbin_2_richbin_0/scaleval\n", + "cluster_bin/bin_zbin_2_richbin_1\n", + "cluster_bin/bin_zbin_2_richbin_1/cluster_id\n", + "cluster_bin/bin_zbin_2_richbin_1/dec\n", + "cluster_bin/bin_zbin_2_richbin_1/ra\n", + "cluster_bin/bin_zbin_2_richbin_1/redshift\n", + "cluster_bin/bin_zbin_2_richbin_1/redshift_err\n", + "cluster_bin/bin_zbin_2_richbin_1/richness\n", + "cluster_bin/bin_zbin_2_richbin_1/richness_err\n", + "cluster_bin/bin_zbin_2_richbin_1/scaleval\n", + "provenance\n", + "[0.25462273 0.23431362 0.28674597 0.34493634 0.24044088]\n" + ] + } + ], + "source": [ + "cluster_tomo = h5py.File(f\"{my_txpipe_dir}/data/example/output_binning/cluster_catalog_tomography.hdf5\")\n", + "#print(cluster_tomo[\"cluster_bin\"][0])\n", + "\n", + "\n", + "with h5py.File(\"/global/homes/e/edujb/TXPipe/cluster_catalog_tomography.hdf5\", \"r\") as f:\n", + " def print_structure(name):\n", + " print(name)\n", + " f.visit(print_structure)\n", + " print(f[\"cluster_bin/bin_zbin_0_richbin_0/redshift\"][:])" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "id": "a7c432f8-da69-4f18-a1d9-2eaf5e55ce74", + "metadata": {}, + "outputs": [], + "source": [ + "step3 = txpipe.extensions.cluster_counts.TXTwoPointRLens.make_stage(\n", + " # This file is the input metadetect shear catalog\n", + " binned_lens_catalog=f\"{my_txpipe_dir}cluster_catalog_tomography.hdf5\",\n", + " # This is an input training set for the tomographic selection\n", + " binned_shear_catalog=f\"{my_txpipe_dir}data/example/output_shearcalib/binned_shear_catalog.hdf5\",\n", + " binned_random_catalog=f\"{my_txpipe_dir}cluster_catalog_tomography.hdf5\",\n", + " patch_centers=f\"{my_txpipe_dir}cluster_catalog_tomography.hdf5\",\n", + " tracer_metadata=f\"{my_txpipe_dir}cluster_catalog_tomography.hdf5\",\n", + " use_randoms= False,\n", + " output_dir=f\"{my_txpipe_dir}data/example/output_binning/\",\n", + " metric=\"Euclidean\",\n", + " var_method=\"shot\"\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "id": "81801925-4343-4e7a-9866-9df35ef8f8f2", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Running with 4 source bins and 6 lens bins\n", + "Running TreeCorr with metric \"Euclidean\"\n", + "DOING SHEAR-POS\n", + "Running 24 calculations: [(0, 0, 1), (0, 1, 1), (0, 2, 1), (0, 3, 1), (0, 4, 1), (0, 5, 1), (1, 0, 1), (1, 1, 1), (1, 2, 1), (1, 3, 1), (1, 4, 1), (1, 5, 1), (2, 0, 1), (2, 1, 1), (2, 2, 1), (2, 3, 1), (2, 4, 1), (2, 5, 1), (3, 0, 1), (3, 1, 1), (3, 2, 1), (3, 3, 1), (3, 4, 1), (3, 5, 1)]\n", + "Rank 0 making patches for shear catalog bin 0\n", + "Patches already done for cache/patches/TXTwoPointRLensbinned_shear_catalog_4206620e42794f3f853ba2e1a68ff289/0\n", + "Rank 0 making patches for position catalog bin 0\n", + "Patches already done for cache/patches/TXTwoPointRLensbinned_lens_catalog_8de35c2d194c44b98232fe45139f53c4/0\n", + "Rank 0 making patches for shear catalog bin 1\n", + "Patches already done for cache/patches/TXTwoPointRLensbinned_shear_catalog_4206620e42794f3f853ba2e1a68ff289/1\n", + "Rank 0 making patches for position catalog bin 1\n", + "Patches already done for cache/patches/TXTwoPointRLensbinned_lens_catalog_8de35c2d194c44b98232fe45139f53c4/1\n", + "Rank 0 making patches for shear catalog bin 2\n", + "Patches already done for cache/patches/TXTwoPointRLensbinned_shear_catalog_4206620e42794f3f853ba2e1a68ff289/2\n", + "Rank 0 making patches for position catalog bin 2\n", + "Patches already done for cache/patches/TXTwoPointRLensbinned_lens_catalog_8de35c2d194c44b98232fe45139f53c4/2\n", + "Rank 0 making patches for shear catalog bin 3\n", + "Patches already done for cache/patches/TXTwoPointRLensbinned_shear_catalog_4206620e42794f3f853ba2e1a68ff289/3\n", + "Rank 0 making patches for position catalog bin 3\n", + "Patches already done for cache/patches/TXTwoPointRLensbinned_lens_catalog_8de35c2d194c44b98232fe45139f53c4/3\n", + "Rank 0 making patches for position catalog bin 4\n", + "Patches already done for cache/patches/TXTwoPointRLensbinned_lens_catalog_8de35c2d194c44b98232fe45139f53c4/4\n", + "Rank 0 making patches for position catalog bin 5\n", + "Patches already done for cache/patches/TXTwoPointRLensbinned_lens_catalog_8de35c2d194c44b98232fe45139f53c4/5\n", + "Calculating shear-position bin pair (0,0): 2164 x 5 objects, 0 randoms\n", + "Processing took 0.1 seconds\n", + "Calculating shear-position bin pair (0,1): 2164 x 3 objects, 0 randoms\n", + "Processing took 0.1 seconds\n", + "Calculating shear-position bin pair (0,2): 2164 x 20 objects, 0 randoms\n", + "Processing took 0.1 seconds\n", + "Calculating shear-position bin pair (0,3): 2164 x 8 objects, 0 randoms\n", + "Processing took 0.0 seconds\n", + "Calculating shear-position bin pair (0,4): 2164 x 13 objects, 0 randoms\n", + "Processing took 0.1 seconds\n", + "Calculating shear-position bin pair (0,5): 2164 x 8 objects, 0 randoms\n", + "Processing took 0.0 seconds\n", + "Calculating shear-position bin pair (1,0): 2205 x 5 objects, 0 randoms\n", + "Processing took 0.0 seconds\n", + "Calculating shear-position bin pair (1,1): 2205 x 3 objects, 0 randoms\n", + "Processing took 0.1 seconds\n", + "Calculating shear-position bin pair (1,2): 2205 x 20 objects, 0 randoms\n", + "Processing took 0.1 seconds\n", + "Calculating shear-position bin pair (1,3): 2205 x 8 objects, 0 randoms\n", + "Processing took 0.1 seconds\n", + "Calculating shear-position bin pair (1,4): 2205 x 13 objects, 0 randoms\n", + "Processing took 0.1 seconds\n", + "Calculating shear-position bin pair (1,5): 2205 x 8 objects, 0 randoms\n", + "Processing took 0.1 seconds\n", + "Calculating shear-position bin pair (2,0): 1847 x 5 objects, 0 randoms\n", + "Processing took 0.0 seconds\n", + "Calculating shear-position bin pair (2,1): 1847 x 3 objects, 0 randoms\n", + "Processing took 0.1 seconds\n", + "Calculating shear-position bin pair (2,2): 1847 x 20 objects, 0 randoms\n", + "Processing took 0.1 seconds\n", + "Calculating shear-position bin pair (2,3): 1847 x 8 objects, 0 randoms\n", + "Processing took 0.0 seconds\n", + "Calculating shear-position bin pair (2,4): 1847 x 13 objects, 0 randoms\n", + "Processing took 0.0 seconds\n", + "Calculating shear-position bin pair (2,5): 1847 x 8 objects, 0 randoms\n", + "Processing took 0.1 seconds\n", + "Calculating shear-position bin pair (3,0): 2883 x 5 objects, 0 randoms\n", + "Processing took 0.1 seconds\n", + "Calculating shear-position bin pair (3,1): 2883 x 3 objects, 0 randoms\n", + "Processing took 0.0 seconds\n", + "Calculating shear-position bin pair (3,2): 2883 x 20 objects, 0 randoms\n", + "Processing took 0.1 seconds\n", + "Calculating shear-position bin pair (3,3): 2883 x 8 objects, 0 randoms\n", + "Processing took 0.1 seconds\n", + "Calculating shear-position bin pair (3,4): 2883 x 13 objects, 0 randoms\n", + "Processing took 0.1 seconds\n", + "Calculating shear-position bin pair (3,5): 2883 x 8 objects, 0 randoms\n", + "Processing took 0.1 seconds\n", + "#i_bin j_bin mean_r_Mpc rlens\n", + "0 0 1.2427598742099695 0.0\n", + "0 0 1.9193831036664843 0.0\n", + "0 0 2.9643952746562157 0.0\n", + "0 0 4.578366521835892 0.0\n", + "0 0 7.071067811865475 0.0\n", + "0 0 10.920925566254217 0.0\n", + "0 0 16.866846478764057 0.0\n", + "0 0 26.050036547934557 0.0\n", + "0 0 40.233033780387636 0.0\n", + "0 1 1.2427598742099695 0.0\n", + "0 1 1.9193831036664843 0.0\n", + "0 1 2.9643952746562157 0.0\n", + "0 1 4.578366521835892 0.0\n", + "0 1 7.071067811865475 0.0\n", + "0 1 10.920925566254217 0.0\n", + "0 1 16.866846478764057 0.0\n", + "0 1 26.050036547934557 0.0\n", + "0 1 40.233033780387636 0.0\n", + "0 2 1.2427598742099695 0.0\n", + "0 2 1.9193831036664843 0.0\n", + "0 2 2.9643952746562157 0.0\n", + "0 2 4.578366521835892 0.0\n", + "0 2 7.071067811865475 0.0\n", + "0 2 10.920925566254217 0.0\n", + "0 2 16.866846478764057 0.0\n", + "0 2 26.050036547934557 0.0\n", + "0 2 40.233033780387636 0.0\n", + "0 3 1.2427598742099695 0.0\n", + "0 3 1.9193831036664843 0.0\n", + "0 3 2.9643952746562157 0.0\n", + "0 3 4.578366521835892 0.0\n", + "0 3 7.071067811865475 0.0\n", + "0 3 10.920925566254217 0.0\n", + "0 3 16.866846478764057 0.0\n", + "0 3 26.050036547934557 0.0\n", + "0 3 40.233033780387636 0.0\n", + "0 4 1.2427598742099695 0.0\n", + "0 4 1.9193831036664843 0.0\n", + "0 4 2.9643952746562157 0.0\n", + "0 4 4.578366521835892 0.0\n", + "0 4 7.071067811865475 0.0\n", + "0 4 10.920925566254217 0.0\n", + "0 4 16.866846478764057 0.0\n", + "0 4 26.050036547934557 0.0\n", + "0 4 40.233033780387636 0.0\n", + "0 5 1.2427598742099695 0.0\n", + "0 5 1.9193831036664843 0.0\n", + "0 5 2.9643952746562157 0.0\n", + "0 5 4.578366521835892 0.0\n", + "0 5 7.071067811865475 0.0\n", + "0 5 10.920925566254217 0.0\n", + "0 5 16.866846478764057 0.0\n", + "0 5 26.050036547934557 0.0\n", + "0 5 40.233033780387636 0.0\n", + "1 0 1.2427598742099695 0.0\n", + "1 0 1.9193831036664843 0.0\n", + "1 0 2.9643952746562157 0.0\n", + "1 0 4.578366521835892 0.0\n", + "1 0 7.071067811865475 0.0\n", + "1 0 10.920925566254217 0.0\n", + "1 0 16.866846478764057 0.0\n", + "1 0 26.050036547934557 0.0\n", + "1 0 40.233033780387636 0.0\n", + "1 1 1.2427598742099695 0.0\n", + "1 1 1.9193831036664843 0.0\n", + "1 1 2.9643952746562157 0.0\n", + "1 1 4.578366521835892 0.0\n", + "1 1 7.071067811865475 0.0\n", + "1 1 10.920925566254217 0.0\n", + "1 1 16.866846478764057 0.0\n", + "1 1 26.050036547934557 0.0\n", + "1 1 40.233033780387636 0.0\n", + "1 2 1.2427598742099695 0.0\n", + "1 2 1.9193831036664843 0.0\n", + "1 2 2.9643952746562157 0.0\n", + "1 2 4.578366521835892 0.0\n", + "1 2 7.071067811865475 0.0\n", + "1 2 10.920925566254217 0.0\n", + "1 2 16.866846478764057 0.0\n", + "1 2 26.050036547934557 0.0\n", + "1 2 40.233033780387636 0.0\n", + "1 3 1.2427598742099695 0.0\n", + "1 3 1.9193831036664843 0.0\n", + "1 3 2.9643952746562157 0.0\n", + "1 3 4.578366521835892 0.0\n", + "1 3 7.071067811865475 0.0\n", + "1 3 10.920925566254217 0.0\n", + "1 3 16.866846478764057 0.0\n", + "1 3 26.050036547934557 0.0\n", + "1 3 40.233033780387636 0.0\n", + "1 4 1.2427598742099695 0.0\n", + "1 4 1.9193831036664843 0.0\n", + "1 4 2.9643952746562157 0.0\n", + "1 4 4.578366521835892 0.0\n", + "1 4 7.071067811865475 0.0\n", + "1 4 10.920925566254217 0.0\n", + "1 4 16.866846478764057 0.0\n", + "1 4 26.050036547934557 0.0\n", + "1 4 40.233033780387636 0.0\n", + "1 5 1.2427598742099695 0.0\n", + "1 5 1.9193831036664843 0.0\n", + "1 5 2.9643952746562157 0.0\n", + "1 5 4.578366521835892 0.0\n", + "1 5 7.071067811865475 0.0\n", + "1 5 10.920925566254217 0.0\n", + "1 5 16.866846478764057 0.0\n", + "1 5 26.050036547934557 0.0\n", + "1 5 40.233033780387636 0.0\n", + "2 0 1.2427598742099695 0.0\n", + "2 0 1.9193831036664843 0.0\n", + "2 0 2.9643952746562157 0.0\n", + "2 0 4.578366521835892 0.0\n", + "2 0 7.071067811865475 0.0\n", + "2 0 10.920925566254217 0.0\n", + "2 0 16.866846478764057 0.0\n", + "2 0 26.050036547934557 0.0\n", + "2 0 40.233033780387636 0.0\n", + "2 1 1.2427598742099695 0.0\n", + "2 1 1.9193831036664843 0.0\n", + "2 1 2.9643952746562157 0.0\n", + "2 1 4.578366521835892 0.0\n", + "2 1 7.071067811865475 0.0\n", + "2 1 10.920925566254217 0.0\n", + "2 1 16.866846478764057 0.0\n", + "2 1 26.050036547934557 0.0\n", + "2 1 40.233033780387636 0.0\n", + "2 2 1.2427598742099695 0.0\n", + "2 2 1.9193831036664843 0.0\n", + "2 2 2.9643952746562157 0.0\n", + "2 2 4.578366521835892 0.0\n", + "2 2 7.071067811865475 0.0\n", + "2 2 10.920925566254217 0.0\n", + "2 2 16.866846478764057 0.0\n", + "2 2 26.050036547934557 0.0\n", + "2 2 40.233033780387636 0.0\n", + "2 3 1.2427598742099695 0.0\n", + "2 3 1.9193831036664843 0.0\n", + "2 3 2.9643952746562157 0.0\n", + "2 3 4.578366521835892 0.0\n", + "2 3 7.071067811865475 0.0\n", + "2 3 10.920925566254217 0.0\n", + "2 3 16.866846478764057 0.0\n", + "2 3 26.050036547934557 0.0\n", + "2 3 40.233033780387636 0.0\n", + "2 4 1.2427598742099695 0.0\n", + "2 4 1.9193831036664843 0.0\n", + "2 4 2.9643952746562157 0.0\n", + "2 4 4.578366521835892 0.0\n", + "2 4 7.071067811865475 0.0\n", + "2 4 10.920925566254217 0.0\n", + "2 4 16.866846478764057 0.0\n", + "2 4 26.050036547934557 0.0\n", + "2 4 40.233033780387636 0.0\n", + "2 5 1.2427598742099695 0.0\n", + "2 5 1.9193831036664843 0.0\n", + "2 5 2.9643952746562157 0.0\n", + "2 5 4.578366521835892 0.0\n", + "2 5 7.071067811865475 0.0\n", + "2 5 10.920925566254217 0.0\n", + "2 5 16.866846478764057 0.0\n", + "2 5 26.050036547934557 0.0\n", + "2 5 40.233033780387636 0.0\n", + "3 0 1.2427598742099695 0.0\n", + "3 0 1.9193831036664843 0.0\n", + "3 0 2.9643952746562157 0.0\n", + "3 0 4.578366521835892 0.0\n", + "3 0 7.071067811865475 0.0\n", + "3 0 10.920925566254217 0.0\n", + "3 0 16.866846478764057 0.0\n", + "3 0 26.050036547934557 0.0\n", + "3 0 40.233033780387636 0.0\n", + "3 1 1.2427598742099695 0.0\n", + "3 1 1.9193831036664843 0.0\n", + "3 1 2.9643952746562157 0.0\n", + "3 1 4.578366521835892 0.0\n", + "3 1 7.071067811865475 0.0\n", + "3 1 10.920925566254217 0.0\n", + "3 1 16.866846478764057 0.0\n", + "3 1 26.050036547934557 0.0\n", + "3 1 40.233033780387636 0.0\n", + "3 2 1.2427598742099695 0.0\n", + "3 2 1.9193831036664843 0.0\n", + "3 2 2.9643952746562157 0.0\n", + "3 2 4.578366521835892 0.0\n", + "3 2 7.071067811865475 0.0\n", + "3 2 10.920925566254217 0.0\n", + "3 2 16.866846478764057 0.0\n", + "3 2 26.050036547934557 0.0\n", + "3 2 40.233033780387636 0.0\n", + "3 3 1.2427598742099695 0.0\n", + "3 3 1.9193831036664843 0.0\n", + "3 3 2.9643952746562157 0.0\n", + "3 3 4.578366521835892 0.0\n", + "3 3 7.071067811865475 0.0\n", + "3 3 10.920925566254217 0.0\n", + "3 3 16.866846478764057 0.0\n", + "3 3 26.050036547934557 0.0\n", + "3 3 40.233033780387636 0.0\n", + "3 4 1.2427598742099695 0.0\n", + "3 4 1.9193831036664843 0.0\n", + "3 4 2.9643952746562157 0.0\n", + "3 4 4.578366521835892 0.0\n", + "3 4 7.071067811865475 0.0\n", + "3 4 10.920925566254217 0.0\n", + "3 4 16.866846478764057 0.0\n", + "3 4 26.050036547934557 0.0\n", + "3 4 40.233033780387636 0.0\n", + "3 5 1.2427598742099695 0.0\n", + "3 5 1.9193831036664843 0.0\n", + "3 5 2.9643952746562157 0.0\n", + "3 5 4.578366521835892 0.0\n", + "3 5 7.071067811865475 0.0\n", + "3 5 10.920925566254217 0.0\n", + "3 5 16.866846478764057 0.0\n", + "3 5 26.050036547934557 0.0\n", + "3 5 40.233033780387636 0.0\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "NOTE/WARNING: Expected output file cluster_shear_catalogs.hdf5 was not generated.\n" + ] + } + ], + "source": [ + "step3.run()\n", + "step3.finalize()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Txpipe_clp", + "language": "python", + "name": "txpipe_clp" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.18" + }, + "nbdime-conflicts": { + "local_diff": [ + { + "diff": [ + { + "diff": [ + { + "key": 0, + "op": "addrange", + "valuelist": [ + "clmm" + ] + }, + { + "key": 0, + "length": 1, + "op": "removerange" + } + ], + "key": "display_name", + "op": "patch" + }, + { + "diff": [ + { + "key": 0, + "op": "addrange", + "valuelist": [ + "clmm" + ] + }, + { + "key": 0, + "length": 1, + "op": "removerange" + } + ], + "key": "name", + "op": "patch" + } + ], + "key": "kernelspec", + "op": "patch" + }, + { + "diff": [ + { + "diff": [ + { + "diff": [ + { + "key": 5, + "length": 1, + "op": "removerange" + } + ], + "key": 0, + "op": "patch" + } + ], + "key": "version", + "op": "patch" + } + ], + "key": "language_info", + "op": "patch" + } + ], + "remote_diff": [ + { + "diff": [ + { + "diff": [ + { + "key": 0, + "op": "addrange", + "valuelist": [ + "Python (firecrown2.0)" + ] + }, + { + "key": 0, + "length": 1, + "op": "removerange" + } + ], + "key": "display_name", + "op": "patch" + }, + { + "diff": [ + { + "key": 0, + "op": "addrange", + "valuelist": [ + "firecrown" + ] + }, + { + "key": 0, + "length": 1, + "op": "removerange" + } + ], + "key": "name", + "op": "patch" + } + ], + "key": "kernelspec", + "op": "patch" + }, + { + "diff": [ + { + "diff": [ + { + "key": 0, + "length": 1, + "op": "removerange" + } + ], + "key": "version", + "op": "patch" + } + ], + "key": "language_info", + "op": "patch" + } + ] + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/notebooks/cluster_counts/gt_treecor_test/config-1deg2-CL.yml b/notebooks/cluster_counts/gt_treecor_test/config-1deg2-CL.yml index a8d00e88..743f98a4 100644 --- a/notebooks/cluster_counts/gt_treecor_test/config-1deg2-CL.yml +++ b/notebooks/cluster_counts/gt_treecor_test/config-1deg2-CL.yml @@ -71,6 +71,9 @@ CLClusterShearCatalogs: subtract_mean_shear: false coordinate_system: celestial #euclidean or celestial, when using shear_true we shoudl use celestial use_true_shear: false +TXShearCalibration: + use_true_shear: true + subtract_mean_shear: false CLClusterEnsembleProfiles: #radial bin definition diff --git a/txpipe/__init__.py b/txpipe/__init__.py index bba2bea7..d25c09db 100755 --- a/txpipe/__init__.py +++ b/txpipe/__init__.py @@ -10,7 +10,7 @@ TXSourceSelectorLensfit, TXSourceSelectorMetadetect, ) -from .lens_selector import TXMeanLensSelector +from .lens_selector import TXMeanLensSelector, TXTruthLensCatalogSplitterWeighted from .photoz_stack import TXPhotozStack, TXPhotozPlot, TXTruePhotozStack from .random_cats import TXRandomCat from .twopoint_fourier import TXTwoPointFourier diff --git a/txpipe/extensions/cluster_counts/rlens.py b/txpipe/extensions/cluster_counts/rlens.py index 6c22d856..af85ba0e 100644 --- a/txpipe/extensions/cluster_counts/rlens.py +++ b/txpipe/extensions/cluster_counts/rlens.py @@ -7,6 +7,9 @@ ) import numpy as np +SHEAR_SHEAR = 0 +SHEAR_POS = 1 +POS_POS = 2 class TXTwoPointRLens(TXTwoPoint): """ @@ -31,9 +34,11 @@ class TXTwoPointRLens(TXTwoPoint): ("binned_shear_catalog", HDFFile), ("binned_random_catalog", HDFFile), ("patch_centers", TextFile), - ("tracer_metadata", HDFFile), ] - outputs = [("rlens_measurement", TextFile)] + outputs = [ + ("rlens_measurement", TextFile), + ("cluster_shear_catalogs", HDFFile), + ] config_options = { # TODO: Allow more fine-grained selection of 2pt subsets to compute @@ -62,21 +67,30 @@ class TXTwoPointRLens(TXTwoPoint): "use_subsampled_randoms": False, } + def read_metadata(self): + return {} + + def get_lens_catalog(self, i): # Override the lens catalog generation. # This is like the parent version except we also add the r_col keyword import treecorr + bin_name = None + with self.open_input("binned_lens_catalog") as f: + for j, zbin_richbin in enumerate(f['cluster_bin'].keys()): + if j == i: + bin_name = zbin_richbin cat = treecorr.Catalog( self.get_input("binned_lens_catalog"), - ext=f"/lens/bin_{i}", + ext=f"/cluster_bin/{bin_name}", ra_col="ra", dec_col="dec", - r_col="comoving_distance", - w_col="weight", + #r_col="comoving_distance", + #w_col="weight", ra_units="degree", dec_units="degree", - patch_centers=self.get_input("patch_centers"), + patch_centers=None,#self.get_input("patch_centers"), save_patch_dir=self.get_patch_dir("binned_lens_catalog", i), ) return cat @@ -94,14 +108,36 @@ def get_random_catalog(self, i): ext=f"/randoms/bin_{i}", ra_col="ra", dec_col="dec", - r_col="comoving_distance", + #r_col="comoving_distance", ra_units="degree", dec_units="degree", - patch_centers=self.get_input("patch_centers"), + patch_centers=None,#self.get_input("patch_centers"), save_patch_dir=self.get_patch_dir("binned_random_catalog", i), ) return rancat + def get_shear_catalog(self, i): + import treecorr + + # Load and calibrate the appropriate bin data + cat = treecorr.Catalog( + self.get_input("binned_shear_catalog"), + ext=f"/shear/bin_{i}", + g1_col="g1", + g2_col="g2", + ra_col="ra", + dec_col="dec", + w_col="weight", + ra_units="degree", + dec_units="degree", + patch_centers=None,#self.get_input("patch_centers"), + save_patch_dir=self.get_patch_dir("binned_shear_catalog", i), + flip_g1=self.config["flip_g1"], + flip_g2=self.config["flip_g2"], + ) + + return cat + def call_treecorr(self, i, j, k): # The parent class uses the label for gamma_t measurements # here, so we overwrite it. @@ -117,3 +153,22 @@ def write_output(self, source_list, lens_list, meta, results): for logr, xi in zip(result.object.meanlogr, result.object.xi): print(result.i, result.j, np.exp(logr), xi) print(result.i, result.j, np.exp(logr), xi, file=f) + + def _read_nbin_from_tomography(self): + if self.get_input("binned_shear_catalog") == "none": + nbin_source = 0 + else: + with self.open_input("binned_shear_catalog") as f: + nbin_source = f["shear"].attrs["nbin_source"] + + if self.get_input("binned_lens_catalog") == "none": + nbin_lens = 0 + else: + with self.open_input("binned_lens_catalog") as f: + nbin_lens = len(f["cluster_bin"]) + + source_list = list(range(nbin_source)) + lens_list = list(range(nbin_lens)) + + return source_list, lens_list + From 414952a465964a61403c2c1e873920ffc0aeaa39 Mon Sep 17 00:00:00 2001 From: eduardojsbarroso Date: Fri, 17 Oct 2025 08:01:40 -0700 Subject: [PATCH 7/7] Created new cluster two point with treecor using euclidean metric --- .../Run_CL_pipeline_treecor.ipynb | 1588 ++++++++++++++--- txpipe/extensions/cluster_counts/__init__.py | 2 +- .../cluster_counts/convert_to_sacc.py | 39 +- txpipe/extensions/cluster_counts/rlens.py | 73 +- txpipe/extensions/cluster_counts/rlens_new.py | 958 ---------- .../two_point_cluster_source.py | 346 ++++ 6 files changed, 1684 insertions(+), 1322 deletions(-) delete mode 100644 txpipe/extensions/cluster_counts/rlens_new.py create mode 100644 txpipe/extensions/cluster_counts/two_point_cluster_source.py diff --git a/notebooks/cluster_counts/gt_treecor_test/Run_CL_pipeline_treecor.ipynb b/notebooks/cluster_counts/gt_treecor_test/Run_CL_pipeline_treecor.ipynb index 21db5a7d..81d13892 100644 --- a/notebooks/cluster_counts/gt_treecor_test/Run_CL_pipeline_treecor.ipynb +++ b/notebooks/cluster_counts/gt_treecor_test/Run_CL_pipeline_treecor.ipynb @@ -101,7 +101,9 @@ "cell_type": "code", "execution_count": 5, "id": "c5e6e548-7d43-43fb-93d7-debaedd9882c", - "metadata": {}, + "metadata": { + "scrolled": true + }, "outputs": [ { "name": "stdout", @@ -119,6 +121,42 @@ { "cell_type": "code", "execution_count": 6, + "id": "a90854f4-6bab-4451-8b01-1b86119d233b", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Keys: ['counts', 'provenance', 'response', 'tomography']\n" + ] + }, + { + "data": { + "text/plain": [ + "[np.float32(0.00090161833),\n", + " np.float32(0.0014274995),\n", + " np.float32(-0.00015233812),\n", + " np.float32(0.00078499416)]" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "shear_tomography_catalog=f\"{my_txpipe_dir}/data/example/outputs_metadetect_20/shear_tomography_catalog.hdf5\"\n", + "with h5py.File(shear_tomography_catalog, 'r') as file:\n", + "# List all groups in the file\n", + " print(\"Keys:\", list(file.keys()))\n", + " #print(file[\"counts\"])\n", + " display(list(file[\"counts\"][\"mean_e1\"]))\n", + " #display(list(file[\"tomography\"][\"bin\"]))\n" + ] + }, + { + "cell_type": "code", + "execution_count": 7, "id": "073b332a-ace9-4e27-9c6e-e1c9807bd2e0", "metadata": {}, "outputs": [], @@ -140,7 +178,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 8, "id": "e4dc452e-f860-4b46-b141-ccb2243dc752", "metadata": {}, "outputs": [], @@ -151,7 +189,82 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 9, + "id": "c59cd0cd-8879-4924-ae84-fd1e011d5ad3", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Keys: ['cluster_bin', 'provenance']\n", + "['cluster_id', 'dec', 'ra', 'redshift', 'redshift_err', 'richness', 'richness_err', 'scaleval']\n", + "5\n", + "[np.float64(0.2546227276325226), np.float64(0.23431362211704254), np.float64(0.28674596548080444), np.float64(0.344936341047287), np.float64(0.24044087529182434)]\n", + "['rich_max', 'rich_min', 'z_max', 'z_min']\n", + "10.0\n", + "[]\n", + "bin_zbin_1_richbin_1\n", + "Tomography contents: ['bin']\n", + "\n", + "Attributes of tomography group:\n", + " catalog_type: metadetect\n", + " nbin: 4\n", + " zmax_0: 0.7\n", + " zmax_1: 0.9\n", + " zmax_2: 1.1\n", + " zmax_3: 2.0\n", + " zmin_0: 0.5\n", + " zmin_1: 0.7\n", + " zmin_2: 0.9\n", + " zmin_3: 1.1\n", + "\n", + "Inspecting first dataset: bin\n", + "Shape: (82200,)\n", + "Dtype: int32\n", + "Attributes:\n", + "First few values: [-1 -1 -1 -1 -1 -1 -1 -1 -1 -1]\n" + ] + } + ], + "source": [ + "cluster_catalog_tomography=f\"{my_txpipe_dir}/data/example/output_binning/cluster_catalog_tomography.hdf5\"\n", + "with h5py.File(cluster_catalog_tomography, 'r') as file:\n", + "# List all groups in the file\n", + " print(\"Keys:\", list(file.keys()))\n", + " print(list(file[\"cluster_bin\"][\"bin_zbin_0_richbin_0\"]))\n", + " print(len(list(file[\"cluster_bin\"][\"bin_zbin_0_richbin_0\"][\"cluster_id\"])))\n", + " print(list(file[\"cluster_bin\"][\"bin_zbin_0_richbin_0\"][\"redshift\"]))\n", + " print(list(file[\"cluster_bin\"][\"bin_zbin_0_richbin_0\"].attrs))\n", + " test = file[\"cluster_bin\"][\"bin_zbin_0_richbin_0\"].attrs\n", + " print(test[\"rich_max\"])\n", + " print(list(file[\"provenance\"].keys()))\n", + " print(list(file[\"cluster_bin\"])[3])\n", + "\n", + "\n", + "cluster_catalog_tomography = f\"{my_txpipe_dir}/data/example/outputs_metadetect/shear_tomography_catalog.hdf5\"\n", + "\n", + "with h5py.File(cluster_catalog_tomography, \"r\") as f:\n", + " tomo = f[\"tomography\"]\n", + " print(\"Tomography contents:\", list(tomo.keys()))\n", + " print(\"\\nAttributes of tomography group:\")\n", + " for a, v in tomo.attrs.items():\n", + " print(f\" {a}: {v}\")\n", + "\n", + " # Now print one dataset info\n", + " first_key = list(tomo.keys())[0]\n", + " print(f\"\\nInspecting first dataset: {first_key}\")\n", + " print(\"Shape:\", tomo[first_key].shape)\n", + " print(\"Dtype:\", tomo[first_key].dtype)\n", + " print(\"Attributes:\")\n", + " for a, v in tomo[first_key].attrs.items():\n", + " print(f\" {a}: {v}\")\n", + " print(\"First few values:\", tomo[first_key][:10])\n" + ] + }, + { + "cell_type": "code", + "execution_count": 10, "id": "3e7eb73a-2bb5-4910-9596-5400ff185862", "metadata": {}, "outputs": [], @@ -166,7 +279,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 11, "id": "87c878d4-41aa-4658-8c9b-d3c5d969c7fa", "metadata": {}, "outputs": [ @@ -213,54 +326,73 @@ }, { "cell_type": "code", - "execution_count": 10, - "id": "cc8203a2-38e5-4414-b1ac-1b7e878d8958", - "metadata": {}, + "execution_count": 12, + "id": "f2db4bee-2ed5-4cfc-bae9-48e973960f75", + "metadata": { + "scrolled": true + }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Redshift bin 1 has 28 clusters.\n" + "\n", + "=== Provenance keys ===\n", + "[]\n", + "\n", + "=== Shear group keys ===\n", + "['bin_0', 'bin_1', 'bin_2', 'bin_3', 'bin_all']\n", + "\n", + "--- bin_0 ---\n", + "Datasets: ['dec', 'g1', 'g2', 'mag_err_i', 'mag_err_r', 'mag_err_z', 'mag_i', 'mag_r', 'mag_z', 'ra', 'weight']\n", + "Attributes:\n", + "\n", + "--- bin_1 ---\n", + "Datasets: ['dec', 'g1', 'g2', 'mag_err_i', 'mag_err_r', 'mag_err_z', 'mag_i', 'mag_r', 'mag_z', 'ra', 'weight']\n", + "Attributes:\n", + "\n", + "--- bin_2 ---\n", + "Datasets: ['dec', 'g1', 'g2', 'mag_err_i', 'mag_err_r', 'mag_err_z', 'mag_i', 'mag_r', 'mag_z', 'ra', 'weight']\n", + "Attributes:\n", + "\n", + "--- bin_3 ---\n", + "Datasets: ['dec', 'g1', 'g2', 'mag_err_i', 'mag_err_r', 'mag_err_z', 'mag_i', 'mag_r', 'mag_z', 'ra', 'weight']\n", + "Attributes:\n", + "\n", + "--- bin_all ---\n", + "Datasets: ['dec', 'g1', 'g2', 'mag_err_i', 'mag_err_r', 'mag_err_z', 'mag_i', 'mag_r', 'mag_z', 'ra', 'weight']\n", + "Attributes:\n" ] } ], "source": [ - "path = \"/global/homes/e/edujb/TXPipe/cluster_catalog_tomography.hdf5\"\n", + "import h5py\n", + "\n", + "binned_shear_catalog = f\"{my_txpipe_dir}data/example/output_shearcalib/binned_shear_catalog.hdf5\"\n", + "\n", + "with h5py.File(binned_shear_catalog, \"r\") as f:\n", + " # Step 1: provenance\n", + " print(\"\\n=== Provenance keys ===\")\n", + " print(list(f[\"provenance\"].keys()))\n", "\n", - "with h5py.File(path, \"r\") as f:\n", - " zbin_groups = {}\n", - " \n", - " # Iterate through all subgroups under cluster_bin\n", - " for name in f[\"cluster_bin\"].keys():\n", - " if name.startswith(\"bin_zbin_\"):\n", - " # Extract redshift bin index\n", - " zbin = name.split(\"_richbin_\")[0].replace(\"bin_zbin_\", \"\")\n", - " if zbin not in zbin_groups:\n", - " zbin_groups[zbin] = []\n", - " zbin_groups[zbin].append(f[\"cluster_bin\"][name])\n", - " \n", - " # Now combine data for each zbin\n", - " clusters_by_zbin = {}\n", - " for zbin, groups in zbin_groups.items():\n", - " combined = {}\n", - " for g in groups:\n", - " for key in g.keys():\n", - " if key not in combined:\n", - " combined[key] = np.array(g[key])\n", - " else:\n", - " combined[key] = np.concatenate([combined[key], np.array(g[key])])\n", - " clusters_by_zbin[zbin] = combined\n", + " # Step 2: shear group structure\n", + " shear = f[\"shear\"]\n", + " print(\"\\n=== Shear group keys ===\")\n", + " print(list(shear.keys()))\n", "\n", - "# Example: Access zbin_1 data\n", - "ra_z1 = clusters_by_zbin[\"1\"][\"ra\"]\n", - "dec_z1 = clusters_by_zbin[\"1\"][\"dec\"]\n", - "print(f\"Redshift bin 1 has {len(ra_z1)} clusters.\")" + " # Step 3: list all subgroups and their contents + attributes\n", + " for b in shear.keys():\n", + " print(f\"\\n--- {b} ---\")\n", + " g = shear[b]\n", + " print(\"Datasets:\", list(g.keys()))\n", + " print(\"Attributes:\")\n", + " for k, v in g.attrs.items():\n", + " print(f\" {k}: {v}\")\n" ] }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 13, "id": "16b401f6-d74e-4813-b87b-85c0802c97cb", "metadata": {}, "outputs": [ @@ -324,6 +456,12 @@ "cluster_bin/bin_zbin_2_richbin_1/richness_err\n", "cluster_bin/bin_zbin_2_richbin_1/scaleval\n", "provenance\n", + "bin_zbin_0_richbin_0\n", + "bin_zbin_0_richbin_1\n", + "bin_zbin_1_richbin_0\n", + "bin_zbin_1_richbin_1\n", + "bin_zbin_2_richbin_0\n", + "bin_zbin_2_richbin_1\n", "[0.25462273 0.23431362 0.28674597 0.34493634 0.24044088]\n" ] } @@ -337,17 +475,19 @@ " def print_structure(name):\n", " print(name)\n", " f.visit(print_structure)\n", + " for j, bin_name in enumerate(f[\"cluster_bin\"].keys()):\n", + " print(bin_name)\n", " print(f[\"cluster_bin/bin_zbin_0_richbin_0/redshift\"][:])" ] }, { "cell_type": "code", - "execution_count": 22, + "execution_count": 14, "id": "a7c432f8-da69-4f18-a1d9-2eaf5e55ce74", "metadata": {}, "outputs": [], "source": [ - "step3 = txpipe.extensions.cluster_counts.TXTwoPointRLens.make_stage(\n", + "step3 = txpipe.extensions.cluster_counts.TXTwoPointClusterSource.make_stage(\n", " # This file is the input metadetect shear catalog\n", " binned_lens_catalog=f\"{my_txpipe_dir}cluster_catalog_tomography.hdf5\",\n", " # This is an input training set for the tomographic selection\n", @@ -355,8 +495,9 @@ " binned_random_catalog=f\"{my_txpipe_dir}cluster_catalog_tomography.hdf5\",\n", " patch_centers=f\"{my_txpipe_dir}cluster_catalog_tomography.hdf5\",\n", " tracer_metadata=f\"{my_txpipe_dir}cluster_catalog_tomography.hdf5\",\n", + " shear_tomography_catalog=f\"{my_txpipe_dir}/data/example/outputs_metadetect/shear_tomography_catalog.hdf5\",\n", " use_randoms= False,\n", - " output_dir=f\"{my_txpipe_dir}data/example/output_binning/\",\n", + " cluster_profiles=f\"{my_txpipe_dir}data/example/output_txcluster/cluster_profiles.pkl\",\n", " metric=\"Euclidean\",\n", " var_method=\"shot\"\n", ")" @@ -364,7 +505,7 @@ }, { "cell_type": "code", - "execution_count": 23, + "execution_count": 15, "id": "81801925-4343-4e7a-9866-9df35ef8f8f2", "metadata": {}, "outputs": [ @@ -375,67 +516,57 @@ "Running with 4 source bins and 6 lens bins\n", "Running TreeCorr with metric \"Euclidean\"\n", "DOING SHEAR-POS\n", - "Running 24 calculations: [(0, 0, 1), (0, 1, 1), (0, 2, 1), (0, 3, 1), (0, 4, 1), (0, 5, 1), (1, 0, 1), (1, 1, 1), (1, 2, 1), (1, 3, 1), (1, 4, 1), (1, 5, 1), (2, 0, 1), (2, 1, 1), (2, 2, 1), (2, 3, 1), (2, 4, 1), (2, 5, 1), (3, 0, 1), (3, 1, 1), (3, 2, 1), (3, 3, 1), (3, 4, 1), (3, 5, 1)]\n", + "Skipping lens bin 2 (z=[0.400, 0.600]) and source bin 0 (z=[0.500, 0.700]) — overlap detected.\n", + "Skipping lens bin 3 (z=[0.400, 0.600]) and source bin 0 (z=[0.500, 0.700]) — overlap detected.\n", + "Skipping lens bin 4 (z=[0.600, 0.800]) and source bin 0 (z=[0.500, 0.700]) — overlap detected.\n", + "Skipping lens bin 5 (z=[0.600, 0.800]) and source bin 0 (z=[0.500, 0.700]) — overlap detected.\n", + "Skipping lens bin 4 (z=[0.600, 0.800]) and source bin 1 (z=[0.700, 0.900]) — overlap detected.\n", + "Skipping lens bin 5 (z=[0.600, 0.800]) and source bin 1 (z=[0.700, 0.900]) — overlap detected.\n", + "Running 18 calculations: [(0, 0, 1), (0, 1, 1), (1, 0, 1), (1, 1, 1), (1, 2, 1), (1, 3, 1), (2, 0, 1), (2, 1, 1), (2, 2, 1), (2, 3, 1), (2, 4, 1), (2, 5, 1), (3, 0, 1), (3, 1, 1), (3, 2, 1), (3, 3, 1), (3, 4, 1), (3, 5, 1)]\n", "Rank 0 making patches for shear catalog bin 0\n", - "Patches already done for cache/patches/TXTwoPointRLensbinned_shear_catalog_4206620e42794f3f853ba2e1a68ff289/0\n", "Rank 0 making patches for position catalog bin 0\n", - "Patches already done for cache/patches/TXTwoPointRLensbinned_lens_catalog_8de35c2d194c44b98232fe45139f53c4/0\n", + "Patches already done for cache/patches/TXTwoPointClusterSourcebinned_lens_catalog_8de35c2d194c44b98232fe45139f53c4/0\n", "Rank 0 making patches for shear catalog bin 1\n", - "Patches already done for cache/patches/TXTwoPointRLensbinned_shear_catalog_4206620e42794f3f853ba2e1a68ff289/1\n", "Rank 0 making patches for position catalog bin 1\n", - "Patches already done for cache/patches/TXTwoPointRLensbinned_lens_catalog_8de35c2d194c44b98232fe45139f53c4/1\n", + "Patches already done for cache/patches/TXTwoPointClusterSourcebinned_lens_catalog_8de35c2d194c44b98232fe45139f53c4/1\n", "Rank 0 making patches for shear catalog bin 2\n", - "Patches already done for cache/patches/TXTwoPointRLensbinned_shear_catalog_4206620e42794f3f853ba2e1a68ff289/2\n", "Rank 0 making patches for position catalog bin 2\n", - "Patches already done for cache/patches/TXTwoPointRLensbinned_lens_catalog_8de35c2d194c44b98232fe45139f53c4/2\n", + "Patches already done for cache/patches/TXTwoPointClusterSourcebinned_lens_catalog_8de35c2d194c44b98232fe45139f53c4/2\n", "Rank 0 making patches for shear catalog bin 3\n", - "Patches already done for cache/patches/TXTwoPointRLensbinned_shear_catalog_4206620e42794f3f853ba2e1a68ff289/3\n", "Rank 0 making patches for position catalog bin 3\n", - "Patches already done for cache/patches/TXTwoPointRLensbinned_lens_catalog_8de35c2d194c44b98232fe45139f53c4/3\n", + "Patches already done for cache/patches/TXTwoPointClusterSourcebinned_lens_catalog_8de35c2d194c44b98232fe45139f53c4/3\n", "Rank 0 making patches for position catalog bin 4\n", - "Patches already done for cache/patches/TXTwoPointRLensbinned_lens_catalog_8de35c2d194c44b98232fe45139f53c4/4\n", + "Patches already done for cache/patches/TXTwoPointClusterSourcebinned_lens_catalog_8de35c2d194c44b98232fe45139f53c4/4\n", "Rank 0 making patches for position catalog bin 5\n", - "Patches already done for cache/patches/TXTwoPointRLensbinned_lens_catalog_8de35c2d194c44b98232fe45139f53c4/5\n", + "Patches already done for cache/patches/TXTwoPointClusterSourcebinned_lens_catalog_8de35c2d194c44b98232fe45139f53c4/5\n", "Calculating shear-position bin pair (0,0): 2164 x 5 objects, 0 randoms\n", - "Processing took 0.1 seconds\n", - "Calculating shear-position bin pair (0,1): 2164 x 3 objects, 0 randoms\n", - "Processing took 0.1 seconds\n", - "Calculating shear-position bin pair (0,2): 2164 x 20 objects, 0 randoms\n", - "Processing took 0.1 seconds\n", - "Calculating shear-position bin pair (0,3): 2164 x 8 objects, 0 randoms\n", "Processing took 0.0 seconds\n", - "Calculating shear-position bin pair (0,4): 2164 x 13 objects, 0 randoms\n", - "Processing took 0.1 seconds\n", - "Calculating shear-position bin pair (0,5): 2164 x 8 objects, 0 randoms\n", + "Calculating shear-position bin pair (0,1): 2164 x 3 objects, 0 randoms\n", "Processing took 0.0 seconds\n", "Calculating shear-position bin pair (1,0): 2205 x 5 objects, 0 randoms\n", - "Processing took 0.0 seconds\n", + "Processing took 0.1 seconds\n", "Calculating shear-position bin pair (1,1): 2205 x 3 objects, 0 randoms\n", "Processing took 0.1 seconds\n", "Calculating shear-position bin pair (1,2): 2205 x 20 objects, 0 randoms\n", "Processing took 0.1 seconds\n", "Calculating shear-position bin pair (1,3): 2205 x 8 objects, 0 randoms\n", "Processing took 0.1 seconds\n", - "Calculating shear-position bin pair (1,4): 2205 x 13 objects, 0 randoms\n", - "Processing took 0.1 seconds\n", - "Calculating shear-position bin pair (1,5): 2205 x 8 objects, 0 randoms\n", - "Processing took 0.1 seconds\n", "Calculating shear-position bin pair (2,0): 1847 x 5 objects, 0 randoms\n", - "Processing took 0.0 seconds\n", - "Calculating shear-position bin pair (2,1): 1847 x 3 objects, 0 randoms\n", "Processing took 0.1 seconds\n", + "Calculating shear-position bin pair (2,1): 1847 x 3 objects, 0 randoms\n", + "Processing took 0.0 seconds\n", "Calculating shear-position bin pair (2,2): 1847 x 20 objects, 0 randoms\n", - "Processing took 0.1 seconds\n", - "Calculating shear-position bin pair (2,3): 1847 x 8 objects, 0 randoms\n", "Processing took 0.0 seconds\n", + "Calculating shear-position bin pair (2,3): 1847 x 8 objects, 0 randoms\n", + "Processing took 0.1 seconds\n", "Calculating shear-position bin pair (2,4): 1847 x 13 objects, 0 randoms\n", - "Processing took 0.0 seconds\n", - "Calculating shear-position bin pair (2,5): 1847 x 8 objects, 0 randoms\n", "Processing took 0.1 seconds\n", + "Calculating shear-position bin pair (2,5): 1847 x 8 objects, 0 randoms\n", + "Processing took 0.0 seconds\n", "Calculating shear-position bin pair (3,0): 2883 x 5 objects, 0 randoms\n", "Processing took 0.1 seconds\n", "Calculating shear-position bin pair (3,1): 2883 x 3 objects, 0 randoms\n", - "Processing took 0.0 seconds\n", + "Processing took 0.1 seconds\n", "Calculating shear-position bin pair (3,2): 2883 x 20 objects, 0 randoms\n", "Processing took 0.1 seconds\n", "Calculating shear-position bin pair (3,3): 2883 x 8 objects, 0 randoms\n", @@ -444,230 +575,1046 @@ "Processing took 0.1 seconds\n", "Calculating shear-position bin pair (3,5): 2883 x 8 objects, 0 randoms\n", "Processing took 0.1 seconds\n", - "#i_bin j_bin mean_r_Mpc rlens\n", - "0 0 1.2427598742099695 0.0\n", - "0 0 1.9193831036664843 0.0\n", - "0 0 2.9643952746562157 0.0\n", - "0 0 4.578366521835892 0.0\n", - "0 0 7.071067811865475 0.0\n", - "0 0 10.920925566254217 0.0\n", - "0 0 16.866846478764057 0.0\n", - "0 0 26.050036547934557 0.0\n", - "0 0 40.233033780387636 0.0\n", - "0 1 1.2427598742099695 0.0\n", - "0 1 1.9193831036664843 0.0\n", - "0 1 2.9643952746562157 0.0\n", - "0 1 4.578366521835892 0.0\n", - "0 1 7.071067811865475 0.0\n", - "0 1 10.920925566254217 0.0\n", - "0 1 16.866846478764057 0.0\n", - "0 1 26.050036547934557 0.0\n", - "0 1 40.233033780387636 0.0\n", - "0 2 1.2427598742099695 0.0\n", - "0 2 1.9193831036664843 0.0\n", - "0 2 2.9643952746562157 0.0\n", - "0 2 4.578366521835892 0.0\n", - "0 2 7.071067811865475 0.0\n", - "0 2 10.920925566254217 0.0\n", - "0 2 16.866846478764057 0.0\n", - "0 2 26.050036547934557 0.0\n", - "0 2 40.233033780387636 0.0\n", - "0 3 1.2427598742099695 0.0\n", - "0 3 1.9193831036664843 0.0\n", - "0 3 2.9643952746562157 0.0\n", - "0 3 4.578366521835892 0.0\n", - "0 3 7.071067811865475 0.0\n", - "0 3 10.920925566254217 0.0\n", - "0 3 16.866846478764057 0.0\n", - "0 3 26.050036547934557 0.0\n", - "0 3 40.233033780387636 0.0\n", - "0 4 1.2427598742099695 0.0\n", - "0 4 1.9193831036664843 0.0\n", - "0 4 2.9643952746562157 0.0\n", - "0 4 4.578366521835892 0.0\n", - "0 4 7.071067811865475 0.0\n", - "0 4 10.920925566254217 0.0\n", - "0 4 16.866846478764057 0.0\n", - "0 4 26.050036547934557 0.0\n", - "0 4 40.233033780387636 0.0\n", - "0 5 1.2427598742099695 0.0\n", - "0 5 1.9193831036664843 0.0\n", - "0 5 2.9643952746562157 0.0\n", - "0 5 4.578366521835892 0.0\n", - "0 5 7.071067811865475 0.0\n", - "0 5 10.920925566254217 0.0\n", - "0 5 16.866846478764057 0.0\n", - "0 5 26.050036547934557 0.0\n", - "0 5 40.233033780387636 0.0\n", - "1 0 1.2427598742099695 0.0\n", - "1 0 1.9193831036664843 0.0\n", - "1 0 2.9643952746562157 0.0\n", - "1 0 4.578366521835892 0.0\n", - "1 0 7.071067811865475 0.0\n", - "1 0 10.920925566254217 0.0\n", - "1 0 16.866846478764057 0.0\n", - "1 0 26.050036547934557 0.0\n", - "1 0 40.233033780387636 0.0\n", - "1 1 1.2427598742099695 0.0\n", - "1 1 1.9193831036664843 0.0\n", - "1 1 2.9643952746562157 0.0\n", - "1 1 4.578366521835892 0.0\n", - "1 1 7.071067811865475 0.0\n", - "1 1 10.920925566254217 0.0\n", - "1 1 16.866846478764057 0.0\n", - "1 1 26.050036547934557 0.0\n", - "1 1 40.233033780387636 0.0\n", - "1 2 1.2427598742099695 0.0\n", - "1 2 1.9193831036664843 0.0\n", - "1 2 2.9643952746562157 0.0\n", - "1 2 4.578366521835892 0.0\n", - "1 2 7.071067811865475 0.0\n", - "1 2 10.920925566254217 0.0\n", - "1 2 16.866846478764057 0.0\n", - "1 2 26.050036547934557 0.0\n", - "1 2 40.233033780387636 0.0\n", - "1 3 1.2427598742099695 0.0\n", - "1 3 1.9193831036664843 0.0\n", - "1 3 2.9643952746562157 0.0\n", - "1 3 4.578366521835892 0.0\n", - "1 3 7.071067811865475 0.0\n", - "1 3 10.920925566254217 0.0\n", - "1 3 16.866846478764057 0.0\n", - "1 3 26.050036547934557 0.0\n", - "1 3 40.233033780387636 0.0\n", - "1 4 1.2427598742099695 0.0\n", - "1 4 1.9193831036664843 0.0\n", - "1 4 2.9643952746562157 0.0\n", - "1 4 4.578366521835892 0.0\n", - "1 4 7.071067811865475 0.0\n", - "1 4 10.920925566254217 0.0\n", - "1 4 16.866846478764057 0.0\n", - "1 4 26.050036547934557 0.0\n", - "1 4 40.233033780387636 0.0\n", - "1 5 1.2427598742099695 0.0\n", - "1 5 1.9193831036664843 0.0\n", - "1 5 2.9643952746562157 0.0\n", - "1 5 4.578366521835892 0.0\n", - "1 5 7.071067811865475 0.0\n", - "1 5 10.920925566254217 0.0\n", - "1 5 16.866846478764057 0.0\n", - "1 5 26.050036547934557 0.0\n", - "1 5 40.233033780387636 0.0\n", - "2 0 1.2427598742099695 0.0\n", - "2 0 1.9193831036664843 0.0\n", - "2 0 2.9643952746562157 0.0\n", - "2 0 4.578366521835892 0.0\n", - "2 0 7.071067811865475 0.0\n", - "2 0 10.920925566254217 0.0\n", - "2 0 16.866846478764057 0.0\n", - "2 0 26.050036547934557 0.0\n", - "2 0 40.233033780387636 0.0\n", - "2 1 1.2427598742099695 0.0\n", - "2 1 1.9193831036664843 0.0\n", - "2 1 2.9643952746562157 0.0\n", - "2 1 4.578366521835892 0.0\n", - "2 1 7.071067811865475 0.0\n", - "2 1 10.920925566254217 0.0\n", - "2 1 16.866846478764057 0.0\n", - "2 1 26.050036547934557 0.0\n", - "2 1 40.233033780387636 0.0\n", - "2 2 1.2427598742099695 0.0\n", - "2 2 1.9193831036664843 0.0\n", - "2 2 2.9643952746562157 0.0\n", - "2 2 4.578366521835892 0.0\n", - "2 2 7.071067811865475 0.0\n", - "2 2 10.920925566254217 0.0\n", - "2 2 16.866846478764057 0.0\n", - "2 2 26.050036547934557 0.0\n", - "2 2 40.233033780387636 0.0\n", - "2 3 1.2427598742099695 0.0\n", - "2 3 1.9193831036664843 0.0\n", - "2 3 2.9643952746562157 0.0\n", - "2 3 4.578366521835892 0.0\n", - "2 3 7.071067811865475 0.0\n", - "2 3 10.920925566254217 0.0\n", - "2 3 16.866846478764057 0.0\n", - "2 3 26.050036547934557 0.0\n", - "2 3 40.233033780387636 0.0\n", - "2 4 1.2427598742099695 0.0\n", - "2 4 1.9193831036664843 0.0\n", - "2 4 2.9643952746562157 0.0\n", - "2 4 4.578366521835892 0.0\n", - "2 4 7.071067811865475 0.0\n", - "2 4 10.920925566254217 0.0\n", - "2 4 16.866846478764057 0.0\n", - "2 4 26.050036547934557 0.0\n", - "2 4 40.233033780387636 0.0\n", - "2 5 1.2427598742099695 0.0\n", - "2 5 1.9193831036664843 0.0\n", - "2 5 2.9643952746562157 0.0\n", - "2 5 4.578366521835892 0.0\n", - "2 5 7.071067811865475 0.0\n", - "2 5 10.920925566254217 0.0\n", - "2 5 16.866846478764057 0.0\n", - "2 5 26.050036547934557 0.0\n", - "2 5 40.233033780387636 0.0\n", - "3 0 1.2427598742099695 0.0\n", - "3 0 1.9193831036664843 0.0\n", - "3 0 2.9643952746562157 0.0\n", - "3 0 4.578366521835892 0.0\n", - "3 0 7.071067811865475 0.0\n", - "3 0 10.920925566254217 0.0\n", - "3 0 16.866846478764057 0.0\n", - "3 0 26.050036547934557 0.0\n", - "3 0 40.233033780387636 0.0\n", - "3 1 1.2427598742099695 0.0\n", - "3 1 1.9193831036664843 0.0\n", - "3 1 2.9643952746562157 0.0\n", - "3 1 4.578366521835892 0.0\n", - "3 1 7.071067811865475 0.0\n", - "3 1 10.920925566254217 0.0\n", - "3 1 16.866846478764057 0.0\n", - "3 1 26.050036547934557 0.0\n", - "3 1 40.233033780387636 0.0\n", - "3 2 1.2427598742099695 0.0\n", - "3 2 1.9193831036664843 0.0\n", - "3 2 2.9643952746562157 0.0\n", - "3 2 4.578366521835892 0.0\n", - "3 2 7.071067811865475 0.0\n", - "3 2 10.920925566254217 0.0\n", - "3 2 16.866846478764057 0.0\n", - "3 2 26.050036547934557 0.0\n", - "3 2 40.233033780387636 0.0\n", - "3 3 1.2427598742099695 0.0\n", - "3 3 1.9193831036664843 0.0\n", - "3 3 2.9643952746562157 0.0\n", - "3 3 4.578366521835892 0.0\n", - "3 3 7.071067811865475 0.0\n", - "3 3 10.920925566254217 0.0\n", - "3 3 16.866846478764057 0.0\n", - "3 3 26.050036547934557 0.0\n", - "3 3 40.233033780387636 0.0\n", - "3 4 1.2427598742099695 0.0\n", - "3 4 1.9193831036664843 0.0\n", - "3 4 2.9643952746562157 0.0\n", - "3 4 4.578366521835892 0.0\n", - "3 4 7.071067811865475 0.0\n", - "3 4 10.920925566254217 0.0\n", - "3 4 16.866846478764057 0.0\n", - "3 4 26.050036547934557 0.0\n", - "3 4 40.233033780387636 0.0\n", - "3 5 1.2427598742099695 0.0\n", - "3 5 1.9193831036664843 0.0\n", - "3 5 2.9643952746562157 0.0\n", - "3 5 4.578366521835892 0.0\n", - "3 5 7.071067811865475 0.0\n", - "3 5 10.920925566254217 0.0\n", - "3 5 16.866846478764057 0.0\n", - "3 5 26.050036547934557 0.0\n", - "3 5 40.233033780387636 0.0\n" + "TreeCorr results saved in pickle format to /global/homes/e/edujb/TXPipe/data/example/output_txcluster/inprogress_cluster_profiles.pkl\n" + ] + } + ], + "source": [ + "step3.run()\n", + "step3.finalize()" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "id": "b010bc11-2249-4644-80ff-a45edf416acd", + "metadata": {}, + "outputs": [], + "source": [ + "step3 = txpipe.extensions.cluster_counts.CLClusterSACC.make_stage(\n", + " # This file is the input metadetect shear catalog\n", + " cluster_profiles=f\"{my_txpipe_dir}data/example/output_txcluster/cluster_profiles.pkl\",\n", + " cluster_sacc_catalog=f\"{my_txpipe_dir}data/example/output_txcluster/cluster_sacc_catalog.sacc\",\n", + " clmm_profile = False,\n", + " survey_name = \"cosmodc2-1deg\",\n", + " area= 1,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "id": "e5331d63-64a2-4c3c-b332-67a973eb0756", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "/global/homes/e/edujb/TXPipe/data/example/output_txcluster/cluster_profiles.pkl\n", + "{'bin_zbin_0_richbin_0': {'radial_bins': array([ 0.20906857, 0.41684557, 1.07068396, 2.14654555,\n", + " 4.88312092, 11.29537732, 26.20185608, 45.97618584,\n", + " 131.11109375]), 'xi': array([-0.13572833, -0.10699241, 0.12930437, 0.0290429 , 0.03569181,\n", + " 0.01701415, -0.00080776, -0.00060735, nan]), 'cov': array([[ 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n", + " [ 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n", + " [ 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n", + " [ 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n", + " [ 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n", + " [ 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n", + " [ 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n", + " [ 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n", + " [nan, nan, nan, nan, nan, nan, nan, nan, nan]]), 'cluster_bin_edges': {'rich_min': np.float64(5.0), 'rich_max': np.float64(10.0), 'z_min': np.float64(0.2), 'z_max': np.float64(0.4)}, 'n_cl': 5}, 'bin_zbin_0_richbin_1': {'radial_bins': array([ 0.15254239, 0.43828747, 0.99741776, 2.26642515,\n", + " 5.00364653, 11.42871645, 25.60546092, 44.26547806,\n", + " 131.11109375]), 'xi': array([-0.24215275, -0.02982692, 0.12603616, 0.10593833, 0.0157024 ,\n", + " -0.00810522, -0.00207486, 0.00065837, nan]), 'cov': array([[ 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n", + " [ 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n", + " [ 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n", + " [ 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n", + " [ 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n", + " [ 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n", + " [ 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n", + " [ 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n", + " [nan, nan, nan, nan, nan, nan, nan, nan, nan]]), 'cluster_bin_edges': {'rich_min': np.float64(10.0), 'rich_max': np.float64(100.0), 'z_min': np.float64(0.2), 'z_max': np.float64(0.4)}, 'n_cl': 3}, 'bin_zbin_1_richbin_0': {'radial_bins': array([ 0.18652928, 0.3686499 , 0.95207127, 2.1601598 ,\n", + " 4.99513208, 11.4277249 , 25.98790542, 45.73001665,\n", + " 131.11109375]), 'xi': array([ 0.14287994, -0.02220425, 0.07015278, -0.00461795, 0.00369406,\n", + " -0.00045234, 0.00146861, -0.00517107, nan]), 'cov': array([[ 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n", + " [ 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n", + " [ 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n", + " [ 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n", + " [ 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n", + " [ 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n", + " [ 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n", + " [ 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n", + " [nan, nan, nan, nan, nan, nan, nan, nan, nan]]), 'cluster_bin_edges': {'rich_min': np.float64(5.0), 'rich_max': np.float64(10.0), 'z_min': np.float64(0.4), 'z_max': np.float64(0.6)}, 'n_cl': 20}, 'bin_zbin_1_richbin_1': {'radial_bins': array([ 0.15254239, 0.35508089, 0.91906734, 2.13971618,\n", + " 5.01351593, 11.40113356, 26.35130234, 45.63581018,\n", + " 131.11109375]), 'xi': array([ nan, 0.04262857, -0.03217405, -0.04794953, 0.00886877,\n", + " 0.01494673, 0.00220333, -0.00458338, nan]), 'cov': array([[nan, nan, nan, nan, nan, nan, nan, nan, nan],\n", + " [ 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n", + " [ 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n", + " [ 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n", + " [ 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n", + " [ 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n", + " [ 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n", + " [ 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n", + " [nan, nan, nan, nan, nan, nan, nan, nan, nan]]), 'cluster_bin_edges': {'rich_min': np.float64(10.0), 'rich_max': np.float64(100.0), 'z_min': np.float64(0.4), 'z_max': np.float64(0.6)}, 'n_cl': 8}, 'bin_zbin_2_richbin_0': {'radial_bins': array([ 0.20854347, 0.43758497, 0.89540293, 2.09733128,\n", + " 4.9740448 , 11.54791321, 25.49140962, 44.70897802,\n", + " 131.11109375]), 'xi': array([ 0.30835395, 0.01770827, 0.02918431, -0.01684315, 0.01727465,\n", + " 0.00130012, 0.00186135, -0.00307779, nan]), 'cov': array([[ 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n", + " [ 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n", + " [ 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n", + " [ 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n", + " [ 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n", + " [ 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n", + " [ 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n", + " [ 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n", + " [nan, nan, nan, nan, nan, nan, nan, nan, nan]]), 'cluster_bin_edges': {'rich_min': np.float64(5.0), 'rich_max': np.float64(10.0), 'z_min': np.float64(0.6), 'z_max': np.float64(0.8)}, 'n_cl': 13}, 'bin_zbin_2_richbin_1': {'radial_bins': array([ 0.2184672 , 0.50461408, 1.01674418, 2.17374328,\n", + " 5.05172601, 11.45203962, 25.67861656, 45.2819483 ,\n", + " 131.11109375]), 'xi': array([-0.88255497, 0.63483215, -0.16239469, -0.00937784, -0.00299003,\n", + " 0.01350242, 0.00449895, -0.00935409, nan]), 'cov': array([[ 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n", + " [ 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n", + " [ 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n", + " [ 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n", + " [ 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n", + " [ 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n", + " [ 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n", + " [ 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n", + " [nan, nan, nan, nan, nan, nan, nan, nan, nan]]), 'cluster_bin_edges': {'rich_min': np.float64(10.0), 'rich_max': np.float64(100.0), 'z_min': np.float64(0.6), 'z_max': np.float64(0.8)}, 'n_cl': 8}}\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "NOTE/WARNING: Expected output file /global/homes/e/edujb/TXPipe/data/example/output_txcluster/cluster_sacc_catalog.sacc was not generated.\n" + ] + } + ], + "source": [ + "step3.run()\n", + "step3.finalize()" + ] + }, + { + "cell_type": "code", + "execution_count": 40, + "id": "f246a8b4-cc5f-4570-acaa-cec1fd103841", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[np.float64(-0.1357283272499406), np.float64(-0.10699240631008075), np.float64(0.1293043701108855), np.float64(0.029042904764608652), np.float64(0.03569180957699113), np.float64(0.017014145190160566), np.float64(-0.0008077585967064613), np.float64(-0.0006073541687937928)]\n", + "-0.1357283272499406\n", + "[np.float64(-0.2421527530381858), np.float64(-0.02982692455591973), np.float64(0.12603615820180855), np.float64(0.10593832903492997), np.float64(0.015702400878801637), np.float64(-0.00810522475485229), np.float64(-0.002074864642488125), np.float64(0.000658366288679954)]\n", + "-0.2421527530381858\n", + "[np.float64(0.14287993877274074), np.float64(-0.0222042527317693), np.float64(0.07015278468730535), np.float64(-0.004617951930959872), np.float64(0.00369406353761691), np.float64(-0.00045233728635367637), np.float64(0.0014686131335628135), np.float64(-0.005171071844522354)]\n", + "0.14287993877274074\n", + "[masked, np.float64(0.04262857372655509), np.float64(-0.03217405099279357), np.float64(-0.04794952762451908), np.float64(0.008868765029656136), np.float64(0.014946734762114944), np.float64(0.0022033283913853287), np.float64(-0.004583383379306379)]\n", + "--\n" + ] + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAksAAAGwCAYAAAC5ACFFAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8fJSN1AAAACXBIWXMAAA9hAAAPYQGoP6dpAABsOklEQVR4nO3deVxUVeMG8GfYF2EUERBFxVDUREVMRV+XTDGXSrPQNMg0ytddMpO3xaU36a3c+2mllWamrZaVmrhriooyuUbmBiqEG4OCst7fH6c7zMAwzMAMM8Dz/XzuZ+7cucsZBp2Hc849RyFJkgQiIiIi0svO2gUgIiIismUMS0REREQGMCwRERERGcCwRERERGQAwxIRERGRAQxLRERERAYwLBEREREZ4GDtAtQGxcXFuHbtGjw8PKBQKKxdHCIiIjKCJEm4c+cO/P39YWdXfv0Rw5IZXLt2DQEBAdYuBhEREVVCWloamjZtWu7rDEtm4OHhAUD8sD09Pa1cGiIiIjJGdnY2AgICNN/j5WFYMgO56c3T05NhiYiIqIapqAsNO3gTERERGcCwRERERGQAwxIRERGRAeyzREREZCbFxcXIz8+3djHoH46OjrC3t6/yeWpcWFqxYgXee+89pKen48EHH8SSJUvQq1cvvfump6fj5ZdfxrFjx3Du3DlMnToVS5Ys0dlnzZo1eP7558sce+/ePbi4uFjiLRARUS2Un5+Pixcvori42NpFIS3169eHn59flcZBrFFh6auvvsL06dOxYsUK9OzZEx999BEGDRqEM2fOoFmzZmX2z8vLQ6NGjfDaa69h8eLF5Z7X09MTKSkpOtsYlIiIyFiSJCE9PR329vYICAgwOMAhVQ9JkpCbm4vMzEwAQOPGjSt9rhoVlhYtWoTx48fjhRdeAAAsWbIEv/76K1auXIn4+Pgy+7do0QJLly4FAHz66aflnlehUMDPz88yhSYiolqvsLAQubm58Pf3h5ubm7WLQ/9wdXUFAGRmZsLHx6fSTXI1Jvrm5+fj2LFjiIiI0NkeERGBgwcPVuncd+/eRfPmzdG0aVMMHToUycnJBvfPy8tDdna2zkJERHVXUVERAMDJycnKJaHS5PBaUFBQ6XPUmLB048YNFBUVwdfXV2e7r68vMjIyKn3eNm3aYM2aNdi8eTM2bNgAFxcX9OzZE+fOnSv3mPj4eCiVSs3CqU6IiAioeHBDqn7m+ExqTFiSlX7TkiRV6QfRvXt3PPvss+jYsSN69eqFr7/+Gq1bt8by5cvLPSYuLg5qtVqzpKWlVfr6REREZNtqTJ8lb29v2Nvbl6lFyszMLFPbVBV2dnZ46KGHDNYsOTs7w9nZ2WzXJCIiIttVY2qWnJycEBYWhoSEBJ3tCQkJ6NGjh9muI0kSVCpVlXrNExER1QR9+/bF9OnTy329RYsWZYbcqYtqTFgCgNjYWKxevRqffvopzp49ixkzZiA1NRUTJkwAIJrHoqOjdY5RqVRQqVS4e/curl+/DpVKhTNnzmhenzdvHn799VdcuHABKpUK48ePh0ql0pzTmm7fBi5dEo9ERETV7ejRo3jxxRfNdr7U1FQ89thjcHd3h7e3N6ZOnWrSIJ4nT55Enz594OrqiiZNmmD+/PmQJMls5StPjWmGA4CRI0fi5s2bmD9/PtLT09G+fXts2bIFzZs3ByAGoUxNTdU5JjQ0VLN+7NgxfPnll2jevDkuXboEAMjKysKLL76IjIwMKJVKhIaGYt++fejatWu1va/yzJoFrF4N/Pe/wGuvWbs0RERU1zRq1Mhs5yoqKsKQIUPQqFEjHDhwADdv3sRzzz0HSZIM9hOWZWdnY8CAAXj44Ydx9OhR/Pnnnxg7dizc3d3x8ssvm62ceklUZWq1WgIgqdVqs5733/+WJECS5swx62mJiMjM7t27J505c0a6d++eJEmSVFwsSXfvWmcpLja+3H369JEmTZokTZo0SVIqlZKXl5f02muvScX/nKR58+bS4sWLNfsDkFatWiUNGzZMcnV1lYKCgqQff/zRqGtt2bJFsrOzk65evarZtmHDBsnZ2dmo788VK1ZISqVSun//vmZbfHy85O/vrymvPqU/G23Gfn/XqGa4usbRUTxWYWgIIiKygtxcoF496yy5uaaVde3atXBwcMDhw4exbNkyLF68GKtXry53/3nz5iEyMhInTpzA4MGDMWbMGNy6davC6xw6dAjt27eHv7+/ZtvAgQORl5eHY8eOGXV8nz59dG6wGjhwIK5du6ZpLbIUhiUbxrBERESWFhAQgMWLFyM4OBhjxozBlClTDE4RNnbsWDzzzDMICgrCggULkJOTgyNHjlR4nYyMjDJ3rzdo0ABOTk5GjZeo73j5eVXGWzRGjeqzVNfIYYkTWBMR1SxubsDdu9a7tim6d++uM15heHg4Fi5cqBmVvLQOHTpo1t3d3eHh4aGZf60i+sZFlEwYL1HfWIvlndecGJZsmDxqPmuWiIhqFoUCcHe3diksw1H+S/4fCoUCxcXFFR7n5+eHw4cP62y7ffs2CgoKjBov0c/PT+9YiwDMOt6iPmyGs2FshiMiIktLTEws87xVq1aVnnS2POHh4Th16hTS09M127Zv3w5nZ2eEhYUZdfy+fft0hhrYvn07/P390aJFC7OWtTSGJRvGsERERJaWlpaG2NhYpKSkYMOGDVi+fDmmTZtm9utERESgXbt2iIqKQnJyMnbu3ImZM2ciJiYGnp6eFR4/evRoODs7Y+zYsTh16hQ2bdqEBQsWIDY2ls1wdRnDEhERWVp0dDTu3buHrl27wt7eHlOmTDHrQJQye3t7/PLLL5g4cSJ69uwJV1dXjB49Gu+//75RxyuVSiQkJGDSpEno0qULGjRogNjYWMTGxpq9rKUxLNkwhiUiIrKkPXv2aNZXrlxZ5vXSt+RLekbLzsrKMvp6zZo1w88//2z0/qWFhIRg3759lT6+stgMZ8MYloiIiKyPYcmGVRSWioqAkycBI25CICIisqgJEyagXr16ehdj5lsdNGhQuccvWLCgGt5B+dgMZ8MqCktvvw3MmQOsXQuUmj+YiIioWs2fPx8zZ87U+5oxHbhXr16Ne/fu6X3Ny8urSmWrKoYlG1ZRWDp/XjwePsywRERE1uXj4wMfH59KH9+kSRMzlsa82AxnwyoKS/LosH/9VT3lISIiqosYlmxYRdOdnDolHkuNJ0ZERERmxLBkw4ytWcrOBn79tXrKREREVNcwLNmwiuaGKywsWR8/Hrhzx/JlIiIiqmsYlmxYRTVL2tuvXgVefdXyZSIiIqprGJZsWEVhSbtmCQBWrgR277ZsmYiIqPbo27cvpk+fXu7rLVq0wJIlS6qtPLaKYcmGGTMopaxzZ/H4wgtATo5ly0VERHXD0aNHzTpPXGpqKh577DG4u7vD29sbU6dORX55dzGVcv/+fYwdOxYhISFwcHDAsGHDzFauijAs2TBTwlJQEBAQAFy4APznP5YvGxER1X6NGjWCm5ubWc5VVFSEIUOGICcnBwcOHMDGjRvx3Xff4eWXXzb6eFdXV0ydOhX9+/c3S5mMxbBkw0wJS2lpwKpVYn35cuDAAcuWjYiIyidJopbfGoueuW4NKiwsxOTJk1G/fn00bNgQr7/+umbC3NLNcAqFAqtXr8bw4cPh5uaGVq1aYfPmzUZdZ/v27Thz5gy++OILhIaGon///li4cCFWrVqF7OzsCo93d3fHypUrERMTAz8/P9PeZBUxLNmwisKS9pxwFy4AAwcC48aJfyjjxgG5uZYvIxERlZWbC9SrZ53F1P/7165dCwcHBxw+fBjLli3D4sWLsXr16nL3nzdvHiIjI3HixAkMHjwYY8aMwa1btyq8zqFDh9C+fXv4+/trtg0cOBB5eXk4duyYaYWuZgxLNqyisKT918Pff4u/KBYuBPz9gXPngDfftHwZiYioZgsICMDixYsRHByMMWPGYMqUKVi8eHG5+48dOxbPPPMMgoKCsGDBAuTk5ODIkSMVXicjIwO+vr462xo0aAAnJydkZGRU+X1YEueGs2GmhCUAuHQJePBB4OOPgaFDgcWLgaeeArp3t2gxiYioFDe3koGDrXFtU3Tv3h0KhULzPDw8HAsXLkSRdl8PLR06dNCsu7u7w8PDA5mZmUZdS/s6MkmS9G63JQxLNkw7LEkSoP27pB2gPD3FKN4XLoiwNGQIEBUFrFsHPP88kJwMuLhUb9mJiOoyhQJwd7d2KSzDUf5y+odCoUCxdr+Qcvj5+eHw4cM6227fvo2CgoIyNU62hs1wNkz797H0mErabdLBweLxwoWSbUuWAH5+wB9/APPmWayIRERUwyWWmmA0MTERrVq1gr29vVmvEx4ejlOnTiE9PV2zbfv27XB2dkZYWJhZr2VuDEs2TDsslW6K0w5L7dqJR+2w5OUlBqkEgPfeA5KSLFNGIiKq2dLS0hAbG4uUlBRs2LABy5cvx7Rp08x+nYiICLRr1w5RUVFITk7Gzp07MXPmTMTExMDT09Ooc5w5cwYqlQq3bt2CWq2GSqWCSqUye1lLYzOcDZPnhgPKhiW1umS9bVvxqB2WAGDYMGDUKGDjRnF3XFKS7jmJiIiio6Nx7949dO3aFfb29pgyZYpZB6KU2dvb45dffsHEiRPRs2dPuLq6YvTo0Xj//feNPsfgwYNx+fJlzfPQ0FAA0Ax1YCkMSzbMUM2Sdlhq00Y8lg5LgBhzaedO4ORJ4O232SRHREQl9uzZo1lfKTdHaLl06ZLOc32hJCsry+jrNWvWDD///LPR+1dUnurCZjgbZmcnFqBsWNIev0uuWbp4sewdct7ewP/9n1hfsACohtpKIiKiWoVhycaVN3zAnTsl64GBIlTduyfGWyrt6aeBESNEJ/Fx43QHsyQiIjKHCRMmoF69enqXCRMmVHj8oEGDyj1+wYIF1fAOysdmOBvn6Ajk5ZUflhQKsU+zZmKcpQsXxF1wpf3f/wHbt4thBPbvB/r0sXjRiYioDpk/fz5mzpyp9zVjOnCvXr0a9+7d0/ual5dXlcpWVQxLNq6imiW5mS4wsCQs9ehR9jy+vkBkJPDJJ8DnnzMsERGRefn4+MDHx6fSxzdp0sSMpTGvGtcMt2LFCgQGBsLFxQVhYWHYv39/ufump6dj9OjRCA4Ohp2dHaZPn653v++++w7t2rWDs7Mz2rVrh02bNlmo9KYzNiy1bCke9XXylkVHi8dvvhFNdkRERFSxGhWWvvrqK0yfPh2vvfYakpOT0atXLwwaNAipqal698/Ly0OjRo3w2muvoWPHjnr3OXToEEaOHImoqCj8/vvviIqKQmRkZJlRRq2lvLCUkyMe5THDjAlL//oX0KKFCFo//mjWYhIREdVaNSosLVq0COPHj8cLL7yAtm3bYsmSJQgICNB7uyMAtGjRAkuXLkV0dDSUSqXefZYsWYIBAwYgLi4Obdq0QVxcHB555BEsWbKk3HLk5eUhOztbZ7GU8sKSPOeQKWHJzg549lmx/vnn5isjERFRbVZjwlJ+fj6OHTuGiIgIne0RERE4ePBgpc976NChMuccOHCgwXPGx8dDqVRqloCAgEpfvyIV1Sw5/NPrzJiwBIg54wDR2dvGJ3kmIiKyCTUmLN24cQNFRUVlJtvz9fVFRhW+9TMyMkw+Z1xcHNRqtWZJS0ur9PUrIoel/Hzd7fJ0J6XD0tWrwP375Z+vdWuge3egqAjYsMG8ZSUiIqqNakxYkikUCp3nkiSV2Wbpczo7O8PT01NnsRR5epLy5oaTw1TDhoCHh1jXGgleL7l2ad0685SRiIhqpr59+5Z78xMgurMY6pZSV9SYsOTt7Q17e/syNT6ZmZllaoZM4efnZ/ZzmlN5zXDy3Wzy6wqF8U1xI0eK45KTxTQoRERE+hw9etSs88RNmzYNYWFhcHZ2RqdOnUw+/uTJk+jTpw9cXV3RpEkTzJ8/3+LzwgE1KCw5OTkhLCwMCQkJOtsTEhLQQ9/AQkYKDw8vc87t27dX6ZzmVFFY0p4YNzBQPFYUlho2BIYMEeusXSIiovI0atQIbm5uZjufJEkYN24cRo4cafKx2dnZGDBgAPz9/XH06FEsX74c77//PhYtWmS28pWnxoQlAIiNjcXq1avx6aef4uzZs5gxYwZSU1M1w6jHxcUhWh5M6B8qlQoqlQp3797F9evXoVKpcObMGc3r06ZNw/bt2/G///0Pf/zxB/73v/9hx44dBqslq1N5YUnul6QdloytWQJKxlxav170XyIiIvORJHEjjjUWUytaCgsLMXnyZNSvXx8NGzbE66+/rqmtKd0Mp1AosHr1agwfPhxubm5o1aoVNm/ebPS1li1bhkmTJqGl/IVlgvXr1+P+/ftYs2YN2rdvjyeffBL/+c9/sGjRIovXLtWoEbxHjhyJmzdvYv78+UhPT0f79u2xZcsWNG/eHIAYhLL0mEuhoaGa9WPHjuHLL79E8+bNNTMX9+jRAxs3bsTrr7+ON954Aw888AC++uordOvWrdrelyEV1Sw5O5dsMyUsDR4MeHkB164Bu3YBAwZUvaxERCTk5gL16lnn2nfvAu7uxu+/du1ajB8/HocPH0ZSUhJefPFFNG/eHDExMXr3nzdvHt5991289957WL58OcaMGYPLly9bfEqSQ4cOoU+fPnDW+uIbOHAg4uLicOnSJQTKzSsWUKNqlgBg4sSJuHTpEvLy8nDs2DH07t1b89qaNWuwZ88enf0lSSqzyEFJ9tRTT+GPP/5Afn4+zp49iyeffLIa3olxygtLeXnisbJhydlZ9F0C2BRHRFSXBQQEYPHixQgODsaYMWMwZcoULF68uNz9x44di2eeeQZBQUFYsGABcnJycOTIEYuXs7y71+XXLKlG1SzVRZUNS5IkOn0bEh0NrFwJfPcdsGKF9f4KIiKqbdzcSgYPtsa1TdG9e3edO8DDw8OxcOFCFJXTR6NDhw6adXd3d3h4eCAzM7NSZTWVvrvX9W03N4YlG1deWJLHXXJxKdnWvLkISHfvAjduAI0aGT53t25Aq1bAuXPA99+X9GMiIqKqUShMawqrSRzlL6Z/KBQKFBcXW/y65d29DsDid7DXuGa4uqaisOTqWrLNxQWQJ22+eLHicysUHHOJiKiuS0xMLPO8VatWsJfn07IR4eHh2LdvH/K1Rmnevn07/P390aJFC4tem2HJxpkSlgDjhw+QyXPF7dwJXLlSuTISEVHNlZaWhtjYWKSkpGDDhg1Yvnw5pk2bZpFr/fXXX1CpVMjIyMC9e/c0d6znl56mQo/Ro0fD2dkZY8eOxalTp7Bp0yYsWLAAsbGxbIar68qb7kQOT6XDUsuWwP79xoelwECgVy9xzPr1wKuvVq28RERUs0RHR+PevXvo2rUr7O3tMWXKFLMORKnthRdewN69ezXP5TvWL168WGHtkFKpREJCAiZNmoQuXbqgQYMGiI2NRWxsrEXKqo1hycaVV7NUWCgeS3fkM+WOOFl0tAhLn38OzJpVccdwIiKqHbTvIF+5cmWZ10vfPa5vPKOsrKxKXa8yQkJCsG/fviqdozLYDGfj9M0NJ0nmDUtPPy3uqjtzRkyBQkRERCUYlmycvpolefRuoOzt/pUJS0ol8MQTYp0dvYmIqDImTJiAevXq6V3kmTYMGTRoULnHL1iwoBreQfnYDGfj9IWl3NyS9fJqltLSRD8n7elQDImOBr7+GvjyS+Ddd0uuS0REZIz58+dj5syZel/z9PSs8PjVq1fjnjw9RSmWHh28IgxLNs7UsOTrKzp937sHpKYCQUHGXSciQozLlJkJbN9eMtEuERGRMXx8fODj41Pp45vIY9/YIDbD2Th9YUk7eJeuOVIoSmqXjBlrSfs6o0eLdTbFERERlWBYsnEV1Szpa2YzdawlmTyC9w8/AGq1accSERHVVgxLNq6isKQ9N5ysMp28ASA0FGjXTsw79+23ph1LRERUWzEs2ThTm+GAyoclhaKkdunzz007loiIqLZiWLJxlWmGq2xYAoAxY0Ro2rcPKDUWGRERUZ3EsGTjqrMZDgCaNgX69RPrX3xh+vFERFRz9O3bF9OnTy/39RYtWmDJkiXVVh5bxbBk4/TNDVdRM5zcwTsrC7h92/RrajfF6RnZnoiI6oijR4+adZ64adOmISwsDM7OzujUqZNJx96/fx9jx45FSEgIHBwcMGzYMLOVqyIMSzZO33Qn2jVL//d/QEaG7jFuboCfn1g3ZfgA2ZNPinOcOwccPmz68UREVDs0atQIbqUH9KsCSZIwbtw4jBw50uRji4qK4OrqiqlTp6J///5mK5MxGJZsXEUdvL/+GtAz92GVmuLq1ROBCeCYS0RElSFJQE6OdRZTWwQKCwsxefJk1K9fHw0bNsTrr7+umTC3dDOcQqHA6tWrMXz4cLi5uaFVq1bYvHmz0ddatmwZJk2ahJbyl5QJ3N3dsXLlSsTExMBPrhGoJgxLNq6iPksAoFKVPa6yYy3J5Ka4jRt1mwCJiKhiubniD09rLKW/Iyqydu1aODg44PDhw1i2bBkWL16M1atXl7v/vHnzEBkZiRMnTmDw4MEYM2YMbt26VcWfmG1jWLJxxoSlkyfLHleVmiVAdPL29wdu3QK2bKncOYiIyPYFBARg8eLFCA4OxpgxYzBlyhQsXry43P3Hjh2LZ555BkFBQViwYAFycnJw5MiRaixx9ePccDauomY4QPRLunMH8PAo2VbVsGRvL4YReO890dG7GvvRERHVeG5uwN271ru2Kbp37w6FQqF5Hh4ejoULF6KoqEjv/h06dNCsu7u7w8PDA5mZmZUqa03BsGTj9IWlnJyy+506BYSHlzyvalgCgKgoEZZ+/hm4eRNo2LDy5yIiqksUCsDd3dqlsAxH+YvpHwqFAsXFxVYqTfVgM5yNMzYslW6Kk8PS5ctAYWHlrh0SAnTqJK799deVOwcREdm2xMTEMs9btWoFe3t7K5XI9jAs2Thjw9KJE7rP/f3FsAOFhcCVK5W/Pqc/ISKq3dLS0hAbG4uUlBRs2LABy5cvx7Rp0yxyrb/++gsqlQoZGRm4d+8eVCoVVCoV8o28k+jMmTNQqVS4desW1Gq15nhLYzOcjatszZKdnbgjLiVF9Glq0aJy13/mGWDmTCAxUYy71KpV5c5DRES2KTo6Gvfu3UPXrl1hb2+PKVOmmHUgSm0vvPAC9u7dq3keGhoKALh48SJaGPFFNXjwYFy+fLnM8ZKFR1BmWLJxptQsSZJoJ5e1bCnC0oULwMMPV+76fn7AwIHA1q1izKX58yt3HiIisj179uzRrK/UM2jfpVKThOoLJVlZWZW6XmWULk91YTOcjavobjgvL3HnWlYWcPWq7rFVHWtJJjfFrVsH1PI+fERERGUwLNk4fXPDaY+z5OsLBAeL9fI6eVc1LD3xhBiW4NIl4LffqnYuIiKqnSZMmIB69erpXSZMmFDh8YMGDSr3+AULFlTDOygfm+FsnL654bRrlurXB5o3B86cEU1xgwaVvGausOTqCjz9NPDpp6Kjd69eVTsfERHVPvPnz8fMmTP1vubp6Vnh8atXr8a90gMJ/sPLy6tKZasqhiUbp68Z7v79kvUGDcQt/hs3Wq5mCRBjLn36KfDNN8CyZSJAERERyXx8fODj41Pp45s0aWLG0pgXm+FsnByWJAmQB1MtXbMkD6ZaevgAuc/SjRtAdnbVytG7N9CsGaBWAz/9VLVzERHVVpa+K4tMZ47PpMaFpRUrViAwMBAuLi4ICwvD/v37De6/d+9ehIWFwcXFBS1btsSHH36o8/qaNWugUCjKLPe1q2+sSHugVLl2Sbto9euLmiUA+OMP3b5Nnp6At7dYv3ixauWwsxO1SwDHXCIiKk0ewNHY8YKo+uT+09G39MjjpqhRzXBfffUVpk+fjhUrVqBnz5746KOPMGjQIJw5cwbNmjUrs//FixcxePBgxMTE4IsvvsBvv/2GiRMnolGjRhgxYoRmP09PT6SkpOgc6+LiYvH3Y4zSYcnevqSGCRBhqVkzEYyys8VQAXJ4AkRT3I0bIix17Fi1skRFAW+/DWzbBmRmAlWobSUiqlUcHBzg5uaG69evw9HREXZ2Na4uotaRJAm5ubnIzMxE/fr1qzQieY0KS4sWLcL48ePxwgsvAACWLFmCX3/9FStXrkR8fHyZ/T/88EM0a9YMS5YsAQC0bdsWSUlJeP/993XCkkKhgJ+fX7W8B1OVDkulaxMbNBBjK4WEiDvVTp7UDUuBgcCRI+bptxQcDHTtKs63YQNgoQFeiYhqHIVCgcaNG+PixYs6gyaS9dWvX7/K3/E1Jizl5+fj2LFjmD17ts72iIgIHDx4UO8xhw4dQkREhM62gQMH4pNPPkFBQYGmSu7u3bto3rw5ioqK0KlTJ7z11luaUUH1ycvLQ15enuZ5dlU7BBmgHYQLCnQ7egOiZgnQDUvazNnJGxC1S0eOiKY4hiUiohJOTk5o1aoVm+JsiKOjo1nmuKsxYenGjRsoKiqCr6+vznZfX19kZGToPSYjI0Pv/oWFhbhx4wYaN26MNm3aYM2aNQgJCUF2djaWLl2Knj174vfff0ercub2iI+Px7x588zzxiqgUIjaJTkolReWyuvkbe6wNGoUMGMGcPw4cPo08OCD5jkvEVFtYGdnZzPdOMh8TGpULSwshIODA06dOmWp8lRIoT2fB0SbZOltFe2vvb179+549tln0bFjR/Tq1Qtff/01WrdujeXLl5d7zri4OKjVas2SlpZW2bdjFO3hA7QHpAR0a5YAy9cseXsDQ4aI9XXrzHNOIiIiW2ZSWHJwcNA0V1U3b29v2Nvbl6lFyszMLFN7JPPz89O7v4ODAxo2bKj3GDs7Ozz00EM4d+5cuWVxdnaGp6enzmJJ2mGp9HhdDRqIRzkspaUBt2+XvC6HpYsXzTdViXxX3Pr1up3NiYiIaiOTu+u//vrriIuLw61btyxRnnI5OTkhLCwMCQkJOtsTEhLQo0cPvceEh4eX2X/79u3o0qVLubcQSpIElUqFxo0bm6fgZqA95Ul5NUtKpbgrDgC0K/6aNgUcHMSx166ZpzxDh4rrXrkCVHFORCIiIptnclhatmwZ9u/fD39/fwQHB6Nz5846iyXFxsZi9erV+PTTT3H27FnMmDEDqampmjln4uLiEC3P+goxT83ly5cRGxuLs2fP4tNPP8Unn3yiMxz7vHnz8Ouvv+LChQtQqVQYP348VCqVUfPYVBdjmuGAktol7X5LDg5iOhTAfE1xzs6i7xLAMZeIiKj2M7mD97BhwyxQDOOMHDkSN2/exPz585Geno727dtjy5YtaP5PGkhPT0dqaqpm/8DAQGzZsgUzZszA//3f/8Hf3x/Lli3TGTYgKysLL774IjIyMqBUKhEaGop9+/aha9eu1f7+yiPPDzdrFjBggO5rSmXJeocOwC+/6O+3dP68aIrr3ds8ZYqKAj78EPjuO2DFCsDd3TznJSIisjUKiWOzV1l2djaUSiXUarVF+i+1agX89ZdYnzBBhBRA1Dhp36G6YQMwejTQo4cYRkD20kvAxx8Db74JmOsmPkkS5Tp/XnT0fvZZ85yXiIiouhj7/c0hRmsA7e5VN2+WrDs76+4nDx9w8qTu4JXmviMOEEMayB29eVccERHVZiaHpaKiIrz//vvo2rUr/Pz84OXlpbOQ+WmHpayskvXSQ3m0bi32vXMH0B5A1hJhCSgJSzt2AFevmvfcREREtsLksDRv3jwsWrQIkZGRUKvViI2NxZNPPgk7OzvMnTvXAkUk7bCkVpesu7qW3a9dO7Gu3cnbUmGpZUvgX/8SQxJ8+aV5z01ERGQrTA5L69evx6pVqzBz5kw4ODjgmWeewerVq/Hmm28iMTHREmWs87TDkvbMKm5uZffVNzilHJYyMsreTVdVcu3S55+XnbeOiIioNjA5LGVkZCDkn2/kevXqQf1PVcfQoUPxyy+/mLd0BEA3LN29W7JuKCxp1yw1aFAyxMDFi+Yt29NPi75Tp04Bv/8utkmS6JC+cSMQFwcsWgQkJ5tvUEwiIqLqZPLQAU2bNkV6ejqaNWuGoKAgbN++HZ07d8bRo0fhXLrHMZmFdljKySlZr1ev7L7anby1tWwp5nO7eNG887nVrw888giwZQsQEwN4egLHjuk2F8q8vIA+fYCHHxbLgw+KjuJERES2zOSwNHz4cOzcuRPdunXDtGnT8Mwzz+CTTz5BamoqZsyYYYky1nnaYUl7uhN9YUmuWfrzT+D+/ZJO4HJYqkq/JUkCLl0SYSgpSTweO1YyvUpSUsm+zs5Ax45Ap05iCpb9+4Fbt4BNm8QCAI0alQSnhx8WHdQZnoiIyNaYHJbeeecdzfpTTz2Fpk2b4uDBgwgKCsLjjz9u1sKRoB2W8vJK1j08yu7r7y9qcG7dAs6eBUJDxfbAQPFobFiSJCA1VTcYJSWJ8+ornyQBhYXA1KnA88+LWiPtchcUiHPs3i2WAweA69eBr78WCwA0bqwbnlq2ZHgiIiLrMzkslda9e3d0797dHGWhcmiHDu1O1PrCkkIhapf27hVNcXJYMnRHnCSJed5KB6MbN/SXJSQE6NIFCAsTj+3bAzNnAsuXiwDUqZP+47p3F0tcnAh9R46UhKdDh4D0dHFXnXxnXbNmuuFJnvuOiIioOlUqLK1btw4ffvghLl68iEOHDqF58+ZYsmQJAgMD8cQTT5i7jHWePN2Ji4toWpNpT3WirUMHEZb0DR9w/rwYE0kORHI4yswsex4HBxGEtINRSEjZwTABIDpahKVNm8QdexUNZO7sDPTqJZY33xTNi4mJIjjt2gUcPixqttauFYv8HrTDk7+/4WsQERGZg8lhaeXKlXjzzTcxffp0vP322ygqKgIA1K9fH0uWLGFYsgC5ZsndXTcsaU+iq83Q8AFnzgBNm5Y9xt5eNJ1pB6MOHcoOfFmesDCgbVvR9Pftt8C4ccYdJ3N1LQlB8+eLjuy//VZS85SUJGrFLlwAPvlEHBMcXHJM376Aj49p1yQiIjKGyWFp+fLlWLVqFYYNG6bTf6lLly6YOXOmWQtHghyWSg9CWVFY0q5Zat4c8PUF/v4bsLMTg1dqB6OOHcue3xTy9Cf/+Y+Y/sTUsFSauzsQESEWQNRW7d9fEp6Sk4GUFLHIc+U9+CDQr58IT336iL5bREREVWVyWLp48SJC5Y4wWpydnZGjfV87mY2pYal9e/GYkSH6HXl7i3MkJYm+SSEhIoyY25gxwGuvAXv2iOlWmjc337k9PYEhQ8QCiDvw9u0rabY7eRI4fVosy5eL8NaxY0nNU+/e5TdbEhERGWLyoJSBgYFQqVRltm/duhXt5Lk2yKzKC0sNG+rfv169kmY37aa4pk1FB2tLBCVAdMDu21esr19vmWvIGjQAnngCWLJE1KBlZgLffANMnCiaAyUJUKmAxYuBxx8XtUxduwKvvgps26Y7uCcREZEhJtcsvfLKK5g0aRLu378PSZJw5MgRbNiwAfHx8Vi9erUlyljnyWGpdMdqQzUlHTqI/j0nToialeoSHS1qez7/XNz1Vl23/jdqBDz1lFgAUau2Z4+oddq9W4wofvSoWN59V3Ref+ihkma7Hj2q1gxJRES1l8lh6fnnn0dhYSFmzZqF3NxcjB49Gk2aNMHSpUsxatQoS5SxzisvLBnqfB0SAvzwQ9mRvC1txAhRu5OSIoJJ167Ve32Znx8wapRYANH8KPd32rVLNBMeOiSWt98Wdxx2717SbNe9u/67/oiIqO6p1NABMTExiImJwY0bN1BcXAwf3oZkUXJY0h5vCTD8ZS5Pe6Ldybs6eHgAw4eLsZLWrbNeWCqtaVPRAV2e+PfixZLwtHu3GE5h3z6xzJsngmjPniXh6aGHyv78iYiobqjSoJTe3t7mKgcZIH9J29vrbpfHX9JHviPu9GmgqKjssZYUHS3C0oYNwMKFhstpLYGBYhk3rmTiX7nWafdu0Qdq506xAKKf17/+VRKeOncWTXlERFT7mdzB+++//0ZUVBT8/f3h4OAAe3t7nYXMTw5LdqU+LUM1S0FBonYkN7dq88FVxiOPiKlLbt4UnaltnUIBtGoFvPgisHGj6O90+jTwwQeiWdHLS4z79OuvwOzZQLduonP9Y48BixaJYQyKi639LoiIyFJM/tt47NixSE1NxRtvvIHGjRtDwcm7LK685h9DNTbyIJPHjol+S61aWaZs+jg4AKNHi1qlzz8Xd6PVJAqFGIeqXTtg0iQRhE6eLGmy27sXUKuBn38WCyDuzuvTR9Q69esnfvb8p0FEVDuYHJYOHDiA/fv3o5O+CcDIIioTlgDRFCeHpSefNH+5DImOFmHpp5/EmEgNGlTv9c3Jzk6M2dSxIzB9umjWVKlKmu327xfv8YcfxAKIu/P69hXhqVMn0WeqcWM23RER1UQm/9cdEBAASXs2V7I4ORT9M7OMRkV3a1mrk7d87Q4dxLW//hp46aXqL4Ol2NuLkc/DwsQEwgUFIpTKNU8HDogJhb/5RiwyOztxl17TpuUv/v68C4+IyNaYHJaWLFmC2bNn46OPPkKLFi0sUCQqTa5Zys/X3W5MzRJQ/cMHyKKjRZiYOVPcZTZiBPDoo4Cbm3XKYymOjmKogQ4dgGeeAa5dE/PaHTokfvY3b4pmu+Ji8dq1a8CRI+Wfz8fHcKBq0qT2/QyJiGyZUWGpQYMGOn2TcnJy8MADD8DNzQ2OpdqIbt26Zd4SUrlhqaJb2eWw9NdfooOypUbuLs/zz4s+SydOiLvjvvxSfMkPGiSC05AhYhoTWyRJ4md2/bpYMjNL1svblptr2jUcHETgtbMTtYZ5eSJQZWaK5fjx8o/18jIcqJo2FcM4EBFR1RkVlpYsWWLhYpAhcijKyyvZZmdXcQdiX19RS5GZCZw5I8YKqk5eXuJOscOHge++E8ulSyXrTk5iotwRI0qmJLEUSQLu3DEcdkpvu3/f9Os4O4v+StpLYaGYo09erl8XwbewUCyVceuWWAw1sdarBwQEiKVJE/2BqkEDdkQnIqqIQmIHpCrLzs6GUqmEWq2GpwWqSr76SoxE3a6dCD2AqJUoKKj42P79xVhBq1cD48ebvWgmkSQRnuSwlJJS8pqDg+gMPWIEMGyYCHoVnUutLht2ygtB16/rhk1jubiIwFk6AJW3rV69isOHXGulHaCMWUr3WTMHR0cx0bKfnwhVgYFi2IlmzUoClbd32WEriIhqA2O/v03us3T8+HE4Ojoi5J82nh9//BGfffYZ2rVrh7lz58LJFkcgrOHkmiXtcGTsl1eHDiIsWavfkjaFQgzm2Lkz8N//iuD37bciOJ08CSQkiOXf/xaT4YaEiC/t/Hz9AciYsFiam5vhsFN6m7u7+WteFAoRqurVA4zt9ldcDGRnGx+srl8Xd+hV9KdQQQGQni6W5OTyy+vuDtSvL4JT48Ylwap1axHig4J4px8R1V4m//f20ksvYfbs2QgJCcGFCxcwcuRIPPnkk/jmm2+Qm5vLJjsLqEpYsnYn7/R00WeqoiYvB4eSJilJEkFKrkUzpF4942p85PWa2jHazk6Elfr1RTAxRlGRCEzlBaq//xbTvGRkiE7o2dn6mx4lCbh7VyxXrohhE8oro7Oz+Ez0BaugIFGD5e0tFjc3NgESUc1gclj6888/NWMsffPNN+jTpw++/PJL/Pbbbxg1ahTDkgXo6+Bt7JeM9vABklS9X07JyeIusdId0w3x9CwZkyk7W3zZa3vgAdHP6emnxbldXc1X3trG3r4kmBgrP1/0hZIDVUYGcP686Gt29aoIWHKwys0V+8u1V8XFwL17Yrl+HTh3ruLylQ5WTZvqBqrSC4dVICJrMDksSZKE4n/mdtixYweGDh0KQIy/dOPGDfOWjgCUhCXtu62MDT3t2om/+OWaBD8/85dPH0kCpkwRX6Z+fkDLlsY1e5X+MkxPBzZtEk11e/aIL+6VK8XSrp3o4zRihAiFrKWoOicn8XmZ8nty9y7wxx9iOXcOuHxZ1ECVDlalmwSLikS/M7VaHGMMuRm1UaPyA5X24uXFCZCJqOpM7uDdr18/BAQEoH///hg/fjzOnDmDoKAg7N27F8899xwuXbpkoaLaLkt38N63T0ylod1U5eIi/oI3RnAw8OefYm6ziAizF0+vjRvFmENubqIjd9OmVT/njRvAjz+K4LRjh26z5AMPlASnhx5icLI1xcXi87tyBUhLE6H3r790g9WtW5Xrh1YRuebK2KVBA3ZoJ6orjP3+NjksnThxAmPGjEFqaipiY2MxZ84cAMCUKVNw8+ZNfPnll1UreQ1k6bB06BDQo4fuNicn4+/uevpp0ZH6/feBl182e/HKyMkB2rQRX4JvvQW8/rr5r5GVJeZl+/ZbEQK1+9oEBIjpXUaMED83zu9cM0iSaHa9cqXskpYGpKaKdVPHszKVnZ2okTIlYHl6MqAT1UQWC0vluX//Puzt7csMUlkXWDosHTsGdOmiu00eyNAY8+cDc+aIEbXXrjV78cp4800Rkry9xfhJRUWAUim+UCp69PAw/a/6u3eBLVtEjdMvv4iwJvP1BYYPF8Gpb1/esVUbZGfrBqmrV8uGK2PHxrWzE810CoX4Pa1szZaDg2nhih3ciWxDtYel6rJixQq89957SE9Px4MPPoglS5agV69e5e6/d+9exMbG4vTp0/D398esWbMwYcIEnX2+++47vPHGGzh//jweeOABvP322xg+fLjRZbJ0WDpxQkziWlpurnEdnH/4QQSG0FDDo0JXxdWrok/R5s1iPrSq/FZ5eJSEJ2MClvajoyOQlCQm8P3pJ9EfRublBTzxhAhO/fuzs3BtlpurP0RpL5mZxp3Lzk78Trq5iRpdhUI0h+fliaBubHN4aS4uFTcHuriIxdm5ZL285w4ODF9EpqqVYemrr75CVFQUVqxYgZ49e+Kjjz7C6tWrcebMGTRr1qzM/hcvXkT79u0RExODl156Cb/99hsmTpyIDRs2YMSIEQCAQ4cOoVevXnjrrbcwfPhwbNq0CW+++SYOHDiAbt26GVUuS4els2dFZ+bS0tKM6wt0/ry4bdvZWfznbo7alfR0EY527xaP+u586tJFDDTZsKEILdnZZR/ldbXavP1V7O1FgHJyEufNztYdLdvJSYzl1LWrWHx8dIOXvM5hw2yDJImaH3Mu9+6JwPT33yVTzFy/LvpW3bwpltJ3Yxri5ib+eCkdXIqLxe9efr5oLr5/3zIDjCoU4vfV0VE8yoFKO1i5uorFzU2MnWVMCDMlsMm1dETmJP/7VyjM362iVoalbt26oXPnzli5cqVmW9u2bTFs2DDEx8eX2f/VV1/F5s2bcfbsWc22CRMm4Pfff8ehQ4cAACNHjkR2dja2bt2q2efRRx9FgwYNsGHDBqPKZemw9NdfQKtWZbcnJwP/jOJgUHGx+PLPyRFjF7Vta3oZMjJEKJIX7dG3AfHXd1CQ6EhuZwccOACEh5t2jfv39Qeq0sGqon3+uVnTLJydTavZ0lcbZm9v/i96Sy+FhdYvg/ZSc/6XqtsUipKw5ugo/v04O4vnrq4lgc3FRQQ2eTE2sBkT4OTaP1tQXFy1f2Pm2Ke6rmPJ8sr//letAl54wbyfkcVG8LaW/Px8HDt2DLNnz9bZHhERgYMHD+o95tChQ4godfvXwIED8cknn6CgoACOjo44dOgQZsyYUWYfQ+NF5eXlIU+rd3V2draJ78Y05XUDu37duOPt7ID27cUcbSdPGheWMjNLgtHu3eK2cG0KhWjW69tX1B517w706ydemzDB9KAElPyH5+Nj+rEyeSoRQ4EqK0uEvVOngIsXy3YYlsMNIJpa5FoHsl12duJzM/fi4FDxPnZ2JbVGubmixionRyzyYJ537pT8TlXE3V00wdWrJ36fi4vLX4qKdB+117WPlaSSxZIkSfwsTBlbzZK0Q5Mx68bQ/hnK66UfyTJ++838YclYNSYs3bhxA0VFRfAtNWmYr68vMjIy9B6TkZGhd//CwkLcuHEDjRs3Lnef8s4JAPHx8Zg3b14l34npygtLpgxrFRIiwtKJE0BkZNnXr18H9u4tCUelR89WKES/KTkc9epVMngkIMY9OnlSbJs/3/hymZv2VCL+/hXvL0miH5c8X92ff5Z8qdnbi7vpHn5YNCna25tWy6VW679j0VJf7Ob68q9pizGTSle3oiIRmrTDU3p6yR19166J2trMTPHv+PZtEeILCkqCFlWdvmBDNVfpG52qk1FhKTY21ugTLlq0qNKFMYai1P+KkiSV2VbR/qW3m3rOuLg4nZ9JdnY2AgICKi58JZkjLMkjecvTnty4IcZvksPRqVP6j5HDUe/eooO0PrdulQwP8NZboo9STaFQAGFhYnn7beD0aRGavv1W/Ez27xeLnZ0IiCNGiGEJmjQx7vz5+eKL05a/2OuKwsKSEcblEFP60dBrFT1qr9tKzYolKBTi91h7kX+3DS3yPvb24hzai8zOruxMA6UDT+lFu/ZMu+lGu6ZNuwZO3k8+pjbT/iPJ0VF3kZss5Uft5szS/dC09y3dvCov2s+1X5fvbtb3/17pbRXtY+z/u5ZgVFhKLjXD5rFjx1BUVITg4GAAYgoUe3t7hIWFmb+E//D29oa9vX2ZGp/MzMwyNUMyPz8/vfs7ODig4T/f6OXtU945AcDZ2RnO1XgrlblqlgBRe9Sxo6hhKq19e91wZOw0GXPmiMDUvj3w0kvGl8nWKBTiPbRvL97Tn3+W1DgdOyZ+dnv3AlOnimZHeRDMwMDyz8kO4obJAcZSoUX70RIDXhpD7p8jd6zW96hvm9xRvDK1g+baR99+tSnsFxeL2t+8PNGMWvpR37by9pF/93JydH/3Sp9HbqbMzxe/kwUF5u1nqU0OitYM7/b2uiFNO1jJfc60+7Np92VzdxeP8n79+pX/R7ulGRWWdu/erVlftGgRPDw8sHbtWjT4px3m9u3beP755w3ewl9VTk5OCAsLQ0JCgs5t/QkJCXjiiSf0HhMeHo6ffvpJZ9v27dvRpUsXzXhQ4eHhSEhI0Om3tH37dvQoPQqkFZkrLCkUomlIDkrt2umGo8r0FTp1SjTBAcDSpbVrHKPWrYG4OLFcugR8/70ITgcPAomJYnnlFdF3Sw5ObdpYu9RVV1ho2dCi/WitACMHFEMBxthHQ6+5uHA0cFtmZ1fyGVpTUZH+0GZKWJPXc3N1F/nfovy63L+udGArKLBMTZsc2PRN0m2q6horUB+T74Zr0qQJtm/fjgcffFBn+6lTpxAREYFr166ZtYDa5KEDPvzwQ4SHh+Pjjz/GqlWrcPr0aTRv3hxxcXG4evUqPv/8cwAlQwe89NJLiImJwaFDhzBhwgSdoQMOHjyI3r174+2338YTTzyBH3/8Ea+//rpNDR2Qn69/TKDISOCrr4w/z8cfi2a4Xr3E9CkGKs+MIknAgAHAzp2iaeq776p2vpri2rWS+er27tX9q7BdO+Cpp0RwkgOqORQUVD68mHqM9hAL1cnU8FLZgOPszABDpI88flhlw5q+Gjb5UTuwyWFNvpZ2YCssLD+0vfyymInCnCw2dICHhwd+/PFH9JNvffrHrl278MQTT+DOnTuVK7GRVqxYgXfffRfp6elo3749Fi9ejN69ewMAxo4di0uXLmHPnj2a/ffu3YsZM2ZoBqV89dVXywxK+e233+L111/HhQsXNINSPvnkk0aXydJhSZL0/+fer58IKtayaZMISc7OYiwoQ81RtdX16yXz1e3cqVtTEhQkQlOrVpWvnZEfrRlgzBVSDJ3DxaV2Ne8QUeUVFuoPZH5+5m+Gs1hYio6Oxt69e7Fw4UJ0794dAJCYmIhXXnkFvXv3xlpr1ZFZkaXDEiCat0rfetyhA/D77xa5XIXu3xe1KBcvAq+9Bvz3v9Yphy3JyhKjhn/3HbBtm/Fz95nKUk1G+pqQGGCIqDazWFjKzc3FzJkz8emnn6Lgnz+jHRwcMH78eLz33ntwd3evWslroOoIS25uZadV8PcXUzpYw4IFIiQ1aSLGLKqDH7tBd+4AW7eK6V/UavPUxshNSAwwRETmYfERvHNycnD+/HlIkoSgoKA6GZJk1RGWPD3FF7A2JydRw1PdX55XrwLBwaItev16YPTo6r0+ERGRORj7/V3pbo7p6elIT09H69at4e7ujho0a0qNpO8us/x8MTpwdXv1VRGUevQAnnmm+q9PRERUnUwOSzdv3sQjjzyC1q1bY/DgwUhPTwcAvPDCC3j55ZfNXkAS7O11n8vDCRg75Ym5HDwoapMUCmDZMjYJERFR7WdyWJoxYwYcHR2RmpoKNzc3zfaRI0di27ZtZi0clZDDkhxO5NpCU8ZaqqriYmDaNLE+bpwY9ZqIiKi2M3kIwe3bt+PXX39F06ZNdba3atUKly9fNlvBSJd2WJIkwMMDuHmzesPSmjVAUpIIam+/XX3XJSIisiaTa5ZycnJ0apRkN27cqNYpQOqa0vPrKJXisbrCUna2GMkaAN58s+oDWhIREdUUJoel3r17a0bIBsQktMXFxXjvvffw8MMPm7VwVKJ0WKpfXzxWV1h66y0xQ3rr1sCUKdVzTSIiIltgcjPce++9h759+yIpKQn5+fmYNWsWTp8+jVu3buG3336zRBkJ1g1Lf/4p5n0DgMWLOTksERHVLSbXLLVr1w4nTpxA165dMWDAAOTk5ODJJ59EcnIyHnjgAUuUkVD2rjN5yPfqCEuxsWIaj8GDxUJERFSXVGqOeD8/P8ybN8/cZSED5LAkPzZsKB4tHZa2bgV++UWM87R4sWWvRUREZItMrln67LPP8M0335TZ/s0339TJeeGspTrCUn4+MH26WJ82TfRXIiIiqmtMDkvvvPMOvL29y2z38fHBggULzFIoKqtlS/Eo1yzJH4Elw9IHH4j+Sj4+wBtvWO46REREtszksHT58mUEBgaW2d68eXOkpqaapVBUVmio7nMfH/FoqRG8//4bkFta4+NLhiogIiKqa0wOSz4+Pjhx4kSZ7b///jsaym1DZHa5ueKxqEg8+vmJx1u3SraZ02uvibGVwsKAsWPNf34iIqKawuSwNGrUKEydOhW7d+9GUVERioqKsGvXLkybNg2jRo2yRBkJZcOSPChkcTGQlWXeax07Bnz6qVhftqxk2AIiIqK6yOS74f773//i8uXLeOSRR+DgIA4vLi5GdHQ0+yxZ0L174lGSxKO7u2gaU6tFvyVzVepJEjB1qngcMwbo0cM85yUiIqqpTA5LTk5O+Oqrr/DWW2/h999/h6urK0JCQtC8eXNLlI/+IdcsyZycRCdvOSwFB5vnOhs2AAcPAm5uwDvvmOecRERENVmlxlkCgNatW6M17yWvNnLNkszZWYSl8+fNd0dcTg4wa5ZY/89/gFJzJRMREdVJJoelcePGGXz9U7mzC5lVQYHuc0dH8w8f8M47wNWrQIsWwMsvm+ecRERENZ3JYen27ds6zwsKCnDq1ClkZWWhX79+ZisY6SosLFl3cBCdruXhA7ZuBcaNKzsliikuXgTee0+sL1wIuLhU/lxERES1iclhadOmTWW2FRcXY+LEiWgpj5xIZqcdlpydxeO4ccDnnwPffQe8/Tbw+uuVP//MmUBeHtCvHzB8eNXKSkREVJuY5aZwOzs7zJgxA4s5eZjFaIclJyfx+K9/iVG2ATHCtp5ZaIyyaxfw/feitmrp0qrVUBEREdU2ZhtB5/z58yjU/kYns9IXlgBgwgQxbxsAPPcccPSo6eeVj//3v4H27atWTiIiotrG5Ga42NhYneeSJCE9PR2//PILnnvuObMVjHTpa4aTLVwo5nDbuhV44gngyBHj72T76CPg1CnAywuYP9985SUiIqotTA5LycnJOs/t7OzQqFEjLFy4sMI75ajyyqtZAgB7e2DjRjGA5OnTwOOPA/v3i4ErDbl5s2SC3LfeEoGJiIiIdJkclnbv3m2JclAFDNUsAYCnJ/Dzz0DXrkByMvDss6Ljt6GpSubMAW7fBkJCgBdfNH+ZiYiIagPO+lVDGKpZkrVoAWzaJF7/4QcxGW55Tp4EVq4U60uXiuEIiIiIqCyjviJDQ0OhMPIWqePHj1epQKSfMWEJAHr2BD75BIiKEoNMBgcDY8fq7iNJolN3cTEwYgTw8MMWKTIREVGtYFRYGjZsmGb9/v37WLFiBdq1a4fw8HAAQGJiIk6fPo2JEydapJBUcTOctmefBf74Q4y99OKLwAMPAL16lby+aROwe7c4z/vvW6a8REREtYVRYWnOnDma9RdeeAFTp07FW2+9VWaftLQ085aONIytWZLNnw+kpADffisGmTxyBGjZUswxJ09l8soroumOiIiIymdyn6VvvvkG0dHRZbY/++yz+O6778xSKCrL1LBkZwesXQuEhYm73oYOBdRqMczApUtiaIHZsy1WXCIiolrD5LDk6uqKAwcOlNl+4MABuHBCMYspKipZr6gZTubmBmzeDDRpApw9K4YUiI8Xr737bsVDCxAREVElwtL06dPx73//G5MnT8YXX3yBL774ApMnT8akSZMwY8YMS5QRgJjANyoqCkqlEkqlElFRUcjKyjJ4jCRJmDt3Lvz9/eHq6oq+ffvi9OnTOvv07dsXCoVCZxk1apTF3kdlmVqzJPP3F4HJzQ3Ytw/IzRWdwG3wLRIREdkkk28Ynz17Nlq2bImlS5fiyy+/BAC0bdsWa9asQWRkpNkLKBs9ejSuXLmCbdu2AQBefPFFREVF4aeffir3mHfffReLFi3CmjVr0Lp1a/z3v//FgAEDkJKSAg8PD81+MTExmK81fLWrq6vF3kdlVTYsAUDnzsC6deLON4WC878RERGZolKj60RGRlo0GJV29uxZbNu2DYmJiejWrRsAYNWqVQgPD0dKSgqCg4PLHCNJEpYsWYLXXnsNTz75JABg7dq18PX1xZdffomXXnpJs6+bmxv8/PyMLk9eXh7y8vI0z7Ozsyv71oxmyt1w+jz5JPDrr2I9LMw8ZSIiIqoLKjUoZVZWFlavXo3//Oc/uHXrFgAxvtLVq1fNWjjZoUOHoFQqNUEJALp37w6lUomDBw/qPebixYvIyMhARESEZpuzszP69OlT5pj169fD29sbDz74IGbOnIk7d+4YLE98fLymOVCpVCIgIKAK765ixcVibCSZqTVLsogIsRAREZHxTK5ZOnHiBPr37w+lUolLly7hhRdegJeXFzZt2oTLly/j888/N3shMzIy4OPjU2a7j48PMjIyyj0GAHx9fXW2+/r64vLly5rnY8aMQWBgIPz8/HDq1CnExcXh999/R0JCQrnliYuL05lQODs726KBSbtWCahczRIRERFVjsk1S7GxsRg7dizOnTunc/fboEGDsG/fPpPONXfu3DKdq0svSUlJAKB3BHFJkiocWbz066WPiYmJQf/+/dG+fXuMGjUK3377LXbs2GFwJHJnZ2d4enrqLJZUOixVtmaJiIiITGdyzdLRo0fx0UcfldnepEmTcmt5yjN58uQK7zxr0aIFTpw4gb///rvMa9evXy9TcyST+yBlZGSgcePGmu2ZmZnlHgMAnTt3hqOjI86dO4fOnTsb8zYsjmGJiIjIekwOSy4uLno7NKekpKBRo0Ymncvb2xve3t4V7hceHg61Wo0jR46ga9euAIDDhw9DrVajR48eeo+Rm9YSEhIQGhoKAMjPz8fevXvxv//9r9xrnT59GgUFBToBy9rYDEdERGQ9JjfDPfHEE5g/fz4KCgoAiGau1NRUzJ49GyNGjDB7AQExNMGjjz6KmJgYJCYmIjExETExMRg6dKjOnXBt2rTBpk2bNOWaPn06FixYgE2bNuHUqVMYO3Ys3NzcMHr0aADA+fPnMX/+fCQlJeHSpUvYsmULnn76aYSGhqJnz54WeS+VwZolIiIi6zE5LL3//vu4fv06fHx8cO/ePfTp0wdBQUHw8PDA22+/bYkyAhB3rIWEhCAiIgIRERHo0KED1q1bp7NPSkoK1Gq15vmsWbMwffp0TJw4EV26dMHVq1exfft2zRhLTk5O2LlzJwYOHIjg4GBMnToVERER2LFjB+zt7S32XkwlhyW5qxXDEhERUfVRSJL2TenG27VrF44fP47i4mJ07twZ/fv3N3fZaozs7GwolUqo1WqLdPZOTQWaNxfzvRUXAytXAhMmmP0yREREdYqx398m9VkqLCyEi4sLVCoV+vXrh379+lW5oFQx1iwRERFZj0nNcA4ODmjevDmKtGd1JYsrHZbYwZuIiKj6mNxn6fXXX0dcXJxm5G6yPNYsERERWY/JQwcsW7YMf/31F/z9/dG8eXO4u7vrvG5oMEeqHN4NR0REZD0mh6Vhw4ZZoBhkCMdZIiIish6Tw9KcOXMsUQ4ygDVLRERE1mNynyWqfgxLRERE1sOwVAPIYUkeEYvNcERERNWHYakGYM0SERGR9TAs1QCla5YYloiIiKoPw1INII8BymY4IiKi6mfU3XCxsbFGn3DRokWVLgzpx5olIiIi6zEqLCUnJxt1MoU8xDSZFTt4ExERWY9RYWn37t2WLgcZwA7eRERE1sM+SzUAwxIREZH1mDyCNwAcPXoU33zzDVJTU5Gfn6/z2vfff2+WglEJhiUiIiLrMblmaePGjejZsyfOnDmDTZs2oaCgAGfOnMGuXbugVCotUcY6Tzss2duLhYiIiKqHyWFpwYIFWLx4MX7++Wc4OTlh6dKlOHv2LCIjI9GsWTNLlLHO0w5LrFUiIiKqXiaHpfPnz2PIkCEAAGdnZ+Tk5EChUGDGjBn4+OOPzV5A0g1LvBOOiIioepkclry8vHDnzh0AQJMmTXDq1CkAQFZWFnJzc81bOgLAmiUiIiJrMrmDd69evZCQkICQkBBERkZi2rRp2LVrFxISEvDII49Yoox1HsMSERGR9Zgclj744APcv38fABAXFwdHR0ccOHAATz75JN544w2zF5DYDEdERGRNJoclLy8vzbqdnR1mzZqFWbNmmbVQpIs1S0RERNZjVFjKzs6Gp6enZt0QeT8yH9YsERERWY9RYalBgwZIT0+Hj48P6tevr3cOOEmSoFAoUFRUZPZC1nWsWSIiIrIeo8LSrl27NM1vnCeu+jEsERERWY9RYalPnz6a9cDAQAQEBJSpXZIkCWlpaeYtHQFgMxwREZE1mTzOUmBgIK5fv15m+61btxAYGGiWQpEu1iwRERFZj8lhSe6bVNrdu3fh4uJilkKRLoYlIiIi6zF66IDY2FgAgEKhwBtvvAE3NzfNa0VFRTh8+DA6depk9gISm+GIiIisyeiwlJycDEDULJ08eRJOWlUcTk5O6NixI2bOnGn+EhJrloiIiKzI6LAk3wX3/PPPY+nSpRxPqRoxLBEREVmPyX2WPvvsM6sEpdu3byMqKgpKpRJKpRJRUVHIysoyeMz333+PgQMHwtvbGwqFAiqVqsw+eXl5mDJlCry9veHu7o7HH38cV65cscybqCTtoavYDEdERFS9TA5LOTk5eOONN9CjRw8EBQWhZcuWOouljB49GiqVCtu2bcO2bdugUqkQFRVVYVl79uyJd955p9x9pk+fjk2bNmHjxo04cOAA7t69i6FDh9rU4JqsWSIiIrIek+eGe+GFF7B3715ERUWhcePGeu+MM7ezZ89i27ZtSExMRLdu3QAAq1atQnh4OFJSUhAcHKz3ODlMXbp0Se/rarUan3zyCdatW4f+/fsDAL744gsEBARgx44dGDhwoPnfTCWwgzcREZH1mByWtm7dil9++QU9e/a0RHn0OnToEJRKpSYoAUD37t2hVCpx8ODBcsNSRY4dO4aCggJERERotvn7+6N9+/Y4ePBguWEpLy8PeXl5mucVzZdXVaxZIiIish6Tm+EaNGigmfqkumRkZMDHx6fMdh8fH2RkZFTpvE5OTmjQoIHOdl9fX4PnjY+P1/SdUiqVCAgIqHQZjMGwREREZD0mh6W33noLb775JnJzc6t88blz50KhUBhckpKSAMDg5L3mVtF54+LioFarNYulp3lhMxwREZH1mNwMt3DhQpw/fx6+vr5o0aIFHB0ddV4/fvy40eeaPHkyRo0aZXCfFi1a4MSJE/j777/LvHb9+nX4+voafb3S/Pz8kJ+fj9u3b+vULmVmZqJHjx7lHufs7AznakwtrFkiIiKyHpPD0rBhw8x2cW9vb3h7e1e4X3h4ONRqNY4cOYKuXbsCAA4fPgy1Wm0w1FQkLCwMjo6OSEhIQGRkJAAgPT0dp06dwrvvvlvp85obwxIREZH1mByW5syZY4lyGNS2bVs8+uijiImJwUcffQQAePHFFzF06FCdzt1t2rRBfHw8hg8fDkBM7puamopr164BAFJSUgCIGiU/Pz8olUqMHz8eL7/8Mho2bAgvLy/MnDkTISEhmrvjbAGb4YiIiKzH5D5LAJCVlYXVq1cjLi4Ot27dAiCa365evWrWwmlbv349QkJCEBERgYiICHTo0AHr1q3T2SclJQVqtVrzfPPmzQgNDcWQIUMAAKNGjUJoaCg+/PBDzT6LFy/GsGHDEBkZiZ49e8LNzQ0//fQT7O3tLfZeTMWaJSIiIutRSJIkmXLAiRMn0L9/fyiVSly6dAkpKSlo2bIl3njjDVy+fBmff/65pcpqs7Kzs6FUKqFWqy0yunn37sDhw2J982bgscfMfgkiIqI6x9jvb5NrlmJjYzF27FicO3cOLi4umu2DBg3Cvn37KldaMki7ZsnCoxQQERFRKSaHpaNHj+Kll14qs71JkyZVGvOIypefX7LerJn1ykFERFQXmRyWXFxc9I5YnZKSgkaNGpmlUKTr/n3x6OwMlBo/k4iIiCzM5LD0xBNPYP78+SgoKAAgBotMTU3F7NmzMWLECLMXkErCko8PUA1T8REREZEWk8PS+++/j+vXr8PHxwf37t1Dnz59EBQUBA8PD7z99tuWKGOdJ09DV4XxN4mIiKiSTB5nydPTEwcOHMCuXbtw/PhxFBcXo3PnzjY1LlFtI/dZatzYuuUgIiKqi0wOS7J+/fqhX79+5iwLleOfFk+GJSIiIiswOSwtW7ZM73aFQgEXFxcEBQWhd+/eNjWoY03HsERERGQ9JoelxYsX4/r168jNzUWDBg0gSRKysrLg5uaGevXqITMzEy1btsTu3bsRwEGBzKKoSDw2aWLdchAREdVFJnfwXrBgAR566CGcO3cON2/exK1bt/Dnn3+iW7duWLp0KVJTU+Hn54cZM2ZYorx1khyWmja1bjmIiIjqIpOnO3nggQfw3XffoVOnTjrbk5OTMWLECFy4cAEHDx7EiBEjkJ6ebs6y2ixLTneiVgP164v1kyeB9u3NenoiIqI6y2LTnaSnp6NQe/6NfxQWFmpG8Pb398edO3dMPTXpkZZWsm6BaeeIiIioAiaHpYcffhgvvfQSkpOTNduSk5Px73//W3N33MmTJxEYGGi+UtZh2mHJodL3LhIREVFlmRyWPvnkE3h5eSEsLAzOzs5wdnZGly5d4OXlhU8++QQAUK9ePSxcuNDsha2LUlNL1hmWiIiIqp/JX79+fn5ISEjAH3/8gT///BOSJKFNmzYIDg7W7PPwww+btZB1GcMSERGRdVX667dNmzZo06aNOctCejAsERERWZfJX79FRUVYs2YNdu7ciczMTBQXF+u8vmvXLrMVjhiWiIiIrM3kr99p06ZhzZo1GDJkCNq3bw+FQmGJctE/2MGbiIjIukz++t24cSO+/vprDB482BLlIS2SBFy7VvKcYYmIiKj6mXw3nJOTE4KCgixRFirlxg0gL0+sKxSAncmfFhEREVWVyV+/L7/8MpYuXQoTB/6mSmATHBERkfWZ/BV84MAB7N69G1u3bsWDDz4IR0dHnde///57sxWurmPnbiIiIusz+Su4fv36GD58uCXKQqWwZomIiMj6TP4K/uyzzyxRDtKDYYmIiMj62GXYhl25UrLOsERERGQdlfoK/vbbb/H1118jNTUV+fn5Oq8dP37cLAUjYN064PnngYgIhiUiIiJrMblmadmyZXj++efh4+OD5ORkdO3aFQ0bNsSFCxcwaNAgS5SxzrK3B7y8xDrDEhERkXWYHJZWrFiBjz/+GB988AGcnJwwa9YsJCQkYOrUqVCr1ZYoY51WWCgeGZaIiIisw+SwlJqaih49egAAXF1dcefOHQBAVFQUNmzYYN7SEcMSERGRlZkclvz8/HDz5k0AQPPmzZGYmAgAuHjxIgeqtAA5LNnbW7ccREREdZXJYalfv3746aefAADjx4/HjBkzMGDAAIwcOZLjL1kAa5aIiIisy+Sv4I8//hjFxcUAgAkTJsDLywsHDhzAY489hgkTJpi9gHUdwxIREZF1mfwVfOXKFQQEBGieR0ZGIjIyEpIkIS0tDc2aNTNrAes6hiUiIiLrMrkZLjAwENevXy+z/datWwgMDDRLofS5ffs2oqKioFQqoVQqERUVhaysLIPHfP/99xg4cCC8vb2hUCigUqnK7NO3b18oFAqdZdSoUZZ5E5XAsERERGRdJoclSZKgUCjKbL979y5cXFzMUih9Ro8eDZVKhW3btmHbtm1QqVSIiooyeExOTg569uyJd955x+B+MTExSE9P1ywfffSROYteJUVF4pFhiYiIyDqM/gqOjY0FACgUCrzxxhtwc3PTvFZUVITDhw+jU6dOZi8gAJw9exbbtm1DYmIiunXrBgBYtWoVwsPDkZKSguDgYL3HyWHq0qVLBs/v5uYGPz8/s5bZXFizREREZF1GfwUnJycDEDVLJ0+ehJOTk+Y1JycndOzYETNnzjR/CQEcOnQISqVSE5QAoHv37lAqlTh48GC5YclY69evxxdffAFfX18MGjQIc+bMgYeHR7n75+XlIS8vT/M8Ozu7Stc3hGGJiIjIuoz+Ct69ezcA4Pnnn8fSpUvh6elpsUKVlpGRAR8fnzLbfXx8kJGRUaVzjxkzBoGBgfDz88OpU6cQFxeH33//HQkJCeUeEx8fj3nz5lXpusZiWCIiIrIuk/ssffbZZ2YLSnPnzi3Tubr0kpSUBAB6+0mV13/KFDExMejfvz/at2+PUaNG4dtvv8WOHTsMTggcFxcHtVqtWdLS0qpUBkMYloiIiKzLql/BkydPrvDOsxYtWuDEiRP4+++/y7x2/fp1+Pr6mrVMnTt3hqOjI86dO4fOnTvr3cfZ2RnOzs5mvW55GJaIiIisy6pfwd7e3vD29q5wv/DwcKjVahw5cgRdu3YFABw+fBhqtVozT525nD59GgUFBWjcuLFZz1tZDEtERETWZXIznDW0bdsWjz76KGJiYpCYmIjExETExMRg6NChOp2727Rpg02bNmme37p1CyqVCmfOnAEApKSkQKVSafo5nT9/HvPnz0dSUhIuXbqELVu24Omnn0ZoaCh69uxZvW+yHAxLRERE1lUjwhIg7lgLCQlBREQEIiIi0KFDB6xbt05nn5SUFKjVas3zzZs3IzQ0FEOGDAEAjBo1CqGhofjwww8BiLv4du7ciYEDByI4OBhTp05FREQEduzYAXsbmbmWYYmIiMi6asxXsJeXF7744guD+0iSpPN87NixGDt2bLn7BwQEYO/eveYonsUwLBEREVlXjalZqqsYloiIiKyLYcnGMSwRERFZF8OSjWNYIiIisi6GJRvHsERERGRdDEs2jmGJiIjIuhiWbBzDEhERkXUxLNk4hiUiIiLrYliycQxLRERE1sWwZOPksGQjA4oTERHVOQxLNo41S0RERNbFsGTjGJaIiIisi2HJxjEsERERWRfDko0rKhKPDEtERETWwbBk41izREREZF0MSzaOYYmIiMi6GJZsHMMSERGRdTEs2TiGJSIiIutiWLJxDEtERETWxbBk4xiWiIiIrIthycYxLBEREVkXw5KNY1giIiKyLoYlG8ewREREZF0MSzaOYYmIiMi6GJZsHMMSERGRdTEs2TiGJSIiIutiWLJxDEtERETWxbBk4xiWiIiIrIthycYxLBEREVkXw5KNY1giIiKyLoYlG8ewREREZF0MSzZODkv29tYtBxERUV3FsGTjWLNERERkXQxLNo5hiYiIyLpqTFi6ffs2oqKioFQqoVQqERUVhaysrHL3LygowKuvvoqQkBC4u7vD398f0dHRuHbtms5+eXl5mDJlCry9veHu7o7HH38cV65csfC7MR7DEhERkXXVmLA0evRoqFQqbNu2Ddu2bYNKpUJUVFS5++fm5uL48eN44403cPz4cXz//ff4888/8fjjj+vsN336dGzatAkbN27EgQMHcPfuXQwdOhRFRUWWfksVkiSguFisMywRERFZh0KSJMnahajI2bNn0a5dOyQmJqJbt24AgMTERISHh+OPP/5AcHCwUec5evQounbtisuXL6NZs2ZQq9Vo1KgR1q1bh5EjRwIArl27hoCAAGzZsgUDBw7Ue568vDzk5eVpnmdnZyMgIABqtRqenp5VfLclCgsBR0exfusW0KCB2U5NRERU52VnZ0OpVFb4/V0japYOHToEpVKpCUoA0L17dyiVShw8eNDo86jVaigUCtSvXx8AcOzYMRQUFCAiIkKzj7+/P9q3b2/wvPHx8ZrmQKVSiYCAANPflBHkJjiANUtERETWUiPCUkZGBnx8fMps9/HxQUZGhlHnuH//PmbPno3Ro0dr0mNGRgacnJzQoFSVja+vr8HzxsXFQa1Wa5a0tDQT3o3xGJaIiIisz6phae7cuVAoFAaXpKQkAIBCoShzvCRJereXVlBQgFGjRqG4uBgrVqyocP+Kzuvs7AxPT0+dxRIYloiIiKzPql/BkydPxqhRowzu06JFC5w4cQJ///13mdeuX78OX19fg8cXFBQgMjISFy9exK5du3SCjZ+fH/Lz83H79m2d2qXMzEz06NHDxHdjftphiYNSEhERWYdVw5K3tze8vb0r3C88PBxqtRpHjhxB165dAQCHDx+GWq02GGrkoHTu3Dns3r0bDRs21Hk9LCwMjo6OSEhIQGRkJAAgPT0dp06dwrvvvluFd2YecliysxMLERERVb8a8RXctm1bPProo4iJiUFiYiISExMRExODoUOH6twJ16ZNG2zatAkAUFhYiKeeegpJSUlYv349ioqKkJGRgYyMDOTn5wMAlEolxo8fj5dffhk7d+5EcnIynn32WYSEhKB///5Wea/aOMYSERGR9dWYr+H169dj6tSpmjvXHn/8cXzwwQc6+6SkpECtVgMArly5gs2bNwMAOnXqpLPf7t270bdvXwDA4sWL4eDggMjISNy7dw+PPPII1qxZA3sbaPdiWCIiIrK+GjHOkq0zdpwGU/31F9CqFeDpCfyTAYmIiMhMatU4S3UVa5aIiIisj2HJhjEsERERWR/Dkg1jWCIiIrI+hiUbxrBERERkfQxLNoxhiYiIyPoYlmwYwxIREZH1MSzZMIYlIiIi62NYsmEMS0RERNbHsGTDGJaIiIisj2HJhslhyQZmXiEiIqqzGJZsGGuWiIiIrI9hyYYxLBEREVkfw5INY1giIiKyPoYlG1ZUJB4ZloiIiKyHYcmGsWaJiIjI+hiWbBjDEhERkfUxLNkwhiUiIiLrY1iyYQxLRERE1sewZMMYloiIiKyPYcmGMSwRERFZH8OSDWNYIiIisj6GJRvGsERERGR9DEs2jGGJiIjI+hiWbBjDEhERkfUxLNkwhiUiIiLrY1iyYQxLRERE1sewZMMYloiIiKyPYcmGMSwRERFZH8OSDWNYIiIisj6GJRtmbw+4uABOTtYuCRERUd2lkCRJsnYharrs7GwolUqo1Wp4enpauzhERERkBGO/v1mzRERERGRAjQlLt2/fRlRUFJRKJZRKJaKiopCVlVXu/gUFBXj11VcREhICd3d3+Pv7Izo6GteuXdPZr2/fvlAoFDrLqFGjLPxuiIiIqKaoMWFp9OjRUKlU2LZtG7Zt2waVSoWoqKhy98/NzcXx48fxxhtv4Pjx4/j+++/x559/4vHHHy+zb0xMDNLT0zXLRx99ZMm3QkRERDVIjbjP6uzZs9i2bRsSExPRrVs3AMCqVasQHh6OlJQUBAcHlzlGqVQiISFBZ9vy5cvRtWtXpKamolmzZprtbm5u8PPzs+ybICIiohqpRtQsHTp0CEqlUhOUAKB79+5QKpU4ePCg0edRq9VQKBSoX7++zvb169fD29sbDz74IGbOnIk7d+4YPE9eXh6ys7N1FiIiIqqdakTNUkZGBnx8fMps9/HxQUZGhlHnuH//PmbPno3Ro0fr9HgfM2YMAgMD4efnh1OnTiEuLg6///57mVopbfHx8Zg3b57pb4SIiIhqHKvWLM2dO7dM5+rSS1JSEgBAoVCUOV6SJL3bSysoKMCoUaNQXFyMFStW6LwWExOD/v37o3379hg1ahS+/fZb7NixA8ePHy/3fHFxcVCr1ZolLS3NxHdORERENYVVa5YmT55c4Z1nLVq0wIkTJ/D333+Xee369evw9fU1eHxBQQEiIyNx8eJF7Nq1q8JxkDp37gxHR0ecO3cOnTt31ruPs7MznJ2dDZ6HiIiIagerhiVvb294e3tXuF94eDjUajWOHDmCrl27AgAOHz4MtVqNHj16lHucHJTOnTuH3bt3o2HDhhVe6/Tp0ygoKEDjxo2NfyNERERUa9WIDt5t27bFo48+ipiYGCQmJiIxMRExMTEYOnSozp1wbdq0waZNmwAAhYWFeOqpp5CUlIT169ejqKgIGRkZyMjIQH5+PgDg/PnzmD9/PpKSknDp0iVs2bIFTz/9NEJDQ9GzZ0+rvFciIiKyLTUiLAHijrWQkBBEREQgIiICHTp0wLp163T2SUlJgVqtBgBcuXIFmzdvxpUrV9CpUyc0btxYs8h30Dk5OWHnzp0YOHAggoODMXXqVERERGDHjh2wt7ev9vdIREREtodzw5kB54YjIiKqeTg3HBEREZEZMCwRERERGVAjBqW0dXJLJkfyJiIiqjnk7+2KeiQxLJmBPD1KQECAlUtCREREprpz5w6USmW5r7ODtxkUFxfj2rVr8PDwMGpE8YpkZ2cjICAAaWlp7DBuA/h52BZ+HraFn4ft4WdiPEmScOfOHfj7+8POrvyeSaxZMgM7Ozs0bdrU7Of19PTkL7oN4edhW/h52BZ+HraHn4lxDNUoydjBm4iIiMgAhiUiIiIiAxiWbJCzszPmzJnDyXptBD8P28LPw7bw87A9/EzMjx28iYiIiAxgzRIRERGRAQxLRERERAYwLBEREREZwLBEREREZADDkg1asWIFAgMD4eLigrCwMOzfv9/aRaoT9u3bh8ceewz+/v5QKBT44YcfdF6XJAlz586Fv78/XF1d0bdvX5w+fdo6ha0D4uPj8dBDD8HDwwM+Pj4YNmwYUlJSdPbhZ1J9Vq5ciQ4dOmgGOgwPD8fWrVs1r/OzsJ74+HgoFApMnz5ds42fh3kxLNmYr776CtOnT8drr72G5ORk9OrVC4MGDUJqaqq1i1br5eTkoGPHjvjggw/0vv7uu+9i0aJF+OCDD3D06FH4+flhwIABmrkBybz27t2LSZMmITExEQkJCSgsLERERARycnI0+/AzqT5NmzbFO++8g6SkJCQlJaFfv3544oknNF/A/Cys4+jRo/j444/RoUMHne38PMxMIpvStWtXacKECTrb2rRpI82ePdtKJaqbAEibNm3SPC8uLpb8/Pykd955R7Pt/v37klKplD788EMrlLDuyczMlABIe/fulSSJn4ktaNCggbR69Wp+FlZy584dqVWrVlJCQoLUp08fadq0aZIk8d+GJbBmyYbk5+fj2LFjiIiI0NkeERGBgwcPWqlUBAAXL15ERkaGzmfj7OyMPn368LOpJmq1GgDg5eUFgJ+JNRUVFWHjxo3IyclBeHg4PwsrmTRpEoYMGYL+/fvrbOfnYX6cSNeG3LhxA0VFRfD19dXZ7uvri4yMDCuVigBofv76PpvLly9bo0h1iiRJiI2Nxb/+9S+0b98eAD8Tazh58iTCw8Nx//591KtXD5s2bUK7du00X8D8LKrPxo0bcfz4cRw9erTMa/y3YX4MSzZIoVDoPJckqcw2sg5+NtYxefJknDhxAgcOHCjzGj+T6hMcHAyVSoWsrCx89913eO6557B3717N6/wsqkdaWhqmTZuG7du3w8XFpdz9+HmYD5vhbIi3tzfs7e3L1CJlZmaW+QuBqpefnx8A8LOxgilTpmDz5s3YvXs3mjZtqtnOz6T6OTk5ISgoCF26dEF8fDw6duyIpUuX8rOoZseOHUNmZibCwsLg4OAABwcH7N27F8uWLYODg4PmZ87Pw3wYlmyIk5MTwsLCkJCQoLM9ISEBPXr0sFKpCAACAwPh5+en89nk5+dj7969/GwsRJIkTJ48Gd9//z127dqFwMBAndf5mVifJEnIy8vjZ1HNHnnkEZw8eRIqlUqzdOnSBWPGjIFKpULLli35eZgZm+FsTGxsLKKiotClSxeEh4fj448/RmpqKiZMmGDtotV6d+/exV9//aV5fvHiRahUKnh5eaFZs2aYPn06FixYgFatWqFVq1ZYsGAB3NzcMHr0aCuWuvaaNGkSvvzyS/z444/w8PDQ/JWsVCrh6uqqGVeGn0n1+M9//oNBgwYhICAAd+7cwcaNG7Fnzx5s27aNn0U18/Dw0PTdk7m7u6Nhw4aa7fw8zMx6N+JRef7v//5Pat68ueTk5CR17txZc6s0Wdbu3bslAGWW5557TpIkcTvunDlzJD8/P8nZ2Vnq3bu3dPLkSesWuhbT91kAkD777DPNPvxMqs+4ceM0/y81atRIeuSRR6Tt27drXudnYV3aQwdIEj8Pc1NIkiRZKacRERER2Tz2WSIiIiIygGGJiIiIyACGJSIiIiIDGJaIiIiIDGBYIiIiIjKAYYmIiIjIAIYlIiIiIgMYloiIiIgMYFgiIpvTt29fTJ8+3Wznmzt3LhQKBRQKBZYsWWK285pTZd6z/J7q169vkTIRkcC54YioTnjwwQexY8cOeHp6Wrsoen3//fdwdHQ06Zj09HR89dVXmDNnjoVKRUQAwxIR1REODg7w8/Or0jmKioqgUChgZ2f+SnkvLy+Tj/Hz84NSqTR7WYhIF5vhiMiqcnJyEB0djXr16qFx48ZYuHChzuv5+fmYNWsWmjRpAnd3d3Tr1g179uzR2WfVqlUICAiAm5sbhg8fjkWLFhnVNLVo0SKEhITA3d0dAQEBmDhxIu7evat5fc2aNahfvz5+/vlntGvXDs7Ozrh8+TLy8vIwa9YsBAQEwNnZGa1atcInn3wCANizZw8UCgV+/fVXhIaGwtXVFf369UNmZia2bt2Ktm3bwtPTE8888wxyc3M11yrdDNeiRQssWLAA48aNg4eHB5o1a4aPP/7Y9B8wEVUZwxIRWdUrr7yC3bt3Y9OmTdi+fTv27NmDY8eOaV5//vnn8dtvv2Hjxo04ceIEnn76aTz66KM4d+4cAOC3337DhAkTMG3aNKhUKgwYMABvv/22Ude2s7PDsmXLcOrUKaxduxa7du3CrFmzdPbJzc1FfHw8Vq9ejdOnT8PHxwfR0dHYuHEjli1bhrNnz+LDDz9EvXr1dI6bO3cuPvjgAxw8eBBpaWmIjIzEkiVL8OWXX+KXX35BQkICli9fbrB8CxcuRJcuXZCcnIyJEyfi3//+N/744w+j3hsRmZFERGQld+7ckZycnKSNGzdqtt28eVNydXWVpk2bJv3111+SQqGQrl69qnPcI488IsXFxUmSJEkjR46UhgwZovP6mDFjJKVSqXk+Z84cqWPHjhWW5+uvv5YaNmyoef7ZZ59JACSVSqXZlpKSIgGQEhIS9J5j9+7dEgBpx44dmm3x8fESAOn8+fOabS+99JI0cOBAzfM+ffpI06ZN0zxv3ry59Oyzz2qeFxcXSz4+PtLKlSt1rvfZZ5/pvFciMj/WLBGR1Zw/fx75+fkIDw/XbPPy8kJwcDAA4Pjx45AkCa1bt0a9evU0y969e3H+/HkAQEpKCrp27apz3tLPy7N7924MGDAATZo0gYeHB6Kjo3Hz5k3k5ORo9nFyckKHDh00z1UqFezt7dGnTx+D59Y+xtfXF25ubmjZsqXOtszMTKPPoVAo4OfnV+ExRGR+7OBNRFYjSZLB14uLi2Fvb49jx47B3t5e5zW52UuSJCgUCpPOCwCXL1/G4MGDMWHCBLz11lvw8vLCgQMHMH78eBQUFGj2c3V11Tm/q6trhecGoHNnm0KhKHOnm0KhQHFxsdHnMPYYIjI/1iwRkdUEBQXB0dERiYmJmm23b9/Gn3/+CQAIDQ1FUVERMjMzERQUpLPId7a1adMGR44c0TlvUlJShddOSkpCYWEhFi5ciO7du6N169a4du1ahceFhISguLgYe/fuNeWtElENxpolIrKaevXqYfz48XjllVfQsGFD+Pr64rXXXtPcmt+6dWuMGTMG0dHRWLhwIUJDQ3Hjxg3s2rULISEhGDx4MKZMmYLevXtj0aJFeOyxx7Br1y5s3bq1TG1TaQ888AAKCwuxfPlyPPbYY/jtt9/w4YcfVljmFi1a4LnnnsO4ceOwbNkydOzYEZcvX0ZmZiYiIyPN8nMhItvCmiUisqr33nsPvXv3xuOPP47+/fvjX//6F8LCwjSvf/bZZ4iOjsbLL7+M4OBgPP744zh8+DACAgIAAD179sSHH36IRYsWoWPHjti2bRtmzJgBFxcXg9ft1KkTFi1ahP/9739o37491q9fj/j4eKPKvHLlSjz11FOYOHEi2rRpg5iYGJ1+TkRUuygkYxr3iYhqkJiYGPzxxx/Yv38/AHEb/w8//ACVSmXdglnAmjVrMH36dGRlZVm7KES1FpvhiKjGe//99zFgwAC4u7tj69atWLt2LVasWKGzz8mTJ1GvXj28++67mDhxopVKal716tVDYWFhhbVoRFQ1rFkiohovMjISe/bswZ07d9CyZUtMmTIFEyZM0Lx+69Yt3Lp1CwDQqFGjWjNFyF9//QUAsLe3R2BgoJVLQ1R7MSwRERERGcAO3kREREQGMCwRERERGcCwRERERGQAwxIRERGRAQxLRERERAYwLBEREREZwLBEREREZADDEhEREZEB/w+X63J3tzmv3wAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "t1 = sacc.Sacc.load_fits(\"/global/homes/e/edujb/TXPipe/data/example/output_txcluster/cluster_sacc_catalog.sacc\")\n", + "for k in range(0,2):\n", + " for j in range(0,2):\n", + " data2 = []\n", + " radius2 = []\n", + " for i in range(0,8):\n", + " trac = ('cosmodc2-1deg', f'bin_rich_{j}', f'bin_z_{k}', f'radius_{i}')\n", + " data2.append(t1.get_data_points(sacc.data_types.standard_types.cluster_shear, trac)[0].value)\n", + " radius = t1.tracers[f'radius_{i}'].center\n", + " radius2.append(radius)\n", + " print(data2)\n", + " print(data2[0])\n", + " plt.plot(radius2, data2, color=\"blue\", label=f'bin_{k}_{j}')\n", + " plt.ylabel(\"tangential reduced shear\")\n", + " plt.xlabel(\"deg[arcmin]\")\n", + " plt.legend(loc='upper right')\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "id": "92e11bb9-30f3-4e4f-b2a4-4147e414041c", + "metadata": {}, + "source": [ + "## 20 deg example" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "0cc1b772-57f4-4fc8-a65f-182b85c1391d", + "metadata": {}, + "outputs": [], + "source": [ + "step1 = txpipe.TXSourceSelectorMetadetect.make_stage(\n", + " # This file is the input metadetect shear catalog\n", + " shear_catalog=f\"/global/cfs/projectdirs/lsst/groups/CL/cl_pipeline_project/TXPipe_data/cosmodc2/20deg2/shear_catalog.hdf5\",\n", + " # This is an input training set for the tomographic selection\n", + " calibration_table=f\"{my_txpipe_dir}data/example/inputs/sample_cosmodc2_w10year_errors.dat\",\n", + "\n", + " # This contains all the options for this stage. You can override them here\n", + " # manually too.\n", + " config=f\"{my_txpipe_dir}/examples/metadetect/config.yml\",\n", + "\n", + " # This is the output file for this stage\n", + " shear_tomography_catalog=f\"{my_txpipe_dir}/data/example/outputs_metadetect_20/shear_tomography_catalog.hdf5\"\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "b5dc1861-424d-4ec0-b7c3-82a29d7c98d9", + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Process 0 running selection for rows 0-100,000\n", + "Process 0 running selection for rows 100,000-200,000\n", + "Process 0 running selection for rows 200,000-300,000\n", + "Process 0 running selection for rows 300,000-400,000\n", + "Process 0 running selection for rows 400,000-500,000\n", + "Process 0 running selection for rows 500,000-600,000\n", + "Process 0 running selection for rows 600,000-700,000\n", + "Process 0 running selection for rows 700,000-800,000\n", + "Process 0 running selection for rows 800,000-900,000\n", + "Process 0 running selection for rows 900,000-1,000,000\n", + "Process 0 running selection for rows 1,000,000-1,100,000\n", + "Process 0 running selection for rows 1,100,000-1,200,000\n", + "Process 0 running selection for rows 1,200,000-1,300,000\n", + "Process 0 running selection for rows 1,300,000-1,400,000\n", + "Process 0 running selection for rows 1,400,000-1,500,000\n", + "Process 0 running selection for rows 1,500,000-1,600,000\n", + "Process 0 running selection for rows 1,600,000-1,700,000\n", + "Process 0 running selection for rows 1,700,000-1,800,000\n", + "Process 0 running selection for rows 1,800,000-1,900,000\n", + "Process 0 running selection for rows 1,900,000-2,000,000\n", + "Process 0 running selection for rows 2,000,000-2,100,000\n", + "Process 0 running selection for rows 2,100,000-2,200,000\n", + "Process 0 running selection for rows 2,200,000-2,300,000\n", + "Process 0 running selection for rows 2,300,000-2,400,000\n", + "Process 0 running selection for rows 2,400,000-2,500,000\n", + "Process 0 running selection for rows 2,500,000-2,600,000\n", + "Process 0 running selection for rows 2,600,000-2,700,000\n", + "Process 0 running selection for rows 2,700,000-2,800,000\n", + "Process 0 running selection for rows 2,800,000-2,900,000\n", + "Process 0 running selection for rows 2,900,000-3,000,000\n", + "Process 0 running selection for rows 3,000,000-3,100,000\n", + "Process 0 running selection for rows 3,100,000-3,200,000\n", + "Process 0 running selection for rows 3,200,000-3,300,000\n", + "Process 0 running selection for rows 3,300,000-3,400,000\n", + "Process 0 running selection for rows 3,400,000-3,500,000\n", + "Process 0 running selection for rows 3,500,000-3,600,000\n", + "Process 0 running selection for rows 3,600,000-3,700,000\n", + "Process 0 running selection for rows 3,700,000-3,800,000\n", + "Process 0 running selection for rows 3,800,000-3,900,000\n", + "Process 0 running selection for rows 3,900,000-4,000,000\n", + "Process 0 running selection for rows 4,000,000-4,100,000\n", + "Process 0 running selection for rows 4,100,000-4,200,000\n", + "Process 0 running selection for rows 4,200,000-4,300,000\n", + "Process 0 running selection for rows 4,300,000-4,400,000\n", + "Process 0 running selection for rows 4,400,000-4,500,000\n", + "Process 0 running selection for rows 4,500,000-4,600,000\n", + "Process 0 running selection for rows 4,600,000-4,700,000\n", + "Process 0 running selection for rows 4,700,000-4,800,000\n", + "Process 0 running selection for rows 4,800,000-4,900,000\n", + "Process 0 running selection for rows 4,900,000-5,000,000\n", + "Process 0 running selection for rows 5,000,000-5,100,000\n", + "Process 0 running selection for rows 5,100,000-5,200,000\n", + "Process 0 running selection for rows 5,200,000-5,300,000\n", + "Process 0 running selection for rows 5,300,000-5,400,000\n", + "Process 0 running selection for rows 5,400,000-5,500,000\n", + "Process 0 running selection for rows 5,500,000-5,600,000\n", + "Process 0 running selection for rows 5,600,000-5,700,000\n", + "Process 0 running selection for rows 5,700,000-5,800,000\n", + "Process 0 running selection for rows 5,800,000-5,900,000\n", + "Process 0 running selection for rows 5,900,000-6,000,000\n", + "Process 0 running selection for rows 6,000,000-6,100,000\n", + "Process 0 running selection for rows 6,100,000-6,200,000\n", + "Process 0 running selection for rows 6,200,000-6,300,000\n", + "Process 0 running selection for rows 6,300,000-6,400,000\n", + "Process 0 running selection for rows 6,400,000-6,500,000\n", + "Process 0 running selection for rows 6,500,000-6,600,000\n", + "Process 0 running selection for rows 6,600,000-6,700,000\n", + "Process 0 running selection for rows 6,700,000-6,800,000\n", + "Process 0 running selection for rows 6,800,000-6,900,000\n", + "Process 0 running selection for rows 6,900,000-7,000,000\n", + "Process 0 running selection for rows 7,000,000-7,100,000\n", + "Process 0 running selection for rows 7,100,000-7,200,000\n", + "Process 0 running selection for rows 7,200,000-7,300,000\n", + "Process 0 running selection for rows 7,300,000-7,400,000\n", + "Process 0 running selection for rows 7,400,000-7,500,000\n", + "Process 0 running selection for rows 7,500,000-7,600,000\n", + "Process 0 running selection for rows 7,600,000-7,700,000\n", + "Process 0 running selection for rows 7,700,000-7,800,000\n", + "Process 0 running selection for rows 7,800,000-7,900,000\n", + "Process 0 running selection for rows 7,900,000-8,000,000\n", + "Process 0 running selection for rows 8,000,000-8,100,000\n", + "Process 0 running selection for rows 8,100,000-8,200,000\n", + "Process 0 running selection for rows 8,200,000-8,300,000\n", + "Process 0 running selection for rows 8,300,000-8,400,000\n", + "Process 0 running selection for rows 8,400,000-8,500,000\n", + "Process 0 running selection for rows 8,500,000-8,600,000\n", + "Process 0 running selection for rows 8,600,000-8,700,000\n", + "Process 0 running selection for rows 8,700,000-8,800,000\n", + "Process 0 running selection for rows 8,800,000-8,900,000\n", + "Process 0 running selection for rows 8,900,000-9,000,000\n", + "Process 0 running selection for rows 9,000,000-9,100,000\n", + "Process 0 running selection for rows 9,100,000-9,200,000\n", + "Process 0 running selection for rows 9,200,000-9,300,000\n", + "Process 0 running selection for rows 9,300,000-9,400,000\n", + "Process 0 running selection for rows 9,400,000-9,500,000\n", + "Process 0 running selection for rows 9,500,000-9,600,000\n", + "Process 0 running selection for rows 9,600,000-9,700,000\n", + "Process 0 running selection for rows 9,700,000-9,800,000\n", + "Process 0 running selection for rows 9,800,000-9,900,000\n", + "Process 0 running selection for rows 9,900,000-10,000,000\n", + "Process 0 running selection for rows 10,000,000-10,100,000\n", + "Process 0 running selection for rows 10,100,000-10,200,000\n", + "Process 0 running selection for rows 10,200,000-10,300,000\n", + "Process 0 running selection for rows 10,300,000-10,400,000\n", + "Process 0 running selection for rows 10,400,000-10,500,000\n", + "Process 0 running selection for rows 10,500,000-10,600,000\n", + "Process 0 running selection for rows 10,600,000-10,700,000\n", + "Process 0 running selection for rows 10,700,000-10,800,000\n", + "Process 0 running selection for rows 10,800,000-10,900,000\n", + "Process 0 running selection for rows 10,900,000-11,000,000\n", + "Process 0 running selection for rows 11,000,000-11,100,000\n", + "Process 0 running selection for rows 11,100,000-11,200,000\n", + "Process 0 running selection for rows 11,200,000-11,300,000\n", + "Process 0 running selection for rows 11,300,000-11,400,000\n", + "Process 0 running selection for rows 11,400,000-11,500,000\n", + "Process 0 running selection for rows 11,500,000-11,600,000\n", + "Process 0 running selection for rows 11,600,000-11,700,000\n", + "Process 0 running selection for rows 11,700,000-11,800,000\n", + "Process 0 running selection for rows 11,800,000-11,900,000\n", + "Process 0 running selection for rows 11,900,000-12,000,000\n", + "Process 0 running selection for rows 12,000,000-12,100,000\n", + "Process 0 running selection for rows 12,100,000-12,200,000\n", + "Process 0 running selection for rows 12,200,000-12,300,000\n", + "Process 0 running selection for rows 12,300,000-12,400,000\n", + "Process 0 running selection for rows 12,400,000-12,500,000\n", + "Process 0 running selection for rows 12,500,000-12,600,000\n", + "Process 0 running selection for rows 12,600,000-12,700,000\n", + "Process 0 running selection for rows 12,700,000-12,800,000\n", + "Process 0 running selection for rows 12,800,000-12,900,000\n", + "Process 0 running selection for rows 12,900,000-13,000,000\n", + "Process 0 running selection for rows 13,000,000-13,100,000\n", + "Process 0 running selection for rows 13,100,000-13,200,000\n", + "Process 0 running selection for rows 13,200,000-13,300,000\n", + "Process 0 running selection for rows 13,300,000-13,400,000\n", + "Process 0 running selection for rows 13,400,000-13,500,000\n", + "Process 0 running selection for rows 13,500,000-13,600,000\n", + "Process 0 running selection for rows 13,600,000-13,700,000\n", + "Process 0 running selection for rows 13,700,000-13,800,000\n", + "Process 0 running selection for rows 13,800,000-13,900,000\n", + "Process 0 running selection for rows 13,900,000-14,000,000\n", + "Process 0 running selection for rows 14,000,000-14,100,000\n", + "Process 0 running selection for rows 14,100,000-14,200,000\n", + "Process 0 running selection for rows 14,200,000-14,300,000\n", + "Process 0 running selection for rows 14,300,000-14,400,000\n", + "Process 0 running selection for rows 14,400,000-14,500,000\n", + "Process 0 running selection for rows 14,500,000-14,600,000\n", + "Process 0 running selection for rows 14,600,000-14,700,000\n", + "Process 0 running selection for rows 14,700,000-14,800,000\n", + "Process 0 running selection for rows 14,800,000-14,900,000\n", + "Process 0 running selection for rows 14,900,000-15,000,000\n", + "Process 0 running selection for rows 15,000,000-15,100,000\n", + "Process 0 running selection for rows 15,100,000-15,200,000\n", + "Process 0 running selection for rows 15,200,000-15,300,000\n", + "Process 0 running selection for rows 15,300,000-15,400,000\n", + "Process 0 running selection for rows 15,400,000-15,500,000\n", + "Process 0 running selection for rows 15,500,000-15,600,000\n", + "Process 0 running selection for rows 15,600,000-15,700,000\n", + "Process 0 running selection for rows 15,700,000-15,800,000\n", + "Process 0 running selection for rows 15,800,000-15,900,000\n", + "Process 0 running selection for rows 15,900,000-16,000,000\n", + "Process 0 running selection for rows 16,000,000-16,100,000\n", + "Process 0 running selection for rows 16,100,000-16,200,000\n", + "Process 0 running selection for rows 16,200,000-16,300,000\n", + "Process 0 running selection for rows 16,300,000-16,400,000\n", + "Process 0 running selection for rows 16,400,000-16,500,000\n", + "Process 0 running selection for rows 16,500,000-16,600,000\n", + "Process 0 running selection for rows 16,600,000-16,700,000\n", + "Process 0 running selection for rows 16,700,000-16,800,000\n", + "Process 0 running selection for rows 16,800,000-16,900,000\n", + "Process 0 running selection for rows 16,900,000-17,000,000\n", + "Process 0 running selection for rows 17,000,000-17,100,000\n", + "Process 0 running selection for rows 17,100,000-17,200,000\n", + "Process 0 running selection for rows 17,200,000-17,300,000\n", + "Process 0 running selection for rows 17,300,000-17,400,000\n", + "Process 0 running selection for rows 17,400,000-17,500,000\n", + "Process 0 running selection for rows 17,500,000-17,600,000\n", + "Process 0 running selection for rows 17,600,000-17,700,000\n", + "Process 0 running selection for rows 17,700,000-17,800,000\n", + "Process 0 running selection for rows 17,800,000-17,900,000\n", + "Process 0 running selection for rows 17,900,000-18,000,000\n", + "Process 0 running selection for rows 18,000,000-18,100,000\n", + "Process 0 running selection for rows 18,100,000-18,200,000\n", + "Process 0 running selection for rows 18,200,000-18,300,000\n", + "Process 0 running selection for rows 18,300,000-18,400,000\n", + "Process 0 running selection for rows 18,400,000-18,500,000\n", + "Process 0 running selection for rows 18,500,000-18,600,000\n", + "Process 0 running selection for rows 18,600,000-18,700,000\n", + "Process 0 running selection for rows 18,700,000-18,800,000\n", + "Process 0 running selection for rows 18,800,000-18,900,000\n", + "Process 0 running selection for rows 18,900,000-19,000,000\n", + "Process 0 running selection for rows 19,000,000-19,100,000\n", + "Process 0 running selection for rows 19,100,000-19,200,000\n", + "Process 0 running selection for rows 19,200,000-19,300,000\n", + "Process 0 running selection for rows 19,300,000-19,400,000\n", + "Process 0 running selection for rows 19,400,000-19,500,000\n", + "Process 0 running selection for rows 19,500,000-19,600,000\n", + "Process 0 running selection for rows 19,600,000-19,700,000\n", + "Process 0 running selection for rows 19,700,000-19,800,000\n", + "Process 0 running selection for rows 19,800,000-19,900,000\n", + "Process 0 running selection for rows 19,900,000-20,000,000\n", + "Process 0 running selection for rows 20,000,000-20,100,000\n", + "Process 0 running selection for rows 20,100,000-20,200,000\n", + "Process 0 running selection for rows 20,200,000-20,300,000\n", + "Process 0 running selection for rows 20,300,000-20,400,000\n", + "Process 0 running selection for rows 20,400,000-20,500,000\n", + "Process 0 running selection for rows 20,500,000-20,600,000\n", + "Process 0 running selection for rows 20,600,000-20,700,000\n", + "Process 0 running selection for rows 20,700,000-20,800,000\n", + "Process 0 running selection for rows 20,800,000-20,900,000\n", + "Process 0 running selection for rows 20,900,000-21,000,000\n", + "Process 0 running selection for rows 21,000,000-21,100,000\n", + "Process 0 running selection for rows 21,100,000-21,200,000\n", + "Process 0 running selection for rows 21,200,000-21,300,000\n", + "Process 0 running selection for rows 21,300,000-21,400,000\n", + "Process 0 running selection for rows 21,400,000-21,500,000\n", + "Process 0 running selection for rows 21,500,000-21,600,000\n", + "Process 0 running selection for rows 21,600,000-21,700,000\n", + "Process 0 running selection for rows 21,700,000-21,800,000\n", + "Process 0 running selection for rows 21,800,000-21,900,000\n", + "Process 0 running selection for rows 21,900,000-22,000,000\n", + "Process 0 running selection for rows 22,000,000-22,100,000\n", + "Process 0 running selection for rows 22,100,000-22,200,000\n", + "Process 0 running selection for rows 22,200,000-22,300,000\n", + "Process 0 running selection for rows 22,300,000-22,400,000\n", + "Process 0 running selection for rows 22,400,000-22,500,000\n", + "Process 0 running selection for rows 22,500,000-22,600,000\n", + "Process 0 running selection for rows 22,600,000-22,700,000\n", + "Process 0 running selection for rows 22,700,000-22,800,000\n", + "Process 0 running selection for rows 22,800,000-22,900,000\n", + "Process 0 running selection for rows 22,900,000-23,000,000\n", + "Process 0 running selection for rows 23,000,000-23,100,000\n", + "Process 0 running selection for rows 23,100,000-23,200,000\n", + "Process 0 running selection for rows 23,200,000-23,300,000\n", + "Process 0 running selection for rows 23,300,000-23,400,000\n", + "Process 0 running selection for rows 23,400,000-23,500,000\n", + "Process 0 running selection for rows 23,500,000-23,600,000\n", + "Process 0 running selection for rows 23,600,000-23,700,000\n", + "Process 0 running selection for rows 23,700,000-23,800,000\n", + "Process 0 running selection for rows 23,800,000-23,900,000\n", + "Process 0 running selection for rows 23,900,000-24,000,000\n", + "Process 0 running selection for rows 24,000,000-24,100,000\n", + "Process 0 running selection for rows 24,100,000-24,200,000\n", + "Process 0 running selection for rows 24,200,000-24,300,000\n", + "Process 0 running selection for rows 24,300,000-24,400,000\n", + "Process 0 running selection for rows 24,400,000-24,500,000\n", + "Process 0 running selection for rows 24,500,000-24,600,000\n", + "Process 0 running selection for rows 24,600,000-24,700,000\n", + "Process 0 running selection for rows 24,700,000-24,800,000\n", + "Process 0 running selection for rows 24,800,000-24,900,000\n", + "Process 0 running selection for rows 24,900,000-25,000,000\n", + "Process 0 running selection for rows 25,000,000-25,100,000\n", + "Process 0 running selection for rows 25,100,000-25,200,000\n", + "Process 0 running selection for rows 25,200,000-25,300,000\n", + "Process 0 running selection for rows 25,300,000-25,400,000\n", + "Process 0 running selection for rows 25,400,000-25,418,721\n" + ] + } + ], + "source": [ + "step1.run()\n", + "step1.finalize()" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "630ab434-849d-47da-9e58-27141bf7bf25", + "metadata": {}, + "outputs": [], + "source": [ + "step2 = txpipe.CLClusterBinningRedshiftRichness.make_stage(\n", + " # This file is the input metadetect shear catalog\n", + " cluster_catalog=\"/global/cfs/projectdirs/lsst/groups/CL/cl_pipeline_project/TXPipe_data/cosmodc2/20deg2/cluster_catalog.hdf5\",\n", + " # This is an input training set for the tomographic selection\n", + "\n", + " # This contains all the options for this stage. You can override them here\n", + " # manually too.\n", + " config=f\"{my_txpipe_dir}/notebooks/cluster_counts/gt_treecor_test/config-1deg2-CL.yml\",\n", + "\n", + " # This is the output file for this stage\n", + " cluster_catalog_tomography=f\"{my_txpipe_dir}/data/example/output_binning_20/cluster_catalog_tomography.hdf5\"\n", + ")\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "id": "6fef9b8a-bf12-4cad-81bf-0df42add864b", + "metadata": {}, + "outputs": [], + "source": [ + "step2.run()\n", + "step2.finalize()" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "id": "31e94cbf-8bd9-40cf-9fcb-a5f17c70da8a", + "metadata": {}, + "outputs": [], + "source": [ + "step3 = txpipe.TXShearCalibration.make_stage(\n", + " # This file is the input metadetect shear catalog\n", + " shear_catalog=\"/global/cfs/projectdirs/lsst/groups/CL/cl_pipeline_project/TXPipe_data/cosmodc2/20deg2/shear_catalog.hdf5\",\n", + " shear_tomography_catalog=f\"{my_txpipe_dir}/data/example/outputs_metadetect_20/shear_tomography_catalog.hdf5\",\n", + " binned_shear_catalog=f\"{my_txpipe_dir}data/example/output_shearcalib_20/binned_shear_catalog.hdf5\",\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "id": "7f44ed87-234c-4381-b5fb-add665311799", + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Copying extra columns: []\n", + "Copying magnitude: ['mag_i', 'mag_r', 'mag_z', 'mag_err_i', 'mag_err_r', 'mag_err_z']\n", + "Using calibration method: MetaDetectCalibrator\n", + "Reading these columns from the 'shear' group:\n", + " - 00/g1\n", + " - 00/g2\n", + " - 00/ra\n", + " - 00/dec\n", + " - 00/weight\n", + " - 00/mag_i\n", + " - 00/mag_r\n", + " - 00/mag_z\n", + " - 00/mag_err_i\n", + " - 00/mag_err_r\n", + " - 00/mag_err_z\n", + "Renaming these columns:\n", + " - 00/g1 -> g1\n", + " - 00/g2 -> g2\n", + " - 00/ra -> ra\n", + " - 00/dec -> dec\n", + " - 00/weight -> weight\n", + " - 00/mag_i -> mag_i\n", + " - 00/mag_r -> mag_r\n", + " - 00/mag_z -> mag_z\n", + " - 00/mag_err_i -> mag_err_i\n", + " - 00/mag_err_r -> mag_err_r\n", + " - 00/mag_err_z -> mag_err_z\n", + "Process 0 collating bins: [0, 1, 2, 3, all]\n", + "Rank 0 processing data 0 - 100,000\n", + "Rank 0 processing data 100,000 - 200,000\n", + "Rank 0 processing data 200,000 - 300,000\n", + "Rank 0 processing data 300,000 - 400,000\n", + "Rank 0 processing data 400,000 - 500,000\n", + "Rank 0 processing data 500,000 - 600,000\n", + "Rank 0 processing data 600,000 - 700,000\n", + "Rank 0 processing data 700,000 - 800,000\n", + "Rank 0 processing data 800,000 - 900,000\n", + "Rank 0 processing data 900,000 - 1,000,000\n", + "Rank 0 processing data 1,000,000 - 1,100,000\n", + "Rank 0 processing data 1,100,000 - 1,200,000\n", + "Rank 0 processing data 1,200,000 - 1,300,000\n", + "Rank 0 processing data 1,300,000 - 1,400,000\n", + "Rank 0 processing data 1,400,000 - 1,500,000\n", + "Rank 0 processing data 1,500,000 - 1,600,000\n", + "Rank 0 processing data 1,600,000 - 1,700,000\n", + "Rank 0 processing data 1,700,000 - 1,800,000\n", + "Rank 0 processing data 1,800,000 - 1,900,000\n", + "Rank 0 processing data 1,900,000 - 2,000,000\n", + "Rank 0 processing data 2,000,000 - 2,100,000\n", + "Rank 0 processing data 2,100,000 - 2,200,000\n", + "Rank 0 processing data 2,200,000 - 2,300,000\n", + "Rank 0 processing data 2,300,000 - 2,400,000\n", + "Rank 0 processing data 2,400,000 - 2,500,000\n", + "Rank 0 processing data 2,500,000 - 2,600,000\n", + "Rank 0 processing data 2,600,000 - 2,700,000\n", + "Rank 0 processing data 2,700,000 - 2,800,000\n", + "Rank 0 processing data 2,800,000 - 2,900,000\n", + "Rank 0 processing data 2,900,000 - 3,000,000\n", + "Rank 0 processing data 3,000,000 - 3,100,000\n", + "Rank 0 processing data 3,100,000 - 3,200,000\n", + "Rank 0 processing data 3,200,000 - 3,300,000\n", + "Rank 0 processing data 3,300,000 - 3,400,000\n", + "Rank 0 processing data 3,400,000 - 3,500,000\n", + "Rank 0 processing data 3,500,000 - 3,600,000\n", + "Rank 0 processing data 3,600,000 - 3,700,000\n", + "Rank 0 processing data 3,700,000 - 3,800,000\n", + "Rank 0 processing data 3,800,000 - 3,900,000\n", + "Rank 0 processing data 3,900,000 - 4,000,000\n", + "Rank 0 processing data 4,000,000 - 4,100,000\n", + "Rank 0 processing data 4,100,000 - 4,200,000\n", + "Rank 0 processing data 4,200,000 - 4,300,000\n", + "Rank 0 processing data 4,300,000 - 4,400,000\n", + "Rank 0 processing data 4,400,000 - 4,500,000\n", + "Rank 0 processing data 4,500,000 - 4,600,000\n", + "Rank 0 processing data 4,600,000 - 4,700,000\n", + "Rank 0 processing data 4,700,000 - 4,800,000\n", + "Rank 0 processing data 4,800,000 - 4,900,000\n", + "Rank 0 processing data 4,900,000 - 5,000,000\n", + "Rank 0 processing data 5,000,000 - 5,100,000\n", + "Rank 0 processing data 5,100,000 - 5,200,000\n", + "Rank 0 processing data 5,200,000 - 5,300,000\n", + "Rank 0 processing data 5,300,000 - 5,400,000\n", + "Rank 0 processing data 5,400,000 - 5,500,000\n", + "Rank 0 processing data 5,500,000 - 5,600,000\n", + "Rank 0 processing data 5,600,000 - 5,700,000\n", + "Rank 0 processing data 5,700,000 - 5,800,000\n", + "Rank 0 processing data 5,800,000 - 5,900,000\n", + "Rank 0 processing data 5,900,000 - 6,000,000\n", + "Rank 0 processing data 6,000,000 - 6,100,000\n", + "Rank 0 processing data 6,100,000 - 6,200,000\n", + "Rank 0 processing data 6,200,000 - 6,300,000\n", + "Rank 0 processing data 6,300,000 - 6,400,000\n", + "Rank 0 processing data 6,400,000 - 6,500,000\n", + "Rank 0 processing data 6,500,000 - 6,600,000\n", + "Rank 0 processing data 6,600,000 - 6,700,000\n", + "Rank 0 processing data 6,700,000 - 6,800,000\n", + "Rank 0 processing data 6,800,000 - 6,900,000\n", + "Rank 0 processing data 6,900,000 - 7,000,000\n", + "Rank 0 processing data 7,000,000 - 7,100,000\n", + "Rank 0 processing data 7,100,000 - 7,200,000\n", + "Rank 0 processing data 7,200,000 - 7,300,000\n", + "Rank 0 processing data 7,300,000 - 7,400,000\n", + "Rank 0 processing data 7,400,000 - 7,500,000\n", + "Rank 0 processing data 7,500,000 - 7,600,000\n", + "Rank 0 processing data 7,600,000 - 7,700,000\n", + "Rank 0 processing data 7,700,000 - 7,800,000\n", + "Rank 0 processing data 7,800,000 - 7,900,000\n", + "Rank 0 processing data 7,900,000 - 8,000,000\n", + "Rank 0 processing data 8,000,000 - 8,100,000\n", + "Rank 0 processing data 8,100,000 - 8,200,000\n", + "Rank 0 processing data 8,200,000 - 8,300,000\n", + "Rank 0 processing data 8,300,000 - 8,400,000\n", + "Rank 0 processing data 8,400,000 - 8,500,000\n", + "Rank 0 processing data 8,500,000 - 8,600,000\n", + "Rank 0 processing data 8,600,000 - 8,700,000\n", + "Rank 0 processing data 8,700,000 - 8,800,000\n", + "Rank 0 processing data 8,800,000 - 8,900,000\n", + "Rank 0 processing data 8,900,000 - 9,000,000\n", + "Rank 0 processing data 9,000,000 - 9,100,000\n", + "Rank 0 processing data 9,100,000 - 9,200,000\n", + "Rank 0 processing data 9,200,000 - 9,300,000\n", + "Rank 0 processing data 9,300,000 - 9,400,000\n", + "Rank 0 processing data 9,400,000 - 9,500,000\n", + "Rank 0 processing data 9,500,000 - 9,600,000\n", + "Rank 0 processing data 9,600,000 - 9,700,000\n", + "Rank 0 processing data 9,700,000 - 9,800,000\n", + "Rank 0 processing data 9,800,000 - 9,900,000\n", + "Rank 0 processing data 9,900,000 - 10,000,000\n", + "Rank 0 processing data 10,000,000 - 10,100,000\n", + "Rank 0 processing data 10,100,000 - 10,200,000\n", + "Rank 0 processing data 10,200,000 - 10,300,000\n", + "Rank 0 processing data 10,300,000 - 10,400,000\n", + "Rank 0 processing data 10,400,000 - 10,500,000\n", + "Rank 0 processing data 10,500,000 - 10,600,000\n", + "Rank 0 processing data 10,600,000 - 10,700,000\n", + "Rank 0 processing data 10,700,000 - 10,800,000\n", + "Rank 0 processing data 10,800,000 - 10,900,000\n", + "Rank 0 processing data 10,900,000 - 11,000,000\n", + "Rank 0 processing data 11,000,000 - 11,100,000\n", + "Rank 0 processing data 11,100,000 - 11,200,000\n", + "Rank 0 processing data 11,200,000 - 11,300,000\n", + "Rank 0 processing data 11,300,000 - 11,400,000\n", + "Rank 0 processing data 11,400,000 - 11,500,000\n", + "Rank 0 processing data 11,500,000 - 11,600,000\n", + "Rank 0 processing data 11,600,000 - 11,700,000\n", + "Rank 0 processing data 11,700,000 - 11,800,000\n", + "Rank 0 processing data 11,800,000 - 11,900,000\n", + "Rank 0 processing data 11,900,000 - 12,000,000\n", + "Rank 0 processing data 12,000,000 - 12,100,000\n", + "Rank 0 processing data 12,100,000 - 12,200,000\n", + "Rank 0 processing data 12,200,000 - 12,300,000\n", + "Rank 0 processing data 12,300,000 - 12,400,000\n", + "Rank 0 processing data 12,400,000 - 12,500,000\n", + "Rank 0 processing data 12,500,000 - 12,600,000\n", + "Rank 0 processing data 12,600,000 - 12,700,000\n", + "Rank 0 processing data 12,700,000 - 12,800,000\n", + "Rank 0 processing data 12,800,000 - 12,900,000\n", + "Rank 0 processing data 12,900,000 - 13,000,000\n", + "Rank 0 processing data 13,000,000 - 13,100,000\n", + "Rank 0 processing data 13,100,000 - 13,200,000\n", + "Rank 0 processing data 13,200,000 - 13,300,000\n", + "Rank 0 processing data 13,300,000 - 13,400,000\n", + "Rank 0 processing data 13,400,000 - 13,500,000\n", + "Rank 0 processing data 13,500,000 - 13,600,000\n", + "Rank 0 processing data 13,600,000 - 13,700,000\n", + "Rank 0 processing data 13,700,000 - 13,800,000\n", + "Rank 0 processing data 13,800,000 - 13,900,000\n", + "Rank 0 processing data 13,900,000 - 14,000,000\n", + "Rank 0 processing data 14,000,000 - 14,100,000\n", + "Rank 0 processing data 14,100,000 - 14,200,000\n", + "Rank 0 processing data 14,200,000 - 14,300,000\n", + "Rank 0 processing data 14,300,000 - 14,400,000\n", + "Rank 0 processing data 14,400,000 - 14,500,000\n", + "Rank 0 processing data 14,500,000 - 14,600,000\n", + "Rank 0 processing data 14,600,000 - 14,700,000\n", + "Rank 0 processing data 14,700,000 - 14,800,000\n", + "Rank 0 processing data 14,800,000 - 14,900,000\n", + "Rank 0 processing data 14,900,000 - 15,000,000\n", + "Rank 0 processing data 15,000,000 - 15,100,000\n", + "Rank 0 processing data 15,100,000 - 15,200,000\n", + "Rank 0 processing data 15,200,000 - 15,300,000\n", + "Rank 0 processing data 15,300,000 - 15,400,000\n", + "Rank 0 processing data 15,400,000 - 15,500,000\n", + "Rank 0 processing data 15,500,000 - 15,600,000\n", + "Rank 0 processing data 15,600,000 - 15,700,000\n", + "Rank 0 processing data 15,700,000 - 15,800,000\n", + "Rank 0 processing data 15,800,000 - 15,900,000\n", + "Rank 0 processing data 15,900,000 - 16,000,000\n", + "Rank 0 processing data 16,000,000 - 16,100,000\n", + "Rank 0 processing data 16,100,000 - 16,200,000\n", + "Rank 0 processing data 16,200,000 - 16,300,000\n", + "Rank 0 processing data 16,300,000 - 16,400,000\n", + "Rank 0 processing data 16,400,000 - 16,500,000\n", + "Rank 0 processing data 16,500,000 - 16,600,000\n", + "Rank 0 processing data 16,600,000 - 16,700,000\n", + "Rank 0 processing data 16,700,000 - 16,800,000\n", + "Rank 0 processing data 16,800,000 - 16,900,000\n", + "Rank 0 processing data 16,900,000 - 17,000,000\n", + "Rank 0 processing data 17,000,000 - 17,100,000\n", + "Rank 0 processing data 17,100,000 - 17,200,000\n", + "Rank 0 processing data 17,200,000 - 17,300,000\n", + "Rank 0 processing data 17,300,000 - 17,400,000\n", + "Rank 0 processing data 17,400,000 - 17,500,000\n", + "Rank 0 processing data 17,500,000 - 17,600,000\n", + "Rank 0 processing data 17,600,000 - 17,700,000\n", + "Rank 0 processing data 17,700,000 - 17,800,000\n", + "Rank 0 processing data 17,800,000 - 17,900,000\n", + "Rank 0 processing data 17,900,000 - 18,000,000\n", + "Rank 0 processing data 18,000,000 - 18,100,000\n", + "Rank 0 processing data 18,100,000 - 18,200,000\n", + "Rank 0 processing data 18,200,000 - 18,300,000\n", + "Rank 0 processing data 18,300,000 - 18,400,000\n", + "Rank 0 processing data 18,400,000 - 18,500,000\n", + "Rank 0 processing data 18,500,000 - 18,600,000\n", + "Rank 0 processing data 18,600,000 - 18,700,000\n", + "Rank 0 processing data 18,700,000 - 18,800,000\n", + "Rank 0 processing data 18,800,000 - 18,900,000\n", + "Rank 0 processing data 18,900,000 - 19,000,000\n", + "Rank 0 processing data 19,000,000 - 19,100,000\n", + "Rank 0 processing data 19,100,000 - 19,200,000\n", + "Rank 0 processing data 19,200,000 - 19,300,000\n", + "Rank 0 processing data 19,300,000 - 19,400,000\n", + "Rank 0 processing data 19,400,000 - 19,500,000\n", + "Rank 0 processing data 19,500,000 - 19,600,000\n", + "Rank 0 processing data 19,600,000 - 19,700,000\n", + "Rank 0 processing data 19,700,000 - 19,800,000\n", + "Rank 0 processing data 19,800,000 - 19,900,000\n", + "Rank 0 processing data 19,900,000 - 20,000,000\n", + "Rank 0 processing data 20,000,000 - 20,100,000\n", + "Rank 0 processing data 20,100,000 - 20,200,000\n", + "Rank 0 processing data 20,200,000 - 20,300,000\n", + "Rank 0 processing data 20,300,000 - 20,400,000\n", + "Rank 0 processing data 20,400,000 - 20,500,000\n", + "Rank 0 processing data 20,500,000 - 20,600,000\n", + "Rank 0 processing data 20,600,000 - 20,700,000\n", + "Rank 0 processing data 20,700,000 - 20,800,000\n", + "Rank 0 processing data 20,800,000 - 20,900,000\n", + "Rank 0 processing data 20,900,000 - 21,000,000\n", + "Rank 0 processing data 21,000,000 - 21,100,000\n", + "Rank 0 processing data 21,100,000 - 21,200,000\n", + "Rank 0 processing data 21,200,000 - 21,300,000\n", + "Rank 0 processing data 21,300,000 - 21,400,000\n", + "Rank 0 processing data 21,400,000 - 21,500,000\n", + "Rank 0 processing data 21,500,000 - 21,600,000\n", + "Rank 0 processing data 21,600,000 - 21,700,000\n", + "Rank 0 processing data 21,700,000 - 21,800,000\n", + "Rank 0 processing data 21,800,000 - 21,900,000\n", + "Rank 0 processing data 21,900,000 - 22,000,000\n", + "Rank 0 processing data 22,000,000 - 22,100,000\n", + "Rank 0 processing data 22,100,000 - 22,200,000\n", + "Rank 0 processing data 22,200,000 - 22,300,000\n", + "Rank 0 processing data 22,300,000 - 22,400,000\n", + "Rank 0 processing data 22,400,000 - 22,500,000\n", + "Rank 0 processing data 22,500,000 - 22,600,000\n", + "Rank 0 processing data 22,600,000 - 22,700,000\n", + "Rank 0 processing data 22,700,000 - 22,800,000\n", + "Rank 0 processing data 22,800,000 - 22,900,000\n", + "Rank 0 processing data 22,900,000 - 23,000,000\n", + "Rank 0 processing data 23,000,000 - 23,100,000\n", + "Rank 0 processing data 23,100,000 - 23,200,000\n", + "Rank 0 processing data 23,200,000 - 23,300,000\n", + "Rank 0 processing data 23,300,000 - 23,400,000\n", + "Rank 0 processing data 23,400,000 - 23,500,000\n", + "Rank 0 processing data 23,500,000 - 23,600,000\n", + "Rank 0 processing data 23,600,000 - 23,700,000\n", + "Rank 0 processing data 23,700,000 - 23,800,000\n", + "Rank 0 processing data 23,800,000 - 23,900,000\n", + "Rank 0 processing data 23,900,000 - 24,000,000\n", + "Rank 0 processing data 24,000,000 - 24,100,000\n", + "Rank 0 processing data 24,100,000 - 24,200,000\n", + "Rank 0 processing data 24,200,000 - 24,300,000\n", + "Rank 0 processing data 24,300,000 - 24,400,000\n", + "Rank 0 processing data 24,400,000 - 24,500,000\n", + "Rank 0 processing data 24,500,000 - 24,600,000\n", + "Rank 0 processing data 24,600,000 - 24,700,000\n", + "Rank 0 processing data 24,700,000 - 24,800,000\n", + "Rank 0 processing data 24,800,000 - 24,900,000\n", + "Rank 0 processing data 24,900,000 - 25,000,000\n", + "Rank 0 processing data 25,000,000 - 25,100,000\n", + "Rank 0 processing data 25,100,000 - 25,200,000\n", + "Rank 0 processing data 25,200,000 - 25,300,000\n", + "Rank 0 processing data 25,300,000 - 25,400,000\n", + "Rank 0 processing data 25,400,000 - 25,418,721\n" + ] + } + ], + "source": [ + "step3.run()\n", + "step3.finalize()" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "id": "b1ad4bd3-000c-46db-9529-5e2469c19c41", + "metadata": {}, + "outputs": [], + "source": [ + "step3 = txpipe.extensions.cluster_counts.TXTwoPointClusterSource.make_stage(\n", + " # This file is the input metadetect shear catalog\n", + " shear_tomography_catalog=f\"{my_txpipe_dir}/data/example/outputs_metadetect_20/shear_tomography_catalog.hdf5\",\n", + " binned_lens_catalog=f\"{my_txpipe_dir}data/example/output_binning_20/cluster_catalog_tomography.hdf5\",\n", + " # This is an input training set for the tomographic selection\n", + " binned_shear_catalog=f\"{my_txpipe_dir}data/example/output_shearcalib_20/binned_shear_catalog.hdf5\",\n", + " binned_random_catalog=f\"{my_txpipe_dir}cluster_catalog_tomography.hdf5\",#idc\n", + " patch_centers=f\"{my_txpipe_dir}cluster_catalog_tomography.hdf5\",#idc\n", + " use_randoms= False,\n", + " cluster_profiles = f\"{my_txpipe_dir}data/example/output_txcluster_20/cluster_profiles.pkl\",\n", + " metric=\"Euclidean\",\n", + " var_method=\"shot\"\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "id": "b5164f86-3f37-4c3d-8a73-ba71647cb626", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Running with 4 source bins and 6 lens bins\n", + "Running TreeCorr with metric \"Euclidean\"\n", + "DOING SHEAR-POS\n", + "Skipping lens bin 2 (z=[0.400, 0.600]) and source bin 0 (z=[0.500, 0.700]) — overlap detected.\n", + "Skipping lens bin 3 (z=[0.400, 0.600]) and source bin 0 (z=[0.500, 0.700]) — overlap detected.\n", + "Skipping lens bin 4 (z=[0.600, 0.800]) and source bin 0 (z=[0.500, 0.700]) — overlap detected.\n", + "Skipping lens bin 5 (z=[0.600, 0.800]) and source bin 0 (z=[0.500, 0.700]) — overlap detected.\n", + "Skipping lens bin 4 (z=[0.600, 0.800]) and source bin 1 (z=[0.700, 0.900]) — overlap detected.\n", + "Skipping lens bin 5 (z=[0.600, 0.800]) and source bin 1 (z=[0.700, 0.900]) — overlap detected.\n", + "Running 18 calculations: [(0, 0, 1), (0, 1, 1), (1, 0, 1), (1, 1, 1), (1, 2, 1), (1, 3, 1), (2, 0, 1), (2, 1, 1), (2, 2, 1), (2, 3, 1), (2, 4, 1), (2, 5, 1), (3, 0, 1), (3, 1, 1), (3, 2, 1), (3, 3, 1), (3, 4, 1), (3, 5, 1)]\n", + "Rank 0 making patches for shear catalog bin 0\n", + "Patches already done for cache/patches/TXTwoPointClusterSourcebinned_shear_catalog_f311bd3690704241938f8d17b48910c0/0\n", + "Rank 0 making patches for position catalog bin 0\n", + "Patches already done for cache/patches/TXTwoPointClusterSourcebinned_lens_catalog_af5ce3f975f24e95a25f28e610b87751/0\n", + "Rank 0 making patches for shear catalog bin 1\n", + "Patches already done for cache/patches/TXTwoPointClusterSourcebinned_shear_catalog_f311bd3690704241938f8d17b48910c0/1\n", + "Rank 0 making patches for position catalog bin 1\n", + "Patches already done for cache/patches/TXTwoPointClusterSourcebinned_lens_catalog_af5ce3f975f24e95a25f28e610b87751/1\n", + "Rank 0 making patches for shear catalog bin 2\n", + "Patches already done for cache/patches/TXTwoPointClusterSourcebinned_shear_catalog_f311bd3690704241938f8d17b48910c0/2\n", + "Rank 0 making patches for position catalog bin 2\n", + "Patches already done for cache/patches/TXTwoPointClusterSourcebinned_lens_catalog_af5ce3f975f24e95a25f28e610b87751/2\n", + "Rank 0 making patches for shear catalog bin 3\n", + "Patches already done for cache/patches/TXTwoPointClusterSourcebinned_shear_catalog_f311bd3690704241938f8d17b48910c0/3\n", + "Rank 0 making patches for position catalog bin 3\n", + "Patches already done for cache/patches/TXTwoPointClusterSourcebinned_lens_catalog_af5ce3f975f24e95a25f28e610b87751/3\n", + "Rank 0 making patches for position catalog bin 4\n", + "Patches already done for cache/patches/TXTwoPointClusterSourcebinned_lens_catalog_af5ce3f975f24e95a25f28e610b87751/4\n", + "Rank 0 making patches for position catalog bin 5\n", + "Patches already done for cache/patches/TXTwoPointClusterSourcebinned_lens_catalog_af5ce3f975f24e95a25f28e610b87751/5\n", + "Calculating shear-position bin pair (0,0): 284313 x 147 objects, 0 randoms\n", + "Processing took 0.2 seconds\n", + "Calculating shear-position bin pair (0,1): 284313 x 52 objects, 0 randoms\n", + "Processing took 0.2 seconds\n", + "Calculating shear-position bin pair (1,0): 352597 x 147 objects, 0 randoms\n", + "Processing took 0.3 seconds\n", + "Calculating shear-position bin pair (1,1): 352597 x 52 objects, 0 randoms\n", + "Processing took 0.3 seconds\n", + "Calculating shear-position bin pair (1,2): 352597 x 376 objects, 0 randoms\n", + "Processing took 0.3 seconds\n", + "Calculating shear-position bin pair (1,3): 352597 x 163 objects, 0 randoms\n", + "Processing took 0.3 seconds\n", + "Calculating shear-position bin pair (2,0): 299930 x 147 objects, 0 randoms\n", + "Processing took 0.3 seconds\n", + "Calculating shear-position bin pair (2,1): 299930 x 52 objects, 0 randoms\n", + "Processing took 0.3 seconds\n", + "Calculating shear-position bin pair (2,2): 299930 x 376 objects, 0 randoms\n", + "Processing took 0.3 seconds\n", + "Calculating shear-position bin pair (2,3): 299930 x 163 objects, 0 randoms\n", + "Processing took 0.3 seconds\n", + "Calculating shear-position bin pair (2,4): 299930 x 293 objects, 0 randoms\n", + "Processing took 0.3 seconds\n", + "Calculating shear-position bin pair (2,5): 299930 x 112 objects, 0 randoms\n", + "Processing took 0.3 seconds\n", + "Calculating shear-position bin pair (3,0): 546236 x 147 objects, 0 randoms\n", + "Processing took 0.4 seconds\n", + "Calculating shear-position bin pair (3,1): 546236 x 52 objects, 0 randoms\n", + "Processing took 0.4 seconds\n", + "Calculating shear-position bin pair (3,2): 546236 x 376 objects, 0 randoms\n", + "Processing took 0.4 seconds\n", + "Calculating shear-position bin pair (3,3): 546236 x 163 objects, 0 randoms\n", + "Processing took 0.3 seconds\n", + "Calculating shear-position bin pair (3,4): 546236 x 293 objects, 0 randoms\n", + "Processing took 0.3 seconds\n", + "Calculating shear-position bin pair (3,5): 546236 x 112 objects, 0 randoms\n", + "Processing took 0.4 seconds\n", + "TreeCorr results saved in pickle format to /global/homes/e/edujb/TXPipe/data/example/output_txcluster_20/inprogress_cluster_profiles.pkl\n" + ] + } + ], + "source": [ + "step3.run()\n", + "step3.finalize()" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "id": "3111a320-896b-4d73-8e52-112355b8d033", + "metadata": {}, + "outputs": [], + "source": [ + "step3 = txpipe.extensions.cluster_counts.CLClusterSACC.make_stage(\n", + " # This file is the input metadetect shear catalog\n", + " cluster_profiles=f\"{my_txpipe_dir}data/example/output_txcluster_20/cluster_profiles.pkl\",\n", + " cluster_sacc_catalog=f\"{my_txpipe_dir}data/example/output_txcluster_20/cluster_sacc_catalog.sacc\",\n", + " clmm_profile = False,\n", + " survey_name = \"cosmodc2-20deg\",\n", + " area= 20,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "id": "b09348c9-4f95-405f-bb21-db4a6fe460bf", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "/global/homes/e/edujb/TXPipe/data/example/output_txcluster_20/cluster_profiles.pkl\n", + "{'bin_zbin_0_richbin_0': {'radial_bins': array([ 0.17407202, 0.39933906, 0.92211672, 2.14368839,\n", + " 5.00536064, 11.62815017, 26.9135804 , 62.03356325,\n", + " 137.55238982]), 'xi': array([ 8.64410958e-04, -5.61097243e-03, 3.26284759e-03, 6.14765393e-03,\n", + " 3.12075319e-03, 1.08827062e-03, 3.14641279e-05, -8.90790203e-05,\n", + " -2.64529054e-04]), 'cov': array([[0., 0., 0., 0., 0., 0., 0., 0., 0.],\n", + " [0., 0., 0., 0., 0., 0., 0., 0., 0.],\n", + " [0., 0., 0., 0., 0., 0., 0., 0., 0.],\n", + " [0., 0., 0., 0., 0., 0., 0., 0., 0.],\n", + " [0., 0., 0., 0., 0., 0., 0., 0., 0.],\n", + " [0., 0., 0., 0., 0., 0., 0., 0., 0.],\n", + " [0., 0., 0., 0., 0., 0., 0., 0., 0.],\n", + " [0., 0., 0., 0., 0., 0., 0., 0., 0.],\n", + " [0., 0., 0., 0., 0., 0., 0., 0., 0.]]), 'cluster_bin_edges': {'rich_min': np.float64(5.0), 'rich_max': np.float64(10.0), 'z_min': np.float64(0.2), 'z_max': np.float64(0.4)}, 'n_cl': 147}, 'bin_zbin_0_richbin_1': {'radial_bins': array([ 0.17959957, 0.39932991, 0.9307726 , 2.15006371,\n", + " 4.97603493, 11.5798078 , 26.82704099, 62.05022668,\n", + " 139.29499849]), 'xi': array([ 1.77569055e-02, 1.01210628e-02, 8.14294949e-03, 1.66330840e-02,\n", + " 9.17624896e-03, 2.80010148e-03, 5.01370456e-04, -1.04279009e-04,\n", + " 3.50864218e-05]), 'cov': array([[0., 0., 0., 0., 0., 0., 0., 0., 0.],\n", + " [0., 0., 0., 0., 0., 0., 0., 0., 0.],\n", + " [0., 0., 0., 0., 0., 0., 0., 0., 0.],\n", + " [0., 0., 0., 0., 0., 0., 0., 0., 0.],\n", + " [0., 0., 0., 0., 0., 0., 0., 0., 0.],\n", + " [0., 0., 0., 0., 0., 0., 0., 0., 0.],\n", + " [0., 0., 0., 0., 0., 0., 0., 0., 0.],\n", + " [0., 0., 0., 0., 0., 0., 0., 0., 0.],\n", + " [0., 0., 0., 0., 0., 0., 0., 0., 0.]]), 'cluster_bin_edges': {'rich_min': np.float64(10.0), 'rich_max': np.float64(100.0), 'z_min': np.float64(0.2), 'z_max': np.float64(0.4)}, 'n_cl': 52}, 'bin_zbin_1_richbin_0': {'radial_bins': array([ 0.16736321, 0.39516446, 0.92077874, 2.15145542,\n", + " 5.00889526, 11.6369617 , 26.95390429, 61.91361977,\n", + " 137.24555351]), 'xi': array([ 4.26457595e-03, 1.13355639e-03, -1.45992690e-03, 2.25231941e-03,\n", + " 1.34835278e-03, 3.04857626e-04, 6.97706651e-04, 1.50808217e-04,\n", + " 5.49834442e-05]), 'cov': array([[0., 0., 0., 0., 0., 0., 0., 0., 0.],\n", + " [0., 0., 0., 0., 0., 0., 0., 0., 0.],\n", + " [0., 0., 0., 0., 0., 0., 0., 0., 0.],\n", + " [0., 0., 0., 0., 0., 0., 0., 0., 0.],\n", + " [0., 0., 0., 0., 0., 0., 0., 0., 0.],\n", + " [0., 0., 0., 0., 0., 0., 0., 0., 0.],\n", + " [0., 0., 0., 0., 0., 0., 0., 0., 0.],\n", + " [0., 0., 0., 0., 0., 0., 0., 0., 0.],\n", + " [0., 0., 0., 0., 0., 0., 0., 0., 0.]]), 'cluster_bin_edges': {'rich_min': np.float64(5.0), 'rich_max': np.float64(10.0), 'z_min': np.float64(0.4), 'z_max': np.float64(0.6)}, 'n_cl': 376}, 'bin_zbin_1_richbin_1': {'radial_bins': array([ 0.17500535, 0.39672799, 0.92679162, 2.1539287 ,\n", + " 5.00503284, 11.62855179, 26.97814338, 61.96537126,\n", + " 137.14620707]), 'xi': array([0.01742153, 0.00470805, 0.00488498, 0.0117704 , 0.00415126,\n", + " 0.00166142, 0.00064411, 0.00018481, 0.00013222]), 'cov': array([[0., 0., 0., 0., 0., 0., 0., 0., 0.],\n", + " [0., 0., 0., 0., 0., 0., 0., 0., 0.],\n", + " [0., 0., 0., 0., 0., 0., 0., 0., 0.],\n", + " [0., 0., 0., 0., 0., 0., 0., 0., 0.],\n", + " [0., 0., 0., 0., 0., 0., 0., 0., 0.],\n", + " [0., 0., 0., 0., 0., 0., 0., 0., 0.],\n", + " [0., 0., 0., 0., 0., 0., 0., 0., 0.],\n", + " [0., 0., 0., 0., 0., 0., 0., 0., 0.],\n", + " [0., 0., 0., 0., 0., 0., 0., 0., 0.]]), 'cluster_bin_edges': {'rich_min': np.float64(10.0), 'rich_max': np.float64(100.0), 'z_min': np.float64(0.4), 'z_max': np.float64(0.6)}, 'n_cl': 163}, 'bin_zbin_2_richbin_0': {'radial_bins': array([ 0.17487655, 0.39468837, 0.93361029, 2.15018524,\n", + " 5.00935388, 11.62448185, 26.8921918 , 61.89934972,\n", + " 137.81140784]), 'xi': array([ 1.51588444e-04, -1.43503075e-02, -6.91990439e-04, 2.86779592e-03,\n", + " 1.93133619e-03, 4.92415389e-04, 6.46357350e-04, 1.57059987e-04,\n", + " -2.89902626e-05]), 'cov': array([[0., 0., 0., 0., 0., 0., 0., 0., 0.],\n", + " [0., 0., 0., 0., 0., 0., 0., 0., 0.],\n", + " [0., 0., 0., 0., 0., 0., 0., 0., 0.],\n", + " [0., 0., 0., 0., 0., 0., 0., 0., 0.],\n", + " [0., 0., 0., 0., 0., 0., 0., 0., 0.],\n", + " [0., 0., 0., 0., 0., 0., 0., 0., 0.],\n", + " [0., 0., 0., 0., 0., 0., 0., 0., 0.],\n", + " [0., 0., 0., 0., 0., 0., 0., 0., 0.],\n", + " [0., 0., 0., 0., 0., 0., 0., 0., 0.]]), 'cluster_bin_edges': {'rich_min': np.float64(5.0), 'rich_max': np.float64(10.0), 'z_min': np.float64(0.6), 'z_max': np.float64(0.8)}, 'n_cl': 293}, 'bin_zbin_2_richbin_1': {'radial_bins': array([ 0.17614558, 0.39720335, 0.91199069, 2.15586169,\n", + " 5.00973388, 11.59801179, 26.89036739, 61.87143233,\n", + " 138.32507855]), 'xi': array([ 3.39100114e-02, -4.02752840e-03, 9.00937265e-03, 9.12471705e-03,\n", + " 3.00461238e-03, 1.00720185e-03, 9.39523511e-04, 1.36335542e-04,\n", + " -6.93974228e-05]), 'cov': array([[0., 0., 0., 0., 0., 0., 0., 0., 0.],\n", + " [0., 0., 0., 0., 0., 0., 0., 0., 0.],\n", + " [0., 0., 0., 0., 0., 0., 0., 0., 0.],\n", + " [0., 0., 0., 0., 0., 0., 0., 0., 0.],\n", + " [0., 0., 0., 0., 0., 0., 0., 0., 0.],\n", + " [0., 0., 0., 0., 0., 0., 0., 0., 0.],\n", + " [0., 0., 0., 0., 0., 0., 0., 0., 0.],\n", + " [0., 0., 0., 0., 0., 0., 0., 0., 0.],\n", + " [0., 0., 0., 0., 0., 0., 0., 0., 0.]]), 'cluster_bin_edges': {'rich_min': np.float64(10.0), 'rich_max': np.float64(100.0), 'z_min': np.float64(0.6), 'z_max': np.float64(0.8)}, 'n_cl': 112}}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "NOTE/WARNING: Expected output file cluster_shear_catalogs.hdf5 was not generated.\n" + "NOTE/WARNING: Expected output file /global/homes/e/edujb/TXPipe/data/example/output_txcluster_20/cluster_sacc_catalog.sacc was not generated.\n" ] } ], @@ -675,6 +1622,65 @@ "step3.run()\n", "step3.finalize()" ] + }, + { + "cell_type": "code", + "execution_count": 39, + "id": "4963f49d-19d0-461d-936d-7bcc29331bd4", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[np.float64(0.0008644109582250803), np.float64(-0.00561097243342368), np.float64(0.0032628475852526403), np.float64(0.006147653932011068), np.float64(0.0031207531930728103), np.float64(0.0010882706235916815), np.float64(3.1464127947719736e-05), np.float64(-8.907902029290128e-05)]\n", + "0.0008644109582250803\n", + "[np.float64(0.0177569055495197), np.float64(0.01012106280328318), np.float64(0.00814294949332071), np.float64(0.01663308397760685), np.float64(0.009176248960324013), np.float64(0.002800101482086858), np.float64(0.0005013704562857116), np.float64(-0.00010427900925364384)]\n", + "0.0177569055495197\n", + "[np.float64(0.004264575949335864), np.float64(0.001133556390755071), np.float64(-0.0014599269019609108), np.float64(0.0022523194088706487), np.float64(0.001348352776352251), np.float64(0.00030485762637805165), np.float64(0.0006977066507798933), np.float64(0.00015080821717624636)]\n", + "0.004264575949335864\n", + "[np.float64(0.01742152912270232), np.float64(0.004708047587156523), np.float64(0.00488497714341866), np.float64(0.011770400393957808), np.float64(0.004151263825991062), np.float64(0.0016614196923270773), np.float64(0.0006441135853125488), np.float64(0.00018481328388700172)]\n", + "0.01742152912270232\n" + ] + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAlQAAAGwCAYAAABvpfsgAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8fJSN1AAAACXBIWXMAAA9hAAAPYQGoP6dpAAB1tklEQVR4nO3dd3hTZfsH8G+6B6VQoC2FUgqUPYQiUJChLEGU5ct8GYL8rMisyMtShlpQpoCAUhR89RVUxIkKypK9WpkiS4rQWqDQQgstbc/vj8eTJmnS5CQnSdN+P9d1riRnPjmgubmf+zyPRpIkCURERERkNTdnN4CIiIjI1TGgIiIiIrIRAyoiIiIiGzGgIiIiIrIRAyoiIiIiGzGgIiIiIrIRAyoiIiIiG3k4uwFlRUFBAa5fv46AgABoNBpnN4eIiIgsIEkS7t69i7CwMLi5mc5DMaBykOvXryM8PNzZzSAiIiIrXL16FdWrVze5nQGVgwQEBAAQfyDly5d3cmuIiIjIEpmZmQgPD9f+jpvCgMpB5G6+8uXLM6AiIiJyMebKdViUTkRERGQjBlRERERENmJARURERGQj1lARERE5UEFBAXJzc53dDPqHp6cn3N3dbT4PAyoiIiIHyc3NxeXLl1FQUODsppCOChUqIDQ01KZxIhlQEREROYAkSUhJSYG7uzvCw8OLHSSSHEOSJGRnZyMtLQ0AULVqVavPxYCKiIjIAfLy8pCdnY2wsDD4+fk5uzn0D19fXwBAWloagoODre7+Y3hMRETkAPn5+QAALy8vJ7eEDMkB7sOHD60+BwMqIiIiB+J8riWPGn8mDKiIiIiIbMSAioiIiMhGDKiIiIjIpE6dOmHSpEkmt9esWRPLli1zWHtKKgZULu7GDeDPP4F795zdEiIiKouOHDmC//u//1PtfMnJyXj66afh7++PypUrY8KECYoGQj158iQ6duwIX19fVKtWDfPmzYMkSaq1zxQOm+Dihg4Ftm8HPv5YvCciInKkKlWqqHau/Px8PPXUU6hSpQr27t2LW7duYcSIEZAkCStWrDB7fGZmJrp27YrHH38cR44cwR9//IGRI0fC398fL7/8smrtNIYZKhcnD5eRl+fcdhARkTKSBGRlOWdRmrDJy8vDuHHjUKFCBVSqVAmzZs3SZn0Mu/w0Gg0SEhLQt29f+Pn5ISoqCt98841F19m2bRvOnDmDjz/+GM2bN0eXLl2wePFirF27FpmZmWaP/+STT/DgwQOsX78ejRs3Rr9+/TBjxgwsWbLE7lkqBlQuzuOfHCMDKiIi15KdDZQr55wlO1tZWzds2AAPDw8cOnQIy5cvx9KlS5GQkGBy/7lz52LAgAE4ceIEevbsiaFDhyI9Pd3sdQ4cOIDGjRsjLCxMu6579+7IycnBsWPHLDq+Y8eO8Pb21jv++vXr+PPPP80ebwsGVC5ODqj+GS+OiIhIdeHh4Vi6dCnq1auHoUOHYvz48Vi6dKnJ/UeOHInBgwejTp06iI+PR1ZWFg4fPmz2OqmpqQgJCdFbV7FiRXh5eSE1NdWq4+XPlhxvC9ZQubiMDPF644Zz20FERMr4+TnvgSKlM9+0adNGb/DLmJgYLF68WDv6u6GmTZtq3/v7+yMgIEA7X545xgbZlCTJ4sE3DfeTu/rsPaAqAyoXd/aseP39d+e2g4iIlNFoAH9/Z7fCPjw9PfU+azQaFBQUmD0uNDQUhw4d0lt3+/ZtPHz4sEjmydTxhpkoOZCz5HhbsMvPxckBN2uoiIjIXg4ePFjkc1RUlNUTCZsSExODU6dOISUlRbtu27Zt8Pb2RnR0tEXH79mzR2+YhW3btiEsLAw1a9ZUta2GGFC5OPnvMmuoiIjIXq5evYq4uDicO3cOn376KVasWIGJEyeqfp1u3bqhYcOGGDZsGBITE/HLL79gypQpGDNmDMqXL2/2+CFDhsDb2xsjR47EqVOnsGXLFsTHxyMuLo5dflQ8t39CYgZURERkL8OHD8f9+/fRqlUruLu7Y/z48aoO5ilzd3fH999/j7Fjx6Jdu3bw9fXFkCFDsGjRIouODwwMxPbt2/HSSy+hZcuWqFixIuLi4hAXF6d6Ww1pJEcMH0rIzMxEYGAgMjIyLIqyLRUZKUZKf+YZ4OuvVTstERGp7MGDB7h8+TIiIyPh4+Pj7OaQjuL+bCz9/WaXn4tjDRUREZHzMaBycayhIiIiVxEbG4ty5coZXWJjY80e36NHD5PHx8fHO+AbmMYaKhfHGioiInIV8+bNw5QpU4xus6QcJiEhAffv3ze6LSgoyKa22YoBlYtjQEVERK4iODgYwcHBVh9frVo1FVujLnb5uThOjkxEROR8DKhcHDNUREREzseAysXJGSoLRvTXc/s20KQJMGuW+m0iIiIqaxhQuThrM1QHDwKnTgHvvgtwJDIiIiLbMKBycdYOm3Drlni9c0cMDEpERETWY0Dl4qwNqG7eLHx//Lh67SEiotKlU6dOmDRpksntNWvWxLJlyxzWnpKKAZWLk7v8lNZQyRkqgAEVERFZ78iRI6rO65ecnIynn34a/v7+qFy5MiZMmIDc3FyLjn3w4AFGjhyJJk2awMPDA3369FGtXeZwHCoXZ2uXH8CAioiIrFelShXVzpWfn4+nnnoKVapUwd69e3Hr1i2MGDECkiRhxYoVFh3v6+uLCRMmYPPmzaq1yxLMULk4j39CYlsDKhamExE5liQBWVnOWZT+Pz8vLw/jxo1DhQoVUKlSJcyaNQvSPycx7PLTaDRISEhA37594efnh6ioKHzzzTcWXWfbtm04c+YMPv74YzRv3hxdunTB4sWLsXbtWmRmZpo93t/fH6tXr8aYMWMQGhqq7EvaiAGVi7P2KT/dgCotDUhJUa9NRERkXnY2UK6cc5bsbGVt3bBhAzw8PHDo0CEsX74cS5cuRUJCgsn9586diwEDBuDEiRPo2bMnhg4divT0dLPXOXDgABo3boywsDDtuu7duyMnJwfHjh1T1mgHY0Dl4qwdh0o3oALY7UdERKaFh4dj6dKlqFevHoYOHYrx48dj6dKlJvcfOXIkBg8ejDp16iA+Ph5ZWVk4fPiw2eukpqYiJCREb13FihXh5eWF1NRUm7+HPbGGysXZGlDVqwecOycCql691G0bERGZ5ucH3LvnvGsr0aZNG2g0Gu3nmJgYLF68GPkmukeaNm2qfe/v74+AgACkpaVZdC3d68gkSTK6viRhQOXibK2h6tq1MKAiIiLH0WgAf39nt8I+PD099T5rNBoUWPAv/9DQUBw6dEhv3e3bt/Hw4cMimauShl1+Ls6aDNWDB4X95127itfERHXbRUREpcfBgweLfI6KioK7/COkkpiYGJw6dQopOoW927Ztg7e3N6Kjo1W9ltoYULk4awIqOTvl7g506CDeJyfrD/ZJREQku3r1KuLi4nDu3Dl8+umnWLFiBSZOnKj6dbp164aGDRti2LBhSExMxC+//IIpU6ZgzJgxKF++vEXnOHPmDJKSkpCeno6MjAwkJSUhKSlJ9bYaYpefi7MloKpUCahQAahTB7hwQWSp5IwVERGRbPjw4bh//z5atWoFd3d3jB8/XtXBPGXu7u74/vvvMXbsWLRr1w6+vr4YMmQIFi1aZPE5evbsiStXrmg/N2/eHAC0wzzYCwMqFyfXUFkbUAFAixYioDp+nAEVERHp27Vrl/b96tWri2z/02BCWGOBy507dyy+Xo0aNfDdd99ZvL+59jgKu/xcnFoBFcDCdCIiImsxoHJxtnb5AcA/2VAWphMRkV3FxsaiXLlyRpfY2Fizx/fo0cPk8fHx8Q74Bqaxy8/FWZOhkovPDQOq8+eBzEzAwro/IiIiRebNm4cpU6YY3WZJ0XlCQgLu379vdFtQUJBNbbMVAyoXp0aXX5UqQHg4cPUqkJRU+OQfERGRmoKDgxEcHGz18dWqVVOxNepil5+LU6PLD2AdFRERkS0YULk4OUOl5GlQBlRERETqcrmAatWqVYiMjISPjw+io6Px66+/Frv/7t27ER0dDR8fH9SqVQtr1qzR23769Gn0798fNWvWhEajwbJly4qcY86cOdBoNHpLaGioml/Lamp0+QEsTCciIrKFSwVUmzZtwqRJkzBz5kwkJiaiffv26NGjB5KTk43uf/nyZfTs2RPt27dHYmIiZsyYgQkTJmDz5s3afbKzs1GrVi0sWLCg2CCpUaNGSElJ0S4nT55U/ftZQ+0M1ZkzhdPSEBERkWVcKqBasmQJRo8ejeeffx4NGjTAsmXLEB4ebnSgMQBYs2YNatSogWXLlqFBgwZ4/vnnMWrUKL0RVx999FEsXLgQgwYNgre3t8lre3h4IDQ0VLtUqVJF9e9nDbUyVGFhQHCwOE8JiRWJiIhchssEVLm5uTh27Bi6deumt75bt27Yv3+/0WMOHDhQZP/u3bvj6NGjePjwoaLrnz9/HmFhYYiMjMSgQYNw6dKlYvfPyclBZmam3mIPSjNU+fnA7dvivW5ApdGwjoqIiIrq1KkTJk2aZHJ7zZo1jZbLlDUuE1DdvHkT+fn5CAkJ0VsfEhKC1NRUo8ekpqYa3T8vLw83FcwE3Lp1a3z00Uf46aefsHbtWqSmpqJt27a4Jad6jJg/fz4CAwO1S3h4uMXXU8LTU7xamqG6c6cw+DIcsoMBFRERKXXkyBFV5/WbOHEioqOj4e3tjUceeUTx8SdPnkTHjh3h6+uLatWqYd68eXafxw9woYBKptFo9D5LklRknbn9ja0vTo8ePdC/f380adIEXbp0wffffw8A2LBhg8ljpk+fjoyMDO1y9epVi6+nhNIMlRwDBgQAXl7621iYTkRESlWpUgV+fn6qnU+SJIwaNQoDBw5UfGxmZia6du2KsLAwHDlyBCtWrMCiRYuwZMkS1dpnissEVJUrV4a7u3uRbFRaWlqRLJQsNDTU6P4eHh6opNvfpZC/vz+aNGmC8+fPm9zH29sb5cuX11vswdqAqnLlotvkDNXJk0Buru1tIyIi0yQJyMpyzqI0YZOXl4dx48ahQoUKqFSpEmbNmqVNUBh2+Wk0GiQkJKBv377w8/NDVFQUvvnmG4uvtXz5crz00kuoVauWskYC+OSTT/DgwQOsX78ejRs3Rr9+/TBjxgwsWbLE7lkqlwmovLy8EB0dje3bt+ut3759O9q2bWv0mJiYmCL7b9u2DS1btoSn3FdmhZycHJw9exZVq1a1+hxq8dAZ696Sbj9jBemyyEggMFAEU2fOqNM+IiIyLjsbKFfOOYvSp7k3bNgADw8PHDp0CMuXL8fSpUuRkJBgcv+5c+diwIABOHHiBHr27ImhQ4ciPT3dxjtm3oEDB9CxY0e9h8y6d++O69ev488//7TrtV0moAKAuLg4JCQk4IMPPsDZs2cxefJkJCcnaydUnD59OoYPH67dPzY2FleuXEFcXBzOnj2LDz74AOvWrdObRyg3NxdJSUlISkpCbm4url27hqSkJFy4cEG7z5QpU7B7925cvnwZhw4dwrPPPovMzEyMGDHCcV/eBN24MD/f/P7FBVQsTCciImPCw8OxdOlS1KtXD0OHDsX48eOxdOlSk/uPHDkSgwcPRp06dRAfH4+srCwcPnzY7u00VTstb7Mnl5rLb+DAgbh16xbmzZuHlJQUNG7cGFu3bkVERAQAICUlRW9MqsjISGzduhWTJ0/Gu+++i7CwMCxfvhz9+/fX7nP9+nU0l4uHACxatAiLFi1Cx44dsWvXLgDAX3/9hcGDB+PmzZuoUqUK2rRpg4MHD2qv60y6Gaq8PP0Ay5jiAipABFQ7d4qAatQoddpIRERF+fkB9+4579pKtGnTRq/2OCYmBosXL0a+iX/JN23aVPve398fAQEBSEtLs6qtSqlRO20NlwqoAGDs2LEYO3as0W3r168vsq5jx444Xky6pWbNmmb7VTdu3KiojY6kZoYKYGE6EZGjaDSAv7+zW2EfhmU1Go0GBUoGTLSSqdppACbrrdXiUl1+VJRhhsocebSI4jJUAJCUZFmARkREpd/BgweLfI6KioK7u7uTWmRcTEwM9uzZg1ydJ6u2bduGsLAw1KxZ067XZkDl4nT/EWBJQGUuQ1W3rkgFZ2cDf/xhe/uIiMj1Xb16FXFxcTh37hw+/fRTrFixAhMnTrTLtS5cuICkpCSkpqbi/v37enXO5gwZMgTe3t4YOXIkTp06hS1btiA+Ph5xcXHs8qPiKc1QmQuo3N2BRx4B9u8XdVQNGtjcRCIicnHDhw/H/fv30apVK7i7u2P8+PGqDuap6/nnn8fu3bu1n+U658uXL5vNMgUGBmL79u146aWX0LJlS1SsWBFxcXGIi4uzS1t1MaBycW46OUY1aqgA0e0nB1RDh9rWPiIicm3yA1oAjM6dazgcgbG65Dt37lh1PWs0adIEe/bssekc1mCXn4vT7b5WI0MFsDCdiIhIKQZULk43Q2UuoJIkyzNUgMhQOWD6IyIiKiNiY2NRrlw5o4s8pmRxevToYfL4+Ph4B3wD09jl5+KUBFTZ2UBOjnhfXEDVsKGY5y8jA7h8GbBi9H8iIqIi5s2bpze4ti5LpmhLSEjA/fv3jW4LCgqyqW22YkDl4pQEVHJ2ytNTTD1gipcX0KQJcOyYyFIxoCIiIjUEBwcjODjY6uOrVaumYmvUxS4/F6dbQ2WuKF23u8/c06Nytx/rqIiIiMxjQOXirMlQFdfdJ+OcfkRERJZjQOXirAmoKlc2f175Sb9jx1iYTkREZA4DKhdnrwxV06aiO/HGDeD6devbR0REVBYwoHJx1tZQmePrWzhKOrv9iIiIiseAysXZK0MFsDCdiIiATp06YdKkSSa316xZE8uWLXNYe0oqBlQuzhEBFTNURERkypEjR1Sd12/ixImIjo6Gt7c3HnnkEUXHPnjwACNHjkSTJk3g4eGBPn36qNYucxhQuTglAdXNm+LV0oBKLkxnQEVERKZUqVIFfn5+qp1PkiSMGjUKAwcOVHxsfn4+fH19MWHCBHTp0kW1NlmCAZWLUzKXn9IMlfwPg6tXRXE6ERGpR5KArCznLEqf3s7Ly8O4ceNQoUIFVKpUCbNmzdJOgmzY5afRaJCQkIC+ffvCz88PUVFR+Oabbyy+1vLly/HSSy+hlhWjSvv7+2P16tUYM2YMQkNDFR9vCwZULk43Q6VmUToAlC8PREWJ96yjIiJSV3a2mLXCGUt2trK2btiwAR4eHjh06BCWL1+OpUuXIiEhweT+c+fOxYABA3DixAn07NkTQ4cORXp6uo13rGRjQOXi7FlDBbAwnYiIgPDwcCxduhT16tXD0KFDMX78eCxdutTk/iNHjsTgwYNRp04dxMfHIysrC4cPH3Zgix2Pc/m5OEsDqrw8MdkxoDyg2rSJdVRERGrz8wPu3XPetZVo06YNNDpzlsXExGDx4sXIN9E10rRpU+17f39/BAQEIC0tzaq2ugoGVC7O0hoq3UxrxYqWn5+F6URE9qHRAP7+zm6FfXh6eup91mg0KCgocFJrHINdfi7O0hoqubuvQgXAQ0EYLQdUFy4UZriIiKhsOXjwYJHPUVFRcNf9V30Zx4DKxVna5WdN/RQg5v2rUUO8T0pSdiwREZUOV69eRVxcHM6dO4dPP/0UK1aswMSJE+1yrQsXLiApKQmpqam4f/8+kpKSkJSUhNzcXIuOP3PmDJKSkpCeno6MjAzt8fbGLj8XZ++AChB1VMnJojC9Y0flxxMRkWsbPnw47t+/j1atWsHd3R3jx49XdTBPXc8//zx2796t/dz8n66Sy5cvo2bNmmaP79mzJ65cuVLkeEnpWBEKMaBycZbWUMkBVeXKyq/RogXw1VesoyIiKot27dqlfb969eoi2//880+9z8YClzt37lh1PWsYtsdR2OXn4pTWUFmToWJhOhERUfEYULk4R3X5AcDZs8oHgyMiIpLFxsaiXLlyRpfY2Fizx/fo0cPk8fHx8Q74Bqaxy8/FKe3ysyagqloVCAkB/v4bOHECaNNG+TmIiIjmzZuHKVOmGN1Wvnx5s8cnJCTg/v37RrcFBQXZ1DZbMaBycY7IUGk0Ikv1ww+iMJ0BFRERWSM4OBjBwcFWH1+tWjUVW6Mudvm5ON2A6uFD0/vdvClerQmogMJuP9ZRERHZxt5Pm5FyavyZMKBycY4oSgdYmE5EZCt5EExLx1Mix8n+p0DYcIR3Jdjl5+J0a6iKy1DZGlDJGaqTJ4HcXMDLy7rzEBGVVR4eHvDz88ONGzfg6ekJNzfmNJxNkiRkZ2cjLS0NFSpUsGnkdwZULs6SLj9Jsj2gqllTTFtz5w5w+nRhxoqIiCyj0WhQtWpVXL58WW/gSXK+ChUqIDQ01KZzMKBycboBlaks8t27hQXr1gZUcmH6jh2iMJ0BFRGRcl5eXoiKimK3Xwni6empypyEDKhcnCU1VHJ2yscH8POz/lpyQHX8ODBqlPXnISIqy9zc3ODj4+PsZpDKFHXg5uXlwcPDA6dOnbJXe0ghS2qobO3uk7EwnYiIyDhFAZWHhwciIiKQX9zjZORQltRQqRVQyYXpSUnFP1FIRERU1ih+xGDWrFmYPn060tPT7dEeUsiSgT1NBVQFBcquFRUF+PsD9+8D584pO5aIiKg0U1xDtXz5cly4cAFhYWGIiIiAv7+/3vbj7A9yKGszVH//DTzyCNC3L7BqlWXXcncXx+zbJwrTGza0psVERESlj+KAqk+fPnZoBllLSVF65cqF6w4dAlJTgY8/Blau1D9PcVq0EAHV8ePA0KHWtZmIiKi0URxQzZ492x7tIBtoNGKsKaUZKkAMqXD5MlC7tmXXYmE6ERFRURymtRTQaMSrkhqq1NTC94mJll9Ld04/pTVYREREpZXigCo/Px+LFi1Cq1atEBoaiqCgIL2FHE/urrMmQwUoC6gaNhTTzmRmiswWERERWRFQzZ07F0uWLMGAAQOQkZGBuLg49OvXD25ubpgzZ44dmkjmyAGVuRoqNTJUnp5A06bKjyMiIirNFAdUn3zyCdauXYspU6bAw8MDgwcPRkJCAl577TUcPHjQHm0kM8x1+d28KV5NZaiSkpRdT7fbj4iIiKwIqFJTU9GkSRMAQLly5ZCRkQEA6NWrF77//nt1W0cWsbWGKiVFP8Ayh4XpRERE+hQHVNWrV0dKSgoAoE6dOti2bRsA4MiRI/D29la3dWQRefoZYwFVbi5w7554byxD5esrXq0tTJckZW0lIiIqjRQHVH379sUvv/wCAJg4cSJeffVVREVFYfjw4RjFGXOdQs5QGauhkrNTbm5AhQrifXa2GC4BAB5/XLwqCaiaNBFB3I0bwLVrVjWZiIioVFE8DtWCBQu075999llUr14d+/fvR506dfDMM8+o2jiyjFyUbixDJQdUFSsW7idnp3x8gA4dgK1bldVR+fqKp/1OnhSBWPXqVjediIioVFAcUBlq06YN2rRpo0ZbyErF1VAVVz8VGlpYD6X0ib0WLURAdfw48PTTyo4lIiIqbawa2PO///0v2rVrh7CwMFy5cgUAsGzZMnz99deqNo4sU1wNVXFjUIWEFAZU588XdgNagoXpREREhRQHVKtXr0ZcXBx69uyJO3fuIP+fwp0KFSpg2bJlarePLFDcOFTmMlRVqgDVqonPv/1m+TU5dAIREVEhxQHVihUrsHbtWsycORPucmoEQMuWLXHy5ElVG0eWURpQ6WaoAOCRR8Srkjoq+Zi//hLF6URERGWZ4oDq8uXLaC739+jw9vZGVlaWKo0iZSwpSjeVoQKsq6MKCADq1lV+HBERUWmkOKCKjIxEkpFUxg8//ICGDRuq0SZSyJIMVeXKhesMM1S2FKYD7PYjIiJS/JTfK6+8gpdeegkPHjyAJEk4fPgwPv30U8yfPx8JCQn2aCOZYWuGSu6+O3VKDATq5WXZdVu0ADZuZEBFRESkOKB67rnnkJeXh6lTpyI7OxtDhgxBtWrV8M4772DQoEH2aCOZIZeyFRQU3WZJDVVkJBAYCGRkAGfPAs2aWXZdPulHREQkWDVswpgxY3DlyhWkpaUhNTUVV69exejRo9VuG1nIlqf8ADGOlZylUtLtJwdUFy+KYIyIiKissiqgklWuXBnBwcFqtYWspGQcqnv3xNQzQGGGCrCujqpSJSAiQrxX8oQgERFRaaM4oPr7778xbNgwhIWFwcPDA+7u7noLOZ6pDFVBQdGASs5O+fsD5coV7mtNhgpgYToRERFgRQ3VyJEjkZycjFdffRVVq1aFRp73hJxGjmMNA6qMjMK6KjmgMqyfkskZqqQkcYybhaF2ixbAli0MqIiIqGxTnKHau3cvPvnkE7z44ovo06cPevfurbfY26pVqxAZGQkfHx9ER0fj119/LXb/3bt3Izo6Gj4+PqhVqxbWrFmjt/306dPo378/atasCY1GY3K0d6XXdSQ5+DEsSpezU/7+gLe3eG9YPyVr0EDsc/cucPmy5ddmYToREZEVAVV4eDgkSbJHW8zatGkTJk2ahJkzZyIxMRHt27dHjx49kJycbHT/y5cvo2fPnmjfvj0SExMxY8YMTJgwAZs3b9buk52djVq1amHBggUINYwyrLyuo3n8k2fMywN0/2gsecJP5ukJNG4s3ivp9pO7/H7/vbA2i4iIqKxRHFAtW7YM06ZNw59//mmH5hRvyZIlGD16NJ5//nk0aNAAy5YtQ3h4OFavXm10/zVr1qBGjRpYtmwZGjRogOeffx6jRo3CokWLtPs8+uijWLhwIQYNGgRvOY1j43UBICcnB5mZmXqLvcjjRhUUFAZMgGVP+OmyZgqaqlXFuQoKgBMnLD+OiIioNLEooKpYsSKCgoIQFBSEQYMGYdeuXahduzYCAgK06+XFXnJzc3Hs2DF069ZNb323bt2wf/9+o8ccOHCgyP7du3fH0aNH8fDhQ7tdFwDmz5+PwMBA7RIeHm7R9ayh+yyAbnedkgwVwBHTiYiIrGVRUbqpuiJHunnzJvLz8xFiEAmEhIQgVU67GEhNTTW6f15eHm7evImqVava5boAMH36dMTFxWk/Z2Zm2i2o0g2oLl0CYmLEe6UZKlsCqq1bGVAREVHZZVFANWLECHu3w2KGTxVKklTsk4bG9je2Xu3rent7m+xCVJvuE3mXLhW+V5qhatpUDPKZkiL2M7aPMSxMJyKisk5xDdXx48dx8uRJ7eevv/4affr0wYwZM5Cbm6tq43RVrlwZ7u7uRbJCaWlpRbJHstDQUKP7e3h4oJJulKHydR1NSUBVXIaqXDkgKkq8V1JHJXf5yXMBEhERlTWKA6oXXngBf/zxBwDg0qVLGDhwIPz8/PD5559j6tSpqjdQ5uXlhejoaGzfvl1v/fbt29G2bVujx8TExBTZf9u2bWjZsiU8PT3tdl1HMxdQVa4sXiWp+AwVYF23X0QEULEi8PAhcPq05ccRERGVFooDqj/++AOP/PM42Oeff46OHTvif//7H9avX683HIE9xMXFISEhAR988AHOnj2LyZMnIzk5GbGxsQBE3dLw4cO1+8fGxuLKlSuIi4vD2bNn8cEHH2DdunWYMmWKdp/c3FwkJSUhKSkJubm5uHbtGpKSknDhwgWLr+tshjVUMsMMVWYm8OCBeK9mQKXRsDCdiIjKNsUjpUuShIJ/RpD8+eef0atXLwBifKqbN2+q2zoDAwcOxK1btzBv3jykpKSgcePG2Lp1KyL+mVAuJSVFb2yoyMhIbN26FZMnT8a7776LsLAwLF++HP3799fuc/36dTSXowgAixYtwqJFi9CxY0fs2rXLous6m26G6to1ICdHDNJpGFDJ2amAAMDPz/i5bClM/+UXEVBxnmwiIiprNJLCUTqfeOIJhIeHo0uXLhg9ejTOnDmDOnXqYPfu3RgxYoRTxqdyBZmZmQgMDERGRgbKly+v6rl79AB+/FGMR5WbC5w7B9StC9SoAVy9Chw+DDz6KLBnD9Cxo6iT+qfXtoi0NJG90mjE1DUBAZa14dNPgSFDgDZtgAMH1PtuREREzmTp77dVA3seP34c48aNw8yZM1GnTh0AwBdffFFiaorKGjlDVaWKeJW7/UxlqIqrpQ8OBsLCRL2VkoE65S6/334rOqcgERFRaae4y69p06Z6T/nJFi5cCHfdYh5yGPm2V6kiuvwuXQLu3y+cCkYOqIp7wk9X8+bA9eui269dO8vaEBUlnhK8d09kyBo2VP49iIiIXJXiDJUpPj4+Fj85R+qSM1Ty03yXLhVmpzw8ADlDaUmGCiicgkZJHZWbW+FxLEwnIqKyRrWAipxHDqjkTJRuQBUUJOqhAGUZKkDZWFRAYbffsWPKjiMiInJ1DKhKATmgkqdS1A2oLB0lXZccUJ06JcaWslTr1uJ10ybR5UhERFRWMKAqBeQaKjmgunxZ+SjpuiIjRTdhbi5w5ozl7ejfXwzymZICrFxp+XFERESujgFVKSBnqCpUEK+ZmYXDIliTodJorKuj8vYG5swR7xcsEMMuEBERlQUWPeUXFxdn8QmXLFlidWPIOnJA5e4uhjy4fh04ckSskwMq3WlnzGWoANHtt2eP8jqqf/8beOst4PffgcWLgXnzlB1PRETkiiwKqBIN0hTHjh1Dfn4+6tWrB0BMR+Pu7o7o6Gj1W0hmyQFVQQFQq5bxgOrOncKJi4ODzZ/T2hHTPTyAN94Ann0WWLIEGDfOsusRERG5MosCqp07d2rfL1myBAEBAdiwYQMqVqwIALh9+zaee+45tG/f3j6tpGLJNVRyQLV3rwiqgKJjUFWoAPj4mD+n7pN+BQX609uY068fEB0tnvabPx9YutTyY4mIiFyR4hqqxYsXY/78+dpgCgAqVqyIN954A4sXL1a1cWQZOdjJzxcBlS4lo6TratBATGWTmQkonU1IowHi48X7VasAnekViYiISiXFAVVmZib+ln+ddaSlpeHu3buqNIqU0e3yi4zU3yYP9mnpE34yT0+gcWPxXmm3HwB07SrmDczNBV5/XfnxRERErkRxQNW3b18899xz+OKLL/DXX3/hr7/+whdffIHRo0ejX79+9mgjmWHY5afL2gwVYH0dFaCfpfrwQ9OTMRMREZUGigOqNWvW4KmnnsK///1vREREICIiAkOHDkWPHj2watUqe7SRzDAsSteldB4/XbYEVADQti3Qq5foinztNevOQURE5AoUB1R+fn5YtWoVbt26hcTERBw/fhzp6elYtWoV/P397dFGMkO3hio0VL/o3JYMlTwWldKhE3S9+aZ43bTJ+sCMiIiopLN6YM+UlBSkpKSgbt268Pf3hyRJaraLFNDNULm56ddRyaOnW5OhatZMdN1dvw6kpVnXtqZNgcGDxftZs6w7BxERUUmnOKC6desWOnfujLp166Jnz55ISUkBADz//PN4+eWXVW8gmadbQwUUdvuVLy+KywHrMlTlygFRUeK9LdmluXNFG7duFUM6EBERlTaKA6rJkyfD09MTycnJ8PPz064fOHAgfvzxR1UbR5bRzVABhQGVNfP4GbK1jgoQQdno0eL99Oli1HYiIqLSRHFAtW3bNrz11luoXr263vqoqChcuXJFtYaR5XRrqIDCLj85oCooKOyyU5KhAtSpowKAV18Vc/3t3Qsw7iYiotJGcUCVlZWll5mS3bx5E97e3qo0ipQxzFA99ph4bdFCvKanA3l54r3SaWDUyFABQPXqYhoaAJg5s7CtREREpYHigKpDhw746KOPtJ81Gg0KCgqwcOFCPP7446o2jixjWEP16KOikFwexUKunwoKEqOfKyFnqM6fB+7ds62d06YBAQEiONu82bZzERERlSSKA6qFCxfivffeQ48ePZCbm4upU6eicePG2LNnD9566y17tJHMMMxQAUDVqoWBlrX1U4DoIqxaVdQ9/fabbe2sXBmQn1t49dXCrBkREZGrUxxQNWzYECdOnECrVq3QtWtXZGVloV+/fkhMTETt2rXt0UYyw7CGypA1T/jp0p0o2VaTJ4varnPnAJ1EJxERkUvzsOag0NBQzJ07V+22kJWMZah02ZKhAkRAtXWrOgNzli8PzJghMlVz5gBDhugPREpEROSKFGeoPvzwQ3z++edF1n/++efYsGGDKo0iZQxrqAyplaFSa6TzF18EqlUDrl4F3ntPnXMSERE5k+KAasGCBahcuXKR9cHBwYiXZ8MlhzLX5WdrhkouTD91Cnj40Lpz6PL1BWbPFu/ffBO4e9f2cxIRETmT4oDqypUriNSd2+QfERERSE5OVqVRpIy5Lj9bM1SRkaKrLjcXOHvWunMYGjkSqFMHuHEDeOcddc5JRETkLIoDquDgYJw4caLI+t9++w2VdIfmJoexdw2Vm1thlkqtbj9PT2DePPF+4ULg1i11zktEROQMigOqQYMGYcKECdi5cyfy8/ORn5+PHTt2YOLEiRg0aJA92khm2LuGClC/jgoABg4UkydnZgJvv63eeYmIiBxNcUD1xhtvoHXr1ujcuTN8fX3h6+uLbt264YknnmANlZMUV0OVn1847Yy1GSpAvSlodLm5iRoqAFi+XAxGSkRE5IoUB1ReXl7YtGkTfv/9d3zyySf48ssvcfHiRXzwwQfwUjoMN6miuC6/W7fEeo0GqFLF+mvojkWl5uTGTz0FtG0LPHgAvPGGeuclIiJyJKvGoQKAunXrom7dumq2haxUXJefXD9VuTLgYfWfNtCwoZi2JiMDuHwZqFXL+nPp0miA+HigUydg7VpgyhT1zk1EROQoin9iR40aVez2Dz74wOrGkHWKy1CpUT8FiCLyxo2B48dFHZWaQU/HjkD37sBPP4nhFP77X/XOTURE5AiKu/xu376tt6SlpWHHjh348ssvcefOHTs0kcwprobK1if8dNmjjkom11J98okY74qIiMiVKM5Qbdmypci6goICjB07FrXYV+MUjshQAfZ50k8WHQ08+yzwxRfArFnAV1+pfw0iIiJ7UZyhMnoSNzdMnjwZS5cuVeN0pJAlNVRqZKjsGVABYlwqNzfg66+BQ4fscw0iIiJ7UCWgAoCLFy8iLy9PrdORApZkqNQIqJo2FUXk168XDsWgpgYNgBEjxPsZM9Q/PxERkb0o7vKLi4vT+yxJElJSUvD9999jhPxrSA5lSQ2VGl1+AQFiupjz50UdVbdutp/T0OzZwMcfAzt2AL/8AnTurP41iIiI1KY4oEo06O9xc3NDlSpVsHjxYrNPAJJ9OCpDBYhuv/PnRbefPQKqiAggNhZYsUJkqQ4eFFkxIiKikkxxQLVz5057tINsYEkNlRoZKkAEVJ99Zr86KgCYORNYtw44fFjUU/XpY79rERERqUG1GipyHlNdfnl5wM2b4r1aGSq1J0k2JiQEmDRJvJ81y3hXJhERUUliUYaqefPm0FjY73L8+HGbGkTKmeryu3FDTBPj5gZUqqTOteQn/c6fB+7dA8qVU+e8hl55BVi1Cjh9Gvjf/4Bhw+xzHSIiIjVYlKHq06cPevfujd69e6N79+64ePEivL290alTJ3Tq1Ak+Pj64ePEiunfvbu/2khGmAiq5fio4uLBb0FYhIUDVqiJQO3FCnXMaU6EC8J//iPezZwO5ufa7FhERka0sylDNnj1b+/7555/HhAkT8PrrrxfZ5+rVq+q2jixiqoZK7fopWfPmQEqK6PZr21bdc+saPx545x0xd2BCAjB2rP2uRUREZAvFNVSff/45hg8fXmT9v//9b2zevFmVRpEypmqo1H7CT+aIOioA8PcXNVQA8PrrQHa2fa9HRERkLcUBla+vL/bu3Vtk/d69e+Hj46NKo0gZU11+9sxQAfaZ08/QmDFAzZriu6xYYf/rERERWUPxsAmTJk3Ciy++iGPHjqFNmzYAgIMHD+KDDz7Aa6+9pnoDyTxzNVSmMlS3bomicm9vZdeTA6qTJ4GHDwFPT2XHK+HlBcydK0ZQf+st4IUXRH0VERFRSaI4QzVt2jR89NFHSExMxIQJEzBhwgQkJiZi/fr1mDZtmj3aSGZYU0N18SJQvTrw9NOiwFyJyEigfHlRKH72rPL2KjV0KNCwIXD7NrBokf2vR0REpJRV41ANGDAA+/btQ3p6OtLT07Fv3z4MGDBA7baRhaypofrhB+DBA2D7dkDpWK1ubkCzZuK9I7r93N2BN94Q75ctK/xeREREJYVVAdWdO3eQkJCAGTNmID09HYAYf+ratWuqNo4sY00N1cGDhe/ffFP5NeVuP3sXpsv69AEefRTIygLi4x1zTSIiIkspDqhOnDiBunXr4q233sLChQtx584dAMCWLVswffp0tdtHFjDV5Vdchko3oNqxQ/+zJRwdUGk0hYHUmjXAlSuOuS4REZElFAdUcXFxGDlyJM6fP6/3VF+PHj2wZ88eVRtHljGWoXr4UBSdA0UzVDduiBoqAOjbV7wqzfroPumntAbLWp07A48/Lmq35s1zzDWJiIgsoTigOnLkCF544YUi66tVq4ZUuY+JHMpYDVVamnj18ACCgvT3P3RIvDZoACxYII7/9ltlI583aCCe7svIAP780+qmK6KbpVq/Hvj9d8dcl4iIyBzFAZWPjw8yMzOLrD937hyqVKmiSqNIGWMZKjm2DQ4u3C6Tu/fatAHq1gX+9S/xWUmWyssLaNxYvHdUtx8g2vzMM+K7cpQOIiIqKRQHVL1798a8efPw8OFDAIBGo0FycjKmTZuG/v37q95AMs9YDZUl9VP/DCMGufTts8+AP/6w/LqOrqOSvfGGyFZ9/jnAubiJiKgkUBxQLVq0CDdu3EBwcDDu37+Pjh07ok6dOggICMCb1jwuRjYrLkNlWD+Vnw8cPizeywFVs2ZAr16iFuqttyy/rqOmoDHUpAkwZIh4P3OmY69NRERkjOKAqnz58ti7dy82b96MBQsWYNy4cdi6dSt2794Nf39/e7SRzDBWQ2UqQ3X2LHD3rhghvVGjwvVyYPLRR0BysmXXdeQUNIbmzhX1YT/+CPBZCCIicjZFAVVeXh48PDxw6tQpPPHEE5gyZQqmTp2KLl262Kt9ZAElGaoDB8Rrq1aFXYWAyFY9/jiQl2f5aOTNmomut2vXxJODjlS7NvD88+L9jBmOe9KQiIjIGEUBlYeHByIiIpBvOCQ3OZWSGirD+ildcpZq7VrLRiMPCADq1BHvHd3tBwCvvgr4+AD79gFbtzr++kRERDLFXX6zZs3C9OnTtSOkk/MZ6/IzlaEqLqB64gmgdWsxJc3SpZZd21l1VAAQFgaMHy/ez5xZdGBTIiIiR1EcUC1fvhy//vorwsLCUK9ePbRo0UJvIccz1uVnLEN15w5w5ox437p10fNoNKL7DABWrRKTEZvjzDoqAPjPf8REzb/9Jp76IyIicgbFAVWfPn0wZcoUTJ8+HUOGDEHv3r31FntbtWoVIiMj4ePjg+joaPz666/F7r97925ER0fDx8cHtWrVwpo1a4rss3nzZjRs2BDe3t5o2LAhtmzZord9zpw50Gg0ekuosfEInMTSGqojR8RrrVpifCpjevUST9HdvQusXGn+2s4aOkFWqRIwZYp4/+qrYoR4IiIih5NcyMaNGyVPT09p7dq10pkzZ6SJEydK/v7+0pUrV4zuf+nSJcnPz0+aOHGidObMGWnt2rWSp6en9MUXX2j32b9/v+Tu7i7Fx8dLZ8+eleLj4yUPDw/p4MGD2n1mz54tNWrUSEpJSdEuaWlpitqekZEhAZAyMjKs+/LFOHtWkgBJCgoSnx88EJ8BSUpPL9xv3jyxbsiQ4s/36aeF57t7t/h9U1PFvhqN+X3tJTNTkqpUEe1Yu9Y5bSAiotLJ0t9vlwqoWrVqJcXGxuqtq1+/vjRt2jSj+0+dOlWqX7++3roXXnhBatOmjfbzgAEDpCeffFJvn+7du0uDBg3Sfp49e7bUrFkzRW198OCBlJGRoV2uXr1qt4Dq3DkRTAQGis9XrojPXl6SVFBQuF/PnmL98uXFny8vT5Lq1BH7Llpk/vqhoWLfffus/go2W7pUtKF6dUm6f9957SAiotLF0oBKcZefs+Tm5uLYsWPo1q2b3vpu3bph//79Ro85cOBAkf27d++Oo0ePakd6N7WP4TnPnz+PsLAwREZGYtCgQbh06VKx7Z0/fz4CAwO1S3h4uEXf0xqGXX5y/VRIiKiLAkS+qriCdF3u7sC0aeL94sWiSL04zq6jAoDYWCA8HPjrL2D1aue1g4iIyiaXCahu3ryJ/Px8hBg8thYSEmJyUubU1FSj++fl5eHmzZvF7qN7ztatW+Ojjz7CTz/9hLVr1yI1NRVt27bFrVu3TLZ3+vTpyMjI0C5Xr15V9H2V0A2o8vOBL7+Uv0fhPhcuAOnpYpiBZs3Mn3PYMBGgpKSIiYiL4+w6KkB8r9mzxfv4eFEDRkRE5CguE1DJNHLK5R+SJBVZZ25/w/XmztmjRw/0798fTZo0QZcuXfD9998DADZs2GDyut7e3ihfvrzeYi/yOFQ5OUCnTsCCBeLz008X7iNnp6KjxcTG5nh5Aa+8It6/9Vbxxd4lIaACgBEjxGTPN29aPuwDERGRGlwmoKpcuTLc3d2LZKPS0tKKZJhkoaGhRvf38PBApUqVit3H1DkBwN/fH02aNMH58+et+SqqkzNUeXnA3r1iWpn33xdPvcnkEdLNdffpGj0aqFIF+PNPYONG0/vJY1GdPOncp+w8PIDXXxfvFy0SgRUREZEjeFiyU1xcnMUnXLJkidWNKY6Xlxeio6Oxfft29O3bV7t++/btJodriImJwbfffqu3btu2bWjZsiU8PT21+2zfvh2TJ0/W26dt27Ym25KTk4OzZ8+iffv2tnwl1fj5Fb7v1An48EOgZk39fSytnzI8b1wcMH06MH8+MHRoYfCmq1YtMWr63bvA77+LYRec5dlnRYCXlCQyawsXOq8tRERUdlgUUCVa2JdTXNebGuLi4jBs2DC0bNkSMTExeP/995GcnIzY2FgAom7p2rVr+OijjwAAsbGxWLlyJeLi4jBmzBgcOHAA69atw6effqo958SJE9GhQwe89dZb6N27N77++mv8/PPP2Lt3r3afKVOm4Omnn0aNGjWQlpaGN954A5mZmRgxYoRdv6+lKlUCEhIAT0/g3/8uGvRkZQEnToj3SgIqABg7VnQhnj0LbNkC9O9fdB83NxHE/Pqr6PZzZkDl5iZqqHr2FONoTZoEVKvmvPYQEVEZ4YhHDtX07rvvShEREZKXl5fUokULaffu3dptI0aMkDp27Ki3/65du6TmzZtLXl5eUs2aNaXVq1cXOefnn38u1atXT/L09JTq168vbd68WW/7wIEDpapVq0qenp5SWFiY1K9fP+n06dOK2m3PcajM2b1bDClQrZp1x8+aJY5v3lx/GAZdEyaIfSZNsr6daikokKTHHhPteeEFZ7eGiIhcmaW/3xpJ+qdKm+wqMzMTgYGByMjIsGuBujFvvy2maOnfH/jiC+XH37wJREQA2dnADz8ATz5ZdJ8PPwRGjRJdjjt32txkm/36K9Chg6irOnu2cBJnIiIiJSz9/baoy8/QkSNH8PnnnyM5ORm5ubl6276Un9mnEsOa+ildlSuLcZ6WLAHefNN4QKU7FpUkFY5/5Szt2wM9eogAcPZs4JNPnNseIiIq3RQ/5bdx40a0a9cOZ86cwZYtW/Dw4UOcOXMGO3bsQGBgoD3aSMVITxdP4Zkad0mSrHvCz9DLL4uhFPbuBfbsKbq9YUNRw3XnjmhPSfDmm+L1008La8iIiIjsQXFAFR8fj6VLl+K7776Dl5cX3nnnHZw9exYDBgxAjRo17NFGKsa//gVERgIGDzNqXb0qJkr28ABatLD+OmFhwHPPiffx8UW3e3kBjRqJ984ej0rWvDkwYIAIKmfNcnZriIioNFMcUF28eBFPPfUUADF4ZVZWFjQaDSZPnoz3339f9QZS8eQhE+7fN75d7u5r1kx/eAVrTJ0qBhH96Sfg6NGi20vKAJ+65s0Tbf7228JMHRERkdoUB1RBQUG4+0//UrVq1XDq1CkAwJ07d5Cdna1u68gsX1/xai6giomx/Vq1agGDB4v3xrJUjz4qXpcsAYoZRN6h6tUDRo4U72fMENkqIiIitSkOqNq3b4/t27cDAAYMGICJEydizJgxGDx4MDp37qx6A6l4lgZUttRP6Zo+Xbxu2QKcOaO/bcQIoHNn8TTgyJFiuXdPneva4rXXRJfkrl3Azz87uzVERFQaKQ6oVq5ciUGDBgEQA2lOmTIFf//9N/r164d169ap3kAqnhxQGUsO5uQAx46J92oFVA0bAv36iffz5+tv8/MT3YGvvy4G2NywQWStTp5U59rWqlFDDFAKMEtFRET2wXGoHMRe41BNngwsWybGmZInRZYdOiQCqcqVgbQ09YYyOHYMaNlS1Cb98YfoCjS0ezcwZAhw/Trg4wMsXw48/7zzhlNISxPtzMoCNm8uDAqJiIiKY+nvt0UZqszMTL33xS3kWMV1+el296kZyERHA927A/n5Yr48Yzp2FGNSPfkk8OAB8H//JwIsZ/0VCQ4W8xIC4om//HzntIOIiEoniwKqihUrIi0tDQBQoUIFVKxYscgiryfHKu4pP7Xrp3TNnCle168Hrl0zvk+VKsD334ugy90d2LhRBGPHj6vfHku8/DIQFCRGTv/4Y+e0gYiISieLRkrfsWMHgoKCAAA7S8K8IqRlaYZKbe3bi+XXX4HFi8WTfca4uYnhFtq3BwYNAi5cEE8cLl4MvPSSY7sAAwOBadNEe2bPFu3x9nbc9YmIqPRSXEOVnJyM8PBwaAx+CSVJwtWrVzm4pwn2qqFatUoEJv36idogWWoqULWqCFju3AHsMX3gjz+K6V38/IArV0StVnHS08XgoN98Iz736wesWwdUqKB+20zJzhbz+qWkACtWAOPGOe7aRETkelStodIVGRmJGzduFFmfnp6OyMhIpacjG5nKUB06JF4bNbJPMAWIOqroaBGkvPOO+f2DgoCvvhJF9J6ewJdfisFA5bY6gp+fGEYBAN54QxSpExER2UpxQCVJUpHsFADcu3cPPj4+qjSKLGcqoLJnd59MoxHDEAAi25ORYdkxEycC+/eLp+7+/BN47DHRBeio501HjRLX/vtv8fQhERGRrSyqoQKAuH8ekdJoNHj11VfhpzOPSX5+Pg4dOoRHHnlE9QZS8UwVpas5Qnpx+vQBGjQQhd6rVhUO/GlOy5aiOP3554EvvgCmTBEDb65fD1SqZMcGQwzyOXcuMGwY8PbbQGwswOcpiIjIFhZnqBITE5GYmAhJknDy5Ent58TERPz+++9o1qwZ1q9fb8emkjHGMlR5ecCRI+K9PTNUgCg6l4OopUuNDzBqSmAg8NlnwOrVojj8u++ARx4B9u61S1P1DB4MNG4s6ssWLrT/9YiIqHRTXJT+3HPP4Z133lG1sLossFdR+t694gm6OnWA8+fFuqQkUZtUvjxw+7YIeuwpLw+IihLdd++8A0yYoPwcv/0GDBggBgp1dxejrf/nP/Zt+9dfiwybnx9w8SIQGmq/axERkWuyW1H6hx9+yGCqBDGWofr9d/HarJn9gykA8PAQwQ8gsj25ucrP0awZcPQoMHSoGHRzxgzxBOHff6vbVl3PPAO0bi2yam++ab/rEBFR6af45zYrKwuvvvoq2rZtizp16qBWrVp6CzmWsYBKfh8Q4Lh2jBwphmn46y/R9WdNgXlAAPDf/wIffCC+17Ztogtwxw61WytoNEB8vHj/3nsiw0ZERGQNi4vSZc8//zx2796NYcOGoWrVqkaf+CPHMRZQ5eSIV0c+dOnjIwbMnDxZDJ75yy/AypVA3brKzqPRiLGqWrUSXYBnzgBduoihDl59VXQHqumJJ8T5f/5ZFKp/+KG65yciorJBcQ1VhQoV8P3336Ndu3b2alOpZK8aqr//Lqz9KSgQAcmyZSKwGTwY+N//VLuUWfn5Ymyn+fNFUOflBbzyiui+03ko1GLZ2cD48SJjBQCdOgGffAKEhanabBw+LLr+3NyAkyeBhg3VPT8REbkuu9VQVaxYUTsNDTmfnKECxCTEgHMyVIDIHs2eDZw6JSZFzs0VtUmNGgHffqv8fH5+YiT1//4X8PcXwyo88gjw00/qtrtVK6BvXxGQyoN+EhERKaE4oHr99dfx2muvIVvJ8/FkN7oBldztJwdWzpqnrk4dYOtWMRVOeLioTXrmGaB3b+vqlP79b+DYMVG4fuOGCNamTwcePlSvza+/LrJ7mzeL4ngiIiIlFAdUixcvxk8//YSQkBA0adIELVq00FvIsTw9C+uKDAMqZw5cr9GIufrOnhVPAHp4iDn8GjYUWSs5i2apevXEYKUvvig+L1ggugCvXlWnvY0aiYE+AaBnT+D//k+0l1PTEBGRJRQXpffp08cOzSBb+PoC9+4VBlRysOKsDJUuf38R/AwfDowdC+zeDcyaBXz0EfDuu6Ig3FI+PmI09k6dgDFjxPQ1jzwiRld/+mnb2zpvnnii8K+/gLVrxeLtDTz+ONCrF/DUU0DNmrZfh4iISh/FRelkHXsVpQNAcLDoCjtxAmjSRAQuq1eLeqY5c1S9lE0kSRTJv/xy4fhSAweKefyqVVN2rosXxbHHjonPkyeLwM3Ly7Y25uSIoO+778Ry+bL+9kaNCoOrmBiReSMiotLLbkXpAHDnzh0kJCRg+vTpSE9PBwAcP34c165ds661ZBPD+fycVZRujkYjBu48d06Mpu7mBmzaBNSvDyxZoqwmqnZtYN8+YNIk8XnpUjHJsmEApJS3N9Ctm5g0+eJFMWzD228DHTuKrtXTp4G33gI6dBCB7NChwKefAv/8Z0BERGWU4oDqxIkTqFu3Lt566y0sWrQId+7cAQBs2bIF0y2dGZdUJRemy88JOLso3ZzAQDFFzdGjYq7Be/dE1io6Wtk8ft7eIpD66iugQgUxf2Hz5qKwXA0ajZj4+ZVXxBOGN26I4GnoUCAoSEzr87//AUOGAFWqiCDr7bdF0MW8LxFR2aI4oIqLi8PIkSNx/vx5+OikQHr06IE9e/ao2jiyjOHgniU1Q2WoeXORZUpIACpVEmNAtW8vRl1PS7P8PL17i/kLY2KAjAzg2WeBceMKA0u1VKwIDBoEfPyxaN/evWIQ0yZNxJALv/4qCvAbNwZq1RJjaP34o/rtICKikkdxQHXkyBG88MILRdZXq1YNqampqjSKlDEMqEp6hkqXmxswerToBhwzRqzbsEE81bd6tRgs1BIREaL2aepU8fndd0WAJU8YrTZ3d6BdOzGI6YkTYjiId98V8w96e4vPK1eKz5UqiUmYExKA69ft0x4iInIuxQGVj48PMjMzi6w/d+4cqlSpokqjSBlrM1Q3bigfvsBeKlUC3n8fOHBAZK7u3BHF9W3aWD4ulKenqG/auhWoXFlkrVq0EN109hYRIdq7dStw65YYcuH//k8U22dnA19/LQLGatVE1+bs2WKE9oIC+7eNiIjsT3FA1bt3b8ybNw8P/6kg1mg0SE5OxrRp09C/f3/VG0jmGRalW5KhunhR/LhHRYkf+5KiTRtRC7ViBVC+vAimWrUS40/dvm3ZOXr0EMFUhw6iPmvIEBHMOGosWn9/MYzDe++JcbISE8XAoW3aiLqs48fFEA2tW4tpdEaNAr78Erh71zHtIyIi9SkOqBYtWoQbN24gODgY9+/fR8eOHVGnTh0EBATgzTfftEcbyQxTRenFZah+/VU8VXf1quiO6ttXvUEybeXuLmqgzp0To6RLErBmjegGXL/esqxOtWpiguZXXxVBTEKCCMzOnLF78/VoNGKsrFmzRPYtNVV8h2efBQICxPARH34I9O8vsnRdu4qC/QsXHNtOIiKyjeKAqnz58ti7dy82b96MBQsWYNy4cdi6dSt2794Nf39/e7SRzDDV5VdchurUKfEaFSXGUvrqKzGK+bJlQF6evVqqTGiomMdv507Rths3gOeeE0MYnDxp/ngPD5EJ2rYNCAkRT989+qgIaJwlOBgYMQL4/HPg5k0R9E2eLP4cHj4Efv5ZDAURFSWGk5gyRXx/NafZISIiO5DIITIyMiQAUkZGhurnjo2VJECSZs8Wn6OixOdffzV9TPfuYp/33pOkkyclqW1b8RmQpBYtJOnIEdWbaZPcXEl6+21J8vcXbXR3l6TJkyUpM9Oy41NSJKlz58LvOGyYJN29a982K3XunCQtWSJJTzwhSR4ehW0FJCkwUJIGDJCkjz6SpBs3nN1SIqKyw9Lfb8UjpS9fvtzoeo1GAx8fH9SpUwcdOnSAuzzBHAGw70jpcXFiPKapU0VRdkQEkJwsip4ffdT4MeHhYoqVffuAtm1FN9q6deIcd+6Ip+/GjRO1Pyo31yZXr4qMjjzWVFiYGBR0wADRvVac/Hwxmvprr4nvW68e8NlnQNOm9m+3UhkZwPbtYrT2rVtFdk6m0Yh6rF69xNKkifnvTkRE1rH491tppFazZk3J399f0mg0UlBQkFSxYkVJo9FI/v7+UkhIiKTRaKTatWtLycnJVsaCpZM9M1QzZogsxvjx4nNwsPh84oTx/W/fLsx83L6tvy01VZKGDi3cHhYmSZs3S1JBgerNtskPP0hS7dqF7ezSRZJ+/92yY3fvFt8LkCRvb0las6bkfT9d+fmSdPCgJM2aJUmPPKKfuQIkqXp1kaX87jtJyspydmuJiEoXS3+/FddQxcfH49FHH8X58+dx69YtpKen448//kDr1q3xzjvvIDk5GaGhoZg8ebL14SApIj/lJxelmxs2QS7Mrl5djDCuKyREDFy5bZuY3uX6dVEw/cwzwJUrqjfdak8+KerA5s4VtWI//ywyNTNnmn+ar0MH8RRgjx7iXsXGAoMHA0ZGAykR3NzEE4Gvvy6eGLx6VTxB+PTTon7ur79E0X6vXqKwvVcv8bmkPGRARFQmKI3UatWqJSUmJhZZf/z4cSkyMlKSJEnat2+fFBoaqvTUpZo9M1SLF4tMxZAh4rO3t/h85Yrx/d97T2x/8sniz5udLbIinp5ifz8/SVq0SJIePlS3/ba6cEGSevYszNhEREjS11+bPy4/X9RlyfVKtWtL0tGjdm+uqrKzJWnrVkkaO1aSatQomr1q2lRkMPfvl6S8PGe3lojI9dgtQ5WSkoI8I4+B5eXlaUdKDwsLw10OquMwuk/5SZL5DJX8hF+jRubP+/rrwG+/iSlhsrPFU2ctWwKHDqnTdjXUri1qjbZsAWrUEJm03r1FBqe4yZLd3MQ8fXv2iOMuXhSjqz/3HBAfLwYEPXRITDNTUufm8/UVmbZ33xWjs588KUZvb9dOfL8TJ8R3adtWPDU5fLioG/tnCk4iIlKJ4oDq8ccfxwsvvIDExETtusTERLz44ot44oknAAAnT55EZGSkeq2kYukGVLm5hetNDZtw+rR4bdzYsvM3aCAmB163TkwK/NtvIvB46SVRPF0SaDRiPK0zZ8T8ep6eIshq2BB4443iR4SPiRFdaX36iOEJ1q8XXYdDhoji75AQMWZUkyYiUJs0CVi+HPj2W3Evs7Ic8x3N0WjEn+m0aWKewbQ00X07aJDo2r15UwxDMXCgmMz58ceBxYvFeF8lNWAkInIVip/yS01NxbBhw/DLL7/A09MTgMhOde7cGf/9738REhKCnTt34uHDh+jWrZtdGu2K7PmU36ZN4kezY0cx6rlcF/XggfGgKiRE/NgW9xSgKTduiKzOhg3ic2ioCC6efbZkPWn2++8i4NuxQ3yOihJZnK5dTR8jSSIIO35cZLYuXRKv166ZDzhCQoDISDEpsuFr9episFJnyssD9u8X3++774CzZ/W3165d+NRghw6Al5dz2klEVNJY+vutOKCS/f777/jjjz8gSRLq16+PevXqWd3YssCeAdU334jMyaOPih/LkBCxvqCgaJBz44YYXBIQ07JYOxbrzp2imPuPP8RnudupJCUmJQnYuFEMKyHP2z1ggBhmoVo1y8+TkyO6EeUAS/f10iXzWToPDzGUhamAKyjI8cHopUvA99+Lvy+7dulnNsuVA7p1E8FVz56Ff5+IiMoiuwdUpIw9A6qffxaZl8aNxY9kRITITMlT0OjatUt09dSqJWqGbJGTI8Z1io8XP8i+vmLS30mTih+l3dEyMkS7VqwQQWa5csCcOcCECaJr0Fa3bxcNtOTXP/80P8p5QIDxQCsyEqhZs7BL117u3RN/h777Tvz9kYNP2aOPFmavmjcvWZlIIiJ7s1tAlZ+fj/Xr1+OXX35BWloaCgwmVtsh97GQHnsGVPv2AY89JrptfvgBqFtXDMZpLHOyciUwfrwo2P7mG3Wuf+6cmLx4507xOSBAZKz69BEZjsBAda5jq99+A8aOFV1fgAhAV60SBff2kp8vhp4wFXClpJg/R9WqpgOuatVE8blaCgpEPZncNXj0aNG2PPWUCK46dxbBKRFRaWa3gGrcuHFYv349nnrqKVStWhUag3+uLl261LoWl3L2DKiOHweio8Wo4T/+KEb+Dg4WE+8aevFFMUbR9Okis6QWSRIFzzNninGRZJ6eIiPWu7dYlHS12UNBgSg6/89/RJE2IJ58W7iwsCvUke7fF1ksUwGXuYdlvbxERtJUwFWxom3tS0kRQfp334mxyXQL8L28xJ9tr14iyCpJ3b1ERGqxW0BVuXJlfPTRR+jZs6fNjSxL7BlQnT0rnmarWFH86D36qJhaJjm56L7t24snwD7+GBg6VNVmABABy9GjYrLlr74qWvz86KMic9W7t2izs7qP0tOBGTOA998XwWCFCsCbbwIvvOD8AnKZJAG3bpkOtpKTzU9kXaGC6dotuWvYUjk5YogJOXt16ZL+9oYNC7sGY2JE7RgRkauzW0AVFhaGXbt2oW7dujY3siyxZ0D155/iR1IeMbx9e/FUm1wwLpMkMZL27dtipPBmzVRthlF//CGePPzqK+DAAf2n5erUEcFVnz5ieAJnBDKHD4us3fHj4nN0tKitqlNHdKEGB5fcmqG8PJENNBVwpaUVf7xGIzKGpgKu0FDT3YmSJLp65eBq717RvSmrWFGMZt+rl3gNClLvexMROZLdAqrFixfj0qVLWLlyZZHuPjLNngFVZmZhndK334r6qCZNxKCOuq5fFz+g7u6iENnUwJ/2kpoq2vf112LiX90ny6pUEdPb9OkjanPsXYitKz9fdIPOnFm07szfXwQXtWvrL3KGR42idnvJyhLBlamAy9wUPT4+oijeVHei7l/j27dFdlSezDk9vXCbm5sYWFTOXjkzM0lEpJTdAqq+ffti586dCAoKQqNGjbRjUcm+/PJL61pcytkzoJIkUQielSXmeHvhBTGa+ZEj+vtt2wZ07w7UqyfGaXKmu3eBn34Smavvv9cfudvfX7SzTx9Rm+Oo7MbffwNvvy2Ksi9eFHPhFfdfh7u7GGFdN8jSDboCAhzTbmtIkhhCw1igdfmy6E40eN6kiEqVjAdaERGi9uqnn0SAdfKk/nEREYXBVadOjg/siYiUsFtA9dxzzxW7/cMPP1RyujLDXgHV2bOizmbECPFjOG8e8Npr4qm/X3/V33fpUjEmU//+wBdfqNYEmz18KGpz5Lor3aJ2d3cxYKlc1B4R4bh25eSI7tRLl0SApbtcumR8WApdVaqYzm5VrVqyszQPH4qA0lTAJRf0m+LmJur4IiNFt+n9+yJIO3NGfxgJPz+gS5fCwvawMPt+LyIipTgOVQljr4Cqa1dRN1WvnqhpGTdODI3QubNYr2v0aOCDD8SYTHPmqNYEVUmSqGeS664MsxvNmxcWtTdt6rygpKBAdGEaBlnye3MBh6+vfrCl+75mzZI/Uvndu6a7Ei9fNh9sygXrhkX19euLP9t+/USWVc0hIYiIrMGAqoSxV0DVs6d4rL11azGR74ABYvLbp54S3S262rQR+3z2GfCvf6nWBLu6eFEEV19/LQqfdbuhatYsLGpv165kPVWWmWk80Lp40Xx3mpzdMZXdkqcWKqkkSQSbpgKuv/6ybO5ADw9xH5o3F12DjRqJjFd4eMn6syai0s2uAdUXX3yBzz77DMnJycjVrSwGcFx+XIr02CugeuYZUejdtaso9O7cGfjlF/Ev/M2bC/crKBCF6/fuiW6XBg1Ua4LD3LghgsSvvhL1YLpZkEqVRLdRnz5i2hQ/P2e10rzcXBFUGetGvHjRfLF4UFDRIEt+HxZW8rM6OTni+xsGWufPi8Xc99doREF8QICov/LyEt9ZksQDBt7eIgPo6yve+/iYXsxtL24fb++Sf6+JyHaW/n4r/nfe8uXLMXPmTIwYMQJff/01nnvuOVy8eBFHjhzBSy+9ZFOjSTn5mQC5AFp+usqw0Dc5WQRTnp5iSABXVKUK8NxzYsnKEgHkV1+JgPLWLTFh84YN4oe0a1cRXPXqJY4rSby8xJ+BsT8HSRLF8aayW2lp4s84Pb3oQweA+HOXC8QNs1s1a6pfAJ6XJ/5e3b1r/aJ7vMG/z4ySJPE0prk5FB3B3V38N+XlJQIseTEMvnx9RZDv5yceuvD1tTyoK267h0fJrsUjKksUB1SrVq3C+++/j8GDB2PDhg2YOnUqatWqhddeew3pus9Kk0PIXR/yFCDyj4zhgI2nT4vX+vVL9qP+lvL3L+zuy8sT3YFy3dWff4ppdb75RmQQHntM1OX06SMCjZJMoxHjP4WGim5MQ3LtkrHs1p9/iqzd2bNFB1SVzx0WJgKr6tVFYXzlyiLjJf+jS2kwZK5Wylo+PuIfCfLi4SGC6Fu3RKZSl5ubCFQkSX8kd0fIzxeLve6DJTw8xH/T8uLlVbgYBnhy5k43wJODvHLligZ75gI6ZumICikOqJKTk9G2bVsAgK+vL+7+MzfGsGHD0KZNG6xcuVLdFlKx5ODIMKAyzEScOiVeGzd2TLscycND1Nh06gQsWSLG3/rqKxFgJSaKJwj37AFeflmMzyUHYq4w0a8kiS4ww0DG01OMKVa+vMh03bsn/uxTUkSG6+ZNMRTF3bvi+IcPxbmuXROL2jw99QMgY0u5cub3kfcrLujPyBDZye+/F8uNG+L7yzw89AMB3QDDy0ts113c3ESmyc1NLBpN0b8XBQXi/hUUiAD+4UOx5OYWvubkiFfddQ8fiv11Bz1VW16eWO7ft981iqPRiPvn4VGYsfPwEPdaDvK8vcVn3T8XOXDTDfD8/cVruXKFAZ7898ZYsMcsHZUkigOq0NBQ3Lp1CxEREYiIiMDBgwfRrFkzXL58Gaxvdzw5QyUPhCnP/WYqQ9WokWPa5SwajRgBvlkz8TTjlSuFmas9e8RTgydPAq+/Loqb5cxVhw7qZO4kSfyw2tLtZbje3HhQ1tBoCn+I5CChOO7uYviDatUK5w6MihK1eI0bO7ZQPjAQePZZsRQUiK7P774TwVViYmE3pG6QZW9ubvrBQfny+p91uwN1s0menvrBiLu7+DMpLrArKCgM2B48EAHz/fvi/YMH4u+f/CrvJwd3coAnB3n5+ZY9IFAcSSo8p7PI90wOjOV7Ki+WZO8MgzvdAM/fvzCokwP/8uULa/lKQ9afbKc4oHriiSfw7bffokWLFhg9ejQmT56ML774AkePHkW/fv3s0UYqhvwfslyELdeglKUMVXEiIsRUMhMmiLqj778XwdWPP4pxllauFEuFCuLJyN69RXeYLQGRvX5YLM3wWJIV8vXV/7GWR1U3Vrv155/ihzglRSxHjxZtW1iY6QFOK1WyXwbBzU084dq6tQiSMzPFn8H9+4VBhvze3GdrtsnBSEGBCGzMFdSrTbcAXzcg8PcX993YNsPPciZJN6iTs3VyYCd368nBU3a2+Duj+yovusGdHNjJi25wJ2f6DAO8ggLlgZ78QII9M4GWkINg+d7J91K+t6a6Z3UzeMV1zxpm8HSzd3KAqBssenkxe+dIip/yKygoQEFBATz+SY189tln2Lt3L+rUqYPY2Fh4lfQBdJzEXk/5vfiimDZlzhxg8eLCDNWcOSJDA4j/yZQrJ/7ndv686xalq+n+fTFO19dfi1orw7ocNfj6Kg+ATAVD/v7Oq1XJzxfBp7Ei+YsXRRBTnPLlTQ8B4cpDIEiSCA7UCMyU7OvMTBAg/ryMBWbFBW3WbJO789zdC++13LV6/74I5O7e1X/VDe7u3xev8n0zDPAMu2d1Azw5OJMDPN0uX1cjB3m6wbKx7J1u16wclBWXvTMM8OTg3DCwM7XO09N1gj27PeX3119/ITw8XPt5wIABGDBgACRJwtWrV1GjRg3rWkxWkX+MHj4UWYJz58Rn3S4/eVRvX1/xBBiJe/H002LJzxcTN3/9tZiHLjvb9kCoXDnXDRQMubuLQvaaNcWwHLokSWT+TA0Bce2aCLiSksRiyMNDnNdYdqtWLfE/7JJKoyn8oXCkvDz7Z94MP+fk6F9fzsY6ikZjXWAWGmpbQFfc5ODZ2eLvtrzIWeqsLPHeMINneF+Ly96Z6p6VAzxJsjyDJ+8rdxWXJHKQZ033rJ+ffkZWXt+/v/N+5xT/Lz8yMhIpKSkIDg7WW5+eno7IyEjk2znnumrVKixcuBApKSlo1KgRli1bhvbt25vcf/fu3YiLi8Pp06cRFhaGqVOnIjY2Vm+fzZs349VXX8XFixdRu3ZtvPnmm+jbt69N13UUucsvL08/oNLt8pO7+xo2FH95SZ+7u3gS8LHHgIULnd0a16LRiK6lSpWAVq2Kbr9/v7Ar0TC7dfmy+AG5cEEsxoSGms5uBQe7zr9w1eThURi0O0pBQWFmyF6ZN2Pb5KBBDmAc3aXq5WU6ELMkMAsIEH9PlRxn6T/ECgoK6+h0F92MnW6QZxjgGd53Y9k7Y7V3ugGereTzWDJciqVyc4EZM9Q7nxKKAypJkqAx8n+xe/fuwcfOs5xu2rQJkyZNwqpVq9CuXTu899576NGjB86cOWM0M3b58mX07NkTY8aMwccff4x9+/Zh7NixqFKlCvr37w8AOHDgAAYOHIjXX38dffv2xZYtWzBgwADs3bsXrVu3tuq6jmSYoZLp/qu5rBSkU8nj6ysC+YYNi27LzweuXzed3bp9W4y4npoK7N9f9Phy5YpmtCIjxTV1a38Ma4HMbbPHsa4e+Lm5Ff7gO4okif+v2TPzZmybbpeqHFQ4cswzd3fLAjhLM20hIeaP8/a27u+onC01tsgPR8iL3AVrmL2T1+v+Geg+WGGqe9bU07MhIbb/GVjL4hqquLg4AMA777yDMWPGwE9nKOr8/HwcOnQI7u7u2Ldvn31aCqB169Zo0aIFVq9erV3XoEED9OnTB/Pnzy+y/3/+8x988803OKszKE9sbCx+++03HDhwAAAwcOBAZGZm4ocfftDu8+STT6JixYr49NNPrbouAOTk5CBHJ0+emZmJ8PBw1WuoZs4E4uNF0bWXF7BokVj/wQdiAEwAGDQI2LQJePtt4JVXVLs0kV3dvm16gFNLp68pSZwRyJWUgNKVjpUDOTkrY/hDbpi9kX/w5fe6AYAcFMi1X7pBgmHw4UwajfGhLNQI6IrbZmtdqLEsXXCw+jNlqF5DlZiYCEBkqE6ePKlXfO7l5YVmzZphypQpNjS5eLm5uTh27BimTZumt75bt27Yb+yfrxDZp27duumt6969O9atW4eHDx/C09MTBw4cwOTJk4vss2zZMquvCwDz58/H3LlzLf16VtPNUNWsWbheN0Mld/kxQ0WupGJFMUFyy5ZFtz14IIbEMMxuJSeL/xYMC4nV+Cy/tzaQkwuanf0kGpUOuplPw1dDul2n8qvu32NJKswOOZL8JKRuLZVuTZXuuGaGNVaG9Vby2GdxceKpX2ewOKDauXMnAOC5557DO++8o2qWxRI3b95Efn4+QgzyeSEhIUhNTTV6TGpqqtH98/LycPPmTVStWtXkPvI5rbkuAEyfPl2b1QMKM1RqM6yhksm9r7m5hXVVZW3IBCq9fHyAevXE4mjyj5EawZk9Az+ey/nntmRfW/8eujL5PqhZLN+0qQsEVLIPP/zQHu2wmGH9lqmaruL2N1xvyTmVXtfb2xveDnj8R85QGQZU8qXPnxfbAgLEI+pEZBvdzAAf8iBbWRqAuXLQqPs5P9/4041yt6jhYLTGZh4wfBJSt2i+bl3n/Vm6zIPdlStXhru7e5GsUFpaWpHskSw0NNTo/h4eHqhUqVKx+8jntOa6jiRnqEwVpesWpLt6YSwRUWmjO0YUuTYnDRWonJeXF6Kjo7F9+3a99du3b9fOLWgoJiamyP7btm1Dy5Yt4flPJGJqH/mc1lzXkXQzVFWrFq6Xn1QpqyOkExEROZLLZKgA8aThsGHD0LJlS8TExOD9999HcnKydlyp6dOn49q1a/joo48AiCf6Vq5cibi4OIwZMwYHDhzAunXrtE/vAcDEiRPRoUMHvPXWW+jduze+/vpr/Pzzz9i7d6/F13Um3QyVn5/4144kFQ66x4J0IiIi+3OpgGrgwIG4desW5s2bh5SUFDRu3Bhbt25FREQEACAlJQXJycna/SMjI7F161ZMnjwZ7777LsLCwrB8+XLtGFQA0LZtW2zcuBGzZs3Cq6++itq1a2PTpk3aMagsua4z6WaoAPHURH5+4bgpcpcfM1RERET2o3guP7KOveby++ADYPRooGdPMfGvh4cIqBYuBF56SQx+WFAgJrUNDVXtskRERGWCpb/fLlNDRcbpDpsAFD5Gm54O/P67CKaCgpw7eiwREVFpx4DKxel2+enOhn7zpn53H5/wIyIish8GVC5Otyhdd/qCGzf4hB8REZGjuFRROhWlm6HSDajS0gq7AfmEHxERkX0xoHJxuhmqBw8K16emAtevi/fMUBEREdkXAyoXp5uh0g2orl4tnB+JGSoiIiL7Yg2VizNVQyUHU6GhwD+z7BAREZGdMKByccYyVLpP9LG7j4iIyP4YULk4Y0XpHjoduezuIyIisj8GVC7OWFG6vA5ghoqIiMgRGFC5OGMZKi+vwu3MUBEREdkfAyoXZyxD5etbuJ0BFRERkf0xoHJxxjJUckBVowag4jzMREREZAIDKhdnLEMVFCRe27d3TpuIiIjKGg7s6eKMZahCQsQ8fpGRzmsXERFRWcKAysXJGSrdcah8fFg7RURE5Ejs8nNxxgb29PZ2XnuIiIjKIgZULk53zKmsLPHq4+OcthAREZVVDKhcnO6o6PfuiVdmqIiIiByLAZWLMxZQMUNFRETkWAyoXJxulx8zVERERM7BgMrFubsXvmeGioiIyDkYULk4jaaw248BFRERkXMwoCoFDAMqdvkRERE5FgOqUkCuo2KGioiIyDkYUJUCzFARERE5FwOqUoAZKiIiIudiQFUKMENFRETkXAyoSgE5oMrOFq/MUBERETkWA6pSQHdwT4AZKiIiIkdjQFUK6E4/AzBDRURE5GgMqEoBZqiIiIiciwFVKcAMFRERkXMxoCoFDDNUDKiIiIgciwFVKWCYoWKXHxERkWMxoCoFmKEiIiJyLgZUpQAzVERERM7FgKoUYIaKiIjIuRhQlQKGGSrDAIuIiIjsiwFVKaAbUPn4ABqN89pCRERUFjGgKgV0M1KsnyIiInI8BlSlgGGGioiIiByLAVUpwAwVERGRczGgKgWYoSIiInIuBlSlgG6GigEVERGR4zGgKgV0M1Ts8iMiInI8BlSlADNUREREzsWAqhRghoqIiMi5GFCVAsxQERERORcDqlKAGSoiIiLnYkBVCnDYBCIiIudiQFUKcGBPIiIi52JAVQowQ0VERORcDKhKAWaoiIiInIsBVSnADBUREZFzMaAqBThsAhERkXMxoCoFOGwCERGRczGgKgWYoSIiInIuBlSlADNUREREzsWAqhRgUToREZFzMaAqBThsAhERkXO5TEB1+/ZtDBs2DIGBgQgMDMSwYcNw586dYo+RJAlz5sxBWFgYfH190alTJ5w+fVpvn5ycHIwfPx6VK1eGv78/nnnmGfz11196+9SsWRMajUZvmTZtmtpf0WrMUBERETmXywRUQ4YMQVJSEn788Uf8+OOPSEpKwrBhw4o95u2338aSJUuwcuVKHDlyBKGhoejatSvu3r2r3WfSpEnYsmULNm7ciL179+LevXvo1asX8vPz9c41b948pKSkaJdZs2bZ5XtagxkqIiIi5/Iwv4vznT17Fj/++CMOHjyI1q1bAwDWrl2LmJgYnDt3DvXq1StyjCRJWLZsGWbOnIl+/foBADZs2ICQkBD873//wwsvvICMjAysW7cO//3vf9GlSxcAwMcff4zw8HD8/PPP6N69u/Z8AQEBCA0NtbjNOTk5yMnJ0X7OzMy06rtbghkqIiIi53KJDNWBAwcQGBioDaYAoE2bNggMDMT+/fuNHnP58mWkpqaiW7du2nXe3t7o2LGj9phjx47h4cOHevuEhYWhcePGRc771ltvoVKlSnjkkUfw5ptvIjc3t9g2z58/X9s9GRgYiPDwcMXf21LMUBERETmXS2SoUlNTERwcXGR9cHAwUlNTTR4DACEhIXrrQ0JCcOXKFe0+Xl5eqFixYpF9dM87ceJEtGjRAhUrVsThw4cxffp0XL58GQkJCSbbPH36dMTFxWk/Z2Zm2i2oYoaKiIjIuZwaUM2ZMwdz584tdp8jR44AADQaTZFtkiQZXa/LcLslxxjuM3nyZO37pk2bomLFinj22We1WStjvL294e2gdBEH9iQiInIupwZU48aNw6BBg4rdp2bNmjhx4gT+/vvvIttu3LhRJAMlk+udUlNTUbVqVe36tLQ07TGhoaHIzc3F7du39bJUaWlpaNu2rck2tWnTBgBw4cIFkwGVI3FgTyIiIudyag1V5cqVUb9+/WIXHx8fxMTEICMjA4cPH9Yee+jQIWRkZJgMfCIjIxEaGort27dr1+Xm5mL37t3aY6Kjo+Hp6am3T0pKCk6dOlVsQJWYmAgAeoGaMzFDRURE5FwuUUPVoEEDPPnkkxgzZgzee+89AMD//d//oVevXnpP+NWvXx/z589H3759odFoMGnSJMTHxyMqKgpRUVGIj4+Hn58fhgwZAgAIDAzE6NGj8fLLL6NSpUoICgrClClT0KRJE+1TfwcOHMDBgwfx+OOPIzAwEEeOHMHkyZPxzDPPoEaNGo6/GUYwQ0VERORcLhFQAcAnn3yCCRMmaJ/Ie+aZZ7By5Uq9fc6dO4eMjAzt56lTp+L+/fsYO3Ysbt++jdatW2Pbtm0ICAjQ7rN06VJ4eHhgwIABuH//Pjp37oz169fD3d0dgKiF2rRpE+bOnYucnBxERERgzJgxmDp1qgO+tWVYlE5ERORcGkmSJGc3oizIzMxEYGAgMjIyUL58eVXPffkyUKuWeJ+TA3h5qXp6IiKiMsvS32+XGIeKiidnqDQa/XoqIiIicgyX6fIj06pVAzp1AsLDRVBFREREjsWAqhRwcwN27nR2K4iIiMoudvkRERER2YgBFREREZGNGFARERER2YgBFREREZGNGFARERER2YgBFREREZGNGFARERER2YgBFREREZGNGFARERER2YgBFREREZGNGFARERER2YgBFREREZGNGFARERER2YgBFREREZGNPJzdgLJCkiQAQGZmppNbQkRERJaSf7fl33FTGFA5yN27dwEA4eHhTm4JERERKXX37l0EBgaa3K6RzIVcpIqCggJcv34dAQEB0Gg0qpwzMzMT4eHhuHr1KsqXL6/KOUsb3iPzeI/M4z0yj/fIPN4jy5S0+yRJEu7evYuwsDC4uZmulGKGykHc3NxQvXp1u5y7fPnyJeIvXUnGe2Qe75F5vEfm8R6Zx3tkmZJ0n4rLTMlYlE5ERERkIwZURERERDZiQOXCvL29MXv2bHh7ezu7KSUW75F5vEfm8R6Zx3tkHu+RZVz1PrEonYiIiMhGzFARERER2YgBFREREZGNGFARERER2YgBFREREZGNGFC5qFWrViEyMhI+Pj6Ijo7Gr7/+6uwmOdWePXvw9NNPIywsDBqNBl999ZXedkmSMGfOHISFhcHX1xedOnXC6dOnndNYJ5g/fz4effRRBAQEIDg4GH369MG5c+f09inr92j16tVo2rSpdjDBmJgY/PDDD9rtZf3+GDN//nxoNBpMmjRJu473CZgzZw40Go3eEhoaqt3OeyRcu3YN//73v1GpUiX4+fnhkUcewbFjx7TbXe0+MaByQZs2bcKkSZMwc+ZMJCYmon379ujRoweSk5Od3TSnycrKQrNmzbBy5Uqj299++20sWbIEK1euxJEjRxAaGoquXbtq51gs7Xbv3o2XXnoJBw8exPbt25GXl4du3bohKytLu09Zv0fVq1fHggULcPToURw9ehRPPPEEevfurf0feFm/P4aOHDmC999/H02bNtVbz/skNGrUCCkpKdrl5MmT2m28R8Dt27fRrl07eHp64ocffsCZM2ewePFiVKhQQbuPy90niVxOq1atpNjYWL119evXl6ZNm+akFpUsAKQtW7ZoPxcUFEihoaHSggULtOsePHggBQYGSmvWrHFCC50vLS1NAiDt3r1bkiTeI1MqVqwoJSQk8P4YuHv3rhQVFSVt375d6tixozRx4kRJkvj3SDZ79mypWbNmRrfxHgn/+c9/pMcee8zkdle8T8xQuZjc3FwcO3YM3bp101vfrVs37N+/30mtKtkuX76M1NRUvXvm7e2Njh07ltl7lpGRAQAICgoCwHtkKD8/Hxs3bkRWVhZiYmJ4fwy89NJLeOqpp9ClSxe99bxPhc6fP4+wsDBERkZi0KBBuHTpEgDeI9k333yDli1b4l//+heCg4PRvHlzrF27VrvdFe8TAyoXc/PmTeTn5yMkJERvfUhICFJTU53UqpJNvi+8Z4IkSYiLi8Njjz2Gxo0bA+A9kp08eRLlypWDt7c3YmNjsWXLFjRs2JD3R8fGjRtx/PhxzJ8/v8g23iehdevW+Oijj/DTTz9h7dq1SE1NRdu2bXHr1i3eo39cunQJq1evRlRUFH766SfExsZiwoQJ+OijjwC45t8lD2c3gKyj0Wj0PkuSVGQd6eM9E8aNG4cTJ05g7969RbaV9XtUr149JCUl4c6dO9i8eTNGjBiB3bt3a7eX9ftz9epVTJw4Edu2bYOPj4/J/cr6ferRo4f2fZMmTRATE4PatWtjw4YNaNOmDQDeo4KCArRs2RLx8fEAgObNm+P06dNYvXo1hg8frt3Ple4TM1QupnLlynB3dy8SoaelpRWJ5EmQn67hPQPGjx+Pb775Bjt37kT16tW163mPBC8vL9SpUwctW7bE/Pnz0axZM7zzzju8P/84duwY0tLSEB0dDQ8PD3h4eGD37t1Yvnw5PDw8tPeirN8nQ/7+/mjSpAnOnz/Pv0v/qFq1Kho2bKi3rkGDBtqHq1zxPjGgcjFeXl6Ijo7G9u3b9dZv374dbdu2dVKrSrbIyEiEhobq3bPc3Fzs3r27zNwzSZIwbtw4fPnll9ixYwciIyP1tvMeGSdJEnJycnh//tG5c2ecPHkSSUlJ2qVly5YYOnQokpKSUKtWLd4nI3JycnD27FlUrVqVf5f+0a5duyJDt/zxxx+IiIgA4KL/T3JWNTxZb+PGjZKnp6e0bt066cyZM9KkSZMkf39/6c8//3R205zm7t27UmJiopSYmCgBkJYsWSIlJiZKV65ckSRJkhYsWCAFBgZKX375pXTy5Elp8ODBUtWqVaXMzEwnt9wxXnzxRSkwMFDatWuXlJKSol2ys7O1+5T1ezR9+nRpz5490uXLl6UTJ05IM2bMkNzc3KRt27ZJksT7Y4ruU36SxPskSZL08ssvS7t27ZIuXbokHTx4UOrVq5cUEBCg/X8075EkHT58WPLw8JDefPNN6fz589Inn3wi+fn5SR9//LF2H1e7TwyoXNS7774rRURESF5eXlKLFi20j7+XVTt37pQAFFlGjBghSZJ4BHf27NlSaGio5O3tLXXo0EE6efKkcxvtQMbuDQDpww8/1O5T1u/RqFGjtP9NValSRercubM2mJIk3h9TDAMq3idJGjhwoFS1alXJ09NTCgsLk/r16yedPn1au533SPj222+lxo0bS97e3lL9+vWl999/X2+7q90njSRJknNyY0RERESlA2uoiIiIiGzEgIqIiIjIRgyoiIiIiGzEgIqIiIjIRgyoiIiIiGzEgIqIiIjIRgyoiIiIiGzEgIqIiIjIRgyoiMglderUCZMmTVLtfHPmzIFGo4FGo8GyZctUO6+arPnO8neqUKGCXdpERIKHsxtARFRSNGrUCD///DPKly/v7KYY9eWXX8LT01PRMSkpKdi0aRNmz55tp1YREcCAiohIy8PDA6GhoTadIz8/HxqNBm5u6ncABAUFKT4mNDQUgYGBqreFiPSxy4+ISrysrCwMHz4c5cqVQ9WqVbF48WK97bm5uZg6dSqqVasGf39/tG7dGrt27dLbZ+3atQgPD4efnx/69u2LJUuWWNQNtmTJEjRp0gT+/v4IDw/H2LFjce/ePe329evXo0KFCvjuu+/QsGFDeHt748qVK8jJycHUqVMRHh4Ob29vREVFYd26dQCAXbt2QaPR4KeffkLz5s3h6+uLJ554Amlpafjhhx/QoEEDlC9fHoMHD0Z2drb2WoZdfjVr1kR8fDxGjRqFgIAA1KhRA++//77yG0xENmNARUQl3iuvvIKdO3diy5Yt2LZtG3bt2oVjx45ptz/33HPYt28fNm7ciBMnTuBf//oXnnzySZw/fx4AsG/fPsTGxmLixIlISkpC165d8eabb1p0bTc3NyxfvhynTp3Chg0bsGPHDkydOlVvn+zsbMyfPx8JCQk4ffo0goODMXz4cGzcuBHLly/H2bNnsWbNGpQrV07vuDlz5mDlypXYv38/rl69igEDBmDZsmX43//+h++//x7bt2/HihUrim3f4sWL0bJlSyQmJmLs2LF48cUX8fvvv1v03YhIRRIRUQl29+5dycvLS9q4caN23a1btyRfX19p4sSJ0oULFySNRiNdu3ZN77jOnTtL06dPlyRJkgYOHCg99dRTetuHDh0qBQYGaj/Pnj1batasmdn2fPbZZ1KlSpW0nz/88EMJgJSUlKRdd+7cOQmAtH37dqPn2LlzpwRA+vnnn7Xr5s+fLwGQLl68qF33wgsvSN27d9d+7tixozRx4kTt54iICOnf//639nNBQYEUHBwsrV69Wu96H374od53JSL1MUNFRCXaxYsXkZubi5iYGO26oKAg1KtXDwBw/PhxSJKEunXroly5ctpl9+7duHjxIgDg3LlzaNWqld55DT+bsnPnTnTt2hXVqlVDQEAAhg8fjlu3biErK0u7j5eXF5o2bar9nJSUBHd3d3Ts2LHYc+seExISAj8/P9SqVUtvXVpamsXn0Gg0CA0NNXsMEamPRelEVKJJklTs9oKCAri7u+PYsWNwd3fX2yZ3sUmSBI1Go+i8AHDlyhX07NkTsbGxeP311xEUFIS9e/di9OjRePjwoXY/X19fvfP7+vqaPTcAvSf2NBpNkSf4NBoNCgoKLD6HpccQkfqYoSKiEq1OnTrw9PTEwYMHtetu376NP/74AwDQvHlz5OfnIy0tDXXq1NFb5Cf26tevj8OHD+ud9+jRo2avffToUeTl5WHx4sVo06YN6tati+vXr5s9rkmTJigoKMDu3buVfFUicmHMUBFRiVauXDmMHj0ar7zyCipVqoSQkBDMnDlTOyxB3bp1MXToUAwfPhyLFy9G8+bNcfPmTezYsQNNmjRBz549MX78eHTo0AFLlizB008/jR07duCHH34okrUyVLt2beTl5WHFihV4+umnsW/fPqxZs8Zsm2vWrIkRI0Zg1KhRWL58OZo1a4YrV64gLS0NAwYMUOW+EFHJwgwVEZV4CxcuRIcOHfDMM8+gS5cueOyxxxAdHa3d/uGHH2L48OF4+eWXUa9ePTzzzDM4dOgQwsPDAQDt2rXDmjVrsGTJEjRr1gw//vgjJk+eDB8fn2Kv+8gjj2DJkiV466230LhxY3zyySeYP3++RW1evXo1nn32WYwdOxb169fHmDFj9OquiKh00UiWFBIQEZUyY8aMwe+//45ff/0VgBjC4KuvvkJSUpJzG2YH69evx6RJk3Dnzh1nN4Wo1GKXHxGVCYsWLULXrl3h7++PH374ARs2bMCqVav09jl58iTKlSuHt99+G2PHjnVSS9VVrlw55OXlmc3GEZFtmKEiojJhwIAB2LVrF+7evYtatWph/PjxiI2N1W5PT09Heno6AKBKlSqlZrqWCxcuAADc3d0RGRnp5NYQlV4MqIiIiIhsxKJ0IiIiIhsxoCIiIiKyEQMqIiIiIhsxoCIiIiKyEQMqIiIiIhsxoCIiIiKyEQMqIiIiIhsxoCIiIiKy0f8DJ70VxFUIMoQAAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "t1 = sacc.Sacc.load_fits(\"/global/homes/e/edujb/TXPipe/data/example/output_txcluster_20/cluster_sacc_catalog.sacc\")\n", + "for k in range(0,2):\n", + " for j in range(0,2):\n", + " data2 = []\n", + " radius2 = []\n", + " for i in range(0,8):\n", + " trac = ('cosmodc2-20deg', f'bin_rich_{j}', f'bin_z_{k}', f'radius_{i}')\n", + " data2.append(t1.get_data_points(sacc.data_types.standard_types.cluster_shear, trac)[0].value)\n", + " radius = t1.tracers[f'radius_{i}'].center\n", + " radius2.append(radius)\n", + " print(data2)\n", + " print(data2[0])\n", + " plt.plot(radius2, data2, color=\"blue\", label=f'bin_{k}_{j}')\n", + " plt.ylabel(\"tangential reduced shear\")\n", + " plt.xlabel(\"deg[arcmin]\")\n", + " plt.legend(loc='upper right')\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a40adb76-75bb-408a-8558-4ae1297a3b18", + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { diff --git a/txpipe/extensions/cluster_counts/__init__.py b/txpipe/extensions/cluster_counts/__init__.py index b69ad61e..70341b3e 100644 --- a/txpipe/extensions/cluster_counts/__init__.py +++ b/txpipe/extensions/cluster_counts/__init__.py @@ -1,6 +1,6 @@ #from .ingest import * from .rlens import TXTwoPointRLens - +from .two_point_cluster_source import TXTwoPointClusterSource #from .select import CLClusterShearCatalogs from .bin_cluster import CLClusterBinningRedshiftRichness from .sources_select_compute import * diff --git a/txpipe/extensions/cluster_counts/convert_to_sacc.py b/txpipe/extensions/cluster_counts/convert_to_sacc.py index 3984c82a..fec2b4e2 100644 --- a/txpipe/extensions/cluster_counts/convert_to_sacc.py +++ b/txpipe/extensions/cluster_counts/convert_to_sacc.py @@ -17,6 +17,7 @@ class CLClusterSACC(PipelineStage): #radial bin definition "r_min" : 0.2, #in Mpc "r_max" : 5.0, #in Mpc + "clmm_profile" : True } def run(self): @@ -65,8 +66,10 @@ def get_bins(self, data: dict): rich_edges = (bin_data['cluster_bin_edges']['rich_min'], bin_data['cluster_bin_edges']['rich_max']) bin_z_dict[bin_z] = z_edges bin_rich_dict[bin_rich] = rich_edges - - radius_centers = np.array(data['bin_zbin_0_richbin_0']['clmm_cluster_ensemble'].stacked_data['radius']) + if self.config["clmm_profile"]: + radius_centers = np.array(data['bin_zbin_0_richbin_0']['clmm_cluster_ensemble'].stacked_data['radius']) + else: + radius_centers = bin_data['radial_bins'] rmin = self.config_options['r_min'] rmax = self.config_options['r_max'] radius_edges = np.logspace(np.log10(rmin), np.log10(rmax), len(radius_centers) + 1) @@ -114,7 +117,11 @@ def add_deltasigma_data(self, sacc_obj, data: dict, survey_name: str): for bin_comb, bin_data in data.items(): bin_z, bin_rich = self.transform_bin_string(bin_comb) for i, bin_radius in enumerate(radius_bins): - tangential_comp = bin_data['clmm_cluster_ensemble'].stacked_data[i]['tangential_comp'] + tangential_comp = None + if self.config["clmm_profile"]: + tangential_comp = bin_data['clmm_cluster_ensemble'].stacked_data[i]['tangential_comp'] + else: + tangential_comp = bin_data["xi"][i] sacc_obj.add_data_point(cluster_shear, (survey_name, bin_rich, bin_z, bin_radius), tangential_comp) def add_covariance_data(self, sacc_obj, data: dict): @@ -125,11 +132,27 @@ def add_covariance_data(self, sacc_obj, data: dict): cluster_count = sacc.standard_types.cluster_counts counts_points = np.array(sacc_obj.get_data_points(cluster_count)) counts_cov = np.array([point.value for point in counts_points]) - - deltasigma_cov = [ - bin_data['clmm_cluster_ensemble'].cov['tan_sc'].diagonal() - for bin_data in data.values() - ] + deltasigma_cov = None + if self.config["clmm_profile"]: + # --- Existing CLMM ensemble method --- + deltasigma_cov = [ + bin_data['clmm_cluster_ensemble'].cov['tan_sc'].diagonal() + for bin_data in data.values() + ] + else: + # --- New TreeCorr stacked output method --- + deltasigma_cov = [] + for key, bin_data in data.items(): + # Check if 'cov' exists + if 'cov' in bin_data: + # Use diagonal of covariance if 2D, otherwise assume already 1D + if bin_data['cov'].ndim == 2: + deltasigma_cov.append(np.diag(bin_data['cov'])) + else: + deltasigma_cov.append(bin_data['cov']) + else: + # If no covariance is available, fill with NaNs + deltasigma_cov.append(np.full(len(bin_data['radial_bins']), np.nan)) diag_cov_vector = np.concatenate([counts_cov.flatten(), np.array(deltasigma_cov).flatten()]) sacc_obj.add_covariance(np.diag(diag_cov_vector)) diff --git a/txpipe/extensions/cluster_counts/rlens.py b/txpipe/extensions/cluster_counts/rlens.py index af85ba0e..4625e784 100644 --- a/txpipe/extensions/cluster_counts/rlens.py +++ b/txpipe/extensions/cluster_counts/rlens.py @@ -7,9 +7,6 @@ ) import numpy as np -SHEAR_SHEAR = 0 -SHEAR_POS = 1 -POS_POS = 2 class TXTwoPointRLens(TXTwoPoint): """ @@ -34,11 +31,9 @@ class TXTwoPointRLens(TXTwoPoint): ("binned_shear_catalog", HDFFile), ("binned_random_catalog", HDFFile), ("patch_centers", TextFile), + ("tracer_metadata", HDFFile), ] - outputs = [ - ("rlens_measurement", TextFile), - ("cluster_shear_catalogs", HDFFile), - ] + outputs = [("rlens_measurement", TextFile)] config_options = { # TODO: Allow more fine-grained selection of 2pt subsets to compute @@ -67,30 +62,21 @@ class TXTwoPointRLens(TXTwoPoint): "use_subsampled_randoms": False, } - def read_metadata(self): - return {} - - def get_lens_catalog(self, i): # Override the lens catalog generation. # This is like the parent version except we also add the r_col keyword import treecorr - bin_name = None - with self.open_input("binned_lens_catalog") as f: - for j, zbin_richbin in enumerate(f['cluster_bin'].keys()): - if j == i: - bin_name = zbin_richbin cat = treecorr.Catalog( self.get_input("binned_lens_catalog"), - ext=f"/cluster_bin/{bin_name}", + ext=f"/lens/bin_{i}", ra_col="ra", dec_col="dec", - #r_col="comoving_distance", - #w_col="weight", + r_col="comoving_distance", + w_col="weight", ra_units="degree", dec_units="degree", - patch_centers=None,#self.get_input("patch_centers"), + patch_centers=self.get_input("patch_centers"), save_patch_dir=self.get_patch_dir("binned_lens_catalog", i), ) return cat @@ -108,36 +94,14 @@ def get_random_catalog(self, i): ext=f"/randoms/bin_{i}", ra_col="ra", dec_col="dec", - #r_col="comoving_distance", + r_col="comoving_distance", ra_units="degree", dec_units="degree", - patch_centers=None,#self.get_input("patch_centers"), + patch_centers=self.get_input("patch_centers"), save_patch_dir=self.get_patch_dir("binned_random_catalog", i), ) return rancat - def get_shear_catalog(self, i): - import treecorr - - # Load and calibrate the appropriate bin data - cat = treecorr.Catalog( - self.get_input("binned_shear_catalog"), - ext=f"/shear/bin_{i}", - g1_col="g1", - g2_col="g2", - ra_col="ra", - dec_col="dec", - w_col="weight", - ra_units="degree", - dec_units="degree", - patch_centers=None,#self.get_input("patch_centers"), - save_patch_dir=self.get_patch_dir("binned_shear_catalog", i), - flip_g1=self.config["flip_g1"], - flip_g2=self.config["flip_g2"], - ) - - return cat - def call_treecorr(self, i, j, k): # The parent class uses the label for gamma_t measurements # here, so we overwrite it. @@ -152,23 +116,4 @@ def write_output(self, source_list, lens_list, meta, results): for result in results: for logr, xi in zip(result.object.meanlogr, result.object.xi): print(result.i, result.j, np.exp(logr), xi) - print(result.i, result.j, np.exp(logr), xi, file=f) - - def _read_nbin_from_tomography(self): - if self.get_input("binned_shear_catalog") == "none": - nbin_source = 0 - else: - with self.open_input("binned_shear_catalog") as f: - nbin_source = f["shear"].attrs["nbin_source"] - - if self.get_input("binned_lens_catalog") == "none": - nbin_lens = 0 - else: - with self.open_input("binned_lens_catalog") as f: - nbin_lens = len(f["cluster_bin"]) - - source_list = list(range(nbin_source)) - lens_list = list(range(nbin_lens)) - - return source_list, lens_list - + print(result.i, result.j, np.exp(logr), xi, file=f) \ No newline at end of file diff --git a/txpipe/extensions/cluster_counts/rlens_new.py b/txpipe/extensions/cluster_counts/rlens_new.py deleted file mode 100644 index 25198d38..00000000 --- a/txpipe/extensions/cluster_counts/rlens_new.py +++ /dev/null @@ -1,958 +0,0 @@ -from ...base_stage import PipelineStage -from ...data_types import ( - HDFFile, - ShearCatalog, - SACCFile, - TextFile, - MapsFile, - QPNOfZFile, -) -from ...utils.patches import PatchMaker -import numpy as np -import collections -import sys -import os -import pathlib -from time import perf_counter -import gc -from ...utils import choose_pixelization - -# This creates a little mini-type, like a struct, -# for holding individual measurements -Measurement = collections.namedtuple("Measurement", ["corr_type", "object", "i", "j"]) - -SHEAR_SHEAR = 0 -SHEAR_POS = 1 -POS_POS = 2 - -#external cross correlations -POS_EXT = 3 -SHEAR_EXT = 4 - - - -from ...twopoint import TXTwoPoint -from ...data_types import ( - HDFFile, - ShearCatalog, - PNGFile, - TextFile, -) -import numpy as np - -class TXTwoPointCluster(PipelineStage): - """ - Make 2pt measurements using TreeCorr - - This stage make the full set of cosmic shear, galaxy-galaxy lensing, - and galaxy density measurements on the tomographic catalog using TreeCorr. - - Results are saved to a sacc file. - """ - name = "TXTwoPointClusterXXX" - inputs = [ - ("cluster_catalog_tomography", HDFFile), - ("shear_tomography_catalog", HDFFile), - ("shear_catalog", HDFFile), - ("binned_random_catalog", HDFFile), - ("patch_centers", TextFile), - ("tracer_metadata", HDFFile), - ] - outputs = [("rlens_measurement", TextFile)] - # Add values to the config file that are not previously defined - config_options = { - # TODO: Allow more fine-grained selection of 2pt subsets to compute - "calcs": [0, 1, 2], - "min_sep": 0.5, - "max_sep": 300.0, - "nbins": 9, - "bin_slop": 0.0, - "sep_units": "arcmin", - "flip_g1": False, - "flip_g2": True, - "cores_per_task": 20, - "verbose": 1, - "source_bins": [-1], - "lens_bins": [-1], - "reduce_randoms_size": 1.0, - "do_shear_shear": True, - "do_shear_pos": True, - "do_pos_pos": True, - "auto_only": False, - "var_method": "jackknife", - "use_randoms": True, - "low_mem": False, - "patch_dir": "./cache/patches", - "chunk_rows": 100_000, - "share_patch_files": False, - "metric": "Euclidean", - "gaussian_sims_factor": [1.], - "use_subsampled_randoms": True, #use subsampled randoms file for RR - } - - def run(self): - """ - Run the analysis for this stage. - """ - import sacc - import healpy - import treecorr - - # Binning information - source_list, lens_list = self.read_nbin() - self.config["num_threads"] = int(os.environ.get("OMP_NUM_THREADS", 1)) - - if self.rank == 0: - # This is a workaround for the fact the the ceci config stuff doesn't - # quite handle the get method properly. - # Which metrics are available, and how they are interpreted, depends on - # whether a distance is in the catalogs returned in get_shear_catalog - # and friends, below. In this base class only the 2D metrics will be - # available, but subclasses can specify to load a distance column too. - metric = self.config["metric"] if "metric" in self.config else "Euclidean" - print(f"Running TreeCorr with metric \"{metric}\"") - - # Calculate metadata like the area and related - # quantities - meta = self.read_metadata() - - # Choose which pairs of bins to calculate - calcs = self.select_calculations(source_list, lens_list) - sys.stdout.flush() - - # Split the catalogs into patch files - self.prepare_patches(calcs, meta) - - results = [] - for i, j, k in calcs: - result = self.call_treecorr(i, j, k) - results.append(result) - - if self.comm: - self.comm.Barrier() - - # Save the results - self.write_output(source_list, lens_list, meta, results) - - def select_calculations(self, source_list, lens_list): - calcs = [] - - # For shear-shear we omit pairs with j>i - if self.config["do_shear_shear"]: - print('DOING SHEAR-SHEAR') - k = SHEAR_SHEAR - for i in source_list: - for j in range(i + 1): - if j in source_list: - calcs.append((i, j, k)) - - # For shear-position we use all pairs - if self.config["do_shear_pos"]: - print('DOING SHEAR-POS') - k = SHEAR_POS - for i in source_list: - for j in lens_list: - calcs.append((i, j, k)) - - # For position-position we omit pairs with j>i - if self.config["do_pos_pos"]: - print('DOING POS-POS') - if not self.config["use_randoms"]: - raise ValueError( - "You need to have a random catalog to calculate position-position correlations" - ) - k = POS_POS - if self.config["auto_only"]: - for i in lens_list: - calcs.append((i, i, k)) - else: - for i in lens_list: - for j in range(i + 1): - if j in lens_list: - calcs.append((i, j, k)) - - if self.rank == 0: - print(f"Running {len(calcs)} calculations: {calcs}") - - return calcs - - def read_nbin(self): - """ - Determine the bins to use in this analysis, either from the input file - or from the configuration. - """ - if self.config["source_bins"] == [-1] and self.config["lens_bins"] == [-1]: - source_list, lens_list = self._read_nbin_from_tomography() - else: - source_list, lens_list = self._read_nbin_from_config() - - ns = len(source_list) - nl = len(lens_list) - if self.rank == 0: - print(f"Running with {ns} source bins and {nl} lens bins") - - return source_list, lens_list - - # These two functions can be combined into a single one. - def _read_nbin_from_tomography(self): - if self.get_input("shear_tomography_catalog") == "none": - nbin_source = 0 - else: - with self.open_input("shear_tomography_catalog") as f: - nbin_source = f["shear"].attrs["nbin_source"] - - if self.get_input("binned_lens_catalog") == "none": - nbin_lens = 0 - else: - with self.open_input("binned_lens_catalog") as f: - nbin_lens = f["lens"].attrs["nbin_lens"] - - source_list = list(range(nbin_source)) - lens_list = list(range(nbin_lens)) - - return source_list, lens_list - - def _read_nbin_from_config(self): - # TODO handle the case where the user only specefies - # bins for only sources or only lenses - source_list = self.config["source_bins"] - lens_list = self.config["lens_bins"] - - # catch bad input - tomo_source_list, tomo_lens_list = self._read_nbin_from_tomography() - tomo_nbin_source = len(tomo_source_list) - tomo_nbin_lens = len(tomo_lens_list) - - nbin_source = len(source_list) - nbin_lens = len(lens_list) - - if source_list == [-1]: - source_list = tomo_source_list - if lens_list == [-1]: - lens_list = tomo_lens_list - - # if more bins are input than exist, raise an error - if not nbin_source <= tomo_nbin_source: - raise ValueError( - f"Requested too many source bins in the config ({nbin_source}): max is {tomo_nbin_source}" - ) - if not nbin_lens <= tomo_nbin_lens: - raise ValueError( - f"Requested too many lens bins in the config ({nbin_lens}): max is {tomo_nbin_lens}" - ) - - # make sure the bin numbers actually exist - for i in source_list: - if i not in tomo_source_list: - raise ValueError( - f"Requested source bin {i} that is not in the input file" - ) - - for i in lens_list: - if i not in tomo_lens_list: - raise ValueError( - f"Requested lens bin {i} that is not in the input file" - ) - - return source_list, lens_list - - - def add_data_points(self, S, results): - import treecorr - import sacc - - XI = "combined" - XIP = sacc.standard_types.galaxy_shear_xi_plus - XIM = sacc.standard_types.galaxy_shear_xi_minus - GAMMAT = sacc.standard_types.galaxy_shearDensity_xi_t - GAMMAX = sacc.standard_types.galaxy_shearDensity_xi_x - WTHETA = sacc.standard_types.galaxy_density_xi - - comb = [] - for index, d in enumerate(results): - # First the tracers and generic tags - tracer1 = f"source_{d.i}" if d.corr_type in [XI, GAMMAT] else f"lens_{d.i}" - tracer2 = f"source_{d.j}" if d.corr_type in [XI] else f"lens_{d.j}" - - # This happens when there is an empty bin. We can't do a covariance - # here, or anything useful, really, so we just skip this bin. - if d.object is None: - continue - - theta = np.exp(d.object.meanlogr) - npair = d.object.npairs - weight = d.object.weight - # xip / xim is a special case because it has two observables. - # the other two are together below - if d.corr_type == XI: - xip = d.object.xip - xim = d.object.xim - xiperr = np.sqrt(d.object.varxip) - ximerr = np.sqrt(d.object.varxim) - n = len(xip) - # add all the data points to the sacc - for i in range(n): - S.add_data_point( - XIP, - (tracer1, tracer2), - xip[i], - theta=theta[i], - error=xiperr[i], - npair=npair[i], - weight=weight[i], - ) - for i in range(n): - S.add_data_point( - XIM, - (tracer1, tracer2), - xim[i], - theta=theta[i], - error=ximerr[i], - npair=npair[i], - weight=weight[i], - ) - else: - if self.config['gaussian_sims_factor'] != [1.]: - # only for gammat and wtheta, for the gaussian simulations we need to scale the measurements up to correct for - # the scaling of the density field when building the simulations. - if 'lens' in tracer2: - if 'lens' in tracer1: - scaling_factor = self.config['gaussian_sims_factor'][int(tracer1[-1])]*self.config['gaussian_sims_factor'][int(tracer2[-1])] - else: - scaling_factor = self.config['gaussian_sims_factor'][int(tracer2[-1])] - - d.object.xi *=scaling_factor - d.object.varxi *=(scaling_factor**2) - - xi = d.object.xi - err = np.sqrt(d.object.varxi) - n = len(xi) - for i in range(n): - S.add_data_point( - d.corr_type, - (tracer1, tracer2), - xi[i], - theta=theta[i], - error=err[i], - weight=weight[i], - ) - - # We build up the comb list to get the covariance of it later - # in the same order as our data points - comb.append(d.object) - - - - # Add the covariance. There are several different jackknife approaches - # available - see the treecorr docs - if treecorr.__version__.startswith("4.2."): - if self.rank == 0: - print("Using old TreeCorr - covariance may be slow. " - "Consider using 4.3 from github main branch.") - cov = treecorr.estimate_multi_cov(comb, self.config["var_method"]) - else: - if self.rank == 0: - print("Using new TreeCorr 4.3 or above") - cov = treecorr.estimate_multi_cov(comb, self.config["var_method"], comm=self.comm) - S.add_covariance(cov) - - def add_gamma_x_data_points(self, S, results): - import treecorr - import sacc - - XI = "combined" - GAMMAT = sacc.standard_types.galaxy_shearDensity_xi_t - GAMMAX = sacc.standard_types.galaxy_shearDensity_xi_x - - covs = [] - for index, d in enumerate(results): - tracer1 = ( - f"source_{d.i}" if d.corr_type in [XI, GAMMAT] else f"lens_{d.i}" - ) - tracer2 = f"source_{d.j}" if d.corr_type in [XI] else f"lens_{d.j}" - - if d.corr_type == GAMMAT: - theta = np.exp(d.object.meanlogr) - npair = d.object.npairs - weight = d.object.weight - xi_x = d.object.xi_im - covX = d.object.estimate_cov("shot") - # TreeCorr v5 returns the diagonal of the covariance matrix - # instead of a full but diagal (so almost all zero) format. - if treecorr.__version_info__[0] >= 5: - covX = np.diag(covX) - covs.append(covX) - err = np.sqrt(np.diag(covX)) - n = len(xi_x) - for i in range(n): - S.add_data_point( - GAMMAX, - (tracer1, tracer2), - xi_x[i], - theta=theta[i], - error=err[i], - weight=weight[i], - ) - S.add_covariance(covs) - - - def write_output(self, source_list, lens_list, meta, results): - import sacc - import treecorr - - XI = "combined" - XIP = sacc.standard_types.galaxy_shear_xi_plus - XIM = sacc.standard_types.galaxy_shear_xi_minus - GAMMAT = sacc.standard_types.galaxy_shearDensity_xi_t - GAMMAX = sacc.standard_types.galaxy_shearDensity_xi_x - WTHETA = sacc.standard_types.galaxy_density_xi - - S = sacc.Sacc() - S2 = sacc.Sacc() - - # We include the n(z) data in the output. - # So here we load it in and add it to the data - - # Load the tracer data N(z) from an input file and - # copy it to the output, for convenience - if self.config["do_shear_pos"] or self.config["do_shear_shear"]: - if source_list: - with self.open_input("shear_photoz_stack", wrapper=True) as f: - for i in source_list: - z, Nz = f.get_bin_n_of_z(i) - S.add_tracer("NZ", f"source_{i}", z, Nz) - if self.config["do_shear_pos"] == True: - S2.add_tracer("NZ", f"source_{i}", z, Nz) - else: - sys.exit("Requesting a measurement that requires source galaxies but no source_list provided") - - if self.config["do_pos_pos"] or self.config["do_shear_pos"]: - if lens_list: - with self.open_input("lens_photoz_stack", wrapper=True) as f: - # For both source and lens - for i in lens_list: - z, Nz = f.get_bin_n_of_z(i) - S.add_tracer("NZ", f"lens_{i}", z, Nz) - if self.config["do_shear_pos"] == True: - S2.add_tracer("NZ", f"lens_{i}", z, Nz) - else: - sys.exit("Requesting a measurement that requires lens galaxies but no lens_list provided") - - # Now build up the collection of data points, adding them all to - # the sacc data one by one. - self.add_data_points(S, results) - - # The other processes are only needed for the covariance estimation. - # They do a bunch of other stuff here that isn't actually needed, but - # it should all be very fast. After this point they are not needed - # at all so return - if self.rank != 0: - return - - - # Our data points may currently be in any order depending on which processes - # ran which calculations. Re-order them. - S.to_canonical_order() - - - self.write_metadata(S, meta) - - - # Finally, save the output to Sacc file - S.save_fits(self.get_output("twopoint_data_real_raw"), overwrite=True) - - - # Adding the gammaX calculation: - if self.config["do_shear_pos"] == True: - self.add_gamma_x_data_points(S2, results) - S2.to_canonical_order() - self.write_metadata(S2, meta) - # always write the file, even if it is empty - S2.save_fits(self.get_output("twopoint_gamma_x"), overwrite=True) - - def write_metadata(self, S, meta): - # We also save the associated metadata to the file - for k, v in meta.items(): - if np.isscalar(v): - S.metadata[k] = v - else: - for i, vi in enumerate(v): - S.metadata[f"{k}_{i}"] = vi - - # Add provenance metadata. In managed formats this is done - # automatically, but because the Sacc library is external - # we do it manually here. - provenance = self.gather_provenance() - provenance.update(SACCFile.generate_provenance()) - for key, value in provenance.items(): - if isinstance(value, str) and "\n" in value: - values = value.split("\n") - for i, v in enumerate(values): - S.metadata[f"provenance/{key}_{i}"] = v - else: - S.metadata[f"provenance/{key}"] = value - - def call_treecorr(self, i, j, k): - """ - This is a wrapper for interaction with treecorr. - """ - import sacc - import pickle - #TODO: fix up the caching code - if self.name == "TXTwoPoint" or self.name == "TXTwoPointPixel": - pickle_filename = self.get_output("twopoint_data_real_raw") + f".checkpoint-{i}-{j}-{k}.pkl" - #pickle_filename = f"treecorr-cache-{i}-{j}-{k}.pkl" - - if os.path.exists(pickle_filename): - print(f"{self.rank} WARNING USING THIS PICKLE FILE I FOUND: {pickle_filename}") - with open(pickle_filename, "rb") as f: - result = pickle.load(f) - return result - - if k == SHEAR_SHEAR: - xx = self.calculate_shear_shear(i, j) - xtype = "combined" - elif k == SHEAR_POS: - xx = self.calculate_shear_pos(i, j) - xtype = sacc.standard_types.galaxy_shearDensity_xi_t - elif k == POS_POS: - xx = self.calculate_pos_pos(i, j) - xtype = sacc.standard_types.galaxy_density_xi - else: - raise ValueError(f"Unknown correlation function {k}") - - # Force garbage collection here to make sure all the - # catalogs are definitely freed - gc.collect() - - # The measurement object collects the results and type info. - # we use it because the ordering will not be simple if we have - # parallelized, so it's good to keep explicit track. - result = Measurement(xtype, xx, i, j) - - sys.stdout.flush() - - if self.comm: - self.comm.Barrier() - - if self.name == "TXTwoPoint" or self.name == "TXTwoPointPixel": - if self.rank == 0: - print(f"Pickling result to {pickle_filename}") - with open(pickle_filename, "wb") as f: - pickle.dump(result, f) - - return result - - def prepare_patches(self, calcs, meta): - """ - For each catalog to be generated, have one process load the catalog - and write its patch files out to disc. These are then re-used later - by all the different processes. - - Parameters - ---------- - - calcs: list - A list of (bin1, bin2, bin_type) where bin1 and bin2 are indices - or bin labels and bin_type is one of the constants SHEAR_SHEAR, - SHEAR_POS, or POS_POS. - - meta: dict - A dict to which the number of patches (or zero, if no patches) will - be added for each catalog type, with keys "npatch_shear", "npatch_pos", - and "npatch_ran". - """ - # Make the full list of catalogs to run - cats = set() - - # Use shear-shear and pos-pos only here as they represent - # catalogs not pairs. - for i, j, k in calcs: - if k == SHEAR_SHEAR: - cats.add((i, SHEAR_SHEAR)) - cats.add((j, SHEAR_SHEAR)) - elif k == SHEAR_POS: - cats.add((i, SHEAR_SHEAR)) - cats.add((j, POS_POS)) - elif k == POS_POS: - cats.add((i, POS_POS)) - cats.add((j, POS_POS)) - cats = list(cats) - cats.sort(key=str) - - chunk_rows = self.config["chunk_rows"] - npatch_shear = 0 - npatch_pos = 0 - npatch_ran = 0 - - self.empty_patch_exists = {} - - # Parallelization is now done at the patch level - for (h, k) in cats: - ktxt = "shear" if k == SHEAR_SHEAR else "position" - print(f"Rank {self.rank} making patches for {ktxt} catalog bin {h}") - - # For shear we just have the one catalog. For position we may - # have randoms also. We explicitly delete catalogs after loading - # them to ensure we don't have two in memory at once. - if k == SHEAR_SHEAR: - cat = self.get_shear_catalog(h) - npatch_shear,contains_empty = PatchMaker.run(cat, chunk_rows, self.comm) - self.empty_patch_exists[cat.save_patch_dir] = contains_empty - del cat - else: - cat = self.get_lens_catalog(h) - npatch_pos,contains_empty = PatchMaker.run(cat, chunk_rows, self.comm) - self.empty_patch_exists[cat.save_patch_dir] = contains_empty - del cat - - ran_cat = self.get_random_catalog(h) - # support use_randoms = False - if ran_cat is None: - continue - npatch_ran,contains_empty = PatchMaker.run(ran_cat, chunk_rows, self.comm) - self.empty_patch_exists[ran_cat.save_patch_dir] = contains_empty - del ran_cat - - if self.config["use_subsampled_randoms"]: - ran_cat = self.get_subsampled_random_catalog(h) - npatch_ran,contains_empty = PatchMaker.run(ran_cat, chunk_rows, self.comm) - self.empty_patch_exists[ran_cat.save_patch_dir] = contains_empty - del ran_cat - - meta["npatch_shear"] = npatch_shear - meta["npatch_pos"] = npatch_pos - meta["npatch_ran"] = npatch_ran - # stop other processes progressing to the rest of the code and - # trying to load things we have not written yet - if self.comm is not None: - self.comm.Barrier() - - def get_patch_dir(self, input_tag, b): - """ - Select a patch directory for the file with the given input tag - and with a bin number/label. - - To ensure that if you change the catalog the patch dir will also - change, the directory path includes the unique ID of the input file. - - Parameters - ---------- - input_tag: str - One of the tags in the class's inputs attribute - b: any - An additional label used as the last component in the returned - directory - - Returns - ------- - str: a directory, which has been created if it did not exist already. - """ - # start from a user-specified base directory - patch_base = self.config["patch_dir"] - - # append the unique identifier for the parent catalog file - with self.open_input(input_tag, wrapper=True) as f: - p = f.read_provenance() - uuid = p["uuid"] - pth = pathlib.Path(f.path).resolve() - ctime = os.stat(pth).st_ctime - - # We expect the input files to be generated within a pipeline and so always - # have input files to have a unique ID. But if for some reason it doesn't - # have one we handle that too. - if uuid == "UNKNOWN": - ident = hash(f"{pth}{ctime}").to_bytes(8, "big", signed=True).hex() - name = f"{input_tag}_{ident}" - else: - name = f"{input_tag}_{uuid}" - - # Include a tag for the current stage name, so that - # if we are running several subclasses at the same time - # they don't interfere with each other. This is a waste of - # disc space, but hopefully we are not short of that. - if not self.config["share_patch_files"]: - name = self.instance_name + name - - # And finally append the bin name or number - patch_dir = pathlib.Path(patch_base) / name / str(b) - - # Make the directory and return it - pathlib.Path(patch_dir).mkdir(exist_ok=True, parents=True) - return patch_dir - - def get_shear_catalog(self, i): - import treecorr - - # Load and calibrate the appropriate bin data - cat = treecorr.Catalog( - self.get_input("shear_tomography_catalog"), - ext=f"/shear/bin_{i}", - g1_col="g1", - g2_col="g2", - ra_col="ra", - dec_col="dec", - w_col="weight", - ra_units="degree", - dec_units="degree", - patch_centers=self.get_input("patch_centers"), - save_patch_dir=self.get_patch_dir("shear_tomography_catalog", i), - flip_g1=self.config["flip_g1"], - flip_g2=self.config["flip_g2"], - ) - - return cat - - - def get_lens_catalog(self, i): - import treecorr - - # Load and calibrate the appropriate bin data - cat = treecorr.Catalog( - self.get_input("binned_lens_catalog"), - ext=f"/lens/bin_{i}", - ra_col="ra", - dec_col="dec", - w_col="weight", - ra_units="degree", - dec_units="degree", - patch_centers=self.get_input("patch_centers"), - save_patch_dir=self.get_patch_dir("binned_lens_catalog", i), - ) - - return cat - - def get_random_catalog(self, i): - import treecorr - - if not self.config["use_randoms"]: - return None - - rancat = treecorr.Catalog( - self.get_input("binned_random_catalog"), - ext=f"/randoms/bin_{i}", - ra_col="ra", - dec_col="dec", - ra_units="degree", - dec_units="degree", - patch_centers=self.get_input("patch_centers"), - save_patch_dir=self.get_patch_dir("binned_random_catalog", i), - ) - - return rancat - - def get_subsampled_random_catalog(self, i): - import treecorr - - if not self.config["use_randoms"]: - return None - - rancat = treecorr.Catalog( - self.get_input("binned_random_catalog_sub"), - ext=f"/randoms/bin_{i}", - ra_col="ra", - dec_col="dec", - ra_units="degree", - dec_units="degree", - patch_centers=self.get_input("patch_centers"), - save_patch_dir=self.get_patch_dir("binned_random_catalog_sub", i), - ) - - return rancat - - def touch_patches(self, cat): - # If any patches were empty for this cat - # run get_patches on rank 0 and bcast - # this will re-make patches but prevents processes conflicting - # in the gg.process - # If no patches are empty returns the cat, unaltered - if cat is None: - return cat - - if self.empty_patch_exists[cat.save_patch_dir]: - if self.rank==0: - cat.get_patches() - if self.comm is not None: - cat = self.comm.bcast(cat, root=0) - - return cat - - def calculate_shear_shear(self, i, j): - import treecorr - - cat_i = self.get_shear_catalog(i) - cat_i = self.touch_patches(cat_i) - n_i = cat_i.nobj - - if i == j: - cat_j = None - n_j = n_i - else: - cat_j = self.get_shear_catalog(j) - cat_j = self.touch_patches(cat_j) - n_j = cat_j.nobj - - - if self.rank == 0: - print( - f"Calculating shear-shear bin pair ({i},{j}): {n_i} x {n_j} objects using MPI" - ) - - if n_i == 0 or n_j == 0: - if self.rank == 0: - print("Empty catalog: returning None") - return None - - gg = treecorr.GGCorrelation(self.config) - t1 = perf_counter() - gg.process(cat_i, cat_j, low_mem=self.config["low_mem"], comm=self.comm) - t2 = perf_counter() - if self.rank == 0: - print(f"Processing took {t2 - t1:.1f} seconds") - - return gg - - def calculate_shear_pos(self, i, j): - import treecorr - - cat_i = self.get_shear_catalog(i) - cat_i = self.touch_patches(cat_i) - n_i = cat_i.nobj - - cat_j = self.get_lens_catalog(j) - cat_j = self.touch_patches(cat_j) - rancat_j = self.get_random_catalog(j) - rancat_j = self.touch_patches(rancat_j) - n_j = cat_j.nobj - n_rand_j = rancat_j.nobj if rancat_j is not None else 0 - - if self.rank == 0: - print( - f"Calculating shear-position bin pair ({i},{j}): {n_i} x {n_j} objects, {n_rand_j} randoms" - ) - - if n_i == 0 or n_j == 0: - if self.rank == 0: - print("Empty catalog: returning None") - return None - - ng = treecorr.NGCorrelation(self.config) - t1 = perf_counter() - ng.process(cat_j, cat_i, comm=self.comm, low_mem=self.config["low_mem"]) - - if rancat_j: - rg = treecorr.NGCorrelation(self.config) - rg.process(rancat_j, cat_i, comm=self.comm, low_mem=self.config["low_mem"]) - else: - rg = None - - ng.calculateXi(rg=rg) - t2 = perf_counter() - if self.rank == 0: - print(f"Processing took {t2 - t1:.1f} seconds") - - return ng - - def calculate_pos_pos(self, i, j): - import treecorr - - cat_i = self.get_lens_catalog(i) - cat_i = self.touch_patches(cat_i) - rancat_i = self.get_random_catalog(i) - rancat_i = self.touch_patches(rancat_i) - n_i = cat_i.nobj - n_rand_i = rancat_i.nobj if rancat_i is not None else 0 - - if i == j: - cat_j = None - rancat_j = rancat_i - n_j = n_i - n_rand_j = n_rand_i - else: - cat_j = self.get_lens_catalog(j) - cat_j = self.touch_patches(cat_j) - rancat_j = self.get_random_catalog(j) - rancat_j = self.touch_patches(rancat_j) - n_j = cat_j.nobj - n_rand_j = rancat_j.nobj - - if self.config['use_subsampled_randoms']: - rancat_sub_i = self.get_subsampled_random_catalog(i) - rancat_sub_i = self.touch_patches(rancat_sub_i) - n_rand_sub_i = rancat_sub_i.nobj if rancat_sub_i is not None else 0 - - if i == j: - rancat_sub_j = rancat_sub_i - n_rand_sub_j = n_rand_sub_i - else: - rancat_sub_j = self.get_subsampled_random_catalog(j) - rancat_sub_j = self.touch_patches(rancat_sub_j) - n_rand_sub_j = rancat_sub_j.nobj if rancat_sub_j is not None else 0 - - - if self.rank == 0: - print( - f"Calculating position-position bin pair ({i}, {j}): {n_i} x {n_j} objects, {n_rand_i} x {n_rand_j} randoms" - ) - if self.config["use_subsampled_randoms"]: - print(f"and for the rr term, {n_rand_sub_i} x {n_rand_sub_j} pairs") - - if n_i == 0 or n_j == 0: - if self.rank == 0: - print("Empty catalog: returning None") - return None - - t1 = perf_counter() - - nn = treecorr.NNCorrelation(self.config) - nn.process(cat_i, cat_j, comm=self.comm, low_mem=self.config["low_mem"]) - - nr = treecorr.NNCorrelation(self.config) - nr.process(cat_i, rancat_j, comm=self.comm, low_mem=self.config["low_mem"]) - - # The next calculation is faster if we explicitly tell TreeCorr - # that its two catalogs here are the same one. - if i == j: - rancat_j = None - rancat_sub_j = None - - rr = treecorr.NNCorrelation(self.config) - if self.config["use_subsampled_randoms"]: - rr.process(rancat_sub_i, rancat_sub_j, comm=self.comm, low_mem=self.config["low_mem"]) - else: - rr.process(rancat_i, rancat_j, comm=self.comm, low_mem=self.config["low_mem"]) - - if i == j: - rn = None - else: - rn = treecorr.NNCorrelation(self.config) - rn.process(rancat_i, cat_j, comm=self.comm, low_mem=self.config["low_mem"]) - - t2 = perf_counter() - nn.calculateXi(rr=rr, dr=nr, rd=rn) - if self.rank == 0: - print(f"Processing took {t2 - t1:.1f} seconds") - - return nn - - def read_metadata(self): - meta_data = self.open_input("tracer_metadata") - area = meta_data["tracers"].attrs["area"] - meta = {} - meta["area"] = area - try: - sigma_e = meta_data["tracers/sigma_e"][:] - N_eff = meta_data["tracers/N_eff"][:] - mean_e1 = meta_data["tracers/mean_e1"][:] - mean_e2 = meta_data["tracers/mean_e2"][:] - - meta["neff"] = N_eff - meta["area"] = area - meta["sigma_e"] = sigma_e - meta["mean_e1"] = mean_e1 - meta["mean_e2"] = mean_e2 - - except KeyError: #will happen for lens only runs - pass - - return meta diff --git a/txpipe/extensions/cluster_counts/two_point_cluster_source.py b/txpipe/extensions/cluster_counts/two_point_cluster_source.py new file mode 100644 index 00000000..f94feeca --- /dev/null +++ b/txpipe/extensions/cluster_counts/two_point_cluster_source.py @@ -0,0 +1,346 @@ +from ...twopoint import TXTwoPoint +from ...data_types import ( + HDFFile, + ShearCatalog, + TomographyCatalog, + PNGFile, + TextFile, + PickleFile, +) +import numpy as np + +SHEAR_SHEAR = 0 +SHEAR_POS = 1 +POS_POS = 2 + +class TXTwoPointClusterSource(TXTwoPoint): + """ + Measure 2-pt shear-position using the Rlens metric + + The Rlens metric uses the impact factor between the vector to the source galaxy + and the location of the lens galaxy as its distance. + + Compared to the parent TXTwoPoint class this: + - does only the shear-position correlation + - loads comoving coordinates as the radial distance in the lens and random catalogs + - removes the sep_unit configuration option since the unit must always be Mpc + - changes the default metric + - does not yet save any results correctly, just prints them out. + + This is mainly an example stage for future CL work. + + """ + name = "TXTwoPointClusterSource" + inputs = [ + ("binned_lens_catalog", HDFFile), + ("binned_shear_catalog", HDFFile), + ("binned_random_catalog", HDFFile), + ("shear_tomography_catalog", TomographyCatalog), + ] + # ("patch_centers", TextFile), + #] + outputs = [ + ("cluster_profiles", PickleFile), + ] + + config_options = { + # TODO: Allow more fine-grained selection of 2pt subsets to compute + "calcs": [0, 1, 2], + "min_sep": 0.1, # arcmin + "max_sep": 200.0, # arcmin + "nbins": 9, + "bin_slop": 0.1, + "sep_units": "arcmin", + "flip_g1": False, + "flip_g2": True, + "cores_per_task": 20, + "verbose": 1, + "source_bins": [-1], + "lens_bins": [-1], + "reduce_randoms_size": 1.0, + "do_shear_shear": False, + "do_shear_pos": True, + "do_pos_pos": False, + "var_method": "jackknife", + "use_randoms": True, + "low_mem": False, + "patch_dir": "./cache/patches", + "chunk_rows": 100_000, + "share_patch_files": False, + "metric": "Euclidean", + "use_subsampled_randoms": False, + } + + def read_metadata(self): + return {} + + + def get_lens_catalog(self, i): + # Override the lens catalog generation. + # This is like the parent version except we also add the r_col keyword + import treecorr + bin_name = None + with self.open_input("binned_lens_catalog") as f: + for j, zbin_richbin in enumerate(f['cluster_bin'].keys()): + if j == i: + bin_name = zbin_richbin + + cat = treecorr.Catalog( + self.get_input("binned_lens_catalog"), + ext=f"/cluster_bin/{bin_name}", + ra_col="ra", + dec_col="dec", + #w_col="weight", + ra_units="degree", + dec_units="degree", + patch_centers=None,#self.get_input("patch_centers"), + save_patch_dir=self.get_patch_dir("binned_lens_catalog", i), + ) + return cat + + def get_random_catalog(self, i): + # As with the lens catalog version, we add the r_col keyword + # compare to the parent class + import treecorr + + if not self.config["use_randoms"]: + return None + + rancat = treecorr.Catalog( + self.get_input("binned_random_catalog"), + ext=f"/randoms/bin_{i}", + ra_col="ra", + dec_col="dec", + ra_units="degree", + dec_units="degree", + patch_centers=None,#self.get_input("patch_centers"), + save_patch_dir=self.get_patch_dir("binned_random_catalog", i), + ) + return rancat + + def get_shear_catalog(self, i): + import treecorr + + # Load and calibrate the appropriate bin data + cat = treecorr.Catalog( + self.get_input("binned_shear_catalog"), + ext=f"/shear/bin_{i}", + g1_col="g1", + g2_col="g2", + ra_col="ra", + dec_col="dec", + w_col="weight", + ra_units="degree", + dec_units="degree", + patch_centers=None,#self.get_input("patch_centers"), + save_patch_dir=self.get_patch_dir("binned_shear_catalog", i), + flip_g1=self.config["flip_g1"], + flip_g2=self.config["flip_g2"], + ) + + return cat + + def call_treecorr(self, i, j, k): + # The parent class uses the label for gamma_t measurements + # here, so we overwrite it. + result = super().call_treecorr(i, j, k) + # choose a better name maybe! + return result._replace(corr_type="galaxy_shearDensity_rlens") + + # def write_output(self, source_list, lens_list, meta, results): + # with self.open_output("cluster_profiles") as f: + # print("#i_bin j_bin mean_r_Mpc rlens") + # #print("#i_bin j_bin mean_r_Mpc rlens", file=f) + # for result in results: + # print(result.object) + # print(result.object.xi) + # for logr, xi in zip(result.object.meanlogr, result.object.xi): + # print(result.i, result.j, np.exp(logr), xi) + # #print(result.i, result.j, np.exp(logr), xi, file=f) + # def write_output(self, source_list, lens_list, meta, results): + # import pickle + # import numpy as np + # # Dictionary to store results for each bin + # binned_cluster_stack = {} + # counts = None + # for result in results: + # for logr, xi in zip(result.object.meanlogr, result.object.xi): + # print(result.i, result.j, np.exp(logr), xi) + # cluster_bin_edges = {} + # with self.open_input("binned_lens_catalog") as f: + # for j, zbin_richbin in enumerate(f['cluster_bin'].keys()): + # bin_name = zbin_richbin + # ext=f"/cluster_bin/{bin_name}", + # metadata = f[ext].attrs + # counts = len(list(f[ext]["cluster_id"])) + # cluster_bin_edges["rich_max"] = metadata["rich_max"] + # cluster_bin_edges["rich_min"] = metadata["rich_min"] + # cluster_bin_edges["z_max"] = metadata["z_max"] + # cluster_bin_edges["z_min"] = metadata["z_min"] + # # Use (i_bin, j_bin) as key + # key = (result.i, result.j) + + # # Save radial bins + # radial_bins = np.exp(result.object.meanlogr) + # xi = result.object.xi + + # # Save metadata for this bin, if available + # bin_meta = meta.get(key, {}) + + # # Store the data + # binned_cluster_stack[key] = { + # "cluster_rich_bin_edges": cluster_bin_edges, + # "n_cl": redshift_edges, + # "radial_bins": radial_bins, + # "xi": xi, + # "counts": counts + # } + + # # Save the dictionary as a pickle file + # output_file = self.get_output("cluster_profiles") + # with open(output_file, "wb") as f: + # pickle.dump(binned_cluster_stack, f) + + # print(f"TreeCorr results saved in pickle format to {output_file}") + + + def write_output(self, source_list, lens_list, meta, results): + import pickle + import numpy as np + + # Dictionary to store stacked results per lens bin (richness-redshift bin) + binned_cluster_stack = {} + + # --- Precompute string keys for all lens bins --- + lens_keys = [] + with self.open_input("binned_lens_catalog") as f: + for j, bin_name in enumerate(f["cluster_bin"].keys()): + lens_keys.append(bin_name) + + # --- Aggregate results over all source bins --- + for result in results: + j = result.j # lens bin index + key = lens_keys[j] # use string key + radial_bins = np.exp(result.object.meanlogr) + xi = result.object.xi + npairs = result.object.npairs + + # TreeCorr covariance for this source-lens pair + try: + cov = result.object.cov['tan_sc'] + except Exception: + cov = np.zeros((len(xi), len(xi))) + + if key not in binned_cluster_stack: + # Initialize with first source bin + binned_cluster_stack[key] = { + "radial_bins": radial_bins, + "xi_sum": xi * npairs, # weighted sum + "npairs_sum": npairs.copy(), # total pairs + "cov_sum": cov * npairs[:, None] # weighted sum of covariance + } + else: + # Aggregate over source bins + binned_cluster_stack[key]["xi_sum"] += xi * npairs + binned_cluster_stack[key]["npairs_sum"] += npairs + binned_cluster_stack[key]["cov_sum"] += cov * npairs[:, None] + + # --- Compute the weighted average xi and covariance for each lens bin --- + for d in binned_cluster_stack.values(): + # xi + with np.errstate(invalid='ignore', divide='ignore'): + d["xi"] = np.divide( + d["xi_sum"], d["npairs_sum"], + out=np.full_like(d["xi_sum"], np.nan), + where=d["npairs_sum"] != 0 + ) + # covariance (diagonal only) + d["cov"] = np.divide( + d["cov_sum"], d["npairs_sum"][:, None], + out=np.full_like(d["cov_sum"], np.nan), + where=d["npairs_sum"][:, None] != 0 + ) + + del d["xi_sum"], d["npairs_sum"], d["cov_sum"] + + # --- Add cluster metadata from the lens catalog --- + with self.open_input("binned_lens_catalog") as f: + for j, bin_name in enumerate(f["cluster_bin"].keys()): + attrs = f[f"/cluster_bin/{bin_name}"].attrs + counts = len(list(f[f"/cluster_bin/{bin_name}"]["cluster_id"])) + key = lens_keys[j] + if key in binned_cluster_stack: + binned_cluster_stack[key]["cluster_bin_edges"] = { + "rich_min": attrs["rich_min"], + "rich_max": attrs["rich_max"], + "z_min": attrs["z_min"], + "z_max": attrs["z_max"] + } + binned_cluster_stack[key]["n_cl"] = counts + + # --- Save the stacked results as a pickle file --- + output_file = self.get_output("cluster_profiles") + with open(output_file, "wb") as f: + pickle.dump(binned_cluster_stack, f) + + print(f"TreeCorr results saved in pickle format to {output_file}") + + + def _read_nbin_from_tomography(self): + if self.get_input("binned_shear_catalog") == "none": + nbin_source = 0 + else: + with self.open_input("binned_shear_catalog") as f: + nbin_source = f["shear"].attrs["nbin_source"] + + if self.get_input("binned_lens_catalog") == "none": + nbin_lens = 0 + else: + with self.open_input("binned_lens_catalog") as f: + nbin_lens = len(f["cluster_bin"]) + + source_list = list(range(nbin_source)) + lens_list = list(range(nbin_lens)) + + return source_list, lens_list + + def select_calculations(self, source_list, lens_list): + calcs = [] + + if self.config["do_shear_pos"]: + print("DOING SHEAR-POS") + k = SHEAR_POS + + # --- Load source bin edges from shear_tomography_catalog --- + with self.open_input("shear_tomography_catalog") as f: + tomo = f["tomography"] + nbin_source = tomo.attrs["nbin"] + source_zmin = [tomo.attrs[f"zmin_{i}"] for i in range(nbin_source)] + source_zmax = [tomo.attrs[f"zmax_{i}"] for i in range(nbin_source)] + + # --- Loop over source and lens bins --- + for i in source_list: + for j in lens_list: + # --- Load lens bin z-range --- + with self.open_input("binned_lens_catalog") as f: + bin_name = list(f["cluster_bin"])[j] + attrs_bin = f[f"cluster_bin/{bin_name}"].attrs + lens_zmin, lens_zmax = attrs_bin["z_min"], attrs_bin["z_max"] + + # --- Check for redshift overlap --- + overlap = not ( + lens_zmax < source_zmin[i] or lens_zmin > source_zmax[i] + ) + + if overlap: + print( + f"Skipping lens bin {j} (z=[{lens_zmin:.3f}, {lens_zmax:.3f}]) " + f"and source bin {i} (z=[{source_zmin[i]:.3f}, {source_zmax[i]:.3f}]) — overlap detected." + ) + continue + calcs.append((i, j, k)) + + if self.rank == 0: + print(f"Running {len(calcs)} calculations: {calcs}") + + return calcs