diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..6784996 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,2 @@ +*.ipynb filter=strip-notebook-output + diff --git a/cosipy/create_dataset/README.md b/cosipy/create_dataset/README.md new file mode 100644 index 0000000..d538c5e --- /dev/null +++ b/cosipy/create_dataset/README.md @@ -0,0 +1,344 @@ +GRB dataset simulation with simulate_random_dataset.py +====================================================== + +This document explains how to use `simulate_random_dataset.py` to create GRB datasets on an HEALPix grid. You can configure: + +- **Flux**: fixed (single value) or uniformly varying within a "min:max" range. +- **Spectrum**: fixed by choosing one of the three predefined spectra, or random among the three. +- **Number of GRBs per pixel**: one per pixel or multiple per pixel. + +For each grid direction, the tool creates a job folder with `GRB.source` and `job.slurm`, and submits the simulations via Docker/SLURM. + +Requirements and important notes +-------------------------------- + +- Ensure the host directory for data is available: it is mounted into the container as `/data` using `--path-data` (default `/cosi-grb/data`). +- The `--path-analysis` argument must point to a host path, typically `$HOME/cosi-grb/data/analysis/`. +- The `--run-name` argument must be the name of the run subfolder (i.e., ``), so that inside the container it corresponds to `/data/analysis/`. +- `--geometry-path` must be a path valid from inside the container, e.g., `/data/models/.../*.geo.setup`, i.e., under one of the mounted directories (`/data`, `/data01`, `/data02`). + +Predefined spectra +------------------ + +Three spectra indexed 0, 1, 2 are available in the code: + +- **0 (soft)**: `Band 10 10000 -1.9 -3.7 230` +- **1 (medium)**: `Band 10 10000 -1 -2.3 699.9` +- **2 (hard)**: `Comptonized 10 10000 -0.5 1500` + +General usage +------------- + +Generic invocation example (adjust paths): + +```bash +RUN="my_run" +BASE="$HOME/cosi-grb/data/analysis/$RUN" + +python3 simulate_random_dataset.py \ + --path-analysis="$BASE" \ + --run-name="$RUN" \ + --nside=8 \ + --grbs-per-pixel=-1 \ + --seed=-1 \ + --random-flux="1:30" \ + --random-spectrum="yes" \ + --limit-grb-number=-1 \ + --geometry-path="/data/models/massmodel-cosi-smex-detailed/COSISMEX.sim_BGOreading.geo.setup" \ + --path-repository="/cosi-grb/codidl/" \ + --path-data="/cosi-grb/data" \ + --noise="true" \ + --max-job=200 +``` + +Meaning of key options +---------------------- + +- **--nside**: HEALPix resolution (number of pixels = 12 × nside²). +- **--grbs-per-pixel**: + - `-1`: a single GRB per pixel. + - `N > 0`: N GRBs for each pixel (replicas with different seeds). +- **--random-flux**: + - Single value (e.g., `--random-flux=10`) for fixed flux. + - Range `"min:max"` (e.g., `"1:30"`) for a uniform draw within the range. +- **--random-spectrum**: + - `"yes"`: randomly picks one among spectra 0/1/2. + - `"0"`, `"1"`, `"2"`: fixes to soft/medium/hard, respectively. +- **--limit-grb-number**: if >0, randomly samples that many coordinates from the grid (useful to limit the dataset size). +- **--seed**: global seed; `-1` generates a different seed per job. +- **--noise**: string forwarded to post-processing (e.g., `true`/`false`). +- **--max-job**: maximum number of simultaneous jobs in the SLURM queue. + +Recipes for the requested datasets +---------------------------------- + +Below are the instructions to create the three main dataset modes and the dataset for Aitoff plots. + +1) Training dataset (Deep Learning) +----------------------------------- + +- **nside**: 128 +- **GRB limit**: 100000 (use `--limit-grb-number=100000`) +- **Spectra**: random (`--random-spectrum="yes"`) +- **Fluxes**: uniform between 1 and 30 (`--random-flux="1:30"`) +- **GRBs per pixel**: one (`--grbs-per-pixel=-1`) + +Example command: + +```bash +RUN="train_n128" +BASE="$HOME/cosi-grb/data/analysis/$RUN" + +python3 simulate_random_dataset.py \ + --path-analysis="$BASE" \ + --run-name="$RUN" \ + --nside=128 \ + --grbs-per-pixel=-1 \ + --seed=-1 \ + --random-flux="1:30" \ + --random-spectrum="yes" \ + --limit-grb-number=100000 \ + --geometry-path="/data/models/massmodel-cosi-smex-detailed/COSISMEX.sim_BGOreading.geo.setup" \ + --path-repository="/cosi-grb/codidl/" \ + --path-data="/cosi-grb/data" \ + --noise="true" \ + --max-job=400 +``` + +2) Test dataset (Deep Learning) +-------------------------------- + +- **nside**: 64 +- **Spectra**: random (`--random-spectrum="yes"`) +- **Fluxes**: uniform between 1 and 30 (`--random-flux="1:30"`) +- **GRBs per pixel**: one (`--grbs-per-pixel=-1`) + +Example command: + +```bash +RUN="test_n64" +BASE="$HOME/cosi-grb/data/analysis/$RUN" + +python3 simulate_random_dataset.py \ + --path-analysis="$BASE" \ + --run-name="$RUN" \ + --nside=64 \ + --grbs-per-pixel=-1 \ + --seed=-1 \ + --random-flux="1:30" \ + --random-spectrum="yes" \ + --limit-grb-number=-1 \ + --geometry-path="/data/models/massmodel-cosi-smex-detailed/COSISMEX.sim_BGOreading.geo.setup" \ + --path-repository="/cosi-grb/codidl/" \ + --path-data="/cosi-grb/data" \ + --noise="true" \ + --max-job=300 +``` + +3) Look-up tables for chi² fit (nside=32, fixed flux=100) +--------------------------------------------------------- + +Create three distinct datasets, one for each spectral model: soft (0), medium (1), hard (2). In all three cases the flux is fixed at 100. + +Example commands: + +```bash +RUN="lut_soft_n32" +BASE="$HOME/cosi-grb/data/analysis/$RUN" +python3 simulate_random_dataset.py \ + --path-analysis="$BASE" \ + --run-name="$RUN" \ + --nside=32 \ + --grbs-per-pixel=-1 \ + --seed=-1 \ + --random-flux=100 \ + --random-spectrum="0" \ + --limit-grb-number=-1 \ + --geometry-path="/data/models/massmodel-cosi-smex-detailed/COSISMEX.sim_BGOreading.geo.setup" \ + --path-repository="/cosi-grb/codidl/" \ + --path-data="/cosi-grb/data" \ + --noise="true" \ + --max-job=200 + +RUN="lut_medium_n32" +BASE="$HOME/cosi-grb/data/analysis/$RUN" +python3 simulate_random_dataset.py \ + --path-analysis="$BASE" \ + --run-name="$RUN" \ + --nside=32 \ + --grbs-per-pixel=-1 \ + --seed=-1 \ + --random-flux=100 \ + --random-spectrum="1" \ + --limit-grb-number=-1 \ + --geometry-path="/data/models/massmodel-cosi-smex-detailed/COSISMEX.sim_BGOreading.geo.setup" \ + --path-repository="/cosi-grb/codidl/" \ + --path-data="/cosi-grb/data" \ + --noise="true" \ + --max-job=200 + +RUN="lut_hard_n32" +BASE="$HOME/cosi-grb/data/analysis/$RUN" +python3 simulate_random_dataset.py \ + --path-analysis="$BASE" \ + --run-name="$RUN" \ + --nside=32 \ + --grbs-per-pixel=-1 \ + --seed=-1 \ + --random-flux=100 \ + --random-spectrum="2" \ + --limit-grb-number=-1 \ + --geometry-path="/data/models/massmodel-cosi-smex-detailed/COSISMEX.sim_BGOreading.geo.setup" \ + --path-repository="/cosi-grb/codidl/" \ + --path-data="/cosi-grb/data" \ + --noise="true" \ + --max-job=200 +``` + +4) Dataset for Aitoff plots (nside=32, flux=10, medium spectrum, 10 GRB/pixel) +------------------------------------------------------------------------------- + +- **nside**: 32 +- **flux**: fixed at 10 (`--random-flux=10`) +- **spectrum**: medium (`--random-spectrum="1"`) +- **GRBs per pixel**: 10 (`--grbs-per-pixel=10`) + +Example command: + +```bash +RUN="aitoff_n32_flux10_medium_10x" +BASE="$HOME/cosi-grb/data/analysis/$RUN" + +python3 simulate_random_dataset.py \ + --path-analysis="$BASE" \ + --run-name="$RUN" \ + --nside=32 \ + --grbs-per-pixel=10 \ + --seed=-1 \ + --random-flux=10 \ + --random-spectrum="1" \ + --limit-grb-number=-1 \ + --geometry-path="/data/models/massmodel-cosi-smex-detailed/COSISMEX.sim_BGOreading.geo.setup" \ + --path-repository="/cosi-grb/codidl/" \ + --path-data="/cosi-grb/data" \ + --noise="true" \ + --max-job=300 +``` + +Tips +---- + +- Always quote flux ranges (e.g., `"1:30"`) to prevent the shell from interpreting the colon. +- To speed up submission, tune `--max-job` according to your cluster resources. +- For very large datasets, consider using `--limit-grb-number` to generate a controlled subset of directions. + +Create datasets with create_dataset.py +====================================== + +After simulations complete, use `create_dataset.py` to build pickle datasets from the generated `.evt.lc` files. The script supports three modes: + +- **files_flat**: iterate over a file glob and produce flat arrays. +- **healpix_flat**: scan simulation folders by HEALPix pixel and produce flat arrays. +- **healpix_matrix**: group by HEALPix pixel and build a fixed-size matrix per pixel. + +The script writes two pickles: `` and `_shared`, separating standard vs "shared" ASIC data. + +General CLI +----------- + +```bash +python3 create_dataset.py \ + --mode \ + --output "/abs/path/to/output.pkl" \ + [--suffix ".evt.lc"] \ + # files_flat only: + [--glob "/abs/path/**/*.evt.lc"] \ + # healpix_* only: + [--root-dir "/abs/path/to/run"] [--nside ] \ + # healpix_matrix only: + [--entries-per-pixel 10] +``` + +Recommended modes per dataset +----------------------------- + +- **Training dataset**: use flat mode (`files_flat`). +- **Test dataset**: use HEALPix mode (`healpix_flat`). +- **Look-up tables (chi² fit)**: use HEALPix mode (`healpix_flat`). +- **Aitoff plots dataset**: use HEALPix matrix mode (`healpix_matrix`). + +1) Training dataset (flat mode) +-------------------------------- + +Assuming simulations are under `$HOME/cosi-grb/data/analysis/train_n128`: + +```bash +RUN="train_n128" +ROOT="$HOME/cosi-grb/data/analysis/$RUN" +OUT="$HOME/cosi-grb/data/datasets/${RUN}.pkl" + +python3 create_dataset.py \ + --mode files_flat \ + --glob "$ROOT/**/*.evt.lc" \ + --output "$OUT" +``` + +2) Test dataset (HEALPix flat) +------------------------------ + +Assuming simulations are under `$HOME/cosi-grb/data/analysis/test_n64` with `nside=64`: + +```bash +RUN="test_n64" +ROOT="$HOME/cosi-grb/data/analysis/$RUN" +OUT="$HOME/cosi-grb/data/datasets/${RUN}.pkl" + +python3 create_dataset.py \ + --mode healpix_flat \ + --root-dir "$ROOT" \ + --nside 64 \ + --output "$OUT" +``` + +3) Look-up tables (HEALPix flat) +--------------------------------- + +For each spectral dataset (soft/medium/hard) at `nside=32`: + +```bash +RUN="lut_soft_n32"; ROOT="$HOME/cosi-grb/data/analysis/$RUN"; OUT="$HOME/cosi-grb/data/datasets/${RUN}.pkl" +python3 create_dataset.py --mode healpix_flat --root-dir "$ROOT" --nside 32 --output "$OUT" + +RUN="lut_medium_n32"; ROOT="$HOME/cosi-grb/data/analysis/$RUN"; OUT="$HOME/cosi-grb/data/datasets/${RUN}.pkl" +python3 create_dataset.py --mode healpix_flat --root-dir "$ROOT" --nside 32 --output "$OUT" + +RUN="lut_hard_n32"; ROOT="$HOME/cosi-grb/data/analysis/$RUN"; OUT="$HOME/cosi-grb/data/datasets/${RUN}.pkl" +python3 create_dataset.py --mode healpix_flat --root-dir "$ROOT" --nside 32 --output "$OUT" +``` + +4) Aitoff plots dataset (HEALPix matrix) +---------------------------------------- + +Assuming simulations are under `$HOME/cosi-grb/data/analysis/aitoff_n32_flux10_medium_10x` with `nside=32` and 10 GRBs per pixel: + +```bash +RUN="aitoff_n32_flux10_medium_10x" +ROOT="$HOME/cosi-grb/data/analysis/$RUN" +OUT="$HOME/cosi-grb/data/datasets/${RUN}.pkl" + +python3 create_dataset.py \ + --mode healpix_matrix \ + --root-dir "$ROOT" \ + --nside 32 \ + --entries-per-pixel 10 \ + --output "$OUT" +``` + +Notes +----- + +- The default `--suffix` is `.evt.lc`, matching the light-curve count files produced by `read_sim_files.py`. +- `create_dataset.py` emits two files: `` and `_shared`. +- Ensure you pass absolute paths (or `$HOME/...`) to avoid path resolution issues. + + diff --git a/cosipy/create_dataset/create_dataset.py b/cosipy/create_dataset/create_dataset.py new file mode 100644 index 0000000..9909270 --- /dev/null +++ b/cosipy/create_dataset/create_dataset.py @@ -0,0 +1,343 @@ +""" +Dataset creator for COSI simulated GRB data +=========================================== + +This script builds pickle datasets from simulated GRB outputs, supporting three +processing modes selected via --mode: +- files_flat: iterate over a glob of input files and produce flat arrays +- healpix_flat: scan simulation folders organized by HEALPix pixel and produce flat arrays +- healpix_matrix: group entries by HEALPix pixel and produce a fixed-size matrix per pixel + +Each dataset entry contains: +- spectrum: value parsed from GRB.source (.Spectrum line) +- flux: float parsed from GRB.source (.Flux line) +- coord: [theta, phi] parsed from the filename +- counts: six BGO counts in a fixed detector order + +Inputs (flags): +- --mode: one of files_flat, healpix_flat, healpix_matrix +- --output: base path for the output pickle (also writes _shared) +- --suffix: file suffix to match (default: .evt.lc) + +Mode-specific flags: +- files_flat: + - --glob: glob pattern to enumerate files (e.g., "/root/**/*.evt.lc") + +- healpix_flat: + - --root-dir: directory containing simulation subfolders named as theta_phi_seed + - --nside: HEALPix nside value (integer) + +- healpix_matrix: + - --root-dir: directory containing simulation subfolders named as theta_phi_seed + - --nside: HEALPix nside value (integer) + - --entries-per-pixel: maximum number of entries per pixel (default: 10) + +Notes: +- "shared" files are detected if the basename (without suffix) ends with "_shared" or contains a token "shared". +- Hidden files (starting with a dot) are ignored. +""" + +import os +import sys +import glob +import pickle +import argparse +from typing import Dict, List, Tuple + +import numpy as np +import healpy as hp + + +DETECTOR_NAMES: List[str] = [ + "bgo_z1[keV]", "bgo_z0[keV]", + "bgo_x1[keV]", "bgo_x0[keV]", + "bgo_y1[keV]", "bgo_y0[keV]", +] + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Dataset creation for COSI GRB simulations") + + parser.add_argument("--mode", choices=["files_flat", "healpix_flat", "healpix_matrix"], required=True, + help="Selection of behavior to use") + parser.add_argument("--output", required=True, help="Base path for output pickle") + parser.add_argument("--suffix", default=".evt.lc", help="File suffix to filter, e.g., .evt.lc") + + # files_flat + parser.add_argument("--glob", dest="glob_pattern", help="Glob pattern of files (for files_flat mode)") + + # healpix_* modes + parser.add_argument("--root-dir", dest="root_dir", help="Root directory with theta_phi_seed subfolders") + parser.add_argument("--nside", type=int, help="HEALPix nside value (required for healpix modes)") + + # healpix_matrix extras + parser.add_argument("--entries-per-pixel", type=int, default=10, + help="Maximum number of entries per pixel (healpix_matrix)") + + args = parser.parse_args() + + if args.mode == "files_flat": + if not args.glob_pattern: + parser.error("--glob is required when --mode=files_flat") + elif args.mode in ("healpix_flat", "healpix_matrix"): + if not args.root_dir or args.nside is None: + parser.error("--root-dir and --nside are required for healpix modes") + + return args + + +def is_hidden_file(file_path: str) -> bool: + return os.path.basename(file_path).startswith('.') + + +def remove_suffix_from_name(filename: str, suffix: str) -> str: + if filename.endswith(suffix): + return filename[: -len(suffix)] + return filename + + +def detect_is_shared(basename_no_suffix: str) -> bool: + if basename_no_suffix.endswith("_shared"): + return True + tokens = basename_no_suffix.split("_") + return "shared" in tokens + + +def extract_theta_phi_from_filename(file_path: str, suffix: str) -> Tuple[str, str]: + base = os.path.basename(file_path) + base_no = remove_suffix_from_name(base, suffix) + tokens = base_no.split("_") + if len(tokens) < 2: + return "", "" + return tokens[0], tokens[1] + + +def parse_source_file(folder_path: str) -> Tuple[str, float]: + source_path = os.path.join(folder_path, "GRB.source") + spectrum_value: str = None + flux_value: float = None + + try: + with open(source_path, "r") as f: + for line in f: + line_stripped = line.strip() + if ".Spectrum" in line_stripped: + try: + _, value = line_stripped.split(maxsplit=1) + spectrum_value = value + except Exception: + pass + if ".Flux" in line_stripped: + try: + _, value = line_stripped.split(maxsplit=1) + flux_value = float(value) + except Exception: + pass + except FileNotFoundError: + # Keep None defaults; caller can handle + pass + + return spectrum_value, (flux_value if flux_value is not None else 0.0) + + +def read_counts_from_file(file_path: str) -> List[int]: + bgo_counts: List[int] = [0] * len(DETECTOR_NAMES) + with open(file_path, "r") as f: + for row in f: + components = row.split() + if len(components) >= 2: + name = components[0] + try: + value = int(components[1]) + except ValueError: + continue + if name in DETECTOR_NAMES: + idx = DETECTOR_NAMES.index(name) + bgo_counts[idx] = value + return bgo_counts + + +def process_one_file(file_path: str, suffix: str) -> Tuple[Dict, bool]: + filename = os.path.basename(file_path) + folder = os.path.dirname(file_path) + + base_no_suffix = remove_suffix_from_name(filename, suffix) + is_shared = detect_is_shared(base_no_suffix) + + theta_str, phi_str = extract_theta_phi_from_filename(file_path, suffix) + counts = read_counts_from_file(file_path) + spectrum, flux = parse_source_file(folder) + + return { + "spectrum": spectrum, + "flux": float(flux), + "coord": [theta_str, phi_str], + "counts": counts, + }, is_shared + + +def is_valid_sim_dir(folder: str) -> bool: + if not os.path.isdir(folder): + return False + parts = os.path.basename(folder).split("_") + if len(parts) != 3: + return False + try: + theta = float(parts[0]) + phi = float(parts[1]) + seed = int(parts[2]) + if not (0.0 <= theta <= 180.0): + return False + if not (0.0 <= phi <= 360.0): + return False + _ = seed + return True + except ValueError: + return False + + +def extract_theta_phi_from_folder(folder_name: str) -> Tuple[float, float]: + try: + theta_str, phi_str, _ = os.path.basename(folder_name).split("_") + return float(theta_str), float(phi_str) + except ValueError: + return None, None + + +def run_files_flat(glob_pattern: str, suffix: str) -> Tuple[np.ndarray, np.ndarray]: + dataset_list: List[Dict] = [] + dataset_shared_list: List[Dict] = [] + + for file_path in glob.glob(glob_pattern, recursive=True): + if is_hidden_file(file_path): + continue + entry, is_shared = process_one_file(file_path, suffix) + if is_shared: + dataset_shared_list.append(entry) + else: + dataset_list.append(entry) + + if len(dataset_list) % 500 == 0 and len(dataset_list) > 0: + print(f"Processed {len(dataset_list)} non-shared files...") + + dataset = np.array(dataset_list, dtype=object) + dataset_shared = np.array(dataset_shared_list, dtype=object) + return dataset, dataset_shared + + +def run_healpix_flat(root_dir: str, suffix: str, nside: int) -> Tuple[np.ndarray, np.ndarray]: + all_entries = glob.glob(os.path.join(root_dir, "*")) + all_folders = [f for f in all_entries if is_valid_sim_dir(f)] + + indexed_folders: List[Tuple[int, str]] = [] + for folder in all_folders: + theta_deg, phi_deg = extract_theta_phi_from_folder(folder) + if theta_deg is None or phi_deg is None: + continue + try: + theta = np.radians(theta_deg) + phi = np.radians(phi_deg) + ipix = hp.ang2pix(nside, theta, phi, nest=True) + indexed_folders.append((ipix, folder)) + except Exception as ex: + print(f"HEALPix index error in {folder}: {ex}") + + indexed_folders.sort(key=lambda x: x[0]) + ordered_folders = [f for _, f in indexed_folders] + + dataset_list: List[Dict] = [] + dataset_shared_list: List[Dict] = [] + + for folder in ordered_folders: + for file_path in glob.glob(os.path.join(folder, f"*{suffix}")): + if is_hidden_file(file_path): + continue + entry, is_shared = process_one_file(file_path, suffix) + if is_shared: + dataset_shared_list.append(entry) + else: + dataset_list.append(entry) + if len(dataset_list) % 500 == 0 and len(dataset_list) > 0: + print(f"Processed {len(dataset_list)} non-shared files...") + + dataset = np.array(dataset_list, dtype=object) + dataset_shared = np.array(dataset_shared_list, dtype=object) + return dataset, dataset_shared + + +def run_healpix_matrix(root_dir: str, suffix: str, nside: int, entries_per_pixel: int) -> Tuple[np.ndarray, np.ndarray]: + all_entries = glob.glob(os.path.join(root_dir, "*")) + all_folders = [f for f in all_entries if is_valid_sim_dir(f)] + + pixel_to_folders: Dict[int, List[str]] = {} + for folder in all_folders: + theta_deg, phi_deg = extract_theta_phi_from_folder(folder) + if theta_deg is None or phi_deg is None: + continue + try: + theta = np.radians(theta_deg) + phi = np.radians(phi_deg) + ipix = hp.ang2pix(nside, theta, phi, nest=True) + pixel_to_folders.setdefault(ipix, []).append(folder) + except Exception as ex: + print(f"HEALPix index error in {folder}: {ex}") + + ordered_pixels = sorted(pixel_to_folders.keys()) + num_pixels = len(ordered_pixels) + print(f"Found {num_pixels} unique HEALPix pixels") + + dataset = np.empty((num_pixels, entries_per_pixel), dtype=object) + dataset_shared = np.empty((num_pixels, entries_per_pixel), dtype=object) + + for pixel_idx, ipix in enumerate(ordered_pixels): + folders = pixel_to_folders[ipix] + pixel_entries: List[Dict] = [] + pixel_entries_shared: List[Dict] = [] + + for folder in folders: + for file_path in glob.glob(os.path.join(folder, f"*{suffix}")): + if is_hidden_file(file_path): + continue + entry, is_shared = process_one_file(file_path, suffix) + if is_shared: + pixel_entries_shared.append(entry) + else: + pixel_entries.append(entry) + + # Fill per-pixel up to entries_per_pixel + for i in range(entries_per_pixel): + dataset[pixel_idx, i] = pixel_entries[i] if i < len(pixel_entries) else None + dataset_shared[pixel_idx, i] = pixel_entries_shared[i] if i < len(pixel_entries_shared) else None + + if pixel_idx % 100 == 0: + print(f"Processed {pixel_idx} pixels...") + + return dataset, dataset_shared + + +def main() -> None: + args = parse_args() + + if args.mode == "files_flat": + dataset, dataset_shared = run_files_flat(args.glob_pattern, args.suffix) + elif args.mode == "healpix_flat": + dataset, dataset_shared = run_healpix_flat(args.root_dir, args.suffix, args.nside) + elif args.mode == "healpix_matrix": + dataset, dataset_shared = run_healpix_matrix(args.root_dir, args.suffix, args.nside, args.entries_per_pixel) + else: + raise ValueError(f"Unsupported mode: {args.mode}") + + with open(args.output, "wb") as f: + pickle.dump(dataset, f) + with open(args.output + "_shared", "wb") as f: + pickle.dump(dataset_shared, f) + + # Minimal completion logs + try: + print(f"Done. Dataset shapes: dataset={getattr(dataset, 'shape', None)}, shared={getattr(dataset_shared, 'shape', None)}") + except Exception: + pass + + +if __name__ == "__main__": + main() diff --git a/cosipy/create_dataset/read_sim_files.py b/cosipy/create_dataset/read_sim_files.py new file mode 100644 index 0000000..7be218e --- /dev/null +++ b/cosipy/create_dataset/read_sim_files.py @@ -0,0 +1,541 @@ +import ROOT as M +import numpy as np +import pandas as pd +import sys + +# Load MEGAlib into ROOT +M.gSystem.Load("$(MEGALIB)/lib/libMEGAlib.so") + +# Initialize MEGAlib +G = M.MGlobal() +G.Initialize() + +GeometryName = sys.argv[1] +Filename = sys.argv[2] +output_file = sys.argv[3] +shared = sys.argv[4] +noise = sys.argv[5] + +print("Ligth curve loaded !") + +# Load geometry: +Geometry = M.MDGeometryQuest() +if Geometry.ScanSetupFile(M.MString(GeometryName)) == True: + print("Geometry " + GeometryName + " loaded!") +else: + print("Unable to load geometry " + GeometryName + " - Aborting!") + quit() + +if noise=="0": + Geometry.ActivateNoising(False) + +Reader = M.MFileEventsSim(Geometry) + +if Reader.Open(M.MString(Filename)) == False: + print("Unable to open file " + Filename + ". Aborting!") + quit() + +#per hit +BGO_Z0_0 = 0. +BGO_Z0_1 = 0. +BGO_Z0_2 = 0. +BGO_Z0_3 = 0. +BGO_Z0_4 = 0. +BGO_Z1_4 = 0. +BGO_Z1_3 = 0. +BGO_Z1_2 = 0. +BGO_Z1_1 = 0. +BGO_Z1_0 = 0. + +BGO_X1_0 = 0. +BGO_X1_1 = 0. +BGO_X1_2 = 0. + +BGO_X0_0 = 0. +BGO_X0_1 = 0. +BGO_X0_2 = 0. + +BGO_Y1_0 = 0. +BGO_Y1_1 = 0. +BGO_Y1_2 = 0. + +BGO_Y0_0 = 0. +BGO_Y0_1 = 0. +BGO_Y0_2 = 0. + +time = 0. +latX = 0. +lonX = 0. +latZ = 0. +lonZ = 0. + +#per event +Time = [] +Energy_Z1 = [] +Energy_Z0 = [] +Energy_X1 = [] +Energy_X0 = [] +Energy_Y1 = [] +Energy_Y0 = [] +LatX = [] +LonX = [] +LatZ = [] +LonZ = [] + +FirstHit = True + +while True: + Event = Reader.GetNextEvent() + + if not Event: + break + M.SetOwnership(Event, True) + + if Event.GetNIAs() > 0: + + for i in range(Event.GetNHTs()): + Hit = Event.GetHTAt(i) + if Hit.GetDetectorType() == 8: + + pos = Hit.GetPosition() + + x=pos.X() + y=pos.Y() + z=pos.Z() + + detector = Geometry.GetDetector(pos).GetName() + + + #bottom + if detector.GetString() == "BGO_Z0_0": + BGO_Z0_0+=(Hit.GetEnergy()) + elif detector.GetString() == "BGO_Z0_1": + BGO_Z0_1+=(Hit.GetEnergy()) + elif detector.GetString() == "BGO_Z0_2": + BGO_Z0_2+=(Hit.GetEnergy()) + elif detector.GetString() == "BGO_Z0_3": + BGO_Z0_3+=(Hit.GetEnergy()) + elif detector.GetString() == "BGO_Z0_4": + BGO_Z0_4+=(Hit.GetEnergy()) + elif detector.GetString() == "BGO_Z1_4": + BGO_Z1_4+=(Hit.GetEnergy()) + elif detector.GetString() == "BGO_Z1_3": + BGO_Z1_3+=(Hit.GetEnergy()) + elif detector.GetString() == "BGO_Z1_2": + BGO_Z1_2+=(Hit.GetEnergy()) + elif detector.GetString() == "BGO_Z1_1": + BGO_Z1_1+=(Hit.GetEnergy()) + elif detector.GetString() == "BGO_Z1_0": + BGO_Z1_0+=(Hit.GetEnergy()) + + + #Y pannel + elif detector.GetString() == "BGO_Y1_0": + BGO_Y1_0+=(Hit.GetEnergy()) + elif detector.GetString() == "BGO_Y1_1": + BGO_Y1_1+=(Hit.GetEnergy()) + elif detector.GetString() == "BGO_Y1_2": + BGO_Y1_2+=(Hit.GetEnergy()) + + #Y neg pannel + elif detector.GetString() == "BGO_Y0_0": + BGO_Y0_0+=(Hit.GetEnergy()) + elif detector.GetString() == "BGO_Y0_1": + BGO_Y0_1+=(Hit.GetEnergy()) + elif detector.GetString() == "BGO_Y0_2": + BGO_Y0_2+=(Hit.GetEnergy()) + + #X pannel + elif detector.GetString() == "BGO_X1_0": + BGO_X1_0+=(Hit.GetEnergy()) + elif detector.GetString() == "BGO_X1_1": + BGO_X1_1+=(Hit.GetEnergy()) + elif detector.GetString() == "BGO_X1_2": + BGO_X1_2+=(Hit.GetEnergy()) + + #X neg pannel + elif detector.GetString() == "BGO_X0_0": + BGO_X0_0+=(Hit.GetEnergy()) + elif detector.GetString() == "BGO_X0_1": + BGO_X0_1+=(Hit.GetEnergy()) + elif detector.GetString() == "BGO_X0_2": + BGO_X0_2+=(Hit.GetEnergy()) + else: + print(detector) + print(str(x)+" "+str(y)+" "+str(z)) + print("coordinate not found") + sys.exit() + + if FirstHit: + #time + time = ( Event.GetTime().GetAsSeconds()) + + # x axis of space craft pointing at GAL latitude + latX=(np.float32(Event.GetGalacticPointingXAxisLatitude())) + + # x axis of space craft pointing at GAL longitude + lonX=(np.float32(Event.GetGalacticPointingXAxisLongitude())) + + # z axis of space craft pointing at GAL latitude + latZ=(np.float32(Event.GetGalacticPointingZAxisLatitude())) + + # z axis of space craft pointing at GAL longitude + lonZ=(np.float32(Event.GetGalacticPointingZAxisLongitude())) + + FirstHit = False + + + #bot neg + if BGO_Z0_0 >= 80. and BGO_Z0_0 <= 2000.: + Energy_Z1.append(np.float32(0)) + Energy_Z0.append(np.float32(BGO_Z0_0)) + Energy_X1.append(np.float32(0)) + Energy_X0.append(np.float32(0)) + Energy_Y1.append(np.float32(0)) + Energy_Y0.append(np.float32(0)) + Time.append(time) + LatX.append(latX) + LonX.append(lonX) + LatZ.append(latZ) + LonZ.append(lonZ) + if BGO_Z0_1 >= 80. and BGO_Z0_1 <= 2000.: + Energy_Z1.append(np.float32(0)) + Energy_Z0.append(np.float32(BGO_Z0_1)) + Energy_X1.append(np.float32(0)) + Energy_X0.append(np.float32(0)) + Energy_Y1.append(np.float32(0)) + Energy_Y0.append(np.float32(0)) + Time.append(time) + LatX.append(latX) + LonX.append(lonX) + LatZ.append(latZ) + LonZ.append(lonZ) + if BGO_Z0_2 >= 80. and BGO_Z0_2 <= 2000.: + Energy_Z1.append(np.float32(0)) + Energy_Z0.append(np.float32(BGO_Z0_2)) + Energy_X1.append(np.float32(0)) + Energy_X0.append(np.float32(0)) + Energy_Y1.append(np.float32(0)) + Energy_Y0.append(np.float32(0)) + Time.append(time) + LatX.append(latX) + LonX.append(lonX) + LatZ.append(latZ) + LonZ.append(lonZ) + if BGO_Z0_3 >= 80. and BGO_Z0_3 <= 2000.: + Energy_Z1.append(np.float32(0)) + Energy_Z0.append(np.float32(BGO_Z0_3)) + Energy_X1.append(np.float32(0)) + Energy_X0.append(np.float32(0)) + Energy_Y1.append(np.float32(0)) + Energy_Y0.append(np.float32(0)) + Time.append(time) + LatX.append(latX) + LonX.append(lonX) + LatZ.append(latZ) + LonZ.append(lonZ) + + + if BGO_Z0_4 >= 80. and BGO_Z0_4 <= 2000.: + Energy_Z1.append(np.float32(0)) + + if shared == "true": + Energy_Y0.append(np.float32(BGO_Z0_4)) ## ASIC shared + Energy_Z0.append(np.float32(0)) + else: + Energy_Z0.append(np.float32(BGO_Z0_4)) + Energy_Y0.append(np.float32(0)) + + Energy_X1.append(np.float32(0)) + Energy_X0.append(np.float32(0)) + Energy_Y1.append(np.float32(0)) + Time.append(time) + LatX.append(latX) + LonX.append(lonX) + LatZ.append(latZ) + LonZ.append(lonZ) + + #bot + if BGO_Z1_4 >= 80. and BGO_Z1_4 <= 2000.: + + if shared == "true": + Energy_Y1.append(np.float32(BGO_Z1_4)) ## ASIC shared + Energy_Z1.append(np.float32(0)) + else: + Energy_Z1.append(np.float32(BGO_Z1_4)) + Energy_Y1.append(np.float32(0)) + + Energy_Z0.append(np.float32(0)) + Energy_X1.append(np.float32(0)) + Energy_X0.append(np.float32(0)) + Energy_Y0.append(np.float32(0)) + Time.append(time) + LatX.append(latX) + LonX.append(lonX) + LatZ.append(latZ) + LonZ.append(lonZ) + + if BGO_Z1_3 >= 80. and BGO_Z1_3 <= 2000.: + Energy_Z0.append(np.float32(0)) + Energy_Z1.append(np.float32(BGO_Z1_3)) + Energy_X1.append(np.float32(0)) + Energy_X0.append(np.float32(0)) + Energy_Y1.append(np.float32(0)) + Energy_Y0.append(np.float32(0)) + Time.append(time) + LatX.append(latX) + LonX.append(lonX) + LatZ.append(latZ) + LonZ.append(lonZ) + if BGO_Z1_2 >= 80. and BGO_Z1_2 <= 2000.: + Energy_Z0.append(np.float32(0)) + Energy_Z1.append(np.float32(BGO_Z1_2)) + Energy_X1.append(np.float32(0)) + Energy_X0.append(np.float32(0)) + Energy_Y1.append(np.float32(0)) + Energy_Y0.append(np.float32(0)) + Time.append(time) + LatX.append(latX) + LonX.append(lonX) + LatZ.append(latZ) + LonZ.append(lonZ) + if BGO_Z1_1 >= 80. and BGO_Z1_1 <= 2000.: + Energy_Z0.append(np.float32(0)) + Energy_Z1.append(np.float32(BGO_Z1_1)) + Energy_X1.append(np.float32(0)) + Energy_X0.append(np.float32(0)) + Energy_Y1.append(np.float32(0)) + Energy_Y0.append(np.float32(0)) + Time.append(time) + LatX.append(latX) + LonX.append(lonX) + LatZ.append(latZ) + LonZ.append(lonZ) + if BGO_Z1_0 >= 80. and BGO_Z1_0 <= 2000.: + Energy_Z0.append(np.float32(0)) + Energy_Z1.append(np.float32(BGO_Z1_0)) + Energy_X1.append(np.float32(0)) + Energy_X0.append(np.float32(0)) + Energy_Y1.append(np.float32(0)) + Energy_Y0.append(np.float32(0)) + Time.append(time) + LatX.append(latX) + LonX.append(lonX) + LatZ.append(latZ) + LonZ.append(lonZ) + + if BGO_X1_0 >= 80. and BGO_X1_0 <= 2000.: + Energy_X1.append(np.float32(BGO_X1_0)) + Energy_Z1.append(np.float32(0)) + Energy_Z0.append(np.float32(0)) + Energy_X0.append(np.float32(0)) + Energy_Y1.append(np.float32(0)) + Energy_Y0.append(np.float32(0)) + Time.append(time) + LatX.append(latX) + LonX.append(lonX) + LatZ.append(latZ) + LonZ.append(lonZ) + + if BGO_X1_1 >= 80. and BGO_X1_1 <= 2000.: + Energy_X1.append(np.float32(BGO_X1_1)) + Energy_Z1.append(np.float32(0)) + Energy_Z0.append(np.float32(0)) + Energy_X0.append(np.float32(0)) + Energy_Y1.append(np.float32(0)) + Energy_Y0.append(np.float32(0)) + Time.append(time) + LatX.append(latX) + LonX.append(lonX) + LatZ.append(latZ) + LonZ.append(lonZ) + + if BGO_X1_2 >= 80. and BGO_X1_2 <= 2000.: + Energy_X1.append(np.float32(BGO_X1_2)) + Energy_Z1.append(np.float32(0)) + Energy_Z0.append(np.float32(0)) + Energy_X0.append(np.float32(0)) + Energy_Y1.append(np.float32(0)) + Energy_Y0.append(np.float32(0)) + Time.append(time) + LatX.append(latX) + LonX.append(lonX) + LatZ.append(latZ) + LonZ.append(lonZ) + + if BGO_Y1_0 >= 80. and BGO_Y1_0 <= 2000.: + Energy_Y1.append(np.float32(BGO_Y1_0)) + Energy_Z1.append(np.float32(0)) + Energy_Z0.append(np.float32(0)) + Energy_X0.append(np.float32(0)) + Energy_X1.append(np.float32(0)) + Energy_Y0.append(np.float32(0)) + Time.append(time) + LatX.append(latX) + LonX.append(lonX) + LatZ.append(latZ) + LonZ.append(lonZ) + + if BGO_Y1_1 >= 80. and BGO_Y1_1 <= 2000.: + Energy_Y1.append(np.float32(BGO_Y1_1)) + Energy_Z1.append(np.float32(0)) + Energy_Z0.append(np.float32(0)) + Energy_X0.append(np.float32(0)) + Energy_X1.append(np.float32(0)) + Energy_Y0.append(np.float32(0)) + Time.append(time) + LatX.append(latX) + LonX.append(lonX) + LatZ.append(latZ) + LonZ.append(lonZ) + + if BGO_Y1_2 >= 80. and BGO_Y1_2 <= 2000.: + Energy_Y1.append(np.float32(BGO_Y1_2)) + Energy_Z1.append(np.float32(0)) + Energy_Z0.append(np.float32(0)) + Energy_X0.append(np.float32(0)) + Energy_X1.append(np.float32(0)) + Energy_Y0.append(np.float32(0)) + Time.append(time) + LatX.append(latX) + LonX.append(lonX) + LatZ.append(latZ) + LonZ.append(lonZ) + + if BGO_X0_0 >= 80. and BGO_X0_0 <= 2000.: + Energy_X0.append(np.float32(BGO_X0_0)) + Energy_Z1.append(np.float32(0)) + Energy_Z0.append(np.float32(0)) + Energy_Y1.append(np.float32(0)) + Energy_X1.append(np.float32(0)) + Energy_Y0.append(np.float32(0)) + Time.append(time) + LatX.append(latX) + LonX.append(lonX) + LatZ.append(latZ) + LonZ.append(lonZ) + + if BGO_X0_1 >= 80. and BGO_X0_1 <= 2000.: + Energy_X0.append(np.float32(BGO_X0_1)) + Energy_Z1.append(np.float32(0)) + Energy_Z0.append(np.float32(0)) + Energy_Y1.append(np.float32(0)) + Energy_X1.append(np.float32(0)) + Energy_Y0.append(np.float32(0)) + Time.append(time) + LatX.append(latX) + LonX.append(lonX) + LatZ.append(latZ) + LonZ.append(lonZ) + + if BGO_X0_2 >= 80. and BGO_X0_2 <= 2000.: + Energy_X0.append(np.float32(BGO_X0_2)) + Energy_Z1.append(np.float32(0)) + Energy_Z0.append(np.float32(0)) + Energy_Y1.append(np.float32(0)) + Energy_X1.append(np.float32(0)) + Energy_Y0.append(np.float32(0)) + Time.append(time) + LatX.append(latX) + LonX.append(lonX) + LatZ.append(latZ) + LonZ.append(lonZ) + + if BGO_Y0_0 >= 80. and BGO_Y0_0 <= 2000.: + Energy_Y0.append(np.float32(BGO_Y0_0)) + Energy_Z1.append(np.float32(0)) + Energy_Z0.append(np.float32(0)) + Energy_Y1.append(np.float32(0)) + Energy_X1.append(np.float32(0)) + Energy_X0.append(np.float32(0)) + Time.append(time) + LatX.append(latX) + LonX.append(lonX) + LatZ.append(latZ) + LonZ.append(lonZ) + + if BGO_Y0_1 >= 80. and BGO_Y0_1 <= 2000.: + Energy_Y0.append(np.float32(BGO_Y0_1)) + Energy_Z1.append(np.float32(0)) + Energy_Z0.append(np.float32(0)) + Energy_Y1.append(np.float32(0)) + Energy_X1.append(np.float32(0)) + Energy_X0.append(np.float32(0)) + Time.append(time) + LatX.append(latX) + LonX.append(lonX) + LatZ.append(latZ) + LonZ.append(lonZ) + + if BGO_Y0_2 >= 80. and BGO_Y0_2 <= 2000.: + Energy_Y0.append(np.float32(BGO_Y0_2)) + Energy_Z1.append(np.float32(0)) + Energy_Z0.append(np.float32(0)) + Energy_Y1.append(np.float32(0)) + Energy_X1.append(np.float32(0)) + Energy_X0.append(np.float32(0)) + Time.append(time) + LatX.append(latX) + LonX.append(lonX) + LatZ.append(latZ) + LonZ.append(lonZ) + + + BGO_Z0_0 = 0. + BGO_Z0_1 = 0. + BGO_Z0_2 = 0. + BGO_Z0_3 = 0. + BGO_Z0_4 = 0. + BGO_Z1_4 = 0. + BGO_Z1_3 = 0. + BGO_Z1_2 = 0. + BGO_Z1_1 = 0. + BGO_Z1_0 = 0. + BGO_X1_0 = 0. + BGO_X1_1 = 0. + BGO_X1_2 = 0. + BGO_X0_0 = 0. + BGO_X0_1 = 0. + BGO_X0_2 = 0. + BGO_Y1_0 = 0. + BGO_Y1_1 = 0. + BGO_Y1_2 = 0. + BGO_Y0_0 = 0. + BGO_Y0_1 = 0. + BGO_Y0_2 = 0. + + Event = 0 + FirstHit = True + + +df = pd.DataFrame({"timestamp[s]": Time,"bgo_z1[keV]":Energy_Z1,"bgo_z0[keV]":Energy_Z0 + ,"bgo_x1[keV]":Energy_X1,"bgo_x0[keV]":Energy_X0, + "bgo_y1[keV]":Energy_Y1,"bgo_y0[keV]":Energy_Y0, + "latX":LatX,"lonX":LonX,"latZ":LatZ,"lonZ":LonZ }) + +df['datetime']= pd.to_datetime(df["timestamp[s]"],unit="s") +df.set_index("datetime",inplace=True) + +if shared == "true": + output_file = output_file.replace(".evt","_shared.evt") +df.to_csv(output_file, index=True) + +# create file with counts for each of the 6 panels + +bgo_data_seconds = {} + +for col in df.columns: + if col.startswith("bgo_"): + filtered_data = df[df[col] > 0][[col]] + times_in_seconds = filtered_data.index.map(lambda x: x.timestamp()) + bgo_data_seconds[col] = np.array(times_in_seconds) + + +output_file_lc = open(output_file+".lc","w") + +for col, times_in_seconds in bgo_data_seconds.items(): + + output_file_lc.write(col+" "+str(len(times_in_seconds))+"\n") + +output_file_lc.close() + \ No newline at end of file diff --git a/cosipy/create_dataset/simulate_random_dataset.py b/cosipy/create_dataset/simulate_random_dataset.py new file mode 100644 index 0000000..bb910cd --- /dev/null +++ b/cosipy/create_dataset/simulate_random_dataset.py @@ -0,0 +1,229 @@ +""" +Simulate random GRB dataset for COSI +==================================== + +This script generates a simulated GRB dataset for COSI by creating, for each +direction in an HEALPix grid (defined by nside), a job directory containing a +MEGAlib source file (GRB.source) and a SLURM script (job.slurm), and submits the +simulations via Docker. You can control the number of replicas per pixel, the +random seed, the average photon flux (fixed value or uniform range), and the +photon spectrum (predefined or random choice). + +Inputs (flags): +- --path-analysis: base directory where job folders are created. +- --run-name: run name used to build paths inside Docker. +- --grbs-per-pixel: number of GRBs to simulate per HEALPix pixel; -1 simulates one GRB per pixel (default: -1). +- --seed: random seed; -1 to generate a different seed per job (default: -1). +- --nside: HEALPix resolution (required). +- --random-flux: average photon flux; accepts "min:max" for a uniform draw or a single value. +- --random-spectrum: "yes" to pick a random predefined spectrum, or 0/1/2 to select a specific predefined spectrum (default: "0"). +- --noise: string passed to post-processing (e.g., "true"/"false"). +- --max-job: maximum number of concurrent jobs in the SLURM queue (default: 100). +- --geometry-path: path to the MEGAlib geometry .geo.setup file (container path). + - --limit-grb-number: sample N coordinates (GRBs) from the grid; -1 uses all (default: -1). + - --path-repository: host absolute path to mount at /deeplearning_cosi (default: /cosi-grb/codidl/). + - --path-data: host absolute path to mount at /data (default: /cosi-grb/data). + +Outputs: +- One directory per job: {theta}_{phi}_{seed} inside `--path-analysis`, + containing `GRB.source`, `job.slurm`, and the generated `.sim`/`.evt` files. + +Example: +python3 simulate_random_dataset.py --path-analysis="/data/analysis/run1" --run-name="run1" \\ + --nside=8 --grbs-per-pixel=1 --seed=-1 --random-flux="1:30" --random-spectrum="yes" \\ + --limit-grb-number=100 \\ + --geometry-path="/data/models/massmodel-cosi-smex-detailed/COSISMEX.sim_BGOreading.geo.setup" \ + --path-repository="/cosi-grb/codidl/" \ + --path-data="/cosi-grb/data" \ + --noise="true" --max-job=200 +""" + +import os,random,subprocess +import argparse +import numpy as np +import healpy as hp +import time +import csv +import yaml + +parser = argparse.ArgumentParser(description="Generate and submit GRB simulations on an HEALPix grid") +parser.add_argument('--path-analysis', dest='path_analysis', required=True, help='Base directory where job folders are created') +parser.add_argument('--run-name', dest='run_name', required=True, help='Run name used to build container paths') +parser.add_argument('--grbs-per-pixel', dest='grbs_per_pixel', type=int, default=-1, help='Number of GRBs to simulate per HEALPix pixel; -1 uses one job per pixel') +parser.add_argument('--seed', dest='seed', type=int, default=-1, help='Global random seed; -1 to generate a different one per job') +parser.add_argument('--nside', dest='nside', type=int, required=True, help='HEALPix resolution') +parser.add_argument('--random-flux', dest='random_flux', required=True, help='Average photon flux: single value or range "min:max"') +parser.add_argument('--random-spectrum', dest='random_spectrum', default='0', help='"yes" random predefined, or fixed index 0/1/2') +parser.add_argument('--noise', dest='noise', required=True, help='Noise parameter for post-processing (e.g., true/false)') +parser.add_argument('--max-job', dest='max_job', type=int, default=100, help='Maximum number of simultaneous jobs in SLURM queue') +parser.add_argument('--geometry-path', dest='geometry_path', required=True, help='Path to the MEGAlib geometry .geo.setup file (container path)') +parser.add_argument('--limit-grb-number', dest='limit_grb_number', type=int, default=-1, help='Sample N coordinates from the grid; -1 uses all') +parser.add_argument('--path-repository', dest='path_repository', default='/cosi-grb/codidl/', help='Host absolute path to mount at /deeplearning_cosi') +parser.add_argument('--path-data', dest='path_data', default='/cosi-grb/data', help='Host absolute path to mount at /data') + +args = parser.parse_args() + +path_analysis = args.path_analysis +run_name = args.run_name +grbs_per_pixel = args.grbs_per_pixel +seed = args.seed +nside = args.nside +random_flux = args.random_flux +random_spectrum = args.random_spectrum +noise = args.noise +max_job = args.max_job +geometry_path = args.geometry_path +limit_grb_number = args.limit_grb_number +path_repository = args.path_repository +path_data = args.path_data + +# Create an HEALPix grid of coordinates given a specific nside density +# Return two arrays: [[l1,l2,l3...],[b1,b2,b3...]] + +def ring_list(nside=0): + npix_healpix = hp.nside2npix(nside) + + theta_ring_healpix = np.zeros(npix_healpix) + phi_ring_healpix = np.zeros(npix_healpix) + + for jp in range(npix_healpix): + theta_rad, phi_rad = hp.pix2ang(nside, jp, nest=True) + theta_deg = np.degrees(theta_rad) + phi_deg = np.degrees(phi_rad) + theta_ring_healpix[jp] = theta_deg + phi_ring_healpix[jp] = phi_deg + + return theta_ring_healpix, phi_ring_healpix + + +coords_hp = ring_list(nside) + +coordinates_list = [] +for i in range(0,len(coords_hp[0])): + + if grbs_per_pixel!=-1: + for j in range(0,grbs_per_pixel): + coordinates_list.append((coords_hp[0][i],coords_hp[1][i])) + else: + coordinates_list.append((coords_hp[0][i],coords_hp[1][i])) + +coordinates = np.array(coordinates_list) + + +if(limit_grb_number!=-1): + print("sample from coordinate") + indices = np.random.choice(coordinates.shape[0], size=limit_grb_number, replace=False) + coordinates = coordinates[indices] + +print("Array of Elements:") +print(coordinates.shape) + +num_coordinates = coordinates.shape[0] + +for coord in coordinates: + + theta= round(coord[0],3) + phi = round(coord[1],3) + + print("create job for "+str(theta)+" "+str(phi)) + #create job + exist = True + while(exist==True): + + if(seed==-1): + random_seed = random.randint(1, 100000) + else: + random_seed = seed + + dir_name=path_analysis+"/"+str(theta)+"_"+str(phi)+"_"+str(random_seed) + + if os.path.exists(dir_name) and os.path.isdir(dir_name): + print("exist") + exist=True + else: + exist=False + + dir_name_docker = "/data/analysis/"+run_name+"/"+str(theta)+"_"+str(phi)+"_"+str(random_seed) + + os.mkdir(dir_name) + os.chdir(dir_name) + + if ":" in random_flux: + min_flux = float(random_flux.split(":")[0]) + max_flux = float(random_flux.split(":")[1]) + flux = np.random.uniform(min_flux, max_flux) + else: + flux = random_flux + + spectra_list = ["Band 10 10000 -1.9 -3.7 230","Band 10 10000 -1 -2.3 699.9","Comptonized 10 10000 -0.5 1500"] + + if random_spectrum == "yes": + spectra = random.choice(spectra_list) + else: + spectra = spectra_list[int(random_spectrum)] + + source_file = """ + # Global Parameters + Version 1 + Geometry """+geometry_path+""" + + # Physics list + PhysicsListEM LivermorePol + + # Output formats + StoreSimulationInfo init-only + + # Store shield counts + PreTriggerMode EveryEventWithHits + + # Run and source parameters + Run GRBSim + GRBSim.FileName GRB + GRBSim.Time 1.0 + GRBSim.Source GRB + GRB.ParticleType 1 + GRB.Beam FarFieldPointSource """+str(theta)+""" """+str(phi)+""" + + # Spectrum + GRB.Spectrum """+spectra+""" + + # Average photon flux in photon/cm2/s + GRB.Flux """+str(flux)+""" + """ + + file_source = open("GRB.source","w") + file_source.write(source_file) + file_source.close() + + job_content = """#!/bin/bash + #SBATCH --partition=large + #SBATCH --job-name=sim_cosi_grb + + docker run --user "$(id -u)" --entrypoint "" --rm -v """+path_data+""":/data -v /data01:/data01 -v /data02:/data02 -v """+path_repository+""":/deeplearning_cosi cosi_mega_bctools:v1.0.4_parmiggiani /bin/bash -i -c 'cd """+dir_name_docker+""" && cosima -u -s """+str(random_seed)+""" GRB.source' + + docker run --user "$(id -u)" --entrypoint "" --rm -v """+path_data+""":/data -v /data01:/data01 -v /data02:/data02 -v """+path_repository+""":/deeplearning_cosi cosi_mega_bctools:v1.0.4_parmiggiani /bin/bash -i -c 'cd """+dir_name_docker+""" && python3 /deeplearning_cosi/create_dataset/read_sim_files.py """+geometry_path+""" """+dir_name_docker+"""/*sim """+dir_name_docker+"""/"""+str(theta)+"""_"""+str(phi)+"""_shared.evt true """+noise+"""' + + docker run --user "$(id -u)" --entrypoint "" --rm -v """+path_data+""":/data -v /data01:/data01 -v /data02:/data02 -v """+path_repository+""":/deeplearning_cosi cosi_mega_bctools:v1.0.4_parmiggiani /bin/bash -i -c 'cd """+dir_name_docker+""" && python3 /deeplearning_cosi/create_dataset/read_sim_files.py """+geometry_path+""" """+dir_name_docker+"""/*sim """+dir_name_docker+"""/"""+str(theta)+"""_"""+str(phi)+""".evt false """+noise+"""'""" + + file_job = open("job.slurm","w") + file_job.write(job_content) + file_job.close() + + + os.system("sbatch job.slurm") #-o /dev/null -e /dev/null + + # Wait until all jobs complete + result = subprocess.run("squeue --format=%j", shell=True, text=True, capture_output=True) + + array = result.stdout.split("\n") + job_number = len(array)-2 + while job_number> max_job: + time.sleep(0.01) + result = subprocess.run("squeue --format=%j", shell=True, text=True, capture_output=True) + array = result.stdout.split("\n") + job_number = len(array)-2 + + + + + diff --git a/cosipy/download_background/README.md b/cosipy/download_background/README.md new file mode 100644 index 0000000..90e78ad --- /dev/null +++ b/cosipy/download_background/README.md @@ -0,0 +1,87 @@ +### Download DC3 background data and extract light curves + +This folder contains utilities to download COSI DC3 BGO background CSV files from Wasabi and extract light-curve count files used by the pipeline. + +Tools: +- `download_background_from_wasabi.py`: downloads the compressed CSVs from Wasabi (uses `cosipy.util.fetch_wasabi_file`). +- `extract_background_data.sh`: converts the CSVs into light-curve count files by calling `extract_lc_from_csv.py` in parallel. + +Prerequisites +------------- +- Run inside the Docker container (recommended), where `/data` is mounted to your host data directory. +- Python environment with `cosipy` available (provided in the container image). + +Where files are stored +---------------------- +- The download script saves files under the container path `/data/background` by default (see `data_dir` inside `download_background_from_wasabi.py`). +- The extraction script reads and writes in a single directory that you can pass as an argument or via `BACKGROUND_DIR`. + +1) Download from Wasabi +----------------------- + +Run the downloader; any missing files will be fetched into the chosen directory. You can pass it as a positional argument, with `--data-dir`, or via `BACKGROUND_DIR` (default: `/data/background`): + +```bash +#enter inside docker container +docker exec --user cosi -it cosi_grb_bgo_container bash + +#activate cosipy env inside docker +source $HOME/deeplearning/bin/activate + +#define the path for the code +export CONTAINER_COSI_CODE="/home/cosi/cosi/cosidl" + +# default destination +cd $CONTAINER_COSI_CODE/download_background +python3 download_background_from_wasabi.py + +# positional directory +python3 download_background_from_wasabi.py /data/background + +# flag form +python3 download_background_from_wasabi.py --data-dir /data/background + +# environment variable +BACKGROUND_DIR=/data/background python3 download_background_from_wasabi.py +``` + +Notes: +- If you prefer a different destination, use the CLI argument or `BACKGROUND_DIR` above; editing the script is no longer necessary. +- The script is idempotent: it skips files that already exist. + +2) Extract light-curve counts +----------------------------- + +Use the shell launcher to convert all downloaded CSVs into LC files. You can provide the base directory explicitly or via `BACKGROUND_DIR`. + +Option A: pass the directory as an argument (matches the downloader default): + +```bash +cd $CONTAINER_COSI_CODE/download_background +sh extract_background_data.sh /data/background +``` + +Option B: use an environment variable and run without arguments: + +```bash +export BACKGROUND_DIR="/data/background" +sh extract_background_data.sh +``` + +What gets generated +------------------- +For each input `*.csv.gz`, the extractor writes: +- a CSV at the same base path without the `.gz` extension, and +- a light-curve count file with suffix `.lc`. + +Example outputs in `/data/background`: +- `AlbedoPhotons_BGOhit_Total.csv` and `AlbedoPhotons_BGOhit_Total.csv.lc` +- `CosmicPhotons_BGOhit_Total.csv` and `CosmicPhotons_BGOhit_Total.csv.lc` +... and similarly for the other components. + +Tips +---- +- If running outside Docker, ensure paths are writable and adjust them consistently in both scripts. +- Keep background data under `/data/...` inside the container so other parts of the pipeline can find them. + + diff --git a/cosipy/download_background/download_background_from_wasabi.py b/cosipy/download_background/download_background_from_wasabi.py new file mode 100644 index 0000000..7dc6953 --- /dev/null +++ b/cosipy/download_background/download_background_from_wasabi.py @@ -0,0 +1,95 @@ +from cosipy.util import fetch_wasabi_file +from pathlib import Path +import os +import argparse + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Download COSI DC3 BGO background CSVs from Wasabi into a local directory" + ) + parser.add_argument( + "data_dir", + nargs="?", + help="Base directory where the files will be stored (positional)", + ) + parser.add_argument( + "--data-dir", + dest="data_dir_flag", + help="Base directory where the files will be stored (optional flag)", + ) + return parser.parse_args() + + +def resolve_data_dir(args: argparse.Namespace) -> Path: + # Precedence: flag > positional > BACKGROUND_DIR env > default + default_dir = "/data/background" + chosen_dir = args.data_dir_flag or args.data_dir or os.environ.get("BACKGROUND_DIR") or default_dir + data_dir = Path(chosen_dir) + data_dir.mkdir(parents=True, exist_ok=True) + return data_dir + + +def main() -> None: + args = parse_args() + data_dir = resolve_data_dir(args) + + albedo_photons = data_dir / "AlbedoPhotons_BGOhit_Total.csv.gz" + if not albedo_photons.exists(): + fetch_wasabi_file( + "COSI-SMEX/DC3/Data/Backgrounds/BGO/AlbedoPhotons_BGOhit_Total.csv.gz", + albedo_photons, + ) + + cosmic_photons = data_dir / "CosmicPhotons_BGOhit_Total.csv.gz" + if not cosmic_photons.exists(): + fetch_wasabi_file( + "COSI-SMEX/DC3/Data/Backgrounds/BGO/CosmicPhotons_BGOhit_Total.csv.gz", + cosmic_photons, + ) + + primary_alphas = data_dir / "PrimaryAlphas_BGOhit_Total.csv.gz" + if not primary_alphas.exists(): + fetch_wasabi_file( + "COSI-SMEX/DC3/Data/Backgrounds/BGO/PrimaryAlphas_BGOhit_Total.csv.gz", + primary_alphas, + ) + + primary_protons = data_dir / "PrimaryProtons_BGOhit_Total.csv.gz" + if not primary_protons.exists(): + fetch_wasabi_file( + "COSI-SMEX/DC3/Data/Backgrounds/BGO/PrimaryProtons_BGOhit_Total.csv.gz", + primary_protons, + ) + + saa_protons_1 = data_dir / "SAAprotons_BGOhit_Total.csv.gz" + if not saa_protons_1.exists(): + fetch_wasabi_file( + "COSI-SMEX/DC3/Data/Backgrounds/BGO/SAAprotons_BGOhit_Total.csv.gz", + saa_protons_1, + ) + + secondary_electrons = data_dir / "SecondaryElectrons_BGOhit_Total.csv.gz" + if not secondary_electrons.exists(): + fetch_wasabi_file( + "COSI-SMEX/DC3/Data/Backgrounds/BGO/SecondaryElectrons_BGOhit_Total.csv.gz", + secondary_electrons, + ) + + secondary_positrons = data_dir / "SecondaryPositrons_BGOhit_Total.csv.gz" + if not secondary_positrons.exists(): + fetch_wasabi_file( + "COSI-SMEX/DC3/Data/Backgrounds/BGO/SecondaryPositrons_BGOhit_Total.csv.gz", + secondary_positrons, + ) + + ori_file = data_dir / "DC3_final_530km_3_month_with_slew_15sbins_GalacticEarth_SAA.ori" + if not ori_file.exists(): + fetch_wasabi_file( + "COSI-SMEX/DC3/Data/Orientation/DC3_final_530km_3_month_with_slew_15sbins_GalacticEarth_SAA.ori", + ori_file, + ) + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/cosipy/download_background/extract_background_data.sh b/cosipy/download_background/extract_background_data.sh new file mode 100644 index 0000000..32b14ad --- /dev/null +++ b/cosipy/download_background/extract_background_data.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env sh + +# Background light-curve extraction launcher +# ----------------------------------------- +# This script launches parallel conversions of several compressed CSV background +# files into light-curve count files using extract_lc_from_csv.py. +# +# Usage: +# ./extract_background_data.sh [BASE_DIR] +# +# Where BASE_DIR is the directory containing the input .csv.gz files and where +# the output files will be written. You can also set the environment variable +# BACKGROUND_DIR; precedence is: CLI argument > $BACKGROUND_DIR > $HOME/cosi-grb/data/background +# +# Example: +# BACKGROUND_DIR="$HOME/cosi-grb/data/background" ./extract_background_data.sh +# ./extract_background_data.sh /data/background + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" + +BASE_DIR="${1:-${BACKGROUND_DIR:-$HOME/cosi-grb/data/background}}" + +ORI_FILE="$BASE_DIR/DC3_final_530km_3_month_with_slew_15sbins_GalacticEarth_SAA.ori" +if [ ! -f "$ORI_FILE" ]; then + echo "ORI file not found: $ORI_FILE" + exit 1 +fi +nohup python "$SCRIPT_DIR/extract_lc_from_csv.py" "$BASE_DIR/AlbedoPhotons_BGOhit_Total.csv.gz" "$BASE_DIR/AlbedoPhotons_BGOhit_Total" 1 1 "$ORI_FILE" & +nohup python "$SCRIPT_DIR/extract_lc_from_csv.py" "$BASE_DIR/CosmicPhotons_BGOhit_Total.csv.gz" "$BASE_DIR/CosmicPhotons_BGOhit_Total" 1 1 "$ORI_FILE" & +nohup python "$SCRIPT_DIR/extract_lc_from_csv.py" "$BASE_DIR/PrimaryAlphas_BGOhit_Total.csv.gz" "$BASE_DIR/PrimaryAlphas_BGOhit_Total" 1 1 "$ORI_FILE" & +nohup python "$SCRIPT_DIR/extract_lc_from_csv.py" "$BASE_DIR/PrimaryProtons_BGOhit_Total.csv.gz" "$BASE_DIR/PrimaryProtons_BGOhit_Total" 1 1 "$ORI_FILE" & +nohup python "$SCRIPT_DIR/extract_lc_from_csv.py" "$BASE_DIR/SAAprotons_BGOhit_Total.csv.gz" "$BASE_DIR/SAAprotons_BGOhit_Total" 1 1 "$ORI_FILE" & +nohup python "$SCRIPT_DIR/extract_lc_from_csv.py" "$BASE_DIR/SecondaryElectrons_BGOhit_Total.csv.gz" "$BASE_DIR/SecondaryElectrons_BGOhit_Total" 1 1 "$ORI_FILE" & +nohup python "$SCRIPT_DIR/extract_lc_from_csv.py" "$BASE_DIR/SecondaryPositrons_BGOhit_Total.csv.gz" "$BASE_DIR/SecondaryPositrons_BGOhit_Total" 1 1 "$ORI_FILE" & diff --git a/cosipy/download_background/extract_lc_from_csv.py b/cosipy/download_background/extract_lc_from_csv.py new file mode 100644 index 0000000..4fd2dd7 --- /dev/null +++ b/cosipy/download_background/extract_lc_from_csv.py @@ -0,0 +1,80 @@ +import pandas as pd +import numpy as np +import os,sys + +""" this is the mapping with the new geometry + bgo_z1 = bottom_1_times + bgo_z0 = bottom_2_times + bgo_x0 = x1_times + bgo_x1 = x2_times + bgo_y1 = y1_times + bgo_y0 = y2_times +""" + +home = os.environ['HOME'] + +file_path = sys.argv[1] +output_path = sys.argv[2] +bin_size = float(sys.argv[3]) +separate_panels = float(sys.argv[4]) +ori_path = sys.argv[5] + +def open_and_read_csv_counts_DC3_newbkg(file): + + col_names = [ + "Type", + "unix_time", + "x1", "x2", "x3", + "x4", "x5", "x6","x7", + "SAA" + ] + + ori_df = pd.read_csv( + ori_path, + sep=r"\s+", + names=col_names, + usecols=["unix_time", "SAA"], + skiprows=1, + skip_blank_lines=True + ) + + ori_df.set_index('unix_time', inplace=True) + + df = pd.read_csv(file, compression='gzip') + + df.set_index('timestamp[s]', inplace=True) + + df_tmp = ori_df.reindex(df.index,method="nearest") + df["SAA"] = df_tmp["SAA"] + df_cut = df[df["SAA"]!=0] + + bgo_times = {} + for col in df_cut.columns: + if col.startswith("bgo_"): + filtered_data = df_cut[df_cut[col] > 0][[col]] + times_in_seconds = filtered_data.index.to_numpy(dtype=float) + bgo_times[col] = np.array(times_in_seconds) + + + extracted_data = {'bottom_1_times':bgo_times['bgo_bottom_1[keV]'], + 'bottom_2_times':bgo_times['bgo_bottom_2[keV]'], + 'x1_times':bgo_times['bgo_x1[keV]'], + 'x2_times':bgo_times['bgo_x2[keV]'], + 'y1_times':bgo_times['bgo_y1[keV]'], + 'y2_times':bgo_times['bgo_y2[keV]']} + + return extracted_data + +data_preprocessed = open_and_read_csv_counts_DC3_newbkg(file_path) + +np.savez(output_path + ,bottom_1_times=data_preprocessed['bottom_1_times'] + ,bottom_2_times=data_preprocessed['bottom_2_times'] + ,x1_times=data_preprocessed['x1_times'] + ,x2_times=data_preprocessed['x2_times'] + ,y1_times=data_preprocessed['y1_times'] + ,y2_times=data_preprocessed['y2_times']) + + + + \ No newline at end of file diff --git a/cosipy/nonimaging/bgo/__init__.py b/cosipy/nonimaging/bgo/__init__.py index e69de29..8b13789 100644 --- a/cosipy/nonimaging/bgo/__init__.py +++ b/cosipy/nonimaging/bgo/__init__.py @@ -0,0 +1 @@ + diff --git a/cosipy/nonimaging/bgo/bc_tools_localization.py b/cosipy/nonimaging/bgo/bc_tools_localization.py new file mode 100644 index 0000000..e865fbd --- /dev/null +++ b/cosipy/nonimaging/bgo/bc_tools_localization.py @@ -0,0 +1,172 @@ +import pickle +import numpy as np +import matplotlib.pyplot as plt +import astropy.units as u +from astropy.coordinates import SkyCoord +from bctools.loc import TSMap, NormLocLike + + +class BGOLocalizerBCT: + """ + Simple class to localize GRBs using three BGO Look-Up Tables (soft/medium/hard). + + Parameters + ---------- + soft_lut_path : str + Path to the pickled LUT for the 'soft' spectrum. + medium_lut_path : str + Path to the pickled LUT for the 'medium' spectrum. + hard_lut_path : str + Path to the pickled LUT for the 'hard' spectrum. + nside : int + HEALPix Nside used to compute TS maps. + + Notes + ----- + The input counts (s_counts and b_counts) must follow the correct order + of BGO detector panels to match the LUT definition: + + ['BGO_X0', 'BGO_X1', 'BGO_Y0', 'BGO_Y1', 'BGO_Z0', 'BGO_Z1'] + """ + + def __init__(self, soft_lut_path, medium_lut_path, hard_lut_path, nside=64): + # Store HEALPix Nside + self.nside = nside + # Load all three LUTs at initialization + self.luts = { + "soft": self._load_pickle(soft_lut_path), + "medium": self._load_pickle(medium_lut_path), + "hard": self._load_pickle(hard_lut_path), + } + + def localize(self, s_counts, b_counts): + """ + Run localization using all LUTs and return the one with the highest TS. + + Parameters + ---------- + s_counts : list or np.ndarray + Source counts in the following order: + ['BGO_X0', 'BGO_X1', 'BGO_Y0', 'BGO_Y1', 'BGO_Z0', 'BGO_Z1'] + b_counts : list or np.ndarray + Background counts in the same order. + + Returns + ------- + dict + Dictionary containing the best localization result. + """ + results = [] + + for label, lut in self.luts.items(): + # Update LUT with data and background + lut.set_background(b_counts) + lut.set_data(s_counts) + + # Compute the TS map for this LUT + ts_map = TSMap(nside=self.nside, coordsys="icrs") + likelihood = NormLocLike(lut) + ts_map.compute(likelihood) + + # Extract localization statistics + ts_value = float(np.max(ts_map)) + best = ts_map.best_loc() + cont_area = ts_map.error_area(cont=0.9).to(u.deg**2) + eq_radius = np.sqrt(cont_area / np.pi).to(u.deg) + + results.append({ + "label": label, + "ts_map": ts_map, + "ts_value": ts_value, + "sqrt_ts": float(np.sqrt(ts_value)), + "ra_deg": best.ra.deg, + "dec_deg": best.dec.deg, + "cont_area_deg2": float(cont_area.value), + "eq_radius_deg": float(eq_radius.value), + }) + + # Return the result with the highest TS value + return max(results, key=lambda r: r["ts_value"]) + + def plot(self, result, true_coord=None, show=True, save_path=None): + """ + Plot the TS map from a localization result. + + Parameters + ---------- + result : dict + Output from `localize()`. Must contain a 'ts_map' (TSMap) and + summary fields like 'label', 'sqrt_ts', 'ra_deg', 'dec_deg', + 'eq_radius_deg'. + true_coord : astropy.coordinates.SkyCoord, optional + True source position to overlay on the map (assumed ICRS RA/Dec). + show : bool + If True, display the plot with plt.show() and return None. + If False, return (img, moll) as given by TSMap.plot(). + save_path : str or None, optional + If provided, save the plot to the given path as a PNG file. + + Returns + ------- + tuple or None + (img, moll) when show=False, otherwise None. + """ + import healpy as hp + import matplotlib.pyplot as plt + import astropy.units as u + from matplotlib.lines import Line2D + + ts_map = result["ts_map"] + img, ax = ts_map.plot() + ax.grid(alpha=0.5) + + if true_coord is not None: + # Actual location of simulated source + ax.scatter( + true_coord.ra.to(u.deg).value, + true_coord.dec.to(u.deg).value, + color="red", + transform=ax.get_transform("world"), + s=2, + label="True source" + ) + + ax.scatter( + ts_map.best_loc().ra.to(u.deg).value, + ts_map.best_loc().dec.to(u.deg).value, + color="blue", + transform=ax.get_transform("world"), + s=2, + label="Best localization" + ) + + # Add legend + ax.legend(loc="upper right", frameon=True) + + # Compose a title with key localization numbers. + title = ( + f"{result['label']} " + f"sqrt(TS)={result['sqrt_ts']:.2f} " + f"RA={result['ra_deg']:.2f} " + f"Dec={result['dec_deg']:.2f} " + f"r90={result['eq_radius_deg']:.2f} deg" + ) + ax.set_title(title) + + # Save to PNG if path provided + if save_path is not None: + plt.savefig(save_path, dpi=300, bbox_inches="tight") + print(f"Plot salvato in: {save_path}") + + if show: + plt.show() + return None + + return img, ax + + + @staticmethod + def _load_pickle(path): + """Load a pickle file.""" + with open(path, "rb") as f: + return pickle.load(f) diff --git a/cosipy/notebooks/.gitattributes b/cosipy/notebooks/.gitattributes new file mode 100644 index 0000000..8425c58 --- /dev/null +++ b/cosipy/notebooks/.gitattributes @@ -0,0 +1 @@ +*.ipynb filter=strip-notebook-output diff --git a/cosipy/notebooks/analyze_fluxes_and_spectra_GBM.ipynb b/cosipy/notebooks/analyze_fluxes_and_spectra_GBM.ipynb new file mode 100644 index 0000000..3a288a7 --- /dev/null +++ b/cosipy/notebooks/analyze_fluxes_and_spectra_GBM.ipynb @@ -0,0 +1,114 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "d8db80cf-f139-41a9-919b-7563671d92ad", + "metadata": {}, + "outputs": [], + "source": [ + "import pandas as pd\n", + "import numpy as np\n", + "file = \"sgrb_fluxes_bis.csv\"\n", + "import matplotlib.pyplot as plt" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bf95a5c0-f573-4a0c-a8eb-d29eba0ab8b9", + "metadata": {}, + "outputs": [], + "source": [ + "\n", + "df = pd.read_csv(file)\n", + "\n", + "\n", + "import ast\n", + "df[\"model parameters\"] = df[\"model parameters\"].apply(ast.literal_eval)\n", + "\n", + "print(df)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "143c9c63-ad8b-44b8-98fa-e08a93feb68c", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "89b2bf38-7004-4949-b596-ebfad0e03f9a", + "metadata": {}, + "outputs": [], + "source": [ + "\n", + "\n", + "df_filtered = df[df[\"t90 (s)\"] < 2]\n", + "\n", + "energy_flux_filtered = df_filtered[\"photon flux (ph/cm2/s for 10-10000 keV)\"]\n", + "\n", + "bins = np.logspace(np.log10(energy_flux_filtered.min()), np.log10(energy_flux_filtered.max()), num=30)\n", + "\n", + "\n", + "plt.hist(energy_flux_filtered, bins=bins, edgecolor='black', alpha=0.7)\n", + "plt.xscale(\"log\")\n", + "\n", + "X = 1 \n", + "Y = 30 \n", + "\n", + "plt.axvline(X, color='red', linestyle='dashed', linewidth=1.5, label=f'{X} ph/cm2/s')\n", + "plt.axvline(Y, color='blue', linestyle='dotted', linewidth=1.5, label=f'{Y} ph/cm2/s')\n", + "\n", + "\n", + "plt.xlabel('Energy Flux (ph/cm²/s)',fontsize=10)\n", + "plt.ylabel('Frequency',fontsize=10)\n", + "plt.title('')\n", + "plt.tick_params(axis='both', labelsize=10) \n", + "plt.legend(fontsize=10)\n", + "\n", + "plt.show()\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "949c21e2-1e9d-45b5-94f6-ed5f1b1965cc", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f716d4a8", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "deeplearning", + "language": "python", + "name": "deeplearning" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.12" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/cosipy/notebooks/background_analysis/analyze_DC3_background.ipynb b/cosipy/notebooks/background_analysis/analyze_DC3_background.ipynb new file mode 100644 index 0000000..70b0218 --- /dev/null +++ b/cosipy/notebooks/background_analysis/analyze_DC3_background.ipynb @@ -0,0 +1,421 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import numpy as np\n", + "import matplotlib.pyplot as plt\n", + "import sys,os\n", + "from pathlib import Path\n", + "\n", + "# Configure here the path to the background data\n", + "data_dir_path = \"/data/background\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def calculate_hist_lc(file,bin_size,g_tmin,g_tmax):\n", + " \n", + " data_loaded = np.load(file)\n", + "\n", + " bgo_z1 = data_loaded['bottom_1_times']\n", + " bgo_z0 = data_loaded['bottom_2_times']\n", + " bgo_x0 = data_loaded['x1_times']\n", + " bgo_x1 = data_loaded['x2_times']\n", + " bgo_y1 = data_loaded['y1_times']\n", + " bgo_y0 = data_loaded['y2_times']\n", + "\n", + " data_array = [bgo_z1,bgo_z0,bgo_x1,bgo_x0,bgo_y1,bgo_y0]\n", + " \n", + " print(\"len data\"+str(len(data_loaded)))\n", + "\n", + " bin_edges = np.arange(g_tmin, g_tmax + bin_size, bin_size)\n", + " \n", + " ligth_curve = np.empty((len(bin_edges-2),6,2))\n", + " \n", + " for j in range(0,len(data_array)):\n", + " \n", + " frequenze, _ = np.histogram(data_array[j], bins=bin_edges)\n", + " \n", + " for i in range(len(frequenze)):\n", + "\n", + " ligth_curve[i][j][0] = bin_edges[i]\n", + " ligth_curve[i][j][1] = frequenze[i]\n", + "\n", + " return ligth_curve\n", + " " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def get_min_max(global_min,global_max,filename):\n", + "\n", + " data_loaded = np.load(filename)\n", + " \n", + " bgo_z1 = data_loaded['bottom_1_times']\n", + " bgo_z0 = data_loaded['bottom_2_times']\n", + " bgo_x0 = data_loaded['x1_times']\n", + " bgo_x1 = data_loaded['x2_times']\n", + " bgo_y1 = data_loaded['y1_times']\n", + " bgo_y0 = data_loaded['y2_times']\n", + " \n", + " print(\"len data\"+str(len(data_loaded)))\n", + " \n", + " all_counts = np.concatenate([bgo_z1,bgo_z0,bgo_x1,bgo_x0,bgo_y1,bgo_y0])\n", + " \n", + " local_min = min(all_counts)\n", + " local_max = max(all_counts)\n", + "\n", + " if local_minglobal_max:\n", + " global_max = local_max\n", + "\n", + " return global_min,global_max\n", + " " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "bin_size = 0.050\n", + "\n", + "data_dir = Path(data_dir_path)\n", + "\n", + "output_path = data_dir/f\"preprocessed_{bin_size}.npz\"\n", + "\n", + "load_from_file = False\n", + "\n", + "if (load_from_file):\n", + " data = np.load(output_path,allow_pickle=True)\n", + "\n", + " albedo_photons = data['albedo_photons']\n", + " cosmic_photons = data['cosmic_photons']\n", + " primary_alphas = data['primary_alphas']\n", + " primary_protons = data['primary_protons']\n", + " saa_protons = data['saa_protons']\n", + " secondary_electrons = data['secondary_electrons']\n", + " secondary_positrons = data['secondary_positrons']\n", + "\n", + "else:\n", + " \n", + " albedo_photons_file = data_dir/\"AlbedoPhotons_BGOhit_Total.npz\"\n", + " cosmic_photons_file = data_dir/\"CosmicPhotons_BGOhit_Total.npz\"\n", + " primary_alphas_file = data_dir/\"PrimaryAlphas_BGOhit_Total.npz\"\n", + " primary_protons_file = data_dir/\"PrimaryProtons_BGOhit_Total.npz\"\n", + " saa_protons_file = data_dir/\"SAAprotons_BGOhit_Total.npz\"\n", + " secondary_electrons_file = data_dir/\"SecondaryElectrons_BGOhit_Total.npz\"\n", + " secondary_positrons_file = data_dir/\"SecondaryPositrons_BGOhit_Total.npz\" \n", + " \n", + " # get global tmin, tmax\n", + " g_tmin = sys.float_info.max\n", + " g_tmax = 0\n", + " g_tmin,g_tmax = get_min_max(g_tmin,g_tmax,albedo_photons_file)\n", + " g_tmin,g_tmax = get_min_max(g_tmin,g_tmax,cosmic_photons_file)\n", + " g_tmin,g_tmax = get_min_max(g_tmin,g_tmax,primary_alphas_file)\n", + " g_tmin,g_tmax = get_min_max(g_tmin,g_tmax,primary_protons_file)\n", + " g_tmin,g_tmax = get_min_max(g_tmin,g_tmax,saa_protons_file)\n", + " g_tmin,g_tmax = get_min_max(g_tmin,g_tmax,secondary_electrons_file)\n", + " g_tmin,g_tmax = get_min_max(g_tmin,g_tmax,secondary_positrons_file)\n", + "\n", + " \n", + " print(\"albedo_photons_file\")\n", + " albedo_photons = calculate_hist_lc(albedo_photons_file,bin_size,g_tmin,g_tmax)\n", + " print(\"cosmic_photons_file\")\n", + " cosmic_photons = calculate_hist_lc(cosmic_photons_file,bin_size,g_tmin,g_tmax)\n", + " print(\"primary_alphas_file\")\n", + " primary_alphas = calculate_hist_lc(primary_alphas_file,bin_size,g_tmin,g_tmax)\n", + " print(\"primary_protons_file\")\n", + " primary_protons = calculate_hist_lc(primary_protons_file,bin_size,g_tmin,g_tmax)\n", + " print(\"saa_protons_file\")\n", + " saa_protons = calculate_hist_lc(saa_protons_file,bin_size,g_tmin,g_tmax)\n", + " print(\"secondary_electrons_file\")\n", + " secondary_electrons = calculate_hist_lc(secondary_electrons_file,bin_size,g_tmin,g_tmax)\n", + " print(\"secondary_positrons_file\")\n", + " secondary_positrons = calculate_hist_lc(secondary_positrons_file,bin_size,g_tmin,g_tmax)\n", + "\n", + " " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "if not load_from_file:\n", + " np.savez(output_path, \n", + " albedo_photons=albedo_photons,\n", + " cosmic_photons=cosmic_photons,\n", + " primary_alphas=primary_alphas,\n", + " primary_protons=primary_protons,\n", + " saa_protons=saa_protons,\n", + " secondary_electrons=secondary_electrons,\n", + " secondary_positrons=secondary_positrons)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "total_counts = np.empty([albedo_photons.shape[0],albedo_photons.shape[1],1]) \n", + "for j in range(0,6):\n", + " \n", + " total = albedo_photons[:,j,1]\n", + " total = total + cosmic_photons[:,j,1]\n", + " total = total + primary_alphas[:,j,1]\n", + " total = total + primary_protons[:,j,1]\n", + " total = total + saa_protons[:,j,1]\n", + " total = total + secondary_electrons[:,j,1]\n", + " total = total + secondary_positrons[:,j,1]\n", + " total_counts[:,j,0] = total\n", + " " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "\n", + "start_orbit = 1078000\n", + "stop_orbit = 1212000\n", + "\n", + "fig, axs = plt.subplots(3, 2, figsize=(15, 15)) # 3 rows, 2 columns\n", + "# Flatten the axs array for easy indexing\n", + "axs = axs.flatten()\n", + "for j in range(0,6):\n", + " \n", + " marker_size=4\n", + "\n", + " axs[j].scatter(albedo_photons[:,j,0][start_orbit:stop_orbit], total_counts[:,j,0][start_orbit:stop_orbit],s=marker_size, alpha=1,color=\"black\",label=\"Total counts\")\n", + " axs[j].scatter(albedo_photons[:,j,0][start_orbit:stop_orbit], albedo_photons[:,j,1][start_orbit:stop_orbit],s=marker_size,alpha=1,label=\"Albedo photons\")\n", + " axs[j].scatter(cosmic_photons[:,j,0][start_orbit:stop_orbit], cosmic_photons[:,j,1][start_orbit:stop_orbit],s=marker_size, alpha=1,label=\"Cosmic photons\")\n", + " axs[j].scatter(primary_alphas[:,j,0][start_orbit:stop_orbit], primary_alphas[:,j,1][start_orbit:stop_orbit],s=marker_size, alpha=1,label=\"Primary alphas\")\n", + " axs[j].scatter(primary_protons[:,j,0][start_orbit:stop_orbit], primary_protons[:,j,1][start_orbit:stop_orbit],s=marker_size, alpha=1,label=\"Primary protons\")\n", + " axs[j].scatter(saa_protons[:,j,0][start_orbit:stop_orbit], saa_protons[:,j,1][start_orbit:stop_orbit],s=marker_size, alpha=1,label=\"SAA protons\")\n", + " axs[j].scatter(secondary_electrons[:,j,0][start_orbit:stop_orbit], secondary_electrons[:,j,1][start_orbit:stop_orbit],s=marker_size, alpha=1,label=\"Secondary electrons\")\n", + " axs[j].scatter(secondary_positrons[:,j,0][start_orbit:stop_orbit], secondary_positrons[:,j,1][start_orbit:stop_orbit],s=marker_size, alpha=1,label=\"Secondary positrons\")\n", + " axs[j].set_xlabel('Time (bins of '+str(bin_size)+' second)')\n", + " axs[j].set_ylabel('Counts')\n", + "\n", + " if j ==0:\n", + " title = \"Bottom Z1\"\n", + " if j ==1:\n", + " title = \"Bottom Z0\"\n", + " if j ==2:\n", + " title = \"X1\"\n", + " if j ==3:\n", + " title = \"X0\"\n", + " if j ==4:\n", + " title = \"Y1\"\n", + " if j ==5:\n", + " title = \"Y0\"\n", + "\n", + " axs[j].set_title(title) \n", + " axs[j].legend(loc=\"upper right\")\n", + "\n", + "plt.xlabel('Asse X', fontsize=14)\n", + "plt.ylabel('Asse Y', fontsize=14)\n", + "plt.tight_layout()\n", + "plt.show()\n", + "\n", + "\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "mean_counts_orbit = np.mean(total_counts[:,0,0][start_orbit:stop_orbit][total_counts[:,0,0][start_orbit:stop_orbit] != 0])\n", + "print(mean_counts_orbit)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# find a window of 500 s with a mean background\n", + "time_series= total_counts[:,2,0][start_orbit:stop_orbit]\n", + "window_size = 10000\n", + "target_mean = mean_counts_orbit\n", + "\n", + "best_start = 0\n", + "min_diff = float('inf')\n", + "\n", + "for i in range(len(time_series) - window_size + 1):\n", + " window = time_series[i:i+window_size]\n", + " mean = np.mean(window)\n", + " diff = abs(mean - target_mean)\n", + " \n", + " if diff < min_diff:\n", + " min_diff = diff\n", + " best_start = i\n", + "\n", + "best_window = time_series[best_start:best_start+window_size]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "print(best_start)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "\n", + "\n", + "vertical_line = start_orbit+best_start\n", + "vertical_line_2 = start_orbit+best_start+10000\n", + "\n", + "fig, axs = plt.subplots(3, 2, figsize=(15, 15)) # 3 rows, 2 columns\n", + "# Flatten the axs array for easy indexing\n", + "axs = axs.flatten()\n", + "for j in range(0,6):\n", + "\n", + " marker_size=4\n", + "\n", + " axs[j].scatter(albedo_photons[:,j,0][start_orbit:stop_orbit], total_counts[:,j,0][start_orbit:stop_orbit],s=marker_size, alpha=1,color=\"black\",label=\"Total counts\")\n", + " axs[j].set_xlabel('Time (bins of '+str(bin_size)+' second)', fontsize=13)\n", + " axs[j].set_ylabel('Counts per 0.05 second', fontsize=13)\n", + " axs[j].tick_params(axis='both', labelsize=13) \n", + " axs[j].xaxis.get_offset_text().set_fontsize(13)\n", + " axs[j].yaxis.get_offset_text().set_fontsize(13)\n", + "\n", + " axs[j].axvline(x=albedo_photons[vertical_line,j,0], color='red', linestyle='--', label='Window Start')\n", + " axs[j].axvline(x=albedo_photons[vertical_line_2,j,0], color='red', linestyle='--', label='Window Stop')\n", + "\n", + " if j ==0:\n", + " title = \"Bottom Z1\"\n", + " if j ==1:\n", + " title = \"Bottom Z0\"\n", + " if j ==2:\n", + " title = \"X1\"\n", + " if j ==3:\n", + " title = \"X0\"\n", + " if j ==4:\n", + " title = \"Y1\"\n", + " if j ==5:\n", + " title = \"Y0\"\n", + "\n", + " axs[j].set_title(title) \n", + " axs[j].legend(fontsize=13)\n", + " \n", + "\n", + "plt.tight_layout()\n", + "plt.show()\n", + "\n", + "\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "for i in range (0,6):\n", + " data = total_counts[:,i,0][vertical_line:vertical_line_2]\n", + " if i ==0:\n", + " print(\"Bottom Z1\") \n", + " if i ==1:\n", + " print(\"Bottom Z0\")\n", + " if i ==2:\n", + " print(\"X1\")\n", + " if i ==3:\n", + " print(\"X0\")\n", + " if i ==4:\n", + " print(\"Y1\")\n", + " if i ==5:\n", + " print(\"Y0\")\n", + " print(np.mean(data))\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.12" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/cosipy/notebooks/bc-tools/README.md b/cosipy/notebooks/bc-tools/README.md new file mode 100644 index 0000000..d720d85 --- /dev/null +++ b/cosipy/notebooks/bc-tools/README.md @@ -0,0 +1,17 @@ +## Running simulations to generate IRF and aggregating IRFs + +Prerequisites: +- Download the COSI mass model from repository X and place it on your system. +- Edit `bgo.config.yaml` to set the correct path to the downloaded mass model. + +1. Change into the directory where you want to generate the simulations. +2. Launch the simulation command, adjusting the number of threads to match your system: + +```bash +nohup bc-rsp [path-to-the-config-file]/bgo.config.yaml fullsky_nside32 --ntriggers 200000 --nthreads 200 & +``` + +After the simulation finishes, use the `aggregate_irf` notebook to aggregate the IRFs following the ACS panel layout. + +After aggregating the IRFs, you can use the `visualizeIRF` notebook to plot the expectation maps of the different BGO panels. + diff --git a/cosipy/notebooks/bc-tools/irf_creation/aggregate_irf.ipynb b/cosipy/notebooks/bc-tools/irf_creation/aggregate_irf.ipynb new file mode 100644 index 0000000..f76bd46 --- /dev/null +++ b/cosipy/notebooks/bc-tools/irf_creation/aggregate_irf.ipynb @@ -0,0 +1,180 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "37871353-0bbb-4e31-a189-b5b9974f8996", + "metadata": {}, + "outputs": [], + "source": [ + "from bctools.sims import DRMList\n", + "import h5py" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "03d024f1-87e1-4c55-9009-c68585bf3d63", + "metadata": {}, + "outputs": [], + "source": [ + "# Open the simulated irf to aggregate the crystals in the 6 final modules\n", + "irf_path = \"/data/models/irf_run10.h5\"\n", + "drm_list = DRMList.open(irf_path)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "065c42e6-14dd-468a-af6d-2f7936303fc3", + "metadata": {}, + "outputs": [], + "source": [ + "drm_list.detectors" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7e382a0f-ca8e-4733-8fb2-682136fa3b88", + "metadata": {}, + "outputs": [], + "source": [ + "\n", + "# Function to recursively print the structure of the HDF5 file\n", + "def print_structure(name, obj):\n", + " print(f\"{name}: {type(obj)}\")\n", + " if isinstance(obj, h5py.Group): # If it's a group, recursively explore it\n", + " for key in obj:\n", + " print_structure(key, obj[key])\n", + "\n", + "# Open the HDF5 file in read-only mode ('r')\n", + "with h5py.File(irf_path, 'r') as h5_file:\n", + " # Start from the root group to print the entire structure\n", + " print_structure('/', h5_file)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4cf3c333-c6b0-4f46-8307-8d3dcb7591d2", + "metadata": {}, + "outputs": [], + "source": [ + "\n", + "# Apri il file originale in modalità lettura\n", + "with h5py.File(irf_path, 'r') as h5_file:\n", + " \n", + " attributes = dict(h5_file.attrs)\n", + "\n", + " \n", + " # Accedi ai dataset esistenti\n", + " BGO_Z0_0 = h5_file['BGO_Z0_0']\n", + " BGO_Z0_1 = h5_file['BGO_Z0_1']\n", + " BGO_Z0_2 = h5_file['BGO_Z0_2']\n", + " BGO_Z0_3 = h5_file['BGO_Z0_3']\n", + " BGO_Z0_4 = h5_file['BGO_Z0_4']\n", + " BGO_Z1_0 = h5_file['BGO_Z1_0']\n", + " BGO_Z1_1 = h5_file['BGO_Z1_1']\n", + " BGO_Z1_2 = h5_file['BGO_Z1_2']\n", + " BGO_Z1_3 = h5_file['BGO_Z1_3']\n", + " BGO_Z1_4 = h5_file['BGO_Z1_4']\n", + "\n", + " # Leggi gli attributi\n", + " attributes_dataset = {}\n", + " for attr_name, attr_value in h5_file['BGO_Z0_0'].attrs.items():\n", + " attributes_dataset[attr_name] = attr_value\n", + "\n", + " BGO_X1_0 = h5_file['BGO_X1_0']\n", + " BGO_X1_1 = h5_file['BGO_X1_1']\n", + " BGO_X1_2 = h5_file['BGO_X1_2']\n", + "\n", + " BGO_X0_0 = h5_file['BGO_X0_0']\n", + " BGO_X0_1 = h5_file['BGO_X0_1']\n", + " BGO_X0_2 = h5_file['BGO_X0_2']\n", + "\n", + " BGO_Y1_0 = h5_file['BGO_Y1_0']\n", + " BGO_Y1_1 = h5_file['BGO_Y1_1']\n", + " BGO_Y1_2 = h5_file['BGO_Y1_2']\n", + "\n", + " BGO_Y0_0 = h5_file['BGO_Y0_0']\n", + " BGO_Y0_1 = h5_file['BGO_Y0_1']\n", + " BGO_Y0_2 = h5_file['BGO_Y0_2']\n", + "\n", + " \n", + " \n", + " # Controlla la forma dei dataset per vedere se sono compatibili\n", + "\n", + " # Somma i due dataset\n", + " BGO_X1 = BGO_X1_0[:] + BGO_X1_1[:] + BGO_X1_2[:]\n", + " BGO_X0 = BGO_X0_0[:] + BGO_X0_1[:] + BGO_X0_2[:]\n", + "\n", + " BGO_Y1 = BGO_Y1_0[:] + BGO_Y1_1[:] + BGO_Y1_2[:]+ BGO_Z1_4[:]\n", + " BGO_Y0 = BGO_Y0_0[:] + BGO_Y0_1[:] + BGO_Y0_2[:] + BGO_Z0_4[:]\n", + "\n", + " BGO_Z1 = BGO_Z1_0[:]+BGO_Z1_1[:]+BGO_Z1_2[:]+BGO_Z1_3[:]\n", + " BGO_Z0 = BGO_Z0_0[:]+BGO_Z0_1[:]+BGO_Z0_2[:]+BGO_Z0_3[:]\n", + " \n", + " # Crea un nuovo file HDF5 per salvare i dati\n", + " with h5py.File(irf_path.replace(\"irf\",\"irf_summed\"), 'w') as new_h5_file:\n", + " # Copia i dataset non sommati nel nuovo file\n", + " #h5_file.copy('BGO_Coinc_sideX', BGO_Coinc_sideX)\n", + "\n", + " for key, value in attributes.items():\n", + " new_h5_file.attrs[key] = value\n", + "\n", + " dataset_BGO_Z1 = new_h5_file.create_dataset('BGO_Z1', data=BGO_Z1)\n", + " # Aggiungi gli attributi\n", + " for attr_name, attr_value in attributes_dataset.items():\n", + " dataset_BGO_Z1.attrs[attr_name] = attr_value\n", + " \n", + " dataset_BGO_Z0 = new_h5_file.create_dataset('BGO_Z0', data=BGO_Z0)\n", + " # Aggiungi gli attributi\n", + " for attr_name, attr_value in attributes_dataset.items():\n", + " dataset_BGO_Z0.attrs[attr_name] = attr_value\n", + "\n", + " # Salva il risultato della somma nel nuovo file\n", + " dataset_BGO_X1 = new_h5_file.create_dataset('BGO_X1', data=BGO_X1)\n", + " # Aggiungi gli attributi\n", + " for attr_name, attr_value in attributes_dataset.items():\n", + " dataset_BGO_X1.attrs[attr_name] = attr_value\n", + " \n", + " dataset_BGO_X0 = new_h5_file.create_dataset('BGO_X0', data=BGO_X0)\n", + " # Aggiungi gli attributi\n", + " for attr_name, attr_value in attributes_dataset.items():\n", + " dataset_BGO_X0.attrs[attr_name] = attr_value\n", + " \n", + " dataset_BGO_Y1 = new_h5_file.create_dataset('BGO_Y1', data=BGO_Y1)\n", + " # Aggiungi gli attributi\n", + " for attr_name, attr_value in attributes_dataset.items():\n", + " dataset_BGO_Y1.attrs[attr_name] = attr_value\n", + " \n", + " dataset_BGO_Y0 = new_h5_file.create_dataset('BGO_Y0', data=BGO_Y0)\n", + " # Aggiungi gli attributi\n", + " for attr_name, attr_value in attributes_dataset.items():\n", + " dataset_BGO_Y0.attrs[attr_name] = attr_value" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "bct", + "language": "python", + "name": "bct" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.12" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/cosipy/notebooks/bc-tools/irf_creation/bgo.config.yaml b/cosipy/notebooks/bc-tools/irf_creation/bgo.config.yaml new file mode 100644 index 0000000..b6bea32 --- /dev/null +++ b/cosipy/notebooks/bc-tools/irf_creation/bgo.config.yaml @@ -0,0 +1,219 @@ +# Example of a configuration file for BurstCube +# Note: Since some parameters are long lists, this is easier to +# read if you deactivate line wrapping e.g. less -S bc_config.yaml +# Stuff related to MEGAlib simulation +simulations: + #Geomega file, relative to this file + geometry: "/home/cosi/cosi/massmodel-cosi-smex-detailed/COSISMEX.sim_BGOreading.geo.setup" + # Spectrum of thrown particle. As long as we have narrow energy bin, + # the result do not strongly depend on this + # See Spectrum class for options + # Note that the normalization is arbitrary. + spectrum: + name: "PowerLaw" + args: + norm: 1 + index: 1 + # Detector effects such as energy resolution and efficiency + # See SimDetectorEffect class for options + detector_effects: + order: ["efficiency", "energy_resolution"] # Names must match entries in 'effects' + effects: + efficiency: + name: "SimEfficiencyFromPoints" + args: + energy: [0. , 11.62855864, 15.41611204, 20.43731452, 27.09397959, 35.91879595, 47.61795505, 63.12766293, 83.68905852, 110.94753378, 147.08440351, 194.99146145, 258.50239135] + unit: keV + efficiency: [0. , 0. , 0. , 0 , 0. ,0. , 0. , 0. , 0.60677697, 0.95142435, 0.96497073, 0.99320829, 1.] + energy_resolution: + name: "SimEnergyResolutionFromFit" + args: + # These coefficients are the ones reported in ICRC 2019 proceedings, + # multiply by 10 so they are in keV (not MeV) and fractional (not %) + # You can also specify one list per detector + coeffs: [0.007, 3.60, 20.98] + unit: keV + + # Energy bins in a standard detector response matrix. This also set the + # minimum and maximum simulated energy + energy_bins: [5.00e+01, 9.70e+01, 1.88e+02, 3.65e+02, 7.07e+02, 1.37e+03, 2.66e+03, 5.16e+03, 1.00e+04] + # HEALPix NSIDE of all-sky detector response + nside: 32 + +# Detector configuration +detector: + type: 'LabeledDetector' + detector_names: ['BGO_Z0_0', 'BGO_Z0_1','BGO_Z0_2','BGO_Z0_3','BGO_Z0_4','BGO_Z1_4','BGO_Z1_3','BGO_Z1_2','BGO_Z1_1','BGO_Z1_0','BGO_X1_0','BGO_X1_1','BGO_X1_2','BGO_X0_0','BGO_X0_1','BGO_X0_2','BGO_Y1_0','BGO_Y1_1','BGO_Y1_2','BGO_Y0_0','BGO_Y0_1','BGO_Y0_2'] + + # Edges of energy channels (keV) + energy_channels: [0,80.0,2.0e+03,20.0e+03] + +io: + headers: + CBD: + PRIMARY: + TELESCOP: ['BURSTCUBE' , 'Telescope mission name'] + INSTRUME: ['CSA' , 'Instrument name'] + DATAMODE: ['CBD' , 'Datamode'] + OBS_ID: ['YYMMDD' , 'Observation ID'] + DATE-OBS: ['yyyy-mm-ddThh:mm:ss', 'Start Date'] + DATE-END: ['yyyy-mm-ddThh:mm:ss', 'Stop Date'] + ORIGIN: ['GSFC' , 'Origin of the FITS file'] + DATE: ['yyyy-mm-ddThh:mm:ss', 'File creation date'] + CREATOR: ['BCTOOLS' , 'Software that create 1st the file'] + # TODO: more missing + CBD: + TELESCOP: ['BURSTCUBE' , 'Telescope mission name'] + INSTRUME: ['CSA' , 'Instrument name'] + DATAMODE: ['CBD' , 'Datamode'] + OBS_ID: ['YYMMDD' , 'Observation ID'] + # TODO: more missing + STDI: + TELESCOP: ['BURSTCUBE' , 'Telescope mission name'] + INSTRUME: ['CBD' , 'Instrument name'] + DATAMODE: ['EVENTS' , 'Datamode'] + OBS_ID: ['YYMMDD' , 'Observation ID'] + # TODO: more missing + TTE: + PRIMARY: + TELESCOP: ['BURSTCUBE' , 'Telescope mission name'] + INSTRUME: ['CSA' , 'Instrument name'] + DATAMODE: ['EVENTS' , 'Datamode'] + OBS_ID: ['YYMMDD' , 'Observation ID'] + DATE-OBS: ['yyyy-mm-ddThh:mm:ss', 'Start Date'] + DATE-END: ['yyyy-mm-ddThh:mm:ss', 'Stop Date'] + ORIGIN: ['GSFC' , 'Origin of the FITS file'] + DATE: ['yyyy-mm-ddThh:mm:ss', 'File creation date'] + CREATOR: ['BCTOOLS' , 'Software that create 1st the file'] + EVENTS: + EXTNAME: ['' , 'name of this binary table extension'] + TELESCOP: ['BURSTCUBE' , 'Telescope mission name'] + INSTRUME: ['CSA' , 'Instrument name'] + OBS_ID: ['YYMMDD' , 'Observation ID'] + EQUINOX: [2000. , 'Equinox for RA and Dec'] + RADECSYS: ['FK5' , 'Stellar reference frame'] + MJDREFI : [59215 , 'MJD of GLAST reference epoch, integer part'] + MJDREFF : [0.00080074074074074 , 'MJD of GLAST reference epoch, fractional part'] + TIMEREF: ['LOCAL' , 'Reference Frame'] + TASSIGN: ['SATELLITE' , 'Time assined'] + TIMESYS: ['TT' , 'Time system'] + TIMEUNIT: ['s' , 'Time since MJDREF, used in TSTART and TSTOP'] + TIMEDEL: [0.0 , 'Integration time'] + CLOCKAPP: [False , 'If clock corrections are applied (T/F)'] + DATE-OBS: ['yyyy-mm-ddThh:mm:ss', 'Start Date'] + DATE-END: ['yyyy-mm-ddThh:mm:ss', 'Stop Date'] + TSTART: [0.0 , 'Start Time'] + TSTOP: [0.0 , 'Stop Time'] + TELAPSE: [0.0 , 'TSTOP-TSTART in sec'] + ONTIME: [0.0 , 'Sum good time interval in'] + EXPOSURE: [0.0 , 'Exposure'] + DEADAPP: [False , 'deadtime applied'] + EVTTYPE: ['T3E' , 'Event Type of T3E or RTTE'] + TRIGGER: [0 , 'Trigger number'] + TRIGTIME: [0.0 , 'Trigger Time sec since 01 Jan 2021 00:00:00'] + TRIGUTC: ['2023-01-17T18:54:0' , 'TIME of Trigger time in UTC'] + HDUCLASS: ['OGIP' , 'format conforms to the OGIP standard'] + HDUCLAS1: ['EVENT', 'hduclass1'] + HDUCLAS2: ['ALL' , 'hduclass2'] + ORIGIN: ['GSFC' , 'Origin of the FITS file'] + CREATOR: ['BCTOOLS' , 'Software that create 1st the file'] + PROCVER: ['00.00.00.00' , 'Processing version'] + CALDBVER: [0 , 'CALDB version'] + SEQPNUM: [1 , 'Number of times the dataset has been processed'] + DATE: ['yyyy-mm-ddThh:mm:ss', 'File creation date'] + STDGTI: + EXTNAME: ['' , 'name of this binary table extension'] + TELESCOP: ['BURSTCUBE' , 'Telescope mission name'] + INSTRUME: ['CSA' , 'Instrument name'] + DATAMODE: ['EVENTS' , 'Datamode'] + OBS_ID: ['YYMMDD' , 'Observation ID'] + MJDREFI : [59215 , 'MJD of GLAST reference epoch, integer part'] + MJDREFF : [0.00080074074074074 , 'MJD of GLAST reference epoch, fractional part'] + TIMEREF: ['LOCAL' , 'Reference Frame'] + TASSIGN: ['SATELLITE' , 'Time assined'] + TIMESYS: ['TT' , 'Time system'] + TIMEUNIT: ['s' , 'Time since MJDREF, used in TSTART and TSTOP'] + CLOCKAPP: [False , 'If clock corrections are applied (T/F)'] + DATE-OBS: ['yyyy-mm-ddThh:mm:ss', 'Start Date'] + DATE-END: ['yyyy-mm-ddThh:mm:ss', 'Stop Date'] + TSTART: [0.0 , 'Start Time'] + TSTOP: [0.0 , 'Stop Time'] + HDUCLASS: ['OGIP' , 'format conforms to the OGIP standard'] + HDUCLAS1: ['GTI', 'hduclass1'] + CREATOR: ['BCTOOLS' , 'Software that create 1st the file'] + PROCVER: ['00.00.00.00' , 'Processing version'] + CALDBVER: [0 , 'CALDB version'] + SEQPNUM: [1 , 'Number of times the dataset has been processed'] + ORIGIN: ['GSFC' , 'Origin of the FITS file'] + DATE: ['yyyy-mm-ddThh:mm:ss', 'File creation date'] + + skymap: + PRIMARY: + TELESCOP: ['BURSTCUBE' , 'Telescope mission name'] + INSTRUME: ['CSA' , 'Instrument name'] + DATAMODE: ['EVENTS' , 'Datamode'] + OBJECT: ['FULLSKY' , 'Object name'] + RA_OBJ: [0.0 , 'R.A. Object'] + DEC_OBJ: [0.0 , 'Dec. Object'] + POSFLAG: [0 , 'Position FLAG 1=good 0=best effort - 1=false'] + EQUINOX: [2000. , 'Equinox for RA and Dec'] + RADECSYS: ['FK5' , 'Stellar reference frame'] + OBS_ID: ['YYMMDD' , 'Observation ID'] + TIMESYS: ['TT' , 'Time system'] + MJDREFI : [59215 , 'MJD of GLAST reference epoch, integer part'] + MJDREFF : [0.00080074074074074 , 'MJD of GLAST reference epoch, fractional part'] + TIMEUNIT: ['s' , 'Time since MJDREF, used in TSTART and TSTOP'] + TIMEREF: ['LOCAL' , 'Reference Frame'] + TASSIGN: ['SATELLITE' , 'Time assined'] + CLOCKAPP: [False , 'If clock corrections are applied (T/F)'] + TRIGGER: [0 , 'Trigger number'] + TRIGTIME: [0.0 , 'Trigger Time sec since 01 Jan 2021 00:00:00'] + TRIGUTC: ['2023-01-17T18:54:0' , 'TIME of Trigger time in UTC'] + TSTART: [0.0 , 'Start Time'] + TSTOP: [0.0 , 'Stop Time'] + DATE-OBS: ['yyyy-mm-ddThh:mm:ss', 'Start Date'] + DATE-END: ['yyyy-mm-ddThh:mm:ss', 'Stop Date'] + EXPOSURE: [0.0 , 'Exposure'] + E_MIN: [0.0 , 'lower energy boundary'] + E_MAX: [0.0 , 'high energy boundary'] + E_UNIT: ['keV' , 'unit for the energy'] + HDUCLASS: ['OGIP' , 'format conforms to the OGIP standard'] + HDUCLAS1: [IMAGE', 'HDUCLAS1'] + HDUCLAS2: [TOTAL', 'HDUCLAS2'] + CREATOR: ['BCTOOLS' , 'Software that create 1st the file'] + PROCVER: ['00.00.00.00' , 'Processing version'] + CALDBVER: [0 , 'CALDB version'] + SEQPNUM: [1 , 'Number of times the dataset has been processed'] + SOFTVER: ['' , 'Software version'] + ORIGIN: ['GSFC' , 'Origin of the FITS file'] + DATE: ['yyyy-mm-ddThh:mm:ss', 'File creation date'] + HEALPIX: + PIXTYPE: ['HEALPIX' , 'HEALPIX pixelization'] + ORDERING: ['' , 'Pixel order scheme'] + COORDSYS: ['' , 'Coordinate system'] + NSIDE: [0 , 'Healpix resolution parameter'] + FIRSTPIX: [0 , 'First pixel'] + LASTPIX: [0 , 'Last pixel'] + INDXSCHM: ['' , 'Indexing : IMPLICIT or EXPLICIT'] + OBJECT: ['FULLSKY' , 'Object name'] + BAD_DATA: [-1.6375e+30 , 'Value given to bad pixels'] + EXTNAME: ['' , 'name of this binary table extension'] + TELESCOP: ['BURSTCUBE' , 'Telescope mission name'] + INSTRUME: ['CSA' , 'Instrument name'] + OBS_ID: ['YYMMDD' , 'Observation ID'] + TRIGGER: [0 , 'Trigger number'] + TRIGTIME: [0.0 , 'Trigger Time sec since 01 Jan 2021 00:00:00'] + TRIGUTC: ['2023-01-17T18:54:0' , 'TIME of Trigger time in UTC'] + DATE-OBS: ['yyyy-mm-ddThh:mm:ss', 'Start Date'] + DATE-END: ['yyyy-mm-ddThh:mm:ss', 'Stop Date'] + EXPOSURE: [0.0 , 'Exposure'] + E_MIN: [0.0 , 'lower energy boundary'] + E_MAX: [0.0 , 'high energy boundary'] + E_UNIT: ['keV' , 'unit for the energy'] + CREATOR: ['BCTOOLS' , 'Software that create 1st the file'] + SEQPNUM: [1 , 'Number of times the dataset has been processed'] + SOFTVER: ['' , 'Software version'] + ORIGIN: ['GSFC' , 'Origin of the FITS file'] + DATE: ['yyyy-mm-ddThh:mm:ss', 'File creation date'] + + diff --git a/cosipy/notebooks/bc-tools/irf_creation/visualizeIRF.ipynb b/cosipy/notebooks/bc-tools/irf_creation/visualizeIRF.ipynb new file mode 100644 index 0000000..826f7ac --- /dev/null +++ b/cosipy/notebooks/bc-tools/irf_creation/visualizeIRF.ipynb @@ -0,0 +1,130 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "66db7bb8-d61d-4ac8-822e-cb930eb9d75d", + "metadata": {}, + "outputs": [], + "source": [ + "import bctools.data as data\n", + "from bctools.io import InstrumentResponse\n", + "from bctools.loc import LocalLocTable\n", + "from bctools.spectra.spectrum import PowerLaw,BandFunction,Comptonized\n", + "\n", + "# Convolve an full instrument response with a hypothetical spectrum\n", + "irf_path = \"/data/models/irf_summed_run8.h5\"\n", + "\n", + "with InstrumentResponse(irf_path) as irf:\n", + "\n", + " # Hypothetical spectrum\n", + " # This normalization corresponds to 1 ph/cm2/s between 50-300 keV\n", + " medium_spectrum = BandFunction._from_megalib(['BandFunction',10,10000,-1,-2.3,699.9],\"10.0\")\n", + " \n", + " # In this case we integrate the rate from all energy channels.\n", + " # You can subdivide the data into multiple energy channel groups\n", + " local_loctable = LocalLocTable.from_irf(irf, medium_spectrum,energy_channels = 1)\n", + "\n", + "\n", + "# The local_loctable contains the expected rates in spacecraft coordinates\n", + "# We now need to use this to estimate the total expected counts in sky coordinate for\n", + "# the full duration of an event.\n", + "# In this case we simply have a 1 second event and specifying the attitude by a quaternion\n", + "# ([0,0,0,1] corresponds to the identity rotation). You can have multiple attitude-duration\n", + "# pairs to correctly model long duration events.\n", + "sky_loctable = local_loctable.to_skyloctable(attitude = [0,0,0,1], duration = 1)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8ff1ec50-639b-436b-9f24-0df4186381b8", + "metadata": {}, + "outputs": [], + "source": [ + "\n", + "\n", + "# Convolve an full instrument response with a hypothetical spectrum\n", + "irf_path_bis = \"/data/models/irf_summed_run10.h5\"\n", + "\n", + "\n", + "with InstrumentResponse(irf_path_bis) as irf_bis:\n", + "\n", + " # Hypothetical spectrum\n", + " # This normalization corresponds to 1 ph/cm2/s between 50-300 keV\n", + " medium_spectrum_bis = BandFunction._from_megalib(['BandFunction',10,10000,-1,-2.3,699.9],\"10.0\")\n", + " \n", + " # In this case we integrate the rate from all energy channels.\n", + " # You can subdivide the data into multiple energy channel groups\n", + " local_loctable_bis = LocalLocTable.from_irf(irf_bis, medium_spectrum_bis,energy_channels = 1)\n", + "\n", + "# The local_loctable contains the expected rates in spacecraft coordinates\n", + "# We now need to use this to estimate the total expected counts in sky coordinate for\n", + "# the full duration of an event.\n", + "# In this case we simply have a 1 second event and specifying the attitude by a quaternion\n", + "# ([0,0,0,1] corresponds to the identity rotation). You can have multiple attitude-duration\n", + "# pairs to correctly model long duration events.\n", + "sky_loctable_bis = local_loctable_bis.to_skyloctable(attitude = [0,0,0,1], duration = 1)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2d779d86-befd-42c9-9682-3933dbefb474", + "metadata": {}, + "outputs": [], + "source": [ + "sky_loctable_bis.get_expectation_map('BGO_Y1').plot();" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cc9dec52-0bfe-46b2-8139-4b1ebcb8588a", + "metadata": {}, + "outputs": [], + "source": [ + "sky_loctable.get_expectation_map('BGO_Y1').plot();" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "35e0c57f-1e76-4052-9952-bafb3d0e3e30", + "metadata": {}, + "outputs": [], + "source": [ + "print(f\"Hard Look-up tables: {sky_loctable.labels}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "91ce59be-ecf8-4d9d-8c72-cd223e47117a", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "bct", + "language": "python", + "name": "bct" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.12" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/cosipy/notebooks/bc-tools/performance_evaluation/bctools_frequentist_coverage.ipynb b/cosipy/notebooks/bc-tools/performance_evaluation/bctools_frequentist_coverage.ipynb new file mode 100644 index 0000000..1e37a43 --- /dev/null +++ b/cosipy/notebooks/bc-tools/performance_evaluation/bctools_frequentist_coverage.ipynb @@ -0,0 +1,948 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "ba72606f-02ac-4c3a-bba3-d9b79ea52966", + "metadata": {}, + "outputs": [], + "source": [ + "from bctools.io import InstrumentResponse\n", + "from bctools.loc import LocalLocTable\n", + "from bctools.spectra.spectrum import BandFunction,Comptonized\n", + "import os\n", + "import math\n", + "import multiprocessing\n", + "import itertools\n", + "import pickle \n", + "import time\n", + "import multiprocessing\n", + "import numpy as np\n", + "from bctools.loc import TSMap, NormLocLike\n", + "import astropy.units as u\n", + "from astropy.coordinates import SkyCoord\n", + "from scipy.stats import chi2\n", + "import numpy as np\n", + "import healpy as hp\n", + "import matplotlib.pyplot as plt\n", + "\n", + "\n", + "\n", + "run_name = \"run10\"\n", + "irf_path = \"/data/models/irf_summed_\"+run_name+\".h5\"\n", + "dir_path = \"/data/test_newrepo/\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6a18cd09-5f40-4911-9f9f-028bc62b06d7", + "metadata": {}, + "outputs": [], + "source": [ + "# Convolve an full instrument response with a hypothetical spectrum\n", + "#irf_path = \"/data/models/irf_summed.h5\"\n", + "#irf_path = \"/data/models/run_1_noeffect_irf_summed.h5\"\n", + "\n", + "# The code is inspired by the bc-tools tutorial.\n", + "\n", + "load_from_file = True\n", + "\n", + "if load_from_file:\n", + "\n", + " with open('/data/models/LUTS/soft_lut_' + run_name + '.pkl', 'rb') as f:\n", + " soft_sky_loctable = pickle.load(f)\n", + " \n", + " with open('/data/models/LUTS/medium_lut_' + run_name + '.pkl', 'rb') as f:\n", + " medium_sky_loctable = pickle.load(f)\n", + " \n", + " with open('/data/models/LUTS/hard_lut_' + run_name + '.pkl', 'rb') as f:\n", + " hard_sky_loctable = pickle.load(f)\n", + " \n", + "else:\n", + "\n", + " with InstrumentResponse(irf_path) as irf: \n", + " \n", + " # Hypothetical spectrum\n", + " # This normalization corresponds to 1 ph/cm2/s between 50-300 keV\n", + " #spectrum = PowerLaw(60,2)\n", + " #spectrum = PowerLaw._from_megalib(['PowerLaw',10,10000,2],\"5.0\")\n", + " soft_spectrum = BandFunction._from_megalib(['BandFunction',10,10000,-1.9,-3.7,230],\"10.0\")\n", + " medium_spectrum = BandFunction._from_megalib(['BandFunction',10,10000,-1,-2.3,699.9],\"10.0\")\n", + " hard_spectrum = Comptonized._from_megalib(['Comptonized',10,10000,-0.5,1500],\"10.0\")\n", + " \n", + " # In this case we integrate the rate from all energy channels. \n", + " # You can subdivide the data into multiple energy channel groups\n", + " soft_local_loctable = LocalLocTable.from_irf(irf, soft_spectrum,energy_channels = 1) # [80,2000]\n", + " medium_local_loctable = LocalLocTable.from_irf(irf, medium_spectrum,energy_channels = 1)\n", + " hard_local_loctable = LocalLocTable.from_irf(irf, hard_spectrum,energy_channels = 1)\n", + " \n", + " # The local_loctable contains the expected rates in spacecraft coordinates\n", + " # We now need to use this to estimate the total expected counts in sky coordinate for\n", + " # the full duration of an event. \n", + " # In this case we simply have a 1 second event and specifying the attitude by a quaternion\n", + " # ([0,0,0,1] corresponds to the identity rotation). You can have multiple attitude-duration\n", + " # pairs to correctly model long duration events.\n", + " soft_sky_loctable = soft_local_loctable.to_skyloctable(attitude = [0,0,0,1], duration = 1)\n", + " medium_sky_loctable = medium_local_loctable.to_skyloctable(attitude = [0,0,0,1], duration = 1)\n", + " hard_sky_loctable = hard_local_loctable.to_skyloctable(attitude = [0,0,0,1], duration = 1)\n", + " \n", + " #Store LUT\n", + " \n", + " import pickle\n", + " \n", + " # Salvataggio su file\n", + " with open('/data/models/LUTS/soft_lut_'+run_name+'.pkl', 'wb') as f:\n", + " pickle.dump(soft_sky_loctable, f)\n", + " with open('/data/models/LUTS/medium_lut_'+run_name+'.pkl', 'wb') as f:\n", + " pickle.dump(medium_sky_loctable, f)\n", + " with open('/data/models/LUTS/hard_lut_'+run_name+'.pkl', 'wb') as f:\n", + " pickle.dump(hard_sky_loctable, f)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "90e862b1-8fb8-4935-b1f6-126b968a3905", + "metadata": {}, + "outputs": [], + "source": [ + "#soft_local_loctable.channel_mask(\"BGO_Y0\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3ab587bb-637b-41ad-ab3a-ca4c1b83b605", + "metadata": {}, + "outputs": [], + "source": [ + "\n", + "def angular_distance(theta1, phi1, theta2, phi2):\n", + " \"\"\"\n", + " Compute the angular (great-circle) distance in degrees between two points\n", + " specified by (theta, phi) coordinates in degrees.\n", + "\n", + " Parameters:\n", + " theta1, phi1: floats, coordinates of the first point (degrees).\n", + " theta2, phi2: floats, coordinates of the second point (degrees).\n", + "\n", + " Returns:\n", + " angular_dist_deg: float, angular distance between the two points in degrees.\n", + " \"\"\"\n", + " # Adjust theta values by shifting -90\n", + " theta1 = 90 - theta1\n", + " theta2 = 90 - theta2\n", + "\n", + " # Convert angles to radians\n", + " theta1_rad = math.radians(theta1)\n", + " phi1_rad = math.radians(phi1)\n", + " theta2_rad = math.radians(theta2)\n", + " phi2_rad = math.radians(phi2)\n", + " \n", + " # Compute the difference in longitude\n", + " delta_phi = abs(phi1_rad - phi2_rad)\n", + "\n", + " # Apply spherical law of cosines\n", + " value = (math.sin(theta1_rad) * math.sin(theta2_rad) +\n", + " math.cos(theta1_rad) * math.cos(theta2_rad) * math.cos(delta_phi))\n", + " \n", + " # Clamp value to [-1, 1] to avoid floating point errors in acos\n", + " value_clamped = max(-1.0, min(1.0, value)) \n", + " \n", + " angular_dist = math.acos(value_clamped)\n", + " \n", + " # Convert angular distance from radians to degrees\n", + " angular_dist_deg = math.degrees(angular_dist)\n", + " \n", + " return angular_dist_deg" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2e60731d-564e-4dad-bfa1-b32280bfb9c8", + "metadata": {}, + "outputs": [], + "source": [ + "print(f\"Soft Look-up tables: {soft_sky_loctable.labels}\")\n", + "print(f\"Medium Look-up tables: {medium_sky_loctable.labels}\")\n", + "print(f\"Hard Look-up tables: {hard_sky_loctable.labels}\")\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9c7a018f-c161-4517-bb66-15fa1531fe32", + "metadata": {}, + "outputs": [], + "source": [ + "hard_sky_loctable.get_expectation_map('BGO_Y0').plot();" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "250a7604-0828-4b7c-8ae5-782d1c1a4134", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "77beda1d-677a-43fc-a8b3-46dad6dcf9f5", + "metadata": {}, + "outputs": [], + "source": [ + "#test with LUTs\n", + "import pickle \n", + "\n", + "file_name_test = \"run57_mix_mega_shared\"\n", + "\n", + "file_path = dir_path+'/'+file_name_test+'_dataset.pkl'\n", + "\n", + "# Load the array from the pickle file\n", + "with open(file_path, 'rb') as file:\n", + " loaded_array_test = pickle.load(file)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ea9d182b-a657-4350-afa7-719b92ff5b49", + "metadata": {}, + "outputs": [], + "source": [ + "loaded_array_test[2]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "60717231-3723-4d30-8ae4-c73baf35dd96", + "metadata": {}, + "outputs": [], + "source": [ + "import numpy as np\n", + "\n", + "filter_flux = 1\n", + "filter_spectra = 1\n", + "\n", + "analysis_spectrum = \"medium\"\n", + "\n", + "# These values lead to a mean sigma between 10-15 and were used for the paper\n", + "if analysis_spectrum == \"soft\":\n", + " min_flux = 14\n", + " max_flux = 16\n", + " spectrum_string = \"230\"\n", + " \n", + "elif analysis_spectrum == \"medium\" :\n", + " min_flux = 1\n", + " max_flux = 3\n", + " spectrum_string = \"699\"\n", + " \n", + "elif analysis_spectrum == \"hard\":\n", + " min_flux = 1\n", + " max_flux = 1.6\n", + " spectrum_string = \"1500\"\n", + "\n", + "filtered_loaded_array_test = np.empty(loaded_array_test.shape[0], dtype=object)\n", + "\n", + "count = 0\n", + "\n", + "if filter_flux == 1:\n", + " for grb in loaded_array_test:\n", + " if grb['flux'] > min_flux and grb['flux'] <= max_flux:\n", + " filtered_loaded_array_test[count] = grb\n", + " count = count+1\n", + "\n", + " loaded_array_test = filtered_loaded_array_test[:count]\n", + "\n", + "filtered_loaded_array_test = np.empty(loaded_array_test.shape[0], dtype=object)\n", + "\n", + "count = 0\n", + "\n", + "if filter_spectra==1:\n", + " for grb in loaded_array_test:\n", + " #if grb['spectra'] == 'medium':\n", + " if spectrum_string in grb['spectrum']:\n", + " filtered_loaded_array_test[count] = grb\n", + " count = count+1\n", + "\n", + " loaded_array_test = filtered_loaded_array_test[:count]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a98f9842-1a7a-4042-8a04-a7fc6f130c33", + "metadata": {}, + "outputs": [], + "source": [ + "loaded_array_test.shape" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "fb781192-43c8-4ad2-8c0a-ecd211f04548", + "metadata": {}, + "outputs": [], + "source": [ + "test_number = 10000\n", + "random_indices = np.random.permutation(len(loaded_array_test))\n", + "\n", + "# Use the random index array to shuffle the original array\n", + "shuffled_loaded_array = loaded_array_test[random_indices]\n", + "\n", + "if(len(loaded_array_test)<=test_number):\n", + " train_dataset = []\n", + "\n", + " test_dataset = shuffled_loaded_array\n", + "else:\n", + " train_dataset = shuffled_loaded_array[:len(shuffled_loaded_array)-test_number]\n", + "\n", + " test_dataset = shuffled_loaded_array[len(shuffled_loaded_array)-test_number:]\n", + "\n", + " " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "683a3d33-3dd3-49af-b5f6-7e13065894e7", + "metadata": {}, + "outputs": [], + "source": [ + "import numpy as np\n", + "\n", + "def process_source(\n", + " grb,\n", + " band=\"auto\", # \"soft\" | \"medium\" | \"hard\" | \"auto\" | or a list/tuple, e.g. (\"soft\",\"hard\")\n", + "):\n", + " \"\"\"\n", + " Parameters\n", + " ----------\n", + " grb : dict\n", + " Must contain: \n", + " - 'coord' = (theta, phi)\n", + " - 'counts' = array/list with at least 6 channels (will be reordered below)\n", + " - optional 'spectrum'\n", + " band : str | Iterable[str]\n", + " \"soft\", \"medium\", \"hard\", \"auto\" (default), or a collection of bands to try.\n", + " With \"auto\" it will compare all three and pick the one with max TS.\n", + " \n", + " Returns\n", + " -------\n", + " list with:\n", + " [theta_real, phi_real, theta_loc, phi_loc, dist, theta_dist, phi_dist,\n", + " max_sqrt_ts, cont_radius, best_ts, original_ts_value, cont_area, b_counts_first, chosen_band]\n", + " \"\"\"\n", + "\n", + " # --- True coordinates from GRB metadata\n", + " theta_real = float(grb['coord'][0])\n", + " phi_real = float(grb['coord'][1])\n", + "\n", + " # --- Reorder input counts as in original code\n", + " s_counts = np.array([grb['counts'][3], grb['counts'][2], grb['counts'][5],\n", + " grb['counts'][4], grb['counts'][1], grb['counts'][0]], dtype=float)\n", + "\n", + " # --- Simulated background counts (different IRF order)\n", + " b_sim = np.array([57.6053, 58.7157, 51.4131, 48.2891, 47.7293, 45.9617], dtype=float)\n", + " b_counts = np.array([b_sim[3], b_sim[2], b_sim[5], b_sim[4], b_sim[1], b_sim[0]], dtype=float) * 20\n", + "\n", + " # --- Optionally add Poisson fluctuations to source counts\n", + " \n", + " random_bkg = np.random.poisson(lam=b_counts)\n", + " s_counts = s_counts + random_bkg\n", + "\n", + " # --- Decide which bands to analyze\n", + " if isinstance(band, str):\n", + " band = band.lower()\n", + " if band == \"auto\":\n", + " bands_to_try = (\"soft\", \"medium\", \"hard\")\n", + " elif band in (\"soft\", \"medium\", \"hard\"):\n", + " bands_to_try = (band,)\n", + " else:\n", + " raise ValueError(f'band=\"{band}\" not valid: use \"soft\", \"medium\", \"hard\" or \"auto\".')\n", + " else:\n", + " # Accept tuple/list of bands\n", + " bands_to_try = tuple(b.lower() for b in band)\n", + " for b in bands_to_try:\n", + " if b not in (\"soft\", \"medium\", \"hard\"):\n", + " raise ValueError(f'Invalid band in list: \"{b}\". Allowed: soft/medium/hard.')\n", + "\n", + " # --- Map band names to localization tables (assume they exist in global scope)\n", + " band_to_table = {\n", + " \"soft\": soft_sky_loctable,\n", + " \"medium\": medium_sky_loctable,\n", + " \"hard\": hard_sky_loctable,\n", + " }\n", + "\n", + " # --- Run localization for each selected band\n", + " results_per_band = {}\n", + " for b in bands_to_try:\n", + " sqrt_ts, ra_loc, dec_loc, cont_radius, ts, loc_tsvalue, cont_area = localize_grb(\n", + " band_to_table[b],\n", + " s_counts,\n", + " b_counts,\n", + " theta_real,\n", + " phi_real\n", + " )\n", + " results_per_band[b] = {\n", + " \"sqrt_ts\": sqrt_ts,\n", + " \"ra_loc\": ra_loc,\n", + " \"dec_loc\": dec_loc,\n", + " \"cont_radius\": cont_radius,\n", + " \"ts\": ts,\n", + " \"loc_tsvalue\": loc_tsvalue,\n", + " \"cont_area\": cont_area,\n", + " }\n", + "\n", + " # --- Choose the best band by maximum sqrt_ts\n", + " best_band = max(results_per_band, key=lambda k: results_per_band[k][\"sqrt_ts\"])\n", + " best = results_per_band[best_band]\n", + "\n", + " # --- Convert RA/DEC of best localization into spherical coordinates\n", + " theta_loc, phi_loc = ra_dec_to_theta_phi(best[\"ra_loc\"], best[\"dec_loc\"])\n", + "\n", + " # --- Angular metrics: distance and coordinate offsets\n", + " dist = angular_distance(theta_loc, phi_loc, theta_real, phi_real)\n", + " theta_dist = float(np.abs(theta_loc - theta_real))\n", + " phi_dist = float(diff_phi(phi_loc, phi_real))\n", + "\n", + " # --- Package results (note: fixed typo b_s_counts -> b_counts[0])\n", + " result = [\n", + " theta_real, # 0 true theta\n", + " phi_real, # 1 true phi\n", + " theta_loc, # 2 localized theta\n", + " phi_loc, # 3 localized phi\n", + " dist, # 4 angular distance\n", + " theta_dist, # 5 theta difference\n", + " phi_dist, # 6 phi difference\n", + " best[\"sqrt_ts\"], # 7 max sqrt TS\n", + " best[\"cont_radius\"], # 8 localization contour radius\n", + " best[\"ts\"], # 9 TS value\n", + " best[\"loc_tsvalue\"], # 10 TS value at localization\n", + " best[\"cont_area\"], # 11 contour area\n", + " float(b_counts[0]), # 12 first background bin\n", + " best_band, # 13 chosen band\n", + " ]\n", + "\n", + " return result\n", + "\n", + "def localize_grb(sky_loctable,s,b,theta_real,phi_real):\n", + " \n", + " sky_loctable.set_background(b)\n", + " sky_loctable.set_data(s)\n", + "\n", + " # Define a map of nside = 32. Note that this is a finer resolution\n", + " #that the underlying look-up table, which will be interpolated\n", + " ts = TSMap(nside = 32, coordsys = 'icrs')\n", + "\n", + " # NormLocLike is a subclass of LocLike and computes\n", + " # a Poisson likelihood for counting instruments. The\n", + " # overall normalization is the only free parameter\n", + " norm_likelihood = NormLocLike(sky_loctable)\n", + " \n", + " # Compute the TS map from one or more LocLikelihood\n", + " ts.compute(norm_likelihood)\n", + " #print(\"#\"+str(theta_real)+\"#\")\n", + " \n", + " # Correggi theta_real prima di convertirlo\n", + " if theta_real < 0:\n", + " print(f\"Invalid theta_real: {theta_real}\")\n", + " theta_real = 0\n", + " elif theta_real > 180:\n", + " print(f\"Invalid theta_real: {theta_real}\")\n", + " theta_real = 180\n", + " \n", + " theta_rad = np.deg2rad(theta_real)\n", + " phi_rad = np.deg2rad(phi_real)\n", + " \n", + " if theta_rad < 0 or theta_rad > np.pi:\n", + " print(f\"theta_rad fuori range: {theta_rad}\")\n", + "\n", + "\n", + " #print(dir(ts))\n", + " ipix = ts.ang2pix(theta_rad, phi_rad)\n", + "\n", + " # Get the value at the given coordinates\n", + " #ipix = ts.ang2pix(ts.nside, theta_real * u.deg.to(u.rad), phi_real * u.deg.to(u.rad))\n", + " original_ts_value = ts._data[ipix]\n", + "\n", + " # Inizializza l'array dei livelli di confidenza\n", + " confidence_levels = np.arange(0, 1.01, 0.01)\n", + " \n", + " # Calcola i raggi di contenimento per ciascun valore di cont\n", + " containment_radius = np.array([\n", + " np.sqrt(ts.error_area(cont=cont) / np.pi).to(u.deg).value\n", + " for cont in confidence_levels\n", + " ])\n", + " \n", + " #original_ts_value = -1\n", + " cont_area = ts.error_area(cont = .9).to(u.deg**2).value\n", + " \n", + " return np.max(ts),ts.best_loc().ra.deg,ts.best_loc().dec.deg,containment_radius,ts,original_ts_value,cont_area\n", + "\n", + "def ra_dec_to_theta_phi(ra, dec):\n", + "\n", + " theta = 90-dec\n", + " \n", + " phi = ra\n", + " \n", + " return theta, phi\n", + "\n", + "def diff_phi(a1, a2):\n", + " # Calcola la differenza diretta\n", + " diff = abs(a1 - a2)\n", + " \n", + " # Trova il percorso più breve tenendo conto del ciclo degli angoli\n", + " if diff > 180:\n", + " diff = 360 - diff\n", + " \n", + " return diff" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3857a3ef-1ac2-4f38-8eda-115cecc9c71e", + "metadata": {}, + "outputs": [], + "source": [ + "\n", + "results_bkg = []\n", + "spectra_fitted = 0\n", + "\n", + "def init_worker():\n", + " seed = int.from_bytes(os.urandom(4), \"little\")\n", + " np.random.seed(seed)\n", + "\n", + "def process_in_parallel(test_dataset,band):\n", + " with multiprocessing.Pool(processes=200, initializer=init_worker) as pool:\n", + " results = pool.starmap(process_source, zip(test_dataset, itertools.repeat(band)))\n", + " return results\n", + "\n", + "\n", + "results_bkg = process_in_parallel(test_dataset,analysis_spectrum)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9f08e0af-ec2f-4e98-8f55-bd90ac55fab2", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "538bf57b-8e53-4a6b-9d02-cff712dacd60", + "metadata": {}, + "outputs": [], + "source": [ + "distances = []\n", + "theta_distances = []\n", + "phi_distances = []\n", + "cont_radius_list = []\n", + "cont_area_list = []\n", + "for res in results_bkg:\n", + " distances.append(res[4])\n", + " theta_distances.append(res[5])\n", + " phi_distances.append(res[6])\n", + " cont_radius_list.append(res[8])\n", + " cont_area_list.append(res[11])\n", + "\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c7b66dff-1fa6-436b-8c51-3423a146dd5d", + "metadata": {}, + "outputs": [], + "source": [ + "\n", + "plt.hist(distances)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5d73bc87-526d-4899-a561-085e65143101", + "metadata": {}, + "outputs": [], + "source": [ + "np.mean(distances)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9abbc20b-d55c-408d-9241-4375d67c5088", + "metadata": {}, + "outputs": [], + "source": [ + "test_labels = []\n", + "for grb in test_dataset:\n", + " theta_real = float(grb['coord'][0])\n", + " phi_real = float(grb['coord'][1])\n", + " test_labels.append([theta_real,phi_real])\n", + "\n", + "test_labels = np.array(test_labels)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "28770484-f4d1-450f-981b-50d2fca8c023", + "metadata": {}, + "outputs": [], + "source": [ + "print(np.mean(distances))\n", + "print(np.mean(theta_distances))\n", + "print(np.mean(phi_distances))" + ] + }, + { + "cell_type": "markdown", + "id": "6d2c7500-9936-4199-9f20-ba2cf734f04c", + "metadata": {}, + "source": [ + "# Frequentist Analysis" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "92af79d2-2663-445e-bd09-153625ff4698", + "metadata": {}, + "outputs": [], + "source": [ + "ts_array = []\n", + "for r in results_bkg:\n", + " ts_localied = r[7] \n", + " ts_array.append(ts_localied)\n", + "ts_array = np.array(ts_array)\n", + "print(\"Mean TS=\"+str(np.sqrt(ts_array).mean()))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "35a196ab-c9b2-4188-95ab-8ae78f9e816f", + "metadata": {}, + "outputs": [], + "source": [ + "def spherical_to_radec_deg(theta_deg, phi_deg):\n", + " \"\"\"\n", + " Convert spherical coordinates (theta, phi) in degrees to RA and Dec.\n", + "\n", + " Parameters:\n", + " - theta_deg: polar angle in degrees (0° at north pole)\n", + " - phi_deg: azimuthal angle in degrees (0° at x-axis)\n", + "\n", + " Returns:\n", + " - ra: Right Ascension in degrees (0 to 360)\n", + " - dec: Declination in degrees (-90 to +90)\n", + " \"\"\"\n", + " dec = 90.0 - theta_deg\n", + " ra = phi_deg % 360.0\n", + " return ra, dec\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6d91ea25-1778-442f-8ee8-939b3fae22e7", + "metadata": {}, + "outputs": [], + "source": [ + "#%matplotlib widget\n", + "import matplotlib.pyplot as plt\n", + "import numpy as np\n", + "\n", + "results = results_bkg[0]\n", + "# result = [theta_real, phi_real, theta_loc, phi_loc, dist, theta_dist, phi_dist, max_sqrt, cont_radius,best_ts,loc_ts_value]\n", + "theta_real = results[0]\n", + "phi_real = results[1]\n", + "theta_loc = results[2]\n", + "phi_loc = results[3]\n", + "ts_max = results[7]\n", + "ts_loc = results[9]\n", + "print(ts_loc)\n", + "\n", + "print(f\"Maximum sqrt(TS) = {np.sqrt(ts_max):.2f}\")\n", + "print(f\"Best estimate: RA = {ts_loc.best_loc().ra:.2f} Dec= {ts_loc.best_loc().dec:.2f}\")\n", + "print(f\"Error area (90% cont.): {ts_loc.error_area(cont = .9).to(u.deg**2):.2f}\")\n", + "print(f\"Equivalent error radius (90% cont.): {np.sqrt(ts_loc.error_area(cont = .9)/np.pi).to(u.deg):.2f}\")\n", + "\n", + "img,ax = ts_loc.plot(cont=0.9)\n", + "ax.grid(alpha = .5)\n", + "\n", + "real_ra,real_dec = spherical_to_radec_deg(theta_real,phi_real)\n", + "\n", + "# Actual location of simulated source\n", + "ax.scatter(real_ra,real_dec,\n", + " color = 'red', transform = ax.get_transform('world'), s = 1);\n", + "\n", + "loc_ra,loc_dec = spherical_to_radec_deg(theta_loc,phi_loc)\n", + "\n", + "# Actual location of simulated source\n", + "ax.scatter(loc_ra,loc_dec,\n", + " color = 'magenta', transform = ax.get_transform('world'), s = 1);" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ed90aa8e-42e3-416f-b54d-7dc84c66606b", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c936f828-cefd-4ebe-8906-5eee08f133ff", + "metadata": {}, + "outputs": [], + "source": [ + "\n", + "values = np.arange(0, 1.01, 0.01)\n", + "\n", + "fraction_array = []\n", + "\n", + "for value in values:\n", + " count = 0 \n", + " for r in results_bkg:\n", + " max_chi2 = r[7]\n", + " original_ts = r[10]\n", + " if original_ts>max_chi2-chi2.ppf(value, 2).flatten():\n", + " count = count + 1\n", + " fraction_array.append(count/len(results_bkg))\n", + " \n", + " " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "76fce1e2-6404-459f-9a55-a41313d45e69", + "metadata": {}, + "outputs": [], + "source": [ + "import matplotlib.pyplot as plt\n", + "import numpy as np\n", + "\n", + "# Plot della curva empirica\n", + "plt.plot(values, fraction_array, label='Model Calibration Curve')\n", + "\n", + "# Aggiunta della unitary line (y = x)\n", + "plt.plot(values, values, linestyle='--', color='gray', label='Perfect Calibration (y=x)')\n", + "\n", + "# Etichette e legenda\n", + "plt.xlabel('Confidence Level')\n", + "plt.ylabel('Fraction < Confidence Level')\n", + "plt.title('Calibration Plot')\n", + "plt.legend()\n", + "plt.grid(True)\n", + "plt.show()\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a76ca367-da43-455d-84cd-4b8d9f9f1877", + "metadata": {}, + "outputs": [], + "source": [ + "fraction_array[90]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "691c53d0-656f-4bc7-8583-3c4602fe97f7", + "metadata": {}, + "outputs": [], + "source": [ + "import numpy as np\n", + "from scipy.stats import chi2\n", + "import matplotlib.pyplot as plt\n", + "\n", + "# Confidence level threshold (fixed)\n", + "delta_chi2_90 = chi2.ppf(0.9, df=2)\n", + "\n", + "# Define flux bins (adjust range and step as needed)\n", + "flux_bins = np.arange(0, 30 + 1, 1)\n", + "flux_centers = (flux_bins[:-1] + flux_bins[1:]) / 2\n", + "coverage_array = []\n", + "\n", + "# For each flux bin\n", + "for i in range(len(flux_bins) - 1):\n", + " count_in = 0\n", + " count_total = 0\n", + " count = 0\n", + " for r in results_bkg:\n", + " flux = float(test_dataset[count]['flux'])\n", + " if flux_bins[i] <= flux < flux_bins[i+1]:\n", + " max_chi2 = r[7]\n", + " original_ts = r[10]\n", + " if original_ts>max_chi2-delta_chi2_90:\n", + " count_in += 1\n", + " count_total += 1\n", + " count = count+1\n", + " \n", + " if count_total > 0:\n", + " coverage = count_in / count_total\n", + " else:\n", + " coverage = 0\n", + " coverage_array.append(coverage)\n", + "\n", + "# Plotting\n", + "plt.plot(flux_centers, coverage_array, marker='o')\n", + "plt.axhline(0.9, color='gray', linestyle='--', label='Target 90%')\n", + "plt.xlabel('Source Flux')\n", + "plt.ylabel('Empirical 90% Coverage')\n", + "plt.title('Coverage vs. Source Flux at Fixed 90% CL')\n", + "plt.legend()\n", + "plt.grid(True)\n", + "plt.show()\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8183aef7-661a-4a99-ab7a-2c723744381a", + "metadata": {}, + "outputs": [], + "source": [ + "import matplotlib.pyplot as plt\n", + "import numpy as np\n", + " \n", + "\n", + "for i in range (0,0):#%matplotlib widget\n", + " \n", + " # result = [theta_real, phi_real, theta_loc, phi_loc, dist, theta_dist, phi_dist, max_sqrt, cont_radius,best_ts,loc_ts_value]\n", + " theta_real = results_bkg[i][0]\n", + " phi_real = results_bkg[i][1]\n", + " theta_loc = results_bkg[i][2]\n", + " phi_loc = results_bkg[i][3]\n", + " ts_max = results_bkg[i][7]\n", + " ts_loc = results_bkg[i][9]\n", + " print(ts_loc)\n", + " \n", + " print(f\"Maximum sqrt(TS) = {np.sqrt(ts_max):.2f}\")\n", + " print(f\"Best estimate: RA = {ts_loc.best_loc().ra:.2f} Dec= {ts_loc.best_loc().dec:.2f}\")\n", + " print(f\"Error area (90% cont.): {ts_loc.error_area(cont = .9).to(u.deg**2):.2f}\")\n", + " print(f\"Equivalent error radius (90% cont.): {np.sqrt(ts_loc.error_area(cont = .9)/np.pi).to(u.deg):.2f}\")\n", + " \n", + " img,ax = ts_loc.plot(cont=0.9)\n", + " ax.grid(alpha = .5)\n", + " \n", + " real_ra,real_dec = spherical_to_radec_deg(theta_real,phi_real)\n", + " \n", + " # Actual location of simulated source\n", + " ax.scatter(real_ra,real_dec,\n", + " color = 'red', transform = ax.get_transform('world'), s = 1);\n", + " \n", + " loc_ra,loc_dec = spherical_to_radec_deg(theta_loc,phi_loc)\n", + " \n", + " # Actual location of simulated source\n", + " ax.scatter(loc_ra,loc_dec,\n", + " color = 'magenta', transform = ax.get_transform('world'), s = 1);" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9d36f327-bec7-42d1-9e82-b58415e96d15", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "48569ec2-576f-40ff-8c7a-d1f8bf24ce58", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "857dc9c4-525f-40d6-a0ff-a461f16dbfb3", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d881814f-a066-40f8-9965-dcbc28a72149", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "906f9af2-5a04-4a8e-941e-bdbec6c7c455", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "05caa286-1b21-4d5b-8219-a7e6bddfc93a", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "187898c7-f3b1-4310-8b44-e0c5f706f7c5", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7314024d-65b0-4d8f-b6db-7ae34b6d5767", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "bct", + "language": "python", + "name": "bct" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.12" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/cosipy/notebooks/bc-tools/performance_evaluation/bctools_localization.ipynb b/cosipy/notebooks/bc-tools/performance_evaluation/bctools_localization.ipynb new file mode 100644 index 0000000..93fb119 --- /dev/null +++ b/cosipy/notebooks/bc-tools/performance_evaluation/bctools_localization.ipynb @@ -0,0 +1,813 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "ba72606f-02ac-4c3a-bba3-d9b79ea52966", + "metadata": {}, + "outputs": [], + "source": [ + "from bctools.io import InstrumentResponse\n", + "from bctools.loc import LocalLocTable\n", + "from bctools.spectra.spectrum import BandFunction,Comptonized\n", + "import os\n", + "import math\n", + "import multiprocessing\n", + "import itertools\n", + "import pickle \n", + "\n", + "run_name = \"run10\"\n", + "irf_path = \"/data/models/irf_summed_\"+run_name+\".h5\"\n", + "dir_path = \"/data/test_newrepo/\"\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6a18cd09-5f40-4911-9f9f-028bc62b06d7", + "metadata": {}, + "outputs": [], + "source": [ + "# Convolve an full instrument response with a hypothetical spectrum\n", + "\n", + "# The code is inspired by the bc-tools tutorial.\n", + "\n", + "import pickle\n", + "\n", + "# The first time this should be False, then you can put True and speedup the loading.\n", + "load_from_file = True\n", + "\n", + "if load_from_file:\n", + "\n", + " with open(dir_path+'/soft_lut_' + run_name + '.pkl', 'rb') as f:\n", + " soft_sky_loctable = pickle.load(f)\n", + " \n", + " with open(dir_path+'/medium_lut_' + run_name + '.pkl', 'rb') as f:\n", + " medium_sky_loctable = pickle.load(f)\n", + " \n", + " with open(dir_path+'/hard_lut_' + run_name + '.pkl', 'rb') as f:\n", + " hard_sky_loctable = pickle.load(f)\n", + " \n", + "else:\n", + "\n", + " with InstrumentResponse(irf_path) as irf: \n", + " \n", + " # Hypothetical spectrum\n", + " # This normalization corresponds to 1 ph/cm2/s between 50-300 keV\n", + " soft_spectrum = BandFunction._from_megalib(['BandFunction',10,10000,-1.9,-3.7,230],\"10.0\")\n", + " medium_spectrum = BandFunction._from_megalib(['BandFunction',10,10000,-1,-2.3,699.9],\"10.0\")\n", + " hard_spectrum = Comptonized._from_megalib(['Comptonized',10,10000,-0.5,1500],\"10.0\")\n", + " \n", + " # In this case we integrate the rate from all energy channels. \n", + " # You can subdivide the data into multiple energy channel groups\n", + " soft_local_loctable = LocalLocTable.from_irf(irf, soft_spectrum,energy_channels = 1) # [80,2000]\n", + " medium_local_loctable = LocalLocTable.from_irf(irf, medium_spectrum,energy_channels = 1)\n", + " hard_local_loctable = LocalLocTable.from_irf(irf, hard_spectrum,energy_channels = 1)\n", + " \n", + " # The local_loctable contains the expected rates in spacecraft coordinates\n", + " # We now need to use this to estimate the total expected counts in sky coordinate for\n", + " # the full duration of an event. \n", + " # In this case we simply have a 1 second event and specifying the attitude by a quaternion\n", + " # ([0,0,0,1] corresponds to the identity rotation). You can have multiple attitude-duration\n", + " # pairs to correctly model long duration events.\n", + " soft_sky_loctable = soft_local_loctable.to_skyloctable(attitude = [0,0,0,1], duration = 1)\n", + " medium_sky_loctable = medium_local_loctable.to_skyloctable(attitude = [0,0,0,1], duration = 1)\n", + " hard_sky_loctable = hard_local_loctable.to_skyloctable(attitude = [0,0,0,1], duration = 1)\n", + " \n", + " import pickle\n", + " \n", + " # Store the LUTS on file\n", + " with open(dir_path+'/soft_lut_' + run_name + '.pkl', 'wb') as f:\n", + " pickle.dump(soft_sky_loctable, f)\n", + " with open(dir_path+'/medium_lut_' + run_name + '.pkl', 'wb') as f:\n", + " pickle.dump(medium_sky_loctable, f)\n", + " with open(dir_path+'/hard_lut_' + run_name + '.pkl', 'wb') as f:\n", + " pickle.dump(hard_sky_loctable, f)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "29f1de1f-90ef-4e02-8092-8d48cec6c472", + "metadata": {}, + "outputs": [], + "source": [ + "hard_sky_loctable" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3ab587bb-637b-41ad-ab3a-ca4c1b83b605", + "metadata": {}, + "outputs": [], + "source": [ + "\n", + "def angular_distance(theta1, phi1, theta2, phi2):\n", + " \"\"\"\n", + " Compute the angular (great-circle) distance in degrees between two points\n", + " specified by (theta, phi) coordinates in degrees.\n", + "\n", + " Parameters:\n", + " theta1, phi1: floats, coordinates of the first point (degrees).\n", + " theta2, phi2: floats, coordinates of the second point (degrees).\n", + "\n", + " Returns:\n", + " angular_dist_deg: float, angular distance between the two points in degrees.\n", + " \"\"\"\n", + " # Adjust theta values by shifting -90\n", + " theta1 = 90 - theta1\n", + " theta2 = 90 - theta2\n", + "\n", + " # Convert angles to radians\n", + " theta1_rad = math.radians(theta1)\n", + " phi1_rad = math.radians(phi1)\n", + " theta2_rad = math.radians(theta2)\n", + " phi2_rad = math.radians(phi2)\n", + " \n", + " # Compute the difference in longitude\n", + " delta_phi = abs(phi1_rad - phi2_rad)\n", + "\n", + " # Apply spherical law of cosines\n", + " value = (math.sin(theta1_rad) * math.sin(theta2_rad) +\n", + " math.cos(theta1_rad) * math.cos(theta2_rad) * math.cos(delta_phi))\n", + " \n", + " # Clamp value to [-1, 1] to avoid floating point errors in acos\n", + " value_clamped = max(-1.0, min(1.0, value)) \n", + " \n", + " angular_dist = math.acos(value_clamped)\n", + " \n", + " # Convert angular distance from radians to degrees\n", + " angular_dist_deg = math.degrees(angular_dist)\n", + " \n", + " return angular_dist_deg" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2e60731d-564e-4dad-bfa1-b32280bfb9c8", + "metadata": {}, + "outputs": [], + "source": [ + "print(f\"Soft Look-up tables: {soft_sky_loctable.labels}\")\n", + "print(f\"Medium Look-up tables: {medium_sky_loctable.labels}\")\n", + "print(f\"Hard Look-up tables: {hard_sky_loctable.labels}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f5bada77-6eb1-4da4-aaf9-f05770d3f522", + "metadata": {}, + "outputs": [], + "source": [ + "medium_sky_loctable.get_expectation_map('BGO_Z1').data" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "60115384-7a38-49ee-8c73-e2689fa8d659", + "metadata": {}, + "outputs": [], + "source": [ + "if False:\n", + " # Store the expectation map to plot it with other tools\n", + " with open(dir_path+\"/exp_medium_Y1.pkl\", \"wb\") as f:\n", + " pickle.dump(medium_sky_loctable.get_expectation_map('BGO_Y1').data, f)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6b2916ee-b8ad-4136-8982-8881d2b6049b", + "metadata": {}, + "outputs": [], + "source": [ + "import healpy as hp\n", + "from astropy.coordinates import SkyCoord\n", + "import astropy.units as u\n", + "import numpy as np\n", + "\n", + "def get_coord_helpix(nside, pix_id):\n", + " \n", + " # Parameters\n", + " nside = 16 # Replace with your NSIDE value\n", + " ipix = 1 # Replace with the HEALPix pixel ID (nested scheme)\n", + " \n", + " # Get the angular coordinates (theta, phi) of the pixel center\n", + " theta, phi = hp.pix2ang(nside, ipix, nest=True)\n", + " \n", + " # Convert to equatorial coordinates (RA, Dec)\n", + " ra = phi * 180.0 / np.pi # phi is longitude in radians (RA)\n", + " dec = 90.0 - theta * 180.0 / np.pi # theta is colatitude (Dec)\n", + " \n", + " # Create the SkyCoord object\n", + " coord = SkyCoord(ra=ra * u.deg, dec=dec * u.deg, frame='icrs')\n", + "\n", + " return coord\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9c7a018f-c161-4517-bb66-15fa1531fe32", + "metadata": {}, + "outputs": [], + "source": [ + "hard_sky_loctable.get_expectation_map('BGO_Z1').plot()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "77beda1d-677a-43fc-a8b3-46dad6dcf9f5", + "metadata": {}, + "outputs": [], + "source": [ + "run_name_test = \"dataset\"\n", + "file_name_test = \"run57_mix_mega_shared\"\n", + "\n", + "file_path ='/data/analysis/'+run_name_test+'/'+file_name_test+'_dataset.pkl'\n", + "\n", + "# Load the array from the pickle file\n", + "with open(file_path, 'rb') as file:\n", + " loaded_array_test = pickle.load(file)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ea9d182b-a657-4350-afa7-719b92ff5b49", + "metadata": {}, + "outputs": [], + "source": [ + "loaded_array_test.shape" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "60717231-3723-4d30-8ae4-c73baf35dd96", + "metadata": {}, + "outputs": [], + "source": [ + "import numpy as np\n", + "\n", + "filter_flux = 1\n", + "filter_spectra = 1\n", + "\n", + "filtered_loaded_array_test = np.empty(loaded_array_test.shape[0], dtype=object)\n", + "\n", + "count = 0\n", + "\n", + "if filter_flux == 1:\n", + " for grb in loaded_array_test:\n", + " if grb['flux'] >14 and grb['flux'] <= 16:\n", + " filtered_loaded_array_test[count] = grb\n", + " count = count+1\n", + "\n", + " loaded_array_test = filtered_loaded_array_test[:count]\n", + "\n", + "filtered_loaded_array_test = np.empty(loaded_array_test.shape[0], dtype=object)\n", + "\n", + "count = 0\n", + "\n", + "if filter_spectra==1:\n", + " for grb in loaded_array_test:\n", + " #if grb['spectra'] == 'medium':\n", + " if '230' in grb['spectrum']:\n", + " filtered_loaded_array_test[count] = grb\n", + " count = count+1\n", + "\n", + " loaded_array_test = filtered_loaded_array_test[:count]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a98f9842-1a7a-4042-8a04-a7fc6f130c33", + "metadata": {}, + "outputs": [], + "source": [ + "loaded_array_test.shape" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "dfd1976a-255b-4e83-9a61-1a8c58cf3540", + "metadata": {}, + "outputs": [], + "source": [ + "test_dataset=loaded_array_test" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1d8a01c1-91a6-4b97-85ad-26478905d405", + "metadata": {}, + "outputs": [], + "source": [ + "len(test_dataset)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3cf7466e-e3f1-400e-8337-aad345f71d4f", + "metadata": {}, + "outputs": [], + "source": [ + "grb_test = test_dataset[0]\n", + "[grb_test['counts'][3],grb_test['counts'][2],grb_test['counts'][5],grb_test['counts'][4],grb_test['counts'][1],grb_test['counts'][0]]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "40ae36f6-eed7-453f-a144-3f01f5290852", + "metadata": {}, + "outputs": [], + "source": [ + "grb_test['coord']" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7efbb20c-a48e-4f9b-983a-de47bdb3d1c8", + "metadata": {}, + "outputs": [], + "source": [ + "grb_test" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d72db1c5-cbd1-430e-ba1e-b430377a0398", + "metadata": {}, + "outputs": [], + "source": [ + "process_lut_source(grb_test,None)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "67d32151-8127-45c5-a888-187dfaf18863", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "683a3d33-3dd3-49af-b5f6-7e13065894e7", + "metadata": {}, + "outputs": [], + "source": [ + "from bctools.loc import TSMap, NormLocLike\n", + "import astropy.units as u\n", + "from astropy.coordinates import SkyCoord\n", + "\n", + "b_sim = np.array([57.6053, 58.7157, 51.4131, 48.2891, 47.7293, 45.9617])\n", + "random_bkg = np.random.poisson(np.array([b_sim[3],b_sim[2],b_sim[5],b_sim[4],b_sim[1],b_sim[0]])*20)\n", + "\n", + "def process_lut_source(grb,random_bkg):\n", + " \n", + " theta_real = float(grb['coord'][0])\n", + " phi_real = float(grb['coord'][1])\n", + " s_counts = np.array([grb['counts'][3],grb['counts'][2],grb['counts'][5],grb['counts'][4],grb['counts'][1],grb['counts'][0]])\n", + " spectra_value = grb['spectrum']\n", + " \n", + " b_sim = np.array([57.6053, 58.7157, 51.4131, 48.2891, 47.7293, 45.9617])\n", + " random_bkg = np.random.poisson(np.array([b_sim[3],b_sim[2],b_sim[5],b_sim[4],b_sim[1],b_sim[0]])*20)\n", + "\n", + " b_counts = np.array([b_sim[3],b_sim[2],b_sim[5],b_sim[4],b_sim[1],b_sim[0]])*20\n", + " s_counts = s_counts+random_bkg\n", + " \n", + " soft_sqrt_ts, soft_ra_loc, soft_dec_loc, soft_cont_radius,soft_ts,soft_loc_tsvalue,soft_cont_area = localize_grb(soft_sky_loctable,s_counts, b_counts,theta_real,phi_real)\n", + " medium_sqrt_ts, medium_ra_loc, medium_dec_loc, medium_cont_radius,medium_ts,medium_loc_tsvalue,medium_cont_area = localize_grb(medium_sky_loctable,s_counts, b_counts,theta_real,phi_real)\n", + " hard_sqrt_ts, hard_ra_loc, hard_dec_loc, hard_cont_radius,hard_ts,hard_loc_tsvalue,hard_cont_area = localize_grb(hard_sky_loctable,s_counts, b_counts,theta_real,phi_real)\n", + "\n", + " max_ts = max(soft_sqrt_ts,medium_sqrt_ts,hard_sqrt_ts)\n", + " spectra_fitted = False\n", + " \n", + " if max_ts == soft_sqrt_ts:\n", + " if spectra_value == \"soft\":\n", + " spectra_fitted = True\n", + " original_ts_value = soft_loc_tsvalue\n", + " best_ts = soft_ts\n", + " ra_loc=soft_ra_loc\n", + " dec_loc=soft_dec_loc\n", + " cont_radius = soft_cont_radius\n", + " cont_area = soft_cont_area\n", + " \n", + " elif max_ts == medium_sqrt_ts:\n", + " if spectra_value == \"medium\":\n", + " spectra_fitted = True\n", + " original_ts_value = medium_loc_tsvalue\n", + " best_ts = medium_ts\n", + " ra_loc=medium_ra_loc\n", + " dec_loc=medium_dec_loc\n", + " cont_radius = medium_cont_radius\n", + " cont_area = medium_cont_area\n", + " \n", + " elif max_ts == hard_sqrt_ts:\n", + " if spectra_value == \"hard\":\n", + " spectra_fitted = True\n", + " original_ts_value = hard_loc_tsvalue\n", + " best_ts = hard_ts\n", + " ra_loc=hard_ra_loc\n", + " dec_loc=hard_dec_loc\n", + " cont_radius = hard_cont_radius\n", + " cont_area = hard_cont_area\n", + " else:\n", + " print(\"ts not found\")\n", + " print(grb)\n", + " \n", + " theta_loc, phi_loc = ra_dec_to_theta_phi(ra_loc, dec_loc)\n", + "\n", + " dist = angular_distance(theta_loc, phi_loc, theta_real, phi_real)\n", + " theta_dist = np.abs(theta_loc - theta_real)\n", + " phi_dist = diff_phi(phi_loc, phi_real)\n", + "\n", + " result = [theta_real, phi_real, theta_loc, phi_loc, dist, theta_dist, phi_dist, max_ts, cont_radius,best_ts,original_ts_value,cont_area,spectra_fitted]\n", + " \n", + " return result\n", + " \n", + "\n", + "\n", + "def localize_grb(sky_loctable,s,b,theta_real,phi_real):\n", + "\n", + " \n", + " sky_loctable.set_background(b)\n", + " sky_loctable.set_data(s)\n", + "\n", + " # Define a map of nside = 32. Note that this is a finer resolution\n", + " #that the underlying look-up table, which will be interpolated\n", + " ts = TSMap(nside = 64, coordsys = 'icrs')\n", + "\n", + " # NormLocLike is a subclass of LocLike and computes\n", + " # a Poisson likelihood for counting instruments. The\n", + " # overall normalization is the only free parameter\n", + " norm_likelihood = NormLocLike(sky_loctable)\n", + " \n", + " # Compute the TS map from one or more LocLikelihood\n", + " ts.compute(norm_likelihood)\n", + " #print(\"#\"+str(theta_real)+\"#\")\n", + " \n", + " # Correggi theta_real prima di convertirlo\n", + " if theta_real < 0:\n", + " print(f\"Invalid theta_real: {theta_real}\")\n", + " theta_real = 0\n", + " elif theta_real > 180:\n", + " print(f\"Invalid theta_real: {theta_real}\")\n", + " theta_real = 180\n", + " \n", + " theta_rad = np.deg2rad(theta_real)\n", + " phi_rad = np.deg2rad(phi_real)\n", + " \n", + " if theta_rad < 0 or theta_rad > np.pi:\n", + " print(f\"theta_rad fuori range: {theta_rad}\")\n", + "\n", + "\n", + " #print(dir(ts))\n", + " ipix = ts.ang2pix(theta_rad, phi_rad)\n", + "\n", + " # Get the value at the given coordinates\n", + " #ipix = ts.ang2pix(ts.nside, theta_real * u.deg.to(u.rad), phi_real * u.deg.to(u.rad))\n", + " original_ts_value = ts._data[ipix]\n", + "\n", + " confidence_levels = np.arange(0, 1.01, 0.01)\n", + " \n", + " containment_radius = np.array([\n", + " np.sqrt(ts.error_area(cont=cont) / np.pi).to(u.deg).value\n", + " for cont in confidence_levels\n", + " ])\n", + " \n", + " #original_ts_value = -1\n", + " cont_area = ts.error_area(cont = .9).to(u.deg**2).value\n", + "\n", + " \n", + "\n", + " return np.max(ts),ts.best_loc().ra.deg,ts.best_loc().dec.deg,containment_radius,ts,original_ts_value,cont_area\n", + "\n", + "def ra_dec_to_theta_phi(ra, dec):\n", + "\n", + " theta = 90-dec\n", + " \n", + " phi = ra\n", + " \n", + " return theta, phi\n", + "\n", + "def diff_phi(a1, a2):\n", + " # Calcola la differenza diretta\n", + " diff = abs(a1 - a2)\n", + " \n", + " # Trova il percorso più breve tenendo conto del ciclo degli angoli\n", + " if diff > 180:\n", + " diff = 360 - diff\n", + " \n", + " return diff" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2ec63933-6dc9-4846-af0c-ea1e46435c9b", + "metadata": {}, + "outputs": [], + "source": [ + "results_bkg = []\n", + "spectra_fitted = 0\n", + "\n", + "\n", + "def init_worker():\n", + " seed = int.from_bytes(os.urandom(4), \"little\")\n", + " np.random.seed(seed)\n", + "\n", + "\n", + "def process_in_parallel(test_dataset,random_bkg):\n", + " with multiprocessing.Pool(processes=200, initializer=init_worker) as pool:\n", + " results = pool.starmap(process_lut_source, zip(test_dataset, itertools.repeat(random_bkg)))\n", + " return results\n", + "\n", + "import math ,time\n", + "print(time.time())\n", + "results_bkg = process_in_parallel(test_dataset,random_bkg)\n", + "print(time.time())" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "538bf57b-8e53-4a6b-9d02-cff712dacd60", + "metadata": {}, + "outputs": [], + "source": [ + "distances = []\n", + "theta_distances = []\n", + "phi_distances = []\n", + "cont_radius_list = []\n", + "cont_area_list = []\n", + "for res in results_bkg:\n", + " distances.append(res[4])\n", + " theta_distances.append(res[5])\n", + " phi_distances.append(res[6])\n", + " cont_radius_list.append(res[8])\n", + " cont_area_list.append(res[11])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5d73bc87-526d-4899-a561-085e65143101", + "metadata": {}, + "outputs": [], + "source": [ + "print(np.mean(distances))\n", + "print(np.mean(cont_area_list))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3c67b8d5", + "metadata": {}, + "outputs": [], + "source": [ + "import numpy as np\n", + "import healpy as hp\n", + "import matplotlib.pyplot as plt\n", + "\n", + "nside = 32\n", + "npix = hp.nside2npix(nside)\n", + "\n", + "\n", + "m = np.array(cont_area_list)\n", + "\n", + "hp.projview(\n", + " m,\n", + " coord=[\"G\"],\n", + " graticule=True,\n", + " graticule_labels=True,\n", + " unit=\"deg2\",\n", + " xlabel=\"longitude\",\n", + " ylabel=\"latitude\",\n", + " cb_orientation=\"vertical\",\n", + " latitude_grid_spacing=30,\n", + " projection_type=\"aitoff\",\n", + " title=\"Aitoff projection\",\n", + " cmap=\"turbo\",\n", + " nest=True\n", + ")\n", + "\n", + "plt.show()\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "316721ab-65f6-42bb-8eff-dd3c86b303fc", + "metadata": {}, + "outputs": [], + "source": [ + "import pickle\n", + "if False:\n", + "\n", + " # Salva l'array in un file usando pickle\n", + " with open(dir_path+\"/bc_\"+file_name_test+\"_distall.pkl\", \"wb\") as f:\n", + " pickle.dump(distances, f)\n", + " \n", + " # Salva l'array in un file usando pickle\n", + " with open(dir_path+\"/bc_\"+file_name_test+\"_cont_area.pkl\", \"wb\") as f:\n", + " pickle.dump(cont_area_list, f)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9abbc20b-d55c-408d-9241-4375d67c5088", + "metadata": {}, + "outputs": [], + "source": [ + "test_labels = []\n", + "for grb in test_dataset:\n", + " theta_real = float(grb['coord'][0])\n", + " phi_real = float(grb['coord'][1])\n", + " test_labels.append([theta_real,phi_real])\n", + "\n", + "test_labels = np.array(test_labels)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "12321b73-8a14-4ff0-bdfb-6dd643a7f7e5", + "metadata": {}, + "outputs": [], + "source": [ + "print(np.mean(distances))\n", + "print(np.mean(theta_distances))\n", + "print(np.mean(phi_distances))\n", + "print(np.mean(cont_area_list))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "554d3620-24a6-4d45-9fb2-a4134feb0433", + "metadata": {}, + "outputs": [], + "source": [ + "# Define the 5-degree intervals\n", + "intervals = np.arange(0, 185, 5)\n", + "\n", + "# Group counts based on 5-degree intervals\n", + "grouped_counts = np.zeros((len(intervals)))\n", + "counts = np.zeros((len(intervals)))\n", + "\n", + "tot_count = 0\n", + "for theta, phi, count in zip(test_labels[:,0], test_labels[:,1], cont_area_list):\n", + " \n", + " if(True): #(phi>125 and phi<145) or \n", + " tot_count += 1\n", + " interval_index = int(theta // 5)\n", + " grouped_counts[interval_index] = grouped_counts[interval_index] + count\n", + " counts[interval_index] += 1\n", + "\n", + "# Compute the average counts in each interval\n", + "grouped_counts = grouped_counts[:tot_count]\n", + "counts = counts[:tot_count]\n", + "\n", + "# Display the histogram\n", + "plt.figure(figsize=(10, 6))\n", + "plt.bar(intervals[:-1], grouped_counts[:-1]/counts[:-1], width=5, align='edge', alpha=1, label=\"loc. error\")\n", + "plt.xlabel('Theta (°)')\n", + "plt.ylabel('Loc. error')\n", + "plt.title('Loc. error as a function of theta')\n", + "plt.grid(True)\n", + "plt.legend()\n", + "plt.show()\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1ca8dfdb-b883-4c88-a212-59c13b5c1e95", + "metadata": {}, + "outputs": [], + "source": [ + "step = 10\n", + "\n", + "# Define the 5-degree intervals\n", + "intervals = np.arange(0, 365, step)\n", + "\n", + "# Group counts based on 5-degree intervals\n", + "grouped_counts_1 = np.zeros((len(intervals)))\n", + "counts_1 = np.zeros((len(intervals)))\n", + "\n", + "grouped_counts_2 = np.zeros((len(intervals)))\n", + "counts_2 = np.zeros((len(intervals)))\n", + "\n", + "grouped_counts_3 = np.zeros((len(intervals)))\n", + "counts_3 = np.zeros((len(intervals)))\n", + "\n", + "grouped_counts_4 = np.zeros((len(intervals)))\n", + "counts_4 = np.zeros((len(intervals)))\n", + "\n", + "total_count_1 = 0\n", + "total_count_2 = 0\n", + "total_count_3 = 0\n", + "total_count_4 = 0\n", + "\n", + "for theta, phi, count in zip(test_labels[:,0], test_labels[:,1], distances):\n", + " \n", + " if(theta > 20 and theta < 55):\n", + " total_count_1 += 1\n", + " interval_index = int(phi // step)\n", + " grouped_counts_1[interval_index] = grouped_counts_1[interval_index] + count\n", + " counts_1[interval_index] += 1\n", + "\n", + " if(theta > 55 and theta < 150):\n", + " total_count_2 += 1\n", + " interval_index = int(phi // step)\n", + " grouped_counts_2[interval_index] = grouped_counts_2[interval_index] + count\n", + " counts_2[interval_index] += 1\n", + "\n", + " if(theta > 150 and theta < 180):\n", + " total_count_3 += 1\n", + " interval_index = int(phi // step)\n", + " grouped_counts_3[interval_index] = grouped_counts_3[interval_index] + count\n", + " counts_3[interval_index] += 1\n", + "\n", + "# Compute the average counts in each interval\n", + "grouped_counts_1 = grouped_counts_1[:total_count_1]\n", + "counts_1 = counts_1[:total_count_1]\n", + "\n", + "grouped_counts_2 = grouped_counts_2[:total_count_2]\n", + "counts_2 = counts_2[:total_count_2]\n", + "\n", + "grouped_counts_3 = grouped_counts_3[:total_count_3]\n", + "counts_3 = counts_3[:total_count_3]\n", + "\n", + "# Display the histogram\n", + "plt.figure(figsize=(10, 6))\n", + "plt.bar(intervals[:-1], grouped_counts_1[:-1]/counts_1[:-1], width=step, align='edge', alpha=0.5, label=\"loc. error theta=[20°,55°]\")\n", + "\n", + "plt.bar(intervals[:-1], grouped_counts_2[:-1]/counts_2[:-1], width=step, align='edge', alpha=0.5, label=\"loc. error theta=[55°,150°]\")\n", + "\n", + "plt.bar(intervals[:-1], grouped_counts_3[:-1]/counts_3[:-1], width=step, align='edge', alpha=0.5, label=\"loc. error theta=[150°,180°]\")\n", + "\n", + "plt.xlabel('Phi (°)')\n", + "plt.ylabel('Loc. error')\n", + "plt.title('Loc. error as a function of phi')\n", + "plt.grid(True)\n", + "plt.legend()\n", + "plt.show()\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0d1ff2c8-fd08-47bc-98ee-0d9acc9e9a77", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2d0b3a49-3fa5-4fb5-8324-c9b0934a3291", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "bct", + "language": "python", + "name": "bct" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.12" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/cosipy/notebooks/bc-tools/performance_evaluation/bctools_plot.ipynb b/cosipy/notebooks/bc-tools/performance_evaluation/bctools_plot.ipynb new file mode 100644 index 0000000..1af7062 --- /dev/null +++ b/cosipy/notebooks/bc-tools/performance_evaluation/bctools_plot.ipynb @@ -0,0 +1,608 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "ba72606f-02ac-4c3a-bba3-d9b79ea52966", + "metadata": {}, + "outputs": [], + "source": [ + "from bctools.io import InstrumentResponse\n", + "from bctools.loc import LocalLocTable\n", + "from bctools.spectra.spectrum import BandFunction,Comptonized\n", + "import os\n", + "import math\n", + "import multiprocessing\n", + "import itertools\n", + "import pickle \n", + "import math\n", + "import numpy as np\n", + "import itertools\n", + "from bctools.loc import TSMap, NormLocLike\n", + "import astropy.units as u\n", + "from astropy.coordinates import SkyCoord\n", + "import matplotlib.pyplot as plt\n", + "import healpy as hp\n", + "\n", + "run_name = \"run10\"\n", + "irf_path = \"/data/models/irf_summed_\"+run_name+\".h5\"\n", + "dir_path = \"/data/test_newrepo/\"\n", + "\n", + "## The plot is generated using the mean spectrum dataset and mean spectrum LUT." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6a18cd09-5f40-4911-9f9f-028bc62b06d7", + "metadata": {}, + "outputs": [], + "source": [ + "load_from_file = True\n", + "\n", + "# The code is inspired by the bc-tools tutorial.\n", + "if load_from_file:\n", + "\n", + " with open('/data/models/LUTS/medium_lut_' + run_name + '.pkl', 'rb') as f:\n", + " medium_sky_loctable = pickle.load(f)\n", + " \n", + "else:\n", + "\n", + " with InstrumentResponse(irf_path) as irf: \n", + " \n", + " # Hypothetical spectrum\n", + " # This normalization corresponds to 1 ph/cm2/s between 50-300 keV\n", + " #spectrum = PowerLaw(60,2)\n", + " #spectrum = PowerLaw._from_megalib(['PowerLaw',10,10000,2],\"5.0\")\n", + "\n", + " medium_spectrum = BandFunction._from_megalib(['BandFunction',10,10000,-1,-2.3,699.9],\"10.0\")\n", + " \n", + " # In this case we integrate the rate from all energy channels. \n", + " # You can subdivide the data into multiple energy channel groups\n", + "\n", + " medium_local_loctable = LocalLocTable.from_irf(irf, medium_spectrum,energy_channels = 1)\n", + " \n", + " \n", + " # The local_loctable contains the expected rates in spacecraft coordinates\n", + " # We now need to use this to estimate the total expected counts in sky coordinate for\n", + " # the full duration of an event. \n", + " # In this case we simply have a 1 second event and specifying the attitude by a quaternion\n", + " # ([0,0,0,1] corresponds to the identity rotation). You can have multiple attitude-duration\n", + " # pairs to correctly model long duration events.\n", + " medium_sky_loctable = medium_local_loctable.to_skyloctable(attitude = [0,0,0,1], duration = 1)\n", + " \n", + " #Store LUT\n", + " \n", + " # Salvataggio su file\n", + " with open('/data/models/LUTS/medium_lut_'+run_name+'.pkl', 'wb') as f:\n", + " pickle.dump(medium_sky_loctable, f)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3ab587bb-637b-41ad-ab3a-ca4c1b83b605", + "metadata": {}, + "outputs": [], + "source": [ + "\n", + "def angular_distance(theta1, phi1, theta2, phi2):\n", + " \"\"\"\n", + " Compute the angular (great-circle) distance in degrees between two points\n", + " specified by (theta, phi) coordinates in degrees.\n", + "\n", + " Parameters:\n", + " theta1, phi1: floats, coordinates of the first point (degrees).\n", + " theta2, phi2: floats, coordinates of the second point (degrees).\n", + "\n", + " Returns:\n", + " angular_dist_deg: float, angular distance between the two points in degrees.\n", + " \"\"\"\n", + " # Adjust theta values by shifting -90\n", + " theta1 = 90 - theta1\n", + " theta2 = 90 - theta2\n", + "\n", + " # Convert angles to radians\n", + " theta1_rad = math.radians(theta1)\n", + " phi1_rad = math.radians(phi1)\n", + " theta2_rad = math.radians(theta2)\n", + " phi2_rad = math.radians(phi2)\n", + " \n", + " # Compute the difference in longitude\n", + " delta_phi = abs(phi1_rad - phi2_rad)\n", + "\n", + " # Apply spherical law of cosines\n", + " value = (math.sin(theta1_rad) * math.sin(theta2_rad) +\n", + " math.cos(theta1_rad) * math.cos(theta2_rad) * math.cos(delta_phi))\n", + " \n", + " # Clamp value to [-1, 1] to avoid floating point errors in acos\n", + " value_clamped = max(-1.0, min(1.0, value)) \n", + " \n", + " angular_dist = math.acos(value_clamped)\n", + " \n", + " # Convert angular distance from radians to degrees\n", + " angular_dist_deg = math.degrees(angular_dist)\n", + " \n", + " return angular_dist_deg" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2e60731d-564e-4dad-bfa1-b32280bfb9c8", + "metadata": {}, + "outputs": [], + "source": [ + "print(f\"Medium Look-up tables: {medium_sky_loctable.labels}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f5bada77-6eb1-4da4-aaf9-f05770d3f522", + "metadata": {}, + "outputs": [], + "source": [ + "medium_sky_loctable.get_expectation_map('BGO_Z1').data" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "60115384-7a38-49ee-8c73-e2689fa8d659", + "metadata": {}, + "outputs": [], + "source": [ + "import pickle\n", + "if False:\n", + "\n", + " # Salva l'array in un file usando pickle\n", + " with open(\"/data/exp_medium_Y1.pkl\", \"wb\") as f:\n", + " pickle.dump(medium_sky_loctable.get_expectation_map('BGO_Y1').data, f)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6b2916ee-b8ad-4136-8982-8881d2b6049b", + "metadata": {}, + "outputs": [], + "source": [ + "import healpy as hp\n", + "from astropy.coordinates import SkyCoord\n", + "import astropy.units as u\n", + "import numpy as np\n", + "\n", + "def get_coord_helpix(nside, pix_id):\n", + " \n", + " # Parameters\n", + " nside = 16 # Replace with your NSIDE value\n", + " ipix = 1 # Replace with the HEALPix pixel ID (nested scheme)\n", + " \n", + " # Get the angular coordinates (theta, phi) of the pixel center\n", + " theta, phi = hp.pix2ang(nside, ipix, nest=True)\n", + " \n", + " # Convert to equatorial coordinates (RA, Dec)\n", + " ra = phi * 180.0 / np.pi # phi is longitude in radians (RA)\n", + " dec = 90.0 - theta * 180.0 / np.pi # theta is colatitude (Dec)\n", + " \n", + " # Create the SkyCoord object\n", + " coord = SkyCoord(ra=ra * u.deg, dec=dec * u.deg, frame='icrs')\n", + "\n", + " return coord\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9c7a018f-c161-4517-bb66-15fa1531fe32", + "metadata": {}, + "outputs": [], + "source": [ + "medium_sky_loctable.get_expectation_map('BGO_Z1').plot()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "77beda1d-677a-43fc-a8b3-46dad6dcf9f5", + "metadata": {}, + "outputs": [], + "source": [ + "#test with LUTs\n", + "import pickle \n", + "shared = True\n", + "numpy_file = 0\n", + "\n", + "run_name_test = \"dataset\"\n", + "file_name_test = \"run57_mix_mega_shared\"\n", + "\n", + "file_path ='/data/analysis/'+run_name_test+'/'+file_name_test+'_dataset.pkl'\n", + "\n", + "# Load the array from the pickle file\n", + "with open(file_path, 'rb') as file:\n", + " loaded_array_test = pickle.load(file)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ea9d182b-a657-4350-afa7-719b92ff5b49", + "metadata": {}, + "outputs": [], + "source": [ + "loaded_array_test.shape" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d45aa57c-b774-4dde-be83-79e8a26c8b55", + "metadata": {}, + "outputs": [], + "source": [ + "import numpy as np\n", + "counts_array = []\n", + "for grb in loaded_array_test:\n", + " counts_array.append(grb['counts'])\n", + "counts_array_np = np.array(counts_array)\n", + "print(counts_array_np.shape)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "40a301da-71ca-460f-86bc-343f734b317f", + "metadata": {}, + "outputs": [], + "source": [ + "np.sum(counts_array_np[:,0])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c0f2263c-327c-497c-8843-225d2a1ee4f0", + "metadata": {}, + "outputs": [], + "source": [ + "plt.hist(counts_array_np[:,0])\n", + "plt.title(\"Counts Hist. for Z1 panel\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "60717231-3723-4d30-8ae4-c73baf35dd96", + "metadata": {}, + "outputs": [], + "source": [ + "import numpy as np\n", + "\n", + "filter_flux = 0\n", + "filter_spectra = 0\n", + "\n", + "filtered_loaded_array_test = np.empty(loaded_array_test.shape[0], dtype=object)\n", + "\n", + "count = 0\n", + "\n", + "if filter_flux == 1:\n", + " for grb in loaded_array_test:\n", + " if grb['flux'] > 10 and grb['flux'] <= 15:\n", + " filtered_loaded_array_test[count] = grb\n", + " count = count+1\n", + "\n", + " loaded_array_test = filtered_loaded_array_test[:count]\n", + "\n", + "filtered_loaded_array_test = np.empty(loaded_array_test.shape[0], dtype=object)\n", + "\n", + "count = 0\n", + "\n", + "if filter_spectra==1:\n", + " for grb in loaded_array_test:\n", + " #if grb['spectra'] == 'medium':\n", + " if '1500' in grb['spectrum']:\n", + " filtered_loaded_array_test[count] = grb\n", + " count = count+1\n", + "\n", + " loaded_array_test = filtered_loaded_array_test[:count]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a98f9842-1a7a-4042-8a04-a7fc6f130c33", + "metadata": {}, + "outputs": [], + "source": [ + "loaded_array_test.shape\n", + "test_dataset=loaded_array_test" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "683a3d33-3dd3-49af-b5f6-7e13065894e7", + "metadata": {}, + "outputs": [], + "source": [ + "b_sim = np.array([57.6053, 58.7157, 51.4131, 48.2891, 47.7293, 45.9617])\n", + "\n", + "def process_lut_source(grb):\n", + " \n", + " theta_real = float(grb['coord'][0])\n", + " phi_real = float(grb['coord'][1])\n", + " counts = grb['counts']\n", + " s_counts = np.array([grb['counts'][3],grb['counts'][2],grb['counts'][5],grb['counts'][4],grb['counts'][1],grb['counts'][0]])\n", + " b_counts = [1,1,1,1,1,1]\n", + " spectra_value = grb['spectrum']\n", + "\n", + " coord_grb = grb['coord']\n", + "\n", + " \n", + " ra = float(coord_grb[1])\n", + " dec = 90.0 - float(coord_grb[0])\n", + " \n", + " \n", + " coord = SkyCoord(ra=ra * u.deg, dec=dec * u.deg, frame='icrs')\n", + "\n", + " #test LUT\n", + " # The source counts are the sum of the expected counts in the LUT and the background counts.\n", + " s_counts = medium_sky_loctable.get_expectation(coord) + np.array([b_sim[3],b_sim[2],b_sim[5],b_sim[4],b_sim[1],b_sim[0]])*20\n", + " b_counts = np.array([b_sim[3],b_sim[2],b_sim[5],b_sim[4],b_sim[1],b_sim[0]])*20\n", + " medium_sqrt_ts, mdium_ra_loc, medium_dec_loc, medium_cont_radius,medium_ts,medium_loc_tsvalue,medium_cont_area = localize_grb(medium_sky_loctable,s_counts, b_counts,theta_real,phi_real)\n", + " \n", + " max_ts = medium_sqrt_ts\n", + " \n", + " if max_ts == medium_sqrt_ts:\n", + " original_ts_value = medium_loc_tsvalue\n", + " best_ts = medium_ts\n", + " ra_loc=mdium_ra_loc\n", + " dec_loc=medium_dec_loc\n", + " cont_radius = medium_cont_radius\n", + " cont_area = medium_cont_area\n", + " \n", + " theta_loc, phi_loc = ra_dec_to_theta_phi(ra_loc, dec_loc)\n", + "\n", + " dist = angular_distance(theta_loc, phi_loc, theta_real, phi_real)\n", + " theta_dist = np.abs(theta_loc - theta_real)\n", + " phi_dist = diff_phi(phi_loc, phi_real)\n", + "\n", + " result = [theta_real, phi_real, theta_loc, phi_loc, dist, theta_dist, phi_dist, max_ts, cont_radius,best_ts,original_ts_value,cont_area]\n", + " \n", + " return result\n", + "\n", + "def localize_grb(sky_table, signal, background, theta_deg, phi_deg):\n", + " # Set background and signal data into the sky table\n", + " sky_table.set_background(background)\n", + " sky_table.set_data(signal)\n", + "\n", + " # Define a TS map with nside = 32 (finer resolution than the lookup table)\n", + " ts_map = TSMap(nside=32, coordsys='icrs')\n", + "\n", + " # Create a normalized likelihood (Poisson likelihood for counting instruments)\n", + " # The only free parameter is the overall normalization\n", + " norm_likelihood = NormLocLike(sky_table)\n", + " \n", + " # Compute the TS map from the likelihood\n", + " ts_map.compute(norm_likelihood)\n", + " \n", + " # Validate theta before converting to radians\n", + " if theta_deg < 0:\n", + " print(f\"Invalid theta value: {theta_deg}\")\n", + " theta_deg = 0\n", + " elif theta_deg > 180:\n", + " print(f\"Invalid theta value: {theta_deg}\")\n", + " theta_deg = 180\n", + " \n", + " # Convert degrees → radians\n", + " theta_rad = np.deg2rad(theta_deg)\n", + " phi_rad = np.deg2rad(phi_deg)\n", + " \n", + " # Check if theta is in a valid radian range\n", + " if theta_rad < 0 or theta_rad > np.pi:\n", + " print(f\"theta_rad out of range: {theta_rad}\")\n", + "\n", + " # Get the pixel index corresponding to the given (theta, phi) coordinates\n", + " pixel_index = ts_map.ang2pix(theta_rad, phi_rad)\n", + "\n", + " # Extract the TS value at the given pixel\n", + " ts_value_at_coords = ts_map._data[pixel_index]\n", + "\n", + " # Define an array of confidence levels (0% → 100%)\n", + " confidence_levels = np.arange(0, 1.01, 0.01)\n", + " \n", + " # Compute containment radii for each confidence level\n", + " containment_radii = np.array([\n", + " np.sqrt(ts_map.error_area(cont=cont) / np.pi).to(u.deg).value\n", + " for cont in confidence_levels\n", + " ])\n", + " \n", + " # Compute containment area at 90% confidence\n", + " containment_area_90 = ts_map.error_area(cont=0.9).to(u.deg**2).value\n", + "\n", + " # Return:\n", + " # - maximum TS value\n", + " # - RA and Dec of the best localization (in degrees)\n", + " # - containment radii\n", + " # - ts_map object\n", + " # - TS value at input coordinates\n", + " # - containment area at 90% confidence\n", + " return (\n", + " np.max(ts_map),\n", + " ts_map.best_loc().ra.deg,\n", + " ts_map.best_loc().dec.deg,\n", + " containment_radii,\n", + " ts_map,\n", + " ts_value_at_coords,\n", + " containment_area_90\n", + " )\n", + "\n", + "\n", + "def ra_dec_to_theta_phi(ra, dec):\n", + "\n", + " theta = 90-dec\n", + " phi = ra\n", + " \n", + " return theta, phi\n", + "\n", + "def diff_phi(a1, a2):\n", + " \n", + " diff = abs(a1 - a2)\n", + " if diff > 180:\n", + " diff = 360 - diff\n", + " \n", + " return diff" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2ec63933-6dc9-4846-af0c-ea1e46435c9b", + "metadata": {}, + "outputs": [], + "source": [ + "results_bkg = []\n", + "spectra_fitted = 0\n", + "\n", + "def init_worker():\n", + " seed = int.from_bytes(os.urandom(4), \"little\")\n", + " np.random.seed(seed)\n", + "\n", + "def process_in_parallel(test_dataset):\n", + " with multiprocessing.Pool(processes=200, initializer=init_worker) as pool:\n", + " results = pool.starmap(process_lut_source, zip(test_dataset))\n", + " return results\n", + "\n", + "results_bkg = process_in_parallel(test_dataset)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "538bf57b-8e53-4a6b-9d02-cff712dacd60", + "metadata": {}, + "outputs": [], + "source": [ + "distances = []\n", + "theta_distances = []\n", + "phi_distances = []\n", + "cont_radius_list = []\n", + "cont_area_list = []\n", + "for res in results_bkg:\n", + " distances.append(res[4])\n", + " theta_distances.append(res[5])\n", + " phi_distances.append(res[6])\n", + " cont_radius_list.append(res[8])\n", + " cont_area_list.append(res[11])\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5d73bc87-526d-4899-a561-085e65143101", + "metadata": {}, + "outputs": [], + "source": [ + "\n", + "hp.projview(\n", + " np.array(cont_area_list),\n", + " coord=[\"G\"],\n", + " projection_type=\"aitoff\", # Cambiato da \"mollweide\" a \"aitoff\"\n", + " graticule=True,\n", + " graticule_labels=True,\n", + " longitude_grid_spacing=60,\n", + " title=file,\n", + " latitude_grid_spacing=30,\n", + " cmap=\"turbo\",\n", + " nest=True,\n", + " unit=\"\", \n", + " fontsize={\n", + " \"xlabel\": 14,\n", + " \"ylabel\": 14,\n", + " \"title\": 16,\n", + " \"xtick_label\": 14,\n", + " \"ytick_label\": 14,\n", + " \"cbar_label\": 14,\n", + " \"cbar_tick_label\": 14 # qui imposti il font size dei numeri della colorbar\n", + " },\n", + " override_plot_properties={\n", + " \"cbar_shrink\": 0.8,\n", + " \"cbar_pad\": 0.05,\n", + " \"cbar_label_pad\": 5\n", + " }\n", + " \n", + ")\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "316721ab-65f6-42bb-8eff-dd3c86b303fc", + "metadata": {}, + "outputs": [], + "source": [ + "import pickle\n", + "if False:\n", + "\n", + " with open(\"/data/bc_\"+file_name_test+\"_distall_plot.pkl\", \"wb\") as f:\n", + " pickle.dump(distances, f)\n", + " \n", + " with open(\"/data/bc_\"+file_name_test+\"_cont_area_plot.pkl\", \"wb\") as f:\n", + " pickle.dump(cont_area_list, f)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "28770484-f4d1-450f-981b-50d2fca8c023", + "metadata": {}, + "outputs": [], + "source": [ + "print(np.mean(distances))\n", + "print(np.mean(theta_distances))\n", + "print(np.mean(phi_distances))\n", + "print(np.mean(cont_area_list))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4eafe9cb-63a7-49ca-8e1a-06e00d8e17c5", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6beeee44-7cfc-44ab-ad3e-ad060c0df3bb", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "bct", + "language": "python", + "name": "bct" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.12" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/cosipy/notebooks/bc-tools/performance_evaluation/bctools_plot_augmented_dataset.ipynb b/cosipy/notebooks/bc-tools/performance_evaluation/bctools_plot_augmented_dataset.ipynb new file mode 100644 index 0000000..4312124 --- /dev/null +++ b/cosipy/notebooks/bc-tools/performance_evaluation/bctools_plot_augmented_dataset.ipynb @@ -0,0 +1,626 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "ba72606f-02ac-4c3a-bba3-d9b79ea52966", + "metadata": {}, + "outputs": [], + "source": [ + "from bctools.io import InstrumentResponse\n", + "from bctools.loc import LocalLocTable\n", + "from bctools.spectra.spectrum import BandFunction,Comptonized\n", + "import os\n", + "import math\n", + "import multiprocessing\n", + "import itertools\n", + "import pickle \n", + "import numpy as np\n", + "from bctools.loc import TSMap, NormLocLike\n", + "import astropy.units as u\n", + "from astropy.coordinates import SkyCoord\n", + "import matplotlib.pyplot as plt\n", + "import healpy as hp\n", + "import time\n", + "\n", + "run_name = \"run10\"\n", + "irf_path = \"/data/models/irf_summed_\"+run_name+\".h5\"\n", + "dir_path = \"/data/test_newrepo/\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6a18cd09-5f40-4911-9f9f-028bc62b06d7", + "metadata": {}, + "outputs": [], + "source": [ + "# Convolve an full instrument response with a hypothetical spectrum\n", + "#irf_path = \"/data/models/irf_summed.h5\"\n", + "#irf_path = \"/data/models/run_1_noeffect_irf_summed.h5\"\n", + "import pickle\n", + "\n", + "load_from_file = True\n", + "\n", + "if load_from_file:\n", + "\n", + " with open('/data/models/LUTS/soft_lut_' + run_name + '.pkl', 'rb') as f:\n", + " soft_sky_loctable = pickle.load(f)\n", + " \n", + " with open('/data/models/LUTS/medium_lut_' + run_name + '.pkl', 'rb') as f:\n", + " medium_sky_loctable = pickle.load(f)\n", + " \n", + " with open('/data/models/LUTS/hard_lut_' + run_name + '.pkl', 'rb') as f:\n", + " hard_sky_loctable = pickle.load(f)\n", + " \n", + "else:\n", + "\n", + " with InstrumentResponse(irf_path) as irf: \n", + " \n", + " # Hypothetical spectrum\n", + " # This normalization corresponds to 1 ph/cm2/s between 50-300 keV\n", + " #spectrum = PowerLaw(60,2)\n", + " #spectrum = PowerLaw._from_megalib(['PowerLaw',10,10000,2],\"5.0\")\n", + " soft_spectrum = BandFunction._from_megalib(['BandFunction',10,10000,-1.9,-3.7,230],\"10.0\")\n", + " medium_spectrum = BandFunction._from_megalib(['BandFunction',10,10000,-1,-2.3,699.9],\"10.0\")\n", + " hard_spectrum = Comptonized._from_megalib(['Comptonized',10,10000,-0.5,1500],\"10.0\")\n", + " \n", + " # In this case we integrate the rate from all energy channels. \n", + " # You can subdivide the data into multiple energy channel groups\n", + " soft_local_loctable = LocalLocTable.from_irf(irf, soft_spectrum,energy_channels = 1) # [80,2000]\n", + " medium_local_loctable = LocalLocTable.from_irf(irf, medium_spectrum,energy_channels = 1)\n", + " hard_local_loctable = LocalLocTable.from_irf(irf, hard_spectrum,energy_channels = 1)\n", + " \n", + " # The local_loctable contains the expected rates in spacecraft coordinates\n", + " # We now need to use this to estimate the total expected counts in sky coordinate for\n", + " # the full duration of an event. \n", + " # In this case we simply have a 1 second event and specifying the attitude by a quaternion\n", + " # ([0,0,0,1] corresponds to the identity rotation). You can have multiple attitude-duration\n", + " # pairs to correctly model long duration events.\n", + " soft_sky_loctable = soft_local_loctable.to_skyloctable(attitude = [0,0,0,1], duration = 1)\n", + " medium_sky_loctable = medium_local_loctable.to_skyloctable(attitude = [0,0,0,1], duration = 1)\n", + " hard_sky_loctable = hard_local_loctable.to_skyloctable(attitude = [0,0,0,1], duration = 1)\n", + " \n", + " #Store LUT\n", + " \n", + " import pickle\n", + " \n", + " # Salvataggio su file\n", + " with open('/data/models/LUTS/soft_lut_'+run_name+'.pkl', 'wb') as f:\n", + " pickle.dump(soft_sky_loctable, f)\n", + " with open('/data/models/LUTS/medium_lut_'+run_name+'.pkl', 'wb') as f:\n", + " pickle.dump(medium_sky_loctable, f)\n", + " with open('/data/models/LUTS/hard_lut_'+run_name+'.pkl', 'wb') as f:\n", + " pickle.dump(hard_sky_loctable, f)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3ab587bb-637b-41ad-ab3a-ca4c1b83b605", + "metadata": {}, + "outputs": [], + "source": [ + "\n", + "def angular_distance(theta1, phi1, theta2, phi2):\n", + " \"\"\"\n", + " Compute the angular (great-circle) distance in degrees between two points\n", + " specified by (theta, phi) coordinates in degrees.\n", + "\n", + " Parameters:\n", + " theta1, phi1: floats, coordinates of the first point (degrees).\n", + " theta2, phi2: floats, coordinates of the second point (degrees).\n", + "\n", + " Returns:\n", + " angular_dist_deg: float, angular distance between the two points in degrees.\n", + " \"\"\"\n", + " # Adjust theta values by shifting -90\n", + " theta1 = 90 - theta1\n", + " theta2 = 90 - theta2\n", + "\n", + " # Convert angles to radians\n", + " theta1_rad = math.radians(theta1)\n", + " phi1_rad = math.radians(phi1)\n", + " theta2_rad = math.radians(theta2)\n", + " phi2_rad = math.radians(phi2)\n", + " \n", + " # Compute the difference in longitude\n", + " delta_phi = abs(phi1_rad - phi2_rad)\n", + "\n", + " # Apply spherical law of cosines\n", + " value = (math.sin(theta1_rad) * math.sin(theta2_rad) +\n", + " math.cos(theta1_rad) * math.cos(theta2_rad) * math.cos(delta_phi))\n", + " \n", + " # Clamp value to [-1, 1] to avoid floating point errors in acos\n", + " value_clamped = max(-1.0, min(1.0, value)) \n", + " \n", + " angular_dist = math.acos(value_clamped)\n", + " \n", + " # Convert angular distance from radians to degrees\n", + " angular_dist_deg = math.degrees(angular_dist)\n", + " \n", + " return angular_dist_deg" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2e60731d-564e-4dad-bfa1-b32280bfb9c8", + "metadata": {}, + "outputs": [], + "source": [ + "print(f\"Soft Look-up tables: {soft_sky_loctable.labels}\")\n", + "print(f\"Medium Look-up tables: {medium_sky_loctable.labels}\")\n", + "print(f\"Hard Look-up tables: {hard_sky_loctable.labels}\")\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f5bada77-6eb1-4da4-aaf9-f05770d3f522", + "metadata": {}, + "outputs": [], + "source": [ + "medium_sky_loctable.get_expectation_map('BGO_Z1').data" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "60115384-7a38-49ee-8c73-e2689fa8d659", + "metadata": {}, + "outputs": [], + "source": [ + "import pickle\n", + "if False:\n", + "\n", + " # Salva l'array in un file usando pickle\n", + " with open(\"/data/exp_medium_Y1.pkl\", \"wb\") as f:\n", + " pickle.dump(medium_sky_loctable.get_expectation_map('BGO_Y1').data, f)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6b2916ee-b8ad-4136-8982-8881d2b6049b", + "metadata": {}, + "outputs": [], + "source": [ + "\n", + "def get_coord_helpix(nside,pix_id):\n", + " \n", + " # Parametri\n", + " nside = 16 # Sostituisci con il tuo valore di NSIDE\n", + " ipix = 1 # Sostituisci con l'ID del pixel HEALPix (in modalità nested)\n", + " \n", + " # Ottieni le coordinate angolari (theta, phi) del centro del pixel\n", + " theta, phi = hp.pix2ang(nside, ipix, nest=True)\n", + " \n", + " # Converti in coordinate equatoriali (RA, Dec)\n", + " ra = phi * 180.0 / np.pi # phi è la longitudine in radianti (RA)\n", + " dec = 90.0 - theta * 180.0 / np.pi # theta è la colatitudine (Dec)\n", + " \n", + " # Crea l'oggetto SkyCoord\n", + " coord = SkyCoord(ra=ra * u.deg, dec=dec * u.deg, frame='icrs')\n", + "\n", + " return coord\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9c7a018f-c161-4517-bb66-15fa1531fe32", + "metadata": {}, + "outputs": [], + "source": [ + "medium_sky_loctable.get_expectation_map('BGO_Z1').plot();" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "77beda1d-677a-43fc-a8b3-46dad6dcf9f5", + "metadata": {}, + "outputs": [], + "source": [ + "\n", + "file_name_test = \"run63_medium_10fact_mega_shared\"\n", + "\n", + "file_path =dir_path+'/'+file_name_test+'_dataset.pkl'\n", + "\n", + "# Load the array from the pickle file\n", + "with open(file_path, 'rb') as file:\n", + " loaded_array_test = pickle.load(file)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ea9d182b-a657-4350-afa7-719b92ff5b49", + "metadata": {}, + "outputs": [], + "source": [ + "loaded_array_test.shape" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "60717231-3723-4d30-8ae4-c73baf35dd96", + "metadata": {}, + "outputs": [], + "source": [ + "import numpy as np\n", + "\n", + "filter_flux = 0\n", + "filter_spectra = 0\n", + "\n", + "filtered_loaded_array_test = np.empty(loaded_array_test.shape[0], dtype=object)\n", + "\n", + "count = 0\n", + "\n", + "if filter_flux == 1:\n", + " for grb in loaded_array_test:\n", + " if grb['flux'] > 10 and grb['flux'] <= 15:\n", + " filtered_loaded_array_test[count] = grb\n", + " count = count+1\n", + "\n", + " loaded_array_test = filtered_loaded_array_test[:count]\n", + "\n", + "filtered_loaded_array_test = np.empty(loaded_array_test.shape[0], dtype=object)\n", + "\n", + "count = 0\n", + "\n", + "if filter_spectra==1:\n", + " for grb in loaded_array_test:\n", + " #if grb['spectra'] == 'medium':\n", + " if '1500' in grb['spectrum']:\n", + " filtered_loaded_array_test[count] = grb\n", + " count = count+1\n", + "\n", + " loaded_array_test = filtered_loaded_array_test[:count]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a98f9842-1a7a-4042-8a04-a7fc6f130c33", + "metadata": {}, + "outputs": [], + "source": [ + "loaded_array_test.shape" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "dfd1976a-255b-4e83-9a61-1a8c58cf3540", + "metadata": {}, + "outputs": [], + "source": [ + "test_dataset=loaded_array_test" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1d8a01c1-91a6-4b97-85ad-26478905d405", + "metadata": {}, + "outputs": [], + "source": [ + "test_dataset[0]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "683a3d33-3dd3-49af-b5f6-7e13065894e7", + "metadata": {}, + "outputs": [], + "source": [ + "b_sim = np.array([57.6053, 58.7157, 51.4131, 48.2891, 47.7293, 45.9617])\n", + "random_bkg = np.random.poisson(np.array([b_sim[3],b_sim[2],b_sim[5],b_sim[4],b_sim[1],b_sim[0]])*20)\n", + "\n", + "def process_lut_source(grb_list,random_bkg):\n", + "\n", + " distance_list = []\n", + " conf_area_list = []\n", + " \n", + " for grb in grb_list:\n", + " \n", + " theta_real = float(grb['coord'][0])\n", + " phi_real = float(grb['coord'][1])\n", + " \n", + " s_counts = np.array([grb['counts'][3],grb['counts'][2],grb['counts'][5],grb['counts'][4],grb['counts'][1],grb['counts'][0]])\n", + "\n", + " b_sim = np.array([57.6053, 58.7157, 51.4131, 48.2891, 47.7293, 45.9617])\n", + " random_bkg = np.random.poisson(np.array([b_sim[3],b_sim[2],b_sim[5],b_sim[4],b_sim[1],b_sim[0]])*20)\n", + " \n", + " s_counts = s_counts+random_bkg\n", + " b_counts = np.array([b_sim[3],b_sim[2],b_sim[5],b_sim[4],b_sim[1],b_sim[0]])*20\n", + " \n", + " medium_sqrt_ts, mdium_ra_loc, medium_dec_loc, medium_cont_radius,medium_ts,medium_loc_tsvalue,medium_cont_area = localize_grb(medium_sky_loctable,s_counts, b_counts,theta_real,phi_real)\n", + " \n", + " max_ts = medium_sqrt_ts\n", + " \n", + " if max_ts == medium_sqrt_ts:\n", + " original_ts_value = medium_loc_tsvalue\n", + " best_ts = medium_ts\n", + " ra_loc=mdium_ra_loc\n", + " dec_loc=medium_dec_loc\n", + " cont_radius = medium_cont_radius\n", + " cont_area = medium_cont_area\n", + " \n", + " theta_loc, phi_loc = ra_dec_to_theta_phi(ra_loc, dec_loc)\n", + " \n", + " dist = angular_distance(theta_loc, phi_loc, theta_real, phi_real)\n", + " theta_dist = np.abs(theta_loc - theta_real)\n", + " phi_dist = diff_phi(phi_loc, phi_real)\n", + "\n", + " distance_list.append(dist)\n", + " conf_area_list.append(cont_area)\n", + "\n", + " distance_avg = np.mean(distance_list)\n", + " cont_area_avg = np.mean(conf_area_list)\n", + " \n", + " result = [distance_avg,cont_area_avg]\n", + " \n", + " return result\n", + " \n", + "\n", + "def localize_grb(sky_loctable,s,b,theta_real,phi_real):\n", + "\n", + " \n", + " sky_loctable.set_background(b)\n", + " sky_loctable.set_data(s)\n", + "\n", + " # Define a map of nside = 32. Note that this is a finer resolution\n", + " #that the underlying look-up table, which will be interpolated\n", + " ts = TSMap(nside = 32, coordsys = 'icrs')\n", + "\n", + " # NormLocLike is a subclass of LocLike and computes\n", + " # a Poisson likelihood for counting instruments. The\n", + " # overall normalization is the only free parameter\n", + " norm_likelihood = NormLocLike(sky_loctable)\n", + " \n", + " # Compute the TS map from one or more LocLikelihood\n", + " ts.compute(norm_likelihood)\n", + " #print(\"#\"+str(theta_real)+\"#\")\n", + " \n", + " # Correggi theta_real prima di convertirlo\n", + " if theta_real < 0:\n", + " print(f\"Invalid theta_real: {theta_real}\")\n", + " theta_real = 0\n", + " elif theta_real > 180:\n", + " print(f\"Invalid theta_real: {theta_real}\")\n", + " theta_real = 180\n", + " \n", + " theta_rad = np.deg2rad(theta_real)\n", + " phi_rad = np.deg2rad(phi_real)\n", + " \n", + " if theta_rad < 0 or theta_rad > np.pi:\n", + " print(f\"theta_rad fuori range: {theta_rad}\")\n", + "\n", + "\n", + " #print(dir(ts))\n", + " ipix = ts.ang2pix(theta_rad, phi_rad)\n", + "\n", + " # Get the value at the given coordinates\n", + " #ipix = ts.ang2pix(ts.nside, theta_real * u.deg.to(u.rad), phi_real * u.deg.to(u.rad))\n", + " original_ts_value = ts._data[ipix]\n", + "\n", + " # Inizializza l'array dei livelli di confidenza\n", + " confidence_levels = np.arange(0, 1.01, 0.01)\n", + " \n", + " # Calcola i raggi di contenimento per ciascun valore di cont\n", + " containment_radius = np.array([\n", + " np.sqrt(ts.error_area(cont=cont) / np.pi).to(u.deg).value\n", + " for cont in confidence_levels\n", + " ])\n", + " \n", + " #original_ts_value = -1\n", + " cont_area = ts.error_area(cont = .9).to(u.deg**2).value\n", + "\n", + " \n", + "\n", + " return np.max(ts),ts.best_loc().ra.deg,ts.best_loc().dec.deg,containment_radius,ts,original_ts_value,cont_area\n", + "\n", + "def ra_dec_to_theta_phi(ra, dec):\n", + "\n", + " theta = 90-dec\n", + " \n", + " phi = ra\n", + " \n", + " return theta, phi\n", + "\n", + "def diff_phi(a1, a2):\n", + " \n", + " diff = abs(a1 - a2)\n", + " \n", + " if diff > 180:\n", + " diff = 360 - diff\n", + " \n", + " return diff" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2ec63933-6dc9-4846-af0c-ea1e46435c9b", + "metadata": {}, + "outputs": [], + "source": [ + "results_bkg = []\n", + "spectra_fitted = 0\n", + "\n", + "\n", + "def init_worker():\n", + " seed = int.from_bytes(os.urandom(4), \"little\")\n", + " np.random.seed(seed)\n", + "\n", + "\n", + "def process_in_parallel(test_dataset,random_bkg):\n", + " with multiprocessing.Pool(processes=200, initializer=init_worker) as pool:\n", + " results = pool.starmap(process_lut_source, zip(test_dataset, itertools.repeat(random_bkg)))\n", + " return results\n", + "\n", + "\n", + "results_bkg = process_in_parallel(test_dataset,random_bkg)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "538bf57b-8e53-4a6b-9d02-cff712dacd60", + "metadata": {}, + "outputs": [], + "source": [ + "distances = []\n", + "area_array = []\n", + "\n", + "for r in results_bkg:\n", + " distances.append(r[0])\n", + " area_array.append(r[1])\n", + "\n", + "distances = np.array(distances)\n", + "area_array = np.array(area_array)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c7b66dff-1fa6-436b-8c51-3423a146dd5d", + "metadata": {}, + "outputs": [], + "source": [ + "plt.hist(distances)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5d73bc87-526d-4899-a561-085e65143101", + "metadata": {}, + "outputs": [], + "source": [ + "import healpy as hp\n", + "import numpy as np\n", + "import matplotlib.pyplot as plt\n", + "import pickle\n", + "hp.projview(\n", + " np.array(area_array),\n", + " coord=[\"G\"],\n", + " projection_type=\"aitoff\", # Cambiato da \"mollweide\" a \"aitoff\"\n", + " graticule=True,\n", + " graticule_labels=True,\n", + " longitude_grid_spacing=60,\n", + " title=file,\n", + " latitude_grid_spacing=30,\n", + " cmap=\"turbo\",\n", + " nest=True,\n", + " unit=\"\", \n", + " fontsize={\n", + " \"xlabel\": 14,\n", + " \"ylabel\": 14,\n", + " \"title\": 16,\n", + " \"xtick_label\": 14,\n", + " \"ytick_label\": 14,\n", + " \"cbar_label\": 14,\n", + " \"cbar_tick_label\": 14 # qui imposti il font size dei numeri della colorbar\n", + " },\n", + " override_plot_properties={\n", + " \"cbar_shrink\": 0.8,\n", + " \"cbar_pad\": 0.05,\n", + " \"cbar_label_pad\": 5\n", + " }\n", + " \n", + ")\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "316721ab-65f6-42bb-8eff-dd3c86b303fc", + "metadata": {}, + "outputs": [], + "source": [ + "import pickle\n", + "if False:\n", + "\n", + " # Salva l'array in un file usando pickle\n", + " with open(\"/data/bc_\"+file_name_test+\"_distall.pkl\", \"wb\") as f:\n", + " pickle.dump(distances, f)\n", + " \n", + " # Salva l'array in un file usando pickle\n", + " with open(\"/data/bc_\"+file_name_test+\"_cont_area.pkl\", \"wb\") as f:\n", + " pickle.dump(cont_area_list, f)\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4c550503-72cf-4498-8cbb-8a09bb3b0811", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b6e21f95-4bb2-4baa-bc25-ab628a51fc9c", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7576ea5d-21e4-4f3b-a3c8-33b2f0d3bf61", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7bf50978-5882-4341-ad4f-13e2bca6ba9f", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3a067916-ac3a-45a1-a0ab-d075c2c85387", + "metadata": {}, + "outputs": [], + "source": [ + " " + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "bct", + "language": "python", + "name": "bct" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.12" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/cosipy/notebooks/chi2/chi2_frequentist_coverage.ipynb b/cosipy/notebooks/chi2/chi2_frequentist_coverage.ipynb new file mode 100644 index 0000000..cf0797d --- /dev/null +++ b/cosipy/notebooks/chi2/chi2_frequentist_coverage.ipynb @@ -0,0 +1,740 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "0", + "metadata": {}, + "outputs": [], + "source": [ + "# import libraries\n", + "\n", + "import pandas as pd\n", + "import numpy as np\n", + "import seaborn as sns\n", + "sns.set(color_codes=True)\n", + "import matplotlib.pyplot as plt\n", + "%matplotlib inline\n", + "import pickle\n", + "import math \n", + "\n", + "#update this path with the path where you stored the datasets\n", + "data_path = \"/data/test_newrepo\" " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "64de671c-4b0a-4805-96f0-0502bd9ca57b", + "metadata": {}, + "outputs": [], + "source": [ + "#Load LUTs\n", + "\n", + "lut_name_soft = \"soft_lut\"\n", + "lut_name_medium = \"medium_lut\"\n", + "lut_name_hard = \"hard_lut\"\n", + "\n", + "lut_soft = np.load(data_path+'/LUT_'+lut_name_soft+'.npy')\n", + "lut_medium = np.load(data_path+'/LUT_'+lut_name_medium+'.npy')\n", + "lut_hard = np.load(data_path+'/LUT_'+lut_name_hard+'.npy')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "52d9a03f-7ee3-4b51-adfd-6de090fa883f", + "metadata": {}, + "outputs": [], + "source": [ + "# Save the LUT to plot it with external scripts\n", + "import pickle\n", + "if False:\n", + " with open(prefix+\"/data/lut_hard_Y1.pkl\", \"wb\") as f:\n", + " pickle.dump(lut_hard[:,4], f)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "84ba1c04-559d-4551-af10-c025420e8d91", + "metadata": {}, + "outputs": [], + "source": [ + "#nside = 32\n", + "\n", + "run_name_test = \"dataset\"\n", + "file_name_test = \"run57_mix_mega_shared\"\n", + "\n", + "file_path = data_path+'/'+file_name_test+'_dataset.pkl'\n", + "\n", + "# Load the array from the pickle file\n", + "with open(file_path, 'rb') as file:\n", + " loaded_array_test = pickle.load(file)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "43d26a53-5d18-4ad1-9bea-ce3e30a821df", + "metadata": {}, + "outputs": [], + "source": [ + "loaded_array_test.shape" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a8fa32a8-21d5-4d6f-81e8-6d2208c85fda", + "metadata": {}, + "outputs": [], + "source": [ + "# Filter the dataset by spectral model and flux\n", + "# for soft spectra use flux = [14,16] and filter the spectra to contain the string \"230\"\n", + "# for medium spectra use flux = [1,3] and filter the spectra to contain the string \"699\"\n", + "# for hard spectra use flux = [1,1,6] and filter the spectra to contain the string \"1500\"\n", + "\n", + "filter_flux = 1\n", + "filter_spectra = 1\n", + "filter_theta=0\n", + "filter_phi=0\n", + "\n", + "filtered_loaded_array_test = np.empty(loaded_array_test.shape[0], dtype=object)\n", + "\n", + "count = 0\n", + "\n", + "if filter_flux == 1:\n", + " for grb in loaded_array_test:\n", + " if grb['flux'] >14 and grb['flux'] <= 16:\n", + " filtered_loaded_array_test[count] = grb\n", + " count = count+1\n", + "\n", + " loaded_array_test = filtered_loaded_array_test[:count]\n", + "\n", + "print(len(loaded_array_test))\n", + "filtered_loaded_array_test = np.empty(loaded_array_test.shape[0], dtype=object)\n", + "\n", + "count = 0\n", + "\n", + "if filter_spectra==1:\n", + " for grb in loaded_array_test:\n", + " if \"230\" in grb['spectrum']: #[\"Band 10 10000 -1.9 -3.7 230\",\"Band 10 10000 -1 -2.3 699.9\",\"Comptonized 10 10000 -0.5 1500\"]\n", + " # if grb['spectra'] == 'soft':\n", + " filtered_loaded_array_test[count] = grb\n", + " count = count+1\n", + "\n", + " loaded_array_test = filtered_loaded_array_test[:count]\n", + "\n", + "print(len(loaded_array_test))\n", + "count = 0\n", + "filtered_loaded_array_test = np.empty(loaded_array_test.shape[0], dtype=object)\n", + "\n", + "if filter_theta==1:\n", + " print(\"filter spectra\")\n", + " for grb in loaded_array_test:\n", + " if float(grb['coord'][0])>50 and float(grb['coord'][0])<130:\n", + " filtered_loaded_array_test[count] = grb\n", + " count = count+1\n", + "\n", + " loaded_array_test = filtered_loaded_array_test[:count]\n", + "\n", + "count = 0\n", + "filtered_loaded_array_test = np.empty(loaded_array_test.shape[0], dtype=object)\n", + "\n", + "if filter_phi==1:\n", + " print(\"filter spectra\")\n", + " for grb in loaded_array_test:\n", + " if float(grb['coord'][1])>100 and float(grb['coord'][1])<170:\n", + " filtered_loaded_array_test[count] = grb\n", + " count = count+1\n", + "\n", + " loaded_array_test = filtered_loaded_array_test[:count]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "eea260a0-39ed-4504-9450-eb0b3a256cce", + "metadata": {}, + "outputs": [], + "source": [ + "test_dataset = loaded_array_test" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a42fbb35-bab4-42b8-8958-09acb15bed07", + "metadata": {}, + "outputs": [], + "source": [ + "test_dataset.shape" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5", + "metadata": {}, + "outputs": [], + "source": [ + "def get_radians(coords):\n", + " # Unpack the list of (theta, phi) pairs\n", + " theta, phi = zip(*coords)\n", + " \n", + " # Convert to numpy arrays\n", + " theta = np.array(theta)\n", + " phi = np.array(phi)\n", + " \n", + " # Wrap phi values >180 into the range (-180, 180]\n", + " mask = phi > 180\n", + " phi[mask] -= 360\n", + " \n", + " # Convert theta into colatitude (90 - theta)\n", + " theta = 90 - theta\n", + "\n", + " # Return values in radians\n", + " return np.radians(theta), np.radians(phi)\n", + "\n", + "def calculate_chi_squared_optimized(s, b, m):\n", + " \"\"\"\n", + " Compute the chi-squared value for each position in the grid in an optimized way.\n", + "\n", + " Parameters:\n", + " s: array of shape (6,) with observed counts (s(j)).\n", + " b: array of shape (12,) with background counts (b(j)).\n", + " m: array of shape (12, 41168) with model counts (m(j, i)).\n", + "\n", + " Returns:\n", + " chi_squared: array of shape (41168,) with the chi-squared value for each position i.\n", + " \"\"\"\n", + " # Use only the first 6 detectors\n", + " indices = np.arange(6)\n", + " \n", + " # Extract model counts for these detectors\n", + " m_subset = m[:, indices] # Shape: (41168, 6)\n", + "\n", + " # Compute numerator and denominator for normalization factor f_i\n", + " with np.errstate(divide='ignore', invalid='ignore'):\n", + " numerator = np.sum(m_subset * (s[indices] - b[indices]) / s[indices], axis=1)\n", + " denominator = np.sum((m_subset**2) / s[indices], axis=1)\n", + "\n", + " # Avoid division by zero\n", + " f_i = np.divide(numerator, denominator, out=np.zeros_like(numerator), where=denominator != 0)\n", + "\n", + " # Expand f_i for broadcasting\n", + " f_i_expanded = f_i[:, np.newaxis] # Shape: (41168, 1)\n", + "\n", + " # Compute chi-squared terms\n", + " chi_numerator = (s[indices] - b[indices] - f_i_expanded * m_subset)**2\n", + " chi_denominator = b[indices] + f_i_expanded * m_subset\n", + "\n", + " # Safe division for chi-squared elements\n", + " chi_squared_elements = np.divide(\n", + " chi_numerator, chi_denominator,\n", + " out=np.full_like(chi_numerator, np.finfo(np.float64).max),\n", + " where=chi_denominator != 0\n", + " )\n", + "\n", + " # Sum over detectors to obtain chi^2 per direction\n", + " chi_squared = np.sum(chi_squared_elements, axis=1)\n", + "\n", + " return chi_squared\n", + "\n", + "\n", + "def angular_distance(theta1, phi1, theta2, phi2):\n", + " \"\"\"\n", + " Compute the angular (great-circle) distance in degrees between two points\n", + " specified by (theta, phi) coordinates in degrees.\n", + "\n", + " Parameters:\n", + " theta1, phi1: floats, coordinates of the first point (degrees).\n", + " theta2, phi2: floats, coordinates of the second point (degrees).\n", + "\n", + " Returns:\n", + " angular_dist_deg: float, angular distance between the two points in degrees.\n", + " \"\"\"\n", + " # Adjust theta values by shifting -90\n", + " theta1 = 90 - theta1\n", + " theta2 = 90 - theta2\n", + "\n", + " # Convert angles to radians\n", + " theta1_rad = math.radians(theta1)\n", + " phi1_rad = math.radians(phi1)\n", + " theta2_rad = math.radians(theta2)\n", + " phi2_rad = math.radians(phi2)\n", + " \n", + " # Compute the difference in longitude\n", + " delta_phi = abs(phi1_rad - phi2_rad)\n", + "\n", + " # Apply spherical law of cosines\n", + " value = (math.sin(theta1_rad) * math.sin(theta2_rad) +\n", + " math.cos(theta1_rad) * math.cos(theta2_rad) * math.cos(delta_phi))\n", + " \n", + " # Clamp value to [-1, 1] to avoid floating point errors in acos\n", + " value_clamped = max(-1.0, min(1.0, value)) \n", + " \n", + " angular_dist = math.acos(value_clamped)\n", + " \n", + " # Convert angular distance from radians to degrees\n", + " angular_dist_deg = math.degrees(angular_dist)\n", + " \n", + " return angular_dist_deg\n", + "\n", + "def diff_phi(a1, a2):\n", + " diff = abs(a1 - a2)\n", + " if diff > 180:\n", + " diff = 360 - diff\n", + " return diff" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7c5de29e-2b49-49cd-b05f-6cf7c41101f7", + "metadata": {}, + "outputs": [], + "source": [ + "def find_closest_pixel(theta_real, phi_real, best_lut):\n", + " # Convert input angles from degrees to radians\n", + " theta_real_rad = np.deg2rad(theta_real)\n", + " phi_real_rad = np.deg2rad(phi_real)\n", + " \n", + " # Extract θ and φ from best_lut (last two columns, in degrees) and convert to radians\n", + " theta_vals_m = np.deg2rad(best_lut[:, -2])\n", + " phi_vals_m = np.deg2rad(best_lut[:, -1])\n", + " \n", + " # Compute the cosine of the spherical angular distance\n", + " cos_dist = (\n", + " np.sin(theta_real_rad) * np.sin(theta_vals_m) * np.cos(phi_real_rad - phi_vals_m)\n", + " + np.cos(theta_real_rad) * np.cos(theta_vals_m)\n", + " )\n", + "\n", + " # Compute the angular distance (in radians)\n", + " angular_distance = np.arccos(cos_dist)\n", + " \n", + " # Find the index of the minimum distance\n", + " min_idx = np.argmin(angular_distance)\n", + " \n", + " # The index of the maximum cosine corresponds to the minimum distance\n", + " # min_idx = np.argmax(cos_dist)\n", + " return min_idx" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bb33c8ff-5905-4ea3-8ea5-8a4ee8539ff7", + "metadata": {}, + "outputs": [], + "source": [ + "def analyze_grbs(test_dataset,spectral_model):\n", + "\n", + " results = []\n", + " spectra_fitted = 0\n", + " good_spectra_fit_distances = []\n", + " bad_spectra_fit_distances=[]\n", + " \n", + " count = 0\n", + " for grb in test_dataset:\n", + " \n", + " #if count == 10:\n", + " # break\n", + " if count % 1000 == 0:\n", + " print(count)\n", + "\n", + " counts = grb['counts']\n", + " spectrum = grb['spectrum']\n", + " if \"230\" in spectrum :\n", + " spectra_value=\"soft\"\n", + " elif \"699.9\" in spectrum:\n", + " spectra_value=\"medium\"\n", + " elif \"Compton\" in spectrum:\n", + " spectra_value=\"hard\"\n", + " else:\n", + " spectra_value=\"random\"\n", + "\n", + " theta_real = float(grb['coord'][0])\n", + " phi_real = float(grb['coord'][1])\n", + "\n", + " # mean counts during a window of 500 seconds\n", + " b_sim = np.array([57.6053, 58.7157, 51.4131, 48.2891, 47.7293, 45.9617])\n", + " global_min = -1\n", + " if spectral_model==\"soft\": \n", + " \n", + " chi_squared_soft = calculate_chi_squared_optimized(np.array(counts)+np.random.poisson(b_sim*20),b_sim*20, lut_soft)\n", + " min_soft = np.min(chi_squared_soft)\n", + " global_min = min_soft\n", + "\n", + " chi2_array = chi_squared_soft\n", + " if spectra_value == \"soft\":\n", + " spectra_fitted = spectra_fitted +1\n", + " spectra_selected = True\n", + " argmin_index = np.argmin(chi_squared_soft)\n", + " best_lut = lut_soft\n", + " \n", + " elif spectral_model==\"medium\":\n", + " \n", + " chi_squared_medium = calculate_chi_squared_optimized(np.array(counts)+np.random.poisson(b_sim*20),b_sim*20, lut_medium)\n", + " min_medium = np.min(chi_squared_medium)\n", + " global_min = min_medium\n", + "\n", + " chi2_array = chi_squared_medium\n", + " if spectra_value == \"medium\":\n", + " spectra_fitted = spectra_fitted +1\n", + " spectra_selected = True\n", + " argmin_index = np.argmin(chi_squared_medium)\n", + " best_lut = lut_medium\n", + " \n", + " elif spectral_model==\"hard\":\n", + " \n", + " chi_squared_hard = calculate_chi_squared_optimized(np.array(counts)+np.random.poisson(b_sim*20),b_sim*20, lut_hard)\n", + " min_hard = np.min(chi_squared_hard)\n", + " global_min = min_hard\n", + "\n", + " chi2_array = chi_squared_hard\n", + " if spectra_value == \"hard\":\n", + " spectra_fitted = spectra_fitted +1\n", + " spectra_selected = True\n", + " argmin_index = np.argmin(chi_squared_hard)\n", + " best_lut = lut_hard\n", + "\n", + " \n", + " if(global_min==-1):\n", + " print(\"error\")\n", + " \n", + " spectra_selected = False\n", + " \n", + " theta_loc = best_lut[argmin_index][6]\n", + " phi_loc = best_lut[argmin_index][7]\n", + " \n", + " theta_real = float(grb['coord'][0])\n", + " phi_real = float(grb['coord'][1])\n", + " \n", + " min_idx = find_closest_pixel(theta_real,phi_real,best_lut)\n", + " \n", + " real_chi2 = chi2_array[min_idx]\n", + " \n", + " dist = angular_distance(theta_loc,phi_loc,theta_real,phi_real)\n", + " theta_dist = np.abs(theta_loc-theta_real)\n", + " phi_dist = diff_phi(phi_loc,phi_real)\n", + " \n", + " if spectra_selected:\n", + " good_spectra_fit_distances.append(dist)\n", + " else:\n", + " bad_spectra_fit_distances.append(dist)\n", + " \n", + " results.append([theta_real,phi_real,theta_loc,phi_loc,dist,theta_dist,phi_dist,global_min,real_chi2,chi2_array])\n", + " \n", + " count = count +1\n", + "\n", + " return results,spectra_fitted, good_spectra_fit_distances,bad_spectra_fit_distances" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1930d232-2ac1-489d-af06-d0437714d64f", + "metadata": {}, + "outputs": [], + "source": [ + "results,spectra_fitted, good_spectra_fit_distances,bad_spectra_fit_distances= analyze_grbs(test_dataset,\"soft\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e4f7970a", + "metadata": {}, + "outputs": [], + "source": [ + "from scipy.stats import chi2\n", + "import healpy as hp\n", + "\n", + "distances = []\n", + "theta_distances = []\n", + "phi_distances = []\n", + "areas_90 = []\n", + "for res in results:\n", + " distances.append(res[4])\n", + " theta_distances.append(res[5])\n", + " phi_distances.append(res[6])\n", + "\n", + " # calculate area \n", + " map = res[9]\n", + "\n", + " limit = res[7]+ chi2.ppf(0.9, 2)\n", + " \n", + " # Maschera i valori maggiori o uguali a X\n", + " nside = hp.npix2nside(len(map))\n", + " \n", + " mask_inside_region = map < limit\n", + " npix_in_region = np.sum(mask_inside_region)\n", + " nside = hp.get_nside(map)\n", + " pix_area = hp.nside2pixarea(nside, degrees=True)\n", + " area_90 = npix_in_region * pix_area\n", + " areas_90.append(area_90)\n", + "\n", + "\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5c43b4b7-e303-42f2-8385-5921e8c39409", + "metadata": {}, + "outputs": [], + "source": [ + "import pickle\n", + "if False:\n", + "\n", + " # Salva l'array in un file usando pickle\n", + " with open(prefix+\"/data/chi2_\"+file_name_test+\"_distall.pkl\", \"wb\") as f:\n", + " pickle.dump(distances, f)\n", + " \n", + " # Salva l'array in un file usando pickle\n", + " with open(prefix+\"/data/chi2_\"+file_name_test+\"_cont_area.pkl\", \"wb\") as f:\n", + " pickle.dump(areas_90, f)\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "168af197-3002-4c45-a680-cc1516a33038", + "metadata": {}, + "outputs": [], + "source": [ + "# Choose the index to plot. Eventually it is possible to implement a for cycle.\n", + "index = 0\n", + "min_chi2 = results[index][7]\n", + "theta_real = float(results[index][0])\n", + "phi_real = float(results[index][1])\n", + "theta_reco=float(results[index][2])\n", + "phi_reco=float(results[index][3])\n", + "\n", + "if phi_real>180:\n", + " phi_real = phi_real-360\n", + "if phi_reco>180:\n", + " phi_reco = phi_reco-360\n", + "\n", + "index = 0\n", + "map = results[index][9]\n", + "\n", + "limit = results[index][7]+ chi2.ppf(0.9, 2)\n", + "\n", + "# Maschera i valori maggiori o uguali a X\n", + "mappa_masked = np.ma.masked_where(map >= limit , map)\n", + "\n", + "hp.projview(\n", + " mappa_masked,\n", + " coord=[\"G\"],\n", + " projection_type=\"aitoff\", \n", + " graticule=True,\n", + " graticule_labels=True,\n", + " longitude_grid_spacing=60,\n", + " title=file,\n", + " latitude_grid_spacing=30,\n", + " cmap=\"viridis\",\n", + " nest=True,\n", + " unit=\"\", \n", + " badcolor=\"antiquewhite\",\n", + " fontsize={\n", + " \"xlabel\": 14,\n", + " \"ylabel\": 14,\n", + " \"title\": 16,\n", + " \"xtick_label\": 14,\n", + " \"ytick_label\": 14,\n", + " \"cbar_label\": 14,\n", + " \"cbar_tick_label\": 14 # qui imposti il font size dei numeri della colorbar\n", + " },\n", + " override_plot_properties={\n", + " \"cbar_shrink\": 0.8,\n", + " \"cbar_pad\": 0.05,\n", + " \"cbar_label_pad\": 0\n", + " }\n", + " \n", + ")\n", + "\n", + "hp.newprojplot(theta=np.radians(theta_real), phi=np.radians(phi_real), marker=\"*\", color=\"magenta\", markersize=13)\n", + "\n", + "hp.newprojplot(theta=np.radians(theta_reco), phi=np.radians(phi_reco), marker=\"X\", color=\"r\", markersize=12)\n", + "\n", + "ax = plt.gca()\n", + "\n", + "fig = plt.gcf()\n", + "axes = fig.get_axes()\n", + "main_ax = plt.gca()\n", + "\n", + "for ax in axes:\n", + " if ax != main_ax:\n", + " cbar_ax = ax\n", + " break\n", + "\n", + "cbar_ax.set_xlabel(\"$\\chi^2$ value (90% c.l.)\", fontsize=14) \n", + "\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f8bfe948", + "metadata": {}, + "outputs": [], + "source": [ + "\n", + "from scipy.stats import chi2\n", + "values = np.arange(0, 1.01, 0.01)\n", + "\n", + "fraction_array = []\n", + "\n", + "for value in values:\n", + " count = 0 \n", + " for r in results:\n", + " min_chi2 = r[7]\n", + " real_chi2 = r[8]\n", + " if real_chi2 0:\n", + " coverage = count_in / count_total\n", + " else:\n", + " coverage = 0\n", + " coverage_array.append(coverage)\n", + "\n", + "# Plotting\n", + "plt.plot(flux_centers, coverage_array, marker='o')\n", + "plt.axhline(0.9, color='gray', linestyle='--', label='Target 90%')\n", + "plt.xlabel('Source Flux')\n", + "plt.ylabel('Empirical 90% Coverage')\n", + "plt.title('Coverage vs. Source Flux at Fixed 90% CL')\n", + "plt.legend()\n", + "plt.grid(True)\n", + "plt.show()\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c9bc4b72-5b47-4f7d-9ea4-7e0bec113b83", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d27c9e32-f2d8-4b24-9256-e7a6f83a6d32", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "885ed89a-4ebd-4ed8-b096-4282fd068392", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "deeplearning", + "language": "python", + "name": "deeplearning" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.12" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/cosipy/notebooks/chi2/chi2_localization.ipynb b/cosipy/notebooks/chi2/chi2_localization.ipynb new file mode 100644 index 0000000..84ad30d --- /dev/null +++ b/cosipy/notebooks/chi2/chi2_localization.ipynb @@ -0,0 +1,1040 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "0", + "metadata": {}, + "outputs": [], + "source": [ + "# import libraries\n", + "import math\n", + "import numpy as np\n", + "import matplotlib.pyplot as plt\n", + "%matplotlib inline\n", + "import pickle\n", + "from scipy.stats import chi2\n", + "import healpy as hp\n", + "\n", + "data_path = \"/data/test_newrepo\" " + ] + }, + { + "cell_type": "raw", + "id": "c91ce832-751b-4e6b-9811-53e7a1a50bc4", + "metadata": { + "vscode": { + "languageId": "raw" + } + }, + "source": [ + "# activate this cell to create the LUTs, then disable it.\n", + "\n", + "#Example\n", + "file_name = \"run52_hard_mega_shared\"\n", + "lut_name = \"hard_lut\"\n", + "\n", + "file_path = data_path+'/'+file_name+'_dataset.pkl'\n", + "# Load the array from the pickle file\n", + "with open(file_path, 'rb') as file:\n", + " loaded_array = pickle.load(file)\n", + " \n", + "print(file_path)\n", + "print(len(loaded_array))\n", + "\n", + "m_tables = np.empty((len(loaded_array),8))\n", + "\n", + "count = -1\n", + "for element in loaded_array:\n", + " count+=1\n", + " m_tables[count] = element['counts']+element['coord']\n", + "\n", + "m_tables = m_tables[:count+1]\n", + "\n", + "np.save(data_path+'/LUT_'+lut_name+'.npy', m_tables) \n", + "print(data_path+'/LUT/LUT_'+lut_name+'.npy')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2bec492e-6f09-46a6-9c4b-70a6dbab85f6", + "metadata": {}, + "outputs": [], + "source": [ + "#Load LUTs\n", + "\n", + "lut_name_soft = \"soft_lut\"\n", + "lut_name_medium = \"medium_lut\"\n", + "lut_name_hard = \"hard_lut\"\n", + "\n", + "lut_soft = np.load(data_path+'/LUT_'+lut_name_soft+'.npy')\n", + "lut_medium = np.load(data_path+'/LUT_'+lut_name_medium+'.npy')\n", + "lut_hard = np.load(data_path+'/LUT_'+lut_name_hard+'.npy')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "52d9a03f-7ee3-4b51-adfd-6de090fa883f", + "metadata": {}, + "outputs": [], + "source": [ + "lut_medium.shape" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f180a1fd-b579-4eb1-8dbf-0d11d955e16d", + "metadata": {}, + "outputs": [], + "source": [ + "import healpy as hp\n", + "import matplotlib.pyplot as plt\n", + "# Plot della mappa\n", + "hp.mollview(lut_medium[:,4], title=\"Medium Y1\", unit=\"counts\", norm='hist',nest=True )\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "eee621c2-e552-4a04-9a20-9fc2ec03bbfc", + "metadata": {}, + "outputs": [], + "source": [ + "lut_plot = lut_medium[:,4]\n", + "\n", + "hp.projview(\n", + " lut_plot,\n", + " coord=[\"G\"],\n", + " projection_type=\"aitoff\", \n", + " graticule=True,\n", + " graticule_labels=True,\n", + " longitude_grid_spacing=60,\n", + " latitude_grid_spacing=30,\n", + " cmap=\"turbo\",\n", + " nest=True,\n", + " unit=\"\", \n", + " badcolor=\"antiquewhite\",\n", + " fontsize={\n", + " \"xlabel\": 14,\n", + " \"ylabel\": 14,\n", + " \"title\": 16,\n", + " \"xtick_label\": 14,\n", + " \"ytick_label\": 14,\n", + " \"cbar_label\": 14,\n", + " \"cbar_tick_label\": 14 # qui imposti il font size dei numeri della colorbar\n", + " },\n", + " override_plot_properties={\n", + " \"cbar_shrink\": 0.8,\n", + " \"cbar_pad\": 0.05,\n", + " \"cbar_label_pad\": 0\n", + " }\n", + " \n", + ")\n", + "\n", + "ax = plt.gca()\n", + "\n", + "fig = plt.gcf()\n", + "axes = fig.get_axes()\n", + "main_ax = plt.gca()\n", + "\n", + "for ax in axes:\n", + " if ax != main_ax:\n", + " cbar_ax = ax\n", + " break\n", + "\n", + "cbar_ax.set_xlabel(\"LUT example\", fontsize=14) \n", + "\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "84ba1c04-559d-4551-af10-c025420e8d91", + "metadata": {}, + "outputs": [], + "source": [ + "#test with LUTs\n", + "test_dataset_name = \"run64\" #change this according to the dataset you want to test\n", + "file_path = data_path+'/'+test_dataset_name+'_dataset.pkl'\n", + "\n", + "# Load the array from the pickle file\n", + "with open(file_path, 'rb') as file:\n", + " loaded_array_test = pickle.load(file)\n", + " \n", + "test_dataset = loaded_array_test" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a8fa32a8-21d5-4d6f-81e8-6d2208c85fda", + "metadata": {}, + "outputs": [], + "source": [ + "# Leave the filters to 0 if you want to test all the dataset otherwise set the filters to 1 and change the parameters\n", + "\n", + "filter_flux = 0\n", + "filter_spectra = 0\n", + "\n", + "filter_theta=0\n", + "filter_phi=0\n", + "\n", + "filtered_loaded_array_test = np.empty(loaded_array_test.shape[0], dtype=object)\n", + "\n", + "count = 0\n", + "\n", + "if filter_flux == 1:\n", + " for grb in loaded_array_test:\n", + " if grb['flux'] >14 and grb['flux'] <= 16:\n", + " filtered_loaded_array_test[count] = grb\n", + " count = count+1\n", + "\n", + " loaded_array_test = filtered_loaded_array_test[:count]\n", + "\n", + "print(len(loaded_array_test))\n", + "filtered_loaded_array_test = np.empty(loaded_array_test.shape[0], dtype=object)\n", + "\n", + "count = 0\n", + "\n", + "if filter_spectra==1:\n", + " for grb in loaded_array_test:\n", + " # filter by spectrum using the spectrum parameter\n", + " if \"230\" in grb['spectrum']: \n", + " #[\"Band 10 10000 -1.9 -3.7 230\",\"Band 10 10000 -1 -2.3 699.9\",\"Comptonized 10 10000 -0.5 1500\"]\n", + " filtered_loaded_array_test[count] = grb\n", + " count = count+1\n", + "\n", + " loaded_array_test = filtered_loaded_array_test[:count]\n", + "\n", + "print(len(loaded_array_test))\n", + "count = 0\n", + "filtered_loaded_array_test = np.empty(loaded_array_test.shape[0], dtype=object)\n", + "\n", + "if filter_theta==1:\n", + " print(\"filter theta\")\n", + " for grb in loaded_array_test:\n", + " if float(grb['coord'][0])>50 and float(grb['coord'][0])<130:\n", + " filtered_loaded_array_test[count] = grb\n", + " count = count+1\n", + "\n", + " loaded_array_test = filtered_loaded_array_test[:count]\n", + "\n", + "count = 0\n", + "filtered_loaded_array_test = np.empty(loaded_array_test.shape[0], dtype=object)\n", + "\n", + "if filter_phi==1:\n", + " print(\"filter phi\")\n", + " for grb in loaded_array_test:\n", + " if float(grb['coord'][1])>100 and float(grb['coord'][1])<170:\n", + " filtered_loaded_array_test[count] = grb\n", + " count = count+1\n", + "\n", + " loaded_array_test = filtered_loaded_array_test[:count]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5", + "metadata": {}, + "outputs": [], + "source": [ + "def get_radians(coords):\n", + " # Unpack the list of (theta, phi) pairs\n", + " theta, phi = zip(*coords)\n", + " \n", + " # Convert to numpy arrays\n", + " theta = np.array(theta)\n", + " phi = np.array(phi)\n", + " \n", + " # Wrap phi values >180 into the range (-180, 180]\n", + " mask = phi > 180\n", + " phi[mask] -= 360\n", + " \n", + " # Convert theta into colatitude (90 - theta)\n", + " theta = 90 - theta\n", + "\n", + " # Return values in radians\n", + " return np.radians(theta), np.radians(phi)\n", + "\n", + "def calculate_chi_squared_optimized(s, b, m):\n", + " \"\"\"\n", + " Compute the chi-squared value for each position in the grid in an optimized way.\n", + "\n", + " Parameters:\n", + " s: array of shape (6,) with observed counts (s(j)).\n", + " b: array of shape (12,) with background counts (b(j)).\n", + " m: array of shape (12, 41168) with model counts (m(j, i)).\n", + "\n", + " Returns:\n", + " chi_squared: array of shape (41168,) with the chi-squared value for each position i.\n", + " \"\"\"\n", + " # Use only the first 6 detectors\n", + " indices = np.arange(6)\n", + " \n", + " # Extract model counts for these detectors\n", + " m_subset = m[:, indices] # Shape: (41168, 6)\n", + "\n", + " # Compute numerator and denominator for normalization factor f_i\n", + " with np.errstate(divide='ignore', invalid='ignore'):\n", + " numerator = np.sum(m_subset * (s[indices] - b[indices]) / s[indices], axis=1)\n", + " denominator = np.sum((m_subset**2) / s[indices], axis=1)\n", + "\n", + " # Avoid division by zero\n", + " f_i = np.divide(numerator, denominator, out=np.zeros_like(numerator), where=denominator != 0)\n", + "\n", + " # Expand f_i for broadcasting\n", + " f_i_expanded = f_i[:, np.newaxis] # Shape: (41168, 1)\n", + "\n", + " # Compute chi-squared terms\n", + " chi_numerator = (s[indices] - b[indices] - f_i_expanded * m_subset)**2\n", + " chi_denominator = b[indices] + f_i_expanded * m_subset\n", + "\n", + " # Safe division for chi-squared elements\n", + " chi_squared_elements = np.divide(\n", + " chi_numerator, chi_denominator,\n", + " out=np.full_like(chi_numerator, np.finfo(np.float64).max),\n", + " where=chi_denominator != 0\n", + " )\n", + "\n", + " # Sum over detectors to obtain chi^2 per direction\n", + " chi_squared = np.sum(chi_squared_elements, axis=1)\n", + "\n", + " return chi_squared\n", + "\n", + "\n", + "def angular_distance(theta1, phi1, theta2, phi2):\n", + " \"\"\"\n", + " Compute the angular (great-circle) distance in degrees between two points\n", + " specified by (theta, phi) coordinates in degrees.\n", + "\n", + " Parameters:\n", + " theta1, phi1: floats, coordinates of the first point (degrees).\n", + " theta2, phi2: floats, coordinates of the second point (degrees).\n", + "\n", + " Returns:\n", + " angular_dist_deg: float, angular distance between the two points in degrees.\n", + " \"\"\"\n", + " # Adjust theta values by shifting -90\n", + " theta1 = 90 - theta1\n", + " theta2 = 90 - theta2\n", + "\n", + " # Convert angles to radians\n", + " theta1_rad = math.radians(theta1)\n", + " phi1_rad = math.radians(phi1)\n", + " theta2_rad = math.radians(theta2)\n", + " phi2_rad = math.radians(phi2)\n", + " \n", + " # Compute the difference in longitude\n", + " delta_phi = abs(phi1_rad - phi2_rad)\n", + "\n", + " # Apply spherical law of cosines\n", + " value = (math.sin(theta1_rad) * math.sin(theta2_rad) +\n", + " math.cos(theta1_rad) * math.cos(theta2_rad) * math.cos(delta_phi))\n", + " \n", + " # Clamp value to [-1, 1] to avoid floating point errors in acos\n", + " value_clamped = max(-1.0, min(1.0, value)) \n", + " \n", + " angular_dist = math.acos(value_clamped)\n", + " \n", + " # Convert angular distance from radians to degrees\n", + " angular_dist_deg = math.degrees(angular_dist)\n", + " \n", + " return angular_dist_deg\n", + "\n", + "def diff_phi(a1, a2):\n", + " diff = abs(a1 - a2)\n", + " if diff > 180:\n", + " diff = 360 - diff\n", + " return diff" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7c5de29e-2b49-49cd-b05f-6cf7c41101f7", + "metadata": {}, + "outputs": [], + "source": [ + "def find_closest_pixel(theta_real, phi_real, best_lut):\n", + " # Convert input angles from degrees to radians\n", + " theta_real_rad = np.deg2rad(theta_real)\n", + " phi_real_rad = np.deg2rad(phi_real)\n", + " \n", + " # Extract θ and φ from best_lut (last two columns, in degrees) and convert to radians\n", + " theta_vals_m = np.deg2rad(best_lut[:, -2])\n", + " phi_vals_m = np.deg2rad(best_lut[:, -1])\n", + " \n", + " # Compute the cosine of the spherical angular distance\n", + " cos_dist = (\n", + " np.sin(theta_real_rad) * np.sin(theta_vals_m) * np.cos(phi_real_rad - phi_vals_m)\n", + " + np.cos(theta_real_rad) * np.cos(theta_vals_m)\n", + " )\n", + "\n", + " # Compute the angular distance (in radians)\n", + " angular_distance = np.arccos(cos_dist)\n", + " \n", + " # Find the index of the minimum distance\n", + " min_idx = np.argmin(angular_distance)\n", + " \n", + " # The index of the maximum cosine corresponds to the minimum distance\n", + " # min_idx = np.argmax(cos_dist)\n", + " return min_idx" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bb33c8ff-5905-4ea3-8ea5-8a4ee8539ff7", + "metadata": {}, + "outputs": [], + "source": [ + "def analyze_grbs(test_dataset, with_background):\n", + " \"\"\"\n", + " Analyze a dataset of GRBs:\n", + " - pick the best spectrum (soft/medium/hard) via chi-squared minimization,\n", + " - localize using the argmin index on the best LUT,\n", + " - compute angular/coordinate errors,\n", + " - collect stats on whether the chosen spectrum matches the true one.\n", + "\n", + " Returns:\n", + " results: list of [theta_real, phi_real, theta_loc, phi_loc, dist, theta_dist, phi_dist, global_min, real_chi2, chi2_array]\n", + " spectra_fitted: int, number of correctly identified spectra\n", + " good_spectra_fit_distances: list of angular distances for correct spectrum choices\n", + " bad_spectra_fit_distances: list of angular distances for incorrect spectrum choices\n", + " \"\"\"\n", + " results = []\n", + " spectra_fitted = 0\n", + " good_spectra_fit_distances = []\n", + " bad_spectra_fit_distances = []\n", + "\n", + " # Map spectrum string to label\n", + " def classify_spectrum(spectrum_str: str) -> str:\n", + " if \"230\" in spectrum_str:\n", + " return \"soft\"\n", + " if \"699.9\" in spectrum_str:\n", + " return \"medium\"\n", + " if \"Compton\" in spectrum_str:\n", + " return \"hard\"\n", + " return \"random\"\n", + "\n", + " # LUTs registry (expects lut_soft, lut_medium, lut_hard to be defined)\n", + " lut_by_key = {\n", + " \"soft\": lut_soft,\n", + " \"medium\": lut_medium,\n", + " \"hard\": lut_hard,\n", + " }\n", + " keys = [\"soft\", \"medium\", \"hard\"]\n", + "\n", + " count = 0\n", + " for grb in test_dataset:\n", + " count += 1\n", + " if count % 1000 == 0:\n", + " print(count)\n", + "\n", + " counts = np.array(grb[\"counts\"], dtype=float)\n", + " spectrum_key_true = classify_spectrum(grb[\"spectrum\"])\n", + "\n", + " theta_real = float(grb[\"coord\"][0])\n", + " phi_real = float(grb[\"coord\"][1])\n", + "\n", + " # Compute chi^2 arrays for each spectrum key\n", + " chi2_by_key = {}\n", + " mins = []\n", + " if with_background:\n", + " # Fixed background vector as in your code\n", + " b_sim = np.array([57.6053, 58.7157, 51.4131, 48.2891, 47.7293, 45.9617], dtype=float)\n", + " b_vec = b_sim * 20.0\n", + "\n", + " for k in keys:\n", + " # Draw independent Poisson noise for each spectrum (same as original behavior)\n", + " noisy_counts = counts + np.random.poisson(b_vec)\n", + " chi2 = calculate_chi_squared_optimized(noisy_counts, b_vec, lut_by_key[k])\n", + " chi2_by_key[k] = chi2\n", + " mins.append(np.min(chi2))\n", + " else:\n", + " zero_b = np.zeros(6, dtype=float)\n", + " for k in keys:\n", + " chi2 = calculate_chi_squared_optimized(counts, zero_b, lut_by_key[k])\n", + " chi2_by_key[k] = chi2\n", + " mins.append(np.min(chi2))\n", + "\n", + " # Pick global best spectrum via argmin\n", + " best_idx = int(np.argmin(mins))\n", + " best_key = keys[best_idx]\n", + " chi2_array = chi2_by_key[best_key]\n", + " best_lut = lut_by_key[best_key]\n", + "\n", + " # Check if predicted spectrum matches the labeled one\n", + " spectra_selected = (spectrum_key_true == best_key)\n", + " if spectra_selected and best_key in {\"soft\", \"medium\", \"hard\"}:\n", + " spectra_fitted += 1\n", + "\n", + " # Best location from LUT argmin\n", + " argmin_index = int(np.argmin(chi2_array))\n", + " theta_loc = best_lut[argmin_index][6]\n", + " phi_loc = best_lut[argmin_index][7]\n", + "\n", + " # Chi^2 at the true direction (closest pixel to the real coordinates)\n", + " min_idx = find_closest_pixel(theta_real, phi_real, best_lut)\n", + " real_chi2 = chi2_array[min_idx]\n", + "\n", + " # Angular and component-wise distances\n", + " dist = angular_distance(theta_loc, phi_loc, theta_real, phi_real)\n", + " theta_dist = np.abs(theta_loc - theta_real)\n", + " phi_dist = diff_phi(phi_loc, phi_real)\n", + "\n", + " if spectra_selected:\n", + " good_spectra_fit_distances.append(dist)\n", + " else:\n", + " bad_spectra_fit_distances.append(dist)\n", + "\n", + " results.append([\n", + " theta_real, phi_real,\n", + " theta_loc, phi_loc,\n", + " dist, theta_dist, phi_dist,\n", + " mins[best_idx], real_chi2, chi2_array\n", + " ])\n", + "\n", + " return results, spectra_fitted, good_spectra_fit_distances, bad_spectra_fit_distances" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1930d232-2ac1-489d-af06-d0437714d64f", + "metadata": {}, + "outputs": [], + "source": [ + "# With True we have background\n", + "results,spectra_fitted, good_spectra_fit_distances,bad_spectra_fit_distances= analyze_grbs(test_dataset,False)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "445bbfba-ec69-40a6-8bd3-5b248904b9fe", + "metadata": {}, + "outputs": [], + "source": [ + "# Collect angular errors and 90% containment areas\n", + "distances = []\n", + "theta_distances = []\n", + "phi_distances = []\n", + "areas_90 = []\n", + "\n", + "# Constant Δchi2 for 90% CL with 2 dof\n", + "delta_chi2_90 = chi2.ppf(0.9, 2)\n", + "\n", + "for res in results:\n", + " # Unpack metrics\n", + " dist = res[4]\n", + " dtheta = res[5]\n", + " dphi = res[6]\n", + " chi2_min = res[7]\n", + " chi2_map = res[9] # array-like chi² map over sky pixels (Healpix)\n", + "\n", + " distances.append(dist)\n", + " theta_distances.append(dtheta)\n", + " phi_distances.append(dphi)\n", + "\n", + " # Threshold for the 90% region: chi2 < chi2_min + Δchi2\n", + " limit = chi2_min + delta_chi2_90\n", + "\n", + " # Healpix bookkeeping\n", + " nside = hp.get_nside(chi2_map)\n", + " pix_area_deg2 = hp.nside2pixarea(nside, degrees=True)\n", + "\n", + " # Pixels inside the 90% region\n", + " mask_inside = chi2_map < limit\n", + " npix_in_region = np.sum(mask_inside)\n", + "\n", + " # 90% containment area (deg²)\n", + " area_90 = npix_in_region * pix_area_deg2\n", + " areas_90.append(area_90)\n", + "\n", + "# Summary statistics\n", + "print(np.mean(areas_90))\n", + "print(np.mean(distances))\n", + "print(np.mean(theta_distances))\n", + "print(np.mean(phi_distances))\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "955f5ace-3844-455e-98c9-5ed721c7d625", + "metadata": {}, + "outputs": [], + "source": [ + "print(np.mean(good_spectra_fit_distances))\n", + "print(np.mean(bad_spectra_fit_distances))\n", + "print(spectra_fitted/len(test_dataset)*100)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8eb13668", + "metadata": {}, + "outputs": [], + "source": [ + "import healpy as hp\n", + "import numpy as np\n", + "import matplotlib.pyplot as plt\n", + "import pickle\n", + "hp.projview(\n", + " np.array(areas_90),\n", + " coord=[\"G\"],\n", + " projection_type=\"aitoff\", \n", + " graticule=True,\n", + " graticule_labels=True,\n", + " longitude_grid_spacing=60,\n", + " title=file,\n", + " latitude_grid_spacing=30,\n", + " cmap=\"turbo\",\n", + " nest=True,\n", + " unit=\"\", \n", + " fontsize={\n", + " \"xlabel\": 14,\n", + " \"ylabel\": 14,\n", + " \"title\": 16,\n", + " \"xtick_label\": 14,\n", + " \"ytick_label\": 14,\n", + " \"cbar_label\": 14,\n", + " \"cbar_tick_label\": 14 \n", + " },\n", + " override_plot_properties={\n", + " \"cbar_shrink\": 0.8,\n", + " \"cbar_pad\": 0.05,\n", + " \"cbar_label_pad\": 5\n", + " }\n", + " \n", + ")\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5a073723-2a85-45da-a106-1b111a153781", + "metadata": {}, + "outputs": [], + "source": [ + "test_labels = []\n", + "for grb in test_dataset:\n", + " theta_real = float(grb['coord'][0])\n", + " phi_real = float(grb['coord'][1])\n", + " test_labels.append([theta_real,phi_real])\n", + "test_labels = np.array(test_labels)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f74989be-5f97-493b-a9b5-260d9754596d", + "metadata": {}, + "outputs": [], + "source": [ + "theta_rad_test,phi_rad_test = get_radians(test_labels)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "74435ec4-60a5-4262-9131-cc93c90f2c90", + "metadata": {}, + "outputs": [], + "source": [ + "def plot_aitoff(phi_rad,theta_rad,dist,title):\n", + "\n", + " plt.figure(figsize=(15,10))\n", + " plt.subplot(111, projection=\"aitoff\")\n", + " plt.title(\"\")\n", + " plt.grid(True)\n", + " \n", + " scatter = plt.scatter(phi_rad, theta_rad, c=dist, cmap='gnuplot2', alpha=0.6)\n", + " # Adding color legend\n", + " cbar = plt.colorbar(scatter, orientation='vertical', aspect=20, shrink=0.5)\n", + " cbar.set_label('Error (°)')\n", + " \n", + " plt.title(title)\n", + " plt.subplots_adjust(top=0.95,bottom=0.0)\n", + " plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "42e15be6-1a05-4d0b-ac9b-d2c2a2069208", + "metadata": {}, + "outputs": [], + "source": [ + "# After reducing the number of GRBs in the dataset it is possible to plot the results\n", + "\n", + "#plot_aitoff(phi_rad_test,theta_rad_test,areas_90,\"Area 90% c.l. results\")\n", + "#plot_aitoff(phi_rad_test,theta_rad_test,distances,\"Error results\")\n", + "#plot_aitoff(phi_rad_test,theta_rad_test,theta_distances,\"Theta errors\")\n", + "#plot_aitoff(phi_rad_test,theta_rad_test,phi_distances,\"Phi errors\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "20e6b6e4", + "metadata": {}, + "outputs": [], + "source": [ + "# Choose the index to plot. Eventually it is possible to implement a for cycle.\n", + "index = 11367\n", + "min_chi2 = results[index][7]\n", + "theta_real = float(results[index][0])\n", + "phi_real = float(results[index][1])\n", + "theta_reco=float(results[index][2])\n", + "phi_reco=float(results[index][3])\n", + "\n", + "if phi_real>180:\n", + " phi_real = phi_real-360\n", + "if phi_reco>180:\n", + " phi_reco = phi_reco-360\n", + "\n", + "m = results[index][9]\n", + "\n", + "limit = min_chi2 + chi2.ppf(0.9, 2)\n", + "\n", + "m_masked = np.ma.masked_where(m >= limit , m)\n", + "\n", + "hp.projview(\n", + " m_masked,\n", + " coord=[\"G\"],\n", + " projection_type=\"aitoff\", \n", + " graticule=True,\n", + " graticule_labels=True,\n", + " longitude_grid_spacing=60,\n", + " title=file,\n", + " latitude_grid_spacing=30,\n", + " cmap=\"viridis\",\n", + " nest=True,\n", + " unit=\"\", \n", + " badcolor=\"antiquewhite\",\n", + " fontsize={\n", + " \"xlabel\": 14,\n", + " \"ylabel\": 14,\n", + " \"title\": 16,\n", + " \"xtick_label\": 14,\n", + " \"ytick_label\": 14,\n", + " \"cbar_label\": 14,\n", + " \"cbar_tick_label\": 14 # qui imposti il font size dei numeri della colorbar\n", + " },\n", + " override_plot_properties={\n", + " \"cbar_shrink\": 0.8,\n", + " \"cbar_pad\": 0.05,\n", + " \"cbar_label_pad\": 0\n", + " }\n", + " \n", + ")\n", + "\n", + "hp.newprojplot(theta=np.radians(theta_real), phi=np.radians(phi_real), marker=\"*\", color=\"magenta\", markersize=13)\n", + "\n", + "hp.newprojplot(theta=np.radians(theta_reco), phi=np.radians(phi_reco), marker=\"X\", color=\"r\", markersize=12)\n", + "\n", + "ax = plt.gca()\n", + "\n", + "fig = plt.gcf()\n", + "axes = fig.get_axes()\n", + "main_ax = plt.gca()\n", + "\n", + "for ax in axes:\n", + " if ax != main_ax:\n", + " cbar_ax = ax\n", + " break\n", + "\n", + "start = int(np.floor(m_masked.min()))\n", + "end = int(np.ceil(m_masked.max()))\n", + "\n", + "cbar_ax.set_xlabel(\"$\\chi^2$ value (90% c.l.)\", fontsize=14) \n", + "\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "eebf88d4", + "metadata": {}, + "outputs": [], + "source": [ + "#Export the data to be plotted with plots/plot_aitoff.py\n", + "if False:\n", + " index = 16257\n", + " # Salva l'array in un file usando pickle\n", + " with open(data_path+\"/areas_chi2_\"+file_name_test+\"_\"+str(index)+\"_\"+str(results[index][0])+\"_\"+str(results[index][1])+\"_\"+str(results[index][2])+\"_\"+str(results[index][3])+\"_\"+str(round(results[index][7],4))+\".pkl\", \"wb\") as f:\n", + " pickle.dump(results[index][9], f)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b5e811f3-1018-4417-aacc-d529e6e8debd", + "metadata": {}, + "outputs": [], + "source": [ + "# Define 5-degree intervals\n", + "intervals = np.arange(0, 185, 5)\n", + "\n", + "# Group counts based on 5-degree intervals\n", + "grouped_counts = np.zeros((len(intervals)))\n", + "counts = np.zeros((len(intervals)))\n", + "\n", + "total_count = 0\n", + "for theta, phi, area in zip(test_labels[:, 0], test_labels[:, 1], areas_90):\n", + " \n", + " if True: # (phi > 125 and phi < 145) or \n", + " total_count += 1\n", + " interval_index = int(theta // 5)\n", + " grouped_counts[interval_index] = grouped_counts[interval_index] + area\n", + " counts[interval_index] += 1\n", + "\n", + "# Compute the mean of counts in each interval\n", + "grouped_counts = grouped_counts[:total_count]\n", + "counts = counts[:total_count]\n", + "\n", + "# Plot the histogram\n", + "plt.figure(figsize=(10, 6))\n", + "plt.bar(\n", + " intervals[:-1],\n", + " grouped_counts[:-1] / counts[:-1],\n", + " width=5,\n", + " align='edge',\n", + " alpha=1,\n", + " label=\"90% c.l. error area\"\n", + ")\n", + "plt.xlabel('Theta (deg)', fontsize=15)\n", + "plt.ylabel('90% c.l. error area (deg$^2$)', fontsize=15)\n", + "plt.title('90% c.l. error area as a function of Theta', fontsize=15)\n", + "plt.tick_params(axis='both', labelsize=15) \n", + "# plt.legend(fontsize=13)\n", + "plt.grid(True)\n", + "# plt.legend()\n", + "plt.show()\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9c6dadfd", + "metadata": {}, + "outputs": [], + "source": [ + "#Export data to create the histogram\n", + "if(False):\n", + "\n", + " training_histo = {'test_labels': test_labels,\"distances\":distances,\"areas_90\":areas_90}\n", + "\n", + " with open(data_path+\"/histo_chi2_area_nobkg.pkl\", \"wb\") as f:\n", + " pickle.dump(training_histo, f)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a1878d20-7d7f-4c16-b892-250699b3ebdb", + "metadata": {}, + "outputs": [], + "source": [ + "# Define 5-degree intervals\n", + "intervals = np.arange(0, 185, 5)\n", + "\n", + "# Group values based on 5-degree intervals\n", + "grouped_values = np.zeros((len(intervals)))\n", + "counts = np.zeros((len(intervals)))\n", + "\n", + "total_count = 0\n", + "for theta, phi, value in zip(test_labels[:, 0], test_labels[:, 1], distances):\n", + " \n", + " if True: # (phi > 125 and phi < 145) or \n", + " total_count += 1\n", + " interval_index = int(theta // 5)\n", + " grouped_values[interval_index] = grouped_values[interval_index] + value\n", + " counts[interval_index] += 1\n", + "\n", + "# Compute the mean of the values in each interval\n", + "grouped_values = grouped_values[:total_count]\n", + "counts = counts[:total_count]\n", + "\n", + "# Plot the histogram\n", + "plt.figure(figsize=(10, 6))\n", + "plt.bar(\n", + " intervals[:-1],\n", + " grouped_values[:-1] / counts[:-1],\n", + " width=5,\n", + " align='edge',\n", + " alpha=1,\n", + " label=\"loc. error\"\n", + ")\n", + "plt.xlabel('Theta (°)', fontsize=15)\n", + "plt.ylabel('Loc. error (°)', fontsize=15)\n", + "plt.title('Loc. error as a function of theta', fontsize=15)\n", + "plt.tick_params(axis='both', labelsize=15) \n", + "plt.grid(True)\n", + "# plt.legend()\n", + "plt.show()\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b074caa0", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "03e9eb91", + "metadata": {}, + "outputs": [], + "source": [ + "step = 10\n", + "\n", + "# Define intervals of 10 degrees\n", + "intervals = np.arange(0, 365, step)\n", + "\n", + "# Group counts based on 10-degree intervals\n", + "grouped_counts_1 = np.zeros((len(intervals)))\n", + "counts_1 = np.zeros((len(intervals)))\n", + "\n", + "grouped_counts_2 = np.zeros((len(intervals)))\n", + "counts_2 = np.zeros((len(intervals)))\n", + "\n", + "grouped_counts_3 = np.zeros((len(intervals)))\n", + "counts_3 = np.zeros((len(intervals)))\n", + "\n", + "grouped_counts_4 = np.zeros((len(intervals)))\n", + "counts_4 = np.zeros((len(intervals)))\n", + "\n", + "total_count_1 = 0\n", + "total_count_2 = 0\n", + "total_count_3 = 0\n", + "total_count_4 = 0\n", + "\n", + "for theta, phi, value in zip(test_labels[:, 0], test_labels[:, 1], areas_90):\n", + " \n", + " if (theta > 20 and theta < 55):\n", + " total_count_1 += 1\n", + " interval_index = int(phi // step)\n", + " grouped_counts_1[interval_index] = grouped_counts_1[interval_index] + value\n", + " counts_1[interval_index] += 1\n", + "\n", + " if (theta > 55 and theta < 130):\n", + " total_count_2 += 1\n", + " interval_index = int(phi // step)\n", + " grouped_counts_2[interval_index] = grouped_counts_2[interval_index] + value\n", + " counts_2[interval_index] += 1\n", + "\n", + " if (theta > 130 and theta < 180):\n", + " total_count_3 += 1\n", + " interval_index = int(phi // step)\n", + " grouped_counts_3[interval_index] = grouped_counts_3[interval_index] + value\n", + " counts_3[interval_index] += 1\n", + "\n", + "# Compute the mean of counts in each interval\n", + "grouped_counts_1 = grouped_counts_1[:total_count_1]\n", + "counts_1 = counts_1[:total_count_1]\n", + "\n", + "grouped_counts_2 = grouped_counts_2[:total_count_2]\n", + "counts_2 = counts_2[:total_count_2]\n", + "\n", + "grouped_counts_3 = grouped_counts_3[:total_count_3]\n", + "counts_3 = counts_3[:total_count_3]\n", + "\n", + "# Plot the histogram\n", + "plt.figure(figsize=(10, 6))\n", + "plt.bar(\n", + " intervals[:-1],\n", + " grouped_counts_1[:-1] / counts_1[:-1],\n", + " width=step,\n", + " align='edge',\n", + " alpha=0.5,\n", + " label=\"loc. error theta=[20°,55°]\"\n", + ")\n", + "\n", + "plt.bar(\n", + " intervals[:-1],\n", + " grouped_counts_2[:-1] / counts_2[:-1],\n", + " width=step,\n", + " align='edge',\n", + " alpha=0.5,\n", + " label=\"loc. error theta=[55°,150°]\"\n", + ")\n", + "\n", + "plt.bar(\n", + " intervals[:-1],\n", + " grouped_counts_3[:-1] / counts_3[:-1],\n", + " width=step,\n", + " align='edge',\n", + " alpha=0.5,\n", + " label=\"loc. error theta=[150°,180°]\"\n", + ")\n", + "\n", + "plt.xlabel('Phi (°)')\n", + "plt.ylabel('Loc. error')\n", + "plt.title('Loc. error as a function of phi')\n", + "plt.grid(True)\n", + "plt.legend()\n", + "plt.show()\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "155da8b3-cda9-4124-abb0-ad6a3a445bde", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "443372c9-83c2-45bf-8efe-4aaf311b52ba", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7dc9aaca-3c0a-41fa-9b11-851a53b7cc6f", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "deeplearning", + "language": "python", + "name": "deeplearning" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.12" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/cosipy/notebooks/chi2/chi2_localization_plot.ipynb b/cosipy/notebooks/chi2/chi2_localization_plot.ipynb new file mode 100644 index 0000000..c6d6920 --- /dev/null +++ b/cosipy/notebooks/chi2/chi2_localization_plot.ipynb @@ -0,0 +1,602 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "0", + "metadata": {}, + "outputs": [], + "source": [ + "# import libraries\n", + "\n", + "import pandas as pd\n", + "import numpy as np\n", + "import seaborn as sns\n", + "sns.set(color_codes=True)\n", + "import matplotlib.pyplot as plt\n", + "%matplotlib inline\n", + "import pickle\n", + "import math \n", + "\n", + "#update this path with the path where you stored the datasets\n", + "data_path = \"/data/test_newrepo\" " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2bec492e-6f09-46a6-9c4b-70a6dbab85f6", + "metadata": {}, + "outputs": [], + "source": [ + "#Load LUTs\n", + "\n", + "lut_name_soft = \"soft_lut\"\n", + "lut_name_medium = \"medium_lut\"\n", + "lut_name_hard = \"hard_lut\"\n", + "\n", + "lut_soft = np.load(data_path+'/LUT_'+lut_name_soft+'.npy')\n", + "lut_medium = np.load(data_path+'/LUT_'+lut_name_medium+'.npy')\n", + "lut_hard = np.load(data_path+'/LUT_'+lut_name_hard+'.npy')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "52d9a03f-7ee3-4b51-adfd-6de090fa883f", + "metadata": {}, + "outputs": [], + "source": [ + "lut_medium.shape" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "84ba1c04-559d-4551-af10-c025420e8d91", + "metadata": {}, + "outputs": [], + "source": [ + "#test with LUTs\n", + "run_name_test = \"dataset\"\n", + "file_name_test = \"run63_medium_10fact_mega_shared\"\n", + "\n", + "file_path = data_path+'/'+file_name_test+'_dataset.pkl'\n", + "\n", + "# Load the array from the pickle file\n", + "with open(file_path, 'rb') as file:\n", + " loaded_array_test = pickle.load(file)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "96bf19b5-89c3-4d70-bdda-309b8e66214a", + "metadata": {}, + "outputs": [], + "source": [ + "loaded_array_test.shape" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a8fa32a8-21d5-4d6f-81e8-6d2208c85fda", + "metadata": {}, + "outputs": [], + "source": [ + "# Use these parameters to filter the dataset. Put them equal to 0 to use all the dataset.\n", + "\n", + "filter_flux = 0\n", + "filter_spectra = 0\n", + "filter_theta=0\n", + "filter_phi=0\n", + "\n", + "filtered_loaded_array_test = np.empty(loaded_array_test.shape[0], dtype=object)\n", + "\n", + "count = 0\n", + "\n", + "if filter_flux == 1:\n", + " for grb in loaded_array_test:\n", + " if grb['flux'] >1 and grb['flux'] <= 1.6:\n", + " filtered_loaded_array_test[count] = grb\n", + " count = count+1\n", + "\n", + " loaded_array_test = filtered_loaded_array_test[:count]\n", + "\n", + "print(len(loaded_array_test))\n", + "filtered_loaded_array_test = np.empty(loaded_array_test.shape[0], dtype=object)\n", + "\n", + "count = 0\n", + "\n", + "if filter_spectra==1:\n", + " for grb in loaded_array_test:\n", + " if \"1500\" in grb['spectrum']: #[\"Band 10 10000 -1.9 -3.7 230\",\"Band 10 10000 -1 -2.3 699.9\",\"Comptonized 10 10000 -0.5 1500\"]\n", + " filtered_loaded_array_test[count] = grb\n", + " count = count+1\n", + "\n", + " loaded_array_test = filtered_loaded_array_test[:count]\n", + "\n", + "print(len(loaded_array_test))\n", + "count = 0\n", + "filtered_loaded_array_test = np.empty(loaded_array_test.shape[0], dtype=object)\n", + "\n", + "if filter_theta==1:\n", + " print(\"filter spectra\")\n", + " for grb in loaded_array_test:\n", + " if float(grb['coord'][0])>50 and float(grb['coord'][0])<130:\n", + " filtered_loaded_array_test[count] = grb\n", + " count = count+1\n", + "\n", + " loaded_array_test = filtered_loaded_array_test[:count]\n", + "\n", + "count = 0\n", + "filtered_loaded_array_test = np.empty(loaded_array_test.shape[0], dtype=object)\n", + "\n", + "if filter_phi==1:\n", + " print(\"filter spectra\")\n", + " for grb in loaded_array_test:\n", + " if float(grb['coord'][1])>100 and float(grb['coord'][1])<170:\n", + " filtered_loaded_array_test[count] = grb\n", + " count = count+1\n", + "\n", + " loaded_array_test = filtered_loaded_array_test[:count]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "eea260a0-39ed-4504-9450-eb0b3a256cce", + "metadata": {}, + "outputs": [], + "source": [ + "test_dataset = loaded_array_test\n", + "print(test_dataset.shape)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5", + "metadata": {}, + "outputs": [], + "source": [ + "def get_radians(coords):\n", + " # Unpack the list of (theta, phi) pairs\n", + " theta, phi = zip(*coords)\n", + " \n", + " # Convert to numpy arrays\n", + " theta = np.array(theta)\n", + " phi = np.array(phi)\n", + " \n", + " # Wrap phi values >180 into the range (-180, 180]\n", + " mask = phi > 180\n", + " phi[mask] -= 360\n", + " \n", + " # Convert theta into colatitude (90 - theta)\n", + " theta = 90 - theta\n", + "\n", + " # Return values in radians\n", + " return np.radians(theta), np.radians(phi)\n", + "\n", + "def calculate_chi_squared_optimized(s, b, m):\n", + " \"\"\"\n", + " Compute the chi-squared value for each position in the grid in an optimized way.\n", + "\n", + " Parameters:\n", + " s: array of shape (6,) with observed counts (s(j)).\n", + " b: array of shape (12,) with background counts (b(j)).\n", + " m: array of shape (12, 41168) with model counts (m(j, i)).\n", + "\n", + " Returns:\n", + " chi_squared: array of shape (41168,) with the chi-squared value for each position i.\n", + " \"\"\"\n", + " # Use only the first 6 detectors\n", + " indices = np.arange(6)\n", + " \n", + " # Extract model counts for these detectors\n", + " m_subset = m[:, indices] # Shape: (41168, 6)\n", + "\n", + " # Compute numerator and denominator for normalization factor f_i\n", + " with np.errstate(divide='ignore', invalid='ignore'):\n", + " numerator = np.sum(m_subset * (s[indices] - b[indices]) / s[indices], axis=1)\n", + " denominator = np.sum((m_subset**2) / s[indices], axis=1)\n", + "\n", + " # Avoid division by zero\n", + " f_i = np.divide(numerator, denominator, out=np.zeros_like(numerator), where=denominator != 0)\n", + "\n", + " # Expand f_i for broadcasting\n", + " f_i_expanded = f_i[:, np.newaxis] # Shape: (41168, 1)\n", + "\n", + " # Compute chi-squared terms\n", + " chi_numerator = (s[indices] - b[indices] - f_i_expanded * m_subset)**2\n", + " chi_denominator = b[indices] + f_i_expanded * m_subset\n", + "\n", + " # Safe division for chi-squared elements\n", + " chi_squared_elements = np.divide(\n", + " chi_numerator, chi_denominator,\n", + " out=np.full_like(chi_numerator, np.finfo(np.float64).max),\n", + " where=chi_denominator != 0\n", + " )\n", + "\n", + " # Sum over detectors to obtain chi^2 per direction\n", + " chi_squared = np.sum(chi_squared_elements, axis=1)\n", + "\n", + " return chi_squared\n", + "\n", + "\n", + "def angular_distance(theta1, phi1, theta2, phi2):\n", + " \"\"\"\n", + " Compute the angular (great-circle) distance in degrees between two points\n", + " specified by (theta, phi) coordinates in degrees.\n", + "\n", + " Parameters:\n", + " theta1, phi1: floats, coordinates of the first point (degrees).\n", + " theta2, phi2: floats, coordinates of the second point (degrees).\n", + "\n", + " Returns:\n", + " angular_dist_deg: float, angular distance between the two points in degrees.\n", + " \"\"\"\n", + " # Adjust theta values by shifting -90\n", + " theta1 = 90 - theta1\n", + " theta2 = 90 - theta2\n", + "\n", + " # Convert angles to radians\n", + " theta1_rad = math.radians(theta1)\n", + " phi1_rad = math.radians(phi1)\n", + " theta2_rad = math.radians(theta2)\n", + " phi2_rad = math.radians(phi2)\n", + " \n", + " # Compute the difference in longitude\n", + " delta_phi = abs(phi1_rad - phi2_rad)\n", + "\n", + " # Apply spherical law of cosines\n", + " value = (math.sin(theta1_rad) * math.sin(theta2_rad) +\n", + " math.cos(theta1_rad) * math.cos(theta2_rad) * math.cos(delta_phi))\n", + " \n", + " # Clamp value to [-1, 1] to avoid floating point errors in acos\n", + " value_clamped = max(-1.0, min(1.0, value)) \n", + " \n", + " angular_dist = math.acos(value_clamped)\n", + " \n", + " # Convert angular distance from radians to degrees\n", + " angular_dist_deg = math.degrees(angular_dist)\n", + " \n", + " return angular_dist_deg\n", + "\n", + "def diff_phi(a1, a2):\n", + " diff = abs(a1 - a2)\n", + " if diff > 180:\n", + " diff = 360 - diff\n", + " return diff" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "362420a6", + "metadata": {}, + "outputs": [], + "source": [ + "def find_closest_pixel(theta_real, phi_real, best_lut):\n", + " # Convert input angles from degrees to radians\n", + " theta_real_rad = np.deg2rad(theta_real)\n", + " phi_real_rad = np.deg2rad(phi_real)\n", + " \n", + " # Extract θ and φ from best_lut (last two columns, in degrees) and convert to radians\n", + " theta_vals_m = np.deg2rad(best_lut[:, -2])\n", + " phi_vals_m = np.deg2rad(best_lut[:, -1])\n", + " \n", + " # Compute the cosine of the spherical angular distance\n", + " cos_dist = (\n", + " np.sin(theta_real_rad) * np.sin(theta_vals_m) * np.cos(phi_real_rad - phi_vals_m)\n", + " + np.cos(theta_real_rad) * np.cos(theta_vals_m)\n", + " )\n", + "\n", + " # Compute the angular distance (in radians)\n", + " angular_distance = np.arccos(cos_dist)\n", + " \n", + " # Find the index of the minimum distance\n", + " min_idx = np.argmin(angular_distance)\n", + " \n", + " # The index of the maximum cosine corresponds to the minimum distance\n", + " # min_idx = np.argmax(cos_dist)\n", + " return min_idx" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0e6fe33d-663a-4123-8872-8b62032e1555", + "metadata": {}, + "outputs": [], + "source": [ + "import healpy as hp\n", + "import numpy as np\n", + "from scipy.stats import chi2\n", + "\n", + "def analyze_grbs(test_dataset,with_background):\n", + "\n", + " results = []\n", + " spectra_fitted = 0\n", + " good_spectra_fit_distances = []\n", + " bad_spectra_fit_distances=[]\n", + " \n", + " # Background counts for the 6 detectors found in DC3 background data\n", + " b_sim = np.array([57.6053, 58.7157, 51.4131, 48.2891, 47.7293, 45.9617])\n", + "\n", + " count = 0\n", + " for grb_list in test_dataset:\n", + "\n", + " dist_list = []\n", + " area_list = []\n", + " \n", + " for grb in grb_list:\n", + " \n", + " count = count +1\n", + " \n", + " if count % 1000 == 0:\n", + " print(count)\n", + " \n", + " counts = grb['counts']\n", + " spectrum = grb['spectrum']\n", + " if \"230\" in spectrum :\n", + " spectra_value=\"soft\"\n", + " elif \"699.9\" in spectrum:\n", + " spectra_value=\"medium\"\n", + " elif \"Compton\" in spectrum:\n", + " spectra_value=\"hard\"\n", + " else:\n", + " spectra_value=\"random\" \n", + " \n", + " theta_real = float(grb['coord'][0])\n", + " phi_real = float(grb['coord'][1])\n", + "\n", + " \n", + " \n", + " if with_background==1: \n", + " \n", + " rand_bkg = np.random.poisson(b_sim*20)\n", + " chi_squared_medium = calculate_chi_squared_optimized(np.array(counts)+rand_bkg,b_sim*20, lut_medium)\n", + " min_medium = np.min(chi_squared_medium)\n", + " \n", + " elif with_background==0:\n", + " \n", + " chi_squared_medium = calculate_chi_squared_optimized(np.array(counts), np.array([0,0,0,0,0,0]), lut_medium)\n", + " min_medium = np.min(chi_squared_medium)\n", + " \n", + " elif with_background==2:\n", + " \n", + " chi_squared_medium = calculate_chi_squared_optimized(np.array(lut_medium[count][:6])+b_sim*20,b_sim*20, lut_medium)\n", + " min_medium = np.min(chi_squared_medium)\n", + " \n", + " \n", + " global_min = min_medium \n", + " \n", + " spectra_selected = False\n", + " \n", + " \n", + " if global_min == min_medium:\n", + " chi2_array = chi_squared_medium\n", + " if spectra_value == \"medium\":\n", + " spectra_fitted = spectra_fitted +1\n", + " spectra_selected = True\n", + " argmin_index = np.argmin(chi_squared_medium)\n", + " best_lut = lut_medium\n", + " \n", + " \n", + " theta_loc = best_lut[argmin_index][6]\n", + " phi_loc = best_lut[argmin_index][7]\n", + " \n", + " theta_real = float(grb['coord'][0])\n", + " phi_real = float(grb['coord'][1])\n", + " \n", + " dist = angular_distance(theta_loc,phi_loc,theta_real,phi_real)\n", + " \n", + " if spectra_selected:\n", + " good_spectra_fit_distances.append(dist)\n", + " else:\n", + " bad_spectra_fit_distances.append(dist)\n", + " \n", + " map = chi2_array\n", + " limit = global_min+ chi2.ppf(0.9, 2)\n", + " \n", + " # Maschera i valori maggiori o uguali a X\n", + " nside = hp.npix2nside(len(chi2_array))\n", + " mappa_masked = np.ma.masked_where(map >= limit , map)\n", + " npix_in_region = np.sum(~mappa_masked.mask)\n", + " pix_area = hp.nside2pixarea(nside, degrees=True) \n", + " area_90 = npix_in_region * pix_area\n", + " \n", + " dist_list.append(dist)\n", + " area_list.append(area_90)\n", + "\n", + " distance = np.mean(dist_list)\n", + " area = np.mean(area_list)\n", + " \n", + " results.append([distance,area])\n", + "\n", + " return results,spectra_fitted, good_spectra_fit_distances,bad_spectra_fit_distances" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1930d232-2ac1-489d-af06-d0437714d64f", + "metadata": {}, + "outputs": [], + "source": [ + "# 0 = no background, 1 = poissonian background, 2 = use LUTs as source and add background\n", + "\n", + "results,spectra_fitted, good_spectra_fit_distances,bad_spectra_fit_distances= analyze_grbs(test_dataset,0)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "965cdcf0-c4c9-43ab-8e34-007267395132", + "metadata": {}, + "outputs": [], + "source": [ + "distances = []\n", + "area_array = []\n", + "\n", + "for r in results:\n", + " distances.append(r[0])\n", + " area_array.append(r[1])\n", + "\n", + "distances = np.array(distances)\n", + "area_array = np.array(area_array)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "825dd4bc", + "metadata": {}, + "outputs": [], + "source": [ + "np.mean(distances)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5c43b4b7-e303-42f2-8385-5921e8c39409", + "metadata": {}, + "outputs": [], + "source": [ + "import pickle\n", + "if False:\n", + "\n", + " # Salva l'array in un file usando pickle\n", + " with open(prefix+\"/data/chi2_\"+file_name_test+\"_distall.pkl\", \"wb\") as f:\n", + " pickle.dump(distances, f)\n", + " \n", + " # Salva l'array in un file usando pickle\n", + " with open(prefix+\"/data/chi2_\"+file_name_test+\"_cont_area.pkl\", \"wb\") as f:\n", + " pickle.dump(area_array, f)\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2cedcd94-1fee-4ffa-b2e5-9ae217a41b42", + "metadata": {}, + "outputs": [], + "source": [ + "import healpy as hp\n", + "import numpy as np\n", + "import matplotlib.pyplot as plt\n", + "import pickle\n", + "hp.projview(\n", + " np.array(area_array),\n", + " coord=[\"G\"],\n", + " projection_type=\"aitoff\", \n", + " graticule=True,\n", + " graticule_labels=True,\n", + " longitude_grid_spacing=60,\n", + " title=file,\n", + " latitude_grid_spacing=30,\n", + " cmap=\"turbo\",\n", + " nest=True,\n", + " unit=\"\", \n", + " fontsize={\n", + " \"xlabel\": 14,\n", + " \"ylabel\": 14,\n", + " \"title\": 16,\n", + " \"xtick_label\": 14,\n", + " \"ytick_label\": 14,\n", + " \"cbar_label\": 14,\n", + " \"cbar_tick_label\": 14 \n", + " },\n", + " override_plot_properties={\n", + " \"cbar_shrink\": 0.8,\n", + " \"cbar_pad\": 0.05,\n", + " \"cbar_label_pad\": 5\n", + " }\n", + " \n", + ")\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ed4a4d1d-3fa0-446a-b24d-ca3a9c9bb775", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e1b0a0d9-b3c4-480e-aff8-e31433ccb845", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "dd0d8a6a-c69f-43c5-bef9-de8a6c7a37c3", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "af0b7d7f-a3c3-44f4-8b4a-d7c1900e39ac", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e08297b6-e584-47dd-945b-5186681f4735", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f5f2990a-9feb-4ed3-9c88-a5b29f100f9b", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a87d20aa-d95e-4f97-bb13-82a2c2277cee", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "deeplearning2", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.12" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/cosipy/notebooks/dl-loc/README.md b/cosipy/notebooks/dl-loc/README.md new file mode 100644 index 0000000..cfb70a8 --- /dev/null +++ b/cosipy/notebooks/dl-loc/README.md @@ -0,0 +1,16 @@ +## Training and inference notebooks + +This folder contains notebooks to train and test deep-learning models for source localization. It includes: + +- **Training notebooks** for three model variants: + - **Degrees model**: predicts position angles in degrees (theta, phi). + - **Sin/Cos model**: predicts labels transformed with sine and cosine of the angles. + - **Cartesian model**: predicts unit-vector Cartesian coordinates (x, y, z) on the sphere. + +- **Inference notebooks** to evaluate trained models and measure localization performance on validation/test data. + +- **Plot inference notebooks** that use the plot dataset to generate HEALPix probability maps from model outputs. + +All notebooks are located in `dl-loc/train_and_test/`. + + diff --git a/cosipy/notebooks/dl-loc/inpsect_dataset.ipynb b/cosipy/notebooks/dl-loc/inpsect_dataset.ipynb new file mode 100644 index 0000000..5600b0e --- /dev/null +++ b/cosipy/notebooks/dl-loc/inpsect_dataset.ipynb @@ -0,0 +1,781 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "5550f091-bcac-4cc2-8b1b-6b700b5b8240", + "metadata": {}, + "outputs": [], + "source": [ + "# import libraries\n", + "\n", + "import numpy as np\n", + "import seaborn as sns\n", + "sns.set(color_codes=True)\n", + "import matplotlib.pyplot as plt\n", + "%matplotlib inline\n", + "import pickle\n", + "import matplotlib.pyplot as plt\n", + "import numpy as np\n", + "\n", + "data_path = \"/data/test_newrepo/\"\n", + "file_name = \"run52_hard_mega_shared\"\n", + "normalize= 0\n", + "numpy_file=0\n", + "number_of_detectors = 6" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "fdf3f54b-5a9c-4633-a007-28df8522c7de", + "metadata": {}, + "outputs": [], + "source": [ + "file_path = data_path+'/'+file_name+'_dataset.pkl'\n", + "\n", + "# Load the array from the pickle file\n", + "with open(file_path, 'rb') as file:\n", + " data_array = pickle.load(file)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "aa3d93ea-c3a9-4bae-b2e8-e42dc12c93e0", + "metadata": {}, + "outputs": [], + "source": [ + "def process_dataset(data_array):\n", + "\n", + " dataset = np.empty((len(data_array),number_of_detectors))\n", + " labels = np.empty((len(data_array),2))\n", + " \n", + " count = -1\n", + " for element in data_array:\n", + " \n", + " count+=1\n", + " \n", + " dataset[count] = element['counts']\n", + " labels[count][0] = float(element['coord'][0])\n", + " labels[count][1] = float(element['coord'][1])\n", + " \n", + " labels = labels[:count+1]\n", + " dataset = dataset[:count+1]\n", + "\n", + " #scale labels\n", + " max_lon = 180.0\n", + " min_lon = 0.0\n", + " \n", + " max_lat = 360.0\n", + " min_lat = 0.0\n", + " \n", + " labels_norm = np.empty((len(dataset),2))\n", + " \n", + " count = -1\n", + " for element in labels:\n", + " \n", + " count+=1\n", + " if normalize==1:\n", + " labels_norm[count][0] = (labels[count][0]-min_lon)/(max_lon-min_lon)\n", + " labels_norm[count][1] = (labels[count][1]-min_lat)/(max_lat-min_lat)\n", + " else:\n", + " labels_norm[count][0] = labels[count][0]\n", + " labels_norm[count][1] = labels[count][1]\n", + "\n", + " # Convert to radians\n", + " coords_rad = []\n", + " \n", + " for theta, phi in labels:\n", + " #if lon > 90 and lon < 270:\n", + " \n", + " coords_rad.append([theta, phi])\n", + " #else:\n", + " # continue\n", + "\n", + " theta, phi = zip(*coords_rad)\n", + " \n", + " \n", + " return dataset, labels, labels_norm, theta, phi, coords_rad\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e1f89ef0-7331-4c15-a95f-9ed955661e71", + "metadata": {}, + "outputs": [], + "source": [ + "dataset, labels, labels, theta, phi, coords_rad = process_dataset(data_array)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "876478df-7f89-4c25-a734-3b4420258274", + "metadata": {}, + "outputs": [], + "source": [ + "counts = np.empty((len(dataset),number_of_detectors))\n", + "index = -1\n", + "for element in dataset:\n", + " index=index+1\n", + " counts[index] = element" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3a78737f-ea7e-40c0-a08f-86e79040d8e4", + "metadata": {}, + "outputs": [], + "source": [ + "# Define 5-degree bins\n", + "bins = np.arange(0, 185, 5)\n", + "\n", + "# Group counts by 5-degree bins\n", + "grouped_counts = np.zeros((len(bins), number_of_detectors))\n", + "counts_per_bin = np.zeros((len(bins), number_of_detectors))\n", + "\n", + "total_count = 0\n", + "for i in range(0, number_of_detectors):\n", + " for theta, phi, count in zip(labels[:, 0], labels[:, 1], counts[:, i]):\n", + " \n", + " if (phi > -5 and phi < 5):\n", + " total_count += 1\n", + " bin_index = int(theta // 5)\n", + " grouped_counts[bin_index][i] = grouped_counts[bin_index][i] + count\n", + " counts_per_bin[bin_index][i] += 1\n", + "\n", + "# Compute the mean counts in each bin\n", + "grouped_counts = grouped_counts[:total_count]\n", + "counts_per_bin = counts_per_bin[:total_count]\n", + "\n", + "# Plot histogram\n", + "plt.figure(figsize=(10, 6))\n", + "#plt.bar(bins, grouped_counts[:,0]/counts_per_bin[:,0], width=5, align='edge', alpha=.5, label=\"z1\")\n", + "#plt.bar(bins, grouped_counts[:,1]/counts_per_bin[:,1], width=5, align='edge', alpha=.5, label=\"z0\")\n", + "plt.bar(bins, grouped_counts[:,2]/counts_per_bin[:,2], width=5, align='edge', alpha=.5, label=\"BGO_X1\")\n", + "plt.bar(bins, grouped_counts[:,3]/counts_per_bin[:,3], width=5, align='edge', alpha=.5, label=\"BGO_X0\")\n", + "#plt.bar(bins, grouped_counts[:,4]/counts_per_bin[:,4], width=5, align='edge', alpha=.5, label=\"BGO_Y1\")\n", + "#plt.bar(bins, grouped_counts[:,5]/counts_per_bin[:,5], width=5, align='edge', alpha=.5, label=\"BGO_Y0\")\n", + "plt.title('')\n", + "plt.grid(True)\n", + "plt.xlabel('Theta (°)', fontsize=15)\n", + "plt.ylabel('Panel mean counts', fontsize=15)\n", + "plt.tick_params(axis='both', labelsize=15) \n", + "plt.legend(fontsize=15)\n", + "plt.show()\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1c748b41-d95b-4a99-bfb2-1b0c2961660e", + "metadata": {}, + "outputs": [], + "source": [ + "import numpy as np\n", + "import matplotlib.pyplot as plt\n", + "\n", + "# Define 5-degree bins\n", + "bins = np.arange(0, 185, 5)\n", + "n_bins = len(bins)\n", + "n_panels = number_of_detectors\n", + "\n", + "# Initialize lists for each bin and each panel\n", + "grouped_counts = [[[] for _ in range(n_panels)] for _ in range(n_bins)]\n", + "\n", + "# Collect counts into bins\n", + "for i in range(n_panels):\n", + " for theta, phi, count in zip(labels[:, 0], labels[:, 1], counts[:, i]):\n", + " if 355 < phi or phi < 5:\n", + " bin_index = int(theta // 5)\n", + " if bin_index < n_bins:\n", + " grouped_counts[bin_index][i].append(count)\n", + "\n", + "# Compute mean and standard deviation for each bin and panel\n", + "mean_counts = np.zeros((n_bins, n_panels))\n", + "std_dev = np.zeros((n_bins, n_panels))\n", + "\n", + "for b in range(n_bins):\n", + " for p in range(n_panels):\n", + " data = grouped_counts[b][p]\n", + " if len(data) > 0:\n", + " mean_counts[b][p] = np.mean(data)\n", + " std_dev[b][p] = 3 * np.std(data) # 3σ error bars\n", + " else:\n", + " mean_counts[b][p] = np.nan\n", + " std_dev[b][p] = 0\n", + "\n", + "# Plot\n", + "plt.figure(figsize=(10, 6))\n", + "width = 5 # slightly less than 5 to separate panels visually\n", + "\n", + "# Offsets for different panels in the bar plot (not used here but can be added)\n", + "offsets = [-2, -1, 0, 1, 2, 3]\n", + "\n", + "panel_labels = [\"z1\", \"z0\", \"BGO_X1\", \"BGO_X0\", \"BGO_Y1\", \"BGO_Y0\"]\n", + "colors = ['b', 'g', 'r', 'c', 'm', 'y']\n", + "\n", + "for i in range(2, 6): # Adjust range if you want to show all panels\n", + " x_pos = bins # + offsets[i] if you want panel separation\n", + " plt.bar(\n", + " x_pos, \n", + " mean_counts[:, i], \n", + " width=width, \n", + " alpha=0.5, \n", + " label=panel_labels[i], \n", + " yerr=std_dev[:, i], \n", + " capsize=3, \n", + " color=colors[i]\n", + " )\n", + "\n", + "plt.xlabel('Theta (°)')\n", + "plt.ylabel('Panel mean counts')\n", + "plt.title('')\n", + "plt.grid(True)\n", + "plt.legend()\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "265e4762-dd91-4741-bd6e-b956bacace13", + "metadata": {}, + "outputs": [], + "source": [ + "# Define 5-degree bins\n", + "bins = np.arange(0, 185, 5)\n", + "\n", + "# Group counts by 5-degree bins\n", + "grouped_counts = np.zeros((len(bins), number_of_detectors))\n", + "counts_per_bin = np.zeros((len(bins), number_of_detectors))\n", + "\n", + "total_count = 0\n", + "for i in range(0, number_of_detectors):\n", + " for theta, phi, count in zip(labels[:, 0], labels[:, 1], counts[:, i]):\n", + " \n", + " if (phi > -5 and phi < 5):\n", + " total_count += 1\n", + " bin_index = int(theta // 5)\n", + " grouped_counts[bin_index][i] = grouped_counts[bin_index][i] + count\n", + " counts_per_bin[bin_index][i] += 1\n", + "\n", + "# Compute the mean counts in each bin\n", + "grouped_counts = grouped_counts[:total_count]\n", + "counts_per_bin = counts_per_bin[:total_count]\n", + "\n", + "# Plot histogram\n", + "plt.figure(figsize=(10, 6))\n", + "plt.bar(bins, grouped_counts[:, 0] / counts_per_bin[:, 0], width=5, align='edge', alpha=.5, label=\"BGO_Z1\")\n", + "plt.bar(bins, grouped_counts[:, 1] / counts_per_bin[:, 1], width=5, align='edge', alpha=.5, label=\"BGO_Z0\")\n", + "#plt.bar(bins, grouped_counts[:, 2] / counts_per_bin[:, 2], width=5, align='edge', alpha=.5, label=\"BGO_X1\")\n", + "#plt.bar(bins, grouped_counts[:, 3] / counts_per_bin[:, 3], width=5, align='edge', alpha=.5, label=\"BGO_X0\")\n", + "#plt.bar(bins, grouped_counts[:, 4] / counts_per_bin[:, 4], width=5, align='edge', alpha=.5, label=\"BGO_Y1\")\n", + "#plt.bar(bins, grouped_counts[:, 5] / counts_per_bin[:, 5], width=5, align='edge', alpha=.5, label=\"BGO_Y0\")\n", + "\n", + "plt.xlabel('Theta (°)', fontsize=15)\n", + "plt.ylabel('Panel mean counts', fontsize=15)\n", + "plt.tick_params(axis='both', labelsize=15) \n", + "plt.legend(fontsize=15, loc=\"upper left\")\n", + "plt.title('')\n", + "plt.grid(True)\n", + "\n", + "plt.show()\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b1002cb6-7aa2-436e-83a0-787e7abce7e4", + "metadata": {}, + "outputs": [], + "source": [ + "for phi_angle in range(0, 360, 10):\n", + " \n", + " print(phi_angle)\n", + " if phi_angle == 360:\n", + " continue\n", + "\n", + " # Define 5-degree bins\n", + " bins = np.arange(0, 185, 5)\n", + "\n", + " # Group counts by 5-degree bins\n", + " grouped_counts = np.zeros((len(bins), number_of_detectors))\n", + " counts_per_bin = np.zeros((len(bins), number_of_detectors))\n", + "\n", + " total_count = 0\n", + " for i in range(0, number_of_detectors):\n", + " for theta, phi, count in zip(labels[:, 0], labels[:, 1], counts[:, i]):\n", + " \n", + " if (phi > phi_angle and phi < phi_angle + 10):\n", + " total_count += 1\n", + " bin_index = int(theta // 5)\n", + " grouped_counts[bin_index][i] = grouped_counts[bin_index][i] + count\n", + " counts_per_bin[bin_index][i] += 1\n", + "\n", + " # Compute the mean counts in each bin\n", + " grouped_counts = grouped_counts[:total_count]\n", + " counts_per_bin = counts_per_bin[:total_count]\n", + "\n", + " # Plot histogram\n", + " plt.figure(figsize=(10, 6))\n", + " plt.bar(bins, grouped_counts[:, 0] / counts_per_bin[:, 0] / total_count, width=5, align='edge', alpha=.5, label=\"BGO_Z1\")\n", + " plt.bar(bins, grouped_counts[:, 1] / counts_per_bin[:, 1] / total_count, width=5, align='edge', alpha=.5, label=\"BGO_Z0\")\n", + " plt.bar(bins, grouped_counts[:, 2] / counts_per_bin[:, 2] / total_count, width=5, align='edge', alpha=.5, label=\"X1\")\n", + " plt.bar(bins, grouped_counts[:, 3] / counts_per_bin[:, 3] / total_count, width=5, align='edge', alpha=.5, label=\"X0\")\n", + " plt.bar(bins, grouped_counts[:, 4] / counts_per_bin[:, 4] / total_count, width=5, align='edge', alpha=.5, label=\"Y1\")\n", + " plt.bar(bins, grouped_counts[:, 5] / counts_per_bin[:, 5] / total_count, width=5, align='edge', alpha=.5, label=\"Y0\")\n", + " plt.xlabel('Theta (°)')\n", + " plt.ylabel('Normalized panel mean counts')\n", + " plt.title('')\n", + " plt.grid(True)\n", + " plt.legend(loc='upper left')\n", + " plt.show()\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ba78ab64-f62f-4386-be3a-d25b8b4b3c34", + "metadata": {}, + "outputs": [], + "source": [ + "# Define 5-degree bins\n", + "bins = np.arange(0, 365, 5)\n", + "\n", + "# Group counts by 5-degree bins\n", + "grouped_counts = np.zeros((len(bins), 6))\n", + "counts_per_bin = np.zeros((len(bins), 6))\n", + "\n", + "total_count = 0\n", + "for i in range(0, 6):\n", + " for theta, phi, count in zip(labels[:, 0], labels[:, 1], counts[:, i]):\n", + " \n", + " if (theta > 85 and theta < 95):\n", + " total_count += 1\n", + " bin_index = int(phi // 5)\n", + " grouped_counts[bin_index][i] = grouped_counts[bin_index][i] + count\n", + " counts_per_bin[bin_index][i] += 1\n", + "\n", + "# Compute the mean counts in each bin\n", + "grouped_counts = grouped_counts[:total_count]\n", + "counts_per_bin = counts_per_bin[:total_count]\n", + "\n", + "# Plot histogram\n", + "plt.figure(figsize=(10, 6))\n", + "#plt.bar(bins, grouped_counts[:, 0] / counts_per_bin[:, 0], width=5, align='edge', alpha=.5, label=\"Bottom\")\n", + "#plt.bar(bins, grouped_counts[:, 1] / counts_per_bin[:, 1], width=5, align='edge', alpha=.5, label=\"X1\")\n", + "plt.bar(bins, grouped_counts[:, 2] / counts_per_bin[:, 2], width=5, align='edge', alpha=.5, label=\"BGO_X1\")\n", + "plt.bar(bins, grouped_counts[:, 3] / counts_per_bin[:, 3], width=5, align='edge', alpha=.5, label=\"BGO_X0\")\n", + "plt.bar(bins, grouped_counts[:, 4] / counts_per_bin[:, 4], width=5, align='edge', alpha=.5, label=\"BGO_Y1\")\n", + "plt.bar(bins, grouped_counts[:, 5] / counts_per_bin[:, 5], width=5, align='edge', alpha=.5, label=\"BGO_Y0\")\n", + "\n", + "plt.xlabel('Phi (°)', fontsize=15)\n", + "plt.ylabel('Panel mean counts', fontsize=15)\n", + "plt.tick_params(axis='both', labelsize=15) \n", + "plt.legend(fontsize=15)\n", + "plt.title('')\n", + "plt.grid(True)\n", + "\n", + "plt.show()\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "29b8b275-6b67-440e-b744-6a1de8ddb4ac", + "metadata": {}, + "outputs": [], + "source": [ + "# Definisci i bin di 5 gradi\n", + "bin_angolari = np.arange(0, 365, 5)\n", + "\n", + "# Raggruppa i conteggi in base ai bin di 5 gradi\n", + "conteggi_raggruppati = np.zeros((len(bin_angolari), 6))\n", + "conteggi_per_bin = np.zeros((len(bin_angolari), 6))\n", + "\n", + "conteggio_totale = 0\n", + "for i in range(0, 6):\n", + " for angolo_theta, angolo_phi, conteggio in zip(labels[:, 0], labels[:, 1], counts[:, i]):\n", + " # X tra 5 e 20, Y tra 20 e 30\n", + " if angolo_theta < 35:\n", + " conteggio_totale += 1\n", + " indice_bin = int(angolo_phi // 5)\n", + " conteggi_raggruppati[indice_bin][i] = conteggi_raggruppati[indice_bin][i] + conteggio\n", + " conteggi_per_bin[indice_bin][i] += 1\n", + "\n", + "# Calcola la media dei conteggi in ciascun bin\n", + "conteggi_raggruppati = conteggi_raggruppati[:conteggio_totale]\n", + "conteggi_per_bin = conteggi_per_bin[:conteggio_totale]\n", + "\n", + "# Visualizza l'istogramma\n", + "plt.figure(figsize=(10, 6))\n", + "#plt.bar(bin_angolari, conteggi_raggruppati[:, 0]/conteggi_per_bin[:, 0], width=5, align='edge', alpha=.5, label=\"Bottom\")\n", + "#plt.bar(bin_angolari, conteggi_raggruppati[:, 1]/conteggi_per_bin[:, 1], width=5, align='edge', alpha=.5, label=\"X1\")\n", + "#plt.bar(bin_angolari, conteggi_raggruppati[:, 2]/conteggi_per_bin[:, 2], width=5, align='edge', alpha=.5, label=\"BGO_X1\")\n", + "#plt.bar(bin_angolari, conteggi_raggruppati[:, 3]/conteggi_per_bin[:, 3], width=5, align='edge', alpha=.5, label=\"BGO_X0\")\n", + "plt.bar(bin_angolari, conteggi_raggruppati[:, 4]/conteggi_per_bin[:, 4], width=5, align='edge', alpha=.5, label=\"BGO_Y1\")\n", + "plt.bar(bin_angolari, conteggi_raggruppati[:, 5]/conteggi_per_bin[:, 5], width=5, align='edge', alpha=.5, label=\"BGO_Y0\")\n", + "\n", + "plt.title('')\n", + "plt.xlabel('Phi (°)', fontsize=15)\n", + "plt.ylabel('Conteggi medi per pannello', fontsize=15)\n", + "plt.tick_params(axis='both', labelsize=15) \n", + "plt.legend(fontsize=15)\n", + "plt.grid(True)\n", + "plt.legend()\n", + "plt.show()\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c5fc69bc-3151-4c19-b088-415737d432b2", + "metadata": {}, + "outputs": [], + "source": [ + "# Define 5-degree bins\n", + "bins = np.arange(0, 185, 5)\n", + "\n", + "# Group counts by 5-degree bins\n", + "grouped_counts = np.zeros((len(bins), number_of_detectors))\n", + "counts_per_bin = np.zeros((len(bins), number_of_detectors))\n", + "\n", + "total_count = 0\n", + "for i in range(0, number_of_detectors):\n", + " for theta, phi, count in zip(labels[:, 0], labels[:, 1], counts[:, i]):\n", + " \n", + " if (phi > 80 and phi < 110):\n", + " total_count += 1\n", + " bin_index = int(theta // 5)\n", + " grouped_counts[bin_index][i] = grouped_counts[bin_index][i] + count\n", + " counts_per_bin[bin_index][i] += 1\n", + "\n", + "# Compute the mean counts in each bin\n", + "grouped_counts = grouped_counts[:total_count]\n", + "counts_per_bin = counts_per_bin[:total_count]\n", + "\n", + "# Plot histogram\n", + "plt.figure(figsize=(10, 6))\n", + "plt.bar(bins, grouped_counts[:, 0] / counts_per_bin[:, 0], width=5, align='edge', alpha=.5, label=\"bottom 1\")\n", + "plt.bar(bins, grouped_counts[:, 1] / counts_per_bin[:, 1], width=5, align='edge', alpha=.5, label=\"bottom 2\")\n", + "#plt.bar(bins, grouped_counts[:, 2] / counts_per_bin[:, 2], width=5, align='edge', alpha=.5, label=\"x1\")\n", + "#plt.bar(bins, grouped_counts[:, 3] / counts_per_bin[:, 3], width=5, align='edge', alpha=.5, label=\"x2\")\n", + "#plt.bar(bins, grouped_counts[:, 4] / counts_per_bin[:, 4], width=5, align='edge', alpha=.5, label=\"y1\")\n", + "#plt.bar(bins, grouped_counts[:, 5] / counts_per_bin[:, 5], width=5, align='edge', alpha=.5, label=\"y2\")\n", + "\n", + "plt.xlabel('Theta (°)')\n", + "plt.ylabel('Panel mean counts')\n", + "plt.title('Mean counts for panels phi=[80°, 110°]')\n", + "plt.grid(True)\n", + "plt.legend()\n", + "plt.show()\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "faf3ab95-8356-41c1-ab52-355a91da5d2f", + "metadata": {}, + "outputs": [], + "source": [ + "# Define 5-degree bins\n", + "bins = np.arange(0, 185, 5)\n", + "\n", + "# Group counts by 5-degree bins\n", + "grouped_counts = np.zeros((len(bins), number_of_detectors))\n", + "counts_per_bin = np.zeros((len(bins), number_of_detectors))\n", + "\n", + "total_count = 0\n", + "for i in range(0, number_of_detectors):\n", + " for theta, phi, count in zip(labels[:, 0], labels[:, 1], counts[:, i]):\n", + " \n", + " if (phi > 35 and phi < 55):\n", + " total_count += 1\n", + " bin_index = int(theta // 5)\n", + " grouped_counts[bin_index][i] = grouped_counts[bin_index][i] + count\n", + " counts_per_bin[bin_index][i] += 1\n", + "\n", + "# Compute the mean counts in each bin\n", + "grouped_counts = grouped_counts[:total_count]\n", + "counts_per_bin = counts_per_bin[:total_count]\n", + "\n", + "# Plot histogram\n", + "plt.figure(figsize=(10, 6))\n", + "#plt.bar(bins, grouped_counts[:, 0] / counts_per_bin[:, 0], width=5, align='edge', alpha=.5, label=\"bottom 1\")\n", + "#plt.bar(bins, grouped_counts[:, 1] / counts_per_bin[:, 1], width=5, align='edge', alpha=.5, label=\"bottom 2\")\n", + "plt.bar(bins, grouped_counts[:, 2] / counts_per_bin[:, 2], width=5, align='edge', alpha=.5, label=\"x1\")\n", + "plt.bar(bins, grouped_counts[:, 3] / counts_per_bin[:, 3], width=5, align='edge', alpha=.5, label=\"x2\")\n", + "plt.bar(bins, grouped_counts[:, 4] / counts_per_bin[:, 4], width=5, align='edge', alpha=.5, label=\"y1\")\n", + "plt.bar(bins, grouped_counts[:, 5] / counts_per_bin[:, 5], width=5, align='edge', alpha=.5, label=\"y2\")\n", + "\n", + "plt.xlabel('Theta (°)')\n", + "plt.ylabel('Panel mean counts')\n", + "plt.title('Mean counts for panels phi=[35°,55°]')\n", + "plt.grid(True)\n", + "plt.legend()\n", + "plt.show()\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ee7cee44-43be-4c2b-8f7e-7d419246363a", + "metadata": {}, + "outputs": [], + "source": [ + "# Define 5-degree bins\n", + "bins = np.arange(0, 185, 5)\n", + "\n", + "# Group counts by 5-degree bins\n", + "grouped_counts = np.zeros((len(bins), number_of_detectors))\n", + "counts_per_bin = np.zeros((len(bins), number_of_detectors))\n", + "\n", + "total_count = 0\n", + "for i in range(0, number_of_detectors):\n", + " for theta, phi, count in zip(labels[:, 0], labels[:, 1], counts[:, i]):\n", + " \n", + " if (phi > 35 and phi < 55):\n", + " total_count += 1\n", + " bin_index = int(theta // 5)\n", + " grouped_counts[bin_index][i] = grouped_counts[bin_index][i] + count\n", + " counts_per_bin[bin_index][i] += 1\n", + "\n", + "# Compute the mean counts in each bin\n", + "grouped_counts = grouped_counts[:total_count]\n", + "counts_per_bin = counts_per_bin[:total_count]\n", + "\n", + "# Plot histogram\n", + "plt.figure(figsize=(10, 6))\n", + "plt.bar(bins, grouped_counts[:, 0] / counts_per_bin[:, 0], width=5, align='edge', alpha=.5, label=\"bottom 1\")\n", + "plt.bar(bins, grouped_counts[:, 1] / counts_per_bin[:, 1], width=5, align='edge', alpha=.5, label=\"bottom 2\")\n", + "#plt.bar(bins, grouped_counts[:, 2] / counts_per_bin[:, 2], width=5, align='edge', alpha=.5, label=\"x1\")\n", + "#plt.bar(bins, grouped_counts[:, 3] / counts_per_bin[:, 3], width=5, align='edge', alpha=.5, label=\"x2\")\n", + "#plt.bar(bins, grouped_counts[:, 4] / counts_per_bin[:, 4], width=5, align='edge', alpha=.5, label=\"y1\")\n", + "#plt.bar(bins, grouped_counts[:, 5] / counts_per_bin[:, 5], width=5, align='edge', alpha=.5, label=\"y2\")\n", + "\n", + "plt.xlabel('Theta (°)')\n", + "plt.ylabel('Panel mean counts')\n", + "plt.title('Mean counts for panels phi=[35°,55°]')\n", + "plt.grid(True)\n", + "plt.legend()\n", + "plt.show()\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2242f56f-e06f-4afd-a3c4-f518dcd1ca3b", + "metadata": {}, + "outputs": [], + "source": [ + "# Define 5-degree bins\n", + "bins = np.arange(0, 185, 5)\n", + "\n", + "# Group counts by 5-degree bins\n", + "grouped_counts = np.zeros((len(bins), number_of_detectors))\n", + "counts_per_bin = np.zeros((len(bins), number_of_detectors))\n", + "\n", + "total_count = 0\n", + "for i in range(0, number_of_detectors):\n", + " for theta, phi, count in zip(labels[:, 0], labels[:, 1], counts[:, i]):\n", + " \n", + " if (phi > 170 and phi < 180):\n", + " total_count += 1\n", + " bin_index = int(theta // 5)\n", + " grouped_counts[bin_index][i] = grouped_counts[bin_index][i] + count\n", + " counts_per_bin[bin_index][i] += 1\n", + "\n", + "# Compute the mean counts in each bin\n", + "grouped_counts = grouped_counts[:total_count]\n", + "counts_per_bin = counts_per_bin[:total_count]\n", + "\n", + "# Plot histogram\n", + "plt.figure(figsize=(10, 6))\n", + "#plt.bar(bins, grouped_counts[:, 0] / counts_per_bin[:, 0], width=5, align='edge', alpha=.5, label=\"bottom 1\")\n", + "#plt.bar(bins, grouped_counts[:, 1] / counts_per_bin[:, 1], width=5, align='edge', alpha=.5, label=\"bottom 2\")\n", + "plt.bar(bins, grouped_counts[:, 2] / counts_per_bin[:, 2], width=5, align='edge', alpha=.5, label=\"x1\")\n", + "plt.bar(bins, grouped_counts[:, 3] / counts_per_bin[:, 3], width=5, align='edge', alpha=.5, label=\"x2\")\n", + "plt.bar(bins, grouped_counts[:, 4] / counts_per_bin[:, 4], width=5, align='edge', alpha=.5, label=\"y1\")\n", + "plt.bar(bins, grouped_counts[:, 5] / counts_per_bin[:, 5], width=5, align='edge', alpha=.5, label=\"y2\")\n", + "\n", + "plt.xlabel('Theta (°)')\n", + "plt.ylabel('Panel mean counts')\n", + "plt.title('Mean counts for panels phi=[170°,180°]')\n", + "plt.grid(True)\n", + "plt.legend()\n", + "plt.show()\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "67c118c3-5214-4736-a343-35e75fd0fd1a", + "metadata": {}, + "outputs": [], + "source": [ + "# Define 5-degree bins\n", + "bins = np.arange(0, 185, 5)\n", + "\n", + "# Group counts by 5-degree bins\n", + "grouped_counts = np.zeros((len(bins), number_of_detectors))\n", + "counts_per_bin = np.zeros((len(bins), number_of_detectors))\n", + "\n", + "total_count = 0\n", + "for i in range(0, number_of_detectors):\n", + " for theta, phi, count in zip(labels[:, 0], labels[:, 1], counts[:, i]):\n", + " \n", + " if (phi > 170 and phi < 180):\n", + " total_count += 1\n", + " bin_index = int(theta // 5)\n", + " grouped_counts[bin_index][i] = grouped_counts[bin_index][i] + count\n", + " counts_per_bin[bin_index][i] += 1\n", + "\n", + "# Compute the mean counts in each bin\n", + "grouped_counts = grouped_counts[:total_count]\n", + "counts_per_bin = counts_per_bin[:total_count]\n", + "\n", + "# Plot histogram\n", + "plt.figure(figsize=(10, 6))\n", + "plt.bar(bins, grouped_counts[:, 0] / counts_per_bin[:, 0], width=5, align='edge', alpha=.5, label=\"bottom 1\")\n", + "plt.bar(bins, grouped_counts[:, 1] / counts_per_bin[:, 1], width=5, align='edge', alpha=.5, label=\"bottom 2\")\n", + "#plt.bar(bins, grouped_counts[:, 2] / counts_per_bin[:, 2], width=5, align='edge', alpha=.5, label=\"x1\")\n", + "#plt.bar(bins, grouped_counts[:, 3] / counts_per_bin[:, 3], width=5, align='edge', alpha=.5, label=\"x2\")\n", + "#plt.bar(bins, grouped_counts[:, 4] / counts_per_bin[:, 4], width=5, align='edge', alpha=.5, label=\"y1\")\n", + "#plt.bar(bins, grouped_counts[:, 5] / counts_per_bin[:, 5], width=5, align='edge', alpha=.5, label=\"y2\")\n", + "\n", + "plt.xlabel('Theta (°)')\n", + "plt.ylabel('Panel mean counts')\n", + "plt.title('Mean counts for panels phi=[170°,180°]')\n", + "plt.grid(True)\n", + "plt.legend()\n", + "plt.show()\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ec783471-1eb6-48dd-a799-5b16bbbbbc7b", + "metadata": {}, + "outputs": [], + "source": [ + "# Define 5-degree bins\n", + "bins = np.arange(0, 365, 5)\n", + "\n", + "# Group counts by 5-degree bins\n", + "grouped_counts = np.zeros((len(bins), 6))\n", + "counts_per_bin = np.zeros((len(bins), 6))\n", + "\n", + "total_count = 0\n", + "for i in range(0, 6):\n", + " for theta, phi, count in zip(labels[:, 0], labels[:, 1], counts[:, i]):\n", + " \n", + " if (theta > 85 and theta < 95):\n", + " total_count += 1\n", + " bin_index = int(phi // 5)\n", + " grouped_counts[bin_index][i] = grouped_counts[bin_index][i] + count\n", + " counts_per_bin[bin_index][i] += 1\n", + "\n", + "# Compute the mean counts in each bin\n", + "grouped_counts = grouped_counts[:total_count]\n", + "counts_per_bin = counts_per_bin[:total_count]\n", + "\n", + "# Plot histogram\n", + "plt.figure(figsize=(10, 6))\n", + "#plt.bar(bins, grouped_counts[:, 0] / counts_per_bin[:, 0], width=5, align='edge', alpha=.5, label=\"bottom\")\n", + "#plt.bar(bins, grouped_counts[:, 1] / counts_per_bin[:, 1], width=5, align='edge', alpha=.5, label=\"x1\")\n", + "plt.bar(bins, grouped_counts[:, 2] / counts_per_bin[:, 2], width=5, align='edge', alpha=.5, label=\"BGO_X1\")\n", + "plt.bar(bins, grouped_counts[:, 3] / counts_per_bin[:, 3], width=5, align='edge', alpha=.5, label=\"BGO_X0\")\n", + "#plt.bar(bins, grouped_counts[:, 4] / counts_per_bin[:, 4], width=5, align='edge', alpha=.5, label=\"y\")\n", + "#plt.bar(bins, grouped_counts[:, 5] / counts_per_bin[:, 5], width=5, align='edge', alpha=.5, label=\"y_negative\")\n", + "\n", + "plt.xlabel('Phi (°)')\n", + "plt.ylabel('Panel mean counts')\n", + "plt.title('Mean counts of each panel')\n", + "plt.grid(True)\n", + "plt.legend()\n", + "plt.show()\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b94cbf4a-a89a-4dc2-812c-02b639d7c076", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0d116ae9-dc64-4429-aa0d-aa5cd6e213fb", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f347fb1b-5ff7-4b5e-954f-e045063b76a3", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d16c7245-003b-4f78-b484-7598d10fc344", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c55afcd5-8c61-4861-8219-9d3db36ed966", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1517a8cd-5c77-4473-9bb4-055d958ddc4b", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3a934e9c-b28c-4494-b89a-a15ee51784c2", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "deeplearning", + "language": "python", + "name": "deeplearning" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.12" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/cosipy/notebooks/dl-loc/train_and_test/SpectralModels.ipynb b/cosipy/notebooks/dl-loc/train_and_test/SpectralModels.ipynb new file mode 100644 index 0000000..a274093 --- /dev/null +++ b/cosipy/notebooks/dl-loc/train_and_test/SpectralModels.ipynb @@ -0,0 +1,740 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Spectral models" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import astropy.units as u\n", + "from astropy.units import Quantity\n", + "import numpy as np\n", + "from abc import ABC, abstractmethod\n", + "from scipy.integrate import quad\n", + "import matplotlib.pyplot as plt" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "plt.rc('font' , size=28) # Controls default text sizes\n", + "plt.rc('figure', titlesize=40) # Fontsize of the figure suptitle\n", + "plt.rc('axes' , titlesize=40) # Fontsize of the title (ax.set_title)\n", + "\n", + "plt.rc('axes' , labelsize=30) # Fontsize of the x and y labels\n", + "plt.rc('xtick' , labelsize=20) # Fontsize of the tick labels\n", + "plt.rc('ytick' , labelsize=20) # Fontsize of the tick labels\n", + "plt.rc('legend', title_fontsize=30) # Legend fontsize title\n", + "plt.rc('legend', fontsize=16) # Legend fontsize text\n", + "\n", + "plt.rc('lines' ,markersize=14) # Markersize\n", + "\n", + "plt.rc('font', family=\"sans-serif\", style=\"normal\", variant=\"normal\",weight=\"normal\")\n", + "# plt.rc('font' , family=\"Times New Roman\", style=\"normal\", variant=\"normal\",weight=\"normal\")\n", + "\n", + "FIGSIZE=(16,12)\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Class Definitions" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Base Spectral Model" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "###################################################################################\n", + "class SpectralModel(ABC):\n", + " \"\"\"\n", + " Base class to deal with spectral model conversions.\n", + " \"\"\"\n", + " unit_energy = u.Unit(\"keV\")\n", + " unit_amplitude = u.Unit(\"s-1 cm-2 keV-1\")\n", + " unit_flux = u.Unit(\"s-1 cm-2\")\n", + " unit_SED = u.Unit(\"erg s-1 cm-2\")\n", + " \n", + " def __init__(self):\n", + " self.N0 = 0.0 * SpectralModel.unit_amplitude\n", + " self.E0 = 1.0 * SpectralModel.unit_energy\n", + "\n", + " def set_N0(self, N0 : Quantity):\n", + " if not N0.unit.is_equivalent(SpectralModel.unit_amplitude):\n", + " raise ValueError\n", + " self.N0 = N0\n", + "\n", + " def set_E0(self, E0 : Quantity):\n", + " if not E0.unit.is_equivalent(SpectralModel.unit_energy):\n", + " raise ValueError\n", + " self.E0 = E0\n", + "\n", + " def get_N0(self):\n", + " return self.N0\n", + " \n", + " def get_E0(self):\n", + " return self.E0\n", + " \n", + " def __str__(self):\n", + " return super().__str__()\n", + " \n", + " def __repr__(self):\n", + " return self.__str__()\n", + " \n", + " @abstractmethod\n", + " def _spectral_model(self, energy_normalised):\n", + " \"\"\"\n", + " Abstract Method to define the spectral models.\n", + " It must return an adimensional value.\n", + " \n", + " Parameters\n", + " ----------\n", + " energy_normalised : `np.array`\n", + " Energy where to compute the model, expressed as adimensional energy / E0.\n", + " \"\"\"\n", + " pass\n", + "\n", + " def _E_spectral_model(self, energy_normalised):\n", + " return energy_normalised * self._spectral_model(energy_normalised)\n", + " \n", + " def get_dNdE(self, energy : Quantity):\n", + " \"\"\"Evaluate dNdE at a given energy.\"\"\"\n", + " if not energy.unit.is_equivalent(SpectralModel.unit_energy):\n", + " raise ValueError\n", + " \n", + " energy_normalised = (energy/self.get_E0()).to('')\n", + " dNdE = self.N0 * self._spectral_model(energy_normalised)\n", + " return dNdE.to(SpectralModel.unit_amplitude)\n", + " \n", + " \n", + " def get_E2dNdE(self, energy : Quantity):\n", + " \"\"\"Evaluate E2dNdE at a given energy.\"\"\"\n", + " if not energy.unit.is_equivalent(SpectralModel.unit_energy):\n", + " raise ValueError\n", + " \n", + " E2dNdE = np.power(energy,2) * self.get_dNdE(energy)\n", + " return E2dNdE.to(SpectralModel.unit_SED)\n", + " \n", + "\n", + " def get_flux(self, E1 : Quantity, E2 : Quantity, return_error=False):\n", + " \"\"\"Evaluate Photon flux Between Two Energies\"\"\"\n", + " if not E1.unit.is_equivalent(SpectralModel.unit_energy):\n", + " raise ValueError\n", + " if not E2.unit.is_equivalent(SpectralModel.unit_energy):\n", + " raise ValueError\n", + " \n", + " E1_normalised = (E1/self.E0).to('')\n", + " E2_normalised = (E2/self.E0).to('')\n", + " \n", + " flux, error = self.N0 * self.E0 * quad(self._spectral_model, E1_normalised, E2_normalised)\n", + "\n", + " if return_error:\n", + " return flux.to(SpectralModel.unit_flux), error.to(SpectralModel.unit_flux)\n", + " return flux.to(SpectralModel.unit_flux)\n", + " \n", + "\n", + "\n", + " def get_eflux(self, E1 : Quantity, E2 : Quantity, return_error=False):\n", + " \"\"\"Evaluate Photon flux Between Two Energies\"\"\"\n", + " if not E1.unit.is_equivalent(SpectralModel.unit_energy):\n", + " raise ValueError\n", + " if not E2.unit.is_equivalent(SpectralModel.unit_energy):\n", + " raise ValueError\n", + " \n", + " E1_normalised = (E1/self.E0).to('')\n", + " E2_normalised = (E2/self.E0).to('')\n", + " \n", + " eflux, error = self.N0 * np.power(self.E0,2) * quad(self._E_spectral_model, E1_normalised, E2_normalised)\n", + "\n", + " if return_error:\n", + " return eflux.to(SpectralModel.unit_SED), error.to(SpectralModel.unit_SED)\n", + " return eflux.to(SpectralModel.unit_SED)\n", + " \n", + "\n", + "\n", + " def estimate_N0_from_flux(self, E1 : Quantity, E2 : Quantity, flux : Quantity):\n", + " \"\"\"Estimate the Amplitude from the flux\"\"\"\n", + " if not E1.unit.is_equivalent(SpectralModel.unit_energy):\n", + " raise ValueError\n", + " if not E2.unit.is_equivalent(SpectralModel.unit_energy):\n", + " raise ValueError\n", + " if not flux.unit.is_equivalent(SpectralModel.unit_flux):\n", + " raise ValueError\n", + " \n", + " E1_normalised = (E1/self.E0).to('')\n", + " E2_normalised = (E2/self.E0).to('')\n", + " \n", + " integral, error = quad(self._spectral_model, E1_normalised, E2_normalised)\n", + "\n", + " N0 = flux / (self.E0 * integral)\n", + "\n", + " return N0.to(SpectralModel.unit_amplitude)\n", + "\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Power Law" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "class PowerLawModel(SpectralModel):\n", + " \n", + " def __init__(self, index : float):\n", + " \"\"\"Set the Index. Amplitude must be set with the appropriate method\"\"\"\n", + " super().__init__()\n", + " self.index = float(index)\n", + "\n", + " def _spectral_model(self, energy_normalised):\n", + " \"\"\"Evaluate the spectral model\"\"\"\n", + " return np.power(energy_normalised,-self.index)\n", + " \n", + " def __str__(self):\n", + " return f\"Power Law Model\\n N0={self.N0:.3e}\\n E0={self.E0}\\n Index={self.index}\\n\"\n", + " \n", + "\n", + " def get_flux(self, E1 : Quantity, E2 : Quantity):\n", + " \"\"\"Redefine to use analytical expressions\"\"\"\n", + " if not E1.unit.is_equivalent(SpectralModel.unit_energy):\n", + " raise ValueError\n", + " if not E2.unit.is_equivalent(SpectralModel.unit_energy):\n", + " raise ValueError\n", + " \n", + " E1_normalised = (E1/self.E0).to('')\n", + " E2_normalised = (E2/self.E0).to('')\n", + " \n", + " if self.index == 1.0:\n", + " integral = np.log(E2_normalised / E1_normalised)\n", + " else:\n", + " integral = np.power(E2_normalised, 1-self.index)-np.power(E1_normalised, 1-self.index) / (1-self.index)\n", + " \n", + " flux = self.N0 * self.E0 * integral\n", + "\n", + " return flux.to(SpectralModel.unit_flux)\n", + " \n", + "\n", + " def get_eflux(self, E1 : Quantity, E2 : Quantity):\n", + " \"\"\"Evaluate Photon flux Between Two Energies\"\"\"\n", + " if not E1.unit.is_equivalent(SpectralModel.unit_energy):\n", + " raise ValueError\n", + " if not E2.unit.is_equivalent(SpectralModel.unit_energy):\n", + " raise ValueError\n", + " \n", + " E1_normalised = (E1/self.E0).to('')\n", + " E2_normalised = (E2/self.E0).to('')\n", + " \n", + " if self.index == 2.0:\n", + " integral = np.log(E2_normalised / E1_normalised)\n", + " else:\n", + " integral = np.power(E2_normalised, 2-self.index)-np.power(E1_normalised, 2-self.index) / (2-self.index)\n", + " \n", + " eflux = self.N0 * np.power(self.E0,2) * integral\n", + "\n", + " return eflux.to(SpectralModel.unit_SED)\n", + " \n", + " def estimate_N0_from_flux(self, E1 : Quantity, E2 : Quantity, flux : Quantity):\n", + " \"\"\"Redefine to use analytical expressions\"\"\"\n", + " if not E1.unit.is_equivalent(SpectralModel.unit_energy):\n", + " raise ValueError\n", + " if not E2.unit.is_equivalent(SpectralModel.unit_energy):\n", + " raise ValueError\n", + " if not flux.unit.is_equivalent(SpectralModel.unit_flux):\n", + " raise ValueError\n", + " \n", + " E1_normalised = (E1/self.E0).to('')\n", + " E2_normalised = (E2/self.E0).to('')\n", + " \n", + " if self.index == 1.0:\n", + " integral = np.log(E2_normalised / E1_normalised)\n", + " else:\n", + " integral = np.power(E2_normalised, 1-self.index)-np.power(E1_normalised, 1-self.index) / (1-self.index)\n", + " \n", + " N0 = flux / (self.E0 * integral)\n", + "\n", + " return N0.to(SpectralModel.unit_amplitude)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Band" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "class BandModel(SpectralModel):\n", + " \n", + " def __init__(self, lower_index : float, upper_index : float, E_break : Quantity):\n", + " \"\"\"Set the Index. Amplitude must be set with the appropriate method\"\"\"\n", + " super().__init__()\n", + " if not E_break.unit.is_equivalent(SpectralModel.unit_energy):\n", + " raise ValueError\n", + " if lower_index - upper_index <=0:\n", + " raise ValueError\n", + " \n", + " self.lower_index = float(lower_index)\n", + " self.upper_index = float(upper_index)\n", + " self.E_break = E_break\n", + "\n", + " def E_transition(self):\n", + " \"\"\"Transition Energy\"\"\"\n", + " return (self.lower_index-self.upper_index)*self.E_break\n", + "\n", + " def _spectral_model(self, energy_normalised):\n", + " \"\"\"Evaluate the spectral model\"\"\"\n", + " E_transition_normalised = (self.E_transition()/self.E0).to('')\n", + " E_break_normalised = (self.E_break/self.E0).to('')\n", + "\n", + " value = np.where(energy_normalised <= E_transition_normalised,\n", + " np.power(energy_normalised, self.lower_index) * np.exp(- energy_normalised / E_break_normalised),\n", + " np.power(energy_normalised, self.upper_index) * np.power(E_break_normalised*(self.lower_index-self.upper_index), self.lower_index-self.upper_index) * np.exp(self.upper_index-self.lower_index)\n", + " )\n", + " return value\n", + " \n", + " def __str__(self):\n", + " string = f\"Band Model\\n N0={self.N0:.3e}\\n E0={self.E0}\\n\"\n", + " string+= f\" Lower Index={self.lower_index}\\n Upper Index={self.upper_index}\\n\"\n", + " string+= f\" Break Energy={self.E_break}\\n Transition Energy={self.E_transition():.3e}\\n\"\n", + " return string\n", + "\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Comptonized" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "class Comptonized(SpectralModel):\n", + " \n", + " def __init__(self, index : float, E_peak : Quantity):\n", + " \"\"\"Set the Index and Peak Energy. Amplitude must be set with the appropriate method\"\"\"\n", + " super().__init__()\n", + " if not E_peak.unit.is_equivalent(SpectralModel.unit_energy):\n", + " raise ValueError\n", + " \n", + " self.index = float(index)\n", + " self.E_peak = E_peak\n", + "\n", + " def E_cutoff(self):\n", + " \"\"\"Energy of the exponential cutoff\"\"\"\n", + " return self.E_peak/(2+self.index)\n", + "\n", + " def _spectral_model(self, energy_normalised):\n", + " \"\"\"Evaluate the spectral model\"\"\"\n", + " E_cutoff_normalised = (self.E_cutoff()/self.E0).to('')\n", + " value = np.power(energy_normalised, self.index) * np.exp(- energy_normalised / E_cutoff_normalised)\n", + " return value\n", + " \n", + " def __str__(self):\n", + " string = f\"Comptonized Model\\n N0={self.N0:.3e}\\n E0={self.E0}\\n\"\n", + " string+= f\" Index={self.index}\\n\"\n", + " string+= f\" Peak Energy={self.E_peak}\\n Cutoff Energy={self.E_cutoff():.3e}\\n\"\n", + " return string\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Examples" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# SOFT: Band 10 10000 -1.9 -3.7 699.9. FLUX=5 ph/s/cm2\n", + "soft = BandModel(lower_index=-1.9, upper_index=-3.7, E_break=230.0*u.keV)\n", + "soft_N0 = soft.estimate_N0_from_flux(E1=10*u.keV, E2=10*u.MeV, flux=5*u.Unit(\"s-1 cm-2\"))\n", + "soft.set_N0(soft_N0)\n", + "\n", + "display(soft)\n", + "display(f\"Flux 10 keV - 10 MeV = {soft.get_flux(E1=10*u.keV, E2=10*u.MeV)}\") # Sanity Check\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# MEDIUM: Band 10 10000 -1 -2.3 230. FLUX=5 ph/s/cm2\n", + "medium = BandModel(lower_index=-1.0, upper_index=-2.3, E_break=699.9*u.keV)\n", + "medium_N0 = medium.estimate_N0_from_flux(E1=10*u.keV, E2=10*u.MeV, flux=5*u.Unit(\"s-1 cm-2\"))\n", + "medium.set_N0(medium_N0)\n", + "\n", + "display(medium)\n", + "display(f\"Flux 10 keV - 10 MeV = {medium.get_flux(E1=10*u.keV, E2=10*u.MeV)}\") # Sanity Check" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# HARD: Compotonized 10 10000 -0.5 1500. FLUX=5 ph/s/cm2\n", + "hard = Comptonized(index=-0.5, E_peak=1500*u.keV)\n", + "hard_N0 = hard.estimate_N0_from_flux(E1=10*u.keV, E2=10*u.MeV, flux=5*u.Unit(\"s-1 cm-2\"))\n", + "hard.set_N0(hard_N0)\n", + "display(hard)\n", + "\n", + "display(f\"Flux 10 keV - 10 MeV = {hard.get_flux(E1=10*u.keV, E2=10*u.MeV)}\") # Sanity Check" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Plot" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "energies = np.logspace(np.log10(10),np.log10(10000), num=51) * u.keV\n", + "# energies" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Spectra as dNdE (ph/s/cm2/keV)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# plt.rc('font' , family=\"Times New Roman\", style=\"normal\", variant=\"normal\",weight=\"normal\")\n", + "\n", + "FIGSIZE=(16,8)\n", + "\n", + "\n", + "fig, ax = plt.subplots(1, figsize=FIGSIZE, constrained_layout=True)\n", + "plt.setp(ax.spines.values(), linewidth=1.5)\n", + "\n", + "x = energies\n", + "\n", + "y = soft.get_dNdE(energies)\n", + "ax.plot(x, y, lw=3, label=f\"Soft {soft}\", color=\"r\")\n", + "\n", + "#y = soft_old.get_dNdE(energies)\n", + "#ax.plot(x, y, lw=3, label=f\"Soft old {soft_old}\", color=\"r\",linestyle='--')\n", + "\n", + "y = medium.get_dNdE(energies)\n", + "ax.plot(x, y, lw=3, label=f\"Medium {medium}\", color=\"b\")\n", + "\n", + "#y = medium_old.get_dNdE(energies)\n", + "#ax.plot(x, y, lw=3, label=f\"Medium old {medium_old}\", color=\"b\",linestyle='--')\n", + "\n", + "y = hard.get_dNdE(energies)\n", + "ax.plot(x, y, lw=3, label=f\"Hard {hard}\", color=\"g\")\n", + "\n", + "#y = hard_old.get_dNdE(energies)\n", + "#ax.plot(x, y, lw=3, label=f\"Hard old {hard_old}\", color=\"g\",linestyle='--')\n", + "\n", + "y = fermi_grb.get_dNdE(energies)\n", + "ax.plot(x, y, lw=3, label=f\"Fermi {fermi_grb}\", color=\"m\")\n", + "\n", + "ax.axvline(80, c='k')\n", + "ax.axvline(2000, c='k')\n", + "\n", + "ax.set_xscale('log')\n", + "ax.set_yscale('log')\n", + "ax.set_xlabel(f\"Energy ({x.unit})\",fontsize=15)\n", + "ax.set_ylabel(f\"dNdE ({y.unit})\",fontsize=15)\n", + "plt.legend(fontsize=15)\n", + "ax.grid(ls=\"--\")\n", + "ax.set_title(\"Spectral Models\")\n", + "ax.legend(bbox_to_anchor=(1.0,1.0))\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# plt.rc('font' , family=\"Times New Roman\", style=\"normal\", variant=\"normal\",weight=\"normal\")\n", + "\n", + "FIGSIZE=(12,8)\n", + "\n", + "\n", + "fig, ax = plt.subplots(1, figsize=FIGSIZE, constrained_layout=True)\n", + "plt.setp(ax.spines.values(), linewidth=1.5)\n", + "\n", + "x = energies\n", + "\n", + "y = soft.get_dNdE(energies)\n", + "ax.plot(x, y, lw=3, label=f\"Soft Band Spectrum\", color=\"r\")\n", + "\n", + "#y = soft_old.get_dNdE(energies)\n", + "#ax.plot(x, y, lw=3, label=f\"Soft old {soft_old}\", color=\"r\",linestyle='--')\n", + "\n", + "y = medium.get_dNdE(energies)\n", + "ax.plot(x, y, lw=3, label=f\"Medium Band Spectrum\", color=\"b\")\n", + "\n", + "#y = medium_old.get_dNdE(energies)\n", + "#ax.plot(x, y, lw=3, label=f\"Medium old {medium_old}\", color=\"b\",linestyle='--')\n", + "\n", + "y = hard.get_dNdE(energies)\n", + "ax.plot(x, y, lw=3, label=f\"Hard Comptonized Spectrum\", color=\"g\")\n", + "\n", + "#y = hard_old.get_dNdE(energies)\n", + "#ax.plot(x, y, lw=3, label=f\"Hard old {hard_old}\", color=\"g\",linestyle='--')\n", + "\n", + "ax.axvline(80, c='k')\n", + "ax.axvline(2000, c='k')\n", + "\n", + "ax.set_xscale('log')\n", + "ax.set_yscale('log')\n", + "ax.set_xlabel(f\"Energy ({x.unit})\",fontsize=22)\n", + "ax.set_ylabel(f\"dNdE ({y.unit})\",fontsize=22)\n", + "ax.tick_params(axis='both', labelsize=22) \n", + "ax.grid(ls=\"--\")\n", + "ax.set_title(\"\")\n", + "ax.legend(fontsize=22,loc='upper right')\n", + "#plt.legend()\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Spectra as SED (erg/s/cm2)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "fig, ax = plt.subplots(1, figsize=FIGSIZE, constrained_layout=True)\n", + "plt.setp(ax.spines.values(), linewidth=1.5)\n", + "\n", + "x = energies\n", + "y = soft.get_E2dNdE(energies)\n", + "ax.plot(x, y, lw=3, label=f\"Soft {soft}\")\n", + "\n", + "y = medium.get_E2dNdE(energies)\n", + "ax.plot(x, y, lw=3, label=f\"Medium {medium}\")\n", + "\n", + "y = hard.get_E2dNdE(energies)\n", + "ax.plot(x, y, lw=3, label=f\"Hard {hard}\")\n", + "\n", + "ax.axvline(80, c='k')\n", + "ax.axvline(10000, c='k')\n", + "\n", + "ax.set_xscale('log')\n", + "ax.set_yscale('log')\n", + "ax.set_xlabel(f\"Energy ({x.unit})\")\n", + "ax.set_ylabel(f\"E2dNdE ({y.unit})\")\n", + "ax.grid(ls=\"--\")\n", + "ax.set_title(\"Spectral Models\")\n", + "ax.legend(bbox_to_anchor=(1.0,1.0))\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# COSI VERSION\n", + "E0 = 300*u.keV\n", + "\n", + "# SOFT\n", + "soft = BandModel(lower_index=-1.9, upper_index=-3.7, E_break=230*u.keV)\n", + "soft.set_E0(E0)\n", + "soft_N0 = soft.estimate_N0_from_flux(E1=10*u.keV, E2=10*u.MeV, flux=500*u.Unit(\"s-1 cm-2\"))\n", + "soft.set_N0(soft_N0)\n", + "#print(f\"Soft Model Flux = {soft.get_flux(E1=10*u.keV, E2=10*u.MeV)}\\n{soft}\\n\")\n", + "\n", + "# MEDIUM\n", + "medium = BandModel(lower_index=-1.0, upper_index=-2.3, E_break=699.9*u.keV)\n", + "medium.set_E0(E0)\n", + "# medium_N0 = medium.estimate_N0_from_flux(E1=10*u.keV, E2=10*u.MeV, flux=200*u.Unit(\"s-1 cm-2\"))\n", + "medium_N0 = soft.get_dNdE(E0)/np.exp(-E0/(699.9*u.keV))\n", + "medium.set_N0(medium_N0)\n", + "#print(f\"Medium Model Flux = {medium.get_flux(E1=10*u.keV, E2=10*u.MeV)}\\n{medium}\\n\")\n", + "\n", + "# HARD\n", + "hard = Comptonized(index=-0.5, E_peak=1500*u.keV)\n", + "hard.set_E0(E0)\n", + "# hard_N0 = hard.estimate_N0_from_flux(E1=10*u.keV, E2=10*u.MeV, flux=100*u.Unit(\"s-1 cm-2\"))\n", + "hard_N0 = soft.get_dNdE(E0)/np.exp(-E0*(2-0.5)/(1500*u.keV))\n", + "hard.set_N0(hard_N0)\n", + "#print(f\"Hard Model Flux = {hard.get_flux(E1=10*u.keV, E2=10*u.MeV)}\\n{hard}\\n\")\n", + "\n", + "print(soft.get_dNdE(E0))\n", + "print(medium.get_dNdE(E0))\n", + "print(hard.get_dNdE(E0))\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# energies\n", + "energies = np.logspace(np.log10(10),np.log10(10000), num=51) * u.keV\n", + "##########################\n", + "\n", + "fig, axs = plt.subplots(1, 2, figsize=(20,4), constrained_layout=True)\n", + "\n", + "########### dNdE\n", + "axs[0].plot(energies, soft.get_dNdE(energies) , lw=2, ls=\"-\", c='r', label=f\"Soft Band Spectrum\")\n", + "axs[0].plot(energies, medium.get_dNdE(energies), lw=2, ls=\"-\", c='b', label=f\"Medium band Spectrum\")\n", + "axs[0].plot(energies, hard.get_dNdE(energies) , lw=2, ls=\"-\", c='g', label=f\"Hard Comptonized Spectrum\")\n", + "axs[0].set_ylabel(f\"dN/dE [{soft.get_dNdE(energies).unit}]\")\n", + "###########\n", + "\n", + "########### E2 dNdE\n", + "axs[1].plot(energies, soft.get_E2dNdE(energies) , lw=2, ls=\"-\", c='r', label=f\"Soft Band Spectrum\")\n", + "axs[1].plot(energies, medium.get_E2dNdE(energies), lw=2, ls=\"-\", c='b', label=f\"Medium band Spectrum\")\n", + "axs[1].plot(energies, hard.get_E2dNdE(energies) , lw=2, ls=\"-\", c='g', label=f\"Hard Comptonized Spectrum\")\n", + "axs[1].set_ylabel(f\"E2 dN/dE [{soft.get_E2dNdE(energies).unit}]\")\n", + "###########\n", + "\n", + "\n", + "for ax in axs:\n", + " # plt.setp(ax.spines.values(), linewidth=1.5)\n", + " ax.axvline(80, c='k')\n", + " ax.axvline(2000, c='k')\n", + " ax.axvline(300,ls=\"--\", c='k')\n", + " ax.set_xscale('log')\n", + " ax.set_yscale('log')\n", + " ax.set_xlabel(f\"Energy [{energies.unit}]\")\n", + " ax.grid(ls=\":\")\n", + " ax.set_title(\"Spectral Models\")\n", + " \n", + "axs[0].legend() \n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "FIGSIZE=(12,8)\n", + "\n", + "\n", + "fig, ax = plt.subplots(1, figsize=FIGSIZE, constrained_layout=True)\n", + "plt.setp(ax.spines.values(), linewidth=1.5)\n", + "\n", + "########### dNdE\n", + "ax.plot(energies, soft.get_dNdE(energies) , lw=3, ls=\"-\", c='r', label=f\"Soft Band Spectrum\")\n", + "ax.plot(energies, medium.get_dNdE(energies), lw=3, ls=\"-\", c='b', label=f\"Medium band Spectrum\")\n", + "ax.plot(energies, hard.get_dNdE(energies) , lw=3, ls=\"-\", c='g', label=f\"Hard Comptonized Spectrum\")\n", + "ax.set_ylabel(f\"dN/dE [{soft.get_dNdE(energies).unit}]\")\n", + "###########\n", + "\n", + "# plt.setp(ax.spines.values(), linewidth=1.5)\n", + "ax.axvline(80, c='k')\n", + "ax.axvline(2000, c='k')\n", + "ax.axvline(300,ls=\"--\", c='k')\n", + "ax.set_xscale('log')\n", + "ax.set_yscale('log')\n", + "ax.set_xlabel(f\"Energy [{energies.unit}]\")\n", + "ax.grid(ls=\":\")\n", + "ax.set_xlabel(f\"Energy ({x.unit})\",fontsize=22)\n", + "ax.set_ylabel(f\"dNdE ({y.unit})\",fontsize=22)\n", + "ax.tick_params(axis='both', labelsize=22) \n", + "\n", + "ax.legend(fontsize=22,loc='upper right') \n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "deeplearning", + "language": "python", + "name": "deeplearning" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.12" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/cosipy/notebooks/dl-loc/train_and_test/inference_cartesian_model.ipynb b/cosipy/notebooks/dl-loc/train_and_test/inference_cartesian_model.ipynb new file mode 100644 index 0000000..6a22f13 --- /dev/null +++ b/cosipy/notebooks/dl-loc/train_and_test/inference_cartesian_model.ipynb @@ -0,0 +1,880 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "0", + "metadata": {}, + "outputs": [], + "source": [ + "# import libraries\n", + "import os\n", + "import pandas as pd\n", + "import numpy as np\n", + "import seaborn as sns\n", + "sns.set(color_codes=True)\n", + "import matplotlib.pyplot as plt\n", + "%matplotlib inline\n", + "import pickle\n", + "import tensorflow as tf\n", + "\n", + "file_name = \"run57_mix_mega_shared\"\n", + "data_dir = \"/data/test_newrepo\"\n", + "model_name = \"run53_mix_mega_shared_cart.keras\"\n", + "test_number = 100000\n", + "normalize = 1\n", + "os.environ[\"CUDA_VISIBLE_DEVICES\"]=\"-1\"\n", + "number_of_detectors = 6\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1", + "metadata": {}, + "outputs": [], + "source": [ + "print(\"Num GPUs Available: \", len(tf.config.list_physical_devices('GPU')))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2", + "metadata": {}, + "outputs": [], + "source": [ + "\n", + "file_path = data_dir+'/'+file_name+'_dataset.pkl'\n", + "\n", + "# Load the array from the pickle file\n", + "with open(file_path, 'rb') as file:\n", + " loaded_array = pickle.load(file)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3", + "metadata": {}, + "outputs": [], + "source": [ + "# Filter the dataset by spectral model and flux\n", + "\n", + "loaded_array_test = loaded_array\n", + "\n", + "filter_flux = 0 #1:30\n", + "filter_spectra = 0\n", + "\n", + "filtered_loaded_array_test = np.empty(loaded_array_test.shape[0], dtype=object)\n", + "\n", + "count = 0\n", + "\n", + "if filter_flux == 1:\n", + " for grb in loaded_array_test:\n", + " if grb['flux'] > 0 and grb['flux'] <= 10:\n", + " filtered_loaded_array_test[count] = grb\n", + " count = count+1\n", + "\n", + " loaded_array_test = filtered_loaded_array_test[:count]\n", + "\n", + "filtered_loaded_array_test = np.empty(loaded_array_test.shape[0], dtype=object)\n", + "\n", + "count = 0\n", + "\n", + "if filter_spectra==1:\n", + " for grb in loaded_array_test:\n", + " if \"1500\" in grb['spectrum']: #[\"Band 10 10000 -1.9 -3.7 230\",\"Band 10 10000 -1 -2.3 699.9\",\"Comptonized 10 10000 -0.5 1500\"]\n", + " filtered_loaded_array_test[count] = grb\n", + " count = count+1\n", + "\n", + " loaded_array_test = filtered_loaded_array_test[:count]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d0cbf4ca-9222-4356-a74c-62afcd4b9629", + "metadata": {}, + "outputs": [], + "source": [ + "loaded_array_test.shape" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1133d511-ce10-49a5-bfde-2873f842cc5f", + "metadata": {}, + "outputs": [], + "source": [ + "filter_len = test_number\n", + "\n", + "random_indices = np.random.permutation(len(loaded_array_test))\n", + "\n", + "shuffled_loaded_array_bkg = loaded_array_test[random_indices]\n", + "\n", + "test_dataset_raw = shuffled_loaded_array_bkg[:filter_len] " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3957a17d-20d1-4140-b3b8-fa212b20bd19", + "metadata": {}, + "outputs": [], + "source": [ + "test_dataset_raw.shape" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9f8eb446-41f5-4501-aba9-b7587b0fd3c2", + "metadata": {}, + "outputs": [], + "source": [ + "\n", + "def l2_normalize(data):\n", + " \"\"\"\n", + " Normalize a NumPy array using the L2 (Euclidean) norm.\n", + "\n", + " Args:\n", + " data (numpy.ndarray): The array to normalize. Can be a 1D vector\n", + " or a 2D matrix (where each row is a vector to normalize).\n", + "\n", + " Returns:\n", + " numpy.ndarray: The L2-normalized array.\n", + " \"\"\"\n", + "\n", + " data = np.array(data)\n", + " if data.ndim == 1:\n", + " # Case: 1D vector\n", + " norm = np.sqrt(np.sum(data**2))\n", + " if norm == 0:\n", + " return data # Avoid division by zero if the vector is null\n", + " return data / norm\n", + " elif data.ndim == 2:\n", + " # Case: 2D matrix (normalize each row)\n", + " norms = np.sqrt(np.sum(data**2, axis=1, keepdims=True))\n", + " # Handle the case of null norms (zero rows)\n", + " norms[norms == 0] = 1\n", + " return data / norms\n", + " else:\n", + " raise ValueError(\"Input must be a 1D or 2D array.\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4", + "metadata": {}, + "outputs": [], + "source": [ + "dataset = np.empty((len(test_dataset_raw),number_of_detectors))\n", + "labels = np.empty((len(test_dataset_raw),2))\n", + "\n", + "count = -1\n", + "for element in test_dataset_raw:\n", + "\n", + " count+=1\n", + " # We are considering a Poissonian background. The mean counts are calculated from the DC3 BGO data.\n", + " b_sim = np.array([57.6053, 58.7157, 51.4131, 48.2891, 47.7293, 45.9617])\n", + " dataset[count] = l2_normalize(np.array(element['counts'])+np.random.poisson(b_sim*20)-b_sim*20)\n", + " #dataset[count] = l2_normalize(element['counts'])\n", + " labels[count][0] = float(element['coord'][0])\n", + " labels[count][1] = float(element['coord'][1])\n", + " \n", + "labels = labels[:count+1]\n", + "dataset = dataset[:count+1]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7", + "metadata": {}, + "outputs": [], + "source": [ + "import matplotlib.pyplot as plt\n", + "from mpl_toolkits.mplot3d import Axes3D\n", + "import numpy as np\n", + "\n", + "# Convert to radians\n", + "coords_deg = []\n", + "for theta, phi in labels:\n", + " coords_deg.append([theta, phi])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8", + "metadata": {}, + "outputs": [], + "source": [ + "coord_array = np.array(coords_deg)\n", + "coord_array.shape" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9", + "metadata": {}, + "outputs": [], + "source": [ + "plt.hist(coord_array[:, 0])\n", + "plt.xlabel(\"Theta\")\n", + "plt.ylabel(\"Counts\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "10", + "metadata": {}, + "outputs": [], + "source": [ + "plt.hist(coord_array[:, 1])\n", + "plt.xlabel(\"Phi\")\n", + "plt.ylabel(\"Counts\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "11", + "metadata": {}, + "outputs": [], + "source": [ + "def cartesian_to_spherical(x, y, z):\n", + " # Calculate theta\n", + " theta = np.degrees(np.arccos(z))\n", + " \n", + " # Calculate phi\n", + " if x == 0 and y == 0:\n", + " phi = 0\n", + " else:\n", + " phi = np.degrees(np.arctan2(y, x))\n", + " if phi < 0:\n", + " phi += 360\n", + " \n", + " return theta, phi\n", + "\n", + "def process_coordinates(coords_deg):\n", + "\n", + " # Separate the longitudes and latitudes in radians\n", + " theta, phi = zip(*coords_deg)\n", + "\n", + " # Convert to Cartesian coordinates\n", + " x = np.sin(np.radians(theta)) * np.cos(np.radians(phi))\n", + " y = np.sin(np.radians(theta)) * np.sin(np.radians(phi))\n", + " z = np.cos(np.radians(theta))\n", + " \n", + " return x,y,z\n", + "\n", + "def process_coordinates_single(coords_deg):\n", + "\n", + " # Separate the longitudes and latitudes in radians\n", + " theta, phi = coords_deg\n", + "\n", + " # Convert to Cartesian coordinates\n", + " x = np.sin(np.radians(theta)) * np.cos(np.radians(phi))\n", + " y = np.sin(np.radians(theta)) * np.sin(np.radians(phi))\n", + " z = np.cos(np.radians(theta))\n", + " \n", + " return x,y,z\n", + "\n", + "x,y,z = process_coordinates(coords_deg)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5b89717c-c91b-406e-ae06-69926d052892", + "metadata": {}, + "outputs": [], + "source": [ + "cartesian_labels = []\n", + "\n", + "for i in range(0,len(test_dataset_raw)):\n", + " cartesian_labels.append([x[i],y[i],z[i]])\n", + " " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "15", + "metadata": {}, + "outputs": [], + "source": [ + "theta, phi = zip(*coords_deg)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "16", + "metadata": {}, + "outputs": [], + "source": [ + "def get_radians(coords):\n", + "\n", + " # Separate the longitudes and latitudes in radians\n", + " theta, phi = zip(*coords)\n", + " \n", + " theta = np.array(theta)\n", + " phi = np.array(phi)\n", + " \n", + " \n", + " mask = phi > 180\n", + " phi[mask] -= 360\n", + " theta = 90-theta\n", + "\n", + " # Convert to Cartesian coordinates\n", + " return np.radians(theta),np.radians(phi)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "20", + "metadata": {}, + "outputs": [], + "source": [ + "# create cartesian labels dataset\n", + "labels_norm = np.empty((len(test_dataset_raw),3))\n", + "count = -1\n", + "for element in cartesian_labels:\n", + " count+=1\n", + " labels_norm[count][0] = element[0]\n", + " labels_norm[count][1] = element[1]\n", + " labels_norm[count][2] = element[2]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "22", + "metadata": {}, + "outputs": [], + "source": [ + "labels_norm.shape" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f7001cb5", + "metadata": {}, + "outputs": [], + "source": [ + "plt.hist(labels_norm[:,0])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f76d96f1", + "metadata": {}, + "outputs": [], + "source": [ + "plt.hist(labels_norm[:,1])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "24", + "metadata": {}, + "outputs": [], + "source": [ + "plt.hist(labels_norm[:,2])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "32", + "metadata": {}, + "outputs": [], + "source": [ + "test_dataset = dataset\n", + "test_labels = labels_norm" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "90af5090-f58f-4588-9e95-858d8f4666d6", + "metadata": {}, + "outputs": [], + "source": [ + "def custom_loss(y_true, y_pred):\n", + " \n", + " lambda_reg = 0.5\n", + "\n", + " mse_loss = MeanAbsoluteError()(y_true,y_pred)\n", + " \n", + " x_pred, y_pred, z_pred = tf.unstack(y_pred, axis=1)\n", + " sphere_constraint_error = tf.square(x_pred**2 + y_pred**2 + z_pred**2 - 1)\n", + " \n", + " loss = mse_loss**2 + lambda_reg * sphere_constraint_error\n", + " \n", + " return loss" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "67e8a640-58b1-4d72-afc0-a55a236e8e61", + "metadata": {}, + "outputs": [], + "source": [ + "from tensorflow.keras.models import load_model\n", + "\n", + "\n", + "# Load the model back from the saved directory\n", + "model = load_model(data_dir+\"/\"+model_name,custom_objects={'custom_loss': custom_loss})" + ] + }, + { + "cell_type": "raw", + "id": "36", + "metadata": {}, + "source": [ + "# Extract x and y coordinates for each array\n", + "x1, y1 = train_labels[:, 0], train_labels[:, 1]\n", + "\n", + "# Create a new figure\n", + "plt.figure()\n", + "\n", + "# Plot the coordinates from array1\n", + "plt.scatter(x1, y1, color='blue', label='Array 1')\n", + "\n", + "# Add labels and legend\n", + "plt.xlabel('X Coordinate')\n", + "plt.ylabel('Y Coordinate')\n", + "plt.title('Scatter Plot of Coordinates')\n", + "plt.legend()\n", + "\n", + "# Show the plot\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "38", + "metadata": {}, + "outputs": [], + "source": [ + "test_dataset.shape" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6291273e-33b4-435f-b657-aeb330ead6a7", + "metadata": {}, + "outputs": [], + "source": [ + "test_dataset[0]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "40", + "metadata": {}, + "outputs": [], + "source": [ + "\n", + "pred_data = model.predict(test_dataset)\n", + " " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "23e21547-ddbc-46aa-b487-b9d26657520c", + "metadata": {}, + "outputs": [], + "source": [ + "pred_data.shape" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "41e34844-f355-475e-af90-82d485335ccf", + "metadata": {}, + "outputs": [], + "source": [ + "test_labels.shape" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "41", + "metadata": {}, + "outputs": [], + "source": [ + "import math\n", + "\n", + "pred_data_renorm = np.empty((len(pred_data),3))\n", + "test_data_renorm = np.empty((len(test_labels),3))\n", + "\n", + " \n", + "for j in range(0,len(pred_data)):\n", + " pred_data_renorm[j][0] = pred_data[j][0]\n", + " pred_data_renorm[j][1] = pred_data[j][1]\n", + " pred_data_renorm[j][2] = pred_data[j][2]\n", + "\n", + " \n", + "min_val = 0\n", + "max_val = 1\n", + "\n", + "for j in range(0,len(test_labels)):\n", + " test_data_renorm[j][0] = test_labels[j][0]\n", + " test_data_renorm[j][1] = test_labels[j][1]\n", + " test_data_renorm[j][2] = test_labels[j][2]\n", + "\n", + " \n", + "pred_data_original = np.empty((len(pred_data),2))\n", + "test_labels_original = np.empty((len(test_labels),2))\n", + " \n", + "for i, element in enumerate(test_data_renorm):\n", + "\n", + " theta,phi = cartesian_to_spherical(element[0], element[1], element[2])\n", + " \n", + " test_labels_original[i][0]=theta\n", + " test_labels_original[i][1]=phi\n", + " \n", + "\n", + "for i, element in enumerate(pred_data_renorm):\n", + "\n", + " theta,phi = cartesian_to_spherical(element[0], element[1], element[2])\n", + " \n", + " pred_data_original[i][0]=theta\n", + " pred_data_original[i][1]=phi\n", + " \n", + "\n", + "\n", + "absolute_diffs_element1 = np.abs(pred_data_original[:, 0] - test_labels_original[:, 0])\n", + "absolute_diffs_element2 = np.abs(pred_data_original[:, 1] - test_labels_original[:, 1])\n", + "\n", + "# Calculate the Mean Absolute Error (MAE) for each element separately\n", + "mae_element1 = np.mean(absolute_diffs_element1)\n", + "mae_element2 = np.mean(absolute_diffs_element2)\n", + "\n", + "print(\"Mean Absolute Error for Element 1:\", mae_element1)\n", + "print(\"Mean Absolute Error for Element 2:\", mae_element2)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "42", + "metadata": {}, + "outputs": [], + "source": [ + "plt.hist((pred_data[:,0]))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "43", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "44", + "metadata": {}, + "outputs": [], + "source": [ + "fig = plt.figure(figsize=(8, 6))\n", + "plt.hist(test_labels_original[:, 0],bins=50,alpha=0.6,color=\"b\",label=\"reco coords\")\n", + "plt.hist(pred_data_original[:,0],bins=50,alpha=0.6,color=\"r\",label=\"test coords\")\n", + "plt.xlabel(\"Theta\")\n", + "plt.legend()\n", + "plt.ylabel(\"Counts\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "45", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "46", + "metadata": {}, + "outputs": [], + "source": [ + "fig = plt.figure(figsize=(8, 6))\n", + "plt.hist(test_labels_original[:, 1],bins=50,alpha=0.6,color=\"b\",label=\"reco coords\")\n", + "plt.hist(pred_data_original[:,1],bins=50,alpha=0.6,color=\"r\",label=\"test coords\")\n", + "plt.xlabel(\"Phi\")\n", + "plt.legend()\n", + "plt.ylabel(\"Counts\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "47", + "metadata": {}, + "outputs": [], + "source": [ + "plt.hist(pred_data_original[:,0])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "48", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "49", + "metadata": {}, + "outputs": [], + "source": [ + "pred_data_original" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "50", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "54", + "metadata": {}, + "outputs": [], + "source": [ + "fig = plt.figure(figsize=(8, 6))\n", + "plt.hist(test_labels[:,0],bins=50,alpha=0.6,color=\"b\",label=\"test coords\")\n", + "plt.hist(pred_data[:, 0],bins=50,alpha=0.6,color=\"r\",label=\"reco coords\")\n", + "plt.xlabel(\"x\")\n", + "plt.legend()\n", + "plt.ylabel(\"Counts\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "55", + "metadata": {}, + "outputs": [], + "source": [ + "fig = plt.figure(figsize=(8, 6))\n", + "plt.hist(test_labels[:,1],bins=50,alpha=0.6,color=\"b\",label=\"test coords\")\n", + "plt.hist(pred_data[:, 1],bins=50,alpha=0.6,color=\"r\",label=\"reco coords\")\n", + "plt.xlabel(\"y\")\n", + "plt.legend()\n", + "plt.ylabel(\"Counts\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "56", + "metadata": {}, + "outputs": [], + "source": [ + "fig = plt.figure(figsize=(8, 6))\n", + "plt.hist(test_labels[:,2],bins=50,alpha=0.6,color=\"b\",label=\"test coords\")\n", + "plt.hist(pred_data[:, 2],bins=50,alpha=0.6,color=\"r\",label=\"reco coords\")\n", + "plt.xlabel(\"z\")\n", + "plt.legend()\n", + "plt.ylabel(\"Counts\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5bc50fe5-2e84-43fd-b035-8de0520b8b8d", + "metadata": {}, + "outputs": [], + "source": [ + "def angular_distance(theta1, phi1, theta2, phi2):\n", + " \"\"\"\n", + " Compute the angular (great-circle) distance in degrees between two points\n", + " specified by (theta, phi) coordinates in degrees.\n", + "\n", + " Parameters:\n", + " theta1, phi1: floats, coordinates of the first point (degrees).\n", + " theta2, phi2: floats, coordinates of the second point (degrees).\n", + "\n", + " Returns:\n", + " angular_dist_deg: float, angular distance between the two points in degrees.\n", + " \"\"\"\n", + " # Adjust theta values by shifting -90\n", + " theta1 = 90 - theta1\n", + " theta2 = 90 - theta2\n", + "\n", + " # Convert angles to radians\n", + " theta1_rad = math.radians(theta1)\n", + " phi1_rad = math.radians(phi1)\n", + " theta2_rad = math.radians(theta2)\n", + " phi2_rad = math.radians(phi2)\n", + " \n", + " # Compute the difference in longitude\n", + " delta_phi = abs(phi1_rad - phi2_rad)\n", + "\n", + " # Apply spherical law of cosines\n", + " value = (math.sin(theta1_rad) * math.sin(theta2_rad) +\n", + " math.cos(theta1_rad) * math.cos(theta2_rad) * math.cos(delta_phi))\n", + " \n", + " # Clamp value to [-1, 1] to avoid floating point errors in acos\n", + " value_clamped = max(-1.0, min(1.0, value)) \n", + " \n", + " angular_dist = math.acos(value_clamped)\n", + " \n", + " # Convert angular distance from radians to degrees\n", + " angular_dist_deg = math.degrees(angular_dist)\n", + " \n", + " return angular_dist_deg\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "58", + "metadata": {}, + "outputs": [], + "source": [ + "# Calculate distances between corresponding coordinates\n", + "#distances = np.linalg.norm(pred_data_original - test_labels_original, axis=1)\n", + "\n", + "distances = []\n", + "theta_dist_arr = []\n", + "phi_dist_arr = []\n", + "\n", + "#print(pred_data_original)\n", + "#print(test_labels_original)\n", + "\n", + "for i in range(0,len(pred_data_original)):\n", + "\n", + " d = angular_distance(pred_data_original[i][0],pred_data_original[i][1],test_labels_original[i][0],test_labels_original[i][1])\n", + " distances.append(d)\n", + " theta_dist = np.abs(pred_data_original[i][0]-test_labels_original[i][0])\n", + " phi_dist = np.abs(pred_data_original[i][1]-test_labels_original[i][1])\n", + " theta_dist_arr.append(theta_dist)\n", + " phi_dist_arr.append(phi_dist)\n", + " \n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8c289cff-a503-4584-a56d-4fbf942892ac", + "metadata": {}, + "outputs": [], + "source": [ + "print(np.mean(distances))\n", + "print(np.mean(theta_dist_arr))\n", + "print(np.mean(phi_dist_arr))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ed00dad5-edf1-4d10-adcd-988177788b4f", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f76fa3ad-9901-428a-a14d-51b88e36b78c", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a4eafecb-a5db-4442-a0d2-436d7e29cba0", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b8be7a21-d4ab-4ee3-b11b-587b9b795d8a", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1c35a63e-d562-4df5-a6b6-bb3881fcc110", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ff84a166-74c8-4daf-baaa-80248afc7c95", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "301e36a8-ab80-4525-8b28-cca6e1516539", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "deeplearning", + "language": "python", + "name": "deeplearning" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.12" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/cosipy/notebooks/dl-loc/train_and_test/inference_cosin_model.ipynb b/cosipy/notebooks/dl-loc/train_and_test/inference_cosin_model.ipynb new file mode 100644 index 0000000..252a07d --- /dev/null +++ b/cosipy/notebooks/dl-loc/train_and_test/inference_cosin_model.ipynb @@ -0,0 +1,874 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "0", + "metadata": {}, + "outputs": [], + "source": [ + "# import libraries\n", + "import os\n", + "import pandas as pd\n", + "import numpy as np\n", + "import seaborn as sns\n", + "sns.set(color_codes=True)\n", + "import matplotlib.pyplot as plt\n", + "%matplotlib inline\n", + "import pickle\n", + "import tensorflow as tf\n", + "\n", + "file_name = \"run57_mix_mega_shared\"\n", + "model_name = \"run53_mix_mega_shared_tanh.keras\"\n", + "normalize= 1\n", + "test_number = 100000\n", + "os.environ[\"CUDA_VISIBLE_DEVICES\"]=\"-1\"\n", + "number_of_detectors = 6\n", + "data_dir = \"/data/test_newrepo\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1", + "metadata": {}, + "outputs": [], + "source": [ + "print(\"Num GPUs Available: \", len(tf.config.list_physical_devices('GPU')))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2", + "metadata": {}, + "outputs": [], + "source": [ + "file_path = data_dir+'/'+file_name+'_dataset.pkl'\n", + "\n", + "# Load the dataset from the pickle file\n", + "with open(file_path, 'rb') as file:\n", + " loaded_array = pickle.load(file)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3", + "metadata": {}, + "outputs": [], + "source": [ + "# Filter the dataset by spectral model and flux\n", + "\n", + "loaded_array_test = loaded_array\n", + "filter_flux = 0 #1:30\n", + "filter_spectra = 0\n", + "\n", + "filtered_loaded_array_test = np.empty(loaded_array_test.shape[0], dtype=object)\n", + "\n", + "count = 0\n", + "\n", + "if filter_flux == 1:\n", + " for grb in loaded_array_test:\n", + " if grb['flux'] > 20 and grb['flux'] <= 30:\n", + " filtered_loaded_array_test[count] = grb\n", + " count = count+1\n", + "\n", + " loaded_array_test = filtered_loaded_array_test[:count]\n", + "\n", + "filtered_loaded_array_test = np.empty(loaded_array_test.shape[0], dtype=object)\n", + "\n", + "count = 0\n", + "\n", + "if filter_spectra==1:\n", + " for grb in loaded_array_test:\n", + " if \"1500\" in grb['spectrum']: #[\"Band 10 10000 -1.9 -3.7 230\",\"Band 10 10000 -1 -2.3 699.9\",\"Comptonized 10 10000 -0.5 1500\"]\n", + " filtered_loaded_array_test[count] = grb\n", + " count = count+1\n", + "\n", + " loaded_array_test = filtered_loaded_array_test[:count]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "798acea4-5f63-4273-8868-1c98293b3283", + "metadata": {}, + "outputs": [], + "source": [ + "loaded_array_test.shape" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "419b96a0-a661-45be-8de6-c5b7dd60c152", + "metadata": {}, + "outputs": [], + "source": [ + "# extract a random sample from the testing dataset\n", + "filter_len = test_number\n", + "random_indices = np.random.permutation(len(loaded_array_test))\n", + "shuffled_loaded_array_bkg = loaded_array_test[random_indices]\n", + "test_dataset_raw = shuffled_loaded_array_bkg[:filter_len] " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "45c62534-8b8f-47a1-85f4-58df0b71db97", + "metadata": {}, + "outputs": [], + "source": [ + "test_dataset_raw.shape" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d0cbf4ca-9222-4356-a74c-62afcd4b9629", + "metadata": {}, + "outputs": [], + "source": [ + "def diff_phi(a1, a2):\n", + " diff = abs(a1 - a2)\n", + " if diff > 180:\n", + " diff = 360 - diff\n", + " return diff\n", + "\n", + "def convert_degrees_to_sin_cos(degrees):\n", + " radians = np.deg2rad(degrees) \n", + " sin = np.sin(radians)\n", + " cos = np.cos(radians)\n", + " return sin, cos\n", + "\n", + "def degrees_from_sin_cos(sin_value, cos_value):\n", + " \n", + " angle_rad = np.arctan2(sin_value, cos_value)\n", + " \n", + " angle_deg = np.degrees(angle_rad)\n", + " \n", + " if angle_deg < 0:\n", + " angle_deg += 360\n", + " return angle_deg\n", + "\n", + "def convert_degress_to_sin_cos_math(degrees):\n", + " \n", + " rad = math.radians(degrees)\n", + "\n", + " \n", + " sin = math.sin(rad)\n", + " cos = math.cos(rad)\n", + "\n", + " print(\"Seno:\", sin)\n", + " print(\"Coseno:\", cos)\n", + "\n", + "\n", + "def angular_distance(theta1, phi1, theta2, phi2):\n", + " \"\"\"\n", + " Compute the angular (great-circle) distance in degrees between two points\n", + " specified by (theta, phi) coordinates in degrees.\n", + "\n", + " Parameters:\n", + " theta1, phi1: floats, coordinates of the first point (degrees).\n", + " theta2, phi2: floats, coordinates of the second point (degrees).\n", + "\n", + " Returns:\n", + " angular_dist_deg: float, angular distance between the two points in degrees.\n", + " \"\"\"\n", + " # Adjust theta values by shifting -90\n", + " theta1 = 90 - theta1\n", + " theta2 = 90 - theta2\n", + "\n", + " # Convert angles to radians\n", + " theta1_rad = math.radians(theta1)\n", + " phi1_rad = math.radians(phi1)\n", + " theta2_rad = math.radians(theta2)\n", + " phi2_rad = math.radians(phi2)\n", + " \n", + " # Compute the difference in longitude\n", + " delta_phi = abs(phi1_rad - phi2_rad)\n", + "\n", + " # Apply spherical law of cosines\n", + " value = (math.sin(theta1_rad) * math.sin(theta2_rad) +\n", + " math.cos(theta1_rad) * math.cos(theta2_rad) * math.cos(delta_phi))\n", + " \n", + " # Clamp value to [-1, 1] to avoid floating point errors in acos\n", + " value_clamped = max(-1.0, min(1.0, value)) \n", + " \n", + " angular_dist = math.acos(value_clamped)\n", + " \n", + " # Convert angular distance from radians to degrees\n", + " angular_dist_deg = math.degrees(angular_dist)\n", + " \n", + " return angular_dist_deg\n", + " \n", + "\n", + "def l2_normalize(data):\n", + " \"\"\"\n", + " Normalize a NumPy array using the L2 (Euclidean) norm.\n", + "\n", + " Args:\n", + " data (numpy.ndarray): The array to normalize. Can be a 1D vector\n", + " or a 2D matrix (where each row is a vector to normalize).\n", + "\n", + " Returns:\n", + " numpy.ndarray: The L2-normalized array.\n", + " \"\"\"\n", + "\n", + " data = np.array(data)\n", + " if data.ndim == 1:\n", + " # Case: 1D vector\n", + " norm = np.sqrt(np.sum(data**2))\n", + " if norm == 0:\n", + " return data # Avoid division by zero if the vector is null\n", + " return data / norm\n", + " elif data.ndim == 2:\n", + " # Case: 2D matrix (normalize each row)\n", + " norms = np.sqrt(np.sum(data**2, axis=1, keepdims=True))\n", + " # Handle the case of null norms (zero rows)\n", + " norms[norms == 0] = 1\n", + " return data / norms\n", + " else:\n", + " raise ValueError(\"Input must be a 1D or 2D array.\")\n", + "\n", + " \n", + "def prepare_dataset(data_array):\n", + "\n", + " dataset = np.empty((len(data_array),number_of_detectors))\n", + " labels = np.empty((len(data_array),2))\n", + " \n", + " count = -1\n", + " for element in data_array:\n", + " \n", + " count+=1\n", + " # We are considering a Poissonian background. The mean counts are calculated from the DC3 BGO data.\n", + " b_sim = np.array([57.6053, 58.7157, 51.4131, 48.2891, 47.7293, 45.9617])\n", + " dataset[count] = l2_normalize(np.array(element['counts'])+np.random.poisson(b_sim*20)-b_sim*20)\n", + " #dataset[count] = l2_normalize(element['counts'])\n", + " labels[count][0] = float(element['coord'][0])\n", + " labels[count][1] = float(element['coord'][1])\n", + " \n", + " labels = labels[:count+1]\n", + " dataset = dataset[:count+1]\n", + "\n", + " # Convert to radians\n", + " coords_rad = []\n", + " \n", + " for theta, phi in labels:\n", + " coords_rad.append([theta, phi])\n", + " \n", + " theta, phi = zip(*coords_rad)\n", + " \n", + " labels_cosin = np.empty((len(labels),4))\n", + " count = -1\n", + " for element in labels:\n", + " \n", + " count=count+1\n", + " \n", + " theta_sin,theta_cos = convert_degrees_to_sin_cos(labels[count][0])\n", + " labels_cosin[count][0] = theta_sin\n", + " labels_cosin[count][1] = theta_cos\n", + " \n", + " phi_sin,phi_cos = convert_degrees_to_sin_cos(labels[count][1])\n", + " labels_cosin[count][2] = phi_sin\n", + " labels_cosin[count][3] = phi_cos\n", + " \n", + " labels_norm = labels_cosin\n", + " \n", + " if normalize == 1:\n", + " \n", + " labels_norm = np.empty((len(labels_cosin),4))\n", + " \n", + " for j in range(0,len(labels_cosin)):\n", + " labels_norm[j][0] = 2 * labels_cosin[j][0] - 1\n", + " labels_norm[j][1] = labels_cosin[j][1]\n", + " labels_norm[j][2] = labels_cosin[j][2]\n", + " labels_norm[j][3] = labels_cosin[j][3]\n", + " \n", + " else:\n", + " labels_norm = labels_cosin\n", + "\n", + " \n", + " \n", + " return dataset, labels, labels_norm, theta, phi, coords_rad\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ee75b63b-3c76-4a3d-b58d-d847d8a6ca3a", + "metadata": {}, + "outputs": [], + "source": [ + "test_dataset, labels, test_labels, theta, phi, coords_rad = prepare_dataset(test_dataset_raw)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3957a17d-20d1-4140-b3b8-fa212b20bd19", + "metadata": {}, + "outputs": [], + "source": [ + "test_dataset.shape" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9", + "metadata": {}, + "outputs": [], + "source": [ + "plt.hist(theta)\n", + "plt.xlabel(\"Theta\")\n", + "plt.ylabel(\"Counts\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "10", + "metadata": {}, + "outputs": [], + "source": [ + "plt.hist(phi)\n", + "plt.xlabel(\"Phi\")\n", + "plt.ylabel(\"Counts\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "20", + "metadata": {}, + "outputs": [], + "source": [ + "#scale labels and convert in cos and sin\n", + "max_lon = 180.0\n", + "min_lon = 0.0\n", + "\n", + "max_lat = 360.0\n", + "min_lat = 0.0\n", + "\n", + "labels_cosin = np.empty((len(labels),4))\n", + "count = -1\n", + "for element in labels:\n", + " \n", + " count=count+1\n", + " \n", + " theta_sin,theta_cos = convert_degrees_to_sin_cos(labels[count][0])\n", + " labels_cosin[count][0] = theta_sin\n", + " labels_cosin[count][1] = theta_cos\n", + " \n", + " phi_sin,phi_cos = convert_degrees_to_sin_cos(labels[count][1])\n", + " labels_cosin[count][2] = phi_sin\n", + " labels_cosin[count][3] = phi_cos\n", + "\n", + "labels_norm = labels_cosin" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "23", + "metadata": {}, + "outputs": [], + "source": [ + "if normalize == 1:\n", + "\n", + " labels_norm = np.empty((len(labels),4))\n", + "\n", + " min_max_values = []\n", + "\n", + "\n", + " for j in range(0,len(labels_cosin)):\n", + " labels_norm[j][0] = 2 * labels_cosin[j][0] - 1\n", + " labels_norm[j][1] = labels_cosin[j][1]\n", + " labels_norm[j][2] = labels_cosin[j][2]\n", + " labels_norm[j][3] = labels_cosin[j][3]\n", + " \n", + "else:\n", + " labels_norm = labels_cosin\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "243519f2", + "metadata": {}, + "outputs": [], + "source": [ + "plt.hist(labels_norm[:,0],bins=100)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "29", + "metadata": {}, + "outputs": [], + "source": [ + "plt.hist(labels_norm[:,1],bins=100)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "30", + "metadata": {}, + "outputs": [], + "source": [ + "plt.hist(labels_norm[:,2],bins=100)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "31", + "metadata": {}, + "outputs": [], + "source": [ + "plt.hist(labels_norm[:,3],bins=100)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "33", + "metadata": {}, + "outputs": [], + "source": [ + "from tensorflow.keras import backend as K\n", + "\n", + "def spherical_loss_2angles(y_true, y_pred):\n", + " # Compute the Mean Squared Error (MSE) between targets and predictions\n", + " mse_loss = K.mean(K.square(y_true - y_pred), axis=-1)\n", + " \n", + " # a the pairs (sinθ, cosθ) and (sinφ, cosφ) from the predictions\n", + " sin_theta, cos_theta = y_pred[:, 0], y_pred[:, 1]\n", + " sin_phi, cos_phi = y_pred[:, 2], y_pred[:, 3]\n", + " \n", + " # Compute the L2 norm of each pair\n", + " # (should ideally be equal to 1 if the network outputs valid sine/cosine pairs)\n", + " norm_theta = K.sqrt(sin_theta**2 + cos_theta**2)\n", + " norm_phi = K.sqrt(sin_phi**2 + cos_phi**2)\n", + " \n", + " # Compute the penalty as the squared deviation of each norm from 1\n", + " # This encourages the network to output normalized sine/cosine pairs\n", + " penalty_theta = K.square(norm_theta - 1.0)\n", + " penalty_phi = K.square(norm_phi - 1.0)\n", + " \n", + " penalty = penalty_theta + penalty_phi\n", + " \n", + " # Weight of the regularization term (can be tuned as a hyperparameter)\n", + " alpha = 0.01\n", + " \n", + " # Final loss = MSE + weighted normalization penalty\n", + " return mse_loss + alpha * penalty" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "67e8a640-58b1-4d72-afc0-a55a236e8e61", + "metadata": {}, + "outputs": [], + "source": [ + "from tensorflow.keras.models import load_model\n", + "\n", + "# Load the model back from the saved directory\n", + "model = load_model(data_dir+\"/\"+model_name,custom_objects={'spherical_loss_2angles': spherical_loss_2angles})" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "40", + "metadata": {}, + "outputs": [], + "source": [ + "pred_data = model.predict(test_dataset)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8cd52a3c-c518-411e-abff-474d0ceb2619", + "metadata": {}, + "outputs": [], + "source": [ + "test_dataset.shape" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "41", + "metadata": {}, + "outputs": [], + "source": [ + "import math\n", + "\n", + "def convert_original(pred_tmp,test_tmp):\n", + "\n", + " pred_data_renorm = np.empty((len(pred_tmp),4))\n", + " test_data_renorm = np.empty((len(test_tmp),4))\n", + " \n", + " \n", + " for j in range(0,len(pred_tmp)):\n", + " pred_data_renorm[j][0] = (pred_tmp[j][0] + 1) / 2\n", + " pred_data_renorm[j][1] = pred_tmp[j][1]\n", + " pred_data_renorm[j][2] = pred_tmp[j][2]\n", + " pred_data_renorm[j][3] = pred_tmp[j][3]\n", + " \n", + " for j in range(0,len(test_tmp)):\n", + " test_data_renorm[j][0] = (test_tmp[j][0] + 1) / 2\n", + " test_data_renorm[j][1] = test_tmp[j][1]\n", + " test_data_renorm[j][2] = test_tmp[j][2]\n", + " test_data_renorm[j][3] = test_tmp[j][3]\n", + " \n", + " pred_data_original_tmp = np.empty((len(pred_tmp),2))\n", + " test_labels_original_tmp = np.empty((len(test_tmp),2))\n", + " \n", + " for i, element in enumerate(test_data_renorm):\n", + " test_labels_original_tmp[i][0]=degrees_from_sin_cos(element[0],element[1])\n", + " test_labels_original_tmp[i][1]=degrees_from_sin_cos(element[2],element[3])\n", + " \n", + " for i, element in enumerate(pred_data_renorm):\n", + " \n", + " pred_data_original_tmp[i][0]=degrees_from_sin_cos(element[0],element[1])\n", + " pred_data_original_tmp[i][1]=degrees_from_sin_cos(element[2],element[3])\n", + " \n", + " absolute_diffs_theta = np.abs(pred_data_original_tmp[:, 0] - test_labels_original_tmp[:, 0])\n", + " absolute_diffs_phi = np.abs(pred_data_original_tmp[:, 1] - test_labels_original_tmp[:, 1])\n", + " \n", + " # Calculate the Mean Absolute Error (MAE) for each element separately\n", + " mae_theta = np.mean(absolute_diffs_theta)\n", + " mae_phi = np.mean(absolute_diffs_phi)\n", + "\n", + " print(\"Mean Absolute Error for Theta:\", mae_theta)\n", + " print(\"Mean Absolute Error for Phi:\", mae_phi)\n", + "\n", + " return pred_data_original_tmp , test_labels_original_tmp " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "99361d7b-e8a9-49c8-9bb5-ffe1e8719d7d", + "metadata": {}, + "outputs": [], + "source": [ + "pred_data_original, test_labels_original = convert_original(pred_data,test_labels)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "43", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "44", + "metadata": {}, + "outputs": [], + "source": [ + "fig = plt.figure(figsize=(8, 6))\n", + "plt.hist(test_labels_original[:, 0],bins=50,alpha=0.6,color=\"b\",label=\"reco coords\")\n", + "plt.hist(pred_data_original[:,0],bins=50,alpha=0.6,color=\"r\",label=\"test coords\")\n", + "plt.xlabel(\"Theta\")\n", + "plt.legend()\n", + "plt.ylabel(\"Counts\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "45", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "46", + "metadata": {}, + "outputs": [], + "source": [ + "fig = plt.figure(figsize=(8, 6))\n", + "plt.hist(test_labels_original[:, 1],bins=50,alpha=0.6,color=\"b\",label=\"reco coords\")\n", + "plt.hist(pred_data_original[:,1],bins=50,alpha=0.6,color=\"r\",label=\"test coords\")\n", + "plt.xlabel(\"Phi\")\n", + "plt.legend()\n", + "plt.ylabel(\"Counts\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "54", + "metadata": {}, + "outputs": [], + "source": [ + "fig = plt.figure(figsize=(8, 6))\n", + "plt.hist(test_labels[:,0],bins=50,alpha=0.6,color=\"b\",label=\"test coords\")\n", + "plt.hist(pred_data[:, 0],bins=50,alpha=0.6,color=\"r\",label=\"reco coords\")\n", + "plt.xlabel(\"Theta sin\")\n", + "plt.legend()\n", + "plt.ylabel(\"Counts\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "55", + "metadata": {}, + "outputs": [], + "source": [ + "fig = plt.figure(figsize=(8, 6))\n", + "plt.hist(test_labels[:,1],bins=50,alpha=0.6,color=\"b\",label=\"test coords\")\n", + "plt.hist(pred_data[:, 1],bins=50,alpha=0.6,color=\"r\",label=\"reco coords\")\n", + "plt.xlabel(\"Theta cos\")\n", + "plt.legend()\n", + "plt.ylabel(\"Counts\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "56", + "metadata": {}, + "outputs": [], + "source": [ + "fig = plt.figure(figsize=(8, 6))\n", + "plt.hist(test_labels[:,2],bins=50,alpha=0.6,color=\"b\",label=\"test coords\")\n", + "plt.hist(pred_data[:, 2],bins=50,alpha=0.6,color=\"r\",label=\"reco coords\")\n", + "plt.xlabel(\"Phi sin\")\n", + "plt.legend()\n", + "plt.ylabel(\"Counts\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "57", + "metadata": {}, + "outputs": [], + "source": [ + "fig = plt.figure(figsize=(8, 6))\n", + "plt.hist(test_labels[:,3],bins=50,alpha=0.6,color=\"b\",label=\"test coords\")\n", + "plt.hist(pred_data[:, 3],bins=50,alpha=0.6,color=\"r\",label=\"reco coords\")\n", + "plt.xlabel(\"Phi cos\")\n", + "plt.legend()\n", + "plt.ylabel(\"Counts\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "58", + "metadata": {}, + "outputs": [], + "source": [ + "# Calculate distances between corresponding coordinates\n", + "\n", + "def angular_distance_phi(phi1, phi2):\n", + " \n", + " diff = abs(phi1 - phi2) % 360\n", + " return min(diff, 360 - diff)\n", + "\n", + "def calculated_distances(pred,test):\n", + " distances = []\n", + " theta_dist_arr = []\n", + " phi_dist_arr = []\n", + "\n", + " \n", + " for i in range(0,len(pred)):\n", + " \n", + " d = angular_distance(pred[i][0],pred[i][1],test[i][0],test[i][1])\n", + " distances.append(d)\n", + " theta_dist = np.abs(pred[i][0]-test[i][0])\n", + " phi_dist = angular_distance_phi(pred[i][1],test[i][1])\n", + " theta_dist_arr.append(theta_dist)\n", + " phi_dist_arr.append(phi_dist)\n", + " \n", + " \n", + " return distances,theta_dist_arr,phi_dist_arr" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "537315d2-8419-4a82-aba9-52c0d49c1a17", + "metadata": {}, + "outputs": [], + "source": [ + "distances,theta_dist_arr,phi_dist_arr = calculated_distances(pred_data_original,test_labels_original)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "60f49450-a027-4191-b84f-41adb97bea5a", + "metadata": {}, + "outputs": [], + "source": [ + "print(np.mean(distances))\n", + "print(np.mean(theta_dist_arr))\n", + "print(np.mean(phi_dist_arr))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "00dfc9a3-2d44-4fa5-ba86-d517627e3994", + "metadata": {}, + "outputs": [], + "source": [ + "plt.hist(distances)\n", + "plt.ylabel(\"Counts\")\n", + "plt.xlabel(\"Error Radius °\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "75", + "metadata": {}, + "outputs": [], + "source": [ + "# Define 10-degree intervals\n", + "intervals = np.arange(0, 365, 10)\n", + "\n", + "# Group counts based on 10-degree intervals\n", + "grouped_counts = np.zeros((len(intervals)))\n", + "counts = np.zeros((len(intervals)))\n", + "\n", + "total_count = 0\n", + "for theta, phi, count in zip(test_labels_original[:,0], test_labels_original[:,1], distances):\n", + " \n", + " if (theta > 20 and theta < 55): # Condition on theta\n", + " total_count += 1\n", + " interval_index = int(phi // 10)\n", + " grouped_counts[interval_index] = grouped_counts[interval_index] + count\n", + " counts[interval_index] += 1\n", + "\n", + "# Compute the average counts in each interval\n", + "grouped_counts = grouped_counts[:total_count]\n", + "counts = counts[:total_count]\n", + "\n", + "# Display histogram\n", + "plt.figure(figsize=(10, 6))\n", + "plt.bar(intervals, grouped_counts / counts, width=10, align='edge', alpha=1, label=\"loc. error\")\n", + "plt.xlabel('Phi (°)')\n", + "plt.ylabel('Loc. error')\n", + "plt.title('Loc. error as a function of phi')\n", + "plt.grid(True)\n", + "plt.legend()\n", + "plt.show()\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "79", + "metadata": {}, + "outputs": [], + "source": [ + "# Define 5-degree intervals\n", + "intervals = np.arange(0, 185, 5)\n", + "\n", + "# Group counts based on 5-degree intervals\n", + "grouped_counts = np.zeros((len(intervals)))\n", + "counts = np.zeros((len(intervals)))\n", + "\n", + "total_count = 0\n", + "for theta, phi, count in zip(test_labels_original[:,0], test_labels_original[:,1], distances):\n", + " \n", + " if(True): # (phi > 125 and phi < 145) or ...\n", + " total_count += 1\n", + " interval_index = int(theta // 5)\n", + " grouped_counts[interval_index] = grouped_counts[interval_index] + count\n", + " counts[interval_index] += 1\n", + "\n", + "# Compute the average counts in each interval\n", + "grouped_counts = grouped_counts[:total_count]\n", + "counts = counts[:total_count]\n", + "\n", + "# Display histogram\n", + "plt.figure(figsize=(10, 6))\n", + "plt.bar(intervals[:-1], grouped_counts[:-1] / counts[:-1], width=5, align='edge', alpha=1, label=\"loc. error\")\n", + "plt.xlabel('Theta (°)')\n", + "plt.ylabel('Loc. error')\n", + "plt.title('Loc. error as a function of theta')\n", + "plt.grid(True)\n", + "plt.legend()\n", + "plt.show()\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "527add68-000a-440a-9a6d-a231b36fe9b1", + "metadata": {}, + "outputs": [], + "source": [ + "print(np.mean(distances))\n", + "print(np.mean(theta_dist_arr))\n", + "print(np.mean(phi_dist_arr))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "634a712e-3df1-47c5-8777-d1da8cc40879", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "id": "1684b6cb-938c-4a70-b2ba-64b64835e615", + "metadata": {}, + "source": [ + "### " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4930f92a-c10a-4af0-8bff-f543f492218a", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "deeplearning", + "language": "python", + "name": "deeplearning" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.12" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/cosipy/notebooks/dl-loc/train_and_test/inference_cosin_plot.ipynb b/cosipy/notebooks/dl-loc/train_and_test/inference_cosin_plot.ipynb new file mode 100644 index 0000000..dea1788 --- /dev/null +++ b/cosipy/notebooks/dl-loc/train_and_test/inference_cosin_plot.ipynb @@ -0,0 +1,743 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "0", + "metadata": {}, + "outputs": [], + "source": [ + "# import libraries\n", + "import os\n", + "import pandas as pd\n", + "import numpy as np\n", + "import seaborn as sns\n", + "sns.set(color_codes=True)\n", + "import matplotlib.pyplot as plt\n", + "%matplotlib inline\n", + "import pickle\n", + "import tensorflow as tf\n", + "import math\n", + "\n", + "file_name = \"run63_medium_10fact_mega_shared\"\n", + "model_name = \"run53_mix_mega_shared_tanh.keras\"\n", + "normalize= 1\n", + "test_number = 100000\n", + "os.environ[\"CUDA_VISIBLE_DEVICES\"]=\"-1\"\n", + "number_of_detectors = 6\n", + "data_dir = \"/data/test_newrepo\"\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8a5b61f6-26b9-4d8f-9cd0-b1c8141de091", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1", + "metadata": {}, + "outputs": [], + "source": [ + "print(\"Num GPUs Available: \", len(tf.config.list_physical_devices('GPU')))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2", + "metadata": {}, + "outputs": [], + "source": [ + "file_path = data_dir+'/'+file_name+'_dataset.pkl'\n", + "\n", + "# Load the dataset from the pickle file\n", + "with open(file_path, 'rb') as file:\n", + " evaluation_array = pickle.load(file)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3", + "metadata": {}, + "outputs": [], + "source": [ + "evaluation_array.shape" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5ad68182-be81-4e9e-8e63-cc202dad3bef", + "metadata": {}, + "outputs": [], + "source": [ + "def diff_phi(a1, a2):\n", + " diff = abs(a1 - a2)\n", + " if diff > 180:\n", + " diff = 360 - diff\n", + " return diff\n", + "\n", + "def convert_degrees_to_sin_cos(degrees):\n", + " radians = np.deg2rad(degrees) \n", + " sin = np.sin(radians)\n", + " cos = np.cos(radians)\n", + " return sin, cos\n", + "\n", + "def degrees_from_sin_cos(sin_value, cos_value):\n", + " angle_rad = np.arctan2(sin_value, cos_value)\n", + " angle_deg = np.degrees(angle_rad)\n", + " if angle_deg < 0:\n", + " angle_deg += 360\n", + " return angle_deg\n", + "\n", + "def convert_degress_to_sin_cos_math(degrees):\n", + " \n", + " rad = math.radians(degrees)\n", + " \n", + " sin = math.sin(rad)\n", + " cos = math.cos(rad)\n", + "\n", + " print(\"Seno:\", sin)\n", + " print(\"Coseno:\", cos)\n", + "\n", + "\n", + "def angular_distance(theta1, phi1, theta2, phi2):\n", + " \"\"\"\n", + " Compute the angular (great-circle) distance in degrees between two points\n", + " specified by (theta, phi) coordinates in degrees.\n", + "\n", + " Parameters:\n", + " theta1, phi1: floats, coordinates of the first point (degrees).\n", + " theta2, phi2: floats, coordinates of the second point (degrees).\n", + "\n", + " Returns:\n", + " angular_dist_deg: float, angular distance between the two points in degrees.\n", + " \"\"\"\n", + " # Adjust theta values by shifting -90\n", + " theta1 = 90 - theta1\n", + " theta2 = 90 - theta2\n", + "\n", + " # Convert angles to radians\n", + " theta1_rad = math.radians(theta1)\n", + " phi1_rad = math.radians(phi1)\n", + " theta2_rad = math.radians(theta2)\n", + " phi2_rad = math.radians(phi2)\n", + " \n", + " # Compute the difference in longitude\n", + " delta_phi = abs(phi1_rad - phi2_rad)\n", + "\n", + " # Apply spherical law of cosines\n", + " value = (math.sin(theta1_rad) * math.sin(theta2_rad) +\n", + " math.cos(theta1_rad) * math.cos(theta2_rad) * math.cos(delta_phi))\n", + " \n", + " # Clamp value to [-1, 1] to avoid floating point errors in acos\n", + " value_clamped = max(-1.0, min(1.0, value)) \n", + " \n", + " angular_dist = math.acos(value_clamped)\n", + " \n", + " # Convert angular distance from radians to degrees\n", + " angular_dist_deg = math.degrees(angular_dist)\n", + " \n", + " return angular_dist_deg\n", + " \n", + "\n", + "def l2_normalize(data):\n", + " \"\"\"\n", + " Normalize a NumPy array using the L2 (Euclidean) norm.\n", + "\n", + " Args:\n", + " data (numpy.ndarray): The array to normalize. Can be a 1D vector\n", + " or a 2D matrix (where each row is a vector to normalize).\n", + "\n", + " Returns:\n", + " numpy.ndarray: The L2-normalized array.\n", + " \"\"\"\n", + "\n", + " data = np.array(data)\n", + " if data.ndim == 1:\n", + " # Case: 1D vector\n", + " norm = np.sqrt(np.sum(data**2))\n", + " if norm == 0:\n", + " return data # Avoid division by zero if the vector is null\n", + " return data / norm\n", + " elif data.ndim == 2:\n", + " # Case: 2D matrix (normalize each row)\n", + " norms = np.sqrt(np.sum(data**2, axis=1, keepdims=True))\n", + " # Handle the case of null norms (zero rows)\n", + " norms[norms == 0] = 1\n", + " return data / norms\n", + " else:\n", + " raise ValueError(\"Input must be a 1D or 2D array.\")\n", + "\n", + " \n", + "def prepare_dataset(data_array):\n", + "\n", + " dataset = np.empty((len(data_array),number_of_detectors))\n", + " labels = np.empty((len(data_array),2))\n", + " \n", + " count = -1\n", + " for element in data_array:\n", + " \n", + " count+=1\n", + " # We are considering a Poissonian background. The mean counts are calculated from the DC3 BGO data.\n", + " b_sim = np.array([57.6053, 58.7157, 51.4131, 48.2891, 47.7293, 45.9617])\n", + " dataset[count] = l2_normalize(np.array(element['counts'])+np.random.poisson(b_sim*20)-b_sim*20)\n", + " #dataset[count] = l2_normalize(element['counts'])\n", + " labels[count][0] = float(element['coord'][0])\n", + " labels[count][1] = float(element['coord'][1])\n", + " \n", + " labels = labels[:count+1]\n", + " dataset = dataset[:count+1]\n", + "\n", + " # Convert to radians\n", + " coords_rad = []\n", + " \n", + " for theta, phi in labels:\n", + " coords_rad.append([theta, phi])\n", + " \n", + " theta, phi = zip(*coords_rad)\n", + " \n", + " labels_cosin = np.empty((len(labels),4))\n", + " count = -1\n", + " for element in labels:\n", + " \n", + " count=count+1\n", + " \n", + " theta_sin,theta_cos = convert_degrees_to_sin_cos(labels[count][0])\n", + " labels_cosin[count][0] = theta_sin\n", + " labels_cosin[count][1] = theta_cos\n", + " \n", + " phi_sin,phi_cos = convert_degrees_to_sin_cos(labels[count][1])\n", + " labels_cosin[count][2] = phi_sin\n", + " labels_cosin[count][3] = phi_cos\n", + " \n", + " labels_norm = labels_cosin\n", + " \n", + " if normalize == 1:\n", + " \n", + " labels_norm = np.empty((len(labels_cosin),4))\n", + " \n", + " for j in range(0,len(labels_cosin)):\n", + " labels_norm[j][0] = 2 * labels_cosin[j][0] - 1\n", + " labels_norm[j][1] = labels_cosin[j][1]\n", + " labels_norm[j][2] = labels_cosin[j][2]\n", + " labels_norm[j][3] = labels_cosin[j][3]\n", + " \n", + " else:\n", + " labels_norm = labels_cosin\n", + "\n", + " \n", + " \n", + " return dataset, labels, labels_norm, theta, phi, coords_rad\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "08718db5", + "metadata": {}, + "outputs": [], + "source": [ + "evaluation_array_flatten = flattened = evaluation_array.flatten()\n", + "print(evaluation_array_flatten.shape)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "efa41147-0aab-4ce5-bec7-b2b1b0382f95", + "metadata": {}, + "outputs": [], + "source": [ + "test_dataset, test_labels_raw, test_labels_norm, test_theta, test_phi, test_coords_rad = prepare_dataset(evaluation_array_flatten)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "17f07fbd-5fb8-417c-b970-881b4a7b60ab", + "metadata": {}, + "outputs": [], + "source": [ + "test_labels_norm.shape" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7", + "metadata": {}, + "outputs": [], + "source": [ + "#split val and test datas\n", + "test_labels = test_labels_norm" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9", + "metadata": {}, + "outputs": [], + "source": [ + "plt.hist(test_theta,alpha=0.5)\n", + "plt.xlabel(\"Theta\")\n", + "plt.ylabel(\"Counts\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "10", + "metadata": {}, + "outputs": [], + "source": [ + "plt.hist(test_phi,alpha=0.5)\n", + "plt.xlabel(\"Phi\")\n", + "plt.ylabel(\"Counts\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "24", + "metadata": {}, + "outputs": [], + "source": [ + "plt.hist(test_labels_norm[:,0],alpha=0.5)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "29", + "metadata": {}, + "outputs": [], + "source": [ + "plt.hist(test_labels_norm[:,1],alpha=0.5)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "30", + "metadata": {}, + "outputs": [], + "source": [ + "plt.hist(test_labels_norm[:,2],alpha=0.5)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "31", + "metadata": {}, + "outputs": [], + "source": [ + "plt.hist(test_labels_norm[:,3],alpha=0.5)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "31781709-2496-46d5-bdf5-0459d0c674b8", + "metadata": {}, + "outputs": [], + "source": [ + "from tensorflow.keras import backend as K\n", + "\n", + "def spherical_loss_2angles(y_true, y_pred):\n", + " # Compute the Mean Squared Error (MSE) between targets and predictions\n", + " mse_loss = K.mean(K.square(y_true - y_pred), axis=-1)\n", + " \n", + " # a the pairs (sinθ, cosθ) and (sinφ, cosφ) from the predictions\n", + " sin_theta, cos_theta = y_pred[:, 0], y_pred[:, 1]\n", + " sin_phi, cos_phi = y_pred[:, 2], y_pred[:, 3]\n", + " \n", + " # Compute the L2 norm of each pair\n", + " # (should ideally be equal to 1 if the network outputs valid sine/cosine pairs)\n", + " norm_theta = K.sqrt(sin_theta**2 + cos_theta**2)\n", + " norm_phi = K.sqrt(sin_phi**2 + cos_phi**2)\n", + " \n", + " # Compute the penalty as the squared deviation of each norm from 1\n", + " # This encourages the network to output normalized sine/cosine pairs\n", + " penalty_theta = K.square(norm_theta - 1.0)\n", + " penalty_phi = K.square(norm_phi - 1.0)\n", + " \n", + " penalty = penalty_theta + penalty_phi\n", + " \n", + " # Weight of the regularization term (can be tuned as a hyperparameter)\n", + " alpha = 0.01\n", + " \n", + " # Final loss = MSE + weighted normalization penalty\n", + " return mse_loss + alpha * penalty" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "38", + "metadata": {}, + "outputs": [], + "source": [ + "from tensorflow.keras.models import load_model\n", + "\n", + "# Load the model back from the saved directory\n", + "model = load_model(data_dir+\"/\"+model_name,custom_objects={'spherical_loss_2angles': spherical_loss_2angles})" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "40", + "metadata": {}, + "outputs": [], + "source": [ + "\n", + "pred_data = model.predict(test_dataset)\n", + " " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e72ef346-610f-4e04-9fc4-bc2a86f6cb8a", + "metadata": {}, + "outputs": [], + "source": [ + "pred_data.shape" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "41", + "metadata": {}, + "outputs": [], + "source": [ + "pred_data_renorm = np.empty((len(pred_data),4))\n", + "test_data_renorm = np.empty((len(test_labels),4))\n", + "\n", + "for j in range(0,len(pred_data)):\n", + " pred_data_renorm[j][0] = (pred_data[j][0] + 1) / 2\n", + " pred_data_renorm[j][1] = pred_data[j][1]\n", + " pred_data_renorm[j][2] = pred_data[j][2]\n", + " pred_data_renorm[j][3] = pred_data[j][3]\n", + " \n", + "min_val = 0\n", + "max_val = 1\n", + "\n", + "for j in range(0,len(test_labels)):\n", + " test_data_renorm[j][0] = (test_labels[j][0] + 1) / 2\n", + " test_data_renorm[j][1] = test_labels[j][1]\n", + " test_data_renorm[j][2] = test_labels[j][2]\n", + " test_data_renorm[j][3] = test_labels[j][3]\n", + " \n", + "pred_data_original = np.empty((len(pred_data),2))\n", + "test_labels_original = np.empty((len(test_labels),2))\n", + " \n", + "for i, element in enumerate(test_data_renorm):\n", + " test_labels_original[i][0]=degrees_from_sin_cos(element[0],element[1])\n", + " test_labels_original[i][1]=degrees_from_sin_cos(element[2],element[3])\n", + " \n", + "\n", + "for i, element in enumerate(pred_data_renorm):\n", + " \n", + " pred_data_original[i][0]=degrees_from_sin_cos(element[0],element[1])\n", + " pred_data_original[i][1]=degrees_from_sin_cos(element[2],element[3])\n", + " \n", + " \n", + "absolute_diffs_theta = np.abs(pred_data_original[:, 0] - test_labels_original[:, 0])\n", + "absolute_diffs_phi = np.abs(pred_data_original[:, 1] - test_labels_original[:, 1])\n", + " \n", + "# Calculate the Mean Absolute Error (MAE) for each element separately\n", + "mae_theta = np.mean(absolute_diffs_theta)\n", + "mae_phi = np.mean(absolute_diffs_phi)\n", + "\n", + "print(\"Mean Absolute Error for Theta:\", mae_theta)\n", + "print(\"Mean Absolute Error for Phi:\", mae_phi)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "42", + "metadata": {}, + "outputs": [], + "source": [ + "plt.hist(test_data_renorm[:,0])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "44", + "metadata": {}, + "outputs": [], + "source": [ + "fig = plt.figure(figsize=(8, 6))\n", + "plt.hist(test_labels_original[:, 0],bins=50,alpha=0.6,color=\"b\",label=\"test coords\")\n", + "plt.hist(pred_data_original[:,0],bins=50,alpha=0.6,color=\"r\",label=\"reco coords\")\n", + "plt.xlabel(\"Theta\")\n", + "plt.legend()\n", + "plt.ylabel(\"Counts\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9e4f9f85", + "metadata": {}, + "outputs": [], + "source": [ + "np.sin(87.612)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0a92b796", + "metadata": {}, + "outputs": [], + "source": [ + "test_labels_original[:, 0]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "46", + "metadata": {}, + "outputs": [], + "source": [ + "fig = plt.figure(figsize=(8, 6))\n", + "plt.hist(test_labels_original[:, 1],bins=50,alpha=0.6,color=\"b\",label=\"reco coords\")\n", + "plt.hist(pred_data_original[:,1],bins=50,alpha=0.6,color=\"r\",label=\"test coords\")\n", + "plt.xlabel(\"Phi\")\n", + "plt.legend()\n", + "plt.ylabel(\"Counts\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "47", + "metadata": {}, + "outputs": [], + "source": [ + "plt.hist(pred_data_original[:,0])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "49", + "metadata": {}, + "outputs": [], + "source": [ + "pred_data_original" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "54", + "metadata": {}, + "outputs": [], + "source": [ + "fig = plt.figure(figsize=(8, 6))\n", + "plt.hist(test_labels[:,0],bins=50,alpha=0.6,color=\"b\",label=\"test coords\")\n", + "plt.hist(pred_data[:, 0],bins=50,alpha=0.6,color=\"r\",label=\"reco coords\")\n", + "plt.xlabel(\"Theta sin\")\n", + "plt.legend()\n", + "plt.ylabel(\"Counts\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "55", + "metadata": {}, + "outputs": [], + "source": [ + "fig = plt.figure(figsize=(8, 6))\n", + "plt.hist(test_labels[:,1],bins=50,alpha=0.6,color=\"b\",label=\"test coords\")\n", + "plt.hist(pred_data[:, 1],bins=50,alpha=0.6,color=\"r\",label=\"reco coords\")\n", + "plt.xlabel(\"Theta cos\")\n", + "plt.legend()\n", + "plt.ylabel(\"Counts\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "56", + "metadata": {}, + "outputs": [], + "source": [ + "fig = plt.figure(figsize=(8, 6))\n", + "plt.hist(test_labels[:,2],bins=50,alpha=0.6,color=\"b\",label=\"test coords\")\n", + "plt.hist(pred_data[:, 2],bins=50,alpha=0.6,color=\"r\",label=\"reco coords\")\n", + "plt.xlabel(\"Phi sin\")\n", + "plt.legend()\n", + "plt.ylabel(\"Counts\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b5b7779b-43f4-4030-bb49-68d05c9d66da", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "57", + "metadata": {}, + "outputs": [], + "source": [ + "fig = plt.figure(figsize=(8, 6))\n", + "plt.hist(test_labels[:,3],bins=50,alpha=0.6,color=\"b\",label=\"test coords\")\n", + "plt.hist(pred_data[:, 3],bins=50,alpha=0.6,color=\"r\",label=\"reco coords\")\n", + "plt.xlabel(\"Phi cos\")\n", + "plt.legend()\n", + "plt.ylabel(\"Counts\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3e43e38c", + "metadata": {}, + "outputs": [], + "source": [ + "pred_data_original_reshaped = pred_data_original.reshape(evaluation_array.shape[0], evaluation_array.shape[1], 2)\n", + "test_labels_original_reshaped = test_labels_original.reshape(evaluation_array.shape[0], evaluation_array.shape[1], 2)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "58", + "metadata": {}, + "outputs": [], + "source": [ + "# Calculate distances between corresponding coordinates\n", + "\n", + "distances = []\n", + "theta_distances = []\n", + "phi_distances = []\n", + "\n", + "for i in range(0,len(pred_data_original_reshaped)):\n", + " \n", + " distances_partial = []\n", + " theta_distances_partial = []\n", + " phi_distances_partial = []\n", + "\n", + " \n", + " for j in range(0,len(pred_data_original_reshaped[i])):\n", + " \n", + "\n", + " d = angular_distance(pred_data_original_reshaped[i][j][0],pred_data_original_reshaped[i][j][1],test_labels_original_reshaped[i][j][0],test_labels_original_reshaped[i][j][1])\n", + " theta_dist = np.abs(pred_data_original_reshaped[i][j][0]-test_labels_original_reshaped[i][j][0])\n", + " phi_dist = diff_phi(pred_data_original_reshaped[i][j][1],test_labels_original_reshaped[i][j][1])\n", + " distances_partial.append(d)\n", + " theta_distances_partial.append(theta_dist)\n", + " phi_distances_partial.append(phi_dist) \n", + " \n", + " distances_avg = np.mean(distances_partial)\n", + " theta_distances_avg = np.mean(theta_distances_partial)\n", + " phi_distances_avg = np.mean(phi_distances_partial)\n", + "\n", + " \n", + " distances.append(distances_avg)\n", + " theta_distances.append(theta_distances_avg)\n", + " phi_distances.append(phi_distances_avg)\n", + " " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "fb0c82c2-5e42-4103-adf5-a0cb24175f68", + "metadata": {}, + "outputs": [], + "source": [ + "import pickle\n", + "if False:\n", + "\n", + " # Store the array for further plotting procedure\n", + " with open(data_dir+\"/\"+evaluation_file+\"_distances_tanh_nobkg_plot.pkl\", \"wb\") as f:\n", + " pickle.dump(distances, f)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c2afbbdc-d1f0-44ca-8c3f-1d5f88e74149", + "metadata": {}, + "outputs": [], + "source": [ + "import numpy as np\n", + "import healpy as hp\n", + "import matplotlib.pyplot as plt\n", + "from matplotlib.colors import PowerNorm\n", + "\n", + "\n", + "nside = 16\n", + "npix = hp.nside2npix(nside)\n", + "\n", + "m = np.array(distances)\n", + "\n", + "new_m = np.sqrt(m)\n", + "\n", + "hp.projview(\n", + " m,\n", + " coord=[\"G\"],\n", + " graticule=True,\n", + " graticule_labels=True,\n", + " unit=\"cbar label\",\n", + " xlabel=\"longitude\",\n", + " ylabel=\"latitude\",\n", + " cb_orientation=\"vertical\",\n", + " latitude_grid_spacing=30,\n", + " projection_type=\"aitoff\",\n", + " title=\"Aitoff projection\",\n", + " cmap=\"turbo\",\n", + " nest=True\n", + ")\n", + "\n", + "plt.show()\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "deeplearning", + "language": "python", + "name": "deeplearning" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.12" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/cosipy/notebooks/dl-loc/train_and_test/inference_simple_model_plot.ipynb b/cosipy/notebooks/dl-loc/train_and_test/inference_simple_model_plot.ipynb new file mode 100644 index 0000000..6b466cd --- /dev/null +++ b/cosipy/notebooks/dl-loc/train_and_test/inference_simple_model_plot.ipynb @@ -0,0 +1,547 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "0", + "metadata": {}, + "outputs": [], + "source": [ + "# import libraries\n", + "import os\n", + "import pandas as pd\n", + "import numpy as np\n", + "import seaborn as sns\n", + "sns.set(color_codes=True)\n", + "import matplotlib.pyplot as plt\n", + "%matplotlib inline\n", + "import pickle\n", + "import matplotlib.pyplot as plt\n", + "import numpy as np\n", + "import tensorflow as tf\n", + "import math\n", + "\n", + "evaluation_file = \"run63_medium_10fact_mega_shared\"\n", + "model_name = \"run53_mix_mega_shared_simple_model\"\n", + "\n", + "normalize= 0\n", + "os.environ[\"CUDA_VISIBLE_DEVICES\"]=\"-1\"\n", + "number_of_detectors = 6\n", + "data_dir = \"/data/test_newrepo\"\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1", + "metadata": {}, + "outputs": [], + "source": [ + "print(\"Num GPUs Available: \", len(tf.config.list_physical_devices('GPU')))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2", + "metadata": {}, + "outputs": [], + "source": [ + "file_path = data_dir+'/'+evaluation_file+'_dataset.pkl'\n", + "\n", + "with open(file_path, 'rb') as file:\n", + " evaluation_array = pickle.load(file)\n", + " " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3", + "metadata": {}, + "outputs": [], + "source": [ + "evaluation_array.shape" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5ad68182-be81-4e9e-8e63-cc202dad3bef", + "metadata": {}, + "outputs": [], + "source": [ + "def diff_phi(a1, a2):\n", + " diff = abs(a1 - a2)\n", + " if diff > 180:\n", + " diff = 360 - diff\n", + " return diff\n", + "\n", + "def angular_distance(theta1, phi1, theta2, phi2):\n", + " \"\"\"\n", + " Compute the angular (great-circle) distance in degrees between two points\n", + " specified by (theta, phi) coordinates in degrees.\n", + "\n", + " Parameters:\n", + " theta1, phi1: floats, coordinates of the first point (degrees).\n", + " theta2, phi2: floats, coordinates of the second point (degrees).\n", + "\n", + " Returns:\n", + " angular_dist_deg: float, angular distance between the two points in degrees.\n", + " \"\"\"\n", + " # Adjust theta values by shifting -90\n", + " theta1 = 90 - theta1\n", + " theta2 = 90 - theta2\n", + "\n", + " # Convert angles to radians\n", + " theta1_rad = math.radians(theta1)\n", + " phi1_rad = math.radians(phi1)\n", + " theta2_rad = math.radians(theta2)\n", + " phi2_rad = math.radians(phi2)\n", + " \n", + " # Compute the difference in longitude\n", + " delta_phi = abs(phi1_rad - phi2_rad)\n", + "\n", + " # Apply spherical law of cosines\n", + " value = (math.sin(theta1_rad) * math.sin(theta2_rad) +\n", + " math.cos(theta1_rad) * math.cos(theta2_rad) * math.cos(delta_phi))\n", + " \n", + " # Clamp value to [-1, 1] to avoid floating point errors in acos\n", + " value_clamped = max(-1.0, min(1.0, value)) \n", + " \n", + " angular_dist = math.acos(value_clamped)\n", + " \n", + " # Convert angular distance from radians to degrees\n", + " angular_dist_deg = math.degrees(angular_dist)\n", + " \n", + " return angular_dist_deg\n", + "\n", + "\n", + "def l2_normalize(data):\n", + " \"\"\"\n", + " Normalize a NumPy array using the L2 (Euclidean) norm.\n", + "\n", + " Args:\n", + " data (numpy.ndarray): The array to normalize. Can be a 1D vector\n", + " or a 2D matrix (where each row is a vector to normalize).\n", + "\n", + " Returns:\n", + " numpy.ndarray: The L2-normalized array.\n", + " \"\"\"\n", + "\n", + " data = np.array(data)\n", + " if data.ndim == 1:\n", + " # Case: 1D vector\n", + " norm = np.sqrt(np.sum(data**2))\n", + " if norm == 0:\n", + " return data # Avoid division by zero if the vector is null\n", + " return data / norm\n", + " elif data.ndim == 2:\n", + " # Case: 2D matrix (normalize each row)\n", + " norms = np.sqrt(np.sum(data**2, axis=1, keepdims=True))\n", + " # Handle the case of null norms (zero rows)\n", + " norms[norms == 0] = 1\n", + " return data / norms\n", + " else:\n", + " raise ValueError(\"Input must be a 1D or 2D array.\")\n", + "\n", + " \n", + "def prepare_dataset(data_array):\n", + " \"\"\"\n", + " Prepare the dataset and corresponding labels for training or evaluation.\n", + "\n", + " Steps performed:\n", + " 1. Normalize the detector counts using L2 normalization.\n", + " 2. Extract longitude and latitude labels.\n", + " 3. Optionally scale labels between 0 and 1 (if `normalize == 1`).\n", + " 4. Convert labels to radians for further processing.\n", + "\n", + " Args:\n", + " data_array (list of dict): Each element must contain:\n", + " - 'counts' (numpy.ndarray): Detector count values.\n", + " - 'coord' (tuple or list): Coordinates (longitude, latitude).\n", + "\n", + " Returns:\n", + " tuple:\n", + " - dataset (numpy.ndarray): Normalized detector counts.\n", + " - labels (numpy.ndarray): Original labels (longitude, latitude).\n", + " - labels_norm (numpy.ndarray): Scaled labels (if normalization applied).\n", + " - theta (tuple): Extracted theta values in degrees.\n", + " - phi (tuple): Extracted phi values in degrees.\n", + " - coords_rad (list): List of coordinates in radians.\n", + " \"\"\"\n", + "\n", + " dataset = np.empty((len(data_array), number_of_detectors))\n", + " labels = np.empty((len(data_array), 2))\n", + " \n", + " count = -1\n", + " for element in data_array:\n", + " count += 1\n", + " dataset[count] = l2_normalize(element['counts'])\n", + " labels[count][0] = float(element['coord'][0])\n", + " labels[count][1] = float(element['coord'][1])\n", + " \n", + " labels = labels[:count+1]\n", + " dataset = dataset[:count+1]\n", + "\n", + " \n", + " labels_norm = np.empty((len(dataset), 2))\n", + " \n", + " count = -1\n", + " for element in labels:\n", + " count += 1\n", + " labels_norm[count][0] = labels[count][0]\n", + " labels_norm[count][1] = labels[count][1]\n", + "\n", + " # Convert to radians\n", + " coords_rad = []\n", + " \n", + " for theta, phi in labels:\n", + " coords_rad.append([theta, phi])\n", + "\n", + " theta, phi = zip(*coords_rad)\n", + " \n", + " return dataset, labels, labels_norm, theta, phi, coords_rad\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "08718db5", + "metadata": {}, + "outputs": [], + "source": [ + "evaluation_array_flatten = flattened = evaluation_array.flatten()\n", + "print(evaluation_array_flatten.shape)\n", + "test_dataset, test_labels_raw, test_labels_norm, test_theta, test_phi, test_coords_rad = prepare_dataset(evaluation_array_flatten)\n", + "print(test_labels_norm.shape)\n", + "test_labels = test_labels_norm" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9", + "metadata": {}, + "outputs": [], + "source": [ + "\n", + "plt.hist(test_theta,alpha=0.5)\n", + "plt.xlabel(\"Theta\")\n", + "plt.ylabel(\"Counts\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "10", + "metadata": {}, + "outputs": [], + "source": [ + "\n", + "plt.hist(test_phi,alpha=0.5)\n", + "plt.xlabel(\"Phi\")\n", + "plt.ylabel(\"Counts\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "24", + "metadata": {}, + "outputs": [], + "source": [ + "\n", + "plt.hist(test_labels_norm[:,0],alpha=0.5)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "29", + "metadata": {}, + "outputs": [], + "source": [ + "\n", + "plt.hist(test_labels_norm[:,1],alpha=0.5)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "38", + "metadata": {}, + "outputs": [], + "source": [ + "from tensorflow.keras.models import load_model\n", + "\n", + "# Load the model back from the saved directory\n", + "model = load_model(data_dir+\"/\"+model_name+\".keras\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "40", + "metadata": {}, + "outputs": [], + "source": [ + "# Process testing dataset\n", + "pred_data = model.predict(test_dataset)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e72ef346-610f-4e04-9fc4-bc2a86f6cb8a", + "metadata": {}, + "outputs": [], + "source": [ + "pred_data.shape" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "41", + "metadata": {}, + "outputs": [], + "source": [ + "max_lon = 180.0\n", + "min_lon = 0.0\n", + " \n", + "max_lat = 360.0\n", + "min_lat = 0.0\n", + "\n", + "pred_data_original = np.empty((len(pred_data),2))\n", + "\n", + "if normalize==1:\n", + " pred_data_original[:, 0] = pred_data[:, 0]*(max_lon-min_lon)+min_lon\n", + " pred_data_original[:, 1] = pred_data[:, 1]*(max_lat-min_lat)+min_lat\n", + "else:\n", + " pred_data_original[:, 0] = pred_data[:, 0]\n", + " pred_data_original[:, 1] = pred_data[:, 1]\n", + "\n", + "test_labels_original = np.empty((len(test_labels),2))\n", + "\n", + "if normalize==1:\n", + " test_labels_original[:, 0] = test_labels[:, 0]*(max_lon-min_lon)+min_lon\n", + " test_labels_original[:, 1] = test_labels[:, 1]*(max_lat-min_lat)+min_lat\n", + "else:\n", + " test_labels_original[:, 0] = test_labels[:, 0]\n", + " test_labels_original[:, 1] = test_labels[:, 1]\n", + "\n", + "\n", + "absolute_diffs_theta = np.abs(pred_data_original[:, 0] - test_labels_original[:, 0])\n", + "absolute_diffs_phi = np.abs(pred_data_original[:, 1] - test_labels_original[:, 1])\n", + "\n", + "# Calculate the Mean Absolute Error (MAE) for each element separately\n", + "mae_theta = np.mean(absolute_diffs_theta)\n", + "mae_phi = np.mean(absolute_diffs_phi)\n", + "\n", + "print(\"Mean Absolute Error for Theta:\", mae_theta)\n", + "print(\"Mean Absolute Error for Phi:\", mae_phi)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "44", + "metadata": {}, + "outputs": [], + "source": [ + "fig = plt.figure(figsize=(8, 6))\n", + "plt.hist(test_labels_original[:, 0],bins=50,alpha=0.6,color=\"b\",label=\"test coords\")\n", + "plt.hist(pred_data_original[:,0],bins=50,alpha=0.6,color=\"r\",label=\"reco coords\")\n", + "plt.xlabel(\"Theta\")\n", + "plt.legend()\n", + "plt.ylabel(\"Counts\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "46", + "metadata": {}, + "outputs": [], + "source": [ + "fig = plt.figure(figsize=(8, 6))\n", + "plt.hist(test_labels_original[:, 1],bins=50,alpha=0.6,color=\"b\",label=\"reco coords\")\n", + "plt.hist(pred_data_original[:,1],bins=50,alpha=0.6,color=\"r\",label=\"test coords\")\n", + "plt.xlabel(\"Phi\")\n", + "plt.legend()\n", + "plt.ylabel(\"Counts\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "54", + "metadata": {}, + "outputs": [], + "source": [ + "fig = plt.figure(figsize=(8, 6))\n", + "plt.hist(test_labels[:,0],bins=50,alpha=0.6,color=\"b\",label=\"test coords\")\n", + "plt.hist(pred_data[:, 0],bins=50,alpha=0.6,color=\"r\",label=\"reco coords\")\n", + "plt.xlabel(\"Theta sin\")\n", + "plt.legend()\n", + "plt.ylabel(\"Counts\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "55", + "metadata": {}, + "outputs": [], + "source": [ + "fig = plt.figure(figsize=(8, 6))\n", + "plt.hist(test_labels[:,1],bins=50,alpha=0.6,color=\"b\",label=\"test coords\")\n", + "plt.hist(pred_data[:, 1],bins=50,alpha=0.6,color=\"r\",label=\"reco coords\")\n", + "plt.xlabel(\"Theta cos\")\n", + "plt.legend()\n", + "plt.ylabel(\"Counts\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3e43e38c", + "metadata": {}, + "outputs": [], + "source": [ + "pred_data_original_reshaped = pred_data_original.reshape(evaluation_array.shape[0], evaluation_array.shape[1], 2)\n", + "test_labels_original_reshaped = test_labels_original.reshape(evaluation_array.shape[0], evaluation_array.shape[1], 2)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "58", + "metadata": {}, + "outputs": [], + "source": [ + "# Calculate the mean distances between corresponding coordinates considering that we have multiple GRBs in the same position.\n", + "\n", + "distances = []\n", + "theta_distances = []\n", + "phi_distances = []\n", + "\n", + "for i in range(0,len(pred_data_original_reshaped)):\n", + " \n", + " distances_partial = []\n", + " theta_distances_partial = []\n", + " phi_distances_partial = []\n", + " \n", + " for j in range(0,len(pred_data_original_reshaped[i])):\n", + " \n", + " d = angular_distance(pred_data_original_reshaped[i][j][0],pred_data_original_reshaped[i][j][1],test_labels_original_reshaped[i][j][0],test_labels_original_reshaped[i][j][1])\n", + " theta_dist = np.abs(pred_data_original_reshaped[i][j][0]-test_labels_original_reshaped[i][j][0])\n", + " phi_dist = diff_phi(pred_data_original_reshaped[i][j][1],test_labels_original_reshaped[i][j][1])\n", + " distances_partial.append(d)\n", + " theta_distances_partial.append(theta_dist)\n", + " phi_distances_partial.append(phi_dist)\n", + " \n", + " distances_avg = np.mean(distances_partial)\n", + " theta_distances_avg = np.mean(theta_distances_partial)\n", + " phi_distances_avg = np.mean(phi_distances_partial)\n", + " \n", + " distances.append(distances_avg)\n", + " theta_distances.append(theta_distances_avg)\n", + " phi_distances.append(phi_distances_avg)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "fb0c82c2-5e42-4103-adf5-a0cb24175f68", + "metadata": {}, + "outputs": [], + "source": [ + "import pickle\n", + "if False:\n", + " # Store the data for further plotting\n", + " with open(data_dir+\"/\"+evaluation_file+\"_distances_model_nobkg_plot.pkl\", \"wb\") as f:\n", + " pickle.dump(distances, f)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c2afbbdc-d1f0-44ca-8c3f-1d5f88e74149", + "metadata": {}, + "outputs": [], + "source": [ + "import numpy as np\n", + "import healpy as hp\n", + "import matplotlib.pyplot as plt\n", + "from matplotlib.colors import PowerNorm\n", + "\n", + "nside = 16\n", + "npix = hp.nside2npix(nside)\n", + "m = np.array(distances)\n", + "new_m = np.sqrt(m)\n", + "hp.projview(\n", + " m,\n", + " \n", + " coord=[\"G\"],\n", + " graticule=True,\n", + " graticule_labels=True,\n", + " unit=\"cbar label\",\n", + " xlabel=\"longitude\",\n", + " ylabel=\"latitude\",\n", + " cb_orientation=\"vertical\",\n", + " latitude_grid_spacing=30,\n", + " projection_type=\"aitoff\",\n", + " title=\"Aitoff projection\",\n", + " cmap=\"turbo\",\n", + " nest=True\n", + ")\n", + "\n", + "plt.show()\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c6be6847-a33e-46b1-8225-72a455bb9a52", + "metadata": {}, + "outputs": [], + "source": [ + "print(np.mean(distances))\n", + "print(np.mean(theta_distances))\n", + "print(np.mean(phi_distances))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1c9246c1-787b-4dd0-ad9f-6d9b70d59647", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "deeplearning", + "language": "python", + "name": "deeplearning" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.12" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/cosipy/notebooks/dl-loc/train_and_test/train_cartesian_model.ipynb b/cosipy/notebooks/dl-loc/train_and_test/train_cartesian_model.ipynb new file mode 100644 index 0000000..07d594d --- /dev/null +++ b/cosipy/notebooks/dl-loc/train_and_test/train_cartesian_model.ipynb @@ -0,0 +1,957 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "0", + "metadata": {}, + "outputs": [], + "source": [ + "# import libraries\n", + "import os\n", + "import pandas as pd\n", + "import numpy as np\n", + "import seaborn as sns\n", + "sns.set(color_codes=True)\n", + "import matplotlib.pyplot as plt\n", + "%matplotlib inline\n", + "import pickle\n", + "from numpy.random import seed\n", + "from tensorflow.random import set_seed\n", + "from tensorflow import keras\n", + "import matplotlib.pyplot as plt\n", + "import numpy as np\n", + "import tensorflow as tf \n", + "import math\n", + "\n", + "# set random seed\n", + "seed(10)\n", + "set_seed(10)\n", + "\n", + "training_dataset_name = \"run53_mix_mega_shared\"\n", + "testing_dataset_name = \"run57_mix_mega_shared\"\n", + "\n", + "model_name = training_dataset_name+\"_cart.keras\"\n", + "\n", + "val_number = 6000\n", + "\n", + "# If not GPU is available, set the following to -1\n", + "os.environ[\"CUDA_VISIBLE_DEVICES\"]=\"0\"\n", + "number_of_detectors = 6\n", + "data_dir = \"/data/test_newrepo/\"\n", + "normalize= 1\n", + "plot=1\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cacd48b2-794d-4114-ab63-aa3d2223c785", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1", + "metadata": {}, + "outputs": [], + "source": [ + "print(\"Num GPUs Available: \", len(tf.config.list_physical_devices('GPU')))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2", + "metadata": {}, + "outputs": [], + "source": [ + "\n", + "# load training dataset\n", + "file_path = data_dir+'/'+training_dataset_name+'_dataset.pkl'\n", + "\n", + "with open(file_path, 'rb') as file:\n", + " training_dataset_array = pickle.load(file)\n", + "\n", + "# load testing dataset\n", + "file_path = data_dir+'/'+testing_dataset_name+'_dataset.pkl'\n", + "\n", + "with open(file_path, 'rb') as file:\n", + " testing_dataset_array = pickle.load(file)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3", + "metadata": {}, + "outputs": [], + "source": [ + "\n", + "def diff_phi(a1, a2):\n", + " # Calcola la differenza diretta\n", + " diff = abs(a1 - a2)\n", + " \n", + " # Trova il percorso più breve tenendo conto del ciclo degli angoli\n", + " if diff > 180:\n", + " diff = 360 - diff\n", + " \n", + " return diff\n", + " \n", + "\n", + "def l2_normalize(data):\n", + " \"\"\"\n", + " Normalize a NumPy array using the L2 (Euclidean) norm.\n", + "\n", + " Args:\n", + " data (numpy.ndarray): The array to normalize. Can be a 1D vector\n", + " or a 2D matrix (where each row is a vector to normalize).\n", + "\n", + " Returns:\n", + " numpy.ndarray: The L2-normalized array.\n", + " \"\"\"\n", + "\n", + " data = np.array(data)\n", + " if data.ndim == 1:\n", + " # Case: 1D vector\n", + " norm = np.sqrt(np.sum(data**2))\n", + " if norm == 0:\n", + " return data # Avoid division by zero if the vector is null\n", + " return data / norm\n", + " elif data.ndim == 2:\n", + " # Case: 2D matrix (normalize each row)\n", + " norms = np.sqrt(np.sum(data**2, axis=1, keepdims=True))\n", + " # Handle the case of null norms (zero rows)\n", + " norms[norms == 0] = 1\n", + " return data / norms\n", + " else:\n", + " raise ValueError(\"Input must be a 1D or 2D array.\")\n", + "\n", + "\n", + "def cartesian_to_spherical(x, y, z):\n", + " # Calculate theta\n", + " theta = np.degrees(np.arccos(z))\n", + " \n", + " # Calculate phi\n", + " if x == 0 and y == 0:\n", + " phi = 0\n", + " else:\n", + " phi = np.degrees(np.arctan2(y, x))\n", + " if phi < 0:\n", + " phi += 360\n", + " \n", + " return theta, phi\n", + "def angular_distance(theta1, phi1, theta2, phi2):\n", + " \"\"\"\n", + " Compute the angular (great-circle) distance in degrees between two points\n", + " specified by (theta, phi) coordinates in degrees.\n", + "\n", + " Parameters:\n", + " theta1, phi1: floats, coordinates of the first point (degrees).\n", + " theta2, phi2: floats, coordinates of the second point (degrees).\n", + "\n", + " Returns:\n", + " angular_dist_deg: float, angular distance between the two points in degrees.\n", + " \"\"\"\n", + " # Adjust theta values by shifting -90\n", + " theta1 = 90 - theta1\n", + " theta2 = 90 - theta2\n", + "\n", + " # Convert angles to radians\n", + " theta1_rad = math.radians(theta1)\n", + " phi1_rad = math.radians(phi1)\n", + " theta2_rad = math.radians(theta2)\n", + " phi2_rad = math.radians(phi2)\n", + " \n", + " # Compute the difference in longitude\n", + " delta_phi = abs(phi1_rad - phi2_rad)\n", + "\n", + " # Apply spherical law of cosines\n", + " value = (math.sin(theta1_rad) * math.sin(theta2_rad) +\n", + " math.cos(theta1_rad) * math.cos(theta2_rad) * math.cos(delta_phi))\n", + " \n", + " # Clamp value to [-1, 1] to avoid floating point errors in acos\n", + " value_clamped = max(-1.0, min(1.0, value)) \n", + " \n", + " angular_dist = math.acos(value_clamped)\n", + " \n", + " # Convert angular distance from radians to degrees\n", + " angular_dist_deg = math.degrees(angular_dist)\n", + " \n", + " return angular_dist_deg\n", + "\n", + "def process_coordinates(coords_rad):\n", + "\n", + " # Separate the longitudes and latitudes in radians\n", + " theta, phi = zip(*coords_rad)\n", + "\n", + " # Convert to Cartesian coordinates\n", + " x = np.sin(np.radians(theta)) * np.cos(np.radians(phi))\n", + " y = np.sin(np.radians(theta)) * np.sin(np.radians(phi))\n", + " z = np.cos(np.radians(theta))\n", + " \n", + " return x,y,z\n", + "\n", + "def process_coordinates_single(coords_rad):\n", + "\n", + " # Separate the longitudes and latitudes in radians\n", + " theta, phi = coords_rad\n", + "\n", + " # Convert to Cartesian coordinates\n", + " x = np.sin(np.radians(theta)) * np.cos(np.radians(phi))\n", + " y = np.sin(np.radians(theta)) * np.sin(np.radians(phi))\n", + " z = np.cos(np.radians(theta))\n", + " \n", + " return x,y,z\n", + "\n", + "\n", + " \n", + "def prepare_dataset(data_array):\n", + "\n", + " dataset = np.empty((len(data_array),number_of_detectors))\n", + " labels = np.empty((len(data_array),2))\n", + " \n", + " count = -1\n", + " for element in data_array:\n", + " \n", + " count+=1\n", + " \n", + " dataset[count] = l2_normalize(element['counts'])\n", + " labels[count][0] = float(element['coord'][0])\n", + " labels[count][1] = float(element['coord'][1])\n", + " \n", + " labels = labels[:count+1]\n", + " dataset = dataset[:count+1]\n", + "\n", + " # Convert to radians\n", + " coords_rad = []\n", + " \n", + " for theta, phi in labels:\n", + " #if lon > 90 and lon < 270:\n", + " \n", + " coords_rad.append([theta, phi])\n", + " #else:\n", + " # continue\n", + "\n", + " x,y,z = process_coordinates(coords_rad)\n", + " \n", + " theta, phi = zip(*coords_rad)\n", + " \n", + " cartesian_labels = []\n", + "\n", + " for i in range(0,len(dataset)):\n", + " cartesian_labels.append([x[i],y[i],z[i]])\n", + " \n", + " \n", + " labels_norm = np.empty((len(dataset),3))\n", + " count = -1\n", + " for element in cartesian_labels:\n", + " count+=1\n", + " labels_norm[count][0] = element[0]\n", + " labels_norm[count][1] = element[1]\n", + " labels_norm[count][2] = element[2]\n", + " \n", + " \n", + " \n", + " return dataset, labels, labels_norm, theta, phi, coords_rad\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4", + "metadata": {}, + "outputs": [], + "source": [ + "# prepare training and validation dataset\n", + "dataset, labels, labels_norm, theta, phi, coords_rad = prepare_dataset(training_dataset_array)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6", + "metadata": {}, + "outputs": [], + "source": [ + "# prepare testing and validation dataset\n", + "test_dataset, test_labels_raw, test_labels_norm, test_theta, test_phi, test_coords_rad = prepare_dataset(testing_dataset_array)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7", + "metadata": {}, + "outputs": [], + "source": [ + "dataset.shape" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8", + "metadata": {}, + "outputs": [], + "source": [ + "test_dataset.shape" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "97c38689-2549-45cf-a7e1-42b03a308d2d", + "metadata": {}, + "outputs": [], + "source": [ + "#split val and test dataset\n", + "random_indices = np.random.permutation(len(dataset))\n", + "\n", + "N = dataset.shape[0]\n", + "validation_indices = random_indices[:val_number]\n", + "train_indices = np.setdiff1d(np.arange(N), validation_indices)\n", + "\n", + "validation_dataset = dataset[validation_indices]\n", + "training_dataset = dataset[train_indices]\n", + "\n", + "validation_labels = labels_norm[validation_indices]\n", + "training_labels = labels_norm[train_indices]\n", + "\n", + "test_labels = test_labels_norm\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9", + "metadata": {}, + "outputs": [], + "source": [ + "plt.hist(theta,alpha=0.5)\n", + "plt.hist(test_theta,alpha=0.5)\n", + "plt.xlabel(\"Theta\")\n", + "plt.ylabel(\"Counts\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "10", + "metadata": {}, + "outputs": [], + "source": [ + "plt.hist(phi,alpha=0.5)\n", + "plt.hist(test_phi,alpha=0.5)\n", + "plt.xlabel(\"Phi\")\n", + "plt.ylabel(\"Counts\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "24", + "metadata": {}, + "outputs": [], + "source": [ + "plt.hist(training_labels[:,0],alpha=0.5)\n", + "plt.hist(test_labels[:,0],alpha=0.5)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "25", + "metadata": {}, + "outputs": [], + "source": [ + "plt.hist(training_labels[:,1],alpha=0.5)\n", + "plt.hist(test_labels[:,1],alpha=0.5)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "27", + "metadata": {}, + "outputs": [], + "source": [ + "plt.hist(training_labels[:,2],alpha=0.5)\n", + "plt.hist(test_labels[:,2],alpha=0.5)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "080c9101-1a8b-4dcc-b564-42c00bf4045f", + "metadata": {}, + "outputs": [], + "source": [ + "def custom_loss(y_true, y_pred):\n", + " \n", + " lambda_reg = 0.5\n", + " \n", + " #mse_loss = CosineSimilarity()(y_true, y_pred) # Calcolo MSE tra coordinate previste e reali\n", + " mse_loss = tf.keras.losses.MeanAbsoluteError()(y_true,y_pred)\n", + " \n", + " # Calcolo dell'errore rispetto all'equazione della sfera\n", + " x_pred, y_pred, z_pred = tf.unstack(y_pred, axis=1)\n", + " sphere_constraint_error = tf.square(x_pred**2 + y_pred**2 + z_pred**2 - 1)\n", + " \n", + " # Combinazione delle due loss con un peso per la regolarizzazione\n", + " loss = mse_loss**2 + lambda_reg * sphere_constraint_error\n", + " \n", + " return loss" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "34", + "metadata": {}, + "outputs": [], + "source": [ + "# Define the model architecture\n", + "model = keras.Sequential([\n", + " keras.layers.Dense(128*2, input_shape=(number_of_detectors,)), \n", + " keras.layers.LeakyReLU(alpha=0.1) ,\n", + " keras.layers.Dropout(0.02),\n", + " keras.layers.Dense(64*2), \n", + " keras.layers.LeakyReLU(alpha=0.1),\n", + " keras.layers.Dropout(0.02),\n", + " keras.layers.Dense(32*2),\n", + " keras.layers.LeakyReLU(alpha=0.1),\n", + " keras.layers.Dropout(0.02),\n", + " keras.layers.Dense(3,activation=\"tanh\")\n", + "])\n", + "\n", + "# Compile the model\n", + "model.compile(optimizer=keras.optimizers.Adam(learning_rate=0.001), loss=custom_loss)\n", + "model.summary()\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "35", + "metadata": {}, + "outputs": [], + "source": [ + "# Define your learning rate schedule function\n", + "def variable_learning_rate(epoch, lr):\n", + " if epoch < 10:\n", + " return 0.01 \n", + " elif epoch >= 4 and epoch < 50:\n", + " return 0.001\n", + " else:\n", + " return 0.0001\n", + "\n", + "\n", + "# Define the LearningRateScheduler callback\n", + "lr_scheduler = tf.keras.callbacks.LearningRateScheduler(variable_learning_rate, verbose=1)\n", + "\n", + "\n", + "history = model.fit(\n", + " x=training_dataset,\n", + " y=training_labels,\n", + " epochs=2000,\n", + " batch_size=256,\n", + " validation_data=(validation_dataset,validation_labels),\n", + " validation_split=0.2,\n", + " shuffle=True,\n", + " callbacks=[\n", + " keras.callbacks.EarlyStopping(monitor=\"val_loss\", patience=5, mode=\"min\")\n", + " ],\n", + ")\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "34acf4f8-b030-4cb9-bf87-902a9e5fabad", + "metadata": {}, + "outputs": [], + "source": [ + "#save model\n", + "save_model =False\n", + "if(save_model):\n", + " model.save(data_dir+\"/\"+model_name)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "37", + "metadata": {}, + "outputs": [], + "source": [ + "fig, ax1 = plt.subplots(1, 1,figsize=(10,5))\n", + "fig.suptitle('Training')\n", + " \n", + "ax1.plot(history.history[\"loss\"], label=\"Training Loss\")\n", + "ax1.plot(history.history[\"val_loss\"], label=\"Validation Loss\")\n", + " \n", + "ax1.legend()\n", + "plt.show() " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "40", + "metadata": {}, + "outputs": [], + "source": [ + "\n", + "# Evaluate testing dataset\n", + "pred_data = model.predict(test_dataset)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "41", + "metadata": {}, + "outputs": [], + "source": [ + "pred_data_renorm = np.empty((len(pred_data),3))\n", + "test_data_renorm = np.empty((len(test_labels),3))\n", + "\n", + "for j in range(0,len(pred_data)):\n", + " pred_data_renorm[j][0] = pred_data[j][0]\n", + " pred_data_renorm[j][1] = pred_data[j][1]\n", + " pred_data_renorm[j][2] = pred_data[j][2]\n", + "\n", + " \n", + "min_val = 0\n", + "max_val = 1\n", + "\n", + "for j in range(0,len(test_labels)):\n", + " test_data_renorm[j][0] = test_labels[j][0]\n", + " test_data_renorm[j][1] = test_labels[j][1]\n", + " test_data_renorm[j][2] = test_labels[j][2]\n", + " \n", + "pred_data_original = np.empty((len(pred_data),2))\n", + "test_labels_original = np.empty((len(test_labels),2))\n", + " \n", + "for i, element in enumerate(test_data_renorm):\n", + "\n", + " theta,phi = cartesian_to_spherical(element[0], element[1], element[2])\n", + " \n", + " test_labels_original[i][0]=theta\n", + " test_labels_original[i][1]=phi\n", + " \n", + "for i, element in enumerate(pred_data_renorm):\n", + "\n", + " theta,phi = cartesian_to_spherical(element[0], element[1], element[2])\n", + " \n", + " pred_data_original[i][0]=theta\n", + " pred_data_original[i][1]=phi\n", + " \n", + "absolute_diffs_theta = np.abs(pred_data_original[:, 0] - test_labels_original[:, 0])\n", + "absolute_diffs_phi = np.abs(pred_data_original[:, 1] - test_labels_original[:, 1])\n", + "\n", + "mae_theta = np.mean(absolute_diffs_theta)\n", + "mae_phi = np.mean(absolute_diffs_phi)\n", + "\n", + "print(\"Mean Absolute Error for Theta:\", mae_theta)\n", + "print(\"Mean Absolute Error for Phi:\", mae_phi)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "44", + "metadata": {}, + "outputs": [], + "source": [ + "fig = plt.figure(figsize=(8, 6))\n", + "plt.hist(test_labels_original[:, 0],bins=50,alpha=0.6,color=\"b\",label=\"reco coords\")\n", + "plt.hist(pred_data_original[:,0],bins=50,alpha=0.6,color=\"r\",label=\"test coords\")\n", + "plt.xlabel(\"Theta\")\n", + "plt.legend()\n", + "plt.ylabel(\"Counts\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "45", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "46", + "metadata": {}, + "outputs": [], + "source": [ + "fig = plt.figure(figsize=(8, 6))\n", + "plt.hist(test_labels_original[:, 1],bins=50,alpha=0.6,color=\"b\",label=\"reco coords\")\n", + "plt.hist(pred_data_original[:,1],bins=50,alpha=0.6,color=\"r\",label=\"test coords\")\n", + "plt.xlabel(\"Phi\")\n", + "plt.legend()\n", + "plt.ylabel(\"Counts\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "53", + "metadata": {}, + "outputs": [], + "source": [ + "\n", + "# Esempio di utilizzo\n", + "theta1 = 90 # Latitudine del primo punto in gradi\n", + "phi1 = 350 # Longitudine del primo punto in gradi\n", + "theta2 = 90 # Latitudine del secondo punto in gradi\n", + "phi2 = 10 # Longitudine del secondo punto in gradi\n", + "\n", + "distance = angular_distance(theta1, phi1, theta2, phi2)\n", + "print(\"Distanza angolare tra i due punti:\", distance, \"gradi\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "54", + "metadata": {}, + "outputs": [], + "source": [ + "fig = plt.figure(figsize=(8, 6))\n", + "plt.hist(test_labels[:,0],bins=50,alpha=0.6,color=\"b\",label=\"test coords\")\n", + "plt.hist(pred_data[:, 0],bins=50,alpha=0.6,color=\"r\",label=\"reco coords\")\n", + "plt.xlabel(\"Theta sin\")\n", + "plt.legend()\n", + "plt.ylabel(\"Counts\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "55", + "metadata": {}, + "outputs": [], + "source": [ + "fig = plt.figure(figsize=(8, 6))\n", + "plt.hist(test_labels[:,1],bins=50,alpha=0.6,color=\"b\",label=\"test coords\")\n", + "plt.hist(pred_data[:, 1],bins=50,alpha=0.6,color=\"r\",label=\"reco coords\")\n", + "plt.xlabel(\"Theta cos\")\n", + "plt.legend()\n", + "plt.ylabel(\"Counts\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "56", + "metadata": {}, + "outputs": [], + "source": [ + "fig = plt.figure(figsize=(8, 6))\n", + "plt.hist(test_labels[:,2],bins=50,alpha=0.6,color=\"b\",label=\"test coords\")\n", + "plt.hist(pred_data[:, 2],bins=50,alpha=0.6,color=\"r\",label=\"reco coords\")\n", + "plt.xlabel(\"Phi sin\")\n", + "plt.legend()\n", + "plt.ylabel(\"Counts\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "58", + "metadata": {}, + "outputs": [], + "source": [ + "distances = []\n", + "theta_distances = []\n", + "phi_distances = []\n", + "\n", + "for i in range(0,len(pred_data_original)):\n", + "\n", + " d = angular_distance(pred_data_original[i][0],pred_data_original[i][1],test_labels_original[i][0],test_labels_original[i][1])\n", + " theta_dist = np.abs(pred_data_original[i][0]-test_labels_original[i][0])\n", + " phi_dist = diff_phi(pred_data_original[i][1],test_labels_original[i][1])\n", + " distances.append(d)\n", + " theta_distances.append(theta_dist)\n", + " phi_distances.append(phi_dist)\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d6741cd0", + "metadata": {}, + "outputs": [], + "source": [ + "import numpy as np\n", + "import healpy as hp\n", + "import matplotlib.pyplot as plt\n", + "\n", + "nside = 32\n", + "npix = hp.nside2npix(nside)\n", + "\n", + "m = np.array(distances)\n", + "\n", + "hp.projview(\n", + " m,\n", + " coord=[\"G\"],\n", + " graticule=True,\n", + " graticule_labels=True,\n", + " unit=\"cbar label\",\n", + " xlabel=\"longitude\",\n", + " ylabel=\"latitude\",\n", + " cb_orientation=\"vertical\",\n", + " latitude_grid_spacing=30,\n", + " projection_type=\"aitoff\",\n", + " title=\"Aitoff projection\",\n", + " cmap=\"turbo\",\n", + " nest=True\n", + ")\n", + "\n", + "plt.show()\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c6be6847-a33e-46b1-8225-72a455bb9a52", + "metadata": {}, + "outputs": [], + "source": [ + "print(np.mean(distances))\n", + "print(np.mean(theta_distances))\n", + "print(np.mean(phi_distances))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "79", + "metadata": {}, + "outputs": [], + "source": [ + "# Definisci gli intervalli di 5 gradi\n", + "intervalli = np.arange(0, 185, 5)\n", + "\n", + "# Raggruppa i conteggi in base agli intervalli di 5 gradi\n", + "conteggi_raggruppati = np.zeros((len(intervalli)))\n", + "conteggi = np.zeros((len(intervalli)))\n", + "\n", + "tot_count = 0\n", + "for theta,phi,conteggio in zip(test_labels_original[:,0],test_labels_original[:,1], distances):\n", + " \n", + " if(True): #(phi>125 and phi<145) or \n", + " tot_count +=1\n", + " indice_intervallo = int(theta // 5)\n", + " conteggi_raggruppati[indice_intervallo] = conteggi_raggruppati[indice_intervallo]+conteggio\n", + " conteggi[indice_intervallo]+=1\n", + "\n", + "# Calcola la media dei conteggi in ciascun intervallo\n", + "conteggi_raggruppati = conteggi_raggruppati[:tot_count]\n", + "conteggi = conteggi[:tot_count]\n", + "\n", + "# Visualizza l'istogramma\n", + "plt.figure(figsize=(10, 6))\n", + "plt.bar(intervalli[:-1], conteggi_raggruppati[:-1]/conteggi[:-1], width=5, align='edge',alpha=1,label=\"loc. error\")\n", + "plt.xlabel('Theta (°)')\n", + "plt.ylabel('Loc. error')\n", + "plt.title('Loc. error in function of theta')\n", + "plt.grid(True)\n", + "plt.legend()\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ac939e97-652f-4773-bddb-465406cdb4b0", + "metadata": {}, + "outputs": [], + "source": [ + "step = 10\n", + "\n", + "# Definisci gli intervalli di 5 gradi\n", + "intervalli = np.arange(0, 365, step)\n", + "\n", + "# Raggruppa i conteggi in base agli intervalli di 5 gradi\n", + "conteggi_raggruppati_1 = np.zeros((len(intervalli)))\n", + "conteggi_1 = np.zeros((len(intervalli)))\n", + "\n", + "conteggi_raggruppati_2 = np.zeros((len(intervalli)))\n", + "conteggi_2 = np.zeros((len(intervalli)))\n", + "\n", + "conteggi_raggruppati_3 = np.zeros((len(intervalli)))\n", + "conteggi_3 = np.zeros((len(intervalli)))\n", + "\n", + "conteggi_raggruppati_4 = np.zeros((len(intervalli)))\n", + "conteggi_4 = np.zeros((len(intervalli)))\n", + "\n", + "tot_count_1 = 0\n", + "tot_count_2 = 0\n", + "tot_count_3 = 0\n", + "tot_count_4 = 0\n", + "\n", + "for theta,phi,conteggio in zip(test_labels_original[:,0],test_labels_original[:,1], distances):\n", + " \n", + " if(theta>20 and theta<55):\n", + " tot_count_1 +=1\n", + " indice_intervallo = int(phi // step)\n", + " conteggi_raggruppati_1[indice_intervallo] = conteggi_raggruppati_1[indice_intervallo]+conteggio\n", + " conteggi_1[indice_intervallo]+=1\n", + "\n", + " if(theta>55 and theta<150):\n", + " tot_count_2 +=1\n", + " indice_intervallo = int(phi // step)\n", + " conteggi_raggruppati_2[indice_intervallo] = conteggi_raggruppati_2[indice_intervallo]+conteggio\n", + " conteggi_2[indice_intervallo]+=1\n", + "\n", + " if(theta>150 and theta<180):\n", + " tot_count_3 +=1\n", + " indice_intervallo = int(phi // step)\n", + " conteggi_raggruppati_3[indice_intervallo] = conteggi_raggruppati_3[indice_intervallo]+conteggio\n", + " conteggi_3[indice_intervallo]+=1\n", + "\n", + "# Calcola la media dei conteggi in ciascun intervallo\n", + "conteggi_raggruppati_1 = conteggi_raggruppati_1[:tot_count_1]\n", + "conteggi_1 = conteggi_1[:tot_count_1]\n", + "\n", + "conteggi_raggruppati_2 = conteggi_raggruppati_2[:tot_count_2]\n", + "conteggi_2 = conteggi_2[:tot_count_2]\n", + "\n", + "conteggi_raggruppati_3 = conteggi_raggruppati_3[:tot_count_3]\n", + "conteggi_3 = conteggi_3[:tot_count_3]\n", + "\n", + "# Visualizza l'istogramma\n", + "plt.figure(figsize=(10, 6))\n", + "plt.bar(intervalli[:-1], conteggi_raggruppati_1[:-1]/conteggi_1[:-1], width=step, align='edge',alpha=0.5,label=\"loc. error theta=[20°,55°]\")\n", + "\n", + "plt.bar(intervalli[:-1], conteggi_raggruppati_2[:-1]/conteggi_2[:-1], width=step, align='edge',alpha=0.5,label=\"loc. error theta=[55°,150°]\")\n", + "\n", + "plt.bar(intervalli[:-1], conteggi_raggruppati_3[:-1]/conteggi_3[:-1], width=step, align='edge',alpha=0.5,label=\"loc. error theta=[150°,180°]\")\n", + "plt.xlabel('Phi (°)')\n", + "plt.ylabel('Loc. error')\n", + "plt.title('Loc. error in function of phi')\n", + "plt.grid(True)\n", + "plt.legend()\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "527add68-000a-440a-9a6d-a231b36fe9b1", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "634a712e-3df1-47c5-8777-d1da8cc40879", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1e25fd17-2d62-466b-b2bc-985468b34f34", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9e9f9c76-3626-48e2-8d53-c49b566d6117", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "46b4537e-1795-4bba-89a1-64e7c8c421ca", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0244c058-1f3d-44e4-96e6-a523f0faf8e5", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9f1e6467-7c96-4110-ab42-79a31b6c137d", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "07b53f7c-3da9-4286-8bd5-fa04867bfd0a", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "959fc986-ce2c-41df-af5d-eae11ce18fe7", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3aab396d-bebe-4ef9-9af1-bbd2ae5119af", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "10686f76-d30a-44ab-b772-c6208128d925", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "deeplearning", + "language": "python", + "name": "deeplearning" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.12" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/cosipy/notebooks/dl-loc/train_and_test/train_cosin_model.ipynb b/cosipy/notebooks/dl-loc/train_and_test/train_cosin_model.ipynb new file mode 100644 index 0000000..c7b0820 --- /dev/null +++ b/cosipy/notebooks/dl-loc/train_and_test/train_cosin_model.ipynb @@ -0,0 +1,975 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "0", + "metadata": {}, + "outputs": [], + "source": [ + "# import libraries\n", + "import os\n", + "import pandas as pd\n", + "import numpy as np\n", + "import seaborn as sns\n", + "sns.set(color_codes=True)\n", + "import matplotlib.pyplot as plt\n", + "%matplotlib inline\n", + "import pickle\n", + "from tensorflow import keras\n", + "import matplotlib.pyplot as plt\n", + "import numpy as np\n", + "import math\n", + "import tensorflow as tf\n", + "from numpy.random import seed\n", + "from tensorflow.random import set_seed\n", + "\n", + "# set random seedindexes_AC1\n", + "seed(10)\n", + "set_seed(10)\n", + "\n", + "training_dataset_name = \"run53_mix_mega_shared\"\n", + "testing_dataset_name = \"run57_mix_mega_shared\"\n", + "\n", + "val_number = 6000\n", + "\n", + "# If not GPU is available, set the following to -1\n", + "os.environ[\"CUDA_VISIBLE_DEVICES\"]=\"0\"\n", + "number_of_detectors = 6\n", + "data_path = \"/data/test_newrepo/\"\n", + "normalize= 1\n", + "plot=1" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1", + "metadata": {}, + "outputs": [], + "source": [ + "print(\"Num GPUs Available: \", len(tf.config.list_physical_devices('GPU')))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2", + "metadata": {}, + "outputs": [], + "source": [ + "\n", + "# load training dataseta\n", + "file_path = data_path+'/'+training_dataset_name+'_dataset.pkl'\n", + "\n", + "with open(file_path, 'rb') as file:\n", + " training_dataset_array = pickle.load(file)\n", + "\n", + "# load testing dataset\n", + "file_path = data_path+'/'+testing_dataset_name+'_dataset.pkl'\n", + "\n", + "with open(file_path, 'rb') as file:\n", + " testing_dataset_array = pickle.load(file)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5ad68182-be81-4e9e-8e63-cc202dad3bef", + "metadata": {}, + "outputs": [], + "source": [ + "def diff_phi(a1, a2):\n", + " diff = abs(a1 - a2)\n", + " if diff > 180:\n", + " diff = 360 - diff\n", + " return diff\n", + "\n", + "def convert_degrees_to_sin_cos(degrees):\n", + " radians = np.deg2rad(degrees) # Converti gradi in radianti\n", + " sin = np.sin(radians)\n", + " cos = np.cos(radians)\n", + " return sin, cos\n", + "\n", + "def degrees_from_sin_cos(sin_value, cos_value):\n", + " # Utilizza arctan2 per ottenere l'angolo nel giusto quadrante\n", + " angle_rad = np.arctan2(sin_value, cos_value)\n", + " # Converti radianti in gradi\n", + " angle_deg = np.degrees(angle_rad)\n", + " # Assicurati che l'angolo sia compreso tra 0 e 360 gradi\n", + " if angle_deg < 0:\n", + " angle_deg += 360\n", + " return angle_deg\n", + "\n", + "def convert_degress_to_sin_cos_math(degrees):\n", + " \n", + " rad = math.radians(degrees)\n", + "\n", + " # Calcolare il seno e il coseno dell'angolo\n", + " sin = math.sin(rad)\n", + " cos = math.cos(rad)\n", + "\n", + " print(\"Seno:\", sin)\n", + " print(\"Coseno:\", cos)\n", + "\n", + "\n", + "def angular_distance(theta1, phi1, theta2, phi2):\n", + " \"\"\"\n", + " Compute the angular (great-circle) distance in degrees between two points\n", + " specified by (theta, phi) coordinates in degrees.\n", + "\n", + " Parameters:\n", + " theta1, phi1: floats, coordinates of the first point (degrees).\n", + " theta2, phi2: floats, coordinates of the second point (degrees).\n", + "\n", + " Returns:\n", + " angular_dist_deg: float, angular distance between the two points in degrees.\n", + " \"\"\"\n", + " # Adjust theta values by shifting -90\n", + " theta1 = 90 - theta1\n", + " theta2 = 90 - theta2\n", + "\n", + " # Convert angles to radians\n", + " theta1_rad = math.radians(theta1)\n", + " phi1_rad = math.radians(phi1)\n", + " theta2_rad = math.radians(theta2)\n", + " phi2_rad = math.radians(phi2)\n", + " \n", + " # Compute the difference in longitude\n", + " delta_phi = abs(phi1_rad - phi2_rad)\n", + "\n", + " # Apply spherical law of cosines\n", + " value = (math.sin(theta1_rad) * math.sin(theta2_rad) +\n", + " math.cos(theta1_rad) * math.cos(theta2_rad) * math.cos(delta_phi))\n", + " \n", + " # Clamp value to [-1, 1] to avoid floating point errors in acos\n", + " value_clamped = max(-1.0, min(1.0, value)) \n", + " \n", + " angular_dist = math.acos(value_clamped)\n", + " \n", + " # Convert angular distance from radians to degrees\n", + " angular_dist_deg = math.degrees(angular_dist)\n", + " \n", + " return angular_dist_deg\n", + " \n", + "\n", + "def l2_normalize(data):\n", + " \"\"\"\n", + " Normalize a NumPy array using the L2 (Euclidean) norm.\n", + "\n", + " Args:\n", + " data (numpy.ndarray): The array to normalize. Can be a 1D vector\n", + " or a 2D matrix (where each row is a vector to normalize).\n", + "\n", + " Returns:\n", + " numpy.ndarray: The L2-normalized array.\n", + " \"\"\"\n", + "\n", + " data = np.array(data)\n", + " if data.ndim == 1:\n", + " # Case: 1D vector\n", + " norm = np.sqrt(np.sum(data**2))\n", + " if norm == 0:\n", + " return data # Avoid division by zero if the vector is null\n", + " return data / norm\n", + " elif data.ndim == 2:\n", + " # Case: 2D matrix (normalize each row)\n", + " norms = np.sqrt(np.sum(data**2, axis=1, keepdims=True))\n", + " # Handle the case of null norms (zero rows)\n", + " norms[norms == 0] = 1\n", + " return data / norms\n", + " else:\n", + " raise ValueError(\"Input must be a 1D or 2D array.\")\n", + "\n", + " \n", + "def prepare_dataset(data_array):\n", + "\n", + " dataset = np.empty((len(data_array),number_of_detectors))\n", + " labels = np.empty((len(data_array),2))\n", + " \n", + " count = -1\n", + " for element in data_array:\n", + " \n", + " \n", + " count+=1\n", + " \n", + " dataset[count] = l2_normalize(element['counts'])\n", + " labels[count][0] = float(element['coord'][0])\n", + " labels[count][1] = float(element['coord'][1])\n", + " \n", + " labels = labels[:count+1]\n", + " dataset = dataset[:count+1]\n", + "\n", + " # Convert to radians\n", + " coords_rad = []\n", + " \n", + " for theta, phi in labels:\n", + " coords_rad.append([theta, phi])\n", + " \n", + " theta, phi = zip(*coords_rad)\n", + " \n", + " labels_cosin = np.empty((len(labels),4))\n", + " count = -1\n", + " for element in labels:\n", + " \n", + " count=count+1\n", + " \n", + " theta_sin,theta_cos = convert_degrees_to_sin_cos(labels[count][0])\n", + " labels_cosin[count][0] = theta_sin\n", + " labels_cosin[count][1] = theta_cos\n", + " \n", + " phi_sin,phi_cos = convert_degrees_to_sin_cos(labels[count][1])\n", + " labels_cosin[count][2] = phi_sin\n", + " labels_cosin[count][3] = phi_cos\n", + " \n", + " labels_norm = labels_cosin\n", + " \n", + " if normalize == 1:\n", + " \n", + " labels_norm = np.empty((len(labels_cosin),4))\n", + " \n", + " for j in range(0,len(labels_cosin)):\n", + " labels_norm[j][0] = 2 * labels_cosin[j][0] - 1\n", + " labels_norm[j][1] = labels_cosin[j][1]\n", + " labels_norm[j][2] = labels_cosin[j][2]\n", + " labels_norm[j][3] = labels_cosin[j][3]\n", + " \n", + " else:\n", + " labels_norm = labels_cosin\n", + "\n", + " \n", + " \n", + " return dataset, labels, labels_norm, theta, phi, coords_rad\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4", + "metadata": {}, + "outputs": [], + "source": [ + "# prepare training and validation dataset\n", + "dataset, labels, labels_norm, theta, phi, coords_rad = prepare_dataset(training_dataset_array)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "efa41147-0aab-4ce5-bec7-b2b1b0382f95", + "metadata": {}, + "outputs": [], + "source": [ + "# prepare testing and validation dataset\n", + "test_dataset, test_labels_raw, test_labels_norm, test_theta, test_phi, test_coords_rad = prepare_dataset(testing_dataset_array)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "17f07fbd-5fb8-417c-b970-881b4a7b60ab", + "metadata": {}, + "outputs": [], + "source": [ + "dataset.shape" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "01aec7b9-516a-43b5-8244-dfa2e6cb0c8e", + "metadata": {}, + "outputs": [], + "source": [ + "test_dataset.shape" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7", + "metadata": {}, + "outputs": [], + "source": [ + "#split val and test dataset\n", + "random_indices = np.random.permutation(len(dataset))\n", + "\n", + "N = dataset.shape[0]\n", + "validation_indices = random_indices[:val_number]\n", + "train_indices = np.setdiff1d(np.arange(N), validation_indices)\n", + "\n", + "validation_dataset = dataset[validation_indices]\n", + "training_dataset = dataset[train_indices]\n", + "\n", + "validation_labels = labels_norm[validation_indices]\n", + "training_labels = labels_norm[train_indices]\n", + "\n", + "test_labels = test_labels_norm\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9", + "metadata": {}, + "outputs": [], + "source": [ + "plt.hist(theta,alpha=0.5)\n", + "plt.hist(test_theta,alpha=0.5)\n", + "plt.xlabel(\"Theta\")\n", + "plt.ylabel(\"Counts\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "10", + "metadata": {}, + "outputs": [], + "source": [ + "plt.hist(phi,alpha=0.5)\n", + "plt.hist(test_phi,alpha=0.5)\n", + "plt.xlabel(\"Phi\")\n", + "plt.ylabel(\"Counts\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "24", + "metadata": {}, + "outputs": [], + "source": [ + "plt.hist(training_labels[:,0],alpha=0.5)\n", + "plt.hist(test_labels_norm[:,0],alpha=0.5)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "29", + "metadata": {}, + "outputs": [], + "source": [ + "plt.hist(training_labels[:,1],alpha=0.5)\n", + "plt.hist(test_labels_norm[:,1],alpha=0.5)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d7e105f5-139a-4a27-b648-eb7d8ef0179b", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "30", + "metadata": {}, + "outputs": [], + "source": [ + "plt.hist(training_labels[:,2],alpha=0.5)\n", + "plt.hist(test_labels_norm[:,2],alpha=0.5)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "31", + "metadata": {}, + "outputs": [], + "source": [ + "plt.hist(training_labels[:,3],alpha=0.5)\n", + "plt.hist(test_labels_norm[:,3],alpha=0.5)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "33", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "31781709-2496-46d5-bdf5-0459d0c674b8", + "metadata": {}, + "outputs": [], + "source": [ + "from tensorflow.keras import backend as K\n", + "\n", + "def spherical_loss_2angles(y_true, y_pred):\n", + " # Compute the Mean Squared Error (MSE) between targets and predictions\n", + " mse_loss = K.mean(K.square(y_true - y_pred), axis=-1)\n", + " \n", + " # Extract the pairs (sinθ, cosθ) and (sinφ, cosφ) from the predictions\n", + " sin_theta, cos_theta = y_pred[:, 0], y_pred[:, 1]\n", + " sin_phi, cos_phi = y_pred[:, 2], y_pred[:, 3]\n", + " \n", + " # Compute the L2 norm of each pair\n", + " # (should ideally be equal to 1 if the network outputs valid sine/cosine pairs)\n", + " norm_theta = K.sqrt(sin_theta**2 + cos_theta**2)\n", + " norm_phi = K.sqrt(sin_phi**2 + cos_phi**2)\n", + " \n", + " # Compute the penalty as the squared deviation of each norm from 1\n", + " # This encourages the network to output normalized sine/cosine pairs\n", + " penalty_theta = K.square(norm_theta - 1.0)\n", + " penalty_phi = K.square(norm_phi - 1.0)\n", + " \n", + " penalty = penalty_theta + penalty_phi\n", + " \n", + " # Weight of the regularization term (can be tuned as a hyperparameter)\n", + " alpha = 0.01\n", + " \n", + " # Final loss = MSE + weighted normalization penalty\n", + " return mse_loss + alpha * penalty" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "34", + "metadata": {}, + "outputs": [], + "source": [ + "# Define the model architecture\n", + "model = keras.Sequential([\n", + " keras.layers.Dense(128*2, input_shape=(number_of_detectors,)),\n", + " keras.layers.LeakyReLU(alpha=0.1) ,\n", + " keras.layers.Dropout(0.02),\n", + " keras.layers.Dense(64*2),\n", + " keras.layers.LeakyReLU(alpha=0.1),\n", + " keras.layers.Dropout(0.02),\n", + " keras.layers.Dense(32*2), \n", + " keras.layers.LeakyReLU(alpha=0.1),\n", + " keras.layers.Dropout(0.02),\n", + " keras.layers.Dense(4,'tanh'), \n", + "\n", + "])\n", + "\n", + "# Compile the model\n", + "model.compile(optimizer=keras.optimizers.Adam(learning_rate=0.001), loss=spherical_loss_2angles)\n", + "model.summary()\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "35", + "metadata": {}, + "outputs": [], + "source": [ + "history = model.fit(\n", + " x=training_dataset,\n", + " y=training_labels,\n", + " epochs=2000,\n", + " batch_size=256,\n", + " validation_data=(validation_dataset,validation_labels),\n", + " shuffle=True,\n", + " callbacks=[\n", + " keras.callbacks.EarlyStopping(monitor=\"val_loss\", patience=5, mode=\"min\")\n", + " ],\n", + ")\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "34acf4f8-b030-4cb9-bf87-902a9e5fabad", + "metadata": {}, + "outputs": [], + "source": [ + "#save model\n", + "save_data = False\n", + "model_name = training_dataset_name+\"_tanh.keras\"\n", + "if(save_data):\n", + " model.save(data_path+\"/\"+model_name)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "37", + "metadata": {}, + "outputs": [], + "source": [ + "fig, ax1 = plt.subplots(1, 1,figsize=(10,5))\n", + "fig.suptitle('Training')\n", + " \n", + "ax1.plot(history.history[\"loss\"], label=\"Training Loss\")\n", + "ax1.plot(history.history[\"val_loss\"], label=\"Validation Loss\")\n", + " \n", + "ax1.legend()\n", + "plt.show() \n", + "\n", + "if(save_data):\n", + " \n", + " training_results = {'loss': history.history[\"loss\"], 'val_loss': history.history[\"val_loss\"]}\n", + "\n", + " with open(data_path+\"/\"+model_name+\"_training_results.pkl\", \"wb\") as f:\n", + " pickle.dump(training_results, f)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "40", + "metadata": {}, + "outputs": [], + "source": [ + "\n", + "# Evaluate the testing dataset\n", + "pred_data = model.predict(test_dataset) " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "41", + "metadata": {}, + "outputs": [], + "source": [ + "import math\n", + "\n", + "pred_data_renorm = np.empty((len(pred_data),4))\n", + "test_data_renorm = np.empty((len(test_labels),4))\n", + "\n", + " \n", + "for j in range(0,len(pred_data)):\n", + " if normalize == 1:\n", + " pred_data_renorm[j][0] = (pred_data[j][0] + 1) / 2\n", + " else:\n", + " pred_data_renorm[j][0] = pred_data[j][0]\n", + " pred_data_renorm[j][1] = pred_data[j][1]\n", + " pred_data_renorm[j][2] = pred_data[j][2]\n", + " pred_data_renorm[j][3] = pred_data[j][3]\n", + " \n", + "min_val = 0\n", + "max_val = 1\n", + "\n", + "for j in range(0,len(test_labels)):\n", + " if normalize == 1: \n", + " test_data_renorm[j][0] = (test_labels[j][0] + 1) / 2\n", + " else:\n", + " test_data_renorm[j][0] = test_labels[j][0]\n", + " test_data_renorm[j][1] = test_labels[j][1]\n", + " test_data_renorm[j][2] = test_labels[j][2]\n", + " test_data_renorm[j][3] = test_labels[j][3]\n", + " \n", + "pred_data_original = np.empty((len(pred_data),2))\n", + "test_labels_original = np.empty((len(test_labels),2))\n", + " \n", + "for i, element in enumerate(test_data_renorm):\n", + " test_labels_original[i][0]=degrees_from_sin_cos(element[0],element[1])\n", + " test_labels_original[i][1]=degrees_from_sin_cos(element[2],element[3])\n", + " \n", + "\n", + "for i, element in enumerate(pred_data_renorm):\n", + " \n", + " pred_data_original[i][0]=degrees_from_sin_cos(element[0],element[1])\n", + " pred_data_original[i][1]=degrees_from_sin_cos(element[2],element[3])\n", + " \n", + "absolute_diffs_theta = np.abs(pred_data_original[:, 0] - test_labels_original[:, 0])\n", + "absolute_diffs_phi = np.abs(pred_data_original[:, 1] - test_labels_original[:, 1])\n", + "\n", + "# Calculate the Mean Absolute Error (MAE) for each element separately\n", + "mae_theta = np.mean(absolute_diffs_theta)\n", + "mae_phi = np.mean(absolute_diffs_phi)\n", + "\n", + "print(\"Mean Absolute Error for Theta:\", mae_theta)\n", + "print(\"Mean Absolute Error for Phi:\", mae_phi)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "44", + "metadata": {}, + "outputs": [], + "source": [ + "fig = plt.figure(figsize=(8, 6))\n", + "plt.hist(test_labels_original[:, 0],bins=50,alpha=0.6,color=\"b\",label=\"test coords\")\n", + "plt.hist(pred_data_original[:,0],bins=50,alpha=0.6,color=\"r\",label=\"reco coords\")\n", + "plt.xlabel(\"Theta\")\n", + "plt.legend()\n", + "plt.ylabel(\"Counts\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "45", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "46", + "metadata": {}, + "outputs": [], + "source": [ + "fig = plt.figure(figsize=(8, 6))\n", + "plt.hist(test_labels_original[:, 1],bins=50,alpha=0.6,color=\"b\",label=\"reco coords\")\n", + "plt.hist(pred_data_original[:,1],bins=50,alpha=0.6,color=\"r\",label=\"test coords\")\n", + "plt.xlabel(\"Phi\")\n", + "plt.legend()\n", + "plt.ylabel(\"Counts\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "54", + "metadata": {}, + "outputs": [], + "source": [ + "fig = plt.figure(figsize=(8, 6))\n", + "plt.hist(test_labels[:,0],bins=50,alpha=0.6,color=\"b\",label=\"test coords\")\n", + "plt.hist(pred_data[:, 0],bins=50,alpha=0.6,color=\"r\",label=\"reco coords\")\n", + "plt.xlabel(\"Theta sin\")\n", + "plt.legend()\n", + "plt.ylabel(\"Counts\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "55", + "metadata": {}, + "outputs": [], + "source": [ + "fig = plt.figure(figsize=(8, 6))\n", + "plt.hist(test_labels[:,1],bins=50,alpha=0.6,color=\"b\",label=\"test coords\")\n", + "plt.hist(pred_data[:, 1],bins=50,alpha=0.6,color=\"r\",label=\"reco coords\")\n", + "plt.xlabel(\"Theta cos\")\n", + "plt.legend()\n", + "plt.ylabel(\"Counts\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "56", + "metadata": {}, + "outputs": [], + "source": [ + "fig = plt.figure(figsize=(8, 6))\n", + "plt.hist(test_labels[:,2],bins=50,alpha=0.6,color=\"b\",label=\"test coords\")\n", + "plt.hist(pred_data[:, 2],bins=50,alpha=0.6,color=\"r\",label=\"reco coords\")\n", + "plt.xlabel(\"Phi sin\")\n", + "plt.legend()\n", + "plt.ylabel(\"Counts\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b5b7779b-43f4-4030-bb49-68d05c9d66da", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "57", + "metadata": {}, + "outputs": [], + "source": [ + "fig = plt.figure(figsize=(8, 6))\n", + "plt.hist(test_labels[:,3],bins=50,alpha=0.6,color=\"b\",label=\"test coords\")\n", + "plt.hist(pred_data[:, 3],bins=50,alpha=0.6,color=\"r\",label=\"reco coords\")\n", + "plt.xlabel(\"Phi cos\")\n", + "plt.legend()\n", + "plt.ylabel(\"Counts\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "58", + "metadata": {}, + "outputs": [], + "source": [ + "# Calculate distances between corresponding coordinates\n", + "\n", + "distances = []\n", + "theta_distances = []\n", + "phi_distances = []\n", + "\n", + "for i in range(0,len(pred_data_original)):\n", + "\n", + " d = angular_distance(pred_data_original[i][0],pred_data_original[i][1],test_labels_original[i][0],test_labels_original[i][1])\n", + " theta_dist = np.abs(pred_data_original[i][0]-test_labels_original[i][0])\n", + " phi_dist = diff_phi(pred_data_original[i][1],test_labels_original[i][1])\n", + " distances.append(d)\n", + " theta_distances.append(theta_dist)\n", + " phi_distances.append(phi_dist)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "fb0c82c2-5e42-4103-adf5-a0cb24175f68", + "metadata": {}, + "outputs": [], + "source": [ + "import pickle\n", + "if save_data:\n", + "\n", + " # Salva l'array in un file usando pickle\n", + " with open(data_path+\"/\"+evaluation_file+\"_distances_tanh_nobkg.pkl\", \"wb\") as f:\n", + " pickle.dump(distances, f)\n", + "\n", + " # Salva l'array in un file usando pickle\n", + " with open(data_path+\"/\"+evaluation_file+\"_conf_area_tanh_nobkg.pkl\", \"wb\") as f:\n", + " pickle.dump(conf_area, f)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c2afbbdc-d1f0-44ca-8c3f-1d5f88e74149", + "metadata": {}, + "outputs": [], + "source": [ + "import numpy as np\n", + "import healpy as hp\n", + "import matplotlib.pyplot as plt\n", + "\n", + "nside = 32\n", + "npix = hp.nside2npix(nside)\n", + "\n", + "\n", + "m = np.array(distances)\n", + "\n", + "hp.projview(\n", + " m,\n", + " coord=[\"G\"],\n", + " graticule=True,\n", + " graticule_labels=True,\n", + " unit=\"cbar label\",\n", + " xlabel=\"longitude\",\n", + " ylabel=\"latitude\",\n", + " cb_orientation=\"vertical\",\n", + " latitude_grid_spacing=30,\n", + " projection_type=\"aitoff\",\n", + " title=\"Aitoff projection\",\n", + " cmap=\"turbo\",\n", + " nest=True\n", + ")\n", + "\n", + "plt.show()\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c6be6847-a33e-46b1-8225-72a455bb9a52", + "metadata": {}, + "outputs": [], + "source": [ + "print(np.mean(distances))\n", + "print(np.mean(theta_distances))\n", + "print(np.mean(phi_distances))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "79", + "metadata": {}, + "outputs": [], + "source": [ + "# Definisci gli intervalli di 5 gradi\n", + "intervalli = np.arange(0, 185, 5)\n", + "\n", + "# Raggruppa i conteggi in base agli intervalli di 5 gradi\n", + "conteggi_raggruppati = np.zeros((len(intervalli)))\n", + "conteggi = np.zeros((len(intervalli)))\n", + "\n", + "tot_count = 0\n", + "for theta,phi,conteggio in zip(test_labels_original[:,0],test_labels_original[:,1], distances):\n", + " \n", + " if(True): #(phi>125 and phi<145) or \n", + " tot_count +=1\n", + " indice_intervallo = int(theta // 5)\n", + " conteggi_raggruppati[indice_intervallo] = conteggi_raggruppati[indice_intervallo]+conteggio\n", + " conteggi[indice_intervallo]+=1\n", + "\n", + "# Calcola la media dei conteggi in ciascun intervallo\n", + "conteggi_raggruppati = conteggi_raggruppati[:tot_count]\n", + "conteggi = conteggi[:tot_count]\n", + "\n", + "# Visualizza l'istogramma\n", + "plt.figure(figsize=(10, 6))\n", + "plt.bar(intervalli[:-1], conteggi_raggruppati[:-1]/conteggi[:-1], width=5, align='edge',alpha=1,label=\"\")\n", + "plt.xlabel('Theta (°)', fontsize=15)\n", + "plt.ylabel('Loc. error (°)', fontsize=15)\n", + "plt.tick_params(axis='both', labelsize=15) \n", + "plt.title('')\n", + "plt.grid(True)\n", + "#plt.legend()\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a4df552b", + "metadata": {}, + "outputs": [], + "source": [ + "if(save_data):\n", + "\n", + " training_histo = {'test_labels_original': test_labels_original[:,0], 'test_labels_original': test_labels_original[:,1],\"distances\":distances}\n", + "\n", + " with open(data_path+\"/\"+model_name+\"_training_histo_tanh_nobkg.pkl\", \"wb\") as f:\n", + " pickle.dump(training_histo, f)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "edaaf41a", + "metadata": {}, + "outputs": [], + "source": [ + "\n", + "if save_data:\n", + " data_to_store ={\"test_labels_0\":test_labels_original[:,0],\"test_labels_1\":test_labels_original[:,1]}\n", + " \n", + " # Salva l'array in un file usando pickle\n", + " with open(data_path+\"/\"+evaluation_file+\"_hist_data_tanh_nobkg.pkl\", \"wb\") as f:\n", + " pickle.dump(data_to_store, f)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ac939e97-652f-4773-bddb-465406cdb4b0", + "metadata": {}, + "outputs": [], + "source": [ + "step = 10\n", + "\n", + "# Definisci gli intervalli di 5 gradi\n", + "intervalli = np.arange(0, 365, step)\n", + "\n", + "# Raggruppa i conteggi in base agli intervalli di 5 gradi\n", + "conteggi_raggruppati_1 = np.zeros((len(intervalli)))\n", + "conteggi_1 = np.zeros((len(intervalli)))\n", + "\n", + "conteggi_raggruppati_2 = np.zeros((len(intervalli)))\n", + "conteggi_2 = np.zeros((len(intervalli)))\n", + "\n", + "conteggi_raggruppati_3 = np.zeros((len(intervalli)))\n", + "conteggi_3 = np.zeros((len(intervalli)))\n", + "\n", + "conteggi_raggruppati_4 = np.zeros((len(intervalli)))\n", + "conteggi_4 = np.zeros((len(intervalli)))\n", + "\n", + "tot_count_1 = 0\n", + "tot_count_2 = 0\n", + "tot_count_3 = 0\n", + "tot_count_4 = 0\n", + "\n", + "for theta,phi,conteggio in zip(test_labels_original[:,0],test_labels_original[:,1], distances):\n", + " \n", + " if(theta>20 and theta<55):\n", + " tot_count_1 +=1\n", + " indice_intervallo = int(phi // step)\n", + " conteggi_raggruppati_1[indice_intervallo] = conteggi_raggruppati_1[indice_intervallo]+conteggio\n", + " conteggi_1[indice_intervallo]+=1\n", + "\n", + " if(theta>55 and theta<150):\n", + " tot_count_2 +=1\n", + " indice_intervallo = int(phi // step)\n", + " conteggi_raggruppati_2[indice_intervallo] = conteggi_raggruppati_2[indice_intervallo]+conteggio\n", + " conteggi_2[indice_intervallo]+=1\n", + "\n", + " if(theta>150 and theta<180):\n", + " tot_count_3 +=1\n", + " indice_intervallo = int(phi // step)\n", + " conteggi_raggruppati_3[indice_intervallo] = conteggi_raggruppati_3[indice_intervallo]+conteggio\n", + " conteggi_3[indice_intervallo]+=1\n", + "\n", + "# Calcola la media dei conteggi in ciascun intervallo\n", + "conteggi_raggruppati_1 = conteggi_raggruppati_1[:tot_count_1]\n", + "conteggi_1 = conteggi_1[:tot_count_1]\n", + "\n", + "conteggi_raggruppati_2 = conteggi_raggruppati_2[:tot_count_2]\n", + "conteggi_2 = conteggi_2[:tot_count_2]\n", + "\n", + "conteggi_raggruppati_3 = conteggi_raggruppati_3[:tot_count_3]\n", + "conteggi_3 = conteggi_3[:tot_count_3]\n", + "\n", + "# Visualizza l'istogramma\n", + "plt.figure(figsize=(10, 6))\n", + "plt.bar(intervalli[:-1], conteggi_raggruppati_1[:-1]/conteggi_1[:-1], width=step, align='edge',alpha=0.5,label=\"loc. error theta=[20°,55°]\")\n", + "\n", + "plt.bar(intervalli[:-1], conteggi_raggruppati_2[:-1]/conteggi_2[:-1], width=step, align='edge',alpha=0.5,label=\"loc. error theta=[55°,150°]\")\n", + "\n", + "plt.bar(intervalli[:-1], conteggi_raggruppati_3[:-1]/conteggi_3[:-1], width=step, align='edge',alpha=0.5,label=\"loc. error theta=[150°,180°]\")\n", + "plt.xlabel('Phi (°)')\n", + "plt.ylabel('Loc. error')\n", + "plt.title('Loc. error in function of phi')\n", + "plt.grid(True)\n", + "plt.legend()\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "527add68-000a-440a-9a6d-a231b36fe9b1", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0244c058-1f3d-44e4-96e6-a523f0faf8e5", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9f1e6467-7c96-4110-ab42-79a31b6c137d", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "deeplearning", + "language": "python", + "name": "deeplearning" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.12" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/cosipy/notebooks/dl-loc/train_and_test/train_simple_model.ipynb b/cosipy/notebooks/dl-loc/train_and_test/train_simple_model.ipynb new file mode 100644 index 0000000..d596989 --- /dev/null +++ b/cosipy/notebooks/dl-loc/train_and_test/train_simple_model.ipynb @@ -0,0 +1,609 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "0", + "metadata": {}, + "outputs": [], + "source": [ + "# import libraries\n", + "import os\n", + "import numpy as np\n", + "import matplotlib.pyplot as plt\n", + "%matplotlib inline\n", + "import pickle\n", + "from numpy.random import seed\n", + "from tensorflow.random import set_seed\n", + "import tensorflow as tf\n", + "from tensorflow import keras\n", + "import matplotlib.pyplot as plt\n", + "import numpy as np\n", + "import math\n", + "\n", + "# set random seedindexes_AC1\n", + "seed(10)\n", + "set_seed(10)\n", + "\n", + "training_dataset_name = \"run53_mix_mega_shared\"\n", + "testing_dataset_name = \"run57_mix_mega_shared\"\n", + "\n", + "val_number = 6000\n", + "# If not GPU is available, set the following to -1\n", + "os.environ[\"CUDA_VISIBLE_DEVICES\"]=\"0\"\n", + "numpy_file=0\n", + "\n", + "number_of_detectors = 6\n", + "data_path = \"/data/test_newrepo/\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1", + "metadata": {}, + "outputs": [], + "source": [ + "print(\"Num GPUs Available: \", len(tf.config.list_physical_devices('GPU')))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2", + "metadata": {}, + "outputs": [], + "source": [ + "\n", + "# load training dataset\n", + "file_path = data_path+'/'+training_dataset_name+'_dataset.pkl'\n", + "\n", + "with open(file_path, 'rb') as file:\n", + " training_dataset_array = pickle.load(file)\n", + "\n", + "# load testing dataset\n", + "file_path = data_path+'/'+testing_dataset_name+'_dataset.pkl'\n", + "\n", + "with open(file_path, 'rb') as file:\n", + " testing_dataset_array = pickle.load(file)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4", + "metadata": {}, + "outputs": [], + "source": [ + "def diff_phi(a1, a2):\n", + " diff = abs(a1 - a2)\n", + " if diff > 180:\n", + " diff = 360 - diff\n", + " return diff\n", + "\n", + "def angular_distance(theta1, phi1, theta2, phi2):\n", + " \"\"\"\n", + " Compute the angular (great-circle) distance in degrees between two points\n", + " specified by (theta, phi) coordinates in degrees.\n", + "\n", + " Parameters:\n", + " theta1, phi1: floats, coordinates of the first point (degrees).\n", + " theta2, phi2: floats, coordinates of the second point (degrees).\n", + "\n", + " Returns:\n", + " angular_dist_deg: float, angular distance between the two points in degrees.\n", + " \"\"\"\n", + " # Adjust theta values by shifting -90\n", + " theta1 = 90 - theta1\n", + " theta2 = 90 - theta2\n", + "\n", + " # Convert angles to radians\n", + " theta1_rad = math.radians(theta1)\n", + " phi1_rad = math.radians(phi1)\n", + " theta2_rad = math.radians(theta2)\n", + " phi2_rad = math.radians(phi2)\n", + " \n", + " # Compute the difference in longitude\n", + " delta_phi = abs(phi1_rad - phi2_rad)\n", + "\n", + " # Apply spherical law of cosines\n", + " value = (math.sin(theta1_rad) * math.sin(theta2_rad) +\n", + " math.cos(theta1_rad) * math.cos(theta2_rad) * math.cos(delta_phi))\n", + " \n", + " # Clamp value to [-1, 1] to avoid floating point errors in acos\n", + " value_clamped = max(-1.0, min(1.0, value)) \n", + " \n", + " angular_dist = math.acos(value_clamped)\n", + " \n", + " # Convert angular distance from radians to degrees\n", + " angular_dist_deg = math.degrees(angular_dist)\n", + " \n", + " return angular_dist_deg\n", + "\n", + "\n", + "def l2_normalize(data):\n", + " \"\"\"\n", + " Normalize a NumPy array using the L2 (Euclidean) norm.\n", + "\n", + " Args:\n", + " data (numpy.ndarray): The array to normalize. Can be a 1D vector\n", + " or a 2D matrix (where each row is a vector to normalize).\n", + "\n", + " Returns:\n", + " numpy.ndarray: The L2-normalized array.\n", + " \"\"\"\n", + "\n", + " data = np.array(data)\n", + " if data.ndim == 1:\n", + " # Case: 1D vector\n", + " norm = np.sqrt(np.sum(data**2))\n", + " if norm == 0:\n", + " return data # Avoid division by zero if the vector is null\n", + " return data / norm\n", + " elif data.ndim == 2:\n", + " # Case: 2D matrix (normalize each row)\n", + " norms = np.sqrt(np.sum(data**2, axis=1, keepdims=True))\n", + " # Handle the case of null norms (zero rows)\n", + " norms[norms == 0] = 1\n", + " return data / norms\n", + " else:\n", + " raise ValueError(\"Input must be a 1D or 2D array.\")\n", + "\n", + " \n", + "def prepare_dataset(data_array):\n", + " \"\"\"\n", + " Prepare the dataset and corresponding labels for training or evaluation.\n", + "\n", + " Steps performed:\n", + " 1. Normalize the detector counts using L2 normalization.\n", + " 2. Extract longitude and latitude labels.\n", + " 3. Optionally scale labels between 0 and 1 (if `normalize == 1`).\n", + " 4. Convert labels to radians for further processing.\n", + "\n", + " Args:\n", + " data_array (list of dict): Each element must contain:\n", + " - 'counts' (numpy.ndarray): Detector count values.\n", + " - 'coord' (tuple or list): Coordinates (longitude, latitude).\n", + "\n", + " Returns:\n", + " tuple:\n", + " - dataset (numpy.ndarray): Normalized detector counts.\n", + " - labels (numpy.ndarray): Original labels (longitude, latitude).\n", + " - labels_norm (numpy.ndarray): Scaled labels (if normalization applied).\n", + " - theta (tuple): Extracted theta values in degrees.\n", + " - phi (tuple): Extracted phi values in degrees.\n", + " - coords_rad (list): List of coordinates in radians.\n", + " \"\"\"\n", + "\n", + " dataset = np.empty((len(data_array), number_of_detectors))\n", + " labels = np.empty((len(data_array), 2))\n", + " \n", + " count = -1\n", + " for element in data_array:\n", + " count += 1\n", + " dataset[count] = l2_normalize(element['counts'])\n", + " labels[count][0] = float(element['coord'][0])\n", + " labels[count][1] = float(element['coord'][1])\n", + " \n", + " labels = labels[:count+1]\n", + " dataset = dataset[:count+1]\n", + "\n", + " \n", + " labels_norm = np.empty((len(dataset), 2))\n", + " \n", + " count = -1\n", + " for element in labels:\n", + " count += 1\n", + " labels_norm[count][0] = labels[count][0]\n", + " labels_norm[count][1] = labels[count][1]\n", + "\n", + " # Convert to radians\n", + " coords_rad = []\n", + " \n", + " for theta, phi in labels:\n", + " coords_rad.append([theta, phi])\n", + "\n", + " theta, phi = zip(*coords_rad)\n", + " \n", + " return dataset, labels, labels_norm, theta, phi, coords_rad\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8900376e-81b7-4789-97d0-4d043a3e928e", + "metadata": {}, + "outputs": [], + "source": [ + "# prepare training and validation dataset\n", + "dataset, labels, labels_norm, theta, phi, coords_rad = prepare_dataset(training_dataset_array)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "057cccd3-b4e3-4aa5-aba9-3225cc579968", + "metadata": {}, + "outputs": [], + "source": [ + "dataset.shape" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3f34c833-26b8-4a56-b76b-f7c13461cc5e", + "metadata": {}, + "outputs": [], + "source": [ + "# prepare testing and validation dataset\n", + "test_dataset, test_labels, test_labels_norm, test_theta, test_phi, test_coords_rad = prepare_dataset(testing_dataset_array)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4eba8616", + "metadata": {}, + "outputs": [], + "source": [ + "test_dataset.shape" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5778a8ed-c05b-485c-b8f4-56edeca89284", + "metadata": {}, + "outputs": [], + "source": [ + "#split val and test dataset\n", + "random_indices = np.random.permutation(len(dataset))\n", + "\n", + "N = dataset.shape[0]\n", + "validation_indices = random_indices[:val_number]\n", + "train_indices = np.setdiff1d(np.arange(N), validation_indices)\n", + "\n", + "validation_dataset = dataset[validation_indices]\n", + "training_dataset = dataset[train_indices]\n", + "\n", + "validation_labels = labels[validation_indices]\n", + "training_labels = labels[train_indices]\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "16", + "metadata": {}, + "outputs": [], + "source": [ + "\n", + "coord_array = np.array(coords_rad)\n", + "coord_array.shape" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "17", + "metadata": {}, + "outputs": [], + "source": [ + "plt.hist(coord_array[:, 0])\n", + "plt.xlabel(\"Theta\")\n", + "plt.ylabel(\"Counts\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "18", + "metadata": {}, + "outputs": [], + "source": [ + "plt.hist(coord_array[:, 1])\n", + "plt.xlabel(\"Phi\")\n", + "plt.ylabel(\"Counts\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "36", + "metadata": {}, + "outputs": [], + "source": [ + "# Define the model architecture\n", + "model = keras.Sequential([\n", + " keras.layers.Dense(128*2, input_shape=(number_of_detectors,)), # activation='relu',#,kernel_regularizer=tf.keras.regularizers.l2(0.01)\n", + " keras.layers.LeakyReLU(alpha=0.1) ,\n", + " keras.layers.Dropout(0.02),\n", + " keras.layers.Dense(64*2), # , activation='relu'#,kernel_regularizer=tf.keras.regularizers.l2(0.01)\n", + " keras.layers.LeakyReLU(alpha=0.1),\n", + " keras.layers.Dropout(0.02),\n", + " keras.layers.Dense(32*2), #activation='relu'\n", + " keras.layers.LeakyReLU(alpha=0.1),\n", + " keras.layers.Dropout(0.02),\n", + " keras.layers.Dense(2)\n", + "])\n", + "\n", + "# Compile the model\n", + "model.compile(optimizer=keras.optimizers.Adam(learning_rate=0.001), loss=\"mse\")\n", + "model.summary()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "37", + "metadata": {}, + "outputs": [], + "source": [ + "history = model.fit(\n", + " x=training_dataset,\n", + " y=training_labels,\n", + " epochs=2000,\n", + " batch_size=256,\n", + " validation_data=(validation_dataset,validation_labels),\n", + " validation_split=0.2,\n", + " shuffle=True,\n", + " callbacks=[\n", + " keras.callbacks.EarlyStopping(monitor=\"val_loss\", patience=5, mode=\"min\")\n", + " ],\n", + ")\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "39", + "metadata": {}, + "outputs": [], + "source": [ + "fig, ax1 = plt.subplots(1, 1,figsize=(10,5))\n", + "fig.suptitle('Training')\n", + " \n", + "ax1.plot(history.history[\"loss\"], label=\"Training Loss\")\n", + "ax1.plot(history.history[\"val_loss\"], label=\"Validation Loss\")\n", + " \n", + "ax1.legend()\n", + "plt.show() " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "40", + "metadata": {}, + "outputs": [], + "source": [ + "#save model for inference when it is ready\n", + "save_model = False\n", + "if(save_model):\n", + " model_name = training_dataset_name+\"_simple_model.keras\"\n", + " model.save(data_path+\"/\"+model_name)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "071fa582-ae86-4327-9b9d-10f13b17078a", + "metadata": {}, + "outputs": [], + "source": [ + "#Perform prediction on the test dataset\n", + "pred_data = model.predict(test_dataset)\n", + " " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "43", + "metadata": {}, + "outputs": [], + "source": [ + "plt.hist(test_labels[:, 0],bins=50,alpha=0.5)\n", + "plt.hist(pred_data[:, 0],bins=50,alpha=0.5)\n", + "plt.xlabel(\"Theta\")\n", + "plt.ylabel(\"Counts\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "44", + "metadata": {}, + "outputs": [], + "source": [ + "plt.hist(test_labels[:, 1],bins=50,alpha=0.5)\n", + "plt.hist(pred_data[:, 1],bins=50,alpha=0.5)\n", + "plt.xlabel(\"Phi\")\n", + "plt.ylabel(\"Counts\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "45", + "metadata": {}, + "outputs": [], + "source": [ + "max_lon = 180.0\n", + "min_lon = 0.0\n", + "\n", + "max_lat = 360.0\n", + "min_lat = 0.0\n", + "\n", + "pred_data_original = np.empty((len(pred_data),2))\n", + "\n", + "pred_data_original[:, 0] = pred_data[:, 0]\n", + "pred_data_original[:, 1] = pred_data[:, 1]\n", + "\n", + "test_labels_original = np.empty((len(test_labels),2))\n", + "\n", + "test_labels_original[:, 0] = test_labels[:, 0]\n", + "test_labels_original[:, 1] = test_labels[:, 1]\n", + "\n", + "absolute_diffs_theta = np.abs(pred_data_original[:, 0] - test_labels_original[:, 0])\n", + "absolute_diffs_phi = np.abs(pred_data_original[:, 1] - test_labels_original[:, 1])\n", + "\n", + "# Calculate the Mean Absolute Error (MAE) for each element separately\n", + "mae_theta = np.mean(absolute_diffs_theta)\n", + "mae_phi = np.mean(absolute_diffs_phi)\n", + "\n", + "print(\"Mean Absolute Error for Theta:\", mae_theta)\n", + "print(\"Mean Absolute Error for Phi:\", mae_phi)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "51", + "metadata": {}, + "outputs": [], + "source": [ + "# Calculate distances between corresponding coordinates\n", + "\n", + "distances = []\n", + "theta_distances = []\n", + "phi_distances = []\n", + "\n", + "for i in range(0,len(pred_data_original)):\n", + "\n", + " d = angular_distance(pred_data_original[i][0],pred_data_original[i][1],test_labels_original[i][0],test_labels_original[i][1])\n", + " theta_dist = np.abs(pred_data_original[i][0]-test_labels_original[i][0])\n", + " phi_dist = diff_phi(pred_data_original[i][1],test_labels_original[i][1])\n", + " distances.append(d)\n", + " theta_distances.append(theta_dist)\n", + " phi_distances.append(phi_dist)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "86deb06f", + "metadata": {}, + "outputs": [], + "source": [ + "print(np.mean(distances))\n", + "print(np.mean(theta_distances))\n", + "print(np.mean(phi_distances))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1cb56bf7-be9a-41ac-ab96-0980d16fd3f2", + "metadata": {}, + "outputs": [], + "source": [ + "import healpy as hp\n", + "import numpy as np\n", + "import matplotlib.pyplot as plt\n", + "hp.projview(\n", + " np.array(distances),\n", + " coord=[\"G\"],\n", + " projection_type=\"aitoff\", \n", + " graticule=True,\n", + " graticule_labels=True,\n", + " longitude_grid_spacing=60,\n", + " title=file,\n", + " latitude_grid_spacing=30,\n", + " cmap=\"turbo\",\n", + " nest=True,\n", + " unit=\"\", \n", + " fontsize={\n", + " \"xlabel\": 14,\n", + " \"ylabel\": 14,\n", + " \"title\": 16,\n", + " \"xtick_label\": 14,\n", + " \"ytick_label\": 14,\n", + " \"cbar_label\": 14,\n", + " \"cbar_tick_label\": 14 \n", + " },\n", + " override_plot_properties={\n", + " \"cbar_shrink\": 0.8,\n", + " \"cbar_pad\": 0.05,\n", + " \"cbar_label_pad\": 5\n", + " }\n", + " \n", + ")\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a565f6a8-f9c9-4080-9791-026b45bd4433", + "metadata": {}, + "outputs": [], + "source": [ + "if False:\n", + "\n", + " # Salva l'array in un file usando pickle\n", + " with open(data_path+\"/\"+testing_dataset_name+\"_distances_simple_model.pkl\", \"wb\") as f:\n", + " pickle.dump(distances, f)\n", + "\n", + " # Salva l'array in un file usando pickle\n", + " with open(data_path+\"/\"+testing_dataset_name+\"_conf_area_simple_model.pkl\", \"wb\") as f:\n", + " pickle.dump(conf_area, f)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "71", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7c895698-f173-465d-8c85-7971f7b2c863", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5baa3d63-7460-412e-8a5f-861c3e9ca87a", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "47e2c040-a777-4f1f-9e99-49453c7dc944", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "deeplearning", + "language": "python", + "name": "deeplearning" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.12" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/cosipy/notebooks/sgrb_fluxes_bis.csv b/cosipy/notebooks/sgrb_fluxes_bis.csv new file mode 100644 index 0000000..7b8343d --- /dev/null +++ b/cosipy/notebooks/sgrb_fluxes_bis.csv @@ -0,0 +1,400 @@ +name,energy flux (erg/cm2/s for 10-10000 keV),photon flux (ph/cm2/s for 10-10000 keV),t90 (s),best fit model,model parameters +bn130204484,7.511685798280614e-06,8.502548104540658,0.192,plaw,[1.332542] +bn101208203,1.5502618530574295e-06,4.046674749151104,0.192,comp,"[-0.4232284, 562.7034]" +bn130112353,1.5041891246709332e-06,4.804817617279744,2.048,comp,"[-0.7090046, 548.3127]" +bn090305052,3.6726300278351423e-06,6.20595125159901,1.856,comp,"[-0.6253819, 1059.131]" +bn170203486,5.213686617190176e-06,8.2099209284794,0.336,plaw,[1.432628] +bn150912600,1.3261562361910868e-06,4.382206600562455,0.32,comp,"[0.04723212, 351.8633]" +bn180404848,2.134421208514859e-06,13.752811161749328,0.544,plaw,[1.872639] +bn150101641,1.7457584432054703e-06,8.080196006384485,0.08,plaw,[1.760916] +bn080919790,3.2092294302931208e-06,5.532369650601615,0.512,plaw,[1.459679] +bn150320462,1.3190107230701346e-05,17.597607222856897,0.064,comp,"[0.03818452, 901.158]" +bn120629565,4.219170359664678e-06,3.9493063348450863,0.704,plaw,[1.273156] +bn160804968,8.362575045634757e-06,8.958363972294755,0.192,comp,"[-0.9808683, 3011.984]" +bn170305256,2.0446719648145637e-06,11.107752574094874,0.448,comp,"[-0.6310343, 273.189]" +bn170802638,1.1356988454778292e-06,3.1201663171053293,2.24,comp,"[-0.813788, 724.0781]" +bn120410585,2.8672945174693685e-06,6.448417899103565,1.088,comp,"[-0.06644946, 549.7534]" +bn121116459,5.957514845505279e-06,8.152468537650293,0.832,plaw,[1.390417] +bn170111760,3.608032262317789e-06,6.121178598123531,0.832,comp,"[-0.4433696, 910.5951]" +bn090412061,3.845973956115478e-06,4.900087557342759,0.896,plaw,[1.368761] +bn140320092,7.256794741319878e-07,1.0375559948041437,2.304,plaw,[1.403644] +bn110916016,3.7745488217631597e-06,4.089195735983891,1.792,plaw,[1.318989] +bn170818137,3.1768168103301167e-06,9.385120054816497,0.576,plaw,[1.620798] +bn170403583,2.5351330647016996e-06,9.915503029463896,0.48,comp,"[-0.9664587, 543.257]" +bn140518709,7.309912697560368e-06,7.799761910176384,0.704,plaw,[1.314269] +bn150208929,4.848637479841042e-06,5.20353589099845,0.128,plaw,[1.316064] +bn110409179,3.490189415235338e-06,9.516219496334694,0.128,comp,"[-0.292223, 498.3796]" +bn120608489,1.230400682296496e-06,5.175226333656584,0.96,comp,"[-0.6051665, 366.4915]" +bn120415891,2.258940228126828e-06,5.068646190907963,0.96,comp,"[-0.6751804, 804.4412]" +bn100223110,1.0346664688923314e-05,13.179797245244858,0.256,comp,"[-0.1722118, 1042.048]" +bn120817168,1.5488546741769854e-05,24.750610035470167,0.16,comp,"[-0.7940158, 1348.356]" +bn081213173,3.923892244530699e-06,5.7526882414147575,0.256,plaw,[1.411187] +bn180913783,1.8402424106480845e-06,5.99835556505174,0.768,comp,"[-0.6424759, 508.0179]" +bn150810485,9.170136824237572e-06,12.82060238414937,1.28,comp,"[-0.6662983, 1360.985]" +bn130503214,5.939364809242622e-06,7.429944700547691,0.88,plaw,[1.363187] +bn121004211,2.264184827041946e-06,4.849568327167719,1.536,plaw,[1.524343] +bn110529034,5.649790523546774e-05,18.07257584731199,0.512,band,"[-0.8673255, -0.8673255, 874.4302092083824]" +bn090907808,1.7432021854676813e-06,5.4064436874627315,0.832,comp,"[-0.08967049, 395.8718]" +bn170127634,2.2678334113717084e-06,5.429878294651053,1.728,comp,"[-0.4318346, 621.9145]" +bn141230871,4.926922440842448e-06,8.51298689610811,0.224,plaw,[1.460363] +bn130912358,4.133352930625488e-06,10.483978286267194,0.512,comp,"[-0.9954004, 998.1555]" +bn180715741,2.121031489614681e-06,5.259107609735459,1.664,comp,"[-0.3213538, 560.7961]" +bn090616157,2.7252920315182215e-06,4.236100205543409,1.152,plaw,[1.428738] +bn171219279,5.6281213092070145e-06,5.693418261418317,1.152,plaw,[1.297632] +bn131217108,7.03556999018713e-06,7.103102812851367,0.768,comp,"[-0.483552, 1634.864]" +bn121112806,8.542590013170422e-06,4.601582264997366,1.28,plaw,[1.085652] +bn180523782,3.5466878475672075e-06,3.686711687486445,1.984,comp,"[-0.3522813, 1434.52]" +bn151231568,2.3847305190325338e-06,8.53467170209779,0.832,comp,"[-0.6703156, 464.5384]" +bn181126162,2.8451238642455797e-06,5.338501973016132,1.664,comp,"[-0.7364213, 1048.92]" +bn101026034,3.4538745535270027e-06,10.392698745302871,0.256,comp,"[-0.4108429, 477.5689]" +bn150101270,4.286208305815974e-06,6.993492386129306,0.48,plaw,[1.443254] +bn090518080,1.0386016346305871e-06,2.7481332445084443,2.048,plaw,[1.587542] +bn150506630,3.842106677704639e-06,7.4377170055066735,0.512,comp,"[-0.7181795, 993.0571]" +bn130929375,5.470069622760402e-07,1.2955466059651675,2.304,plaw,[1.554339] +bn081122614,9.570119762797254e-07,8.553019595678222,0.192,comp,"[-1.067436, 179.2807]" +bn120822628,3.171729991330128e-06,4.837100446253387,1.536,plaw,[1.423034] +bn081101491,9.788741762001454e-07,5.63088061679261,0.128,comp,"[-0.3365863, 221.3938]" +bn081230871,4.80854053217107e-06,5.209263539572954,0.512,plaw,[1.318982] +bn150923297,4.051987591142892e-06,7.832444943210134,0.192,plaw,[1.493789] +bn100117879,1.981515361470747e-06,7.399592817053263,0.256,comp,"[-0.0955883, 325.4309]" +bn120616630,6.257795303561133e-06,7.1739572813588675,0.048,plaw,[1.336461] +bn180625941,2.391692137425415e-06,5.370322230216104,0.704,comp,"[-0.1671652, 576.0172]" +bn150522944,5.956178232278473e-07,3.3318039771959675,1.024,comp,"[-0.3862028, 233.5066]" +bn131004904,5.77066513629537e-07,7.15546139696295,1.152,comp,"[-1.363461, 117.9107]" +bn110307972,4.3325904415060854e-06,4.900792259792692,2.304,plaw,[1.332334] +bn100516396,5.040054869120579e-06,4.384721202390004,0.64,plaw,[1.249766] +bn180201780,1.0347145360950688e-05,83.03503804575423,0.64,plaw,[1.953521] +bn180402406,1.2179664618937408e-05,13.821782491614094,0.448,comp,"[-0.3720512, 1326.791]" +bn130504314,2.5101377742565105e-05,29.428534092592898,0.384,comp,"[-0.4199932, 1327.161]" +bn141105406,1.867858451737139e-06,6.256447113855943,1.28,comp,"[-0.3955756, 420.3897]" +bn180626392,2.3938566356434295e-06,8.90609228068897,0.96,comp,"[-0.6310148, 431.2351]" +bn150923864,8.202809049419525e-07,6.629045435591429,1.792,comp,"[-0.2397291, 143.4936]" +bn140501139,5.512753612497556e-06,6.789640495777994,0.256,plaw,[1.358435] +bn110422029,4.3289701978965824e-06,6.165934685965113,0.32,plaw,[1.402498] +bn180227211,6.480386373496658e-06,6.341762098710512,0.288,plaw,[1.287217] +bn150506972,2.272407612013544e-06,4.274663726212857,0.384,comp,"[-0.7717279, 1086.463]" +bn140402007,4.573051305795568e-06,4.970849352965056,0.32,comp,"[0.1942518, 1045.689]" +bn140428906,7.739426218954876e-06,15.196936198325442,0.32,comp,"[-0.6376418, 905.1552]" +bn100107074,1.0060337138389053e-05,8.584829022471048,0.576,plaw,[1.243538] +bn140216331,4.898075366212136e-07,1.29315132542092,2.432,comp,"[-0.837336, 779.2198]" +bn110717180,6.455469891351073e-06,20.329419711776694,0.112,comp,"[-0.4280015, 458.5691]" +bn120619884,5.447431419655468e-06,4.285076439567267,0.96,plaw,[1.217004] +bn111117510,2.486747028686732e-06,7.007605452962691,0.432,comp,"[-0.50224, 543.6155]" +bn130701761,6.814253851504495e-06,9.503017623774587,1.6,comp,"[-0.5327687, 1205.375]" +bn100417166,1.004180532707992e-06,5.008626540265062,0.192,comp,"[-0.190245, 245.8652]" +bn141122087,2.0374398683275006e-06,6.063880851797552,1.28,comp,"[-0.07643487, 411.3504]" +bn130626452,3.9217676070558315e-06,9.37218555146553,1.728,plaw,[1.557022] +bn120205285,5.759263384063646e-06,3.6668079059692777,0.576,plaw,[1.145513] +bn100616773,3.336713251539189e-06,7.862662526461025,0.192,plaw,[1.552819] +bn101027230,2.6074517257787334e-06,5.424292656273528,1.344,plaw,[1.515657] +bn081012045,3.7898457507459915e-06,5.894487888634544,1.216,comp,"[-0.6467478, 1185.219]" +bn120830297,6.2686535489898845e-06,8.441088244853258,0.896,comp,"[-0.3467566, 1089.863]" +bn120916085,4.1416074380534134e-06,5.93438798997467,1.28,plaw,[1.404296] +bn170918139,2.3534242390436926e-06,6.423358998762294,0.128,plaw,[1.596876] +bn111103948,4.814757320015597e-06,6.524917417061572,0.32,comp,"[-0.5644991, 1277.969]" +bn131128629,1.084372897795605e-06,4.540308988148106,1.984,plaw,[1.728761] +bn170219002,1.4557953172522263e-05,19.492552507055027,0.096,comp,"[-0.9051238, 1944.159]" +bn081119184,2.998637004322343e-06,5.013006760683758,0.32,plaw,[1.450514] +bn200415367,2.5857933068303638e-05,33.66281744474779,0.144,comp,"[0.01191045, 934.1315]" +bn171108656,8.578507726003322e-07,10.761165139065808,0.032,comp,"[-0.9947428, 100.2242]" +bn120814201,3.768329546426759e-06,7.92802236750434,0.896,comp,"[-0.8407089, 1030.139]" +bn161015400,6.123684065743604e-06,5.9108658910460035,0.192,plaw,[1.282882] +bn150214293,1.939615724962764e-06,4.70580024861154,0.192,comp,"[-0.6361677, 711.1631]" +bn120302722,4.1809851766450654e-06,6.6722533420533745,1.6,plaw,[1.436625] +bn110131780,3.6798929571355218e-06,4.557104181519238,0.384,plaw,[1.360104] +bn160428412,2.2407328563569204e-06,3.8059029542439413,0.576,comp,"[-0.9152142, 1474.281]" +bn101204343,3.6623964127620294e-06,8.53907560064391,0.128,comp,"[-0.27564, 584.4052]" +bn180128215,9.412297149541725e-06,8.96460100911737,0.208,plaw,[1.278659] +bn151229486,5.361483749019815e-06,6.4478069556509165,0.16,plaw,[1.351151] +bn140605377,3.3743146364486964e-06,6.726439625433339,0.512,comp,"[-0.2490734, 680.4542]" +bn150604434,3.1203137218318098e-06,7.761246210280928,0.896,comp,"[-0.7699804, 780.5557]" +bn160224911,2.9287284390690814e-06,4.665559510145033,0.384,plaw,[1.436095] +bn110420946,2.537483456559018e-06,9.349237308650254,0.128,comp,"[-0.2095776, 345.8381]" +bn140209313,8.56085433504862e-05,17.653419739217302,1.408,band,"[-0.6056056, -0.6056056, 103.1994247825436]" +bn130808253,2.3058206791225485e-06,14.149779488191507,0.256,plaw,[1.855503] +bn090225009,1.4244322627181296e-06,1.6767851689281772,2.176,plaw,[1.344596] +bn100525744,2.3992305135456737e-06,6.599435766428264,1.472,comp,"[-0.4946739, 555.7353]" +bn150906944,3.2436373013270346e-06,10.528589017453793,0.32,comp,"[-0.7777976, 570.8503]" +bn160714097,3.339344069466145e-06,5.328063964320228,0.32,plaw,[1.436566] +bn171030729,1.1425067680904638e-05,11.520389240499078,0.096,plaw,[1.296621] +bn170826369,3.0218941928067865e-06,10.45664090669741,0.256,comp,"[-1.062518, 722.4983]" +bn180311074,6.956080956777807e-06,11.267001633067895,0.96,plaw,[1.441067] +bn160603719,1.3288222664497853e-06,4.060910898310765,0.384,comp,"[-0.4932332, 493.4194]" +bn170222209,5.2659047641465415e-06,10.717488580790716,1.664,comp,"[-0.58921, 834.1014]" +bn090219074,1.0585992540909487e-05,14.762343576078008,0.448,plaw,[1.396118] +bn140724533,2.611480461575051e-06,4.46611457496338,0.896,plaw,[1.457297] +bn130706900,1.9843005882091704e-06,6.811455900159717,0.128,comp,"[-0.4525248, 422.3404]" +bn121119579,1.4383831996015302e-07,1.935593916262986,2.304,comp,"[-1.082056, 91.3795]" +bn180313978,4.15797838793094e-06,16.335726321967208,0.08,comp,"[-0.01932918, 299.7309]" +bn140720158,1.5879845920822258e-06,5.92526608750943,0.32,comp,"[-0.4203456, 377.5847]" +bn150312403,1.2911139565775668e-06,6.049100305426595,0.32,comp,"[-0.4839129, 300.8777]" +bn110213876,6.2247250149830175e-06,5.1186053072062165,0.448,plaw,[1.231522] +bn150923429,6.397134635254728e-06,6.984820031841003,0.192,plaw,[1.321413] +bn170325331,2.425476474811363e-06,6.452141673545806,0.576,comp,"[-0.4901589, 575.1304]" +bn140105748,1.1108308829158114e-06,3.3541903471812256,0.576,comp,"[-0.08398413, 406.3376]" +bn170816599,1.1331297885075644e-05,14.881342921795719,1.6,comp,"[-0.5768291, 1337.131]" +bn101116481,1.5619130153311176e-06,4.434998615788587,0.576,comp,"[-0.1046184, 437.3415]" +bn111024896,1.0167557864799006e-06,4.48530547570211,0.96,comp,"[-0.3349986, 299.6304]" +bn081113230,9.704247803454592e-07,7.108954561665115,0.576,comp,"[-0.9729986, 227.7972]" +bn160314473,2.7624416819003028e-06,6.18160435914157,1.664,plaw,[1.537395] +bn080815917,3.904039921105156e-06,5.857461652680574,0.832,plaw,[1.418133] +bn150728151,1.1438038846983404e-06,4.9257976453925805,1.728,comp,"[-0.8060713, 411.7273]" +bn100301068,2.369209374869409e-06,8.11060343956219,0.96,comp,"[-0.3499259, 400.5231]" +bn140624423,2.6348880224558985e-06,12.733326517613111,0.096,comp,"[-0.7897792, 351.0793]" +bn170121067,2.134165979644497e-06,2.757163578156054,2.304,comp,"[-0.7465572, 1626.416]" +bn160826938,1.1660815977610424e-06,3.8075931905414757,1.792,plaw,[1.651295] +bn111207512,3.84486911496208e-06,4.074666876347628,0.768,plaw,[1.312151] +bn130804023,4.0124827738067466e-06,13.803027165547551,0.96,comp,"[-0.7317937, 511.2381]" +bn131020113,1.1567077744424124e-06,1.4498880373030307,2.24,plaw,[1.363794] +bn141031998,5.4233817831760525e-06,6.24717961880644,0.16,plaw,[1.337932] +bn160408268,5.03285068652707e-06,10.70797802880623,1.056,comp,"[-0.7176537, 890.0989]" +bn101129652,1.1604226984839183e-05,10.525864603805966,0.384,comp,"[-0.07567174, 1404.938]" +bn130722990,2.362559433815048e-07,3.247751028360847,2.304,comp,"[-0.9295732, 84.58176]" +bn141202470,3.946143151347294e-06,8.017082139468652,1.408,comp,"[-0.2250504, 658.6006]" +bn180131528,4.281729559665601e-06,3.916578655337399,0.96,plaw,[1.265828] +bn090108322,5.770314008907182e-06,7.2445309528555795,0.192,plaw,[1.364285] +bn110605780,5.68936457390035e-07,3.9328347299752298,1.536,comp,"[-0.765664, 216.8002]" +bn150901924,4.514034336882669e-06,6.3923584800021995,0.256,plaw,[1.400751] +bn140329272,2.197547559047359e-06,7.258802451722647,0.064,comp,"[-0.2441292, 395.5108]" +bn161230298,2.350650721916021e-06,7.39901182577629,0.448,comp,"[-0.6488175, 531.7714]" +bn090909854,3.842871166682418e-06,6.234451172915105,1.152,plaw,[1.441548] +bn141213300,8.743770823151358e-07,9.692208284925536,0.768,comp,"[-0.9211113, 118.9579]" +bn081229187,1.5201249594567954e-06,4.279937855515393,0.768,comp,"[-0.4205929, 516.9091]" +bn150601904,4.048995190672093e-06,3.1190147633179652,0.768,plaw,[1.210097] +bn160406503,3.99717614814262e-06,10.251640772232028,0.432,comp,"[-0.7133458, 713.2415]" +bn120509619,6.485061898579682e-06,4.134344336501147,0.704,plaw,[1.145971] +bn091223191,3.7442125949076486e-06,4.0510235179926,0.576,plaw,[1.318583] +bn130628860,6.793929839446144e-06,10.94648349387138,0.512,comp,"[-0.6597789, 1153.93]" +bn090308734,2.605816760215615e-06,6.7420624258020885,1.664,comp,"[-0.5395959, 613.8003]" +bn100719311,4.0310335687076755e-06,4.186056054203643,1.6,plaw,[1.305823] +bn170101374,2.5085906196795605e-07,1.58274995811883,2.304,comp,"[-0.1335389, 185.1384]" +bn150805746,7.441464493981277e-07,4.730642227304868,1.408,plaw,[1.867881] +bn150412507,1.0042080550529435e-06,4.708703894993706,0.576,plaw,[1.76513] +bn151202565,3.907173239537126e-06,7.220082555061988,0.704,comp,"[-0.8062927, 1154.13]" +bn100216422,1.8990489364126118e-06,5.719459438330105,0.192,comp,"[0.06048958, 386.0227]" +bn120506128,7.051762259171903e-07,1.5967489639268102,2.304,plaw,[1.540919] +bn120212353,1.0895596762596946e-05,5.999028617132631,0.864,plaw,[1.093697] +bn180206203,5.251927212106203e-06,6.704602555452073,0.448,plaw,[1.369361] +bn170915520,1.8233704916475548e-06,4.656641198457837,0.64,comp,"[-0.4608204, 590.2501]" +bn180715755,4.415871372659054e-06,6.281829743353457,0.704,comp,"[-0.112533, 902.2797]" +bn081024245,6.902563131276899e-06,7.531305519417018,0.832,plaw,[1.321192] +bn111011094,2.4080742034367353e-06,10.825617730692727,1.472,comp,"[-0.6625118, 351.1842]" +bn100811108,1.3839592492434447e-05,16.24056519515673,0.384,comp,"[-0.1667924, 1130.748]" +bn101021063,4.9031019130146005e-06,5.001192979999022,1.536,plaw,[1.300222] +bn100827455,7.7672389210591e-06,15.126560058526735,0.576,comp,"[-0.3420875, 735.2537]" +bn110509475,4.06466692961516e-06,6.020216931614912,0.64,plaw,[1.414255] +bn091012783,5.862423188523224e-06,13.422011011865232,0.704,comp,"[-0.4491792, 660.5104]" +bn090520832,2.7684113188009148e-06,5.754285615344392,0.768,plaw,[1.515406] +bn171230119,4.355761886020974e-06,5.567912149159679,1.28,comp,"[-0.8616033, 1922.165]" +bn140105065,2.2726453585235782e-06,5.348171318202458,1.088,comp,"[-0.9611638, 1047.484]" +bn090331681,2.1598818068113607e-06,4.883089822869466,0.832,comp,"[-0.6064378, 751.839]" +bn141208632,3.412169047269015e-06,5.086406221148495,0.96,plaw,[1.416187] +bn160624477,5.122536783491786e-06,5.45882856336548,0.384,plaw,[1.313872] +bn161210524,1.0831163035805301e-06,1.746744200093874,2.304,plaw,[1.439766] +bn120101354,9.952644188638323e-07,6.229002859730241,0.128,comp,"[-0.3315439, 200.2687]" +bn141102112,5.885360754312782e-06,9.55816497917782,0.016,plaw,[1.441864] +bn081107321,6.521617006109458e-07,9.360728593471363,1.664,comp,"[-0.3759126, 70.81098]" +bn101129726,4.082647750062381e-06,9.529197229278875,0.576,comp,"[-0.7622867, 835.1282]" +bn111112908,6.365070455086943e-06,14.871139674290413,0.224,comp,"[-0.864512, 933.1935]" +bn140511095,1.4834663173173027e-06,8.290548746821386,1.408,comp,"[-0.9484671, 328.4363]" +bn090717111,3.827742543840784e-06,4.619734214810109,0.384,plaw,[1.352241] +bn160822672,7.692077068949601e-06,22.921091035051717,0.04,comp,"[-0.822139, 661.2325]" +bn180417689,4.9807426485905015e-06,8.399750160779313,0.256,plaw,[1.453125] +bn150715136,1.5148278871064944e-06,3.4527422512147563,0.384,comp,"[-0.8518311, 947.4437]" +bn180225417,2.5561226154965535e-06,4.4821828031130755,0.896,comp,"[-0.5777379, 974.7769]" +bn100629801,1.788722744254467e-06,12.243006051290664,0.832,comp,"[-0.7815721, 221.6978]" +bn180204109,4.704307164820773e-06,12.046521407223988,1.152,comp,"[-0.8839226, 852.8797]" +bn161026373,8.631373813837978e-07,4.8804360017727175,0.112,comp,"[-0.777012, 284.2669]" +bn090819607,2.0978352886489993e-06,6.293774349474611,0.192,comp,"[-0.06216746, 405.5841]" +bn170926782,5.430369677227797e-07,5.858679352282125,0.896,comp,"[-0.5998119, 110.7527]" +bn090510016,2.723157463355152e-05,17.66484335543551,0.96,comp,"[-0.8582567, 4727.058]" +bn091006360,5.312972163520974e-07,4.058054782347826,0.192,comp,"[-0.6492926, 179.138]" +bn090108020,1.0163038065033836e-06,9.929064959929724,0.704,comp,"[-0.6679534, 129.8776]" +bn170817529,2.3784091225450274e-06,3.6894966785049634,2.048,plaw,[1.428136] +bn140912664,2.555143915094892e-07,1.3205346777771874,2.304,comp,"[-0.4830689, 268.0318]" +bn170726249,5.282477370598211e-07,3.415678943142375,1.792,comp,"[-0.4985754, 206.3862]" +bn120603439,1.1188469659790979e-06,3.333831090918437,0.384,comp,"[-0.8006601, 647.3687]" +bn170121133,4.928810299018839e-06,5.774309952493094,1.6,plaw,[1.343128] +bn141205337,6.95177872844904e-06,6.736727763209073,1.28,plaw,[1.284128] +bn090621922,4.311222373189026e-06,10.376067271522526,0.384,comp,"[-0.09899004, 519.2835]" +bn121012724,3.4893217691728954e-06,9.22704393613825,0.448,comp,"[-0.4912427, 579.3709]" +bn110705151,1.7634789444447982e-05,23.952722932420865,0.192,comp,"[-0.2600401, 1023.676]" +bn090228204,1.4820998499059196e-05,29.75889745683513,0.448,comp,"[-0.6096338, 861.4447]" +bn081024891,2.4715345823608016e-06,3.710398138601457,0.64,comp,"[-0.9356987, 1772.778]" +bn170127067,4.341188922957119e-05,57.57768980107842,0.128,comp,"[0.1977072, 853.9544]" +bn111022854,1.4326695874909403e-06,6.019478447001055,0.192,comp,"[-0.6220712, 370.9891]" +bn121127914,7.71863270641981e-06,13.89500241396338,0.64,comp,"[-0.5996025, 964.7147]" +bn161110179,3.773304995786646e-06,3.48646053134935,1.792,plaw,[1.269036] +bn120429003,6.080155483902179e-07,3.944465037022686,1.664,comp,"[-1.017845, 281.5007]" +bn101031625,1.5437428479497544e-06,8.274247710322697,0.384,comp,"[-0.8882931, 330.9054]" +bn110401920,5.951656678063295e-06,9.64580770396461,2.368,comp,"[-0.7648572, 1281.506]" +bn170604603,5.676992480128125e-06,9.53568828015814,0.32,comp,"[-0.5403653, 990.7679]" +bn160211119,2.521494129266772e-06,3.425046637764223,0.96,plaw,[1.388179] +bn081115891,4.0479982323533495e-06,3.729966140672248,0.32,plaw,[1.268158] +bn150922234,4.324965420199967e-06,12.902970444994159,0.144,comp,"[-0.5193607, 515.6516]" +bn150705588,1.5880702082820336e-06,6.3831682942160555,0.704,plaw,[1.715829] +bn090814368,4.404425688597497e-06,9.296170555989146,0.192,comp,"[-0.4337014, 714.4023]" +bn120519721,5.214733997304044e-06,9.788991626010024,1.056,comp,"[-0.4478355, 819.0082]" +bn090802235,5.087301299921192e-06,21.677857489516967,0.048,comp,"[-0.5641558, 351.8253]" +bn100411516,4.907973323759041e-07,0.9324027503290443,0.512,plaw,[1.488628] +bn130919173,7.105624140368293e-07,7.306032513220299,0.96,comp,"[-0.5474918, 116.3509]" +bn151222340,7.966478837372408e-06,10.743723428426726,0.768,comp,"[-0.5932693, 1318.725]" +bn090109332,1.1370391955002628e-06,3.3781868245001756,1.728,plaw,[1.622516] +bn130307126,9.321458715559792e-06,11.724484491102297,0.384,comp,"[-0.7818725, 1752.764]" +bn170912985,4.766952735158694e-06,6.890502699019298,0.48,plaw,[1.406932] +bn090120627,6.018815055856888e-06,4.751380890275479,1.856,comp,"[-0.6475313, 2520.527]" +bn090924625,3.1716441593246864e-06,7.9354642171547685,0.352,comp,"[-0.5020451, 620.6631]" +bn121102064,4.780988232261797e-06,5.661046805059226,2.048,plaw,[1.346392] +bn140901821,3.524144346126918e-05,39.75336292122223,0.176,comp,"[-0.3904769, 1352.723]" +bn120129312,2.4082764890793086e-06,3.2269020898099177,1.28,comp,"[-0.4861467, 1211.39]" +bn140619490,2.0895135255033814e-06,11.787922284643544,0.448,comp,"[-0.4909284, 242.4708]" +bn140224382,1.2494008106923993e-06,2.524778963038089,2.304,comp,"[-0.9904461, 1329.403]" +bn140626843,7.752876928885368e-07,6.3226512445527066,1.792,comp,"[-0.796208, 176.1007]" +bn090328713,1.1280076845696971e-05,14.94251116481627,0.192,comp,"[-0.9078408, 1978.796]" +bn150828901,1.1222234413912515e-06,1.3850772863001402,2.048,plaw,[1.359079] +bn111222619,2.0067762505525974e-05,40.01563935569644,0.288,comp,"[-0.3949346, 740.759]" +bn121023322,4.647199599252981e-06,8.19428801298668,0.512,comp,"[-0.8776175, 1335.68]" +bn100706693,2.6685477351994237e-06,4.374164053461535,0.128,comp,"[-0.2465212, 834.7422]" +bn090418816,1.7330996018214715e-06,2.0553955200482443,0.32,plaw,[1.346881] +bn160806584,9.743127678835907e-07,8.811332110604694,1.664,comp,"[-0.5626919, 138.6533]" +bn170304003,8.443964645551153e-07,11.774179595392862,0.16,comp,"[-1.107594, 86.40084]" +bn161218222,1.908528659240739e-05,14.904950160936124,0.32,comp,"[-0.3889576, 1986.234]" +bn120314412,3.6712986024353624e-06,3.8806558932038597,1.28,plaw,[1.311345] +bn170511648,9.687660422126237e-07,3.2428358483317408,1.28,plaw,[1.658912] +bn120915000,2.5338013877271933e-06,4.903951739579248,0.576,comp,"[-0.4281524, 781.8129]" +bn080806584,4.3648483350205274e-07,1.6939517533921045,2.304,plaw,[1.704795] +bn141111435,6.760293802767975e-06,4.036557283836257,1.728,plaw,[1.122932] +bn130219626,1.4482151366022629e-06,6.440144638734237,1.536,comp,"[-0.0639023, 266.4166]" +bn110112934,1.8570347300403856e-06,6.008926779970664,2.304,comp,"[-0.7354548, 552.0621]" +bn130924910,1.3822630951008314e-06,3.5919137218765713,1.792,plaw,[1.58211] +bn150721431,3.95020325084157e-06,5.319548946238618,0.32,plaw,[1.385565] +bn150609316,5.128765409649946e-06,7.373943563399044,0.256,plaw,[1.405322] +bn150325696,1.6831618568394608e-06,6.076718306017567,0.08,comp,"[0.09633534, 315.3902]" +bn180525151,4.7808421541146275e-06,7.602489545306558,0.544,comp,"[-0.4917646, 1011.928]" +bn180511364,2.4329076729112164e-06,6.6327987935507124,0.128,comp,"[-0.8308135, 744.0004]" +bn150605782,3.46700839239169e-06,9.886502177008941,0.176,plaw,[1.610093] +bn161115745,7.994091181947968e-06,8.298356325208982,0.032,plaw,[1.305704] +bn100929916,5.7062656550770355e-06,10.083597013137497,0.32,comp,"[-0.7349309, 1122.155]" +bn081102365,1.2457860736931309e-06,3.770498343491197,1.728,comp,"[-0.8125097, 642.5339]" +bn100302061,6.860184303516307e-06,5.703677353611136,2.24,comp,"[-0.7538889, 2739.251]" +bn180602938,2.5157485928843142e-06,10.12140140559982,0.008,comp,"[-0.5978773, 384.5051]" +bn130705398,4.560675385684189e-06,5.1679688544962,0.128,plaw,[1.332882] +bn120814803,5.395545548322132e-06,4.748667142973954,0.192,plaw,[1.253489] +bn091126333,2.0409577605918817e-06,7.767136858199383,0.192,comp,"[-0.2651718, 342.1027]" +bn160821937,1.0499302653306153e-06,6.5417118934651555,1.088,plaw,[1.860818] +bn140526571,4.803730029507273e-06,4.595943684375109,0.064,plaw,[1.280087] +bn150819440,9.299814010696024e-06,43.092477641719015,0.96,comp,"[-1.136249, 523.8768]" +bn150128624,5.6076885009743775e-06,15.282488800538374,0.096,plaw,[1.596424] +bn150811849,1.4547612174774599e-05,15.287626741246589,0.64,comp,"[-0.3058705, 1375.016]" +bn160829334,2.172073575102614e-06,4.868549299846377,0.512,comp,"[-0.192975, 584.2841]" +bn100612545,6.673582647924178e-06,8.551624394216564,0.576,comp,"[-0.1934675, 1047.508]" +bn110227009,6.519522240316543e-07,2.9976051301789997,1.728,comp,"[0.08758307, 244.7061]" +bn170711713,7.597394726694033e-06,6.195713863432896,1.152,comp,"[-0.6164615, 2343.02]" +bn090429753,1.606078769930178e-06,3.6816756260545134,0.64,comp,"[-0.8408305, 928.933]" +bn090206620,2.50125675806739e-06,7.552766067228084,0.32,comp,"[-0.3500916, 459.8798]" +bn100101988,1.9553240458975394e-06,3.3318245480803967,1.984,comp,"[-0.2395344, 798.4311]" +bn080725541,4.024099523717524e-06,5.443477067057302,0.96,comp,"[-0.711663, 1482.592]" +bn130515056,4.7033430118025084e-06,13.06351674708823,0.256,comp,"[-0.3230098, 496.4844]" +bn171207055,2.64205200883046e-06,5.38506473802634,0.176,comp,"[-0.2967678, 682.2257]" +bn100816026,6.57126795395459e-07,5.592725900662463,2.045,comp,"[-0.3651878, 140.0713]" +bn130802730,7.588845836904922e-06,7.575361828038543,0.064,plaw,[1.293457] +bn160612842,7.289962561692848e-06,10.311352309772994,0.288,comp,"[-0.6814442, 1364.698]" +bn180617872,6.465490905546305e-07,4.984306702458314,1.92,comp,"[-0.3685844, 158.3149]" +bn100516369,5.011154097569832e-06,8.192954971089124,2.112,plaw,[1.443861] +bn101214748,1.9016991389921502e-06,8.046186762673193,2.24,comp,"[-0.5182318, 345.2993]" +bn080802386,3.0149364145086394e-06,8.835070563745324,0.576,comp,"[-0.7368949, 622.389]" +bn100206563,4.367225988574533e-06,11.875006671528357,0.176,comp,"[-0.4039798, 531.8227]" +bn170827818,8.846746961837848e-07,4.692474225907562,0.832,comp,"[-0.8261869, 319.8343]" +bn110424758,5.799234873166692e-06,5.781559963506882,0.672,plaw,[1.293057] +bn141128962,1.0743794669012112e-06,9.312802243456707,0.272,comp,"[-0.9102901, 171.3933]" +bn140807500,5.559176887651662e-06,14.241205504658438,0.512,comp,"[-0.8318772, 803.6211]" +bn130404877,3.660400542722187e-06,4.900575373170143,0.96,plaw,[1.383797] +bn080905499,5.915248121256069e-06,6.763702645547836,0.96,plaw,[1.335663] +bn171223818,1.9092098419497564e-05,16.50025935241779,0.384,plaw,[1.247638] +bn130622615,4.6101561973772115e-07,3.447105281109781,0.96,comp,"[-0.5288881, 174.6805]" +bn130416770,5.013707596638266e-06,9.581554610112281,0.192,comp,"[-0.7738269, 1069.092]" +bn141126233,2.7063930972911776e-06,3.6365574219784444,0.896,plaw,[1.384899] +bn140610487,5.61224628788064e-06,5.108766236877147,0.96,plaw,[1.26428] +bn081226509,1.521899430816986e-06,6.069514702453806,0.192,comp,"[-0.3654584, 340.5381]" +bn141113346,3.512971105544402e-06,4.659668676732043,0.448,plaw,[1.380981] +bn100204858,3.5837999522192034e-06,4.857149022632113,1.92,plaw,[1.387503] +bn131126163,1.0324284464414294e-05,21.258650582403963,0.128,comp,"[-0.09535138, 610.2769]" +bn081130212,1.0055588491883326e-07,1.41124919996547,2.24,plaw,[2.205672] +bn170714049,4.511443345291077e-06,6.554852105615145,0.224,plaw,[1.408482] +bn120222021,1.5935907301680633e-06,15.46879619091421,1.088,comp,"[-0.6461664, 129.9966]" +bn180402481,2.4215765767358383e-06,8.910900313351664,0.256,comp,"[-0.1321826, 335.3777]" +bn180123820,2.117716229805567e-06,3.3493970216528166,0.32,plaw,[1.433941] +bn120831901,4.494455291900937e-06,10.5554942875019,0.384,comp,"[-0.6001447, 716.5452]" +bn090320045,8.737321125143794e-07,1.6534208838812394,2.368,plaw,[1.487465] +bn120612687,5.982332103815928e-06,10.242219799217482,0.256,comp,"[-0.5411253, 970.8893]" +bn151228129,6.36657645737176e-06,9.622152248981415,0.256,comp,"[-0.09345973, 840.04]" +bn140710537,4.9425171533511565e-06,5.62460869030308,0.384,plaw,[1.334197] +bn091018957,2.267167169835969e-06,9.007143221774937,0.192,comp,"[-0.1345362, 309.0007]" +bn130617564,4.4450638801239026e-06,7.976633205276828,0.768,plaw,[1.471646] +bn120524134,6.584014570933995e-07,10.947257572892662,0.704,comp,"[-0.9848641, 61.6312]" +bn180703949,3.4804233871197234e-06,33.707464669437364,1.536,comp,"[-0.7720429, 136.8747]" +bn170709334,2.259346319526777e-06,6.213786639045233,1.856,comp,"[-0.4967802, 556.5977]" +bn150629564,3.630229244556662e-06,5.813567669216119,1.92,comp,"[-1.032268, 1937.749]" +bn170124528,2.4694475664909945e-06,6.6445182394439595,0.448,comp,"[0.03220855, 438.6344]" +bn100328141,4.772902209618571e-06,8.064763894940846,0.384,comp,"[-0.7529942, 1205.029]" +bn110526715,2.8580423957303242e-06,9.194259183269248,0.448,comp,"[-0.7617428, 568.8406]" +bn090427644,1.238548653460064e-06,1.6913095034339571,1.024,plaw,[1.389781] +bn090514726,6.928992202063025e-07,4.250427164010498,2.24,comp,"[-0.7059953, 244.8738]" +bn170206453,0.0001916010973767384,33.20997577952277,1.168,band,"[-0.4577412, -0.4577412, 225.6376167216553]" +bn160820496,2.754988917573291e-06,7.2483029845158775,0.448,comp,"[-0.8093778, 759.6443]" +bn150301045,3.737513462216555e-06,8.455633727649039,0.416,plaw,[1.540661] +bn140109771,2.958018118861477e-06,6.407811709503917,0.704,plaw,[1.527717] +bn171011810,3.8925708298786635e-06,3.9214425057404196,0.48,plaw,[1.296333] +bn160818230,1.1507538261150572e-06,5.955131697306231,2.176,comp,"[-0.3959783, 256.7397]" +bn081216531,8.46356967197507e-06,15.00733037879702,0.768,comp,"[-0.7304916, 1112.488]" +bn151022577,1.1406160552944655e-06,6.394943595409979,0.32,comp,"[-0.6362693, 263.5302]" +bn081209981,1.038856952406531e-05,16.671774033621666,0.192,comp,"[-0.6273371, 1123.715]" +bn121124606,2.8460143567309584e-06,4.745685624486662,0.256,plaw,[1.449749] +bn081223419,1.0712170952482402e-06,8.02608594486468,0.576,comp,"[-0.6269119, 181.7943]" +bn161001045,6.194008584084935e-07,3.136170848586572,2.24,comp,"[-0.939295, 373.1742]" +bn180201706,4.734266383230645e-06,6.561145769516364,0.192,plaw,[1.394244] +bn090620901,3.2634937437412737e-06,3.774166288504364,0.96,plaw,[1.339153] +bn110212550,5.7169533912993614e-06,17.55383585599886,0.064,comp,"[-0.4480852, 477.3155]" +bn150316400,1.543296746310281e-06,2.923072583009608,1.984,plaw,[1.487729] +bn110517453,5.101111804934765e-06,5.061804257638168,0.576,plaw,[1.291586] +bn100625773,4.474724099692356e-06,14.780863668086287,0.24,comp,"[-0.5947551, 483.189]" +bn120327418,1.211746967065948e-06,6.383718274630897,0.256,comp,"[-0.305056, 241.8336]" +bn130127743,5.061165241313139e-06,8.297148112104045,0.144,comp,"[0.06320828, 722.8948]" +bn100208386,5.469001512924745e-06,4.923430622368748,0.192,plaw,[1.260736] +bn121014638,2.339752961267045e-06,4.195556360913291,0.576,plaw,[1.471425] +bn090531775,4.27356443987383e-06,4.797059224111441,0.768,comp,"[-0.6363609, 1682.377]" +bn091019750,1.3641837231816815e-06,7.8603682401867525,0.208,comp,"[-0.260731, 214.5296]" +bn110728056,1.5574420570424373e-06,4.542614966359054,0.704,comp,"[-0.7102456, 611.406]" +bn100717446,1.6849059580785438e-06,4.133435647874339,2.432,comp,"[-0.6197731, 692.8386]" +bn160804180,3.637834477335076e-06,8.316955346956814,0.64,comp,"[-0.5215968, 695.5443]" +bn150628767,1.3288494060366223e-06,3.541249933226383,0.64,comp,"[-0.1990396, 487.7806]" +bn140129499,6.00091243975545e-06,7.740566610162724,0.128,plaw,[1.372511] +bn180130744,2.9596679303128704e-06,6.459821474339156,0.256,plaw,[1.52996] +bn141124277,4.133184598594645e-06,5.536193700166678,0.512,plaw,[1.383942] +bn110801335,3.5642652082921073e-06,11.966718203097102,0.384,plaw,[1.659831] +bn160726065,2.226868193360931e-06,10.952245398110485,0.768,comp,"[-0.8683226, 365.0328]" +bn081226044,1.7553940051605527e-06,5.722835248927925,0.832,comp,"[-0.8072595, 583.4851]" +bn120210650,5.617866943517487e-07,3.7000107904811848,1.344,plaw,[1.880428] +bn100805300,1.4257415130331756e-05,28.326649555826062,0.064,plaw,[1.501964] +bn131006367,6.130195954927256e-06,5.778729668687307,0.128,plaw,[1.275394] +bn180511437,4.173414344932197e-07,4.936501443540984,1.984,comp,"[-0.9226517, 107.5333]" +bn130716442,1.326838910624794e-05,5.694522967051237,0.768,plaw,[0.9977308] +bn081105614,1.2541252307451164e-05,9.200848932045211,1.28,plaw,[1.193876] +bn081204517,1.5107822673263308e-06,7.716538109349071,0.192,comp,"[-0.657518, 299.4318]" +bn101224227,4.0020859447075936e-06,5.9838418662234485,1.728,plaw,[1.417095] +bn080831053,3.902537238730734e-06,5.571610226271248,0.576,plaw,[1.403205] +bn141011282,6.65527536932161e-06,18.45352308285459,0.08,comp,"[-0.6444664, 614.1506]" +bn090927422,2.3778258406910913e-06,5.197689796354085,0.512,plaw,[1.530408] +bn150118927,4.841256460035596e-06,17.278630242911465,0.288,comp,"[-1.041165, 672.6705]" +bn150412931,1.6894647498026266e-06,4.079542323380114,0.64,comp,"[-0.2993239, 570.0925]" +bn080723913,9.435581515444458e-07,4.412365839451427,0.192,comp,"[-0.5945316, 320.8395]" +bn130325005,2.8446116642254787e-06,7.269031848748188,0.64,plaw,[1.577079] +bn120811014,1.1562939323576781e-05,13.087020313403075,0.448,comp,"[-0.2913616, 1261.286]" +bn111001804,3.4847841473201643e-06,4.263837866430446,0.384,plaw,[1.356429] +bn091224373,8.75651827187935e-06,6.67300678602336,0.768,plaw,[1.20653] +bn090617208,5.620805752401871e-06,11.837152280645057,0.192,comp,"[-0.3540218, 681.0817]" +bn090405663,3.978019090773316e-06,7.6247517451912605,0.448,comp,"[-0.6396907, 931.821]" +bn160411062,2.4854700500197835e-06,6.067170876885757,0.672,plaw,[1.56337] +bn120609580,1.8480854802320165e-06,4.676272222115803,1.792,plaw,[1.574127] +bn101216721,9.366006479919954e-07,8.637438120407946,1.917,comp,"[-0.8634751, 152.902]" +bn170506169,4.166133580475353e-06,7.866750494100488,0.832,comp,"[-0.6169903, 928.6597]" diff --git a/cosipy/plots/plot_aitoff.py b/cosipy/plots/plot_aitoff.py new file mode 100644 index 0000000..22a04d3 --- /dev/null +++ b/cosipy/plots/plot_aitoff.py @@ -0,0 +1,285 @@ +import argparse +import os +import pickle +import numpy as np +import healpy as hp +import matplotlib.pyplot as plt +from scipy.stats import chi2 + +""" +Aitoff projection utility for HEALPix maps. + +If --plot-mask=1, the input filename must end with the following numeric +suffix to encode geometry and chi2 thresholding info: + + _{theta_real}_{phi_real}_{theta_reco}_{phi_reco}_{min_chi2}.pkl + +Example filename: + areas_chi2_run57_mix_mega_shared_2438_51.318_28.828_55.771_28.125_0.5221.pkl + +Notes: +- Angles are in degrees. Phi values in [0, 360) are converted to [-180, 180] for plotting. +- Maps are assumed to be in NESTED ordering. +""" + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Plot HEALPix map in Aitoff projection with optional chi2 masking" + ) + parser.add_argument( + "--input", + "-i", + required=True, + help="Path to input pickle file containing a 1D HEALPix map", + ) + parser.add_argument( + "--plot-mask", + type=int, + default=0, + help="If 1, mask values >= min_chi2 + chi2.ppf(0.9,2) parsed from filename", + ) + parser.add_argument( + "--title", + default=None, + help="Figure title; defaults to the input filename", + ) + parser.add_argument( + "--downsample-nside", + type=int, + default=0, + help="If >0, downsample the map to this nside and plot a second panel", + ) + parser.add_argument( + "--coord", + default="G", + help="Coordinate system passed to healpy (e.g., G)", + ) + parser.add_argument( + "--cmap", + default="turbo", + help="Colormap for the base map (e.g., turbo, viridis)", + ) + parser.add_argument( + "--mask-cmap", + default="viridis", + help="Colormap for the masked chi2 map", + ) + parser.add_argument( + "--cbar-label", + default="", + help="Colorbar label for the base map", + ) + parser.add_argument( + "--downsample-cbar-label", + default=None, + help="Colorbar label for the downsampled map (defaults to --cbar-label)", + ) + parser.add_argument( + "--save", + default=None, + help=( + "If set, save figure(s) to this path. If downsampling or mask is requested, " + "suffixes _nside and _masked are appended to the base name." + ), + ) + return parser.parse_args() + + +def read_map(path: str) -> np.ndarray: + with open(path, "rb") as f: + m = pickle.load(f) + return np.array(m) + + +def _set_colorbar_label(label: str) -> None: + if not label: + return + fig = plt.gcf() + axes = fig.get_axes() + main_ax = plt.gca() + for ax in axes: + if ax is not main_ax: + ax.set_xlabel(label, fontsize=14) + break + + +def plot_aitoff( + m: np.ndarray, + title: str, + cmap: str, + coord: str, + cbar_label: str, + badcolor: str = None, +) -> None: + hp.projview( + m, + coord=[coord], + projection_type="aitoff", + graticule=True, + graticule_labels=True, + longitude_grid_spacing=60, + latitude_grid_spacing=30, + title=title, + cmap=cmap, + nest=True, + unit="", + badcolor=badcolor if badcolor else None, + fontsize={ + "xlabel": 14, + "ylabel": 14, + "title": 16, + "xtick_label": 14, + "ytick_label": 14, + "cbar_label": 14, + "cbar_tick_label": 14, + }, + override_plot_properties={ + "cbar_shrink": 0.8, + "cbar_pad": 0.05, + "cbar_label_pad": 5, + }, + ) + # Improve x tick contrast and set colorbar label + ax = plt.gca() + ax.tick_params(axis="x", colors="white") + _set_colorbar_label(cbar_label) + + +def parse_filename_for_mask(basename: str): + """ + Parse theta/phi real/reco and min_chi2 from the filename suffix. + + Expected pattern: + ..._{theta_real}_{phi_real}_{theta_reco}_{phi_reco}_{min_chi2}.pkl + """ + stem = os.path.basename(basename) + if not stem.endswith(".pkl"): + raise ValueError("Input must be a .pkl file when --plot-mask=1") + tokens = stem[:-4].split("_") + if len(tokens) < 5: + raise ValueError( + "Filename must end with _thetaReal_phiReal_thetaReco_phiReco_minChi2.pkl" + ) + try: + min_chi2 = float(tokens[-1]) + theta_real = float(tokens[-5]) + phi_real = float(tokens[-4]) + theta_reco = float(tokens[-3]) + phi_reco = float(tokens[-2]) + except Exception as ex: + raise ValueError("Could not parse numeric values from filename suffix") from ex + + # Map phi from [0, 360) to [-180, 180] for display + if phi_real > 180: + phi_real -= 360 + if phi_reco > 180: + phi_reco -= 360 + return theta_real, phi_real, theta_reco, phi_reco, min_chi2 + + +def plot_masked( + m: np.ndarray, + title: str, + cmap: str, + coord: str, + input_path: str, +) -> None: + theta_real, phi_real, theta_reco, phi_reco, min_chi2 = parse_filename_for_mask( + input_path + ) + # Mask values outside the 90% c.l. region for 2 d.o.f. + limit = min_chi2 + chi2.ppf(0.9, 2) + masked = np.ma.masked_where(m >= limit, m) + + plot_aitoff( + masked, + title=title, + cmap=cmap, + coord=coord, + cbar_label="$\\chi^2$ value (90% c.l.)", + badcolor="antiquewhite", + ) + + # Overlay true (magenta star) and reconstructed (red X) directions + hp.newprojplot( + theta=np.radians(theta_real), + phi=np.radians(phi_real), + marker="*", + color="magenta", + markersize=13, + ) + hp.newprojplot( + theta=np.radians(theta_reco), + phi=np.radians(phi_reco), + marker="X", + color="red", + markersize=12, + ) + + +def main() -> None: + args = parse_args() + + title = args.title or os.path.basename(args.input) + m = read_map(args.input) + + # Base map + plot_aitoff( + m, + title=title, + cmap=args.cmap, + coord=args.coord, + cbar_label=args.cbar_label, + ) + if args.save: + base, _ = os.path.splitext(args.save) + plt.savefig(f"{base}.png", dpi=200, bbox_inches="tight") + + # Optional downsampled map + if args.downsample_nside and args.downsample_nside > 0: + m_down = hp.ud_grade( + m, + nside_out=args.downsample_nside, + order_in="NESTED", + order_out="NESTED", + power=0, + ) + plot_aitoff( + m_down, + title=f"{title} (nside={args.downsample_nside})", + cmap=args.cmap, + coord=args.coord, + cbar_label=( + args.downsample_cbar_label + if args.downsample_cbar_label is not None + else args.cbar_label + ), + ) + if args.save: + base, _ = os.path.splitext(args.save) + plt.savefig( + f"{base}_nside{args.downsample_nside}.png", + dpi=200, + bbox_inches="tight", + ) + + # Optional masked chi2 map (requires filename suffix pattern) + if args.plot_mask == 1: + plot_masked( + m, + title=f"{title} (masked 90% c.l.)", + cmap=args.mask_cmap, + coord=args.coord, + input_path=args.input, + ) + if args.save: + base, _ = os.path.splitext(args.save) + plt.savefig(f"{base}_masked.png", dpi=200, bbox_inches="tight") + + if not args.save: + plt.show() + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 0000000..a08ac64 --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,105 @@ +# Dockerfile for COSItools +# +# Build the docker with: +# docker build -t cosi-main - < Dockerfile + +FROM ubuntu:22.04 + +MAINTAINER Andreas Zoglauer + +# Install all the COSItools prerequisites +RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -yq gosu vim nano less gzip git git-lfs gawk dpkg-dev make g++ gcc gfortran gdb valgrind binutils libx11-dev libxpm-dev libxft-dev libxext-dev libssl-dev libpcre3-dev libglu1-mesa-dev libglew-dev libftgl-dev libmysqlclient-dev libfftw3-dev libgraphviz-dev libavahi-compat-libdnssd-dev libldap2-dev python3 python3-pip python3-dev python3-tk python3-venv libxml2-dev libkrb5-dev libgsl-dev cmake libxmu-dev curl doxygen libblas-dev liblapack-dev expect dos2unix libncurses5-dev libboost-all-dev libcfitsio-dev mlocate libxerces-c-dev libhealpix-cxx-dev libbz2-dev wget + +# Add COSI user +RUN groupadd -g 1111 cosi && useradd -u 1111 -g 1111 -ms /bin/bash cosi +#RUN ln -s /usr/bin/python3 /usr/bin/python + +# Switch to user cosi +USER cosi + +# Setup +#RUN cd /home/cosi && /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/cositools/cosi-setup/main/setup.sh)" +#RUN echo . /home/cosi/COSItools/source.sh >> /home/cosi/.bashrc +#switch megalib branch +# === MEGAlib === + +# Needs a lot of memory. Check your docker preferences if it fails + +RUN cd /home/cosi/ && git clone https://github.com/zoglauer/megalib && cd megalib && git checkout develop-cosi + +#COPY build-root.sh $MEGALIBDIR/config/build-root.sh + +RUN cd /home/cosi/megalib && \ + bash config/configure_packages && \ + bash setup.sh --branch=develop-cosi && \ + echo . /home/cosi/megalib/bin/source-megalib.sh >> $HOME/.bashrc + +#ENV PATH="/root/.local/bin:${PATH}" + +RUN cd /home/cosi && python3 -m venv bct + +RUN cd /home/cosi \ + && git clone https://gitlab.com/burstcube/bc-tools.git \ + && . bct/bin/activate \ + && cd bc-tools && git checkout develop && pip install . && cd ..\ + && wget https://fermi.gsfc.nasa.gov/ssc/data/analysis/gbm/gbm_data_tools/gbm_data_tools-1.1.1.tar.gz \ + && python3 -m pip install basemap ipykernel "matplotlib>=3.7.1" \ + && pip3 install ./gbm_data_tools-1.1.1.tar.gz \ + && python -m ipykernel install --user --name=bct \ + && sed -i 's/trapz/trapezoid/g' bct/lib/python3.10/site-packages/gdt/core/data_primitives.py + + +RUN cd /home/cosi && python3 -m venv deeplearning + +RUN cd /home/cosi \ + . deeplearning/bin/activate \ + && python3 -m pip install tensorflow[and-cuda]==2.19.0 keras matplotlib scipy plotly mhealpy pandas astropy scikit-learn seaborn ipykernel \ + && python3 -m ipykernel install --user --name=deeplearning + +RUN cd /home/cosi && python3 -m venv cosipy + +RUN cd /home/cosi \ + . cosipy/bin/activate \ + && python3 -m pip install --use-pep517 cosipy \ + && python3 -m pip install ipykernel \ + && python3 -m ipykernel install --user --name=cosipy + + +RUN pip3 install --user pytest-cov + +RUN pip3 install --user sphinx sphinx_rtd_theme + +RUN pip3 install --user mhealpy + +#To build tutorials +RUN pip3 install --user nbsphinx jupyter_client ipykernel +#ADD --chown=user:user https://github.com/jgm/pandoc/releases/download/2.9.2.1/pandoc-2.9.2.1-linux-amd64.tar.gz $HOME +RUN cd $HOME && wget https://github.com/jgm/pandoc/releases/download/2.9.2.1/pandoc-2.9.2.1-linux-amd64.tar.gz +RUN tar xvzf $HOME/pandoc-2.9.2.1-linux-amd64.tar.gz --strip-components 1 -C $HOME/.local + +# For full instrument response +RUN pip3 install --user h5py + +#RUN cd /home/cosi && git clone https://github.com/cositools/grb-simulations.git + + +# Switch back to ROOT +USER root + +# For full instrument response +RUN pip3 install notebook pandas + +# Create entry-point script - changes the UID of cosi to the local USER and group for full access to the exchange directory on all machines +RUN cd /usr/local/bin \ + && echo '#!/bin/bash' >> entrypoint.sh \ + && echo 'if [ "${USERID}" != "" ]; then usermod -u ${USERID} cosi; fi' >> entrypoint.sh \ + && echo 'if [ "${GROUPID}" != "" ]; then groupmod -g ${GROUPID} cosi; fi' >> entrypoint.sh \ + && echo 'if [ "${USERID}" != "" ] || [ "${GROUPID}" != "" ]; then chown -R cosi:cosi /home/cosi; fi' >> entrypoint.sh \ + && echo ". /home/cosi/megalib/bin/source-megalib.sh" >> entrypoint.sh \ + && echo 'gosu cosi tail -f /dev/null' >> entrypoint.sh \ + && chmod a+rx /usr/local/bin/entrypoint.sh + +# The working directory is the COSItools directory +WORKDIR /home/cosi + +ENTRYPOINT ["/usr/local/bin/entrypoint.sh"] \ No newline at end of file diff --git a/docker/README.md b/docker/README.md new file mode 100644 index 0000000..efbc23a --- /dev/null +++ b/docker/README.md @@ -0,0 +1,76 @@ +### COSI Docker for GRB simulations and BCtools IRF + +This Docker setup provides a reproducible environment to run COSI GRB simulations (COSIMa) and post-processing tools (BCtools). It includes a non-root user `cosi` and mounts host folders for source code and data. + +### Environment variables (recommended) + +Define these variables in your shell to customize host and container paths: + +```bash +export HOST_COSI_CODE="$HOME/cosi-grb" # host folder containing this repository +export HOST_COSI_DATA_PATH="$HOME/cosi-grb/data" # host folder containing data/models/outputs +export CONTAINER_COSI_CODE="/home/cosi/cosi" # container path where code is mounted +export PORT=8888 # port for Jupyter (customize) +``` + +Notes: +- You may change `HOST_COSI_CODE` and `HOST_COSI_DATA_PATH`, but you must install/clone this repository under `HOST_COSI_CODE` because it is mounted into the container at `CONTAINER_COSI_CODE`. +- Scripts in this repo expect geometry/models under the container path `/data/...`; therefore keep your models under `HOST_COSI_DATA_PATH` so they are visible in the container as `/data/...`. + +### Build the Docker image + +```bash +cd cosidl/docker +docker build -t cosi_grb_bgo:v1.1.0 . +``` + +### Bootstrap the image for your user + +```bash +cd .. +sh bootstrap.sh cosi_grb_bgo:v1.1.0 +``` + +What bootstrap does: +- Re-tags the base image temporarily and builds a new image `cosi_grb_bgo:v1.1.0_`. +- Aligns the `cosi` user inside the image to your host UID/GID, so files created in mounted volumes are owned by you on the host. + +### Run the container + +```bash +docker run -d --entrypoint="" \ + -v "$HOST_COSI_CODE":"$CONTAINER_COSI_CODE" \ + -v "$HOST_COSI_DATA_PATH":"/data" \ + --name cosi_grb_bgo_container \ + -p $PORT:$PORT \ + cosi_grb_bgo:v1.1.0_ \ + tail -f /dev/null +``` + +### Enter the container + +```bash +docker exec --user cosi -it cosi_grb_bgo_container bash +``` + +### Start Jupyter Notebook in the container + +```bash +export CONTAINER_COSI_CODE="/home/cosi/cosi" # container path where code is mounted +export PORT=8888 +cd "$CONTAINER_COSI_CODE" +nohup jupyter notebook --ip=* --port=$PORT --NotebookApp.token='' --no-browser & +``` + +### SSH port forwarding from your local machine + +```bash +ssh -N -L $PORT:localhost:$PORT @ +``` + +### Tips and troubleshooting + +- If you change the mount points, update both the host and container paths consistently. The scripts expect geometry paths like `/data/models/.../*.geo.setup` inside the container. +- Use a unique container name if you run multiple instances. +- If you see file permission issues on the host, ensure you ran the bootstrap step so container user IDs match your host. +- Keep `HOST_COSI_DATA_PATH` on a fast disk if possible; it stores large simulation outputs. diff --git a/docker/bootstrap.sh b/docker/bootstrap.sh new file mode 100644 index 0000000..56c8745 --- /dev/null +++ b/docker/bootstrap.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +: ' +This script is meant to be used to bootstrap the cosi docker container. +' + +set -u +set -v +set -e + +if [ $# -ne 2 ]; then + printf "\n\33[31m > bootstrap.sh \33[0m\n\n" +else + + MY_UID="$(id -u)" + MY_GID="$(id -g)" + SDE_TAG=$1 + INTERMEDIATE_TAG=local/delete_me_soon:latest + USERNAME=$2 + + docker tag $SDE_TAG $INTERMEDIATE_TAG + docker build \ + --tag $SDE_TAG\_$USERNAME \ + - <=61"] build-backend = "setuptools.build_meta" [project] name = "cosipy.nonimaging" dynamic = ["version"] -dependencies = [ - "cosipy", - ] +dependencies = ["cosipy"] description = "cosipy extension to analyze COSI's non-imaging data: BGO, BTO and Ge's single-site events" -authors = [ - {name = 'COSI Team', email = 'imc@umd.edu'}, - ] +authors = [{ name = "COSI Team", email = "imc@umd.edu" }] [tool.setuptools.dynamic] version = { attr = "cosipy.nonimaging.__version__" } +[tool.setuptools.packages.find] +include = ["cosipy.nonimaging*"] +exclude = ["docker*"] \ No newline at end of file