From de0d8485b40efe52c0d7c8ef562dc4d86eba18bf Mon Sep 17 00:00:00 2001 From: Christoph Deil Date: Tue, 8 Jan 2019 18:34:44 +0100 Subject: [PATCH 1/5] Support RGB in HEALpix -> HiPS --- hips/draw/healpix.py | 45 ++++++++++-- hips/draw/tests/test_healpix.py | 117 ++++++++++++++++++++++---------- 2 files changed, 120 insertions(+), 42 deletions(-) diff --git a/hips/draw/healpix.py b/hips/draw/healpix.py index 00619ee..94995ca 100644 --- a/hips/draw/healpix.py +++ b/hips/draw/healpix.py @@ -12,7 +12,11 @@ def healpix_to_hips_tile( - hpx_data: np.ndarray, tile_width: int, tile_idx: int, file_format: str, frame: str + hpx_data: np.ndarray, + tile_width: int, + tile_idx: int, + file_format: str, + frame: str = "icrs", ) -> HipsTile: """Create single HiPS tile from HEALPix data. @@ -39,13 +43,38 @@ def healpix_to_hips_tile( offset_ipix = tile_idx * tile_width ** 2 ipix = hpx_ipix + offset_ipix - data = hpx_data[ipix] + + hpx_data = np.asarray(hpx_data) + if file_format == "fits": + if hpx_data.ndim != 1: + raise ValueError( + f"Invalid hpx_data.ndim = {hpx_data.ndim}." + " Must be ndim = 1 for file_format='fits'." + ) + data = hpx_data[ipix] + elif file_format == "jpg": + if hpx_data.ndim != 2 or hpx_data.shape[1] != 3: + raise ValueError( + f"Invalid hpx_data.shape = {hpx_data.shape}." + " Must be shape = (npix, 3) to represent RGB color images." + ) + data = hpx_data[ipix, :] + elif file_format == "png": + if hpx_data.ndim != 2 or hpx_data.shape[1] != 4: + raise ValueError( + f"Invalid hpx_data.shape = {hpx_data.shape}." + " Must be shape = (npix, 4) to represent RGBA color images." + ) + data = hpx_data[ipix, :] + else: + raise ValueError(f"Invalid file_format: {file_format!r}") # np.rot90 returns a rotated view so we make a copy here # because the view information is lost on fits io data = np.rot90(data).copy() - hpx_nside = hp.npix2nside(hpx_data.size / tile_width ** 2) + hpx_npix = hpx_data.shape[0] + hpx_nside = hp.npix2nside(hpx_npix / tile_width ** 2) hpx_order = int(np.log2(hpx_nside)) meta = HipsTileMeta( @@ -59,7 +88,13 @@ def healpix_to_hips_tile( return HipsTile.from_numpy(meta=meta, data=data) -def healpix_to_hips(hpx_data, tile_width, base_path, file_format, frame): +def healpix_to_hips( + hpx_data: np.ndarray, + tile_width: int, + base_path: str, + file_format: str, + frame: str = "icrs", +): """Convert HEALPix image to HiPS. This function writes the HiPS to disk. @@ -91,7 +126,7 @@ def healpix_to_hips(hpx_data, tile_width, base_path, file_format, frame): } ).write(path) - n_tiles = hpx_data.size // tile_width ** 2 + n_tiles = hpx_data.shape[0] // tile_width ** 2 for tile_idx in range(n_tiles): tile = healpix_to_hips_tile( diff --git a/hips/draw/tests/test_healpix.py b/hips/draw/tests/test_healpix.py index 0a16ab4..040ff8e 100644 --- a/hips/draw/tests/test_healpix.py +++ b/hips/draw/tests/test_healpix.py @@ -1,61 +1,104 @@ # Licensed under a 3-clause BSD style license - see LICENSE.rst import pytest import numpy as np -from PIL import Image -from astropy.io import fits -from numpy.testing import assert_allclose, assert_equal +from numpy.testing import assert_equal from astropy_healpix import healpy as hp +from ...tiles import HipsTile, HipsTileMeta from ..healpix import healpix_to_hips, healpix_to_hips_tile -def test_healpix_to_hips_tile(): - nside, tile_width = 4, 2 - npix = hp.nside2npix(nside) - hpx_data = np.arange(npix, dtype="uint8") +def make_hpx_data(file_format): + npix = hp.nside2npix(4) + data = np.arange(npix, dtype="uint8") + + if file_format == "fits": + return data + elif file_format == "jpg": + return np.moveaxis([data, data + 1, data + 2], 0, -1) + elif file_format == "png": + return np.moveaxis([data, data + 1, data + 2, data + 3], 0, -1) + else: + raise ValueError() + + +def test_healpix_to_hips_tile_fits(): + hpx_data = make_hpx_data("fits") + tile = healpix_to_hips_tile( - hpx_data=hpx_data, - tile_width=tile_width, - tile_idx=0, - file_format="fits", - frame="galactic", + hpx_data=hpx_data, tile_width=2, tile_idx=0, file_format="fits" ) - assert_equal(tile.data, [[1, 3], [0, 2]]) assert tile.meta.order == 1 assert tile.meta.ipix == 0 assert tile.meta.file_format == "fits" - assert tile.meta.frame == "galactic" + assert tile.meta.frame == "icrs" + assert tile.meta.width == 2 + + assert tile.data.dtype == np.uint8 + assert tile.data.shape == (2, 2) + assert_equal(tile.data, [[1, 3], [0, 2]]) + + +def test_healpix_to_hips_tile_jpg(): + hpx_data = make_hpx_data("jpg") + + tile = healpix_to_hips_tile( + hpx_data=hpx_data, tile_width=2, tile_idx=0, file_format="jpg" + ) + + assert tile.meta.order == 1 + assert tile.meta.ipix == 0 + assert tile.meta.file_format == "jpg" assert tile.meta.width == 2 + assert tile.data.dtype == np.uint8 + assert tile.data.shape == (2, 2, 3) + + # Note: we don't assert on the tile data for JPEG, + # because JPEG encoding noise is large and results + # can vary depending on JPEG lib and machine used. + + +def test_healpix_to_hips_tile_png(): + hpx_data = make_hpx_data("png") + + tile = healpix_to_hips_tile( + hpx_data=hpx_data, tile_width=2, tile_idx=0, file_format="png" + ) + + assert tile.meta.order == 1 + assert tile.meta.ipix == 0 + assert tile.meta.file_format == "png" + assert tile.meta.frame == "icrs" + assert tile.meta.width == 2 + + assert tile.data.dtype == np.uint8 + assert tile.data.shape == (2, 2, 4) + assert_equal(tile.data[..., 0], [[1, 3], [0, 2]]) + @pytest.mark.parametrize("file_format", ["fits", "png"]) def test_healpix_to_hips(tmpdir, file_format): - nside, tile_width = 4, 2 - npix = hp.nside2npix(nside) - hpx_data = np.arange(npix, dtype="uint8") + hpx_data = make_hpx_data(file_format) healpix_to_hips( - hpx_data=hpx_data, - tile_width=tile_width, - base_path=tmpdir, - file_format=file_format, - frame="galactic", + hpx_data=hpx_data, tile_width=2, base_path=tmpdir, file_format=file_format ) - # The test data is filled with np.arange(), here we reproduce the sum of the - # indices in the nested scheme manually for comparison - desired = hpx_data.reshape((-1, tile_width, tile_width)) - - for idx, val in enumerate(desired): - filename = str(tmpdir / f"Norder1/Dir0/Npix{idx}.{file_format}") - if file_format is "fits": - data = fits.getdata(filename) - data = np.rot90(data, k=-1) - else: - data = np.array(Image.open(filename)) - data = data.T - assert_allclose(val, data) - properties = (tmpdir / "properties").read_text(encoding=None) assert file_format in properties - assert "galactic" in properties + assert "icrs" in properties + + # Check one tile + filename = str(tmpdir / f"Norder1/Dir0/Npix2.{file_format}") + meta = HipsTileMeta(order=1, ipix=2, file_format=file_format, width=2) + tile = HipsTile.read(meta, filename) + + if file_format == "fits": + assert tile.data.dtype == np.uint8 + assert tile.data.shape == (2, 2) + assert_equal(tile.data, [[9, 11], [8, 10]]) + elif file_format == "png": + assert tile.data.dtype == np.uint8 + assert tile.data.shape == (2, 2, 4) + assert_equal(tile.data[..., 0], [[9, 11], [8, 10]]) From da470d192573733b3aacec640ba27abf083c987e Mon Sep 17 00:00:00 2001 From: Christoph Deil Date: Tue, 8 Jan 2019 21:33:59 +0100 Subject: [PATCH 2/5] Improve healpix_to_hips and add docs --- docs/hipsgen.py | 27 ++++++++++ docs/hipsgen.rst | 72 +++++++++++++++++++++++++ docs/index.rst | 1 + hips/draw/__init__.py | 1 + hips/draw/healpix.py | 45 +++++----------- hips/draw/hipsgen.py | 93 +++++++++++++++++++++++++++++++++ hips/draw/tests/test_healpix.py | 31 +---------- hips/draw/tests/test_hipsgen.py | 34 ++++++++++++ 8 files changed, 242 insertions(+), 62 deletions(-) create mode 100644 docs/hipsgen.py create mode 100644 docs/hipsgen.rst create mode 100644 hips/draw/hipsgen.py create mode 100644 hips/draw/tests/test_hipsgen.py diff --git a/docs/hipsgen.py b/docs/hipsgen.py new file mode 100644 index 0000000..9194cfd --- /dev/null +++ b/docs/hipsgen.py @@ -0,0 +1,27 @@ +"""Example how to generate HiPS from HEALPix. + +We will use +""" +import logging +import numpy as np +import hips +from astropy.coordinates import SkyCoord +from astropy_healpix import HEALPix + + +def make_healpix_data(): + """Silly example of HEALPix data. + Angular distance to + """ + healpix = HEALPix(nside=4) + ipix = np.arange(healpix.npix) + lon, lat = healpix.healpix_to_lonlat(ipix) + coord = SkyCoord(lon, lat) + center = SkyCoord(0, 0, unit='deg') + return coord.separation(center).deg + + +if __name__ == '__main__': + logging.basicConfig(level=logging.INFO) + data = make_healpix_data() + hips.healpix_to_hips(data, tile_width=4, base_path='test123', file_format='fits') diff --git a/docs/hipsgen.rst b/docs/hipsgen.rst new file mode 100644 index 0000000..663c1f0 --- /dev/null +++ b/docs/hipsgen.rst @@ -0,0 +1,72 @@ +.. include:: references.txt + +.. _hipsgen: + +************* +Generate HiPS +************* + +The `~hips.healpix_to_hips` function can be used to generate HiPS data from all-sky HEALPix images. +We give an example below. + +Note that this functionality is very limited, and for most applications you will want to use the +Java ``hipsgen`` tool or the graphical user interface to ``hipsgen`` from the Aladin desktop application. + +To use `~hips.healpix_to_hips` you have to pass in the all-sky HEALPix data as a Numpy array +(so it has to completely fit into memory). For grayscale images, the HEALPix data array is +one-dimensional, RGB images with array shape ``(npix, 3)`` can be converted to JPEG HiPS tiles, +and RGBA (where A is the transparency channel) images with array shape ``(npix, 4)`` can be +converted to PNG HiPS tiles. + +Some data is distributed directly in HEALPix format, e.g. from all-sky surveys like Planck. +Some data from high-energy telescopes like Fermi-LAT consists of event lists can be used +to compute HEALPix images. And WCS-based images can also be reprojected to HEALPix images +using e.g. `reproject `__. +For these use cases `~hips.healpix_to_hips` can be used for the final all-sky HEALPix +map to HiPS conversion step, giving you full flexibility over the processing and pixel +color scales etc from Python in the pre-processing steps. + +Limitations +----------- + +Currently the following limitations exist: + +- No mosaic functionality from WCS images + (a lot of work to implement) +- Only one HiPS resolution is generated at the moment + (not hard to write a script to make multiple resolutions) +- No ``Allsky.fits`` generated yet + (shouldn't be hard to generate) + +Dataset +======= + +bla bla + +Example +======= + +Generate HiPS: + +.. code-block:: bash + + $ python docs/hipsgen.py + +Open HiPS in Aladin Lite: + +.. code-block:: bash + + $ cd test123 + $ python -m http.server + # Then open http://localhost:8000/ in your webbrowser + +Or open HiPS in Aladin Desktop: + +.. code-block:: bash + + $ TODO: how to load the folder in Aladin + +Here's the ``hipsgen.py`` script: + +.. literalinclude:: hipsgen.py + diff --git a/docs/index.rst b/docs/index.rst index f152c9b..01a7d52 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -15,6 +15,7 @@ A Python astronomy package for HiPS : Hierarchical Progressive Surveys. about installation getting_started + hipsgen api changelog diff --git a/hips/draw/__init__.py b/hips/draw/__init__.py index 14575cd..fcb29b8 100644 --- a/hips/draw/__init__.py +++ b/hips/draw/__init__.py @@ -3,3 +3,4 @@ from .paint import * from .ui import * from .healpix import * +from .hipsgen import * diff --git a/hips/draw/healpix.py b/hips/draw/healpix.py index 94995ca..878d848 100644 --- a/hips/draw/healpix.py +++ b/hips/draw/healpix.py @@ -1,12 +1,12 @@ # Licensed under a 3-clause BSD style license - see LICENSE.rst +from typing import List import logging -from pathlib import Path import numpy as np from astropy_healpix import healpy as hp -from ..tiles import HipsTile, HipsTileMeta, HipsSurveyProperties +from ..tiles import HipsTile, HipsTileMeta from ..utils.healpix import hips_tile_healpix_ipix_array -__all__ = ["healpix_to_hips_tile", "healpix_to_hips"] +__all__ = ["healpix_to_hips_tile", "healpix_to_hips_tiles"] log = logging.getLogger(__name__) @@ -88,56 +88,37 @@ def healpix_to_hips_tile( return HipsTile.from_numpy(meta=meta, data=data) -def healpix_to_hips( +def healpix_to_hips_tiles( hpx_data: np.ndarray, tile_width: int, - base_path: str, file_format: str, frame: str = "icrs", -): - """Convert HEALPix image to HiPS. - - This function writes the HiPS to disk. - If you don't want that, use `healpix_to_hips_tile` directly. +) -> List[HipsTile]: + """Convert HEALPix image to HiPS tiles, Parameters ---------- hpx_data : `~numpy.ndarray` - Healpix data stored in the "nested" scheme. + HEALPix data stored in the "nested" scheme. tile_width : int Width of the hips tiles. - base_path : str or `~pathlib.Path` - Base path. file_format : {'fits', 'jpg', 'png'} HiPS tile file format frame : {'icrs', 'galactic', 'ecliptic'} Sky coordinate frame - """ - base_path = Path(base_path) - base_path.mkdir(exist_ok=True, parents=True) - - path = base_path / "properties" - log.info(f"Writing {path}") - HipsSurveyProperties( - { - "hips_tile_format": file_format, - "hips_tile_width": tile_width, - "hips_frame": frame, - } - ).write(path) + Returns + ------- + tiles : Generator of HipsTile + HiPS tiles + """ n_tiles = hpx_data.shape[0] // tile_width ** 2 for tile_idx in range(n_tiles): - tile = healpix_to_hips_tile( + yield healpix_to_hips_tile( hpx_data=hpx_data, tile_width=tile_width, tile_idx=tile_idx, file_format=file_format, frame=frame, ) - - path = base_path / tile.meta.tile_default_path - log.info(f"Writing {path}") - path.parent.mkdir(exist_ok=True, parents=True) - tile.write(path) diff --git a/hips/draw/hipsgen.py b/hips/draw/hipsgen.py new file mode 100644 index 0000000..77cd826 --- /dev/null +++ b/hips/draw/hipsgen.py @@ -0,0 +1,93 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst +import logging +import numpy as np +from ..tiles import HipsSurveyProperties +from .healpix import healpix_to_hips_tiles +from pathlib import Path + +__all__ = ["healpix_to_hips"] + +log = logging.getLogger(__name__) + +HTML_TEMPLATE = r""" + + + + +
+ + +""" + + +# TODO: change to class with each step as a method? +def healpix_to_hips( + hpx_data: np.ndarray, + tile_width: int, + base_path: str, + file_format: str, + frame: str = "icrs", +): + """Convert HEALPix image to HiPS. + + Directly writes files to output folder. + + Parameters + ---------- + hpx_data : `~numpy.ndarray` + Healpix data stored in the "nested" scheme. + tile_width : int + Width of the hips tiles. + base_path : str or `~pathlib.Path` + Base path. + file_format : {'fits', 'jpg', 'png'} + HiPS tile file format + frame : {'icrs', 'galactic', 'ecliptic'} + Sky coordinate frame + """ + # Make and write properties file + base_path = Path(base_path) + base_path.mkdir(exist_ok=True, parents=True) + path = base_path / "properties" + log.info(f"Writing {path}") + HipsSurveyProperties( + { + "hips_tile_format": file_format, + "hips_tile_width": tile_width, + "hips_frame": frame, + } + ).write(path) + + # Make and write index.html + txt = HTML_TEMPLATE.format_map({ + "name": "test123", + "imgFormat": "fits", + }) + path = base_path / "index.html" + log.info(f"Writing {path}") + path.write_text(txt) + + # TODO: make and write Allsky.{file_format} + + # Make and write tiles + for tile in healpix_to_hips_tiles(hpx_data, tile_width, file_format, frame): + path = base_path / tile.meta.tile_default_path + log.info(f"Writing {path}") + path.parent.mkdir(exist_ok=True, parents=True) + tile.write(path) diff --git a/hips/draw/tests/test_healpix.py b/hips/draw/tests/test_healpix.py index 040ff8e..31882cd 100644 --- a/hips/draw/tests/test_healpix.py +++ b/hips/draw/tests/test_healpix.py @@ -1,10 +1,8 @@ # Licensed under a 3-clause BSD style license - see LICENSE.rst -import pytest import numpy as np from numpy.testing import assert_equal from astropy_healpix import healpy as hp -from ...tiles import HipsTile, HipsTileMeta -from ..healpix import healpix_to_hips, healpix_to_hips_tile +from ..healpix import healpix_to_hips_tile def make_hpx_data(file_format): @@ -75,30 +73,3 @@ def test_healpix_to_hips_tile_png(): assert tile.data.dtype == np.uint8 assert tile.data.shape == (2, 2, 4) assert_equal(tile.data[..., 0], [[1, 3], [0, 2]]) - - -@pytest.mark.parametrize("file_format", ["fits", "png"]) -def test_healpix_to_hips(tmpdir, file_format): - hpx_data = make_hpx_data(file_format) - - healpix_to_hips( - hpx_data=hpx_data, tile_width=2, base_path=tmpdir, file_format=file_format - ) - - properties = (tmpdir / "properties").read_text(encoding=None) - assert file_format in properties - assert "icrs" in properties - - # Check one tile - filename = str(tmpdir / f"Norder1/Dir0/Npix2.{file_format}") - meta = HipsTileMeta(order=1, ipix=2, file_format=file_format, width=2) - tile = HipsTile.read(meta, filename) - - if file_format == "fits": - assert tile.data.dtype == np.uint8 - assert tile.data.shape == (2, 2) - assert_equal(tile.data, [[9, 11], [8, 10]]) - elif file_format == "png": - assert tile.data.dtype == np.uint8 - assert tile.data.shape == (2, 2, 4) - assert_equal(tile.data[..., 0], [[9, 11], [8, 10]]) diff --git a/hips/draw/tests/test_hipsgen.py b/hips/draw/tests/test_hipsgen.py new file mode 100644 index 0000000..224d8e7 --- /dev/null +++ b/hips/draw/tests/test_hipsgen.py @@ -0,0 +1,34 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst +import pytest +import numpy as np +from numpy.testing import assert_equal +from ...tiles import HipsTile, HipsTileMeta +from ..hipsgen import healpix_to_hips +from .test_healpix import make_hpx_data + + +@pytest.mark.parametrize("file_format", ["fits", "png"]) +def test_healpix_to_hips(tmpdir, file_format): + hpx_data = make_hpx_data(file_format) + + healpix_to_hips( + hpx_data=hpx_data, tile_width=2, base_path=tmpdir, file_format=file_format + ) + + properties = (tmpdir / "properties").read_text(encoding=None) + assert file_format in properties + assert "icrs" in properties + + # Check one tile + filename = str(tmpdir / f"Norder1/Dir0/Npix2.{file_format}") + meta = HipsTileMeta(order=1, ipix=2, file_format=file_format, width=2) + tile = HipsTile.read(meta, filename) + + if file_format == "fits": + assert tile.data.dtype == np.uint8 + assert tile.data.shape == (2, 2) + assert_equal(tile.data, [[9, 11], [8, 10]]) + elif file_format == "png": + assert tile.data.dtype == np.uint8 + assert tile.data.shape == (2, 2, 4) + assert_equal(tile.data[..., 0], [[9, 11], [8, 10]]) From 92bc8dfabcb2bfb05340d1a590d092d5fd93d2fc Mon Sep 17 00:00:00 2001 From: Christoph Deil Date: Thu, 10 Jan 2019 11:48:53 +0100 Subject: [PATCH 3/5] Remove old resolved TODO in allsky.py --- hips/tiles/allsky.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/hips/tiles/allsky.py b/hips/tiles/allsky.py index 829bbb5..ef9958b 100644 --- a/hips/tiles/allsky.py +++ b/hips/tiles/allsky.py @@ -92,9 +92,6 @@ def from_tiles(cls, tiles: List[HipsTile]) -> 'HipsTileAllskyArray': """Create all-sky image from list of tiles.""" meta = tiles[0].meta.copy() data = cls.tiles_to_allsky_array(tiles) - # TODO: check return type here. - # Pycharm warns that a `HipsTile` is returned here, not a `HipsTileAllskyArray` - # Is this true or a bug in their static code analysis? return cls.from_numpy(meta, data) @staticmethod From 866748ac20b0ab91c7829a34b37e3b288124eba5 Mon Sep 17 00:00:00 2001 From: Christoph Deil Date: Thu, 10 Jan 2019 17:23:52 +0100 Subject: [PATCH 4/5] Improve hipsgen --- docs/hipsgen.py | 30 +++++++++++++---- hips/draw/hipsgen.py | 77 +++++++++++++++++++++++++++++++++++++------- 2 files changed, 89 insertions(+), 18 deletions(-) diff --git a/docs/hipsgen.py b/docs/hipsgen.py index 9194cfd..ae25459 100644 --- a/docs/hipsgen.py +++ b/docs/hipsgen.py @@ -9,19 +9,35 @@ from astropy_healpix import HEALPix -def make_healpix_data(): +def make_healpix_data(file_format): """Silly example of HEALPix data. - Angular distance to + + Angular distance in deg to (0, 0) as pixel values. """ healpix = HEALPix(nside=4) ipix = np.arange(healpix.npix) lon, lat = healpix.healpix_to_lonlat(ipix) coord = SkyCoord(lon, lat) - center = SkyCoord(0, 0, unit='deg') - return coord.separation(center).deg + center = SkyCoord(0, 0, unit="deg") + data = coord.separation(center).deg + + if file_format == "fits": + return data + elif file_format == "jpg": + data = data.astype('uint8') + return np.moveaxis([data, data + 1, data + 2], 0, -1) + elif file_format == "png": + data = data.astype('uint8') + return np.moveaxis([data, data + 1, data + 2, data + 3], 0, -1) + else: + raise ValueError() -if __name__ == '__main__': +if __name__ == "__main__": logging.basicConfig(level=logging.INFO) - data = make_healpix_data() - hips.healpix_to_hips(data, tile_width=4, base_path='test123', file_format='fits') + + file_format = "png" + data = make_healpix_data(file_format) + hips.healpix_to_hips( + data, tile_width=4, base_path="test123", file_format=file_format + ) diff --git a/hips/draw/hipsgen.py b/hips/draw/hipsgen.py index 77cd826..0b07222 100644 --- a/hips/draw/hipsgen.py +++ b/hips/draw/hipsgen.py @@ -1,11 +1,11 @@ # Licensed under a 3-clause BSD style license - see LICENSE.rst import logging import numpy as np -from ..tiles import HipsSurveyProperties +from ..tiles import HipsSurveyProperties, HipsTileAllskyArray, HipsTile, HipsTileMeta from .healpix import healpix_to_hips_tiles from pathlib import Path -__all__ = ["healpix_to_hips"] +__all__ = ["healpix_to_hips", "HipsSurvey"] log = logging.getLogger(__name__) @@ -28,7 +28,7 @@ aladin.setImageSurvey( aladin.createImageSurvey( - '{name}', '{name}', '', 'galactic', 1, {{imgFormat: '{imgFormat}'}} + '{name}', '{name}', '.', 'galactic', 1, {{imgFormat: '{imgFormat}'}} ) ); @@ -66,28 +66,83 @@ def healpix_to_hips( base_path.mkdir(exist_ok=True, parents=True) path = base_path / "properties" log.info(f"Writing {path}") - HipsSurveyProperties( + properties = HipsSurveyProperties( { "hips_tile_format": file_format, "hips_tile_width": tile_width, "hips_frame": frame, } - ).write(path) + ) + properties.write(path) # Make and write index.html - txt = HTML_TEMPLATE.format_map({ - "name": "test123", - "imgFormat": "fits", - }) + txt = HTML_TEMPLATE.format_map({"name": "test123", "imgFormat": "fits"}) path = base_path / "index.html" log.info(f"Writing {path}") path.write_text(txt) - # TODO: make and write Allsky.{file_format} - # Make and write tiles for tile in healpix_to_hips_tiles(hpx_data, tile_width, file_format, frame): path = base_path / tile.meta.tile_default_path log.info(f"Writing {path}") path.parent.mkdir(exist_ok=True, parents=True) tile.write(path) + + # Make and write allsky file + hips_survey = HipsSurvey(base_path) + allsky = hips_survey.make_allsky(order=0, file_format=file_format) + path = base_path / f"Norder0/Allsky.{file_format}" + log.info(f"Writing {path}") + allsky.write(path) + + +# TODO: generalise and move to a better location +class HipsSurvey: + """HiPS survey container. + + Represents one HiPS survey, acting as a manager + to interact with the various pieces that make up a HiPS: + + - One `HipsSurveyProperties` + - Many `HipsTile` + - One `HipsTileAllskyArray` per order + + TODO: do we need several classes, for the different cases, + or can they be handled by one `HipsSurvey` class? + + - in memory + - on local disk + - on server + + Functionality should include: + + - locate, read, write files + - scan and print summary of contents (e.g. which orders / tiles are present) + - copy / clone HiPS (possibly only small parts, with selections) locally and from servers + - generate allsky from tiles? put this functionality here or somewhere else? + + Parameters + ---------- + base_path : `pathlib.Path` + Base path (folder where the ``properties`` file is located) + """ + + def __init__(self, base_path): + self.base_path = base_path + + def make_allsky(self, order: int, file_format: str) -> HipsTileAllskyArray: + tiles = self.read_tiles(order, file_format) + # TODO: make from_tiles work with generators. For now we have to make a list + tiles = list(tiles) + return HipsTileAllskyArray.from_tiles(tiles) + + def read_tiles(self, order: int, file_format: str): + properties = HipsSurveyProperties.read(self.base_path / 'properties') + tile_path = self.base_path / f'Norder{order}' + + for path in tile_path.glob(f"Dir*/Npix*.{file_format}"): + ipix = int(path.as_posix().split('/')[-1].split('.')[0][4:]) + frame = properties.hips_frame + width = properties.tile_width + meta = HipsTileMeta(order, ipix, file_format, frame, width) + yield HipsTile.read(meta, path) From 6df49aa70a72f94b9449d6273b4752a16c5a2c54 Mon Sep 17 00:00:00 2001 From: Christoph Deil Date: Thu, 10 Jan 2019 17:49:00 +0100 Subject: [PATCH 5/5] Improve hipsgen --- docs/hipsgen.py | 21 +++++++++++++++------ hips/draw/healpix.py | 16 +++++++++------- hips/draw/hipsgen.py | 30 ++++++++++++++++++------------ 3 files changed, 42 insertions(+), 25 deletions(-) diff --git a/docs/hipsgen.py b/docs/hipsgen.py index ae25459..d3db83f 100644 --- a/docs/hipsgen.py +++ b/docs/hipsgen.py @@ -9,12 +9,13 @@ from astropy_healpix import HEALPix -def make_healpix_data(file_format): +def make_healpix_data(hips_order, tile_width, file_format): """Silly example of HEALPix data. Angular distance in deg to (0, 0) as pixel values. """ - healpix = HEALPix(nside=4) + nside = tile_width * (2 ** hips_order) + healpix = HEALPix(nside=nside) ipix = np.arange(healpix.npix) lon, lat = healpix.healpix_to_lonlat(ipix) coord = SkyCoord(lon, lat) @@ -24,10 +25,10 @@ def make_healpix_data(file_format): if file_format == "fits": return data elif file_format == "jpg": - data = data.astype('uint8') + data = data.astype("uint8") return np.moveaxis([data, data + 1, data + 2], 0, -1) elif file_format == "png": - data = data.astype('uint8') + data = data.astype("uint8") return np.moveaxis([data, data + 1, data + 2, data + 3], 0, -1) else: raise ValueError() @@ -37,7 +38,15 @@ def make_healpix_data(file_format): logging.basicConfig(level=logging.INFO) file_format = "png" - data = make_healpix_data(file_format) + hips_order = 3 + tile_width = 4 + + hpx_data = make_healpix_data(hips_order, tile_width, file_format) + hips.healpix_to_hips( - data, tile_width=4, base_path="test123", file_format=file_format + hpx_data=hpx_data, + hips_order=hips_order, + tile_width=tile_width, + base_path="test123", + file_format=file_format, ) diff --git a/hips/draw/healpix.py b/hips/draw/healpix.py index 878d848..f4795c1 100644 --- a/hips/draw/healpix.py +++ b/hips/draw/healpix.py @@ -2,7 +2,6 @@ from typing import List import logging import numpy as np -from astropy_healpix import healpy as hp from ..tiles import HipsTile, HipsTileMeta from ..utils.healpix import hips_tile_healpix_ipix_array @@ -13,6 +12,7 @@ def healpix_to_hips_tile( hpx_data: np.ndarray, + hips_order: int, tile_width: int, tile_idx: int, file_format: str, @@ -24,6 +24,8 @@ def healpix_to_hips_tile( ---------- hpx_data : `~numpy.ndarray` Healpix data stored in the "nested" scheme. + hips_order : int + HEALPix order for the HiPS tiles tile_width : int Width of the hips tile. tile_idx : int @@ -73,12 +75,8 @@ def healpix_to_hips_tile( # because the view information is lost on fits io data = np.rot90(data).copy() - hpx_npix = hpx_data.shape[0] - hpx_nside = hp.npix2nside(hpx_npix / tile_width ** 2) - hpx_order = int(np.log2(hpx_nside)) - meta = HipsTileMeta( - order=hpx_order, + order=hips_order, ipix=tile_idx, file_format=file_format, frame=frame, @@ -90,6 +88,7 @@ def healpix_to_hips_tile( def healpix_to_hips_tiles( hpx_data: np.ndarray, + hips_order: int, tile_width: int, file_format: str, frame: str = "icrs", @@ -100,6 +99,8 @@ def healpix_to_hips_tiles( ---------- hpx_data : `~numpy.ndarray` HEALPix data stored in the "nested" scheme. + hips_order : int + HEALPix order for the HiPS tiles tile_width : int Width of the hips tiles. file_format : {'fits', 'jpg', 'png'} @@ -112,11 +113,12 @@ def healpix_to_hips_tiles( tiles : Generator of HipsTile HiPS tiles """ - n_tiles = hpx_data.shape[0] // tile_width ** 2 + n_tiles = 12 * (2 ** hips_order) for tile_idx in range(n_tiles): yield healpix_to_hips_tile( hpx_data=hpx_data, + hips_order=hips_order, tile_width=tile_width, tile_idx=tile_idx, file_format=file_format, diff --git a/hips/draw/hipsgen.py b/hips/draw/hipsgen.py index 0b07222..9413fc2 100644 --- a/hips/draw/hipsgen.py +++ b/hips/draw/hipsgen.py @@ -38,11 +38,12 @@ # TODO: change to class with each step as a method? def healpix_to_hips( - hpx_data: np.ndarray, - tile_width: int, - base_path: str, - file_format: str, - frame: str = "icrs", + hpx_data: np.ndarray, + hips_order: int, + tile_width: int, + base_path: str, + file_format: str, + frame: str = "icrs", ): """Convert HEALPix image to HiPS. @@ -52,6 +53,8 @@ def healpix_to_hips( ---------- hpx_data : `~numpy.ndarray` Healpix data stored in the "nested" scheme. + hips_order : int + HEALPix order for the HiPS tiles tile_width : int Width of the hips tiles. base_path : str or `~pathlib.Path` @@ -70,19 +73,22 @@ def healpix_to_hips( { "hips_tile_format": file_format, "hips_tile_width": tile_width, + "hips_order": hips_order, "hips_frame": frame, } ) properties.write(path) # Make and write index.html - txt = HTML_TEMPLATE.format_map({"name": "test123", "imgFormat": "fits"}) + txt = HTML_TEMPLATE.format_map({"name": "test123", "imgFormat": file_format}) path = base_path / "index.html" log.info(f"Writing {path}") path.write_text(txt) # Make and write tiles - for tile in healpix_to_hips_tiles(hpx_data, tile_width, file_format, frame): + for tile in healpix_to_hips_tiles( + hpx_data, hips_order, tile_width, file_format, frame + ): path = base_path / tile.meta.tile_default_path log.info(f"Writing {path}") path.parent.mkdir(exist_ok=True, parents=True) @@ -90,8 +96,8 @@ def healpix_to_hips( # Make and write allsky file hips_survey = HipsSurvey(base_path) - allsky = hips_survey.make_allsky(order=0, file_format=file_format) - path = base_path / f"Norder0/Allsky.{file_format}" + allsky = hips_survey.make_allsky(order=hips_order, file_format=file_format) + path = base_path / f"Norder{hips_order}/Allsky.{file_format}" log.info(f"Writing {path}") allsky.write(path) @@ -137,11 +143,11 @@ def make_allsky(self, order: int, file_format: str) -> HipsTileAllskyArray: return HipsTileAllskyArray.from_tiles(tiles) def read_tiles(self, order: int, file_format: str): - properties = HipsSurveyProperties.read(self.base_path / 'properties') - tile_path = self.base_path / f'Norder{order}' + properties = HipsSurveyProperties.read(self.base_path / "properties") + tile_path = self.base_path / f"Norder{order}" for path in tile_path.glob(f"Dir*/Npix*.{file_format}"): - ipix = int(path.as_posix().split('/')[-1].split('.')[0][4:]) + ipix = int(path.as_posix().split("/")[-1].split(".")[0][4:]) frame = properties.hips_frame width = properties.tile_width meta = HipsTileMeta(order, ipix, file_format, frame, width)