Merge pull request #64 from desihub/tilefile

enable non-standard tilesfile

py/desimodel/io.py

@@ -121,8 +121,8 @@ def load_fiberpos():
 #
 #
 #
-_tiles = None
-def load_tiles(onlydesi=True, extra=False):
+_tiles = dict()
+def load_tiles(onlydesi=True, extra=False, tilesfile=None, cache=True):
     """Return DESI tiles structure from desimodel/data/footprint/desi-tiles.fits.
 
     Parameters
@@ -131,37 +131,59 @@ def load_tiles(onlydesi=True, extra=False):
         If ``True``, trim to just the tiles in the DESI footprint.
     extra : :class:`bool`, (default False)
         If ``True``, include extra layers with PROGRAM='EXTRA'.
+    tilesfile : (str)
+        Name of tiles file to load; or None for default.
+        Without path, look in $DESIMODEL/data/footprint, otherwise load file.
+    cache : :class:`bool`, (default True)
+        Use cache of tiles data.
     """
     global _tiles
-    if _tiles is None:
-        footprint = os.path.join(os.environ['DESIMODEL'],'data','footprint','desi-tiles.fits')
-        with fits.open(footprint) as hdulist:
-            _tiles = hdulist[1].data
+
+    if tilesfile is None:
+        tilesfile = 'desi-tiles.fits'
+
+    #- Check if tilesfile includes a path (absolute or relative)
+    tilespath, filename = os.path.split(tilesfile)
+    if tilespath == '':
+        tilesfile = os.path.join(os.environ['DESIMODEL'],'data','footprint',filename)
+
+    #- standarize path location
+    tilesfile = os.path.abspath(tilesfile)
+
+    if cache and tilesfile in _tiles:
+        tiledata = _tiles[tilesfile]
+    else:
+        with fits.open(tilesfile, memmap=False) as hdulist:
+            tiledata = hdulist[1].data
         #
         # Temporary workaround for problem identified in
         # https://github.com/desihub/desimodel/issues/30
         #
-        if any([c.bzero is not None for c in _tiles.columns]):
-            foo = [_tiles[k].dtype for k in _tiles.dtype.names]
+        if any([c.bzero is not None for c in tiledata.columns]):
+            foo = [_tiles[k].dtype for k in tiledata.dtype.names]
 
         #- Check for out-of-date tiles file
-        if np.issubdtype(_tiles['OBSCONDITIONS'].dtype, 'u2'):
+        if np.issubdtype(tiledata['OBSCONDITIONS'].dtype, 'u2'):
             import warnings
             warnings.warn('old desi-tiles.fits with uint16 OBSCONDITIONS; please update your $DESIMODEL checkout', DeprecationWarning)
 
+        #- load cache for next time
+        if cache:
+            _tiles[tilesfile] = tiledata
+
     #- Filter to only the DESI footprint if requested
-    subset = np.ones(len(_tiles), dtype=bool)
+    subset = np.ones(len(tiledata), dtype=bool)
     if onlydesi:
-        subset &= _tiles['IN_DESI'] > 0
+        subset &= tiledata['IN_DESI'] > 0
 
     #- Filter out PROGRAM=EXTRA tiles if requested
     if not extra:
-        subset &= ~np.char.startswith(_tiles['PROGRAM'], 'EXTRA')
+        subset &= ~np.char.startswith(tiledata['PROGRAM'], 'EXTRA')
 
     if np.all(subset):
-        return _tiles
+        return tiledata
     else:
-        return _tiles[subset]
+        return tiledata[subset]
 
 _platescale = None
 def load_platescale():

py/desimodel/test/test_footprint.py

@@ -22,6 +22,10 @@ def test_get_tile_radec(self):
         """Test grabbing tile information by tileID.
         """
         io_tile_cache = io._tiles
+        io._tiles = dict()
+        tx = io.load_tiles()
+        tilefile = list(io._tiles.keys())[0]
+
         tiles = np.zeros((4,), dtype=[('TILEID', 'i2'),
                                       ('RA', 'f8'),
                                       ('DEC', 'f8'),
@@ -34,7 +38,7 @@ def test_get_tile_radec(self):
         tiles['DEC'] = [-2.0, -1.0, 1.0, 2.0]
         tiles['IN_DESI'] = [0, 1, 1, 0]
         tiles['PROGRAM'] = 'DARK'
-        io._tiles = tiles
+        io._tiles[tilefile] = tiles
         ra, dec = footprint.get_tile_radec(1)
         self.assertEqual((ra, dec), (0.0, 0.0))
         ra, dec, = footprint.get_tile_radec(2)

py/desimodel/test/test_io.py

@@ -3,7 +3,9 @@
 """Test desimodel.io.
 """
 import os
+import sys
 import uuid
+import tempfile
 import numpy as np
 from astropy.table import Table
 import unittest
@@ -38,13 +40,15 @@ def setUpClass(cls):
         global specter_available, desimodel_available
         cls.specter_available = specter_available
         cls.desimodel_available = desimodel_available
-        cls.trimdir = 'test-'+uuid.uuid4().hex
+        cls.tempdir = tempfile.mkdtemp(prefix='testio-')
+        cls.trimdir = os.path.join(cls.tempdir, 'trim')
+        cls.testfile = os.path.join(cls.tempdir, 'test-abc123.fits')
 
     @classmethod
     def tearDownClass(cls):
-        if os.path.exists(cls.trimdir):
+        if os.path.exists(cls.tempdir):
             import shutil
-            shutil.rmtree(cls.trimdir)
+            shutil.rmtree(cls.tempdir)
 
     def setUp(self):
         """Ensure that any desimodel.io caches are clear before running
@@ -55,7 +59,9 @@ def setUp(self):
         io._params = None
         io._gfa = None
         io._fiberpos = None
-        io._tiles = None
+        io._tiles = dict()
+        if os.path.exists(self.testfile):
+            os.remove(self.testfile)
 
     def tearDown(self):
         pass
@@ -135,12 +141,18 @@ def test_load_pix_file(self):
     def test_load_tiles(self):
         """Test loading of tile files.
         """
-        self.assertIsNone(io._tiles)
+        # starting clean
+        self.assertEqual(io._tiles, {})
+        t0 = io.load_tiles(cache=False)
+        self.assertEqual(len(io._tiles), 0)
+        # loading tiles fills the cache with one items
         t1 = io.load_tiles(onlydesi=False)
-        tile_cache_id1 = id(io._tiles)
-        self.assertIsNotNone(io._tiles)
+        self.assertEqual(len(io._tiles), 1)
+        tile_cache_id1 = id(list(io._tiles.items())[0])
+        # reloading, even with a filter, shouldn't change cache
         t2 = io.load_tiles(onlydesi=True)
-        tile_cache_id2 = id(io._tiles)
+        self.assertEqual(len(io._tiles), 1)
+        tile_cache_id2 = id(list(io._tiles.items())[0])
         self.assertEqual(tile_cache_id1, tile_cache_id2)
         #- Temporarily support OBSCONDITIONS as u2 (old) or i4 (new)
         self.assertTrue(np.issubdtype(t1['OBSCONDITIONS'].dtype, 'i4') or \
@@ -153,7 +165,7 @@ def test_load_tiles(self):
         # I think this is the exact same test as above, except using set theory.
         self.assertEqual(len(set(t2.TILEID) - set(t1.TILEID)), 0)
         t3 = io.load_tiles(onlydesi=False)
-        tile_cache_id3 = id(io._tiles)
+        tile_cache_id3 = id(list(io._tiles.items())[0])
         self.assertEqual(tile_cache_id1, tile_cache_id3)
         self.assertTrue(np.issubdtype(t3['OBSCONDITIONS'].dtype, 'i4') or \
                         np.issubdtype(t3['OBSCONDITIONS'].dtype, 'u2') )
@@ -164,6 +176,27 @@ def test_load_tiles(self):
         self.assertGreater(np.sum(np.char.startswith(b['PROGRAM'], 'EXTRA')), 0)
         self.assertLess(len(a), len(b))
 
+    @unittest.skipUnless(desimodel_available, desimodel_message)
+    def test_load_tiles_alt(self):
+        # starting clean
+        self.assertEqual(io._tiles, {})
+        t1 = Table(io.load_tiles())
+        t1.write(self.testfile)
+        #- no path; should fail since that file isn't in $DESIMODEL/data/footprint/
+        if sys.version_info.major == 2:
+            with self.assertRaises(IOError):
+                t2 = io.load_tiles(tilesfile=os.path.basename(self.testfile))
+        else:
+            with self.assertRaises(FileNotFoundError):
+                t2 = io.load_tiles(tilesfile=os.path.basename(self.testfile))
+
+        #- with path, should work:
+        t2 = Table(io.load_tiles(tilesfile=self.testfile))
+        self.assertTrue(np.all(t1 == t2))
+
+        #- cache should have two items now
+        self.assertEqual(len(io._tiles), 2)
+
     @unittest.skipUnless(desimodel_available, desimodel_message)
     def test_tiles_consistency(self):
         """Test consistency of tile files.