Sonify in any viewer (spacetelescope#3690)

* Moving code from viewer to sonify plugin

More work moving sonification code to plugin

Finish moving get_sonified_cube to plugin

Moving more things to plugin from viewer

Trying to figure out how much more should go in the plugin

Missed this one

Only do this when needed

No errors but sound isn't playing

* Fix cherry pick

* Debugging

* Update sonified cube based on viewer

* Codestyle

* Changelog

* Remove defunct code

* Fix existing test and add new checks for adding to second viewer

* Codestyle

* Restore removal of callback before adding

* Allow the user to add the sonified data to any viewer right off the bat. Remove unused viewer select disable props

Author: rosteen

CHANGES.rst

@@ -40,6 +40,8 @@ Cubeviz
 
 - Add sonified layer for each cube created by the Sonify Data plugin. [#3430, #3660]
 
+- Sonified data can now be added to any image viewer after initial sonification. [#3690]
+
 - Renamed ``Spectral Extraction`` plugin to ``3D Spectral Extraction``. [#3691]
 
 Imviz

jdaviz/components/plugin_add_results.vue

@@ -32,7 +32,6 @@
         :api_hint="add_results_api_hint && add_results_api_hint+'.viewer ='"
         :api_hints_enabled="api_hints_enabled && add_results_api_hint"
         :hint="add_to_viewer_hint ? add_to_viewer_hint : 'Plot results in the specified viewer.  Data entry will be available in the data dropdown for all applicable viewers.'"
-        :add_to_viewer_disabled="add_to_viewer_disabled"
       ></plugin-viewer-select>
     </div>
 
@@ -95,8 +94,7 @@
     props: ['add_results_api_hint',
             'label', 'label_default', 'label_auto', 'label_invalid_msg', 'label_overwrite', 'label_label', 'label_hint',
             'add_to_viewer_items', 'add_to_viewer_selected', 'add_to_viewer_hint', 'auto_update_result',
-            'action_disabled', 'action_spinner', 'action_label', 'action_api_hint', 'action_tooltip', 'api_hints_enabled',
-            'add_to_viewer_disabled'],
+            'action_disabled', 'action_spinner', 'action_label', 'action_api_hint', 'action_tooltip', 'api_hints_enabled'],
     computed: {
       actionButtonText() {
         if (this.api_hints_enabled && this.action_api_hint) {

jdaviz/components/plugin_viewer_select.vue

@@ -1,6 +1,6 @@
 <template>
   <div>
-    <v-row v-if="show_multiselect_toggle && api_hints_enabled && api_hint_multiselect"> 
+    <v-row v-if="show_multiselect_toggle && api_hints_enabled && api_hint_multiselect">
       <span :class="api_hints_enabled && api_hint_multiselect ? 'api-hint' : null">
         {{  api_hint_multiselect }} {{  multiselect ? 'True' : 'False' }}
       </span>
@@ -31,7 +31,6 @@
       :chips="multiselect && !api_hints_enabled"
       item-text="label"
       item-value="label"
-      :disabled="add_to_viewer_disabled"
       persistent-hint
     >
     <template v-slot:selection="{ item, index }">
@@ -91,7 +90,7 @@
 module.exports = {
   props: ['items', 'selected', 'label', 'hint', 'rules', 'show_if_single_entry', 'multiselect',
           'show_multiselect_toggle', 'icon_checktoradial', 'icon_radialtocheck',
-          'api_hint', 'api_hint_multiselect', 'api_hints_enabled', 'add_to_viewer_disabled']
+          'api_hint', 'api_hint_multiselect', 'api_hints_enabled']
 };
 </script>
 

jdaviz/configs/cubeviz/plugins/sonify_data/sonify_data.py

@@ -1,15 +1,21 @@
 import time
-from traitlets import Bool, List, Unicode, observe
+
+from astropy.nddata import CCDData
 import astropy.units as u
+from astropy.wcs import WCS
+import numpy as np
+from scipy.special import erf
+from traitlets import Bool, List, Unicode, observe
 
 from jdaviz.core.custom_traitlets import IntHandleEmpty, FloatHandleEmpty
 from jdaviz.core.registries import tray_registry
 from jdaviz.core.template_mixin import (PluginTemplateMixin, DatasetSelectMixin,
                                         SpectralSubsetSelectMixin, with_spinner,
                                         AddResultsMixin)
 from jdaviz.core.user_api import PluginUserApi
-from jdaviz.core.events import SnackbarMessage
-
+from jdaviz.core.events import SnackbarMessage, AddDataMessage
+from jdaviz.configs.cubeviz.plugins.cube_listener import CubeListenerData, INT_MAX
+from jdaviz.core.sonified_layers import SonifiedLayerState
 
 __all__ = ['SonifyData']
 
@@ -60,8 +66,6 @@ class SonifyData(PluginTemplateMixin, DatasetSelectMixin, SpectralSubsetSelectMi
     sound_devices_items = List().tag(sync=True)
     sound_devices_selected = Unicode('').tag(sync=True)
 
-    add_to_viewer_enabled = Bool(False).tag(sync=True)
-
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
 
@@ -77,10 +81,23 @@ def __init__(self, *args, **kwargs):
             self.sound_device_indexes = None
             self.refresh_device_list()
 
+        self.add_results.viewer.add_filter('is_image_viewer')
         self.add_to_viewer_selected = 'flux-viewer'
+        self.sonified_cube = None
+        self.sonified_viewers = []
+        self.sonification_wl_ranges = None
+        self.sonification_wl_unit = None
+        self.stream = None
+        # Dictionary that contains keys with UUIDs for each
+        # sonified data layer. The value of each key is another dictionary containing
+        # coordinates as keys and arrays representing sounds as the value.
+        self.data_lookup = {}
 
         self._update_label_default(None)
 
+        self.hub.subscribe(self, AddDataMessage,
+                           handler=self._data_added_to_viewer)
+
     @property
     def user_api(self):
         expose = ['sonify_cube']
@@ -94,6 +111,36 @@ def _update_label_default(self, event):
         # Modify default label to avoid vue error from re-using label
         self.results_label_default = self.app.return_unique_name('Sonified data', typ='data')
 
+    def _data_added_to_viewer(self, msg):
+        # Keep track of the volume attribute for each layer.
+        msg.viewer.sonified_layers_enabled = []
+        for layer in msg.viewer.state.layers:
+            if not isinstance(layer, SonifiedLayerState):
+                continue
+
+            # Add viewer to sonified_viewers if it isn't there already
+            if msg.viewer not in self.sonified_viewers:
+                self.sonified_viewers.append(msg.viewer)
+
+            # Find layer, add volume check to dictionary and add callback to volume changing and
+            # audible changing
+            msg.viewer.layer_volume[layer.layer.label] = layer.volume
+            msg.viewer.sonified_layers_enabled += ([layer.layer.label] if
+                                                   getattr(layer, 'audible', False) else [])  # noqa
+
+            layer.remove_callback('volume', msg.viewer.recalculate_combined_sonified_grid)
+            layer.remove_callback('audible', msg.viewer.recalculate_combined_sonified_grid)
+            layer.add_callback('volume', msg.viewer.recalculate_combined_sonified_grid)
+            layer.add_callback('audible', msg.viewer.recalculate_combined_sonified_grid)
+
+    def start_stream(self):
+        if self.stream and not self.stream.closed:
+            self.stream.start()
+
+    def stop_stream(self):
+        if self.stream and not self.stream.closed:
+            self.stream.stop()
+
     def sonify_cube(self):
         """
         Create a sonified grid in the flux viewer so that sound plays when mousing over the viewer.
@@ -112,30 +159,141 @@ def sonify_cube(self):
             display_unit = self.spectrum_viewer.state.x_display_unit
             min_wavelength = self.spectral_subset.selected_obj.lower.to_value(u.Unit(display_unit))
             max_wavelength = self.spectral_subset.selected_obj.upper.to_value(u.Unit(display_unit))
-            self.flux_viewer.update_listener_wls((min_wavelength, max_wavelength), display_unit)
+            self.sonification_wl_ranges = ([min_wavelength, max_wavelength],)
+            self.sonification_wl_unit = display_unit
 
         # Ensure the current spectral region bounds are up-to-date at render time
         self.update_wavelength_range(None)
 
         previous_label = self.results_label
 
         # generate the sonified cube
-        sonified_cube = self.flux_viewer.get_sonified_cube(self.sample_rate, self.buffer_size,
-                                                           selected_device_index, self.assidx,
-                                                           self.ssvidx, self.pccut, self.audfrqmin,
-                                                           self.audfrqmax, self.eln,
-                                                           self.use_pccut, self.results_label)
+        sonified_cube = self.get_sonified_cube(self.sample_rate, self.buffer_size,
+                                               selected_device_index, self.assidx,
+                                               self.ssvidx, self.pccut, self.audfrqmin,
+                                               self.audfrqmax, self.eln,
+                                               self.use_pccut, self.results_label)
         sonified_cube.meta['Sonified'] = True
-        self.add_results.add_results_from_plugin(sonified_cube, replace=False)
 
-        self.flux_viewer.recalculate_combined_sonified_grid()
+        # For now, this still initially adds the sonified data to flux-viewer
+        self.add_results.add_results_from_plugin(sonified_cube, replace=False)
+        self.add_results.viewer.selected_obj.recalculate_combined_sonified_grid()
 
         t1 = time.time()
         msg = SnackbarMessage(f"'{previous_label}' sonified successfully in {t1-t0} seconds.",
                               color='success',
                               sender=self)
         self.app.hub.broadcast(msg)
 
+    def get_sonified_cube(self, sample_rate, buffer_size, device, assidx, ssvidx,
+                          pccut, audfrqmin, audfrqmax, eln, use_pccut, results_label):
+        spectrum = self.dataset.selected_obj
+        wlens = spectrum.wavelength.to('m').value
+        flux = spectrum.flux.value
+        self.sample_rate = sample_rate
+        self.buffer_size = buffer_size
+
+        if self.sonification_wl_ranges:
+            wdx = np.zeros(wlens.size).astype(bool)
+            for r in self.sonification_wl_ranges:
+                # index just the spectral subregion
+                wdx = np.logical_or(wdx,
+                                    np.logical_and(wlens >= r[0].to_value(u.m),
+                                                   wlens <= r[1].to_value(u.m)))
+            wlens = wlens[wdx]
+            flux_slices = [slice(None),]*3
+            flux_slices[spectrum.spectral_axis_index] = wdx
+            flux = flux[*flux_slices]
+
+        pc_cube = np.percentile(np.nan_to_num(flux), np.clip(pccut, 0, 99),
+                                axis=spectrum.spectral_axis_index)
+
+        # clip zeros and remove NaNs
+        clipped_arr = np.nan_to_num(np.clip(flux, 0, np.inf), copy=False)
+
+        # make a rough white-light image from the clipped array
+        whitelight = np.expand_dims(clipped_arr.sum(spectrum.spectral_axis_index),
+                                    axis=spectrum.spectral_axis_index)
+
+        if use_pccut:
+            # subtract any percentile cut
+            clipped_arr -= np.expand_dims(pc_cube, axis=spectrum.spectral_axis_index)
+
+            # and re-clip
+            clipped_arr = np.clip(clipped_arr, 0, np.inf)
+
+        self.sonified_cube = CubeListenerData(clipped_arr ** assidx, wlens, duration=0.8,
+                                              samplerate=sample_rate, buffsize=buffer_size,
+                                              wl_unit=self.sonification_wl_unit,
+                                              audfrqmin=audfrqmin, audfrqmax=audfrqmax,
+                                              eln=eln, vol=self.volume,
+                                              spectral_axis_index=spectrum.spectral_axis_index)
+
+        self.sonified_cube.sonify_cube()
+        self.sonified_cube.sigcube = (
+                self.sonified_cube.sigcube * pow(whitelight / whitelight.max(),
+                                                 ssvidx)).astype('int16')
+        self.stream = sd.OutputStream(samplerate=sample_rate, blocksize=buffer_size, device=device,
+                                      channels=1, dtype='int16', latency='low',
+                                      callback=self.sonified_cube.player_callback)
+        self.sonified_cube.cbuff = True
+
+        spatial_inds = [0, 1, 2]
+        spatial_inds.remove(spectrum.spectral_axis_index)
+        x_size = self.sonified_cube.sigcube.shape[spatial_inds[0]]
+        y_size = self.sonified_cube.sigcube.shape[spatial_inds[1]]
+
+        # Create a new entry for the sonified layer in data_lookup. The value is a dictionary
+        # containing (x_size * y_size) keys with values being arrays that represent sounds
+        if spectrum.spectral_axis_index == 2:
+            self.data_lookup[results_label] = {(x, y): self.sonified_cube.sigcube[x, y, :]
+                                               for x in range(0, x_size)
+                                               for y in range(0, y_size)}
+        elif spectrum.spectral_axis_index == 0:
+            # This looks wrong but it's because in this case x_size is actually the y axis and vice
+            # versa, wasn't sure about the best way to handle the spatial_inds thing above.
+            self.data_lookup[results_label] = {(y, x): self.sonified_cube.sigcube[:, x, y]
+                                               for x in range(0, x_size)
+                                               for y in range(0, y_size)}
+
+        # Create a 2D array with coordinates starting at (0, 0) and going until (x_size, y_size)
+        a = np.arange(1, x_size * y_size + 1).reshape((x_size, y_size))
+
+        # Attempt to copy the spatial WCS information from the cube
+        if hasattr(spectrum.wcs, 'celestial'):
+            wcs = spectrum.wcs.celestial
+        elif hasattr(spectrum.wcs, 'to_fits_sip'):
+            # GWCS case
+            wcs = WCS(spectrum.wcs.to_fits_sip())
+        else:
+            wcs = None
+
+        sonified_cube = CCDData(a * u.Unit(''), wcs=wcs)
+        return sonified_cube
+
+    def update_sonified_cube_with_coord(self, viewer, coord, vollim='buff'):
+        # Set newsig to the combined sound array at coord
+        if (int(coord[0]), int(coord[1])) not in viewer.combined_sonified_grid:
+            return
+
+        # use cached version of combined sonified grid
+        compsig = viewer.combined_sonified_grid[int(coord[0]), int(coord[1])]
+
+        # Adjust volume to remove clipping
+        if vollim == 'sig':
+            # sigmoidal volume limiting
+            self.sonified_cube.newsig = (erf(compsig/INT_MAX) * INT_MAX).astype('int16')
+        elif vollim == 'clip':
+            # hard-clipped volume limiting
+            self.sonified_cube.newsig = np.clip(compsig, -INT_MAX, INT_MAX).astype('int16')
+        elif vollim == 'buff':
+            # renormalise buffer
+            sigmax = abs(compsig).max()
+            if sigmax > INT_MAX:
+                compsig = ((INT_MAX/abs(compsig).max()) * compsig)
+            self.sonified_cube.newsig = compsig.astype('int16')
+        self.sonified_cube.cbuff = True
+
     @with_spinner()
     def vue_sonify_cube(self, *args):
         self.sonify_cube()
@@ -153,17 +311,26 @@ def update_wavelength_range(self, event):
             wlranges = self.spectral_subset.selected_obj.subregions
         else:
             wlranges = None
-        self.flux_viewer.update_listener_wls(wlranges, display_unit)
+        self.sonification_wl_ranges = wlranges
+        self.sonification_wl_unit = display_unit
 
     @observe('volume')
     def update_volume_level(self, event):
-        self.flux_viewer.update_volume_level(event['new'])
+        for viewer in self.sonified_viewers:
+            viewer.update_volume_level(event['new'])
 
     @observe('sound_devices_selected')
     def update_sound_device(self, event):
+        # This might get called before the plugin is fully initialized
+        if not hasattr(self, 'sonified_cube') or not self.sonified_cube:
+            return
         if event['new'] != event['old']:
             didx = dict(zip(*self.build_device_lists()))[event['new']]
-            self.flux_viewer.update_sound_device(didx)
+            # This was moved here from viewers.py now that the stream is handled here.
+            self.stop_stream()
+            self.stream = sd.OutputStream(samplerate=self.sample_rate, blocksize=self.buffer_size,
+                                          device=didx, channels=1, dtype='int16', latency='low',
+                                          callback=self.sonified_cube.player_callback)
 
     def refresh_device_list(self):
         devices, indexes = self.build_device_lists()

jdaviz/configs/cubeviz/plugins/sonify_data/sonify_data.vue

@@ -147,12 +147,11 @@
           label_hint="Label for the sonified data"
           :add_to_viewer_items="add_to_viewer_items"
           :add_to_viewer_selected.sync="add_to_viewer_selected"
-          :add_to_viewer_enabled="false"
+          add_to_viewer_hint="Add sonified layer to selected viewer. The sonified data will be available to add to all relevant viewers after creation."
           action_label="Sonify data"
-          action_tooltip="Create sonified data and add to flux viewer"
+          action_tooltip="Create sonified data and add to selected viewer"
           :action_spinner="spinner"
           action_api_hint='plg.sonify_cube()'
-          :add_to_viewer_disabled="true"
           @click:action="sonify_cube"
       ></plugin-add-results>
  </j-tray-plugin>

jdaviz/configs/cubeviz/plugins/sonify_data/tests/test_sonify_data.py

@@ -1,6 +1,7 @@
 import os
 
 import astropy.units as u
+from numpy.testing import assert_allclose
 import pytest
 from specutils import SpectralRegion
 
@@ -16,8 +17,8 @@ def test_sonify_data(cubeviz_helper, spectrum1d_cube_larger):
 
     # Create sonified data cube
     sonify_plg.vue_sonify_cube()
-    assert sonify_plg.flux_viewer.sonification_wl_ranges is None
-    assert sonify_plg.flux_viewer.sonified_cube is not None
+    assert sonify_plg.sonification_wl_ranges is None
+    assert sonify_plg.sonified_cube is not None
 
     # Test changing volume
     sonify_plg.volume = 90
@@ -29,8 +30,8 @@ def test_sonify_data(cubeviz_helper, spectrum1d_cube_larger):
     subset_plugin.import_region(spec_region)
     sonify_plg.spectral_subset_selected = 'Subset 1'
     sonify_plg.vue_sonify_cube()
-    ranges = (sonify_plg.flux_viewer.sonification_wl_ranges[0][0].value,
-              sonify_plg.flux_viewer.sonification_wl_ranges[0][1].value)
+    ranges = (sonify_plg.sonification_wl_ranges[0][0].value,
+              sonify_plg.sonification_wl_ranges[0][1].value)
     assert ranges == (4.62360028e-07, 4.62920561e-07)
 
     # Stop/start stream
@@ -39,6 +40,21 @@ def test_sonify_data(cubeviz_helper, spectrum1d_cube_larger):
     sonify_plg.vue_start_stop_stream()
     assert sonify_plg.stream_active
 
+    # Add sonified data to uncert-viewer
+    uncert_viewer = cubeviz_helper.viewers['uncert-viewer']._obj
+    uncert_viewer.data_menu.add_data('Sonified data')
+    assert 'Sonified data' in uncert_viewer.data_menu.data_labels_loaded
+
+    event_data = {'event': 'mousemove', 'domain': {'x': 1, 'y': 1}}
+    uncert_viewer._viewer_mouse_event(event_data)
+
+    compsig = uncert_viewer.combined_sonified_grid[(1, 1)]
+    sigmax = abs(compsig).max()
+    INT_MAX = 2**15 - 1
+    if sigmax > INT_MAX:
+        compsig = ((INT_MAX/abs(compsig).max()) * compsig)
+    assert_allclose(sonify_plg.sonified_cube.newsig, compsig)
+
 
 @pytest.mark.skipif(not IN_GITHUB_ACTIONS, reason="Plugin disabled only in CI")
 def test_sonify_data_disabled(cubeviz_helper, spectrum1d_cube_larger):