Merge pull request #51 from observatorycontrolsystem/feature/integrate_heroic

Added settings to enable sending new structures and updates to instru…

Failing test looks like its due to github server issues right now

Author: jnation3406

README.md

@@ -33,6 +33,11 @@ This project is configured using environment variables.
 | `OAUTH_TOKEN_URL`     | OAuth2 token URL, set this to use OAuth2 authentication                            | `""`                            |
 | `OAUTH_PROFILE_URL`   | Observation portal profile endpoint, used to retrieve details on user accounts     | `""`                            |
 | `OAUTH_SERVER_KEY`    | Observation portal server secret key to authenticate calls from the server         | `""`                            |
+| `HEROIC_API_URL`    | HEROIC server api url, required for submitting your observatory updates to the HEROIC service         | `""`                            |
+| `HEROIC_API_TOKEN`    | HEROIC server api token, required for submitting your observatory updates to the HEROIC service         | `""`                            |
+| `HEROIC_OBSERVATORY`    | HEROIC server observatory code, required for submitting your observatory updates to the HEROIC service         | `""`                            |
+| `HEROIC_EXCLUDE_SITES`    | Comma delimited list of site codes to ignore when sending updates to HEROIC         | `""`                            |
+| `HEROIC_EXCLUDE_TELESCOPES`    | Comma delimited list of site.enclosure.telescope codes to ignore when sending updates to HEROIC      | `""`                            |
 
 ## Local Development
 
@@ -145,3 +150,7 @@ Return a specific camera's configuration
 Return all instruments that are in the SCHEDULABLE state
 
     GET /instruments/?state=SCHEDULABLE
+
+## Sending data to HEROIC
+
+HEROIC is a service provided by Scimma through the NSF that accepts and stores observatory information, including instrument configuration and telescope status. By default, no data will be sent to the HEROIC service. If you want to send your observatory updates to HEROIC, you must set all the `HEROIC_*` environment variables. You must login to the HEROIC server, retrieve your API token, and request that an Observatory is created for you with your account as the admin for that observatory. Afterwards, by setting the appropriate environment variables your configuration database should automatically send updates to HEROIC when updates are made through the API or admin interface.

configdb/hardware/admin.py

@@ -11,6 +11,7 @@
 from reversion.admin import VersionAdmin
 from reversion.errors import RegistrationError
 
+from configdb.hardware.heroic import update_heroic_instrument_capabilities
 from configdb.hardware.models import (
     Site, Enclosure, GenericMode, ModeType, GenericModeGroup, Telescope, Instrument, Camera, CameraType,
     OpticalElementGroup, OpticalElement, InstrumentType, InstrumentCategory, ConfigurationType, ConfigurationTypeProperties
@@ -165,12 +166,27 @@ class InstrumentAdmin(HardwareAdmin):
     def science_camera_codes(self, obj):
         return ','.join([science_camera.code for science_camera in obj.science_cameras.all()])
 
+    def save_related(self, request, form, formsets, change):
+        # This is the best way to trigger on saving m2m admin relationships on the Instrument, like Science Cameras
+        finished = super().save_related(request, form, formsets, change)
+        update_heroic_instrument_capabilities(form.instance)
+        return finished
+
 @admin.register(Camera)
 class CameraAdmin(HardwareAdmin):
     form = CameraAdminForm
     list_display = ('code', 'camera_type')
     search_fields = ('code',)
 
+    def save_related(self, request, form, formsets, change):
+        # This is the best way to trigger on saving m2m admin relationships on the Camera, like Optical Element Groups
+        old_oegs = set(form.instance.optical_element_groups.all())
+        finished = super().save_related(request, form, formsets, change)
+        if old_oegs != set(form.instance.optical_element_groups.all()):
+            for instrument in form.instance.instrument_set.all():
+                update_heroic_instrument_capabilities(instrument)
+        return finished
+
 
 @admin.register(InstrumentType)
 class InstrumentTypeAdmin(HardwareAdmin):
@@ -193,13 +209,47 @@ class GenericModeGroupAdmin(HardwareAdmin):
     search_fields = ('instrument_type', 'type')
     list_filter = ('type', 'instrument_type')
 
+    def save_model(self, request, obj, form, change):
+        old_instrument_type = None
+        if obj.pk:
+            old_obj = self.model.objects.get(pk=obj.pk)
+            if old_obj.instrument_type != obj.instrument_type:
+                old_instrument_type = old_obj.instrument_type
+        # Now update the model so the new model details are saved
+        finished = super().save_model(request, obj, form, change)
+
+        if old_instrument_type:
+            # The instrument_type has changed so update heroic for the old instrument type
+            for instrument in old_instrument_type.instrument_set.all():
+                update_heroic_instrument_capabilities(instrument)
+        return finished
+
+    def save_related(self, request, form, formsets, change):
+        # This is the best way to trigger on saving m2m admin relationships on the GenericModeGroup,
+        # like when its members change
+        finished = super().save_related(request, form, formsets, change)
+        if form.instance.instrument_type:
+            for instrument in form.instance.instrument_type.instrument_set.all():
+                update_heroic_instrument_capabilities(instrument)
+        return finished
+
 
 @admin.register(GenericMode)
 class GenericModeAdmin(HardwareAdmin):
     form = GenericModeAdminForm
     list_display = ('name', 'code', 'overhead', 'schedulable')
     search_fields = ('name', 'code')
 
+    def save_related(self, request, form, formsets, change):
+        # This is the best way to trigger on saving m2m admin relationships on the GenericMode,
+        # like when its membersships change
+        finished = super().save_related(request, form, formsets, change)
+        for gmg in form.instance.genericmodegroup_set.all():
+            if gmg.instrument_type:
+                for instrument in gmg.instrument_type.instrument_set.all():
+                    update_heroic_instrument_capabilities(instrument)
+        return finished
+
 
 @admin.register(OpticalElementGroup)
 class OpticalElementGroupAdmin(HardwareAdmin):
@@ -208,12 +258,31 @@ class OpticalElementGroupAdmin(HardwareAdmin):
     search_fields = ('name', 'type')
     list_filter = ('type',)
 
+    def save_related(self, request, form, formsets, change):
+        # This is the best way to trigger on saving m2m admin relationships on the OpticalElementGroup,
+        # like when its members change
+        finished = super().save_related(request, form, formsets, change)
+        for camera in form.instance.camera_set.all():
+            for instrument in camera.instrument_set.all():
+                update_heroic_instrument_capabilities(instrument)
+        return finished
+
 
 @admin.register(OpticalElement)
 class OpticalElementAdmin(HardwareAdmin):
     list_display = ('name', 'code', 'schedulable')
     search_fields = ('name', 'code')
 
+    def save_related(self, request, form, formsets, change):
+        # This is the best way to trigger on saving m2m admin relationships on the OpticalElement,
+        # like when its memberships change
+        finished = super().save_related(request, form, formsets, change)
+        for oeg in form.instance.opticalelementgroup_set.all():
+            for camera in oeg.camera_set.all():
+                for instrument in camera.instrument_set.all():
+                    update_heroic_instrument_capabilities(instrument)
+        return finished
+
 
 @admin.register(LogEntry)
 class LogEntryAdmin(admin.ModelAdmin):

configdb/hardware/apps.py

@@ -0,0 +1,16 @@
+from django.apps import AppConfig
+from django.conf import settings
+
+
+def can_submit_to_heroic():
+    return settings.HEROIC_API_URL and settings.HEROIC_API_TOKEN and settings.HEROIC_OBSERVATORY
+
+
+class HardwareConfig(AppConfig):
+    name = 'configdb.hardware'
+
+    def ready(self):
+        # Only load the heroic communication signals if heroic settings are set
+        if can_submit_to_heroic():
+            import configdb.hardware.signals.handlers  # noqa
+        super().ready()

configdb/hardware/heroic.py

@@ -0,0 +1,214 @@
+from django.conf import settings
+from urllib.parse import urljoin
+import requests
+import logging
+
+from configdb.hardware.models import Instrument, Telescope, Site
+from configdb.hardware.apps import can_submit_to_heroic
+
+
+logger = logging.getLogger()
+
+
+def instrument_status_conversion(state: str):
+    ''' Converts instrument state to HEROIC instrument status
+    '''
+    if state == 'DISABLED' or state == 'MANUAL':
+        return 'UNAVAILABLE'
+    elif state == 'SCHEDULABLE':
+        return 'SCHEDULABLE'
+    else:
+        return 'AVAILABLE'
+
+
+def telescope_status_conversion(telescope: Telescope):
+    return 'SCHEDULABLE' if telescope.active and telescope.enclosure.active and telescope.enclosure.site.active else 'UNAVAILABLE'
+
+
+def heroic_site_id(site: Site):
+    ''' Extract a HEROIC id for the site.
+        This concatenates the observatory.site for human readability
+    '''
+    return f"{settings.HEROIC_OBSERVATORY}.{site.code}"
+
+
+def heroic_telescope_id(telescope: Telescope):
+    ''' Extract a HEROIC id for the telescope
+        This concatenates the observatory.site.telescope for human readability
+    '''
+    return f"{heroic_site_id(telescope.enclosure.site)}.{telescope.enclosure.code}-{telescope.code}"
+
+
+def heroic_instrument_id(instrument: Instrument):
+    ''' Extract a HEROIC id for the instrument
+        This concatenates the observatory.site.telescope.instrument
+        for human-readability.
+    '''
+    return f"{heroic_telescope_id(instrument.telescope)}.{instrument.code}"
+
+
+def heroic_optical_element_groups(instrument: Instrument):
+    ''' Puts the optical element groups of an instrument in the format for reporting to HEROIC
+    '''
+    optical_element_groups = {}
+    for camera in instrument.science_cameras.all():
+        for optical_element_group in camera.optical_element_groups.all():
+            optical_element_groups[optical_element_group.type] = {'options': []}
+            if optical_element_group.default:
+                optical_element_groups[optical_element_group.type]['default'] = optical_element_group.default.code
+            for optical_element in optical_element_group.optical_elements.all():
+                optical_element_groups[optical_element_group.type]['options'].append({
+                    'id': optical_element.code,
+                    'name': optical_element.name,
+                    'schedulable': optical_element.schedulable
+                })
+    return optical_element_groups
+
+
+def heroic_operation_modes(instrument: Instrument):
+    ''' Puts the generic mode groups of an instrument in the format for reporting to HEROIC
+    '''
+    operation_modes = {}
+    for generic_mode_group in instrument.instrument_type.mode_types.all():
+        operation_modes[generic_mode_group.type.id] = {'options': []}
+        if generic_mode_group.default:
+            operation_modes[generic_mode_group.type.id]['default'] = generic_mode_group.default.code
+        for mode in generic_mode_group.modes.all():
+            operation_modes[generic_mode_group.type.id]['options'].append({
+                'id': mode.code,
+                'name': mode.name,
+                'schedulable': mode.schedulable
+            })
+    return operation_modes
+
+
+def instrument_to_heroic_instrument_capabilities(instrument: Instrument):
+    ''' Extracts the current instrument capabilities of an instrument to send to HEROIC
+    '''
+    capabilities = {
+        'instrument': heroic_instrument_id(instrument),
+        'status': instrument_status_conversion(instrument.state),
+        'optical_element_groups': heroic_optical_element_groups(instrument),
+        'operation_modes': heroic_operation_modes(instrument)
+    }
+    return capabilities
+
+
+def telescope_to_heroic_telescope_properties(telescope: Telescope):
+    ''' Extracts the current telescope properties of a telescope to send to HEROIC
+    '''
+    telescope_payload = {
+        'name': f"{telescope.name} - {telescope.enclosure.name}",
+        'site': heroic_site_id(telescope.enclosure.site),
+        'aperture': telescope.aperture,
+        'latitude': telescope.lat,
+        'longitude': telescope.long,
+        'horizon': telescope.horizon,
+        'negative_ha_limit': telescope.ha_limit_neg,
+        'positive_ha_limit': telescope.ha_limit_pos,
+        'zenith_blind_spot': telescope.zenith_blind_spot
+    }
+    return telescope_payload
+
+
+def send_to_heroic(api_endpoint: str, payload: dict, update: bool = False):
+    ''' Function to send data to HEROIC API endpoints
+    '''
+    headers = {'Authorization': f'Token {settings.HEROIC_API_TOKEN}'}
+    url = urljoin(settings.HEROIC_API_URL, api_endpoint)
+    if update:
+        response = requests.patch(url, headers=headers, json=payload)
+    else:
+        response = requests.post(url, headers=headers, json=payload)
+    logger.warning(response.json())
+    response.raise_for_status()
+
+
+
+def create_heroic_instrument(instrument: Instrument):
+    ''' Create a new instrument payload and send it to HEROIC
+    '''
+    if (instrument.telescope.enclosure.site.code not in settings.HEROIC_EXCLUDE_SITES and str(instrument.telescope) not in settings.HEROIC_EXCLUDE_TELESCOPES):
+        instrument_payload = {
+            'id': heroic_instrument_id(instrument),
+            'name': f"{instrument.instrument_type.name} - {instrument.code}",
+            'telescope': heroic_telescope_id(instrument.telescope),
+            'available': True
+        }
+        try:
+            send_to_heroic('instruments/', instrument_payload)
+        except Exception as e:
+            logger.error(f'Failed to create heroic instrument {str(instrument)}: {repr(e)}')
+
+
+def update_heroic_instrument_capabilities(instrument: Instrument):
+    ''' Send the current instrument capabilities of an instrument to HEROIC
+        if it is not DISABLED and heroic is set up in settings.py
+    '''
+    if can_submit_to_heroic() and instrument.state != 'DISABLED' and instrument.telescope.enclosure.site.code not in settings.HEROIC_EXCLUDE_SITES and str(instrument.telescope) not in settings.HEROIC_EXCLUDE_TELESCOPES:
+        capabilities = instrument_to_heroic_instrument_capabilities(instrument)
+        try:
+            send_to_heroic('instrument-capabilities/', capabilities)
+        except Exception as e:
+            logger.error(f'Failed to create heroic instrument {str(instrument)} capability update: {repr(e)}')
+
+
+def create_heroic_telescope(telescope: Telescope):
+    ''' Create a new telescope payload and send it to HEROIC
+    '''
+    if telescope.enclosure.site.code not in settings.HEROIC_EXCLUDE_SITES and str(telescope) not in settings.HEROIC_EXCLUDE_TELESCOPES:
+        telescope_payload = telescope_to_heroic_telescope_properties(telescope)
+        telescope_payload['id'] = heroic_telescope_id(telescope)
+        telescope_payload['status'] = telescope_status_conversion(telescope)
+        if telescope_payload['status'] != 'SCHEDULABLE':
+            telescope_payload['reason'] = 'Telescope is currently marked as inactive to prevent usage'
+        try:
+            send_to_heroic('telescopes/', telescope_payload)
+        except Exception as e:
+            logger.error(f'Failed to create heroic telescope {str(telescope)}: {repr(e)}')
+
+
+def update_heroic_telescope_properties(telescope: Telescope):
+    ''' Send updated telescope properties to HEROIC when they change
+    '''
+    if telescope.enclosure.site.code not in settings.HEROIC_EXCLUDE_SITES and str(telescope) not in settings.HEROIC_EXCLUDE_TELESCOPES:
+        telescope_update_payload = telescope_to_heroic_telescope_properties(telescope)
+        try:
+            send_to_heroic(f'telescopes/{heroic_telescope_id(telescope)}/', telescope_update_payload, update=True)
+        except Exception as e:
+            logger.error(f'Failed to update heroic telescope {str(telescope)}: {repr(e)}')
+
+
+def site_to_heroic_site_properties(site: Site):
+    ''' Extracts the current site properties of a site to send to HEROIC
+    '''
+    site_payload = {
+        'name': site.name,
+        'observatory': settings.HEROIC_OBSERVATORY,
+        'elevation': site.elevation,
+        'timezone': site.tz
+    }
+    return site_payload
+
+
+def create_heroic_site(site: Site):
+    ''' Create a new site payload and send it to HEROIC
+    '''
+    if site.code not in settings.HEROIC_EXCLUDE_SITES:
+        site_payload = site_to_heroic_site_properties(site)
+        site_payload['id'] = heroic_site_id(site)
+        try:
+            send_to_heroic('sites/', site_payload)
+        except Exception as e:
+            logger.error(f'Failed to create heroic site {str(site)}: {repr(e)}')
+
+
+def update_heroic_site(site: Site):
+    ''' Send updated site properties to HEROIC when they change
+    '''
+    if site.code not in settings.HEROIC_EXCLUDE_SITES:
+        site_payload = site_to_heroic_site_properties(site)
+        try:
+            send_to_heroic(f'sites/{heroic_site_id(site)}/', site_payload, update=True)
+        except Exception as e:
+            logger.error(f'Failed to update heroic site {str(site)}: {repr(e)}')