Scicat instrument preferences

conda-build/meta.yaml

@@ -40,6 +40,8 @@ requirements:
     - pyscicat
 
 test:
+  source_files:
+    - tests/**
   imports:
     - rfi_file_monitor
     - rfi_file_monitor.application
@@ -70,6 +72,7 @@ test:
     - rfi_file_monitor.operations.s3_downloader
     - rfi_file_monitor.operations.s3_uploader
     - rfi_file_monitor.operations.sftp_uploader
+    - rfi_file_monitor.operations.scicataloguer
     - rfi_file_monitor.preferences
     - rfi_file_monitor.preferenceswindow
     - rfi_file_monitor.queue_manager
@@ -84,6 +87,7 @@ test:
     - rfi_file_monitor.version
   commands:
     - pip check
+    - nose2 -v --log-level DEBUG
     - test -f $SP_DIR/rfi_file_monitor/docs/queue_manager.pango  # [unix]
     - test -f $SP_DIR/rfi_file_monitor/operations/docs/template.pango  # [unix]
     - test -f $SP_DIR/rfi_file_monitor/engines/docs/template.pango  # [unix]
@@ -92,6 +96,7 @@ test:
     - if not exist %SP_DIR%\\rfi_file_monitor\\engines\\docs\\\template.pango exit 1  # [win]
   requires:
     - pip
+    - nose2
 
 about:
   home: https://github.com/RosalindFranklinInstitute/rfi-file-monitor

instrument-config/test-instrument-prefs.yml

@@ -0,0 +1,4 @@
+test-instrument:
+  techniques:
+    test-instrument-technique:
+      - None

rfi_file_monitor/operations/docs/scicataloguer.pango

@@ -3,8 +3,8 @@
 <span size="x-large">Purpose</span>
 
 This operation creates metadata entries for data taken on the instrument and uploads them into the catalogue. 
-It automatically captures the required data from the previous operations. It can be used for both raw and derived datasets. 
-Uses pyscicat module to establish connection to SciCat. SciCat documentation can be found <a href="https://scicatproject.github.io/documentation/">here</a>.
+It automatically captures the required data from the previous operations. It can be used for both raw and derived datasets.
+SciCat documentation can be found <a href="https://scicatproject.github.io/documentation/">here</a>.
 
 <span size="x-large">Options</span>
 
@@ -16,10 +16,9 @@ Uses pyscicat module to establish connection to SciCat. SciCat documentation can
 * <b>Owner Group</b>
 * <b>Email</b>: contact email for the owner of the dataset
 * <b>Orcid</b>: orcid of the owner of the dataset
-* <b>Principal Invesigator</b>
+* <b>Principal Investigator</b>
 * <b>Experiment name</b>: name of the experiment 
-* <b>Instrument</b>: instrument used
-* <b>Technique</b>: the techique used by the instrument
+* <b>Technique</b>: the techique used by the instrument set in preferences
 * <b>Input Datasets</b>: for derived datasets only, comma-separated list of SciCat locations for raw datasets used
 * <b>Used Software</b>: for derived datasets only, comma-separated list of software used to derive analysis results
 

rfi_file_monitor/operations/scicataloguer.py

@@ -1,19 +1,21 @@
 import gi
 
 gi.require_version("Gtk", "3.0")
-from gi.repository import Gtk
+from gi.repository import Gtk, Gio
 from datetime import datetime
+from munch import Munch
 from ..operation import Operation
 from ..file import File
 from ..files.directory import Directory
 from ..files.regular_file import RegularFile
 from ..utils.decorators import supported_filetypes, with_pango_docs
+from ..preferences import Preference, InstrumentSetup
 from pathlib import PurePath, Path, PurePosixPath
 from pyscicat.client import ScicatClient
 from pyscicat.model import Dataset, RawDataset, DerivedDataset
 import logging
 from urllib.parse import urlparse
-from typing import Dict, Optional, List
+from typing import Dict, Optional, List, Any
 from ..version import __version__ as core_version
 
 logger = logging.getLogger(__name__)
@@ -27,6 +29,13 @@ class SciCataloguer(Operation):
     def __init__(self, *args, **kwargs):
         Operation.__init__(self, *args, **kwargs)
 
+        current_app = Gio.Application.get_default()
+        instrument_prefs: Munch[
+            Preference, Any
+        ] = current_app.get_preferences().settings
+        self.instrument_choice = instrument_prefs[InstrumentSetup]
+        self.instr_dict = InstrumentSetup.values[self.instrument_choice]
+
         self._grid = Gtk.Grid(
             row_spacing=5,
             halign=Gtk.Align.FILL,
@@ -284,32 +293,6 @@ def __init__(self, *args, **kwargs):
         )
         self._grid.attach(self._exp_name_entry, 3, 4, 1, 1)
 
-        # Instrument
-        # TO DO - this is temporary until instrument preferences configured
-        self._grid.attach(
-            Gtk.Label(
-                label=" Instrument ",
-                halign=Gtk.Align.CENTER,
-                valign=Gtk.Align.CENTER,
-                hexpand=False,
-                vexpand=False,
-            ),
-            0,
-            5,
-            1,
-            1,
-        )
-        self._instrument_entry = self.register_widget(
-            Gtk.Entry(
-                halign=Gtk.Align.FILL,
-                valign=Gtk.Align.CENTER,
-                hexpand=True,
-                vexpand=False,
-            ),
-            "instrument_choice",
-        )
-        self._grid.attach(self._instrument_entry, 1, 5, 1, 1)
-
         # Technique
         self._grid.attach(
             Gtk.Label(
@@ -319,21 +302,20 @@ def __init__(self, *args, **kwargs):
                 hexpand=False,
                 vexpand=False,
             ),
-            2,
+            0,
             5,
             1,
             1,
         )
-        self._technique_entry = self.register_widget(
-            Gtk.Entry(
-                halign=Gtk.Align.FILL,
-                valign=Gtk.Align.CENTER,
-                hexpand=True,
-                vexpand=False,
-            ),
-            "technique",
-        )
-        self._grid.attach(self._technique_entry, 3, 5, 1, 1)
+        # create combo box
+        combo = Gtk.ComboBoxText.new()
+        for k in self.instr_dict["techniques"].keys():
+            combo.append_text(k)
+        if len(self.instr_dict["techniques"].keys()) == 1:
+            combo.set_active(0)
+
+        widget = self.register_widget(combo, "technique")
+        self._grid.attach(widget, 1, 5, 1, 1)
 
         # Input boxes for derived dataset specific fields
         self._grid.attach(
@@ -397,7 +379,6 @@ def __init__(self, *args, **kwargs):
 
     # Makes input datasets/used software boxes editable if dataset is derived
     def checkbox_toggled(self, checkbox):
-        # Set class attribute for derived/raw dataset
         if checkbox.get_active() == True:
             self.params.derived_dataset = True
             self._input_datasets_entry.set_sensitive(True)
@@ -424,6 +405,13 @@ def _check_required_fields(params):
         if not params.owner_group:
             raise RequiredInfoNotFound("Owner group required")
 
+    @staticmethod
+    def _check_instrument(instrument_choice):
+        if instrument_choice == "test-instrument":
+            raise RequiredInfoNotFound(
+                "An instrument is required. Please provide an instrument via the instrument-prefs.yml file in the instrument-config directory instead of using the default test instrument. Instrument choice can be changed in Preferences."
+            )
+
     def preflight_check(self):
         self._preflight_check(self.params)
 
@@ -520,6 +508,11 @@ def create_payload(self, file, params):
             payload = RawPayload(**default_payload.dict())
             payload = self.is_raw_payload(payload, data_format)
 
+        # Check for instrument id
+        # Add instrument detail
+        if "id" in self.instr_dict:
+            payload.instrumentId = str(self.instr_dict["id"])
+
         # Remove unneeded metadata defaults
         del payload.scientificMetadataDefaults
         return payload
@@ -583,7 +576,7 @@ def is_dir_payload(self, _payload, file):
 
     # Adds raw dataset specific data
     def is_raw_payload(self, _payload, _data_format):
-        _payload.creationLocation = str(self.params.instrument_choice)
+        _payload.creationLocation = str(self.instrument_choice)
         _payload.principalInvestigator = self.params.investigator
         _payload.endTime = _payload.creationTime
         _payload.dataFormat = _data_format
@@ -605,26 +598,23 @@ def insert_payload(self, payload, scicat_session):
         except Exception as e:
             raise Exception(f"Could not catalogue payload in scicat: {e}")
 
-    # Upserts dataset in Scicat
-    # This won't work with upserting until features added into PySciCat
+    # Updates or inserts dataset in Scicat
     def upsert_payload(self, payload, scicat_session):
         # Ensure that raw/derived datasets don't overwrite each other
         query_results = scicat_session.get_datasets(
             {"datasetName": payload.datasetName, "type": payload.type}
         )
         if query_results:
             if query_results[0]["datasetName"] == payload.datasetName:
-                logger.info("pretending to upsert payload")
-                # try:
-                # if payload.type == "raw":
-                # r = scicat_session.upsert_raw_dataset(payload, {"datasetName": payload.datasetName, "type": payload.type})
-                # else:
-                # r = scicat_session.upsert_derived_dataset(payload, {"datasetName": payload.datasetName, "type": payload.type})
-                # if r:
-                #    logger.info(f"Payload upserted, PID: {r}")
-                # except Exception as e:
-                #   raise Exception(f"Could not catalogue payload in scicat: {e}")
-                #   return str(e)
+                try:
+                    r = scicat_session.update_dataset(
+                        payload,
+                        query_results[0]["pid"],
+                    )
+                    if r:
+                        logger.info(f"Payload updated, PID: {r}")
+                except Exception as e:
+                    raise Exception(f"Could not update payload in scicat: {e}")
             else:
                 self.insert_payload(payload, scicat_session)
         else:
@@ -693,9 +683,8 @@ def get_host_location(cls, file: File, operations_list, operation):
     def scientific_metadata_concatenation(
         cls, scientific_metadata, defaults, additional
     ):
-        scientific_metadata |= defaults
-        scientific_metadata |= additional
-        return scientific_metadata
+        _metadata = {**scientific_metadata, **defaults, **additional}
+        return _metadata
 
 
 class ParserNotFound(Exception):

rfi_file_monitor/preferences.py

@@ -9,6 +9,10 @@
 
 from .operation import Operation
 from .engine import Engine
+from rfi_file_monitor.utils import (
+    INSTRUMENT_CONFIG_FILEPATH,
+    TEST_INSTRUMENT_CONFIG_FILEPATH,
+)
 
 
 class Preference(ABC):
@@ -174,6 +178,29 @@ def __init__(
     default="*.swp,*.swx,*.DS_Store",
 )
 
+if INSTRUMENT_CONFIG_FILEPATH.is_file():
+    InstrumentSetup = DictPreference.from_file(
+        key="Instrument Setup",
+        yaml_file=INSTRUMENT_CONFIG_FILEPATH,
+        description="This is the instrument description. Instruments can be specified in a instrument-prefs.yml file inside the instrument-config directory.",
+    )
+elif TEST_INSTRUMENT_CONFIG_FILEPATH.is_file():
+    InstrumentSetup = DictPreference.from_file(
+        key="Instrument Setup",
+        yaml_file=TEST_INSTRUMENT_CONFIG_FILEPATH,
+        description="This is a default instrument. Instruments can be specified in a instrument-prefs.yml file inside the instrument-config directory.",
+    )
+else:
+    InstrumentSetup = DictPreference(
+        key="Instrument Setup",
+        values={
+            "test-instrument": {
+                "techniques": {"test-instrument-technique": ["None"]}
+            },
+        },
+        description="This is a default instrument. Instruments can be specified in a instrument-prefs.yml file inside the instrument-config directory.",
+    )
+
 
 class Preferences(NamedTuple):
     settings: Munch[Preference, Any]

rfi_file_monitor/utils/__init__.py

@@ -41,6 +41,14 @@
     GLib.get_user_config_dir(), "rfi-file-monitor", "prefs.yml"
 )
 
+INSTRUMENT_CONFIG_FILEPATH = Path(
+    GLib.get_current_dir(), "instrument-config", "instrument-prefs.yml"
+)
+
+TEST_INSTRUMENT_CONFIG_FILEPATH = Path(
+    GLib.get_current_dir(), "instrument-config", "test-instrument-prefs.yml"
+)
+
 PATTERN_PLACEHOLDER_TEXT = "e.g *.txt, *.csv or *temp* or *log*"
 
 DEFAULT_TIMEOUT = 5  # seconds

setup.cfg

@@ -62,6 +62,7 @@ rfi_file_monitor.engines =
 rfi_file_monitor.preferences =
     AllowedFilePatternsPreference = rfi_file_monitor.preferences:AllowedFilePatternsPreference
     IgnoredFilePatternsPreference = rfi_file_monitor.preferences:IgnoredFilePatternsPreference
+    InstrumentSetup = rfi_file_monitor.preferences:InstrumentSetup
 rfi_file_monitor.files =
     RegularFile = rfi_file_monitor.files.regular_file:RegularFile
     WeightedRegularFile = rfi_file_monitor.files.regular_file:WeightedRegularFile

tests/tests.py

@@ -173,7 +173,7 @@ def test_payload_file_generator(self):
         derived_payload.usedSoftware = self.usedSoftware
         derived_payload.scientificMetadata = (
             PayloadHelpers.scientific_metadata_concatenation(
-                scientificMetadata, payload.scientificMetadataDefaults
+                scientificMetadata, payload.scientificMetadataDefaults, {}
             )
         )
         del derived_payload.scientificMetadataDefaults
@@ -255,7 +255,7 @@ def test_payload_dir_generator(self):
         derived_payload.usedSoftware = self.usedSoftware
         derived_payload.scientificMetadata = (
             PayloadHelpers.scientific_metadata_concatenation(
-                scientificMetadata, payload.scientificMetadataDefaults
+                scientificMetadata, payload.scientificMetadataDefaults, {}
             )
         )
         del derived_payload.scientificMetadataDefaults