diff --git a/conda-build/meta.yaml b/conda-build/meta.yaml index 23e3a76..41b5774 100644 --- a/conda-build/meta.yaml +++ b/conda-build/meta.yaml @@ -40,6 +40,8 @@ requirements: - pyscicat test: + source_files: + - tests/** imports: - rfi_file_monitor - rfi_file_monitor.application @@ -70,6 +72,7 @@ test: - rfi_file_monitor.operations.s3_downloader - rfi_file_monitor.operations.s3_uploader - rfi_file_monitor.operations.sftp_uploader + - rfi_file_monitor.operations.scicataloguer - rfi_file_monitor.preferences - rfi_file_monitor.preferenceswindow - rfi_file_monitor.queue_manager @@ -84,6 +87,7 @@ test: - rfi_file_monitor.version commands: - pip check + - nose2 -v --log-level DEBUG - test -f $SP_DIR/rfi_file_monitor/docs/queue_manager.pango # [unix] - test -f $SP_DIR/rfi_file_monitor/operations/docs/template.pango # [unix] - test -f $SP_DIR/rfi_file_monitor/engines/docs/template.pango # [unix] @@ -92,6 +96,7 @@ test: - if not exist %SP_DIR%\\rfi_file_monitor\\engines\\docs\\\template.pango exit 1 # [win] requires: - pip + - nose2 about: home: https://github.com/RosalindFranklinInstitute/rfi-file-monitor diff --git a/instrument-config/test-instrument-prefs.yml b/instrument-config/test-instrument-prefs.yml new file mode 100644 index 0000000..ea2412e --- /dev/null +++ b/instrument-config/test-instrument-prefs.yml @@ -0,0 +1,4 @@ +test-instrument: + techniques: + test-instrument-technique: + - None \ No newline at end of file diff --git a/rfi_file_monitor/operations/docs/scicataloguer.pango b/rfi_file_monitor/operations/docs/scicataloguer.pango index db180fa..d40a191 100644 --- a/rfi_file_monitor/operations/docs/scicataloguer.pango +++ b/rfi_file_monitor/operations/docs/scicataloguer.pango @@ -3,8 +3,8 @@ Purpose This operation creates metadata entries for data taken on the instrument and uploads them into the catalogue. -It automatically captures the required data from the previous operations. It can be used for both raw and derived datasets. -Uses pyscicat module to establish connection to SciCat. SciCat documentation can be found here. +It automatically captures the required data from the previous operations. It can be used for both raw and derived datasets. +SciCat documentation can be found here. Options @@ -16,10 +16,9 @@ Uses pyscicat module to establish connection to SciCat. SciCat documentation can * Owner Group * Email: contact email for the owner of the dataset * Orcid: orcid of the owner of the dataset -* Principal Invesigator +* Principal Investigator * Experiment name: name of the experiment -* Instrument: instrument used -* Technique: the techique used by the instrument +* Technique: the techique used by the instrument set in preferences * Input Datasets: for derived datasets only, comma-separated list of SciCat locations for raw datasets used * Used Software: for derived datasets only, comma-separated list of software used to derive analysis results diff --git a/rfi_file_monitor/operations/scicataloguer.py b/rfi_file_monitor/operations/scicataloguer.py index 10d1072..0570d65 100644 --- a/rfi_file_monitor/operations/scicataloguer.py +++ b/rfi_file_monitor/operations/scicataloguer.py @@ -1,19 +1,21 @@ import gi gi.require_version("Gtk", "3.0") -from gi.repository import Gtk +from gi.repository import Gtk, Gio from datetime import datetime +from munch import Munch from ..operation import Operation from ..file import File from ..files.directory import Directory from ..files.regular_file import RegularFile from ..utils.decorators import supported_filetypes, with_pango_docs +from ..preferences import Preference, InstrumentSetup from pathlib import PurePath, Path, PurePosixPath from pyscicat.client import ScicatClient from pyscicat.model import Dataset, RawDataset, DerivedDataset import logging from urllib.parse import urlparse -from typing import Dict, Optional, List +from typing import Dict, Optional, List, Any from ..version import __version__ as core_version logger = logging.getLogger(__name__) @@ -27,6 +29,13 @@ class SciCataloguer(Operation): def __init__(self, *args, **kwargs): Operation.__init__(self, *args, **kwargs) + current_app = Gio.Application.get_default() + instrument_prefs: Munch[ + Preference, Any + ] = current_app.get_preferences().settings + self.instrument_choice = instrument_prefs[InstrumentSetup] + self.instr_dict = InstrumentSetup.values[self.instrument_choice] + self._grid = Gtk.Grid( row_spacing=5, halign=Gtk.Align.FILL, @@ -284,32 +293,6 @@ def __init__(self, *args, **kwargs): ) self._grid.attach(self._exp_name_entry, 3, 4, 1, 1) - # Instrument - # TO DO - this is temporary until instrument preferences configured - self._grid.attach( - Gtk.Label( - label=" Instrument ", - halign=Gtk.Align.CENTER, - valign=Gtk.Align.CENTER, - hexpand=False, - vexpand=False, - ), - 0, - 5, - 1, - 1, - ) - self._instrument_entry = self.register_widget( - Gtk.Entry( - halign=Gtk.Align.FILL, - valign=Gtk.Align.CENTER, - hexpand=True, - vexpand=False, - ), - "instrument_choice", - ) - self._grid.attach(self._instrument_entry, 1, 5, 1, 1) - # Technique self._grid.attach( Gtk.Label( @@ -319,21 +302,20 @@ def __init__(self, *args, **kwargs): hexpand=False, vexpand=False, ), - 2, + 0, 5, 1, 1, ) - self._technique_entry = self.register_widget( - Gtk.Entry( - halign=Gtk.Align.FILL, - valign=Gtk.Align.CENTER, - hexpand=True, - vexpand=False, - ), - "technique", - ) - self._grid.attach(self._technique_entry, 3, 5, 1, 1) + # create combo box + combo = Gtk.ComboBoxText.new() + for k in self.instr_dict["techniques"].keys(): + combo.append_text(k) + if len(self.instr_dict["techniques"].keys()) == 1: + combo.set_active(0) + + widget = self.register_widget(combo, "technique") + self._grid.attach(widget, 1, 5, 1, 1) # Input boxes for derived dataset specific fields self._grid.attach( @@ -397,7 +379,6 @@ def __init__(self, *args, **kwargs): # Makes input datasets/used software boxes editable if dataset is derived def checkbox_toggled(self, checkbox): - # Set class attribute for derived/raw dataset if checkbox.get_active() == True: self.params.derived_dataset = True self._input_datasets_entry.set_sensitive(True) @@ -424,6 +405,13 @@ def _check_required_fields(params): if not params.owner_group: raise RequiredInfoNotFound("Owner group required") + @staticmethod + def _check_instrument(instrument_choice): + if instrument_choice == "test-instrument": + raise RequiredInfoNotFound( + "An instrument is required. Please provide an instrument via the instrument-prefs.yml file in the instrument-config directory instead of using the default test instrument. Instrument choice can be changed in Preferences." + ) + def preflight_check(self): self._preflight_check(self.params) @@ -520,6 +508,11 @@ def create_payload(self, file, params): payload = RawPayload(**default_payload.dict()) payload = self.is_raw_payload(payload, data_format) + # Check for instrument id + # Add instrument detail + if "id" in self.instr_dict: + payload.instrumentId = str(self.instr_dict["id"]) + # Remove unneeded metadata defaults del payload.scientificMetadataDefaults return payload @@ -583,7 +576,7 @@ def is_dir_payload(self, _payload, file): # Adds raw dataset specific data def is_raw_payload(self, _payload, _data_format): - _payload.creationLocation = str(self.params.instrument_choice) + _payload.creationLocation = str(self.instrument_choice) _payload.principalInvestigator = self.params.investigator _payload.endTime = _payload.creationTime _payload.dataFormat = _data_format @@ -605,8 +598,7 @@ def insert_payload(self, payload, scicat_session): except Exception as e: raise Exception(f"Could not catalogue payload in scicat: {e}") - # Upserts dataset in Scicat - # This won't work with upserting until features added into PySciCat + # Updates or inserts dataset in Scicat def upsert_payload(self, payload, scicat_session): # Ensure that raw/derived datasets don't overwrite each other query_results = scicat_session.get_datasets( @@ -614,17 +606,15 @@ def upsert_payload(self, payload, scicat_session): ) if query_results: if query_results[0]["datasetName"] == payload.datasetName: - logger.info("pretending to upsert payload") - # try: - # if payload.type == "raw": - # r = scicat_session.upsert_raw_dataset(payload, {"datasetName": payload.datasetName, "type": payload.type}) - # else: - # r = scicat_session.upsert_derived_dataset(payload, {"datasetName": payload.datasetName, "type": payload.type}) - # if r: - # logger.info(f"Payload upserted, PID: {r}") - # except Exception as e: - # raise Exception(f"Could not catalogue payload in scicat: {e}") - # return str(e) + try: + r = scicat_session.update_dataset( + payload, + query_results[0]["pid"], + ) + if r: + logger.info(f"Payload updated, PID: {r}") + except Exception as e: + raise Exception(f"Could not update payload in scicat: {e}") else: self.insert_payload(payload, scicat_session) else: @@ -693,9 +683,8 @@ def get_host_location(cls, file: File, operations_list, operation): def scientific_metadata_concatenation( cls, scientific_metadata, defaults, additional ): - scientific_metadata |= defaults - scientific_metadata |= additional - return scientific_metadata + _metadata = {**scientific_metadata, **defaults, **additional} + return _metadata class ParserNotFound(Exception): diff --git a/rfi_file_monitor/preferences.py b/rfi_file_monitor/preferences.py index 2e8c92b..85f2ca2 100644 --- a/rfi_file_monitor/preferences.py +++ b/rfi_file_monitor/preferences.py @@ -9,6 +9,10 @@ from .operation import Operation from .engine import Engine +from rfi_file_monitor.utils import ( + INSTRUMENT_CONFIG_FILEPATH, + TEST_INSTRUMENT_CONFIG_FILEPATH, +) class Preference(ABC): @@ -174,6 +178,29 @@ def __init__( default="*.swp,*.swx,*.DS_Store", ) +if INSTRUMENT_CONFIG_FILEPATH.is_file(): + InstrumentSetup = DictPreference.from_file( + key="Instrument Setup", + yaml_file=INSTRUMENT_CONFIG_FILEPATH, + description="This is the instrument description. Instruments can be specified in a instrument-prefs.yml file inside the instrument-config directory.", + ) +elif TEST_INSTRUMENT_CONFIG_FILEPATH.is_file(): + InstrumentSetup = DictPreference.from_file( + key="Instrument Setup", + yaml_file=TEST_INSTRUMENT_CONFIG_FILEPATH, + description="This is a default instrument. Instruments can be specified in a instrument-prefs.yml file inside the instrument-config directory.", + ) +else: + InstrumentSetup = DictPreference( + key="Instrument Setup", + values={ + "test-instrument": { + "techniques": {"test-instrument-technique": ["None"]} + }, + }, + description="This is a default instrument. Instruments can be specified in a instrument-prefs.yml file inside the instrument-config directory.", + ) + class Preferences(NamedTuple): settings: Munch[Preference, Any] diff --git a/rfi_file_monitor/utils/__init__.py b/rfi_file_monitor/utils/__init__.py index f42cfbd..e7b5560 100644 --- a/rfi_file_monitor/utils/__init__.py +++ b/rfi_file_monitor/utils/__init__.py @@ -41,6 +41,14 @@ GLib.get_user_config_dir(), "rfi-file-monitor", "prefs.yml" ) +INSTRUMENT_CONFIG_FILEPATH = Path( + GLib.get_current_dir(), "instrument-config", "instrument-prefs.yml" +) + +TEST_INSTRUMENT_CONFIG_FILEPATH = Path( + GLib.get_current_dir(), "instrument-config", "test-instrument-prefs.yml" +) + PATTERN_PLACEHOLDER_TEXT = "e.g *.txt, *.csv or *temp* or *log*" DEFAULT_TIMEOUT = 5 # seconds diff --git a/setup.cfg b/setup.cfg index fb5298a..3b52137 100644 --- a/setup.cfg +++ b/setup.cfg @@ -62,6 +62,7 @@ rfi_file_monitor.engines = rfi_file_monitor.preferences = AllowedFilePatternsPreference = rfi_file_monitor.preferences:AllowedFilePatternsPreference IgnoredFilePatternsPreference = rfi_file_monitor.preferences:IgnoredFilePatternsPreference + InstrumentSetup = rfi_file_monitor.preferences:InstrumentSetup rfi_file_monitor.files = RegularFile = rfi_file_monitor.files.regular_file:RegularFile WeightedRegularFile = rfi_file_monitor.files.regular_file:WeightedRegularFile diff --git a/tests/tests.py b/tests/tests.py index 9336601..29148d7 100644 --- a/tests/tests.py +++ b/tests/tests.py @@ -173,7 +173,7 @@ def test_payload_file_generator(self): derived_payload.usedSoftware = self.usedSoftware derived_payload.scientificMetadata = ( PayloadHelpers.scientific_metadata_concatenation( - scientificMetadata, payload.scientificMetadataDefaults + scientificMetadata, payload.scientificMetadataDefaults, {} ) ) del derived_payload.scientificMetadataDefaults @@ -255,7 +255,7 @@ def test_payload_dir_generator(self): derived_payload.usedSoftware = self.usedSoftware derived_payload.scientificMetadata = ( PayloadHelpers.scientific_metadata_concatenation( - scientificMetadata, payload.scientificMetadataDefaults + scientificMetadata, payload.scientificMetadataDefaults, {} ) ) del derived_payload.scientificMetadataDefaults