Merge pull request #273 from cta-observatory/link_datacheck

Make link of the updated longterm datacheck file in a common directory

src/osa/conftest.py

@@ -482,7 +482,14 @@ def datacheck_dl1_files(base_test_dir):
 
 @pytest.fixture(scope="session")
 def longterm_dir(base_test_dir):
-    directory = base_test_dir / "OSA" / "DL1DataCheck_LongTerm" / "v0.1.0" / "20200117"
+    directory = base_test_dir / "OSA" / "DL1DataCheck_LongTerm" / prod_id / date_to_dir(date)
+    directory.mkdir(parents=True, exist_ok=True)
+    return directory
+
+
+@pytest.fixture(scope="session")
+def longterm_link_latest_dir(base_test_dir):
+    directory = base_test_dir / "OSA" / "DL1DataCheck_LongTerm" / "night_wise" / "all"
     directory.mkdir(parents=True, exist_ok=True)
     return directory
 

src/osa/paths.py

@@ -18,6 +18,7 @@
 from osa.utils import utils
 from osa.utils.logging import myLogger
 
+
 log = myLogger(logging.getLogger(__name__))
 
 __all__ = [
@@ -346,3 +347,26 @@ def create_source_directories(source_list: list, cuts_dir: Path):
         if source is not None:
             source_dir = cuts_dir / source
             source_dir.mkdir(parents=True, exist_ok=True)
+
+
+def get_latest_version_file(longterm_files: List[str]) -> Path:
+    """Get the latest version path of the produced longterm DL1 datacheck files for a given date."""
+    return max(
+        longterm_files,
+        key=lambda path: int(path.parents[1].name.split(".")[1])
+        if path.parents[1].name.startswith("v")
+        else "",
+    )
+
+
+def create_longterm_symlink():
+    """If the created longterm DL1 datacheck file corresponds to the latest 
+    version available, make symlink to it in the "all" common directory."""
+    nightdir = utils.date_to_dir(options.date)
+    longterm_dir = Path(cfg.get("LST1", "LONGTERM_DIR"))
+    linked_longterm_file = longterm_dir / f"night_wise/all/DL1_datacheck_{nightdir}.h5"
+    all_longterm_files = longterm_dir.rglob(f"v*/{nightdir}/DL1_datacheck_{nightdir}.h5")
+    latest_version_file = get_latest_version_file(all_longterm_files)
+
+    log.info("Symlink the latest version longterm DL1 datacheck file in the common directory.")
+    linked_longterm_file.symlink_to(latest_version_file)

src/osa/scripts/closer.py

@@ -24,7 +24,7 @@
 )
 from osa.nightsummary.extract import extract_runs, extract_sequences
 from osa.nightsummary.nightsummary import run_summary_table
-from osa.paths import destination_dir
+from osa.paths import destination_dir, create_longterm_symlink
 from osa.raw import is_raw_data_available
 from osa.report import start
 from osa.utils.cliopts import closercliparsing
@@ -165,6 +165,7 @@ def post_process(seq_tuple):
         list_job_id = merge_dl1_datacheck(seq_list)
         longterm_job_id = daily_datacheck(daily_longterm_cmd(list_job_id))
         cherenkov_transparency(cherenkov_transparency_cmd(longterm_job_id))
+        create_longterm_symlink()
 
     # Extract the provenance info
     extract_provenance(seq_list)

src/osa/scripts/tests/test_osa_scripts.py

@@ -179,6 +179,9 @@ def test_closer(
     drs4_time_calibration_files,
     systematic_correction_files,
     merged_run_summary,
+    longterm_dir,
+    longterm_link_latest_dir,
+    daily_datacheck_dl1_files,
 ):
     # First assure that the end of night flag is not set and remove it otherwise
     night_finished_flag = Path(
@@ -198,6 +201,10 @@ def test_closer(
     for obs_file in test_observed_data:
         assert obs_file.exists()
     assert merged_run_summary.exists()
+    assert longterm_dir.exists()
+    assert longterm_link_latest_dir.exists()
+    for check_file in daily_datacheck_dl1_files:
+        assert check_file.exists()
 
     run_program("closer", "-y", "-v", "-t", "-d", "2020-01-17", "LST1")
     closed_seq_file = running_analysis_dir / "sequence_LST1_01809.closed"