Merge pull request #293 from cta-observatory/gainsel_automation

Improvements for gain selection automation

src/osa/configs/sequencer.cfg

@@ -55,6 +55,7 @@ dl1_to_dl2: lstchain_dl1_to_dl2
 dl1a_config: /software/lstchain/data/lstchain_standard_config.json
 store_image_dl1ab: True
 merge_dl1_datacheck: True
+use_ff_heuristic_gain_selection: False
 dl1b_config: /software/lstchain/data/lstchain_standard_config.json
 dl2_config: /software/lstchain/data/lstchain_standard_config.json
 rf_models: /data/models/prod5/zenith_20deg/20201023_v0.6.3
@@ -71,7 +72,8 @@ electron: /path/to/DL2/electron_mc_testing.h5
 PARTITION_PEDCALIB: short, long
 PARTITION_DATA: short, long
 MEMSIZE_PEDCALIB: 3GB
-MEMSIZE_DATA: 16GB
+MEMSIZE_DATA: 6GB
+MEMSIZE_GAINSEL: 2GB
 WALLTIME: 1:15:00
 # Days from current day up to which the jobs are fetched from the queue.
 # Default is None (left empty).

src/osa/job.py

@@ -653,7 +653,7 @@ def get_squeue_output(squeue_output: StringIO) -> pd.DataFrame:
     return df
 
 
-def run_sacct() -> StringIO:
+def run_sacct(job_id: str = None) -> StringIO:
     """Run sacct to obtain the job information."""
     if shutil.which("sacct") is None:
         log.warning("No job info available since sacct command is not available")
@@ -668,13 +668,18 @@ def run_sacct() -> StringIO:
         "-o",
         ",".join(FORMAT_SLURM),
     ]
+
+    if job_id:
+        sacct_cmd.append("--jobs")
+        sacct_cmd.append(job_id)
+
     if cfg.get("SLURM", "STARTTIME_DAYS_SACCT"):
         days = int(cfg.get("SLURM", "STARTTIME_DAYS_SACCT"))
         start_date = (datetime.date.today() - datetime.timedelta(days=days)).isoformat()
         sacct_cmd.extend(["--starttime", start_date])
 
     return StringIO(sp.check_output(sacct_cmd).decode())
-
+    
 
 def get_sacct_output(sacct_output: StringIO) -> pd.DataFrame:
     """
@@ -809,3 +814,12 @@ def update_sequence_state(sequence, filtered_job_info: pd.DataFrame) -> None:
         sequence.exit = "0:15"
     elif any("RUNNING" in job for job in filtered_job_info.State):
         sequence.state = "RUNNING"
+
+
+def job_finished_in_timeout(job_id: str) -> bool:
+    """Return True if the input job_id finished in TIMEOUT state."""
+    job_status = get_sacct_output(run_sacct(job_id=job_id))["State"]
+    if job_id and job_status.item() == "TIMEOUT":
+        return True
+    else:
+        return False