add py3.12, slight updates wrt slurm

.github/workflows/main.yml

@@ -17,7 +17,7 @@ jobs:
     strategy:
       max-parallel: 5
       matrix:
-        python: [3.8,3.9,'3.10', '3.11']
+        python: [3.8, 3.9,'3.10', '3.11']
       fail-fast: false
 
 

pyproject.toml

@@ -6,7 +6,7 @@ build-backend = "poetry.core.masonry.api"
 #maintainer ?#maintainer email
 [tool.poetry]
 name = "sequana_pipetools"
-version = "1.0.5"
+version = "1.0.6"
 description = "A set of tools to help building or using Sequana pipelines"
 authors = ["Sequana Team"]
 license = "BSD-3"
@@ -25,6 +25,7 @@ classifiers = [
         "Programming Language :: Python :: 3.9",
         "Programming Language :: Python :: 3.10",
         "Programming Language :: Python :: 3.11",
+        "Programming Language :: Python :: 3.12",
         "Topic :: Software Development :: Libraries :: Python Modules",
         "Topic :: Scientific/Engineering :: Bio-Informatics",
         "Topic :: Scientific/Engineering :: Information Analysis",
@@ -55,7 +56,7 @@ sequana_pipetools = "sequana_pipetools.scripts.main:main"
 
 [tool.poetry.group.dev.dependencies]
 black = "^23.7.0"
-pytest = "^7.4.0"
+pytest = "^8.0.0"
 mock = "^5.1.0"
 pytest-mock = "^3.11.1"
 sequana-rnaseq = "^0.18.0"

sequana_pipetools/snaketools/pipeline_manager.py

@@ -183,7 +183,7 @@ def teardown(self, extra_dirs_to_remove=[], extra_files_to_remove=[], outdir="."
         # for HPC with slurm only
         try:
             slurm_stats = SlurmStats(outdir)
-            slurm_stats.to_csv(f"{outdir}.sequana/slurm_stats.txt")
+            slurm_stats.to_csv(f"{outdir}/.sequana/slurm_stats.txt")
         except Exception:
             pass
 

sequana_pipetools/snaketools/slurm.py

@@ -64,9 +64,9 @@ def __init__(self, working_directory, logs_directory="logs", pattern="*/*slurm*.
 
     def to_csv(self, outfile):
         with open(outfile, "w") as fout:
-            fout.write(",".join(self.columns))
+            fout.write(",".join(self.columns) + "\n")
             for result in self.results:
-                fout.write(",".join([str(x) for x in result]))
+                fout.write(",".join([str(x) for x in result]) + "\n")
 
     def _parse_sacct_output(self, output):
         """Function to parse sacct output
@@ -82,7 +82,7 @@ def _parse_sacct_output(self, output):
         job_info = []
 
         # Regex to match the values
-        value_regex = re.compile(r"(\S+)?\s+(\d+)\s+(\d{2}:\d{2}:\d{2})\s+(\d{2}:\d{2}:\d{2})")
+        value_regex = re.compile(r"(\S+)?\s+(\d+)\s+((?:\d+-)?\d{2}:\d{2}:\d{2})\s+((?:\d+-)?\d{2}:\d{2}:\d{2})")
 
         for i, line in enumerate(lines):
             match = value_regex.search(line)
@@ -96,22 +96,27 @@ def _parse_sacct_output(self, output):
                 # Only keep the second line (main job)
                 if i == 1:
                     # Convert MaxRSS from KB to GB
-                    maxrss_gb = self._kb_to_gb(maxrss)
+                    maxrss_gb = self._convert_memory_to_gb(maxrss)
                     # Append parsed values to the job_info list
                     job_info = [maxrss_gb, alloccpus, elapsed, cputime]
                     break
 
         # Return the list of job information
         return job_info
 
-    def _kb_to_gb(self, kb_str):
-        # Remove the 'K' and convert to float
-        kb = float(kb_str.replace("K", ""))
+    def _convert_memory_to_gb(self, memory_str):
+        """Convert memory string to gigabytes (GB).
 
-        # Convert kilobytes to gigabytes (1 GB = 1024^2 KB)
-        gb = kb / (1024**2)
-
-        return round(gb, 6)  # Round to six decimal places for precision
+        Handles memory units in kilobytes (K), megabytes (M), gigabytes (G), and terabytes (T).
+        """
+        units = {"K": 1 / (1024**2), "M": 1 / 1024, "G": 1, "T": 1024}
+
+        match = re.match(r"(\d+(?:\.\d+)?)([KMGT])?", memory_str)
+        if match:
+            value = float(match.group(1))
+            unit = match.group(2) if match.group(2) else "K"  # Default to KB if no unit is specified
+            return round(value * units[unit], 6)
+        return 0.0  # Return 0 GB if parsing fails
 
 
 class SlurmParsing(SlurmData):

tests/snaketools/test_slurm.py

@@ -45,6 +45,19 @@ def test_parse_sacct_output(mock_sacct_output):
     assert job_info == [0.252117, 1, "00:00:24", "00:00:24"]
 
 
+def test_parse_sacct_output_generic():
+    slurm_stats = SlurmStats(working_directory=".", logs_directory="logs")
+    mock_sacct_output = """
+    MaxRSS  AllocCPUS    Elapsed    CPUTime
+---------- ---------- ---------- ----------
+                    4   06:07:19 1-00:29:16
+ 18410.50M          4   06:07:19 1-00:29:16
+         0          4   06:07:19 1-00:29:16
+"""
+    job_info = slurm_stats._parse_sacct_output(mock_sacct_output)
+    assert job_info == [17.979004, 4, "06:07:19", "1-00:29:16"]
+
+
 @patch("subprocess.run")
 def test_slurm_stats_with_mocked_sacct(mock_run, mock_sacct_output, tmpdir):
     # Mock the subprocess.run method