Fix job status 'broken' with slurm when job is in fact not broken

simexpal/base.py

@@ -453,16 +453,17 @@ def query_slurm(self):
 		if not self.slurm_queried:
 			# -h omits the header line.
 			# -r outputs one job array element per line.
-			output = subprocess.check_output(['squeue', '-h', '-r'])
+			# --format=%i,%t outputs jobid,status
+			output = subprocess.check_output(['squeue', '-h', '-r', '--format=%i,%t'])
 			output = output.decode().splitlines()
 
 			self.slurm_queried = True
 
 			if len(output) > 0:
-				output = [entry.split() for entry in output]
+				output = [entry.split(',') for entry in output]
 				for entry in output:
 					entry_jobid = entry[0]
-					entry_state = entry[4]
+					entry_state = entry[1]
 					if entry_state == 'PD':
 						status = Status.SUBMITTED
 					elif entry_state in ['R', 'CG']:

tests/examples/slurm-pr-183/experiments.yml

@@ -0,0 +1,17 @@
+experiments:
+  - name: slow
+    args: [python, 'slow.py', '@EXTRA_ARGS@']
+    stdout: out
+
+instances:
+  - repo: local
+    items: 
+      - name: bar
+        files: []
+        extra_args: []
+
+variants:
+  - axis: 'foo'
+    items:
+      - name: '1'
+        extra_args: ['1']

tests/examples/slurm-pr-183/readme.md

@@ -0,0 +1,4 @@
+# Test setup for PR #183
+Issue: `simex e list` lists experiments as 'broken' even though they are running fine in slurm.
+We currently do not have integration tests in simex - but: we can use the experiment setup in this folder manually to test for this bug and similar problems if the need arises.
+Usage: just run `runtest.sh` and confirm that the output has correct status (i.e. not 'broken')

tests/examples/slurm-pr-183/runtest.sh

@@ -0,0 +1,16 @@
+simex e purge --failed -f
+
+rm -r output
+rm -r aux
+rm .simex.cache
+
+simex e list
+simex e launch --launcher scaling
+simex e list
+
+squeue
+squeue -r
+
+sleep 3s
+
+simex e list

tests/examples/slurm-pr-183/slow.py

@@ -0,0 +1,3 @@
+import time
+
+time.sleep(20)