continue a separate time point series after a failed time point

src/main/java/org/micromanager/lightsheetmanager/LightSheetManagerPlugin.java

@@ -15,7 +15,7 @@ public class LightSheetManagerPlugin implements MenuPlugin, SciJavaPlugin {
     public static final String copyright = "Applied Scientific Instrumentation (ASI), 2022-2026";
     public static final String description = "A plugin to control various types of light sheet microscopes.";
     public static final String menuName = "Light Sheet Manager";
-    public static final String version = "0.8.18";
+    public static final String version = "0.8.19";
 
     private Studio studio_;
 

src/main/java/org/micromanager/lightsheetmanager/model/acquisitions/AcquisitionEngine.java

@@ -293,6 +293,31 @@ protected void setAwaitingTimePoint(final boolean awaiting) {
         awaitingTimePoint_ = awaiting;
     }
 
+    /**
+     * Stops the sequence of every camera that is still running one.
+     *
+     * @param cameras the cameras to check
+     * @return one entry per camera checked: "name was still running" if it was stopped, or
+     *     "name had finished"
+     */
+    protected List<String> stopCameraSequences(final CameraBase[] cameras) {
+        final List<String> states = new ArrayList<>();
+        try {
+            for (CameraBase camera : cameras) {
+                final String name = camera.getDeviceName();
+                if (core_.isSequenceRunning(name)) {
+                    core_.stopSequenceAcquisition(name);
+                    states.add(name + " was still running");
+                } else {
+                    states.add(name + " had finished");
+                }
+            }
+        } catch (Exception e) {
+            studio_.logs().logError("Could not stop camera sequences: " + e.getMessage());
+        }
+        return states;
+    }
+
     /**
      * Names the stage the run is in, for a log line reading "... during &lt;stage&gt;".
      */

src/main/java/org/micromanager/lightsheetmanager/model/acquisitions/AcquisitionEngineDispim.java

@@ -521,15 +521,7 @@ void finish() {
         // --- Job A ---
 
         // Stop all cameras sequences
-        try {
-            for (CameraBase camera : cameras) {
-                if (core_.isSequenceRunning(camera.getDeviceName())) {
-                    core_.stopSequenceAcquisition(camera.getDeviceName());
-                }
-            }
-        } catch (Exception e) {
-            studio_.logs().logError("Could not stop camera sequences: " + e.getMessage());
-        }
+        stopCameraSequences(cameras);
 
         // clean up controller settings after acquisition
         // want to do this, even with demo cameras, so we can test everything else

src/main/java/org/micromanager/lightsheetmanager/model/acquisitions/AcquisitionEngineScape.java

@@ -60,6 +60,7 @@
 import java.util.Locale;
 import java.util.Map;
 import java.util.Objects;
+import java.util.stream.Collectors;
 
 /**
  * Manages the acquisition for SCAPE microscopes.
@@ -1027,8 +1028,10 @@ private void submitEvents(final ScapeAcquisitionSettings settings, final String[
      * <p>Time point t is due at start + t * interval on a monotonic clock, so a late time point
      * does not delay the later ones: they run back to back until the schedule catches up.
      *
-     * <p>Any abort or failure ends the series. There is no camera timeout, so a starved time
-     * point hangs the series.
+     * <p>A time point that fails is marked incomplete and the series goes on; the time points
+     * that failed are reported once, when the series ends. A stop, or an abort without a
+     * failure, ends the series. There is no camera timeout, so a starved time point hangs the
+     * series.
      *
      * @return true if at least one dataset was completed
      */
@@ -1047,6 +1050,8 @@ private boolean runSeparateTimePoints(final ScapeAcquisitionSettings settings,
         final long seriesStartNs = System.nanoTime();
         // directory names of the completed datasets
         final List<String> datasetNames = new ArrayList<>();
+        // the time points that failed, reported together when the series ends
+        final List<Integer> failedTimePoints = new ArrayList<>();
 
         // the last dataset's window stays open until the next time point starts, so a series
         // that stops in the wait still shows it
@@ -1078,8 +1083,8 @@ private boolean runSeparateTimePoints(final ScapeAcquisitionSettings settings,
 
                 final String datasetDir = datastore_.getSavePath();
                 previousStore = datastore_;
-                final boolean keepGoing =
-                        finishTimePoint(settings, timeIndex, datasetDir, datasetNames);
+                final boolean keepGoing = finishTimePoint(settings, timeIndex, datasetDir,
+                        datasetNames, failedTimePoints);
                 inFlightIndex = -1; // accounted for by finishTimePoint
                 if (!keepGoing) {
                     break;
@@ -1090,10 +1095,12 @@ private boolean runSeparateTimePoints(final ScapeAcquisitionSettings settings,
             final String inFlightDir = datastore_ == null ? null : datastore_.getSavePath();
             if (inFlightIndex >= 0 && inFlightDir != null) {
                 writeIncompleteMarker(inFlightDir, inFlightIndex, "failed");
+                failedTimePoints.add(inFlightIndex);
             }
             final int completed = datasetNames.size();
             studio_.logs().logMessage("separate time points: " + completed + " of " + numTimePoints
                     + " datasets complete in " + seriesDir);
+            reportFailedTimePoints(failedTimePoints, completed, numTimePoints, seriesDir);
         }
         return !datasetNames.isEmpty();
     }
@@ -1188,51 +1195,84 @@ private String createSeriesDirectory(final String saveDir, final String saveName
         return seriesDir;
     }
 
+    /** What became of one time point of a series. */
+    private enum TimePointResult {
+        /** Every image arrived and the dataset is whole. */
+        COMPLETE,
+        /** The acquisition failed: AcqEngJ aborted it and kept the exception. */
+        FAILED,
+        /** A stop or a closed window aborted it, without an exception. */
+        ABORTED
+    }
+
     /**
      * Ends one time point of a series, after waitForCompletion(), and says whether the series
      * should go on. Records the dataset as complete or marks it incomplete, and leaves its window
      * for the caller to close.
      *
+     * <p>A failed time point is logged and marked incomplete, any camera it left running is
+     * stopped, and the series goes on unless a stop was requested.
+     *
      * @param settings the run snapshot
      * @param timeIndex the time point that just ended
      * @param datasetDir where this time point was written
-     * @param datasetNames the completed datasets, appended to when this one is whole
+     * @param datasetNames the completed datasets, appended to if this one is whole
+     * @param failedTimePoints the time points that failed, appended to if this one fails
      * @return true if the next time point should run
      */
     private boolean finishTimePoint(final ScapeAcquisitionSettings settings, final int timeIndex,
-            final String datasetDir, final List<String> datasetNames) {
+            final String datasetDir, final List<String> datasetNames,
+            final List<Integer> failedTimePoints) {
 
-        // check both: some aborts set only the flag, others reach only this Acquisition
-        final boolean aborted = currentAcquisition_.isAbortRequested();
-        boolean keepGoing = !aborted && !isStopRequested();
-        if (!keepGoing) {
-            studio_.logs().logMessage("separate time points: the series stopped at time point "
-                    + timeIndex);
-        }
-        // null while the dataset is whole; a stop after it finished ends the series but does not
-        // make it incomplete
-        String incompleteReason = aborted ? "aborted" : null;
+        Exception failure = null;
         try {
             currentAcquisition_.checkForExceptions();
         } catch (Exception e) {
-            // shown, not only logged, so a long series that stops unattended says why
-            model_.logging().reportError(e, "The acquisition failed at time point " + timeIndex
-                    + " and the series stopped there. Earlier time points are complete on disk.");
-            incompleteReason = "failed";
-            keepGoing = false;
+            failure = e;
+        }
+        // AcqEngJ aborts a failing acquisition too, so the exception is what tells a failure from
+        // a stop or a closed window
+        final TimePointResult result;
+        if (failure != null) {
+            result = TimePointResult.FAILED;
+        } else if (currentAcquisition_.isAbortRequested()) {
+            result = TimePointResult.ABORTED;
+        } else {
+            result = TimePointResult.COMPLETE;
+        }
+        // check both: some aborts set only the flag, others reach only this Acquisition. A stop
+        // ends the series whatever became of this time point, and leaves a whole dataset whole.
+        final boolean keepGoing = result != TimePointResult.ABORTED && !isStopRequested();
+        if (!keepGoing) {
+            studio_.logs().logMessage("separate time points: the series stopped at time point "
+                    + timeIndex);
         }
 
         // an incomplete dataset is marked inside its folder, so it is not mistaken for a whole one
-        if (incompleteReason == null) {
-            datasetNames.add(new File(datasetDir).getName());
-        } else {
-            writeIncompleteMarker(datasetDir, timeIndex, incompleteReason);
+        switch (result) {
+            case COMPLETE:
+                datasetNames.add(new File(datasetDir).getName());
+                break;
+            case FAILED:
+                failedTimePoints.add(timeIndex);
+                // logged, not shown: the time points that failed are reported together at the end
+                studio_.logs().logError(failure, "separate time points: time point " + timeIndex
+                        + " failed and is marked incomplete"
+                        + (keepGoing ? "; the series goes on" : ""));
+                stopCamerasLeftRunning(timeIndex);
+                writeIncompleteMarker(datasetDir, timeIndex, "failed");
+                break;
+            case ABORTED:
+                writeIncompleteMarker(datasetDir, timeIndex, "aborted");
+                break;
+            default:
+                break;
         }
 
-        // stage scan only: let the stage finish retracing before the next time point moves it.
-        // finish() does this after the last one.
-        if (keepGoing && settings.stageScan().enabled() && model_.devices().isUsingPLogic()
-                && controller_ != null) {
+        // stage scan: let the stage finish retracing before the next time point moves it. After a
+        // failure, in any mode: the volume may still be running. finish() does this after the last.
+        if (keepGoing && (settings.stageScan().enabled() || result == TimePointResult.FAILED)
+                && model_.devices().isUsingPLogic() && controller_ != null) {
             controller_.stopSPIMStateMachines();
         }
 
@@ -1242,6 +1282,45 @@ private boolean finishTimePoint(final ScapeAcquisitionSettings settings, final i
         return keepGoing;
     }
 
+    /**
+     * Stops any camera a failed time point left running, since the next time point cannot start
+     * a camera that is still running, and logs what each camera was doing.
+     *
+     * @param timeIndex the time point that failed
+     */
+    private void stopCamerasLeftRunning(final int timeIndex) {
+        // which cameras were still running, and what is left in the buffer, tell a frame that
+        // never came from one that came late
+        final int imagesLeft = core_.getRemainingImageCount();
+        final List<String> cameras = stopCameraSequences(model_.devices().imagingCameras());
+        studio_.logs().logMessage("separate time points: after the failure at time point "
+                + timeIndex + ": " + String.join(", ", cameras)
+                + "; images left in the buffer: " + imagesLeft);
+    }
+
+    /**
+     * Reports the time points of the series that failed, once, so a series that ran unattended
+     * does not look whole.
+     *
+     * @param failedTimePoints the time points that failed
+     * @param completed the number of complete datasets
+     * @param numTimePoints the number of time points the series asked for
+     * @param seriesDir the folder holding the series
+     */
+    private void reportFailedTimePoints(final List<Integer> failedTimePoints,
+            final int completed, final int numTimePoints, final String seriesDir) {
+        if (failedTimePoints.isEmpty()) {
+            return;
+        }
+        final String which = failedTimePoints.stream()
+                .map(String::valueOf)
+                .collect(Collectors.joining(", "));
+        model_.logging().reportError("Time points that failed: " + which + ". Each is marked "
+                + "incomplete in its folder.\n\n" + completed + " of " + numTimePoints
+                + " datasets are complete in " + seriesDir
+                + ". The CoreLog has the error for each.");
+    }
+
     /**
      * Marks a dataset directory whose time point did not finish.
      *
@@ -1297,15 +1376,7 @@ void finish() {
         // --- Job A ---
 
         // Stop all cameras sequences
-        try {
-            for (CameraBase camera : cameras) {
-                if (core_.isSequenceRunning(camera.getDeviceName())) {
-                    core_.stopSequenceAcquisition(camera.getDeviceName());
-                }
-            }
-        } catch (Exception e) {
-            studio_.logs().logError("Could not stop camera sequences: " + e.getMessage());
-        }
+        stopCameraSequences(cameras);
 
         // clean up controller settings after acquisition
         // want to do this, even with demo cameras, so we can test everything else