Skip to content

Commit fb52c0e

Browse files
committed
Address review feedback on snapshot orphan overlay fix
1 parent 5ce4411 commit fb52c0e

4 files changed

Lines changed: 28 additions & 66 deletions

File tree

‎agent/src/main/java/com/cloud/agent/Agent.java‎

Lines changed: 2 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -666,29 +666,9 @@ protected void reconnect(final Link link, String preferredMSHost, boolean forTra
666666
protected void waitForCommandsAndReconnect(final Link link) {
667667
int inProgress = commandsInProgress.get();
668668
if (inProgress > 0) {
669-
logger.warn("Agent disconnected with {} command(s) still in progress. Waiting for them to complete before reconnecting.", inProgress);
670-
671-
long maxWaitMs = 300000;
672-
long waitedMs = 0;
673-
while (commandsInProgress.get() > 0 && waitedMs < maxWaitMs) {
674-
try {
675-
Thread.sleep(1000);
676-
} catch (InterruptedException e) {
677-
Thread.currentThread().interrupt();
678-
break;
679-
}
680-
waitedMs += 1000;
681-
}
682-
683-
if (commandsInProgress.get() > 0) {
684-
logger.warn("Timed out waiting for {} command(s) to complete after {} ms. Proceeding with reconnect — "
685-
+ "in-progress commands (e.g. snapshot merges) may not have completed.", commandsInProgress.get(), waitedMs);
686-
} else {
687-
logger.info("All in-progress commands completed. Proceeding with reconnect.");
688-
}
669+
logger.warn("Agent disconnected with {} command(s) still in progress. Reconnecting — local operations (e.g. blockcommit) will continue in their threads.",
670+
inProgress);
689671
}
690-
691-
logger.info("Reconnecting instead of exiting to avoid losing in-flight work (e.g. snapshot overlays).");
692672
reconnect(link);
693673
}
694674

‎agent/src/test/java/com/cloud/agent/AgentTest.java‎

Lines changed: 4 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -268,31 +268,20 @@ public void testWaitForCommandsAndReconnectNoCommandsInProgress() throws Configu
268268
spyAgent.waitForCommandsAndReconnect(mockLink);
269269

270270
verify(spyAgent).reconnect(mockLink);
271-
verify(logger).info(eq("Reconnecting instead of exiting to avoid losing in-flight work (e.g. snapshot overlays)."));
272271
}
273272

274273
@Test
275-
public void testWaitForCommandsAndReconnectCommandsCompleteBeforeTimeout() throws ConfigurationException {
274+
public void testWaitForCommandsAndReconnectWithCommandsInProgress() throws ConfigurationException {
276275
Agent spyAgent = spy(new Agent(shell, 1, serverResource));
277276
ReflectionTestUtils.setField(spyAgent, "logger", logger);
278277
doNothing().when(spyAgent).reconnect(any(Link.class));
279278

280279
Link mockLink = mock(Link.class);
281-
spyAgent.commandsInProgress.set(1);
282-
283-
Thread decrementThread = new Thread(() -> {
284-
try {
285-
Thread.sleep(500);
286-
} catch (InterruptedException e) {
287-
Thread.currentThread().interrupt();
288-
}
289-
spyAgent.commandsInProgress.set(0);
290-
});
291-
decrementThread.start();
292-
280+
spyAgent.commandsInProgress.set(3);
293281
spyAgent.waitForCommandsAndReconnect(mockLink);
294282

295283
verify(spyAgent).reconnect(mockLink);
296-
verify(logger).info(eq("All in-progress commands completed. Proceeding with reconnect."));
284+
verify(logger).warn(eq("Agent disconnected with {} command(s) still in progress. Reconnecting — local operations (e.g. blockcommit) will continue in their threads."),
285+
eq(3));
297286
}
298287
}

‎plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java‎

Lines changed: 17 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -302,6 +302,9 @@ public boolean configure(final String name, final Map<String, Object> params) th
302302

303303
incrementalSnapshotTimeout = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.INCREMENTAL_SNAPSHOT_TIMEOUT) * 1000;
304304
incrementalSnapshotRetryRebaseWait = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.INCREMENTAL_SNAPSHOT_RETRY_REBASE_WAIT) * 1000;
305+
306+
reconcileSnapshotMarkers();
307+
305308
return true;
306309
}
307310

@@ -2587,7 +2590,6 @@ private SnapshotObjectTO takeFullVolumeSnapshotOfRunningVm(CreateObjectCommand c
25872590

25882591
validateConvertResult(convertResult, snapshotPath);
25892592
} catch (LibvirtException e) {
2590-
removeSnapshotMarker(primaryPool, diskPath, snapshotName);
25912593
if (!e.getMessage().contains(LIBVIRT_OPERATION_NOT_SUPPORTED_MESSAGE)) {
25922594
throw e;
25932595
}
@@ -3780,7 +3782,7 @@ protected void reconcileSingleMarker(File marker) {
37803782
}
37813783

37823784
if (StringUtils.isNotEmpty(vmName)) {
3783-
if (reconcileWithRunningDomain(vmName, diskLabel, volumePath, overlayPath, marker)) {
3785+
if (reconcileWithRunningDomain(vmName, diskLabel, volumePath, overlayPath, snapshotName, marker)) {
37843786
return;
37853787
}
37863788
}
@@ -3793,7 +3795,7 @@ protected void reconcileSingleMarker(File marker) {
37933795
}
37943796

37953797
private boolean reconcileWithRunningDomain(String vmName, String diskLabel, String volumePath,
3796-
String overlayPath, File marker) {
3798+
String overlayPath, String snapshotName, File marker) {
37973799
try {
37983800
Connect conn = LibvirtConnection.getConnection();
37993801
Domain vm = conn.domainLookupByName(vmName);
@@ -3813,7 +3815,7 @@ private boolean reconcileWithRunningDomain(String vmName, String diskLabel, Stri
38133815

38143816
String actualDiskLabel = StringUtils.isNotEmpty(diskLabel) ? diskLabel : disk.getDiskLabel();
38153817
try {
3816-
resource.mergeDeltaIntoBaseFile(vm, actualDiskLabel, volumePath, null, true, "reconcile", null, conn);
3818+
resource.mergeDeltaIntoBaseFile(vm, actualDiskLabel, volumePath, null, true, snapshotName, null, conn);
38173819
logger.info("Successfully reconciled overlay [{}] back to base [{}] for VM [{}].", overlayPath, volumePath, vmName);
38183820
marker.delete();
38193821
return true;
@@ -3825,8 +3827,13 @@ private boolean reconcileWithRunningDomain(String vmName, String diskLabel, Stri
38253827
}
38263828

38273829
if (currentSource != null && currentSource.equals(volumePath)) {
3828-
logger.info("VM [{}] disk [{}] is already on base [{}]. Overlay [{}] is orphaned. Removing marker.",
3830+
logger.info("VM [{}] disk [{}] is already on base [{}]. Overlay [{}] is orphaned. Cleaning up.",
38293831
vmName, disk.getDiskLabel(), volumePath, overlayPath);
3832+
File orphanOverlay = new File(overlayPath);
3833+
if (orphanOverlay.exists()) {
3834+
orphanOverlay.delete();
3835+
logger.debug("Deleted orphaned overlay file [{}].", overlayPath);
3836+
}
38303837
marker.delete();
38313838
return true;
38323839
}
@@ -3854,21 +3861,11 @@ private void reconcileOffline(String volumePath, String overlayPath, File marker
38543861
return;
38553862
}
38563863

3857-
long markerTimestamp = 0;
3858-
try {
3859-
String content = new String(Files.readAllBytes(marker.toPath())).trim();
3860-
String[] parts = content.split("\\|", -1);
3861-
if (parts.length > 6) {
3862-
markerTimestamp = Long.parseLong(parts[6]);
3863-
}
3864-
} catch (Exception e) {
3865-
logger.debug("Could not parse marker timestamp", e);
3866-
}
3867-
3868-
if (markerTimestamp > 0 && baseFile.lastModified() > markerTimestamp) {
3869-
logger.error("Base file [{}] has been modified since snapshot marker was written (base modified={}, marker={}). "
3870-
+ "The overlay [{}] cannot be safely committed. Manual intervention required — keeping marker.",
3871-
volumePath, baseFile.lastModified(), markerTimestamp, overlayPath);
3864+
long overlayLastModified = overlayFile.lastModified();
3865+
if (overlayLastModified > 0 && baseFile.lastModified() > overlayLastModified) {
3866+
logger.error("Base file [{}] was modified after overlay [{}] was created (base mtime={}, overlay mtime={}). "
3867+
+ "Cannot safely commit — another process may have written to the base. Manual intervention required, keeping marker.",
3868+
volumePath, overlayPath, baseFile.lastModified(), overlayLastModified);
38723869
return;
38733870
}
38743871

‎plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessorTest.java‎

Lines changed: 5 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -899,33 +899,29 @@ public void testGetVmBlockJobLockReturnsSameLockForSameVm() throws Exception {
899899
}
900900

901901
@Test
902-
public void testReconcileOfflineBaseModifiedSinceMarker() throws Exception {
902+
public void testReconcileOfflineBaseModifiedAfterOverlay() throws Exception {
903903
File tempDir = Files.createTempDirectory("reconcile_offline_test").toFile();
904904
try {
905905
File baseFile = new File(tempDir, "base.qcow2");
906906
File overlayFile = new File(tempDir, "overlay.qcow2");
907907

908-
try (FileOutputStream fos = new FileOutputStream(baseFile)) { fos.write("base".getBytes()); }
909908
try (FileOutputStream fos = new FileOutputStream(overlayFile)) { fos.write("overlay".getBytes()); }
909+
overlayFile.setLastModified(System.currentTimeMillis() - 60000);
910910

911-
long markerTimestamp = System.currentTimeMillis() - 60000;
912-
911+
try (FileOutputStream fos = new FileOutputStream(baseFile)) { fos.write("base".getBytes()); }
913912
baseFile.setLastModified(System.currentTimeMillis());
914913

915914
File marker = new File(tempDir, ".snapshot_marker_test");
916-
String content = String.join("|",
917-
"pool-uuid", baseFile.getAbsolutePath(), "snap", "vm1", "vda",
918-
overlayFile.getAbsolutePath(), String.valueOf(markerTimestamp));
919915
try (FileOutputStream fos = new FileOutputStream(marker)) {
920-
fos.write(content.getBytes());
916+
fos.write("marker".getBytes());
921917
}
922918

923919
java.lang.reflect.Method method = KVMStorageProcessor.class.getDeclaredMethod(
924920
"reconcileOffline", String.class, String.class, File.class);
925921
method.setAccessible(true);
926922
method.invoke(storageProcessorSpy, baseFile.getAbsolutePath(), overlayFile.getAbsolutePath(), marker);
927923

928-
Assert.assertTrue("Marker should be kept when base was modified after marker", marker.exists());
924+
Assert.assertTrue("Marker should be kept when base was modified after overlay was created", marker.exists());
929925
} finally {
930926
File[] files = tempDir.listFiles();
931927
if (files != null) for (File f : files) f.delete();

0 commit comments

Comments
 (0)