Skip to content

Commit d9ea9b8

Browse files
bhouse-nexthopxxxxxxxxxxxxx
authored andcommitted
Fix defects found in review of per-offering overcommit
Two of these made the branch worse than what it replaced. An exempt VM was not charged for what it holds. The ratio was recorded on the VM only when it differed from 1, so an offering asking for a ratio of 1 in a cluster overcommitted ten times recorded nothing, and the charge fell back to the cluster's ratio - a tenth of what the VM occupies. The memory pinning still applied, so the host really did hold the memory it was not charged for. - record the ratio whenever it differs from the cluster's, which is exactly when the charge needs it Memory reclaim was disabled on every VM of every default installation. The trigger was "the resolved ratio is 1", and clusters ship with a ratio of 1, so every guest would have lost free page reporting at its next start on installations that opted into nothing. - trigger on the offering having explicitly asked, not on the resolved value happening to be 1 Capacity checks asked in different units to the charge. An offering that overrides its cluster's ratio occupies a different share of a host than its size suggests, but the allocator and the planner still asked for the unscaled size. A VM could pass the check and then fail to start, and hosts were ranked on the wrong numbers. - scale the request wherever room is checked, resolved from the offering since the VM has not started yet Also: an unreadable ratio no longer breaks a start or silently skips a refund. Tests: 13 new cases for what gets recorded on the VM and when reclaim is turned off, including the two cases above. Signed-off-by: Brad House <bhouse@nexthop.ai>
1 parent 866a37e commit d9ea9b8

7 files changed

Lines changed: 323 additions & 33 deletions

File tree

‎engine/components-api/src/main/java/com/cloud/capacity/CapacityManager.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -162,6 +162,12 @@ boolean checkIfHostHasCapacity(Host host, Integer cpu, long ram, boolean checkFr
162162

163163
void updateCapacityForHost(Host host);
164164

165+
/**
166+
* Expresses a VM's request in the units capacity is counted in, taking account of any overcommit
167+
* ratio its service offering sets in place of its cluster's.
168+
*/
169+
long scaleRequestForOffering(long serviceOfferingId, long clusterId, long requested, boolean forCpu);
170+
165171
/**
166172
* @param pool storage pool
167173
* @param templateForVmCreation template that will be used for vm creation

‎engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java‎

Lines changed: 51 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -1860,54 +1860,63 @@ private void addToNetworkNameMap(long networkId, long dataCenterId, Map<Long, St
18601860
private void updateOverCommitRatioForVmProfile(VirtualMachineProfile vmProfile, long clusterId) {
18611861
final ClusterDetailsVO clusterDetailCpu = _clusterDetailsDao.findDetail(clusterId, VmDetailConstants.CPU_OVER_COMMIT_RATIO);
18621862
final ClusterDetailsVO clusterDetailRam = _clusterDetailsDao.findDetail(clusterId, VmDetailConstants.MEMORY_OVER_COMMIT_RATIO);
1863+
final float clusterCpuRatio = Float.parseFloat(clusterDetailCpu.getValue());
1864+
final float clusterRamRatio = Float.parseFloat(clusterDetailRam.getValue());
18631865

1864-
final float cpuRatio = overCommitRatioForVm(vmProfile, VmDetailConstants.CPU_OVER_COMMIT_RATIO,
1865-
Float.parseFloat(clusterDetailCpu.getValue()));
1866-
final float ramRatio = overCommitRatioForVm(vmProfile, VmDetailConstants.MEMORY_OVER_COMMIT_RATIO,
1867-
Float.parseFloat(clusterDetailRam.getValue()));
1866+
final Float offeringCpuRatio = offeringOverCommitRatio(vmProfile, VmDetailConstants.CPU_OVER_COMMIT_RATIO);
1867+
final Float offeringRamRatio = offeringOverCommitRatio(vmProfile, VmDetailConstants.MEMORY_OVER_COMMIT_RATIO);
18681868

1869-
persistOverCommitRatio(vmProfile.getId(), VmDetailConstants.CPU_OVER_COMMIT_RATIO, cpuRatio);
1870-
persistOverCommitRatio(vmProfile.getId(), VmDetailConstants.MEMORY_OVER_COMMIT_RATIO, ramRatio);
1871-
persistMemoryReclaimFlag(vmProfile.getId(), ramRatio);
1869+
final float cpuRatio = offeringCpuRatio != null ? offeringCpuRatio : clusterCpuRatio;
1870+
final float ramRatio = offeringRamRatio != null ? offeringRamRatio : clusterRamRatio;
1871+
1872+
persistOverCommitRatio(vmProfile.getId(), VmDetailConstants.CPU_OVER_COMMIT_RATIO, cpuRatio, clusterCpuRatio);
1873+
persistOverCommitRatio(vmProfile.getId(), VmDetailConstants.MEMORY_OVER_COMMIT_RATIO, ramRatio, clusterRamRatio);
1874+
persistMemoryReclaimFlag(vmProfile.getId(), offeringRamRatio);
18721875

18731876
vmProfile.setCpuOvercommitRatio(cpuRatio);
18741877
vmProfile.setMemoryOvercommitRatio(ramRatio);
18751878
}
18761879

18771880
/**
1878-
* The overcommit ratio a VM should run at: its service offering's if that offering sets one,
1879-
* otherwise its cluster's.
1881+
* The overcommit ratio a VM's service offering asks for, or null when it does not ask for one
1882+
* and should inherit its cluster's.
18801883
*
1881-
* An offering setting a ratio of 1 in an overcommitted cluster is how infrastructure VMs are
1884+
* An offering setting a ratio of 1 on an overcommitted cluster is how infrastructure VMs are
18821885
* kept off the overcommit: they are charged their full request and hold all of it.
18831886
*/
1884-
private float overCommitRatioForVm(VirtualMachineProfile vmProfile, String key, float clusterRatio) {
1887+
protected Float offeringOverCommitRatio(VirtualMachineProfile vmProfile, String key) {
18851888
String offeringRatio = _serviceOfferingDetailsDao.getDetail(vmProfile.getServiceOfferingId(), key);
18861889
if (offeringRatio == null) {
1887-
return clusterRatio;
1890+
return null;
18881891
}
18891892
try {
18901893
float ratio = Float.parseFloat(offeringRatio);
18911894
if (ratio <= 0) {
18921895
logger.warn("Ignoring {} of {} on service offering {}: it must be greater than zero.",
18931896
key, offeringRatio, vmProfile.getServiceOfferingId());
1894-
return clusterRatio;
1897+
return null;
18951898
}
18961899
return ratio;
18971900
} catch (NumberFormatException e) {
18981901
logger.warn("Ignoring {} of {} on service offering {}: it is not a number.",
18991902
key, offeringRatio, vmProfile.getServiceOfferingId());
1900-
return clusterRatio;
1903+
return null;
19011904
}
19021905
}
19031906

19041907
/**
1905-
* A VM that is not overcommitted on memory should hold what it was given rather than lend the
1906-
* unused part back, otherwise it is only exempt in the books. The hypervisor reads this when
1907-
* the VM starts.
1908+
* Turns off returning the guest's unused pages to the host, for a VM whose offering has
1909+
* explicitly opted out of overcommit. Otherwise the exemption would hold in the books only.
1910+
*
1911+
* Only an explicit choice counts. An offering that says nothing inherits its cluster, and a
1912+
* cluster with no overcommit configured is the default everywhere - reading that as "pin the
1913+
* memory of every VM" would change behaviour for installations that opted into nothing.
1914+
*
1915+
* @param offeringRatio
1916+
* the ratio the offering asked for, or null if it asked for nothing
19081917
*/
1909-
private void persistMemoryReclaimFlag(long vmId, float memoryRatio) {
1910-
boolean disable = memoryRatio <= 1f;
1918+
protected void persistMemoryReclaimFlag(long vmId, Float offeringRatio) {
1919+
boolean disable = offeringRatio != null && offeringRatio <= 1f;
19111920
VMInstanceDetailVO existing = vmInstanceDetailsDao.findDetail(vmId, VmDetailConstants.MEMORY_RECLAIM_DISABLED);
19121921
if (disable && existing == null) {
19131922
vmInstanceDetailsDao.addDetail(vmId, VmDetailConstants.MEMORY_RECLAIM_DISABLED, "true", true);
@@ -1917,20 +1926,36 @@ private void persistMemoryReclaimFlag(long vmId, float memoryRatio) {
19171926
}
19181927

19191928
/**
1920-
* Records the ratio the VM is running at, so that capacity accounting can scale it correctly
1921-
* even after the cluster or the offering is changed underneath it.
1929+
* Records the ratio the VM is running at.
1930+
*
1931+
* Capacity is counted in cluster-overcommitted units, and the charge scales a VM by this detail,
1932+
* so it has to be present whenever the VM's ratio differs from its cluster's - otherwise the VM
1933+
* is charged as though it shared the cluster's ratio. It is also kept while the cluster is
1934+
* overcommitted at all, so that capacity accounting survives the cluster ratio being changed
1935+
* beneath a running VM.
19221936
*/
1923-
private void persistOverCommitRatio(long vmId, String key, float ratio) {
1937+
protected void persistOverCommitRatio(long vmId, String key, float ratio, float clusterRatio) {
1938+
boolean needed = ratio != clusterRatio || clusterRatio > 1f;
19241939
VMInstanceDetailVO existing = vmInstanceDetailsDao.findDetail(vmId, key);
1925-
if (existing == null) {
1926-
if (ratio != 1f) {
1927-
vmInstanceDetailsDao.addDetail(vmId, key, String.valueOf(ratio), true);
1940+
if (!needed) {
1941+
if (existing != null) {
1942+
vmInstanceDetailsDao.removeDetail(vmId, key);
19281943
}
1929-
} else if (Float.parseFloat(existing.getValue()) != ratio) {
1944+
return;
1945+
}
1946+
if (existing == null || parseRatioOrDefault(existing.getValue(), Float.NaN) != ratio) {
19301947
vmInstanceDetailsDao.addDetail(vmId, key, String.valueOf(ratio), true);
19311948
}
19321949
}
19331950

1951+
private float parseRatioOrDefault(String value, float fallback) {
1952+
try {
1953+
return Float.parseFloat(value);
1954+
} catch (NumberFormatException | NullPointerException e) {
1955+
return fallback;
1956+
}
1957+
}
1958+
19341959
/**
19351960
* Setting pod id to null can result in migration of Volumes across pods. This is not desirable for VMs which
19361961
* have a volume in Ready state (happens when a VM is shutdown and started again).
Lines changed: 191 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,191 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
package com.cloud.vm;
18+
19+
import org.junit.Test;
20+
import org.junit.runner.RunWith;
21+
import org.mockito.InjectMocks;
22+
import org.mockito.Mock;
23+
import org.mockito.Mockito;
24+
import org.mockito.junit.MockitoJUnitRunner;
25+
26+
import com.cloud.service.dao.ServiceOfferingDetailsDao;
27+
import com.cloud.vm.dao.VMInstanceDetailsDao;
28+
29+
import static org.junit.Assert.assertEquals;
30+
import static org.junit.Assert.assertNull;
31+
32+
/**
33+
* A service offering may override its cluster's overcommit ratio. These cover what gets recorded on
34+
* the VM as a result, which is what capacity accounting and the hypervisor then read.
35+
*/
36+
@RunWith(MockitoJUnitRunner.class)
37+
public class VirtualMachineManagerOverCommitTest {
38+
39+
private static final long VM_ID = 1L;
40+
private static final long OFFERING_ID = 5L;
41+
private static final String CPU = VmDetailConstants.CPU_OVER_COMMIT_RATIO;
42+
private static final String MEMORY = VmDetailConstants.MEMORY_OVER_COMMIT_RATIO;
43+
private static final String RECLAIM = VmDetailConstants.MEMORY_RECLAIM_DISABLED;
44+
45+
@Mock
46+
private VMInstanceDetailsDao vmInstanceDetailsDao;
47+
48+
@Mock
49+
private ServiceOfferingDetailsDao serviceOfferingDetailsDao;
50+
51+
@InjectMocks
52+
private VirtualMachineManagerImpl vmManager = new VirtualMachineManagerImpl();
53+
54+
private VirtualMachineProfile profile() {
55+
VirtualMachineProfile profile = Mockito.mock(VirtualMachineProfile.class);
56+
Mockito.lenient().when(profile.getId()).thenReturn(VM_ID);
57+
Mockito.lenient().when(profile.getServiceOfferingId()).thenReturn(OFFERING_ID);
58+
return profile;
59+
}
60+
61+
private void offeringSays(String key, String value) {
62+
Mockito.when(serviceOfferingDetailsDao.getDetail(OFFERING_ID, key)).thenReturn(value);
63+
}
64+
65+
private void vmDetailIs(String key, String value) {
66+
VMInstanceDetailVO detail = value == null ? null : Mockito.mock(VMInstanceDetailVO.class);
67+
if (detail != null) {
68+
Mockito.lenient().when(detail.getValue()).thenReturn(value);
69+
}
70+
Mockito.lenient().when(vmInstanceDetailsDao.findDetail(VM_ID, key)).thenReturn(detail);
71+
}
72+
73+
// --- what the offering asks for ---
74+
75+
@Test
76+
public void testOfferingWithNoDetailInheritsTheCluster() {
77+
offeringSays(CPU, null);
78+
assertNull(vmManager.offeringOverCommitRatio(profile(), CPU));
79+
}
80+
81+
@Test
82+
public void testOfferingRatioIsUsed() {
83+
offeringSays(CPU, "1");
84+
assertEquals(1.0f, vmManager.offeringOverCommitRatio(profile(), CPU), 1e-6);
85+
}
86+
87+
@Test
88+
public void testNonsenseOfferingRatioInheritsTheCluster() {
89+
offeringSays(CPU, "not a number");
90+
assertNull(vmManager.offeringOverCommitRatio(profile(), CPU));
91+
}
92+
93+
@Test
94+
public void testNonPositiveOfferingRatioInheritsTheCluster() {
95+
offeringSays(CPU, "0");
96+
assertNull(vmManager.offeringOverCommitRatio(profile(), CPU));
97+
}
98+
99+
// --- what gets recorded on the VM ---
100+
101+
@Test
102+
public void testAnExemptOfferingRecordsItsRatioSoCapacityCanScaleIt() {
103+
// ratio 1 in a cluster overcommitted 10 times: without the detail the VM would be charged
104+
// a tenth of what it actually holds
105+
vmDetailIs(MEMORY, null);
106+
107+
vmManager.persistOverCommitRatio(VM_ID, MEMORY, 1.0f, 10.0f);
108+
109+
Mockito.verify(vmInstanceDetailsDao).addDetail(VM_ID, MEMORY, "1.0", true);
110+
}
111+
112+
@Test
113+
public void testAnOvercommittedClusterRecordsItsRatio() {
114+
vmDetailIs(CPU, null);
115+
116+
vmManager.persistOverCommitRatio(VM_ID, CPU, 10.0f, 10.0f);
117+
118+
Mockito.verify(vmInstanceDetailsDao).addDetail(VM_ID, CPU, "10.0", true);
119+
}
120+
121+
@Test
122+
public void testNothingIsRecordedWhenNobodyOvercommits() {
123+
vmDetailIs(CPU, null);
124+
125+
vmManager.persistOverCommitRatio(VM_ID, CPU, 1.0f, 1.0f);
126+
127+
Mockito.verify(vmInstanceDetailsDao, Mockito.never())
128+
.addDetail(Mockito.anyLong(), Mockito.anyString(), Mockito.anyString(), Mockito.anyBoolean());
129+
}
130+
131+
@Test
132+
public void testAStaleRatioIsRewritten() {
133+
vmDetailIs(CPU, "4.0");
134+
135+
vmManager.persistOverCommitRatio(VM_ID, CPU, 10.0f, 10.0f);
136+
137+
Mockito.verify(vmInstanceDetailsDao).addDetail(VM_ID, CPU, "10.0", true);
138+
}
139+
140+
@Test
141+
public void testAnUnreadableStoredRatioIsRewritten() {
142+
vmDetailIs(CPU, "rubbish");
143+
144+
vmManager.persistOverCommitRatio(VM_ID, CPU, 10.0f, 10.0f);
145+
146+
Mockito.verify(vmInstanceDetailsDao).addDetail(VM_ID, CPU, "10.0", true);
147+
}
148+
149+
// --- memory reclaim ---
150+
151+
@Test
152+
public void testReclaimIsDisabledOnlyWhenTheOfferingAsksForIt() {
153+
vmDetailIs(RECLAIM, null);
154+
155+
vmManager.persistMemoryReclaimFlag(VM_ID, 1.0f);
156+
157+
Mockito.verify(vmInstanceDetailsDao).addDetail(VM_ID, RECLAIM, "true", true);
158+
}
159+
160+
@Test
161+
public void testReclaimIsUntouchedWhenTheOfferingSaysNothing() {
162+
// the default everywhere: no offering override, and clusters ship with a ratio of 1.
163+
// reading that as "pin every VM's memory" would change behaviour for installations that
164+
// opted into nothing at all
165+
vmDetailIs(RECLAIM, null);
166+
167+
vmManager.persistMemoryReclaimFlag(VM_ID, null);
168+
169+
Mockito.verify(vmInstanceDetailsDao, Mockito.never())
170+
.addDetail(Mockito.anyLong(), Mockito.anyString(), Mockito.anyString(), Mockito.anyBoolean());
171+
}
172+
173+
@Test
174+
public void testReclaimStaysOnForAnOfferingThatStillOvercommits() {
175+
vmDetailIs(RECLAIM, null);
176+
177+
vmManager.persistMemoryReclaimFlag(VM_ID, 4.0f);
178+
179+
Mockito.verify(vmInstanceDetailsDao, Mockito.never())
180+
.addDetail(Mockito.anyLong(), Mockito.anyString(), Mockito.anyString(), Mockito.anyBoolean());
181+
}
182+
183+
@Test
184+
public void testReclaimIsRestoredWhenAnOfferingStopsExemptingItself() {
185+
vmDetailIs(RECLAIM, "true");
186+
187+
vmManager.persistMemoryReclaimFlag(VM_ID, null);
188+
189+
Mockito.verify(vmInstanceDetailsDao).removeDetail(VM_ID, RECLAIM);
190+
}
191+
}

0 commit comments

Comments
 (0)