Skip to content

Commit

Permalink
Merge branch 'master' into docs/observability
Browse files Browse the repository at this point in the history
  • Loading branch information
Rawven authored Oct 20, 2024
2 parents 2a595bb + 7ce173e commit 36ea6d4
Show file tree
Hide file tree
Showing 20 changed files with 408 additions and 76 deletions.
3 changes: 3 additions & 0 deletions CHANGES.md
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,9 @@ Apollo 2.4.0
* [Fix link namespace published items show missing some items](https://github.com/apolloconfig/apollo/pull/5240)
* [Feature: Add limit and whitelist for namespace count per appid+cluster](https://github.com/apolloconfig/apollo/pull/5228)
* [Feature support the observe status access-key for pre-check and logging only](https://github.com/apolloconfig/apollo/pull/5236)
* [Feature add limit for items count per namespace](https://github.com/apolloconfig/apollo/pull/5227)
* [Feature: Add ConfigService cache record stats function](https://github.com/apolloconfig/apollo/pull/5247)
* [RefreshAdminServerAddressTask supports dynamic configuration of time interval](https://github.com/apolloconfig/apollo/pull/5248)

------------------
All issues and pull requests are [here](https://github.com/apolloconfig/apollo/milestone/15?closed=1)
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
package com.ctrip.framework.apollo.adminservice.controller;

import com.ctrip.framework.apollo.adminservice.aop.PreAcquireNamespaceLock;
import com.ctrip.framework.apollo.biz.config.BizConfig;
import com.ctrip.framework.apollo.biz.entity.Commit;
import com.ctrip.framework.apollo.biz.entity.Item;
import com.ctrip.framework.apollo.biz.entity.Namespace;
Expand Down Expand Up @@ -58,13 +59,14 @@ public class ItemController {
private final NamespaceService namespaceService;
private final CommitService commitService;
private final ReleaseService releaseService;
private final BizConfig bizConfig;


public ItemController(final ItemService itemService, final NamespaceService namespaceService, final CommitService commitService, final ReleaseService releaseService) {
public ItemController(final ItemService itemService, final NamespaceService namespaceService, final CommitService commitService, final ReleaseService releaseService, final BizConfig bizConfig) {
this.itemService = itemService;
this.namespaceService = namespaceService;
this.commitService = commitService;
this.releaseService = releaseService;
this.bizConfig = bizConfig;
}

@PreAcquireNamespaceLock
Expand All @@ -78,6 +80,14 @@ public ItemDTO create(@PathVariable("appId") String appId,
if (managedEntity != null) {
throw BadRequestException.itemAlreadyExists(entity.getKey());
}

if (bizConfig.isItemNumLimitEnabled()) {
int itemCount = itemService.findNonEmptyItemCount(entity.getNamespaceId());
if (itemCount >= bizConfig.itemNumLimit()) {
throw new BadRequestException("The maximum number of items (" + bizConfig.itemNumLimit() + ") for this namespace has been reached. Current item count is " + itemCount + ".");
}
}

entity = itemService.save(entity);
dto = BeanUtils.transform(ItemDTO.class, entity);
commitService.createCommit(appId, clusterName, namespaceName, new ConfigChangeContentBuilder().createItem(entity).build(),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,8 @@ public class BizConfig extends RefreshableConfig {

private static final int DEFAULT_MAX_NAMESPACE_NUM = 200;

private static final int DEFAULT_MAX_ITEM_NUM = 1000;

private static final int DEFAULT_APPNAMESPACE_CACHE_REBUILD_INTERVAL = 60; //60s
private static final int DEFAULT_GRAY_RELEASE_RULE_SCAN_INTERVAL = 60; //60s
private static final int DEFAULT_APPNAMESPACE_CACHE_SCAN_INTERVAL = 1; //1s
Expand Down Expand Up @@ -117,6 +119,15 @@ public Set<String> namespaceNumLimitWhite() {
return Sets.newHashSet(getArrayProperty("namespace.num.limit.white", new String[0]));
}

public boolean isItemNumLimitEnabled() {
return getBooleanProperty("item.num.limit.enabled", false);
}

public int itemNumLimit() {
int limit = getIntProperty("item.num.limit", DEFAULT_MAX_ITEM_NUM);
return checkInt(limit, 5, Integer.MAX_VALUE, DEFAULT_MAX_ITEM_NUM);
}

public Map<Long, Integer> namespaceValueLengthLimitOverride() {
String namespaceValueLengthOverrideString = getValue("namespace.value.length.limit.override");
Map<Long, Integer> namespaceValueLengthOverride = Maps.newHashMap();
Expand Down Expand Up @@ -223,6 +234,10 @@ public boolean isConfigServiceCacheEnabled() {
return getBooleanProperty("config-service.cache.enabled", false);
}

public boolean isConfigServiceCacheStatsEnabled() {
return getBooleanProperty("config-service.cache.stats.enabled", false);
}

public boolean isConfigServiceCacheKeyIgnoreCase() {
return getBooleanProperty("config-service.cache.key.ignore-case", false);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -64,4 +64,7 @@ public interface ItemRepository extends PagingAndSortingRepository<Item, Long> {
@Query("update Item set IsDeleted = true, DeletedAt = ROUND(UNIX_TIMESTAMP(NOW(4))*1000), DataChange_LastModifiedBy = ?2 where NamespaceId = ?1 and IsDeleted = false")
int deleteByNamespaceId(long namespaceId, String operator);

@Query("select count(*) from Item where namespaceId = :namespaceId and key <>''")
int countByNamespaceIdAndFilterKeyEmpty(@Param("namespaceId") long namespaceId);

}
Original file line number Diff line number Diff line change
Expand Up @@ -135,6 +135,10 @@ public List<Item> findItemsModifiedAfterDate(long namespaceId, Date date) {
return itemRepository.findByNamespaceIdAndDataChangeLastModifiedTimeGreaterThan(namespaceId, date);
}

public int findNonEmptyItemCount(long namespaceId) {
return itemRepository.countByNamespaceIdAndFilterKeyEmpty(namespaceId);
}

public Page<Item> findItemsByKey(String key, Pageable pageable) {
return itemRepository.findByKey(key, pageable);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
*/
package com.ctrip.framework.apollo.biz.service;

import com.ctrip.framework.apollo.biz.config.BizConfig;
import com.ctrip.framework.apollo.biz.entity.Audit;
import com.ctrip.framework.apollo.biz.entity.Item;
import com.ctrip.framework.apollo.biz.entity.Namespace;
Expand All @@ -25,6 +26,7 @@
import com.ctrip.framework.apollo.common.exception.BadRequestException;
import com.ctrip.framework.apollo.common.exception.NotFoundException;
import com.ctrip.framework.apollo.common.utils.BeanUtils;
import com.ctrip.framework.apollo.core.utils.StringUtils;
import java.util.List;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
Expand All @@ -38,20 +40,23 @@ public class ItemSetService {
private final CommitService commitService;
private final ItemService itemService;
private final NamespaceService namespaceService;
private final BizConfig bizConfig;

public ItemSetService(
final AuditService auditService,
final CommitService commitService,
final ItemService itemService,
final NamespaceService namespaceService) {
final NamespaceService namespaceService,
final BizConfig bizConfig) {
this.auditService = auditService;
this.commitService = commitService;
this.itemService = itemService;
this.namespaceService = namespaceService;
this.bizConfig = bizConfig;
}

@Transactional
public ItemChangeSets updateSet(Namespace namespace, ItemChangeSets changeSets){
public ItemChangeSets updateSet(Namespace namespace, ItemChangeSets changeSets) {
return updateSet(namespace.getAppId(), namespace.getClusterName(), namespace.getNamespaceName(), changeSets);
}

Expand All @@ -64,6 +69,16 @@ public ItemChangeSets updateSet(String appId, String clusterName,
throw NotFoundException.namespaceNotFound(appId, clusterName, namespaceName);
}

if (bizConfig.isItemNumLimitEnabled()) {
int itemCount = itemService.findNonEmptyItemCount(namespace.getId());
int createItemCount = (int) changeSet.getCreateItems().stream().filter(item -> !StringUtils.isEmpty(item.getKey())).count();
int deleteItemCount = (int) changeSet.getDeleteItems().stream().filter(item -> !StringUtils.isEmpty(item.getKey())).count();
itemCount = itemCount + createItemCount - deleteItemCount;
if (itemCount > bizConfig.itemNumLimit()) {
throw new BadRequestException("The maximum number of items (" + bizConfig.itemNumLimit() + ") for this namespace has been reached. Current item count is " + itemCount + ".");
}
}

String operator = changeSet.getDataChangeLastModifiedBy();
ConfigChangeContentBuilder configChangeContentBuilder = new ConfigChangeContentBuilder();

Expand All @@ -84,7 +99,7 @@ public ItemChangeSets updateSet(String appId, String clusterName,

if (configChangeContentBuilder.hasContent()) {
commitService.createCommit(appId, clusterName, namespaceName, configChangeContentBuilder.build(),
changeSet.getDataChangeLastModifiedBy());
changeSet.getDataChangeLastModifiedBy());
}

return changeSet;
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,147 @@
/*
* Copyright 2024 Apollo Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package com.ctrip.framework.apollo.biz.service;

import static org.mockito.Mockito.when;

import com.ctrip.framework.apollo.biz.AbstractIntegrationTest;
import com.ctrip.framework.apollo.biz.config.BizConfig;
import com.ctrip.framework.apollo.biz.entity.Item;
import com.ctrip.framework.apollo.biz.entity.Namespace;
import com.ctrip.framework.apollo.common.dto.ItemChangeSets;
import com.ctrip.framework.apollo.common.dto.ItemDTO;
import com.ctrip.framework.apollo.common.exception.BadRequestException;
import org.junit.Assert;
import org.junit.Test;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.mock.mockito.MockBean;
import org.springframework.test.context.jdbc.Sql;

public class ItemSetServiceTest extends AbstractIntegrationTest {

@MockBean
private BizConfig bizConfig;

@Autowired
private ItemService itemService;
@Autowired
private NamespaceService namespaceService;

@Autowired
private ItemSetService itemSetService;

@Test
@Sql(scripts = "/sql/itemset-test.sql", executionPhase = Sql.ExecutionPhase.BEFORE_TEST_METHOD)
@Sql(scripts = "/sql/clean.sql", executionPhase = Sql.ExecutionPhase.AFTER_TEST_METHOD)
public void testUpdateSetWithoutItemNumLimit() {

when(bizConfig.itemKeyLengthLimit()).thenReturn(128);
when(bizConfig.itemValueLengthLimit()).thenReturn(20000);

when(bizConfig.isItemNumLimitEnabled()).thenReturn(false);
when(bizConfig.itemNumLimit()).thenReturn(5);

Namespace namespace = namespaceService.findOne(1L);

ItemChangeSets changeSets = new ItemChangeSets();
changeSets.addCreateItem(buildNormalItem(0L, namespace.getId(), "k6", "v6", "test item num limit", 6));
changeSets.addCreateItem(buildNormalItem(0L, namespace.getId(), "k7", "v7", "test item num limit", 7));

try {
itemSetService.updateSet(namespace, changeSets);
} catch (Exception e) {
Assert.fail();
}

int size = itemService.findNonEmptyItemCount(namespace.getId());
Assert.assertEquals(7, size);

}

@Test
@Sql(scripts = "/sql/itemset-test.sql", executionPhase = Sql.ExecutionPhase.BEFORE_TEST_METHOD)
@Sql(scripts = "/sql/clean.sql", executionPhase = Sql.ExecutionPhase.AFTER_TEST_METHOD)
public void testUpdateSetWithItemNumLimit() {

when(bizConfig.itemKeyLengthLimit()).thenReturn(128);
when(bizConfig.itemValueLengthLimit()).thenReturn(20000);

when(bizConfig.isItemNumLimitEnabled()).thenReturn(true);
when(bizConfig.itemNumLimit()).thenReturn(5);

Namespace namespace = namespaceService.findOne(1L);
Item item9901 = itemService.findOne(9901);
Item item9902 = itemService.findOne(9902);

ItemChangeSets changeSets = new ItemChangeSets();
changeSets.addUpdateItem(buildNormalItem(item9901.getId(), item9901.getNamespaceId(), item9901.getKey(), item9901.getValue() + " update", item9901.getComment(), item9901.getLineNum()));
changeSets.addDeleteItem(buildNormalItem(item9902.getId(), item9902.getNamespaceId(), item9902.getKey(), item9902.getValue() + " update", item9902.getComment(), item9902.getLineNum()));
changeSets.addCreateItem(buildNormalItem(0L, item9901.getNamespaceId(), "k6", "v6", "test item num limit", 6));
changeSets.addCreateItem(buildNormalItem(0L, item9901.getNamespaceId(), "k7", "v7", "test item num limit", 7));

try {
itemSetService.updateSet(namespace, changeSets);
Assert.fail();
} catch (Exception e) {
Assert.assertTrue(e instanceof BadRequestException);
}

int size = itemService.findNonEmptyItemCount(namespace.getId());
Assert.assertEquals(5, size);

}

@Test
@Sql(scripts = "/sql/itemset-test.sql", executionPhase = Sql.ExecutionPhase.BEFORE_TEST_METHOD)
@Sql(scripts = "/sql/clean.sql", executionPhase = Sql.ExecutionPhase.AFTER_TEST_METHOD)
public void testUpdateSetWithItemNumLimit2() {

when(bizConfig.itemKeyLengthLimit()).thenReturn(128);
when(bizConfig.itemValueLengthLimit()).thenReturn(20000);

when(bizConfig.isItemNumLimitEnabled()).thenReturn(true);
when(bizConfig.itemNumLimit()).thenReturn(5);

Namespace namespace = namespaceService.findOne(1L);
Item item9901 = itemService.findOne(9901);
Item item9902 = itemService.findOne(9902);

ItemChangeSets changeSets = new ItemChangeSets();
changeSets.addUpdateItem(buildNormalItem(item9901.getId(), item9901.getNamespaceId(), item9901.getKey(), item9901.getValue() + " update", item9901.getComment(), item9901.getLineNum()));
changeSets.addDeleteItem(buildNormalItem(item9902.getId(), item9902.getNamespaceId(), item9902.getKey(), item9902.getValue() + " update", item9902.getComment(), item9902.getLineNum()));
changeSets.addCreateItem(buildNormalItem(0L, item9901.getNamespaceId(), "k6", "v6", "test item num limit", 6));

try {
itemSetService.updateSet(namespace, changeSets);
} catch (Exception e) {
Assert.fail();
}

int size = itemService.findNonEmptyItemCount(namespace.getId());
Assert.assertEquals(5, size);

}


private ItemDTO buildNormalItem(Long id, Long namespaceId, String key, String value, String comment, int lineNum) {
ItemDTO item = new ItemDTO(key, value, comment, lineNum);
item.setId(id);
item.setNamespaceId(namespaceId);
return item;
}

}
25 changes: 25 additions & 0 deletions apollo-biz/src/test/resources/sql/itemset-test.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
--
-- Copyright 2024 Apollo Authors
--
-- Licensed under the Apache License, Version 2.0 (the "License");
-- you may not use this file except in compliance with the License.
-- You may obtain a copy of the License at
--
-- http://www.apache.org/licenses/LICENSE-2.0
--
-- Unless required by applicable law or agreed to in writing, software
-- distributed under the License is distributed on an "AS IS" BASIS,
-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-- See the License for the specific language governing permissions and
-- limitations under the License.
--

INSERT INTO "Namespace" (`Id`, `AppId`, `ClusterName`, `NamespaceName`, `IsDeleted`, `DataChange_CreatedBy`, `DataChange_LastModifiedBy`)VALUES(1,'testApp', 'default', 'application', 0, 'apollo', 'apollo');

INSERT INTO "Item" (`Id`, `NamespaceId`, "Key", "Type", "Value", `Comment`, `LineNum`)
VALUES
(9901, 1, 'k1', 0, 'v1', '', 1),
(9902, 1, 'k2', 2, 'v2', '', 2),
(9903, 1, 'k3', 0, 'v3', '', 3),
(9904, 1, 'k4', 0, 'v4', '', 4),
(9905, 1, 'k5', 0, 'v5', '', 5);
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@
import com.ctrip.framework.apollo.configservice.service.config.ConfigServiceWithCache;
import com.ctrip.framework.apollo.configservice.service.config.DefaultConfigService;
import com.ctrip.framework.apollo.configservice.util.AccessKeyUtil;
import io.micrometer.core.instrument.MeterRegistry;
import org.springframework.boot.web.servlet.FilterRegistrationBean;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
Expand All @@ -47,15 +48,18 @@ public class ConfigServiceAutoConfiguration {
private final ReleaseService releaseService;
private final ReleaseMessageService releaseMessageService;
private final GrayReleaseRuleRepository grayReleaseRuleRepository;
private final MeterRegistry meterRegistry;

public ConfigServiceAutoConfiguration(final BizConfig bizConfig,
final ReleaseService releaseService,
final ReleaseMessageService releaseMessageService,
final GrayReleaseRuleRepository grayReleaseRuleRepository) {
final ReleaseService releaseService,
final ReleaseMessageService releaseMessageService,
final GrayReleaseRuleRepository grayReleaseRuleRepository,
final MeterRegistry meterRegistry) {
this.bizConfig = bizConfig;
this.releaseService = releaseService;
this.releaseMessageService = releaseMessageService;
this.grayReleaseRuleRepository = grayReleaseRuleRepository;
this.meterRegistry = meterRegistry;
}

@Bean
Expand All @@ -67,7 +71,7 @@ public GrayReleaseRulesHolder grayReleaseRulesHolder() {
public ConfigService configService() {
if (bizConfig.isConfigServiceCacheEnabled()) {
return new ConfigServiceWithCache(releaseService, releaseMessageService,
grayReleaseRulesHolder(), bizConfig);
grayReleaseRulesHolder(), bizConfig, meterRegistry);
}
return new DefaultConfigService(releaseService, grayReleaseRulesHolder());
}
Expand Down
Loading

0 comments on commit 36ea6d4

Please sign in to comment.