From 3f3ea5edee4f4069858b2a58b4f3472f21e6834b Mon Sep 17 00:00:00 2001 From: EmanuelGi <35410958+EmanuelGi@users.noreply.github.com> Date: Thu, 29 Sep 2022 15:42:08 +0800 Subject: [PATCH 1/7] [ISSUE #8309] Sync services and endpoints from k8s (#9220) * [ISSUE#8309] Nacos supports synchronizing service metadata from k8s service discovery. (#7211) * Synchronize k8s services. * Synchronize k8s endpoints. * ApiClient uses the default configuration instead * Start registered Informers after registering handlers * Compare oldServicePorts and newServicePorts to determine portChange * Delete the logic of service renaming judgment * Fix error in the process of registering instances * Use independent log * add k8s sync switch and start informer optimization * start informer optimization --- console/pom.xml | 4 + .../src/main/resources/application.properties | 6 + distribution/conf/application.properties | 8 + .../conf/application.properties.example | 8 + distribution/conf/nacos-logback.xml | 22 + k8s-sync/pom.xml | 59 +++ .../alibaba/nacos/k8s/sync/K8sSyncConfig.java | 49 ++ .../alibaba/nacos/k8s/sync/K8sSyncServer.java | 438 ++++++++++++++++++ .../com/alibaba/nacos/k8s/sync/Loggers.java | 30 ++ pom.xml | 18 + 10 files changed, 642 insertions(+) create mode 100644 k8s-sync/pom.xml create mode 100644 k8s-sync/src/main/java/com/alibaba/nacos/k8s/sync/K8sSyncConfig.java create mode 100644 k8s-sync/src/main/java/com/alibaba/nacos/k8s/sync/K8sSyncServer.java create mode 100644 k8s-sync/src/main/java/com/alibaba/nacos/k8s/sync/Loggers.java diff --git a/console/pom.xml b/console/pom.xml index fb3785c88c5..27b8875c295 100644 --- a/console/pom.xml +++ b/console/pom.xml @@ -50,6 +50,10 @@ ${project.groupId} nacos-istio + + ${project.groupId} + nacos-k8s-sync + diff --git a/console/src/main/resources/application.properties b/console/src/main/resources/application.properties index 273d6dcd38e..7d7f6fd0448 100644 --- a/console/src/main/resources/application.properties +++ b/console/src/main/resources/application.properties @@ -151,7 +151,13 @@ nacos.core.auth.plugin.nacos.token.secret.key=SecretKey0123456789012345678901234 ### If turn on the MCP server: nacos.istio.mcp.server.enabled=false +#*************** K8s Related Configurations ***************# +### If turn on the K8s sync: +nacos.k8s.sync.enabled=false +### If use the Java API from an application outside a kubernetes cluster +#nacos.k8s.sync.outsideCluster=false +#nacos.k8s.sync.kubeConfig=/.kube/config ###*************** Add from 1.3.0 ***************### diff --git a/distribution/conf/application.properties b/distribution/conf/application.properties index 60b24d8232e..d5e71aa755b 100644 --- a/distribution/conf/application.properties +++ b/distribution/conf/application.properties @@ -177,6 +177,14 @@ nacos.core.auth.plugin.nacos.token.secret.key=SecretKey0123456789012345678901234 ### If turn on the MCP server: nacos.istio.mcp.server.enabled=false +#*************** K8s Related Configurations ***************# +### If turn on the K8s sync: +nacos.k8s.sync.enabled=false + +### If use the Java API from an application outside a kubernetes cluster +#nacos.k8s.sync.outsideCluster=false +#nacos.k8s.sync.kubeConfig=/.kube/config + #*************** Core Related Configurations ***************# ### set the WorkerID manually diff --git a/distribution/conf/application.properties.example b/distribution/conf/application.properties.example index e4359aa0842..1e93fcd707a 100644 --- a/distribution/conf/application.properties.example +++ b/distribution/conf/application.properties.example @@ -178,6 +178,14 @@ nacos.core.auth.server.identity.value=security ### If turn on the MCP server: nacos.istio.mcp.server.enabled=false +#*************** K8s Related Configurations ***************# +### If turn on the K8s sync: +nacos.k8s.sync.enabled=false + +### If use the Java API from an application outside a kubernetes cluster +#nacos.k8s.sync.outsideCluster=false +#nacos.k8s.sync.kubeConfig=/.kube/config + #*************** Core Related Configurations ***************# ### set the WorkerID manually diff --git a/distribution/conf/nacos-logback.xml b/distribution/conf/nacos-logback.xml index 15f7623a2ab..a540a7bfc8c 100644 --- a/distribution/conf/nacos-logback.xml +++ b/distribution/conf/nacos-logback.xml @@ -582,6 +582,23 @@ + + ${LOG_HOME}/k8s-sync-main.log + true + + ${LOG_HOME}/k8s-sync-main.log.%d{yyyy-MM-dd}.%i + 2GB + 7 + 7GB + true + + + %date %level %msg%n%n + UTF-8 + + + @@ -702,6 +719,11 @@ + + + + + diff --git a/k8s-sync/pom.xml b/k8s-sync/pom.xml new file mode 100644 index 00000000000..10fe2050bfc --- /dev/null +++ b/k8s-sync/pom.xml @@ -0,0 +1,59 @@ + + + + + + nacos-all + com.alibaba.nacos + ${revision} + ../pom.xml + + + 4.0.0 + nacos-k8s-sync + jar + + nacos-k8s-sync ${project.version} + http://nacos.io + + + + ${project.groupId} + nacos-core + + + ${project.groupId} + nacos-naming + + + + io.kubernetes + client-java-api + + + io.kubernetes + client-java + + + org.springframework + spring-beans + 5.3.20 + compile + + + \ No newline at end of file diff --git a/k8s-sync/src/main/java/com/alibaba/nacos/k8s/sync/K8sSyncConfig.java b/k8s-sync/src/main/java/com/alibaba/nacos/k8s/sync/K8sSyncConfig.java new file mode 100644 index 00000000000..95fe9fe36f8 --- /dev/null +++ b/k8s-sync/src/main/java/com/alibaba/nacos/k8s/sync/K8sSyncConfig.java @@ -0,0 +1,49 @@ +/* + * Copyright 1999-2018 Alibaba Group Holding Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.nacos.k8s.sync; + +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +/** + * Configurations for k8s integration. + * + * @author EmanuelGi + */ +@Component +public class K8sSyncConfig { + @Value("${nacos.k8s.sync.enabled:false}") + private boolean enabled = false; + + @Value("${nacos.k8s.sync.outsideCluster:false}") + private boolean outsideCluster = false; + + @Value("${nacos.k8s.sync.kubeConfig:}") + private String kubeConfig; + + public boolean isEnabled() { + return enabled; + } + + public boolean isOutsideCluster() { + return outsideCluster; + } + + public String getKubeConfig() { + return kubeConfig; + } +} diff --git a/k8s-sync/src/main/java/com/alibaba/nacos/k8s/sync/K8sSyncServer.java b/k8s-sync/src/main/java/com/alibaba/nacos/k8s/sync/K8sSyncServer.java new file mode 100644 index 00000000000..65c40ceec45 --- /dev/null +++ b/k8s-sync/src/main/java/com/alibaba/nacos/k8s/sync/K8sSyncServer.java @@ -0,0 +1,438 @@ +/* + * Copyright 1999-2022 Alibaba Group Holding Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.nacos.k8s.sync; + +import com.alibaba.nacos.api.common.Constants; +import com.alibaba.nacos.api.exception.NacosException; +import com.alibaba.nacos.api.naming.pojo.Instance; +import com.alibaba.nacos.naming.core.InstanceOperatorClientImpl; +import com.alibaba.nacos.naming.core.ServiceOperatorV2Impl; +import com.alibaba.nacos.naming.core.v2.ServiceManager; +import com.alibaba.nacos.naming.core.v2.pojo.Service; +import io.kubernetes.client.informer.ResourceEventHandler; +import io.kubernetes.client.informer.SharedIndexInformer; +import io.kubernetes.client.informer.SharedInformerFactory; +import io.kubernetes.client.informer.cache.Lister; +import io.kubernetes.client.openapi.ApiClient; +import io.kubernetes.client.openapi.Configuration; +import io.kubernetes.client.openapi.apis.CoreV1Api; +import io.kubernetes.client.openapi.models.V1EndpointAddress; +import io.kubernetes.client.openapi.models.V1EndpointSubset; +import io.kubernetes.client.openapi.models.V1Endpoints; +import io.kubernetes.client.openapi.models.V1EndpointsList; +import io.kubernetes.client.openapi.models.V1Service; +import io.kubernetes.client.openapi.models.V1ServiceList; +import io.kubernetes.client.openapi.models.V1ServicePort; +import io.kubernetes.client.util.CallGeneratorParams; +import io.kubernetes.client.util.ClientBuilder; +import io.kubernetes.client.util.KubeConfig; +import okhttp3.OkHttpClient; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import javax.annotation.PostConstruct; +import java.io.FileReader; +import java.io.IOException; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +/** + * Start and stop k8s-sync. + * + * @author EmanuelGi + */ +@Component +public class K8sSyncServer { + + @Autowired + private K8sSyncConfig k8sSyncConfig; + + @Autowired + private ServiceOperatorV2Impl serviceOperatorV2; + + @Autowired + private InstanceOperatorClientImpl instanceOperatorClient; + + private SharedInformerFactory factory; + + /** + * start. + * + * @throws IOException io exception + */ + @PostConstruct + public void start() throws IOException { + if (!k8sSyncConfig.isEnabled()) { + Loggers.MAIN.info("The Nacos k8s-sync is disabled."); + return; + } + Loggers.MAIN.info("Starting Nacos k8s-sync ..."); + startInformer(); + Runtime.getRuntime().addShutdownHook(new Thread() { + @Override + public void run() { + Loggers.MAIN.info("Stopping Nacos k8s-sync ..."); + K8sSyncServer.this.stop(); + Loggers.MAIN.info("Nacos k8s-sync stopped..."); + } + }); + } + + /** + * start informer. + * + * @throws IOException io exception + */ + public void startInformer() throws IOException { + ApiClient apiClient; + CoreV1Api coreV1Api; + + if (k8sSyncConfig.isOutsideCluster()) { + apiClient = getOutsideApiClient(); + coreV1Api = new CoreV1Api(); + } else { + coreV1Api = new CoreV1Api(); + apiClient = coreV1Api.getApiClient(); + } + + OkHttpClient httpClient = apiClient.getHttpClient().newBuilder().build(); + apiClient.setHttpClient(httpClient); + + factory = new SharedInformerFactory(apiClient); + SharedIndexInformer serviceInformer = + factory.sharedIndexInformerFor( + (CallGeneratorParams params) -> { + return coreV1Api.listServiceForAllNamespacesCall( + null, + null, + null, + null, + null, + null, + params.resourceVersion, + null, + params.timeoutSeconds, + params.watch, + null); + }, + V1Service.class, + V1ServiceList.class); + + SharedIndexInformer endpointInformer = + factory.sharedIndexInformerFor( + (CallGeneratorParams params) -> { + return coreV1Api.listEndpointsForAllNamespacesCall( + null, + null, + null, + null, + null, + null, + params.resourceVersion, + null, + params.timeoutSeconds, + params.watch, + null); + }, + V1Endpoints.class, + V1EndpointsList.class); + + serviceInformer.addEventHandler( + new ResourceEventHandler() { + @Override + public void onAdd(V1Service service) { + if (service.getMetadata() == null || service.getSpec() == null) { + return; + } + String serviceName = service.getMetadata().getName(); + String namespace = service.getMetadata().getNamespace(); + List servicePorts = service.getSpec().getPorts(); + try { + registerService(namespace, serviceName, servicePorts, false, endpointInformer); + Loggers.MAIN.info("add service, namespace:" + namespace + " serviceName: " + serviceName); + } catch (Exception e) { + Loggers.MAIN.warn("add service fail, message:" + e.getMessage() + " namespace:" + + namespace + " serviceName: " + serviceName); + } + } + + @Override + public void onUpdate(V1Service oldService, V1Service newService) { + if (oldService.getMetadata() == null || oldService.getSpec() == null + || newService.getMetadata() == null || newService.getSpec() == null) { + return; + } + List oldServicePorts = oldService.getSpec().getPorts(); + String serviceName = newService.getMetadata().getName(); + String namespace = newService.getMetadata().getNamespace(); + List newServicePorts = newService.getSpec().getPorts(); + boolean portChanged = compareServicePorts(oldServicePorts, newServicePorts); + try { + registerService(namespace, serviceName, newServicePorts, portChanged, endpointInformer); + Loggers.MAIN.info("update service, namespace: " + namespace + " serviceName: " + serviceName); + } catch (Exception e) { + Loggers.MAIN.warn("update service fail, message: " + e.getMessage() + " namespace: " + + namespace + " serviceName: " + serviceName); + } + } + + @Override + public void onDelete(V1Service service, boolean deletedFinalStateUnknown) { + if (service.getMetadata() == null) { + return; + } + String serviceName = service.getMetadata().getName(); + String namespace = service.getMetadata().getNamespace(); + try { + unregisterService(namespace, serviceName); + Loggers.MAIN.info("delete service, namespace:" + namespace + " serviceName:" + serviceName); + } catch (Exception e) { + Loggers.MAIN.warn("delete service fail, message: " + e.getMessage() + + " namespace:" + namespace + " serviceName:" + serviceName); + } + } + }); + + endpointInformer.addEventHandler(new ResourceEventHandler() { + @Override + public void onAdd(V1Endpoints obj) { + if (obj.getMetadata() == null) { + return; + } + String serviceName = obj.getMetadata().getName(); + String namespace = obj.getMetadata().getNamespace(); + Set addIpSet = getIpFromEndpoints(obj); + + //TODO 因为需要指定namespace，这里servicelister需要重新new，是否可以优化,比如说作为单例的放到map中 + Lister serviceLister = new Lister<>(serviceInformer.getIndexer(), namespace); + V1Service service = serviceLister.get(serviceName); + List servicePorts = service.getSpec().getPorts(); + try { + registerInstances(addIpSet, namespace, serviceName, servicePorts); + Loggers.MAIN.info("add instances, namespace:" + namespace + " serviceName: " + serviceName); + } catch (NacosException e) { + Loggers.MAIN.warn("add instances fail, message:" + e.getMessage() + " namespace:" + namespace + ", serviceName: " + serviceName); + } + } + + @Override + public void onUpdate(V1Endpoints oldObj, V1Endpoints newObj) { + if (newObj.getMetadata() == null) { + return; + } + String serviceName = newObj.getMetadata().getName(); + String namespace = newObj.getMetadata().getNamespace(); + Lister serviceLister = new Lister<>(serviceInformer.getIndexer(), namespace); + V1Service service = serviceLister.get(serviceName); + List servicePorts = service.getSpec().getPorts(); + try { + registerService(namespace, serviceName, servicePorts, false, endpointInformer); + Loggers.MAIN.info("update instances, namespace:" + namespace + " serviceName: " + serviceName); + } catch (NacosException e) { + Loggers.MAIN.warn("update instances fail, message:" + e.getMessage() + " namespace:" + + namespace + ", serviceName: " + serviceName); + } + } + + @Override + public void onDelete(V1Endpoints obj, boolean deletedFinalStateUnknown) { + if (obj.getMetadata() == null) { + return; + } + String serviceName = obj.getMetadata().getName(); + String namespace = obj.getMetadata().getNamespace(); + Set deleteIpSet = getIpFromEndpoints(obj); + try { + List oldInstanceList = instanceOperatorClient.listAllInstances(namespace, serviceName); + unregisterInstances(deleteIpSet, namespace, serviceName, oldInstanceList); + Loggers.MAIN.info("delete instances, namespace:" + namespace + ", serviceName: " + serviceName); + } catch (NacosException e) { + Loggers.MAIN.info("delete instances fail, namespace:" + namespace + ", serviceName: " + serviceName); + } + } + }); + factory.startAllRegisteredInformers(); + } + + /** + * create instance. + * + * @param ip instance ip + * @param targetPort instance port + * @param serviceName service name + * @param port service port + * @return instance + */ + public Instance createInstance(String ip, int targetPort, String serviceName, int port) { + Instance instance = new Instance(); + instance.setIp(ip); + instance.setPort(targetPort); + instance.setClusterName(serviceName); + instance.setEphemeral(false); + instance.setHealthy(true); + instance.addMetadata("servicePort", String.valueOf(port)); + return instance; + } + + /** + * register service. + * + * @param namespace service namespace + * @param serviceName service name + * @param servicePorts service ports + * @param portChanged port is changed or not + * @throws NacosException nacos exception during registering + */ + public void registerService(String namespace, String serviceName, List servicePorts, boolean portChanged, + SharedIndexInformer endpointInformer) throws NacosException { + //TODO defaultnamespace 常量 + + Service service = Service.newService(namespace, Constants.DEFAULT_GROUP, serviceName, false); + ServiceManager.getInstance().getSingleton(service); + + //NotifyCenter.publishEvent(new NamingTraceEvent.RegisterServiceTraceEvent(System.currentTimeMillis(), + // namespace, Constants.DEFAULT_GROUP, serviceName)); + + Set oldIpSet = new HashSet<>(); + List oldInstanceList = instanceOperatorClient.listAllInstances(namespace, serviceName); + for (Instance instance:oldInstanceList) { + oldIpSet.add(instance.getIp()); + } + Lister endpointLister = new Lister<>(endpointInformer.getIndexer(), namespace); + V1Endpoints endpoints = endpointLister.get(serviceName); + Set newIpSet = getIpFromEndpoints(endpoints); + + //unregister deleted instance + Set deleteIpSet = new HashSet<>(); + deleteIpSet.addAll(oldIpSet); + deleteIpSet.removeAll(newIpSet); + unregisterInstances(deleteIpSet, namespace, serviceName, oldInstanceList); + //register added instance + Set addIpSet = new HashSet<>(); + addIpSet.addAll(newIpSet); + if (!portChanged) { + addIpSet.removeAll(oldIpSet); + } + registerInstances(addIpSet, namespace, serviceName, servicePorts); + } + + /** + * unregister service. + * + * @param namespace service namespace + * @param serviceName service name + * @throws NacosException nacos exception during unregistering + */ + public void unregisterService(String namespace, String serviceName) throws NacosException { + List instancelist = instanceOperatorClient.listAllInstances(namespace, serviceName); + for (Instance instance:instancelist) { + instanceOperatorClient.removeInstance(namespace, serviceName, instance); + } + serviceOperatorV2.delete(namespace, serviceName); + } + + /** + * register instances. + * + * @param addIpSet add ip set + * @param namespace service namespace + * @param serviceName service name + * @param servicePorts servie ports + * @throws NacosException nacos exception during registering instances + */ + public void registerInstances(Set addIpSet, String namespace, String serviceName, + List servicePorts) throws NacosException { + for (V1ServicePort servicePort:servicePorts) { + int port = servicePort.getPort(); + if (!servicePort.getTargetPort().isInteger()) { + continue; + } + int targetPort = servicePort.getTargetPort().getIntValue(); + for (String ip:addIpSet) { + Instance instance = createInstance(ip, targetPort, serviceName, port); + instanceOperatorClient.registerInstance(namespace, serviceName, instance); + } + } + //TODO：register instance后是否需要发布事件 + } + + /** + * unregister instances. + * + * @param deleteIpSet delete ip set + * @param namespace service namespace + * @param serviceName service name + * @param oldInstanceList old instance list from nacos service + */ + public void unregisterInstances(Set deleteIpSet, String namespace, String serviceName, + List oldInstanceList) { + for (Instance instance:oldInstanceList) { + if (deleteIpSet.contains(instance.getIp())) { + instanceOperatorClient.removeInstance(namespace, serviceName, instance); + } + } + } + + public Set getIpFromEndpoints(V1Endpoints endpoints) { + Set ipSet = new HashSet<>(); + List endpointSubsetList = endpoints.getSubsets(); + for (V1EndpointSubset endpointSubset:endpointSubsetList) { + for (V1EndpointAddress endpointAddress:endpointSubset.getAddresses()) { + ipSet.add(endpointAddress.getIp()); + } + } + return ipSet; + } + + /** + * compare oldServicePorts and newServicePorts. + * + * @param oldServicePorts old service ports list + * @param newServicePorts new service ports list + */ + public boolean compareServicePorts(List oldServicePorts, List newServicePorts) { + if (oldServicePorts.size() != newServicePorts.size()) { + return false; + } + return oldServicePorts.containsAll(newServicePorts) && newServicePorts.containsAll(oldServicePorts); + } + + /** + * use the Java API from an application outside a kubernetes cluster. + * you should load a kubeConfig to generate apiClient instead of getting it from coreV1api. + */ + public ApiClient getOutsideApiClient() throws IOException { + String kubeConfigPath = k8sSyncConfig.getKubeConfig(); + + // loading the out-of-cluster config, a kubeconfig from file-system + ApiClient apiClient = ClientBuilder.kubeconfig(KubeConfig.loadKubeConfig(new FileReader(kubeConfigPath))).build(); + + // set the global default api-client to the in-cluster one from above + Configuration.setDefaultApiClient(apiClient); + return apiClient; + } + + /** + * stop. + */ + public void stop() { + if (factory != null) { + factory.stopAllRegisteredInformers(); + } + } +} diff --git a/k8s-sync/src/main/java/com/alibaba/nacos/k8s/sync/Loggers.java b/k8s-sync/src/main/java/com/alibaba/nacos/k8s/sync/Loggers.java new file mode 100644 index 00000000000..b4c028d2f09 --- /dev/null +++ b/k8s-sync/src/main/java/com/alibaba/nacos/k8s/sync/Loggers.java @@ -0,0 +1,30 @@ +/* + * Copyright 1999-2018 Alibaba Group Holding Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.nacos.k8s.sync; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Loggers Holder. + * + * @author EmanuelGi + */ +public class Loggers { + + public static final Logger MAIN = LoggerFactory.getLogger("com.alibaba.nacos.k8s.sync.main"); +} diff --git a/pom.xml b/pom.xml index 72403f46669..072cd1727b9 100644 --- a/pom.xml +++ b/pom.xml @@ -595,6 +595,7 @@ sys plugin plugin-default-impl + k8s-sync @@ -698,6 +699,11 @@ nacos-istio ${project.version} + + ${project.groupId} + nacos-k8s-sync + ${project.version} + ${project.groupId} nacos-consistency @@ -975,6 +981,18 @@ 2.9.0 compile + + + io.kubernetes + client-java-api + 14.0.0 + + + + io.kubernetes + client-java + 14.0.0 + From a4eb614fa0735d2e473ca86f7c44cbce754fa014 Mon Sep 17 00:00:00 2001 From: RocketEngine26 Date: Fri, 21 Oct 2022 14:08:01 +0800 Subject: [PATCH 2/7] =?UTF-8?q?=E3=80=90summer=5Fissue#8313=E3=80=91Ecolog?= =?UTF-8?q?ical=20Construction=20and=20optimization=20of=20NACOS=20Service?= =?UTF-8?q?=20Mesh=20(#9158)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Change the polling mechanism to listen for event changes;Implement the update design of the data layer、Debounce、CDE, EDS, incremental EDS, incremental MCP. * Remove unnecessary changes and logs * Simplify the code * Simplify * Add the delta API description * Fixed a bug * Update application.properties * Modify the event merge mechanism; remove unnecessary classes * Initialize --- .../alibaba/nacos/istio/api/ApiGenerator.java | 15 +- .../nacos/istio/api/ApiGeneratorFactory.java | 6 + .../alibaba/nacos/istio/common/Debounce.java | 119 +++++++++ .../com/alibaba/nacos/istio/common/Event.java | 39 --- .../nacos/istio/common/EventProcessor.java | 33 +-- .../alibaba/nacos/istio/common/EventType.java | 33 --- .../istio/common/NacosResourceManager.java | 12 +- .../NacosServiceInfoResourceWatcher.java | 184 ++++++++++--- .../nacos/istio/common/ResourceSnapshot.java | 50 ++-- .../nacos/istio/common/WatchedStatus.java | 23 ++ .../nacos/istio/mcp/EmptyMcpGenerator.java | 9 +- .../nacos/istio/mcp/NacosMcpService.java | 49 ++-- .../istio/mcp/ServiceEntryMcpGenerator.java | 35 ++- .../alibaba/nacos/istio/misc/IstioConfig.java | 25 +- .../nacos/istio/model/IstioEndpoint.java | 133 ++++++++++ .../nacos/istio/model/IstioResources.java | 42 +++ .../nacos/istio/model/IstioService.java | 74 +++--- .../nacos/istio/model/PushRequest.java | 89 +++++++ .../nacos/istio/server/IstioServer.java | 10 +- .../nacos/istio/util/IstioCrdUtil.java | 123 +++++---- .../nacos/istio/util/IstioExecutor.java | 33 +-- .../alibaba/nacos/istio/xds/CdsGenerator.java | 94 +++++++ .../nacos/istio/xds/DeltaConnection.java | 58 ++++ .../alibaba/nacos/istio/xds/EdsGenerator.java | 171 ++++++++++++ .../nacos/istio/xds/EmptyXdsGenerator.java | 14 +- .../nacos/istio/xds/NacosXdsService.java | 248 +++++++++++++++--- .../istio/xds/ServiceEntryXdsGenerator.java | 82 +++++- .../core/InstanceOperatorClientImpl.java | 5 +- .../naming/core/ServiceOperatorV2Impl.java | 5 +- .../v2/event/metadata/InfoChangeEvent.java | 66 +++++ 30 files changed, 1514 insertions(+), 365 deletions(-) create mode 100644 istio/src/main/java/com/alibaba/nacos/istio/common/Debounce.java delete mode 100644 istio/src/main/java/com/alibaba/nacos/istio/common/Event.java delete mode 100644 istio/src/main/java/com/alibaba/nacos/istio/common/EventType.java create mode 100644 istio/src/main/java/com/alibaba/nacos/istio/model/IstioEndpoint.java create mode 100644 istio/src/main/java/com/alibaba/nacos/istio/model/IstioResources.java create mode 100644 istio/src/main/java/com/alibaba/nacos/istio/model/PushRequest.java create mode 100644 istio/src/main/java/com/alibaba/nacos/istio/xds/CdsGenerator.java create mode 100644 istio/src/main/java/com/alibaba/nacos/istio/xds/DeltaConnection.java create mode 100644 istio/src/main/java/com/alibaba/nacos/istio/xds/EdsGenerator.java create mode 100644 naming/src/main/java/com/alibaba/nacos/naming/core/v2/event/metadata/InfoChangeEvent.java diff --git a/istio/src/main/java/com/alibaba/nacos/istio/api/ApiGenerator.java b/istio/src/main/java/com/alibaba/nacos/istio/api/ApiGenerator.java index 568ee349d42..8be7aaccacc 100644 --- a/istio/src/main/java/com/alibaba/nacos/istio/api/ApiGenerator.java +++ b/istio/src/main/java/com/alibaba/nacos/istio/api/ApiGenerator.java @@ -16,7 +16,8 @@ package com.alibaba.nacos.istio.api; -import com.alibaba.nacos.istio.common.ResourceSnapshot; +import com.alibaba.nacos.istio.model.PushRequest; +import io.envoyproxy.envoy.service.discovery.v3.Resource; import java.util.List; @@ -30,8 +31,16 @@ public interface ApiGenerator { /** * Generate data based on resource snapshot. * - * @param resourceSnapshot Resource snapshot + * @param pushRequest Push Request * @return data */ - List generate(ResourceSnapshot resourceSnapshot); + List generate(PushRequest pushRequest); + + /** + * Delta generate data based on resource snapshot. + * + * @param pushRequest Push Request + * @return data + */ + List deltaGenerate(PushRequest pushRequest); } diff --git a/istio/src/main/java/com/alibaba/nacos/istio/api/ApiGeneratorFactory.java b/istio/src/main/java/com/alibaba/nacos/istio/api/ApiGeneratorFactory.java index d54dd78a8de..75665c56ec9 100644 --- a/istio/src/main/java/com/alibaba/nacos/istio/api/ApiGeneratorFactory.java +++ b/istio/src/main/java/com/alibaba/nacos/istio/api/ApiGeneratorFactory.java @@ -18,6 +18,8 @@ import com.alibaba.nacos.istio.mcp.EmptyMcpGenerator; import com.alibaba.nacos.istio.mcp.ServiceEntryMcpGenerator; +import com.alibaba.nacos.istio.xds.CdsGenerator; +import com.alibaba.nacos.istio.xds.EdsGenerator; import com.alibaba.nacos.istio.xds.EmptyXdsGenerator; import com.alibaba.nacos.istio.xds.ServiceEntryXdsGenerator; import org.springframework.stereotype.Component; @@ -41,6 +43,10 @@ public ApiGeneratorFactory() { apiGeneratorMap.put(SERVICE_ENTRY_PROTO_PACKAGE, ServiceEntryXdsGenerator.getInstance()); // TODO Support other api generator + //xds + apiGeneratorMap.put(CLUSTER_TYPE, CdsGenerator.getInstance()); + apiGeneratorMap.put(ENDPOINT_TYPE, EdsGenerator.getInstance()); + // mcp apiGeneratorMap.put(SERVICE_ENTRY_COLLECTION, ServiceEntryMcpGenerator.getInstance()); } diff --git a/istio/src/main/java/com/alibaba/nacos/istio/common/Debounce.java b/istio/src/main/java/com/alibaba/nacos/istio/common/Debounce.java new file mode 100644 index 00000000000..a65ebf12895 --- /dev/null +++ b/istio/src/main/java/com/alibaba/nacos/istio/common/Debounce.java @@ -0,0 +1,119 @@ +/* + * Copyright 1999-2022 Alibaba Group Holding Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.alibaba.nacos.istio.common; + +import com.alibaba.nacos.istio.misc.IstioConfig; +import com.alibaba.nacos.istio.model.PushRequest; + +import java.util.Date; +import java.util.Queue; +import java.util.Timer; +import java.util.TimerTask; +import java.util.concurrent.Callable; + +/**. + * @author RocketEngine26 + * @date 2022/8/20 9:05 + */ +public class Debounce implements Callable { + private Date startDebounce; + + private Date lastConfigUpdateTime; + + private final IstioConfig istioConfig; + + private final Queue pushRequestQueue; + + private PushRequest pushRequest; + + private int debouncedEvents = 0; + + private boolean free = true; + + private boolean flag = false; + + public Debounce(Queue pushRequestQueue, IstioConfig istioConfig) { + this.pushRequestQueue = pushRequestQueue; + this.istioConfig = istioConfig; + } + + @Override + public PushRequest call() throws Exception { + while (true) { + if (flag) { + return pushRequest; + } + + PushRequest otherRequest = pushRequestQueue.poll(); + + if (otherRequest != null) { + lastConfigUpdateTime = new Date(); + if (debouncedEvents == 0) { + startDebounce = lastConfigUpdateTime; + pushRequest = otherRequest; + new Timer().schedule(new TimerTask() { + @Override + public void run() { + if (free) { + try { + pushWorker(); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + } + }, istioConfig.getDebounceAfter()); + } else { + merge(otherRequest); + } + debouncedEvents++; + } + } + } + + private void pushWorker() { + long eventDelay = System.currentTimeMillis() - startDebounce.getTime(); + long quietTime = System.currentTimeMillis() - lastConfigUpdateTime.getTime(); + + if (eventDelay > istioConfig.getDebounceMax() || quietTime > istioConfig.getDebounceAfter()) { + if (pushRequest != null) { + free = false; + flag = true; + debouncedEvents = 0; + } + } else { + new Timer().schedule(new TimerTask() { + @Override + public void run() { + if (free) { + try { + pushWorker(); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + } + }, istioConfig.getDebounceAfter() - quietTime); + } + } + + private void merge(PushRequest otherRequest) { + pushRequest.getReason().addAll(otherRequest.getReason()); + pushRequest.setFull(pushRequest.isFull() || otherRequest.isFull()); + } +} diff --git a/istio/src/main/java/com/alibaba/nacos/istio/common/Event.java b/istio/src/main/java/com/alibaba/nacos/istio/common/Event.java deleted file mode 100644 index a681d4d96ec..00000000000 --- a/istio/src/main/java/com/alibaba/nacos/istio/common/Event.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright 1999-2018 Alibaba Group Holding Ltd. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.alibaba.nacos.istio.common; - -/** - * @author special.fy - */ -public class Event { - - public static final Event SERVICE_UPDATE_EVENT = new Event(EventType.Service); - - private EventType type; - - public Event(EventType type) { - this.type = type; - } - - public EventType getType() { - return type; - } - - public void setType(EventType type) { - this.type = type; - } -} diff --git a/istio/src/main/java/com/alibaba/nacos/istio/common/EventProcessor.java b/istio/src/main/java/com/alibaba/nacos/istio/common/EventProcessor.java index b7b7222b73d..ccdba5b53ff 100644 --- a/istio/src/main/java/com/alibaba/nacos/istio/common/EventProcessor.java +++ b/istio/src/main/java/com/alibaba/nacos/istio/common/EventProcessor.java @@ -18,6 +18,7 @@ import com.alibaba.nacos.istio.mcp.NacosMcpService; import com.alibaba.nacos.istio.misc.Loggers; +import com.alibaba.nacos.istio.model.PushRequest; import com.alibaba.nacos.istio.util.IstioExecutor; import com.alibaba.nacos.istio.xds.NacosXdsService; import com.alibaba.nacos.sys.utils.ApplicationUtils; @@ -48,22 +49,22 @@ public class EventProcessor implements ApplicationListener events; + private final BlockingQueue requests; public EventProcessor() { - events = new ArrayBlockingQueue<>(20); + requests = new ArrayBlockingQueue<>(20); } /** * notify. * - * @param event event + * @param pushRequest push request */ - public void notify(Event event) { + public void notify(PushRequest pushRequest) { try { - events.put(event); + requests.put(pushRequest); } catch (InterruptedException e) { - Loggers.MAIN.warn("There are too many events, this event {} will be ignored.", event.getType()); + Loggers.MAIN.warn("There are too many events, this event {} will be ignored.", pushRequest.getReason()); // set the interrupted flag Thread.currentThread().interrupt(); } @@ -92,15 +93,15 @@ private class Consumer extends Thread { public void run() { Future task = null; boolean hasNewEvent = false; - Event lastEvent = null; + PushRequest lastEvent = null; while (true) { try { // Today we only care about service event, // so we simply ignore event until the last task has been completed. - Event event = events.poll(MAX_WAIT_EVENT_TIME, TimeUnit.MILLISECONDS); - if (event != null) { + PushRequest pushRequest = requests.poll(MAX_WAIT_EVENT_TIME, TimeUnit.MILLISECONDS); + if (pushRequest != null) { hasNewEvent = true; - lastEvent = event; + lastEvent = pushRequest; } if (hasClientConnection() && needNewTask(hasNewEvent, task)) { task = IstioExecutor.asyncHandleEvent(new EventHandleTask(lastEvent)); @@ -126,17 +127,19 @@ private boolean needNewTask(boolean hasNewEvent, Future task) { private class EventHandleTask implements Callable { - private final Event event; + private final PushRequest pushRequest; - EventHandleTask(Event event) { - this.event = event; + EventHandleTask(PushRequest pushRequest) { + this.pushRequest = pushRequest; } @Override public Void call() throws Exception { ResourceSnapshot snapshot = resourceManager.createResourceSnapshot(); - nacosXdsService.handleEvent(snapshot, event); - nacosMcpService.handleEvent(snapshot, event); + pushRequest.setResourceSnapshot(snapshot); + nacosXdsService.handleEvent(pushRequest); + nacosXdsService.handleDeltaEvent(pushRequest); + nacosMcpService.handleEvent(pushRequest); return null; } } diff --git a/istio/src/main/java/com/alibaba/nacos/istio/common/EventType.java b/istio/src/main/java/com/alibaba/nacos/istio/common/EventType.java deleted file mode 100644 index 0062b8e72f5..00000000000 --- a/istio/src/main/java/com/alibaba/nacos/istio/common/EventType.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright 1999-2018 Alibaba Group Holding Ltd. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.alibaba.nacos.istio.common; - -/** - * @author special.fy - */ -public enum EventType { - - /** - * The service info of nacos changes. - */ - Service, - - /** - * The endpoints of service change. - */ - Endpoint; -} diff --git a/istio/src/main/java/com/alibaba/nacos/istio/common/NacosResourceManager.java b/istio/src/main/java/com/alibaba/nacos/istio/common/NacosResourceManager.java index ef3f2a1928d..f3127b318d8 100644 --- a/istio/src/main/java/com/alibaba/nacos/istio/common/NacosResourceManager.java +++ b/istio/src/main/java/com/alibaba/nacos/istio/common/NacosResourceManager.java @@ -18,7 +18,6 @@ import com.alibaba.nacos.istio.misc.IstioConfig; import com.alibaba.nacos.istio.model.IstioService; -import com.alibaba.nacos.istio.util.IstioExecutor; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; @@ -39,12 +38,7 @@ public class NacosResourceManager { private IstioConfig istioConfig; public NacosResourceManager() { - resourceSnapshot = new ResourceSnapshot(); - } - - public void start() { - IstioExecutor.registerNacosResourceWatcher(serviceInfoResourceWatcher, istioConfig.getMcpPushInterval() * 2L, - istioConfig.getMcpPushInterval()); + resourceSnapshot = new ResourceSnapshot(istioConfig); } public Map services() { @@ -67,9 +61,9 @@ public void initResourceSnapshot() { ResourceSnapshot resourceSnapshot = getResourceSnapshot(); resourceSnapshot.initResourceSnapshot(this); } - + public ResourceSnapshot createResourceSnapshot() { - ResourceSnapshot resourceSnapshot = new ResourceSnapshot(); + ResourceSnapshot resourceSnapshot = new ResourceSnapshot(istioConfig); resourceSnapshot.initResourceSnapshot(this); setResourceSnapshot(resourceSnapshot); return resourceSnapshot; diff --git a/istio/src/main/java/com/alibaba/nacos/istio/common/NacosServiceInfoResourceWatcher.java b/istio/src/main/java/com/alibaba/nacos/istio/common/NacosServiceInfoResourceWatcher.java index 296586d7d6b..ba588e6d8b3 100644 --- a/istio/src/main/java/com/alibaba/nacos/istio/common/NacosServiceInfoResourceWatcher.java +++ b/istio/src/main/java/com/alibaba/nacos/istio/common/NacosServiceInfoResourceWatcher.java @@ -17,85 +17,185 @@ package com.alibaba.nacos.istio.common; import com.alibaba.nacos.api.naming.pojo.ServiceInfo; +import com.alibaba.nacos.common.notify.NotifyCenter; +import com.alibaba.nacos.common.notify.listener.SmartSubscriber; +import com.alibaba.nacos.istio.misc.IstioConfig; import com.alibaba.nacos.istio.model.IstioService; +import com.alibaba.nacos.istio.model.PushRequest; import com.alibaba.nacos.istio.util.IstioCrdUtil; import com.alibaba.nacos.naming.core.v2.ServiceManager; +import com.alibaba.nacos.naming.core.v2.event.client.ClientOperationEvent; +import com.alibaba.nacos.naming.core.v2.event.metadata.InfoChangeEvent; +import com.alibaba.nacos.naming.core.v2.event.publisher.NamingEventPublisherFactory; import com.alibaba.nacos.naming.core.v2.index.ServiceStorage; import com.alibaba.nacos.naming.core.v2.pojo.Service; import org.springframework.beans.factory.annotation.Autowired; import java.util.HashMap; -import java.util.HashSet; +import java.util.LinkedList; +import java.util.List; import java.util.Map; +import java.util.Queue; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; + +import static com.alibaba.nacos.istio.util.IstioExecutor.cycleDebounce; +import static com.alibaba.nacos.istio.util.IstioExecutor.debouncePushChange; /** * @author special.fy */ @org.springframework.stereotype.Service -public class NacosServiceInfoResourceWatcher implements Runnable { +public class NacosServiceInfoResourceWatcher extends SmartSubscriber { private final Map serviceInfoMap = new ConcurrentHashMap<>(16); - + + private final Queue pushRequestQueue = new ConcurrentLinkedQueue<>(); + + private boolean isInitial = true; + + @Autowired + private IstioConfig istioConfig; + @Autowired private ServiceStorage serviceStorage; @Autowired private EventProcessor eventProcessor; - + + public NacosServiceInfoResourceWatcher() { + NotifyCenter.registerSubscriber(this, NamingEventPublisherFactory.getInstance()); + } + + public Map snapshot() { + return new HashMap<>(serviceInfoMap); + } + @Override - public void run() { - boolean changed = false; - - // Query all services to see if any of them have changes. + public List> subscribeTypes() { + List> result = new LinkedList<>(); + result.add(ClientOperationEvent.ClientRegisterServiceEvent.class); + result.add(ClientOperationEvent.ClientDeregisterServiceEvent.class); + result.add(InfoChangeEvent.ServiceInfoChangeEvent.class); + result.add(InfoChangeEvent.InstanceInfoChangeEvent.class); + return result; + } + + public void onEvent(com.alibaba.nacos.common.notify.Event event) { + if (isInitial) { + init(); + isInitial = false; + cycleDebounce(new ToNotify()); + } + + if (event instanceof ClientOperationEvent.ClientRegisterServiceEvent) { + // If service changed, push to all subscribers. + ClientOperationEvent.ClientRegisterServiceEvent clientRegisterServiceEvent = (ClientOperationEvent.ClientRegisterServiceEvent) event; + Service service = clientRegisterServiceEvent.getService(); + String serviceName = IstioCrdUtil.buildServiceName(service); + + IstioService old = serviceInfoMap.get(serviceName); + PushRequest pushRequest; + + boolean full = update(serviceName, service); + if (old != null) { + pushRequest = new PushRequest(serviceName, full); + } else { + pushRequest = new PushRequest(serviceName, true); + } + pushRequestQueue.add(pushRequest); + } else if (event instanceof ClientOperationEvent.ClientDeregisterServiceEvent) { + ClientOperationEvent.ClientDeregisterServiceEvent clientDeregisterServiceEvent = (ClientOperationEvent + .ClientDeregisterServiceEvent) event; + Service service = clientDeregisterServiceEvent.getService(); + String serviceName = IstioCrdUtil.buildServiceName(service); + PushRequest pushRequest; + + boolean full = update(serviceName, service); + if (serviceStorage.getPushData(service).ipCount() <= 0) { + pushRequest = new PushRequest(serviceName, true); + serviceInfoMap.remove(serviceName); + } else { + pushRequest = new PushRequest(serviceName, full); + } + pushRequestQueue.add(pushRequest); + } else if (event instanceof InfoChangeEvent.ServiceInfoChangeEvent) { + InfoChangeEvent.ServiceInfoChangeEvent serviceInfoChangeEvent = (InfoChangeEvent.ServiceInfoChangeEvent) event; + Service service = serviceInfoChangeEvent.getService(); + String serviceName = IstioCrdUtil.buildServiceName(service); + PushRequest pushRequest = new PushRequest(serviceName, true); + + update(serviceName, service); + pushRequestQueue.add(pushRequest); + + } else if (event instanceof InfoChangeEvent.InstanceInfoChangeEvent) { + InfoChangeEvent.InstanceInfoChangeEvent instanceInfoChangeEvent = (InfoChangeEvent.InstanceInfoChangeEvent) event; + Service service = instanceInfoChangeEvent.getService(); + String serviceName = IstioCrdUtil.buildServiceName(service); + + boolean full = update(serviceName, service); + PushRequest pushRequest = new PushRequest(serviceName, full); + pushRequestQueue.add(pushRequest); + } + } + + private void init() { Set namespaces = ServiceManager.getInstance().getAllNamespaces(); - Set allServices = new HashSet<>(); for (String namespace : namespaces) { - Set services = ServiceManager.getInstance().getSingletons(namespace); + Set services = ServiceManager.getInstance().getSingletons(namespace); if (services.isEmpty()) { continue; } - + for (Service service : services) { - String serviceName = IstioCrdUtil.buildServiceNameForServiceEntry(service); - allServices.add(serviceName); - - IstioService old = serviceInfoMap.get(serviceName); - // Service not changed - if (old != null && old.getRevision().equals(service.getRevision())) { - continue; - } - - // Update the resource - changed = true; + String serviceName = IstioCrdUtil.buildServiceName(service); ServiceInfo serviceInfo = serviceStorage.getPushData(service); if (!serviceInfo.isValid()) { - serviceInfoMap.remove(serviceName); continue; } - - if (old != null) { - serviceInfoMap.put(serviceName, new IstioService(service, serviceInfo, old)); - } else { - serviceInfoMap.put(serviceName, new IstioService(service, serviceInfo)); - } + serviceInfoMap.put(serviceName, new IstioService(service, serviceInfo)); + pushRequestQueue.add(new PushRequest(serviceName, true)); } } - - for (String key : serviceInfoMap.keySet()) { - if (!allServices.contains(key)) { - changed = true; - serviceInfoMap.remove(key); - } + } + + private boolean update(String serviceName, Service service) { + ServiceInfo serviceInfo = serviceStorage.getPushData(service); + if (!serviceInfo.isValid()) { + serviceInfoMap.remove(serviceName); + return true; } - - if (changed) { - eventProcessor.notify(Event.SERVICE_UPDATE_EVENT); + + IstioService old = serviceInfoMap.get(serviceName); + if (old != null) { + serviceInfoMap.put(serviceName, new IstioService(service, serviceInfo, old)); + } else { + serviceInfoMap.put(serviceName, new IstioService(service, serviceInfo)); } + return false; } - - public Map snapshot() { - return new HashMap<>(serviceInfoMap); + + private class ToNotify implements Runnable { + @Override + public void run() { + while (true) { + if (pushRequestQueue.size() > 0) { + PushRequest updatePush; + Future futureUpdate = debouncePushChange(new Debounce(pushRequestQueue, istioConfig)); + + try { + updatePush = futureUpdate.get(); + if (updatePush != null) { + eventProcessor.notify(updatePush); + } + } catch (InterruptedException | ExecutionException e) { + throw new RuntimeException(e); + } + } + } + } } -} +} \ No newline at end of file diff --git a/istio/src/main/java/com/alibaba/nacos/istio/common/ResourceSnapshot.java b/istio/src/main/java/com/alibaba/nacos/istio/common/ResourceSnapshot.java index 49e8c28bcd4..70cb6b61930 100644 --- a/istio/src/main/java/com/alibaba/nacos/istio/common/ResourceSnapshot.java +++ b/istio/src/main/java/com/alibaba/nacos/istio/common/ResourceSnapshot.java @@ -16,15 +16,13 @@ package com.alibaba.nacos.istio.common; +import com.alibaba.nacos.istio.misc.IstioConfig; +import com.alibaba.nacos.istio.model.IstioResources; import com.alibaba.nacos.istio.model.IstioService; -import com.alibaba.nacos.istio.model.ServiceEntryWrapper; -import com.alibaba.nacos.istio.util.IstioCrdUtil; import java.text.SimpleDateFormat; -import java.util.ArrayList; import java.util.Date; -import java.util.List; -import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicLong; /** @@ -32,16 +30,19 @@ */ public class ResourceSnapshot { private static AtomicLong versionSuffix = new AtomicLong(0); - - private final List serviceEntries; + + private final IstioResources istioResources; + + private IstioConfig istioConfig; private boolean isCompleted; private String version; - - public ResourceSnapshot() { + + public ResourceSnapshot(IstioConfig istioConfig) { isCompleted = false; - serviceEntries = new ArrayList<>(); + istioResources = new IstioResources(new ConcurrentHashMap(16)); + this.istioConfig = istioConfig; } public synchronized void initResourceSnapshot(NacosResourceManager manager) { @@ -49,7 +50,7 @@ public synchronized void initResourceSnapshot(NacosResourceManager manager) { return; } - initServiceEntry(manager); + initIstioResources(manager); generateVersion(); @@ -60,22 +61,19 @@ private void generateVersion() { String time = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssXXX").format(new Date()); version = time + "/" + versionSuffix.getAndIncrement(); } - - private void initServiceEntry(NacosResourceManager manager) { - Map serviceInfoMap = manager.services(); - for (String serviceName : serviceInfoMap.keySet()) { - ServiceEntryWrapper serviceEntryWrapper = IstioCrdUtil.buildServiceEntry(serviceName, manager.getIstioConfig().getDomainSuffix(), serviceInfoMap.get(serviceName)); - if (serviceEntryWrapper != null) { - serviceEntries.add(serviceEntryWrapper); - } - } - + + private void initIstioResources(NacosResourceManager manager) { + istioResources.setIstioServiceMap(manager.services()); } - - public List getServiceEntries() { - return serviceEntries; + + public IstioResources getIstioResources() { + return istioResources; } - + + public IstioConfig getIstioConfig() { + return istioConfig; + } + public boolean isCompleted() { return isCompleted; } @@ -83,4 +81,4 @@ public boolean isCompleted() { public String getVersion() { return version; } -} +} \ No newline at end of file diff --git a/istio/src/main/java/com/alibaba/nacos/istio/common/WatchedStatus.java b/istio/src/main/java/com/alibaba/nacos/istio/common/WatchedStatus.java index 44a472a6e5a..02b5ed84303 100644 --- a/istio/src/main/java/com/alibaba/nacos/istio/common/WatchedStatus.java +++ b/istio/src/main/java/com/alibaba/nacos/istio/common/WatchedStatus.java @@ -16,12 +16,19 @@ package com.alibaba.nacos.istio.common; +import java.util.HashSet; +import java.util.Set; + /** * @author special.fy */ public class WatchedStatus { private String type; + + private boolean lastAckOrNack; + + private Set lastSubscribe; private String latestVersion; @@ -70,4 +77,20 @@ public String getAckedNonce() { public void setAckedNonce(String ackedNonce) { this.ackedNonce = ackedNonce; } + + public boolean isLastAckOrNack() { + return lastAckOrNack; + } + + public void setLastAckOrNack(boolean lastAckOrNack) { + this.lastAckOrNack = lastAckOrNack; + } + + public Set getLastSubscribe() { + return lastSubscribe; + } + + public void setLastSubscribe(Set lastSubscribe) { + this.lastSubscribe = new HashSet<>(lastSubscribe); + } } diff --git a/istio/src/main/java/com/alibaba/nacos/istio/mcp/EmptyMcpGenerator.java b/istio/src/main/java/com/alibaba/nacos/istio/mcp/EmptyMcpGenerator.java index b9cd664ac5d..4c90e6e6b0a 100644 --- a/istio/src/main/java/com/alibaba/nacos/istio/mcp/EmptyMcpGenerator.java +++ b/istio/src/main/java/com/alibaba/nacos/istio/mcp/EmptyMcpGenerator.java @@ -17,7 +17,7 @@ package com.alibaba.nacos.istio.mcp; import com.alibaba.nacos.istio.api.ApiGenerator; -import com.alibaba.nacos.istio.common.ResourceSnapshot; +import com.alibaba.nacos.istio.model.PushRequest; import istio.mcp.v1alpha1.ResourceOuterClass.Resource; import java.util.ArrayList; @@ -42,7 +42,12 @@ public static EmptyMcpGenerator getInstance() { } @Override - public List generate(ResourceSnapshot resourceSnapshot) { + public List generate(PushRequest pushRequest) { + return new ArrayList<>(); + } + + @Override + public List deltaGenerate(PushRequest pushRequest) { return new ArrayList<>(); } } diff --git a/istio/src/main/java/com/alibaba/nacos/istio/mcp/NacosMcpService.java b/istio/src/main/java/com/alibaba/nacos/istio/mcp/NacosMcpService.java index afa6cfeba4f..4f718ef7b44 100644 --- a/istio/src/main/java/com/alibaba/nacos/istio/mcp/NacosMcpService.java +++ b/istio/src/main/java/com/alibaba/nacos/istio/mcp/NacosMcpService.java @@ -19,11 +19,10 @@ import com.alibaba.nacos.istio.api.ApiGenerator; import com.alibaba.nacos.istio.api.ApiGeneratorFactory; import com.alibaba.nacos.istio.common.AbstractConnection; -import com.alibaba.nacos.istio.common.Event; import com.alibaba.nacos.istio.common.NacosResourceManager; -import com.alibaba.nacos.istio.common.ResourceSnapshot; import com.alibaba.nacos.istio.common.WatchedStatus; import com.alibaba.nacos.istio.misc.Loggers; +import com.alibaba.nacos.istio.model.PushRequest; import com.alibaba.nacos.istio.util.NonceGenerator; import io.grpc.stub.StreamObserver; import istio.mcp.v1alpha1.Mcp; @@ -105,8 +104,10 @@ private void process(Mcp.RequestResources requestResources, AbstractConnection connection : connections.values()) { - WatchedStatus watchedStatus = connection.getWatchedStatusByType(SERVICE_ENTRY_COLLECTION); - if (watchedStatus != null) { - connection.push(serviceEntryMcpResponse, watchedStatus); - } - } - break; - default: - Loggers.MAIN.warn("Invalid event {}, ignore it.", event.getType()); + public void handleEvent(PushRequest pushRequest) { + if (connections.size() == 0) { + return; + } + + Loggers.MAIN.info("mcp: event {} trigger push.", pushRequest.getReason()); + + Mcp.Resources serviceEntryMcpResponse = buildMcpResourcesResponse(SERVICE_ENTRY_COLLECTION, pushRequest); + + for (AbstractConnection connection : connections.values()) { + WatchedStatus watchedStatus = connection.getWatchedStatusByType(SERVICE_ENTRY_COLLECTION); + if (watchedStatus != null) { + connection.push(serviceEntryMcpResponse, watchedStatus); + } } } - private Mcp.Resources buildMcpResourcesResponse(String type, ResourceSnapshot resourceSnapshot) { + private Mcp.Resources buildMcpResourcesResponse(String type, PushRequest pushRequest) { @SuppressWarnings("unchecked") ApiGenerator serviceEntryGenerator = (ApiGenerator) apiGeneratorFactory.getApiGenerator(type); - List rawResources = serviceEntryGenerator.generate(resourceSnapshot); + List rawResources = serviceEntryGenerator.generate(pushRequest); String nonce = NonceGenerator.generateNonce(); return Mcp.Resources.newBuilder() .setCollection(type) .addAllResources(rawResources) - .setSystemVersionInfo(resourceSnapshot.getVersion()) + .setSystemVersionInfo(pushRequest.getResourceSnapshot().getVersion()) .setNonce(nonce).build(); } } diff --git a/istio/src/main/java/com/alibaba/nacos/istio/mcp/ServiceEntryMcpGenerator.java b/istio/src/main/java/com/alibaba/nacos/istio/mcp/ServiceEntryMcpGenerator.java index 3cb4cca9767..160a0f5ac79 100644 --- a/istio/src/main/java/com/alibaba/nacos/istio/mcp/ServiceEntryMcpGenerator.java +++ b/istio/src/main/java/com/alibaba/nacos/istio/mcp/ServiceEntryMcpGenerator.java @@ -18,6 +18,9 @@ import com.alibaba.nacos.istio.api.ApiGenerator; import com.alibaba.nacos.istio.common.ResourceSnapshot; +import com.alibaba.nacos.istio.misc.IstioConfig; +import com.alibaba.nacos.istio.model.IstioService; +import com.alibaba.nacos.istio.model.PushRequest; import com.alibaba.nacos.istio.model.ServiceEntryWrapper; import com.google.protobuf.Any; import istio.mcp.v1alpha1.MetadataOuterClass; @@ -26,15 +29,19 @@ import java.util.ArrayList; import java.util.List; +import java.util.Map; import static com.alibaba.nacos.istio.api.ApiConstants.SERVICE_ENTRY_PROTO; +import static com.alibaba.nacos.istio.util.IstioCrdUtil.buildServiceEntry; /** * @author special.fy */ public class ServiceEntryMcpGenerator implements ApiGenerator { - - private volatile static ServiceEntryMcpGenerator singleton = null; + + private List serviceEntries; + + private static volatile ServiceEntryMcpGenerator singleton = null; public static ServiceEntryMcpGenerator getInstance() { if (singleton == null) { @@ -48,10 +55,23 @@ public static ServiceEntryMcpGenerator getInstance() { } @Override - public List generate(ResourceSnapshot resourceSnapshot) { + public List generate(PushRequest pushRequest) { List result = new ArrayList<>(); - - List serviceEntries = resourceSnapshot.getServiceEntries(); + serviceEntries = new ArrayList<>(16); + ResourceSnapshot resourceSnapshot = pushRequest.getResourceSnapshot(); + + IstioConfig istioConfig = resourceSnapshot.getIstioConfig(); + Map serviceInfoMap = resourceSnapshot.getIstioResources().getIstioServiceMap(); + + for (Map.Entry entry : serviceInfoMap.entrySet()) { + String serviceName = entry.getKey(); + + ServiceEntryWrapper serviceEntryWrapper = buildServiceEntry(serviceName, serviceName + istioConfig.getDomainSuffix(), serviceInfoMap.get(serviceName)); + if (serviceEntryWrapper != null) { + serviceEntries.add(serviceEntryWrapper); + } + } + for (ServiceEntryWrapper serviceEntryWrapper : serviceEntries) { MetadataOuterClass.Metadata metadata = serviceEntryWrapper.getMetadata(); ServiceEntryOuterClass.ServiceEntry serviceEntry = serviceEntryWrapper.getServiceEntry(); @@ -63,4 +83,9 @@ public List generate(ResourceSnapshot resourceSnapshot) { return result; } + + @Override + public List deltaGenerate(PushRequest pushRequest) { + return new ArrayList<>(); + } } diff --git a/istio/src/main/java/com/alibaba/nacos/istio/misc/IstioConfig.java b/istio/src/main/java/com/alibaba/nacos/istio/misc/IstioConfig.java index 433018d8eb5..68030c01a73 100644 --- a/istio/src/main/java/com/alibaba/nacos/istio/misc/IstioConfig.java +++ b/istio/src/main/java/com/alibaba/nacos/istio/misc/IstioConfig.java @@ -32,8 +32,16 @@ public class IstioConfig { private boolean serverEnabled = false; @Value("${nacos.istio.mcp.server.port:18848}") private int serverPort = 18848; - @Value("${nacos.istio.mcp.push.interval:3000}") - private int mcpPushInterval; + + @Value("${nacos.istio.server.full:true}") + private boolean fullEnabled = true; + + @Value("${nacos.istio.debounce.max:5000}") + private long debounceMax; + + @Value("${nacos.istio.debounce.after:100}") + private long debounceAfter; + @Value("${nacos.istio.domain.suffix:nacos}") private String domainSuffix; @@ -48,9 +56,16 @@ public int getServerPort() { public String getDomainSuffix() { return domainSuffix; } - - public int getMcpPushInterval() { - return mcpPushInterval; + + public boolean isFullEnabled() { + return fullEnabled; + } + + public long getDebounceMax() { + return debounceMax; } + public long getDebounceAfter() { + return debounceAfter; + } } diff --git a/istio/src/main/java/com/alibaba/nacos/istio/model/IstioEndpoint.java b/istio/src/main/java/com/alibaba/nacos/istio/model/IstioEndpoint.java new file mode 100644 index 00000000000..d7ba0bcc393 --- /dev/null +++ b/istio/src/main/java/com/alibaba/nacos/istio/model/IstioEndpoint.java @@ -0,0 +1,133 @@ +/* + * + * Copyright 1999-2022 Alibaba Group Holding Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.alibaba.nacos.istio.model; + +import com.alibaba.nacos.api.naming.pojo.Instance; +import com.google.protobuf.UInt32Value; +import io.envoyproxy.envoy.config.core.v3.Address; +import io.envoyproxy.envoy.config.core.v3.Locality; +import io.envoyproxy.envoy.config.core.v3.SocketAddress; +import io.envoyproxy.envoy.config.endpoint.v3.Endpoint; +import io.envoyproxy.envoy.config.endpoint.v3.LbEndpoint; +import org.apache.commons.lang.StringUtils; + +import java.util.Map; + +import static com.alibaba.nacos.istio.util.IstioCrdUtil.ISTIO_HOSTNAME; + +/**. + * @author RocketEngine26 + * @date 2022/8/9 10:29 + */ +public class IstioEndpoint { + private LbEndpoint lbEndpoint; + + private Instance instance; + + private Locality locality; + + private String protocol; + + private String hostName; + + private String clusterName; + + public IstioEndpoint(Instance instance) { + this.instance = instance; + this.hostName = StringUtils.isNotEmpty(instance.getMetadata().get(ISTIO_HOSTNAME)) ? instance.getMetadata().get(ISTIO_HOSTNAME) : ""; + this.clusterName = StringUtils.isNotEmpty(instance.getClusterName()) ? instance.getClusterName() : ""; + + if (StringUtils.isNotEmpty(instance.getMetadata().get("protocol"))) { + this.protocol = instance.getMetadata().get("protocol"); + + if ("triple".equals(this.protocol) || "tri".equals(this.protocol)) { + this.protocol = "grpc"; + } + } else { + this.protocol = "http"; + } + + buildLocality(); + } + + private void buildLocality() { + String region = instance.getMetadata().getOrDefault("region", ""); + String zone = instance.getMetadata().getOrDefault("zone", ""); + String subzone = instance.getMetadata().getOrDefault("subzone", ""); + + this.locality = Locality.newBuilder().setRegion(region).setZone(zone).setSubZone(subzone).build(); + } + + private LbEndpoint buildLbEndpoint() { + Address adder = Address.newBuilder().setSocketAddress(SocketAddress.newBuilder().setAddress(instance.getIp()) + .setPortValue(this.instance.getPort()).setProtocol(SocketAddress.Protocol.TCP).build()).build(); + this.lbEndpoint = LbEndpoint.newBuilder().setLoadBalancingWeight(UInt32Value.newBuilder().setValue( + (int) this.instance.getWeight())).setEndpoint(Endpoint.newBuilder().setAddress(adder).build()).build(); + + return this.lbEndpoint; + } + + public Map getLabels() { + return instance.getMetadata(); + } + + public String getAdder() { + return instance.getIp(); + } + + public LbEndpoint getLbEndpoint() { + return buildLbEndpoint(); + } + + public String getStringLocality() { + return locality.getRegion() + "." + locality.getZone() + "." + locality.getSubZone(); + } + + public Locality getLocality() { + return locality; + } + + public int getPort() { + return instance.getPort(); + } + + public String getProtocol() { + return protocol; + } + + public int getWeight() { + return (int) instance.getWeight(); + } + + public String getHostName() { + return hostName; + } + + public String getClusterName() { + return clusterName; + } + + public boolean isHealthy() { + return instance.isHealthy(); + } + + public boolean isEnabled() { + return instance.isEnabled(); + } +} \ No newline at end of file diff --git a/istio/src/main/java/com/alibaba/nacos/istio/model/IstioResources.java b/istio/src/main/java/com/alibaba/nacos/istio/model/IstioResources.java new file mode 100644 index 00000000000..d6343ccc434 --- /dev/null +++ b/istio/src/main/java/com/alibaba/nacos/istio/model/IstioResources.java @@ -0,0 +1,42 @@ +/* + * + * Copyright 1999-2022 Alibaba Group Holding Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.alibaba.nacos.istio.model; + +import java.util.Map; + +/**. + * @author RocketEngine26 + * @date 2022/8/9 16:26 + */ +public class IstioResources { + //TODO: crd + private Map istioServiceMap; + + public IstioResources(Map istioServiceMap) { + this.istioServiceMap = istioServiceMap; + } + + public Map getIstioServiceMap() { + return istioServiceMap; + } + + public void setIstioServiceMap(Map istioServiceMap) { + this.istioServiceMap = istioServiceMap; + } +} \ No newline at end of file diff --git a/istio/src/main/java/com/alibaba/nacos/istio/model/IstioService.java b/istio/src/main/java/com/alibaba/nacos/istio/model/IstioService.java index be2a67d3ec2..cc10e670caf 100644 --- a/istio/src/main/java/com/alibaba/nacos/istio/model/IstioService.java +++ b/istio/src/main/java/com/alibaba/nacos/istio/model/IstioService.java @@ -28,19 +28,23 @@ * @author special.fy */ public class IstioService { - - private String name; - - private String groupName; - - private String namespace; - - private Long revision; - - private List hosts; - - private Date createTimeStamp; - + + private final String name; + + private final String groupName; + + private final String namespace; + + private final Long revision; + + private int port = 0; + + private String protocol; + + private final List hosts; + + private final Date createTimeStamp; + public IstioService(Service service, ServiceInfo serviceInfo) { this.name = serviceInfo.getName(); this.groupName = serviceInfo.getGroupName(); @@ -49,7 +53,7 @@ public IstioService(Service service, ServiceInfo serviceInfo) { // Record the create time of service to avoid trigger istio pull push. // See https://github.com/istio/istio/pull/30684 createTimeStamp = new Date(); - + this.hosts = sanitizeServiceInfo(serviceInfo); } @@ -61,35 +65,35 @@ public IstioService(Service service, ServiceInfo serviceInfo, IstioService old) // set the create time of service as old time to avoid trigger istio pull push. // See https://github.com/istio/istio/pull/30684 createTimeStamp = old.getCreateTimeStamp(); - + this.hosts = sanitizeServiceInfo(serviceInfo); } - private List sanitizeServiceInfo(ServiceInfo serviceInfo) { - List hosts = new ArrayList<>(); + private List sanitizeServiceInfo(ServiceInfo serviceInfo) { + List hosts = new ArrayList<>(); for (Instance instance : serviceInfo.getHosts()) { if (instance.isHealthy() && instance.isEnabled()) { - hosts.add(instance); + IstioEndpoint istioEndpoint = new IstioEndpoint(instance); + if (port == 0) { + port = istioEndpoint.getPort(); + protocol = istioEndpoint.getProtocol(); + } + hosts.add(istioEndpoint); } } // Panic mode, all instances are invalid, to push all instances to istio. if (hosts.isEmpty()) { - hosts = serviceInfo.getHosts(); + for (Instance instance : serviceInfo.getHosts()) { + IstioEndpoint istioEndpoint = new IstioEndpoint(instance); + hosts.add(istioEndpoint); + } } return hosts; } - - public String getName() { - return name; - } - - public String getGroupName() { - return groupName; - } - + public String getNamespace() { return namespace; } @@ -97,12 +101,20 @@ public String getNamespace() { public Long getRevision() { return revision; } - - public List getHosts() { + + public int getPort() { + return port; + } + + public String getProtocol() { + return protocol; + } + + public List getHosts() { return hosts; } public Date getCreateTimeStamp() { return createTimeStamp; } -} +} \ No newline at end of file diff --git a/istio/src/main/java/com/alibaba/nacos/istio/model/PushRequest.java b/istio/src/main/java/com/alibaba/nacos/istio/model/PushRequest.java new file mode 100644 index 00000000000..cf26c196a94 --- /dev/null +++ b/istio/src/main/java/com/alibaba/nacos/istio/model/PushRequest.java @@ -0,0 +1,89 @@ +/* + * Copyright 1999-2022 Alibaba Group Holding Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.alibaba.nacos.istio.model; + +import com.alibaba.nacos.istio.common.ResourceSnapshot; + +import java.util.HashSet; +import java.util.Set; + +/** + * @author RocketEngine26 + * @date 2022/8/21 下午1:09 + */ +public class PushRequest { + private ResourceSnapshot resourceSnapshot; + + private final Set reason = new HashSet<>(); + + private Set subscribe; + + private final Set removed = new HashSet<>(); + + private boolean full; + + public PushRequest(ResourceSnapshot snapshot, boolean full) { + this.resourceSnapshot = snapshot; + this.full = full; + } + + public PushRequest(String reason, boolean full) { + this.full = full; + this.reason.add(reason); + } + + public ResourceSnapshot getResourceSnapshot() { + return resourceSnapshot; + } + + public boolean isFull() { + return full; + } + + public void setFull(boolean full) { + this.full = full; + } + + public void setResourceSnapshot(ResourceSnapshot resourceSnapshot) { + this.resourceSnapshot = resourceSnapshot; + } + + public Set getReason() { + return reason; + } + + public void addReason(String reason) { + this.reason.add(reason); + } + + public Set getRemoved() { + return removed; + } + + public void addRemoved(String remove) { + this.removed.add(remove); + } + + public Set getSubscribe() { + return subscribe; + } + + public void setSubscribe(Set subscribe) { + this.subscribe = subscribe; + } +} diff --git a/istio/src/main/java/com/alibaba/nacos/istio/server/IstioServer.java b/istio/src/main/java/com/alibaba/nacos/istio/server/IstioServer.java index 82a707c66e8..f7ea6a22a57 100644 --- a/istio/src/main/java/com/alibaba/nacos/istio/server/IstioServer.java +++ b/istio/src/main/java/com/alibaba/nacos/istio/server/IstioServer.java @@ -16,7 +16,6 @@ package com.alibaba.nacos.istio.server; -import com.alibaba.nacos.istio.common.NacosResourceManager; import com.alibaba.nacos.istio.mcp.NacosMcpService; import com.alibaba.nacos.istio.misc.IstioConfig; import com.alibaba.nacos.istio.misc.Loggers; @@ -49,10 +48,6 @@ public class IstioServer { @Autowired private NacosXdsService nacosXdsService; - - @Autowired - private NacosResourceManager nacosResourceManager; - /** * Start. * @@ -65,10 +60,9 @@ public void start() throws IOException { Loggers.MAIN.info("The Nacos Istio server is disabled."); return; } - nacosResourceManager.start(); - + Loggers.MAIN.info("Nacos Istio server, starting Nacos Istio server..."); - + server = ServerBuilder.forPort(istioConfig.getServerPort()).addService(ServerInterceptors.intercept(nacosMcpService, serverInterceptor)) .addService(ServerInterceptors.intercept(nacosXdsService, serverInterceptor)).build(); server.start(); diff --git a/istio/src/main/java/com/alibaba/nacos/istio/util/IstioCrdUtil.java b/istio/src/main/java/com/alibaba/nacos/istio/util/IstioCrdUtil.java index 038ac55d5f4..bb2864efc72 100644 --- a/istio/src/main/java/com/alibaba/nacos/istio/util/IstioCrdUtil.java +++ b/istio/src/main/java/com/alibaba/nacos/istio/util/IstioCrdUtil.java @@ -17,19 +17,23 @@ package com.alibaba.nacos.istio.util; import com.alibaba.nacos.api.common.Constants; -import com.alibaba.nacos.api.naming.pojo.Instance; +import com.alibaba.nacos.common.utils.StringUtils; +import com.alibaba.nacos.istio.model.IstioEndpoint; import com.alibaba.nacos.istio.model.IstioService; import com.alibaba.nacos.istio.model.ServiceEntryWrapper; import com.alibaba.nacos.naming.core.v2.pojo.Service; import com.google.protobuf.Timestamp; -import istio.mcp.v1alpha1.MetadataOuterClass.Metadata; +import io.envoyproxy.envoy.config.core.v3.TrafficDirection; +import istio.mcp.v1alpha1.MetadataOuterClass; import istio.networking.v1alpha3.GatewayOuterClass; -import istio.networking.v1alpha3.ServiceEntryOuterClass.ServiceEntry; +import istio.networking.v1alpha3.ServiceEntryOuterClass; +import istio.networking.v1alpha3.WorkloadEntryOuterClass; import istio.networking.v1alpha3.WorkloadEntryOuterClass.WorkloadEntry; -import org.apache.commons.lang.StringUtils; +import java.util.ArrayList; import java.util.Date; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.regex.Pattern; @@ -40,86 +44,89 @@ public class IstioCrdUtil { public static final String VALID_DEFAULT_GROUP_NAME = "DEFAULT-GROUP"; - private static final String ISTIO_HOSTNAME = "istio.hostname"; + public static final String ISTIO_HOSTNAME = "istio.hostname"; public static final String VALID_LABEL_KEY_FORMAT = "^([a-zA-Z0-9](?:[-a-zA-Z0-9]*[a-zA-Z0-9])?(?:\\.[a-zA-Z0-9](?:[-a-zA-Z0-9]*[a-zA-Z0-9])?)*/)?((?:[A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$"; + public static final String VALID_LABEL_VALUE_FORMAT = "^((?:[A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?$"; - - public static String buildServiceNameForServiceEntry(Service service) { + + public static String buildClusterName(TrafficDirection direction, String subset, String hostName, int port) { + return direction.toString().toLowerCase() + "|" + port + "|" + subset + "|" + hostName; + } + + public static String buildServiceName(Service service) { String group = !Constants.DEFAULT_GROUP.equals(service.getGroup()) ? service.getGroup() : VALID_DEFAULT_GROUP_NAME; // DEFAULT_GROUP is invalid for istio,because the istio host only supports: [0-9],[A-Z],[a-z],-,* return service.getName() + "." + group + "." + service.getNamespace(); } - - public static ServiceEntryWrapper buildServiceEntry(String serviceName, String domainSuffix,IstioService istioService) { + + public static String parseServiceEntryNameToServiceName(String serviceEntryName, String domain) { + return serviceEntryName.substring(0, serviceEntryName.length() - domain.length() - 1); + } + + public static String parseClusterNameToServiceName(String clusterName, String domain) { + String str = clusterName.split("\\|", 4)[3]; + return str.substring(0, str.length() - domain.length() - 1); + } + + public static ServiceEntryWrapper buildServiceEntry(String serviceName, String hostName, IstioService istioService) { if (istioService.getHosts().isEmpty()) { return null; } - ServiceEntry.Builder serviceEntryBuilder = ServiceEntry - .newBuilder().setResolution(ServiceEntry.Resolution.STATIC) - .setLocation(ServiceEntry.Location.MESH_INTERNAL); + ServiceEntryOuterClass.ServiceEntry.Builder serviceEntryBuilder = ServiceEntryOuterClass.ServiceEntry + .newBuilder().setResolution(ServiceEntryOuterClass.ServiceEntry.Resolution.STATIC) + .setLocation(ServiceEntryOuterClass.ServiceEntry.Location.MESH_INTERNAL); int port = 0; String protocol = "http"; - String hostname = serviceName; - - for (Instance instance : istioService.getHosts()) { - if (port == 0) { - port = instance.getPort(); - } - - if (StringUtils.isNotEmpty(instance.getMetadata().get("protocol"))) { - protocol = instance.getMetadata().get("protocol"); - - if (protocol.equals("triple")||protocol.equals("tri")){ - protocol = "grpc"; - } - } - - String metaHostname = instance.getMetadata().get(ISTIO_HOSTNAME); - if (StringUtils.isNotEmpty(metaHostname)) { - hostname = metaHostname; - } - - if (!instance.isHealthy() || !instance.isEnabled()) { + List endpoints = buildWorkloadEntry(istioService.getHosts()); + + serviceEntryBuilder.addHosts(hostName).addPorts(GatewayOuterClass.Port.newBuilder().setNumber(port) + .setName(protocol).setProtocol(protocol.toUpperCase()).build()).addAllEndpoints(endpoints); + ServiceEntryOuterClass.ServiceEntry serviceEntry = serviceEntryBuilder.build(); + + Date createTimestamp = istioService.getCreateTimeStamp(); + MetadataOuterClass.Metadata metadata = MetadataOuterClass.Metadata.newBuilder() + .setName(istioService.getNamespace() + "/" + serviceName) + .putAnnotations("virtual", "1") + .putLabels("registryType", "nacos") + .setCreateTime(Timestamp.newBuilder().setSeconds(createTimestamp.getTime() / 1000).build()) + .setVersion(String.valueOf(istioService.getRevision())).build(); + + return new ServiceEntryWrapper(metadata, serviceEntry); + } + + public static List buildWorkloadEntry(List istioEndpointList) { + List result = new ArrayList<>(); + + for (IstioEndpoint istioEndpoint : istioEndpointList) { + if (!istioEndpoint.isHealthy() || !istioEndpoint.isEnabled()) { continue; } Map metadata = new HashMap<>(1 << 3); - if (StringUtils.isNotEmpty(instance.getClusterName())) { - metadata.put("cluster", instance.getClusterName()); + if (StringUtils.isNotEmpty(istioEndpoint.getClusterName())) { + metadata.put("cluster", istioEndpoint.getClusterName()); } - for (Map.Entry entry : instance.getMetadata().entrySet()){ - if (!Pattern.matches(VALID_LABEL_KEY_FORMAT, entry.getKey())){ + for (Map.Entry entry : istioEndpoint.getLabels().entrySet()) { + if (!Pattern.matches(VALID_LABEL_KEY_FORMAT, entry.getKey())) { continue; } - if (!Pattern.matches(VALID_LABEL_VALUE_FORMAT, entry.getValue())){ + if (!Pattern.matches(VALID_LABEL_VALUE_FORMAT, entry.getValue())) { continue; } - metadata.put(entry.getKey(), entry.getValue()); + metadata.put(entry.getKey().toLowerCase(), entry.getValue()); } - - WorkloadEntry workloadEntry = WorkloadEntry.newBuilder() - .setAddress(instance.getIp()).setWeight((int) instance.getWeight()) - .putAllLabels(metadata).putPorts(protocol, instance.getPort()).build(); - serviceEntryBuilder.addEndpoints(workloadEntry); + + WorkloadEntryOuterClass.WorkloadEntry workloadEntry = WorkloadEntryOuterClass.WorkloadEntry.newBuilder() + .setAddress(istioEndpoint.getAdder()).setWeight(istioEndpoint.getWeight()) + .putAllLabels(metadata).putPorts(istioEndpoint.getProtocol(), istioEndpoint.getPort()).build(); + + result.add(workloadEntry); } - - serviceEntryBuilder.addHosts(hostname + "." + domainSuffix).addPorts( - GatewayOuterClass.Port.newBuilder().setNumber(port).setName(protocol).setProtocol(protocol.toUpperCase()).build()); - ServiceEntry serviceEntry = serviceEntryBuilder.build(); - - Date createTimestamp = istioService.getCreateTimeStamp(); - Metadata metadata = Metadata.newBuilder() - .setName(istioService.getNamespace() + "/" + serviceName) - .putAnnotations("virtual", "1") - .putLabels("registryType", "nacos") - .setCreateTime(Timestamp.newBuilder().setSeconds(createTimestamp.getTime() / 1000).build()) - .setVersion(String.valueOf(istioService.getRevision())).build(); - - return new ServiceEntryWrapper(metadata, serviceEntry); + return result; } } diff --git a/istio/src/main/java/com/alibaba/nacos/istio/util/IstioExecutor.java b/istio/src/main/java/com/alibaba/nacos/istio/util/IstioExecutor.java index ee703d70fc4..b91b3c94b8a 100644 --- a/istio/src/main/java/com/alibaba/nacos/istio/util/IstioExecutor.java +++ b/istio/src/main/java/com/alibaba/nacos/istio/util/IstioExecutor.java @@ -20,35 +20,36 @@ import com.alibaba.nacos.common.executor.NameThreadFactory; import com.alibaba.nacos.core.utils.ClassUtils; import com.alibaba.nacos.istio.IstioApp; -import com.alibaba.nacos.sys.env.EnvUtil; import java.util.concurrent.Callable; import java.util.concurrent.ExecutorService; import java.util.concurrent.Future; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; - /** * @author special.fy */ public class IstioExecutor { - - private static final ScheduledExecutorService NACOS_RESOURCE_WATCHER = ExecutorFactory.Managed - .newScheduledExecutorService(ClassUtils.getCanonicalName(IstioApp.class), - EnvUtil.getAvailableProcessors(2), - new NameThreadFactory("com.alibaba.nacos.istio.resource.watcher")); - private static final ExecutorService EVENT_HANDLE_EXECUTOR = ExecutorFactory.Managed .newSingleExecutorService(ClassUtils.getCanonicalName(IstioApp.class), - new NameThreadFactory("com.alibaba.nacos.istio.event.handle")); - - - public static void registerNacosResourceWatcher(Runnable watcher, long initialDelay, long period) { - NACOS_RESOURCE_WATCHER.scheduleAtFixedRate(watcher, initialDelay, period, TimeUnit.MILLISECONDS); - } + new NameThreadFactory("com.alibaba.nacos.istio.event.handle")); + + private static final ExecutorService PUSH_CHANGE_EXECUTOR = ExecutorFactory.Managed + .newSingleExecutorService(ClassUtils.getCanonicalName(IstioApp.class), + new NameThreadFactory("com.alibaba.nacos.istio.pushchange.debounce")); + + private static final ExecutorService CYCLE_DEBOUNCE_EXECUTOR = ExecutorFactory.Managed + .newSingleExecutorService(ClassUtils.getCanonicalName(IstioApp.class), + new NameThreadFactory("com.alibaba.nacos.istio.cycle.debounce")); public static Future asyncHandleEvent(Callable task) { return EVENT_HANDLE_EXECUTOR.submit(task); } + + public static Future debouncePushChange(Callable debounce) { + return PUSH_CHANGE_EXECUTOR.submit(debounce); + } + + public static void cycleDebounce(Runnable toNotify) { + CYCLE_DEBOUNCE_EXECUTOR.submit(toNotify); + } } diff --git a/istio/src/main/java/com/alibaba/nacos/istio/xds/CdsGenerator.java b/istio/src/main/java/com/alibaba/nacos/istio/xds/CdsGenerator.java new file mode 100644 index 00000000000..758fdd23a80 --- /dev/null +++ b/istio/src/main/java/com/alibaba/nacos/istio/xds/CdsGenerator.java @@ -0,0 +1,94 @@ +/* + * Copyright 1999-2022 Alibaba Group Holding Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.alibaba.nacos.istio.xds; + +import com.alibaba.nacos.istio.api.ApiGenerator; +import com.alibaba.nacos.istio.misc.IstioConfig; +import com.alibaba.nacos.istio.misc.Loggers; +import com.alibaba.nacos.istio.model.IstioService; +import com.alibaba.nacos.istio.model.PushRequest; +import com.google.protobuf.Any; +import io.envoyproxy.envoy.config.cluster.v3.Cluster; +import io.envoyproxy.envoy.config.core.v3.AggregatedConfigSource; +import io.envoyproxy.envoy.config.core.v3.ConfigSource; +import io.envoyproxy.envoy.config.core.v3.Http1ProtocolOptions; +import io.envoyproxy.envoy.config.core.v3.Http2ProtocolOptions; +import io.envoyproxy.envoy.config.core.v3.TrafficDirection; +import io.envoyproxy.envoy.service.discovery.v3.Resource; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import static com.alibaba.nacos.istio.api.ApiConstants.CLUSTER_TYPE; +import static com.alibaba.nacos.istio.util.IstioCrdUtil.buildClusterName; +import static io.envoyproxy.envoy.config.core.v3.ApiVersion.V2_VALUE; + +/** + * @author RocketEngine26 + * @date 2022/8/17 + */ +public final class CdsGenerator implements ApiGenerator { + + private static volatile CdsGenerator singleton = null; + + public static CdsGenerator getInstance() { + if (singleton == null) { + synchronized (CdsGenerator.class) { + if (singleton == null) { + singleton = new CdsGenerator(); + } + } + } + return singleton; + } + + @Override + public List generate(PushRequest pushRequest) { + if (!pushRequest.isFull()) { + return null; + } + List result = new ArrayList<>(); + IstioConfig istioConfig = pushRequest.getResourceSnapshot().getIstioConfig(); + Map istioServiceMap = pushRequest.getResourceSnapshot().getIstioResources().getIstioServiceMap(); + for (Map.Entry entry : istioServiceMap.entrySet()) { + String name = buildClusterName(TrafficDirection.OUTBOUND, "", + entry.getKey() + '.' + istioConfig.getDomainSuffix(), entry.getValue().getPort()); + + Cluster.Builder cluster = Cluster.newBuilder().setName(name).setType(Cluster.DiscoveryType.EDS) + .setEdsClusterConfig(Cluster.EdsClusterConfig.newBuilder().setServiceName(name).setEdsConfig( + ConfigSource.newBuilder().setAds(AggregatedConfigSource.newBuilder()) + .setResourceApiVersionValue(V2_VALUE).build()).build()); + if ("grpc".equals(entry.getValue().getProtocol())) { + cluster.setHttp2ProtocolOptions(Http2ProtocolOptions.newBuilder().build()); + } else { + cluster.setHttpProtocolOptions(Http1ProtocolOptions.newBuilder().build()); + } + + result.add(Any.newBuilder().setValue(cluster.build().toByteString()).setTypeUrl(CLUSTER_TYPE).build()); + } + + return result; + } + + @Override + public List deltaGenerate(PushRequest pushRequest) { + Loggers.MAIN.info("Delta Cds Not supported"); + return null; + } +} \ No newline at end of file diff --git a/istio/src/main/java/com/alibaba/nacos/istio/xds/DeltaConnection.java b/istio/src/main/java/com/alibaba/nacos/istio/xds/DeltaConnection.java new file mode 100644 index 00000000000..fbf91168837 --- /dev/null +++ b/istio/src/main/java/com/alibaba/nacos/istio/xds/DeltaConnection.java @@ -0,0 +1,58 @@ +/* + * Copyright 1999-2022 Alibaba Group Holding Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.alibaba.nacos.istio.xds; + +import com.alibaba.nacos.istio.common.AbstractConnection; +import com.alibaba.nacos.istio.common.WatchedStatus; +import com.alibaba.nacos.istio.misc.Loggers; +import io.envoyproxy.envoy.service.discovery.v3.DeltaDiscoveryResponse; +import io.grpc.stub.StreamObserver; + +/** + * @author RocketEngine26 + * @date 2022/8/20 下午10:46 + */ +public class DeltaConnection extends AbstractConnection { + + public DeltaConnection(StreamObserver streamObserver) { + super(streamObserver); + } + + @Override + public void push(DeltaDiscoveryResponse response, WatchedStatus watchedStatus) { + if (Loggers.MAIN.isDebugEnabled()) { + Loggers.MAIN.debug("DeltaDiscoveryResponse: {}", response.toString()); + } + + Loggers.MAIN.info("DeltaDiscoveryResponse: {}", response.toString()); + + this.streamObserver.onNext(response); + + // Update watched status + watchedStatus.setLatestVersion(response.getSystemVersionInfo()); + watchedStatus.setLatestNonce(response.getNonce()); + + Loggers.MAIN.info("delta: push, type: {}, connection-id {}, version {}, nonce {}, resource size {}.", + watchedStatus.getType(), + getConnectionId(), + response.getSystemVersionInfo(), + response.getNonce(), + response.getResourcesCount()); + } +} + diff --git a/istio/src/main/java/com/alibaba/nacos/istio/xds/EdsGenerator.java b/istio/src/main/java/com/alibaba/nacos/istio/xds/EdsGenerator.java new file mode 100644 index 00000000000..17eb9413708 --- /dev/null +++ b/istio/src/main/java/com/alibaba/nacos/istio/xds/EdsGenerator.java @@ -0,0 +1,171 @@ +/* + * Copyright 1999-2022 Alibaba Group Holding Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.alibaba.nacos.istio.xds; + +import com.alibaba.nacos.istio.api.ApiGenerator; +import com.alibaba.nacos.istio.misc.IstioConfig; +import com.alibaba.nacos.istio.model.IstioEndpoint; +import com.alibaba.nacos.istio.model.IstioService; +import com.alibaba.nacos.istio.model.PushRequest; +import com.google.protobuf.Any; +import com.google.protobuf.UInt32Value; +import io.envoyproxy.envoy.config.core.v3.TrafficDirection; +import io.envoyproxy.envoy.config.endpoint.v3.ClusterLoadAssignment; +import io.envoyproxy.envoy.config.endpoint.v3.LbEndpoint; +import io.envoyproxy.envoy.config.endpoint.v3.LocalityLbEndpoints; +import io.envoyproxy.envoy.service.discovery.v3.Resource; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import static com.alibaba.nacos.istio.api.ApiConstants.ENDPOINT_TYPE; +import static com.alibaba.nacos.istio.util.IstioCrdUtil.buildClusterName; +import static com.alibaba.nacos.istio.util.IstioCrdUtil.parseClusterNameToServiceName; + +/**. + * @author RocketEngine26 + * @date 2022/7/24 15:28 + */ +public final class EdsGenerator implements ApiGenerator { + + private static volatile EdsGenerator singleton = null; + + public static EdsGenerator getInstance() { + if (singleton == null) { + synchronized (EdsGenerator.class) { + if (singleton == null) { + singleton = new EdsGenerator(); + } + } + } + return singleton; + } + + @Override + public List generate(PushRequest pushRequest) { + List result = new ArrayList<>(); + IstioConfig istioConfig = pushRequest.getResourceSnapshot().getIstioConfig(); + Map istioServiceMap = pushRequest.getResourceSnapshot().getIstioResources().getIstioServiceMap(); + if (pushRequest.getReason().size() != 0) { + for (String reason : pushRequest.getReason()) { + IstioService istioService = istioServiceMap.get(reason); + String name = buildClusterName(TrafficDirection.OUTBOUND, "", + reason + '.' + istioConfig.getDomainSuffix(), istioService.getPort()); + Any any = buildEndpoint(name, istioService); + if (any != null) { + result.add(any); + } + } + } else { + for (Map.Entry entry : istioServiceMap.entrySet()) { + String name = buildClusterName(TrafficDirection.OUTBOUND, "", + entry.getKey() + '.' + istioConfig.getDomainSuffix(), entry.getValue().getPort()); + Any any = buildEndpoint(name, entry.getValue()); + if (any != null) { + result.add(any); + } + } + } + return result; + } + + @Override + public List deltaGenerate(PushRequest pushRequest) { + if (pushRequest.isFull()) { + return null; + } + + List result = new ArrayList<>(); + Set reason = pushRequest.getReason(); + IstioConfig istioConfig = pushRequest.getResourceSnapshot().getIstioConfig(); + Map istioServiceMap = pushRequest.getResourceSnapshot().getIstioResources().getIstioServiceMap(); + + if (pushRequest.getSubscribe().size() != 0) { + for (String subscribe : pushRequest.getSubscribe()) { + String serviceName = parseClusterNameToServiceName(subscribe, istioConfig.getDomainSuffix()); + if (reason.contains(serviceName)) { + if (istioServiceMap.containsKey(serviceName)) { + Any any = buildEndpoint(subscribe, istioServiceMap.get(serviceName)); + if (any != null) { + result.add(Resource.newBuilder().setResource(any).setVersion(pushRequest.getResourceSnapshot().getVersion()).build()); + } else { + pushRequest.addRemoved(subscribe); + } + } else { + pushRequest.addRemoved(subscribe); + } + } + } + } else { + for (Map.Entry entry : istioServiceMap.entrySet()) { + String name = buildClusterName(TrafficDirection.OUTBOUND, "", + entry.getKey() + '.' + istioConfig.getDomainSuffix(), entry.getValue().getPort()); + Any any = buildEndpoint(name, entry.getValue()); + if (any != null) { + result.add(Resource.newBuilder().setResource(any).setVersion(pushRequest.getResourceSnapshot().getVersion()).build()); + } else { + pushRequest.addRemoved(name); + } + } + } + + return result; + } + + private static Any buildEndpoint(String name, IstioService istioService) { + if (istioService.getHosts().isEmpty()) { + return null; + } + + List istioEndpoints = istioService.getHosts(); + Map llbEndpointsBuilder = new HashMap<>(istioEndpoints.size()); + + for (IstioEndpoint istioEndpoint : istioEndpoints) { + String label = istioEndpoint.getStringLocality(); + LbEndpoint lbEndpoint = istioEndpoint.getLbEndpoint(); + + if (!llbEndpointsBuilder.containsKey(label)) { + LocalityLbEndpoints.Builder llbEndpointBuilder = LocalityLbEndpoints.newBuilder() + .setLocality(istioEndpoint.getLocality()).addLbEndpoints(lbEndpoint); + llbEndpointsBuilder.put(label, llbEndpointBuilder); + } else { + llbEndpointsBuilder.get(label).addLbEndpoints(lbEndpoint); + } + } + + List listlle = new ArrayList<>(); + for (LocalityLbEndpoints.Builder builder : llbEndpointsBuilder.values()) { + int weight = 0; + for (LbEndpoint lbEndpoint : builder.getLbEndpointsList()) { + weight += lbEndpoint.getLoadBalancingWeight().getValue(); + } + LocalityLbEndpoints lle = builder.setLoadBalancingWeight(UInt32Value.newBuilder().setValue(weight)).build(); + listlle.add(lle); + } + + if (listlle.size() == 0) { + return null; + } + + ClusterLoadAssignment cla = ClusterLoadAssignment.newBuilder().setClusterName(name).addAllEndpoints(listlle).build(); + return Any.newBuilder().setValue(cla.toByteString()).setTypeUrl(ENDPOINT_TYPE).build(); + } +} \ No newline at end of file diff --git a/istio/src/main/java/com/alibaba/nacos/istio/xds/EmptyXdsGenerator.java b/istio/src/main/java/com/alibaba/nacos/istio/xds/EmptyXdsGenerator.java index 275a1ea4d97..99e288c3462 100644 --- a/istio/src/main/java/com/alibaba/nacos/istio/xds/EmptyXdsGenerator.java +++ b/istio/src/main/java/com/alibaba/nacos/istio/xds/EmptyXdsGenerator.java @@ -17,8 +17,9 @@ package com.alibaba.nacos.istio.xds; import com.alibaba.nacos.istio.api.ApiGenerator; -import com.alibaba.nacos.istio.common.ResourceSnapshot; +import com.alibaba.nacos.istio.model.PushRequest; import com.google.protobuf.Any; +import io.envoyproxy.envoy.service.discovery.v3.Resource; import java.util.ArrayList; import java.util.List; @@ -28,7 +29,7 @@ */ public class EmptyXdsGenerator implements ApiGenerator { - private volatile static EmptyXdsGenerator singleton = null; + private static volatile EmptyXdsGenerator singleton = null; public static EmptyXdsGenerator getInstance() { if (singleton == null) { @@ -40,9 +41,14 @@ public static EmptyXdsGenerator getInstance() { } return singleton; } - + + @Override + public List generate(PushRequest pushRequest) { + return new ArrayList<>(); + } + @Override - public List generate(ResourceSnapshot resourceSnapshot) { + public List deltaGenerate(PushRequest pushRequest) { return new ArrayList<>(); } } diff --git a/istio/src/main/java/com/alibaba/nacos/istio/xds/NacosXdsService.java b/istio/src/main/java/com/alibaba/nacos/istio/xds/NacosXdsService.java index 8df95908459..27674056a01 100644 --- a/istio/src/main/java/com/alibaba/nacos/istio/xds/NacosXdsService.java +++ b/istio/src/main/java/com/alibaba/nacos/istio/xds/NacosXdsService.java @@ -19,22 +19,32 @@ import com.alibaba.nacos.istio.api.ApiGenerator; import com.alibaba.nacos.istio.api.ApiGeneratorFactory; import com.alibaba.nacos.istio.common.*; +import com.alibaba.nacos.istio.misc.IstioConfig; import com.alibaba.nacos.istio.misc.Loggers; +import com.alibaba.nacos.istio.model.PushRequest; import com.alibaba.nacos.istio.util.NonceGenerator; import com.google.protobuf.Any; import io.envoyproxy.envoy.service.discovery.v3.AggregatedDiscoveryServiceGrpc; +import io.envoyproxy.envoy.service.discovery.v3.DeltaDiscoveryRequest; +import io.envoyproxy.envoy.service.discovery.v3.DeltaDiscoveryResponse; import io.envoyproxy.envoy.service.discovery.v3.DiscoveryRequest; import io.envoyproxy.envoy.service.discovery.v3.DiscoveryResponse; +import io.envoyproxy.envoy.service.discovery.v3.Resource; import io.grpc.stub.StreamObserver; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.concurrent.ConcurrentHashMap; +import static com.alibaba.nacos.istio.api.ApiConstants.CLUSTER_TYPE; +import static com.alibaba.nacos.istio.api.ApiConstants.ENDPOINT_TYPE; import static com.alibaba.nacos.istio.api.ApiConstants.MESH_CONFIG_PROTO_PACKAGE; import static com.alibaba.nacos.istio.api.ApiConstants.SERVICE_ENTRY_PROTO_PACKAGE; +import static com.alibaba.nacos.istio.util.IstioCrdUtil.parseClusterNameToServiceName; /** * @author special.fy @@ -43,11 +53,13 @@ public class NacosXdsService extends AggregatedDiscoveryServiceGrpc.AggregatedDiscoveryServiceImplBase { private final Map> connections = new ConcurrentHashMap<>(16); + + private final Map> deltaConnections = new ConcurrentHashMap<>(16); public boolean hasClientConnection() { - return connections.size() != 0; + return connections.size() != 0 || deltaConnections.size() != 0; } - + @Autowired ApiGeneratorFactory apiGeneratorFactory; @@ -100,8 +112,17 @@ public void process(DiscoveryRequest discoveryRequest, AbstractConnection resourceNames = new HashSet<>(discoveryRequest.getResourceNamesList()); + PushRequest pushRequest = new PushRequest(resourceManager.getResourceSnapshot(), true); + IstioConfig istioConfig = pushRequest.getResourceSnapshot().getIstioConfig(); + + for (String resourceName : resourceNames) { + String reason = parseClusterNameToServiceName(resourceName, istioConfig.getDomainSuffix()); + pushRequest.addReason(reason); + } + + DiscoveryResponse response = buildDiscoveryResponse(discoveryRequest.getTypeUrl(), pushRequest); connection.push(response, connection.getWatchedStatusByType(discoveryRequest.getTypeUrl())); } @@ -161,46 +182,203 @@ private boolean shouldPush(DiscoveryRequest discoveryRequest, AbstractConnection return false; } - public void handleEvent(ResourceSnapshot resourceSnapshot, Event event) { - switch (event.getType()) { - case Service: - if (connections.size() == 0) { - return; - } - - Loggers.MAIN.info("xds: event {} trigger push.", event.getType()); - - // Service Entry via MCP - DiscoveryResponse serviceEntryResponse = buildDiscoveryResponse(SERVICE_ENTRY_PROTO_PACKAGE, resourceSnapshot); - // TODO CDS, EDS - - for (AbstractConnection connection : connections.values()) { - // Service Entry via MCP - WatchedStatus watchedStatus = connection.getWatchedStatusByType(SERVICE_ENTRY_PROTO_PACKAGE); - if (watchedStatus != null) { - connection.push(serviceEntryResponse, watchedStatus); - } - // TODO CDS, EDS + public void handleEvent(PushRequest pushRequest) { + if (connections.size() == 0) { + return; + } + + for (AbstractConnection connection : connections.values()) { + //mcp + WatchedStatus watchedStatus = connection.getWatchedStatusByType(SERVICE_ENTRY_PROTO_PACKAGE); + if (watchedStatus != null) { + DiscoveryResponse serviceEntryResponse = buildDiscoveryResponse(SERVICE_ENTRY_PROTO_PACKAGE, pushRequest); + connection.push(serviceEntryResponse, watchedStatus); + } + //CDS + WatchedStatus cdsWatchedStatus = connection.getWatchedStatusByType(CLUSTER_TYPE); + if (cdsWatchedStatus != null) { + DiscoveryResponse cdsResponse = buildDiscoveryResponse(CLUSTER_TYPE, pushRequest); + if (cdsResponse != null) { + connection.push(cdsResponse, cdsWatchedStatus); } - break; - case Endpoint: - Loggers.MAIN.warn("Currently, endpoint event is not supported."); - break; - default: - Loggers.MAIN.warn("Invalid event {}, ignore it.", event.getType()); + } + //EDS + WatchedStatus edsWatchedStatus = connection.getWatchedStatusByType(ENDPOINT_TYPE); + if (edsWatchedStatus != null) { + DiscoveryResponse edsResponse = buildDiscoveryResponse(ENDPOINT_TYPE, pushRequest); + connection.push(edsResponse, edsWatchedStatus); + } } } - private DiscoveryResponse buildDiscoveryResponse(String type, ResourceSnapshot resourceSnapshot) { + private DiscoveryResponse buildDiscoveryResponse(String type, PushRequest pushRequest) { @SuppressWarnings("unchecked") - ApiGenerator serviceEntryGenerator = (ApiGenerator) apiGeneratorFactory.getApiGenerator(type); - List rawResources = serviceEntryGenerator.generate(resourceSnapshot); - + ApiGenerator generator = (ApiGenerator) apiGeneratorFactory.getApiGenerator(type); + List rawResources = generator.generate(pushRequest); + if (rawResources == null) { + return null; + } + String nonce = NonceGenerator.generateNonce(); return DiscoveryResponse.newBuilder() .setTypeUrl(type) .addAllResources(rawResources) - .setVersionInfo(resourceSnapshot.getVersion()) + .setVersionInfo(pushRequest.getResourceSnapshot().getVersion()) + .setNonce(nonce).build(); + } + + @Override + public StreamObserver deltaAggregatedResources(StreamObserver responseObserver) { + // Init snapshot of nacos service info. + resourceManager.initResourceSnapshot(); + AbstractConnection newConnection = new DeltaConnection(responseObserver); + + return new StreamObserver() { + private boolean initRequest = true; + + @Override + public void onNext(DeltaDiscoveryRequest deltaDiscoveryRequest) { + // init connection + if (initRequest) { + newConnection.setConnectionId(deltaDiscoveryRequest.getNode().getId()); + deltaConnections.put(newConnection.getConnectionId(), newConnection); + initRequest = false; + } + + deltaProcess(deltaDiscoveryRequest, newConnection); + } + + @Override + public void onError(Throwable throwable) { + Loggers.MAIN.error("delta xds: {} stream error.", newConnection.getConnectionId(), throwable); + clear(); + } + + @Override + public void onCompleted() { + Loggers.MAIN.info("delta xds: {} stream close.", newConnection.getConnectionId()); + responseObserver.onCompleted(); + clear(); + } + + private void clear() { + deltaConnections.remove(newConnection.getConnectionId()); + } + }; + } + + public void deltaProcess(DeltaDiscoveryRequest deltaDiscoveryRequest, AbstractConnection connection) { + if (!deltaShouldPush(deltaDiscoveryRequest, connection)) { + return; + } + ResourceSnapshot resourceSnapshot = resourceManager.getResourceSnapshot(); + PushRequest pushRequest = new PushRequest(resourceSnapshot, true); + + Set subscribe = new HashSet<>(deltaDiscoveryRequest.getResourceNamesSubscribeList()); + pushRequest.setSubscribe(subscribe); + connection.getWatchedStatusByType(deltaDiscoveryRequest.getTypeUrl()).setLastSubscribe(subscribe); + + DeltaDiscoveryResponse response = buildDeltaDiscoveryResponse(deltaDiscoveryRequest.getTypeUrl(), pushRequest); + connection.push(response, connection.getWatchedStatusByType(deltaDiscoveryRequest.getTypeUrl())); + } + + private boolean deltaShouldPush(DeltaDiscoveryRequest deltaDiscoveryRequest, AbstractConnection connection) { + String type = deltaDiscoveryRequest.getTypeUrl(); + String connectionId = connection.getConnectionId(); + + // Suitable for bug of istio + // See https://github.com/istio/istio/pull/34633 + if (type.equals(MESH_CONFIG_PROTO_PACKAGE)) { + Loggers.MAIN.info("delta xds: type {} should be ignored.", type); + return false; + } + + WatchedStatus watchedStatus; + if (deltaDiscoveryRequest.getResponseNonce().isEmpty()) { + Loggers.MAIN.info("delta xds: init request, type {}, connection-id {}", + type, connectionId); + watchedStatus = new WatchedStatus(); + watchedStatus.setType(deltaDiscoveryRequest.getTypeUrl()); + connection.addWatchedResource(deltaDiscoveryRequest.getTypeUrl(), watchedStatus); + + return true; + } + + watchedStatus = connection.getWatchedStatusByType(deltaDiscoveryRequest.getTypeUrl()); + if (watchedStatus == null) { + Loggers.MAIN.info("delta xds: reconnect, type {}, connection-id {}, nonce {}.", + type, connectionId, deltaDiscoveryRequest.getResponseNonce()); + watchedStatus = new WatchedStatus(); + watchedStatus.setType(deltaDiscoveryRequest.getTypeUrl()); + connection.addWatchedResource(deltaDiscoveryRequest.getTypeUrl(), watchedStatus); + + return true; + } + + if (deltaDiscoveryRequest.getErrorDetail().getCode() != 0) { + Loggers.MAIN.error("delta xds: ACK error, connection-id: {}, code: {}, message: {}", + connectionId, + deltaDiscoveryRequest.getErrorDetail().getCode(), + deltaDiscoveryRequest.getErrorDetail().getMessage()); + watchedStatus.setLastAckOrNack(true); + return false; + } + + if (!watchedStatus.getLatestNonce().equals(deltaDiscoveryRequest.getResponseNonce())) { + Loggers.MAIN.warn("delta xds: request dis match, type {}, connection-id {}", + deltaDiscoveryRequest.getTypeUrl(), + connection.getConnectionId()); + return false; + } + + // This request is ack, we should record version and nonce. + //TODO: setAckedVersion + watchedStatus.setAckedNonce(deltaDiscoveryRequest.getResponseNonce()); + watchedStatus.setLastSubscribe(new HashSet<>(deltaDiscoveryRequest.getResourceNamesSubscribeList())); + Loggers.MAIN.info("delta xds: ack, type {}, connection-id {}, nonce {}", type, connectionId, deltaDiscoveryRequest.getResponseNonce()); + return false; + } + + public void handleDeltaEvent(PushRequest pushRequest) { + if (deltaConnections.size() == 0) { + return; + } + pushRequest.setFull(pushRequest.getResourceSnapshot().getIstioConfig().isFullEnabled()); + for (AbstractConnection connection : deltaConnections.values()) { + WatchedStatus watchedStatus = connection.getWatchedStatusByType(SERVICE_ENTRY_PROTO_PACKAGE); + if (watchedStatus != null && watchedStatus.isLastAckOrNack()) { + pushRequest.setSubscribe(watchedStatus.getLastSubscribe()); + DeltaDiscoveryResponse serviceEntryResponse = buildDeltaDiscoveryResponse(SERVICE_ENTRY_PROTO_PACKAGE, pushRequest); + if (serviceEntryResponse != null) { + connection.push(serviceEntryResponse, watchedStatus); + } + } + + WatchedStatus edsWatchedStatus = connection.getWatchedStatusByType(ENDPOINT_TYPE); + if (edsWatchedStatus != null && edsWatchedStatus.isLastAckOrNack()) { + pushRequest.setSubscribe(edsWatchedStatus.getLastSubscribe()); + DeltaDiscoveryResponse edsResponse = buildDeltaDiscoveryResponse(ENDPOINT_TYPE, pushRequest); + if (edsResponse != null) { + connection.push(edsResponse, edsWatchedStatus); + } + } + } + } + + private DeltaDiscoveryResponse buildDeltaDiscoveryResponse(String type, PushRequest pushRequest) { + @SuppressWarnings("unchecked") + ApiGenerator generator = (ApiGenerator) apiGeneratorFactory.getApiGenerator(type); + List rawResources = generator.deltaGenerate(pushRequest); + if (rawResources == null) { + return null; + } + + String nonce = NonceGenerator.generateNonce(); + return DeltaDiscoveryResponse.newBuilder() + .setTypeUrl(type) + .addAllResources(rawResources) + .addAllRemovedResources(pushRequest.getRemoved()) + .setSystemVersionInfo(pushRequest.getResourceSnapshot().getVersion()) .setNonce(nonce).build(); } } diff --git a/istio/src/main/java/com/alibaba/nacos/istio/xds/ServiceEntryXdsGenerator.java b/istio/src/main/java/com/alibaba/nacos/istio/xds/ServiceEntryXdsGenerator.java index cd7e0495e24..b12741dd72b 100644 --- a/istio/src/main/java/com/alibaba/nacos/istio/xds/ServiceEntryXdsGenerator.java +++ b/istio/src/main/java/com/alibaba/nacos/istio/xds/ServiceEntryXdsGenerator.java @@ -17,24 +17,33 @@ package com.alibaba.nacos.istio.xds; import com.alibaba.nacos.istio.api.ApiGenerator; -import com.alibaba.nacos.istio.common.ResourceSnapshot; +import com.alibaba.nacos.istio.misc.IstioConfig; +import com.alibaba.nacos.istio.model.IstioService; +import com.alibaba.nacos.istio.model.PushRequest; import com.alibaba.nacos.istio.model.ServiceEntryWrapper; import com.google.protobuf.Any; import istio.mcp.v1alpha1.MetadataOuterClass.Metadata; import istio.mcp.v1alpha1.ResourceOuterClass.Resource; +import istio.networking.v1alpha3.ServiceEntryOuterClass; import istio.networking.v1alpha3.ServiceEntryOuterClass.ServiceEntry; import java.util.ArrayList; import java.util.List; +import java.util.Map; +import java.util.Set; import static com.alibaba.nacos.istio.api.ApiConstants.*; +import static com.alibaba.nacos.istio.util.IstioCrdUtil.buildServiceEntry; +import static com.alibaba.nacos.istio.util.IstioCrdUtil.parseServiceEntryNameToServiceName; /** * @author special.fy */ public final class ServiceEntryXdsGenerator implements ApiGenerator { - private volatile static ServiceEntryXdsGenerator singleton = null; + private static volatile ServiceEntryXdsGenerator singleton = null; + + private List serviceEntries; public static ServiceEntryXdsGenerator getInstance() { if (singleton == null) { @@ -48,10 +57,20 @@ public static ServiceEntryXdsGenerator getInstance() { } @Override - public List generate(ResourceSnapshot resourceSnapshot) { + public List generate(PushRequest pushRequest) { List resources = new ArrayList<>(); - - List serviceEntries = resourceSnapshot.getServiceEntries(); + serviceEntries = new ArrayList<>(16); + IstioConfig istioConfig = pushRequest.getResourceSnapshot().getIstioConfig(); + Map serviceInfoMap = pushRequest.getResourceSnapshot().getIstioResources().getIstioServiceMap(); + + for (Map.Entry entry : serviceInfoMap.entrySet()) { + String serviceName = entry.getKey(); + + ServiceEntryWrapper serviceEntryWrapper = buildServiceEntry(serviceName, serviceName + istioConfig.getDomainSuffix(), serviceInfoMap.get(serviceName)); + if (serviceEntryWrapper != null) { + serviceEntries.add(serviceEntryWrapper); + } + } for (ServiceEntryWrapper serviceEntryWrapper : serviceEntries) { Metadata metadata = serviceEntryWrapper.getMetadata(); ServiceEntry serviceEntry = serviceEntryWrapper.getServiceEntry(); @@ -65,7 +84,60 @@ public List generate(ResourceSnapshot resourceSnapshot) { for (Resource resource : resources) { result.add(Any.newBuilder().setValue(resource.toByteString()).setTypeUrl(MCP_RESOURCE_PROTO).build()); } + + return result; + } + + @Override + public List deltaGenerate(PushRequest pushRequest) { + if (pushRequest.isFull()) { + return null; + } + + List result = new ArrayList<>(); + serviceEntries = new ArrayList<>(); + Set reason = pushRequest.getReason(); + IstioConfig istioConfig = pushRequest.getResourceSnapshot().getIstioConfig(); + Map istioServiceMap = pushRequest.getResourceSnapshot().getIstioResources().getIstioServiceMap(); + + if (pushRequest.getSubscribe().size() != 0) { + for (String subscribe : pushRequest.getSubscribe()) { + String serviceName = parseServiceEntryNameToServiceName(subscribe, istioConfig.getDomainSuffix()); + if (reason.contains(serviceName)) { + if (istioServiceMap.containsKey(serviceName)) { + ServiceEntryWrapper serviceEntryWrapper = buildServiceEntry(serviceName, subscribe, istioServiceMap.get(serviceName)); + if (serviceEntryWrapper != null) { + serviceEntries.add(serviceEntryWrapper); + } else { + pushRequest.addRemoved(subscribe); + } + } else { + pushRequest.addRemoved(subscribe); + } + } + } + } else { + for (Map.Entry entry : istioServiceMap.entrySet()) { + String hostName = entry.getKey() + "." + istioConfig.getDomainSuffix(); + ServiceEntryWrapper serviceEntryWrapper = buildServiceEntry(entry.getKey(), hostName, entry.getValue()); + if (serviceEntryWrapper != null) { + serviceEntries.add(serviceEntryWrapper); + } else { + pushRequest.addRemoved(hostName); + } + } + } + + + for (ServiceEntryWrapper serviceEntryWrapper : serviceEntries) { + ServiceEntryOuterClass.ServiceEntry serviceEntry = serviceEntryWrapper.getServiceEntry(); + + Any any = Any.newBuilder().setValue(serviceEntry.toByteString()).setTypeUrl(SERVICE_ENTRY_PROTO).build(); + result.add(io.envoyproxy.envoy.service.discovery.v3.Resource.newBuilder().setResource(any).setVersion( + pushRequest.getResourceSnapshot().getVersion()).build()); + } + return result; } } diff --git a/naming/src/main/java/com/alibaba/nacos/naming/core/InstanceOperatorClientImpl.java b/naming/src/main/java/com/alibaba/nacos/naming/core/InstanceOperatorClientImpl.java index 20135482aa0..73a60bcb47b 100644 --- a/naming/src/main/java/com/alibaba/nacos/naming/core/InstanceOperatorClientImpl.java +++ b/naming/src/main/java/com/alibaba/nacos/naming/core/InstanceOperatorClientImpl.java @@ -22,6 +22,7 @@ import com.alibaba.nacos.api.naming.pojo.Instance; import com.alibaba.nacos.api.naming.pojo.ServiceInfo; import com.alibaba.nacos.api.naming.utils.NamingUtils; +import com.alibaba.nacos.common.notify.NotifyCenter; import com.alibaba.nacos.common.utils.ConvertUtils; import com.alibaba.nacos.common.utils.InternetAddressUtil; import com.alibaba.nacos.naming.core.v2.ServiceManager; @@ -30,6 +31,7 @@ import com.alibaba.nacos.naming.core.v2.client.impl.IpPortBasedClient; import com.alibaba.nacos.naming.core.v2.client.manager.ClientManager; import com.alibaba.nacos.naming.core.v2.client.manager.ClientManagerDelegate; +import com.alibaba.nacos.naming.core.v2.event.metadata.InfoChangeEvent; import com.alibaba.nacos.naming.core.v2.index.ServiceStorage; import com.alibaba.nacos.naming.core.v2.metadata.InstanceMetadata; import com.alibaba.nacos.naming.core.v2.metadata.NamingMetadataManager; @@ -130,6 +132,7 @@ public void updateInstance(String namespaceId, String serviceName, Instance inst String metadataId = InstancePublishInfo .genMetadataId(instance.getIp(), instance.getPort(), instance.getClusterName()); metadataOperateService.updateInstanceMetadata(service, metadataId, buildMetadata(instance)); + NotifyCenter.publishEvent(new InfoChangeEvent.InstanceInfoChangeEvent(service, instance)); } private InstanceMetadata buildMetadata(Instance instance) { @@ -336,4 +339,4 @@ private Service getService(String namespaceId, String serviceName, boolean ephem return Service.newService(namespaceId, groupName, serviceNameNoGrouped, ephemeral); } -} +} \ No newline at end of file diff --git a/naming/src/main/java/com/alibaba/nacos/naming/core/ServiceOperatorV2Impl.java b/naming/src/main/java/com/alibaba/nacos/naming/core/ServiceOperatorV2Impl.java index b34d54c215b..d315f08c200 100644 --- a/naming/src/main/java/com/alibaba/nacos/naming/core/ServiceOperatorV2Impl.java +++ b/naming/src/main/java/com/alibaba/nacos/naming/core/ServiceOperatorV2Impl.java @@ -19,9 +19,11 @@ import com.alibaba.nacos.api.common.Constants; import com.alibaba.nacos.api.exception.NacosException; import com.alibaba.nacos.api.naming.utils.NamingUtils; +import com.alibaba.nacos.common.notify.NotifyCenter; import com.alibaba.nacos.common.utils.JacksonUtils; import com.alibaba.nacos.naming.constants.FieldsConstants; import com.alibaba.nacos.naming.core.v2.ServiceManager; +import com.alibaba.nacos.naming.core.v2.event.metadata.InfoChangeEvent; import com.alibaba.nacos.naming.core.v2.index.ServiceStorage; import com.alibaba.nacos.naming.core.v2.metadata.ClusterMetadata; import com.alibaba.nacos.naming.core.v2.metadata.NamingMetadataManager; @@ -90,6 +92,7 @@ public void update(Service service, ServiceMetadata metadata) throws NacosExcept String.format("service %s not found!", service.getGroupedServiceName())); } metadataOperateService.updateServiceMetadata(service, metadata); + NotifyCenter.publishEvent(new InfoChangeEvent.ServiceInfoChangeEvent(service)); } @Override @@ -248,4 +251,4 @@ public Collection searchServiceName(String namespaceId, String expr, boo } return result; } -} +} \ No newline at end of file diff --git a/naming/src/main/java/com/alibaba/nacos/naming/core/v2/event/metadata/InfoChangeEvent.java b/naming/src/main/java/com/alibaba/nacos/naming/core/v2/event/metadata/InfoChangeEvent.java new file mode 100644 index 00000000000..da85e091081 --- /dev/null +++ b/naming/src/main/java/com/alibaba/nacos/naming/core/v2/event/metadata/InfoChangeEvent.java @@ -0,0 +1,66 @@ +/* + * Copyright 1999-2022 Alibaba Group Holding Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.alibaba.nacos.naming.core.v2.event.metadata; + +import com.alibaba.nacos.api.naming.pojo.Instance; +import com.alibaba.nacos.common.notify.Event; +import com.alibaba.nacos.naming.core.v2.pojo.Service; + +/**. + * @author RocketEngine26 + * @date 2022/9/14 下午6:12 + */ +public class InfoChangeEvent extends Event { + private static final long serialVersionUID = 2222222222222L; + + private final Service service; + + public InfoChangeEvent(Service service) { + this.service = service; + } + + public Service getService() { + return service; + } + + public static class ServiceInfoChangeEvent extends InfoChangeEvent { + + private static final long serialVersionUID = 3333333333333L; + + public ServiceInfoChangeEvent(Service service) { + super(service); + service.renewUpdateTime(); + } + } + + public static class InstanceInfoChangeEvent extends InfoChangeEvent { + + private static final long serialVersionUID = 4444444444444L; + + private final Instance instance; + + public InstanceInfoChangeEvent(Service service, Instance instance) { + super(service); + this.instance = instance; + } + + public Instance getInstance() { + return instance; + } + } +} \ No newline at end of file From f62cf16aedae4813b31905b15cf94e7ef5dfcff9 Mon Sep 17 00:00:00 2001 From: KomachiSion Date: Tue, 11 Jul 2023 14:52:57 +0800 Subject: [PATCH 3/7] Upgrade to 3.0.0-SNAPSHOT. --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 7ae1c82cee7..2bb1bc528bd 100644 --- a/pom.xml +++ b/pom.xml @@ -88,7 +88,7 @@ - 2.3.0-SNAPSHOT + 3.0.0-SNAPSHOT UTF-8 UTF-8 From 678b90a91149b5757e4ee684c4be37f35eb2bc51 Mon Sep 17 00:00:00 2001 From: PoisonGravity Date: Mon, 30 Oct 2023 17:55:04 +0800 Subject: [PATCH 4/7] Summer ospp#10389 (#11208) * Notify IstioConfigChangeEvent by parsing config tags * Parse Configs and Create Virtual Service * function complete * Style Fixed * Delete Personal Info * add some comments and regex process func * Nacos Istio LDS & RDS xds protocol * reset configs --- .../model/event/IstioConfigChangeEvent.java | 59 ++++ .../service/ConfigOperationService.java | 8 + .../config/server/utils/ConfigTagUtil.java | 74 +++++ .../main/resources/META-INF/logback/nacos.xml | 1 - .../alibaba/nacos/istio/api/ApiConstants.java | 5 + .../nacos/istio/api/ApiGeneratorFactory.java | 4 + .../istio/common/IstioConfigProcessor.java | 137 ++++++++ .../NacosServiceInfoResourceWatcher.java | 2 +- .../nacos/istio/model/DestinationRule.java | 144 +++++++++ .../nacos/istio/model/VirtualService.java | 303 ++++++++++++++++++ .../nacos/istio/server/IstioServer.java | 3 +- .../alibaba/nacos/istio/xds/LdsGenerator.java | 245 ++++++++++++++ .../nacos/istio/xds/NacosXdsService.java | 66 +++- .../alibaba/nacos/istio/xds/RdsGenerator.java | 246 ++++++++++++++ 14 files changed, 1291 insertions(+), 6 deletions(-) create mode 100644 config/src/main/java/com/alibaba/nacos/config/server/model/event/IstioConfigChangeEvent.java create mode 100644 config/src/main/java/com/alibaba/nacos/config/server/utils/ConfigTagUtil.java create mode 100644 istio/src/main/java/com/alibaba/nacos/istio/common/IstioConfigProcessor.java create mode 100644 istio/src/main/java/com/alibaba/nacos/istio/model/DestinationRule.java create mode 100644 istio/src/main/java/com/alibaba/nacos/istio/model/VirtualService.java create mode 100644 istio/src/main/java/com/alibaba/nacos/istio/xds/LdsGenerator.java create mode 100644 istio/src/main/java/com/alibaba/nacos/istio/xds/RdsGenerator.java diff --git a/config/src/main/java/com/alibaba/nacos/config/server/model/event/IstioConfigChangeEvent.java b/config/src/main/java/com/alibaba/nacos/config/server/model/event/IstioConfigChangeEvent.java new file mode 100644 index 00000000000..218ae03eac0 --- /dev/null +++ b/config/src/main/java/com/alibaba/nacos/config/server/model/event/IstioConfigChangeEvent.java @@ -0,0 +1,59 @@ +/* + * Copyright 1999-2023 Alibaba Group Holding Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.nacos.config.server.model.event; + +public class IstioConfigChangeEvent extends ConfigDataChangeEvent { + + public final String content; + + public final String type; + + public IstioConfigChangeEvent(String dataId, String group, long gmtModified, String content, String type) { + super(dataId, group, gmtModified); + this.content = content; + this.type = type; + } + + public IstioConfigChangeEvent(boolean isBeta, String dataId, String group, String tenant, long gmtModified, + String content, String type) { + super(isBeta, dataId, group, tenant, gmtModified); + this.content = content; + this.type = type; + } + + public IstioConfigChangeEvent(boolean isBeta, String dataId, String group, long gmtModified, String content, + String type) { + super(isBeta, dataId, group, gmtModified); + this.content = content; + this.type = type; + } + + public IstioConfigChangeEvent(boolean isBeta, String dataId, String group, String tenant, String tag, + long gmtModified, String content, String type) { + super(isBeta, dataId, group, tenant, tag, gmtModified); + this.content = content; + this.type = type; + } + + public IstioConfigChangeEvent(String dataId, String group, String tenant, boolean isBatch, long gmtModified, + String content, String type) { + super(dataId, group, tenant, isBatch, gmtModified); + this.content = content; + this.type = type; + } + +} diff --git a/config/src/main/java/com/alibaba/nacos/config/server/service/ConfigOperationService.java b/config/src/main/java/com/alibaba/nacos/config/server/service/ConfigOperationService.java index 265732b0c33..a5530cd129a 100644 --- a/config/src/main/java/com/alibaba/nacos/config/server/service/ConfigOperationService.java +++ b/config/src/main/java/com/alibaba/nacos/config/server/service/ConfigOperationService.java @@ -25,11 +25,13 @@ import com.alibaba.nacos.config.server.model.ConfigOperateResult; import com.alibaba.nacos.config.server.model.event.ConfigDataChangeEvent; import com.alibaba.nacos.config.server.model.ConfigRequestInfo; +import com.alibaba.nacos.config.server.model.event.IstioConfigChangeEvent; import com.alibaba.nacos.config.server.model.form.ConfigForm; import com.alibaba.nacos.config.server.service.repository.ConfigInfoBetaPersistService; import com.alibaba.nacos.config.server.service.repository.ConfigInfoPersistService; import com.alibaba.nacos.config.server.service.repository.ConfigInfoTagPersistService; import com.alibaba.nacos.config.server.service.trace.ConfigTraceService; +import com.alibaba.nacos.config.server.utils.ConfigTagUtil; import com.alibaba.nacos.config.server.utils.ParamUtils; import com.alibaba.nacos.config.server.utils.TimeUtils; import com.alibaba.nacos.sys.utils.InetUtils; @@ -102,6 +104,12 @@ public Boolean publishConfig(ConfigForm configForm, ConfigRequestInfo configRequ ConfigChangePublisher.notifyConfigChange( new ConfigDataChangeEvent(false, configForm.getDataId(), configForm.getGroup(), configForm.getNamespaceId(), configOperateResult.getLastModified())); + if (ConfigTagUtil.isIstio(configForm.getConfigTags())) { + ConfigChangePublisher.notifyConfigChange( + new IstioConfigChangeEvent(configForm.getDataId(), configForm.getGroup(), + configOperateResult.getLastModified(), configForm.getContent(), + ConfigTagUtil.getIstioType(configForm.getConfigTags()))); + } } else { persistEvent = ConfigTraceService.PERSISTENCE_EVENT_TAG + "-" + configForm.getTag(); configOperateResult = configInfoTagPersistService.insertOrUpdateTag(configInfo, configForm.getTag(), diff --git a/config/src/main/java/com/alibaba/nacos/config/server/utils/ConfigTagUtil.java b/config/src/main/java/com/alibaba/nacos/config/server/utils/ConfigTagUtil.java new file mode 100644 index 00000000000..0fda7e843dc --- /dev/null +++ b/config/src/main/java/com/alibaba/nacos/config/server/utils/ConfigTagUtil.java @@ -0,0 +1,74 @@ +/* + * Copyright 1999-2023 Alibaba Group Holding Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.nacos.config.server.utils; + +import java.util.Arrays; + +public class ConfigTagUtil { + + public static final String VIRTUAL_SERVICE = "virtual-service"; + + public static final String DESTINATION_RULE = "destination-rule"; + + private static final String TAGS_DELIMITER = ","; + + private static final String HYPHEN = "-"; + + /** + *

Checks if config tags contains "virtual-service" or "destination-rule".

+ * @param configTags the tags to check + * @return {@code true} if the config tags contains "virtual-service" or "destination-rule". + * @throws IllegalArgumentException if configTags is null. + */ + public static boolean isIstio(String configTags) { + if (configTags == null) { + throw new IllegalArgumentException("configTags cannot be null."); + } + + if (configTags.isEmpty()) { + return false; + } + + return Arrays.stream(configTags.split(TAGS_DELIMITER)) + .map(tag -> tag.trim().replaceAll(HYPHEN, "")) + .anyMatch(tag -> tag.equalsIgnoreCase(VIRTUAL_SERVICE.replaceAll(HYPHEN, "")) + || tag.equalsIgnoreCase(DESTINATION_RULE.replaceAll(HYPHEN, ""))); + } + + /** + *

Gets the type of Istio from the config tags.

+ * @param configTags the tags to check + * @return the type of Istio if it is found, {@code null} otherwise. + * @throws IllegalArgumentException if configTags is null. + */ + public static String getIstioType(String configTags) { + if (configTags == null) { + throw new IllegalArgumentException("configTags cannot be null."); + } + + if (configTags.isEmpty()) { + return null; + } + + return Arrays.stream(configTags.split(TAGS_DELIMITER)) + .map(tag -> tag.trim().replaceAll(HYPHEN, "")) + .filter(tag -> tag.equalsIgnoreCase(VIRTUAL_SERVICE.replaceAll(HYPHEN, "")) + || tag.equalsIgnoreCase(DESTINATION_RULE.replaceAll(HYPHEN, ""))) + .findFirst() + .orElse(null); + } +} diff --git a/core/src/main/resources/META-INF/logback/nacos.xml b/core/src/main/resources/META-INF/logback/nacos.xml index ca543a8bdd1..1296290bad8 100644 --- a/core/src/main/resources/META-INF/logback/nacos.xml +++ b/core/src/main/resources/META-INF/logback/nacos.xml @@ -275,7 +275,6 @@ - diff --git a/istio/src/main/java/com/alibaba/nacos/istio/api/ApiConstants.java b/istio/src/main/java/com/alibaba/nacos/istio/api/ApiConstants.java index edafe3fbbfe..e1c8d4fecc3 100644 --- a/istio/src/main/java/com/alibaba/nacos/istio/api/ApiConstants.java +++ b/istio/src/main/java/com/alibaba/nacos/istio/api/ApiConstants.java @@ -51,4 +51,9 @@ public class ApiConstants { */ public static final String CLUSTER_TYPE = API_TYPE_PREFIX + "envoy.config.cluster.v3.Cluster"; public static final String ENDPOINT_TYPE = API_TYPE_PREFIX + "envoy.config.endpoint.v3.ClusterLoadAssignment"; + + public static final String LISTENER_TYPE = API_TYPE_PREFIX + "envoy.config.listener.v3.Listener"; + + public static final String ROUTE_TYPE = API_TYPE_PREFIX + "envoy.config.route.v3.RouteConfiguration"; + } diff --git a/istio/src/main/java/com/alibaba/nacos/istio/api/ApiGeneratorFactory.java b/istio/src/main/java/com/alibaba/nacos/istio/api/ApiGeneratorFactory.java index 75665c56ec9..7a02f74b382 100644 --- a/istio/src/main/java/com/alibaba/nacos/istio/api/ApiGeneratorFactory.java +++ b/istio/src/main/java/com/alibaba/nacos/istio/api/ApiGeneratorFactory.java @@ -21,6 +21,8 @@ import com.alibaba.nacos.istio.xds.CdsGenerator; import com.alibaba.nacos.istio.xds.EdsGenerator; import com.alibaba.nacos.istio.xds.EmptyXdsGenerator; +import com.alibaba.nacos.istio.xds.LdsGenerator; +import com.alibaba.nacos.istio.xds.RdsGenerator; import com.alibaba.nacos.istio.xds.ServiceEntryXdsGenerator; import org.springframework.stereotype.Component; @@ -46,6 +48,8 @@ public ApiGeneratorFactory() { //xds apiGeneratorMap.put(CLUSTER_TYPE, CdsGenerator.getInstance()); apiGeneratorMap.put(ENDPOINT_TYPE, EdsGenerator.getInstance()); + apiGeneratorMap.put(LISTENER_TYPE, LdsGenerator.getInstance()); + apiGeneratorMap.put(ROUTE_TYPE, RdsGenerator.getInstance()); // mcp apiGeneratorMap.put(SERVICE_ENTRY_COLLECTION, ServiceEntryMcpGenerator.getInstance()); diff --git a/istio/src/main/java/com/alibaba/nacos/istio/common/IstioConfigProcessor.java b/istio/src/main/java/com/alibaba/nacos/istio/common/IstioConfigProcessor.java new file mode 100644 index 00000000000..2c0d1e685b7 --- /dev/null +++ b/istio/src/main/java/com/alibaba/nacos/istio/common/IstioConfigProcessor.java @@ -0,0 +1,137 @@ + +/* + * Copyright 1999-2023 Alibaba Group Holding Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.nacos.istio.common; + +import com.alibaba.nacos.common.notify.Event; +import com.alibaba.nacos.common.notify.NotifyCenter; +import com.alibaba.nacos.common.notify.listener.Subscriber; +import com.alibaba.nacos.config.server.model.event.IstioConfigChangeEvent; +import com.alibaba.nacos.istio.misc.Loggers; +import com.alibaba.nacos.istio.model.DestinationRule; +import com.alibaba.nacos.istio.model.PushRequest; +import com.alibaba.nacos.istio.model.VirtualService; +import com.alibaba.nacos.istio.xds.NacosXdsService; +import com.alibaba.nacos.sys.utils.ApplicationUtils; +import org.springframework.stereotype.Component; +import org.springframework.stereotype.Service; +import org.yaml.snakeyaml.Yaml; +import java.util.Map; + +/** + * Listener for IstioConfig. + * + * @author junwei + */ +@Service +@Component +public class IstioConfigProcessor { + + private NacosXdsService nacosXdsService; + + private NacosResourceManager resourceManager; + + public static final String CONFIG_REASON = "config"; + + private static final String VIRTUAL_SERVICE = "VirtualService"; + + private static final String DESTINATION_RULE = "DestinationRule"; + + private static final String API_VERSION = "networking.istio.io/v1alpha3"; + + Yaml yaml = new Yaml(); + + public IstioConfigProcessor() { + NotifyCenter.registerSubscriber(new Subscriber() { + @Override + public void onEvent(Event event) { + if (event instanceof IstioConfigChangeEvent) { + IstioConfigChangeEvent istioConfigChangeEvent = (IstioConfigChangeEvent) event; + String content = istioConfigChangeEvent.content; + if (isContentValid(content) && tryParseContent(content)) { + PushRequest pushRequest = new PushRequest(content, true); + if (null == nacosXdsService) { + nacosXdsService = ApplicationUtils.getBean(NacosXdsService.class); + } + if (null == resourceManager) { + resourceManager = ApplicationUtils.getBean(NacosResourceManager.class); + } + pushRequest.addReason(CONFIG_REASON); + ResourceSnapshot snapshot = resourceManager.createResourceSnapshot(); + pushRequest.setResourceSnapshot(snapshot); + nacosXdsService.handleConfigEvent(pushRequest); + } + } + + } + + @Override + public Class subscribeType() { + return IstioConfigChangeEvent.class; + } + }); + } + + public boolean isContentValid(String content) { + if (content == null || content.trim().isEmpty()) { + Loggers.MAIN.warn("Configuration content is null or empty."); + return false; + } + + Map obj; + try { + obj = yaml.load(content); + } catch (Exception e) { + Loggers.MAIN.error("Invalid YAML content.", e); + return false; + } + + String apiVersion = obj.containsKey("apiVersion") ? (String) obj.get("apiVersion") : ""; + String kind = obj.containsKey("kind") ? (String) obj.get("kind") : ""; + + return API_VERSION.equals(apiVersion) && (VIRTUAL_SERVICE.equals(kind) + || DESTINATION_RULE.equals(kind)) && obj.containsKey("metadata") && obj.containsKey("spec"); + } + + public boolean tryParseContent(String content) { + + if (content == null || content.trim().isEmpty()) { + Loggers.MAIN.warn("Configuration content is null or empty."); + return false; + } + + try { + Map obj = yaml.load(content); + String kind = (String) obj.get("kind"); + if (VIRTUAL_SERVICE.equals(kind)) { + VirtualService virtualService = yaml.loadAs(content, VirtualService.class); + Loggers.MAIN.info("Configuration Content was successfully parsed as VirtualService."); + } else if (DESTINATION_RULE.equals(kind)) { + DestinationRule destinationRule = yaml.loadAs(content, DestinationRule.class); + Loggers.MAIN.info("Configuration Content was successfully parsed as DestinationRule."); + } else { + Loggers.MAIN.warn("Unknown Config : Unknown 'kind' field in content: {}", kind); + return false; + } + return true; + } catch (Exception e) { + Loggers.MAIN.error("Error parsing configuration content: {}", e.getMessage(), e); + return false; + } + } + +} diff --git a/istio/src/main/java/com/alibaba/nacos/istio/common/NacosServiceInfoResourceWatcher.java b/istio/src/main/java/com/alibaba/nacos/istio/common/NacosServiceInfoResourceWatcher.java index ba588e6d8b3..121cebcd814 100644 --- a/istio/src/main/java/com/alibaba/nacos/istio/common/NacosServiceInfoResourceWatcher.java +++ b/istio/src/main/java/com/alibaba/nacos/istio/common/NacosServiceInfoResourceWatcher.java @@ -135,7 +135,7 @@ public void onEvent(com.alibaba.nacos.common.notify.Event event) { InfoChangeEvent.InstanceInfoChangeEvent instanceInfoChangeEvent = (InfoChangeEvent.InstanceInfoChangeEvent) event; Service service = instanceInfoChangeEvent.getService(); String serviceName = IstioCrdUtil.buildServiceName(service); - + boolean full = update(serviceName, service); PushRequest pushRequest = new PushRequest(serviceName, full); pushRequestQueue.add(pushRequest); diff --git a/istio/src/main/java/com/alibaba/nacos/istio/model/DestinationRule.java b/istio/src/main/java/com/alibaba/nacos/istio/model/DestinationRule.java new file mode 100644 index 00000000000..30271aaef1d --- /dev/null +++ b/istio/src/main/java/com/alibaba/nacos/istio/model/DestinationRule.java @@ -0,0 +1,144 @@ +/* + * Copyright 1999-2023 Alibaba Group Holding Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.nacos.istio.model; + +import java.util.List; + +public class DestinationRule { + + private String apiVersion; + + private String kind; + + private Metadata metadata; + + private Spec spec; + + public static class Metadata { + + private String name; + + private String namespace; + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getNamespace() { + return namespace; + } + + public void setNamespace(String namespace) { + this.namespace = namespace; + } + } + + public static class Spec { + + private String host; + + private List subsets; + + public static class Subset { + + private String name; + + private Labels labels; + + public static class Labels { + + private String version; + + public String getVersion() { + return version; + } + + public void setVersion(String version) { + this.version = version; + } + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public Labels getLabels() { + return labels; + } + + public void setLabels(Labels labels) { + this.labels = labels; + } + } + + public String getHost() { + return host; + } + + public void setHost(String host) { + this.host = host; + } + + public List getSubsets() { + return subsets; + } + + public void setSubsets(List subsets) { + this.subsets = subsets; + } + } + + public String getApiVersion() { + return apiVersion; + } + + public void setApiVersion(String apiVersion) { + this.apiVersion = apiVersion; + } + + public String getKind() { + return kind; + } + + public void setKind(String kind) { + this.kind = kind; + } + + public Metadata getMetadata() { + return metadata; + } + + public void setMetadata(Metadata metadata) { + this.metadata = metadata; + } + + public Spec getSpec() { + return spec; + } + + public void setSpec(Spec spec) { + this.spec = spec; + } +} diff --git a/istio/src/main/java/com/alibaba/nacos/istio/model/VirtualService.java b/istio/src/main/java/com/alibaba/nacos/istio/model/VirtualService.java new file mode 100644 index 00000000000..8c3855078aa --- /dev/null +++ b/istio/src/main/java/com/alibaba/nacos/istio/model/VirtualService.java @@ -0,0 +1,303 @@ +package com.alibaba.nacos.istio.model; +/* + * Copyright 1999-2023 Alibaba Group Holding Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.util.List; + +public class VirtualService { + + private String apiVersion; + + private String kind; + + private Metadata metadata; + + private Spec spec; + + public VirtualService() {} + + public static class Metadata { + + private String name; + + private String namespace; + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getNamespace() { + return namespace; + } + + public void setNamespace(String namespace) { + this.namespace = namespace; + } + } + + public static class Spec { + + private List hosts; + + private List http; + + public static class Http { + + private String name; + + private List match; + + private Rewrite rewrite; + + private List route; + + private Redirect redirect; + + public static class Match { + + private Uri uri; + + public static class Uri { + + private String prefix; + + private String exact; + + private String regex; + + public String getPrefix() { + return prefix; + } + + public void setPrefix(String prefix) { + this.prefix = prefix; + } + + public String getExact() { + return exact; + } + + public void setExact(String exact) { + this.exact = exact; + } + + public String getRegex() { + return regex; + } + + public void setRegex(String regex) { + this.regex = regex; + } + } + + public Uri getUri() { + return uri; + } + + public void setUri(Uri uri) { + this.uri = uri; + } + } + + public static class Rewrite { + + private String uri; + + public String getUri() { + return uri; + } + + public void setUri(String uri) { + this.uri = uri; + } + } + + public static class Route { + + private Destination destination; + + public static class Destination { + + private String host; + + private String subset; + + private Port port; + + public static class Port { + + private int number; + + public int getNumber() { + return number; + } + + public void setNumber(int number) { + this.number = number; + } + } + + public Port getPort() { + return port; + } + + public void setPort(Port port) { + this.port = port; + } + + public String getHost() { + return host; + } + + public void setHost(String host) { + this.host = host; + } + + public String getSubset() { + return subset; + } + + public void setSubset(String subset) { + this.subset = subset; + } + } + + public Destination getDestination() { + return destination; + } + + public void setDestination(Destination destination) { + this.destination = destination; + } + } + + public static class Redirect { + + private String uri; + + private String authority; + + public String getUri() { + return uri; + } + + public void setUri(String uri) { + this.uri = uri; + } + + public String getAuthority() { + return authority; + } + + public void setAuthority(String authority) { + this.authority = authority; + } + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public List getMatch() { + return match; + } + + public void setMatch(List match) { + this.match = match; + } + + public Rewrite getRewrite() { + return rewrite; + } + + public void setRewrite(Rewrite rewrite) { + this.rewrite = rewrite; + } + + public List getRoute() { + return route; + } + + public void setRoute(List route) { + this.route = route; + } + + public Redirect getRedirect() { + return redirect; + } + + public void setRedirect(Redirect redirect) { + this.redirect = redirect; + } + } + + public List getHosts() { + return hosts; + } + + public void setHosts(List hosts) { + this.hosts = hosts; + } + + public List getHttp() { + return http; + } + + public void setHttp(List http) { + this.http = http; + } + } + + public String getApiVersion() { + return apiVersion; + } + + public void setApiVersion(String apiVersion) { + this.apiVersion = apiVersion; + } + + public String getKind() { + return kind; + } + + public void setKind(String kind) { + this.kind = kind; + } + + public Metadata getMetadata() { + return metadata; + } + + public void setMetadata(Metadata metadata) { + this.metadata = metadata; + } + + public Spec getSpec() { + return spec; + } + + public void setSpec(Spec spec) { + this.spec = spec; + } +} diff --git a/istio/src/main/java/com/alibaba/nacos/istio/server/IstioServer.java b/istio/src/main/java/com/alibaba/nacos/istio/server/IstioServer.java index f7ea6a22a57..ba3a808da64 100644 --- a/istio/src/main/java/com/alibaba/nacos/istio/server/IstioServer.java +++ b/istio/src/main/java/com/alibaba/nacos/istio/server/IstioServer.java @@ -48,6 +48,7 @@ public class IstioServer { @Autowired private NacosXdsService nacosXdsService; + /** * Start. * @@ -71,9 +72,7 @@ public void start() throws IOException { @Override public void run() { - System.out.println("Stopping Nacos Istio server..."); IstioServer.this.stop(); - System.out.println("Nacos Istio server stopped..."); } }); } diff --git a/istio/src/main/java/com/alibaba/nacos/istio/xds/LdsGenerator.java b/istio/src/main/java/com/alibaba/nacos/istio/xds/LdsGenerator.java new file mode 100644 index 00000000000..89eab9c973f --- /dev/null +++ b/istio/src/main/java/com/alibaba/nacos/istio/xds/LdsGenerator.java @@ -0,0 +1,245 @@ +/* + * Copyright 1999-2023 Alibaba Group Holding Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.nacos.istio.xds; + +import com.alibaba.nacos.istio.api.ApiGenerator; +import com.alibaba.nacos.istio.misc.Loggers; +import com.alibaba.nacos.istio.model.IstioService; +import com.alibaba.nacos.istio.model.PushRequest; +import com.google.protobuf.Any; +import com.google.protobuf.ByteString; +import io.envoyproxy.envoy.config.accesslog.v3.AccessLog; +import io.envoyproxy.envoy.config.core.v3.Address; +import io.envoyproxy.envoy.config.core.v3.AggregatedConfigSource; +import io.envoyproxy.envoy.config.core.v3.ConfigSource; +import io.envoyproxy.envoy.config.core.v3.SocketAddress; +import io.envoyproxy.envoy.config.listener.v3.Filter; +import io.envoyproxy.envoy.config.listener.v3.FilterChain; +import io.envoyproxy.envoy.config.listener.v3.Listener; +import io.envoyproxy.envoy.config.route.v3.Route; +import io.envoyproxy.envoy.config.route.v3.RouteAction; +import io.envoyproxy.envoy.config.route.v3.RouteConfiguration; +import io.envoyproxy.envoy.config.route.v3.RouteMatch; +import io.envoyproxy.envoy.config.route.v3.VirtualHost; +import io.envoyproxy.envoy.extensions.filters.http.router.v3.Router; +import io.envoyproxy.envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager; +import io.envoyproxy.envoy.extensions.filters.network.http_connection_manager.v3.HttpFilter; +import io.envoyproxy.envoy.extensions.filters.network.http_connection_manager.v3.Rds; +import io.envoyproxy.envoy.service.discovery.v3.Resource; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import static com.alibaba.nacos.istio.api.ApiConstants.LISTENER_TYPE; +import static io.envoyproxy.envoy.config.core.v3.ApiVersion.V2_VALUE; + +public class LdsGenerator implements ApiGenerator { + + public static final String INIT_LISTENER = "bootstrap_listener"; + + private static final String INIT_LISTENER_NAME = "listener_0"; + + private static final String INIT_LISTENER_ADDRESS = "0.0.0.0"; + + private static final int INIT_LISTENER_PORT = 80; + + private static final String ACCESS_LOGGER_NAME = "envoy.access_loggers.stdout"; + + private static final String TYPE_URL_ACCESS_LOG = "type.googleapis.com/envoy.extensions.access_loggers.stream.v3.StdoutAccessLog"; + + private static final int DEFAULT_PORT_INCREMENT = 1; + + public static final String ROUTE_CONFIGURATION_SUFFIX = "_route_config"; + + public static final String BOOTSTRAP_UPSTREAM_CLUSTER = "nacos_xds"; + + public static final String DEFAULT_FILTER_CHAIN_NAME = "filter_chain_0"; + + public static final String DEFAULT_FILTER_TYPE = "envoy.filters.network.http_connection_manager"; + + public static final String DEFAULT_HTTPMANAGER_PREFIX = "ingress_http"; + + public static final String DEFAULT_HTTP_ROUTER_TYPE = "envoy.filters.http.router"; + + private static volatile LdsGenerator singleton = null; + + public static LdsGenerator getInstance() { + if (singleton == null) { + synchronized (LdsGenerator.class) { + if (singleton == null) { + singleton = new LdsGenerator(); + } + } + } + return singleton; + } + + @Override + public List generate(PushRequest pushRequest) { + if (!pushRequest.isFull()) { + return null; + } + Map istioServiceMap = pushRequest.getResourceSnapshot().getIstioResources() + .getIstioServiceMap(); + List result = new ArrayList<>(); + result.add(buildBootstrapListener()); + + if (pushRequest.isFull()) { + for (Map.Entry entry : istioServiceMap.entrySet()) { + IstioService istioService = entry.getValue(); + if (istioService != null) { + String rdsName = entry.getKey() + ROUTE_CONFIGURATION_SUFFIX; + result.add(buildDynamicListener(entry.getKey(), INIT_LISTENER_ADDRESS, istioService.getPort(), rdsName)); + } else { + Loggers.MAIN.error("Attempt to create listener for non-existent service"); + } + } + } + return result; + } + + @Override + public List deltaGenerate(PushRequest pushRequest) { + return null; + } + + /** + * Constructs a default Envoy listener configuration with specified parameters. + * This method prepares the necessary configurations for both bootstrap and non-bootstrap scenarios. + */ + private static Any buildDefaultListener(String listenerName, String listenerAddress, int listenerPort, String rdsName, boolean isBootstrap) { + if (!isBootstrap && ((listenerName == null) || (listenerPort == 0) || (rdsName == null))) { + Loggers.MAIN.error("Listener name, Listener port and RDS name cannot be null."); + return null; + } + + Listener.Builder listenerBuilder = Listener.newBuilder().setName(listenerName); + + listenerAddress = (listenerAddress == null) ? INIT_LISTENER_ADDRESS : listenerAddress; + int portValue = isBootstrap ? INIT_LISTENER_PORT : listenerPort + DEFAULT_PORT_INCREMENT; + listenerBuilder.setAddress(Address.newBuilder() + .setSocketAddress(SocketAddress.newBuilder().setAddress(listenerAddress).setPortValue(portValue))); + + String routeConfigName = isBootstrap ? INIT_LISTENER + ROUTE_CONFIGURATION_SUFFIX : listenerName + ROUTE_CONFIGURATION_SUFFIX; + String virtualHostName = isBootstrap ? INIT_LISTENER : listenerName; + + RouteConfiguration routeConfiguration = RouteConfiguration.newBuilder() + .setName(routeConfigName) + .addVirtualHosts( + VirtualHost.newBuilder() + .setName(virtualHostName) + .addDomains("*") + .addRoutes( + Route.newBuilder() + .setMatch(RouteMatch.newBuilder().setPrefix("/").build()) + .setRoute(RouteAction.newBuilder().setCluster( + BOOTSTRAP_UPSTREAM_CLUSTER).build()) + .build() + ) + .build() + ) + .build(); + + HttpConnectionManager httpConnectionManager = HttpConnectionManager.newBuilder().setStatPrefix(DEFAULT_HTTPMANAGER_PREFIX) + .addAccessLog(buildAccessLog()).setCodecType(HttpConnectionManager.CodecType.AUTO) + .setRouteConfig(routeConfiguration).addHttpFilters(createHttpFilter()).build(); + + listenerBuilder.addFilterChains(createFilterChain(httpConnectionManager)); + + Listener listener = listenerBuilder.build(); + + return Any.newBuilder().setValue(listener.toByteString()).setTypeUrl(LISTENER_TYPE).build(); + } + + /** + * Creates the initial bootstrap listener configuration. + * This is used during the startup phase of the Envoy server to construct the base configuration. + */ + private static Any buildBootstrapListener() { + return buildDefaultListener(INIT_LISTENER_NAME, INIT_LISTENER_ADDRESS, INIT_LISTENER_PORT, null, true); + } + + /** + * Constructs a default listener configuration for static environments. + * This method is designed to provide configurations for scenarios where dynamic discovery services might not be used. + */ + private static Any buildDefaultStaticListener(String listenerName, String listenerAddress, int listenerPort, String rdsName) { + if (INIT_LISTENER.equals(listenerName)) { + return buildBootstrapListener(); + } + return buildDefaultListener(listenerName, listenerAddress, listenerPort, rdsName, false); + } + + /** + * Constructs a listener configuration for environments using dynamic service discovery. + * This method prepares the listener to utilize dynamic routing and service discovery services with rds. + */ + private static Any buildDynamicListener(String listenerName, String listenerAddress, int listenerPort, + String rdsName) { + if (INIT_LISTENER.equals(listenerName)) { + return buildBootstrapListener(); + } + + if ((listenerName == null) || (listenerPort == 0) || (rdsName == null)) { + Loggers.MAIN.error("Listener name, Listener port and RDS name cannot be null."); + return null; + } + + listenerAddress = (listenerAddress == null) ? INIT_LISTENER_ADDRESS : listenerAddress; + Listener.Builder listenerBuilder = Listener.newBuilder().setName(listenerName); + + listenerBuilder.setAddress(Address.newBuilder() + .setSocketAddress(SocketAddress.newBuilder().setAddress(listenerAddress).setPortValue(listenerPort + DEFAULT_PORT_INCREMENT))); + ConfigSource configSource = createConfigSource(); + + Rds rds = Rds.newBuilder().setConfigSource(configSource).setRouteConfigName(rdsName).build(); + + HttpConnectionManager httpConnectionManager = HttpConnectionManager.newBuilder().setStatPrefix(DEFAULT_HTTPMANAGER_PREFIX) + .addAccessLog(buildAccessLog()).setCodecType(HttpConnectionManager.CodecType.AUTO).setRds(rds) + .addHttpFilters(createHttpFilter()).build(); + + listenerBuilder.addFilterChains(createFilterChain(httpConnectionManager)); + + Listener listener = listenerBuilder.build(); + + return Any.newBuilder().setValue(listener.toByteString()).setTypeUrl(LISTENER_TYPE).build(); + + } + + private static AccessLog buildAccessLog() { + return AccessLog.newBuilder().setName(ACCESS_LOGGER_NAME) + .setTypedConfig(Any.newBuilder().setTypeUrl(TYPE_URL_ACCESS_LOG).setValue(ByteString.EMPTY).build()) + .build(); + } + + private static ConfigSource createConfigSource() { + return ConfigSource.newBuilder().setAds(AggregatedConfigSource.newBuilder()) + .setResourceApiVersionValue(V2_VALUE).build(); + } + + private static HttpFilter createHttpFilter() { + return HttpFilter.newBuilder().setName(DEFAULT_HTTP_ROUTER_TYPE) + .setTypedConfig(Any.pack(Router.newBuilder().build())).build(); + } + + private static FilterChain createFilterChain(HttpConnectionManager httpConnectionManager) { + return FilterChain.newBuilder().setName(DEFAULT_FILTER_CHAIN_NAME).addFilters( + Filter.newBuilder().setName(DEFAULT_FILTER_TYPE) + .setTypedConfig(Any.pack(httpConnectionManager))).build(); + } +} diff --git a/istio/src/main/java/com/alibaba/nacos/istio/xds/NacosXdsService.java b/istio/src/main/java/com/alibaba/nacos/istio/xds/NacosXdsService.java index 27674056a01..1180089c0db 100644 --- a/istio/src/main/java/com/alibaba/nacos/istio/xds/NacosXdsService.java +++ b/istio/src/main/java/com/alibaba/nacos/istio/xds/NacosXdsService.java @@ -42,9 +42,13 @@ import static com.alibaba.nacos.istio.api.ApiConstants.CLUSTER_TYPE; import static com.alibaba.nacos.istio.api.ApiConstants.ENDPOINT_TYPE; +import static com.alibaba.nacos.istio.api.ApiConstants.LISTENER_TYPE; import static com.alibaba.nacos.istio.api.ApiConstants.MESH_CONFIG_PROTO_PACKAGE; +import static com.alibaba.nacos.istio.api.ApiConstants.ROUTE_TYPE; import static com.alibaba.nacos.istio.api.ApiConstants.SERVICE_ENTRY_PROTO_PACKAGE; import static com.alibaba.nacos.istio.util.IstioCrdUtil.parseClusterNameToServiceName; +import static com.alibaba.nacos.istio.xds.LdsGenerator.INIT_LISTENER; +import static com.alibaba.nacos.istio.xds.RdsGenerator.DEFAULT_ROUTE_CONFIGURATION; /** * @author special.fy @@ -117,9 +121,24 @@ public void process(DiscoveryRequest discoveryRequest, AbstractConnection connection : connections.values()) { + //RDS + WatchedStatus watchedStatus; + watchedStatus = connection.getWatchedStatusByType(ROUTE_TYPE); + if (watchedStatus == null) { + watchedStatus = new WatchedStatus(); + watchedStatus.setType(ROUTE_TYPE); + connection.addWatchedResource(ROUTE_TYPE, watchedStatus); + } + WatchedStatus rdsWatchedStatus = connection.getWatchedStatusByType(ROUTE_TYPE); + if (rdsWatchedStatus != null) { + DiscoveryResponse rdsResponse = buildDiscoveryResponse(ROUTE_TYPE, pushRequest); + connection.push(rdsResponse, rdsWatchedStatus); + } } } diff --git a/istio/src/main/java/com/alibaba/nacos/istio/xds/RdsGenerator.java b/istio/src/main/java/com/alibaba/nacos/istio/xds/RdsGenerator.java new file mode 100644 index 00000000000..4fcab4c287a --- /dev/null +++ b/istio/src/main/java/com/alibaba/nacos/istio/xds/RdsGenerator.java @@ -0,0 +1,246 @@ +/* + * Copyright 1999-2023 Alibaba Group Holding Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.nacos.istio.xds; + +import com.alibaba.nacos.istio.api.ApiGenerator; +import com.alibaba.nacos.istio.misc.Loggers; +import com.alibaba.nacos.istio.model.IstioService; +import com.alibaba.nacos.istio.model.PushRequest; +import com.alibaba.nacos.istio.model.VirtualService; +import com.google.protobuf.Any; +import io.envoyproxy.envoy.config.core.v3.TrafficDirection; +import io.envoyproxy.envoy.config.route.v3.RedirectAction; +import io.envoyproxy.envoy.config.route.v3.Route; +import io.envoyproxy.envoy.config.route.v3.RouteAction; +import io.envoyproxy.envoy.config.route.v3.RouteConfiguration; +import io.envoyproxy.envoy.config.route.v3.RouteMatch; +import io.envoyproxy.envoy.config.route.v3.VirtualHost; +import io.envoyproxy.envoy.service.discovery.v3.Resource; +import io.envoyproxy.envoy.type.matcher.v3.RegexMatcher; +import org.yaml.snakeyaml.Yaml; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import static com.alibaba.nacos.istio.api.ApiConstants.ROUTE_TYPE; +import static com.alibaba.nacos.istio.util.IstioCrdUtil.buildClusterName; + +public class RdsGenerator implements ApiGenerator { + + public static final String DEFAULT_ROUTE_CONFIGURATION = "default_route_configuration"; + + public static final String CONFIG_REASON = "config"; + + public static final String DOMAIN_SUFFIX = ".nacos"; + + public static final String ROUTE_CONFIGURATION_SUFFIX = "_route_config"; + + public static final String BOOTSTRAP_UPSTREAM_CLUSTER = "nacos_xds"; + + private final Yaml yaml = new Yaml(); + + private static volatile RdsGenerator singleton = null; + + public static RdsGenerator getInstance() { + if (singleton == null) { + synchronized (RdsGenerator.class) { + if (singleton == null) { + singleton = new RdsGenerator(); + } + } + } + return singleton; + } + + @Override + public List generate(PushRequest pushRequest) { + + List result = new ArrayList<>(); + Set reasons = pushRequest.getReason(); + if (reasons.contains(DEFAULT_ROUTE_CONFIGURATION)) { + reasons.stream() + .filter(reason -> !DEFAULT_ROUTE_CONFIGURATION.equals(reason)) + .forEach(reason -> { + result.add(buildDefaultRouteConfiguration(reason)); + }); + } else if (reasons.contains(CONFIG_REASON)) { + reasons.stream() + .filter(reason -> !CONFIG_REASON.equals(reason)) + .forEach(reason -> { + VirtualService vs = parseContent(reason, VirtualService.class); + result.add(generateRdsFromVirtualService(vs, pushRequest)); + }); + } else { + reasons.forEach(reason -> { + result.add(buildDefaultRouteConfiguration(reason)); + }); + } + + return result; + } + + @Override + public List deltaGenerate(PushRequest pushRequest) { + return null; + } + + private static Any buildDefaultRouteConfiguration(String routeConfigurationName) { + if (routeConfigurationName == null) { + throw new IllegalArgumentException("routeConfigurationName cannot be null"); + } + String virtualHostName = routeConfigurationName; + if (routeConfigurationName.endsWith(ROUTE_CONFIGURATION_SUFFIX)) { + virtualHostName = routeConfigurationName.substring(0, routeConfigurationName.length() - ROUTE_CONFIGURATION_SUFFIX.length()); + } + RouteConfiguration routeConfiguration = RouteConfiguration.newBuilder() + .setName(routeConfigurationName) + .addVirtualHosts( + VirtualHost.newBuilder() + .setName(virtualHostName) + .addDomains("*") + .addRoutes( + Route.newBuilder() + .setMatch(RouteMatch.newBuilder().setPrefix("/").build()) + .setRoute(RouteAction.newBuilder().setCluster(BOOTSTRAP_UPSTREAM_CLUSTER).build()) + .build() + ) + .build() + ) + .build(); + + return Any.newBuilder().setValue(routeConfiguration.toByteString()).setTypeUrl(ROUTE_TYPE).build(); + } + + /*** + *

generate Rds From VirtualService.

+ * @param virtualService VirtualService Parsed + * @param pushRequest PushRequest + * @return + */ + public static Any generateRdsFromVirtualService(VirtualService virtualService, PushRequest pushRequest) { + List httpRoutes = virtualService.getSpec().getHttp(); + List hosts = virtualService.getSpec().getHosts(); + Map istioServiceMap = pushRequest.getResourceSnapshot().getIstioResources().getIstioServiceMap(); + List hostnames = getMatchingHostnames(hosts, pushRequest); + String virtualHostName = virtualService.getMetadata().getName(); + for (Map.Entry entry : istioServiceMap.entrySet()) { + if (entry.getKey().contains(virtualHostName)) { + virtualHostName = buildClusterName(TrafficDirection.OUTBOUND, "", entry.getKey() + DOMAIN_SUFFIX, + entry.getValue().getPort()); + Loggers.MAIN.info("Setting virtualHostName: {}", virtualHostName); + } + } + VirtualHost.Builder virtualHostBuilder = VirtualHost.newBuilder() + .setName(virtualHostName) + .addAllDomains(hostnames); + + for (VirtualService.Spec.Http httpRoute : httpRoutes) { + processHttpRoute(httpRoute, virtualHostBuilder, pushRequest); + } + + RouteConfiguration routeConfiguration = RouteConfiguration.newBuilder() + .setName(virtualService.getMetadata().getName() + ROUTE_CONFIGURATION_SUFFIX).addVirtualHosts(virtualHostBuilder) + .build(); + return Any.newBuilder().setValue(routeConfiguration.toByteString()).setTypeUrl(ROUTE_TYPE).build(); + + } + + private T parseContent(String content, Class valueType) { + return yaml.loadAs(content, valueType); + } + + private static List getMatchingHostnames(List hosts, PushRequest pushRequest) { + List hostnames = new ArrayList<>(); + Map istioServiceMap = pushRequest.getResourceSnapshot().getIstioResources().getIstioServiceMap(); + for (String host: hosts) { + if ("*".equals(host)) { + hostnames.add("*"); + break; + } + for (Map.Entry entry : istioServiceMap.entrySet()) { + if (entry.getKey().contains(host)) { + String hostname = buildClusterName(TrafficDirection.OUTBOUND, "", + host + DOMAIN_SUFFIX, entry.getValue().getPort()); + Loggers.MAIN.info("Matching hostname: {}", hostname); + hostnames.add(hostname); + } + } + } + return hostnames; + } + + private static void processHttpRoute(VirtualService.Spec.Http httpRoute, VirtualHost.Builder virtualHostBuilder, PushRequest pushRequest) { + Route.Builder routeBuilder = Route.newBuilder(); + + if (httpRoute.getName() != null) { + routeBuilder.setName(httpRoute.getName()); + } + + for (VirtualService.Spec.Http.Match match : httpRoute.getMatch()) { + RouteMatch.Builder routeMatchBuilder = RouteMatch.newBuilder(); + if (match.getUri().getPrefix() != null) { + routeMatchBuilder.setPrefix(match.getUri().getPrefix()); + } else if (match.getUri().getExact() != null) { + routeMatchBuilder.setPath(match.getUri().getExact()); + } else if (match.getUri().getRegex() != null) { // 检查是否定义了正则表达式 + RegexMatcher regexMatcher = RegexMatcher.newBuilder() + .setRegex(match.getUri().getRegex()) + .build(); + routeMatchBuilder.setSafeRegex(regexMatcher); + } + routeBuilder.setMatch(routeMatchBuilder); + } + + if (httpRoute.getRedirect() != null) { + setRedirectAction(httpRoute.getRedirect(), routeBuilder); + } else { + setRouteAction(httpRoute, routeBuilder, pushRequest); + } + virtualHostBuilder.addRoutes(routeBuilder); + } + + private static void setRouteAction(VirtualService.Spec.Http httpRoute, Route.Builder routeBuilder, PushRequest pushRequest) { + Map istioServiceMap = pushRequest.getResourceSnapshot().getIstioResources().getIstioServiceMap(); + RouteAction.Builder routeActionBuilder = RouteAction.newBuilder(); + String hostName = httpRoute.getRoute().get(0).getDestination().getHost(); + if (httpRoute.getRewrite() != null) { + routeActionBuilder.setPrefixRewrite(httpRoute.getRewrite().getUri()); + } + String destName = hostName; + for (Map.Entry entry : istioServiceMap.entrySet()) { + if (entry.getKey().contains(hostName)) { + destName = buildClusterName(TrafficDirection.OUTBOUND, "", entry.getKey() + DOMAIN_SUFFIX, + entry.getValue().getPort()); + Loggers.MAIN.info("Setting route action to destination: {}", destName); + } + } + routeBuilder.setRoute(routeActionBuilder.setCluster(destName)); + } + + private static void setRedirectAction(VirtualService.Spec.Http.Redirect redirect, Route.Builder routeBuilder) { + RedirectAction.Builder redirectBuilder = RedirectAction.newBuilder(); + if (redirect.getUri() != null) { + redirectBuilder.setPathRedirect(redirect.getUri()); + } + if (redirect.getAuthority() != null) { + redirectBuilder.setHostRedirect(redirect.getAuthority()); + } + routeBuilder.setRedirect(redirectBuilder); + } +} From ed532b2d3c5c475125dfbf9c5a30aa2408829673 Mon Sep 17 00:00:00 2001 From: KomachiSion Date: Tue, 23 Jul 2024 16:04:26 +0800 Subject: [PATCH 5/7] abstract kubernetes client version to property. --- pom.xml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index ac9463f83c6..6584568154a 100644 --- a/pom.xml +++ b/pom.xml @@ -151,6 +151,7 @@ 2.0 5.3.34 5.7.12 + 14.0.0 5.10.2 @@ -1037,13 +1038,13 @@ io.kubernetes client-java-api - 14.0.0 + ${kubernetes.client.version} io.kubernetes client-java - 14.0.0 + ${kubernetes.client.version} From d4d00c29796f83ef83d7d5356b882dbf6897e84c Mon Sep 17 00:00:00 2001 From: KomachiSion Date: Tue, 23 Jul 2024 16:41:06 +0800 Subject: [PATCH 6/7] configTags is an optional parameters, fix istio force required problem. --- .../com/alibaba/nacos/config/server/utils/ConfigTagUtil.java | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/config/src/main/java/com/alibaba/nacos/config/server/utils/ConfigTagUtil.java b/config/src/main/java/com/alibaba/nacos/config/server/utils/ConfigTagUtil.java index 0fda7e843dc..1ffcf4a9c55 100644 --- a/config/src/main/java/com/alibaba/nacos/config/server/utils/ConfigTagUtil.java +++ b/config/src/main/java/com/alibaba/nacos/config/server/utils/ConfigTagUtil.java @@ -32,17 +32,14 @@ public class ConfigTagUtil { *

Checks if config tags contains "virtual-service" or "destination-rule".

* @param configTags the tags to check * @return {@code true} if the config tags contains "virtual-service" or "destination-rule". - * @throws IllegalArgumentException if configTags is null. */ public static boolean isIstio(String configTags) { if (configTags == null) { - throw new IllegalArgumentException("configTags cannot be null."); + return false; } - if (configTags.isEmpty()) { return false; } - return Arrays.stream(configTags.split(TAGS_DELIMITER)) .map(tag -> tag.trim().replaceAll(HYPHEN, "")) .anyMatch(tag -> tag.equalsIgnoreCase(VIRTUAL_SERVICE.replaceAll(HYPHEN, "")) From e2d44f2fd2b12124d844d152b7025e3f758d00e6 Mon Sep 17 00:00:00 2001 From: "blake.qiu" <46370663+Bo-Qiu@users.noreply.github.com> Date: Thu, 31 Oct 2024 14:54:14 +0800 Subject: [PATCH 7/7] fix(#12769): remove config history by derby. (#12796) --- .../datasource/impl/derby/HistoryConfigInfoMapperByDerby.java | 4 ++-- .../impl/derby/HistoryConfigInfoMapperByDerbyTest.java | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/plugin/datasource/src/main/java/com/alibaba/nacos/plugin/datasource/impl/derby/HistoryConfigInfoMapperByDerby.java b/plugin/datasource/src/main/java/com/alibaba/nacos/plugin/datasource/impl/derby/HistoryConfigInfoMapperByDerby.java index 08942c9a4fe..041d580ff7e 100644 --- a/plugin/datasource/src/main/java/com/alibaba/nacos/plugin/datasource/impl/derby/HistoryConfigInfoMapperByDerby.java +++ b/plugin/datasource/src/main/java/com/alibaba/nacos/plugin/datasource/impl/derby/HistoryConfigInfoMapperByDerby.java @@ -33,8 +33,8 @@ public class HistoryConfigInfoMapperByDerby extends AbstractMapperByDerby implem @Override public MapperResult removeConfigHistory(MapperContext context) { - String sql = "DELETE FROM his_config_info WHERE id IN( " - + "SELECT id FROM his_config_info WHERE gmt_modified < ? OFFSET 0 ROWS FETCH NEXT ? ROWS ONLY)"; + String sql = "DELETE FROM his_config_info WHERE nid IN( " + + "SELECT nid FROM his_config_info WHERE gmt_modified < ? OFFSET 0 ROWS FETCH NEXT ? ROWS ONLY)"; return new MapperResult(sql, CollectionUtils.list(context.getWhereParameter(FieldConstant.START_TIME), context.getWhereParameter(FieldConstant.LIMIT_SIZE))); } diff --git a/plugin/datasource/src/test/java/com/alibaba/nacos/plugin/datasource/impl/derby/HistoryConfigInfoMapperByDerbyTest.java b/plugin/datasource/src/test/java/com/alibaba/nacos/plugin/datasource/impl/derby/HistoryConfigInfoMapperByDerbyTest.java index 434a4d583fc..61d16757141 100644 --- a/plugin/datasource/src/test/java/com/alibaba/nacos/plugin/datasource/impl/derby/HistoryConfigInfoMapperByDerbyTest.java +++ b/plugin/datasource/src/test/java/com/alibaba/nacos/plugin/datasource/impl/derby/HistoryConfigInfoMapperByDerbyTest.java @@ -63,7 +63,7 @@ void setUp() throws Exception { void testRemoveConfigHistory() { MapperResult mapperResult = historyConfigInfoMapperByDerby.removeConfigHistory(context); assertEquals(mapperResult.getSql(), - "DELETE FROM his_config_info WHERE id IN( SELECT id FROM his_config_info WHERE gmt_modified < ? " + "DELETE FROM his_config_info WHERE nid IN( SELECT nid FROM his_config_info WHERE gmt_modified < ? " + "OFFSET 0 ROWS FETCH NEXT ? ROWS ONLY)"); assertArrayEquals(new Object[] {startTime, limitSize}, mapperResult.getParamList().toArray()); }