Merge remote-tracking branch 'upstream/main' into access

lhpqaq · Jan 3, 2025 · aa449bf · aa449bf
2 parents b6b3346 + 5529204
commit aa449bf
Show file tree

Hide file tree

Showing 87 changed files with 1,651 additions and 568 deletions.
diff --git a/...manager-agent/src/main/java/org/apache/bigtop/manager/agent/metrics/MetricsCollector.java b/...manager-agent/src/main/java/org/apache/bigtop/manager/agent/metrics/MetricsCollector.java
@@ -47,7 +47,7 @@ public class MetricsCollector {
     @Qualifier("diskIOMultiGauge") private MultiGauge diskIOMultiGauge;
 
     @Async
-    @Scheduled(cron = "*/10 * *  * * ?")
+    @Scheduled(cron = "0,30 * *  * * ?")
     public void collect() {
         // refresh agent host monitoring data
         scrape();

diff --git a/...sistant/src/main/java/org/apache/bigtop/manager/ai/assistant/GeneralAssistantFactory.java b/...sistant/src/main/java/org/apache/bigtop/manager/ai/assistant/GeneralAssistantFactory.java
@@ -20,7 +20,6 @@
 
 import org.apache.bigtop.manager.ai.assistant.config.GeneralAssistantConfig;
 import org.apache.bigtop.manager.ai.assistant.provider.ChatMemoryStoreProvider;
-import org.apache.bigtop.manager.ai.assistant.provider.LocSystemPromptProvider;
 import org.apache.bigtop.manager.ai.core.AbstractAIAssistantFactory;
 import org.apache.bigtop.manager.ai.core.config.AIAssistantConfig;
 import org.apache.bigtop.manager.ai.core.enums.PlatformType;
@@ -50,15 +49,14 @@ public class GeneralAssistantFactory extends AbstractAIAssistantFactory {
     private ChatMemoryStoreProvider chatMemoryStoreProvider;
 
     private void configureSystemPrompt(AIAssistant.Builder builder, SystemPrompt systemPrompt, String locale) {
-        LocSystemPromptProvider locSystemPromptProvider = (LocSystemPromptProvider) systemPromptProvider;
         List<String> systemPrompts = new ArrayList<>();
         if (systemPrompt != null) {
-            systemPrompts.add(locSystemPromptProvider.getSystemMessage(systemPrompt));
+            systemPrompts.add(systemPromptProvider.getSystemMessage(systemPrompt));
         }
         if (locale != null) {
-            systemPrompts.add(locSystemPromptProvider.getLanguagePrompt(locale));
+            systemPrompts.add(systemPromptProvider.getLanguagePrompt(locale));
         }
-        builder.withSystemPrompt(locSystemPromptProvider.getSystemMessages(systemPrompts));
+        builder.withSystemPrompt(systemPromptProvider.getSystemMessages(systemPrompts));
     }
 
     private AIAssistant.Builder initializeBuilder(PlatformType platformType) {
@@ -82,7 +80,7 @@ public AIAssistant createWithPrompt(
         AIAssistant.Builder builder = initializeBuilder(platformType);
         builder.id(id)
                 .memoryStore(chatMemoryStoreProvider.createPersistentChatMemoryStore())
-                .withConfigProvider(generalAssistantConfig)
+                .withConfig(generalAssistantConfig)
                 .withToolProvider(toolProvider);
 
         configureSystemPrompt(builder, systemPrompt, generalAssistantConfig.getLanguage());
@@ -98,7 +96,7 @@ public AIAssistant createForTest(AIAssistantConfig config, ToolProvider toolProv
 
         builder.id(null)
                 .memoryStore(chatMemoryStoreProvider.createInMemoryChatMemoryStore())
-                .withConfigProvider(generalAssistantConfig)
+                .withConfig(generalAssistantConfig)
                 .withToolProvider(toolProvider);
 
         return builder.build();

diff --git a/...rc/main/java/org/apache/bigtop/manager/ai/assistant/provider/LocSystemPromptProvider.java b/...rc/main/java/org/apache/bigtop/manager/ai/assistant/provider/LocSystemPromptProvider.java
@@ -72,6 +72,7 @@ private String loadPromptFromFile(String fileName) {
         return Objects.requireNonNullElse(text, "You are a helpful assistant.");
     }
 
+    @Override
     public String getLanguagePrompt(String locale) {
         final String filePath = SystemPrompt.LANGUAGE_PROMPT.getValue() + '-' + locale + ".st";
         String text = loadTextFromFile(filePath);

diff --git a/...ant/src/test/java/org/apache/bigtop/manager/ai/assistant/GeneralAssistantFactoryTest.java b/...ant/src/test/java/org/apache/bigtop/manager/ai/assistant/GeneralAssistantFactoryTest.java
@@ -57,7 +57,7 @@ void testCreateAIAssistant() {
         AIAssistant.Builder mockBuilder = mock(OpenAIAssistant.Builder.class);
         when(mockBuilder.id(any())).thenReturn(mockBuilder);
         when(mockBuilder.memoryStore(any())).thenReturn(mockBuilder);
-        when(mockBuilder.withConfigProvider(any())).thenReturn(mockBuilder);
+        when(mockBuilder.withConfig(any())).thenReturn(mockBuilder);
         when(mockBuilder.withToolProvider(any())).thenReturn(mockBuilder);
         when(mockBuilder.withSystemPrompt(any())).thenReturn(mockBuilder);
         when(mockBuilder.build()).thenReturn(mock(AIAssistant.class));

diff --git a/...-manager-ai-core/src/main/java/org/apache/bigtop/manager/ai/core/AbstractAIAssistant.java b/...-manager-ai-core/src/main/java/org/apache/bigtop/manager/ai/core/AbstractAIAssistant.java
@@ -61,7 +61,7 @@ public abstract static class Builder implements AIAssistant.Builder {
         protected Object id;
 
         protected ChatMemoryStore chatMemoryStore;
-        protected AIAssistantConfig configProvider;
+        protected AIAssistantConfig config;
 
         protected ToolProvider toolProvider;
         protected String systemPrompt;
@@ -78,8 +78,8 @@ public Builder withSystemPrompt(String systemPrompt) {
             return this;
         }
 
-        public Builder withConfigProvider(AIAssistantConfig configProvider) {
-            this.configProvider = configProvider;
+        public Builder withConfig(AIAssistantConfig config) {
+            this.config = config;
             return this;
         }
 

diff --git a/...-manager-ai-core/src/main/java/org/apache/bigtop/manager/ai/core/factory/AIAssistant.java b/...-manager-ai-core/src/main/java/org/apache/bigtop/manager/ai/core/factory/AIAssistant.java
@@ -74,7 +74,7 @@ interface Builder {
 
         Builder memoryStore(ChatMemoryStore memoryStore);
 
-        Builder withConfigProvider(AIAssistantConfig configProvider);
+        Builder withConfig(AIAssistantConfig configProvider);
 
         Builder withToolProvider(ToolProvider toolProvider);
 

diff --git a/...i-core/src/main/java/org/apache/bigtop/manager/ai/core/provider/SystemPromptProvider.java b/...i-core/src/main/java/org/apache/bigtop/manager/ai/core/provider/SystemPromptProvider.java
@@ -29,5 +29,7 @@ public interface SystemPromptProvider {
     // return default system prompt
     String getSystemMessage();
 
+    String getLanguagePrompt(String locale);
+
     String getSystemMessages(List<String> systemPrompts);
 }
diff --git a/...ai-dashscope/src/main/java/org/apache/bigtop/manager/ai/dashscope/DashScopeAssistant.java b/...ai-dashscope/src/main/java/org/apache/bigtop/manager/ai/dashscope/DashScopeAssistant.java
@@ -65,17 +65,17 @@ public AIAssistant build() {
 
         @Override
         public ChatLanguageModel getChatLanguageModel() {
-            String model = ValidationUtils.ensureNotNull(configProvider.getModel(), "model");
-            String apiKey = ValidationUtils.ensureNotNull(
-                    configProvider.getCredentials().get("apiKey"), "apiKey");
+            String model = ValidationUtils.ensureNotNull(config.getModel(), "model");
+            String apiKey =
+                    ValidationUtils.ensureNotNull(config.getCredentials().get("apiKey"), "apiKey");
             return QwenChatModel.builder().apiKey(apiKey).modelName(model).build();
         }
 
         @Override
         public StreamingChatLanguageModel getStreamingChatLanguageModel() {
-            String model = ValidationUtils.ensureNotNull(configProvider.getModel(), "model");
-            String apiKey = ValidationUtils.ensureNotNull(
-                    configProvider.getCredentials().get("apiKey"), "apiKey");
+            String model = ValidationUtils.ensureNotNull(config.getModel(), "model");
+            String apiKey =
+                    ValidationUtils.ensureNotNull(config.getCredentials().get("apiKey"), "apiKey");
             return QwenStreamingChatModel.builder()
                     .apiKey(apiKey)
                     .modelName(model)

diff --git a/...-manager-ai-openai/src/main/java/org/apache/bigtop/manager/ai/openai/OpenAIAssistant.java b/...-manager-ai-openai/src/main/java/org/apache/bigtop/manager/ai/openai/OpenAIAssistant.java
@@ -51,9 +51,9 @@ public static class Builder extends AbstractAIAssistant.Builder {
 
         @Override
         public ChatLanguageModel getChatLanguageModel() {
-            String model = ValidationUtils.ensureNotNull(configProvider.getModel(), "model");
-            String apiKey = ValidationUtils.ensureNotNull(
-                    configProvider.getCredentials().get("apiKey"), "apiKey");
+            String model = ValidationUtils.ensureNotNull(config.getModel(), "model");
+            String apiKey =
+                    ValidationUtils.ensureNotNull(config.getCredentials().get("apiKey"), "apiKey");
             return OpenAiChatModel.builder()
                     .apiKey(apiKey)
                     .baseUrl(BASE_URL)
@@ -63,9 +63,9 @@ public ChatLanguageModel getChatLanguageModel() {
 
         @Override
         public StreamingChatLanguageModel getStreamingChatLanguageModel() {
-            String model = ValidationUtils.ensureNotNull(configProvider.getModel(), "model");
-            String apiKey = ValidationUtils.ensureNotNull(
-                    configProvider.getCredentials().get("apiKey"), "apiKey");
+            String model = ValidationUtils.ensureNotNull(config.getModel(), "model");
+            String apiKey =
+                    ValidationUtils.ensureNotNull(config.getCredentials().get("apiKey"), "apiKey");
             return OpenAiStreamingChatModel.builder()
                     .apiKey(apiKey)
                     .baseUrl(BASE_URL)

diff --git a/...nager-ai-qianfan/src/main/java/org/apache/bigtop/manager/ai/qianfan/QianFanAssistant.java b/...nager-ai-qianfan/src/main/java/org/apache/bigtop/manager/ai/qianfan/QianFanAssistant.java
@@ -65,11 +65,11 @@ public AIAssistant build() {
 
         @Override
         public ChatLanguageModel getChatLanguageModel() {
-            String model = ValidationUtils.ensureNotNull(configProvider.getModel(), "model");
-            String apiKey = ValidationUtils.ensureNotNull(
-                    configProvider.getCredentials().get("apiKey"), "apiKey");
-            String secretKey = ValidationUtils.ensureNotNull(
-                    configProvider.getCredentials().get("secretKey"), "secretKey");
+            String model = ValidationUtils.ensureNotNull(config.getModel(), "model");
+            String apiKey =
+                    ValidationUtils.ensureNotNull(config.getCredentials().get("apiKey"), "apiKey");
+            String secretKey =
+                    ValidationUtils.ensureNotNull(config.getCredentials().get("secretKey"), "secretKey");
             return QianfanChatModel.builder()
                     .apiKey(apiKey)
                     .secretKey(secretKey)
@@ -79,11 +79,11 @@ public ChatLanguageModel getChatLanguageModel() {
 
         @Override
         public StreamingChatLanguageModel getStreamingChatLanguageModel() {
-            String model = ValidationUtils.ensureNotNull(configProvider.getModel(), "model");
-            String apiKey = ValidationUtils.ensureNotNull(
-                    configProvider.getCredentials().get("apiKey"), "apiKey");
-            String secretKey = ValidationUtils.ensureNotNull(
-                    configProvider.getCredentials().get("secretKey"), "secretKey");
+            String model = ValidationUtils.ensureNotNull(config.getModel(), "model");
+            String apiKey =
+                    ValidationUtils.ensureNotNull(config.getCredentials().get("apiKey"), "apiKey");
+            String secretKey =
+                    ValidationUtils.ensureNotNull(config.getCredentials().get("secretKey"), "secretKey");
             return QianfanStreamingChatModel.builder()
                     .apiKey(apiKey)
                     .secretKey(secretKey)

diff --git a/bigtop-manager-common/src/main/java/org/apache/bigtop/manager/common/enums/Command.java b/bigtop-manager-common/src/main/java/org/apache/bigtop/manager/common/enums/Command.java
@@ -28,23 +28,18 @@
 @AllArgsConstructor
 @Getter
 public enum Command {
+    // Available for: Cluster, Host, Service, Component
     ADD("add", "Add"),
-
     REMOVE("remove", "Remove"),
-
     START("start", "Start"),
-
     STOP("stop", "Stop"),
-
-    STATUS("status", "Status"),
-
     RESTART("restart", "Restart"),
 
-    CONFIGURE("configure", "Configure"),
-
+    // Available for: Service, Component
+    STATUS("status", "Status"),
     CHECK("check", "Check"),
-
     CUSTOM("custom", "Custom"),
+    CONFIGURE("configure", "Configure"),
     ;
 
     private final String code;

diff --git a/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/po/HostPO.java b/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/po/HostPO.java
@@ -39,6 +39,9 @@ public class HostPO extends BasePO implements Serializable {
     @Column(name = "hostname")
     private String hostname;
 
+    @Column(name = "agent_dir")
+    private String agentDir;
+
     @Column(name = "ssh_user")
     private String sshUser;
 

diff --git a/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/po/JobPO.java b/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/po/JobPO.java
@@ -20,14 +20,12 @@
 
 import lombok.Data;
 import lombok.EqualsAndHashCode;
-import lombok.ToString;
 
 import jakarta.persistence.Column;
 import jakarta.persistence.Id;
 import jakarta.persistence.Lob;
 import jakarta.persistence.Table;
 import java.io.Serializable;
-import java.util.List;
 
 @Data
 @EqualsAndHashCode(callSuper = true)
@@ -50,7 +48,4 @@ public class JobPO extends BasePO implements Serializable {
 
     @Column(name = "cluster_id")
     private Long clusterId;
-
-    @ToString.Exclude
-    private List<StagePO> stages;
 }
diff --git a/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/po/ServicePO.java b/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/po/ServicePO.java
@@ -53,8 +53,8 @@ public class ServicePO extends BasePO implements Serializable {
     @Column(name = "stack")
     private String stack;
 
-    @Column(name = "need_restart")
-    private Boolean needRestart;
+    @Column(name = "restart_flag")
+    private Boolean restartFlag;
 
     @Column(name = "cluster_id")
     private Long clusterId;

diff --git a/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/po/StagePO.java b/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/po/StagePO.java
@@ -20,13 +20,11 @@
 
 import lombok.Data;
 import lombok.EqualsAndHashCode;
-import lombok.ToString;
 
 import jakarta.persistence.Column;
 import jakarta.persistence.Id;
 import jakarta.persistence.Table;
 import java.io.Serializable;
-import java.util.List;
 
 @Data
 @EqualsAndHashCode(callSuper = true)
@@ -60,7 +58,4 @@ public class StagePO extends BasePO implements Serializable {
 
     @Column(name = "job_id", nullable = false)
     private Long jobId;
-
-    @ToString.Exclude
-    private List<TaskPO> tasks;
 }
diff --git a/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/query/ServiceQuery.java b/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/query/ServiceQuery.java
@@ -33,7 +33,7 @@ public class ServiceQuery {
 
     private Long clusterId;
 
-    private Boolean needRestart;
+    private Boolean restartFlag;
 
     private Integer status;
 }
diff --git a/bigtop-manager-dao/src/main/resources/mapper/mysql/HostMapper.xml b/bigtop-manager-dao/src/main/resources/mapper/mysql/HostMapper.xml
@@ -24,11 +24,13 @@
 <mapper namespace="org.apache.bigtop.manager.dao.repository.HostDao">
 
     <sql id="baseColumns">
-        id, hostname, ipv4, ipv6, os, arch, available_processors, free_memory_size, total_memory_size, free_disk, total_disk, status, cluster_id
+        id,  hostname, agent_dir, ssh_user, ssh_port, auth_type, ssh_password, ssh_key_string, ssh_key_filename, ssh_key_password,
+        grpc_port, ipv4, ipv6, os, arch, available_processors, free_memory_size, total_memory_size,
+        free_disk, total_disk, desc, status, err_info, cluster_id
     </sql>
 
     <sql id="baseColumnsV2">
-        ${alias}.id,  ${alias}.hostname,  ${alias}.ssh_user, ${alias}.ssh_port, ${alias}.auth_type,
+        ${alias}.id,  ${alias}.hostname, ${alias}.agent_dir, ${alias}.ssh_user, ${alias}.ssh_port, ${alias}.auth_type,
         ${alias}.ssh_password, ${alias}.ssh_key_string, ${alias}.ssh_key_filename, ${alias}.ssh_key_password,
         ${alias}.grpc_port, ${alias}.ipv4, ${alias}.ipv6, ${alias}.os, ${alias}.arch,
         ${alias}.available_processors, ${alias}.free_memory_size, ${alias}.total_memory_size,

diff --git a/bigtop-manager-dao/src/main/resources/mapper/mysql/ServiceMapper.xml b/bigtop-manager-dao/src/main/resources/mapper/mysql/ServiceMapper.xml
@@ -24,11 +24,11 @@
 <mapper namespace="org.apache.bigtop.manager.dao.repository.ServiceDao">
 
     <sql id="baseColumns">
-        id, name, display_name, `desc`, user, version, stack, need_restart, cluster_id, status
+        id, name, display_name, `desc`, user, version, stack, restart_flag, cluster_id, status
     </sql>
 
     <sql id="baseColumnsV2">
-        ${alias}.id, ${alias}.name, ${alias}.display_name, ${alias}.`desc`, ${alias}.user, ${alias}.version, ${alias}.stack, ${alias}.need_restart, ${alias}.cluster_id, ${alias}.status
+        ${alias}.id, ${alias}.name, ${alias}.display_name, ${alias}.`desc`, ${alias}.user, ${alias}.version, ${alias}.stack, ${alias}.restart_flag, ${alias}.cluster_id, ${alias}.status
     </sql>
 
     <select id="findByQuery" resultType="org.apache.bigtop.manager.dao.po.ServicePO">
@@ -43,8 +43,8 @@
             <if test="query.clusterId != null">
                 and s.cluster_id = #{query.clusterId}
             </if>
-            <if test="query.needRestart != null">
-                and s.need_restart = #{query.needRestart}
+            <if test="query.restartFlag != null">
+                and s.restart_flag = #{query.restartFlag}
             </if>
             <if test="query.status != null">
                 and s.status = #{query.status}

diff --git a/bigtop-manager-dao/src/main/resources/mapper/postgresql/HostMapper.xml b/bigtop-manager-dao/src/main/resources/mapper/postgresql/HostMapper.xml
@@ -24,11 +24,13 @@
 <mapper namespace="org.apache.bigtop.manager.dao.repository.HostDao">
 
     <sql id="baseColumns">
-        id, hostname, ipv4, ipv6, os, arch, available_processors, free_memory_size, total_memory_size, free_disk, total_disk, state, cluster_id
+        id,  hostname, agent_dir, ssh_user, ssh_port, auth_type, ssh_password, ssh_key_string, ssh_key_filename, ssh_key_password,
+        grpc_port, ipv4, ipv6, os, arch, available_processors, free_memory_size, total_memory_size,
+        free_disk, total_disk, "desc", status, err_info, cluster_id
     </sql>
 
     <sql id="baseColumnsV2">
-        ${alias}.id,  ${alias}.hostname,  ${alias}.ssh_user, ${alias}.ssh_port, ${alias}.auth_type,
+        ${alias}.id,  ${alias}.hostname,  ${alias}.agent_dir, ${alias}.ssh_user, ${alias}.ssh_port, ${alias}.auth_type,
         ${alias}.ssh_password, ${alias}.ssh_key_string, ${alias}.ssh_key_filename, ${alias}.ssh_key_password,
         ${alias}.grpc_port, ${alias}.ipv4, ${alias}.ipv6, ${alias}.os, ${alias}.arch,
         ${alias}.available_processors, ${alias}.free_memory_size, ${alias}.total_memory_size,

diff --git a/bigtop-manager-dao/src/main/resources/mapper/postgresql/ServiceMapper.xml b/bigtop-manager-dao/src/main/resources/mapper/postgresql/ServiceMapper.xml
@@ -24,11 +24,11 @@
 <mapper namespace="org.apache.bigtop.manager.dao.repository.ServiceDao">
 
     <sql id="baseColumns">
-        id, name, display_name, "desc", "user", version, stack, need_restart, cluster_id, status
+        id, name, display_name, "desc", "user", version, stack, restart_flag, cluster_id, status
     </sql>
 
     <sql id="baseColumnsV2">
-        ${alias}.id, ${alias}.name, ${alias}.display_name, ${alias}."desc", ${alias}."user", ${alias}.version, ${alias}.stack, ${alias}.need_restart, ${alias}.cluster_id, ${alias}.status
+        ${alias}.id, ${alias}.name, ${alias}.display_name, ${alias}."desc", ${alias}."user", ${alias}.version, ${alias}.stack, ${alias}.restart_flag, ${alias}.cluster_id, ${alias}.status
     </sql>
 
     <select id="findByQuery" resultType="org.apache.bigtop.manager.dao.po.ServicePO">
@@ -43,8 +43,8 @@
             <if test="query.clusterId != null">
                 and s.cluster_id = #{query.clusterId}
             </if>
-            <if test="query.needRestart != null">
-                and s.need_restart = #{query.needRestart}
+            <if test="query.restartFlag != null">
+                and s.restart_flag = #{query.restartFlag}
             </if>
             <if test="query.status != null">
                 and s.status = #{query.status}