From 8df8fa7451839e84b00e41f33746730a031f5f0a Mon Sep 17 00:00:00 2001 From: lhpqaq <657407891@qq.com> Date: Fri, 20 Sep 2024 18:25:07 +0800 Subject: [PATCH] BIGTOP-4226: Support soft deletion in Chatbot --- .../store/PersistentChatMemoryStore.java | 28 +- .../{MessageSender.java => MessageType.java} | 12 +- .../ai/dashscope/DashScopeAssistant.java | 20 +- .../bigtop/manager/dao/enums/DBType.java | 3 +- .../dao/interceptor/AuditingInterceptor.java | 67 +-- .../bigtop/manager/dao/po/ChatMessagePO.java | 9 +- .../bigtop/manager/dao/po/ChatThreadPO.java | 9 +- .../apache/bigtop/manager/dao/po/HostPO.java | 2 +- .../apache/bigtop/manager/dao/po/JobPO.java | 4 +- .../manager/dao/po/PlatformAuthorizedPO.java | 5 +- .../apache/bigtop/manager/dao/po/RepoPO.java | 2 +- .../apache/bigtop/manager/dao/po/StackPO.java | 4 +- .../apache/bigtop/manager/dao/po/StagePO.java | 6 +- .../apache/bigtop/manager/dao/po/TaskPO.java | 2 +- .../manager/dao/repository/BaseDao.java | 27 +- .../dao/repository/ChatMessageDao.java | 2 - .../manager/dao/repository/ChatThreadDao.java | 4 +- .../manager/dao/repository/HostDao.java | 2 - .../dao/repository/PlatformAuthorizedDao.java | 4 + .../manager/dao/repository/RepoDao.java | 2 - .../manager/dao/sql/BaseSqlProvider.java | 53 +- .../bigtop/manager/dao/sql/SQLBuilder.java | 522 ++++++++++++++++-- .../mapper/mysql/ChatMessageMapper.xml | 6 +- .../mapper/mysql/ChatThreadMapper.xml | 11 +- .../resources/mapper/mysql/HostMapper.xml | 8 - .../mapper/mysql/PlatformAuthorizedMapper.xml | 12 +- .../resources/mapper/mysql/RepoMapper.xml | 8 - .../mapper/postgresql/ChatMessageMapper.xml | 32 ++ .../mapper/postgresql/ChatThreadMapper.xml | 65 +++ .../mapper/postgresql/ClusterMapper.xml | 81 +++ .../mapper/postgresql/ComponentMapper.xml | 109 ++++ .../mapper/postgresql/HostComponentMapper.xml | 222 ++++++++ .../mapper/postgresql/HostMapper.xml | 103 ++++ .../resources/mapper/postgresql/JobMapper.xml | 142 +++++ .../postgresql/PlatformAuthorizedMapper.xml | 49 ++ .../mapper/postgresql/PlatformMapper.xml | 38 ++ .../mapper/postgresql/RepoMapper.xml | 56 ++ .../mapper/postgresql/ServiceConfigMapper.xml | 118 ++++ .../mapper/postgresql/ServiceMapper.xml | 133 +++++ .../mapper/postgresql/StackMapper.xml | 46 ++ .../mapper/postgresql/StageMapper.xml | 39 ++ .../mapper/postgresql/TaskMapper.xml | 39 ++ .../mapper/postgresql/UserMapper.xml | 43 ++ .../server/command/job/AbstractJob.java | 10 +- .../server/command/job/ClusterCreateJob.java | 8 +- .../server/command/stage/AbstractStage.java | 11 +- .../server/command/task/AbstractTask.java | 10 +- .../command/task/ComponentInstallTask.java | 2 +- .../command/task/ComponentStartTask.java | 2 +- .../command/task/ComponentStopTask.java | 2 +- .../manager/server/config/MyBatisConfig.java | 6 +- .../server/controller/ChatbotController.java | 8 +- .../model/converter/ChatMessageConverter.java | 6 +- .../server/model/req/ChatbotMessageReq.java | 11 +- .../server/model/vo/ChatMessageVO.java | 6 +- .../scheduler/ComponentStatusScheduler.java | 4 +- .../server/scheduler/HostInfoScheduler.java | 2 +- .../service/impl/ChatbotServiceImpl.java | 24 +- .../service/impl/ClusterServiceImpl.java | 4 +- .../service/impl/ConfigServiceImpl.java | 2 +- .../server/service/impl/HostServiceImpl.java | 2 +- .../server/service/impl/JobServiceImpl.java | 6 +- .../server/service/impl/UserServiceImpl.java | 2 +- .../main/resources/ddl/MySQL-DDL-CREATE.sql | 5 +- .../resources/ddl/PostgreSQL-DDL-CREATE.sql | 394 +++++++++++++ .../controller/ChatbotControllerTest.java | 9 +- dev-support/README.md | 23 +- dev-support/docker/centos7/Dockerfile | 32 -- .../docker/centos7/build-containers.sh | 111 ---- dev-support/docker/centos7/build-image.sh | 30 - .../docker/centos7/clear-containers.sh | 38 -- .../docker/centos7/distribute-scripts.sh | 35 -- dev-support/docker/containers/build.sh | 226 ++++++++ .../Dockerfile => image/Dockerfile.template} | 6 +- dev-support/docker/image/build.sh | 78 +++ dev-support/docker/openeuler22/Dockerfile | 33 -- .../docker/openeuler22/build-containers.sh | 133 ----- dev-support/docker/openeuler22/build-image.sh | 30 - .../docker/openeuler22/clear-containers.sh | 40 -- .../docker/openeuler22/distribute-scripts.sh | 35 -- .../docker/openeuler22/restart-containers.sh | 43 -- dev-support/docker/rocky8/build-containers.sh | 132 ----- dev-support/docker/rocky8/build-image.sh | 30 - dev-support/docker/rocky8/clear-containers.sh | 40 -- .../docker/rocky8/distribute-scripts.sh | 35 -- .../docker/rocky8/restart-containers.sh | 43 -- 86 files changed, 2780 insertions(+), 1078 deletions(-) rename bigtop-manager-ai/bigtop-manager-ai-core/src/main/java/org/apache/bigtop/manager/ai/core/enums/{MessageSender.java => MessageType.java} (83%) create mode 100644 bigtop-manager-dao/src/main/resources/mapper/postgresql/ChatMessageMapper.xml create mode 100644 bigtop-manager-dao/src/main/resources/mapper/postgresql/ChatThreadMapper.xml create mode 100644 bigtop-manager-dao/src/main/resources/mapper/postgresql/ClusterMapper.xml create mode 100644 bigtop-manager-dao/src/main/resources/mapper/postgresql/ComponentMapper.xml create mode 100644 bigtop-manager-dao/src/main/resources/mapper/postgresql/HostComponentMapper.xml create mode 100644 bigtop-manager-dao/src/main/resources/mapper/postgresql/HostMapper.xml create mode 100644 bigtop-manager-dao/src/main/resources/mapper/postgresql/JobMapper.xml create mode 100644 bigtop-manager-dao/src/main/resources/mapper/postgresql/PlatformAuthorizedMapper.xml create mode 100644 bigtop-manager-dao/src/main/resources/mapper/postgresql/PlatformMapper.xml create mode 100644 bigtop-manager-dao/src/main/resources/mapper/postgresql/RepoMapper.xml create mode 100644 bigtop-manager-dao/src/main/resources/mapper/postgresql/ServiceConfigMapper.xml create mode 100644 bigtop-manager-dao/src/main/resources/mapper/postgresql/ServiceMapper.xml create mode 100644 bigtop-manager-dao/src/main/resources/mapper/postgresql/StackMapper.xml create mode 100644 bigtop-manager-dao/src/main/resources/mapper/postgresql/StageMapper.xml create mode 100644 bigtop-manager-dao/src/main/resources/mapper/postgresql/TaskMapper.xml create mode 100644 bigtop-manager-dao/src/main/resources/mapper/postgresql/UserMapper.xml rename dev-support/example/bigtop_manager/user.sql => bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/req/ChatbotMessageReq.java (79%) create mode 100644 bigtop-manager-server/src/main/resources/ddl/PostgreSQL-DDL-CREATE.sql delete mode 100644 dev-support/docker/centos7/Dockerfile delete mode 100755 dev-support/docker/centos7/build-containers.sh delete mode 100755 dev-support/docker/centos7/build-image.sh delete mode 100755 dev-support/docker/centos7/clear-containers.sh delete mode 100755 dev-support/docker/centos7/distribute-scripts.sh create mode 100644 dev-support/docker/containers/build.sh rename dev-support/docker/{rocky8/Dockerfile => image/Dockerfile.template} (97%) create mode 100644 dev-support/docker/image/build.sh delete mode 100644 dev-support/docker/openeuler22/Dockerfile delete mode 100755 dev-support/docker/openeuler22/build-containers.sh delete mode 100755 dev-support/docker/openeuler22/build-image.sh delete mode 100755 dev-support/docker/openeuler22/clear-containers.sh delete mode 100755 dev-support/docker/openeuler22/distribute-scripts.sh delete mode 100755 dev-support/docker/openeuler22/restart-containers.sh delete mode 100755 dev-support/docker/rocky8/build-containers.sh delete mode 100755 dev-support/docker/rocky8/build-image.sh delete mode 100755 dev-support/docker/rocky8/clear-containers.sh delete mode 100755 dev-support/docker/rocky8/distribute-scripts.sh delete mode 100755 dev-support/docker/rocky8/restart-containers.sh diff --git a/bigtop-manager-ai/bigtop-manager-ai-assistant/src/main/java/org/apache/bigtop/manager/ai/assistant/store/PersistentChatMemoryStore.java b/bigtop-manager-ai/bigtop-manager-ai-assistant/src/main/java/org/apache/bigtop/manager/ai/assistant/store/PersistentChatMemoryStore.java index ba4fd95ef..d5895ea5f 100644 --- a/bigtop-manager-ai/bigtop-manager-ai-assistant/src/main/java/org/apache/bigtop/manager/ai/assistant/store/PersistentChatMemoryStore.java +++ b/bigtop-manager-ai/bigtop-manager-ai-assistant/src/main/java/org/apache/bigtop/manager/ai/assistant/store/PersistentChatMemoryStore.java @@ -18,7 +18,7 @@ */ package org.apache.bigtop.manager.ai.assistant.store; -import org.apache.bigtop.manager.ai.core.enums.MessageSender; +import org.apache.bigtop.manager.ai.core.enums.MessageType; import org.apache.bigtop.manager.dao.po.ChatMessagePO; import org.apache.bigtop.manager.dao.po.ChatThreadPO; import org.apache.bigtop.manager.dao.repository.ChatMessageDao; @@ -27,7 +27,6 @@ import dev.langchain4j.data.message.AiMessage; import dev.langchain4j.data.message.ChatMessage; import dev.langchain4j.data.message.ChatMessageType; -import dev.langchain4j.data.message.SystemMessage; import dev.langchain4j.data.message.UserMessage; import dev.langchain4j.store.memory.chat.ChatMemoryStore; @@ -47,12 +46,10 @@ public PersistentChatMemoryStore(ChatThreadDao chatThreadDao, ChatMessageDao cha private ChatMessage convertToChatMessage(ChatMessagePO chatMessagePO) { String sender = chatMessagePO.getSender().toLowerCase(); - if (sender.equals(MessageSender.AI.getValue())) { + if (sender.equals(MessageType.AI.getValue())) { return new AiMessage(chatMessagePO.getMessage()); - } else if (sender.equals(MessageSender.USER.getValue())) { + } else if (sender.equals(MessageType.USER.getValue())) { return new UserMessage(chatMessagePO.getMessage()); - } else if (sender.equals(MessageSender.SYSTEM.getValue())) { - return new SystemMessage(chatMessagePO.getMessage()); } else { return null; } @@ -61,21 +58,17 @@ private ChatMessage convertToChatMessage(ChatMessagePO chatMessagePO) { private ChatMessagePO convertToChatMessagePO(ChatMessage chatMessage, Long chatThreadId) { ChatMessagePO chatMessagePO = new ChatMessagePO(); if (chatMessage.type().equals(ChatMessageType.AI)) { - chatMessagePO.setSender(MessageSender.AI.getValue()); + chatMessagePO.setSender(MessageType.AI.getValue()); AiMessage aiMessage = (AiMessage) chatMessage; chatMessagePO.setMessage(aiMessage.text()); } else if (chatMessage.type().equals(ChatMessageType.USER)) { - chatMessagePO.setSender(MessageSender.USER.getValue()); + chatMessagePO.setSender(MessageType.USER.getValue()); UserMessage userMessage = (UserMessage) chatMessage; chatMessagePO.setMessage(userMessage.singleText()); - } else if (chatMessage.type().equals(ChatMessageType.SYSTEM)) { - chatMessagePO.setSender(MessageSender.SYSTEM.getValue()); - SystemMessage systemMessage = (SystemMessage) chatMessage; - chatMessagePO.setMessage(systemMessage.text()); } else { - chatMessagePO.setSender(chatMessage.type().toString()); + return null; } - ChatThreadPO chatThreadPO = chatThreadDao.findById(chatThreadId); + ChatThreadPO chatThreadPO = chatThreadDao.findByThreadId(chatThreadId); chatMessagePO.setUserId(chatThreadPO.getUserId()); chatMessagePO.setThreadId(chatThreadId); return chatMessagePO; @@ -94,11 +87,16 @@ public List getMessages(Object threadId) { @Override public void updateMessages(Object threadId, List messages) { ChatMessagePO chatMessagePO = convertToChatMessagePO(messages.get(messages.size() - 1), (Long) threadId); + if (chatMessagePO == null) { + return; + } chatMessageDao.save(chatMessagePO); } @Override public void deleteMessages(Object threadId) { - chatMessageDao.deleteByThreadId((Long) threadId); + List chatMessagePOS = chatMessageDao.findAllByThreadId((Long) threadId); + chatMessagePOS.forEach(chatMessage -> chatMessage.setIsDeleted(true)); + chatMessageDao.partialUpdateByIds(chatMessagePOS); } } diff --git a/bigtop-manager-ai/bigtop-manager-ai-core/src/main/java/org/apache/bigtop/manager/ai/core/enums/MessageSender.java b/bigtop-manager-ai/bigtop-manager-ai-core/src/main/java/org/apache/bigtop/manager/ai/core/enums/MessageType.java similarity index 83% rename from bigtop-manager-ai/bigtop-manager-ai-core/src/main/java/org/apache/bigtop/manager/ai/core/enums/MessageSender.java rename to bigtop-manager-ai/bigtop-manager-ai-core/src/main/java/org/apache/bigtop/manager/ai/core/enums/MessageType.java index 1c93085a8..9f67b66d4 100644 --- a/bigtop-manager-ai/bigtop-manager-ai-core/src/main/java/org/apache/bigtop/manager/ai/core/enums/MessageSender.java +++ b/bigtop-manager-ai/bigtop-manager-ai-core/src/main/java/org/apache/bigtop/manager/ai/core/enums/MessageType.java @@ -26,14 +26,14 @@ import java.util.stream.Collectors; @Getter -public enum MessageSender { +public enum MessageType { USER("user"), AI("ai"), SYSTEM("system"); private final String value; - MessageSender(String value) { + MessageType(String value) { this.value = value; } @@ -41,13 +41,13 @@ public static List getSenders() { return Arrays.stream(values()).map(item -> item.value).collect(Collectors.toList()); } - public static MessageSender getMessageSender(String value) { + public static MessageType getMessageSender(String value) { if (Objects.isNull(value) || value.isEmpty()) { return null; } - for (MessageSender messageSender : MessageSender.values()) { - if (messageSender.value.equals(value)) { - return messageSender; + for (MessageType messageType : MessageType.values()) { + if (messageType.value.equals(value)) { + return messageType; } } diff --git a/bigtop-manager-ai/bigtop-manager-ai-dashscope/src/main/java/org/apache/bigtop/manager/ai/dashscope/DashScopeAssistant.java b/bigtop-manager-ai/bigtop-manager-ai-dashscope/src/main/java/org/apache/bigtop/manager/ai/dashscope/DashScopeAssistant.java index 441fd38d3..6ea7fd4fd 100644 --- a/bigtop-manager-ai/bigtop-manager-ai-dashscope/src/main/java/org/apache/bigtop/manager/ai/dashscope/DashScopeAssistant.java +++ b/bigtop-manager-ai/bigtop-manager-ai-dashscope/src/main/java/org/apache/bigtop/manager/ai/dashscope/DashScopeAssistant.java @@ -19,7 +19,7 @@ package org.apache.bigtop.manager.ai.dashscope; import org.apache.bigtop.manager.ai.core.AbstractAIAssistant; -import org.apache.bigtop.manager.ai.core.enums.MessageSender; +import org.apache.bigtop.manager.ai.core.enums.MessageType; import org.apache.bigtop.manager.ai.core.enums.PlatformType; import org.apache.bigtop.manager.ai.core.factory.AIAssistant; @@ -88,13 +88,13 @@ private String getValueFromAssistantStreamMessage(AssistantStreamMessage assista return streamMessage.toString(); } - private void saveMessage(String message, MessageSender sender) { + private void saveMessage(String message, MessageType sender) { ChatMessage chatMessage; - if (sender.equals(MessageSender.AI)) { + if (sender.equals(MessageType.AI)) { chatMessage = new AiMessage(message); - } else if (sender.equals(MessageSender.USER)) { + } else if (sender.equals(MessageType.USER)) { chatMessage = new UserMessage(message); - } else if (sender.equals(MessageSender.SYSTEM)) { + } else if (sender.equals(MessageType.SYSTEM)) { chatMessage = new SystemMessage(message); } else { return; @@ -131,7 +131,7 @@ public void setSystemPrompt(String systemPrompt) { } catch (NoApiKeyException | InputRequiredException | InvalidateParameter e) { throw new RuntimeException(e); } - saveMessage(systemPrompt, MessageSender.SYSTEM); + saveMessage(systemPrompt, MessageType.SYSTEM); } public static Builder builder() { @@ -140,7 +140,7 @@ public static Builder builder() { @Override public Flux streamAsk(String userMessage) { - saveMessage(userMessage, MessageSender.USER); + saveMessage(userMessage, MessageType.USER); TextMessageParam textMessageParam = TextMessageParam.builder() .apiKey(dashScopeThreadParam.getApiKey()) .role(Role.USER.getValue()) @@ -174,13 +174,13 @@ public Flux streamAsk(String userMessage) { return message; }) .doOnComplete(() -> { - saveMessage(finalMessage.toString(), MessageSender.AI); + saveMessage(finalMessage.toString(), MessageType.AI); }); } @Override public String ask(String userMessage) { - saveMessage(userMessage, MessageSender.USER); + saveMessage(userMessage, MessageType.USER); TextMessageParam textMessageParam = TextMessageParam.builder() .apiKey(dashScopeThreadParam.getApiKey()) .role(Role.USER.getValue()) @@ -244,7 +244,7 @@ public String ask(String userMessage) { ContentText contentText = (ContentText) content; finalMessage.append(contentText.getText().getValue()); } - saveMessage(finalMessage.toString(), MessageSender.AI); + saveMessage(finalMessage.toString(), MessageType.AI); return finalMessage.toString(); } diff --git a/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/enums/DBType.java b/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/enums/DBType.java index b2e8bac23..111fa663f 100644 --- a/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/enums/DBType.java +++ b/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/enums/DBType.java @@ -25,7 +25,8 @@ @Getter public enum DBType { - MYSQL("mysql", "MYSQL"), + MYSQL("mysql", "MySQL"), + POSTGRESQL("postgresql", "PostgreSQL"), DM("dm", "DaMeng"); DBType(String code, String desc) { diff --git a/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/interceptor/AuditingInterceptor.java b/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/interceptor/AuditingInterceptor.java index 4cc536ca7..d1782fc17 100644 --- a/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/interceptor/AuditingInterceptor.java +++ b/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/interceptor/AuditingInterceptor.java @@ -24,6 +24,7 @@ import org.apache.bigtop.manager.dao.annotations.CreateTime; import org.apache.bigtop.manager.dao.annotations.UpdateBy; import org.apache.bigtop.manager.dao.annotations.UpdateTime; +import org.apache.bigtop.manager.dao.po.BasePO; import org.apache.ibatis.binding.MapperMethod; import org.apache.ibatis.executor.Executor; @@ -68,22 +69,27 @@ public Object intercept(Invocation invocation) throws Throwable { Object parameter = invocation.getArgs()[1]; log.debug("sqlCommandType {}", sqlCommandType); - Collection objects; - if (parameter instanceof MapperMethod.ParamMap) { - MapperMethod.ParamMap paramMap = ((MapperMethod.ParamMap) parameter); - if (paramMap.get("param1") instanceof Collection) { - objects = ((Collection) paramMap.get("param1")); + if (SqlCommandType.INSERT == sqlCommandType || SqlCommandType.UPDATE == sqlCommandType) { + Collection objects; + if (parameter instanceof MapperMethod.ParamMap) { + MapperMethod.ParamMap paramMap = ((MapperMethod.ParamMap) parameter); + if (!paramMap.containsKey("param1") && paramMap.containsKey("arg0")) { + objects = ((Collection) paramMap.get("arg0")); + } else if (paramMap.get("param1") instanceof Collection) { + objects = ((Collection) paramMap.get("param1")); + } else { + objects = Collections.singletonList(paramMap.get("param1")); + } } else { - objects = Collections.singletonList(paramMap.get("param1")); + objects = Collections.singletonList(parameter); } - } else { - objects = Collections.singletonList(parameter); - } - for (Object o : objects) { - setAuditFields(o, sqlCommandType); + for (Object o : objects) { + if (o instanceof BasePO) { + setAuditFields(o, sqlCommandType); + } + } } - return invocation.proceed(); } @@ -92,26 +98,25 @@ private void setAuditFields(Object object, SqlCommandType sqlCommandType) throws Timestamp timestamp = new Timestamp(System.currentTimeMillis()); List fields = ClassUtils.getFields(object.getClass()); - if (SqlCommandType.INSERT == sqlCommandType || SqlCommandType.UPDATE == sqlCommandType) { - for (Field field : fields) { - boolean accessible = field.canAccess(object); - field.setAccessible(true); - if (field.isAnnotationPresent(CreateBy.class) - && SqlCommandType.INSERT == sqlCommandType - && userId != null) { - field.set(object, userId); - } - if (field.isAnnotationPresent(CreateTime.class) && SqlCommandType.INSERT == sqlCommandType) { - field.set(object, timestamp); - } - if (field.isAnnotationPresent(UpdateBy.class) && userId != null) { - field.set(object, userId); - } - if (field.isAnnotationPresent(UpdateTime.class)) { - field.set(object, timestamp); - } - field.setAccessible(accessible); + + for (Field field : fields) { + boolean accessible = field.canAccess(object); + field.setAccessible(true); + if (field.isAnnotationPresent(CreateBy.class) + && SqlCommandType.INSERT == sqlCommandType + && userId != null) { + field.set(object, userId); + } + if (field.isAnnotationPresent(CreateTime.class) && SqlCommandType.INSERT == sqlCommandType) { + field.set(object, timestamp); + } + if (field.isAnnotationPresent(UpdateBy.class) && userId != null) { + field.set(object, userId); + } + if (field.isAnnotationPresent(UpdateTime.class)) { + field.set(object, timestamp); } + field.setAccessible(accessible); } } } diff --git a/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/po/ChatMessagePO.java b/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/po/ChatMessagePO.java index 423681077..e84414bda 100644 --- a/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/po/ChatMessagePO.java +++ b/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/po/ChatMessagePO.java @@ -37,12 +37,15 @@ public class ChatMessagePO extends BasePO implements Serializable { @Column(name = "message", nullable = false, length = 255) private String message; - @Column(name = "sender") + @Column(name = "sender", nullable = false) private String sender; - @Column(name = "user_id") + @Column(name = "user_id", nullable = false) private Long userId; - @Column(name = "thread_id") + @Column(name = "thread_id", nullable = false) private Long threadId; + + @Column(name = "is_deleted") + private Boolean isDeleted; } diff --git a/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/po/ChatThreadPO.java b/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/po/ChatThreadPO.java index f17cfb33e..08a9e4b0b 100644 --- a/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/po/ChatThreadPO.java +++ b/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/po/ChatThreadPO.java @@ -38,12 +38,15 @@ public class ChatThreadPO extends BasePO implements Serializable { @Column(name = "model", nullable = false, length = 255) private String model; - @Column(name = "thread_info", columnDefinition = "json", nullable = false) + @Column(name = "thread_info", columnDefinition = "json") private Map threadInfo; - @Column(name = "user_id") + @Column(name = "user_id", nullable = false) private Long userId; - @Column(name = "platform_id") + @Column(name = "platform_id", nullable = false) private Long platformId; + + @Column(name = "is_deleted") + private Boolean isDeleted; } diff --git a/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/po/HostPO.java b/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/po/HostPO.java index f0b3054da..5b4131742 100644 --- a/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/po/HostPO.java +++ b/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/po/HostPO.java @@ -69,7 +69,7 @@ public class HostPO extends BasePO implements Serializable { @Column(name = "state") private String state; - @Column(name = "cluster_id") + @Column(name = "cluster_id", nullable = false) private Long clusterId; @Transient diff --git a/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/po/JobPO.java b/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/po/JobPO.java index 30adbb6a6..22938e28e 100644 --- a/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/po/JobPO.java +++ b/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/po/JobPO.java @@ -39,14 +39,14 @@ public class JobPO extends BasePO implements Serializable { @Column(name = "id") private Long id; - @Column(name = "state") + @Column(name = "state", nullable = false) private String state; @Column(name = "name") private String name; @Lob - @Column(name = "context") + @Column(name = "context", nullable = false) private String context; @Column(name = "cluster_id") diff --git a/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/po/PlatformAuthorizedPO.java b/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/po/PlatformAuthorizedPO.java index f27fcd437..ecd943bcd 100644 --- a/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/po/PlatformAuthorizedPO.java +++ b/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/po/PlatformAuthorizedPO.java @@ -39,6 +39,9 @@ public class PlatformAuthorizedPO extends BasePO implements Serializable { @Column(name = "credentials", columnDefinition = "json", nullable = false) private Map credentials; - @Column(name = "platform_id") + @Column(name = "platform_id", nullable = false) private Long platformId; + + @Column(name = "is_deleted") + private Boolean isDeleted; } diff --git a/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/po/RepoPO.java b/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/po/RepoPO.java index 00030d2c5..53e513ceb 100644 --- a/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/po/RepoPO.java +++ b/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/po/RepoPO.java @@ -53,6 +53,6 @@ public class RepoPO extends BasePO implements Serializable { @Column(name = "repo_type") private String repoType; - @Column(name = "cluster_id") + @Column(name = "cluster_id", nullable = false) private Long clusterId; } diff --git a/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/po/StackPO.java b/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/po/StackPO.java index 76db3ea9e..5b7add256 100644 --- a/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/po/StackPO.java +++ b/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/po/StackPO.java @@ -35,9 +35,9 @@ public class StackPO extends BasePO implements Serializable { @Column(name = "id") private Long id; - @Column(name = "stack_name") + @Column(name = "stack_name", nullable = false) private String stackName; - @Column(name = "stack_version") + @Column(name = "stack_version", nullable = false) private String stackVersion; } diff --git a/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/po/StagePO.java b/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/po/StagePO.java index 22a2d94c1..dd17195c0 100644 --- a/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/po/StagePO.java +++ b/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/po/StagePO.java @@ -37,10 +37,10 @@ public class StagePO extends BasePO implements Serializable { @Column(name = "id") private Long id; - @Column(name = "name") + @Column(name = "name", nullable = false) private String name; - @Column(name = "state") + @Column(name = "state", nullable = false) private String state; @Column(name = "order") @@ -55,7 +55,7 @@ public class StagePO extends BasePO implements Serializable { @Column(name = "context") private String context; - @Column(name = "job_id") + @Column(name = "job_id", nullable = false) private Long jobId; @Column(name = "cluster_id") diff --git a/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/po/TaskPO.java b/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/po/TaskPO.java index 54b507173..7fad46310 100644 --- a/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/po/TaskPO.java +++ b/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/po/TaskPO.java @@ -38,7 +38,7 @@ public class TaskPO extends BasePO implements Serializable { @Column(name = "name") private String name; - @Column(name = "context") + @Column(name = "context", nullable = false) private String context; @Column(name = "state") diff --git a/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/repository/BaseDao.java b/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/repository/BaseDao.java index 2c85e6f26..686ece108 100644 --- a/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/repository/BaseDao.java +++ b/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/repository/BaseDao.java @@ -42,11 +42,36 @@ public interface BaseDao { int save(Entity entity); /** - * Update the entity by primary key. + * Insert all entities. + */ + @Options(useGeneratedKeys = true, keyProperty = "id", keyColumn = "id") + @InsertProvider(type = BaseSqlProvider.class, method = "insertList") + int saveAll(List entities); + + /** + * Partially update the entity by primary key. + */ + @UpdateProvider(type = BaseSqlProvider.class, method = "partialUpdateById") + int partialUpdateById(Entity entity); + + /** + * Fully update the entity by primary key. */ @UpdateProvider(type = BaseSqlProvider.class, method = "updateById") int updateById(Entity entity); + /** + * Partially update the entities by primary key. + */ + @UpdateProvider(type = BaseSqlProvider.class, method = "partialUpdateByIds") + int partialUpdateByIds(List entities); + + /** + * Fully update the entities by primary key. + */ + @UpdateProvider(type = BaseSqlProvider.class, method = "updateByIds") + int updateByIds(List entities); + /** * Query the entity by primary key. */ diff --git a/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/repository/ChatMessageDao.java b/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/repository/ChatMessageDao.java index 59721afdf..a81e9c6c2 100644 --- a/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/repository/ChatMessageDao.java +++ b/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/repository/ChatMessageDao.java @@ -26,6 +26,4 @@ public interface ChatMessageDao extends BaseDao { List findAllByThreadId(@Param("threadId") Long threadId); - - void deleteByThreadId(@Param("threadId") Long threadId); } diff --git a/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/repository/ChatThreadDao.java b/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/repository/ChatThreadDao.java index 895cba8f5..6c1e70b68 100644 --- a/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/repository/ChatThreadDao.java +++ b/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/repository/ChatThreadDao.java @@ -27,12 +27,12 @@ public interface ChatThreadDao extends BaseDao { List findAllByUserId(@Param("userId") Long userId); - ChatThreadPO findById(Long id); - ChatThreadPO findByThreadId(@Param("id") Long id); List findAllByPlatformAuthorizedIdAndUserId( @Param("platformId") Long platformAuthorizedId, @Param("userId") Long userId); void saveWithThreadInfo(ChatThreadPO chatThreadPO); + + List findAllByPlatformId(@Param("platformId") Long platformId); } diff --git a/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/repository/HostDao.java b/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/repository/HostDao.java index 335a5e564..be95624b2 100644 --- a/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/repository/HostDao.java +++ b/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/repository/HostDao.java @@ -28,8 +28,6 @@ public interface HostDao extends BaseDao { - int saveAll(@Param("hosts") List hosts); - HostPO findByHostname(@Param("hostname") String hostname); List findAllByHostnameIn(@Param("hostnames") Collection hostnames); diff --git a/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/repository/PlatformAuthorizedDao.java b/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/repository/PlatformAuthorizedDao.java index fd801d76c..46da6eca0 100644 --- a/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/repository/PlatformAuthorizedDao.java +++ b/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/repository/PlatformAuthorizedDao.java @@ -22,8 +22,12 @@ import org.apache.ibatis.annotations.Param; +import java.util.List; + public interface PlatformAuthorizedDao extends BaseDao { PlatformAuthorizedPO findByPlatformId(@Param("id") Long platformId); void saveWithCredentials(PlatformAuthorizedPO platformAuthorizedPO); + + List findAllPlatform(); } diff --git a/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/repository/RepoDao.java b/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/repository/RepoDao.java index c951dfb35..3c8d30215 100644 --- a/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/repository/RepoDao.java +++ b/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/repository/RepoDao.java @@ -30,7 +30,5 @@ public interface RepoDao extends BaseDao { Optional findByRepoName(@Param("repoName") String clusterName); - int saveAll(@Param("clusters") List repos); - List findAllByClusterId(@Param("clusterId") Long clusterId); } diff --git a/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/sql/BaseSqlProvider.java b/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/sql/BaseSqlProvider.java index 09eaf94db..d2f92968c 100644 --- a/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/sql/BaseSqlProvider.java +++ b/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/sql/BaseSqlProvider.java @@ -31,6 +31,7 @@ import java.io.Serializable; import java.lang.reflect.Type; import java.util.Collection; +import java.util.List; @Slf4j public class BaseSqlProvider { @@ -46,6 +47,43 @@ public String insert(Entity entity, ProviderContext context) { return SQLBuilder.insert(tableMetaData, entity, databaseId); } + public String insertList(List entities, ProviderContext context) { + Assert.notNull(entities, "entities must not be null"); + Assert.notEmpty(entities, "entities list must not be empty"); + + String databaseId = context.getDatabaseId(); + + Class entityClass = entities.get(0).getClass(); + + TableMetaData tableMetaData = TableMetaData.forClass(entityClass); + + return SQLBuilder.insertList(tableMetaData, entities, databaseId); + } + + public String partialUpdateById(Entity entity, ProviderContext context) { + Assert.notNull(entity, "entity must not null"); + + String databaseId = context.getDatabaseId(); + + Class entityClass = entity.getClass(); + TableMetaData tableMetaData = TableMetaData.forClass(entityClass); + + return SQLBuilder.update(tableMetaData, entity, databaseId, true); + } + + public String partialUpdateByIds(List entities, ProviderContext context) { + Assert.notNull(entities, "entities must not be null"); + Assert.notEmpty(entities, "entities list must not be empty"); + + String databaseId = context.getDatabaseId(); + + Class entityClass = entities.get(0).getClass(); + + TableMetaData tableMetaData = TableMetaData.forClass(entityClass); + + return SQLBuilder.updateList(tableMetaData, entities, databaseId, true); + } + public String updateById(Entity entity, ProviderContext context) { Assert.notNull(entity, "entity must not null"); @@ -54,7 +92,20 @@ public String updateById(Entity entity, ProviderContext context) { Class entityClass = entity.getClass(); TableMetaData tableMetaData = TableMetaData.forClass(entityClass); - return SQLBuilder.update(tableMetaData, entity, databaseId); + return SQLBuilder.update(tableMetaData, entity, databaseId, false); + } + + public String updateByIds(List entities, ProviderContext context) { + Assert.notNull(entities, "entities must not be null"); + Assert.notEmpty(entities, "entities list must not be empty"); + + String databaseId = context.getDatabaseId(); + + Class entityClass = entities.get(0).getClass(); + + TableMetaData tableMetaData = TableMetaData.forClass(entityClass); + + return SQLBuilder.updateList(tableMetaData, entities, databaseId, false); } public String selectById(Serializable id, ProviderContext context) { diff --git a/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/sql/SQLBuilder.java b/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/sql/SQLBuilder.java index ce729c77c..4321f223b 100644 --- a/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/sql/SQLBuilder.java +++ b/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/sql/SQLBuilder.java @@ -20,7 +20,11 @@ package org.apache.bigtop.manager.dao.sql; import org.apache.bigtop.manager.common.utils.ClassUtils; +import org.apache.bigtop.manager.dao.annotations.CreateBy; +import org.apache.bigtop.manager.dao.annotations.CreateTime; import org.apache.bigtop.manager.dao.annotations.QueryCondition; +import org.apache.bigtop.manager.dao.annotations.UpdateBy; +import org.apache.bigtop.manager.dao.annotations.UpdateTime; import org.apache.bigtop.manager.dao.enums.DBType; import org.apache.ibatis.jdbc.SQL; @@ -31,11 +35,13 @@ import lombok.extern.slf4j.Slf4j; +import jakarta.persistence.Column; import java.beans.PropertyDescriptor; import java.io.Serializable; import java.lang.reflect.Field; -import java.text.MessageFormat; +import java.util.ArrayList; import java.util.Collection; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Objects; @@ -54,7 +60,7 @@ public static String insert(TableMetaData tableMetaData, Entity entity, SQL sql = new SQL(); switch (DBType.toType(databaseId)) { case MYSQL: { - sql.INSERT_INTO(tableMetaData.getTableName()); + sql.INSERT_INTO(keywordsFormat(tableMetaData.getTableName(), DBType.MYSQL)); for (Map.Entry entry : fieldColumnMap.entrySet()) { // Ignore primary key if (Objects.equals(entry.getKey(), tableMetaData.getPkProperty())) { @@ -66,11 +72,30 @@ public static String insert(TableMetaData tableMetaData, Entity entity, } Object value = ReflectionUtils.invokeMethod(ps.getReadMethod(), entity); if (!ObjectUtils.isEmpty(value)) { - sql.VALUES("`" + entry.getValue() + "`", getTokenParam(entry.getKey())); + sql.VALUES(keywordsFormat(entry.getValue(), DBType.MYSQL), getTokenParam(entry.getKey())); } } break; } + case POSTGRESQL: { + sql.INSERT_INTO(keywordsFormat(tableMetaData.getTableName(), DBType.POSTGRESQL)); + for (Map.Entry entry : fieldColumnMap.entrySet()) { + // Ignore primary key + if (Objects.equals(entry.getKey(), tableMetaData.getPkProperty())) { + continue; + } + PropertyDescriptor ps = BeanUtils.getPropertyDescriptor(entityClass, entry.getKey()); + if (ps == null || ps.getReadMethod() == null) { + continue; + } + Object value = ReflectionUtils.invokeMethod(ps.getReadMethod(), entity); + if (!ObjectUtils.isEmpty(value)) { + sql.VALUES(keywordsFormat(entry.getValue(), DBType.POSTGRESQL), getTokenParam(entry.getKey())); + } + } + break; + } + default: { log.error("Unsupported data source"); } @@ -79,14 +104,106 @@ public static String insert(TableMetaData tableMetaData, Entity entity, return sql.toString(); } - public static String update(TableMetaData tableMetaData, Entity entity, String databaseId) { + public static String insertList(TableMetaData tableMetaData, List entities, String databaseId) { + if (entities == null || entities.isEmpty()) { + throw new IllegalArgumentException("Entities list must not be null or empty"); + } + + Class entityClass = entities.get(0).getClass(); + Map fieldColumnMap = tableMetaData.getFieldColumnMap(); + + SQL sql = new SQL(); + switch (DBType.toType(databaseId)) { + case MYSQL: { + sql.INSERT_INTO(keywordsFormat(tableMetaData.getTableName(), DBType.MYSQL)); + + boolean firstRow = true; + int idx = 0; + for (Entity entity : entities) { + List values = new ArrayList<>(); + for (Map.Entry entry : fieldColumnMap.entrySet()) { + // Ignore primary key + if (Objects.equals(entry.getKey(), tableMetaData.getPkProperty())) { + continue; + } + PropertyDescriptor ps = BeanUtils.getPropertyDescriptor(entityClass, entry.getKey()); + if (ps == null || ps.getReadMethod() == null) { + continue; + } + Object value = ReflectionUtils.invokeMethod(ps.getReadMethod(), entity); + if (!ObjectUtils.isEmpty(value)) { + if (firstRow) { + sql.VALUES( + keywordsFormat(entry.getValue(), DBType.MYSQL), + getTokenParam("arg0[" + idx + "]." + entry.getKey())); + } + values.add(getTokenParam("arg0[" + idx + "]." + entry.getKey())); + } + } + if (firstRow) { + firstRow = false; + } else { + sql.ADD_ROW(); + sql.INTO_VALUES(values.toArray(new String[0])); + } + idx++; + } + break; + } + case POSTGRESQL: { + sql.INSERT_INTO(keywordsFormat(tableMetaData.getTableName(), DBType.POSTGRESQL)); + + boolean firstRow = true; + List columns = new ArrayList<>(); + int idx = 0; + for (Entity entity : entities) { + List values = new ArrayList<>(); + for (Map.Entry entry : fieldColumnMap.entrySet()) { + // Ignore primary key + if (Objects.equals(entry.getKey(), tableMetaData.getPkProperty())) { + continue; + } + PropertyDescriptor ps = BeanUtils.getPropertyDescriptor(entityClass, entry.getKey()); + if (ps == null || ps.getReadMethod() == null) { + continue; + } + Object value = ReflectionUtils.invokeMethod(ps.getReadMethod(), entity); + if (!ObjectUtils.isEmpty(value)) { + if (firstRow) { + sql.VALUES( + keywordsFormat(entry.getValue(), DBType.POSTGRESQL), + getTokenParam("arg0[" + idx + "]." + entry.getKey())); + } + values.add(getTokenParam("arg0[" + idx + "]." + entry.getKey())); + } + } + if (firstRow) { + firstRow = false; + } else { + sql.ADD_ROW(); + sql.INTO_VALUES(values.toArray(new String[0])); + } + idx++; + } + } + + default: { + log.error("Unsupported data source"); + } + } + + return sql.toString(); + } + + public static String update( + TableMetaData tableMetaData, Entity entity, String databaseId, boolean partial) { Class entityClass = entity.getClass(); Map fieldColumnMap = tableMetaData.getFieldColumnMap(); SQL sql = new SQL(); switch (DBType.toType(databaseId)) { case MYSQL: { - sql.UPDATE(tableMetaData.getTableName()); + sql.UPDATE(keywordsFormat(tableMetaData.getTableName(), DBType.MYSQL)); for (Map.Entry entry : fieldColumnMap.entrySet()) { // Ignore primary key if (Objects.equals(entry.getKey(), tableMetaData.getPkProperty())) { @@ -97,12 +214,49 @@ public static String update(TableMetaData tableMetaData, Entity entity, continue; } Object value = ReflectionUtils.invokeMethod(ps.getReadMethod(), entity); - if (!ObjectUtils.isEmpty(value)) { - sql.SET("`" + getEquals(entry.getValue() + "`", entry.getKey())); + if (ObjectUtils.isEmpty(value) && partial) { + continue; } + Field field = ReflectionUtils.findField(entityClass, entry.getKey()); + if (field != null) { + Column column = field.getAnnotation(Column.class); + if (column != null && !column.nullable() && value == null) { + continue; + } + } + sql.SET(getEquals(keywordsFormat(entry.getValue(), DBType.MYSQL), entry.getKey())); } - sql.WHERE(getEquals(tableMetaData.getPkColumn(), tableMetaData.getPkProperty())); + sql.WHERE(getEquals( + keywordsFormat(tableMetaData.getPkColumn(), DBType.MYSQL), tableMetaData.getPkProperty())); + break; + } + case POSTGRESQL: { + sql.UPDATE(keywordsFormat(tableMetaData.getTableName(), DBType.POSTGRESQL)); + for (Map.Entry entry : fieldColumnMap.entrySet()) { + // Ignore primary key + if (Objects.equals(entry.getKey(), tableMetaData.getPkProperty())) { + continue; + } + PropertyDescriptor ps = BeanUtils.getPropertyDescriptor(entityClass, entry.getKey()); + if (ps == null || ps.getReadMethod() == null) { + continue; + } + Object value = ReflectionUtils.invokeMethod(ps.getReadMethod(), entity); + if (ObjectUtils.isEmpty(value) && partial) { + continue; + } + Field field = ReflectionUtils.findField(entityClass, entry.getKey()); + if (field != null) { + Column column = field.getAnnotation(Column.class); + if (column != null && !column.nullable() && value == null) { + continue; + } + } + sql.SET(getEquals(keywordsFormat(entry.getValue(), DBType.POSTGRESQL), entry.getKey())); + } + sql.WHERE(getEquals( + keywordsFormat(tableMetaData.getPkColumn(), DBType.POSTGRESQL), tableMetaData.getPkProperty())); break; } default: { @@ -113,6 +267,189 @@ public static String update(TableMetaData tableMetaData, Entity entity, return sql.toString(); } + public static String updateList( + TableMetaData tableMetaData, List entities, String databaseId, boolean partial) { + if (entities == null || entities.isEmpty()) { + throw new IllegalArgumentException("Entities list must not be null or empty"); + } + + Class entityClass = entities.get(0).getClass(); + Map fieldColumnMap = tableMetaData.getFieldColumnMap(); + + StringBuilder sqlBuilder = new StringBuilder(); + switch (DBType.toType(databaseId)) { + case MYSQL: { + sqlBuilder + .append("UPDATE ") + .append(keywordsFormat(tableMetaData.getTableName(), DBType.MYSQL)) + .append(" SET "); + Map setClauses = new LinkedHashMap<>(); + String primaryKey = keywordsFormat("id", DBType.MYSQL); + for (Map.Entry entry : fieldColumnMap.entrySet()) { + // Ignore primary key + if (Objects.equals(entry.getKey(), tableMetaData.getPkProperty())) { + primaryKey = keywordsFormat(entry.getValue(), DBType.MYSQL); + continue; + } + + StringBuilder caseClause = new StringBuilder(); + caseClause + .append(keywordsFormat(entry.getValue(), DBType.MYSQL)) + .append(" = CASE "); + for (Entity entity : entities) { + PropertyDescriptor ps = BeanUtils.getPropertyDescriptor(entityClass, entry.getKey()); + if (ps == null || ps.getReadMethod() == null) { + continue; + } + Field field = ReflectionUtils.findField(entityClass, entry.getKey()); + if (field == null || checkBaseField(field)) { + continue; + } + Object value = ReflectionUtils.invokeMethod(ps.getReadMethod(), entity); + PropertyDescriptor pkPs = + BeanUtils.getPropertyDescriptor(entityClass, tableMetaData.getPkProperty()); + if (pkPs == null || pkPs.getReadMethod() == null) { + continue; + } + Object pkValue = ReflectionUtils.invokeMethod(pkPs.getReadMethod(), entity); + + if (!ObjectUtils.isEmpty(value)) { + caseClause + .append("WHEN ") + .append(primaryKey) + .append(" = '") + .append(pkValue) + .append("' THEN '") + .append(escapeSingleQuote(value.toString())) + .append("' "); + } else if (!partial) { + Column column = field.getAnnotation(Column.class); + if (column != null && !column.nullable() && value == null) { + continue; + } + caseClause + .append("WHEN ") + .append(primaryKey) + .append(" = '") + .append(pkValue) + .append("' THEN NULL "); + } + } + caseClause + .append("ELSE ") + .append(keywordsFormat(entry.getValue(), DBType.MYSQL)) + .append(" "); + caseClause.append("END"); + setClauses.put(entry.getValue(), caseClause); + } + sqlBuilder.append(String.join(", ", setClauses.values())); + + sqlBuilder.append(" WHERE ").append(primaryKey).append(" IN ("); + String pkValues = entities.stream() + .map(entity -> { + PropertyDescriptor pkPs = + BeanUtils.getPropertyDescriptor(entityClass, tableMetaData.getPkProperty()); + Object pkValue = ReflectionUtils.invokeMethod(pkPs.getReadMethod(), entity); + return "'" + pkValue.toString() + "'"; + }) + .collect(Collectors.joining(", ")); + + sqlBuilder.append(pkValues).append(")"); + break; + } + case POSTGRESQL: { + sqlBuilder + .append("UPDATE ") + .append("\"") + .append(tableMetaData.getTableName()) + .append("\"") + .append(" SET "); + Map setClauses = new LinkedHashMap<>(); + String primaryKey = keywordsFormat("id", DBType.POSTGRESQL); + for (Map.Entry entry : fieldColumnMap.entrySet()) { + // Ignore primary key + if (Objects.equals(entry.getKey(), tableMetaData.getPkProperty())) { + primaryKey = keywordsFormat(entry.getValue(), DBType.POSTGRESQL); + continue; + } + Field field = ReflectionUtils.findField(entityClass, entry.getKey()); + if (field == null || checkBaseField(field)) { + continue; + } + StringBuilder caseClause = new StringBuilder(); + caseClause + .append(keywordsFormat(entry.getValue(), DBType.POSTGRESQL)) + .append(" = CASE "); + + for (Entity entity : entities) { + PropertyDescriptor ps = BeanUtils.getPropertyDescriptor(entityClass, entry.getKey()); + if (ps == null || ps.getReadMethod() == null) { + continue; + } + Object value = ReflectionUtils.invokeMethod(ps.getReadMethod(), entity); + PropertyDescriptor pkPs = + BeanUtils.getPropertyDescriptor(entityClass, tableMetaData.getPkProperty()); + if (pkPs == null || pkPs.getReadMethod() == null) { + continue; + } + Object pkValue = ReflectionUtils.invokeMethod(pkPs.getReadMethod(), entity); + + if (!ObjectUtils.isEmpty(value)) { + caseClause + .append("WHEN ") + .append(primaryKey) + .append(" = '") + .append(pkValue) + .append("' THEN '") + .append(escapeSingleQuote(value.toString())) + .append("' "); + } else if (!partial) { + Column column = field.getAnnotation(Column.class); + if (column != null && !column.nullable() && value == null) { + continue; + } + caseClause + .append("WHEN ") + .append(primaryKey) + .append(" = '") + .append(pkValue) + .append("' THEN NULL "); + } + } + if (caseClause.toString().endsWith("CASE ")) { + caseClause.append("WHEN TRUE THEN "); + } else { + caseClause.append("ELSE "); + } + caseClause + .append(keywordsFormat(entry.getValue(), DBType.POSTGRESQL)) + .append(" "); + caseClause.append("END"); + setClauses.put(entry.getValue(), caseClause); + } + sqlBuilder.append(String.join(", ", setClauses.values())); + + sqlBuilder.append(" WHERE ").append(primaryKey).append(" IN ("); + String pkValues = entities.stream() + .map(entity -> { + PropertyDescriptor pkPs = + BeanUtils.getPropertyDescriptor(entityClass, tableMetaData.getPkProperty()); + Object pkValue = ReflectionUtils.invokeMethod(pkPs.getReadMethod(), entity); + return "'" + pkValue.toString() + "'"; + }) + .collect(Collectors.joining(", ")); + + sqlBuilder.append(pkValues).append(")"); + break; + } + default: { + log.error("Unsupported data source"); + } + } + + return sqlBuilder.toString(); + } + public static String selectById(TableMetaData tableMetaData, String databaseId, Serializable id) { SQL sql = new SQL(); @@ -120,7 +457,17 @@ public static String selectById(TableMetaData tableMetaData, String databaseId, case MYSQL: { sql.SELECT(tableMetaData.getBaseColumns()); sql.FROM(tableMetaData.getTableName()); - sql.WHERE(tableMetaData.getPkColumn() + " = '" + id + "'"); + sql.WHERE(getEquals(tableMetaData.getPkColumn(), tableMetaData.getPkProperty())); + break; + } + case POSTGRESQL: { + String baseColumns = tableMetaData.getBaseColumns(); + if (baseColumns.toLowerCase().contains("user.")) { + baseColumns = baseColumns.replace("user.", "\"user\"."); + } + sql.SELECT(baseColumns); + sql.FROM(keywordsFormat(tableMetaData.getTableName(), DBType.POSTGRESQL)); + sql.WHERE(tableMetaData.getPkColumn() + " = " + id); break; } default: { @@ -137,10 +484,31 @@ public static String selectByIds( SQL sql = new SQL(); switch (DBType.toType(databaseId)) { case MYSQL: { - String idsStr = ids.stream().map(String::valueOf).collect(Collectors.joining("', '")); sql.SELECT(tableMetaData.getBaseColumns()); sql.FROM(tableMetaData.getTableName()); - sql.WHERE(tableMetaData.getPkColumn() + " in ('" + idsStr + "')"); + if (ids == null || ids.isEmpty()) { + sql.WHERE("1 = 0"); + break; + } + + StringBuilder idStr = new StringBuilder(); + for (int i = 0; i < ids.size(); i++) { + idStr.append(getTokenParam("arg0[" + i + "]")).append(","); + } + idStr.deleteCharAt(idStr.lastIndexOf(",")); + + sql.WHERE(tableMetaData.getPkColumn() + " IN ( " + idStr + " )"); + break; + } + case POSTGRESQL: { + String idsStr = ids.stream().map(String::valueOf).collect(Collectors.joining(",")); + String baseColumns = tableMetaData.getBaseColumns(); + if (baseColumns.toLowerCase().contains("user.")) { + baseColumns = baseColumns.replace("user.", "\"user\"."); + } + sql.SELECT(baseColumns); + sql.FROM(keywordsFormat(tableMetaData.getTableName(), DBType.POSTGRESQL)); + sql.WHERE(tableMetaData.getPkColumn() + " in (" + idsStr + ")"); break; } default: { @@ -155,6 +523,14 @@ public static String selectAll(TableMetaData tableMetaData, String databaseId) { SQL sql = new SQL(); switch (DBType.toType(databaseId)) { + case POSTGRESQL: + String baseColumns = tableMetaData.getBaseColumns(); + if (baseColumns.toLowerCase().contains("user.")) { + baseColumns = baseColumns.replace("user.", "\"user\"."); + } + sql.SELECT(baseColumns); + sql.FROM(keywordsFormat(tableMetaData.getTableName(), DBType.POSTGRESQL)); + break; case MYSQL: { sql.SELECT(tableMetaData.getBaseColumns()); sql.FROM(tableMetaData.getTableName()); @@ -173,7 +549,12 @@ public static String deleteById(TableMetaData tableMetaData, String databaseId, switch (DBType.toType(databaseId)) { case MYSQL: { sql.DELETE_FROM(tableMetaData.getTableName()); - sql.WHERE(tableMetaData.getPkColumn() + " = '" + id + "'"); + sql.WHERE(getEquals(tableMetaData.getPkColumn(), tableMetaData.getPkProperty())); + break; + } + case POSTGRESQL: { + sql.FROM(keywordsFormat(tableMetaData.getTableName(), DBType.POSTGRESQL)); + sql.WHERE(tableMetaData.getPkColumn() + " = " + id); break; } default: { @@ -189,9 +570,24 @@ public static String deleteByIds( SQL sql = new SQL(); switch (DBType.toType(databaseId)) { case MYSQL: { - String idsStr = ids.stream().map(String::valueOf).collect(Collectors.joining("', '")); + if (ids == null || ids.isEmpty()) { + break; + } sql.DELETE_FROM(tableMetaData.getTableName()); - sql.WHERE(tableMetaData.getPkColumn() + " in ('" + idsStr + "')"); + + StringBuilder idStr = new StringBuilder(); + for (int i = 0; i < ids.size(); i++) { + idStr.append(getTokenParam("arg0[" + i + "]")).append(","); + } + idStr.deleteCharAt(idStr.lastIndexOf(",")); + + sql.WHERE(tableMetaData.getPkColumn() + " IN ( " + idStr + " )"); + break; + } + case POSTGRESQL: { + String idsStr = ids.stream().map(String::valueOf).collect(Collectors.joining(", ")); + sql.DELETE_FROM(keywordsFormat(tableMetaData.getTableName(), DBType.POSTGRESQL)); + sql.WHERE(tableMetaData.getPkColumn() + " in (" + idsStr + ")"); break; } default: { @@ -208,8 +604,10 @@ public static String findByCondition( log.info("databaseId: {}", databaseId); SQL sql = new SQL(); switch (DBType.toType(databaseId)) { + case POSTGRESQL: + tableName = "\"" + tableName + "\""; case MYSQL: { - sql = mysqlCondition(condition, tableName); + sql = mysqlCondition(condition, tableMetaData); break; } default: { @@ -220,6 +618,14 @@ public static String findByCondition( return sql.toString(); } + private static String keywordsFormat(String keyword, DBType dbType) { + return switch (dbType) { + case MYSQL -> "`" + keyword + "`"; + case POSTGRESQL -> "\"" + keyword + "\""; + default -> keyword; + }; + } + private static String getEquals(String column, String property) { return column + " = " + getTokenParam(property); } @@ -228,7 +634,22 @@ private static String getTokenParam(String property) { return "#{" + property + "}"; } - private static SQL mysqlCondition(Condition condition, String tableName) throws IllegalAccessException { + private static String escapeSingleQuote(String input) { + if (input != null) { + return input.replace("'", "''"); + } + return null; + } + + private static boolean checkBaseField(Field field) { + return field.isAnnotationPresent(CreateBy.class) + || field.isAnnotationPresent(CreateTime.class) + || field.isAnnotationPresent(UpdateBy.class) + || field.isAnnotationPresent(UpdateTime.class); + } + + private static SQL mysqlCondition(Condition condition, TableMetaData tableMetaData) + throws IllegalAccessException { Class loadClass; try { @@ -241,7 +662,7 @@ private static SQL mysqlCondition(Condition condition, String tableN /* Prepare SQL */ SQL sql = new SQL(); sql.SELECT("*"); - sql.FROM(tableName); + sql.FROM(tableMetaData.getTableName()); for (Field field : fieldList) { field.setAccessible(true); String fieldName = field.getName(); @@ -249,75 +670,72 @@ private static SQL mysqlCondition(Condition condition, String tableN if (field.isAnnotationPresent(QueryCondition.class) && Objects.nonNull(field.get(condition))) { QueryCondition annotation = field.getAnnotation(QueryCondition.class); - String queryKey = fieldName; + String property = fieldName; if (!annotation.queryKey().isEmpty()) { - queryKey = annotation.queryKey(); + property = annotation.queryKey(); } - log.info( - "[queryKey] {}, [queryType] {}, [queryValue] {}", - queryKey, - annotation.queryType().toString(), - field.get(condition)); - Object value = field.get(condition); - if (value != null) { + Map fieldColumnMap = tableMetaData.getFieldColumnMap(); + + if (value != null && fieldColumnMap.containsKey(property)) { + String columnName = fieldColumnMap.get(property); + + log.info( + "[queryKey] {}, [queryType] {}, [queryValue] {}", + property, + annotation.queryType().toString(), + field.get(condition)); switch (annotation.queryType()) { case EQ: - sql.WHERE(MessageFormat.format("{0} = ''{1}''", queryKey, value)); + sql.WHERE(getEquals(columnName, fieldName)); break; case NOT_EQ: - sql.WHERE(MessageFormat.format("{0} != ''{1}''", queryKey, value)); + sql.WHERE(columnName + " != " + getTokenParam(fieldName)); break; case IN: - sql.WHERE(MessageFormat.format( - "{0} IN (''{1}'')", - queryKey, - String.join("','", value.toString().split(annotation.multipleDelimiter())))); + sql.WHERE(columnName + " IN ( REPLACE( " + getTokenParam(fieldName) + ", '" + + annotation.multipleDelimiter() + "', ',') )"); break; case NOT_IN: - sql.WHERE(MessageFormat.format( - "{0} NOT IN (''{1}'')", - queryKey, - String.join("','", value.toString().split(annotation.multipleDelimiter())))); + sql.WHERE(columnName + " NOT IN ( REPLACE( " + getTokenParam(fieldName) + ", '" + + annotation.multipleDelimiter() + "', ',') )"); break; case GT: - sql.WHERE(MessageFormat.format("{0} > ''{1}''", queryKey, value)); + sql.WHERE(columnName + " > " + getTokenParam(fieldName)); break; case GTE: - sql.WHERE(MessageFormat.format("{0} >= ''{1}''", queryKey, value)); + sql.WHERE(columnName + " >= " + getTokenParam(fieldName)); break; case LT: - sql.WHERE(MessageFormat.format("{0} < ''{1}''", queryKey, value)); + sql.WHERE(columnName + " < " + getTokenParam(fieldName)); break; case LTE: - sql.WHERE(MessageFormat.format("{0} <= ''{1}''", queryKey, value)); + sql.WHERE(columnName + " <= " + getTokenParam(fieldName)); break; case BETWEEN: - String[] valueArr = field.get(condition).toString().split(annotation.pairDelimiter()); - if (valueArr.length == 2) { - sql.WHERE(MessageFormat.format( - "{0} BETWEEN ''{1}'' AND ''{2}''", queryKey, valueArr[0], valueArr[1])); - } + sql.WHERE(columnName + " BETWEEN SUBSTRING_INDEX( " + getTokenParam(fieldName) + ", '" + + annotation.pairDelimiter() + "', 1) AND SUBSTRING_INDEX( " + + getTokenParam(fieldName) + ", '" + + annotation.pairDelimiter() + "', 2)"); break; case PREFIX_LIKE: - sql.WHERE(MessageFormat.format("{0} LIKE CONCAT(''{1}'', ''%'')", queryKey, value)); + sql.WHERE(columnName + " LIKE CONCAT( " + getTokenParam(fieldName) + ", '%')"); break; case SUFFIX_LIKE: - sql.WHERE(MessageFormat.format("{0} LIKE CONCAT(''%'', ''{1}'')", queryKey, value)); + sql.WHERE(columnName + " LIKE CONCAT('%', " + getTokenParam(fieldName) + ")"); break; case LIKE: - sql.WHERE(MessageFormat.format("{0} LIKE CONCAT(''%'', ''{1}'', ''%'')", queryKey, value)); + sql.WHERE(columnName + " LIKE CONCAT('%', " + getTokenParam(fieldName) + ", '%')"); break; case NOT_LIKE: - sql.WHERE(MessageFormat.format( - "{0} NOT LIKE CONCAT(''%'', ''{1}'', ''%'')", queryKey, value)); + sql.WHERE(columnName + " NOT LIKE CONCAT('%', " + getTokenParam(fieldName) + ", '%')"); break; case NULL: - sql.WHERE(queryKey + " IS NULL"); + sql.WHERE(columnName + " IS NULL"); break; case NOT_NULL: - sql.WHERE(queryKey + " IS NOT NULL"); + sql.WHERE(columnName + " IS NOT NULL"); break; default: log.warn("Unknown query type: {}", annotation.queryType()); diff --git a/bigtop-manager-dao/src/main/resources/mapper/mysql/ChatMessageMapper.xml b/bigtop-manager-dao/src/main/resources/mapper/mysql/ChatMessageMapper.xml index 108410b1c..190434432 100644 --- a/bigtop-manager-dao/src/main/resources/mapper/mysql/ChatMessageMapper.xml +++ b/bigtop-manager-dao/src/main/resources/mapper/mysql/ChatMessageMapper.xml @@ -26,11 +26,7 @@ SELECT * FROM llm_chat_message WHERE thread_id = #{threadId} + AND is_deleted = 0 - - DELETE FROM llm_chat_message - WHERE thread_id = #{threadId} - - \ No newline at end of file diff --git a/bigtop-manager-dao/src/main/resources/mapper/mysql/ChatThreadMapper.xml b/bigtop-manager-dao/src/main/resources/mapper/mysql/ChatThreadMapper.xml index 5ecd9a2ae..adecfc118 100644 --- a/bigtop-manager-dao/src/main/resources/mapper/mysql/ChatThreadMapper.xml +++ b/bigtop-manager-dao/src/main/resources/mapper/mysql/ChatThreadMapper.xml @@ -33,16 +33,17 @@ SELECT * FROM llm_chat_thread WHERE user_id = #{userId} + AND is_deleted = 0 @@ -55,4 +56,10 @@ thread_info = VALUES(thread_info) + + \ No newline at end of file diff --git a/bigtop-manager-dao/src/main/resources/mapper/mysql/HostMapper.xml b/bigtop-manager-dao/src/main/resources/mapper/mysql/HostMapper.xml index 4c475a71d..ea168aff4 100644 --- a/bigtop-manager-dao/src/main/resources/mapper/mysql/HostMapper.xml +++ b/bigtop-manager-dao/src/main/resources/mapper/mysql/HostMapper.xml @@ -100,12 +100,4 @@ - - insert into host (hostname, ipv4, ipv6, os, arch, available_processors, free_memory_size, total_memory_size, free_disk, total_disk, state, cluster_id, create_by, update_by, create_time, update_time) - values - - (#{host.hostname}, #{host.ipv4}, #{host.ipv6}, #{host.os}, #{host.arch}, #{host.availableProcessors}, #{host.freeMemorySize}, #{host.totalMemorySize}, #{host.freeDisk}, #{host.totalDisk}, #{host.state}, #{host.clusterId} ,#{host.createBy},#{host.updateBy},#{host.createTime},#{host.updateTime}) - - - \ No newline at end of file diff --git a/bigtop-manager-dao/src/main/resources/mapper/mysql/PlatformAuthorizedMapper.xml b/bigtop-manager-dao/src/main/resources/mapper/mysql/PlatformAuthorizedMapper.xml index f065a4c7b..e9fa2bf49 100644 --- a/bigtop-manager-dao/src/main/resources/mapper/mysql/PlatformAuthorizedMapper.xml +++ b/bigtop-manager-dao/src/main/resources/mapper/mysql/PlatformAuthorizedMapper.xml @@ -31,15 +31,19 @@ + + INSERT INTO llm_platform_authorized (platform_id, credentials) VALUES (#{platformId}, #{credentials, typeHandler=org.apache.bigtop.manager.dao.handler.JsonTypeHandler}) - ON DUPLICATE KEY UPDATE - platform_id = VALUES(platform_id), - credentials = VALUES(credentials) + ON DUPLICATE KEY UPDATE + platform_id = VALUES(platform_id), + credentials = VALUES(credentials) \ No newline at end of file diff --git a/bigtop-manager-dao/src/main/resources/mapper/mysql/RepoMapper.xml b/bigtop-manager-dao/src/main/resources/mapper/mysql/RepoMapper.xml index cee5e8685..d1f0d358d 100644 --- a/bigtop-manager-dao/src/main/resources/mapper/mysql/RepoMapper.xml +++ b/bigtop-manager-dao/src/main/resources/mapper/mysql/RepoMapper.xml @@ -40,14 +40,6 @@ limit 1 - - insert into repo (base_url, os, arch, repo_id, repo_name, repo_type, cluster_id, create_by, update_by, create_time, update_time) - values - - (#{cluster.baseUrl},#{cluster.os},#{cluster.arch},#{cluster.repoId},#{cluster.repoName},#{cluster.repoType},#{cluster.clusterId},#{cluster.createBy},#{cluster.updateBy},#{cluster.createTime},#{cluster.updateTime}) - - - + SELECT * + FROM llm_chat_message + WHERE thread_id = #{threadId} + AND is_deleted = false + + + \ No newline at end of file diff --git a/bigtop-manager-dao/src/main/resources/mapper/postgresql/ChatThreadMapper.xml b/bigtop-manager-dao/src/main/resources/mapper/postgresql/ChatThreadMapper.xml new file mode 100644 index 000000000..3775c50cc --- /dev/null +++ b/bigtop-manager-dao/src/main/resources/mapper/postgresql/ChatThreadMapper.xml @@ -0,0 +1,65 @@ + + + + + + id, user_id, platform_id, model + + + + + + + + + + + + + + INSERT INTO llm_chat_thread (platform_id, user_id, model, thread_info) + VALUES (#{platformId}, #{userId}, #{model}, #{threadInfo, typeHandler=org.apache.bigtop.manager.dao.handler.JsonTypeHandler}) + ON DUPLICATE KEY UPDATE + platform_id = VALUES(platform_id), + user_id = VALUES(user_id), + model = VALUES(model), + thread_info = VALUES(thread_info) + + + + + \ No newline at end of file diff --git a/bigtop-manager-dao/src/main/resources/mapper/postgresql/ClusterMapper.xml b/bigtop-manager-dao/src/main/resources/mapper/postgresql/ClusterMapper.xml new file mode 100644 index 000000000..75caa42aa --- /dev/null +++ b/bigtop-manager-dao/src/main/resources/mapper/postgresql/ClusterMapper.xml @@ -0,0 +1,81 @@ + + + + + + + + id, cluster_name, cluster_type, root, user_group, packages, repo_template, state, selected, stack_id + + + ${alias}.id, ${alias}.cluster_name, ${alias}.cluster_type, ${alias}.root, ${alias}.user_group, ${alias}.packages, ${alias}.repo_template, ${alias}.state, ${alias}.selected, ${alias}.stack_id + + + + + + + + + + + \ No newline at end of file diff --git a/bigtop-manager-dao/src/main/resources/mapper/postgresql/ComponentMapper.xml b/bigtop-manager-dao/src/main/resources/mapper/postgresql/ComponentMapper.xml new file mode 100644 index 000000000..cc1c93d65 --- /dev/null +++ b/bigtop-manager-dao/src/main/resources/mapper/postgresql/ComponentMapper.xml @@ -0,0 +1,109 @@ + + + + + + + + id, component_name, display_name, command_script, custom_commands, category, quick_link, cardinality, service_id, cluster_id + + + ${alias}.id, ${alias}.component_name, ${alias}.display_name, ${alias}.command_script, ${alias}.custom_commands, ${alias}.category, ${alias}.quick_link, ${alias}.cardinality, ${alias}.service_id, ${alias}.cluster_id + + + + + + + + + + + \ No newline at end of file diff --git a/bigtop-manager-dao/src/main/resources/mapper/postgresql/HostComponentMapper.xml b/bigtop-manager-dao/src/main/resources/mapper/postgresql/HostComponentMapper.xml new file mode 100644 index 000000000..214dbc902 --- /dev/null +++ b/bigtop-manager-dao/src/main/resources/mapper/postgresql/HostComponentMapper.xml @@ -0,0 +1,222 @@ + + + + + + + + id, state, host_id, component_id + + + ${alias}.id, ${alias}.state, ${alias}.host_id, ${alias}.component_id + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/bigtop-manager-dao/src/main/resources/mapper/postgresql/HostMapper.xml b/bigtop-manager-dao/src/main/resources/mapper/postgresql/HostMapper.xml new file mode 100644 index 000000000..ea168aff4 --- /dev/null +++ b/bigtop-manager-dao/src/main/resources/mapper/postgresql/HostMapper.xml @@ -0,0 +1,103 @@ + + + + + + + + id, hostname, ipv4, ipv6, os, arch, available_processors, free_memory_size, total_memory_size, free_disk, total_disk, state, cluster_id + + + + ${alias}.id, ${alias}.hostname, ${alias}.ipv4, ${alias}.ipv6, ${alias}.os, ${alias}.arch, ${alias}.available_processors, ${alias}.free_memory_size, ${alias}.total_memory_size, ${alias}.free_disk, ${alias}.total_disk, ${alias}.state, ${alias}.cluster_id + + + + + + + + + + + \ No newline at end of file diff --git a/bigtop-manager-dao/src/main/resources/mapper/postgresql/JobMapper.xml b/bigtop-manager-dao/src/main/resources/mapper/postgresql/JobMapper.xml new file mode 100644 index 000000000..7805bba8c --- /dev/null +++ b/bigtop-manager-dao/src/main/resources/mapper/postgresql/JobMapper.xml @@ -0,0 +1,142 @@ + + + + + + + + id, state, name, context, cluster_id, create_time, update_time + + + + ${alias}.id, ${alias}.state, ${alias}.name, ${alias}.context, ${alias}.cluster_id, ${alias}.create_time, ${alias}.update_time + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/bigtop-manager-dao/src/main/resources/mapper/postgresql/PlatformAuthorizedMapper.xml b/bigtop-manager-dao/src/main/resources/mapper/postgresql/PlatformAuthorizedMapper.xml new file mode 100644 index 000000000..63e841a31 --- /dev/null +++ b/bigtop-manager-dao/src/main/resources/mapper/postgresql/PlatformAuthorizedMapper.xml @@ -0,0 +1,49 @@ + + + + + + + id, credentials, platfotrm_id + + + + + + + + + + + + INSERT INTO llm_platform_authorized (platform_id, credentials) + VALUES (#{platformId}, #{credentials, typeHandler=org.apache.bigtop.manager.dao.handler.JsonTypeHandler}) + ON DUPLICATE KEY UPDATE + platform_id = VALUES(platform_id), + credentials = VALUES(credentials) + + + \ No newline at end of file diff --git a/bigtop-manager-dao/src/main/resources/mapper/postgresql/PlatformMapper.xml b/bigtop-manager-dao/src/main/resources/mapper/postgresql/PlatformMapper.xml new file mode 100644 index 000000000..885ae6d19 --- /dev/null +++ b/bigtop-manager-dao/src/main/resources/mapper/postgresql/PlatformMapper.xml @@ -0,0 +1,38 @@ + + + + + + + "id", "name", "credential", "support_models" + + + + + + + + + + \ No newline at end of file diff --git a/bigtop-manager-dao/src/main/resources/mapper/postgresql/RepoMapper.xml b/bigtop-manager-dao/src/main/resources/mapper/postgresql/RepoMapper.xml new file mode 100644 index 000000000..d1f0d358d --- /dev/null +++ b/bigtop-manager-dao/src/main/resources/mapper/postgresql/RepoMapper.xml @@ -0,0 +1,56 @@ + + + + + + + + id, base_url, os, arch, repo_id, repo_name, repo_type, cluster_id + + + + + + + \ No newline at end of file diff --git a/bigtop-manager-dao/src/main/resources/mapper/postgresql/ServiceConfigMapper.xml b/bigtop-manager-dao/src/main/resources/mapper/postgresql/ServiceConfigMapper.xml new file mode 100644 index 000000000..e40c4f9e6 --- /dev/null +++ b/bigtop-manager-dao/src/main/resources/mapper/postgresql/ServiceConfigMapper.xml @@ -0,0 +1,118 @@ + + + + + + + + id, config_desc, version, selected, service_id, cluster_id, create_time, update_time + + + + ${alias}.id, ${alias}.config_desc, ${alias}.version, ${alias}.selected, ${alias}.service_id, ${alias}.cluster_id, ${alias}.create_time, ${alias}.update_time + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/bigtop-manager-dao/src/main/resources/mapper/postgresql/ServiceMapper.xml b/bigtop-manager-dao/src/main/resources/mapper/postgresql/ServiceMapper.xml new file mode 100644 index 000000000..5a4056514 --- /dev/null +++ b/bigtop-manager-dao/src/main/resources/mapper/postgresql/ServiceMapper.xml @@ -0,0 +1,133 @@ + + + + + + + + id, service_name, display_name, service_desc, service_version, package_specifics, service_user, required_services, cluster_id + + + + ${alias}.id, ${alias}.service_name, ${alias}.display_name, ${alias}.service_desc, ${alias}.service_version, ${alias}.package_specifics, ${alias}.service_user, ${alias}.required_services, ${alias}.cluster_id + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/bigtop-manager-dao/src/main/resources/mapper/postgresql/StackMapper.xml b/bigtop-manager-dao/src/main/resources/mapper/postgresql/StackMapper.xml new file mode 100644 index 000000000..d9da21ba5 --- /dev/null +++ b/bigtop-manager-dao/src/main/resources/mapper/postgresql/StackMapper.xml @@ -0,0 +1,46 @@ + + + + + + + + "id", "stack_name", "stack_version" + + + + + \ No newline at end of file diff --git a/bigtop-manager-dao/src/main/resources/mapper/postgresql/StageMapper.xml b/bigtop-manager-dao/src/main/resources/mapper/postgresql/StageMapper.xml new file mode 100644 index 000000000..35d6ac583 --- /dev/null +++ b/bigtop-manager-dao/src/main/resources/mapper/postgresql/StageMapper.xml @@ -0,0 +1,39 @@ + + + + + + + + UPDATE "stage" + SET "state" = CASE + + WHEN id = #{item.id} THEN #{item.state} + + END + WHERE "id" IN + + #{item.id} + + + + \ No newline at end of file diff --git a/bigtop-manager-dao/src/main/resources/mapper/postgresql/TaskMapper.xml b/bigtop-manager-dao/src/main/resources/mapper/postgresql/TaskMapper.xml new file mode 100644 index 000000000..a746f0dba --- /dev/null +++ b/bigtop-manager-dao/src/main/resources/mapper/postgresql/TaskMapper.xml @@ -0,0 +1,39 @@ + + + + + + + + UPDATE task + SET "state" = CASE + + WHEN id = #{item.id} THEN #{item.state} + + END + WHERE id IN + + #{item.id} + + + + \ No newline at end of file diff --git a/bigtop-manager-dao/src/main/resources/mapper/postgresql/UserMapper.xml b/bigtop-manager-dao/src/main/resources/mapper/postgresql/UserMapper.xml new file mode 100644 index 000000000..24a32e289 --- /dev/null +++ b/bigtop-manager-dao/src/main/resources/mapper/postgresql/UserMapper.xml @@ -0,0 +1,43 @@ + + + + + + + + "id", "username", "password", "nickname", "status" + + + + + \ No newline at end of file diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/AbstractJob.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/AbstractJob.java index 902311b0c..6ae1ee859 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/AbstractJob.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/AbstractJob.java @@ -33,10 +33,13 @@ import org.apache.bigtop.manager.server.command.task.Task; import org.apache.bigtop.manager.server.holder.SpringContextHolder; +import lombok.extern.slf4j.Slf4j; + import java.util.ArrayList; import java.util.List; import java.util.concurrent.LinkedBlockingQueue; +@Slf4j public abstract class AbstractJob implements Job { protected StackDao stackDao; @@ -85,7 +88,7 @@ protected void beforeCreateStages() { @Override public void beforeRun() { jobPO.setState(JobState.PROCESSING.getName()); - jobDao.updateById(jobPO); + jobDao.partialUpdateById(jobPO); } @Override @@ -106,6 +109,7 @@ public void run() { } } } catch (Exception e) { + log.error(e.getMessage(), e); success = false; } @@ -120,7 +124,7 @@ public void run() { public void onSuccess() { JobPO jobPO = getJobPO(); jobPO.setState(JobState.SUCCESSFUL.getName()); - jobDao.updateById(jobPO); + jobDao.partialUpdateById(jobPO); } @Override @@ -150,7 +154,7 @@ public void onFailure() { if (!stagePOList.isEmpty()) { stageDao.updateStateByIds(stagePOList); } - jobDao.updateById(jobPO); + jobDao.partialUpdateById(jobPO); } @Override diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/ClusterCreateJob.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/ClusterCreateJob.java index 47a274473..91a4c6099 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/ClusterCreateJob.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/ClusterCreateJob.java @@ -77,22 +77,22 @@ public void onSuccess() { // Update cluster state to installed clusterPO.setState(MaintainState.INSTALLED.getName()); - clusterDao.updateById(clusterPO); + clusterDao.partialUpdateById(clusterPO); // Link job to cluster after cluster successfully added JobPO jobPO = getJobPO(); jobPO.setClusterId(clusterPO.getId()); - jobDao.updateById(jobPO); + jobDao.partialUpdateById(jobPO); for (Stage stage : getStages()) { StagePO stagePO = stage.getStagePO(); stagePO.setClusterId(clusterPO.getId()); - stageDao.updateById(stagePO); + stageDao.partialUpdateById(stagePO); for (Task task : stage.getTasks()) { TaskPO taskPO = task.getTaskPO(); taskPO.setClusterId(clusterPO.getId()); - taskDao.updateById(taskPO); + taskDao.partialUpdateById(taskPO); } } } diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/AbstractStage.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/AbstractStage.java index 2081f7c36..07259a39a 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/AbstractStage.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/AbstractStage.java @@ -25,10 +25,13 @@ import org.apache.bigtop.manager.server.command.task.Task; import org.apache.bigtop.manager.server.holder.SpringContextHolder; +import lombok.extern.slf4j.Slf4j; + import java.util.ArrayList; import java.util.List; import java.util.concurrent.CompletableFuture; +@Slf4j public abstract class AbstractStage implements Stage { protected StageDao stageDao; @@ -73,7 +76,7 @@ protected String getComponentName() { @Override public void beforeRun() { stagePO.setState(JobState.PROCESSING.getName()); - stageDao.updateById(stagePO); + stageDao.partialUpdateById(stagePO); } @Override @@ -93,6 +96,7 @@ public Boolean run() { try { return future.get(); } catch (Exception e) { + log.error("stage failed,", e); return false; } }) @@ -100,6 +104,7 @@ public Boolean run() { allTaskSuccess = taskResults.stream().allMatch(Boolean::booleanValue); } catch (Exception e) { + log.error("stage failed", e); allTaskSuccess = false; } @@ -116,14 +121,14 @@ public Boolean run() { public void onSuccess() { StagePO stagePO = getStagePO(); stagePO.setState(JobState.SUCCESSFUL.getName()); - stageDao.updateById(stagePO); + stageDao.partialUpdateById(stagePO); } @Override public void onFailure() { StagePO stagePO = getStagePO(); stagePO.setState(JobState.FAILED.getName()); - stageDao.updateById(stagePO); + stageDao.partialUpdateById(stagePO); } @Override diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/task/AbstractTask.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/task/AbstractTask.java index fc0871a84..e8767cbbf 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/task/AbstractTask.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/task/AbstractTask.java @@ -31,6 +31,9 @@ import org.apache.bigtop.manager.server.grpc.GrpcClient; import org.apache.bigtop.manager.server.holder.SpringContextHolder; +import lombok.extern.slf4j.Slf4j; + +@Slf4j public abstract class AbstractTask implements Task { protected TaskDao taskDao; @@ -65,7 +68,7 @@ protected String getCustomCommand() { @Override public void beforeRun() { taskPO.setState(JobState.PROCESSING.getName()); - taskDao.updateById(taskPO); + taskDao.partialUpdateById(taskPO); } @Override @@ -85,6 +88,7 @@ public Boolean run() { taskSuccess = reply != null && reply.getCode() == MessageConstants.SUCCESS_CODE; } catch (Exception e) { + log.error("task failed", e); taskSuccess = false; } @@ -102,7 +106,7 @@ public void onSuccess() { TaskPO taskPO = getTaskPO(); taskPO.setContent(ProtobufUtil.toJson(commandRequest)); taskPO.setState(JobState.SUCCESSFUL.getName()); - taskDao.updateById(taskPO); + taskDao.partialUpdateById(taskPO); } @Override @@ -110,7 +114,7 @@ public void onFailure() { TaskPO taskPO = getTaskPO(); taskPO.setContent(ProtobufUtil.toJson(commandRequest)); taskPO.setState(JobState.FAILED.getName()); - taskDao.updateById(taskPO); + taskDao.partialUpdateById(taskPO); } @Override diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/task/ComponentInstallTask.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/task/ComponentInstallTask.java index 91f003cbb..176a8e740 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/task/ComponentInstallTask.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/task/ComponentInstallTask.java @@ -43,7 +43,7 @@ public void onSuccess() { HostComponentPO hostComponentPO = hostComponentDao.findByClusterIdAndComponentNameAndHostname(clusterId, componentName, hostname); hostComponentPO.setState(MaintainState.INSTALLED.getName()); - hostComponentDao.updateById(hostComponentPO); + hostComponentDao.partialUpdateById(hostComponentPO); } @Override diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/task/ComponentStartTask.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/task/ComponentStartTask.java index fca02aa87..aed3898cd 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/task/ComponentStartTask.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/task/ComponentStartTask.java @@ -43,7 +43,7 @@ public void onSuccess() { HostComponentPO hostComponentPO = hostComponentDao.findByClusterIdAndComponentNameAndHostname(clusterId, componentName, hostname); hostComponentPO.setState(MaintainState.STARTED.getName()); - hostComponentDao.updateById(hostComponentPO); + hostComponentDao.partialUpdateById(hostComponentPO); } @Override diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/task/ComponentStopTask.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/task/ComponentStopTask.java index b779aa448..2b7ea999b 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/task/ComponentStopTask.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/task/ComponentStopTask.java @@ -43,7 +43,7 @@ public void onSuccess() { HostComponentPO hostComponentPO = hostComponentDao.findByClusterIdAndComponentNameAndHostname(clusterId, componentName, hostname); hostComponentPO.setState(MaintainState.STOPPED.getName()); - hostComponentDao.updateById(hostComponentPO); + hostComponentDao.partialUpdateById(hostComponentPO); } @Override diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/config/MyBatisConfig.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/config/MyBatisConfig.java index 17a3bdad3..b9b58f307 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/config/MyBatisConfig.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/config/MyBatisConfig.java @@ -43,8 +43,10 @@ public class MyBatisConfig { public SqlSessionFactory sqlSessionFactory(DataSource dataSource) throws Exception { SqlSessionFactoryBean sessionFactory = new SqlSessionFactoryBean(); sessionFactory.setDataSource(dataSource); - sessionFactory.setMapperLocations( - new PathMatchingResourcePatternResolver().getResources("classpath*:mapper/**/*Mapper.xml")); + sessionFactory.setMapperLocations(new PathMatchingResourcePatternResolver() + .getResources(String.format( + "classpath*:mapper/%s/**/*Mapper.xml", + new ProductNameDatabaseIdProvider().getDatabaseId(dataSource)))); org.apache.ibatis.session.Configuration configuration = new org.apache.ibatis.session.Configuration(); configuration.setMapUnderscoreToCamelCase(true); diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/controller/ChatbotController.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/controller/ChatbotController.java index 777119394..c09680a4f 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/controller/ChatbotController.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/controller/ChatbotController.java @@ -20,6 +20,7 @@ import org.apache.bigtop.manager.server.model.converter.PlatformConverter; import org.apache.bigtop.manager.server.model.dto.PlatformDTO; +import org.apache.bigtop.manager.server.model.req.ChatbotMessageReq; import org.apache.bigtop.manager.server.model.req.PlatformReq; import org.apache.bigtop.manager.server.model.vo.ChatMessageVO; import org.apache.bigtop.manager.server.model.vo.ChatThreadVO; @@ -66,7 +67,7 @@ public ResponseEntity> authorizedPlatforms() { return ResponseEntity.success(chatbotService.authorizedPlatforms()); } - @Operation(summary = "platforms", description = "Get authorized platforms") + @Operation(summary = "platforms", description = "Get platform auth credentials") @GetMapping("/platforms/{platformId}/auth/credential") public ResponseEntity> platformsAuthCredential(@PathVariable Long platformId) { return ResponseEntity.success(chatbotService.platformsAuthCredential(platformId)); @@ -106,8 +107,9 @@ public ResponseEntity> getAllChatThreads( @Operation(summary = "talk", description = "Talk with Chatbot") @PostMapping("platforms/{platformId}/threads/{threadId}/talk") - public SseEmitter talk(@PathVariable Long platformId, @PathVariable Long threadId, @RequestParam String message) { - return chatbotService.talk(platformId, threadId, message); + public SseEmitter talk( + @PathVariable Long platformId, @PathVariable Long threadId, @RequestBody ChatbotMessageReq messageReq) { + return chatbotService.talk(platformId, threadId, messageReq.getMessage()); } @Operation(summary = "history", description = "Get chat records") diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/converter/ChatMessageConverter.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/converter/ChatMessageConverter.java index b2c17238d..714f2d1e3 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/converter/ChatMessageConverter.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/converter/ChatMessageConverter.java @@ -18,7 +18,7 @@ */ package org.apache.bigtop.manager.server.model.converter; -import org.apache.bigtop.manager.ai.core.enums.MessageSender; +import org.apache.bigtop.manager.ai.core.enums.MessageType; import org.apache.bigtop.manager.dao.po.ChatMessagePO; import org.apache.bigtop.manager.server.config.MapStructSharedConfig; import org.apache.bigtop.manager.server.model.vo.ChatMessageVO; @@ -32,12 +32,12 @@ public interface ChatMessageConverter { ChatMessageVO fromPO2VO(ChatMessagePO chatMessagePO); - default MessageSender mapStringToMessageSender(String sender) { + default MessageType mapStringToMessageSender(String sender) { if (sender == null) { return null; } try { - return MessageSender.valueOf(sender.toUpperCase()); + return MessageType.valueOf(sender.toUpperCase()); } catch (IllegalArgumentException e) { return null; } diff --git a/dev-support/example/bigtop_manager/user.sql b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/req/ChatbotMessageReq.java similarity index 79% rename from dev-support/example/bigtop_manager/user.sql rename to bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/req/ChatbotMessageReq.java index 9786777df..3bcbe0cfc 100644 --- a/dev-support/example/bigtop_manager/user.sql +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/req/ChatbotMessageReq.java @@ -16,5 +16,14 @@ * specific language governing permissions and limitations * under the License. */ +package org.apache.bigtop.manager.server.model.req; -INSERT INTO bigtop_manager.user (id, create_time, update_time, nickname, password, status, username)VALUES (1, now(), now(), 'Administrator', '21232f297a57a5a743894a0e4a801fc3', true, 'admin'); \ No newline at end of file +import lombok.Data; + +import jakarta.validation.constraints.NotEmpty; + +@Data +public class ChatbotMessageReq { + @NotEmpty + private String message; +} diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/vo/ChatMessageVO.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/vo/ChatMessageVO.java index da226da07..7ad366a87 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/vo/ChatMessageVO.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/model/vo/ChatMessageVO.java @@ -18,19 +18,19 @@ */ package org.apache.bigtop.manager.server.model.vo; -import org.apache.bigtop.manager.ai.core.enums.MessageSender; +import org.apache.bigtop.manager.ai.core.enums.MessageType; import lombok.Data; @Data public class ChatMessageVO { - private MessageSender sender; + private MessageType sender; private String message; private String createTime; - public ChatMessageVO(MessageSender sender, String messageText, String createTime) { + public ChatMessageVO(MessageType sender, String messageText, String createTime) { this.sender = sender; this.message = messageText; this.createTime = createTime; diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/scheduler/ComponentStatusScheduler.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/scheduler/ComponentStatusScheduler.java index 4c0d99caf..141362c1f 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/scheduler/ComponentStatusScheduler.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/scheduler/ComponentStatusScheduler.java @@ -71,13 +71,13 @@ public void execute() { if (reply.getStatus() == 0 && MaintainState.fromString(hostComponentPO.getState()) == MaintainState.STOPPED) { hostComponentPO.setState(MaintainState.STARTED.getName()); - hostComponentDao.updateById(hostComponentPO); + hostComponentDao.partialUpdateById(hostComponentPO); } if (reply.getStatus() != 0 && MaintainState.fromString(hostComponentPO.getState()) == MaintainState.STARTED) { hostComponentPO.setState(MaintainState.STOPPED.getName()); - hostComponentDao.updateById(hostComponentPO); + hostComponentDao.partialUpdateById(hostComponentPO); } } } diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/scheduler/HostInfoScheduler.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/scheduler/HostInfoScheduler.java index f314661d3..33ebab048 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/scheduler/HostInfoScheduler.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/scheduler/HostInfoScheduler.java @@ -75,6 +75,6 @@ private void getHostInfo(HostPO hostPO) { hostPO.setState(MaintainState.STOPPED.getName()); } - hostDao.updateById(hostPO); + hostDao.partialUpdateById(hostPO); } } diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/service/impl/ChatbotServiceImpl.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/service/impl/ChatbotServiceImpl.java index c338340e4..da363aee7 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/service/impl/ChatbotServiceImpl.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/service/impl/ChatbotServiceImpl.java @@ -21,7 +21,7 @@ import org.apache.bigtop.manager.ai.assistant.GeneralAssistantFactory; import org.apache.bigtop.manager.ai.assistant.provider.AIAssistantConfig; import org.apache.bigtop.manager.ai.assistant.store.PersistentChatMemoryStore; -import org.apache.bigtop.manager.ai.core.enums.MessageSender; +import org.apache.bigtop.manager.ai.core.enums.MessageType; import org.apache.bigtop.manager.ai.core.enums.PlatformType; import org.apache.bigtop.manager.ai.core.factory.AIAssistant; import org.apache.bigtop.manager.ai.core.factory.AIAssistantFactory; @@ -137,7 +137,7 @@ public List platforms() { @Override public List authorizedPlatforms() { List authorizedPlatforms = new ArrayList<>(); - List authorizedPlatformPOs = platformAuthorizedDao.findAll(); + List authorizedPlatformPOs = platformAuthorizedDao.findAllPlatform(); for (PlatformAuthorizedPO authorizedPlatformPO : authorizedPlatformPOs) { PlatformPO platformPO = platformDao.findById(authorizedPlatformPO.getPlatformId()); authorizedPlatforms.add(PlatformAuthorizedConverter.INSTANCE.fromPO2VO(authorizedPlatformPO, platformPO)); @@ -209,10 +209,14 @@ public List platformsAuthCredential(Long platformId) { @Override public boolean deleteAuthorizedPlatform(Long platformId) { - List authorizedPlatformPOs = platformAuthorizedDao.findAll(); + List authorizedPlatformPOs = platformAuthorizedDao.findAllPlatform(); for (PlatformAuthorizedPO authorizedPlatformPO : authorizedPlatformPOs) { if (authorizedPlatformPO.getId().equals(platformId)) { - platformAuthorizedDao.deleteById(authorizedPlatformPO.getId()); + authorizedPlatformPO.setIsDeleted(true); + platformAuthorizedDao.partialUpdateById(authorizedPlatformPO); + List chatThreadPOS = chatThreadDao.findAllByPlatformId(authorizedPlatformPO.getId()); + chatThreadPOS.forEach(chatThread -> chatThread.setIsDeleted(true)); + chatThreadDao.partialUpdateByIds(chatThreadPOS); return true; } } @@ -253,7 +257,11 @@ public boolean deleteChatThreads(Long platformId, Long threadId) { for (ChatThreadPO chatThreadPO : chatThreadPOS) { if (chatThreadPO.getId().equals(threadId) && chatThreadPO.getPlatformId().equals(platformId)) { - chatThreadDao.deleteById(threadId); + chatThreadPO.setIsDeleted(true); + chatThreadDao.partialUpdateById(chatThreadPO); + List chatMessagePOS = chatMessageDao.findAllByThreadId(threadId); + chatMessagePOS.forEach(chatMessage -> chatMessage.setIsDeleted(true)); + chatMessageDao.partialUpdateByIds(chatMessagePOS); return true; } } @@ -312,7 +320,7 @@ public SseEmitter talk(Long platformId, Long threadId, String message) { @Override public List history(Long platformId, Long threadId) { List chatMessages = new ArrayList<>(); - ChatThreadPO chatThreadPO = chatThreadDao.findById(threadId); + ChatThreadPO chatThreadPO = chatThreadDao.findByThreadId(threadId); if (chatThreadPO == null) { throw new ApiException(ApiExceptionEnum.CHAT_THREAD_NOT_FOUND); } @@ -323,11 +331,11 @@ public List history(Long platformId, Long threadId) { List chatMessagePOs = chatMessageDao.findAllByThreadId(threadId); for (ChatMessagePO chatMessagePO : chatMessagePOs) { ChatMessageVO chatMessageVO = ChatMessageConverter.INSTANCE.fromPO2VO(chatMessagePO); - MessageSender sender = chatMessageVO.getSender(); + MessageType sender = chatMessageVO.getSender(); if (sender == null) { continue; } - if (sender.equals(MessageSender.USER) || sender.equals(MessageSender.AI)) { + if (sender.equals(MessageType.USER) || sender.equals(MessageType.AI)) { chatMessages.add(chatMessageVO); } } diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/service/impl/ClusterServiceImpl.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/service/impl/ClusterServiceImpl.java index 19604a130..d41223621 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/service/impl/ClusterServiceImpl.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/service/impl/ClusterServiceImpl.java @@ -87,7 +87,7 @@ public ClusterVO save(ClusterDTO clusterDTO) { clusterDao.findByClusterName(clusterDTO.getClusterName()).orElse(new ClusterPO()); if (oldClusterPO.getId() != null) { clusterPO.setId(oldClusterPO.getId()); - clusterDao.updateById(clusterPO); + clusterDao.partialUpdateById(clusterPO); } else { clusterDao.save(clusterPO); } @@ -126,7 +126,7 @@ public ClusterVO get(Long id) { public ClusterVO update(Long id, ClusterDTO clusterDTO) { ClusterPO clusterPO = ClusterConverter.INSTANCE.fromDTO2PO(clusterDTO); clusterPO.setId(id); - clusterDao.updateById(clusterPO); + clusterDao.partialUpdateById(clusterPO); return ClusterConverter.INSTANCE.fromEntity2VO(clusterPO); } diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/service/impl/ConfigServiceImpl.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/service/impl/ConfigServiceImpl.java index 1f5ac9fcc..1db6526e7 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/service/impl/ConfigServiceImpl.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/service/impl/ConfigServiceImpl.java @@ -94,7 +94,7 @@ private void upsertServiceConfig( if (shouldUpdateConfig(existConfigs, configs)) { // Unselect current config currentConfig.setSelected(false); - serviceConfigDao.updateById(currentConfig); + serviceConfigDao.partialUpdateById(currentConfig); // Create a new config String configDesc = "Update config for " + servicePO.getServiceName(); diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/service/impl/HostServiceImpl.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/service/impl/HostServiceImpl.java index 7127d0e0e..8bc79a367 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/service/impl/HostServiceImpl.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/service/impl/HostServiceImpl.java @@ -99,7 +99,7 @@ public HostVO get(Long id) { public HostVO update(Long id, HostDTO hostDTO) { HostPO hostPO = HostConverter.INSTANCE.fromDTO2PO(hostDTO); hostPO.setId(id); - hostDao.updateById(hostPO); + hostDao.partialUpdateById(hostPO); return HostConverter.INSTANCE.fromPO2VO(hostPO); } diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/service/impl/JobServiceImpl.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/service/impl/JobServiceImpl.java index b6a30a3c3..fbf0bbeb2 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/service/impl/JobServiceImpl.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/service/impl/JobServiceImpl.java @@ -110,15 +110,15 @@ private void resetJobStatusInDB(JobPO jobPO) { for (StagePO stagePO : jobPO.getStages()) { for (TaskPO taskPO : stagePO.getTasks()) { taskPO.setState(JobState.PENDING.getName()); - taskDao.updateById(taskPO); + taskDao.partialUpdateById(taskPO); } stagePO.setState(JobState.PENDING.getName()); - stageDao.updateById(stagePO); + stageDao.partialUpdateById(stagePO); } jobPO.setState(JobState.PENDING.getName()); - jobDao.updateById(jobPO); + jobDao.partialUpdateById(jobPO); } private Job recreateJob(JobPO jobPO) { diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/service/impl/UserServiceImpl.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/service/impl/UserServiceImpl.java index eaa01d26b..61568d198 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/service/impl/UserServiceImpl.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/service/impl/UserServiceImpl.java @@ -50,7 +50,7 @@ public UserVO update(UserDTO userDTO) { Long id = SessionUserHolder.getUserId(); UserPO userPO = userDao.findOptionalById(id).orElseThrow(() -> new ApiException(ApiExceptionEnum.NEED_LOGIN)); userPO.setNickname(userDTO.getNickname()); - userDao.updateById(userPO); + userDao.partialUpdateById(userPO); return UserConverter.INSTANCE.fromPO2VO(userPO); } } diff --git a/bigtop-manager-server/src/main/resources/ddl/MySQL-DDL-CREATE.sql b/bigtop-manager-server/src/main/resources/ddl/MySQL-DDL-CREATE.sql index f702df793..dc67ea794 100644 --- a/bigtop-manager-server/src/main/resources/ddl/MySQL-DDL-CREATE.sql +++ b/bigtop-manager-server/src/main/resources/ddl/MySQL-DDL-CREATE.sql @@ -328,6 +328,7 @@ CREATE TABLE `llm_platform_authorized` `id` BIGINT(20) UNSIGNED NOT NULL AUTO_INCREMENT, `platform_id` BIGINT(20) UNSIGNED NOT NULL, `credentials` JSON NOT NULL, + `is_deleted` TINYINT(1) DEFAULT 0 NULL, `create_time` DATETIME DEFAULT CURRENT_TIMESTAMP, `update_time` DATETIME DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, `create_by` BIGINT DEFAULT NULL, @@ -342,6 +343,7 @@ CREATE TABLE `llm_chat_thread` `platform_id` BIGINT(20) UNSIGNED NOT NULL, `user_id` BIGINT(20) UNSIGNED NOT NULL, `model` VARCHAR(255) NOT NULL, + `is_deleted` TINYINT(1) DEFAULT 0 NULL, `thread_info` JSON DEFAULT NULL, `create_time` DATETIME DEFAULT CURRENT_TIMESTAMP, `update_time` DATETIME DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, @@ -359,6 +361,7 @@ CREATE TABLE `llm_chat_message` `user_id` BIGINT(20) UNSIGNED NOT NULL, `message` TEXT NOT NULL, `sender` VARCHAR(50) NOT NULL, + `is_deleted` TINYINT(1) DEFAULT 0 NULL, `create_time` DATETIME DEFAULT CURRENT_TIMESTAMP, `update_time` DATETIME DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, `create_by` BIGINT DEFAULT NULL, @@ -376,5 +379,5 @@ VALUES (1, now(), now(), 'Administrator', '21232f297a57a5a743894a0e4a801fc3', tr INSERT INTO bigtop_manager.llm_platform (id, credential, name, support_models) VALUES (1, '{"apiKey": "API Key"}', 'OpenAI', 'gpt-3.5-turbo,gpt-4,gpt-4o,gpt-3.5-turbo-16k,gpt-4-turbo-preview,gpt-4-32k,gpt-4o-mini'), -(2, '{"apiKey": "API Key"}', 'DashScope', 'qwen-max,qwen-plus,qwen-turbo'), +(2, '{"apiKey": "API Key"}', 'DashScope', 'qwen-1.8b-chat,qwen-max,qwen-plus,qwen-turbo'), (3, '{"apiKey": "API Key", "secretKey": "Secret Key"}', 'QianFan','Yi-34B-Chat,ERNIE-4.0-8K,ERNIE-3.5-128K,ERNIE-Speed-8K,Llama-2-7B-Chat,Fuyu-8B'); diff --git a/bigtop-manager-server/src/main/resources/ddl/PostgreSQL-DDL-CREATE.sql b/bigtop-manager-server/src/main/resources/ddl/PostgreSQL-DDL-CREATE.sql new file mode 100644 index 000000000..68e64e0ae --- /dev/null +++ b/bigtop-manager-server/src/main/resources/ddl/PostgreSQL-DDL-CREATE.sql @@ -0,0 +1,394 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +CREATE TABLE audit_log +( + id BIGINT NOT NULL GENERATED ALWAYS AS IDENTITY, + args TEXT, + create_by BIGINT, + create_time TIMESTAMP(0), + operation_desc VARCHAR(255), + operation_summary VARCHAR(255), + tag_desc VARCHAR(255), + tag_name VARCHAR(255), + update_by BIGINT, + update_time TIMESTAMP(0), + uri VARCHAR(255), + user_id BIGINT, + PRIMARY KEY (id) +); + +CREATE TABLE "user" +( + id BIGINT CHECK (id > 0) NOT NULL GENERATED ALWAYS AS IDENTITY, + username VARCHAR(32) DEFAULT NULL, + password VARCHAR(32) DEFAULT NULL, + nickname VARCHAR(32) DEFAULT NULL, + status BOOLEAN DEFAULT TRUE, + create_time TIMESTAMP(0) DEFAULT NULL, + update_time TIMESTAMP(0) DEFAULT NULL, + create_by BIGINT, + update_by BIGINT, + PRIMARY KEY (id), + CONSTRAINT uk_username UNIQUE (username) +); + +COMMENT ON COLUMN "user".status IS '0-Disable, 1-Enable'; + +CREATE TABLE cluster +( + id BIGINT CHECK (id > 0) NOT NULL GENERATED ALWAYS AS IDENTITY, + cluster_name VARCHAR(255) DEFAULT NULL, + cluster_desc VARCHAR(255) DEFAULT NULL, + cluster_type SMALLINT CHECK (cluster_type > 0) DEFAULT 1, + selected BOOLEAN DEFAULT TRUE, + create_time TIMESTAMP(0) DEFAULT NULL, + update_time TIMESTAMP(0) DEFAULT NULL, + create_by BIGINT, + packages VARCHAR(255), + repo_template VARCHAR(255), + root VARCHAR(255), + state VARCHAR(255), + update_by BIGINT, + user_group VARCHAR(255), + stack_id BIGINT, + PRIMARY KEY (id), + CONSTRAINT uk_cluster_name UNIQUE (cluster_name) +); + +COMMENT ON COLUMN cluster.cluster_name IS 'Cluster Name'; +COMMENT ON COLUMN cluster.cluster_desc IS 'Cluster Description'; +COMMENT ON COLUMN cluster.cluster_type IS '1-Physical Machine, 2-Kubernetes'; +COMMENT ON COLUMN cluster.selected IS '0-Disable, 1-Enable'; + +DROP INDEX IF EXISTS idx_cluster_stack_id; +CREATE INDEX idx_cluster_stack_id ON cluster (stack_id); + +CREATE TABLE component +( + id BIGINT NOT NULL GENERATED ALWAYS AS IDENTITY, + category VARCHAR(255), + command_script VARCHAR(255), + component_name VARCHAR(255), + create_by BIGINT, + create_time TIMESTAMP(0), + custom_commands TEXT, + display_name VARCHAR(255), + quick_link VARCHAR(255), + cardinality VARCHAR(255), + update_by BIGINT, + update_time TIMESTAMP(0), + cluster_id BIGINT, + service_id BIGINT, + PRIMARY KEY (id), + CONSTRAINT uk_component_name UNIQUE (component_name, cluster_id) +); + +DROP INDEX IF EXISTS idx_component_cluster_id; +DROP INDEX IF EXISTS idx_component_service_id; +CREATE INDEX idx_component_cluster_id ON component (cluster_id); +CREATE INDEX idx_component_service_id ON component (service_id); + +CREATE TABLE host_component +( + id BIGINT NOT NULL GENERATED ALWAYS AS IDENTITY, + create_by BIGINT, + create_time TIMESTAMP(0), + state VARCHAR(255), + update_by BIGINT, + update_time TIMESTAMP(0), + component_id BIGINT, + host_id BIGINT, + PRIMARY KEY (id) +); + +DROP INDEX IF EXISTS idx_hc_component_id; +DROP INDEX IF EXISTS idx_hc_host_id; +CREATE INDEX idx_hc_component_id ON host_component (component_id); +CREATE INDEX idx_hc_host_id ON host_component (host_id); + +CREATE TABLE host +( + id BIGINT CHECK (id > 0) NOT NULL GENERATED ALWAYS AS IDENTITY, + cluster_id BIGINT CHECK (cluster_id > 0) NOT NULL, + hostname VARCHAR(255) DEFAULT NULL, + ipv4 VARCHAR(32) DEFAULT NULL, + ipv6 VARCHAR(32) DEFAULT NULL, + arch VARCHAR(32) DEFAULT NULL, + os VARCHAR(32) DEFAULT NULL, + processor_count INT DEFAULT NULL, + physical_memory BIGINT DEFAULT NULL, + state VARCHAR(32) DEFAULT NULL, + create_time TIMESTAMP(0) DEFAULT NULL, + update_time TIMESTAMP(0) DEFAULT NULL, + available_processors INTEGER, + create_by BIGINT, + free_disk BIGINT, + free_memory_size BIGINT, + total_disk BIGINT, + total_memory_size BIGINT, + update_by BIGINT, + PRIMARY KEY (id), + CONSTRAINT uk_hostname UNIQUE (hostname, cluster_id) +); + +COMMENT ON COLUMN host.physical_memory IS 'Total Physical Memory(Bytes)'; + +DROP INDEX IF EXISTS idx_host_cluster_id; +CREATE INDEX idx_host_cluster_id ON host (cluster_id); + +CREATE TABLE repo +( + id BIGINT CHECK (id > 0) NOT NULL GENERATED ALWAYS AS IDENTITY, + cluster_id BIGINT CHECK (cluster_id > 0) NOT NULL, + os VARCHAR(32) DEFAULT NULL, + arch VARCHAR(32) DEFAULT NULL, + base_url VARCHAR(64) DEFAULT NULL, + repo_id VARCHAR(32) DEFAULT NULL, + repo_name VARCHAR(64) DEFAULT NULL, + repo_type VARCHAR(64) DEFAULT NULL, + create_time TIMESTAMP(0) DEFAULT NULL, + update_time TIMESTAMP(0) DEFAULT NULL, + create_by BIGINT, + update_by BIGINT, + PRIMARY KEY (id), + CONSTRAINT uk_repo_id UNIQUE (repo_id, os, arch, cluster_id) +); + +DROP INDEX IF EXISTS idx_cluster_id; +CREATE INDEX idx_cluster_id ON repo (cluster_id); + +CREATE TABLE stack +( + id BIGINT CHECK (id > 0) NOT NULL GENERATED ALWAYS AS IDENTITY, + stack_name VARCHAR(32) NOT NULL, + stack_version VARCHAR(32) NOT NULL, + create_time TIMESTAMP(0) DEFAULT NULL, + update_time TIMESTAMP(0) DEFAULT NULL, + create_by BIGINT, + update_by BIGINT, + PRIMARY KEY (id), + CONSTRAINT uk_stack UNIQUE (stack_name, stack_version) +); + +CREATE TABLE task +( + id BIGINT NOT NULL GENERATED ALWAYS AS IDENTITY, + command VARCHAR(255), + component_name VARCHAR(255), + content TEXT, + context TEXT NOT NULL, + create_by BIGINT, + create_time TIMESTAMP(0), + custom_command VARCHAR(255), + hostname VARCHAR(255), + name VARCHAR(255), + service_name VARCHAR(255), + service_user VARCHAR(255), + stack_name VARCHAR(255), + stack_version VARCHAR(255), + state VARCHAR(255), + update_by BIGINT, + update_time TIMESTAMP(0), + cluster_id BIGINT, + job_id BIGINT, + stage_id BIGINT, + PRIMARY KEY (id) +); + +DROP INDEX IF EXISTS idx_task_cluster_id; +DROP INDEX IF EXISTS idx_task_job_id; +DROP INDEX IF EXISTS idx_task_stage_id; +CREATE INDEX idx_task_cluster_id ON task (cluster_id); +CREATE INDEX idx_task_job_id ON task (job_id); +CREATE INDEX idx_task_stage_id ON task (stage_id); + +CREATE TABLE job +( + id BIGINT CHECK (id > 0) NOT NULL GENERATED ALWAYS AS IDENTITY, + cluster_id BIGINT CHECK (cluster_id > 0) DEFAULT NULL, + state VARCHAR(32) NOT NULL, + context TEXT NOT NULL, + create_time TIMESTAMP(0) DEFAULT NULL, + update_time TIMESTAMP(0) DEFAULT NULL, + create_by BIGINT, + name VARCHAR(255), + update_by BIGINT, + PRIMARY KEY (id) +); + +CREATE INDEX idx_job_cluster_id ON job (cluster_id); + +CREATE TABLE type_config +( + id BIGINT NOT NULL GENERATED ALWAYS AS IDENTITY, + create_by BIGINT, + create_time TIMESTAMP(0), + properties_json TEXT, + type_name VARCHAR(255), + update_by BIGINT, + update_time TIMESTAMP(0), + service_config_id BIGINT, + PRIMARY KEY (id) +); + +CREATE TABLE service +( + id BIGINT NOT NULL GENERATED ALWAYS AS IDENTITY, + create_by BIGINT, + create_time TIMESTAMP(0), + display_name VARCHAR(255), + package_specifics VARCHAR(1024), + required_services VARCHAR(255), + service_desc VARCHAR(1024), + service_name VARCHAR(255), + service_user VARCHAR(255), + service_version VARCHAR(255), + update_by BIGINT, + update_time TIMESTAMP(0), + cluster_id BIGINT, + PRIMARY KEY (id), + CONSTRAINT uk_service_name UNIQUE (service_name, cluster_id) +); + +CREATE INDEX idx_service_cluster_id ON service (cluster_id); + +CREATE TABLE service_config +( + id BIGINT NOT NULL GENERATED ALWAYS AS IDENTITY, + config_desc VARCHAR(255), + create_by BIGINT, + create_time TIMESTAMP(0), + selected BOOLEAN default FALSE, + update_by BIGINT, + update_time TIMESTAMP(0), + version INTEGER, + cluster_id BIGINT, + service_id BIGINT, + PRIMARY KEY (id) +); + +CREATE INDEX idx_sc_cluster_id ON service_config (cluster_id); +CREATE INDEX idx_sc_service_id ON service_config (service_id); + +CREATE TABLE setting +( + id BIGINT NOT NULL GENERATED ALWAYS AS IDENTITY, + config_data TEXT, + create_by BIGINT, + create_time TIMESTAMP(0), + type_name VARCHAR(255), + update_by BIGINT, + update_time TIMESTAMP(0), + PRIMARY KEY (id) +); + +CREATE TABLE stage +( + id BIGINT CHECK (id > 0) NOT NULL GENERATED ALWAYS AS IDENTITY, + name VARCHAR(32) NOT NULL, + cluster_id BIGINT CHECK (cluster_id > 0) DEFAULT NULL, + job_id BIGINT CHECK (job_id > 0) NOT NULL, + state VARCHAR(32) NOT NULL, + create_time TIMESTAMP(0) DEFAULT NULL, + update_time TIMESTAMP(0) DEFAULT NULL, + component_name VARCHAR(255), + context TEXT, + create_by BIGINT, + "order" INTEGER, + service_name VARCHAR(255), + update_by BIGINT, + PRIMARY KEY (id) +); + +CREATE INDEX idx_stage_cluster_id ON stage (cluster_id); +CREATE INDEX idx_job_id ON stage (job_id); + +CREATE TABLE llm_platform +( + id BIGINT CHECK (id > 0) NOT NULL GENERATED ALWAYS AS IDENTITY, + name VARCHAR(255) NOT NULL, + credential JSON DEFAULT NULL, + support_models VARCHAR(255) DEFAULT NULL, + create_time TIMESTAMP(0) DEFAULT CURRENT_TIMESTAMP, + update_time TIMESTAMP(0) DEFAULT CURRENT_TIMESTAMP /* ON UPDATE CURRENT_TIMESTAMP */, + create_by BIGINT DEFAULT NULL, + update_by BIGINT DEFAULT NULL, + PRIMARY KEY (id) +); + +CREATE TABLE llm_platform_authorized +( + id BIGINT CHECK (id > 0) NOT NULL GENERATED ALWAYS AS IDENTITY, + platform_id BIGINT CHECK (platform_id > 0) NOT NULL, + credentials JSON NOT NULL, + is_deleted BOOLEAN DEFAULT FALSE, + create_time TIMESTAMP(0) DEFAULT CURRENT_TIMESTAMP, + update_time TIMESTAMP(0) DEFAULT CURRENT_TIMESTAMP /* ON UPDATE CURRENT_TIMESTAMP */, + create_by BIGINT DEFAULT NULL, + update_by BIGINT DEFAULT NULL, + PRIMARY KEY (id) +); + +CREATE INDEX idx_authorized_platform_id ON llm_platform_authorized (platform_id); + +CREATE TABLE llm_chat_thread +( + id BIGINT CHECK (id > 0) NOT NULL GENERATED ALWAYS AS IDENTITY, + platform_id BIGINT CHECK (platform_id > 0) NOT NULL, + user_id BIGINT CHECK (user_id > 0) NOT NULL, + model VARCHAR(255) NOT NULL, + thread_info JSON DEFAULT NULL, + is_deleted BOOLEAN DEFAULT FALSE, + create_time TIMESTAMP(0) DEFAULT CURRENT_TIMESTAMP, + update_time TIMESTAMP(0) DEFAULT CURRENT_TIMESTAMP /* ON UPDATE CURRENT_TIMESTAMP */, + create_by BIGINT DEFAULT NULL, + update_by BIGINT DEFAULT NULL, + PRIMARY KEY (id) +); + +CREATE INDEX idx_chatthread_platform_id ON llm_chat_thread (platform_id); +CREATE INDEX idx_chatthread_user_id ON llm_chat_thread (user_id); + +CREATE TABLE llm_chat_message +( + id BIGINT CHECK (id > 0) NOT NULL GENERATED ALWAYS AS IDENTITY, + thread_id BIGINT CHECK (thread_id > 0) NOT NULL, + user_id BIGINT CHECK (user_id > 0) NOT NULL, + message TEXT NOT NULL, + sender VARCHAR(50) NOT NULL, + is_deleted BOOLEAN DEFAULT FALSE, + create_time TIMESTAMP(0) DEFAULT CURRENT_TIMESTAMP, + update_time TIMESTAMP(0) DEFAULT CURRENT_TIMESTAMP /* ON UPDATE CURRENT_TIMESTAMP */, + create_by BIGINT DEFAULT NULL, + update_by BIGINT DEFAULT NULL, + PRIMARY KEY (id) +); + +CREATE INDEX idx_thread_id ON llm_chat_message (thread_id); +CREATE INDEX idx_message_user_id ON llm_chat_message (user_id); + +INSERT INTO "user" (create_time, update_time, nickname, password, status, username) +VALUES (now(), now(), 'Administrator', '21232f297a57a5a743894a0e4a801fc3', true, 'admin'); + +INSERT INTO llm_platform (credential, NAME, support_models) +VALUES +('{"apiKey": "API Key"}','OpenAI','gpt-3.5-turbo,gpt-4,gpt-4o,gpt-3.5-turbo-16k,gpt-4-turbo-preview,gpt-4-32k,gpt-4o-mini'), +('{"apiKey": "API Key"}','DashScope','qwen-max,qwen-plus,qwen-turbo'), +('{"apiKey": "API Key", "secretKey": "Secret Key"}','QianFan','Yi-34B-Chat,ERNIE-4.0-8K,ERNIE-3.5-128K,ERNIE-Speed-8K,Llama-2-7B-Chat,Fuyu-8B'); diff --git a/bigtop-manager-server/src/test/java/org/apache/bigtop/manager/server/controller/ChatbotControllerTest.java b/bigtop-manager-server/src/test/java/org/apache/bigtop/manager/server/controller/ChatbotControllerTest.java index d5f12ba84..04148917a 100644 --- a/bigtop-manager-server/src/test/java/org/apache/bigtop/manager/server/controller/ChatbotControllerTest.java +++ b/bigtop-manager-server/src/test/java/org/apache/bigtop/manager/server/controller/ChatbotControllerTest.java @@ -19,6 +19,7 @@ package org.apache.bigtop.manager.server.controller; import org.apache.bigtop.manager.server.model.dto.PlatformDTO; +import org.apache.bigtop.manager.server.model.req.ChatbotMessageReq; import org.apache.bigtop.manager.server.model.req.PlatformReq; import org.apache.bigtop.manager.server.model.vo.ChatMessageVO; import org.apache.bigtop.manager.server.model.vo.ChatThreadVO; @@ -174,12 +175,14 @@ void getAllChatThreads() { void talk() { Long platformId = 1L; Long threadId = 1L; - String message = "Hello"; + ChatbotMessageReq messageReq = new ChatbotMessageReq(); + messageReq.setMessage("Hello"); SseEmitter emitter = new SseEmitter(); - when(chatbotService.talk(eq(platformId), eq(threadId), eq(message))).thenReturn(emitter); + when(chatbotService.talk(eq(platformId), eq(threadId), eq(messageReq.getMessage()))) + .thenReturn(emitter); - SseEmitter result = chatbotController.talk(platformId, threadId, message); + SseEmitter result = chatbotController.talk(platformId, threadId, messageReq); assertEquals(emitter, result); } diff --git a/dev-support/README.md b/dev-support/README.md index 4f9f3d4a7..5a7d7356e 100644 --- a/dev-support/README.md +++ b/dev-support/README.md @@ -30,38 +30,31 @@ yum install -y git docker ``` ### **Step 2**: Download Bigtop Manager source ```shell -git clone https://github.com/kevinw66/bigtop-manager.git +git clone https://github.com/apache/bigtop-manager.git ``` > You need to change the `node.version` in the `pom.xml` file under the `bigtop-manager-ui` module to `16.x` if you are using CentOS-7. -### **Step 3**: Enter workspace -**RHEL (Rocky 8) :** -```shell -cd bigtop-manager/dev-support/docker/rocky8/ -``` -### **Step 4**: Build develop basic image +### **Step 3**: Build develop basic image Run the setup command, you will get `bigtop-manager/develop:trunk-rocky-8` image. It has the enviroment needed to compile Bigtop-Manager and run servers such as Bigtop-Manager Server, Bigtop-Manager Agent, Mysql, etc. **RHEL (Rocky 8) :** ```shell -./build-image.sh +/bin/bash dev-support/docker/image/build.sh trunk-rocky-8 ``` -### **Step 5**: Build source & create cluster +### **Step 4**: Build source & create cluster * Bigtop Manager UI、Bigtop Manager Server Debug Port、MariaDB Server are also exposed to local ports: 8080、5005、3306. * Docker hostnames are: bigtop-manager-server、bigtop-manager-agent-01、bigtop-manager-agent-02. **RHEL (Rocky 8) :** ```shell -./build-containers.sh +/bin/bash dev-support/docker/containers/build.sh -e postgres -c 3 -o trunk-rocky-8 [--skip-compile] ``` -### **Step 6**: Insert data to Database -Copy SQL on `dev-support/example/bigtop_manager/user.sql` and run on mysql database `bigtop_manager` which is installed in container bigtop-manager-server. -### **Step 7**: Access Web UI +### **Step 5**: Access Web UI Now you can access Web UI which exposes on `http://localhost:8080`. Log in with username `admin` and password `admin`. -### **Step 8**: Clear cluster +### **Step 6**: Clear cluster Clean up the containers when you are done developing or testing. **RHEL (Rocky 8) :** ```shell -./clear-containers.sh +/bin/bash dev-support/docker/containers/build.sh -d ``` diff --git a/dev-support/docker/centos7/Dockerfile b/dev-support/docker/centos7/Dockerfile deleted file mode 100644 index cc061043d..000000000 --- a/dev-support/docker/centos7/Dockerfile +++ /dev/null @@ -1,32 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -FROM centos:7 - -RUN yum -y install sudo wget openssh-clients openssh-server vim mariadb mariadb-server net-tools chrony krb5-server krb5-libs krb5-workstation git rpm-build -RUN wget https://download.java.net/java/GA/jdk17.0.2/dfd4a8d0985749f896bed50d7138ee7f/8/GPL/openjdk-17.0.2_linux-x64_bin.tar.gz -O /tmp/jdk.tar.gz --no-check-certificate \ - && mkdir -p /usr/local/java && tar -xzf /tmp/jdk.tar.gz -C /usr/local/java --strip-components=1 \ - && rm -f /tmp/jdk.tar.gz \ - && ln -s /usr/local/java/bin/java /usr/bin/java -RUN wget https://archive.apache.org/dist/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz -O /tmp/apache-maven.tar.gz --no-check-certificate \ - && mkdir -p /usr/share/maven && tar -xzf /tmp/apache-maven.tar.gz -C /usr/share/maven --strip-components=1 \ - && rm -f /tmp/apache-maven.tar.gz \ - && ln -s /usr/share/maven/bin/mvn /usr/bin/mvn - -RUN /bin/sed -i 's,# StrictHostKeyChecking ask,StrictHostKeyChecking no,g' /etc/ssh/ssh_config - -RUN ssh-keygen -f "/root/.ssh/id_rsa" -N "" - -EXPOSE 1-65535 diff --git a/dev-support/docker/centos7/build-containers.sh b/dev-support/docker/centos7/build-containers.sh deleted file mode 100755 index cd700a09a..000000000 --- a/dev-support/docker/centos7/build-containers.sh +++ /dev/null @@ -1,111 +0,0 @@ -#!/bin/bash - -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -BIN_DIR=$(dirname $0) -cd $BIN_DIR -echo $PWD - -echo -e "\033[32mStarting container bigtop-manager-build\033[0m" -if [[ -z $(docker ps -a --format "table {{.Names}}" | grep "bigtop-manager-build") ]];then - docker run -it -d --name bigtop-manager-build --privileged=true -e "container=docker" \ - -v /sys/fs/cgroup:/sys/fs/cgroup:ro -v $PWD/../../../:/opt/bigtop-manager/ \ - -w /opt/bigtop-manager \ - bigtop-manager/develop:trunk-centos-7 -else - docker start bigtop-manager-build -fi - -echo -e "\033[32mCompiling bigtop-manager\033[0m" -docker exec bigtop-manager-build bash -c "mvn clean package -DskipTests" -docker stop bigtop-manager-build - -echo -e "\033[32mCreating network bigtop-manager\033[0m" -docker network create --driver bridge bigtop-manager - -echo -e "\033[32mCreating container bigtop-manager-server\033[0m" -docker run -d -p 3306:3306 -p 5005:5005 -p 8080:8080 --name bigtop-manager-server --hostname bigtop-manager-server --network bigtop-manager --privileged -e "container=docker" -v /sys/fs/cgroup:/sys/fs/cgroup:ro bigtop-manager/develop:trunk-centos-7 /usr/sbin/init -docker cp ../../../bigtop-manager-server/target/bigtop-manager-server bigtop-manager-server:/opt/ -docker cp ../../../bigtop-manager-agent/target/bigtop-manager-agent bigtop-manager-server:/opt/ -SERVER_PUB_KEY=`docker exec bigtop-manager-server /bin/cat /root/.ssh/id_rsa.pub` -docker exec bigtop-manager-server bash -c "echo '$SERVER_PUB_KEY' > /root/.ssh/authorized_keys" -docker exec bigtop-manager-server /bin/systemctl enable sshd -docker exec bigtop-manager-server /bin/systemctl start sshd - -echo -e "\033[32mSetting up mariadb-server\033[0m" -docker exec bigtop-manager-server /bin/systemctl enable mariadb -docker exec bigtop-manager-server /bin/systemctl start mariadb -docker exec bigtop-manager-server bash -c "mysql -e \"UPDATE mysql.user SET Password = PASSWORD('root') WHERE User = 'root'\"" -docker exec bigtop-manager-server bash -c "mysql -e \"GRANT ALL PRIVILEGES ON *.* TO 'root'@'%' IDENTIFIED BY 'root' WITH GRANT OPTION\"" -docker exec bigtop-manager-server bash -c "mysql -e \"DROP USER ''@'localhost'\"" -docker exec bigtop-manager-server bash -c "mysql -e \"DROP DATABASE test\"" -docker exec bigtop-manager-server bash -c "mysql -e \"CREATE DATABASE bigtop_manager\"" - -docker exec bigtop-manager-server bash -c "mysql -e \"FLUSH PRIVILEGES\"" - -echo -e "\033[32mCreating container bigtop-manager-agent-01\033[0m" -docker run -d --name bigtop-manager-agent-01 --hostname bigtop-manager-agent-01 --network bigtop-manager --privileged -e "container=docker" -v /sys/fs/cgroup:/sys/fs/cgroup:ro bigtop-manager/develop:trunk-centos-7 /usr/sbin/init -docker cp ../../../bigtop-manager-agent/target/bigtop-manager-agent bigtop-manager-agent-01:/opt/ -docker exec bigtop-manager-agent-01 bash -c "echo '$SERVER_PUB_KEY' > /root/.ssh/authorized_keys" -docker exec bigtop-manager-agent-01 /bin/systemctl enable sshd -docker exec bigtop-manager-agent-01 /bin/systemctl start sshd - -echo -e "\033[32mCreating container bigtop-manager-agent-02\033[0m" -docker run -d --name bigtop-manager-agent-02 --hostname bigtop-manager-agent-02 --network bigtop-manager --privileged -e "container=docker" -v /sys/fs/cgroup:/sys/fs/cgroup:ro bigtop-manager/develop:trunk-centos-7 /usr/sbin/init -docker cp ../../../bigtop-manager-agent/target/bigtop-manager-agent bigtop-manager-agent-02:/opt/ -docker exec bigtop-manager-agent-02 bash -c "echo '$SERVER_PUB_KEY' > /root/.ssh/authorized_keys" -docker exec bigtop-manager-agent-02 /bin/systemctl enable sshd -docker exec bigtop-manager-agent-02 /bin/systemctl start sshd - -echo -e "\033[32mConfiguring hosts file\033[0m" -BIGTOP_MANAGER_SERVER_IP=`docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' bigtop-manager-server` -BIGTOP_MANAGER_AGENT_01_IP=`docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' bigtop-manager-agent-01` -BIGTOP_MANAGER_AGENT_02_IP=`docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' bigtop-manager-agent-02` -docker exec bigtop-manager-server bash -c "echo '$BIGTOP_MANAGER_AGENT_01_IP bigtop-manager-agent-01' >> /etc/hosts" -docker exec bigtop-manager-server bash -c "echo '$BIGTOP_MANAGER_AGENT_02_IP bigtop-manager-agent-02' >> /etc/hosts" -docker exec bigtop-manager-agent-01 bash -c "echo '$BIGTOP_MANAGER_SERVER_IP bigtop-manager-server' >> /etc/hosts" -docker exec bigtop-manager-agent-01 bash -c "echo '$BIGTOP_MANAGER_AGENT_02_IP bigtop-manager-agent-02' >> /etc/hosts" -docker exec bigtop-manager-agent-02 bash -c "echo '$BIGTOP_MANAGER_SERVER_IP bigtop-manager-server' >> /etc/hosts" -docker exec bigtop-manager-agent-02 bash -c "echo '$BIGTOP_MANAGER_AGENT_01_IP bigtop-manager-agent-01' >> /etc/hosts" - -echo -e "\033[32mSynchronize Chrony\033[0m" -docker exec bigtop-manager-server bash -c "systemctl enable chronyd; systemctl start chronyd; chronyc tracking" -docker exec bigtop-manager-agent-01 bash -c "systemctl enable chronyd; systemctl start chronyd; chronyc tracking" -docker exec bigtop-manager-agent-02 bash -c "systemctl enable chronyd; systemctl start chronyd; chronyc tracking" - -echo -e "\033[32mServer Ip Setting\033[0m" -docker exec bigtop-manager-server bash -c "sed -i 's/bigtop.manager.server.host=localhost/bigtop.manager.server.host=$BIGTOP_MANAGER_SERVER_IP/' /opt/bigtop-manager-agent/conf/application.yml" -docker exec bigtop-manager-agent-01 bash -c "sed -i 's/bigtop.manager.server.host=localhost/bigtop.manager.server.host=$BIGTOP_MANAGER_SERVER_IP/' /opt/bigtop-manager-agent/conf/application.yml" -docker exec bigtop-manager-agent-02 bash -c "sed -i 's/bigtop.manager.server.host=localhost/bigtop.manager.server.host=$BIGTOP_MANAGER_SERVER_IP/' /opt/bigtop-manager-agent/conf/application.yml" - -docker exec bigtop-manager-server bash -c "nohup /bin/bash /opt/bigtop-manager-server/bin/start.sh > /dev/null 2>&1 &" -docker exec bigtop-manager-server bash -c "nohup /bin/bash /opt/bigtop-manager-agent/bin/start.sh > /dev/null 2>&1 &" -docker exec bigtop-manager-agent-01 bash -c "nohup /bin/bash /opt/bigtop-manager-agent/bin/start.sh > /dev/null 2>&1 &" -docker exec bigtop-manager-agent-02 bash -c "nohup /bin/bash /opt/bigtop-manager-agent/bin/start.sh > /dev/null 2>&1 &" - -echo -e "\033[32mPrint Bigtop-Manager Server RSA Private Key\033[0m" -docker exec bigtop-manager-server bash -c "cat ~/.ssh/id_rsa" - -# MySQL HOST: bigtop-manager-server -# MySQL PORT: 3306 -# DATABASE NAME: bigtop_manager -# DATABASE USER NAME: root -# DATABASE PASSWORD: root - diff --git a/dev-support/docker/centos7/build-image.sh b/dev-support/docker/centos7/build-image.sh deleted file mode 100755 index 0871b46d3..000000000 --- a/dev-support/docker/centos7/build-image.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/bash - -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -BIN_DIR=$(dirname $0) -cd $BIN_DIR -echo $PWD - -echo -e "\033[32mRemoving image bigtop-manager:trunk-centos-7\033[0m" -docker rmi bigtop-manager/develop:trunk-centos-7 - -echo -e "\033[32mBuilding image bigtop-manager:trunk-centos-7\033[0m" -docker build -t bigtop-manager/develop:trunk-centos-7 . diff --git a/dev-support/docker/centos7/clear-containers.sh b/dev-support/docker/centos7/clear-containers.sh deleted file mode 100755 index 65f3d7f94..000000000 --- a/dev-support/docker/centos7/clear-containers.sh +++ /dev/null @@ -1,38 +0,0 @@ -#!/bin/bash - -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -echo -e "\033[32mStopping container bigtop-manager-build and maven process\033[0m" -if [ `docker inspect --format '{{.State.Running}}' bigtop-manager-build` == true ];then - docker exec bigtop-manager-build bash -c "pkill -KILL -f maven" - docker stop bigtop-manager-build -fi - -echo -e "\033[32mRemoving container bigtop-manager-server\033[0m" -docker rm -f bigtop-manager-server - -echo -e "\033[32mRemoving container bigtop-manager-agent-01\033[0m" -docker rm -f bigtop-manager-agent-01 - -echo -e "\033[32mRemoving container bigtop-manager-agent-02\033[0m" -docker rm -f bigtop-manager-agent-02 - -echo -e "\033[32mRemoving network bigtop-manager\033[0m" -docker network rm bigtop-manager \ No newline at end of file diff --git a/dev-support/docker/centos7/distribute-scripts.sh b/dev-support/docker/centos7/distribute-scripts.sh deleted file mode 100755 index e35c10534..000000000 --- a/dev-support/docker/centos7/distribute-scripts.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/bin/bash - -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -echo -e "\033[32mSynchronizing script to bigtop-manager-server\033[0m" -docker cp ../../../bigtop-manager-server/src/main/resources/stacks/ bigtop-manager-server:/opt/bigtop-manager-server/ - -docker exec bigtop-manager-server bash -c "ps -ef | grep 'org.apache.bigtop.manager.server.ServerApplication' | grep -v 'grep' | awk '{print $2}' | xargs kill" -docker exec bigtop-manager-server bash -c "ps -ef | grep 'org.apache.bigtop.manager.agent.AgentApplication' | grep -v 'grep' | awk '{print $2}' | xargs kill" -docker exec bigtop-manager-agent-01 bash -c "ps -ef | grep 'org.apache.bigtop.manager.agent.AgentApplication' | grep -v 'grep' | awk '{print $2}' | xargs kill" -docker exec bigtop-manager-agent-02 bash -c "ps -ef | grep 'org.apache.bigtop.manager.agent.AgentApplication' | grep -v 'grep' | awk '{print $2}' | xargs kill" - -docker exec bigtop-manager-server bash -c "nohup /bin/bash /opt/bigtop-manager-server/bin/start.sh > /dev/null 2>&1 &" -docker exec bigtop-manager-server bash -c "nohup /bin/bash /opt/bigtop-manager-agent/bin/start.sh > /dev/null 2>&1 &" -docker exec bigtop-manager-agent-01 bash -c "nohup /bin/bash /opt/bigtop-manager-agent/bin/start.sh > /dev/null 2>&1 &" -docker exec bigtop-manager-agent-02 bash -c "nohup /bin/bash /opt/bigtop-manager-agent/bin/start.sh > /dev/null 2>&1 &" - -echo -e "\033[32mDone!\033[0m" \ No newline at end of file diff --git a/dev-support/docker/containers/build.sh b/dev-support/docker/containers/build.sh new file mode 100644 index 000000000..afe8090d4 --- /dev/null +++ b/dev-support/docker/containers/build.sh @@ -0,0 +1,226 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +usage() { + echo "usage: $PROG args" + echo " commands:" + echo " -c NUM_INSTANCES, --create NUM_INSTANCES - Create Docker containers based bigtop-manager cluster, defaults to 3" + echo " -e, --database - The specified database, defaults to postgres" + echo " -o, --os - Specify the operating system, default is trunk-rocky-8" + echo " --skip-compile - Skip Compile" + echo " -d, --destroy - Destroy all containers" + echo " -h, --help" + exit 1 +} + +log() { + echo -e "\033[32m[LOG] $1\033[0m" +} + +build() { + log "Build on docker $SKIP_BUILD" + if ! $SKIP_BUILD; then + log "Compiling bigtop-manager" + docker run -it --rm -u $(id -u):$(id -g) \ + -v $PWD/../../../:/opt/develop/bigtop-manager/ \ + -v /$USER/.m2:/$USER/.m2 \ + -w /opt/develop/bigtop-manager \ + bigtop-manager/develop:${OS} bash -c "mvn clean package -DskipTests" + else + log "Skip Compile!!!" + fi + log "Build Success!!!" +} + +destroy() { + log "Destroy Containers!!!" + docker rm -f $(docker network inspect bigtop-manager -f '{{range .Containers}}{{.Name}}{{" "}}{{end}}') + docker network rm bigtop-manager + exit 0 +} + +create() { + log "Create Containers!!!" + docker network create --driver bridge bigtop-manager + create_db + create_container +} + +create_container() { + for ((i=1;i<=$NUM_INSTANCES;i+=1)) + do + container_name="bm-$i" + log "Create ${container_name}" + if [ $i -eq 1 ]; then + docker run -itd -p 15005:5005 -p 15006:5006 -p 18080:8080 --name ${container_name} --hostname ${container_name} --network bigtop-manager --cap-add=SYS_TIME bigtop-manager/develop:${OS} + docker cp ../../../bigtop-manager-server/target/bigtop-manager-server ${container_name}:/opt/ + SERVER_PUB_KEY=`docker exec ${container_name} /bin/cat /root/.ssh/id_rsa.pub` + else + docker run -itd --name ${container_name} --hostname ${container_name} --network bigtop-manager --cap-add=SYS_TIME bigtop-manager/develop:${OS} + fi + docker cp ../../../bigtop-manager-agent/target/bigtop-manager-agent ${container_name}:/opt/ + docker exec ${container_name} bash -c "echo '$SERVER_PUB_KEY' > /root/.ssh/authorized_keys" + docker exec ${container_name} ssh-keygen -N '' -t rsa -b 2048 -f /etc/ssh/ssh_host_rsa_key + docker exec ${container_name} ssh-keygen -N '' -t ecdsa -b 256 -f /etc/ssh/ssh_host_ecdsa_key + docker exec ${container_name} ssh-keygen -N '' -t ed25519 -b 256 -f /etc/ssh/ssh_host_ed25519_key + docker exec ${container_name} /bin/systemctl start sshd + + docker exec ${container_name} bash -c "systemctl start chronyd && chronyc tracking" + done + + containers=($(docker network inspect bigtop-manager -f '{{range .Containers}}{{.Name}}{{" "}}{{end}}')) + for container in ${containers[@]}; do + container_ip=$(docker inspect --format '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' $container) + log "container: ${container}; container_ip: ${container_ip}" + for container2 in ${containers[@]}; do + if [ ${container2} != $container ]; then + docker exec ${container2} bash -c "echo '${container_ip} ${container}' >> /etc/hosts" + fi + done + done + + # wait database started + for container in ${containers[@]}; do + if [ ${container} == "bm-1" ]; then + if [ $DATABASE == "mysql" ]; then + log "docker exec ${container} bash -c \"mysql -h bm-mysql -P 3306 -uroot -proot -e 'create database bigtop_manager'\"" + docker exec ${container} bash -c "mysql -h bm-mysql -P 3306 -uroot -proot -e 'create database bigtop_manager'" + docker exec ${container} bash -c "mysql -h bm-mysql -P 3306 -uroot -proot -Dbigtop_manager < /opt/bigtop-manager-server/ddl/MySQL-DDL-CREATE.sql" + + docker exec ${container} bash -c "wget https://repo1.maven.org/maven2/com/mysql/mysql-connector-j/8.0.33/mysql-connector-j-8.0.33.jar -O /opt/bigtop-manager-server/libs/mysql-connector-java-8.0.33.jar" + docker exec ${container} bash -c "sed -i 's/org.postgresql.Driver/com.mysql.cj.jdbc.Driver/' /opt/bigtop-manager-server/conf/application.yml" + docker exec ${container} bash -c "sed -i 's/postgresql/mysql/' /opt/bigtop-manager-server/conf/application.yml" + docker exec ${container} bash -c "sed -i 's/localhost:5432/bm-mysql:3306/' /opt/bigtop-manager-server/conf/application.yml" + docker exec ${container} bash -c "sed -i 's/postgres/root/' /opt/bigtop-manager-server/conf/application.yml" + elif [ $DATABASE == "postgres" ]; then + docker exec ${container} bash -c "yum install postgresql -y" + docker exec ${container} bash -c "PGPASSWORD=postgres psql -h bm-postgres -p5432 -U postgres -c 'create database bigtop_manager'" + docker exec ${container} bash -c "PGPASSWORD=postgres psql -h bm-postgres -p5432 -U postgres -d bigtop_manager -f /opt/bigtop-manager-server/ddl/PostgreSQL-DDL-CREATE.sql" + docker exec ${container} bash -c "sed -i 's/localhost:5432/bm-postgres:5432/' /opt/bigtop-manager-server/conf/application.yml" + fi + docker exec ${container} bash -c "nohup /bin/bash /opt/bigtop-manager-server/bin/start.sh --debug > /dev/null 2>&1 &" + fi + docker exec ${container} bash -c "nohup /bin/bash /opt/bigtop-manager-agent/bin/start.sh --debug > /dev/null 2>&1 &" + log "All Service Started!!!" + done + +} + +create_db() { + if [ $DATABASE == "mysql" ]; then + docker run --restart=always -it -d \ + -p 3306:3306 \ + --cap-add=SYS_TIME \ + --network bigtop-manager \ + --name bm-mysql \ + --hostname bm-mysql \ + -e MYSQL_ROOT_PASSWORD=root \ + mysql:8.0 + # Loop check log + while true; do + # Use the Docker logs command to retrieve the latest log content + logs=$(docker logs bm-mysql | tail -n 100) + log "$logs" + + # Check if the log contains specific strings + if echo "$logs" | grep -q "/usr/sbin/mysqld: ready for connections"; then + echo "MySQL is ready for connections." + break + else + echo "MySQL is not ready yet, waiting..." + # Wait for a while and check again + sleep 5 + fi + done + elif [ $DATABASE == "postgres" ]; then + docker run --restart=always -d \ + -p 5432:5432 \ + --network bigtop-manager \ + --name bm-postgres \ + --hostname bm-postgres \ + -e POSTGRES_PASSWORD=postgres \ + postgres:16 + fi +} + + +BIN_DIR=$(dirname $0) +cd $BIN_DIR +echo $PWD + +PROG=`basename $0` + +DATABASE=postgres +OS=trunk-rocky-8 +NUM_INSTANCES=3 +SKIP_BUILD=false + +while [ $# -gt 0 ]; do + case "$1" in + -e|--database) + if [ $# -lt 2 ]; then + echo "Requires a db" 1>&2 + usage + fi + if [ $2 != "postgres" ] && [ $2 != "mysql" ]; then + echo "The Database should be [postgres], or [mysql]" 1>&2 + usage + fi + DATABASE=$2 + shift 2;; + -o|--os) + if [ $# -lt 2 ]; then + echo "Requires a os" 1>&2 + usage + fi + if [ $2 != "trunk-rocky-8" ] && [ $2 != "trunk-centos-7" ] && [ $2 != "trunk-openeuler-22" ]; then + echo "The OS should be [trunk-rocky-8], [trunk-centos-7], or [trunk-openeuler-22]" 1>&2 + usage + fi + OS=$2 + shift 2;; + -c|--create) + if [ $# -lt 2 ]; then + echo "Requires a container number" 1>&2 + usage + fi + if [ $2 -gt 10 ] || [ $2 -lt 3 ]; then + echo "NUM-INSTANCES should be between [3-10]" 1>&2 + usage + fi + NUM_INSTANCES=$2 + shift 2;; + --skip-compile) + SKIP_BUILD=true + shift;; + -d|--destroy) + destroy + shift;; + -h|--help) + usage + shift;; + *) + echo "Unknown argument: '$1'" 1>&2 + usage;; + esac +done + +log "DATABASE: $DATABASE; OS: $OS; NUM_INSTANCES: $NUM_INSTANCES; SKIP_BUILD: $SKIP_BUILD; " + +build + +create diff --git a/dev-support/docker/rocky8/Dockerfile b/dev-support/docker/image/Dockerfile.template similarity index 97% rename from dev-support/docker/rocky8/Dockerfile rename to dev-support/docker/image/Dockerfile.template index 1e12fe29b..1b3659489 100644 --- a/dev-support/docker/rocky8/Dockerfile +++ b/dev-support/docker/image/Dockerfile.template @@ -13,7 +13,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM rockylinux:8 +FROM OS:VERSION + +MAINTAINER dev@bigtop.apache.org RUN yum -y install sudo wget openssh-clients openssh-server vim mariadb mariadb-server net-tools chrony krb5-server krb5-libs krb5-workstation git rpm-build python3 procps-ng RUN wget https://download.java.net/java/GA/jdk17.0.2/dfd4a8d0985749f896bed50d7138ee7f/8/GPL/openjdk-17.0.2_linux-x64_bin.tar.gz -O /tmp/jdk.tar.gz --no-check-certificate \ @@ -29,5 +31,3 @@ RUN wget https://raw.githubusercontent.com/gdraheim/docker-systemctl-replacement RUN /bin/sed -i 's,# StrictHostKeyChecking ask,StrictHostKeyChecking no,g' /etc/ssh/ssh_config RUN ssh-keygen -f "/root/.ssh/id_rsa" -N "" - -EXPOSE 1-65535 diff --git a/dev-support/docker/image/build.sh b/dev-support/docker/image/build.sh new file mode 100644 index 000000000..cc0d380d9 --- /dev/null +++ b/dev-support/docker/image/build.sh @@ -0,0 +1,78 @@ +#!/bin/sh + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex + +log() { + echo -e "\n[LOG] $1\n" +} + +exception() { + echo "[ERROR] Unsupported [${OS}-${VERSION}]" + exit 1 +} + +BIN_DIR=$(dirname $0) +cd $BIN_DIR +echo $PWD + +if [ $# != 1 ]; then + echo "Creates bigtop-manager/develop image" + echo + echo "Usage: build.sh " + echo + echo "Example: build.sh trunk-rocky-8" + echo " : build.sh 1.0.0-rocky-8" + exit 1 +fi + +PREFIX=$(echo "$1" | cut -d '-' -f 1) +OS=$(echo "$1" | cut -d '-' -f 2) +VERSION=$(echo "$1" | cut -d '-' -f 3-) + + +# Decimals are not supported. Either use integers only +# e.g. 16.04 -> 16 +VERSION_INT=$(echo "$VERSION" | cut -d '.' -f 1) + +log "PREFIX: ${PREFIX}; OS: ${OS}; VERSION: ${VERSION}; VERSION_INT: ${VERSION_INT}" +DOCKER_OS=${OS} +DOCKER_VERSION=${VERSION} +CUSTOM_REPO="" +case ${OS}-${VERSION_INT} in + centos-7) + CUSTOM_REPO="RUN mv /etc/yum.repos.d/CentOS-Base.repo /etc/yum.repos.d/CentOS-Base.repo.back \ + && curl -o /etc/yum.repos.d/CentOS-Base.repo http://mirrors.aliyun.com/repo/Centos-7.repo" + ;; + rocky-8) + DOCKER_OS=rockylinux + ;; + openeuler-22) + DOCKER_OS=openeuler/openeuler + DOCKER_VERSION=22.03 + ;; + *) + echo "Unsupported OS ${OS}-${VERSION}." + exit 1 +esac + +# generate Dockerfile for build +sed -e "s|OS|${DOCKER_OS}|;s|VERSION|${DOCKER_VERSION}|" Dockerfile.template | + sed -e "/MAINTAINER dev@bigtop.apache.org/a\\$CUSTOM_REPO" > Dockerfile + +docker build --rm --no-cache -t bigtop-manager/develop:${PREFIX}-${OS}-${VERSION} -f Dockerfile ../.. +rm -f Dockerfile diff --git a/dev-support/docker/openeuler22/Dockerfile b/dev-support/docker/openeuler22/Dockerfile deleted file mode 100644 index 631f5eaba..000000000 --- a/dev-support/docker/openeuler22/Dockerfile +++ /dev/null @@ -1,33 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -FROM openeuler/openeuler:22.03 - -RUN dnf -y install sudo wget openssh-clients openssh-server vim mariadb mariadb-server net-tools chrony krb5-server krb5-libs krb5-workstation git rpm-build python3 procps-ng -RUN wget https://download.java.net/java/GA/jdk17.0.2/dfd4a8d0985749f896bed50d7138ee7f/8/GPL/openjdk-17.0.2_linux-x64_bin.tar.gz -O /tmp/jdk.tar.gz --no-check-certificate \ - && mkdir -p /usr/local/java && tar -xzf /tmp/jdk.tar.gz -C /usr/local/java --strip-components=1 \ - && rm -f /tmp/jdk.tar.gz \ - && ln -s /usr/local/java/bin/java /usr/bin/java -RUN wget https://archive.apache.org/dist/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz -O /tmp/apache-maven.tar.gz --no-check-certificate \ - && mkdir -p /usr/share/maven && tar -xzf /tmp/apache-maven.tar.gz -C /usr/share/maven --strip-components=1 \ - && rm -f /tmp/apache-maven.tar.gz \ - && ln -s /usr/share/maven/bin/mvn /usr/bin/mvn -RUN wget https://raw.githubusercontent.com/gdraheim/docker-systemctl-replacement/master/files/docker/systemctl3.py -O /usr/bin/systemctl - -RUN /bin/sed -i 's,# StrictHostKeyChecking ask,StrictHostKeyChecking no,g' /etc/ssh/ssh_config - -RUN ssh-keygen -f "/root/.ssh/id_rsa" -N "" - -EXPOSE 1-65535 diff --git a/dev-support/docker/openeuler22/build-containers.sh b/dev-support/docker/openeuler22/build-containers.sh deleted file mode 100755 index 23844dd13..000000000 --- a/dev-support/docker/openeuler22/build-containers.sh +++ /dev/null @@ -1,133 +0,0 @@ -#!/bin/bash - -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -BIN_DIR=$(dirname $0) -cd $BIN_DIR -echo $PWD - -SKIP_BUILD=false - -for arg in "$@" -do - if [ "$arg" == "--skip-build" ]; then - SKIP_BUILD=true - fi -done - -if ! $SKIP_BUILD; then - echo -e "\033[32mBuild on docker\033[0m" - echo -e "\033[32mStarting container bigtop-manager-build-oe22\033[0m" - if [[ -z $(docker ps -a --format "table {{.Names}}" | grep "bigtop-manager-build-oe22") ]];then - docker run -it -d --name bigtop-manager-build-oe22 -u $(id -u):$(id -g) -v $PWD/../../../:/opt/develop/bigtop-manager/ -w /opt/develop/bigtop-manager bigtop-manager/develop:trunk-openeuler-22 - else - docker start bigtop-manager-build-oe22 - fi - - echo -e "\033[32mCompiling bigtop-manager\033[0m" - docker exec bigtop-manager-build-oe22 bash -c "mvn clean package -DskipTests" - docker stop bigtop-manager-build-oe22 -fi - -echo -e "\033[32mCreating network bigtop-manager\033[0m" -docker network create --driver bridge bigtop-manager - -echo -e "\033[32mCreating container bigtop-manager-server\033[0m" -docker run -it -d -p 13306:3306 -p 15005:5005 -p 15006:5006 -p 18080:8080 --name bigtop-manager-server --hostname bigtop-manager-server --network bigtop-manager --cap-add=SYS_TIME bigtop-manager/develop:trunk-openeuler-22 -docker cp ../../../bigtop-manager-server/target/bigtop-manager-server bigtop-manager-server:/opt/ -docker cp ../../../bigtop-manager-agent/target/bigtop-manager-agent bigtop-manager-server:/opt/ -SERVER_PUB_KEY=`docker exec bigtop-manager-server /bin/cat /root/.ssh/id_rsa.pub` -docker exec bigtop-manager-server bash -c "echo '$SERVER_PUB_KEY' > /root/.ssh/authorized_keys" -docker exec bigtop-manager-server ssh-keygen -N '' -t rsa -b 2048 -f /etc/ssh/ssh_host_rsa_key -docker exec bigtop-manager-server ssh-keygen -N '' -t ecdsa -b 256 -f /etc/ssh/ssh_host_ecdsa_key -docker exec bigtop-manager-server ssh-keygen -N '' -t ed25519 -b 256 -f /etc/ssh/ssh_host_ed25519_key -docker exec bigtop-manager-server /bin/systemctl start sshd - -echo -e "\033[32mSetting up mariadb-server\033[0m" -docker exec bigtop-manager-server bash -c "systemctl start mariadb" -docker exec bigtop-manager-server bash -c "mysql -e \"UPDATE mysql.user SET Password = PASSWORD('root') WHERE User = 'root'\"" -docker exec bigtop-manager-server bash -c "mysql -e \"GRANT ALL PRIVILEGES ON *.* TO 'root'@'%' IDENTIFIED BY 'root' WITH GRANT OPTION\"" -docker exec bigtop-manager-server bash -c "mysql -e \"CREATE DATABASE bigtop_manager\"" -docker exec bigtop-manager-server bash -c "mysql -e \"FLUSH PRIVILEGES\"" -docker exec bigtop-manager-server bash -c "mysql -e \"DROP USER 'root'@'localhost'\"" - -echo -e "\033[32mCreating container bigtop-manager-agent-01\033[0m" -docker run -it -d --name bigtop-manager-agent-01 --hostname bigtop-manager-agent-01 --network bigtop-manager --cap-add=SYS_TIME bigtop-manager/develop:trunk-openeuler-22 -docker cp ../../../bigtop-manager-agent/target/bigtop-manager-agent bigtop-manager-agent-01:/opt/ -docker exec bigtop-manager-agent-01 bash -c "echo '$SERVER_PUB_KEY' > /root/.ssh/authorized_keys" -docker exec bigtop-manager-agent-01 ssh-keygen -N '' -t rsa -b 2048 -f /etc/ssh/ssh_host_rsa_key -docker exec bigtop-manager-agent-01 ssh-keygen -N '' -t ecdsa -b 256 -f /etc/ssh/ssh_host_ecdsa_key -docker exec bigtop-manager-agent-01 ssh-keygen -N '' -t ed25519 -b 256 -f /etc/ssh/ssh_host_ed25519_key -docker exec bigtop-manager-agent-01 /bin/systemctl start sshd - -echo -e "\033[32mCreating container bigtop-manager-agent-02\033[0m" -docker run -it -d --name bigtop-manager-agent-02 --hostname bigtop-manager-agent-02 --network bigtop-manager --cap-add=SYS_TIME bigtop-manager/develop:trunk-openeuler-22 -docker cp ../../../bigtop-manager-agent/target/bigtop-manager-agent bigtop-manager-agent-02:/opt/ -docker exec bigtop-manager-agent-02 bash -c "echo '$SERVER_PUB_KEY' > /root/.ssh/authorized_keys" -docker exec bigtop-manager-agent-02 ssh-keygen -N '' -t rsa -b 2048 -f /etc/ssh/ssh_host_rsa_key -docker exec bigtop-manager-agent-02 ssh-keygen -N '' -t ecdsa -b 256 -f /etc/ssh/ssh_host_ecdsa_key -docker exec bigtop-manager-agent-02 ssh-keygen -N '' -t ed25519 -b 256 -f /etc/ssh/ssh_host_ed25519_key -docker exec bigtop-manager-agent-02 /bin/systemctl start sshd - -echo -e "\033[32mConfiguring hosts file\033[0m" -BIGTOP_MANAGER_SERVER_IP=`docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' bigtop-manager-server` -BIGTOP_MANAGER_AGENT_01_IP=`docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' bigtop-manager-agent-01` -BIGTOP_MANAGER_AGENT_02_IP=`docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' bigtop-manager-agent-02` -docker exec bigtop-manager-server bash -c "echo '$BIGTOP_MANAGER_AGENT_01_IP bigtop-manager-agent-01' >> /etc/hosts" -docker exec bigtop-manager-server bash -c "echo '$BIGTOP_MANAGER_AGENT_02_IP bigtop-manager-agent-02' >> /etc/hosts" -docker exec bigtop-manager-agent-01 bash -c "echo '$BIGTOP_MANAGER_SERVER_IP bigtop-manager-server' >> /etc/hosts" -docker exec bigtop-manager-agent-01 bash -c "echo '$BIGTOP_MANAGER_AGENT_02_IP bigtop-manager-agent-02' >> /etc/hosts" -docker exec bigtop-manager-agent-02 bash -c "echo '$BIGTOP_MANAGER_SERVER_IP bigtop-manager-server' >> /etc/hosts" -docker exec bigtop-manager-agent-02 bash -c "echo '$BIGTOP_MANAGER_AGENT_01_IP bigtop-manager-agent-01' >> /etc/hosts" - -echo -e "\033[32mSynchronize Chrony\033[0m" -docker exec bigtop-manager-server bash -c "systemctl start chronyd" -docker exec bigtop-manager-server bash -c "chronyc tracking" -docker exec bigtop-manager-agent-01 bash -c "systemctl start chronyd" -docker exec bigtop-manager-agent-01 bash -c "chronyc tracking" -docker exec bigtop-manager-agent-02 bash -c "systemctl start chronyd" -docker exec bigtop-manager-agent-02 bash -c "chronyc tracking" - -echo -e "\033[32mServer Settings\033[0m" -docker exec bigtop-manager-server bash -c "wget https://repo1.maven.org/maven2/com/mysql/mysql-connector-j/8.0.33/mysql-connector-j-8.0.33.jar -O /opt/bigtop-manager-server/libs/mysql-connector-java-8.0.33.jar" -docker exec bigtop-manager-server bash -c "sed -i 's/org.postgresql.Driver/com.mysql.cj.jdbc.Driver/' /opt/bigtop-manager-server/conf/application.yml" -docker exec bigtop-manager-server bash -c "sed -i 's/postgresql/mysql/' /opt/bigtop-manager-server/conf/application.yml" -docker exec bigtop-manager-server bash -c "sed -i 's/localhost:5432/localhost:3306/' /opt/bigtop-manager-server/conf/application.yml" -docker exec bigtop-manager-server bash -c "sed -i 's/postgres/root/' /opt/bigtop-manager-server/conf/application.yml" - -docker exec bigtop-manager-server bash -c "sed -i 's/host: localhost/host: $BIGTOP_MANAGER_SERVER_IP/' /opt/bigtop-manager-agent/conf/application.yml" -docker exec bigtop-manager-agent-01 bash -c "sed -i 's/host: localhost/host: $BIGTOP_MANAGER_SERVER_IP/' /opt/bigtop-manager-agent/conf/application.yml" -docker exec bigtop-manager-agent-02 bash -c "sed -i 's/host: localhost/host: $BIGTOP_MANAGER_SERVER_IP/' /opt/bigtop-manager-agent/conf/application.yml" - -docker exec bigtop-manager-server bash -c "mysql -uroot -proot -Dbigtop_manager < /opt/bigtop-manager-server/ddl/MySQL-DDL-CREATE.sql" -docker exec bigtop-manager-server bash -c "nohup /bin/bash /opt/bigtop-manager-server/bin/start.sh --debug > /dev/null 2>&1 &" -docker exec bigtop-manager-server bash -c "nohup /bin/bash /opt/bigtop-manager-agent/bin/start.sh --debug > /dev/null 2>&1 &" -docker exec bigtop-manager-agent-01 bash -c "nohup /bin/bash /opt/bigtop-manager-agent/bin/start.sh > /dev/null 2>&1 &" -docker exec bigtop-manager-agent-02 bash -c "nohup /bin/bash /opt/bigtop-manager-agent/bin/start.sh > /dev/null 2>&1 &" - -echo -e "\033[32mPrint Bigtop-Manager Server RSA Private Key\033[0m" -docker exec bigtop-manager-server bash -c "cat ~/.ssh/id_rsa" - -# MySQL HOST: bigtop-manager-server -# MySQL PORT: 3306 -# DATABASE NAME: bigtop_manager -# DATABASE USER NAME: root -# DATABASE PASSWORD: root - diff --git a/dev-support/docker/openeuler22/build-image.sh b/dev-support/docker/openeuler22/build-image.sh deleted file mode 100755 index 5694daff8..000000000 --- a/dev-support/docker/openeuler22/build-image.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/bash - -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -BIN_DIR=$(dirname $0) -cd $BIN_DIR -echo $PWD - -echo -e "\033[32mRemoving image bigtop-manager:trunk-openeuler-22\033[0m" -docker rmi bigtop-manager/develop:trunk-openeuler-22 - -echo -e "\033[32mBuilding image bigtop-manager:trunk-openeuler-22\033[0m" -docker build -t bigtop-manager/develop:trunk-openeuler-22 . diff --git a/dev-support/docker/openeuler22/clear-containers.sh b/dev-support/docker/openeuler22/clear-containers.sh deleted file mode 100755 index 896296045..000000000 --- a/dev-support/docker/openeuler22/clear-containers.sh +++ /dev/null @@ -1,40 +0,0 @@ -#!/bin/bash - -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -if docker ps -a | grep -q 'bigtop-manager-build-oe22'; then - echo -e "\033[32mStopping container bigtop-manager-build-oe22 and maven process\033[0m" - if [ `docker inspect --format '{{.State.Running}}' bigtop-manager-build-oe22` == true ];then - docker exec bigtop-manager-build-oe22 bash -c "pkill -KILL -f maven" - docker stop bigtop-manager-build-oe22 - fi -fi - -echo -e "\033[32mRemoving container bigtop-manager-server\033[0m" -docker rm -f bigtop-manager-server - -echo -e "\033[32mRemoving container bigtop-manager-agent-01\033[0m" -docker rm -f bigtop-manager-agent-01 - -echo -e "\033[32mRemoving container bigtop-manager-agent-02\033[0m" -docker rm -f bigtop-manager-agent-02 - -echo -e "\033[32mRemoving network bigtop-manager\033[0m" -docker network rm bigtop-manager \ No newline at end of file diff --git a/dev-support/docker/openeuler22/distribute-scripts.sh b/dev-support/docker/openeuler22/distribute-scripts.sh deleted file mode 100755 index e35c10534..000000000 --- a/dev-support/docker/openeuler22/distribute-scripts.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/bin/bash - -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -echo -e "\033[32mSynchronizing script to bigtop-manager-server\033[0m" -docker cp ../../../bigtop-manager-server/src/main/resources/stacks/ bigtop-manager-server:/opt/bigtop-manager-server/ - -docker exec bigtop-manager-server bash -c "ps -ef | grep 'org.apache.bigtop.manager.server.ServerApplication' | grep -v 'grep' | awk '{print $2}' | xargs kill" -docker exec bigtop-manager-server bash -c "ps -ef | grep 'org.apache.bigtop.manager.agent.AgentApplication' | grep -v 'grep' | awk '{print $2}' | xargs kill" -docker exec bigtop-manager-agent-01 bash -c "ps -ef | grep 'org.apache.bigtop.manager.agent.AgentApplication' | grep -v 'grep' | awk '{print $2}' | xargs kill" -docker exec bigtop-manager-agent-02 bash -c "ps -ef | grep 'org.apache.bigtop.manager.agent.AgentApplication' | grep -v 'grep' | awk '{print $2}' | xargs kill" - -docker exec bigtop-manager-server bash -c "nohup /bin/bash /opt/bigtop-manager-server/bin/start.sh > /dev/null 2>&1 &" -docker exec bigtop-manager-server bash -c "nohup /bin/bash /opt/bigtop-manager-agent/bin/start.sh > /dev/null 2>&1 &" -docker exec bigtop-manager-agent-01 bash -c "nohup /bin/bash /opt/bigtop-manager-agent/bin/start.sh > /dev/null 2>&1 &" -docker exec bigtop-manager-agent-02 bash -c "nohup /bin/bash /opt/bigtop-manager-agent/bin/start.sh > /dev/null 2>&1 &" - -echo -e "\033[32mDone!\033[0m" \ No newline at end of file diff --git a/dev-support/docker/openeuler22/restart-containers.sh b/dev-support/docker/openeuler22/restart-containers.sh deleted file mode 100755 index 7e0a49784..000000000 --- a/dev-support/docker/openeuler22/restart-containers.sh +++ /dev/null @@ -1,43 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -echo -e "\033[32mRestarting containers\033[0m" -docker restart bigtop-manager-server -docker restart bigtop-manager-agent-01 -docker restart bigtop-manager-agent-02 - -echo -e "\033[32mRe-enabling systemctl servers\033[0m" -docker exec bigtop-manager-server bash -c "systemctl start mariadb" -docker exec bigtop-manager-server bash -c "systemctl start sshd" -docker exec bigtop-manager-server bash -c "systemctl start chronyd" -docker exec bigtop-manager-agent-01 bash -c "systemctl start sshd" -docker exec bigtop-manager-agent-01 bash -c "systemctl start chronyd" -docker exec bigtop-manager-agent-02 bash -c "systemctl start sshd" -docker exec bigtop-manager-agent-02 bash -c "systemctl start chronyd" - -echo -e "\033[32mSynchronize Chrony\033[0m" -docker exec bigtop-manager-server bash -c "chronyc tracking" -docker exec bigtop-manager-agent-01 bash -c "chronyc tracking" -docker exec bigtop-manager-agent-02 bash -c "chronyc tracking" - -echo -e "\033[32mRestarting bigtop-manager servers\033[0m" -docker exec bigtop-manager-server bash -c "nohup /bin/bash /opt/bigtop-manager-server/bin/start.sh --debug > /dev/null 2>&1 &" -docker exec bigtop-manager-server bash -c "nohup /bin/bash /opt/bigtop-manager-agent/bin/start.sh > /dev/null 2>&1 &" -docker exec bigtop-manager-agent-01 bash -c "nohup /bin/bash /opt/bigtop-manager-agent/bin/start.sh > /dev/null 2>&1 &" -docker exec bigtop-manager-agent-02 bash -c "nohup /bin/bash /opt/bigtop-manager-agent/bin/start.sh > /dev/null 2>&1 &" diff --git a/dev-support/docker/rocky8/build-containers.sh b/dev-support/docker/rocky8/build-containers.sh deleted file mode 100755 index 86f4184eb..000000000 --- a/dev-support/docker/rocky8/build-containers.sh +++ /dev/null @@ -1,132 +0,0 @@ -#!/bin/bash - -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -BIN_DIR=$(dirname $0) -cd $BIN_DIR -echo $PWD - -SKIP_BUILD=false - -for arg in "$@" -do - if [ "$arg" == "--skip-build" ]; then - SKIP_BUILD=true - fi -done - -if ! $SKIP_BUILD; then - echo -e "\033[32mBuild on docker\033[0m" - echo -e "\033[32mStarting container bigtop-manager-build-r8\033[0m" - if [[ -z $(docker ps -a --format "table {{.Names}}" | grep "bigtop-manager-build-r8") ]];then - docker run -it -d --name bigtop-manager-build-r8 -u $(id -u):$(id -g) -v $PWD/../../../:/opt/develop/bigtop-manager/ -w /opt/develop/bigtop-manager bigtop-manager/develop:trunk-rocky-8 - else - docker start bigtop-manager-build-r8 - fi - - echo -e "\033[32mCompiling bigtop-manager\033[0m" - docker exec bigtop-manager-build-r8 bash -c "mvn clean package -DskipTests" - docker stop bigtop-manager-build-r8 -fi - -echo -e "\033[32mCreating network bigtop-manager\033[0m" -docker network create --driver bridge bigtop-manager - -echo -e "\033[32mCreating container bigtop-manager-server\033[0m" -docker run -it -d -p 13306:3306 -p 15005:5005 -p 15006:5006 -p 18080:8080 --name bigtop-manager-server --hostname bigtop-manager-server --network bigtop-manager --cap-add=SYS_TIME bigtop-manager/develop:trunk-rocky-8 -docker cp ../../../bigtop-manager-server/target/bigtop-manager-server bigtop-manager-server:/opt/ -docker cp ../../../bigtop-manager-agent/target/bigtop-manager-agent bigtop-manager-server:/opt/ -SERVER_PUB_KEY=`docker exec bigtop-manager-server /bin/cat /root/.ssh/id_rsa.pub` -docker exec bigtop-manager-server bash -c "echo '$SERVER_PUB_KEY' > /root/.ssh/authorized_keys" -docker exec bigtop-manager-server ssh-keygen -N '' -t rsa -b 2048 -f /etc/ssh/ssh_host_rsa_key -docker exec bigtop-manager-server ssh-keygen -N '' -t ecdsa -b 256 -f /etc/ssh/ssh_host_ecdsa_key -docker exec bigtop-manager-server ssh-keygen -N '' -t ed25519 -b 256 -f /etc/ssh/ssh_host_ed25519_key -docker exec bigtop-manager-server /bin/systemctl start sshd - -echo -e "\033[32mSetting up mariadb-server\033[0m" -docker exec bigtop-manager-server bash -c "systemctl start mariadb" -docker exec bigtop-manager-server bash -c "mysql -e \"UPDATE mysql.user SET Password = PASSWORD('root') WHERE User = 'root'\"" -docker exec bigtop-manager-server bash -c "mysql -e \"GRANT ALL PRIVILEGES ON *.* TO 'root'@'%' IDENTIFIED BY 'root' WITH GRANT OPTION\"" -docker exec bigtop-manager-server bash -c "mysql -e \"CREATE DATABASE bigtop_manager\"" -docker exec bigtop-manager-server bash -c "mysql -e \"FLUSH PRIVILEGES\"" - -echo -e "\033[32mCreating container bigtop-manager-agent-01\033[0m" -docker run -it -d --name bigtop-manager-agent-01 --hostname bigtop-manager-agent-01 --network bigtop-manager --cap-add=SYS_TIME bigtop-manager/develop:trunk-rocky-8 -docker cp ../../../bigtop-manager-agent/target/bigtop-manager-agent bigtop-manager-agent-01:/opt/ -docker exec bigtop-manager-agent-01 bash -c "echo '$SERVER_PUB_KEY' > /root/.ssh/authorized_keys" -docker exec bigtop-manager-agent-01 ssh-keygen -N '' -t rsa -b 2048 -f /etc/ssh/ssh_host_rsa_key -docker exec bigtop-manager-agent-01 ssh-keygen -N '' -t ecdsa -b 256 -f /etc/ssh/ssh_host_ecdsa_key -docker exec bigtop-manager-agent-01 ssh-keygen -N '' -t ed25519 -b 256 -f /etc/ssh/ssh_host_ed25519_key -docker exec bigtop-manager-agent-01 /bin/systemctl start sshd - -echo -e "\033[32mCreating container bigtop-manager-agent-02\033[0m" -docker run -it -d --name bigtop-manager-agent-02 --hostname bigtop-manager-agent-02 --network bigtop-manager --cap-add=SYS_TIME bigtop-manager/develop:trunk-rocky-8 -docker cp ../../../bigtop-manager-agent/target/bigtop-manager-agent bigtop-manager-agent-02:/opt/ -docker exec bigtop-manager-agent-02 bash -c "echo '$SERVER_PUB_KEY' > /root/.ssh/authorized_keys" -docker exec bigtop-manager-agent-02 ssh-keygen -N '' -t rsa -b 2048 -f /etc/ssh/ssh_host_rsa_key -docker exec bigtop-manager-agent-02 ssh-keygen -N '' -t ecdsa -b 256 -f /etc/ssh/ssh_host_ecdsa_key -docker exec bigtop-manager-agent-02 ssh-keygen -N '' -t ed25519 -b 256 -f /etc/ssh/ssh_host_ed25519_key -docker exec bigtop-manager-agent-02 /bin/systemctl start sshd - -echo -e "\033[32mConfiguring hosts file\033[0m" -BIGTOP_MANAGER_SERVER_IP=`docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' bigtop-manager-server` -BIGTOP_MANAGER_AGENT_01_IP=`docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' bigtop-manager-agent-01` -BIGTOP_MANAGER_AGENT_02_IP=`docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' bigtop-manager-agent-02` -docker exec bigtop-manager-server bash -c "echo '$BIGTOP_MANAGER_AGENT_01_IP bigtop-manager-agent-01' >> /etc/hosts" -docker exec bigtop-manager-server bash -c "echo '$BIGTOP_MANAGER_AGENT_02_IP bigtop-manager-agent-02' >> /etc/hosts" -docker exec bigtop-manager-agent-01 bash -c "echo '$BIGTOP_MANAGER_SERVER_IP bigtop-manager-server' >> /etc/hosts" -docker exec bigtop-manager-agent-01 bash -c "echo '$BIGTOP_MANAGER_AGENT_02_IP bigtop-manager-agent-02' >> /etc/hosts" -docker exec bigtop-manager-agent-02 bash -c "echo '$BIGTOP_MANAGER_SERVER_IP bigtop-manager-server' >> /etc/hosts" -docker exec bigtop-manager-agent-02 bash -c "echo '$BIGTOP_MANAGER_AGENT_01_IP bigtop-manager-agent-01' >> /etc/hosts" - -echo -e "\033[32mSynchronize Chrony\033[0m" -docker exec bigtop-manager-server bash -c "systemctl start chronyd" -docker exec bigtop-manager-server bash -c "chronyc tracking" -docker exec bigtop-manager-agent-01 bash -c "systemctl start chronyd" -docker exec bigtop-manager-agent-01 bash -c "chronyc tracking" -docker exec bigtop-manager-agent-02 bash -c "systemctl start chronyd" -docker exec bigtop-manager-agent-02 bash -c "chronyc tracking" - -echo -e "\033[32mServer Settings\033[0m" -docker exec bigtop-manager-server bash -c "wget https://repo1.maven.org/maven2/com/mysql/mysql-connector-j/8.0.33/mysql-connector-j-8.0.33.jar -O /opt/bigtop-manager-server/libs/mysql-connector-java-8.0.33.jar" -docker exec bigtop-manager-server bash -c "sed -i 's/org.postgresql.Driver/com.mysql.cj.jdbc.Driver/' /opt/bigtop-manager-server/conf/application.yml" -docker exec bigtop-manager-server bash -c "sed -i 's/postgresql/mysql/' /opt/bigtop-manager-server/conf/application.yml" -docker exec bigtop-manager-server bash -c "sed -i 's/localhost:5432/localhost:3306/' /opt/bigtop-manager-server/conf/application.yml" -docker exec bigtop-manager-server bash -c "sed -i 's/postgres/root/' /opt/bigtop-manager-server/conf/application.yml" - -docker exec bigtop-manager-server bash -c "sed -i 's/host: localhost/host: $BIGTOP_MANAGER_SERVER_IP/' /opt/bigtop-manager-agent/conf/application.yml" -docker exec bigtop-manager-agent-01 bash -c "sed -i 's/host: localhost/host: $BIGTOP_MANAGER_SERVER_IP/' /opt/bigtop-manager-agent/conf/application.yml" -docker exec bigtop-manager-agent-02 bash -c "sed -i 's/host: localhost/host: $BIGTOP_MANAGER_SERVER_IP/' /opt/bigtop-manager-agent/conf/application.yml" - -docker exec bigtop-manager-server bash -c "mysql -uroot -proot -Dbigtop_manager < /opt/bigtop-manager-server/ddl/MySQL-DDL-CREATE.sql" -docker exec bigtop-manager-server bash -c "nohup /bin/bash /opt/bigtop-manager-server/bin/start.sh --debug > /dev/null 2>&1 &" -docker exec bigtop-manager-server bash -c "nohup /bin/bash /opt/bigtop-manager-agent/bin/start.sh --debug > /dev/null 2>&1 &" -docker exec bigtop-manager-agent-01 bash -c "nohup /bin/bash /opt/bigtop-manager-agent/bin/start.sh > /dev/null 2>&1 &" -docker exec bigtop-manager-agent-02 bash -c "nohup /bin/bash /opt/bigtop-manager-agent/bin/start.sh > /dev/null 2>&1 &" - -echo -e "\033[32mPrint Bigtop-Manager Server RSA Private Key\033[0m" -docker exec bigtop-manager-server bash -c "cat ~/.ssh/id_rsa" - -# MySQL HOST: bigtop-manager-server -# MySQL PORT: 3306 -# DATABASE NAME: bigtop_manager -# DATABASE USER NAME: root -# DATABASE PASSWORD: root - diff --git a/dev-support/docker/rocky8/build-image.sh b/dev-support/docker/rocky8/build-image.sh deleted file mode 100755 index 7725364cd..000000000 --- a/dev-support/docker/rocky8/build-image.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/bash - -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -BIN_DIR=$(dirname $0) -cd $BIN_DIR -echo $PWD - -echo -e "\033[32mRemoving image bigtop-manager:trunk-rocky-8\033[0m" -docker rmi bigtop-manager/develop:trunk-rocky-8 - -echo -e "\033[32mBuilding image bigtop-manager:trunk-rocky-8\033[0m" -docker build -t bigtop-manager/develop:trunk-rocky-8 . diff --git a/dev-support/docker/rocky8/clear-containers.sh b/dev-support/docker/rocky8/clear-containers.sh deleted file mode 100755 index 86b6ae249..000000000 --- a/dev-support/docker/rocky8/clear-containers.sh +++ /dev/null @@ -1,40 +0,0 @@ -#!/bin/bash - -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -if docker ps -a | grep -q 'bigtop-manager-build-r8'; then - echo -e "\033[32mStopping container bigtop-manager-build-r8 and maven process\033[0m" - if [ `docker inspect --format '{{.State.Running}}' bigtop-manager-build-r8` == true ];then - docker exec bigtop-manager-build-r8 bash -c "pkill -KILL -f maven" - docker stop bigtop-manager-build-r8 - fi -fi - -echo -e "\033[32mRemoving container bigtop-manager-server\033[0m" -docker rm -f bigtop-manager-server - -echo -e "\033[32mRemoving container bigtop-manager-agent-01\033[0m" -docker rm -f bigtop-manager-agent-01 - -echo -e "\033[32mRemoving container bigtop-manager-agent-02\033[0m" -docker rm -f bigtop-manager-agent-02 - -echo -e "\033[32mRemoving network bigtop-manager\033[0m" -docker network rm bigtop-manager \ No newline at end of file diff --git a/dev-support/docker/rocky8/distribute-scripts.sh b/dev-support/docker/rocky8/distribute-scripts.sh deleted file mode 100755 index e35c10534..000000000 --- a/dev-support/docker/rocky8/distribute-scripts.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/bin/bash - -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -echo -e "\033[32mSynchronizing script to bigtop-manager-server\033[0m" -docker cp ../../../bigtop-manager-server/src/main/resources/stacks/ bigtop-manager-server:/opt/bigtop-manager-server/ - -docker exec bigtop-manager-server bash -c "ps -ef | grep 'org.apache.bigtop.manager.server.ServerApplication' | grep -v 'grep' | awk '{print $2}' | xargs kill" -docker exec bigtop-manager-server bash -c "ps -ef | grep 'org.apache.bigtop.manager.agent.AgentApplication' | grep -v 'grep' | awk '{print $2}' | xargs kill" -docker exec bigtop-manager-agent-01 bash -c "ps -ef | grep 'org.apache.bigtop.manager.agent.AgentApplication' | grep -v 'grep' | awk '{print $2}' | xargs kill" -docker exec bigtop-manager-agent-02 bash -c "ps -ef | grep 'org.apache.bigtop.manager.agent.AgentApplication' | grep -v 'grep' | awk '{print $2}' | xargs kill" - -docker exec bigtop-manager-server bash -c "nohup /bin/bash /opt/bigtop-manager-server/bin/start.sh > /dev/null 2>&1 &" -docker exec bigtop-manager-server bash -c "nohup /bin/bash /opt/bigtop-manager-agent/bin/start.sh > /dev/null 2>&1 &" -docker exec bigtop-manager-agent-01 bash -c "nohup /bin/bash /opt/bigtop-manager-agent/bin/start.sh > /dev/null 2>&1 &" -docker exec bigtop-manager-agent-02 bash -c "nohup /bin/bash /opt/bigtop-manager-agent/bin/start.sh > /dev/null 2>&1 &" - -echo -e "\033[32mDone!\033[0m" \ No newline at end of file diff --git a/dev-support/docker/rocky8/restart-containers.sh b/dev-support/docker/rocky8/restart-containers.sh deleted file mode 100755 index 7e0a49784..000000000 --- a/dev-support/docker/rocky8/restart-containers.sh +++ /dev/null @@ -1,43 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -echo -e "\033[32mRestarting containers\033[0m" -docker restart bigtop-manager-server -docker restart bigtop-manager-agent-01 -docker restart bigtop-manager-agent-02 - -echo -e "\033[32mRe-enabling systemctl servers\033[0m" -docker exec bigtop-manager-server bash -c "systemctl start mariadb" -docker exec bigtop-manager-server bash -c "systemctl start sshd" -docker exec bigtop-manager-server bash -c "systemctl start chronyd" -docker exec bigtop-manager-agent-01 bash -c "systemctl start sshd" -docker exec bigtop-manager-agent-01 bash -c "systemctl start chronyd" -docker exec bigtop-manager-agent-02 bash -c "systemctl start sshd" -docker exec bigtop-manager-agent-02 bash -c "systemctl start chronyd" - -echo -e "\033[32mSynchronize Chrony\033[0m" -docker exec bigtop-manager-server bash -c "chronyc tracking" -docker exec bigtop-manager-agent-01 bash -c "chronyc tracking" -docker exec bigtop-manager-agent-02 bash -c "chronyc tracking" - -echo -e "\033[32mRestarting bigtop-manager servers\033[0m" -docker exec bigtop-manager-server bash -c "nohup /bin/bash /opt/bigtop-manager-server/bin/start.sh --debug > /dev/null 2>&1 &" -docker exec bigtop-manager-server bash -c "nohup /bin/bash /opt/bigtop-manager-agent/bin/start.sh > /dev/null 2>&1 &" -docker exec bigtop-manager-agent-01 bash -c "nohup /bin/bash /opt/bigtop-manager-agent/bin/start.sh > /dev/null 2>&1 &" -docker exec bigtop-manager-agent-02 bash -c "nohup /bin/bash /opt/bigtop-manager-agent/bin/start.sh > /dev/null 2>&1 &"