From 35fa7e7010965a43d21dc8bb2350bb7996e62ecf Mon Sep 17 00:00:00 2001 From: alexfu Date: Thu, 12 Sep 2024 21:49:35 +0800 Subject: [PATCH 01/13] BIGTOP-4223: Support PostgreSQL for mybatis on Bigtop Manager --- .../bigtop/manager/dao/enums/DBType.java | 1 + .../mapper/postgresql/ChatMessageMapper.xml | 36 ++ .../mapper/postgresql/ChatThreadMapper.xml | 58 +++ .../mapper/postgresql/ClusterMapper.xml | 81 ++++ .../mapper/postgresql/ComponentMapper.xml | 109 +++++ .../mapper/postgresql/HostComponentMapper.xml | 222 ++++++++++ .../mapper/postgresql/HostMapper.xml | 111 +++++ .../resources/mapper/postgresql/JobMapper.xml | 130 ++++++ .../postgresql/PlatformAuthorizedMapper.xml | 45 ++ .../mapper/postgresql/PlatformMapper.xml | 38 ++ .../mapper/postgresql/RepoMapper.xml | 64 +++ .../mapper/postgresql/ServiceConfigMapper.xml | 118 +++++ .../mapper/postgresql/ServiceMapper.xml | 133 ++++++ .../mapper/postgresql/StackMapper.xml | 46 ++ .../mapper/postgresql/StageMapper.xml | 39 ++ .../mapper/postgresql/TaskMapper.xml | 39 ++ .../mapper/postgresql/UserMapper.xml | 43 ++ .../manager/server/config/MyBatisConfig.java | 2 +- .../resources/ddl/PostgreSQL-DDL-CREATE.sql | 403 ++++++++++++++++++ 19 files changed, 1717 insertions(+), 1 deletion(-) create mode 100644 bigtop-manager-dao/src/main/resources/mapper/postgresql/ChatMessageMapper.xml create mode 100644 bigtop-manager-dao/src/main/resources/mapper/postgresql/ChatThreadMapper.xml create mode 100644 bigtop-manager-dao/src/main/resources/mapper/postgresql/ClusterMapper.xml create mode 100644 bigtop-manager-dao/src/main/resources/mapper/postgresql/ComponentMapper.xml create mode 100644 bigtop-manager-dao/src/main/resources/mapper/postgresql/HostComponentMapper.xml create mode 100644 bigtop-manager-dao/src/main/resources/mapper/postgresql/HostMapper.xml create mode 100644 bigtop-manager-dao/src/main/resources/mapper/postgresql/JobMapper.xml create mode 100644 bigtop-manager-dao/src/main/resources/mapper/postgresql/PlatformAuthorizedMapper.xml create mode 100644 bigtop-manager-dao/src/main/resources/mapper/postgresql/PlatformMapper.xml create mode 100644 bigtop-manager-dao/src/main/resources/mapper/postgresql/RepoMapper.xml create mode 100644 bigtop-manager-dao/src/main/resources/mapper/postgresql/ServiceConfigMapper.xml create mode 100644 bigtop-manager-dao/src/main/resources/mapper/postgresql/ServiceMapper.xml create mode 100644 bigtop-manager-dao/src/main/resources/mapper/postgresql/StackMapper.xml create mode 100644 bigtop-manager-dao/src/main/resources/mapper/postgresql/StageMapper.xml create mode 100644 bigtop-manager-dao/src/main/resources/mapper/postgresql/TaskMapper.xml create mode 100644 bigtop-manager-dao/src/main/resources/mapper/postgresql/UserMapper.xml create mode 100644 bigtop-manager-server/src/main/resources/ddl/PostgreSQL-DDL-CREATE.sql diff --git a/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/enums/DBType.java b/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/enums/DBType.java index b2e8bac2..f961f750 100644 --- a/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/enums/DBType.java +++ b/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/enums/DBType.java @@ -26,6 +26,7 @@ @Getter public enum DBType { MYSQL("mysql", "MYSQL"), + POSTGRESQL("postgresql", "PostgreSQL"), DM("dm", "DaMeng"); DBType(String code, String desc) { diff --git a/bigtop-manager-dao/src/main/resources/mapper/postgresql/ChatMessageMapper.xml b/bigtop-manager-dao/src/main/resources/mapper/postgresql/ChatMessageMapper.xml new file mode 100644 index 00000000..108410b1 --- /dev/null +++ b/bigtop-manager-dao/src/main/resources/mapper/postgresql/ChatMessageMapper.xml @@ -0,0 +1,36 @@ + + + + + + + + + DELETE FROM llm_chat_message + WHERE thread_id = #{threadId} + + + \ No newline at end of file diff --git a/bigtop-manager-dao/src/main/resources/mapper/postgresql/ChatThreadMapper.xml b/bigtop-manager-dao/src/main/resources/mapper/postgresql/ChatThreadMapper.xml new file mode 100644 index 00000000..5ecd9a2a --- /dev/null +++ b/bigtop-manager-dao/src/main/resources/mapper/postgresql/ChatThreadMapper.xml @@ -0,0 +1,58 @@ + + + + + + id, user_id, platform_id, model + + + + + + + + + + + + + + INSERT INTO llm_chat_thread (platform_id, user_id, model, thread_info) + VALUES (#{platformId}, #{userId}, #{model}, #{threadInfo, typeHandler=org.apache.bigtop.manager.dao.handler.JsonTypeHandler}) + ON DUPLICATE KEY UPDATE + platform_id = VALUES(platform_id), + user_id = VALUES(user_id), + model = VALUES(model), + thread_info = VALUES(thread_info) + + + \ No newline at end of file diff --git a/bigtop-manager-dao/src/main/resources/mapper/postgresql/ClusterMapper.xml b/bigtop-manager-dao/src/main/resources/mapper/postgresql/ClusterMapper.xml new file mode 100644 index 00000000..75caa42a --- /dev/null +++ b/bigtop-manager-dao/src/main/resources/mapper/postgresql/ClusterMapper.xml @@ -0,0 +1,81 @@ + + + + + + + + id, cluster_name, cluster_type, root, user_group, packages, repo_template, state, selected, stack_id + + + ${alias}.id, ${alias}.cluster_name, ${alias}.cluster_type, ${alias}.root, ${alias}.user_group, ${alias}.packages, ${alias}.repo_template, ${alias}.state, ${alias}.selected, ${alias}.stack_id + + + + + + + + + + + \ No newline at end of file diff --git a/bigtop-manager-dao/src/main/resources/mapper/postgresql/ComponentMapper.xml b/bigtop-manager-dao/src/main/resources/mapper/postgresql/ComponentMapper.xml new file mode 100644 index 00000000..cc1c93d6 --- /dev/null +++ b/bigtop-manager-dao/src/main/resources/mapper/postgresql/ComponentMapper.xml @@ -0,0 +1,109 @@ + + + + + + + + id, component_name, display_name, command_script, custom_commands, category, quick_link, cardinality, service_id, cluster_id + + + ${alias}.id, ${alias}.component_name, ${alias}.display_name, ${alias}.command_script, ${alias}.custom_commands, ${alias}.category, ${alias}.quick_link, ${alias}.cardinality, ${alias}.service_id, ${alias}.cluster_id + + + + + + + + + + + \ No newline at end of file diff --git a/bigtop-manager-dao/src/main/resources/mapper/postgresql/HostComponentMapper.xml b/bigtop-manager-dao/src/main/resources/mapper/postgresql/HostComponentMapper.xml new file mode 100644 index 00000000..51060006 --- /dev/null +++ b/bigtop-manager-dao/src/main/resources/mapper/postgresql/HostComponentMapper.xml @@ -0,0 +1,222 @@ + + + + + + + + id, state, host_id, component_id + + + ${alias}.id, ${alias}.state, ${alias}.host_id, ${alias}.component_id + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/bigtop-manager-dao/src/main/resources/mapper/postgresql/HostMapper.xml b/bigtop-manager-dao/src/main/resources/mapper/postgresql/HostMapper.xml new file mode 100644 index 00000000..4c475a71 --- /dev/null +++ b/bigtop-manager-dao/src/main/resources/mapper/postgresql/HostMapper.xml @@ -0,0 +1,111 @@ + + + + + + + + id, hostname, ipv4, ipv6, os, arch, available_processors, free_memory_size, total_memory_size, free_disk, total_disk, state, cluster_id + + + + ${alias}.id, ${alias}.hostname, ${alias}.ipv4, ${alias}.ipv6, ${alias}.os, ${alias}.arch, ${alias}.available_processors, ${alias}.free_memory_size, ${alias}.total_memory_size, ${alias}.free_disk, ${alias}.total_disk, ${alias}.state, ${alias}.cluster_id + + + + + + + + + + + + insert into host (hostname, ipv4, ipv6, os, arch, available_processors, free_memory_size, total_memory_size, free_disk, total_disk, state, cluster_id, create_by, update_by, create_time, update_time) + values + + (#{host.hostname}, #{host.ipv4}, #{host.ipv6}, #{host.os}, #{host.arch}, #{host.availableProcessors}, #{host.freeMemorySize}, #{host.totalMemorySize}, #{host.freeDisk}, #{host.totalDisk}, #{host.state}, #{host.clusterId} ,#{host.createBy},#{host.updateBy},#{host.createTime},#{host.updateTime}) + + + + \ No newline at end of file diff --git a/bigtop-manager-dao/src/main/resources/mapper/postgresql/JobMapper.xml b/bigtop-manager-dao/src/main/resources/mapper/postgresql/JobMapper.xml new file mode 100644 index 00000000..6efb75bc --- /dev/null +++ b/bigtop-manager-dao/src/main/resources/mapper/postgresql/JobMapper.xml @@ -0,0 +1,130 @@ + + + + + + + + id, state, name, context, cluster_id, create_time, update_time + + + + ${alias}.id, ${alias}.state, ${alias}.name, ${alias}.context, ${alias}.cluster_id, ${alias}.create_time, ${alias}.update_time + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/bigtop-manager-dao/src/main/resources/mapper/postgresql/PlatformAuthorizedMapper.xml b/bigtop-manager-dao/src/main/resources/mapper/postgresql/PlatformAuthorizedMapper.xml new file mode 100644 index 00000000..f065a4c7 --- /dev/null +++ b/bigtop-manager-dao/src/main/resources/mapper/postgresql/PlatformAuthorizedMapper.xml @@ -0,0 +1,45 @@ + + + + + + + id, credentials, platfotrm_id + + + + + + + + + + INSERT INTO llm_platform_authorized (platform_id, credentials) + VALUES (#{platformId}, #{credentials, typeHandler=org.apache.bigtop.manager.dao.handler.JsonTypeHandler}) + ON DUPLICATE KEY UPDATE + platform_id = VALUES(platform_id), + credentials = VALUES(credentials) + + + \ No newline at end of file diff --git a/bigtop-manager-dao/src/main/resources/mapper/postgresql/PlatformMapper.xml b/bigtop-manager-dao/src/main/resources/mapper/postgresql/PlatformMapper.xml new file mode 100644 index 00000000..e4ca9039 --- /dev/null +++ b/bigtop-manager-dao/src/main/resources/mapper/postgresql/PlatformMapper.xml @@ -0,0 +1,38 @@ + + + + + + + id, name, credential, support_models + + + + + + + + + + \ No newline at end of file diff --git a/bigtop-manager-dao/src/main/resources/mapper/postgresql/RepoMapper.xml b/bigtop-manager-dao/src/main/resources/mapper/postgresql/RepoMapper.xml new file mode 100644 index 00000000..b51ab044 --- /dev/null +++ b/bigtop-manager-dao/src/main/resources/mapper/postgresql/RepoMapper.xml @@ -0,0 +1,64 @@ + + + + + + + + id, base_url, os, arch, repo_id, repo_name, cluster_id + + + + + + insert into repo (base_url, os, arch, repo_id, repo_name, cluster_id, create_by, update_by, create_time, update_time) + values + + (#{cluster.baseUrl},#{cluster.os},#{cluster.arch},#{cluster.repoId},#{cluster.repoName},#{cluster.clusterId},#{cluster.createBy},#{cluster.updateBy},#{cluster.createTime},#{cluster.updateTime}) + + + + + + \ No newline at end of file diff --git a/bigtop-manager-dao/src/main/resources/mapper/postgresql/ServiceConfigMapper.xml b/bigtop-manager-dao/src/main/resources/mapper/postgresql/ServiceConfigMapper.xml new file mode 100644 index 00000000..52a30d61 --- /dev/null +++ b/bigtop-manager-dao/src/main/resources/mapper/postgresql/ServiceConfigMapper.xml @@ -0,0 +1,118 @@ + + + + + + + + id, config_desc, version, selected, service_id, cluster_id, create_time, update_time + + + + ${alias}.id, ${alias}.config_desc, ${alias}.version, ${alias}.selected, ${alias}.service_id, ${alias}.cluster_id, ${alias}.create_time, ${alias}.update_time + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/bigtop-manager-dao/src/main/resources/mapper/postgresql/ServiceMapper.xml b/bigtop-manager-dao/src/main/resources/mapper/postgresql/ServiceMapper.xml new file mode 100644 index 00000000..5a405651 --- /dev/null +++ b/bigtop-manager-dao/src/main/resources/mapper/postgresql/ServiceMapper.xml @@ -0,0 +1,133 @@ + + + + + + + + id, service_name, display_name, service_desc, service_version, package_specifics, service_user, required_services, cluster_id + + + + ${alias}.id, ${alias}.service_name, ${alias}.display_name, ${alias}.service_desc, ${alias}.service_version, ${alias}.package_specifics, ${alias}.service_user, ${alias}.required_services, ${alias}.cluster_id + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/bigtop-manager-dao/src/main/resources/mapper/postgresql/StackMapper.xml b/bigtop-manager-dao/src/main/resources/mapper/postgresql/StackMapper.xml new file mode 100644 index 00000000..d654df90 --- /dev/null +++ b/bigtop-manager-dao/src/main/resources/mapper/postgresql/StackMapper.xml @@ -0,0 +1,46 @@ + + + + + + + + id, stack_name, stack_version + + + + + \ No newline at end of file diff --git a/bigtop-manager-dao/src/main/resources/mapper/postgresql/StageMapper.xml b/bigtop-manager-dao/src/main/resources/mapper/postgresql/StageMapper.xml new file mode 100644 index 00000000..6b567ce2 --- /dev/null +++ b/bigtop-manager-dao/src/main/resources/mapper/postgresql/StageMapper.xml @@ -0,0 +1,39 @@ + + + + + + + + UPDATE stage + SET `state` = CASE + + WHEN id = #{item.id} THEN #{item.state} + + END + WHERE id IN + + #{item.id} + + + + \ No newline at end of file diff --git a/bigtop-manager-dao/src/main/resources/mapper/postgresql/TaskMapper.xml b/bigtop-manager-dao/src/main/resources/mapper/postgresql/TaskMapper.xml new file mode 100644 index 00000000..7f9d74b8 --- /dev/null +++ b/bigtop-manager-dao/src/main/resources/mapper/postgresql/TaskMapper.xml @@ -0,0 +1,39 @@ + + + + + + + + UPDATE task + SET `state` = CASE + + WHEN id = #{item.id} THEN #{item.state} + + END + WHERE id IN + + #{item.id} + + + + \ No newline at end of file diff --git a/bigtop-manager-dao/src/main/resources/mapper/postgresql/UserMapper.xml b/bigtop-manager-dao/src/main/resources/mapper/postgresql/UserMapper.xml new file mode 100644 index 00000000..c2f463da --- /dev/null +++ b/bigtop-manager-dao/src/main/resources/mapper/postgresql/UserMapper.xml @@ -0,0 +1,43 @@ + + + + + + + + id, username, password, nickname, status + + + + + \ No newline at end of file diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/config/MyBatisConfig.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/config/MyBatisConfig.java index 17a3bdad..b3c791f3 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/config/MyBatisConfig.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/config/MyBatisConfig.java @@ -44,7 +44,7 @@ public SqlSessionFactory sqlSessionFactory(DataSource dataSource) throws Excepti SqlSessionFactoryBean sessionFactory = new SqlSessionFactoryBean(); sessionFactory.setDataSource(dataSource); sessionFactory.setMapperLocations( - new PathMatchingResourcePatternResolver().getResources("classpath*:mapper/**/*Mapper.xml")); + new PathMatchingResourcePatternResolver().getResources(String.format("classpath*:mapper/%s/**/*Mapper.xml", new ProductNameDatabaseIdProvider().getDatabaseId(dataSource)))); org.apache.ibatis.session.Configuration configuration = new org.apache.ibatis.session.Configuration(); configuration.setMapUnderscoreToCamelCase(true); diff --git a/bigtop-manager-server/src/main/resources/ddl/PostgreSQL-DDL-CREATE.sql b/bigtop-manager-server/src/main/resources/ddl/PostgreSQL-DDL-CREATE.sql new file mode 100644 index 00000000..38202217 --- /dev/null +++ b/bigtop-manager-server/src/main/resources/ddl/PostgreSQL-DDL-CREATE.sql @@ -0,0 +1,403 @@ +/* SQLINES DEMO *** e Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + + +-- delimiter ; + +-- CREATE DATABASE `bigtop_manager` /*!40100 DEFAULT CHARACTER SET utf8 */; +-- +-- CREATE USER 'bigtop_manager' IDENTIFIED BY 'bigdata'; + +-- USE @schema; + +-- SQLINES DEMO *** ge_engine to InnoDB +-- SQLINES DEMO *** riable should be used for versions prior to MySQL 5.6 +-- @version_short := substring_index(@@version, '.', 2); +-- @major := cast(substring_index(@version_short, '.', 1) as SIGNED); +-- @minor := cast(substring_index(@version_short, '.', -1) as SIGNED); +-- @engine_stmt := CASE WHEN (@major >= 5 AND @minor>=6) or @major >= 8 THEN 'SET default_storage_engine=INNODB' ELSE 'SET storage_engine=INNODB' END; +-- execute statement; +-- DEALLOCATE PREPARE statement; + +-- SQLINES FOR EVALUATION USE ONLY (14 DAYS) +CREATE TABLE audit_log +( + id BIGINT NOT NULL GENERATED ALWAYS AS IDENTITY, + args TEXT, + create_by BIGINT, + create_time TIMESTAMP(0), + operation_desc VARCHAR(255), + operation_summary VARCHAR(255), + tag_desc VARCHAR(255), + tag_name VARCHAR(255), + update_by BIGINT, + update_time TIMESTAMP(0), + uri VARCHAR(255), + user_id BIGINT, + PRIMARY KEY (id) +) ; + +CREATE TABLE "user" +( + id BIGINT CHECK (id > 0) NOT NULL GENERATED ALWAYS AS IDENTITY, + username VARCHAR(32) DEFAULT NULL, + password VARCHAR(32) DEFAULT NULL, + nickname VARCHAR(32) DEFAULT NULL, + status int DEFAULT 1, + create_time TIMESTAMP(0) DEFAULT NULL, + update_time TIMESTAMP(0) DEFAULT NULL, + create_by BIGINT, + update_by BIGINT, + PRIMARY KEY (id), + CONSTRAINT uk_username UNIQUE (username) +) ; + +COMMENT ON COLUMN "user".status IS '0-Disable, 1-Enable'; + +CREATE TABLE cluster +( + id BIGINT CHECK (id > 0) NOT NULL GENERATED ALWAYS AS IDENTITY, + cluster_name VARCHAR(255) DEFAULT NULL, + cluster_desc VARCHAR(255) DEFAULT NULL, + cluster_type SMALLINT CHECK (cluster_type > 0) DEFAULT 1, + selected int DEFAULT 1, + create_time TIMESTAMP(0) DEFAULT NULL, + update_time TIMESTAMP(0) DEFAULT NULL, + create_by BIGINT, + packages VARCHAR(255), + repo_template VARCHAR(255), + root VARCHAR(255), + state VARCHAR(255), + update_by BIGINT, + user_group VARCHAR(255), + stack_id BIGINT, + PRIMARY KEY (id), + CONSTRAINT uk_cluster_name UNIQUE (cluster_name) +) ; + +COMMENT ON COLUMN cluster.cluster_name IS 'Cluster Name'; +COMMENT ON COLUMN cluster.cluster_desc IS 'Cluster Description'; +COMMENT ON COLUMN cluster.cluster_type IS '1-Physical Machine, 2-Kubernetes'; +COMMENT ON COLUMN cluster.selected IS '0-Disable, 1-Enable'; + +CREATE INDEX idx_cluster_stack_id ON cluster (stack_id); + +CREATE TABLE component +( + id BIGINT NOT NULL GENERATED ALWAYS AS IDENTITY, + category VARCHAR(255), + command_script VARCHAR(255), + component_name VARCHAR(255), + create_by BIGINT, + create_time TIMESTAMP(0), + custom_commands TEXT, + display_name VARCHAR(255), + quick_link VARCHAR(255), + cardinality VARCHAR(255), + update_by BIGINT, + update_time TIMESTAMP(0), + cluster_id BIGINT, + service_id BIGINT, + PRIMARY KEY (id) + , + CONSTRAINT uk_component_name UNIQUE (component_name, cluster_id) +) ; + +CREATE INDEX idx_component_cluster_id ON component (cluster_id); +CREATE INDEX idx_component_service_id ON component (service_id); + +CREATE TABLE host_component +( + id BIGINT NOT NULL GENERATED ALWAYS AS IDENTITY, + create_by BIGINT, + create_time TIMESTAMP(0), + state VARCHAR(255), + update_by BIGINT, + update_time TIMESTAMP(0), + component_id BIGINT, + host_id BIGINT, + PRIMARY KEY (id) +) ; + +CREATE INDEX idx_hc_component_id ON host_component (component_id); +CREATE INDEX idx_hc_host_id ON host_component (host_id); + +CREATE TABLE host +( + id BIGINT CHECK (id > 0) NOT NULL GENERATED ALWAYS AS IDENTITY, + cluster_id BIGINT CHECK (cluster_id > 0) NOT NULL, + hostname VARCHAR(255) DEFAULT NULL, + ipv4 VARCHAR(32) DEFAULT NULL, + ipv6 VARCHAR(32) DEFAULT NULL, + arch VARCHAR(32) DEFAULT NULL, + os VARCHAR(32) DEFAULT NULL, + processor_count INT DEFAULT NULL, + physical_memory BIGINT DEFAULT NULL, + state VARCHAR(32) DEFAULT NULL, + create_time TIMESTAMP(0) DEFAULT NULL, + update_time TIMESTAMP(0) DEFAULT NULL, + available_processors INTEGER, + create_by BIGINT, + free_disk BIGINT, + free_memory_size BIGINT, + total_disk BIGINT, + total_memory_size BIGINT, + update_by BIGINT, + PRIMARY KEY (id), + CONSTRAINT uk_hostname UNIQUE (hostname, cluster_id) +) ; + +COMMENT ON COLUMN host.physical_memory IS 'Total Physical Memory(Bytes)'; + +CREATE INDEX idx_host_cluster_id ON host (cluster_id); + +CREATE TABLE repo +( + id BIGINT CHECK (id > 0) NOT NULL GENERATED ALWAYS AS IDENTITY, + cluster_id BIGINT CHECK (cluster_id > 0) NOT NULL, + os VARCHAR(32) DEFAULT NULL, + arch VARCHAR(32) DEFAULT NULL, + base_url VARCHAR(64) DEFAULT NULL, + repo_id VARCHAR(32) DEFAULT NULL, + repo_name VARCHAR(64) DEFAULT NULL, + repo_type VARCHAR(64) DEFAULT NULL, + create_time TIMESTAMP(0) DEFAULT NULL, + update_time TIMESTAMP(0) DEFAULT NULL, + create_by BIGINT, + update_by BIGINT, + PRIMARY KEY (id) + , + CONSTRAINT uk_repo_id UNIQUE (repo_id, os, arch, cluster_id) +) ; + +CREATE INDEX idx_cluster_id ON repo (cluster_id); + +CREATE TABLE stack +( + id BIGINT CHECK (id > 0) NOT NULL GENERATED ALWAYS AS IDENTITY, + stack_name VARCHAR(32) NOT NULL, + stack_version VARCHAR(32) NOT NULL, + create_time TIMESTAMP(0) DEFAULT NULL, + update_time TIMESTAMP(0) DEFAULT NULL, + create_by BIGINT, + update_by BIGINT, + PRIMARY KEY (id), + CONSTRAINT uk_stack UNIQUE (stack_name, stack_version) +) ; + +CREATE TABLE task +( + id BIGINT NOT NULL GENERATED ALWAYS AS IDENTITY, + command VARCHAR(255), + component_name VARCHAR(255), + content TEXT, + context TEXT NOT NULL, + create_by BIGINT, + create_time TIMESTAMP(0), + custom_command VARCHAR(255), + hostname VARCHAR(255), + name VARCHAR(255), + service_name VARCHAR(255), + service_user VARCHAR(255), + stack_name VARCHAR(255), + stack_version VARCHAR(255), + state VARCHAR(255), + update_by BIGINT, + update_time TIMESTAMP(0), + cluster_id BIGINT, + job_id BIGINT, + stage_id BIGINT, + PRIMARY KEY (id) +) ; + +CREATE INDEX idx_task_cluster_id ON task (cluster_id); +CREATE INDEX idx_task_job_id ON task (job_id); +CREATE INDEX idx_task_stage_id ON task (stage_id); + +CREATE TABLE job +( + id BIGINT CHECK (id > 0) NOT NULL GENERATED ALWAYS AS IDENTITY, + cluster_id BIGINT CHECK (cluster_id > 0) DEFAULT NULL, + state VARCHAR(32) NOT NULL, + context TEXT NOT NULL, + create_time TIMESTAMP(0) DEFAULT NULL, + update_time TIMESTAMP(0) DEFAULT NULL, + create_by BIGINT, + name VARCHAR(255), + update_by BIGINT, + PRIMARY KEY (id) +) ; + +CREATE INDEX idx_cluster_id ON job (cluster_id); + +CREATE TABLE type_config +( + id BIGINT NOT NULL GENERATED ALWAYS AS IDENTITY, + create_by BIGINT, + create_time TIMESTAMP(0), + properties_json TEXT, + type_name VARCHAR(255), + update_by BIGINT, + update_time TIMESTAMP(0), + service_config_id BIGINT, + PRIMARY KEY (id) +) ; + +CREATE TABLE service +( + id BIGINT NOT NULL GENERATED ALWAYS AS IDENTITY, + create_by BIGINT, + create_time TIMESTAMP(0), + display_name VARCHAR(255), + package_specifics VARCHAR(1024), + required_services VARCHAR(255), + service_desc VARCHAR(1024), + service_name VARCHAR(255), + service_user VARCHAR(255), + service_version VARCHAR(255), + update_by BIGINT, + update_time TIMESTAMP(0), + cluster_id BIGINT, + PRIMARY KEY (id) + , + CONSTRAINT uk_service_name UNIQUE (service_name, cluster_id) +) ; + +CREATE INDEX idx_service_cluster_id ON service (cluster_id); + +CREATE TABLE service_config +( + id BIGINT NOT NULL GENERATED ALWAYS AS IDENTITY, + config_desc VARCHAR(255), + create_by BIGINT, + create_time TIMESTAMP(0), + selected SMALLINT default 0, + update_by BIGINT, + update_time TIMESTAMP(0), + version INTEGER, + cluster_id BIGINT, + service_id BIGINT, + PRIMARY KEY (id) +) ; + +CREATE INDEX idx_sc_cluster_id ON service_config (cluster_id); +CREATE INDEX idx_sc_service_id ON service_config (service_id); + +CREATE TABLE setting +( + id BIGINT NOT NULL GENERATED ALWAYS AS IDENTITY, + config_data TEXT, + create_by BIGINT, + create_time TIMESTAMP(0), + type_name VARCHAR(255), + update_by BIGINT, + update_time TIMESTAMP(0), + PRIMARY KEY (id) +) ; + +CREATE TABLE stage +( + id BIGINT CHECK (id > 0) NOT NULL GENERATED ALWAYS AS IDENTITY, + name VARCHAR(32) NOT NULL, + cluster_id BIGINT CHECK (cluster_id > 0) DEFAULT NULL, + job_id BIGINT CHECK (job_id > 0) NOT NULL, + state VARCHAR(32) NOT NULL, + create_time TIMESTAMP(0) DEFAULT NULL, + update_time TIMESTAMP(0) DEFAULT NULL, + component_name VARCHAR(255), + context TEXT, + create_by BIGINT, + "order" INTEGER, + service_name VARCHAR(255), + update_by BIGINT, + PRIMARY KEY (id) +) ; + +CREATE INDEX idx_cluster_id ON stage (cluster_id); +CREATE INDEX idx_job_id ON stage (job_id); + +CREATE TABLE llm_platform +( + id BIGINT CHECK (id > 0) NOT NULL GENERATED ALWAYS AS IDENTITY, + name VARCHAR(255) NOT NULL, + credential JSON DEFAULT NULL, + support_models VARCHAR(255) DEFAULT NULL, + create_time TIMESTAMP(0) DEFAULT CURRENT_TIMESTAMP, + update_time TIMESTAMP(0) DEFAULT CURRENT_TIMESTAMP /* ON UPDATE CURRENT_TIMESTAMP */, + create_by BIGINT DEFAULT NULL, + update_by BIGINT DEFAULT NULL, + PRIMARY KEY (id) +) ; + +CREATE TABLE llm_platform_authorized +( + id BIGINT CHECK (id > 0) NOT NULL GENERATED ALWAYS AS IDENTITY, + platform_id BIGINT CHECK (platform_id > 0) NOT NULL, + credentials JSON NOT NULL, + create_time TIMESTAMP(0) DEFAULT CURRENT_TIMESTAMP, + update_time TIMESTAMP(0) DEFAULT CURRENT_TIMESTAMP /* ON UPDATE CURRENT_TIMESTAMP */, + create_by BIGINT DEFAULT NULL, + update_by BIGINT DEFAULT NULL, + PRIMARY KEY (id) +) ; + +CREATE INDEX idx_platform_id ON llm_platform_authorized (platform_id); + +CREATE TABLE llm_chat_thread +( + id BIGINT CHECK (id > 0) NOT NULL GENERATED ALWAYS AS IDENTITY, + platform_id BIGINT CHECK (platform_id > 0) NOT NULL, + user_id BIGINT CHECK (user_id > 0) NOT NULL, + model VARCHAR(255) NOT NULL, + thread_info JSON DEFAULT NULL, + create_time TIMESTAMP(0) DEFAULT CURRENT_TIMESTAMP, + update_time TIMESTAMP(0) DEFAULT CURRENT_TIMESTAMP /* ON UPDATE CURRENT_TIMESTAMP */, + create_by BIGINT DEFAULT NULL, + update_by BIGINT DEFAULT NULL, + PRIMARY KEY (id) +) ; + +CREATE INDEX idx_platform_id ON llm_chat_thread (platform_id); +CREATE INDEX idx_user_id ON llm_chat_thread (user_id); + +CREATE TABLE llm_chat_message +( + id BIGINT CHECK (id > 0) NOT NULL GENERATED ALWAYS AS IDENTITY, + thread_id BIGINT CHECK (thread_id > 0) NOT NULL, + user_id BIGINT CHECK (user_id > 0) NOT NULL, + message TEXT NOT NULL, + sender VARCHAR(50) NOT NULL, + create_time TIMESTAMP(0) DEFAULT CURRENT_TIMESTAMP, + update_time TIMESTAMP(0) DEFAULT CURRENT_TIMESTAMP /* ON UPDATE CURRENT_TIMESTAMP */, + create_by BIGINT DEFAULT NULL, + update_by BIGINT DEFAULT NULL, + PRIMARY KEY (id) +) ; + +CREATE INDEX idx_thread_id ON llm_chat_message (thread_id); +CREATE INDEX idx_user_id ON llm_chat_message (user_id); + +-- SQLINES DEMO *** min user +INSERT INTO "user" (create_time, update_time, nickname, password, status, username) +VALUES (now(), now(), 'Administrator', '21232f297a57a5a743894a0e4a801fc3', 1, 'admin'); + +-- SQLINES DEMO *** chat platform +INSERT INTO llm_platform (credential,NAME,support_models) +VALUES + ('{"apiKey": "API Key"}','OpenAI','gpt-3.5-turbo,gpt-4,gpt-4o,gpt-3.5-turbo-16k,gpt-4-turbo-preview,gpt-4-32k,gpt-4o-mini'), + ('{"apiKey": "API Key"}','DashScope','qwen-max,qwen-plus,qwen-turbo'); From 0b89afc006840ad943e0dd035099f77421362efe Mon Sep 17 00:00:00 2001 From: alexfu Date: Thu, 12 Sep 2024 21:54:56 +0800 Subject: [PATCH 02/13] BIGTOP-4223: Support PostgreSQL for mybatis on Bigtop Manager --- .../apache/bigtop/manager/server/config/MyBatisConfig.java | 4 +++- .../src/main/resources/ddl/PostgreSQL-DDL-CREATE.sql | 3 ++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/config/MyBatisConfig.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/config/MyBatisConfig.java index b3c791f3..8114a130 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/config/MyBatisConfig.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/config/MyBatisConfig.java @@ -44,7 +44,9 @@ public SqlSessionFactory sqlSessionFactory(DataSource dataSource) throws Excepti SqlSessionFactoryBean sessionFactory = new SqlSessionFactoryBean(); sessionFactory.setDataSource(dataSource); sessionFactory.setMapperLocations( - new PathMatchingResourcePatternResolver().getResources(String.format("classpath*:mapper/%s/**/*Mapper.xml", new ProductNameDatabaseIdProvider().getDatabaseId(dataSource)))); + new PathMatchingResourcePatternResolver().getResources( + String.format("classpath*:mapper/%s/**/*Mapper.xml", + new ProductNameDatabaseIdProvider().getDatabaseId(dataSource)))); org.apache.ibatis.session.Configuration configuration = new org.apache.ibatis.session.Configuration(); configuration.setMapUnderscoreToCamelCase(true); diff --git a/bigtop-manager-server/src/main/resources/ddl/PostgreSQL-DDL-CREATE.sql b/bigtop-manager-server/src/main/resources/ddl/PostgreSQL-DDL-CREATE.sql index 38202217..8678baa2 100644 --- a/bigtop-manager-server/src/main/resources/ddl/PostgreSQL-DDL-CREATE.sql +++ b/bigtop-manager-server/src/main/resources/ddl/PostgreSQL-DDL-CREATE.sql @@ -1,4 +1,5 @@ -/* SQLINES DEMO *** e Apache Software Foundation (ASF) under one +/* + * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file From 10dd2560bdb5e28aa6cfa16b44e826efaf995f34 Mon Sep 17 00:00:00 2001 From: alexfu Date: Thu, 12 Sep 2024 21:58:51 +0800 Subject: [PATCH 03/13] BIGTOP-4223: Support PostgreSQL for mybatis on Bigtop Manager --- .../bigtop/manager/server/config/MyBatisConfig.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/config/MyBatisConfig.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/config/MyBatisConfig.java index 8114a130..b9b58f30 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/config/MyBatisConfig.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/config/MyBatisConfig.java @@ -43,10 +43,10 @@ public class MyBatisConfig { public SqlSessionFactory sqlSessionFactory(DataSource dataSource) throws Exception { SqlSessionFactoryBean sessionFactory = new SqlSessionFactoryBean(); sessionFactory.setDataSource(dataSource); - sessionFactory.setMapperLocations( - new PathMatchingResourcePatternResolver().getResources( - String.format("classpath*:mapper/%s/**/*Mapper.xml", - new ProductNameDatabaseIdProvider().getDatabaseId(dataSource)))); + sessionFactory.setMapperLocations(new PathMatchingResourcePatternResolver() + .getResources(String.format( + "classpath*:mapper/%s/**/*Mapper.xml", + new ProductNameDatabaseIdProvider().getDatabaseId(dataSource)))); org.apache.ibatis.session.Configuration configuration = new org.apache.ibatis.session.Configuration(); configuration.setMapUnderscoreToCamelCase(true); From 75b9491dfec8651e8989bed418e3e7277823c40f Mon Sep 17 00:00:00 2001 From: alexfu Date: Thu, 12 Sep 2024 23:40:24 +0800 Subject: [PATCH 04/13] BIGTOP-4223: Support PostgreSQL for mybatis --- .../resources/ddl/PostgreSQL-DDL-CREATE.sql | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/bigtop-manager-server/src/main/resources/ddl/PostgreSQL-DDL-CREATE.sql b/bigtop-manager-server/src/main/resources/ddl/PostgreSQL-DDL-CREATE.sql index 8678baa2..6299d756 100644 --- a/bigtop-manager-server/src/main/resources/ddl/PostgreSQL-DDL-CREATE.sql +++ b/bigtop-manager-server/src/main/resources/ddl/PostgreSQL-DDL-CREATE.sql @@ -17,24 +17,6 @@ * under the License. */ - --- delimiter ; - --- CREATE DATABASE `bigtop_manager` /*!40100 DEFAULT CHARACTER SET utf8 */; --- --- CREATE USER 'bigtop_manager' IDENTIFIED BY 'bigdata'; - --- USE @schema; - --- SQLINES DEMO *** ge_engine to InnoDB --- SQLINES DEMO *** riable should be used for versions prior to MySQL 5.6 --- @version_short := substring_index(@@version, '.', 2); --- @major := cast(substring_index(@version_short, '.', 1) as SIGNED); --- @minor := cast(substring_index(@version_short, '.', -1) as SIGNED); --- @engine_stmt := CASE WHEN (@major >= 5 AND @minor>=6) or @major >= 8 THEN 'SET default_storage_engine=INNODB' ELSE 'SET storage_engine=INNODB' END; --- execute statement; --- DEALLOCATE PREPARE statement; - -- SQLINES FOR EVALUATION USE ONLY (14 DAYS) CREATE TABLE audit_log ( From 8940fe83e32f2106813650a73c0bcba1e39b864f Mon Sep 17 00:00:00 2001 From: alexfu Date: Thu, 12 Sep 2024 23:40:46 +0800 Subject: [PATCH 05/13] BIGTOP-4223: Support PostgreSQL for mybatis --- .../src/main/resources/ddl/PostgreSQL-DDL-CREATE.sql | 1 - 1 file changed, 1 deletion(-) diff --git a/bigtop-manager-server/src/main/resources/ddl/PostgreSQL-DDL-CREATE.sql b/bigtop-manager-server/src/main/resources/ddl/PostgreSQL-DDL-CREATE.sql index 6299d756..cfada606 100644 --- a/bigtop-manager-server/src/main/resources/ddl/PostgreSQL-DDL-CREATE.sql +++ b/bigtop-manager-server/src/main/resources/ddl/PostgreSQL-DDL-CREATE.sql @@ -17,7 +17,6 @@ * under the License. */ --- SQLINES FOR EVALUATION USE ONLY (14 DAYS) CREATE TABLE audit_log ( id BIGINT NOT NULL GENERATED ALWAYS AS IDENTITY, From 1db679739ab21e05bb936ba7a8a4978745bdf4c8 Mon Sep 17 00:00:00 2001 From: alexfu Date: Fri, 13 Sep 2024 00:01:52 +0800 Subject: [PATCH 06/13] BIGTOP-4223: Support PostgreSQL for mybatis --- .../resources/ddl/PostgreSQL-DDL-CREATE.sql | 276 +++++++++--------- 1 file changed, 138 insertions(+), 138 deletions(-) diff --git a/bigtop-manager-server/src/main/resources/ddl/PostgreSQL-DDL-CREATE.sql b/bigtop-manager-server/src/main/resources/ddl/PostgreSQL-DDL-CREATE.sql index cfada606..e0489f65 100644 --- a/bigtop-manager-server/src/main/resources/ddl/PostgreSQL-DDL-CREATE.sql +++ b/bigtop-manager-server/src/main/resources/ddl/PostgreSQL-DDL-CREATE.sql @@ -32,34 +32,34 @@ CREATE TABLE audit_log uri VARCHAR(255), user_id BIGINT, PRIMARY KEY (id) -) ; +); CREATE TABLE "user" ( id BIGINT CHECK (id > 0) NOT NULL GENERATED ALWAYS AS IDENTITY, - username VARCHAR(32) DEFAULT NULL, - password VARCHAR(32) DEFAULT NULL, - nickname VARCHAR(32) DEFAULT NULL, - status int DEFAULT 1, - create_time TIMESTAMP(0) DEFAULT NULL, - update_time TIMESTAMP(0) DEFAULT NULL, + username VARCHAR(32) DEFAULT NULL, + password VARCHAR(32) DEFAULT NULL, + nickname VARCHAR(32) DEFAULT NULL, + status int DEFAULT 1, + create_time TIMESTAMP(0) DEFAULT NULL, + update_time TIMESTAMP(0) DEFAULT NULL, create_by BIGINT, update_by BIGINT, PRIMARY KEY (id), CONSTRAINT uk_username UNIQUE (username) -) ; +); COMMENT ON COLUMN "user".status IS '0-Disable, 1-Enable'; CREATE TABLE cluster ( id BIGINT CHECK (id > 0) NOT NULL GENERATED ALWAYS AS IDENTITY, - cluster_name VARCHAR(255) DEFAULT NULL, - cluster_desc VARCHAR(255) DEFAULT NULL, + cluster_name VARCHAR(255) DEFAULT NULL, + cluster_desc VARCHAR(255) DEFAULT NULL, cluster_type SMALLINT CHECK (cluster_type > 0) DEFAULT 1, - selected int DEFAULT 1, - create_time TIMESTAMP(0) DEFAULT NULL, - update_time TIMESTAMP(0) DEFAULT NULL, + selected int DEFAULT 1, + create_time TIMESTAMP(0) DEFAULT NULL, + update_time TIMESTAMP(0) DEFAULT NULL, create_by BIGINT, packages VARCHAR(255), repo_template VARCHAR(255), @@ -70,7 +70,7 @@ CREATE TABLE cluster stack_id BIGINT, PRIMARY KEY (id), CONSTRAINT uk_cluster_name UNIQUE (cluster_name) -) ; +); COMMENT ON COLUMN cluster.cluster_name IS 'Cluster Name'; COMMENT ON COLUMN cluster.cluster_desc IS 'Cluster Description'; @@ -95,13 +95,12 @@ CREATE TABLE component update_time TIMESTAMP(0), cluster_id BIGINT, service_id BIGINT, - PRIMARY KEY (id) - , + PRIMARY KEY (id), CONSTRAINT uk_component_name UNIQUE (component_name, cluster_id) -) ; +); -CREATE INDEX idx_component_cluster_id ON component (cluster_id); -CREATE INDEX idx_component_service_id ON component (service_id); +CREATE INDEX idx_component_cluster_id ON component (cluster_id); +CREATE INDEX idx_component_service_id ON component (service_id); CREATE TABLE host_component ( @@ -114,14 +113,14 @@ CREATE TABLE host_component component_id BIGINT, host_id BIGINT, PRIMARY KEY (id) -) ; +); -CREATE INDEX idx_hc_component_id ON host_component (component_id); -CREATE INDEX idx_hc_host_id ON host_component (host_id); +CREATE INDEX idx_hc_component_id ON host_component (component_id); +CREATE INDEX idx_hc_host_id ON host_component (host_id); CREATE TABLE host ( - id BIGINT CHECK (id > 0) NOT NULL GENERATED ALWAYS AS IDENTITY, + id BIGINT CHECK (id > 0) NOT NULL GENERATED ALWAYS AS IDENTITY, cluster_id BIGINT CHECK (cluster_id > 0) NOT NULL, hostname VARCHAR(255) DEFAULT NULL, ipv4 VARCHAR(32) DEFAULT NULL, @@ -131,8 +130,8 @@ CREATE TABLE host processor_count INT DEFAULT NULL, physical_memory BIGINT DEFAULT NULL, state VARCHAR(32) DEFAULT NULL, - create_time TIMESTAMP(0) DEFAULT NULL, - update_time TIMESTAMP(0) DEFAULT NULL, + create_time TIMESTAMP(0) DEFAULT NULL, + update_time TIMESTAMP(0) DEFAULT NULL, available_processors INTEGER, create_by BIGINT, free_disk BIGINT, @@ -142,90 +141,89 @@ CREATE TABLE host update_by BIGINT, PRIMARY KEY (id), CONSTRAINT uk_hostname UNIQUE (hostname, cluster_id) -) ; +); COMMENT ON COLUMN host.physical_memory IS 'Total Physical Memory(Bytes)'; -CREATE INDEX idx_host_cluster_id ON host (cluster_id); +CREATE INDEX idx_host_cluster_id ON host (cluster_id); CREATE TABLE repo ( - id BIGINT CHECK (id > 0) NOT NULL GENERATED ALWAYS AS IDENTITY, + id BIGINT CHECK (id > 0) NOT NULL GENERATED ALWAYS AS IDENTITY, cluster_id BIGINT CHECK (cluster_id > 0) NOT NULL, - os VARCHAR(32) DEFAULT NULL, - arch VARCHAR(32) DEFAULT NULL, - base_url VARCHAR(64) DEFAULT NULL, - repo_id VARCHAR(32) DEFAULT NULL, - repo_name VARCHAR(64) DEFAULT NULL, - repo_type VARCHAR(64) DEFAULT NULL, - create_time TIMESTAMP(0) DEFAULT NULL, - update_time TIMESTAMP(0) DEFAULT NULL, + os VARCHAR(32) DEFAULT NULL, + arch VARCHAR(32) DEFAULT NULL, + base_url VARCHAR(64) DEFAULT NULL, + repo_id VARCHAR(32) DEFAULT NULL, + repo_name VARCHAR(64) DEFAULT NULL, + repo_type VARCHAR(64) DEFAULT NULL, + create_time TIMESTAMP(0) DEFAULT NULL, + update_time TIMESTAMP(0) DEFAULT NULL, create_by BIGINT, update_by BIGINT, - PRIMARY KEY (id) - , + PRIMARY KEY (id), CONSTRAINT uk_repo_id UNIQUE (repo_id, os, arch, cluster_id) -) ; +); -CREATE INDEX idx_cluster_id ON repo (cluster_id); +CREATE INDEX idx_cluster_id ON repo (cluster_id); CREATE TABLE stack ( - id BIGINT CHECK (id > 0) NOT NULL GENERATED ALWAYS AS IDENTITY, - stack_name VARCHAR(32) NOT NULL, - stack_version VARCHAR(32) NOT NULL, - create_time TIMESTAMP(0) DEFAULT NULL, - update_time TIMESTAMP(0) DEFAULT NULL, - create_by BIGINT, - update_by BIGINT, + id BIGINT CHECK (id > 0) NOT NULL GENERATED ALWAYS AS IDENTITY, + stack_name VARCHAR(32) NOT NULL, + stack_version VARCHAR(32) NOT NULL, + create_time TIMESTAMP(0) DEFAULT NULL, + update_time TIMESTAMP(0) DEFAULT NULL, + create_by BIGINT, + update_by BIGINT, PRIMARY KEY (id), - CONSTRAINT uk_stack UNIQUE (stack_name, stack_version) -) ; + CONSTRAINT uk_stack UNIQUE (stack_name, stack_version) +); CREATE TABLE task ( - id BIGINT NOT NULL GENERATED ALWAYS AS IDENTITY, - command VARCHAR(255), - component_name VARCHAR(255), - content TEXT, - context TEXT NOT NULL, - create_by BIGINT, - create_time TIMESTAMP(0), - custom_command VARCHAR(255), - hostname VARCHAR(255), - name VARCHAR(255), - service_name VARCHAR(255), - service_user VARCHAR(255), - stack_name VARCHAR(255), - stack_version VARCHAR(255), - state VARCHAR(255), - update_by BIGINT, - update_time TIMESTAMP(0), - cluster_id BIGINT, - job_id BIGINT, - stage_id BIGINT, + id BIGINT NOT NULL GENERATED ALWAYS AS IDENTITY, + command VARCHAR(255), + component_name VARCHAR(255), + content TEXT, + context TEXT NOT NULL, + create_by BIGINT, + create_time TIMESTAMP(0), + custom_command VARCHAR(255), + hostname VARCHAR(255), + name VARCHAR(255), + service_name VARCHAR(255), + service_user VARCHAR(255), + stack_name VARCHAR(255), + stack_version VARCHAR(255), + state VARCHAR(255), + update_by BIGINT, + update_time TIMESTAMP(0), + cluster_id BIGINT, + job_id BIGINT, + stage_id BIGINT, PRIMARY KEY (id) -) ; +); -CREATE INDEX idx_task_cluster_id ON task (cluster_id); -CREATE INDEX idx_task_job_id ON task (job_id); -CREATE INDEX idx_task_stage_id ON task (stage_id); +CREATE INDEX idx_task_cluster_id ON task (cluster_id); +CREATE INDEX idx_task_job_id ON task (job_id); +CREATE INDEX idx_task_stage_id ON task (stage_id); CREATE TABLE job ( id BIGINT CHECK (id > 0) NOT NULL GENERATED ALWAYS AS IDENTITY, cluster_id BIGINT CHECK (cluster_id > 0) DEFAULT NULL, - state VARCHAR(32) NOT NULL, - context TEXT NOT NULL, - create_time TIMESTAMP(0) DEFAULT NULL, - update_time TIMESTAMP(0) DEFAULT NULL, + state VARCHAR(32) NOT NULL, + context TEXT NOT NULL, + create_time TIMESTAMP(0) DEFAULT NULL, + update_time TIMESTAMP(0) DEFAULT NULL, create_by BIGINT, name VARCHAR(255), update_by BIGINT, PRIMARY KEY (id) -) ; +); -CREATE INDEX idx_cluster_id ON job (cluster_id); +CREATE INDEX idx_cluster_id ON job (cluster_id); CREATE TABLE type_config ( @@ -238,7 +236,7 @@ CREATE TABLE type_config update_time TIMESTAMP(0), service_config_id BIGINT, PRIMARY KEY (id) -) ; +); CREATE TABLE service ( @@ -255,12 +253,11 @@ CREATE TABLE service update_by BIGINT, update_time TIMESTAMP(0), cluster_id BIGINT, - PRIMARY KEY (id) - , + PRIMARY KEY (id), CONSTRAINT uk_service_name UNIQUE (service_name, cluster_id) -) ; +); -CREATE INDEX idx_service_cluster_id ON service (cluster_id); +CREATE INDEX idx_service_cluster_id ON service (cluster_id); CREATE TABLE service_config ( @@ -275,10 +272,10 @@ CREATE TABLE service_config cluster_id BIGINT, service_id BIGINT, PRIMARY KEY (id) -) ; +); -CREATE INDEX idx_sc_cluster_id ON service_config (cluster_id); -CREATE INDEX idx_sc_service_id ON service_config (service_id); +CREATE INDEX idx_sc_cluster_id ON service_config (cluster_id); +CREATE INDEX idx_sc_service_id ON service_config (service_id); CREATE TABLE setting ( @@ -290,96 +287,99 @@ CREATE TABLE setting update_by BIGINT, update_time TIMESTAMP(0), PRIMARY KEY (id) -) ; +); CREATE TABLE stage ( - id BIGINT CHECK (id > 0) NOT NULL GENERATED ALWAYS AS IDENTITY, - name VARCHAR(32) NOT NULL, + id BIGINT CHECK (id > 0) NOT NULL GENERATED ALWAYS AS IDENTITY, + name VARCHAR(32) NOT NULL, cluster_id BIGINT CHECK (cluster_id > 0) DEFAULT NULL, job_id BIGINT CHECK (job_id > 0) NOT NULL, - state VARCHAR(32) NOT NULL, - create_time TIMESTAMP(0) DEFAULT NULL, - update_time TIMESTAMP(0) DEFAULT NULL, + state VARCHAR(32) NOT NULL, + create_time TIMESTAMP(0) DEFAULT NULL, + update_time TIMESTAMP(0) DEFAULT NULL, component_name VARCHAR(255), context TEXT, create_by BIGINT, - "order" INTEGER, + "order" INTEGER, service_name VARCHAR(255), update_by BIGINT, PRIMARY KEY (id) -) ; +); -CREATE INDEX idx_cluster_id ON stage (cluster_id); -CREATE INDEX idx_job_id ON stage (job_id); +CREATE INDEX idx_cluster_id ON stage (cluster_id); +CREATE INDEX idx_job_id ON stage (job_id); CREATE TABLE llm_platform ( id BIGINT CHECK (id > 0) NOT NULL GENERATED ALWAYS AS IDENTITY, - name VARCHAR(255) NOT NULL, - credential JSON DEFAULT NULL, - support_models VARCHAR(255) DEFAULT NULL, - create_time TIMESTAMP(0) DEFAULT CURRENT_TIMESTAMP, - update_time TIMESTAMP(0) DEFAULT CURRENT_TIMESTAMP /* ON UPDATE CURRENT_TIMESTAMP */, - create_by BIGINT DEFAULT NULL, - update_by BIGINT DEFAULT NULL, + name VARCHAR(255) NOT NULL, + credential JSON DEFAULT NULL, + support_models VARCHAR(255) DEFAULT NULL, + create_time TIMESTAMP(0) DEFAULT CURRENT_TIMESTAMP, + update_time TIMESTAMP(0) DEFAULT CURRENT_TIMESTAMP /* ON UPDATE CURRENT_TIMESTAMP */, + create_by BIGINT DEFAULT NULL, + update_by BIGINT DEFAULT NULL, PRIMARY KEY (id) -) ; +); CREATE TABLE llm_platform_authorized ( - id BIGINT CHECK (id > 0) NOT NULL GENERATED ALWAYS AS IDENTITY, + id BIGINT CHECK (id > 0) NOT NULL GENERATED ALWAYS AS IDENTITY, platform_id BIGINT CHECK (platform_id > 0) NOT NULL, - credentials JSON NOT NULL, - create_time TIMESTAMP(0) DEFAULT CURRENT_TIMESTAMP, - update_time TIMESTAMP(0) DEFAULT CURRENT_TIMESTAMP /* ON UPDATE CURRENT_TIMESTAMP */, - create_by BIGINT DEFAULT NULL, - update_by BIGINT DEFAULT NULL, + credentials JSON NOT NULL, + create_time TIMESTAMP(0) DEFAULT CURRENT_TIMESTAMP, + update_time TIMESTAMP(0) DEFAULT CURRENT_TIMESTAMP /* ON UPDATE CURRENT_TIMESTAMP */, + create_by BIGINT DEFAULT NULL, + update_by BIGINT DEFAULT NULL, PRIMARY KEY (id) -) ; +); CREATE INDEX idx_platform_id ON llm_platform_authorized (platform_id); CREATE TABLE llm_chat_thread ( - id BIGINT CHECK (id > 0) NOT NULL GENERATED ALWAYS AS IDENTITY, + id BIGINT CHECK (id > 0) NOT NULL GENERATED ALWAYS AS IDENTITY, platform_id BIGINT CHECK (platform_id > 0) NOT NULL, - user_id BIGINT CHECK (user_id > 0) NOT NULL, - model VARCHAR(255) NOT NULL, - thread_info JSON DEFAULT NULL, - create_time TIMESTAMP(0) DEFAULT CURRENT_TIMESTAMP, - update_time TIMESTAMP(0) DEFAULT CURRENT_TIMESTAMP /* ON UPDATE CURRENT_TIMESTAMP */, - create_by BIGINT DEFAULT NULL, - update_by BIGINT DEFAULT NULL, + user_id BIGINT CHECK (user_id > 0) NOT NULL, + model VARCHAR(255) NOT NULL, + thread_info JSON DEFAULT NULL, + create_time TIMESTAMP(0) DEFAULT CURRENT_TIMESTAMP, + update_time TIMESTAMP(0) DEFAULT CURRENT_TIMESTAMP /* ON UPDATE CURRENT_TIMESTAMP */, + create_by BIGINT DEFAULT NULL, + update_by BIGINT DEFAULT NULL, PRIMARY KEY (id) -) ; +); -CREATE INDEX idx_platform_id ON llm_chat_thread (platform_id); -CREATE INDEX idx_user_id ON llm_chat_thread (user_id); +CREATE INDEX idx_platform_id ON llm_chat_thread (platform_id); +CREATE INDEX idx_user_id ON llm_chat_thread (user_id); CREATE TABLE llm_chat_message ( - id BIGINT CHECK (id > 0) NOT NULL GENERATED ALWAYS AS IDENTITY, + id BIGINT CHECK (id > 0) NOT NULL GENERATED ALWAYS AS IDENTITY, thread_id BIGINT CHECK (thread_id > 0) NOT NULL, - user_id BIGINT CHECK (user_id > 0) NOT NULL, - message TEXT NOT NULL, - sender VARCHAR(50) NOT NULL, - create_time TIMESTAMP(0) DEFAULT CURRENT_TIMESTAMP, - update_time TIMESTAMP(0) DEFAULT CURRENT_TIMESTAMP /* ON UPDATE CURRENT_TIMESTAMP */, - create_by BIGINT DEFAULT NULL, - update_by BIGINT DEFAULT NULL, + user_id BIGINT CHECK (user_id > 0) NOT NULL, + message TEXT NOT NULL, + sender VARCHAR(50) NOT NULL, + create_time TIMESTAMP(0) DEFAULT CURRENT_TIMESTAMP, + update_time TIMESTAMP(0) DEFAULT CURRENT_TIMESTAMP /* ON UPDATE CURRENT_TIMESTAMP */, + create_by BIGINT DEFAULT NULL, + update_by BIGINT DEFAULT NULL, PRIMARY KEY (id) -) ; +); -CREATE INDEX idx_thread_id ON llm_chat_message (thread_id); -CREATE INDEX idx_user_id ON llm_chat_message (user_id); +CREATE INDEX idx_thread_id ON llm_chat_message (thread_id); +CREATE INDEX idx_user_id ON llm_chat_message (user_id); -- SQLINES DEMO *** min user INSERT INTO "user" (create_time, update_time, nickname, password, status, username) VALUES (now(), now(), 'Administrator', '21232f297a57a5a743894a0e4a801fc3', 1, 'admin'); -- SQLINES DEMO *** chat platform -INSERT INTO llm_platform (credential,NAME,support_models) -VALUES - ('{"apiKey": "API Key"}','OpenAI','gpt-3.5-turbo,gpt-4,gpt-4o,gpt-3.5-turbo-16k,gpt-4-turbo-preview,gpt-4-32k,gpt-4o-mini'), - ('{"apiKey": "API Key"}','DashScope','qwen-max,qwen-plus,qwen-turbo'); +INSERT INTO llm_platform (credential, NAME, support_models) +VALUES ('{ + "apiKey": "API Key" +}', 'OpenAI', 'gpt-3.5-turbo,gpt-4,gpt-4o,gpt-3.5-turbo-16k,gpt-4-turbo-preview,gpt-4-32k,gpt-4o-mini'), + ('{ + "apiKey": "API Key" + }', 'DashScope', 'qwen-max,qwen-plus,qwen-turbo'); From 616f89a9d2c5469b4cf132f31fbbe031fc3ca32c Mon Sep 17 00:00:00 2001 From: alexfu Date: Fri, 13 Sep 2024 00:05:37 +0800 Subject: [PATCH 07/13] BIGTOP-4223: Support PostgreSQL for mybatis --- .../src/main/resources/ddl/PostgreSQL-DDL-CREATE.sql | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/bigtop-manager-server/src/main/resources/ddl/PostgreSQL-DDL-CREATE.sql b/bigtop-manager-server/src/main/resources/ddl/PostgreSQL-DDL-CREATE.sql index e0489f65..96ffae62 100644 --- a/bigtop-manager-server/src/main/resources/ddl/PostgreSQL-DDL-CREATE.sql +++ b/bigtop-manager-server/src/main/resources/ddl/PostgreSQL-DDL-CREATE.sql @@ -377,9 +377,5 @@ VALUES (now(), now(), 'Administrator', '21232f297a57a5a743894a0e4a801fc3', 1, 'a -- SQLINES DEMO *** chat platform INSERT INTO llm_platform (credential, NAME, support_models) -VALUES ('{ - "apiKey": "API Key" -}', 'OpenAI', 'gpt-3.5-turbo,gpt-4,gpt-4o,gpt-3.5-turbo-16k,gpt-4-turbo-preview,gpt-4-32k,gpt-4o-mini'), - ('{ - "apiKey": "API Key" - }', 'DashScope', 'qwen-max,qwen-plus,qwen-turbo'); +VALUES ('{"apiKey": "API Key"}', 'OpenAI', 'gpt-3.5-turbo,gpt-4,gpt-4o,gpt-3.5-turbo-16k,gpt-4-turbo-preview,gpt-4-32k,gpt-4o-mini'), + ('{"apiKey": "API Key"}', 'DashScope', 'qwen-max,qwen-plus,qwen-turbo'); From 929f041d4bdd68c49f01835ce5cdbe1f9ca8a278 Mon Sep 17 00:00:00 2001 From: alexfu Date: Fri, 13 Sep 2024 00:07:48 +0800 Subject: [PATCH 08/13] BIGTOP-4223: Support PostgreSQL for mybatis --- .../main/java/org/apache/bigtop/manager/dao/enums/DBType.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/enums/DBType.java b/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/enums/DBType.java index f961f750..111fa663 100644 --- a/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/enums/DBType.java +++ b/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/enums/DBType.java @@ -25,7 +25,7 @@ @Getter public enum DBType { - MYSQL("mysql", "MYSQL"), + MYSQL("mysql", "MySQL"), POSTGRESQL("postgresql", "PostgreSQL"), DM("dm", "DaMeng"); From b9463a2c59ee115f2ad85cf59e710de50969d0dc Mon Sep 17 00:00:00 2001 From: alexfu Date: Fri, 13 Sep 2024 00:08:24 +0800 Subject: [PATCH 09/13] BIGTOP-4223: Support PostgreSQL for mybatis --- .../src/main/resources/ddl/PostgreSQL-DDL-CREATE.sql | 2 -- 1 file changed, 2 deletions(-) diff --git a/bigtop-manager-server/src/main/resources/ddl/PostgreSQL-DDL-CREATE.sql b/bigtop-manager-server/src/main/resources/ddl/PostgreSQL-DDL-CREATE.sql index 96ffae62..860d3426 100644 --- a/bigtop-manager-server/src/main/resources/ddl/PostgreSQL-DDL-CREATE.sql +++ b/bigtop-manager-server/src/main/resources/ddl/PostgreSQL-DDL-CREATE.sql @@ -371,11 +371,9 @@ CREATE TABLE llm_chat_message CREATE INDEX idx_thread_id ON llm_chat_message (thread_id); CREATE INDEX idx_user_id ON llm_chat_message (user_id); --- SQLINES DEMO *** min user INSERT INTO "user" (create_time, update_time, nickname, password, status, username) VALUES (now(), now(), 'Administrator', '21232f297a57a5a743894a0e4a801fc3', 1, 'admin'); --- SQLINES DEMO *** chat platform INSERT INTO llm_platform (credential, NAME, support_models) VALUES ('{"apiKey": "API Key"}', 'OpenAI', 'gpt-3.5-turbo,gpt-4,gpt-4o,gpt-3.5-turbo-16k,gpt-4-turbo-preview,gpt-4-32k,gpt-4o-mini'), ('{"apiKey": "API Key"}', 'DashScope', 'qwen-max,qwen-plus,qwen-turbo'); From 5ad2b3ea7d6be1b29168407fdf3c4692382053a0 Mon Sep 17 00:00:00 2001 From: alexfu Date: Fri, 13 Sep 2024 00:12:29 +0800 Subject: [PATCH 10/13] BIGTOP-4223: Support PostgreSQL for mybatis --- .../src/main/resources/ddl/PostgreSQL-DDL-CREATE.sql | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/bigtop-manager-server/src/main/resources/ddl/PostgreSQL-DDL-CREATE.sql b/bigtop-manager-server/src/main/resources/ddl/PostgreSQL-DDL-CREATE.sql index 860d3426..de449024 100644 --- a/bigtop-manager-server/src/main/resources/ddl/PostgreSQL-DDL-CREATE.sql +++ b/bigtop-manager-server/src/main/resources/ddl/PostgreSQL-DDL-CREATE.sql @@ -375,5 +375,7 @@ INSERT INTO "user" (create_time, update_time, nickname, password, status, userna VALUES (now(), now(), 'Administrator', '21232f297a57a5a743894a0e4a801fc3', 1, 'admin'); INSERT INTO llm_platform (credential, NAME, support_models) -VALUES ('{"apiKey": "API Key"}', 'OpenAI', 'gpt-3.5-turbo,gpt-4,gpt-4o,gpt-3.5-turbo-16k,gpt-4-turbo-preview,gpt-4-32k,gpt-4o-mini'), - ('{"apiKey": "API Key"}', 'DashScope', 'qwen-max,qwen-plus,qwen-turbo'); +VALUES +('{"apiKey": "API Key"}','OpenAI','gpt-3.5-turbo,gpt-4,gpt-4o,gpt-3.5-turbo-16k,gpt-4-turbo-preview,gpt-4-32k,gpt-4o-mini'), +('{"apiKey": "API Key"}','DashScope','qwen-max,qwen-plus,qwen-turbo'), +('{"apiKey": "API Key", "secretKey": "Secret Key"}','QianFan','Yi-34B-Chat,ERNIE-4.0-8K,ERNIE-3.5-128K,ERNIE-Speed-8K,Llama-2-7B-Chat,Fuyu-8B'); From d0629881353396c83da8d6735f6568db56b7bc1a Mon Sep 17 00:00:00 2001 From: alexfu Date: Sat, 14 Sep 2024 15:56:22 +0800 Subject: [PATCH 11/13] BIGTOP-4223: Support PostgreSQL for mybatis --- .../bigtop/manager/dao/sql/SQLBuilder.java | 80 +++++++++++++++++++ .../resources/mapper/postgresql/JobMapper.xml | 30 ++++--- .../mapper/postgresql/PlatformMapper.xml | 2 +- .../mapper/postgresql/ServiceConfigMapper.xml | 4 +- .../mapper/postgresql/StackMapper.xml | 2 +- .../mapper/postgresql/StageMapper.xml | 6 +- .../mapper/postgresql/TaskMapper.xml | 2 +- .../mapper/postgresql/UserMapper.xml | 2 +- .../server/command/job/AbstractJob.java | 4 + .../server/command/stage/AbstractStage.java | 5 ++ .../server/command/task/AbstractTask.java | 4 + .../resources/ddl/PostgreSQL-DDL-CREATE.sql | 8 +- 12 files changed, 127 insertions(+), 22 deletions(-) diff --git a/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/sql/SQLBuilder.java b/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/sql/SQLBuilder.java index ce729c77..8a6d7ba5 100644 --- a/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/sql/SQLBuilder.java +++ b/bigtop-manager-dao/src/main/java/org/apache/bigtop/manager/dao/sql/SQLBuilder.java @@ -71,6 +71,25 @@ public static String insert(TableMetaData tableMetaData, Entity entity, } break; } + case POSTGRESQL: { + sql.INSERT_INTO("\"" + tableMetaData.getTableName() + "\""); + for (Map.Entry entry : fieldColumnMap.entrySet()) { + // Ignore primary key + if (Objects.equals(entry.getKey(), tableMetaData.getPkProperty())) { + continue; + } + PropertyDescriptor ps = BeanUtils.getPropertyDescriptor(entityClass, entry.getKey()); + if (ps == null || ps.getReadMethod() == null) { + continue; + } + Object value = ReflectionUtils.invokeMethod(ps.getReadMethod(), entity); + if (!ObjectUtils.isEmpty(value)) { + sql.VALUES("\"" + entry.getValue() + "\"", getTokenParam(entry.getKey())); + } + } + break; + } + default: { log.error("Unsupported data source"); } @@ -105,6 +124,25 @@ public static String update(TableMetaData tableMetaData, Entity entity, sql.WHERE(getEquals(tableMetaData.getPkColumn(), tableMetaData.getPkProperty())); break; } + case POSTGRESQL: { + sql.UPDATE("\"" + tableMetaData.getTableName() + "\""); + for (Map.Entry entry : fieldColumnMap.entrySet()) { + // Ignore primary key + if (Objects.equals(entry.getKey(), tableMetaData.getPkProperty())) { + continue; + } + PropertyDescriptor ps = BeanUtils.getPropertyDescriptor(entityClass, entry.getKey()); + if (ps == null || ps.getReadMethod() == null) { + continue; + } + Object value = ReflectionUtils.invokeMethod(ps.getReadMethod(), entity); + if (!ObjectUtils.isEmpty(value)) { + sql.SET("\"" + getEquals(entry.getValue() + "\"", entry.getKey())); + } + } + sql.WHERE(getEquals(tableMetaData.getPkColumn(), tableMetaData.getPkProperty())); + break; + } default: { log.error("Unsupported data source"); } @@ -123,6 +161,16 @@ public static String selectById(TableMetaData tableMetaData, String databaseId, sql.WHERE(tableMetaData.getPkColumn() + " = '" + id + "'"); break; } + case POSTGRESQL: { + String baseColumns = tableMetaData.getBaseColumns(); + if (baseColumns.toLowerCase().contains("user.")) { + baseColumns = baseColumns.replace("user.", "\"user\"."); + } + sql.SELECT(baseColumns); + sql.FROM("\"" + tableMetaData.getTableName() + "\""); + sql.WHERE(tableMetaData.getPkColumn() + " = " + id); + break; + } default: { log.error("Unsupported data source"); } @@ -143,6 +191,17 @@ public static String selectByIds( sql.WHERE(tableMetaData.getPkColumn() + " in ('" + idsStr + "')"); break; } + case POSTGRESQL: { + String idsStr = ids.stream().map(String::valueOf).collect(Collectors.joining(",")); + String baseColumns = tableMetaData.getBaseColumns(); + if (baseColumns.toLowerCase().contains("user.")) { + baseColumns = baseColumns.replace("user.", "\"user\"."); + } + sql.SELECT(baseColumns); + sql.FROM("\"" + tableMetaData.getTableName() + "\""); + sql.WHERE(tableMetaData.getPkColumn() + " in (" + idsStr + ")"); + break; + } default: { log.error("Unsupported data source"); } @@ -155,6 +214,14 @@ public static String selectAll(TableMetaData tableMetaData, String databaseId) { SQL sql = new SQL(); switch (DBType.toType(databaseId)) { + case POSTGRESQL: + String baseColumns = tableMetaData.getBaseColumns(); + if (baseColumns.toLowerCase().contains("user.")) { + baseColumns = baseColumns.replace("user.", "\"user\"."); + } + sql.SELECT(baseColumns); + sql.FROM("\"" + tableMetaData.getTableName() + "\""); + break; case MYSQL: { sql.SELECT(tableMetaData.getBaseColumns()); sql.FROM(tableMetaData.getTableName()); @@ -176,6 +243,11 @@ public static String deleteById(TableMetaData tableMetaData, String databaseId, sql.WHERE(tableMetaData.getPkColumn() + " = '" + id + "'"); break; } + case POSTGRESQL: { + sql.FROM("\"" + tableMetaData.getTableName() + "\""); + sql.WHERE(tableMetaData.getPkColumn() + " = " + id); + break; + } default: { log.error("Unsupported data source"); } @@ -194,6 +266,12 @@ public static String deleteByIds( sql.WHERE(tableMetaData.getPkColumn() + " in ('" + idsStr + "')"); break; } + case POSTGRESQL: { + String idsStr = ids.stream().map(String::valueOf).collect(Collectors.joining(", ")); + sql.DELETE_FROM("\"" + tableMetaData.getTableName() + "\""); + sql.WHERE(tableMetaData.getPkColumn() + " in (" + idsStr + ")"); + break; + } default: { log.error("Unsupported data source"); } @@ -208,6 +286,8 @@ public static String findByCondition( log.info("databaseId: {}", databaseId); SQL sql = new SQL(); switch (DBType.toType(databaseId)) { + case POSTGRESQL: + tableName = "\"" + tableName + "\""; case MYSQL: { sql = mysqlCondition(condition, tableName); break; diff --git a/bigtop-manager-dao/src/main/resources/mapper/postgresql/JobMapper.xml b/bigtop-manager-dao/src/main/resources/mapper/postgresql/JobMapper.xml index 6efb75bc..7805bba8 100644 --- a/bigtop-manager-dao/src/main/resources/mapper/postgresql/JobMapper.xml +++ b/bigtop-manager-dao/src/main/resources/mapper/postgresql/JobMapper.xml @@ -73,7 +73,7 @@ - select @@ -83,8 +83,11 @@ from (select * from job - - cluster_id = #{clusterId} + + id in + + #{id} + ) h @@ -92,6 +95,7 @@ on h.id = st.job_id inner join task tk on tk.job_id = h.id and st.id = tk.stage_id + order by h.id desc + + \ No newline at end of file diff --git a/bigtop-manager-dao/src/main/resources/mapper/postgresql/PlatformMapper.xml b/bigtop-manager-dao/src/main/resources/mapper/postgresql/PlatformMapper.xml index e4ca9039..885ae6d1 100644 --- a/bigtop-manager-dao/src/main/resources/mapper/postgresql/PlatformMapper.xml +++ b/bigtop-manager-dao/src/main/resources/mapper/postgresql/PlatformMapper.xml @@ -23,7 +23,7 @@ - id, name, credential, support_models + "id", "name", "credential", "support_models" diff --git a/bigtop-manager-dao/src/main/resources/mapper/postgresql/ServiceConfigMapper.xml b/bigtop-manager-dao/src/main/resources/mapper/postgresql/ServiceConfigMapper.xml index 52a30d61..e40c4f9e 100644 --- a/bigtop-manager-dao/src/main/resources/mapper/postgresql/ServiceConfigMapper.xml +++ b/bigtop-manager-dao/src/main/resources/mapper/postgresql/ServiceConfigMapper.xml @@ -78,7 +78,7 @@ from (select * from service_config - selected = 1 + selected is true and cluster_id = #{clusterId} @@ -103,7 +103,7 @@ from (select * from service_config - selected = 1 + selected is true and cluster_id = #{clusterId} diff --git a/bigtop-manager-dao/src/main/resources/mapper/postgresql/StackMapper.xml b/bigtop-manager-dao/src/main/resources/mapper/postgresql/StackMapper.xml index d654df90..d9da21ba 100644 --- a/bigtop-manager-dao/src/main/resources/mapper/postgresql/StackMapper.xml +++ b/bigtop-manager-dao/src/main/resources/mapper/postgresql/StackMapper.xml @@ -24,7 +24,7 @@ - id, stack_name, stack_version + "id", "stack_name", "stack_version" diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/AbstractJob.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/AbstractJob.java index 902311b0..42f1248e 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/AbstractJob.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/job/AbstractJob.java @@ -33,10 +33,13 @@ import org.apache.bigtop.manager.server.command.task.Task; import org.apache.bigtop.manager.server.holder.SpringContextHolder; +import lombok.extern.slf4j.Slf4j; + import java.util.ArrayList; import java.util.List; import java.util.concurrent.LinkedBlockingQueue; +@Slf4j public abstract class AbstractJob implements Job { protected StackDao stackDao; @@ -106,6 +109,7 @@ public void run() { } } } catch (Exception e) { + log.error(e.getMessage(), e); success = false; } diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/AbstractStage.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/AbstractStage.java index 2081f7c3..f0a75671 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/AbstractStage.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/stage/AbstractStage.java @@ -25,10 +25,13 @@ import org.apache.bigtop.manager.server.command.task.Task; import org.apache.bigtop.manager.server.holder.SpringContextHolder; +import lombok.extern.slf4j.Slf4j; + import java.util.ArrayList; import java.util.List; import java.util.concurrent.CompletableFuture; +@Slf4j public abstract class AbstractStage implements Stage { protected StageDao stageDao; @@ -93,6 +96,7 @@ public Boolean run() { try { return future.get(); } catch (Exception e) { + log.error("stage failed,", e); return false; } }) @@ -100,6 +104,7 @@ public Boolean run() { allTaskSuccess = taskResults.stream().allMatch(Boolean::booleanValue); } catch (Exception e) { + log.error("stage failed", e); allTaskSuccess = false; } diff --git a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/task/AbstractTask.java b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/task/AbstractTask.java index fc0871a8..41f840c5 100644 --- a/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/task/AbstractTask.java +++ b/bigtop-manager-server/src/main/java/org/apache/bigtop/manager/server/command/task/AbstractTask.java @@ -31,6 +31,9 @@ import org.apache.bigtop.manager.server.grpc.GrpcClient; import org.apache.bigtop.manager.server.holder.SpringContextHolder; +import lombok.extern.slf4j.Slf4j; + +@Slf4j public abstract class AbstractTask implements Task { protected TaskDao taskDao; @@ -85,6 +88,7 @@ public Boolean run() { taskSuccess = reply != null && reply.getCode() == MessageConstants.SUCCESS_CODE; } catch (Exception e) { + log.error("task failed", e); taskSuccess = false; } diff --git a/bigtop-manager-server/src/main/resources/ddl/PostgreSQL-DDL-CREATE.sql b/bigtop-manager-server/src/main/resources/ddl/PostgreSQL-DDL-CREATE.sql index de449024..35fc719c 100644 --- a/bigtop-manager-server/src/main/resources/ddl/PostgreSQL-DDL-CREATE.sql +++ b/bigtop-manager-server/src/main/resources/ddl/PostgreSQL-DDL-CREATE.sql @@ -40,7 +40,7 @@ CREATE TABLE "user" username VARCHAR(32) DEFAULT NULL, password VARCHAR(32) DEFAULT NULL, nickname VARCHAR(32) DEFAULT NULL, - status int DEFAULT 1, + status BOOLEAN DEFAULT TRUE, create_time TIMESTAMP(0) DEFAULT NULL, update_time TIMESTAMP(0) DEFAULT NULL, create_by BIGINT, @@ -57,7 +57,7 @@ CREATE TABLE cluster cluster_name VARCHAR(255) DEFAULT NULL, cluster_desc VARCHAR(255) DEFAULT NULL, cluster_type SMALLINT CHECK (cluster_type > 0) DEFAULT 1, - selected int DEFAULT 1, + selected BOOLEAN DEFAULT TRUE, create_time TIMESTAMP(0) DEFAULT NULL, update_time TIMESTAMP(0) DEFAULT NULL, create_by BIGINT, @@ -265,7 +265,7 @@ CREATE TABLE service_config config_desc VARCHAR(255), create_by BIGINT, create_time TIMESTAMP(0), - selected SMALLINT default 0, + selected BOOLEAN default FALSE, update_by BIGINT, update_time TIMESTAMP(0), version INTEGER, @@ -372,7 +372,7 @@ CREATE INDEX idx_thread_id ON llm_chat_message (thread_id); CREATE INDEX idx_user_id ON llm_chat_message (user_id); INSERT INTO "user" (create_time, update_time, nickname, password, status, username) -VALUES (now(), now(), 'Administrator', '21232f297a57a5a743894a0e4a801fc3', 1, 'admin'); +VALUES (now(), now(), 'Administrator', '21232f297a57a5a743894a0e4a801fc3', true, 'admin'); INSERT INTO llm_platform (credential, NAME, support_models) VALUES From ca4229d6d53a7864f90e8c6061b8023e41f8a795 Mon Sep 17 00:00:00 2001 From: alexfu Date: Sat, 14 Sep 2024 16:15:05 +0800 Subject: [PATCH 12/13] BIGTOP-4223: Support PostgreSQL for mybatis --- .../resources/ddl/PostgreSQL-DDL-CREATE.sql | 22 ++++++++++++++----- 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/bigtop-manager-server/src/main/resources/ddl/PostgreSQL-DDL-CREATE.sql b/bigtop-manager-server/src/main/resources/ddl/PostgreSQL-DDL-CREATE.sql index 35fc719c..3a94b405 100644 --- a/bigtop-manager-server/src/main/resources/ddl/PostgreSQL-DDL-CREATE.sql +++ b/bigtop-manager-server/src/main/resources/ddl/PostgreSQL-DDL-CREATE.sql @@ -77,6 +77,7 @@ COMMENT ON COLUMN cluster.cluster_desc IS 'Cluster Description'; COMMENT ON COLUMN cluster.cluster_type IS '1-Physical Machine, 2-Kubernetes'; COMMENT ON COLUMN cluster.selected IS '0-Disable, 1-Enable'; +DROP INDEX IF EXISTS idx_cluster_stack_id; CREATE INDEX idx_cluster_stack_id ON cluster (stack_id); CREATE TABLE component @@ -99,6 +100,8 @@ CREATE TABLE component CONSTRAINT uk_component_name UNIQUE (component_name, cluster_id) ); +DROP INDEX IF EXISTS idx_component_cluster_id; +DROP INDEX IF EXISTS idx_component_service_id; CREATE INDEX idx_component_cluster_id ON component (cluster_id); CREATE INDEX idx_component_service_id ON component (service_id); @@ -115,6 +118,8 @@ CREATE TABLE host_component PRIMARY KEY (id) ); +DROP INDEX IF EXISTS idx_hc_component_id; +DROP INDEX IF EXISTS idx_hc_host_id; CREATE INDEX idx_hc_component_id ON host_component (component_id); CREATE INDEX idx_hc_host_id ON host_component (host_id); @@ -145,6 +150,7 @@ CREATE TABLE host COMMENT ON COLUMN host.physical_memory IS 'Total Physical Memory(Bytes)'; +DROP INDEX IF EXISTS idx_host_cluster_id; CREATE INDEX idx_host_cluster_id ON host (cluster_id); CREATE TABLE repo @@ -165,6 +171,7 @@ CREATE TABLE repo CONSTRAINT uk_repo_id UNIQUE (repo_id, os, arch, cluster_id) ); +DROP INDEX IF EXISTS idx_cluster_id; CREATE INDEX idx_cluster_id ON repo (cluster_id); CREATE TABLE stack @@ -205,6 +212,9 @@ CREATE TABLE task PRIMARY KEY (id) ); +DROP INDEX IF EXISTS idx_task_cluster_id; +DROP INDEX IF EXISTS idx_task_job_id; +DROP INDEX IF EXISTS idx_task_stage_id; CREATE INDEX idx_task_cluster_id ON task (cluster_id); CREATE INDEX idx_task_job_id ON task (job_id); CREATE INDEX idx_task_stage_id ON task (stage_id); @@ -223,7 +233,7 @@ CREATE TABLE job PRIMARY KEY (id) ); -CREATE INDEX idx_cluster_id ON job (cluster_id); +CREATE INDEX idx_job_cluster_id ON job (cluster_id); CREATE TABLE type_config ( @@ -307,7 +317,7 @@ CREATE TABLE stage PRIMARY KEY (id) ); -CREATE INDEX idx_cluster_id ON stage (cluster_id); +CREATE INDEX idx_stage_cluster_id ON stage (cluster_id); CREATE INDEX idx_job_id ON stage (job_id); CREATE TABLE llm_platform @@ -335,7 +345,7 @@ CREATE TABLE llm_platform_authorized PRIMARY KEY (id) ); -CREATE INDEX idx_platform_id ON llm_platform_authorized (platform_id); +CREATE INDEX idx_authorized_platform_id ON llm_platform_authorized (platform_id); CREATE TABLE llm_chat_thread ( @@ -351,8 +361,8 @@ CREATE TABLE llm_chat_thread PRIMARY KEY (id) ); -CREATE INDEX idx_platform_id ON llm_chat_thread (platform_id); -CREATE INDEX idx_user_id ON llm_chat_thread (user_id); +CREATE INDEX idx_chatthread_platform_id ON llm_chat_thread (platform_id); +CREATE INDEX idx_chatthread_user_id ON llm_chat_thread (user_id); CREATE TABLE llm_chat_message ( @@ -369,7 +379,7 @@ CREATE TABLE llm_chat_message ); CREATE INDEX idx_thread_id ON llm_chat_message (thread_id); -CREATE INDEX idx_user_id ON llm_chat_message (user_id); +CREATE INDEX idx_message_user_id ON llm_chat_message (user_id); INSERT INTO "user" (create_time, update_time, nickname, password, status, username) VALUES (now(), now(), 'Administrator', '21232f297a57a5a743894a0e4a801fc3', true, 'admin'); From 1b60ce67c10ca4c92a0b999e07efaa7d5a242f18 Mon Sep 17 00:00:00 2001 From: alexfu Date: Sat, 14 Sep 2024 22:28:16 +0800 Subject: [PATCH 13/13] BIGTOP-4223: Support PostgreSQL for mybatis --- .../resources/mapper/postgresql/HostComponentMapper.xml | 8 ++++---- .../src/main/resources/mapper/postgresql/RepoMapper.xml | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/bigtop-manager-dao/src/main/resources/mapper/postgresql/HostComponentMapper.xml b/bigtop-manager-dao/src/main/resources/mapper/postgresql/HostComponentMapper.xml index 51060006..214dbc90 100644 --- a/bigtop-manager-dao/src/main/resources/mapper/postgresql/HostComponentMapper.xml +++ b/bigtop-manager-dao/src/main/resources/mapper/postgresql/HostComponentMapper.xml @@ -115,7 +115,7 @@ - , c.component_name + , c.component_name, h.hostname from host_component hc inner join (select * from host @@ -134,7 +134,7 @@ on hc.host_id = h.id inner join (select * from component - + component_name = #{componentName} @@ -166,7 +166,7 @@ and id = #{hostId} - ) ht + ) ht on hc.host_id = ht.id @@ -176,7 +176,7 @@ - , c.service_id, c.component_name, c.display_name, c.category, s.service_name, clus.cluster_name, ht.hostname + , c.service_id, c.component_name, c.display_name, c.category, s.service_name, clus.cluster_name, ht.hostname from host_component hc inner join (select * from component diff --git a/bigtop-manager-dao/src/main/resources/mapper/postgresql/RepoMapper.xml b/bigtop-manager-dao/src/main/resources/mapper/postgresql/RepoMapper.xml index b51ab044..cee5e868 100644 --- a/bigtop-manager-dao/src/main/resources/mapper/postgresql/RepoMapper.xml +++ b/bigtop-manager-dao/src/main/resources/mapper/postgresql/RepoMapper.xml @@ -24,7 +24,7 @@ - id, base_url, os, arch, repo_id, repo_name, cluster_id + id, base_url, os, arch, repo_id, repo_name, repo_type, cluster_id - insert into repo (base_url, os, arch, repo_id, repo_name, cluster_id, create_by, update_by, create_time, update_time) + insert into repo (base_url, os, arch, repo_id, repo_name, repo_type, cluster_id, create_by, update_by, create_time, update_time) values - (#{cluster.baseUrl},#{cluster.os},#{cluster.arch},#{cluster.repoId},#{cluster.repoName},#{cluster.clusterId},#{cluster.createBy},#{cluster.updateBy},#{cluster.createTime},#{cluster.updateTime}) + (#{cluster.baseUrl},#{cluster.os},#{cluster.arch},#{cluster.repoId},#{cluster.repoName},#{cluster.repoType},#{cluster.clusterId},#{cluster.createBy},#{cluster.updateBy},#{cluster.createTime},#{cluster.updateTime})