From 801d55d032953ddf32c6b1ee8689fa060a154097 Mon Sep 17 00:00:00 2001 From: limeng Date: Wed, 30 Mar 2022 09:59:00 +0800 Subject: [PATCH 1/3] dataAssets dataWarehouse dataModel --- .../governance/DataAssetsRemoteClient.scala | 1 - .../impl/LinkisDataAssetsRemoteClient.scala | 1 - .../TestDataAssetsRemoteClient.scala | 23 +- .../data-assets-server/pom.xml | 20 +- .../data/governance/atlas/AtlasClient.java | 20 +- .../data/governance/atlas/AtlasService.java | 1 - .../data/governance/conf/GovernanceConf.java | 2 +- .../data/governance/dao/PartInfoMapper.java | 23 ++ .../dao/TableStorageInfoMapper.java | 36 ++ .../governance/dao/TableStorageMapper.java | 35 ++ .../governance/entity/HiveTblDetailInfo.java | 1 - .../DSSDataGovernanceAssetRestful.java | 308 +++++++++--------- .../data/governance/service/AssetService.java | 11 + .../service/AuthenticationClientStrategy.java | 2 +- .../service/impl/AssetServiceImpl.java | 107 ++++-- .../src/main/resources/application.yml | 15 +- .../resources/atlas-application.properties | 67 ++++ .../src/main/resources/linkis.properties | 10 +- .../governance/DataAssetsApplication.scala | 3 +- .../dss-datamodel-center-common/pom.xml | 5 + .../center/common/event/UnBindLabelEvent.java | 1 - .../src/main/resources/application.yml | 2 + .../src/main/resources/linkis.properties | 10 +- .../restful/DimensionRestfulApi.java | 57 ++-- .../restful/IndicatorRestfulApi.java | 116 +++---- .../measure/restful/MeasureRestfulApi.java | 61 ++-- .../datamodel/table/dao/TableQueryMapper.java | 2 +- .../materialized/HiveSchemaDdlBuilder.java | 2 +- .../table/restful/TableRestfulApi.java | 267 +++++++-------- .../client/GovernanceDwRemoteClient.scala | 1 + .../linkis/mybatis/DataSourceConfig.java | 2 +- .../mybatis/MyBatisMapperScannerConfig.java | 3 +- .../mybatis/MybatisConfigurationFactory.java | 2 +- .../mybatis/conf/MybatisConfiguration.java | 2 +- .../warehouse/restful/DwDsRestfulApi.java | 43 ++- .../warehouse/restful/DwLayerRestfulApi.java | 116 +++---- .../restful/DwModifierRestfulApi.java | 116 +++---- .../DwStatisticalPeriodRestfulApi.java | 121 ++++--- .../restful/DwThemeDomainRestfulApi.java | 114 +++---- .../src/main/resources/application.yml | 2 + .../src/main/resources/linkis.properties | 2 +- .../service/impl/DwLayerServiceImpl.java | 1 - .../governance-dao/pom.xml | 6 + 43 files changed, 959 insertions(+), 781 deletions(-) create mode 100644 dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/dao/PartInfoMapper.java create mode 100644 dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/dao/TableStorageInfoMapper.java create mode 100644 dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/dao/TableStorageMapper.java create mode 100644 dss-apps/dss-dataasset-management/data-assets-server/src/main/resources/atlas-application.properties diff --git a/dss-apps/dss-dataasset-management/data-assets-client/src/main/scala/com/webank/wedatasphere/dss/data/governance/DataAssetsRemoteClient.scala b/dss-apps/dss-dataasset-management/data-assets-client/src/main/scala/com/webank/wedatasphere/dss/data/governance/DataAssetsRemoteClient.scala index 17144448bf..3446f45a81 100644 --- a/dss-apps/dss-dataasset-management/data-assets-client/src/main/scala/com/webank/wedatasphere/dss/data/governance/DataAssetsRemoteClient.scala +++ b/dss-apps/dss-dataasset-management/data-assets-client/src/main/scala/com/webank/wedatasphere/dss/data/governance/DataAssetsRemoteClient.scala @@ -1,6 +1,5 @@ package com.webank.wedatasphere.dss.data.governance - import com.webank.wedatasphere.dss.data.governance.request._ import com.webank.wedatasphere.dss.data.governance.response._ diff --git a/dss-apps/dss-dataasset-management/data-assets-client/src/main/scala/com/webank/wedatasphere/dss/data/governance/impl/LinkisDataAssetsRemoteClient.scala b/dss-apps/dss-dataasset-management/data-assets-client/src/main/scala/com/webank/wedatasphere/dss/data/governance/impl/LinkisDataAssetsRemoteClient.scala index 136216f2cb..2335bcca3c 100644 --- a/dss-apps/dss-dataasset-management/data-assets-client/src/main/scala/com/webank/wedatasphere/dss/data/governance/impl/LinkisDataAssetsRemoteClient.scala +++ b/dss-apps/dss-dataasset-management/data-assets-client/src/main/scala/com/webank/wedatasphere/dss/data/governance/impl/LinkisDataAssetsRemoteClient.scala @@ -1,6 +1,5 @@ package com.webank.wedatasphere.dss.data.governance.impl - import com.webank.wedatasphere.dss.data.governance.request._ import com.webank.wedatasphere.dss.data.governance.response._ import com.webank.wedatasphere.dss.data.governance.{AbstractRemoteClient, DataAssetsRemoteClient} diff --git a/dss-apps/dss-dataasset-management/data-assets-client/src/test/scala/com/webank/wedatasphere/dss/data/governance/TestDataAssetsRemoteClient.scala b/dss-apps/dss-dataasset-management/data-assets-client/src/test/scala/com/webank/wedatasphere/dss/data/governance/TestDataAssetsRemoteClient.scala index 67a61673f4..f259f590ca 100644 --- a/dss-apps/dss-dataasset-management/data-assets-client/src/test/scala/com/webank/wedatasphere/dss/data/governance/TestDataAssetsRemoteClient.scala +++ b/dss-apps/dss-dataasset-management/data-assets-client/src/test/scala/com/webank/wedatasphere/dss/data/governance/TestDataAssetsRemoteClient.scala @@ -1,7 +1,8 @@ package com.webank.wedatasphere.dss.data.governance import com.webank.wedatasphere.dss.data.governance.impl.LinkisDataAssetsRemoteClient -import com.webank.wedatasphere.dss.data.governance.request.{GetHiveTblPartitionAction, GetTblPartInfoByNameAction} +import com.webank.wedatasphere.dss.data.governance.request.{GetHiveTblPartitionAction, GetTblPartInfoByNameAction, SearchHiveDbAction} +import org.apache.linkis.httpclient.dws.authentication.TokenAuthenticationStrategy import org.apache.linkis.httpclient.dws.config.DWSClientConfigBuilder import java.util.concurrent.TimeUnit @@ -11,7 +12,7 @@ import scala.Console.println object TestDataAssetsRemoteClient { def main(args: Array[String]): Unit = { val clientConfig = DWSClientConfigBuilder.newBuilder() - .addServerUrl("http://localhost:20082") + .addServerUrl("http://192.168.0.25:8088") .connectionTimeout(30000) .discoveryEnabled(false) .discoveryFrequency(1,TimeUnit.MINUTES) @@ -19,9 +20,9 @@ object TestDataAssetsRemoteClient { .maxConnectionSize(5) .retryEnabled(false) .readTimeout(30000) - .setAuthenticationStrategy(null) + .setAuthenticationStrategy(new TokenAuthenticationStrategy()) .setAuthTokenKey("hdfs") - .setAuthTokenValue("hdfs") + .setAuthTokenValue("BML-AUTH") .setDWSVersion("v1") .build() @@ -30,16 +31,18 @@ object TestDataAssetsRemoteClient { // val searchHiveTblResult = dataAssetsClient.searchHiveTbl(SearchHiveTblAction.builder().setUser("hdfs").setQuery("").setLimit(10).setOffset(0).setOwner("undefined").build()).getHiveList // println(searchHiveTblResult) // -// val searchHiveDbResult = dataAssetsClient.searchHiveDb(SearchHiveDbAction.builder().setUser("hdfs").setQuery("").setLimit(10).setOffset(0).setOwner("undefined").build()).getHiveList -// println(searchHiveDbResult) + val searchHiveDbResult = dataAssetsClient.searchHiveDb(SearchHiveDbAction.builder().setUser("hdfs").setQuery("").setLimit(10).setOffset(0).setOwner("undefined").build()).getHiveList + println(searchHiveDbResult) // // val hiveTblBasicResult = dataAssetsClient // .getHiveTblBasic(GetHiveTblBasicAction.builder().setUser("hdfs").setGuid("27920dc8-1eef-4d7d-9423-b5967d9e2d33").build()) // .result // println(hiveTblBasicResult) // - val hiveTblPartitionResult = dataAssetsClient.getHiveTblPartition(GetHiveTblPartitionAction.builder().setUser("hdfs").setGuid("a3be4a97-6465-4c3d-adee-76dfa662e531").build()).result - println(hiveTblPartitionResult) +// val hiveTblPartitionResult = dataAssetsClient.getHiveTblPartition(GetHiveTblPartitionAction.builder().setUser("hdfs").setGuid("a3be4a97-6465-4c3d-adee-76dfa662e531").build()).result +// println(hiveTblPartitionResult) + + // // val hiveTblCreateResult = dataAssetsClient.getHiveTblCreate(GetHiveTblCreateAction.builder().setUser("hdfs").setGuid("a3be4a97-6465-4c3d-adee-76dfa662e531").build()).result // println(hiveTblCreateResult) @@ -102,7 +105,7 @@ object TestDataAssetsRemoteClient { // // println(hiveTableStatsResult.getResult) - val partInfo = dataAssetsClient.getHiveTblPartInfoByNameResult(GetTblPartInfoByNameAction.builder().setUser("hdfs").setDbName("linkis_db").setTableName("linkis_partitions").build()) - println(partInfo.getInfo) +// val partInfo = dataAssetsClient.getHiveTblPartInfoByNameResult(GetTblPartInfoByNameAction.builder().setUser("hdfs").setDbName("linkis_db").setTableName("linkis_partitions").build()) +// println(partInfo.getInfo) } } diff --git a/dss-apps/dss-dataasset-management/data-assets-server/pom.xml b/dss-apps/dss-dataasset-management/data-assets-server/pom.xml index 1bd85be8ee..f5fbae9980 100644 --- a/dss-apps/dss-dataasset-management/data-assets-server/pom.xml +++ b/dss-apps/dss-dataasset-management/data-assets-server/pom.xml @@ -15,6 +15,7 @@ 2.2.0 8 8 + 2.7.2 @@ -31,6 +32,10 @@ com.fasterxml.jackson.core jackson-core + + com.sun.jersey + jersey-server + @@ -48,6 +53,7 @@ ${linkis.version} + org.apache.atlas atlas-client-v2 @@ -57,8 +63,13 @@ com.google.guava guava + + com.sun.jersey + jersey-server + + org.projectlombok lombok @@ -77,6 +88,13 @@ dss-framework-workspace-client 1.0.1 + + + mysql + mysql-connector-java + 5.1.49 + + @@ -115,7 +133,7 @@ false - data-assets-server + data-assets false false diff --git a/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/atlas/AtlasClient.java b/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/atlas/AtlasClient.java index fbcda8ddc4..0af0a64f0e 100644 --- a/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/atlas/AtlasClient.java +++ b/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/atlas/AtlasClient.java @@ -138,7 +138,7 @@ public String createSubClassification(String name, String superType) throws Atla atlasTypesDef.setClassificationDefs(Lists.newArrayList(atlasClassificationDef)); MultivaluedMap params = new MultivaluedMapImpl(); params.add("type","classification"); - return callAPI(AtlasClientV2.API_V2.CREATE_TYPE_DEFS, String.class, gson.toJson(atlasTypesDef),params); + return callAPI(API_V2.CREATE_TYPE_DEFS, String.class, gson.toJson(atlasTypesDef),params); } // Glossary APIs @@ -233,7 +233,23 @@ public String basicSearchPostForString(final String typeName, final String class searchParameters.setIncludeClassificationAttributes(true); searchParameters.setExcludeDeletedEntities(excludeDeletedEntities); searchParameters.setAttributes(returnColumnsParams); - return callAPI(AtlasClientV2.API_V2.FACETED_SEARCH, String.class, gson.toJson(searchParameters),new MultivaluedMapImpl()); + return callAPI(API_V2.FACETED_SEARCH, String.class, gson.toJson(searchParameters),new MultivaluedMapImpl()); + } + + /** + * 根据关键字检索实体 + */ + public String basicSearchPostForLabel(final String typeName, final String termName,final boolean excludeDeletedEntities) throws AtlasServiceException { + SearchParameters searchParameters = new SearchParameters(); + Set returnColumnsParams = Sets.newHashSet( "parameters", "lastAccessTime"); + searchParameters.setTypeName(typeName); + searchParameters.setTermName(termName); + searchParameters.setIncludeSubClassifications(true); + searchParameters.setIncludeSubTypes(true); + searchParameters.setIncludeClassificationAttributes(true); + searchParameters.setExcludeDeletedEntities(excludeDeletedEntities); + searchParameters.setAttributes(returnColumnsParams); + return callAPI(API_V2.FACETED_SEARCH, String.class, gson.toJson(searchParameters),new MultivaluedMapImpl()); } /** diff --git a/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/atlas/AtlasService.java b/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/atlas/AtlasService.java index 3a5046ee5f..1ca71c6c5a 100644 --- a/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/atlas/AtlasService.java +++ b/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/atlas/AtlasService.java @@ -116,7 +116,6 @@ public List searchHiveTable0(String classification, String qu //atlasClient.createAtlasTypeDefs() return atlasSearchResult.getEntities(); } - /** * 创建子类型 * diff --git a/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/conf/GovernanceConf.java b/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/conf/GovernanceConf.java index 9769d05482..3845cb15fa 100644 --- a/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/conf/GovernanceConf.java +++ b/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/conf/GovernanceConf.java @@ -18,8 +18,8 @@ package com.webank.wedatasphere.dss.data.governance.conf; -import org.apache.linkis.common.conf.CommonVars; +import org.apache.linkis.common.conf.CommonVars; public interface GovernanceConf { CommonVars ATLAS_REST_ADDRESS = CommonVars.apply("atlas.rest.address"); diff --git a/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/dao/PartInfoMapper.java b/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/dao/PartInfoMapper.java new file mode 100644 index 0000000000..62fd7af1fb --- /dev/null +++ b/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/dao/PartInfoMapper.java @@ -0,0 +1,23 @@ +package com.webank.wedatasphere.dss.data.governance.dao; + + +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import com.webank.wedatasphere.dss.data.governance.entity.PartInfo; +import org.apache.ibatis.annotations.Param; +import org.apache.ibatis.annotations.Select; +import java.util.List; + +public interface PartInfoMapper extends BaseMapper { + + String sql = "select b.PART_NAME,b.CREATE_TIME,\n" + + " MAX(CASE c.PARAM_KEY WHEN 'transient_lastDdlTime' THEN c.PARAM_VALUE ELSE null END) last_access_time ,\n" + + " MAX(CASE c.PARAM_KEY WHEN 'numRows' THEN c.PARAM_VALUE ELSE null END) reord_cnt,\n" + + " MAX(CASE c.PARAM_KEY WHEN 'totalSize' THEN c.PARAM_VALUE ELSE null END) store,\n" + + " MAX(CASE c.PARAM_KEY WHEN 'numFiles' THEN c.PARAM_VALUE ELSE null END) file_count\n" + + " from TBLS a,PARTITIONS b,PARTITION_PARAMS c,DBS d \n" + + " where a.TBL_NAME= #{tableName} AND d.NAME= #{dbName} AND a.TBL_ID=b.TBL_ID AND a.DB_ID=d.DB_ID AND b.PART_ID=c.PART_ID \n" + + " GROUP BY c.PART_ID"; + @Select(sql) + List query(@Param("dbName") String dbName, @Param("tableName") String tableName) ; + +} diff --git a/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/dao/TableStorageInfoMapper.java b/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/dao/TableStorageInfoMapper.java new file mode 100644 index 0000000000..a17c5efa44 --- /dev/null +++ b/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/dao/TableStorageInfoMapper.java @@ -0,0 +1,36 @@ +package com.webank.wedatasphere.dss.data.governance.dao; + +import com.baomidou.mybatisplus.core.conditions.Wrapper; +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import com.baomidou.mybatisplus.core.toolkit.Constants; +import com.webank.wedatasphere.dss.data.governance.entity.TableInfo; +import org.apache.ibatis.annotations.Param; +import org.apache.ibatis.annotations.Select; +import java.util.List; + +public interface TableStorageInfoMapper extends BaseMapper { + + //普通表获取表名和存储量 + String sql= "select CONCAT(DBS.NAME, '.', TBLS.TBL_NAME) AS table_name, CASE WHEN TABLE_PARAMS.PARAM_KEY = 'totalSize' THEN TABLE_PARAMS.PARAM_VALUE ELSE 0 END as totalSize\n" + + "from DBS, TBLS,TABLE_PARAMS\n" + + "where TBLS.TBL_ID=TABLE_PARAMS.TBL_ID AND TBLS.DB_ID=DBS.DB_ID AND TABLE_PARAMS.PARAM_KEY = 'totalSize'"; + + //分区表获取表名和存储量 + String sql2="select CONCAT(DBS.NAME, '.', TBLS.TBL_NAME) AS table_name,SUM(CASE WHEN PARTITION_PARAMS.PARAM_KEY = 'totalSize' THEN PARTITION_PARAMS.PARAM_VALUE ELSE 0 END) as totalSize\n" + + " from DBS,TBLS,PARTITIONS ,PARTITION_PARAMS \n" + + " where DBS.DB_ID=TBLS.DB_ID AND TBLS.TBL_ID=PARTITIONS.TBL_ID AND PARTITIONS.PART_ID =PARTITION_PARAMS.PART_ID AND PARTITION_PARAMS.PARAM_KEY = 'totalSize'\n" + + " group by table_name"; + + //合并分区表和普通表查询结果，并根据"DB.Table"过滤 + String queryWrapperSql = "select table_name as tableName,totalSize as storage FROM (( " + sql + ")" + " UNION " +"(" + sql2 +")) as q ${ew.customSqlSegment}"; + + //合并分区表和普通表查询结果 + String querySql = "select table_name as tableName,totalSize as storage FROM (( " + sql + ")" + " UNION " +"(" + sql2 +")) as q"; + + @Select(querySql) + List query(); + + @Select(queryWrapperSql) + List queryByTableName(@Param(Constants.WRAPPER) Wrapper queryWrapper); + +} diff --git a/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/dao/TableStorageMapper.java b/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/dao/TableStorageMapper.java new file mode 100644 index 0000000000..0ba7aff6b3 --- /dev/null +++ b/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/dao/TableStorageMapper.java @@ -0,0 +1,35 @@ +package com.webank.wedatasphere.dss.data.governance.dao; + +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import org.apache.ibatis.annotations.Mapper; +import org.apache.ibatis.annotations.Param; +import org.apache.ibatis.annotations.Select; + +import java.util.List; + + +@Mapper +public interface TableStorageMapper extends BaseMapper { + + String sql="select SUM(PARAM_VALUE) as storage from TABLE_PARAMS WHERE PARAM_KEY='totalSize'"; + + String sql2="select SUM(PARAM_VALUE) as storage from PARTITION_PARAMS WHERE PARAM_KEY='totalSize'"; + + String querySql = "select sum(storage) as storage from((" + sql + ") union all (" + sql2 + ")) as q"; + + @Select(querySql) + List getTableStorage(); + + String commonSql="select TABLE_PARAMS.PARAM_VALUE as totalSize from DBS, TBLS,TABLE_PARAMS where TBLS.TBL_ID=TABLE_PARAMS.TBL_ID AND TBLS.DB_ID=DBS.DB_ID AND TABLE_PARAMS.PARAM_KEY='totalSize' AND DBS.NAME= #{dbName} AND TBLS.TBL_NAME=#{tableName}"; + @Select(commonSql) + List getTableInfo(@Param("dbName") String dbName,@Param("tableName") String tableName); + + + String partitionSql="select SUM(PARTITION_PARAMS.PARAM_VALUE) as totalSize\n" + + "from DBS,TBLS,PARTITIONS ,PARTITION_PARAMS\n" + + "where DBS.DB_ID=TBLS.DB_ID AND TBLS.TBL_ID=PARTITIONS.TBL_ID AND PARTITIONS.PART_ID =PARTITION_PARAMS.PART_ID AND PARTITION_PARAMS.PARAM_KEY='totalSize' AND DBS.NAME= #{dbName} AND TBLS.TBL_NAME= #{tableName}\n" + + "group by TBLS.TBL_NAME"; + + @Select(partitionSql) + List getPartTableInfo(@Param("dbName") String dbName,@Param("tableName") String tableName); +} diff --git a/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/entity/HiveTblDetailInfo.java b/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/entity/HiveTblDetailInfo.java index 97cae23c02..dcf135ad66 100644 --- a/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/entity/HiveTblDetailInfo.java +++ b/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/entity/HiveTblDetailInfo.java @@ -29,5 +29,4 @@ public static class HiveColumnInfo { private String guid; private String comment; } - } diff --git a/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/restful/DSSDataGovernanceAssetRestful.java b/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/restful/DSSDataGovernanceAssetRestful.java index 18c022b646..8c8101bdf1 100644 --- a/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/restful/DSSDataGovernanceAssetRestful.java +++ b/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/restful/DSSDataGovernanceAssetRestful.java @@ -15,26 +15,20 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; import org.springframework.util.CollectionUtils; -import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.*; import javax.annotation.Resource; import javax.servlet.http.HttpServletRequest; -import javax.ws.rs.*; -import javax.ws.rs.core.Context; -import javax.ws.rs.core.MediaType; -import javax.ws.rs.core.Response; import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Set; -@Path("/data-assets/asset") -@Consumes(MediaType.APPLICATION_JSON) -@Produces(MediaType.APPLICATION_JSON) -@Component + +@RestController +@RequestMapping(value = "/data-assets/asset", produces = {"application/json;charset=utf-8"}) public class DSSDataGovernanceAssetRestful implements AuthenticationClientStrategy { private static final Logger logger = LoggerFactory.getLogger(DSSDataGovernanceAssetRestful.class); @@ -54,26 +48,26 @@ public class DSSDataGovernanceAssetRestful implements AuthenticationClientStrate /** * 获取数据资产概要：hivedb数、hivetable数据、总存储量 */ - @GET - @Path("/hiveSummary") - public Response getHiveSummary(@Context HttpServletRequest req) throws Exception { + @RequestMapping( value = "/hiveSummary", method = RequestMethod.GET) + public Message getHiveSummary(HttpServletRequest req) throws Exception { - return Message.messageToResponse(Message.ok().data("result", assetService.getHiveSummary())); + return Message.ok().data("result", assetService.getHiveSummary()); } /** * 搜索hive表 */ - @GET - @Path("/hiveTbl/search") - public Response searchHiveTbl(@QueryParam("classification") String classification, - @QueryParam("query") String query, - @QueryParam("label") String label, - @QueryParam("type") String type, - @QueryParam("precise")@DefaultValue("0") int precise, - @QueryParam("owner") @DefaultValue("") String owner, - @QueryParam("limit") @DefaultValue(DEFAULT_LIMIT) int limit, - @QueryParam("offset") @DefaultValue(DEFAULT_OFFSET) int offset) throws Exception { + + @RequestMapping( value = "/hiveTbl/search", method = RequestMethod.GET) + public Message searchHiveTbl(HttpServletRequest req, + @RequestParam(value = "classification",required = false) String classification, + @RequestParam(value = "query",required = false) String query, + @RequestParam(value = "label",required = false) String label, + @RequestParam(value="type",required = false) String type, + @RequestParam(value="precise",defaultValue="0") int precise, + @RequestParam(value="owner",defaultValue="") String owner, + @RequestParam(value="limit",defaultValue=DEFAULT_LIMIT ) int limit, + @RequestParam(value="offset",defaultValue=DEFAULT_OFFSET ) int offset) throws Exception { //适配模型 if (ClassificationConstant.isTypeScope(type)) { @@ -93,7 +87,7 @@ public Response searchHiveTbl(@QueryParam("classification") String classificatio } List hiveTblBasicList = assetService.searchHiveTable(classification, query,label,limit, offset); if (StringUtils.isBlank(owner) || owner.equals("undefined")|| CollectionUtils.isEmpty(hiveTblBasicList)) { - return Message.messageToResponse(Message.ok().data("result", hiveTblBasicList)); + return Message.ok().data("result", hiveTblBasicList); } else { List res = new ArrayList<>(); for (HiveTblSimpleInfo hiveTblSimpleInfo : hiveTblBasicList) { @@ -101,33 +95,33 @@ public Response searchHiveTbl(@QueryParam("classification") String classificatio res.add(hiveTblSimpleInfo); } } - return Message.messageToResponse(Message.ok().data("result", res)); + return Message.ok().data("result", res); } } /** * 搜索hive表统计信息 */ - @GET - @Path("/hiveTbl/stats") - public Response searchHiveTblStats(@QueryParam("dbName") String dbName, - @QueryParam("tableName") String tableName, - @QueryParam("guid") String guid) throws Exception { + + @RequestMapping( value = "/hiveTbl/stats", method = RequestMethod.GET) + public Message searchHiveTblStats(@RequestParam(value = "dbName",required = false) String dbName, + @RequestParam(value = "tableName",required = false) String tableName, + @RequestParam(value = "guid",required = false) String guid) throws Exception { logger.info("searchHiveTblStats dbName : {}, tableName : {}, guid : {}", dbName, tableName, guid); - return Message.messageToResponse(Message.ok().data("result", assetService.hiveTblStats(dbName, tableName, guid))); + return Message.ok().data("result", assetService.hiveTblStats(dbName, tableName, guid)); } /** * 搜索hive表容量 */ - @GET - @Path("/hiveTbl/size") - public Response searchHiveTblSize(@QueryParam("dbName") String dbName, - @QueryParam("tableName") String tableName, - @QueryParam("guid") String guid) throws Exception { + + @RequestMapping( value = "/hiveTbl/size", method = RequestMethod.GET) + public Message searchHiveTblSize(@RequestParam(value = "dbName",required = false) String dbName, + @RequestParam(value = "tableName",required = false) String tableName, + @RequestParam(value = "guid",required = false) String guid) throws Exception { logger.info("searchHiveTblSize dbName : {}, tableName : {}, guid : {}", dbName, tableName, guid); - return Message.messageToResponse(Message.ok().data("result", assetService.hiveTblSize(dbName, tableName, guid))); + return Message.ok().data("result", assetService.hiveTblSize(dbName, tableName, guid)); } @@ -135,17 +129,17 @@ public Response searchHiveTblSize(@QueryParam("dbName") String dbName, /** * 搜索hive库 */ - @GET - @Path("/hiveDb/search") - public Response searchHiveDb(@QueryParam("classification") String classification, - @QueryParam("query") String query, - @QueryParam("owner") @DefaultValue("") String owner, - @QueryParam("limit") @DefaultValue(DEFAULT_LIMIT) int limit, - @QueryParam("offset") @DefaultValue(DEFAULT_OFFSET) int offset) throws Exception { + + @RequestMapping( value = "/hiveDb/search", method = RequestMethod.GET) + public Message searchHiveDb(@RequestParam(value = "classification",required = false) String classification, + @RequestParam(value = "query",required = false) String query, + @RequestParam(value="owner",defaultValue="") String owner, + @RequestParam(value="limit",defaultValue=DEFAULT_LIMIT) int limit, + @RequestParam(value="offset",defaultValue=DEFAULT_OFFSET) int offset) throws Exception { List hiveTblBasicList = assetService.searchHiveDb(classification, '*' + query + '*', limit, offset); if (StringUtils.isBlank(owner) || owner.equals("undefined")) { - return Message.messageToResponse(Message.ok().data("result", hiveTblBasicList)); + return Message.ok().data("result", hiveTblBasicList); } else { List res = new ArrayList<>(); for (HiveTblSimpleInfo hiveTblSimpleInfo : hiveTblBasicList) { @@ -153,110 +147,126 @@ public Response searchHiveDb(@QueryParam("classification") String classification res.add(hiveTblSimpleInfo); } } - return Message.messageToResponse(Message.ok().data("result", res)); + return Message.ok().data("result", res); } } /** * 获取单个表的详细信息，包括：基本信息、字段信息 */ - @GET - @Path("/hiveTbl/{guid}/basic") - public Response getHiveTblBasic(@PathParam("guid") String guid) throws Exception { - return Message.messageToResponse(Message.ok().data("result", assetService.getHiveTblDetail(guid))); + + @RequestMapping( value = "/hiveTbl/{guid}/basic", method = RequestMethod.GET) + public Message getHiveTblBasic(@PathVariable("guid") String guid) throws Exception { + return Message.ok().data("result", assetService.getHiveTblDetail(guid)); } /** * 获取表分区信息 */ - @GET - @Path("/hiveTbl/{guid}/partition") - public Response getHiveTblPartition(@PathParam("guid") String guid) throws Exception { + + @RequestMapping( value = "/hiveTbl/{guid}/partition", method = RequestMethod.GET) + public Message getHiveTblPartition(@PathVariable("guid") String guid) throws Exception { List hiveTblPartition = assetService.getHiveTblPartition(guid); if (hiveTblPartition.size() > 0) { - return Message.messageToResponse(Message.ok().data("result", hiveTblPartition)); + return Message.ok().data("result", hiveTblPartition); } else { - return Message.messageToResponse(Message.ok().data("result", null)); + return Message.ok().data("result", null); } } /** * 根据表名获取表分区信息 */ - @GET - @Path("/hiveTbl/partition/name") - public Response getHiveTblPartitionByName(@QueryParam("dbName") String dbName,@QueryParam("tableName") String tableName) throws Exception { + + @RequestMapping( value = "/hiveTbl/partition/name", method = RequestMethod.GET) + public Message getHiveTblPartitionByName(@RequestParam("dbName") String dbName,@RequestParam("tableName") String tableName) throws Exception { logger.info("getHiveTblPartitionByName dbName : {}, tableName : {}", dbName, tableName); List hiveTblPartition = assetService.getHiveTblPartitionByName(dbName,tableName); - return Message.messageToResponse(Message.ok().data("result", hiveTblPartition)); + return Message.ok().data("result", hiveTblPartition); } /** * 获取表的血缘信息 */ - @GET - @Path("/hiveTbl/{guid}/lineage") - public Response getHiveTblLineage(@PathParam("guid") String guid, - @QueryParam("direction") @DefaultValue(DEFAULT_DIRECTION) AtlasLineageInfo.LineageDirection direction, - @QueryParam("depth") @DefaultValue(DEFAULT_DEPTH) int depth) throws Exception { - return Message.messageToResponse(Message.ok().data("result", assetService.getHiveTblLineage(guid, direction, depth))); + + @RequestMapping( value = "/hiveTbl/{guid}/lineage", method = RequestMethod.GET) + public Message getHiveTblLineage(@PathVariable("guid") String guid, + @RequestParam(value = "direction",defaultValue = DEFAULT_DIRECTION) AtlasLineageInfo.LineageDirection direction, + @RequestParam(value = "depth",defaultValue = DEFAULT_DEPTH) int depth) throws Exception { + return Message.ok().data("result", assetService.getHiveTblLineage(guid, direction, depth)); } /** * 获取表的select语句 */ - @GET - @Path("/hiveTbl/{guid}/select") - public Response getHiveTblSelect(@PathParam("guid") String guid) throws Exception { + @RequestMapping( value = "/hiveTbl/{guid}/select", method = RequestMethod.GET) + public Message getHiveTblSelect(@PathVariable("guid") String guid) throws Exception { - return Message.messageToResponse(Message.ok().data("result", assetService.getTbSelect(guid))); + + return Message.ok().data("result", assetService.getTbSelect(guid)); } /** * 获取表的create语句 */ - @GET - @Path("/hiveTbl/{guid}/create") - public Response getHiveTblCreate(@PathParam("guid") String guid) throws Exception { - return Message.messageToResponse(Message.ok().data("result", assetService.getTbCreate(guid))); + + @RequestMapping( value = "/hiveTbl/{guid}/create", method = RequestMethod.GET) + public Message getHiveTblCreate(@PathVariable("guid") String guid) throws Exception { + return Message.ok().data("result", assetService.getTbCreate(guid)); } /** * 获取存储量前10的表信息 */ - @GET - @Path("/hiveTbl/topStorage") - public Response getTop10Storage(@Context HttpServletRequest req) throws Exception { + + @RequestMapping( value = "/hiveTbl/topStorage", method = RequestMethod.GET) + public Message getTop10Storage(HttpServletRequest req) throws Exception { List top10Table = assetService.getTop10Table(); - return Message.messageToResponse(Message.ok().data("result", top10Table)); + return Message.ok().data("result", top10Table); + } + + /** + * 根据标签和日期获取存储量前10的表信息 + */ + + @RequestMapping( value = "/hiveTbl/topStorageByLabel", method = RequestMethod.GET) + public Message getTop10StorageByLabel(@RequestParam(value = "label",defaultValue = "") String label, + @RequestParam(value = "startDate",required = false) String startDate, + @RequestParam(value = "endDate",required = false) String endDate) { + try{ + List top10Table = assetService.getTop10TableByLabelDay(label,startDate,endDate); + return Message.ok().data("result", top10Table); + }catch (Exception e){ + logger.error("topStorageByLabel msg:{}", e.getMessage()); + } + return Message.ok(); } /** * 修改单个表或单个列注释 */ - @PUT - @Path("/comment/{guid}") - public Response modifyComment(@PathParam("guid") String guid, @QueryParam("comment") String comment) throws Exception { + + @RequestMapping( value = "/comment/{guid}", method = RequestMethod.PUT) + public Message modifyComment(@PathVariable("guid") String guid, @RequestParam(value = "comment",required = false) String comment) throws Exception { comment = "\"" + comment + "\""; assetService.modifyComment(guid, comment); - return Message.messageToResponse(Message.ok().data("result", "修改成功")); + return Message.ok().data("result", "修改成功"); } /** * 批量修改多个个表或列注释 */ - @PUT - @Path("/comment/bulk") - public Response modifyComment(@RequestBody Map commentMap) throws Exception { + @RequestMapping( value = "/comment/bulk", method = RequestMethod.PUT) + public Message modifyComment(@RequestBody Map commentMap) throws Exception { for (Map.Entry stringStringEntry : commentMap.entrySet()) { stringStringEntry.setValue("\"" + stringStringEntry.getValue() + "\""); } assetService.bulkModifyComment(commentMap); - return Message.messageToResponse(Message.ok().data("result", "修改成功")); + return Message.ok().data("result", "修改成功"); } /** @@ -267,11 +277,11 @@ public Response modifyComment(@RequestBody Map commentMap) throw * @return * @throws Exception */ - @POST - @Path("/model/type") - public Response createModelType(@Context HttpServletRequest req, @RequestBody CreateModelTypeVO vo) throws Exception { + + @RequestMapping( value = "/model/type", method = RequestMethod.POST) + public Message createModelType(HttpServletRequest req, @RequestBody CreateModelTypeVO vo) throws Exception { logger.info("createModelType : {}", vo); - return Message.messageToResponse(Message.ok().data("result", assetService.createModelType(vo))); + return Message.ok().data("result", assetService.createModelType(vo)); } /** @@ -282,12 +292,12 @@ public Response createModelType(@Context HttpServletRequest req, @RequestBody Cr * @return * @throws Exception */ - @POST - @Path("/model/type/delete") - public Response deleteModelType(@Context HttpServletRequest req, @RequestBody DeleteModelTypeVO vo) throws Exception { + + @RequestMapping( value = "/model/type/delete", method = RequestMethod.POST) + public Message deleteModelType(HttpServletRequest req, @RequestBody DeleteModelTypeVO vo) throws Exception { logger.info("deleteModelTypeVO : {}", vo); assetService.deleteModelType(vo); - return Message.messageToResponse(Message.ok().data("result", "删除成功")); + return Message.ok().data("result", "删除成功"); } @@ -299,12 +309,12 @@ public Response deleteModelType(@Context HttpServletRequest req, @RequestBody De * @return * @throws Exception */ - @POST - @Path("/model/bind") - public Response bindModelType(@Context HttpServletRequest req, @RequestBody BindModelVO vo) throws Exception { + + @RequestMapping( value = "/model/bind", method = RequestMethod.POST) + public Message bindModelType( HttpServletRequest req, @RequestBody BindModelVO vo) throws Exception { logger.info("bindModelVO : {}", vo); assetService.bindModelType(vo); - return Message.messageToResponse(Message.ok().data("result", "绑定成功")); + return Message.ok().data("result", "绑定成功"); } /** @@ -315,12 +325,12 @@ public Response bindModelType(@Context HttpServletRequest req, @RequestBody Bind * @return * @throws Exception */ - @POST - @Path("/model/unbind") - public Response unBindModelType(@Context HttpServletRequest req, @RequestBody UnBindModelVO vo) throws Exception { + + @RequestMapping( value = "/model/unbind", method = RequestMethod.POST) + public Message unBindModelType(HttpServletRequest req, @RequestBody UnBindModelVO vo) throws Exception { logger.info("unBindModelVO : {}", vo); assetService.unBindModel(vo); - return Message.messageToResponse(Message.ok().data("result", "解绑成功")); + return Message.ok().data("result", "解绑成功"); } @@ -332,11 +342,11 @@ public Response unBindModelType(@Context HttpServletRequest req, @RequestBody Un * @return * @throws Exception */ - @POST - @Path("/model/type/modify") - public Response updateModelType(@Context HttpServletRequest req, @RequestBody UpdateModelTypeVO vo) throws Exception { + + @RequestMapping( value = "/model/type/modify", method = RequestMethod.POST) + public Message updateModelType(HttpServletRequest req, @RequestBody UpdateModelTypeVO vo) throws Exception { logger.info("updateModelTypeVO : {}", vo); - return Message.messageToResponse(Message.ok().data("result", assetService.updateModelType(vo))); + return Message.ok().data("result", assetService.updateModelType(vo)); } /** @@ -347,11 +357,11 @@ public Response updateModelType(@Context HttpServletRequest req, @RequestBody Up * @return * @throws Exception */ - @POST - @Path("/labels") - public Response createLabel(@Context HttpServletRequest req, @RequestBody CreateLabelVO vo) throws Exception { + + @RequestMapping( value = "/labels", method = RequestMethod.POST) + public Message createLabel(HttpServletRequest req, @RequestBody CreateLabelVO vo) throws Exception { logger.info("createLabel vo : {}", vo); - return Message.messageToResponse(Message.ok().data("result", assetService.createLabel(vo))); + return Message.ok().data("result", assetService.createLabel(vo)); } /** @@ -362,11 +372,11 @@ public Response createLabel(@Context HttpServletRequest req, @RequestBody Create * @return * @throws Exception */ - @POST - @Path("/labels/modify") - public Response updateLabel(@Context HttpServletRequest req, @RequestBody UpdateLabelVO vo) throws Exception { + + @RequestMapping( value = "/labels/modify", method = RequestMethod.POST) + public Message updateLabel(HttpServletRequest req, @RequestBody UpdateLabelVO vo) throws Exception { logger.info("updateLabel vo : {}", vo); - return Message.messageToResponse(Message.ok().data("result", assetService.updateLabel(vo))); + return Message.ok().data("result", assetService.updateLabel(vo)); } /** @@ -377,12 +387,12 @@ public Response updateLabel(@Context HttpServletRequest req, @RequestBody Update * @return * @throws Exception */ - @POST - @Path("/labels/delete") - public Response deleteLabel(@Context HttpServletRequest req, @RequestBody DeleteLabelVO vo) throws Exception { + + @RequestMapping( value = "/labels/delete", method = RequestMethod.POST) + public Message deleteLabel(HttpServletRequest req, @RequestBody DeleteLabelVO vo) throws Exception { logger.info("deleteLabel vo : {}", vo); assetService.deleteLabel(vo); - return Message.messageToResponse(Message.ok().data("result", "删除成功")); + return Message.ok().data("result", "删除成功"); } /** @@ -393,12 +403,12 @@ public Response deleteLabel(@Context HttpServletRequest req, @RequestBody Delete * @return * @throws Exception */ - @POST - @Path("/labels/bind") - public Response bindLabel(@Context HttpServletRequest req, @RequestBody BindLabelVO vo) throws Exception { + + @RequestMapping( value = "/labels/bind", method = RequestMethod.POST) + public Message bindLabel(HttpServletRequest req, @RequestBody BindLabelVO vo) throws Exception { logger.info("bindLabel vo : {}", vo); assetService.bindLabel(vo); - return Message.messageToResponse(Message.ok().data("result", "绑定成功")); + return Message.ok().data("result", "绑定成功"); } @@ -410,12 +420,12 @@ public Response bindLabel(@Context HttpServletRequest req, @RequestBody BindLabe * @return * @throws Exception */ - @POST - @Path("/labels/unbind") - public Response unBindLabel(@Context HttpServletRequest req, @RequestBody UnBindLabelVO vo) throws Exception { + + @RequestMapping( value = "/labels/unbind", method = RequestMethod.POST) + public Message unBindLabel(HttpServletRequest req, @RequestBody UnBindLabelVO vo) throws Exception { logger.info("unBindLabel vo : {}", vo); assetService.unBindLabel(vo); - return Message.messageToResponse(Message.ok().data("result", "解绑成功")); + return Message.ok().data("result", "解绑成功"); } @@ -427,37 +437,37 @@ public Response unBindLabel(@Context HttpServletRequest req, @RequestBody UnBind * @return * @throws Exception */ - @GET - @Path("/labels/search") - public Response searchLabel(@Context HttpServletRequest req - , @QueryParam("query") String query - , @QueryParam("limit") @DefaultValue(DEFAULT_LIMIT) int limit - , @QueryParam("offset") @DefaultValue(DEFAULT_OFFSET) int offset) throws Exception { + + @RequestMapping( value = "/labels/search", method = RequestMethod.GET) + public Message searchLabel(HttpServletRequest req + , @RequestParam(value = "query",required = false) String query + , @RequestParam(value = "limit", defaultValue=DEFAULT_LIMIT) int limit + , @RequestParam(value = "offset",defaultValue=DEFAULT_OFFSET) int offset) throws Exception { logger.info("searchLabel query : {}", query); - return Message.messageToResponse(Message.ok().data("result",assetService.listLabels(query,limit,offset))); + return Message.ok().data("result",assetService.listLabels(query,limit,offset)); } /** * 设置单个表或单个列的标签 */ - @POST - @Path("/label/{guid}") - public Response setLabels(@PathParam("guid") String guid, @RequestBody Set labels) throws Exception { + + @RequestMapping( value = "/label/{guid}", method = RequestMethod.POST) + public Message setLabels(@PathVariable("guid") String guid, @RequestBody Set labels) throws Exception { logger.info("setLabels guid : {}, labels : {}", guid, labels); assetService.setLabels(guid, labels); - return Message.messageToResponse(Message.ok().data("result", "设置成功")); + return Message.ok().data("result", "设置成功"); } /** * 获取工作空间下所有用户名 */ - @GET - @Path("getWorkspaceUsers/{workspaceId}/{search}") - public Response getWorkspaceUsers(@PathParam("workspaceId") int workspaceId, @PathParam("search") String search) throws Exception { + + @RequestMapping( value = "getWorkspaceUsers/{workspaceId}/{search}", method = RequestMethod.GET) + public Message getWorkspaceUsers(@PathVariable("workspaceId") int workspaceId, @PathVariable("search") String search) throws Exception { String searchs = "%" + search + "%"; List workspaceUsers = workspaceInfoService.getWorkspaceUsers(workspaceId, searchs); - return Message.messageToResponse(Message.ok().data("result", workspaceUsers)); + return Message.ok().data("result", workspaceUsers); } @@ -468,12 +478,12 @@ public Response getWorkspaceUsers(@PathParam("workspaceId") int workspaceId, @Pa * @param workspaceId * @return */ - @GET - @Path("/users/{workspaceId}") - public Response users(@Context HttpServletRequest req, @PathParam("workspaceId") String workspaceId) throws ErrorException { + + @RequestMapping( value = "/users/{workspaceId}", method = RequestMethod.GET) + public Message users(HttpServletRequest req, @PathVariable("workspaceId") String workspaceId) throws ErrorException { logger.info("users workspaceId : {}", workspaceId); GetWorkspaceUsersResult result = linkisWorkSpaceRemoteClient.getWorkspaceUsers(GetWorkspaceUsersAction.builder().setUser(getStrategyUser(req)).setWorkspaceId(workspaceId).build()); - return Message.messageToResponse(Message.ok().data("users", result.getWorkspaceUserList())); + return Message.ok().data("users", result.getWorkspaceUserList()); } @@ -484,12 +494,12 @@ public Response users(@Context HttpServletRequest req, @PathParam("workspaceId") * @param workspaceId * @return */ - @GET - @Path("/roles/{workspaceId}") - public Response roles(@Context HttpServletRequest req, @PathParam("workspaceId") String workspaceId) throws ErrorException { + + @RequestMapping( value = "/roles/{workspaceId}", method = RequestMethod.GET) + public Message roles(HttpServletRequest req, @PathVariable("workspaceId") String workspaceId) throws ErrorException { logger.info("roles workspaceId : {}", workspaceId); GetWorkspaceUsersResult result = linkisWorkSpaceRemoteClient.getWorkspaceUsers(GetWorkspaceUsersAction.builder().setUser(getStrategyUser(req)).setWorkspaceId(workspaceId).build()); - return Message.messageToResponse(Message.ok().data("users", result.getWorkspaceRoleList())); + return Message.ok().data("users", result.getWorkspaceRoleList()); } diff --git a/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/service/AssetService.java b/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/service/AssetService.java index d0c0ded84b..2d5ff70227 100644 --- a/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/service/AssetService.java +++ b/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/service/AssetService.java @@ -46,6 +46,17 @@ public List searchHiveDb(String classification, String query, public List getTop10Table () throws DataGovernanceException, SQLException, DAOException; + /** + * 根据标签和日期获取存储量前10的表 + * @param label + * @param startDate + * @param endDate + * @return + * @throws DataGovernanceException + * @throws SQLException + * @throws DAOException + */ + public List getTop10TableByLabelDay (final String label,final String startDate,final String endDate) throws DataGovernanceException, SQLException, DAOException; /** * 创建模型 diff --git a/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/service/AuthenticationClientStrategy.java b/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/service/AuthenticationClientStrategy.java index 4a036cf58a..79d7ef6a3f 100644 --- a/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/service/AuthenticationClientStrategy.java +++ b/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/service/AuthenticationClientStrategy.java @@ -3,9 +3,9 @@ import com.webank.wedatasphere.dss.data.governance.conf.ClientStrategy; +import org.apache.commons.lang.StringUtils; import org.apache.linkis.common.conf.CommonVars; import org.apache.linkis.server.security.SecurityFilter; -import org.apache.commons.lang.StringUtils; import javax.servlet.http.HttpServletRequest; diff --git a/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/service/impl/AssetServiceImpl.java b/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/service/impl/AssetServiceImpl.java index 773318a16c..c52277f081 100644 --- a/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/service/impl/AssetServiceImpl.java +++ b/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/service/impl/AssetServiceImpl.java @@ -5,13 +5,11 @@ import com.google.gson.internal.LinkedTreeMap; import com.webank.wedatasphere.dss.data.governance.atlas.AtlasService; import com.webank.wedatasphere.dss.data.governance.dao.*; -import com.webank.wedatasphere.dss.data.governance.dao.impl.MetaInfoMapperImpl; import com.webank.wedatasphere.dss.data.governance.dto.HiveTblStatsDTO; import com.webank.wedatasphere.dss.data.governance.dto.SearchLabelDTO; import com.webank.wedatasphere.dss.data.governance.entity.*; import com.webank.wedatasphere.dss.data.governance.exception.DAOException; import com.webank.wedatasphere.dss.data.governance.exception.DataGovernanceException; -import com.webank.wedatasphere.dss.data.governance.restful.DSSDataGovernanceAssetRestful; import com.webank.wedatasphere.dss.data.governance.service.AssetService; import com.webank.wedatasphere.dss.data.governance.utils.DateUtil; import com.webank.wedatasphere.dss.data.governance.vo.*; @@ -31,6 +29,8 @@ import org.springframework.util.CollectionUtils; import javax.annotation.Resource; +import java.text.ParseException; +import java.text.SimpleDateFormat; import java.util.*; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; @@ -40,22 +40,32 @@ @Service public class AssetServiceImpl implements AssetService { private static final Logger logger = LoggerFactory.getLogger(AssetServiceImpl.class); + @Resource private AtlasService atlasService; - private MetaInfoMapper metaInfoMapper; private WorkspaceInfoMapper workspaceInfoMapper; + private static SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy MM-dd HH:mm:ss"); + @Resource private TableColumnCountQueryMapper tableColumnCountQueryMapper; @Resource private TableSizeInfoMapper tableSizeInfoMapper; + @Resource + private TableStorageMapper tableStorageMapper; + + @Resource + private PartInfoMapper partInfoMapper; + + @Resource + private TableStorageInfoMapper tableStorageInfoMapper; + @Resource private TableSizePartitionInfoMapper tableSizePartitionInfoMapper; public AssetServiceImpl(AtlasService atlasService) { this.atlasService = atlasService; - this.metaInfoMapper = new MetaInfoMapperImpl(); } @Override @@ -65,10 +75,10 @@ public Map getHiveSummary() throws DataGovernanceException { result.put("hiveDb", atlasService.getHiveDbCnt()); result.put("hiveTable", atlasService.getHiveTableCnt()); - result.put("hiveStore", metaInfoMapper.getTableStorage()); + result.put("hiveStore", tableStorageMapper.getTableStorage().get(0)); return result; - } catch (AtlasServiceException | DAOException exception) { + } catch (AtlasServiceException exception) { throw new DataGovernanceException(23000, exception.getMessage()); } } @@ -287,14 +297,16 @@ public HiveTblDetailInfo getHiveTblDetail(String guid) throws DataGovernanceExce AtlasEntity atlasEntity = atlasService.getHiveTbl(guid); Map hiveTblNameAndIsPartById = atlasService.getHiveTblNameAndIsPartById(guid); Boolean isPartTable = (Boolean) hiveTblNameAndIsPartById.get("isPartition"); - int storage = 0; + long storage = 0; String db_name = String.valueOf(atlasEntity.getAttributes().get("qualifiedName")).split("@")[0]; String tableName = db_name.split("\\.")[1]; String dbName = db_name.split("\\.")[0]; - try { - storage = metaInfoMapper.getTableInfo(dbName, tableName, isPartTable); - } catch (DAOException e) { - e.printStackTrace(); + if (isPartTable){ + List partTableInfo = tableStorageMapper.getPartTableInfo(dbName, tableName); + storage = partTableInfo.isEmpty()?0:partTableInfo.get(0); + }else{ + List tableInfo = tableStorageMapper.getTableInfo(dbName, tableName); + storage = tableInfo.isEmpty()?0:tableInfo.get(0); } List guids = new ArrayList<>(); List> columns = (List>) atlasEntity.getAttributes().get("columns"); @@ -334,7 +346,7 @@ public HiveTblDetailInfo getHiveTblDetail(String guid) throws DataGovernanceExce basic.setName(tableName); basic.setQualifiedName(atlasEntity.getAttribute("qualifiedName").toString()); basic.setOwner(String.valueOf(atlasEntity.getAttributes().get("owner"))); - basic.setCreateTime(new java.text.SimpleDateFormat("yyyy MM-dd HH:mm:ss").format(atlasEntity.getCreateTime())); + basic.setCreateTime(new SimpleDateFormat("yyyy MM-dd HH:mm:ss").format(atlasEntity.getCreateTime())); basic.setStore(String.valueOf(storage)); //设置标签 Set labels = Sets.newHashSet(); @@ -349,8 +361,17 @@ public HiveTblDetailInfo getHiveTblDetail(String guid) throws DataGovernanceExce basic.setIsParTbl(isPartTable); basic.setGuid(guid); +// if (!CollectionUtils.isEmpty(atlasEntity.getClassifications())) { +// basic.setClassifications(atlasEntity.getClassifications().stream().map(AtlasStruct::getTypeName).collect(Collectors.toList())); +// } if (!CollectionUtils.isEmpty(atlasEntity.getClassifications())) { - basic.setClassifications(atlasEntity.getClassifications().stream().map(AtlasStruct::getTypeName).collect(Collectors.toList())); + basic.setClassifications(atlasEntity.getClassifications().stream() + .map(AtlasStruct::getTypeName) + .filter(typeName ->{ + String[] split = typeName.split("_"); + int modeType = ClassificationConstant.valueOf(split[0].toUpperCase(Locale.ROOT)).getType(); + return (modeType == 3 || modeType == 4); + }).collect(Collectors.toList())); } Object comment = atlasEntity.getAttribute("comment"); if (comment != null) { @@ -409,12 +430,7 @@ public List getHiveTblPartition(String guid) throws DataGovernanceExce private List getPartInfos(String db_name) { String tableName = db_name.split("\\.")[1]; String dbName = db_name.split("\\.")[0]; - List partInfo = new ArrayList<>(); - try { - partInfo = metaInfoMapper.getPartInfo(dbName, tableName); - } catch (DAOException e) { - e.printStackTrace(); - } + List partInfo = partInfoMapper.query(dbName, tableName); return partInfo; } @@ -586,7 +602,56 @@ public AtlasLineageInfo getHiveTblLineage(final String guid, final AtlasLineageI @Override public List getTop10Table() throws DAOException { - return metaInfoMapper.getTop10Table(); + List tableInfoList = tableStorageInfoMapper.query(); + // 根据存储量进行排序，并获取Top10 + tableInfoList.sort((tb1,tb2)-> Long.compare(Long.parseLong(tb2.getStorage()), Long.parseLong(tb1.getStorage()))); + return tableInfoList.stream().limit(10).collect(Collectors.toList()); + } + + @Override + public List getTop10TableByLabelDay(String label,String startDate,String endDate) throws DAOException, DataGovernanceException { + + if (!StringUtils.isBlank(label)){ + label = GlossaryConstant.LABEL.formatQuery(label); + } + + //根据标签和日期去Atlas获取tableName列表 + List tableNameList = null; + try { + List atlasEntityHeaders = atlasService.searchHiveTable0(null,null,label,true,0,0); + //对Atlas查询结果根据day进行过滤，并对tableName进行格式化处理 + + if (atlasEntityHeaders != null) { + tableNameList = atlasEntityHeaders.stream() + .filter(atlasEntityHeader -> { + long createTime = 0; + Object ct = atlasEntityHeader.getAttribute("createTime"); + if (ct != null) { + try { + createTime = simpleDateFormat.parse(DateUtil.unixToTimeStr((Double) ct)).getTime()/1000; + } catch (ParseException e) { + e.printStackTrace(); + } + } + return Long.parseLong(startDate) < createTime && createTime < Long.parseLong(endDate); + }).map(atlasEntityHeader -> { + String qualifiedName = atlasEntityHeader.getAttribute("qualifiedName").toString(); + return StringUtils.substringBefore(qualifiedName,"@"); + }).collect(Collectors.toList()); + } + } catch (AtlasServiceException ex) { + throw new DataGovernanceException(23000, ex.getMessage()); + } + + if (!CollectionUtils.isEmpty(tableNameList)){ + //根据表名（DB.TABLE）列表去数据库获取存储量 + List tableInfoList = tableStorageInfoMapper.queryByTableName(Wrappers.lambdaQuery().in(TableInfo::getTableName, tableNameList)); + + tableInfoList.sort((tb1,tb2)-> Long.compare(Long.parseLong(tb2.getStorage()), Long.parseLong(tb1.getStorage()))); + + return tableInfoList.stream().limit(10).collect(Collectors.toList()); + } + return null; } @@ -818,7 +883,7 @@ public List listLabels(String query, Integer limit, Integer offs } public List getHiveTblPartitionByName(String dbName,String tableName) throws Exception { - return metaInfoMapper.getPartInfo(dbName,tableName); + return partInfoMapper.query(dbName,tableName); } diff --git a/dss-apps/dss-dataasset-management/data-assets-server/src/main/resources/application.yml b/dss-apps/dss-dataasset-management/data-assets-server/src/main/resources/application.yml index 257ead5789..570a130c6a 100644 --- a/dss-apps/dss-dataasset-management/data-assets-server/src/main/resources/application.yml +++ b/dss-apps/dss-dataasset-management/data-assets-server/src/main/resources/application.yml @@ -1,19 +1,22 @@ server: - port: 20082 + port: 9355 + servlet: + context-path: /api/rest_j/v1 spring: application: - name: data-assets-server + name: data-assets eureka: client: serviceUrl: defaultZone: http://localhost:20303/eureka/ + instance: + metadata-map: + test: wedatasphere + management: endpoints: web: exposure: include: refresh,info logging: - config: classpath:log4j2.xml -#mybatis-plus: -# configuration: -# log-impl: org.apache.ibatis.logging.stdout.StdOutImpl \ No newline at end of file + config: classpath:log4j2.xml \ No newline at end of file diff --git a/dss-apps/dss-dataasset-management/data-assets-server/src/main/resources/atlas-application.properties b/dss-apps/dss-dataasset-management/data-assets-server/src/main/resources/atlas-application.properties new file mode 100644 index 0000000000..1998e2af9c --- /dev/null +++ b/dss-apps/dss-dataasset-management/data-assets-server/src/main/resources/atlas-application.properties @@ -0,0 +1,67 @@ +# Generated by Apache Ambari. Sat Feb 12 10:44:52 2022 + +atlas.audit.hbase.tablename=ATLAS_ENTITY_AUDIT_EVENTS +atlas.audit.hbase.zookeeper.quorum=hdp +atlas.audit.zookeeper.session.timeout.ms=60000 +atlas.authentication.keytab=/etc/security/keytabs/atlas.service.keytab +atlas.authentication.method.file=true +atlas.authentication.method.file.filename=/usr/hdp/current/atlas-server/conf/users-credentials.properties +atlas.authentication.method.kerberos=false +atlas.authentication.method.ldap=false +atlas.authentication.method.ldap.ad.base.dn= +atlas.authentication.method.ldap.ad.bind.dn= +atlas.authentication.method.ldap.ad.bind.password= +atlas.authentication.method.ldap.ad.default.role=ROLE_USER +atlas.authentication.method.ldap.ad.domain= +atlas.authentication.method.ldap.ad.referral=ignore +atlas.authentication.method.ldap.ad.url= +atlas.authentication.method.ldap.ad.user.searchfilter=(sAMAccountName={0}) +atlas.authentication.method.ldap.base.dn= +atlas.authentication.method.ldap.bind.dn= +atlas.authentication.method.ldap.bind.password= +atlas.authentication.method.ldap.default.role=ROLE_USER +atlas.authentication.method.ldap.groupRoleAttribute=cn +atlas.authentication.method.ldap.groupSearchBase= +atlas.authentication.method.ldap.groupSearchFilter= +atlas.authentication.method.ldap.referral=ignore +atlas.authentication.method.ldap.type=ldap +atlas.authentication.method.ldap.url= +atlas.authentication.method.ldap.user.searchfilter= +atlas.authentication.method.ldap.userDNpattern=uid= +atlas.authentication.principal=atlas +atlas.authorizer.impl=ranger +atlas.cluster.name=bigdata_all_in_one +atlas.enableTLS=false +atlas.graph.index.search.solr.mode=cloud +atlas.graph.index.search.solr.wait-searcher=true +atlas.graph.index.search.solr.zookeeper-url=hdp:2181/infra-solr +atlas.graph.storage.hbase.table=atlas_janus +atlas.graph.storage.hostname=hdp +atlas.kafka.auto.commit.enable=false +atlas.kafka.bootstrap.servers=hdp:6667 +atlas.kafka.hook.group.id=atlas +atlas.kafka.zookeeper.connect=hdp:2181 +atlas.kafka.zookeeper.connection.timeout.ms=30000 +atlas.kafka.zookeeper.session.timeout.ms=60000 +atlas.kafka.zookeeper.sync.time.ms=20 +atlas.lineage.schema.query.hive_table=hive_table where __guid='%s'\, columns +atlas.lineage.schema.query.Table=Table where __guid='%s'\, columns +atlas.notification.create.topics=true +atlas.notification.embedded=false +atlas.notification.replicas=1 +atlas.notification.topics=ATLAS_HOOK,ATLAS_ENTITIES +atlas.proxyusers= +atlas.rest.address=http://hdp:21000 +atlas.server.address.id1=hdp:21000 +atlas.server.bind.address=0.0.0.0 +atlas.server.ha.enabled=false +atlas.server.http.port=21000 +atlas.server.https.port=21443 +atlas.server.ids=id1 +atlas.simple.authz.policy.file=/usr/hdp/current/atlas-server/conf/atlas-simple-authz-policy.json +atlas.solr.kerberos.enable=false +atlas.ssl.exclude.protocols=TLSv1.2 +atlas.sso.knox.browser.useragent= +atlas.sso.knox.enabled=false +atlas.sso.knox.providerurl= +atlas.sso.knox.publicKey= \ No newline at end of file diff --git a/dss-apps/dss-dataasset-management/data-assets-server/src/main/resources/linkis.properties b/dss-apps/dss-dataasset-management/data-assets-server/src/main/resources/linkis.properties index 7aaada8c63..c1af7215ec 100644 --- a/dss-apps/dss-dataasset-management/data-assets-server/src/main/resources/linkis.properties +++ b/dss-apps/dss-dataasset-management/data-assets-server/src/main/resources/linkis.properties @@ -60,12 +60,6 @@ atlas.root.label=datamodel_label atlas.root.collection=datamodel_collection -# hive metadata config -metastore.datasource.driver=com.mysql.jdbc.Driver -metastore.datasource.url=jdbc:mysql://localhost:3306/hive?characterEncoding=UTF-8 -metastore.datasource.username= -metastore.datasource.password= - wds.workspace.client.serverurl=http://localhost:29001 @@ -75,8 +69,8 @@ wds.workspace.client.authenticationStrategy=org.apache.linkis.httpclient.dws.aut wds.workspace.client.discovery.enabled=false -wds.workspace.client.authtoken.value=hdfs +wds.workspace.client.authtoken.value=BML-AUTH wds.workspace.client.dws.version=v1 -wds.wedatasphere.data.assert.client.strategy=static \ No newline at end of file +wds.wedatasphere.data.assert.client.strategy=token \ No newline at end of file diff --git a/dss-apps/dss-dataasset-management/data-assets-server/src/main/scala/com/webank/wedatasphere/dss/data/governance/DataAssetsApplication.scala b/dss-apps/dss-dataasset-management/data-assets-server/src/main/scala/com/webank/wedatasphere/dss/data/governance/DataAssetsApplication.scala index d2f77b9db9..1ea733fc96 100644 --- a/dss-apps/dss-dataasset-management/data-assets-server/src/main/scala/com/webank/wedatasphere/dss/data/governance/DataAssetsApplication.scala +++ b/dss-apps/dss-dataasset-management/data-assets-server/src/main/scala/com/webank/wedatasphere/dss/data/governance/DataAssetsApplication.scala @@ -18,9 +18,8 @@ package com.webank.wedatasphere.dss.data.governance +import org.apache.linkis.DataWorkCloudApplication import org.apache.linkis.common.utils.Logging -import org.apache.linkis.{DataWorkCloudApplication, LinkisBaseServerApp} - object DataAssetsApplication extends Logging { diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-center-common/pom.xml b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/pom.xml index acc086314b..db5293654b 100644 --- a/dss-apps/dss-datamodel-center/dss-datamodel-center-common/pom.xml +++ b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/pom.xml @@ -73,6 +73,11 @@ compile + + mysql + mysql-connector-java + 5.1.49 + diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/event/UnBindLabelEvent.java b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/event/UnBindLabelEvent.java index 5c8066aa17..05ae7acc0c 100644 --- a/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/event/UnBindLabelEvent.java +++ b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/event/UnBindLabelEvent.java @@ -1,6 +1,5 @@ package com.webank.wedatasphere.dss.datamodel.center.common.event; -import javafx.application.Application; import lombok.Getter; import lombok.ToString; import org.springframework.context.ApplicationEvent; diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-center-server/src/main/resources/application.yml b/dss-apps/dss-datamodel-center/dss-datamodel-center-server/src/main/resources/application.yml index bddac1e886..dae02dbbe8 100644 --- a/dss-apps/dss-datamodel-center/dss-datamodel-center-server/src/main/resources/application.yml +++ b/dss-apps/dss-datamodel-center/dss-datamodel-center-server/src/main/resources/application.yml @@ -1,5 +1,7 @@ server: port: 9321 + servlet: + context-path: /api/rest_j/v1 spring: application: name: datamodel-server diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-center-server/src/main/resources/linkis.properties b/dss-apps/dss-datamodel-center/dss-datamodel-center-server/src/main/resources/linkis.properties index 4dcb6b7d90..c751b6828e 100644 --- a/dss-apps/dss-datamodel-center/dss-datamodel-center-server/src/main/resources/linkis.properties +++ b/dss-apps/dss-datamodel-center/dss-datamodel-center-server/src/main/resources/linkis.properties @@ -39,7 +39,7 @@ wds.wedatasphere.linkis.discovery.enabled=false wds.wedatasphere.linkis.authenticationStrategy=org.apache.linkis.httpclient.dws.authentication.TokenAuthenticationStrategy -wds.wedatasphere.linkis.authtoken.value=hdfs +wds.wedatasphere.linkis.authtoken.value=BML-AUTH wds.wedatasphere.linkis.dws.version=v1 @@ -66,7 +66,7 @@ wds.wedatasphere.warehouse.client.authtoken.key=hdfs wds.wedatasphere.warehouse.client.authenticationStrategy=org.apache.linkis.httpclient.dws.authentication.TokenAuthenticationStrategy -wds.wedatasphere.warehouse.client.authtoken.value=hdfs +wds.wedatasphere.warehouse.client.authtoken.value=BML-AUTH wds.wedatasphere.warehouse.client.discovery.enabled=false @@ -81,7 +81,7 @@ wds.wedatasphere.assets.client.authenticationStrategy=org.apache.linkis.httpclie wds.wedatasphere.assets.client.discovery.enabled=false -wds.wedatasphere.assets.client.authtoken.value=hdfs +wds.wedatasphere.assets.client.authtoken.value=BML-AUTH wds.wedatasphere.assets.client.dws.version=v1 @@ -95,10 +95,10 @@ wds.workspace.client.authenticationStrategy=org.apache.linkis.httpclient.dws.aut wds.workspace.client.discovery.enabled=false -wds.workspace.client.authtoken.value=hdfs +wds.workspace.client.authtoken.value=BML-AUTH wds.workspace.client.dws.version=v1 -wds.wedatasphere.data.model.client.strategy=static +wds.wedatasphere.data.model.client.strategy=token diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-dimension-management/src/main/java/com/webank/wedatasphere/dss/datamodel/dimension/restful/DimensionRestfulApi.java b/dss-apps/dss-datamodel-center/dss-datamodel-dimension-management/src/main/java/com/webank/wedatasphere/dss/datamodel/dimension/restful/DimensionRestfulApi.java index 54f2f9ec25..cd4e07be27 100644 --- a/dss-apps/dss-datamodel-center/dss-datamodel-dimension-management/src/main/java/com/webank/wedatasphere/dss/datamodel/dimension/restful/DimensionRestfulApi.java +++ b/dss-apps/dss-datamodel-center/dss-datamodel-dimension-management/src/main/java/com/webank/wedatasphere/dss/datamodel/dimension/restful/DimensionRestfulApi.java @@ -10,20 +10,15 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; -import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.*; import javax.servlet.http.HttpServletRequest; -import javax.ws.rs.*; -import javax.ws.rs.core.Context; -import javax.ws.rs.core.MediaType; -import javax.ws.rs.core.Response; +import javax.validation.Valid; import java.io.IOException; -@Component -@Path("/datamodel/") -@Produces(MediaType.APPLICATION_JSON) -@Consumes(MediaType.APPLICATION_JSON) + +@RestController +@RequestMapping(value = "datamodel", produces = {"application/json;charset=utf-8"}) public class DimensionRestfulApi { private static final Logger LOGGER = LoggerFactory.getLogger(DimensionRestfulApi.class); @@ -38,11 +33,10 @@ public class DimensionRestfulApi { * @return * @throws IOException */ - @POST - @Path("/dimensions") - public Response add(@Context HttpServletRequest req, @RequestBody DimensionAddVO vo) throws ErrorException { + @RequestMapping( value = "/dimensions", method = RequestMethod.POST) + public Message add(HttpServletRequest req, @Valid @RequestBody DimensionAddVO vo) throws ErrorException { LOGGER.info("dimensionAddVO : {}", vo); - return Message.messageToResponse(Message.ok().data("id", dimensionService.addDimension(vo))); + return Message.ok().data("id", dimensionService.addDimension(vo)); } /** @@ -52,11 +46,10 @@ public Response add(@Context HttpServletRequest req, @RequestBody DimensionAddVO * @param vo * @return */ - @PUT - @Path("/dimensions/enable/{id}") - public Response enable(@Context HttpServletRequest req, @PathParam("id") Long id, @RequestBody DimensionEnableVO vo) { + @RequestMapping( value = "/dimensions/enable/{id}", method = RequestMethod.PUT) + public Message enable(HttpServletRequest req, @PathVariable("id") Long id, @RequestBody DimensionEnableVO vo) { LOGGER.info("enable id : {}, vo : {}", id, vo); - return Message.messageToResponse(Message.ok().data("count", dimensionService.enableDimension(id, vo))); + return Message.ok().data("count", dimensionService.enableDimension(id, vo)); } /** @@ -66,11 +59,10 @@ public Response enable(@Context HttpServletRequest req, @PathParam("id") Long id * @param vo * @return */ - @PUT - @Path("/dimensions/{id}") - public Response update(@Context HttpServletRequest req, @PathParam("id") Long id, @RequestBody DimensionUpdateVO vo) throws ErrorException{ + @RequestMapping( value = "/dimensions/{id}", method = RequestMethod.PUT) + public Message update(HttpServletRequest req, @PathVariable("id") Long id, @RequestBody DimensionUpdateVO vo) throws ErrorException{ LOGGER.info("update id : {}, vo : {}", id, vo); - return Message.messageToResponse(Message.ok().data("count",dimensionService.updateDimension(id,vo))); + return Message.ok().data("count",dimensionService.updateDimension(id,vo)); } /** @@ -79,11 +71,10 @@ public Response update(@Context HttpServletRequest req, @PathParam("id") Long id * @param id * @return */ - @DELETE - @Path("/dimensions/{id}") - public Response delete(@Context HttpServletRequest req, @PathParam("id") Long id) throws ErrorException { + @RequestMapping( value = "/dimensions/{id}", method = RequestMethod.DELETE) + public Message delete(HttpServletRequest req, @PathVariable("id") Long id) throws ErrorException { LOGGER.info("delete id : {}", id); - return Message.messageToResponse(Message.ok().data("count",dimensionService.deleteDimension(id))); + return Message.ok().data("count",dimensionService.deleteDimension(id)); } /** @@ -91,11 +82,10 @@ public Response delete(@Context HttpServletRequest req, @PathParam("id") Long id * @param req * @return */ - @POST - @Path("/dimensions/list") - public Response list(@Context HttpServletRequest req,@RequestBody DimensionQueryVO vo){ + @RequestMapping( value = "/dimensions/list", method = RequestMethod.POST) + public Message list(HttpServletRequest req,@Valid @RequestBody DimensionQueryVO vo){ LOGGER.info("list vo : {}",vo); - return Message.messageToResponse(dimensionService.listDimensions(vo)); + return dimensionService.listDimensions(vo); } @@ -105,10 +95,9 @@ public Response list(@Context HttpServletRequest req,@RequestBody DimensionQuery * @param id * @return */ - @GET - @Path("/dimensions/{id}") - public Response query(@Context HttpServletRequest req, @PathParam("id") Long id) throws ErrorException { + @RequestMapping( value = "/dimensions/{id}", method = RequestMethod.GET) + public Message query(HttpServletRequest req, @PathVariable("id") Long id) throws ErrorException { LOGGER.info("query id : {}", id); - return Message.messageToResponse(Message.ok().data("detail",dimensionService.queryById(id))); + return Message.ok().data("detail",dimensionService.queryById(id)); } } diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-indicator-management/src/main/java/com/webank/wedatasphere/dss/datamodel/indicator/restful/IndicatorRestfulApi.java b/dss-apps/dss-datamodel-center/dss-datamodel-indicator-management/src/main/java/com/webank/wedatasphere/dss/datamodel/indicator/restful/IndicatorRestfulApi.java index ef93875f64..a52f837879 100644 --- a/dss-apps/dss-datamodel-center/dss-datamodel-indicator-management/src/main/java/com/webank/wedatasphere/dss/datamodel/indicator/restful/IndicatorRestfulApi.java +++ b/dss-apps/dss-datamodel-center/dss-datamodel-indicator-management/src/main/java/com/webank/wedatasphere/dss/datamodel/indicator/restful/IndicatorRestfulApi.java @@ -13,21 +13,15 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; -import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.*; import javax.servlet.http.HttpServletRequest; -import javax.ws.rs.*; -import javax.ws.rs.core.Context; -import javax.ws.rs.core.MediaType; -import javax.ws.rs.core.Response; +import javax.validation.Valid; import java.io.IOException; -@Component -@Path("/datamodel") -@Produces(MediaType.APPLICATION_JSON) -@Consumes(MediaType.APPLICATION_JSON) +@RestController +@RequestMapping(value = "datamodel", produces = {"application/json;charset=utf-8"}) public class IndicatorRestfulApi implements AuthenticationClientStrategy { @@ -47,11 +41,10 @@ public class IndicatorRestfulApi implements AuthenticationClientStrategy { * @return * @throws IOException */ - @POST - @Path("/indicators") - public Response add(@Context HttpServletRequest req, @RequestBody IndicatorAddVO vo) throws Exception { + @RequestMapping( value = "/indicators", method = RequestMethod.POST) + public Message add(HttpServletRequest req, @Valid @RequestBody IndicatorAddVO vo) throws Exception { LOGGER.info("indicatorAddVO : {}", vo); - return Message.messageToResponse(Message.ok().data("count",indicatorService.addIndicator(vo,"1" ))); + return Message.ok().data("count",indicatorService.addIndicator(vo,"1" )); } @@ -63,11 +56,10 @@ public Response add(@Context HttpServletRequest req, @RequestBody IndicatorAddVO * @return * @throws Exception */ - @PUT - @Path("/indicators/{id}") - public Response update(@Context HttpServletRequest req, @PathParam("id") Long id , @RequestBody IndicatorUpdateVO vo) throws Exception { + @RequestMapping( value = "/indicators/{id}", method = RequestMethod.PUT) + public Message update(HttpServletRequest req, @PathVariable("id") Long id , @RequestBody IndicatorUpdateVO vo) throws Exception { LOGGER.info("update id : {}, indicatorUpdateVO : {}", id, vo); - return Message.messageToResponse(Message.ok().data("count",indicatorService.updateIndicator(id,vo))); + return Message.ok().data("count",indicatorService.updateIndicator(id,vo)); } @@ -78,11 +70,10 @@ public Response update(@Context HttpServletRequest req, @PathParam("id") Long id * @param vo * @return */ - @PUT - @Path("/indicators/enable/{id}") - public Response enable(@Context HttpServletRequest req, @PathParam("id") Long id, @RequestBody IndicatorEnableVO vo) { + @RequestMapping( value = "/indicators/enable/{id}", method = RequestMethod.PUT) + public Message enable(HttpServletRequest req, @PathVariable("id") Long id, @RequestBody IndicatorEnableVO vo) { LOGGER.info("enable id : {}, vo : {}", id, vo); - return Message.messageToResponse(Message.ok().data("count", indicatorService.enableIndicator(id, vo))); + return Message.ok().data("count", indicatorService.enableIndicator(id, vo)); } @@ -91,11 +82,10 @@ public Response enable(@Context HttpServletRequest req, @PathParam("id") Long id * @param req * @return */ - @POST - @Path("/indicators/list") - public Response list(@Context HttpServletRequest req, @RequestBody IndicatorQueryVO vo){ + @RequestMapping( value = "/indicators/list", method = RequestMethod.POST) + public Message list(HttpServletRequest req, @RequestBody IndicatorQueryVO vo){ LOGGER.info("list vo : {}",vo); - return Message.messageToResponse(indicatorService.listIndicators(vo)); + return indicatorService.listIndicators(vo); } @@ -106,11 +96,10 @@ public Response list(@Context HttpServletRequest req, @RequestBody IndicatorQuer * @param id * @return */ - @GET - @Path("/indicators/{id}") - public Response query(@Context HttpServletRequest req, @PathParam("id") Long id) throws ErrorException { + @RequestMapping( value = "/indicators/{id}", method = RequestMethod.GET) + public Message query(HttpServletRequest req, @PathVariable("id") Long id) throws ErrorException { LOGGER.info("query id : {}", id); - return Message.messageToResponse(indicatorService.queryById(id)); + return indicatorService.queryById(id); } /** @@ -119,11 +108,11 @@ public Response query(@Context HttpServletRequest req, @PathParam("id") Long id) * @param id * @return */ - @DELETE - @Path("/indicators/{id}") - public Response delete(@Context HttpServletRequest req, @PathParam("id") Long id) throws ErrorException { + + @RequestMapping( value = "/indicators/{id}", method = RequestMethod.DELETE) + public Message delete(HttpServletRequest req, @PathVariable("id") Long id) throws ErrorException { LOGGER.info("delete id : {}", id); - return Message.messageToResponse(Message.ok().data("count",indicatorService.deleteIndicator(id))); + return Message.ok().data("count",indicatorService.deleteIndicator(id)); } @@ -135,11 +124,10 @@ public Response delete(@Context HttpServletRequest req, @PathParam("id") Long id * @return * @throws IOException */ - @POST - @Path("/indicators/versions/{id}") - public Response addVersion(@Context HttpServletRequest req, @PathParam("id") Long id,@RequestBody IndicatorVersionAddVO vo) throws Exception { + @RequestMapping( value = "/indicators/versions/{id}", method = RequestMethod.POST) + public Message addVersion(HttpServletRequest req, @PathVariable("id") Long id,@Valid @RequestBody IndicatorVersionAddVO vo) throws Exception { LOGGER.info("indicatorVersionAddVO : {}", vo); - return Message.messageToResponse(Message.ok().data("count",indicatorService.addIndicatorVersion(id,vo))); + return Message.ok().data("count",indicatorService.addIndicatorVersion(id,vo)); } @@ -152,11 +140,10 @@ public Response addVersion(@Context HttpServletRequest req, @PathParam("id") Lon * @return * @throws IOException */ - @POST - @Path("/indicators/versions/rollback") - public Response versionRollBack(@Context HttpServletRequest req,IndicatorVersionRollBackVO vo) throws Exception { + @RequestMapping( value = "/indicators/versions/rollback", method = RequestMethod.POST) + public Message versionRollBack(HttpServletRequest req,@Valid @RequestBody IndicatorVersionRollBackVO vo) throws Exception { LOGGER.info("indicatorVersionRollBackVO : {}", vo); - return Message.messageToResponse(Message.ok().data("count",indicatorService.versionRollBack(vo))); + return Message.ok().data("count",indicatorService.versionRollBack(vo)); } @@ -166,11 +153,10 @@ public Response versionRollBack(@Context HttpServletRequest req,IndicatorVersion * @param req * @return */ - @POST - @Path("/indicators/versions/list") - public Response indicatorVersionsList(@Context HttpServletRequest req, @RequestBody IndicatorVersionQueryVO vo){ + @RequestMapping( value = "/indicators/versions/list", method = RequestMethod.POST) + public Message indicatorVersionsList(HttpServletRequest req, @RequestBody IndicatorVersionQueryVO vo){ LOGGER.info("version list vo : {}",vo); - return Message.messageToResponse(indicatorService.listIndicatorVersions(vo)); + return indicatorService.listIndicatorVersions(vo); } @@ -179,11 +165,10 @@ public Response indicatorVersionsList(@Context HttpServletRequest req, @RequestB * @param req * @return */ - @POST - @Path("/themes/list") - public Response themesList(@Context HttpServletRequest req){ + @RequestMapping( value = "/themes/list", method = RequestMethod.POST) + public Message themesList(HttpServletRequest req){ ListDwThemeDomainAction action = ListDwThemeDomainAction.builder().setUser(getStrategyUser(req)).setIsAvailable(true).build(); - return Message.messageToResponse(Message.ok().data("list",governanceDwRemoteClient.listThemeDomains(action).getAll())); + return Message.ok().data("list",governanceDwRemoteClient.listThemeDomains(action).getAll()); } /** @@ -191,12 +176,11 @@ public Response themesList(@Context HttpServletRequest req){ * @param req * @return */ - @POST - @Path("/layers/list") - public Response layerList(@Context HttpServletRequest req,@RequestBody LayerVO vo){ + @RequestMapping( value = "/layers/list", method = RequestMethod.POST) + public Message layerList(HttpServletRequest req,@RequestBody LayerVO vo){ LOGGER.info("layerList vo : {}",vo); ListDwLayerAction action = ListDwLayerAction.builder().setIsAvailable(true).setDb(vo.getDbName()).setUser(getStrategyUser(req)).build(); - return Message.messageToResponse(Message.ok().data("list",governanceDwRemoteClient.listLayers(action).getAll())); + return Message.ok().data("list",governanceDwRemoteClient.listLayers(action).getAll()); } /** @@ -204,12 +188,12 @@ public Response layerList(@Context HttpServletRequest req,@RequestBody LayerVO v * @param req * @return */ - @POST - @Path("/cycles/list") - public Response cycleList(@Context HttpServletRequest req,@RequestBody CycleVO vo){ + + @RequestMapping( value = "/cycles/list", method = RequestMethod.POST) + public Message cycleList(HttpServletRequest req,@RequestBody CycleVO vo){ LOGGER.info("cycleList vo : {}",vo); ListDwStatisticalPeriodAction action =ListDwStatisticalPeriodAction.builder().setUser(getStrategyUser(req)).setLayer(vo.getLayer()).setIsAvailable(true).setTheme(vo.getTheme()).build(); - return Message.messageToResponse(Message.ok().data("list",governanceDwRemoteClient.listStatisticalPeriods(action).getAll())); + return Message.ok().data("list",governanceDwRemoteClient.listStatisticalPeriods(action).getAll()); } /** @@ -217,18 +201,18 @@ public Response cycleList(@Context HttpServletRequest req,@RequestBody CycleVO v * @param req * @return */ - @POST - @Path("/modifiers/list") - public Response modifierList(@Context HttpServletRequest req,@RequestBody ModifierVO vo){ + + @RequestMapping( value = "/modifiers/list", method = RequestMethod.POST) + public Message modifierList(HttpServletRequest req,@RequestBody ModifierVO vo){ LOGGER.info("modifierList vo : {}",vo); ListDwModifierAction action = ListDwModifierAction.builder().setUser(getStrategyUser(req)).setLayer(vo.getLayer()).setIsAvailable(true).setTheme(vo.getTheme()).build(); - return Message.messageToResponse(Message.ok().data("list",governanceDwRemoteClient.listModifiers(action).getAll())); + return Message.ok().data("list",governanceDwRemoteClient.listModifiers(action).getAll()); } - @POST - @Path("/current/user") - public Response currentUser(@Context HttpServletRequest req){ - return Message.messageToResponse(Message.ok().data("user",getStrategyUser(req))); + + @RequestMapping( value = "/current/user", method = RequestMethod.POST) + public Message currentUser(HttpServletRequest req){ + return Message.ok().data("user",getStrategyUser(req)); } } diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-measure-management/src/main/java/com/webank/wedatasphere/dss/datamodel/measure/restful/MeasureRestfulApi.java b/dss-apps/dss-datamodel-center/dss-datamodel-measure-management/src/main/java/com/webank/wedatasphere/dss/datamodel/measure/restful/MeasureRestfulApi.java index 61351fd339..8d2fa1a514 100644 --- a/dss-apps/dss-datamodel-center/dss-datamodel-measure-management/src/main/java/com/webank/wedatasphere/dss/datamodel/measure/restful/MeasureRestfulApi.java +++ b/dss-apps/dss-datamodel-center/dss-datamodel-measure-management/src/main/java/com/webank/wedatasphere/dss/datamodel/measure/restful/MeasureRestfulApi.java @@ -1,7 +1,6 @@ package com.webank.wedatasphere.dss.datamodel.measure.restful; -import com.google.common.collect.Lists; -import com.webank.wedatasphere.dss.datamodel.center.common.exception.DSSDatamodelCenterException; + import com.webank.wedatasphere.dss.datamodel.measure.service.MeasureService; import com.webank.wedatasphere.dss.datamodel.measure.vo.MeasureAddVO; import com.webank.wedatasphere.dss.datamodel.measure.vo.MeasureEnableVO; @@ -9,25 +8,17 @@ import com.webank.wedatasphere.dss.datamodel.measure.vo.MeasureUpdateVO; import org.apache.linkis.common.exception.ErrorException; import org.apache.linkis.server.Message; -import org.apache.linkis.server.security.SecurityFilter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; -import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.*; import javax.servlet.http.HttpServletRequest; -import javax.ws.rs.*; -import javax.ws.rs.core.Context; -import javax.ws.rs.core.MediaType; -import javax.ws.rs.core.Response; import java.io.IOException; -@Component -@Path("/datamodel/") -@Produces(MediaType.APPLICATION_JSON) -@Consumes(MediaType.APPLICATION_JSON) +@RestController +@RequestMapping(value = "datamodel", produces = {"application/json;charset=utf-8"}) public class MeasureRestfulApi { @@ -44,12 +35,10 @@ public class MeasureRestfulApi { * @return * @throws IOException */ - @POST - @Path("/measures") - public Response add(@Context HttpServletRequest req, @RequestBody MeasureAddVO vo) throws ErrorException{ + @RequestMapping( value = "/measures", method = RequestMethod.POST) + public Message add(HttpServletRequest req, @RequestBody MeasureAddVO vo) throws ErrorException{ LOGGER.info("measureAddVO : {}", vo); - //String userName = SecurityFilter.getLoginUsername(req); - return Message.messageToResponse(Message.ok().data("id", measureService.addMeasure(vo))); + return Message.ok().data("id", measureService.addMeasure(vo)); } /** @@ -59,11 +48,10 @@ public Response add(@Context HttpServletRequest req, @RequestBody MeasureAddVO v * @param vo * @return */ - @PUT - @Path("/measures/enable/{id}") - public Response enable(@Context HttpServletRequest req, @PathParam("id") Long id, @RequestBody MeasureEnableVO vo) { + @RequestMapping( value = "/measures/enable/{id}", method = RequestMethod.PUT) + public Message enable(HttpServletRequest req, @PathVariable("id") Long id, @RequestBody MeasureEnableVO vo) { LOGGER.info("enable id : {}, vo : {}", id, vo); - return Message.messageToResponse(Message.ok().data("count", measureService.enableMeasure(id, vo))); + return Message.ok().data("count", measureService.enableMeasure(id, vo)); } /** @@ -73,11 +61,10 @@ public Response enable(@Context HttpServletRequest req, @PathParam("id") Long id * @param vo * @return */ - @PUT - @Path("/measures/{id}") - public Response update(@Context HttpServletRequest req, @PathParam("id") Long id, @RequestBody MeasureUpdateVO vo) throws ErrorException{ + @RequestMapping( value = "/measures/{id}", method = RequestMethod.PUT) + public Message update(HttpServletRequest req, @PathVariable("id") Long id, @RequestBody MeasureUpdateVO vo) throws ErrorException{ LOGGER.info("update id : {}, vo : {}", id, vo); - return Message.messageToResponse(Message.ok().data("count",measureService.updateMeasure(id,vo))); + return Message.ok().data("count",measureService.updateMeasure(id,vo)); } @@ -87,11 +74,11 @@ public Response update(@Context HttpServletRequest req, @PathParam("id") Long id * @param id * @return */ - @GET - @Path("/measures/{id}") - public Response query(@Context HttpServletRequest req, @PathParam("id") Long id) throws ErrorException { + + @RequestMapping( value = "/measures/{id}", method = RequestMethod.GET) + public Message query(HttpServletRequest req, @PathVariable("id") Long id) throws ErrorException { LOGGER.info("query id : {}", id); - return Message.messageToResponse(Message.ok().data("detail",measureService.queryById(id))); + return Message.ok().data("detail",measureService.queryById(id)); } /** @@ -100,11 +87,10 @@ public Response query(@Context HttpServletRequest req, @PathParam("id") Long id) * @param id * @return */ - @DELETE - @Path("/measures/{id}") - public Response delete(@Context HttpServletRequest req, @PathParam("id") Long id) throws ErrorException { + @RequestMapping( value = "/measures/{id}", method = RequestMethod.DELETE) + public Message delete(HttpServletRequest req, @PathVariable("id") Long id) throws ErrorException { LOGGER.info("delete id : {}", id); - return Message.messageToResponse(Message.ok().data("count",measureService.deleteMeasure(id))); + return Message.ok().data("count",measureService.deleteMeasure(id)); } /** @@ -112,10 +98,9 @@ public Response delete(@Context HttpServletRequest req, @PathParam("id") Long id * @param req * @return */ - @POST - @Path("/measures/list") - public Response list(@Context HttpServletRequest req,@RequestBody MeasureQueryVO vo){ + @RequestMapping( value = "/measures/list", method = RequestMethod.POST) + public Message list(HttpServletRequest req,@RequestBody MeasureQueryVO vo){ LOGGER.info("list vo : {}",vo); - return Message.messageToResponse(measureService.listMeasures(vo)); + return measureService.listMeasures(vo); } } diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/dao/TableQueryMapper.java b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/dao/TableQueryMapper.java index d703a816de..bb7d3f3b29 100644 --- a/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/dao/TableQueryMapper.java +++ b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/dao/TableQueryMapper.java @@ -11,7 +11,7 @@ import java.util.List; public interface TableQueryMapper extends BaseMapper { - String querySql = "SELECT a.*, b.model_type,b.model_name FROM dss_datamodel_table AS a LEFT JOIN dss_datamodel_table_columns AS b ON b.table_id = a.id " + + String querySql = "SELECT a.*, any_value(b.model_type) as model_type,any_value(b.model_name) as model_name FROM dss_datamodel_table AS a LEFT JOIN dss_datamodel_table_columns AS b ON b.table_id = a.id " + " ${ew.customSqlSegment}"; String wrapperSql = "SELECT * from ( " + querySql + " ) AS q ${ew.customSqlSegment}"; diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/materialized/HiveSchemaDdlBuilder.java b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/materialized/HiveSchemaDdlBuilder.java index 6ae6d0f646..ab86c10572 100644 --- a/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/materialized/HiveSchemaDdlBuilder.java +++ b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/materialized/HiveSchemaDdlBuilder.java @@ -88,7 +88,7 @@ public String createTableString() { builder.append("( "); boolean isFirstColumn = true; - for (HiveSchema.Column c : hiveSchema.getColumns()) { + for (Column c : hiveSchema.getColumns()) { if (!c.isPartition()) { if (isFirstColumn) { isFirstColumn = false; diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/restful/TableRestfulApi.java b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/restful/TableRestfulApi.java index b019157044..1897fd1cd1 100644 --- a/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/restful/TableRestfulApi.java +++ b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/restful/TableRestfulApi.java @@ -14,22 +14,16 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; -import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.*; import javax.annotation.Resource; import javax.servlet.http.HttpServletRequest; -import javax.ws.rs.*; -import javax.ws.rs.core.Context; -import javax.ws.rs.core.MediaType; -import javax.ws.rs.core.Response; import java.io.IOException; -@Component -@Path("/datamodel/") -@Produces(MediaType.APPLICATION_JSON) -@Consumes(MediaType.APPLICATION_JSON) + +@RestController +@RequestMapping(value = "datamodel", produces = {"application/json;charset=utf-8"}) public class TableRestfulApi implements AuthenticationClientStrategy { @@ -62,12 +56,11 @@ public class TableRestfulApi implements AuthenticationClientStrategy { * @return * @throws IOException */ - @POST - @Path("/tables") - public Response add(@Context HttpServletRequest req, @RequestBody TableAddVO vo) throws ErrorException { + @RequestMapping( value = "/tables", method = RequestMethod.POST) + public Message add(HttpServletRequest req, @RequestBody TableAddVO vo) throws ErrorException { vo.setCreator(getStrategyUser(req)); LOGGER.info("tablesAddVO : {}", vo); - return Message.messageToResponse(Message.ok().data("id", tableService.addTable(vo))); + return Message.ok().data("id", tableService.addTable(vo)); } @@ -79,12 +72,11 @@ public Response add(@Context HttpServletRequest req, @RequestBody TableAddVO vo) * @return * @throws IOException */ - @PUT - @Path("/tables/{id}") - public Response update(@Context HttpServletRequest req, @PathParam("id") Long id, @RequestBody TableUpdateVO vo) throws ErrorException { + @RequestMapping( value = "/tables/{id}", method = RequestMethod.PUT) + public Message update(HttpServletRequest req, @PathVariable("id") Long id, @RequestBody TableUpdateVO vo) throws ErrorException { vo.setCreator(getStrategyUser(req)); LOGGER.info("update id : {}, tableUpdateVO : {}", id, vo); - return Message.messageToResponse(Message.ok().data("count", tableService.updateTable(id, vo))); + return Message.ok().data("count", tableService.updateTable(id, vo)); } /** @@ -94,11 +86,10 @@ public Response update(@Context HttpServletRequest req, @PathParam("id") Long id * @return * @throws IOException */ - @DELETE - @Path("/tables/{id}") - public Response delete(@Context HttpServletRequest req, @PathParam("id") Long id) throws ErrorException { + @RequestMapping( value = "/tables/{id}", method = RequestMethod.DELETE) + public Message delete(HttpServletRequest req, @PathVariable("id") Long id) throws ErrorException { LOGGER.info("delete id : {}", id); - return Message.messageToResponse(Message.ok().data("count", tableService.deleteTable(id))); + return Message.ok().data("count", tableService.deleteTable(id)); } @@ -109,12 +100,11 @@ public Response delete(@Context HttpServletRequest req, @PathParam("id") Long id * @return * @throws IOException */ - @PUT - @Path("/tables/bind/{id}") - public Response bind(@Context HttpServletRequest req, @PathParam("id") Long id) throws ErrorException { + @RequestMapping( value = "/tables/bind/{id}", method = RequestMethod.PUT) + public Message bind(HttpServletRequest req, @PathVariable("id") Long id) throws ErrorException { LOGGER.info("bind id : {}", id); tableService.tryBind(id); - return Message.messageToResponse(Message.ok()); + return Message.ok(); } @@ -125,11 +115,10 @@ public Response bind(@Context HttpServletRequest req, @PathParam("id") Long id) * @param id * @return */ - @GET - @Path("/tables/{id}") - public Response query(@Context HttpServletRequest req, @PathParam("id") Long id) throws ErrorException { + @RequestMapping( value = "/tables/{id}", method = RequestMethod.GET) + public Message query(HttpServletRequest req, @PathVariable("id") Long id) throws ErrorException { LOGGER.info("query id : {}", id); - return Message.messageToResponse(Message.ok().data("detail", tableService.queryById(id))); + return Message.ok().data("detail", tableService.queryById(id)); } @@ -140,12 +129,11 @@ public Response query(@Context HttpServletRequest req, @PathParam("id") Long id) * @param vo * @return */ - @POST - @Path("/tables/name") - public Response queryName(@Context HttpServletRequest req, @RequestBody TableQueryOneVO vo) throws ErrorException { + @RequestMapping( value = "/tables/name", method = RequestMethod.POST) + public Message queryName(HttpServletRequest req, @RequestBody TableQueryOneVO vo) throws ErrorException { vo.setUser(getStrategyUser(req)); LOGGER.info("query vo : {}", vo); - return Message.messageToResponse(Message.ok().data("detail", tableService.queryByName(vo))); + return Message.ok().data("detail", tableService.queryByName(vo)); } @@ -155,12 +143,11 @@ public Response queryName(@Context HttpServletRequest req, @RequestBody TableQue * @param req * @return */ - @POST - @Path("/tables/list") - public Response list(@Context HttpServletRequest req, @RequestBody TableListVO vo) { + @RequestMapping( value = "/tables/list", method = RequestMethod.POST) + public Message list(HttpServletRequest req, @RequestBody TableListVO vo) { vo.setUser(getStrategyUser(req)); LOGGER.info("list vo : {}", vo); - return Message.messageToResponse(tableService.list(vo)); + return tableService.list(vo); } @@ -172,12 +159,11 @@ public Response list(@Context HttpServletRequest req, @RequestBody TableListVO v * @return * @throws IOException */ - @POST - @Path("/tables/versions/{id}") - public Response addVersion(@Context HttpServletRequest req, @PathParam("id") Long id, @RequestBody TableVersionAddVO vo) throws Exception { + @RequestMapping( value = "/tables/versions/{id}", method = RequestMethod.POST) + public Message addVersion(HttpServletRequest req, @PathVariable("id") Long id, @RequestBody TableVersionAddVO vo) throws Exception { vo.setCreator(getStrategyUser(req)); LOGGER.info("tableVersionAddVO : {}", vo); - return Message.messageToResponse(Message.ok().data("count", tableService.addTableVersion(id, vo))); + return Message.ok().data("count", tableService.addTableVersion(id, vo)); } @@ -189,12 +175,11 @@ public Response addVersion(@Context HttpServletRequest req, @PathParam("id") Lon * @return * @throws IOException */ - @POST - @Path("/tables/versions/rollback") - public Response versionRollBack(@Context HttpServletRequest req, @RequestBody TableVersionRollBackVO vo) throws Exception { + @RequestMapping( value = "/tables/versions/rollback", method = RequestMethod.POST) + public Message versionRollBack(HttpServletRequest req, @RequestBody TableVersionRollBackVO vo) throws Exception { vo.setUser(getStrategyUser(req)); LOGGER.info("tableVersionRollBackVO : {}", vo); - return Message.messageToResponse(Message.ok().data("count", tableService.versionRollBack(vo))); + return Message.ok().data("count", tableService.versionRollBack(vo)); } @@ -204,11 +189,10 @@ public Response versionRollBack(@Context HttpServletRequest req, @RequestBody Ta * @param req * @return */ - @POST - @Path("/tables/versions/list") - public Response tableVersionsList(@Context HttpServletRequest req, @RequestBody TableVersionQueryVO vo) { + @RequestMapping( value = "/tables/versions/list", method = RequestMethod.POST) + public Message tableVersionsList(HttpServletRequest req, @RequestBody TableVersionQueryVO vo) { LOGGER.info("version list vo : {}", vo); - return Message.messageToResponse(tableService.listTableVersions(vo)); + return tableService.listTableVersions(vo); } @@ -232,11 +216,11 @@ public Response tableVersionsList(@Context HttpServletRequest req, @RequestBody * @param req * @return */ - @GET - @Path("/themes/reference/{name}") - public Response themesReference(@Context HttpServletRequest req, @PathParam("name") String name) { + + @RequestMapping( value = "/themes/reference/{name}", method = RequestMethod.GET) + public Message themesReference(HttpServletRequest req, @PathVariable("name") String name) { LOGGER.info("themes reference name : {}", name); - return Message.messageToResponse(Message.ok().data("result", dataWarehouseReferenceService.themeReferenceCount(name))); + return Message.ok().data("result", dataWarehouseReferenceService.themeReferenceCount(name)); } // /** @@ -258,11 +242,10 @@ public Response themesReference(@Context HttpServletRequest req, @PathParam("nam * @param req * @return */ - @GET - @Path("/layers/reference/{name}") - public Response layersReference(@Context HttpServletRequest req, @PathParam("name") String name) { + @RequestMapping( value = "/layers/reference/{name}", method = RequestMethod.GET) + public Message layersReference(HttpServletRequest req, @PathVariable("name") String name) { LOGGER.info("layers reference name : {}", name); - return Message.messageToResponse(Message.ok().data("result", dataWarehouseReferenceService.layerReferenceCount(name))); + return Message.ok().data("result", dataWarehouseReferenceService.layerReferenceCount(name)); } /** @@ -271,11 +254,10 @@ public Response layersReference(@Context HttpServletRequest req, @PathParam("nam * @param req * @return */ - @GET - @Path("/cycles/reference/{name}") - public Response cycleReference(@Context HttpServletRequest req, @PathParam("name") String name) { + @RequestMapping( value = "/cycles/reference/{name}", method = RequestMethod.GET) + public Message cycleReference(HttpServletRequest req, @PathVariable("name") String name) { LOGGER.info("cycles reference name : {}", name); - return Message.messageToResponse(Message.ok().data("result", dataWarehouseReferenceService.cycleReferenceCount(name))); + return Message.ok().data("result", dataWarehouseReferenceService.cycleReferenceCount(name)); } /** @@ -284,11 +266,10 @@ public Response cycleReference(@Context HttpServletRequest req, @PathParam("name * @param req * @return */ - @GET - @Path("/modifiers/reference/{name}") - public Response modifiersReference(@Context HttpServletRequest req, @PathParam("name") String name) { + @RequestMapping( value = "/modifiers/reference/{name}", method = RequestMethod.GET) + public Message modifiersReference(HttpServletRequest req, @PathVariable("name") String name) { LOGGER.info("modifiers reference name : {}", name); - return Message.messageToResponse(Message.ok().data("result", dataWarehouseReferenceService.modifierReferenceCount(name))); + return Message.ok().data("result", dataWarehouseReferenceService.modifierReferenceCount(name)); } /** @@ -297,12 +278,11 @@ public Response modifiersReference(@Context HttpServletRequest req, @PathParam(" * @param req * @return */ - @POST - @Path("/tables/databases/list") - public Response tableDataBasesList(@Context HttpServletRequest req, TableDatabasesQueryVO vo) { + @RequestMapping( value = "/tables/databases/list", method = RequestMethod.POST) + public Message tableDataBasesList(HttpServletRequest req,@RequestBody TableDatabasesQueryVO vo) { vo.setUser(getStrategyUser(req)); LOGGER.info("table databases vo : {}", vo); - return Message.messageToResponse(tableService.listDataBases(vo)); + return tableService.listDataBases(vo); } @@ -312,12 +292,11 @@ public Response tableDataBasesList(@Context HttpServletRequest req, TableDatabas * @param req * @return */ - @POST - @Path("/tables/collect/") - public Response tableCollect(@Context HttpServletRequest req, @RequestBody TableCollectVO vo) throws ErrorException { + @RequestMapping( value = "/tables/collect", method = RequestMethod.POST) + public Message tableCollect(HttpServletRequest req, @RequestBody TableCollectVO vo) throws ErrorException { vo.setUser(getStrategyUser(req)); LOGGER.info("table collection vo : {}", vo); - return Message.messageToResponse(Message.ok().data("count", tableService.tableCollect(vo))); + return Message.ok().data("count", tableService.tableCollect(vo)); } /** @@ -326,12 +305,11 @@ public Response tableCollect(@Context HttpServletRequest req, @RequestBody Table * @param req * @return */ - @POST - @Path("/tables/collect/cancel") - public Response tableCancelCollect(@Context HttpServletRequest req, @RequestBody TableCollectCancelVO vo) throws ErrorException { + @RequestMapping( value = "/tables/collect/cancel", method = RequestMethod.POST) + public Message tableCancelCollect(HttpServletRequest req, @RequestBody TableCollectCancelVO vo) throws ErrorException { vo.setUser(getStrategyUser(req)); LOGGER.info("table collection cancel vo : {}", vo); - return Message.messageToResponse(Message.ok().data("count", tableService.tableCancel(vo))); + return Message.ok().data("count", tableService.tableCancel(vo)); } @@ -341,12 +319,11 @@ public Response tableCancelCollect(@Context HttpServletRequest req, @RequestBody * @param req * @return */ - @POST - @Path("/tables/collect/list") - public Response tableCollectList(@Context HttpServletRequest req, @RequestBody TableCollectQueryVO vo) throws ErrorException { + @RequestMapping( value = "/tables/collect/list", method = RequestMethod.POST) + public Message tableCollectList(HttpServletRequest req, @RequestBody TableCollectQueryVO vo) throws ErrorException { vo.setUser(getStrategyUser(req)); LOGGER.info("table collection list vo : {}", vo); - return Message.messageToResponse(tableService.tableCollections(vo)); + return tableService.tableCollections(vo); } /** @@ -356,12 +333,11 @@ public Response tableCollectList(@Context HttpServletRequest req, @RequestBody T * @param vo * @return */ - @POST - @Path("/tables/data/preview/") - public Response tableDataPreview(@Context HttpServletRequest req, @RequestBody TableDataPreviewVO vo) throws ErrorException { + @RequestMapping( value = "/tables/data/preview", method = RequestMethod.POST) + public Message tableDataPreview(HttpServletRequest req, @RequestBody TableDataPreviewVO vo) throws ErrorException { vo.setUser(getStrategyUser(req)); LOGGER.info("table data preview : {}", vo); - return Message.messageToResponse(tableService.previewData(vo)); + return tableService.previewData(vo); } @@ -372,12 +348,11 @@ public Response tableDataPreview(@Context HttpServletRequest req, @RequestBody T * @param vo * @return */ - @POST - @Path("/tables/create/") - public Response tableCreate(@Context HttpServletRequest req, @RequestBody TableCreateVO vo) throws ErrorException { + @RequestMapping( value = "/tables/create", method = RequestMethod.POST) + public Message tableCreate(HttpServletRequest req, @RequestBody TableCreateVO vo) throws ErrorException { vo.setUser(getStrategyUser(req)); LOGGER.info("table create vo : {}", vo); - return Message.messageToResponse(Message.ok().data("count", tableService.tableCreate(vo))); + return Message.ok().data("count", tableService.tableCreate(vo)); } /** @@ -387,12 +362,11 @@ public Response tableCreate(@Context HttpServletRequest req, @RequestBody TableC * @param vo * @return */ - @POST - @Path("/tables/create/sql") - public Response tableCreateSql(@Context HttpServletRequest req, @RequestBody TableCreateSqlVO vo) throws ErrorException { + @RequestMapping( value = "/tables/create/sql", method = RequestMethod.POST) + public Message tableCreateSql(HttpServletRequest req, @RequestBody TableCreateSqlVO vo) throws ErrorException { vo.setUser(getStrategyUser(req)); LOGGER.info("table create sql vo : {}", vo); - return Message.messageToResponse(Message.ok().data("detail", tableService.tableCreateSql(vo))); + return Message.ok().data("detail", tableService.tableCreateSql(vo)); } @@ -403,11 +377,10 @@ public Response tableCreateSql(@Context HttpServletRequest req, @RequestBody Tab * @param vo * @return */ - @POST - @Path("/tables/dictionaries/list") - public Response tableDictionaryList(@Context HttpServletRequest req, @RequestBody TableDictionaryListVO vo) { + @RequestMapping( value = "/tables/dictionaries/list", method = RequestMethod.POST) + public Message tableDictionaryList(HttpServletRequest req, @RequestBody TableDictionaryListVO vo) { LOGGER.info("table dictionaries list vo : {}", vo); - return Message.messageToResponse(tableService.dictionaryList(vo)); + return tableService.dictionaryList(vo); } @@ -418,11 +391,10 @@ public Response tableDictionaryList(@Context HttpServletRequest req, @RequestBod * @param vo * @return */ - @POST - @Path("/tables/columns/add") - public Response tableColumnsAdd(@Context HttpServletRequest req, @RequestBody TableColumnsAddVO vo) throws ErrorException { + @RequestMapping( value = "/tables/columns/add", method = RequestMethod.POST) + public Message tableColumnsAdd(HttpServletRequest req, @RequestBody TableColumnsAddVO vo) throws ErrorException { LOGGER.info("table column add vo : {}", vo); - return Message.messageToResponse(Message.ok().data("count", tableService.addTableColumn(vo))); + return Message.ok().data("count", tableService.addTableColumn(vo)); } /** @@ -432,11 +404,11 @@ public Response tableColumnsAdd(@Context HttpServletRequest req, @RequestBody Ta * @param vo * @return */ - @POST - @Path("/tables/columns/bind/{columnId}") - public Response tableColumnBind(@Context HttpServletRequest req, @PathParam("columnId") Long columnId, @RequestBody TableColumnBindVO vo) throws ErrorException { + + @RequestMapping( value = "/tables/columns/bind/{columnId}", method = RequestMethod.POST) + public Message tableColumnBind(HttpServletRequest req, @PathVariable("columnId") Long columnId, @RequestBody TableColumnBindVO vo) throws ErrorException { LOGGER.info("table column bind model columnId : {}, vo : {}", columnId, vo); - return Message.messageToResponse(Message.ok().data("count", tableService.tableColumnBind(columnId, vo))); + return Message.ok().data("count", tableService.tableColumnBind(columnId, vo)); } /** @@ -445,12 +417,11 @@ public Response tableColumnBind(@Context HttpServletRequest req, @PathParam("col * @param vo * @return */ - @POST - @Path("/tables/partition/stats/") - public Response tblPartitionStats(@Context HttpServletRequest req, TblPartitionStatsVO vo) { + @RequestMapping( value = "/tables/partition/stats", method = RequestMethod.POST) + public Message tblPartitionStats(HttpServletRequest req,@RequestBody TblPartitionStatsVO vo) { vo.setUser(getStrategyUser(req)); LOGGER.info("table partition stats vo : {}", vo); - return Message.messageToResponse(tableService.listTablePartitionStats(vo)); + return tableService.listTablePartitionStats(vo); } @@ -460,10 +431,10 @@ public Response tblPartitionStats(@Context HttpServletRequest req, TblPartitionS * @param req * @return */ - @GET - @Path("/current/user") - public Response currentUser(@Context HttpServletRequest req) { - return Message.messageToResponse(Message.ok().data("user", getStrategyUser(req))); + + @RequestMapping( value = "/current/user", method = RequestMethod.GET) + public Message currentUser(HttpServletRequest req) { + return Message.ok().data("user", getStrategyUser(req)); } @@ -473,12 +444,11 @@ public Response currentUser(@Context HttpServletRequest req) { * @param req * @return */ - @POST - @Path("/tables/check/data") - public Response tableCheckData(@Context HttpServletRequest req, @RequestBody TableCheckDataVO vo) throws ErrorException { + @RequestMapping( value = "/tables/check/data", method = RequestMethod.POST) + public Message tableCheckData(HttpServletRequest req, @RequestBody TableCheckDataVO vo) throws ErrorException { vo.setUser(getStrategyUser(req)); LOGGER.info("table partition stats vo : {}", vo); - return Message.messageToResponse(Message.ok().data("status", tableService.tableCheckData(vo))); + return Message.ok().data("status", tableService.tableCheckData(vo)); } @@ -490,11 +460,11 @@ public Response tableCheckData(@Context HttpServletRequest req, @RequestBody Tab * @return * @throws IOException */ - @POST - @Path("/labels") - public Response addLabels(@Context HttpServletRequest req, @RequestBody LabelAddVO vo) throws ErrorException { + + @RequestMapping( value = "/labels", method = RequestMethod.POST) + public Message addLabels(HttpServletRequest req, @RequestBody LabelAddVO vo) throws ErrorException { LOGGER.info("addLabels vo : {}", vo); - return Message.messageToResponse(Message.ok().data("id", labelService.add(vo))); + return Message.ok().data("id", labelService.add(vo)); } @@ -506,11 +476,10 @@ public Response addLabels(@Context HttpServletRequest req, @RequestBody LabelAdd * @return * @throws IOException */ - @PUT - @Path("/labels/{id}") - public Response updateLabels(@Context HttpServletRequest req, @PathParam("id") Long id, @RequestBody LabelUpdateVO vo) throws ErrorException { + @RequestMapping( value = "/labels/{id}", method = RequestMethod.PUT) + public Message updateLabels(HttpServletRequest req, @PathVariable("id") Long id, @RequestBody LabelUpdateVO vo) throws ErrorException { LOGGER.info("updateLabels id : {}, vo : {}", id, vo); - return Message.messageToResponse(Message.ok().data("count", labelService.update(id, vo))); + return Message.ok().data("count", labelService.update(id, vo)); } /** @@ -520,11 +489,10 @@ public Response updateLabels(@Context HttpServletRequest req, @PathParam("id") L * @return * @throws IOException */ - @DELETE - @Path("/labels/{id}") - public Response deleteLabels(@Context HttpServletRequest req, @PathParam("id") Long id) throws ErrorException { + @RequestMapping( value = "/labels/{id}", method = RequestMethod.DELETE) + public Message deleteLabels(HttpServletRequest req, @PathVariable("id") Long id) throws ErrorException { LOGGER.info("deleteLabels id : {}", id); - return Message.messageToResponse(Message.ok().data("count", labelService.delete(id))); + return Message.ok().data("count", labelService.delete(id)); } /** @@ -534,11 +502,11 @@ public Response deleteLabels(@Context HttpServletRequest req, @PathParam("id") L * @param id * @return */ - @GET - @Path("/labels/{id}") - public Response queryLabels(@Context HttpServletRequest req, @PathParam("id") Long id) throws ErrorException { + + @RequestMapping( value = "/labels/{id}", method = RequestMethod.GET) + public Message queryLabels(HttpServletRequest req, @PathVariable("id") Long id) throws ErrorException { LOGGER.info("queryLabels id : {}", id); - return Message.messageToResponse(Message.ok().data("detail", labelService.query(id))); + return Message.ok().data("detail", labelService.query(id)); } @@ -548,11 +516,11 @@ public Response queryLabels(@Context HttpServletRequest req, @PathParam("id") Lo * @param req * @return */ - @POST - @Path("/labels/list") - public Response listLabels(@Context HttpServletRequest req, @RequestBody LabelsQueryVO vo) throws ErrorException { + + @RequestMapping( value = "/labels/list", method = RequestMethod.POST) + public Message listLabels(HttpServletRequest req, @RequestBody LabelsQueryVO vo) throws ErrorException { LOGGER.info("listLabels vo : {}", vo); - return Message.messageToResponse(labelService.list(vo)); + return labelService.list(vo); } /** @@ -563,11 +531,10 @@ public Response listLabels(@Context HttpServletRequest req, @RequestBody LabelsQ * @param vo * @return */ - @PUT - @Path("/labels/enable/{id}") - public Response enableLabel(@Context HttpServletRequest req, @PathParam("id") Long id, @RequestBody LabelEnableVO vo) throws ErrorException { + @RequestMapping( value = "/labels/enable/{id}", method = RequestMethod.PUT) + public Message enableLabel(HttpServletRequest req, @PathVariable("id") Long id, @RequestBody LabelEnableVO vo) throws ErrorException { LOGGER.info("enableLabel id : {}, vo : {}", id, vo); - return Message.messageToResponse(Message.ok().data("count", labelService.enable(id, vo))); + return Message.ok().data("count", labelService.enable(id, vo)); } @@ -578,12 +545,12 @@ public Response enableLabel(@Context HttpServletRequest req, @PathParam("id") Lo * @param workspaceId * @return */ - @GET - @Path("/users/{workspaceId}") - public Response users(@Context HttpServletRequest req, @PathParam("workspaceId") String workspaceId) throws ErrorException { + + @RequestMapping( value = "/users/{workspaceId}", method = RequestMethod.GET) + public Message users(HttpServletRequest req, @PathVariable("workspaceId") String workspaceId) throws ErrorException { LOGGER.info("users workspaceId : {}", workspaceId); GetWorkspaceUsersResult result = linkisWorkSpaceRemoteClient.getWorkspaceUsers(GetWorkspaceUsersAction.builder().setUser(getStrategyUser(req)).setWorkspaceId(workspaceId).build()); - return Message.messageToResponse(Message.ok().data("users", result.getWorkspaceUserList())); + return Message.ok().data("users", result.getWorkspaceUserList()); } @@ -594,12 +561,12 @@ public Response users(@Context HttpServletRequest req, @PathParam("workspaceId") * @param workspaceId * @return */ - @GET - @Path("/roles/{workspaceId}") - public Response roles(@Context HttpServletRequest req, @PathParam("workspaceId") String workspaceId) throws ErrorException { + + @RequestMapping( value = "/roles/{workspaceId}", method = RequestMethod.GET) + public Message roles(HttpServletRequest req, @PathVariable("workspaceId") String workspaceId) throws ErrorException { LOGGER.info("roles workspaceId : {}", workspaceId); GetWorkspaceUsersResult result = linkisWorkSpaceRemoteClient.getWorkspaceUsers(GetWorkspaceUsersAction.builder().setUser(getStrategyUser(req)).setWorkspaceId(workspaceId).build()); - return Message.messageToResponse(Message.ok().data("users", result.getWorkspaceRoleList())); + return Message.ok().data("users", result.getWorkspaceRoleList()); } } diff --git a/dss-apps/dss-datawarehouse-design/data-warehouse-client/src/main/scala/com/webank/wedatasphere/warehouse/client/GovernanceDwRemoteClient.scala b/dss-apps/dss-datawarehouse-design/data-warehouse-client/src/main/scala/com/webank/wedatasphere/warehouse/client/GovernanceDwRemoteClient.scala index b0996493d0..ded899dea4 100644 --- a/dss-apps/dss-datawarehouse-design/data-warehouse-client/src/main/scala/com/webank/wedatasphere/warehouse/client/GovernanceDwRemoteClient.scala +++ b/dss-apps/dss-datawarehouse-design/data-warehouse-client/src/main/scala/com/webank/wedatasphere/warehouse/client/GovernanceDwRemoteClient.scala @@ -1,4 +1,5 @@ package com.webank.wedatasphere.warehouse.client + import org.apache.linkis.httpclient.dws.DWSHttpClient import org.apache.linkis.httpclient.dws.config.DWSClientConfig import com.webank.wedatasphere.warehouse.client.action.{ListDwLayerAction, ListDwModifierAction, ListDwStatisticalPeriodAction, ListDwThemeDomainAction} diff --git a/dss-apps/dss-datawarehouse-design/data-warehouse-mybatis/src/main/java/com/webank/wedatasphere/linkis/mybatis/DataSourceConfig.java b/dss-apps/dss-datawarehouse-design/data-warehouse-mybatis/src/main/java/com/webank/wedatasphere/linkis/mybatis/DataSourceConfig.java index 033fb8cff1..2f73054d1a 100644 --- a/dss-apps/dss-datawarehouse-design/data-warehouse-mybatis/src/main/java/com/webank/wedatasphere/linkis/mybatis/DataSourceConfig.java +++ b/dss-apps/dss-datawarehouse-design/data-warehouse-mybatis/src/main/java/com/webank/wedatasphere/linkis/mybatis/DataSourceConfig.java @@ -14,7 +14,7 @@ * limitations under the License. */ -package org.apache.linkis.mybatis; +package com.webank.wedatasphere.linkis.mybatis; import org.apache.linkis.common.utils.JavaLog; import org.apache.linkis.mybatis.conf.MybatisConfiguration; diff --git a/dss-apps/dss-datawarehouse-design/data-warehouse-mybatis/src/main/java/com/webank/wedatasphere/linkis/mybatis/MyBatisMapperScannerConfig.java b/dss-apps/dss-datawarehouse-design/data-warehouse-mybatis/src/main/java/com/webank/wedatasphere/linkis/mybatis/MyBatisMapperScannerConfig.java index 804c562b61..18e50f37d5 100644 --- a/dss-apps/dss-datawarehouse-design/data-warehouse-mybatis/src/main/java/com/webank/wedatasphere/linkis/mybatis/MyBatisMapperScannerConfig.java +++ b/dss-apps/dss-datawarehouse-design/data-warehouse-mybatis/src/main/java/com/webank/wedatasphere/linkis/mybatis/MyBatisMapperScannerConfig.java @@ -14,8 +14,9 @@ * limitations under the License. */ -package org.apache.linkis.mybatis; +package com.webank.wedatasphere.linkis.mybatis; +import org.apache.linkis.mybatis.MybatisConfigurationFactory; import org.apache.linkis.mybatis.conf.MybatisConfiguration; import org.mybatis.spring.mapper.MapperScannerConfigurer; import org.springframework.boot.autoconfigure.AutoConfigureAfter; diff --git a/dss-apps/dss-datawarehouse-design/data-warehouse-mybatis/src/main/java/com/webank/wedatasphere/linkis/mybatis/MybatisConfigurationFactory.java b/dss-apps/dss-datawarehouse-design/data-warehouse-mybatis/src/main/java/com/webank/wedatasphere/linkis/mybatis/MybatisConfigurationFactory.java index be56464746..70dc22aba2 100644 --- a/dss-apps/dss-datawarehouse-design/data-warehouse-mybatis/src/main/java/com/webank/wedatasphere/linkis/mybatis/MybatisConfigurationFactory.java +++ b/dss-apps/dss-datawarehouse-design/data-warehouse-mybatis/src/main/java/com/webank/wedatasphere/linkis/mybatis/MybatisConfigurationFactory.java @@ -14,7 +14,7 @@ * limitations under the License. */ -package org.apache.linkis.mybatis; +package com.webank.wedatasphere.linkis.mybatis; import com.baomidou.mybatisplus.annotation.DbType; import com.baomidou.mybatisplus.extension.plugins.MybatisPlusInterceptor; diff --git a/dss-apps/dss-datawarehouse-design/data-warehouse-mybatis/src/main/java/com/webank/wedatasphere/linkis/mybatis/conf/MybatisConfiguration.java b/dss-apps/dss-datawarehouse-design/data-warehouse-mybatis/src/main/java/com/webank/wedatasphere/linkis/mybatis/conf/MybatisConfiguration.java index 43b91c5d80..56c864a2af 100644 --- a/dss-apps/dss-datawarehouse-design/data-warehouse-mybatis/src/main/java/com/webank/wedatasphere/linkis/mybatis/conf/MybatisConfiguration.java +++ b/dss-apps/dss-datawarehouse-design/data-warehouse-mybatis/src/main/java/com/webank/wedatasphere/linkis/mybatis/conf/MybatisConfiguration.java @@ -14,7 +14,7 @@ * limitations under the License. */ -package org.apache.linkis.mybatis.conf; +package com.webank.wedatasphere.linkis.mybatis.conf; import org.apache.linkis.common.conf.CommonVars; diff --git a/dss-apps/dss-datawarehouse-design/data-warehouse-server/src/main/java/com/webank/wedatasphere/warehouse/restful/DwDsRestfulApi.java b/dss-apps/dss-datawarehouse-design/data-warehouse-server/src/main/java/com/webank/wedatasphere/warehouse/restful/DwDsRestfulApi.java index 1804ffa1be..28fd2225d4 100644 --- a/dss-apps/dss-datawarehouse-design/data-warehouse-server/src/main/java/com/webank/wedatasphere/warehouse/restful/DwDsRestfulApi.java +++ b/dss-apps/dss-datawarehouse-design/data-warehouse-server/src/main/java/com/webank/wedatasphere/warehouse/restful/DwDsRestfulApi.java @@ -3,18 +3,17 @@ import org.apache.linkis.server.Message; import com.webank.wedatasphere.warehouse.service.DwDsService; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RestController; import javax.servlet.http.HttpServletRequest; -import javax.ws.rs.*; -import javax.ws.rs.core.Context; -import javax.ws.rs.core.MediaType; -import javax.ws.rs.core.Response; - -@Component -@Produces(MediaType.APPLICATION_JSON) -@Consumes(MediaType.APPLICATION_JSON) -@Path("/data-warehouse") + + + +@RestController +@RequestMapping(value = "/data-warehouse", produces = {"application/json;charset=utf-8"}) public class DwDsRestfulApi { private final DwDsService dwDsService; @@ -25,25 +24,25 @@ public DwDsRestfulApi(DwDsService dwDsService) { } // list all hive dbs - @GET - @Path("/dbs/hive") - public Response getAllHiveDbs(@Context HttpServletRequest request) throws Exception { + + @RequestMapping( value = "/dbs/hive", method = RequestMethod.GET) + public Message getAllHiveDbs(HttpServletRequest request) throws Exception { Message message = this.dwDsService.getAllHiveDbs(request); - return Message.messageToResponse(message); + return message; } - @GET - @Path("/workspace/{id}/principal_users") - public Response getAllAvailableUsers(@Context HttpServletRequest request, @PathParam(value = "id") String id) throws Exception { + + @RequestMapping( value = "/workspace/{id}/principal_users", method = RequestMethod.GET) + public Message getAllAvailableUsers(HttpServletRequest request, @PathVariable(value = "id") String id) throws Exception { Message message = this.dwDsService.getPrincipalUsers(request, id); - return Message.messageToResponse(message); + return message; } - @GET - @Path("/workspace/{id}/principal_roles") - public Response getAllAvailableRoles(@Context HttpServletRequest request, @PathParam(value = "id") String id) throws Exception { + + @RequestMapping( value = "/workspace/{id}/principal_roles", method = RequestMethod.GET) + public Message getAllAvailableRoles(HttpServletRequest request, @PathVariable(value = "id") String id) throws Exception { Message message = this.dwDsService.getPrincipalRoles(request, id); - return Message.messageToResponse(message); + return message; } } diff --git a/dss-apps/dss-datawarehouse-design/data-warehouse-server/src/main/java/com/webank/wedatasphere/warehouse/restful/DwLayerRestfulApi.java b/dss-apps/dss-datawarehouse-design/data-warehouse-server/src/main/java/com/webank/wedatasphere/warehouse/restful/DwLayerRestfulApi.java index ceba2655ee..ea0046e038 100644 --- a/dss-apps/dss-datawarehouse-design/data-warehouse-server/src/main/java/com/webank/wedatasphere/warehouse/restful/DwLayerRestfulApi.java +++ b/dss-apps/dss-datawarehouse-design/data-warehouse-server/src/main/java/com/webank/wedatasphere/warehouse/restful/DwLayerRestfulApi.java @@ -7,18 +7,14 @@ import com.webank.wedatasphere.warehouse.exception.DwException; import com.webank.wedatasphere.warehouse.service.DwLayerService; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; +import org.springframework.web.bind.annotation.*; import javax.servlet.http.HttpServletRequest; -import javax.ws.rs.*; -import javax.ws.rs.core.Context; -import javax.ws.rs.core.MediaType; -import javax.ws.rs.core.Response; - -@Component -@Produces(MediaType.APPLICATION_JSON) -@Consumes(MediaType.APPLICATION_JSON) -@Path("/data-warehouse") + + + +@RestController +@RequestMapping(value = "/data-warehouse", produces = {"application/json;charset=utf-8"}) public class DwLayerRestfulApi { private final DwLayerService dwLayerService; @@ -29,29 +25,29 @@ public DwLayerRestfulApi(DwLayerService dwLayerService) { } // list all preset layers - @GET - @Path("/layers/preset") - public Response getAllPresetLayers(@Context HttpServletRequest request) throws DwException { + + @RequestMapping( value = "/layers/preset", method = RequestMethod.GET) + public Message getAllPresetLayers(HttpServletRequest request) throws DwException { Message message = this.dwLayerService.getAllPresetLayers(request); - return Message.messageToResponse(message); + return message; } - @GET - @Path("/layers/all") - public Response getAllLayers(@Context HttpServletRequest request, @QueryParam(value = "isAvailable") Boolean isAvailable, @QueryParam(value = "db") String db) throws DwException { + + @RequestMapping( value = "/layers/all", method = RequestMethod.GET) + public Message getAllLayers(HttpServletRequest request, @RequestParam(value = "isAvailable",required = false) Boolean isAvailable, @RequestParam(value = "db",required = false) String db) throws DwException { Message message = this.dwLayerService.getAllLayers(request, isAvailable, db); - return Message.messageToResponse(message); + return message; } // query paged custom layers - @GET - @Path("/layers/custom") - public Response queryPagedCustomLayers( - @Context HttpServletRequest request, - @QueryParam("page") Integer page, - @QueryParam("size") Integer size, - @QueryParam("name") String name, - @QueryParam("enabled") Boolean enabled + + @RequestMapping( value = "/layers/custom", method = RequestMethod.GET) + public Message queryPagedCustomLayers( + HttpServletRequest request, + @RequestParam(value = "page",required = false) Integer page, + @RequestParam(value = "size",required = false) Integer size, + @RequestParam(value = "name",required = false) String name, + @RequestParam(value = "enabled",required = false) Boolean enabled )throws DwException { final DwLayerQueryCommand command = new DwLayerQueryCommand(); command.setName(name); @@ -59,71 +55,67 @@ public Response queryPagedCustomLayers( command.setPage(page); command.setSize(size); Message message = this.dwLayerService.queryPagedCustomLayers(request, command); - return Message.messageToResponse(message); + return message; } // create custom layer - @POST - @Path("/layers/custom") - public Response createDwCustomLayer(@Context HttpServletRequest request, DwLayerCreateCommand command) throws DwException { + @RequestMapping( value = "/layers/custom", method = RequestMethod.POST) + public Message createDwCustomLayer( HttpServletRequest request,@RequestBody DwLayerCreateCommand command) throws DwException { Message message = this.dwLayerService.createDwCustomLayer(request, command); - return Message.messageToResponse(message); + return message; } // get layer by id - @GET - @Path("/layers/{id}") - public Response getLayerById( - @Context HttpServletRequest request, - @PathParam("id") Long id + @RequestMapping( value = "/layers/{id}", method = RequestMethod.GET) + public Message getLayerById( + HttpServletRequest request, + @PathVariable("id") Long id ) throws DwException { Message message = this.dwLayerService.getLayerById(request, id); - return Message.messageToResponse(message); + return message; } // delete layer - @DELETE - @Path("/layers/{id}") - public Response deleteById( - @Context HttpServletRequest request, - @PathParam("id") Long id + + @RequestMapping( value = "/layers/{id}", method = RequestMethod.DELETE) + public Message deleteById( + HttpServletRequest request, + @PathVariable("id") Long id ) throws DwException { Message message = this.dwLayerService.deleteById(request, id); - return Message.messageToResponse(message); + return message; } // update layer - @PUT - @Path("/layers/{id}") - public Response update( - @Context HttpServletRequest request, - @PathParam("id") Long id, - DwLayerUpdateCommand command + @RequestMapping( value = "/layers/{id}", method = RequestMethod.PUT) + public Message update( + HttpServletRequest request, + @PathVariable("id") Long id, + @RequestBody DwLayerUpdateCommand command ) throws DwException { command.setId(id); Message message = this.dwLayerService.update(request, command); - return Message.messageToResponse(message); + return message; } // enable layer - @PUT - @Path("/layers/{id}/enable") - public Response enable( - @Context HttpServletRequest request, - @PathParam("id") Long id + @RequestMapping( value = "/layers/{id}/enable", method = RequestMethod.PUT) + public Message enable( + HttpServletRequest request, + @PathVariable("id") Long id ) throws DwException { Message message = this.dwLayerService.enable(request, id); - return Message.messageToResponse(message); + return message; } // disable layer - @PUT - @Path("/layers/{id}/disable") - public Response disable( - @Context HttpServletRequest request, - @PathParam("id") Long id + + @RequestMapping( value = "/layers/{id}/disable", method = RequestMethod.PUT) + public Message disable( + HttpServletRequest request, + @PathVariable("id") Long id ) throws DwException { Message message = this.dwLayerService.disable(request, id); - return Message.messageToResponse(message); + return message; } } diff --git a/dss-apps/dss-datawarehouse-design/data-warehouse-server/src/main/java/com/webank/wedatasphere/warehouse/restful/DwModifierRestfulApi.java b/dss-apps/dss-datawarehouse-design/data-warehouse-server/src/main/java/com/webank/wedatasphere/warehouse/restful/DwModifierRestfulApi.java index 31ac57af1e..1d520f4946 100644 --- a/dss-apps/dss-datawarehouse-design/data-warehouse-server/src/main/java/com/webank/wedatasphere/warehouse/restful/DwModifierRestfulApi.java +++ b/dss-apps/dss-datawarehouse-design/data-warehouse-server/src/main/java/com/webank/wedatasphere/warehouse/restful/DwModifierRestfulApi.java @@ -6,19 +6,13 @@ import com.webank.wedatasphere.warehouse.exception.DwException; import com.webank.wedatasphere.warehouse.service.DwModifierService; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; +import org.springframework.web.bind.annotation.*; import javax.servlet.http.HttpServletRequest; -import javax.ws.rs.*; -import javax.ws.rs.core.Context; -import javax.ws.rs.core.MediaType; -import javax.ws.rs.core.Response; -import java.util.Optional; - -@Component -@Produces(MediaType.APPLICATION_JSON) -@Consumes(MediaType.APPLICATION_JSON) -@Path("/data-warehouse") + + +@RestController +@RequestMapping(value = "/data-warehouse", produces = {"application/json;charset=utf-8"}) public class DwModifierRestfulApi { private final DwModifierService dwModifierService; @@ -28,14 +22,14 @@ public DwModifierRestfulApi(DwModifierService dwModifierService) { this.dwModifierService = dwModifierService; } - @GET - @Path("/modifiers/all") - public Response queryAllModifiers( - @Context HttpServletRequest request, - @QueryParam("typeName") String typeName, - @QueryParam(value = "isAvailable") Boolean isAvailable, - @QueryParam(value = "theme") String theme, - @QueryParam(value = "layer") String layer + + @RequestMapping( value = "/modifiers/all", method = RequestMethod.GET) + public Message queryAllModifiers( + HttpServletRequest request, + @RequestParam(value = "typeName",required = false) String typeName, + @RequestParam(value = "isAvailable",required = false) Boolean isAvailable, + @RequestParam(value = "theme",required = false) String theme, + @RequestParam(value = "layer",required = false) String layer )throws DwException { final DwModifierQueryCommand command = new DwModifierQueryCommand(); command.setName(typeName); @@ -43,18 +37,17 @@ public Response queryAllModifiers( command.setTheme(theme); command.setLayer(layer); Message message = this.dwModifierService.queryAllModifiers(request, command); - return Message.messageToResponse(message); + return message; } // query paged modifiers - @GET - @Path("/modifiers") - public Response queryPagedDecorations( - @Context HttpServletRequest request, - @QueryParam("page") Integer page, - @QueryParam("size") Integer size, - @QueryParam("name") String typeName, - @QueryParam("enabled") String enabled + @RequestMapping( value = "/modifiers", method = RequestMethod.GET) + public Message queryPagedDecorations( + HttpServletRequest request, + @RequestParam(value = "page",required = false) Integer page, + @RequestParam(value = "size",required = false) Integer size, + @RequestParam(value = "name",required = false) String typeName, + @RequestParam(value = "enabled",required = false) String enabled )throws DwException { final DwModifierQueryCommand command = new DwModifierQueryCommand(); command.setName(typeName); @@ -64,71 +57,66 @@ public Response queryPagedDecorations( command.setEnabled(Boolean.parseBoolean(enabled)); } Message message = this.dwModifierService.queryPage(request, command); - return Message.messageToResponse(message); + return message; } // create dw decoration word - @POST - @Path("/modifiers") - public Response create(@Context HttpServletRequest request, DwModifierCreateCommand command) throws DwException { + @RequestMapping( value = "/modifiers", method = RequestMethod.POST) + public Message create(HttpServletRequest request,@RequestBody DwModifierCreateCommand command) throws DwException { Message message = this.dwModifierService.create(request, command); - return Message.messageToResponse(message); + return message; } // fetch one decoration details by id - @GET - @Path("/modifiers/{id}") - public Response getById( - @Context HttpServletRequest request, - @PathParam("id") Long id + @RequestMapping( value = "/modifiers/{id}", method = RequestMethod.GET) + public Message getById( + HttpServletRequest request, + @PathVariable("id") Long id ) throws DwException { Message message = this.dwModifierService.getById(request, id); - return Message.messageToResponse(message); + return message; } // remove decoration logic - @DELETE - @Path("/modifiers/{id}") - public Response deleteById( - @Context HttpServletRequest request, - @PathParam("id") Long id + @RequestMapping( value = "/modifiers/{id}", method = RequestMethod.DELETE) + public Message deleteById( + HttpServletRequest request, + @PathVariable("id") Long id ) throws DwException { Message message = this.dwModifierService.deleteById(request, id); - return Message.messageToResponse(message); + return message; } // update - @PUT - @Path("/modifiers/{id}") - public Response update( - @Context HttpServletRequest request, - @PathParam("id") Long id, - DwModifierUpdateCommand command + @RequestMapping( value = "/modifiers/{id}", method = RequestMethod.PUT) + public Message update( + HttpServletRequest request, + @PathVariable("id") Long id, + @RequestBody DwModifierUpdateCommand command ) throws DwException { command.setId(id); Message message = this.dwModifierService.update(request, command); - return Message.messageToResponse(message); + return message; } // enable modifier - @PUT - @Path("/modifiers/{id}/enable") - public Response enable( - @Context HttpServletRequest request, - @PathParam("id") Long id + + @RequestMapping( value = "/modifiers/{id}/enable", method = RequestMethod.PUT) + public Message enable( + HttpServletRequest request, + @PathVariable("id") Long id ) throws DwException { Message message = this.dwModifierService.enable(request, id); - return Message.messageToResponse(message); + return message; } // disable modifier - @PUT - @Path("/modifiers/{id}/disable") - public Response disable( - @Context HttpServletRequest request, - @PathParam("id") Long id + @RequestMapping( value = "/modifiers/{id}/disable", method = RequestMethod.PUT) + public Message disable( + HttpServletRequest request, + @PathVariable("id") Long id ) throws DwException { Message message = this.dwModifierService.disable(request, id); - return Message.messageToResponse(message); + return message; } } diff --git a/dss-apps/dss-datawarehouse-design/data-warehouse-server/src/main/java/com/webank/wedatasphere/warehouse/restful/DwStatisticalPeriodRestfulApi.java b/dss-apps/dss-datawarehouse-design/data-warehouse-server/src/main/java/com/webank/wedatasphere/warehouse/restful/DwStatisticalPeriodRestfulApi.java index 9948a9cabc..f5eaf9a51a 100644 --- a/dss-apps/dss-datawarehouse-design/data-warehouse-server/src/main/java/com/webank/wedatasphere/warehouse/restful/DwStatisticalPeriodRestfulApi.java +++ b/dss-apps/dss-datawarehouse-design/data-warehouse-server/src/main/java/com/webank/wedatasphere/warehouse/restful/DwStatisticalPeriodRestfulApi.java @@ -6,19 +6,13 @@ import com.webank.wedatasphere.warehouse.exception.DwException; import com.webank.wedatasphere.warehouse.service.DwStatisticalPeriodService; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; +import org.springframework.web.bind.annotation.*; import javax.servlet.http.HttpServletRequest; -import javax.ws.rs.*; -import javax.ws.rs.core.Context; -import javax.ws.rs.core.MediaType; -import javax.ws.rs.core.Response; -import java.util.Optional; - -@Component -@Produces(MediaType.APPLICATION_JSON) -@Consumes(MediaType.APPLICATION_JSON) -@Path("/data-warehouse") + + +@RestController +@RequestMapping(value = "/data-warehouse", produces = {"application/json;charset=utf-8"}) public class DwStatisticalPeriodRestfulApi { private final DwStatisticalPeriodService dwStatisticalPeriodService; @@ -28,14 +22,14 @@ public DwStatisticalPeriodRestfulApi(final DwStatisticalPeriodService dwStatisti this.dwStatisticalPeriodService = dwStatisticalPeriodService; } - @GET - @Path("/statistical_periods/all") - public Response queryAll( - @Context HttpServletRequest request, - @QueryParam("name") String name, - @QueryParam(value = "isAvailable") Boolean isAvailable, - @QueryParam(value = "theme") String theme, - @QueryParam(value = "layer") String layer + + @RequestMapping( value = "/statistical_periods/all", method = RequestMethod.GET) + public Message queryAll( + HttpServletRequest request, + @RequestParam(value = "name",required = false) String name, + @RequestParam(value = "isAvailable",required = false) Boolean isAvailable, + @RequestParam(value = "theme",required = false) String theme, + @RequestParam(value = "layer",required = false) String layer )throws DwException { final DwStatisticalPeriodQueryCommand command = new DwStatisticalPeriodQueryCommand(); command.setName(name); @@ -43,18 +37,18 @@ public Response queryAll( command.setTheme(theme); command.setLayer(layer); Message message = this.dwStatisticalPeriodService.queryAll(request, command); - return Message.messageToResponse(message); + return message; } // query paged statistical periods - @GET - @Path("/statistical_periods") - public Response queryPagedDecorations( - @Context HttpServletRequest request, - @QueryParam("page") Integer page, - @QueryParam("size") Integer size, - @QueryParam("name") String name, - @QueryParam(value = "enabled") String enabled + + @RequestMapping( value = "/statistical_periods", method = RequestMethod.GET) + public Message queryPagedDecorations( + HttpServletRequest request, + @RequestParam(value = "page",required = false) Integer page, + @RequestParam(value = "size",required = false) Integer size, + @RequestParam(value = "name",required = false) String name, + @RequestParam(value = "enabled",required = false) String enabled )throws DwException { final DwStatisticalPeriodQueryCommand command = new DwStatisticalPeriodQueryCommand(); command.setName(name); @@ -64,72 +58,71 @@ public Response queryPagedDecorations( command.setEnabled(Boolean.parseBoolean(enabled)); } Message message = this.dwStatisticalPeriodService.queryPage(request, command); - return Message.messageToResponse(message); + return message; } // create dw statistical_periods - @POST - @Path("/statistical_periods") - public Response create(@Context HttpServletRequest request, DwStatisticalPeriodCreateCommand command) throws DwException { + + @RequestMapping( value = "/statistical_periods", method = RequestMethod.POST) + public Message create(HttpServletRequest request, @RequestBody DwStatisticalPeriodCreateCommand command) throws DwException { Message message = this.dwStatisticalPeriodService.create(request, command); - return Message.messageToResponse(message); + return message; } // fetch one statistical_periods details by id - @GET - @Path("/statistical_periods/{id}") - public Response getById( - @Context HttpServletRequest request, - @PathParam("id") Long id + + @RequestMapping( value = "/statistical_periods/{id}", method = RequestMethod.GET) + public Message getById( + HttpServletRequest request, + @PathVariable("id") Long id ) throws DwException { Message message = this.dwStatisticalPeriodService.getById(request, id); - return Message.messageToResponse(message); + return message; } // remove statistical_periods logic - @DELETE - @Path("/statistical_periods/{id}") - public Response deleteById( - @Context HttpServletRequest request, - @PathParam("id") Long id + + @RequestMapping( value = "/statistical_periods/{id}", method = RequestMethod.DELETE) + public Message deleteById( + HttpServletRequest request, + @PathVariable("id") Long id ) throws DwException { Message message = this.dwStatisticalPeriodService.deleteById(request, id); - return Message.messageToResponse(message); + return message; } // update statistical_periods - @PUT - @Path("/statistical_periods/{id}") - public Response update( - @Context HttpServletRequest request, - @PathParam("id") Long id, - DwStatisticalPeriodUpdateCommand command + + @RequestMapping( value = "/statistical_periods/{id}", method = RequestMethod.PUT) + public Message update( + HttpServletRequest request, + @PathVariable("id") Long id, + @RequestBody DwStatisticalPeriodUpdateCommand command ) throws DwException { command.setId(id); Message message = this.dwStatisticalPeriodService.update(request, command); - return Message.messageToResponse(message); + return message; } // enable statistical_periods - @PUT - @Path("/statistical_periods/{id}/enable") - public Response enable( - @Context HttpServletRequest request, - @PathParam("id") Long id + + @RequestMapping( value = "/statistical_periods/{id}/enable", method = RequestMethod.PUT) + public Message enable( + HttpServletRequest request, + @PathVariable("id") Long id ) throws DwException { Message message = this.dwStatisticalPeriodService.enable(request, id); - return Message.messageToResponse(message); + return message; } // disable statistical_periods - @PUT - @Path("/statistical_periods/{id}/disable") - public Response disable( - @Context HttpServletRequest request, - @PathParam("id") Long id + @RequestMapping( value = "/statistical_periods/{id}/disable", method = RequestMethod.PUT) + public Message disable( + HttpServletRequest request, + @PathVariable("id") Long id ) throws DwException { Message message = this.dwStatisticalPeriodService.disable(request, id); - return Message.messageToResponse(message); + return message; } } diff --git a/dss-apps/dss-datawarehouse-design/data-warehouse-server/src/main/java/com/webank/wedatasphere/warehouse/restful/DwThemeDomainRestfulApi.java b/dss-apps/dss-datawarehouse-design/data-warehouse-server/src/main/java/com/webank/wedatasphere/warehouse/restful/DwThemeDomainRestfulApi.java index 71bc18573e..5ad0f1a6eb 100644 --- a/dss-apps/dss-datawarehouse-design/data-warehouse-server/src/main/java/com/webank/wedatasphere/warehouse/restful/DwThemeDomainRestfulApi.java +++ b/dss-apps/dss-datawarehouse-design/data-warehouse-server/src/main/java/com/webank/wedatasphere/warehouse/restful/DwThemeDomainRestfulApi.java @@ -5,19 +5,12 @@ import com.webank.wedatasphere.warehouse.exception.DwException; import com.webank.wedatasphere.warehouse.service.DwThemeDomainService; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; +import org.springframework.web.bind.annotation.*; import javax.servlet.http.HttpServletRequest; -import javax.ws.rs.*; -import javax.ws.rs.core.Context; -import javax.ws.rs.core.MediaType; -import javax.ws.rs.core.Response; -import java.util.Optional; - -@Component -@Produces(MediaType.APPLICATION_JSON) -@Consumes(MediaType.APPLICATION_JSON) -@Path("/data-warehouse") + +@RestController +@RequestMapping(value = "/data-warehouse", produces = {"application/json;charset=utf-8"}) public class DwThemeDomainRestfulApi { private final DwThemeDomainService dwThemeDomainService; @@ -28,14 +21,14 @@ public DwThemeDomainRestfulApi(DwThemeDomainService dwThemeDomainService) { } // query paged theme domains - @GET - @Path("/themedomains") - public Response queryPagedCustomLayers( - @Context HttpServletRequest request, - @QueryParam("page") Integer page, - @QueryParam("size") Integer size, - @QueryParam("name") String name, - @QueryParam("enabled") Boolean enabled + + @RequestMapping( value = "/themedomains", method = RequestMethod.GET) + public Message queryPagedCustomLayers( + HttpServletRequest request, + @RequestParam(value = "page",required = false) Integer page, + @RequestParam(value = "size",required = false) Integer size, + @RequestParam(value = "name",required = false) String name, + @RequestParam(value = "enabled",required = false) Boolean enabled )throws DwException { final DwThemeDomainQueryCommand command = new DwThemeDomainQueryCommand(); command.setName(name); @@ -43,85 +36,82 @@ public Response queryPagedCustomLayers( command.setSize(size); command.setEnabled(enabled); Message message = this.dwThemeDomainService.queryPage(request, command); - return Message.messageToResponse(message); + return message; } - @GET - @Path("/themedomains/all") - public Response queryAllThemeDomains( - @Context HttpServletRequest request, - @QueryParam("name") String name, - @QueryParam(value = "isAvailable") Boolean isAvailable + + @RequestMapping( value = "/themedomains/all", method = RequestMethod.GET) + public Message queryAllThemeDomains( + HttpServletRequest request, + @RequestParam(value = "name",required = false) String name, + @RequestParam(value = "isAvailable",required = false) Boolean isAvailable )throws DwException { final DwThemeDomainQueryCommand command = new DwThemeDomainQueryCommand(); command.setName(name); command.setEnabled(isAvailable); Message message = this.dwThemeDomainService.queryAllThemeDomains(request, command); - return Message.messageToResponse(message); + return message; } // create theme domain - @POST - @Path("/themedomains") - public Response createDwCustomLayer(@Context HttpServletRequest request, DwThemeDomainCreateCommand command) throws DwException { + @RequestMapping( value = "/themedomains", method = RequestMethod.POST) + public Message createDwCustomLayer(HttpServletRequest request,@RequestBody DwThemeDomainCreateCommand command) throws DwException { Message message = this.dwThemeDomainService.create(request, command); - return Message.messageToResponse(message); + return message; } // get theme domain by id - @GET - @Path("/themedomains/{id}") - public Response getLayerById( - @Context HttpServletRequest request, - @PathParam("id") Long id + @RequestMapping( value = "/themedomains/{id}", method = RequestMethod.GET) + public Message getLayerById( + HttpServletRequest request, + @PathVariable("id") Long id ) throws DwException { Message message = this.dwThemeDomainService.getById(request, id); - return Message.messageToResponse(message); + return message; } // delete theme domain - @DELETE - @Path("/themedomains/{id}") - public Response deleteById( - @Context HttpServletRequest request, - @PathParam("id") Long id + + @RequestMapping( value = "/themedomains/{id}", method = RequestMethod.DELETE) + public Message deleteById( + HttpServletRequest request, + @PathVariable("id") Long id ) throws DwException { Message message = this.dwThemeDomainService.deleteById(request, id); - return Message.messageToResponse(message); + return message; } // update theme domain - @PUT - @Path("/themedomains/{id}") - public Response update( - @Context HttpServletRequest request, - @PathParam("id") Long id, - DwThemeDomainUpdateCommand command + + @RequestMapping( value = "/themedomains/{id}", method = RequestMethod.PUT) + public Message update( + HttpServletRequest request, + @PathVariable("id") Long id, + @RequestBody DwThemeDomainUpdateCommand command ) throws DwException { command.setId(id); Message message = this.dwThemeDomainService.update(request, command); - return Message.messageToResponse(message); + return message; } // enable theme domain - @PUT - @Path("/themedomains/{id}/enable") - public Response enable( - @Context HttpServletRequest request, - @PathParam("id") Long id + + @RequestMapping( value = "/themedomains/{id}/enable", method = RequestMethod.PUT) + public Message enable( + HttpServletRequest request, + @PathVariable("id") Long id ) throws DwException { Message message = this.dwThemeDomainService.enable(request, id); - return Message.messageToResponse(message); + return message; } // disable theme domain - @PUT - @Path("/themedomains/{id}/disable") - public Response disable( - @Context HttpServletRequest request, - @PathParam("id") Long id + @RequestMapping( value = "/themedomains/{id}/disable", method = RequestMethod.PUT) + public Message disable( + HttpServletRequest request, + @PathVariable("id") Long id ) throws DwException { Message message = this.dwThemeDomainService.disable(request, id); - return Message.messageToResponse(message); + return message; } } diff --git a/dss-apps/dss-datawarehouse-design/data-warehouse-server/src/main/resources/application.yml b/dss-apps/dss-datawarehouse-design/data-warehouse-server/src/main/resources/application.yml index 191013fa28..97b323808f 100644 --- a/dss-apps/dss-datawarehouse-design/data-warehouse-server/src/main/resources/application.yml +++ b/dss-apps/dss-datawarehouse-design/data-warehouse-server/src/main/resources/application.yml @@ -1,5 +1,7 @@ server: port: 20080 + servlet: + context-path: /api/rest_j/v1 spring: application: name: data-warehouse-server diff --git a/dss-apps/dss-datawarehouse-design/data-warehouse-server/src/main/resources/linkis.properties b/dss-apps/dss-datawarehouse-design/data-warehouse-server/src/main/resources/linkis.properties index 0c5ead7d62..6c10965f5e 100644 --- a/dss-apps/dss-datawarehouse-design/data-warehouse-server/src/main/resources/linkis.properties +++ b/dss-apps/dss-datawarehouse-design/data-warehouse-server/src/main/resources/linkis.properties @@ -31,7 +31,7 @@ wds.linkis.server.version=v1 ## datasource client wds.datawarehouse.datasource.client.serverurl=http://dss.shineweng.com:8088 wds.datawarehouse.datasource.client.authtoken.key=hdfs -wds.datawarehouse.datasource.client.authtoken.value=hdfs +wds.datawarehouse.datasource.client.authtoken.value=BML-AUTH wds.datawarehouse.datasource.client.dws.version=v1 ##restful diff --git a/dss-apps/dss-datawarehouse-design/data-warehouse-service/src/main/java/com/webank/wedatasphere/warehouse/service/impl/DwLayerServiceImpl.java b/dss-apps/dss-datawarehouse-design/data-warehouse-service/src/main/java/com/webank/wedatasphere/warehouse/service/impl/DwLayerServiceImpl.java index baaaf8f272..3b265e7f92 100644 --- a/dss-apps/dss-datawarehouse-design/data-warehouse-service/src/main/java/com/webank/wedatasphere/warehouse/service/impl/DwLayerServiceImpl.java +++ b/dss-apps/dss-datawarehouse-design/data-warehouse-service/src/main/java/com/webank/wedatasphere/warehouse/service/impl/DwLayerServiceImpl.java @@ -28,7 +28,6 @@ import com.webank.wedatasphere.warehouse.dto.PageInfo; import com.webank.wedatasphere.warehouse.exception.DwException; import com.webank.wedatasphere.warehouse.dao.mapper.DwLayerMapper; -//import com.webank.wedatasphere.warehouse.mapper.DwLayerModelMapper; import com.webank.wedatasphere.warehouse.exception.DwExceptionCode; import com.webank.wedatasphere.warehouse.service.DwDomainReferenceAdapter; import com.webank.wedatasphere.warehouse.service.DwLayerService; diff --git a/dss-apps/dss-datawarehouse-design/governance-dao/pom.xml b/dss-apps/dss-datawarehouse-design/governance-dao/pom.xml index 0baa851007..5886d38f09 100644 --- a/dss-apps/dss-datawarehouse-design/governance-dao/pom.xml +++ b/dss-apps/dss-datawarehouse-design/governance-dao/pom.xml @@ -45,6 +45,12 @@ lombok provided + + + mysql + mysql-connector-java + 5.1.49 + From 09580ee75aba678e54197a9e74e68a76974ca947 Mon Sep 17 00:00:00 2001 From: limeng Date: Wed, 30 Mar 2022 12:02:24 +0800 Subject: [PATCH 2/3] dataAssets dataWarehouse dataModel --- .../wedatasphere/warehouse/service/impl/DwDsServiceImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dss-apps/dss-datawarehouse-design/data-warehouse-service/src/main/java/com/webank/wedatasphere/warehouse/service/impl/DwDsServiceImpl.java b/dss-apps/dss-datawarehouse-design/data-warehouse-service/src/main/java/com/webank/wedatasphere/warehouse/service/impl/DwDsServiceImpl.java index 7a4fc07eeb..30ba1cb9d6 100644 --- a/dss-apps/dss-datawarehouse-design/data-warehouse-service/src/main/java/com/webank/wedatasphere/warehouse/service/impl/DwDsServiceImpl.java +++ b/dss-apps/dss-datawarehouse-design/data-warehouse-service/src/main/java/com/webank/wedatasphere/warehouse/service/impl/DwDsServiceImpl.java @@ -36,7 +36,7 @@ public Message getAllHiveDbs(HttpServletRequest request) throws Exception { GetMetadataSourceAllDatabasesAction action = GetMetadataSourceAllDatabasesAction.builder().setUser(userName).build(); GetMetadataSourceAllDatabasesResult result = client.getAllDBMetaDataSource(action); - List dbs = result.getDbs(); + List dbs = result.allDbs(); return Message.ok().data("list", dbs); } catch (Exception e) { if (e instanceof ErrorException) { From c7ae963aee5afd78f30b5ebc2b0aebef4acafa34 Mon Sep 17 00:00:00 2001 From: limeng Date: Thu, 31 Mar 2022 21:00:34 +0800 Subject: [PATCH 3/3] dataAssets dataWarehouse dataModel --- .../dao/impl/MetaInfoMapperImpl.java | 140 ------------------ .../data/governance/utils/DataSourceUtil.java | 27 ---- .../data/governance/utils/RestfulUtils.java | 50 ------- .../warehouse/client/RemoteClientTest.java | 5 - .../linkis/mybatis/DataSourceConfig.java | 2 +- .../mybatis/MyBatisMapperScannerConfig.java | 3 +- .../mybatis/MybatisConfigurationFactory.java | 2 +- .../mybatis/conf/MybatisConfiguration.java | 2 +- .../data-warehouse-service/pom.xml | 21 +++ .../service/impl/DwDsServiceImpl.java | 12 +- .../warehouse/LinkisRemoteClientHolder.scala | 7 +- .../warehouse/RemoteClientTest.java | 25 ---- .../warehouse/DwLayerRemoteClientTest.scala | 6 - 13 files changed, 36 insertions(+), 266 deletions(-) delete mode 100644 dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/dao/impl/MetaInfoMapperImpl.java delete mode 100644 dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/utils/DataSourceUtil.java delete mode 100644 dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/utils/RestfulUtils.java rename dss-apps/dss-datawarehouse-design/data-warehouse-mybatis/src/main/java/{com/webank/wedatasphere => org/apache}/linkis/mybatis/DataSourceConfig.java (98%) rename dss-apps/dss-datawarehouse-design/data-warehouse-mybatis/src/main/java/{com/webank/wedatasphere => org/apache}/linkis/mybatis/MyBatisMapperScannerConfig.java (93%) rename dss-apps/dss-datawarehouse-design/data-warehouse-mybatis/src/main/java/{com/webank/wedatasphere => org/apache}/linkis/mybatis/MybatisConfigurationFactory.java (99%) rename dss-apps/dss-datawarehouse-design/data-warehouse-mybatis/src/main/java/{com/webank/wedatasphere => org/apache}/linkis/mybatis/conf/MybatisConfiguration.java (98%) diff --git a/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/dao/impl/MetaInfoMapperImpl.java b/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/dao/impl/MetaInfoMapperImpl.java deleted file mode 100644 index 6ec20f2b07..0000000000 --- a/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/dao/impl/MetaInfoMapperImpl.java +++ /dev/null @@ -1,140 +0,0 @@ -package com.webank.wedatasphere.dss.data.governance.dao.impl; - -import com.webank.wedatasphere.dss.data.governance.dao.MetaInfoMapper; -import com.webank.wedatasphere.dss.data.governance.entity.PartInfo; -import com.webank.wedatasphere.dss.data.governance.entity.TableInfo; -import com.webank.wedatasphere.dss.data.governance.exception.DAOException; -import com.webank.wedatasphere.dss.data.governance.utils.DataSourceUtil; -import com.webank.wedatasphere.dss.data.governance.utils.DateUtil; - -import javax.sql.DataSource; -import java.sql.Connection; -import java.sql.PreparedStatement; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Comparator; -import java.util.List; -import java.util.stream.Collectors; - - -public class MetaInfoMapperImpl implements MetaInfoMapper { - @Override - public Long getTableStorage() throws DAOException { - DataSource dataSource = DataSourceUtil.getDataSource(); - - long num = 0; - PreparedStatement ps = null; - ResultSet rs = null; - try (Connection con = dataSource.getConnection()) { - String sql = "select SUM(PARAM_VALUE) from TABLE_PARAMS WHERE PARAM_KEY='totalSize'"; - ps = con.prepareStatement(sql); - rs = ps.executeQuery(); - while (rs.next()) { - num = rs.getLong(1); - } - String sql2 = "select SUM(PARAM_VALUE) from PARTITION_PARAMS WHERE PARAM_KEY='totalSize'"; - ps = con.prepareStatement(sql2); - rs = ps.executeQuery(); - while (rs.next()) { - num = num + rs.getLong(1); - } - } catch (SQLException e) { - throw new DAOException(23001, e.getMessage()); - } - return num; - } - - @Override - public List getTop10Table() throws DAOException { - DataSource dataSource = DataSourceUtil.getDataSource(); - - PreparedStatement ps = null; - ResultSet rs = null; - List tableInfos = new ArrayList<>(); - try (Connection con = dataSource.getConnection();) { - String sql = "select DBS.NAME ,TBLS.TBL_NAME,TABLE_PARAMS.PARAM_VALUE as totalSize from DBS, TBLS,TABLE_PARAMS where TBLS.TBL_ID=TABLE_PARAMS.TBL_ID AND TBLS.DB_ID=DBS.DB_ID AND TABLE_PARAMS.PARAM_KEY='totalSize' order by totalSize DESC limit 10"; - ps = con.prepareStatement(sql); - rs = ps.executeQuery(); - while (rs.next()) { - TableInfo tableinfo = new TableInfo(); - tableinfo.setTableName(rs.getString(1) + "." + rs.getString(2)); - tableinfo.setStorage(rs.getString(3)); - tableInfos.add(tableinfo); - } - String sql2 = "select DBS.NAME ,TBLS.TBL_NAME,SUM(PARTITION_PARAMS.PARAM_VALUE) as totalSize from DBS,TBLS,PARTITIONS ,PARTITION_PARAMS where DBS.DB_ID=TBLS.DB_ID AND TBLS.TBL_ID=PARTITIONS.TBL_ID AND PARTITIONS.PART_ID =PARTITION_PARAMS.PART_ID AND PARTITION_PARAMS.PARAM_KEY='totalSize' group by TBLS.TBL_NAME order by totalSize desc limit 10"; - ps = con.prepareStatement(sql2); - rs = ps.executeQuery(); - while (rs.next()) { - TableInfo tableinfo = new TableInfo(); - tableinfo.setTableName(rs.getString(1) + "." + rs.getString(2)); - tableinfo.setStorage(rs.getString(3)); - tableInfos.add(tableinfo); - } - - tableInfos.sort((o1, o2) -> { - long result = Long.parseLong(o2.getStorage()) - Long.parseLong(o1.getStorage()); - return Long.compare(result, 0L); - }); - } catch (SQLException e) { - throw new DAOException(23001, e.getMessage()); - } - - return tableInfos.stream().limit(10).collect(Collectors.toList()); - } - - @Override - public int getTableInfo(String dbName, String tableName, Boolean isPartTable) throws DAOException { - DataSource dataSource = DataSourceUtil.getDataSource(); - PreparedStatement ps = null; - ResultSet rs = null; - int res = 0; - try (Connection con = dataSource.getConnection()) { - String sql = null; - if (isPartTable == false) { - sql = "select TABLE_PARAMS.PARAM_VALUE as totalSize from DBS, TBLS,TABLE_PARAMS where TBLS.TBL_ID=TABLE_PARAMS.TBL_ID AND TBLS.DB_ID=DBS.DB_ID AND TABLE_PARAMS.PARAM_KEY='totalSize' AND DBS.NAME=" + "'" + dbName + "' AND TBLS.TBL_NAME=" + "'" + tableName + "'"; - } else { - - sql = "select SUM(PARTITION_PARAMS.PARAM_VALUE) as totalSize from DBS,TBLS,PARTITIONS ,PARTITION_PARAMS where DBS.DB_ID=TBLS.DB_ID AND TBLS.TBL_ID=PARTITIONS.TBL_ID AND PARTITIONS.PART_ID =PARTITION_PARAMS.PART_ID AND PARTITION_PARAMS.PARAM_KEY='totalSize' AND DBS.NAME=" + "'" + dbName + "' AND TBLS.TBL_NAME=" + "'" + tableName + "' group by TBLS.TBL_NAME"; - } - ps = con.prepareStatement(sql); - rs = ps.executeQuery(); - while (rs.next()) { - res = rs.getInt(1); - } - } catch (SQLException e) { - throw new DAOException(23001, e.getMessage()); - } - return res; - } - - @Override - public List getPartInfo(String dbName, String tableName) throws DAOException { - DataSource dataSource = DataSourceUtil.getDataSource(); - - PreparedStatement ps = null; - ResultSet rs = null; - List PartInfos = new ArrayList<>(); - try (Connection con = dataSource.getConnection()) { - String sql = "select b.PART_NAME,b.CREATE_TIME,MAX(CASE c.PARAM_KEY WHEN 'transient_lastDdlTime' THEN c.PARAM_VALUE ELSE null END) transient_lastDdlTime ,MAX(CASE c.PARAM_KEY WHEN 'numRows' THEN c.PARAM_VALUE ELSE null END) numRows,MAX(CASE c.PARAM_KEY WHEN 'totalSize' THEN c.PARAM_VALUE ELSE null END) totalSize, MAX(CASE c.PARAM_KEY WHEN 'numFiles' THEN c.PARAM_VALUE ELSE null END) numFiles from TBLS a,PARTITIONS b,PARTITION_PARAMS c,DBS d where a.TBL_NAME=" + "'" + tableName + "'" + "AND d.NAME=" + "'" + dbName + "'" + "AND a.TBL_ID=b.TBL_ID AND a.DB_ID=d.DB_ID AND b.PART_ID=c.PART_ID GROUP BY c.PART_ID"; - ps = con.prepareStatement(sql); - rs = ps.executeQuery(); - while (rs.next()) { - PartInfo part = new PartInfo(); - part.setPartName(rs.getString(1)); - part.setCreateTime(DateUtil.unixToTimeStr(Long.valueOf(rs.getInt(2)) * 1000)); - part.setLastAccessTime(DateUtil.unixToTimeStr(Long.valueOf(rs.getInt(3)) * 1000)); - part.setReordCnt(rs.getInt(4)); - part.setStore(rs.getInt(5)); - part.setFileCount(rs.getInt(6)); - PartInfos.add(part); - } - - } catch (SQLException e) { - throw new DAOException(23001, e.getMessage()); - } - - return PartInfos; - } -} diff --git a/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/utils/DataSourceUtil.java b/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/utils/DataSourceUtil.java deleted file mode 100644 index 92a4a2316d..0000000000 --- a/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/utils/DataSourceUtil.java +++ /dev/null @@ -1,27 +0,0 @@ -package com.webank.wedatasphere.dss.data.governance.utils; - -import com.alibaba.druid.pool.DruidDataSource; -import com.webank.wedatasphere.dss.data.governance.conf.GovernanceConf; - - -public class DataSourceUtil { - private static DruidDataSource druidDataSource =null; - - private DataSourceUtil(){} - - public static DruidDataSource getDataSource(){ - if(druidDataSource ==null) { - synchronized (DataSourceUtil.class){ - if(druidDataSource ==null){ - druidDataSource = new DruidDataSource(); - druidDataSource.setDriverClassName(GovernanceConf.METASTORE_DATASOURCE_DRIVER.getValue()); - druidDataSource.setUrl(GovernanceConf.METASTORE_DATASOURCE_URL.getValue()); - druidDataSource.setUsername(GovernanceConf.METASTORE_DATASOURCE_USERNAME.getValue()); - druidDataSource.setPassword(GovernanceConf.METASTORE_DATASOURCE_PASSWORD.getValue()); - } - } - } - return druidDataSource; - } - -} diff --git a/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/utils/RestfulUtils.java b/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/utils/RestfulUtils.java deleted file mode 100644 index e2f0d966d0..0000000000 --- a/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/utils/RestfulUtils.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * - * * Copyright 2019 WeBank - * * - * * Licensed under the Apache License, Version 2.0 (the "License"); - * * you may not use this file except in compliance with the License. - * * You may obtain a copy of the License at - * * - * * http://www.apache.org/licenses/LICENSE-2.0 - * * - * * Unless required by applicable law or agreed to in writing, software - * * distributed under the License is distributed on an "AS IS" BASIS, - * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * * See the License for the specific language governing permissions and - * * limitations under the License. - * - */ - -package com.webank.wedatasphere.dss.data.governance.utils; - -import org.apache.linkis.server.Message; -import org.apache.commons.math3.util.Pair; - -import javax.ws.rs.core.Response; -import java.util.Arrays; - - -public class RestfulUtils { - - public static Response dealError(String reason){ - Message message = Message.error(reason); - return Message.messageToResponse(message); - } - - public static Response dealOk(String msg){ - Message message = Message.ok(msg); - return Message.messageToResponse(message); - } - - - - @SafeVarargs - public static Response dealOk(String msg, Pair... data){ - Message message = Message.ok(msg); - Arrays.stream(data).forEach(p -> message.data(p.getKey(), p.getValue())); - return Message.messageToResponse(message); - } - - -} diff --git a/dss-apps/dss-datawarehouse-design/data-warehouse-client/src/test/java/com/webank/wedatasphere/warehouse/client/RemoteClientTest.java b/dss-apps/dss-datawarehouse-design/data-warehouse-client/src/test/java/com/webank/wedatasphere/warehouse/client/RemoteClientTest.java index 810bb4f1f9..28f524089c 100644 --- a/dss-apps/dss-datawarehouse-design/data-warehouse-client/src/test/java/com/webank/wedatasphere/warehouse/client/RemoteClientTest.java +++ b/dss-apps/dss-datawarehouse-design/data-warehouse-client/src/test/java/com/webank/wedatasphere/warehouse/client/RemoteClientTest.java @@ -18,11 +18,6 @@ public class RemoteClientTest { public static void main(String[] args) { -// LinkisMetadataSourceRemoteClient client = getMetadataSourceRemoteClient(); -// GetMetadataSourceAllDatabasesAction action = GetMetadataSourceAllDatabasesAction.builder().setUser("hdfs").build(); -// GetMetadataSourceAllDatabasesResult result = client.getAllDBMetaDataSource(action); -// List dbs = result.getDbs(); -// System.out.println(dbs); DWSClientConfig clientConfig = ((DWSClientConfigBuilder) (DWSClientConfigBuilder.newBuilder().addServerUrl("http://dss.shineweng.com:8088") .connectionTimeout(30000).discoveryEnabled(true) .discoveryFrequency(1, TimeUnit.MINUTES) diff --git a/dss-apps/dss-datawarehouse-design/data-warehouse-mybatis/src/main/java/com/webank/wedatasphere/linkis/mybatis/DataSourceConfig.java b/dss-apps/dss-datawarehouse-design/data-warehouse-mybatis/src/main/java/org/apache/linkis/mybatis/DataSourceConfig.java similarity index 98% rename from dss-apps/dss-datawarehouse-design/data-warehouse-mybatis/src/main/java/com/webank/wedatasphere/linkis/mybatis/DataSourceConfig.java rename to dss-apps/dss-datawarehouse-design/data-warehouse-mybatis/src/main/java/org/apache/linkis/mybatis/DataSourceConfig.java index 2f73054d1a..033fb8cff1 100644 --- a/dss-apps/dss-datawarehouse-design/data-warehouse-mybatis/src/main/java/com/webank/wedatasphere/linkis/mybatis/DataSourceConfig.java +++ b/dss-apps/dss-datawarehouse-design/data-warehouse-mybatis/src/main/java/org/apache/linkis/mybatis/DataSourceConfig.java @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.webank.wedatasphere.linkis.mybatis; +package org.apache.linkis.mybatis; import org.apache.linkis.common.utils.JavaLog; import org.apache.linkis.mybatis.conf.MybatisConfiguration; diff --git a/dss-apps/dss-datawarehouse-design/data-warehouse-mybatis/src/main/java/com/webank/wedatasphere/linkis/mybatis/MyBatisMapperScannerConfig.java b/dss-apps/dss-datawarehouse-design/data-warehouse-mybatis/src/main/java/org/apache/linkis/mybatis/MyBatisMapperScannerConfig.java similarity index 93% rename from dss-apps/dss-datawarehouse-design/data-warehouse-mybatis/src/main/java/com/webank/wedatasphere/linkis/mybatis/MyBatisMapperScannerConfig.java rename to dss-apps/dss-datawarehouse-design/data-warehouse-mybatis/src/main/java/org/apache/linkis/mybatis/MyBatisMapperScannerConfig.java index 18e50f37d5..804c562b61 100644 --- a/dss-apps/dss-datawarehouse-design/data-warehouse-mybatis/src/main/java/com/webank/wedatasphere/linkis/mybatis/MyBatisMapperScannerConfig.java +++ b/dss-apps/dss-datawarehouse-design/data-warehouse-mybatis/src/main/java/org/apache/linkis/mybatis/MyBatisMapperScannerConfig.java @@ -14,9 +14,8 @@ * limitations under the License. */ -package com.webank.wedatasphere.linkis.mybatis; +package org.apache.linkis.mybatis; -import org.apache.linkis.mybatis.MybatisConfigurationFactory; import org.apache.linkis.mybatis.conf.MybatisConfiguration; import org.mybatis.spring.mapper.MapperScannerConfigurer; import org.springframework.boot.autoconfigure.AutoConfigureAfter; diff --git a/dss-apps/dss-datawarehouse-design/data-warehouse-mybatis/src/main/java/com/webank/wedatasphere/linkis/mybatis/MybatisConfigurationFactory.java b/dss-apps/dss-datawarehouse-design/data-warehouse-mybatis/src/main/java/org/apache/linkis/mybatis/MybatisConfigurationFactory.java similarity index 99% rename from dss-apps/dss-datawarehouse-design/data-warehouse-mybatis/src/main/java/com/webank/wedatasphere/linkis/mybatis/MybatisConfigurationFactory.java rename to dss-apps/dss-datawarehouse-design/data-warehouse-mybatis/src/main/java/org/apache/linkis/mybatis/MybatisConfigurationFactory.java index 70dc22aba2..be56464746 100644 --- a/dss-apps/dss-datawarehouse-design/data-warehouse-mybatis/src/main/java/com/webank/wedatasphere/linkis/mybatis/MybatisConfigurationFactory.java +++ b/dss-apps/dss-datawarehouse-design/data-warehouse-mybatis/src/main/java/org/apache/linkis/mybatis/MybatisConfigurationFactory.java @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.webank.wedatasphere.linkis.mybatis; +package org.apache.linkis.mybatis; import com.baomidou.mybatisplus.annotation.DbType; import com.baomidou.mybatisplus.extension.plugins.MybatisPlusInterceptor; diff --git a/dss-apps/dss-datawarehouse-design/data-warehouse-mybatis/src/main/java/com/webank/wedatasphere/linkis/mybatis/conf/MybatisConfiguration.java b/dss-apps/dss-datawarehouse-design/data-warehouse-mybatis/src/main/java/org/apache/linkis/mybatis/conf/MybatisConfiguration.java similarity index 98% rename from dss-apps/dss-datawarehouse-design/data-warehouse-mybatis/src/main/java/com/webank/wedatasphere/linkis/mybatis/conf/MybatisConfiguration.java rename to dss-apps/dss-datawarehouse-design/data-warehouse-mybatis/src/main/java/org/apache/linkis/mybatis/conf/MybatisConfiguration.java index 56c864a2af..43b91c5d80 100644 --- a/dss-apps/dss-datawarehouse-design/data-warehouse-mybatis/src/main/java/com/webank/wedatasphere/linkis/mybatis/conf/MybatisConfiguration.java +++ b/dss-apps/dss-datawarehouse-design/data-warehouse-mybatis/src/main/java/org/apache/linkis/mybatis/conf/MybatisConfiguration.java @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.webank.wedatasphere.linkis.mybatis.conf; +package org.apache.linkis.mybatis.conf; import org.apache.linkis.common.conf.CommonVars; diff --git a/dss-apps/dss-datawarehouse-design/data-warehouse-service/pom.xml b/dss-apps/dss-datawarehouse-design/data-warehouse-service/pom.xml index 895604f8db..29984d6647 100644 --- a/dss-apps/dss-datawarehouse-design/data-warehouse-service/pom.xml +++ b/dss-apps/dss-datawarehouse-design/data-warehouse-service/pom.xml @@ -114,6 +114,27 @@ + + org.apache.linkis + linkis-computation-client + ${linkis.version} + + + org.apache.linkis + linkis-gateway-httpclient-support + + + org.apache.linkis + linkis-bml + + + com.google.code.gson + gson + + + + + org.projectlombok lombok diff --git a/dss-apps/dss-datawarehouse-design/data-warehouse-service/src/main/java/com/webank/wedatasphere/warehouse/service/impl/DwDsServiceImpl.java b/dss-apps/dss-datawarehouse-design/data-warehouse-service/src/main/java/com/webank/wedatasphere/warehouse/service/impl/DwDsServiceImpl.java index 30ba1cb9d6..06390c1474 100644 --- a/dss-apps/dss-datawarehouse-design/data-warehouse-service/src/main/java/com/webank/wedatasphere/warehouse/service/impl/DwDsServiceImpl.java +++ b/dss-apps/dss-datawarehouse-design/data-warehouse-service/src/main/java/com/webank/wedatasphere/warehouse/service/impl/DwDsServiceImpl.java @@ -7,7 +7,6 @@ import com.webank.wedatasphere.dss.framework.workspace.client.response.GetWorkspaceUsersResult; import org.apache.linkis.common.exception.ErrorException; import org.apache.linkis.datasource.client.impl.LinkisMetadataSourceRemoteClient; -import org.apache.linkis.datasource.client.request.GetMetadataSourceAllDatabasesAction; import org.apache.linkis.datasource.client.response.GetMetadataSourceAllDatabasesResult; import org.apache.linkis.server.Message; import org.apache.linkis.server.security.SecurityFilter; @@ -15,6 +14,9 @@ import com.webank.wedatasphere.warehouse.exception.DwException; import com.webank.wedatasphere.warehouse.exception.DwExceptionCode; import com.webank.wedatasphere.warehouse.service.DwDsService; +import org.apache.linkis.ujes.client.UJESClient; +import org.apache.linkis.ujes.client.request.GetDBSAction; +import org.apache.linkis.ujes.client.response.GetDBSResult; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Service; @@ -32,11 +34,11 @@ public Message getAllHiveDbs(HttpServletRequest request) throws Exception { String userName = SecurityFilter.getLoginUsername(request); LOGGER.info("listAllDataSources userName:" + userName); try { - LinkisMetadataSourceRemoteClient client = LinkisRemoteClientHolder.getMetadataSourceRemoteClient(); + UJESClient client = LinkisRemoteClientHolder.getMetadataSourceRemoteClient(); - GetMetadataSourceAllDatabasesAction action = GetMetadataSourceAllDatabasesAction.builder().setUser(userName).build(); - GetMetadataSourceAllDatabasesResult result = client.getAllDBMetaDataSource(action); - List dbs = result.allDbs(); + GetDBSAction action = GetDBSAction.builder().setUser(userName).build(); + GetDBSResult result = client.getDBS(action); + List dbs = result.getDBSName(); return Message.ok().data("list", dbs); } catch (Exception e) { if (e instanceof ErrorException) { diff --git a/dss-apps/dss-datawarehouse-design/data-warehouse-service/src/main/scala/com/webank/wedatasphere/warehouse/LinkisRemoteClientHolder.scala b/dss-apps/dss-datawarehouse-design/data-warehouse-service/src/main/scala/com/webank/wedatasphere/warehouse/LinkisRemoteClientHolder.scala index eae22a948a..1cea04443d 100644 --- a/dss-apps/dss-datawarehouse-design/data-warehouse-service/src/main/scala/com/webank/wedatasphere/warehouse/LinkisRemoteClientHolder.scala +++ b/dss-apps/dss-datawarehouse-design/data-warehouse-service/src/main/scala/com/webank/wedatasphere/warehouse/LinkisRemoteClientHolder.scala @@ -3,10 +3,11 @@ package com.webank.wedatasphere.warehouse import com.webank.wedatasphere.dss.data.governance.impl.LinkisDataAssetsRemoteClient import com.webank.wedatasphere.dss.datamodel.center.client.impl.LinkisDatamodelCenterRemoteClient import com.webank.wedatasphere.dss.framework.workspace.client.impl.LinkisWorkSpaceRemoteClient -import org.apache.linkis.datasource.client.impl.{LinkisDataSourceRemoteClient, LinkisMetadataSourceRemoteClient} +import org.apache.linkis.datasource.client.impl.LinkisDataSourceRemoteClient import org.apache.linkis.httpclient.dws.authentication.TokenAuthenticationStrategy import org.apache.linkis.httpclient.dws.config.{DWSClientConfig, DWSClientConfigBuilder} import com.webank.wedatasphere.warehouse.client.DwDataSourceConfiguration +import org.apache.linkis.ujes.client.UJESClient import java.lang import java.util.concurrent.TimeUnit @@ -42,7 +43,7 @@ object LinkisRemoteClientHolder { val dataSourceClient = new LinkisDataSourceRemoteClient(clientConfig) - val metadataSourceRemoteClient = new LinkisMetadataSourceRemoteClient(clientConfig) + val metadataSourceRemoteClient = UJESClient.apply(clientConfig) var dataModelRemoteClient = new LinkisDatamodelCenterRemoteClient(clientConfig) @@ -66,7 +67,7 @@ object LinkisRemoteClientHolder { dataAssetsRemoteClient } - def getMetadataSourceRemoteClient: LinkisMetadataSourceRemoteClient = { + def getMetadataSourceRemoteClient: UJESClient = { metadataSourceRemoteClient } } diff --git a/dss-apps/dss-datawarehouse-design/data-warehouse-service/src/test/java/com/webank/wedatasphere/warehouse/RemoteClientTest.java b/dss-apps/dss-datawarehouse-design/data-warehouse-service/src/test/java/com/webank/wedatasphere/warehouse/RemoteClientTest.java index 0db5085132..43a0dccd91 100644 --- a/dss-apps/dss-datawarehouse-design/data-warehouse-service/src/test/java/com/webank/wedatasphere/warehouse/RemoteClientTest.java +++ b/dss-apps/dss-datawarehouse-design/data-warehouse-service/src/test/java/com/webank/wedatasphere/warehouse/RemoteClientTest.java @@ -1,34 +1,9 @@ package com.webank.wedatasphere.warehouse; -import org.apache.linkis.datasource.client.impl.LinkisMetadataSourceRemoteClient; -import org.apache.linkis.datasource.client.request.GetMetadataSourceAllDatabasesAction; -import org.apache.linkis.datasource.client.response.GetMetadataSourceAllDatabasesResult; -import org.apache.linkis.httpclient.dws.authentication.StaticAuthenticationStrategy; -import org.apache.linkis.httpclient.dws.config.DWSClientConfig; -import org.apache.linkis.httpclient.dws.config.DWSClientConfigBuilder; -import java.util.List; -import java.util.concurrent.TimeUnit; public class RemoteClientTest { - public static void main(String[] args) { - LinkisMetadataSourceRemoteClient client = getMetadataSourceRemoteClient(); - GetMetadataSourceAllDatabasesAction action = GetMetadataSourceAllDatabasesAction.builder().setUser("hdfs").build(); - GetMetadataSourceAllDatabasesResult result = client.getAllDBMetaDataSource(action); - List dbs = result.getDbs(); - System.out.println(dbs); - } - private static LinkisMetadataSourceRemoteClient getMetadataSourceRemoteClient(){ - DWSClientConfig clientConfig = ((DWSClientConfigBuilder) (DWSClientConfigBuilder.newBuilder().addServerUrl("http://gateway:8088") - .connectionTimeout(30000).discoveryEnabled(true) - .discoveryFrequency(1, TimeUnit.MINUTES) - .loadbalancerEnabled(true).maxConnectionSize(5) - .retryEnabled(false).readTimeout(30000) - .setAuthenticationStrategy(new StaticAuthenticationStrategy()).setAuthTokenKey("hdfs") - .setAuthTokenValue("hdfs"))).setDWSVersion("v1").build(); - return new LinkisMetadataSourceRemoteClient(clientConfig); - } } diff --git a/dss-apps/dss-datawarehouse-design/data-warehouse-service/src/test/scala/com/webank/wedatasphere/warehouse/DwLayerRemoteClientTest.scala b/dss-apps/dss-datawarehouse-design/data-warehouse-service/src/test/scala/com/webank/wedatasphere/warehouse/DwLayerRemoteClientTest.scala index 77c6e79307..32a8983a6d 100644 --- a/dss-apps/dss-datawarehouse-design/data-warehouse-service/src/test/scala/com/webank/wedatasphere/warehouse/DwLayerRemoteClientTest.scala +++ b/dss-apps/dss-datawarehouse-design/data-warehouse-service/src/test/scala/com/webank/wedatasphere/warehouse/DwLayerRemoteClientTest.scala @@ -1,7 +1,5 @@ package com.webank.wedatasphere.warehouse -import org.apache.linkis.datasource.client.impl.{LinkisMetaDataRemoteClient, LinkisMetadataSourceRemoteClient} -import org.apache.linkis.datasource.client.request.{GetMetadataSourceAllDatabasesAction, MetadataGetDatabasesAction} import org.apache.linkis.httpclient.dws.authentication.StaticAuthenticationStrategy import org.apache.linkis.httpclient.dws.config.{DWSClientConfig, DWSClientConfigBuilder} @@ -25,11 +23,7 @@ object DwLayerRemoteClientTest { .setDWSVersion("v1") .build(); - val client = new LinkisMetadataSourceRemoteClient(clientCfg) - val action = GetMetadataSourceAllDatabasesAction.builder().setUser("hdfs").build() - val result = client.getAllDBMetaDataSource(action) - println(result.dbs) // val client = new LinkisMetaDataRemoteClient(clientCfg) // val action = new ListDwLayerAction