diff --git a/dss-apps/dss-dataasset-management/data-assets-client/pom.xml b/dss-apps/dss-dataasset-management/data-assets-client/pom.xml new file mode 100644 index 0000000000..9025f3e8c5 --- /dev/null +++ b/dss-apps/dss-dataasset-management/data-assets-client/pom.xml @@ -0,0 +1,58 @@ + + + + dss-dataasset-management + com.webank.wedatasphere.dss + 1.0.1 + + 4.0.0 + + data-assets-client + + + 8 + 8 + + + + + org.apache.linkis + linkis-gateway-httpclient-support + ${linkis.version} + + + org.projectlombok + lombok + 1.18.16 + compile + + + + + + + + org.apache.maven.plugins + maven-deploy-plugin + + + + net.alchim31.maven + scala-maven-plugin + + + org.apache.maven.plugins + maven-jar-plugin + + + + + src/main/resources + + + ${project.artifactId}-${project.version} + + + \ No newline at end of file diff --git a/dss-apps/dss-dataasset-management/data-assets-client/src/main/java/com/webank/wedatasphere/dss/data/governance/entity/ClassificationConstant.java b/dss-apps/dss-dataasset-management/data-assets-client/src/main/java/com/webank/wedatasphere/dss/data/governance/entity/ClassificationConstant.java new file mode 100644 index 0000000000..c00620f1b3 --- /dev/null +++ b/dss-apps/dss-dataasset-management/data-assets-client/src/main/java/com/webank/wedatasphere/dss/data/governance/entity/ClassificationConstant.java @@ -0,0 +1,77 @@ +package com.webank.wedatasphere.dss.data.governance.entity; + + +import org.apache.commons.lang.StringUtils; + +import java.util.Optional; + +public enum ClassificationConstant { + /** + * 指标 + */ + INDICATOR(1, "indicator"), + /** + * 度量 + */ + MEASURE(2,"measure"), + /** + * 维度 + */ + DIMENSION(0, "dimension"), + /** + * 主题 + */ + THEME(3, "theme"), + /** + * 分层 + */ + LAYER(4, "layer"); + + private int type; + + public static final String SEPARATOR = "_"; + + private String typeCode; + + ClassificationConstant(int type, String typeCode) { + this.type = type; + + this.typeCode = typeCode; + } + + public static boolean isTypeScope(int type) { + return type >= 0 && type < values().length; + } + + public static boolean isTypeScope(String type) { + return getClassificationConstantByTypeCode(type).isPresent(); + } + + public static Optional getClassificationConstantByTypeCode(String typeCode) { + for (ClassificationConstant c : values()) { + if (StringUtils.equals(c.typeCode,typeCode)) { + return Optional.of(c); + } + } + return Optional.empty(); + } + + public static Optional getClassificationConstantByType(int type) { + for (ClassificationConstant c : values()) { + if (c.type == type) { + return Optional.of(c); + } + } + return Optional.empty(); + } + + + + public int getType() { + return type; + } + + public String getTypeCode() { + return typeCode; + } +} diff --git a/dss-apps/dss-dataasset-management/data-assets-client/src/main/java/com/webank/wedatasphere/dss/data/governance/entity/CreateLabelInfo.java b/dss-apps/dss-dataasset-management/data-assets-client/src/main/java/com/webank/wedatasphere/dss/data/governance/entity/CreateLabelInfo.java new file mode 100644 index 0000000000..9bf1a6139d --- /dev/null +++ b/dss-apps/dss-dataasset-management/data-assets-client/src/main/java/com/webank/wedatasphere/dss/data/governance/entity/CreateLabelInfo.java @@ -0,0 +1,10 @@ +package com.webank.wedatasphere.dss.data.governance.entity; + +import lombok.Data; + +@Data +public class CreateLabelInfo { + private String name; + + private String guid; +} diff --git a/dss-apps/dss-dataasset-management/data-assets-client/src/main/java/com/webank/wedatasphere/dss/data/governance/entity/CreateModelTypeInfo.java b/dss-apps/dss-dataasset-management/data-assets-client/src/main/java/com/webank/wedatasphere/dss/data/governance/entity/CreateModelTypeInfo.java new file mode 100644 index 0000000000..cf1fa62bb4 --- /dev/null +++ b/dss-apps/dss-dataasset-management/data-assets-client/src/main/java/com/webank/wedatasphere/dss/data/governance/entity/CreateModelTypeInfo.java @@ -0,0 +1,12 @@ +package com.webank.wedatasphere.dss.data.governance.entity; + + +import lombok.Data; + +@Data +public class CreateModelTypeInfo { + private String name; + + private String guid; + +} diff --git a/dss-apps/dss-dataasset-management/data-assets-client/src/main/java/com/webank/wedatasphere/dss/data/governance/entity/HiveSimpleInfo.java b/dss-apps/dss-dataasset-management/data-assets-client/src/main/java/com/webank/wedatasphere/dss/data/governance/entity/HiveSimpleInfo.java new file mode 100644 index 0000000000..2aeaedfbbb --- /dev/null +++ b/dss-apps/dss-dataasset-management/data-assets-client/src/main/java/com/webank/wedatasphere/dss/data/governance/entity/HiveSimpleInfo.java @@ -0,0 +1,23 @@ +package com.webank.wedatasphere.dss.data.governance.entity; + +import lombok.Data; + +import java.io.Serializable; +import java.util.List; + + +@Data +public class HiveSimpleInfo implements Serializable { + private String guid; + private String name; + private String qualifiedName; + private String createTime; + private String owner; + private String aliases; + private String lastAccessTime; + private String comment; + private List classifications; + private String totalSize; + private String external; + +} diff --git a/dss-apps/dss-dataasset-management/data-assets-client/src/main/java/com/webank/wedatasphere/dss/data/governance/entity/HiveTblStatsDTO.java b/dss-apps/dss-dataasset-management/data-assets-client/src/main/java/com/webank/wedatasphere/dss/data/governance/entity/HiveTblStatsDTO.java new file mode 100644 index 0000000000..82f916c2ef --- /dev/null +++ b/dss-apps/dss-dataasset-management/data-assets-client/src/main/java/com/webank/wedatasphere/dss/data/governance/entity/HiveTblStatsDTO.java @@ -0,0 +1,18 @@ +package com.webank.wedatasphere.dss.data.governance.entity; + + +import lombok.Data; + +@Data +public class HiveTblStatsDTO { + + private Integer columnCount = 0; + + private Long totalSize = 0L; + + private Integer numFiles = 0; + + private Integer partitionCount = 0; + + private Integer accessCount = 0; +} diff --git a/dss-apps/dss-dataasset-management/data-assets-client/src/main/java/com/webank/wedatasphere/dss/data/governance/entity/PartInfo.java b/dss-apps/dss-dataasset-management/data-assets-client/src/main/java/com/webank/wedatasphere/dss/data/governance/entity/PartInfo.java new file mode 100644 index 0000000000..1a5d85685a --- /dev/null +++ b/dss-apps/dss-dataasset-management/data-assets-client/src/main/java/com/webank/wedatasphere/dss/data/governance/entity/PartInfo.java @@ -0,0 +1,14 @@ +package com.webank.wedatasphere.dss.data.governance.entity; + +import lombok.Data; + + +@Data +public class PartInfo { + private String partName; + private int reordCnt; + private int store; + private String createTime; + private String lastAccessTime; + private int fileCount; +} \ No newline at end of file diff --git a/dss-apps/dss-dataasset-management/data-assets-client/src/main/java/com/webank/wedatasphere/dss/data/governance/entity/QueryType.java b/dss-apps/dss-dataasset-management/data-assets-client/src/main/java/com/webank/wedatasphere/dss/data/governance/entity/QueryType.java new file mode 100644 index 0000000000..f54a0e6cd5 --- /dev/null +++ b/dss-apps/dss-dataasset-management/data-assets-client/src/main/java/com/webank/wedatasphere/dss/data/governance/entity/QueryType.java @@ -0,0 +1,18 @@ +package com.webank.wedatasphere.dss.data.governance.entity; + + +public enum QueryType { + PRECISE(1), + FUZZY(0); + private int code; + + QueryType(int code) { + this.code = code; + } + + public int getCode() { + return code; + } + + +} diff --git a/dss-apps/dss-dataasset-management/data-assets-client/src/main/java/com/webank/wedatasphere/dss/data/governance/entity/SearchLabelInfo.java b/dss-apps/dss-dataasset-management/data-assets-client/src/main/java/com/webank/wedatasphere/dss/data/governance/entity/SearchLabelInfo.java new file mode 100644 index 0000000000..c33b928d19 --- /dev/null +++ b/dss-apps/dss-dataasset-management/data-assets-client/src/main/java/com/webank/wedatasphere/dss/data/governance/entity/SearchLabelInfo.java @@ -0,0 +1,9 @@ +package com.webank.wedatasphere.dss.data.governance.entity; + +import lombok.Data; + +@Data +public class SearchLabelInfo { + private String name; + private String guid; +} diff --git a/dss-apps/dss-dataasset-management/data-assets-client/src/main/java/com/webank/wedatasphere/dss/data/governance/entity/UpdateLabelInfo.java b/dss-apps/dss-dataasset-management/data-assets-client/src/main/java/com/webank/wedatasphere/dss/data/governance/entity/UpdateLabelInfo.java new file mode 100644 index 0000000000..c04b5c79c4 --- /dev/null +++ b/dss-apps/dss-dataasset-management/data-assets-client/src/main/java/com/webank/wedatasphere/dss/data/governance/entity/UpdateLabelInfo.java @@ -0,0 +1,10 @@ +package com.webank.wedatasphere.dss.data.governance.entity; + +import lombok.Data; + +@Data +public class UpdateLabelInfo { + private String name; + + private String guid; +} diff --git a/dss-apps/dss-dataasset-management/data-assets-client/src/main/java/com/webank/wedatasphere/dss/data/governance/entity/UpdateModelTypeInfo.java b/dss-apps/dss-dataasset-management/data-assets-client/src/main/java/com/webank/wedatasphere/dss/data/governance/entity/UpdateModelTypeInfo.java new file mode 100644 index 0000000000..33cee5d07b --- /dev/null +++ b/dss-apps/dss-dataasset-management/data-assets-client/src/main/java/com/webank/wedatasphere/dss/data/governance/entity/UpdateModelTypeInfo.java @@ -0,0 +1,11 @@ +package com.webank.wedatasphere.dss.data.governance.entity; + + +import lombok.Data; + +@Data +public class UpdateModelTypeInfo { + private String name; + + private String guid; +} diff --git a/dss-apps/dss-dataasset-management/data-assets-client/src/main/scala/com/webank/wedatasphere/dss/data/governance/AbstractRemoteClient.scala b/dss-apps/dss-dataasset-management/data-assets-client/src/main/scala/com/webank/wedatasphere/dss/data/governance/AbstractRemoteClient.scala new file mode 100644 index 0000000000..87b84d4a21 --- /dev/null +++ b/dss-apps/dss-dataasset-management/data-assets-client/src/main/scala/com/webank/wedatasphere/dss/data/governance/AbstractRemoteClient.scala @@ -0,0 +1,15 @@ +package com.webank.wedatasphere.dss.data.governance + +import org.apache.linkis.httpclient.dws.DWSHttpClient +import org.apache.linkis.httpclient.request.Action +import org.apache.linkis.httpclient.response.Result + +abstract class AbstractRemoteClient extends RemoteClient { + protected val dwsHttpClient:DWSHttpClient + + override def execute(action: Action): Result = action match { + case action: Action => dwsHttpClient.execute(action) + } + + override def close(): Unit = dwsHttpClient.close() +} diff --git a/dss-apps/dss-dataasset-management/data-assets-client/src/main/scala/com/webank/wedatasphere/dss/data/governance/DataAssetsRemoteClient.scala b/dss-apps/dss-dataasset-management/data-assets-client/src/main/scala/com/webank/wedatasphere/dss/data/governance/DataAssetsRemoteClient.scala new file mode 100644 index 0000000000..17144448bf --- /dev/null +++ b/dss-apps/dss-dataasset-management/data-assets-client/src/main/scala/com/webank/wedatasphere/dss/data/governance/DataAssetsRemoteClient.scala @@ -0,0 +1,47 @@ +package com.webank.wedatasphere.dss.data.governance + + +import com.webank.wedatasphere.dss.data.governance.request._ +import com.webank.wedatasphere.dss.data.governance.response._ + + +trait DataAssetsRemoteClient extends RemoteClient { + def searchHiveTbl(action: SearchHiveTblAction): SearchHiveTblResult + + def searchHiveDb(action: SearchHiveDbAction): SearchHiveDbResult + + def getHiveTblPartition(action: GetHiveTblPartitionAction): GetHiveTblPartitionResult + + def getHiveTblBasic(action: GetHiveTblBasicAction): GetHiveTblBasicResult + + def getHiveTblCreate(action: GetHiveTblCreateAction): GetHiveTblCreateResult + + def createModelType(action: CreateModelTypeAction): CreateModelTypeResult + + def bindModelType(action: BindModelTypeAction): BindModelTypeResult + + def updateModelType(action: UpdateModelTypeAction): UpdateModelTypeResult + + def unBindModelType(action: UnBindModelTypeAction): UnBindModelTypeResult + + def createLabel(action: CreateLabelAction): CreateLabelResult + + def updateLabel(action: UpdateLabelAction): UpdateLabelResult + + def deleteLabel(action: DeleteLabelAction): DeleteLabelResult + + def bindLabel(action: BindLabelAction): BindLabelResult + + def unBindLabel(action: UnBindLabelAction): UnBindLabelResult + + def searchLabel(action: SearchLabelAction): SearchLabelResult + + def deleteModelType(action: DeleteModelTypeAction): DeleteModelTypeResult + + def searchHiveTblSize(action: HiveTblSizeAction): HiveTblSizeResult + + def searchHiveTblStats(action: HiveTblStatsAction): HiveTblStatsResult + + def getHiveTblPartInfoByNameResult(action: GetTblPartInfoByNameAction): GetHiveTblPartInfoByNameResult + +} diff --git a/dss-apps/dss-dataasset-management/data-assets-client/src/main/scala/com/webank/wedatasphere/dss/data/governance/RemoteClient.scala b/dss-apps/dss-dataasset-management/data-assets-client/src/main/scala/com/webank/wedatasphere/dss/data/governance/RemoteClient.scala new file mode 100644 index 0000000000..9d1e5028d7 --- /dev/null +++ b/dss-apps/dss-dataasset-management/data-assets-client/src/main/scala/com/webank/wedatasphere/dss/data/governance/RemoteClient.scala @@ -0,0 +1,11 @@ +package com.webank.wedatasphere.dss.data.governance + +import org.apache.linkis.httpclient.request.Action +import org.apache.linkis.httpclient.response.Result +import java.io.Closeable + +trait RemoteClient extends Closeable{ + protected def execute(action: Action): Result + + override def close(): Unit +} diff --git a/dss-apps/dss-dataasset-management/data-assets-client/src/main/scala/com/webank/wedatasphere/dss/data/governance/exception/DataAssetsClientBuilderException.scala b/dss-apps/dss-dataasset-management/data-assets-client/src/main/scala/com/webank/wedatasphere/dss/data/governance/exception/DataAssetsClientBuilderException.scala new file mode 100644 index 0000000000..5c9b205855 --- /dev/null +++ b/dss-apps/dss-dataasset-management/data-assets-client/src/main/scala/com/webank/wedatasphere/dss/data/governance/exception/DataAssetsClientBuilderException.scala @@ -0,0 +1,5 @@ +package com.webank.wedatasphere.dss.data.governance.exception + +import org.apache.linkis.common.exception.ErrorException + +class DataAssetsClientBuilderException (errorDesc: String) extends ErrorException(23000, errorDesc) diff --git a/dss-apps/dss-dataasset-management/data-assets-client/src/main/scala/com/webank/wedatasphere/dss/data/governance/impl/LinkisDataAssetsRemoteClient.scala b/dss-apps/dss-dataasset-management/data-assets-client/src/main/scala/com/webank/wedatasphere/dss/data/governance/impl/LinkisDataAssetsRemoteClient.scala new file mode 100644 index 0000000000..136216f2cb --- /dev/null +++ b/dss-apps/dss-dataasset-management/data-assets-client/src/main/scala/com/webank/wedatasphere/dss/data/governance/impl/LinkisDataAssetsRemoteClient.scala @@ -0,0 +1,51 @@ +package com.webank.wedatasphere.dss.data.governance.impl + + +import com.webank.wedatasphere.dss.data.governance.request._ +import com.webank.wedatasphere.dss.data.governance.response._ +import com.webank.wedatasphere.dss.data.governance.{AbstractRemoteClient, DataAssetsRemoteClient} +import org.apache.linkis.httpclient.dws.DWSHttpClient +import org.apache.linkis.httpclient.dws.config.DWSClientConfig + +class LinkisDataAssetsRemoteClient(clientConfig: DWSClientConfig) extends AbstractRemoteClient with DataAssetsRemoteClient { + override protected val dwsHttpClient: DWSHttpClient = new DWSHttpClient(clientConfig, "DataAssets-Client") + + override def searchHiveTbl(action: SearchHiveTblAction): SearchHiveTblResult = execute(action).asInstanceOf[SearchHiveTblResult] + + override def getHiveTblPartition(action: GetHiveTblPartitionAction): GetHiveTblPartitionResult = execute(action).asInstanceOf[GetHiveTblPartitionResult] + + override def getHiveTblBasic(action: GetHiveTblBasicAction): GetHiveTblBasicResult = execute(action).asInstanceOf[GetHiveTblBasicResult] + + override def getHiveTblCreate(action: GetHiveTblCreateAction): GetHiveTblCreateResult = execute(action).asInstanceOf[GetHiveTblCreateResult] + + override def searchHiveDb(action: SearchHiveDbAction): SearchHiveDbResult = execute(action).asInstanceOf[SearchHiveDbResult] + + override def createModelType(action: CreateModelTypeAction): CreateModelTypeResult = execute(action).asInstanceOf[CreateModelTypeResult] + + override def bindModelType(action: BindModelTypeAction): BindModelTypeResult = execute(action).asInstanceOf[BindModelTypeResult] + + override def updateModelType(action: UpdateModelTypeAction): UpdateModelTypeResult = execute(action).asInstanceOf[UpdateModelTypeResult] + + override def unBindModelType(action: UnBindModelTypeAction): UnBindModelTypeResult = execute(action).asInstanceOf[UnBindModelTypeResult] + + override def deleteModelType(action: DeleteModelTypeAction): DeleteModelTypeResult = execute(action).asInstanceOf[DeleteModelTypeResult] + + override def searchHiveTblSize(action: HiveTblSizeAction): HiveTblSizeResult = execute(action).asInstanceOf[HiveTblSizeResult] + + override def searchHiveTblStats(action: HiveTblStatsAction): HiveTblStatsResult = execute(action).asInstanceOf[HiveTblStatsResult] + + override def createLabel(action: CreateLabelAction): CreateLabelResult = execute(action).asInstanceOf[CreateLabelResult] + + override def updateLabel(action: UpdateLabelAction): UpdateLabelResult = execute(action).asInstanceOf[UpdateLabelResult] + + override def deleteLabel(action: DeleteLabelAction): DeleteLabelResult = execute(action).asInstanceOf[DeleteLabelResult] + + override def bindLabel(action: BindLabelAction): BindLabelResult = execute(action).asInstanceOf[BindLabelResult] + + override def unBindLabel(action: UnBindLabelAction): UnBindLabelResult = execute(action).asInstanceOf[UnBindLabelResult] + + override def searchLabel(action: SearchLabelAction): SearchLabelResult = execute(action).asInstanceOf[SearchLabelResult] + + override def getHiveTblPartInfoByNameResult(action: GetTblPartInfoByNameAction): GetHiveTblPartInfoByNameResult = execute(action).asInstanceOf[GetHiveTblPartInfoByNameResult] + +} diff --git a/dss-apps/dss-dataasset-management/data-assets-client/src/main/scala/com/webank/wedatasphere/dss/data/governance/request/BindLabelAction.scala b/dss-apps/dss-dataasset-management/data-assets-client/src/main/scala/com/webank/wedatasphere/dss/data/governance/request/BindLabelAction.scala new file mode 100644 index 0000000000..c163338f7d --- /dev/null +++ b/dss-apps/dss-dataasset-management/data-assets-client/src/main/scala/com/webank/wedatasphere/dss/data/governance/request/BindLabelAction.scala @@ -0,0 +1,77 @@ +package com.webank.wedatasphere.dss.data.governance.request + +import com.webank.wedatasphere.dss.data.governance.exception.DataAssetsClientBuilderException +import org.apache.linkis.httpclient.dws.DWSHttpClient +import org.apache.linkis.httpclient.request.POSTAction + +class BindLabelAction extends POSTAction with DataAssetsAction { + + private var user: String = _ + + override def getRequestPayload: String = DWSHttpClient.jacksonJson.writeValueAsString(getRequestPayloads) + + override def suffixURLs: Array[String] = Array("data-assets", "asset", "labels", "bind") + + override def setUser(user: String): Unit = this.user = user + + override def getUser: String = this.user + +} + +object BindLabelAction { + def builder(): Builder = new Builder + + class Builder private[BindLabelAction] { + private var user: String = _ + + private var label: String = _ + + private var labelGuid: String = _ + + private var tableName: String = _ + + private var tableGuid: String = _ + + def setUser(user: String): Builder = { + this.user = user + this + } + + def setTableName(tableName: String): Builder = { + this.tableName = tableName; + this + } + + def setLabel(label : String) :Builder={ + this.label = label + this + } + + def setTableGuid(tableGuid : String) : Builder = { + this.tableGuid = tableGuid + this + } + + def setLabelGuid(labelGuid : String) : Builder = { + this.labelGuid = labelGuid + this + } + + def build(): BindLabelAction = { + var action = new BindLabelAction; + if ((tableName == null || label == null) && (labelGuid == null || tableGuid == null)) + throw new DataAssetsClientBuilderException("tableName and label ,or tableGuid and labelGuid are needed!") + + action.setUser(user) + action.addRequestPayload("label", label) + action.addRequestPayload("tableName", tableName) + action.addRequestPayload("tableGuid", tableGuid) + action.addRequestPayload("labelGuid", labelGuid) + action + } + } +} + + + + diff --git a/dss-apps/dss-dataasset-management/data-assets-client/src/main/scala/com/webank/wedatasphere/dss/data/governance/request/BindModelTypeAction.scala b/dss-apps/dss-dataasset-management/data-assets-client/src/main/scala/com/webank/wedatasphere/dss/data/governance/request/BindModelTypeAction.scala new file mode 100644 index 0000000000..f99cab178b --- /dev/null +++ b/dss-apps/dss-dataasset-management/data-assets-client/src/main/scala/com/webank/wedatasphere/dss/data/governance/request/BindModelTypeAction.scala @@ -0,0 +1,84 @@ +package com.webank.wedatasphere.dss.data.governance.request + +import com.webank.wedatasphere.dss.data.governance.entity.ClassificationConstant +import com.webank.wedatasphere.dss.data.governance.exception.DataAssetsClientBuilderException +import org.apache.linkis.httpclient.dws.DWSHttpClient +import org.apache.linkis.httpclient.request.POSTAction + +class BindModelTypeAction extends POSTAction with DataAssetsAction { + + private var user: String = _ + + private var modelName: String = _ + + private var modelType: String = _ + + private var tableName: String = _ + + private var guid: String = _ + + override def getRequestPayload: String = DWSHttpClient.jacksonJson.writeValueAsString(getRequestPayloads) + + override def setUser(user: String): Unit = this.user = user + + override def getUser: String = this.user + + override def suffixURLs: Array[String] = Array("data-assets", "asset", "model", "bind") +} + +object BindModelTypeAction { + def builder(): Builder = new Builder + + class Builder private[BindModelTypeAction] { + private var user: String = _ + + private var modelName: String = _ + + private var modelType: ClassificationConstant = _ + + private var tableName: String = _ + + private var guid: String = _ + + def setUser(user: String): Builder = { + this.user = user + this + } + + def setModelName(modelName: String): Builder = { + this.modelName = modelName; + this + } + + def setModelType(modelType: ClassificationConstant): Builder = { + this.modelType = modelType; + this + } + + def setTableName(tableName: String): Builder = { + this.tableName = tableName; + this + } + + def setGuid(guid: String): Builder = { + this.guid = guid; + this + } + + def build(): BindModelTypeAction = { + var action = new BindModelTypeAction; + if (tableName == null && guid == null) throw new DataAssetsClientBuilderException("tableName or guid is needed!") + if (modelType == null || modelName == null) throw new DataAssetsClientBuilderException("modelType and modelName is needed!") + action.setUser(user) + action.guid = guid + action.modelName = modelName + action.modelType = modelType.getTypeCode + action.tableName = tableName + action.addRequestPayload("guid", action.guid) + action.addRequestPayload("modelName", action.modelName) + action.addRequestPayload("modelType", action.modelType) + action.addRequestPayload("tableName", action.tableName) + action + } + } +} diff --git a/dss-apps/dss-dataasset-management/data-assets-client/src/main/scala/com/webank/wedatasphere/dss/data/governance/request/CreateLabelAction.scala b/dss-apps/dss-dataasset-management/data-assets-client/src/main/scala/com/webank/wedatasphere/dss/data/governance/request/CreateLabelAction.scala new file mode 100644 index 0000000000..5c10666625 --- /dev/null +++ b/dss-apps/dss-dataasset-management/data-assets-client/src/main/scala/com/webank/wedatasphere/dss/data/governance/request/CreateLabelAction.scala @@ -0,0 +1,50 @@ +package com.webank.wedatasphere.dss.data.governance.request + +import com.webank.wedatasphere.dss.data.governance.exception.DataAssetsClientBuilderException +import org.apache.linkis.httpclient.dws.DWSHttpClient +import org.apache.linkis.httpclient.request.POSTAction + +class CreateLabelAction extends POSTAction with DataAssetsAction { + + private var user: String = _ + + override def getRequestPayload: String = DWSHttpClient.jacksonJson.writeValueAsString(getRequestPayloads) + + override def suffixURLs: Array[String] = Array("data-assets", "asset", "labels") + + override def setUser(user: String): Unit = this.user = user + + override def getUser: String = this.user + +} + +object CreateLabelAction { + def builder(): Builder = new Builder + + class Builder private[CreateLabelAction]() { + private var user: String = _ + private var name: String = _ + + def setUser(user: String): Builder = { + this.user = user + this + } + + def setName(name: String): Builder = { + this.name = name + this + } + + + def build(): CreateLabelAction = { + val action = new CreateLabelAction + if (name == null) throw new DataAssetsClientBuilderException("name is needed!") + action.setUser(user) + action.addRequestPayload("name", this.name) + action + + } + } +} + + diff --git a/dss-apps/dss-dataasset-management/data-assets-client/src/main/scala/com/webank/wedatasphere/dss/data/governance/request/CreateModelTypeAction.scala b/dss-apps/dss-dataasset-management/data-assets-client/src/main/scala/com/webank/wedatasphere/dss/data/governance/request/CreateModelTypeAction.scala new file mode 100644 index 0000000000..4ac84dedd2 --- /dev/null +++ b/dss-apps/dss-dataasset-management/data-assets-client/src/main/scala/com/webank/wedatasphere/dss/data/governance/request/CreateModelTypeAction.scala @@ -0,0 +1,61 @@ +package com.webank.wedatasphere.dss.data.governance.request + +import com.webank.wedatasphere.dss.data.governance.entity.ClassificationConstant +import com.webank.wedatasphere.dss.data.governance.exception.DataAssetsClientBuilderException +import org.apache.linkis.httpclient.dws.DWSHttpClient +import org.apache.linkis.httpclient.request.POSTAction + +class CreateModelTypeAction extends POSTAction with DataAssetsAction { + + private var name:String= _ + + private var `type`:String= _ + + private var user:String = _ + + override def getRequestPayload: String = DWSHttpClient.jacksonJson.writeValueAsString(getRequestPayloads) + + override def setUser(user: String): Unit = this.user = user + + override def getUser: String = user + + override def suffixURLs: Array[String] = Array("data-assets", "asset", "model","type") +} + +object CreateModelTypeAction{ + def builder(): Builder = new Builder + + class Builder private[CreateModelTypeAction]() { + private var user: String = _ + private var name:String= _ + private var `type`:ClassificationConstant= _ + + def setUser(user: String): Builder ={ + this.user = user + this + } + + def setName(name:String):Builder = { + this.name = name + this + } + + def setType(`type`:ClassificationConstant):Builder = { + this.`type` = `type`; + this; + } + + def build(): CreateModelTypeAction = { + val action = new CreateModelTypeAction + if(`type` == null) throw new DataAssetsClientBuilderException("type is needed!") + if (name == null) throw new DataAssetsClientBuilderException("name is needed!") + action.name = name + action.`type` = `type`.getTypeCode + action.setUser(user) + action.addRequestPayload("name",action.name) + action.addRequestPayload("type",action.`type`) + action + + } + } +} diff --git a/dss-apps/dss-dataasset-management/data-assets-client/src/main/scala/com/webank/wedatasphere/dss/data/governance/request/DataAssetsAction.scala b/dss-apps/dss-dataasset-management/data-assets-client/src/main/scala/com/webank/wedatasphere/dss/data/governance/request/DataAssetsAction.scala new file mode 100644 index 0000000000..2f4a5a90bb --- /dev/null +++ b/dss-apps/dss-dataasset-management/data-assets-client/src/main/scala/com/webank/wedatasphere/dss/data/governance/request/DataAssetsAction.scala @@ -0,0 +1,5 @@ +package com.webank.wedatasphere.dss.data.governance.request + +import org.apache.linkis.httpclient.dws.request.DWSHttpAction + +trait DataAssetsAction extends DWSHttpAction with org.apache.linkis.httpclient.request.UserAction diff --git a/dss-apps/dss-dataasset-management/data-assets-client/src/main/scala/com/webank/wedatasphere/dss/data/governance/request/DeleteLabelAction.scala b/dss-apps/dss-dataasset-management/data-assets-client/src/main/scala/com/webank/wedatasphere/dss/data/governance/request/DeleteLabelAction.scala new file mode 100644 index 0000000000..d70c551e1d --- /dev/null +++ b/dss-apps/dss-dataasset-management/data-assets-client/src/main/scala/com/webank/wedatasphere/dss/data/governance/request/DeleteLabelAction.scala @@ -0,0 +1,49 @@ +package com.webank.wedatasphere.dss.data.governance.request + +import com.webank.wedatasphere.dss.data.governance.exception.DataAssetsClientBuilderException +import org.apache.linkis.httpclient.dws.DWSHttpClient +import org.apache.linkis.httpclient.request.POSTAction + +class DeleteLabelAction extends POSTAction with DataAssetsAction { + + private var user: String = _ + + override def getRequestPayload: String = DWSHttpClient.jacksonJson.writeValueAsString(getRequestPayloads) + + override def suffixURLs: Array[String] = Array("data-assets", "asset", "labels","delete") + + override def setUser(user: String): Unit = this.user = user + + override def getUser: String = this.user + +} +object DeleteLabelAction { + def builder(): Builder = new Builder + + class Builder private[DeleteLabelAction]() { + private var user: String = _ + private var name: String = _ + + def setUser(user: String): Builder = { + this.user = user + this + } + + def setName(name: String): Builder = { + this.name = name + this + } + + + def build(): DeleteLabelAction = { + val action = new DeleteLabelAction + if (name == null) throw new DataAssetsClientBuilderException("name is needed!") + action.setUser(user) + action.addRequestPayload("name", this.name) + action + } + } +} + + + diff --git a/dss-apps/dss-dataasset-management/data-assets-client/src/main/scala/com/webank/wedatasphere/dss/data/governance/request/DeleteModelTypeAction.scala b/dss-apps/dss-dataasset-management/data-assets-client/src/main/scala/com/webank/wedatasphere/dss/data/governance/request/DeleteModelTypeAction.scala new file mode 100644 index 0000000000..4d4254385f --- /dev/null +++ b/dss-apps/dss-dataasset-management/data-assets-client/src/main/scala/com/webank/wedatasphere/dss/data/governance/request/DeleteModelTypeAction.scala @@ -0,0 +1,62 @@ +package com.webank.wedatasphere.dss.data.governance.request + +import com.webank.wedatasphere.dss.data.governance.entity.ClassificationConstant +import com.webank.wedatasphere.dss.data.governance.exception.DataAssetsClientBuilderException +import org.apache.linkis.httpclient.dws.DWSHttpClient +import org.apache.linkis.httpclient.request.POSTAction + +class DeleteModelTypeAction extends POSTAction with DataAssetsAction{ + + + private var name:String= _ + + private var `type`:String= _ + + private var user:String = _ + + override def getRequestPayload: String = DWSHttpClient.jacksonJson.writeValueAsString(getRequestPayloads) + + override def setUser(user: String): Unit = this.user = user + + override def getUser: String = user + + override def suffixURLs: Array[String] = Array("data-assets", "asset", "model","type","delete") +} + +object DeleteModelTypeAction{ + def builder(): Builder = new Builder + + class Builder private[DeleteModelTypeAction]() { + private var user: String = _ + private var name:String= _ + private var `type`:ClassificationConstant= _ + + def setUser(user: String): Builder ={ + this.user = user + this + } + + def setName(name:String):Builder = { + this.name = name + this + } + + def setType(`type`:ClassificationConstant):Builder = { + this.`type` = `type`; + this; + } + + def build(): DeleteModelTypeAction = { + val action = new DeleteModelTypeAction + if(`type` == null) throw new DataAssetsClientBuilderException("type is needed!") + if (name == null) throw new DataAssetsClientBuilderException("name is needed!") + action.name = name + action.`type` = `type`.getTypeCode + action.setUser(user) + action.addRequestPayload("name",action.name) + action.addRequestPayload("type",action.`type`) + action + + } + } +} diff --git a/dss-apps/dss-dataasset-management/data-assets-client/src/main/scala/com/webank/wedatasphere/dss/data/governance/request/GetHiveTblBasicAction.scala b/dss-apps/dss-dataasset-management/data-assets-client/src/main/scala/com/webank/wedatasphere/dss/data/governance/request/GetHiveTblBasicAction.scala new file mode 100644 index 0000000000..994cf0981a --- /dev/null +++ b/dss-apps/dss-dataasset-management/data-assets-client/src/main/scala/com/webank/wedatasphere/dss/data/governance/request/GetHiveTblBasicAction.scala @@ -0,0 +1,42 @@ +package com.webank.wedatasphere.dss.data.governance.request + +import com.webank.wedatasphere.dss.data.governance.exception.DataAssetsClientBuilderException +import org.apache.linkis.httpclient.request.GetAction + +class GetHiveTblBasicAction extends GetAction with DataAssetsAction{ + + private var guid:String=_ + + private var user:String = _ + + override def setUser(user: String): Unit = this.user = user + + override def getUser: String = this.user + + override def suffixURLs: Array[String] = Array("data-assets", "asset", "hiveTbl",guid,"basic") +} +object GetHiveTblBasicAction{ + def builder(): Builder = new Builder + + class Builder private[GetHiveTblBasicAction]() { + private var guid:String = _ + private var user: String = _ + + def setUser(user: String): Builder ={ + this.user = user + this + } + def setGuid(guid:String): Builder = { + this.guid = guid + this + } + + def build(): GetHiveTblBasicAction = { + val action = new GetHiveTblBasicAction + if(guid == null) throw new DataAssetsClientBuilderException("guid is needed!") + action.guid = guid + action.setUser(user) + action + } + } +} diff --git a/dss-apps/dss-dataasset-management/data-assets-client/src/main/scala/com/webank/wedatasphere/dss/data/governance/request/GetHiveTblCreateAction.scala b/dss-apps/dss-dataasset-management/data-assets-client/src/main/scala/com/webank/wedatasphere/dss/data/governance/request/GetHiveTblCreateAction.scala new file mode 100644 index 0000000000..b3003e13f3 --- /dev/null +++ b/dss-apps/dss-dataasset-management/data-assets-client/src/main/scala/com/webank/wedatasphere/dss/data/governance/request/GetHiveTblCreateAction.scala @@ -0,0 +1,41 @@ +package com.webank.wedatasphere.dss.data.governance.request + +import com.webank.wedatasphere.dss.data.governance.exception.DataAssetsClientBuilderException +import org.apache.linkis.httpclient.request.GetAction + +class GetHiveTblCreateAction extends GetAction with DataAssetsAction{ + private var guid:String=_ + + private var user:String = _ + + override def setUser(user: String): Unit = this.user = user + + override def getUser: String = this.user + + override def suffixURLs: Array[String] = Array("data-assets", "asset", "hiveTbl",guid,"create") +} +object GetHiveTblCreateAction{ + def builder(): Builder = new Builder + class Builder private[GetHiveTblCreateAction]() { + private var guid:String = _ + private var user: String = _ + + def setUser(user: String): Builder ={ + this.user = user + this + } + + def setGuid(guid:String): Builder = { + this.guid = guid + this + } + + def build(): GetHiveTblCreateAction = { + val action = new GetHiveTblCreateAction + if(guid == null) throw new DataAssetsClientBuilderException("guid is needed!") + action.guid = guid + action.setUser(user) + action + } + } +} diff --git a/dss-apps/dss-dataasset-management/data-assets-client/src/main/scala/com/webank/wedatasphere/dss/data/governance/request/GetHiveTblPartitionAction.scala b/dss-apps/dss-dataasset-management/data-assets-client/src/main/scala/com/webank/wedatasphere/dss/data/governance/request/GetHiveTblPartitionAction.scala new file mode 100644 index 0000000000..9517512e88 --- /dev/null +++ b/dss-apps/dss-dataasset-management/data-assets-client/src/main/scala/com/webank/wedatasphere/dss/data/governance/request/GetHiveTblPartitionAction.scala @@ -0,0 +1,43 @@ +package com.webank.wedatasphere.dss.data.governance.request + +import com.webank.wedatasphere.dss.data.governance.exception.DataAssetsClientBuilderException +import org.apache.linkis.httpclient.request.GetAction + +class GetHiveTblPartitionAction extends GetAction with DataAssetsAction{ + + private var guid:String=_ + + private var user:String = _ + + override def setUser(user: String): Unit = this.user = user + + override def getUser: String = this.user + + override def suffixURLs: Array[String] = Array("data-assets", "asset", "hiveTbl",guid,"partition") +} +object GetHiveTblPartitionAction{ + def builder(): Builder = new Builder + + class Builder private[GetHiveTblPartitionAction]() { + private var guid:String = _ + private var user: String = _ + + def setUser(user: String): Builder ={ + this.user = user + this + } + + def setGuid(guid:String): Builder = { + this.guid = guid + this + } + + def build(): GetHiveTblPartitionAction = { + val action = new GetHiveTblPartitionAction + if(guid == null) throw new DataAssetsClientBuilderException("guid is needed!") + action.guid = guid + action.setUser(user) + action + } + } +} diff --git a/dss-apps/dss-dataasset-management/data-assets-client/src/main/scala/com/webank/wedatasphere/dss/data/governance/request/GetTblPartInfoByNameAction.scala b/dss-apps/dss-dataasset-management/data-assets-client/src/main/scala/com/webank/wedatasphere/dss/data/governance/request/GetTblPartInfoByNameAction.scala new file mode 100644 index 0000000000..b29b8b3160 --- /dev/null +++ b/dss-apps/dss-dataasset-management/data-assets-client/src/main/scala/com/webank/wedatasphere/dss/data/governance/request/GetTblPartInfoByNameAction.scala @@ -0,0 +1,53 @@ +package com.webank.wedatasphere.dss.data.governance.request + +import com.webank.wedatasphere.dss.data.governance.exception.DataAssetsClientBuilderException +import org.apache.linkis.httpclient.request.GetAction + +class GetTblPartInfoByNameAction extends GetAction with DataAssetsAction { + + + private var user: String = _ + + override def suffixURLs: Array[String] = Array("data-assets", "asset", "hiveTbl", "partition", "name") + + override def setUser(user: String): Unit = this.user = user + + override def getUser: String = this.user +} + +object GetTblPartInfoByNameAction { + def builder(): Builder = new Builder + + class Builder private[GetTblPartInfoByNameAction]() { + private var user: String = _ + private var dbName: String = _ + private var tableName: String = _ + + def setUser(user: String): Builder = { + this.user = user + this + } + + def setDbName(dbName: String): Builder = { + this.dbName = dbName + this + } + + def setTableName(tableName: String): Builder = { + this.tableName = tableName + this + } + + + def build(): GetTblPartInfoByNameAction = { + val action = new GetTblPartInfoByNameAction + if (tableName == null || dbName == null) + throw new DataAssetsClientBuilderException("tableName or dbName can not be null") + + action.setParameter("tableName", tableName) + action.setParameter("dbName", dbName) + action.setUser(user) + action + } + } +} diff --git a/dss-apps/dss-dataasset-management/data-assets-client/src/main/scala/com/webank/wedatasphere/dss/data/governance/request/HiveTblSizeAction.scala b/dss-apps/dss-dataasset-management/data-assets-client/src/main/scala/com/webank/wedatasphere/dss/data/governance/request/HiveTblSizeAction.scala new file mode 100644 index 0000000000..e8b0fb9ef5 --- /dev/null +++ b/dss-apps/dss-dataasset-management/data-assets-client/src/main/scala/com/webank/wedatasphere/dss/data/governance/request/HiveTblSizeAction.scala @@ -0,0 +1,62 @@ +package com.webank.wedatasphere.dss.data.governance.request + +import com.webank.wedatasphere.dss.data.governance.exception.DataAssetsClientBuilderException +import org.apache.linkis.httpclient.request.GetAction + +class HiveTblSizeAction extends GetAction with DataAssetsAction { + + + private var user: String = _ + + override def suffixURLs: Array[String] = Array("data-assets", "asset", "hiveTbl", "size") + + override def setUser(user: String): Unit = this.user = user + + override def getUser: String = this.user +} + +object HiveTblSizeAction { + def builder(): Builder = new Builder + + class Builder private[HiveTblSizeAction]() { + private var guid: String = _ + private var user: String = _ + private var dbName: String = _ + private var tableName: String = _ + + def setUser(user: String): Builder = { + this.user = user + this + } + + def setDbName(dbName: String): Builder = { + this.dbName = dbName + this + } + + def setTableName(tableName: String): Builder = { + this.tableName = tableName + this + } + + def setGuid(guid: String): Builder = { + this.guid = guid + this + } + + def build(): HiveTblSizeAction = { + val action = new HiveTblSizeAction + if (guid == null && (tableName == null || dbName == null)) + throw new DataAssetsClientBuilderException("That tableName or dbName being null and guid being null could not happen in the same time!") + if (guid != null) { + action.setParameter("guid", guid) + } + if (tableName != null && dbName != null) { + action.setParameter("tableName", tableName) + action.setParameter("dbName", dbName) + } + action.setUser(user) + action + } + } +} diff --git a/dss-apps/dss-dataasset-management/data-assets-client/src/main/scala/com/webank/wedatasphere/dss/data/governance/request/HiveTblStatsAction.scala b/dss-apps/dss-dataasset-management/data-assets-client/src/main/scala/com/webank/wedatasphere/dss/data/governance/request/HiveTblStatsAction.scala new file mode 100644 index 0000000000..cedbdc8473 --- /dev/null +++ b/dss-apps/dss-dataasset-management/data-assets-client/src/main/scala/com/webank/wedatasphere/dss/data/governance/request/HiveTblStatsAction.scala @@ -0,0 +1,68 @@ +package com.webank.wedatasphere.dss.data.governance.request + +import com.webank.wedatasphere.dss.data.governance.exception.DataAssetsClientBuilderException +import org.apache.linkis.httpclient.request.GetAction + +class HiveTblStatsAction extends GetAction with DataAssetsAction { + + private var dbName: String = _ + + private var tableName: String = _ + + private var guid: String = _ + + private var user: String = _ + + override def suffixURLs: Array[String] = Array("data-assets", "asset", "hiveTbl", "stats") + + override def setUser(user: String): Unit = this.user = user + + override def getUser: String = this.user +} + +object HiveTblStatsAction { + def builder(): Builder = new Builder + + class Builder private[HiveTblStatsAction]() { + private var guid: String = _ + private var user: String = _ + private var dbName: String = _ + private var tableName: String = _ + + def setUser(user: String): Builder = { + this.user = user + this + } + + def setDbName(dbName: String): Builder = { + this.dbName = dbName + this + } + + def setTableName(tableName: String): Builder = { + this.tableName = tableName + this + } + + def setGuid(guid: String): Builder = { + this.guid = guid + this + } + + def build(): HiveTblStatsAction = { + val action = new HiveTblStatsAction + if (guid == null && (tableName == null || dbName == null)) + throw new DataAssetsClientBuilderException("That tableName or dbName being null and guid being null could not happen in the same time!") + if (guid != null) { + action.setParameter("guid", guid) + } + if (tableName != null && dbName != null) { + action.setParameter("tableName", tableName) + action.setParameter("dbName", dbName) + } + action.setUser(user) + action + } + } +} + diff --git a/dss-apps/dss-dataasset-management/data-assets-client/src/main/scala/com/webank/wedatasphere/dss/data/governance/request/SearchHiveDbAction.scala b/dss-apps/dss-dataasset-management/data-assets-client/src/main/scala/com/webank/wedatasphere/dss/data/governance/request/SearchHiveDbAction.scala new file mode 100644 index 0000000000..b3e57864fc --- /dev/null +++ b/dss-apps/dss-dataasset-management/data-assets-client/src/main/scala/com/webank/wedatasphere/dss/data/governance/request/SearchHiveDbAction.scala @@ -0,0 +1,72 @@ +package com.webank.wedatasphere.dss.data.governance.request + +import org.apache.linkis.httpclient.request.GetAction + +class SearchHiveDbAction extends GetAction with DataAssetsAction{ + override def suffixURLs: Array[String] = Array("data-assets", "asset", "hiveDb","search") + + private var user:String = _ + + override def setUser(user: String): Unit = this.user = user + + override def getUser: String = this.user + +} +object SearchHiveDbAction{ + def builder(): Builder = new Builder + + class Builder private[SearchHiveDbAction]() { + private var classification:String=_ + private var query:String=_ + private var owner:String=_ + private var limit:Int=10 + private var offset:Int=0 + private var user: String = _ + + def setUser(user: String): Builder ={ + this.user = user + this + } + + def setClassification(classification:String): Builder ={ + this.classification = classification + this + } + + def setQuery(query:String): Builder ={ + this.query = query + this + } + + def setOwner(owner:String):Builder={ + this.owner=owner + this + } + + def setLimit(limit:Int):Builder={ + this.limit=limit + this + } + + def setOffset(offset:Int):Builder={ + this.offset=offset + this + } + + def build(): SearchHiveDbAction = { + val action = new SearchHiveDbAction + + if(classification != null) action.setParameter("classification",classification) + if(query!=null) action.setParameter("query",query) + if(owner == null) action.setParameter("owner","") + else action.setParameter("owner",owner) + + action.setParameter("limit",limit) + action.setParameter("offset",offset) + + action.setUser(user) + action + } + } +} + diff --git a/dss-apps/dss-dataasset-management/data-assets-client/src/main/scala/com/webank/wedatasphere/dss/data/governance/request/SearchHiveTblAction.scala b/dss-apps/dss-dataasset-management/data-assets-client/src/main/scala/com/webank/wedatasphere/dss/data/governance/request/SearchHiveTblAction.scala new file mode 100644 index 0000000000..1a96c1d44d --- /dev/null +++ b/dss-apps/dss-dataasset-management/data-assets-client/src/main/scala/com/webank/wedatasphere/dss/data/governance/request/SearchHiveTblAction.scala @@ -0,0 +1,79 @@ +package com.webank.wedatasphere.dss.data.governance.request + +import com.webank.wedatasphere.dss.data.governance.entity.QueryType +import org.apache.linkis.httpclient.request.GetAction + +class SearchHiveTblAction extends GetAction with DataAssetsAction{ + override def suffixURLs: Array[String] = Array("data-assets", "asset", "hiveTbl","search") + + private var user:String = _ + + override def setUser(user: String): Unit = this.user = user + + override def getUser: String = this.user + +} +object SearchHiveTblAction{ + def builder(): Builder = new Builder + + class Builder private[SearchHiveTblAction]() { + private var classification:String=_ + private var query:String=_ + private var owner:String=_ + private var limit:Int=10 + private var offset:Int=0 + private var precise: QueryType = QueryType.FUZZY + private var user: String = _ + + def setUser(user: String): Builder ={ + this.user = user + this + } + + def setClassification(classification:String): Builder ={ + this.classification = classification + this + } + + def setQuery(query:String): Builder ={ + this.query = query + this + } + + def setOwner(owner:String):Builder={ + this.owner=owner + this + } + + def setPrecise(precise : QueryType):Builder={ + this.precise = precise + this + } + + def setLimit(limit:Int):Builder={ + this.limit=limit + this + } + + def setOffset(offset:Int):Builder={ + this.offset=offset + this + } + + def build(): SearchHiveTblAction = { + val action = new SearchHiveTblAction + + if(classification != null) action.setParameter("classification",classification) + if(query!=null) action.setParameter("query",query) + if(owner == null) action.setParameter("owner","") + else action.setParameter("owner",owner) + + action.setParameter("limit",limit) + action.setParameter("offset",offset) + action.setParameter("precise",precise.getCode) + + action.setUser(user) + action + } + } +} diff --git a/dss-apps/dss-dataasset-management/data-assets-client/src/main/scala/com/webank/wedatasphere/dss/data/governance/request/SearchLabelAction.scala b/dss-apps/dss-dataasset-management/data-assets-client/src/main/scala/com/webank/wedatasphere/dss/data/governance/request/SearchLabelAction.scala new file mode 100644 index 0000000000..031dc1ce32 --- /dev/null +++ b/dss-apps/dss-dataasset-management/data-assets-client/src/main/scala/com/webank/wedatasphere/dss/data/governance/request/SearchLabelAction.scala @@ -0,0 +1,59 @@ +package com.webank.wedatasphere.dss.data.governance.request + +import org.apache.linkis.httpclient.request.GetAction + +class SearchLabelAction extends GetAction with DataAssetsAction{ + override def suffixURLs: Array[String] = Array("data-assets", "asset", "labels","search") + + private var user:String = _ + + override def setUser(user: String): Unit = this.user = user + + override def getUser: String = this.user + +} +object SearchLabelAction{ + def builder(): Builder = new Builder + + class Builder private[SearchLabelAction]() { + + private var query:String=_ + + private var limit:Int=10 + private var offset:Int=0 + private var user: String = _ + + def setUser(user: String): Builder ={ + this.user = user + this + } + + + def setQuery(query:String): Builder ={ + this.query = query + this + } + + + + def setLimit(limit:Int):Builder={ + this.limit=limit + this + } + + def setOffset(offset:Int):Builder={ + this.offset=offset + this + } + + def build(): SearchLabelAction = { + val action = new SearchLabelAction + if(query!=null) action.setParameter("query",query) + action.setParameter("limit",limit) + action.setParameter("offset",offset) + action.setUser(user) + action + } + } +} + diff --git a/dss-apps/dss-dataasset-management/data-assets-client/src/main/scala/com/webank/wedatasphere/dss/data/governance/request/UnBindLabelAction.scala b/dss-apps/dss-dataasset-management/data-assets-client/src/main/scala/com/webank/wedatasphere/dss/data/governance/request/UnBindLabelAction.scala new file mode 100644 index 0000000000..1118c4bba2 --- /dev/null +++ b/dss-apps/dss-dataasset-management/data-assets-client/src/main/scala/com/webank/wedatasphere/dss/data/governance/request/UnBindLabelAction.scala @@ -0,0 +1,86 @@ +package com.webank.wedatasphere.dss.data.governance.request + +import com.webank.wedatasphere.dss.data.governance.exception.DataAssetsClientBuilderException +import org.apache.linkis.httpclient.dws.DWSHttpClient +import org.apache.linkis.httpclient.request.POSTAction + +class UnBindLabelAction extends POSTAction with DataAssetsAction { + + private var user: String = _ + + override def getRequestPayload: String = DWSHttpClient.jacksonJson.writeValueAsString(getRequestPayloads) + + override def suffixURLs: Array[String] = Array("data-assets", "asset", "labels", "unbind") + + override def setUser(user: String): Unit = this.user = user + + override def getUser: String = this.user + +} + +object UnBindLabelAction { + def builder(): Builder = new Builder + + class Builder private[UnBindLabelAction] { + private var user: String = _ + + private var label: String = _ + + private var labelGuid: String = _ + + private var tableName: String = _ + + private var tableGuid: String = _ + + private var relationGuid: String = _ + + def setUser(user: String): Builder = { + this.user = user + this + } + + def setTableName(tableName: String): Builder = { + this.tableName = tableName; + this + } + + def setLabel(label : String) :Builder={ + this.label = label + this + } + + def setTableGuid(tableGuid : String) : Builder = { + this.tableGuid = tableGuid + this + } + + def setLabelGuid(labelGuid : String) : Builder = { + this.labelGuid = labelGuid + this + } + + def setRelationGuid(relationGuid : String) : Builder = { + this.relationGuid = relationGuid + this + } + + def build(): UnBindLabelAction = { + var action = new UnBindLabelAction; + if ((tableName == null || label == null) && (labelGuid == null || tableGuid == null || relationGuid == null)) throw new DataAssetsClientBuilderException("tableName and label ,or tableGuid and labelGuid and relationGuid are needed!") + + action.setUser(user) + action.addRequestPayload("label", label) + action.addRequestPayload("tableName", tableName) + action.addRequestPayload("tableGuid", tableGuid) + action.addRequestPayload("labelGuid", labelGuid) + action.addRequestPayload("relationGuid", relationGuid) + action + } + } +} + + + + + + diff --git a/dss-apps/dss-dataasset-management/data-assets-client/src/main/scala/com/webank/wedatasphere/dss/data/governance/request/UnBindModelTypeAction.scala b/dss-apps/dss-dataasset-management/data-assets-client/src/main/scala/com/webank/wedatasphere/dss/data/governance/request/UnBindModelTypeAction.scala new file mode 100644 index 0000000000..ab123b4c73 --- /dev/null +++ b/dss-apps/dss-dataasset-management/data-assets-client/src/main/scala/com/webank/wedatasphere/dss/data/governance/request/UnBindModelTypeAction.scala @@ -0,0 +1,87 @@ +package com.webank.wedatasphere.dss.data.governance.request + +import com.webank.wedatasphere.dss.data.governance.entity.ClassificationConstant +import com.webank.wedatasphere.dss.data.governance.exception.DataAssetsClientBuilderException +import org.apache.linkis.httpclient.dws.DWSHttpClient +import org.apache.linkis.httpclient.request.POSTAction + +class UnBindModelTypeAction extends POSTAction with DataAssetsAction { + + private var user: String = _ + + private var modelName: String = _ + + private var modelType: String = _ + + private var tableName: String = _ + + private var guid: String = _ + + override def getRequestPayload: String = DWSHttpClient.jacksonJson.writeValueAsString(getRequestPayloads) + + override def setUser(user: String): Unit = this.user = user + + override def getUser: String = this.user + + override def suffixURLs: Array[String] = Array("data-assets", "asset", "model", "unbind") +} + +object UnBindModelTypeAction { + + def builder() = new Builder + + class Builder private[UnBindModelTypeAction] { + private var user: String = _ + + private var modelName: String = _ + + private var modelType: ClassificationConstant = _ + + private var tableName: String = _ + + private var guid: String = _ + + def setUser(user: String): Builder = { + this.user = user + this + } + + def setModelName(modelName: String): Builder = { + this.modelName = modelName; + this + } + + def setModelType(modelType: ClassificationConstant): Builder = { + this.modelType = modelType; + this + } + + def setTableName(tableName: String): Builder = { + this.tableName = tableName; + this + } + + def setGuid(guid: String): Builder = { + this.guid = guid; + this + } + + def build(): UnBindModelTypeAction = { + var action = new UnBindModelTypeAction; + if (tableName == null && guid == null) throw new DataAssetsClientBuilderException("tableName or guid is needed!") + if (modelType == null || modelName == null) throw new DataAssetsClientBuilderException("modelType and modelName is needed!") + action.setUser(user) + action.guid = guid + action.modelName = modelName + action.modelType = modelType.getTypeCode + action.tableName = tableName + action.addRequestPayload("guid", action.guid) + action.addRequestPayload("modelName", action.modelName) + action.addRequestPayload("modelType", action.modelType) + action.addRequestPayload("tableName", action.tableName) + action + } + } +} + + diff --git a/dss-apps/dss-dataasset-management/data-assets-client/src/main/scala/com/webank/wedatasphere/dss/data/governance/request/UpdateLabelAction.scala b/dss-apps/dss-dataasset-management/data-assets-client/src/main/scala/com/webank/wedatasphere/dss/data/governance/request/UpdateLabelAction.scala new file mode 100644 index 0000000000..e40e246ece --- /dev/null +++ b/dss-apps/dss-dataasset-management/data-assets-client/src/main/scala/com/webank/wedatasphere/dss/data/governance/request/UpdateLabelAction.scala @@ -0,0 +1,57 @@ +package com.webank.wedatasphere.dss.data.governance.request + +import com.webank.wedatasphere.dss.data.governance.exception.DataAssetsClientBuilderException +import org.apache.linkis.httpclient.dws.DWSHttpClient +import org.apache.linkis.httpclient.request.POSTAction + +class UpdateLabelAction extends POSTAction with DataAssetsAction { + + private var user: String = _ + + override def getRequestPayload: String = DWSHttpClient.jacksonJson.writeValueAsString(getRequestPayloads) + + override def suffixURLs: Array[String] = Array("data-assets", "asset", "labels","modify") + + override def setUser(user: String): Unit = this.user = user + + override def getUser: String = this.user + +} +object UpdateLabelAction { + def builder(): Builder = new Builder + + class Builder private[UpdateLabelAction]() { + private var user: String = _ + private var name: String = _ + private var orgName: String = _ + + def setUser(user: String): Builder = { + this.user = user + this + } + + def setName(name: String): Builder = { + this.name = name + this + } + + def setOrgName(orgName: String) : Builder = { + this.orgName = orgName + this + } + + def build(): UpdateLabelAction = { + val action = new UpdateLabelAction + if (name == null) throw new DataAssetsClientBuilderException("name is needed!") + if (orgName == null) throw new DataAssetsClientBuilderException("orgName is needed!") + action.setUser(user) + action.addRequestPayload("name", this.name) + action.addRequestPayload("orgName", this.orgName) + action + + } + } +} + + + diff --git a/dss-apps/dss-dataasset-management/data-assets-client/src/main/scala/com/webank/wedatasphere/dss/data/governance/request/UpdateModelTypeAction.scala b/dss-apps/dss-dataasset-management/data-assets-client/src/main/scala/com/webank/wedatasphere/dss/data/governance/request/UpdateModelTypeAction.scala new file mode 100644 index 0000000000..66819ad6d7 --- /dev/null +++ b/dss-apps/dss-dataasset-management/data-assets-client/src/main/scala/com/webank/wedatasphere/dss/data/governance/request/UpdateModelTypeAction.scala @@ -0,0 +1,72 @@ +package com.webank.wedatasphere.dss.data.governance.request + +import com.webank.wedatasphere.dss.data.governance.entity.ClassificationConstant +import com.webank.wedatasphere.dss.data.governance.exception.DataAssetsClientBuilderException +import org.apache.linkis.httpclient.dws.DWSHttpClient +import org.apache.linkis.httpclient.request.POSTAction + +class UpdateModelTypeAction extends POSTAction with DataAssetsAction{ + + + private var name:String= _ + + private var `type`:String= _ + + private var user:String = _ + + private var orgName:String = _ + + override def getRequestPayload: String = DWSHttpClient.jacksonJson.writeValueAsString(getRequestPayloads) + + override def setUser(user: String): Unit = this.user = user + + override def getUser: String = user + + override def suffixURLs: Array[String] = Array("data-assets", "asset", "model","type","modify") +} + +object UpdateModelTypeAction{ + def builder(): Builder = new Builder + + class Builder private[UpdateModelTypeAction]() { + private var user: String = _ + private var name:String= _ + private var `type`:ClassificationConstant= _ + private var orgName:String = _ + def setUser(user: String): Builder ={ + this.user = user + this + } + + def setName(name:String):Builder = { + this.name = name + this + } + + def setOrgName(orgName:String):Builder = { + this.orgName = orgName + this + } + + def setType(`type`:ClassificationConstant):Builder = { + this.`type` = `type`; + this; + } + + def build(): UpdateModelTypeAction = { + val action = new UpdateModelTypeAction + if(`type` == null) throw new DataAssetsClientBuilderException("type is needed!") + if (name == null) throw new DataAssetsClientBuilderException("name is needed!") + if (orgName == null) throw new DataAssetsClientBuilderException("orgName is needed!") + action.name = name + action.`type` = `type`.getTypeCode + action.orgName = orgName + action.setUser(user) + action.addRequestPayload("name",action.name) + action.addRequestPayload("type",action.`type`) + action.addRequestPayload("orgName",action.orgName) + action + } + } + +} diff --git a/dss-apps/dss-dataasset-management/data-assets-client/src/main/scala/com/webank/wedatasphere/dss/data/governance/response/BindLabelResult.scala b/dss-apps/dss-dataasset-management/data-assets-client/src/main/scala/com/webank/wedatasphere/dss/data/governance/response/BindLabelResult.scala new file mode 100644 index 0000000000..750148ceb3 --- /dev/null +++ b/dss-apps/dss-dataasset-management/data-assets-client/src/main/scala/com/webank/wedatasphere/dss/data/governance/response/BindLabelResult.scala @@ -0,0 +1,11 @@ +package com.webank.wedatasphere.dss.data.governance.response + +import org.apache.linkis.httpclient.dws.annotation.DWSHttpMessageResult +import org.apache.linkis.httpclient.dws.response.DWSResult + +import scala.beans.BeanProperty + +@DWSHttpMessageResult("/api/rest_j/v\\d+/data-assets/asset/labels/bind") +class BindLabelResult extends DWSResult{ + @BeanProperty var result: Object = _ +} diff --git a/dss-apps/dss-dataasset-management/data-assets-client/src/main/scala/com/webank/wedatasphere/dss/data/governance/response/BindModelTypeResult.scala b/dss-apps/dss-dataasset-management/data-assets-client/src/main/scala/com/webank/wedatasphere/dss/data/governance/response/BindModelTypeResult.scala new file mode 100644 index 0000000000..c933afaf16 --- /dev/null +++ b/dss-apps/dss-dataasset-management/data-assets-client/src/main/scala/com/webank/wedatasphere/dss/data/governance/response/BindModelTypeResult.scala @@ -0,0 +1,11 @@ +package com.webank.wedatasphere.dss.data.governance.response + +import org.apache.linkis.httpclient.dws.annotation.DWSHttpMessageResult +import org.apache.linkis.httpclient.dws.response.DWSResult + +import scala.beans.BeanProperty + +@DWSHttpMessageResult("/api/rest_j/v\\d+/data-assets/asset/model/bind") +class BindModelTypeResult extends DWSResult{ + @BeanProperty var result: Object = _ +} diff --git a/dss-apps/dss-dataasset-management/data-assets-client/src/main/scala/com/webank/wedatasphere/dss/data/governance/response/CreateLabelResult.scala b/dss-apps/dss-dataasset-management/data-assets-client/src/main/scala/com/webank/wedatasphere/dss/data/governance/response/CreateLabelResult.scala new file mode 100644 index 0000000000..0184edda51 --- /dev/null +++ b/dss-apps/dss-dataasset-management/data-assets-client/src/main/scala/com/webank/wedatasphere/dss/data/governance/response/CreateLabelResult.scala @@ -0,0 +1,20 @@ +package com.webank.wedatasphere.dss.data.governance.response + +import com.webank.wedatasphere.dss.data.governance.entity.CreateLabelInfo +import org.apache.linkis.httpclient.dws.DWSHttpClient +import org.apache.linkis.httpclient.dws.annotation.DWSHttpMessageResult +import org.apache.linkis.httpclient.dws.response.DWSResult + +import scala.beans.BeanProperty + + +@DWSHttpMessageResult("/api/rest_j/v\\d+/data-assets/asset/labels") +class CreateLabelResult extends DWSResult{ + @BeanProperty var result: java.util.Map[String, Any] = _ + + + def getInfo : CreateLabelInfo ={ + val str = DWSHttpClient.jacksonJson.writeValueAsString(result) + DWSHttpClient.jacksonJson.readValue(str, classOf[CreateLabelInfo]) + } +} \ No newline at end of file diff --git a/dss-apps/dss-dataasset-management/data-assets-client/src/main/scala/com/webank/wedatasphere/dss/data/governance/response/CreateModelTypeResult.scala b/dss-apps/dss-dataasset-management/data-assets-client/src/main/scala/com/webank/wedatasphere/dss/data/governance/response/CreateModelTypeResult.scala new file mode 100644 index 0000000000..b7de74ce42 --- /dev/null +++ b/dss-apps/dss-dataasset-management/data-assets-client/src/main/scala/com/webank/wedatasphere/dss/data/governance/response/CreateModelTypeResult.scala @@ -0,0 +1,19 @@ +package com.webank.wedatasphere.dss.data.governance.response + +import com.webank.wedatasphere.dss.data.governance.entity.{CreateModelTypeInfo, HiveSimpleInfo} +import org.apache.linkis.httpclient.dws.DWSHttpClient +import org.apache.linkis.httpclient.dws.annotation.DWSHttpMessageResult +import org.apache.linkis.httpclient.dws.response.DWSResult + +import scala.beans.BeanProperty + +@DWSHttpMessageResult("/api/rest_j/v\\d+/data-assets/asset/model/type") +class CreateModelTypeResult extends DWSResult{ + @BeanProperty var result: java.util.Map[String, Any] = _ + + + def getInfo : CreateModelTypeInfo ={ + val str = DWSHttpClient.jacksonJson.writeValueAsString(result) + DWSHttpClient.jacksonJson.readValue(str, classOf[CreateModelTypeInfo]) + } +} diff --git a/dss-apps/dss-dataasset-management/data-assets-client/src/main/scala/com/webank/wedatasphere/dss/data/governance/response/DeleteLabelResult.scala b/dss-apps/dss-dataasset-management/data-assets-client/src/main/scala/com/webank/wedatasphere/dss/data/governance/response/DeleteLabelResult.scala new file mode 100644 index 0000000000..6190890933 --- /dev/null +++ b/dss-apps/dss-dataasset-management/data-assets-client/src/main/scala/com/webank/wedatasphere/dss/data/governance/response/DeleteLabelResult.scala @@ -0,0 +1,11 @@ +package com.webank.wedatasphere.dss.data.governance.response + +import org.apache.linkis.httpclient.dws.annotation.DWSHttpMessageResult +import org.apache.linkis.httpclient.dws.response.DWSResult + +import scala.beans.BeanProperty + +@DWSHttpMessageResult("/api/rest_j/v\\d+/data-assets/asset/labels/delete") +class DeleteLabelResult extends DWSResult{ + @BeanProperty var result:Object = _ +} diff --git a/dss-apps/dss-dataasset-management/data-assets-client/src/main/scala/com/webank/wedatasphere/dss/data/governance/response/DeleteModelTypeResult.scala b/dss-apps/dss-dataasset-management/data-assets-client/src/main/scala/com/webank/wedatasphere/dss/data/governance/response/DeleteModelTypeResult.scala new file mode 100644 index 0000000000..381538aa17 --- /dev/null +++ b/dss-apps/dss-dataasset-management/data-assets-client/src/main/scala/com/webank/wedatasphere/dss/data/governance/response/DeleteModelTypeResult.scala @@ -0,0 +1,11 @@ +package com.webank.wedatasphere.dss.data.governance.response + +import org.apache.linkis.httpclient.dws.annotation.DWSHttpMessageResult +import org.apache.linkis.httpclient.dws.response.DWSResult + +import scala.beans.BeanProperty + +@DWSHttpMessageResult("/api/rest_j/v\\d+/data-assets/asset/model/type/delete") +class DeleteModelTypeResult extends DWSResult{ + @BeanProperty var result:Object = _ +} diff --git a/dss-apps/dss-dataasset-management/data-assets-client/src/main/scala/com/webank/wedatasphere/dss/data/governance/response/GetHiveTblBasicResult.scala b/dss-apps/dss-dataasset-management/data-assets-client/src/main/scala/com/webank/wedatasphere/dss/data/governance/response/GetHiveTblBasicResult.scala new file mode 100644 index 0000000000..7eb1dc6d94 --- /dev/null +++ b/dss-apps/dss-dataasset-management/data-assets-client/src/main/scala/com/webank/wedatasphere/dss/data/governance/response/GetHiveTblBasicResult.scala @@ -0,0 +1,11 @@ +package com.webank.wedatasphere.dss.data.governance.response + +import org.apache.linkis.httpclient.dws.annotation.DWSHttpMessageResult +import org.apache.linkis.httpclient.dws.response.DWSResult + + +import scala.beans.BeanProperty +@DWSHttpMessageResult("/api/rest_j/v\\d+/data-assets/asset/hiveTbl/(\\S+)/basic") +class GetHiveTblBasicResult extends DWSResult{ + @BeanProperty var result:Object = _ +} diff --git a/dss-apps/dss-dataasset-management/data-assets-client/src/main/scala/com/webank/wedatasphere/dss/data/governance/response/GetHiveTblCreateResult.scala b/dss-apps/dss-dataasset-management/data-assets-client/src/main/scala/com/webank/wedatasphere/dss/data/governance/response/GetHiveTblCreateResult.scala new file mode 100644 index 0000000000..4089d43511 --- /dev/null +++ b/dss-apps/dss-dataasset-management/data-assets-client/src/main/scala/com/webank/wedatasphere/dss/data/governance/response/GetHiveTblCreateResult.scala @@ -0,0 +1,10 @@ +package com.webank.wedatasphere.dss.data.governance.response + +import org.apache.linkis.httpclient.dws.annotation.DWSHttpMessageResult +import org.apache.linkis.httpclient.dws.response.DWSResult + +import scala.beans.BeanProperty +@DWSHttpMessageResult("/api/rest_j/v\\d+/data-assets/asset/hiveTbl/(\\S+)/create") +class GetHiveTblCreateResult extends DWSResult{ + @BeanProperty var result:String = _ +} diff --git a/dss-apps/dss-dataasset-management/data-assets-client/src/main/scala/com/webank/wedatasphere/dss/data/governance/response/GetHiveTblPartInfoByNameResult.scala b/dss-apps/dss-dataasset-management/data-assets-client/src/main/scala/com/webank/wedatasphere/dss/data/governance/response/GetHiveTblPartInfoByNameResult.scala new file mode 100644 index 0000000000..7f5db0c3f0 --- /dev/null +++ b/dss-apps/dss-dataasset-management/data-assets-client/src/main/scala/com/webank/wedatasphere/dss/data/governance/response/GetHiveTblPartInfoByNameResult.scala @@ -0,0 +1,22 @@ +package com.webank.wedatasphere.dss.data.governance.response + +import com.webank.wedatasphere.dss.data.governance.entity.{HiveSimpleInfo, PartInfo} +import org.apache.linkis.httpclient.dws.DWSHttpClient +import org.apache.linkis.httpclient.dws.annotation.DWSHttpMessageResult +import org.apache.linkis.httpclient.dws.response.DWSResult + +import java.util +import scala.beans.BeanProperty + +@DWSHttpMessageResult("/api/rest_j/v\\d+/data-assets/asset/hiveTbl/partition/name") +class GetHiveTblPartInfoByNameResult extends DWSResult{ + @BeanProperty var result:util.List[java.util.Map[String, Any]] = _ + + def getInfo: util.List[PartInfo] ={ + import scala.collection.JavaConverters._ + result.asScala.map(x=>{ + val str = DWSHttpClient.jacksonJson.writeValueAsString(x) + DWSHttpClient.jacksonJson.readValue(str, classOf[PartInfo]) + }).asJava + } +} diff --git a/dss-apps/dss-dataasset-management/data-assets-client/src/main/scala/com/webank/wedatasphere/dss/data/governance/response/GetHiveTblPartitionResult.scala b/dss-apps/dss-dataasset-management/data-assets-client/src/main/scala/com/webank/wedatasphere/dss/data/governance/response/GetHiveTblPartitionResult.scala new file mode 100644 index 0000000000..ec7280f139 --- /dev/null +++ b/dss-apps/dss-dataasset-management/data-assets-client/src/main/scala/com/webank/wedatasphere/dss/data/governance/response/GetHiveTblPartitionResult.scala @@ -0,0 +1,11 @@ +package com.webank.wedatasphere.dss.data.governance.response + +import org.apache.linkis.httpclient.dws.annotation.DWSHttpMessageResult +import org.apache.linkis.httpclient.dws.response.DWSResult + +import java.util +import scala.beans.BeanProperty +@DWSHttpMessageResult("/api/rest_j/v\\d+/data-assets/asset/hiveTbl/(\\S+)/partition") +class GetHiveTblPartitionResult extends DWSResult{ + @BeanProperty var result:util.List[java.util.Map[String, Any]] = _ +} diff --git a/dss-apps/dss-dataasset-management/data-assets-client/src/main/scala/com/webank/wedatasphere/dss/data/governance/response/HiveTblSizeResult.scala b/dss-apps/dss-dataasset-management/data-assets-client/src/main/scala/com/webank/wedatasphere/dss/data/governance/response/HiveTblSizeResult.scala new file mode 100644 index 0000000000..fcd7a0edc1 --- /dev/null +++ b/dss-apps/dss-dataasset-management/data-assets-client/src/main/scala/com/webank/wedatasphere/dss/data/governance/response/HiveTblSizeResult.scala @@ -0,0 +1,12 @@ +package com.webank.wedatasphere.dss.data.governance.response + +import org.apache.linkis.httpclient.dws.annotation.DWSHttpMessageResult +import org.apache.linkis.httpclient.dws.response.DWSResult + +import java.util +import scala.beans.BeanProperty + +@DWSHttpMessageResult("/api/rest_j/v\\d+/data-assets/asset/hiveTbl/size") +class HiveTblSizeResult extends DWSResult{ + @BeanProperty var result: Long = _ +} diff --git a/dss-apps/dss-dataasset-management/data-assets-client/src/main/scala/com/webank/wedatasphere/dss/data/governance/response/HiveTblStatsResult.scala b/dss-apps/dss-dataasset-management/data-assets-client/src/main/scala/com/webank/wedatasphere/dss/data/governance/response/HiveTblStatsResult.scala new file mode 100644 index 0000000000..07ec55fcfa --- /dev/null +++ b/dss-apps/dss-dataasset-management/data-assets-client/src/main/scala/com/webank/wedatasphere/dss/data/governance/response/HiveTblStatsResult.scala @@ -0,0 +1,19 @@ +package com.webank.wedatasphere.dss.data.governance.response + +import com.webank.wedatasphere.dss.data.governance.entity.{CreateModelTypeInfo, HiveTblStatsDTO} +import org.apache.linkis.httpclient.dws.DWSHttpClient +import org.apache.linkis.httpclient.dws.annotation.DWSHttpMessageResult +import org.apache.linkis.httpclient.dws.response.DWSResult + +import scala.beans.BeanProperty + +@DWSHttpMessageResult("/api/rest_j/v\\d+/data-assets/asset/hiveTbl/stats") +class HiveTblStatsResult extends DWSResult{ + @BeanProperty var result: java.util.Map[String, Any] = _ + + + def getInfo : HiveTblStatsDTO ={ + val str = DWSHttpClient.jacksonJson.writeValueAsString(result) + DWSHttpClient.jacksonJson.readValue(str, classOf[HiveTblStatsDTO]) + } +} diff --git a/dss-apps/dss-dataasset-management/data-assets-client/src/main/scala/com/webank/wedatasphere/dss/data/governance/response/SearchHiveDbResult.scala b/dss-apps/dss-dataasset-management/data-assets-client/src/main/scala/com/webank/wedatasphere/dss/data/governance/response/SearchHiveDbResult.scala new file mode 100644 index 0000000000..b3760fc36c --- /dev/null +++ b/dss-apps/dss-dataasset-management/data-assets-client/src/main/scala/com/webank/wedatasphere/dss/data/governance/response/SearchHiveDbResult.scala @@ -0,0 +1,22 @@ +package com.webank.wedatasphere.dss.data.governance.response + +import com.webank.wedatasphere.dss.data.governance.entity.HiveSimpleInfo +import org.apache.linkis.httpclient.dws.DWSHttpClient +import org.apache.linkis.httpclient.dws.annotation.DWSHttpMessageResult +import org.apache.linkis.httpclient.dws.response.DWSResult + +import java.util +import scala.beans.BeanProperty + +@DWSHttpMessageResult("/api/rest_j/v\\d+/data-assets/asset/hiveDb/search") +class SearchHiveDbResult extends DWSResult{ + @BeanProperty var result:util.List[java.util.Map[String, Any]] = _ + + def getHiveList: util.List[HiveSimpleInfo] ={ + import scala.collection.JavaConverters._ + result.asScala.map(x=>{ + val str = DWSHttpClient.jacksonJson.writeValueAsString(x) + DWSHttpClient.jacksonJson.readValue(str, classOf[HiveSimpleInfo]) + }).asJava + } +} diff --git a/dss-apps/dss-dataasset-management/data-assets-client/src/main/scala/com/webank/wedatasphere/dss/data/governance/response/SearchHiveTblResult.scala b/dss-apps/dss-dataasset-management/data-assets-client/src/main/scala/com/webank/wedatasphere/dss/data/governance/response/SearchHiveTblResult.scala new file mode 100644 index 0000000000..d79db99515 --- /dev/null +++ b/dss-apps/dss-dataasset-management/data-assets-client/src/main/scala/com/webank/wedatasphere/dss/data/governance/response/SearchHiveTblResult.scala @@ -0,0 +1,22 @@ +package com.webank.wedatasphere.dss.data.governance.response + +import com.webank.wedatasphere.dss.data.governance.entity.HiveSimpleInfo +import org.apache.linkis.httpclient.dws.DWSHttpClient +import org.apache.linkis.httpclient.dws.annotation.DWSHttpMessageResult +import org.apache.linkis.httpclient.dws.response.DWSResult + +import java.util +import scala.beans.BeanProperty + +@DWSHttpMessageResult("/api/rest_j/v\\d+/data-assets/asset/hiveTbl/search") +class SearchHiveTblResult extends DWSResult{ + @BeanProperty var result:util.List[java.util.Map[String, Any]] = _ + + def getHiveList: util.List[HiveSimpleInfo] ={ + import scala.collection.JavaConverters._ + result.asScala.map(x=>{ + val str = DWSHttpClient.jacksonJson.writeValueAsString(x) + DWSHttpClient.jacksonJson.readValue(str, classOf[HiveSimpleInfo]) + }).asJava + } +} diff --git a/dss-apps/dss-dataasset-management/data-assets-client/src/main/scala/com/webank/wedatasphere/dss/data/governance/response/SearchLabelResult.scala b/dss-apps/dss-dataasset-management/data-assets-client/src/main/scala/com/webank/wedatasphere/dss/data/governance/response/SearchLabelResult.scala new file mode 100644 index 0000000000..51f17913c1 --- /dev/null +++ b/dss-apps/dss-dataasset-management/data-assets-client/src/main/scala/com/webank/wedatasphere/dss/data/governance/response/SearchLabelResult.scala @@ -0,0 +1,22 @@ +package com.webank.wedatasphere.dss.data.governance.response + +import com.webank.wedatasphere.dss.data.governance.entity.{HiveSimpleInfo, SearchLabelInfo} +import org.apache.linkis.httpclient.dws.DWSHttpClient +import org.apache.linkis.httpclient.dws.annotation.DWSHttpMessageResult +import org.apache.linkis.httpclient.dws.response.DWSResult + +import java.util +import scala.beans.BeanProperty + +@DWSHttpMessageResult("/api/rest_j/v\\d+/data-assets/asset/labels/search") +class SearchLabelResult extends DWSResult{ + @BeanProperty var result:util.List[java.util.Map[String, Any]] = _ + + def getLabelList: util.List[SearchLabelInfo] ={ + import scala.collection.JavaConverters._ + result.asScala.map(x=>{ + val str = DWSHttpClient.jacksonJson.writeValueAsString(x) + DWSHttpClient.jacksonJson.readValue(str, classOf[SearchLabelInfo]) + }).asJava + } +} diff --git a/dss-apps/dss-dataasset-management/data-assets-client/src/main/scala/com/webank/wedatasphere/dss/data/governance/response/UnBindLabelResult.scala b/dss-apps/dss-dataasset-management/data-assets-client/src/main/scala/com/webank/wedatasphere/dss/data/governance/response/UnBindLabelResult.scala new file mode 100644 index 0000000000..a0b1fb7121 --- /dev/null +++ b/dss-apps/dss-dataasset-management/data-assets-client/src/main/scala/com/webank/wedatasphere/dss/data/governance/response/UnBindLabelResult.scala @@ -0,0 +1,11 @@ +package com.webank.wedatasphere.dss.data.governance.response + +import org.apache.linkis.httpclient.dws.annotation.DWSHttpMessageResult +import org.apache.linkis.httpclient.dws.response.DWSResult + +import scala.beans.BeanProperty + +@DWSHttpMessageResult("/api/rest_j/v\\d+/data-assets/asset/labels/unbind") +class UnBindLabelResult extends DWSResult{ + @BeanProperty var result: Object = _ +} diff --git a/dss-apps/dss-dataasset-management/data-assets-client/src/main/scala/com/webank/wedatasphere/dss/data/governance/response/UnBindModelTypeResult.scala b/dss-apps/dss-dataasset-management/data-assets-client/src/main/scala/com/webank/wedatasphere/dss/data/governance/response/UnBindModelTypeResult.scala new file mode 100644 index 0000000000..a0be9615c2 --- /dev/null +++ b/dss-apps/dss-dataasset-management/data-assets-client/src/main/scala/com/webank/wedatasphere/dss/data/governance/response/UnBindModelTypeResult.scala @@ -0,0 +1,11 @@ +package com.webank.wedatasphere.dss.data.governance.response + +import org.apache.linkis.httpclient.dws.annotation.DWSHttpMessageResult +import org.apache.linkis.httpclient.dws.response.DWSResult + +import scala.beans.BeanProperty + +@DWSHttpMessageResult("/api/rest_j/v\\d+/data-assets/asset/model/unbind") +class UnBindModelTypeResult extends DWSResult{ + @BeanProperty var result: Object = _ +} diff --git a/dss-apps/dss-dataasset-management/data-assets-client/src/main/scala/com/webank/wedatasphere/dss/data/governance/response/UpdateLabelResult.scala b/dss-apps/dss-dataasset-management/data-assets-client/src/main/scala/com/webank/wedatasphere/dss/data/governance/response/UpdateLabelResult.scala new file mode 100644 index 0000000000..913ee3adb3 --- /dev/null +++ b/dss-apps/dss-dataasset-management/data-assets-client/src/main/scala/com/webank/wedatasphere/dss/data/governance/response/UpdateLabelResult.scala @@ -0,0 +1,20 @@ +package com.webank.wedatasphere.dss.data.governance.response + +import com.webank.wedatasphere.dss.data.governance.entity.UpdateLabelInfo +import org.apache.linkis.httpclient.dws.DWSHttpClient +import org.apache.linkis.httpclient.dws.annotation.DWSHttpMessageResult +import org.apache.linkis.httpclient.dws.response.DWSResult + +import scala.beans.BeanProperty + + +@DWSHttpMessageResult("/api/rest_j/v\\d+/data-assets/asset/labels/modify") +class UpdateLabelResult extends DWSResult{ + @BeanProperty var result: java.util.Map[String, Any] = _ + + + def getInfo : UpdateLabelInfo ={ + val str = DWSHttpClient.jacksonJson.writeValueAsString(result) + DWSHttpClient.jacksonJson.readValue(str, classOf[UpdateLabelInfo]) + } +} \ No newline at end of file diff --git a/dss-apps/dss-dataasset-management/data-assets-client/src/main/scala/com/webank/wedatasphere/dss/data/governance/response/UpdateModelTypeResult.scala b/dss-apps/dss-dataasset-management/data-assets-client/src/main/scala/com/webank/wedatasphere/dss/data/governance/response/UpdateModelTypeResult.scala new file mode 100644 index 0000000000..2d9359e5c8 --- /dev/null +++ b/dss-apps/dss-dataasset-management/data-assets-client/src/main/scala/com/webank/wedatasphere/dss/data/governance/response/UpdateModelTypeResult.scala @@ -0,0 +1,19 @@ +package com.webank.wedatasphere.dss.data.governance.response + +import com.webank.wedatasphere.dss.data.governance.entity.{CreateModelTypeInfo, UpdateModelTypeInfo} +import org.apache.linkis.httpclient.dws.DWSHttpClient +import org.apache.linkis.httpclient.dws.annotation.DWSHttpMessageResult +import org.apache.linkis.httpclient.dws.response.DWSResult + +import scala.beans.BeanProperty + +@DWSHttpMessageResult("/api/rest_j/v\\d+/data-assets/asset/model/type/modify") +class UpdateModelTypeResult extends DWSResult{ + @BeanProperty var result: java.util.Map[String, Any] = _ + + + def getInfo : UpdateModelTypeInfo ={ + val str = DWSHttpClient.jacksonJson.writeValueAsString(result) + DWSHttpClient.jacksonJson.readValue(str, classOf[UpdateModelTypeInfo]) + } +} diff --git a/dss-apps/dss-dataasset-management/data-assets-client/src/test/scala/com/webank/wedatasphere/dss/data/governance/TestDataAssetsRemoteClient.scala b/dss-apps/dss-dataasset-management/data-assets-client/src/test/scala/com/webank/wedatasphere/dss/data/governance/TestDataAssetsRemoteClient.scala new file mode 100644 index 0000000000..67a61673f4 --- /dev/null +++ b/dss-apps/dss-dataasset-management/data-assets-client/src/test/scala/com/webank/wedatasphere/dss/data/governance/TestDataAssetsRemoteClient.scala @@ -0,0 +1,108 @@ +package com.webank.wedatasphere.dss.data.governance + +import com.webank.wedatasphere.dss.data.governance.impl.LinkisDataAssetsRemoteClient +import com.webank.wedatasphere.dss.data.governance.request.{GetHiveTblPartitionAction, GetTblPartInfoByNameAction} +import org.apache.linkis.httpclient.dws.config.DWSClientConfigBuilder + +import java.util.concurrent.TimeUnit +import scala.Console.println + + +object TestDataAssetsRemoteClient { + def main(args: Array[String]): Unit = { + val clientConfig = DWSClientConfigBuilder.newBuilder() + .addServerUrl("http://localhost:20082") + .connectionTimeout(30000) + .discoveryEnabled(false) + .discoveryFrequency(1,TimeUnit.MINUTES) + .loadbalancerEnabled(true) + .maxConnectionSize(5) + .retryEnabled(false) + .readTimeout(30000) + .setAuthenticationStrategy(null) + .setAuthTokenKey("hdfs") + .setAuthTokenValue("hdfs") + .setDWSVersion("v1") + .build() + + val dataAssetsClient = new LinkisDataAssetsRemoteClient(clientConfig) +// +// val searchHiveTblResult = dataAssetsClient.searchHiveTbl(SearchHiveTblAction.builder().setUser("hdfs").setQuery("").setLimit(10).setOffset(0).setOwner("undefined").build()).getHiveList +// println(searchHiveTblResult) +// +// val searchHiveDbResult = dataAssetsClient.searchHiveDb(SearchHiveDbAction.builder().setUser("hdfs").setQuery("").setLimit(10).setOffset(0).setOwner("undefined").build()).getHiveList +// println(searchHiveDbResult) +// +// val hiveTblBasicResult = dataAssetsClient +// .getHiveTblBasic(GetHiveTblBasicAction.builder().setUser("hdfs").setGuid("27920dc8-1eef-4d7d-9423-b5967d9e2d33").build()) +// .result +// println(hiveTblBasicResult) +// + val hiveTblPartitionResult = dataAssetsClient.getHiveTblPartition(GetHiveTblPartitionAction.builder().setUser("hdfs").setGuid("a3be4a97-6465-4c3d-adee-76dfa662e531").build()).result + println(hiveTblPartitionResult) +// +// val hiveTblCreateResult = dataAssetsClient.getHiveTblCreate(GetHiveTblCreateAction.builder().setUser("hdfs").setGuid("a3be4a97-6465-4c3d-adee-76dfa662e531").build()).result +// println(hiveTblCreateResult) +// +// val deleteModelTypeResult = dataAssetsClient.deleteModelType(DeleteModelTypeAction.builder().setUser("hdfs").setType(ClassificationConstant.INDICATOR).setName("test004").build()) +// println(deleteModelTypeResult.getResult) +// +// val createModelTypeResult = dataAssetsClient.createModelType(CreateModelTypeAction.builder().setUser("hdfs").setType(ClassificationConstant.INDICATOR).setName("test004").build()).getInfo +// println(createModelTypeResult) +// +// val updateModelTypeResult =dataAssetsClient.updateModelType(UpdateModelTypeAction.builder().setUser("hdfs").setType(ClassificationConstant.INDICATOR).setName("test000NEW").setOrgName("test000").build()) +// println(updateModelTypeResult.getInfo.getGuid) +// println(updateModelTypeResult.getInfo.getName) +// +// val bindResult = dataAssetsClient.bindModelType(BindModelTypeAction.builder() +// .setUser("hdfs") +// .setTableName("default.test02") +// .setModelName("test001") +// .setModelType(ClassificationConstant.INDICATOR).build()) +// println(bindResult.getResult) + +// val unBindResult = dataAssetsClient.unBindModelType(UnBindModelTypeAction.builder() +// .setUser("hdfs") +// .setTableName("default.test02") +// .setModelName("test001") +// .setModelType(ClassificationConstant.INDICATOR).build()) +// println(unBindResult.getResult) + +// val hiveTableSizeResult = dataAssetsClient.searchHiveTblSize(HiveTblSizeAction.builder() +// .setUser("hdfs") +// .setTableName("test04") +// .setDbName("default").build()) +// println(hiveTableSizeResult.getResult) +// val hiveTableStatsResult = dataAssetsClient.searchHiveTblStats(HiveTblStatsAction.builder() +// .setUser("hdfs") +// .setTableName("test04") +// .setDbName("default").build()) +// println(hiveTableStatsResult.getResult) + +// val searchLabelResult = dataAssetsClient.searchLabel(SearchLabelAction.builder().setUser("hdfs").setQuery("t").build()); +// println(searchLabelResult.getLabelList) +// +// val createLabelResult = dataAssetsClient.createLabel(CreateLabelAction.builder().setUser("hdfs").setName("label007").build()) +// println(createLabelResult.getInfo) +// +// TimeUnit.SECONDS.sleep(5) +// +// val bindLabelResult = dataAssetsClient.bindLabel(BindLabelAction.builder().setUser("hdfs").setLabel("term1").setTableName("linkis_db.linkis_test01").build()) +// println(bindLabelResult.getResult) +//// +// val unbindLabelResult = dataAssetsClient.unBindLabel(UnBindLabelAction.builder().setUser("hdfs").setLabel("term1").setTableName("linkis_db.linkis_test01").build()) +// println(unbindLabelResult.getResult) +// +// val updateLabelResult = dataAssetsClient.updateLabel(UpdateLabelAction.builder().setUser("hdfs").setName("label008").setOrgName("label007").build()) +// println(updateLabelResult.getInfo) +// +// val deleteLabelResult = dataAssetsClient.deleteLabel(DeleteLabelAction.builder().setUser("hdfs").setName("label008").build()) +// println(deleteLabelResult.getResult) + +// +// println(hiveTableStatsResult.getResult) + + val partInfo = dataAssetsClient.getHiveTblPartInfoByNameResult(GetTblPartInfoByNameAction.builder().setUser("hdfs").setDbName("linkis_db").setTableName("linkis_partitions").build()) + println(partInfo.getInfo) + } +} diff --git a/dss-apps/dss-dataasset-management/data-assets-server/bin/start-assets.sh b/dss-apps/dss-dataasset-management/data-assets-server/bin/start-assets.sh new file mode 100755 index 0000000000..b5390ff9c1 --- /dev/null +++ b/dss-apps/dss-dataasset-management/data-assets-server/bin/start-assets.sh @@ -0,0 +1,38 @@ +#!/bin/bash + +cd `dirname $0` +cd .. +HOME=`pwd` + +export SERVER_PID=$HOME/bin/linkis.pid +export SERVER_LOG_PATH=$HOME/logs +export SERVER_CLASS=com.webank.wedatasphere.dss.data.governance.DataAssetsApplication + +if test -z "$SERVER_HEAP_SIZE" +then + export SERVER_HEAP_SIZE="512M" +fi + +if test -z "$SERVER_JAVA_OPTS" +then + export SERVER_JAVA_OPTS=" -Xmx$SERVER_HEAP_SIZE -XX:+UseG1GC -Xloggc:$HOME/logs/linkis-gc.log" +fi + +if [[ -f "${SERVER_PID}" ]]; then + pid=$(cat ${SERVER_PID}) + if kill -0 ${pid} >/dev/null 2>&1; then + echo "Server is already running." + exit 1 + fi +fi + +nohup java $SERVER_JAVA_OPTS -cp $HOME/conf:$HOME/lib/* $SERVER_CLASS 2>&1 > $SERVER_LOG_PATH/linkis.out & +pid=$! +if [[ -z "${pid}" ]]; then + echo "server $SERVER_NAME start failed!" + exit 1 +else + echo "server $SERVER_NAME start succeeded!" + echo $pid > $SERVER_PID + sleep 1 +fi \ No newline at end of file diff --git a/dss-apps/dss-dataasset-management/data-assets-server/bin/stop-assets.sh b/dss-apps/dss-dataasset-management/data-assets-server/bin/stop-assets.sh new file mode 100755 index 0000000000..f032887111 --- /dev/null +++ b/dss-apps/dss-dataasset-management/data-assets-server/bin/stop-assets.sh @@ -0,0 +1,47 @@ +#!/bin/bash + +cd `dirname $0` +cd .. +HOME=`pwd` + +export SERVER_PID=$HOME/bin/linkis.pid + +function wait_for_server_to_die() { + local pid + local count + pid=$1 + timeout=$2 + count=0 + timeoutTime=$(date "+%s") + let "timeoutTime+=$timeout" + currentTime=$(date "+%s") + forceKill=1 + + while [[ $currentTime -lt $timeoutTime ]]; do + $(kill ${pid} > /dev/null 2> /dev/null) + if kill -0 ${pid} > /dev/null 2>&1; then + sleep 3 + else + forceKill=0 + break + fi + currentTime=$(date "+%s") + done + + if [[ forceKill -ne 0 ]]; then + $(kill -9 ${pid} > /dev/null 2> /dev/null) + fi +} + +if [[ ! -f "${SERVER_PID}" ]]; then + echo "server $SERVER_NAME is not running" +else + pid=$(cat ${SERVER_PID}) + if [[ -z "${pid}" ]]; then + echo "server $SERVER_NAME is not running" + else + wait_for_server_to_die $pid 40 + $(rm -f ${SERVER_PID}) + echo "server $SERVER_NAME is stopped." + fi +fi \ No newline at end of file diff --git a/dss-apps/dss-dataasset-management/data-assets-server/pom.xml b/dss-apps/dss-dataasset-management/data-assets-server/pom.xml new file mode 100644 index 0000000000..1bd85be8ee --- /dev/null +++ b/dss-apps/dss-dataasset-management/data-assets-server/pom.xml @@ -0,0 +1,153 @@ + + + + dss-dataasset-management + com.webank.wedatasphere.dss + 1.0.1 + + 4.0.0 + + data-assets-server + + + 2.2.0 + 8 + 8 + + + + + org.apache.linkis + linkis-module + ${linkis.version} + + + com.fasterxml.jackson.core + jackson-databind + + + com.fasterxml.jackson.core + jackson-core + + + + + org.apache.commons + commons-math3 + + + xstream + com.thoughtworks.xstream + + + + org.apache.linkis + linkis-mybatis + ${linkis.version} + + + + org.apache.atlas + atlas-client-v2 + ${atlas.version} + + + com.google.guava + guava + + + + + org.projectlombok + lombok + 1.18.16 + compile + + + + com.alibaba + druid + 1.1.9 + + + + com.webank.wedatasphere.dss + dss-framework-workspace-client + 1.0.1 + + + + + + + org.apache.maven.plugins + maven-deploy-plugin + + + + net.alchim31.maven + scala-maven-plugin + + + org.apache.maven.plugins + maven-jar-plugin + + + org.apache.maven.plugins + maven-assembly-plugin + 2.3 + false + + + make-assembly + package + + single + + + + src/main/assembly/distribution.xml + + + + + + false + data-assets-server + false + false + + src/main/assembly/distribution.xml + + + + + org.apache.maven.plugins + maven-compiler-plugin + + 8 + 8 + + + + + + src/main/java + + **/*.xml + + + + src/main/resources + + **/*.xml + **/*.properties + **/*.yml + + + + + + \ No newline at end of file diff --git a/dss-apps/dss-dataasset-management/data-assets-server/src/main/assembly/distribution.xml b/dss-apps/dss-dataasset-management/data-assets-server/src/main/assembly/distribution.xml new file mode 100644 index 0000000000..eed259d1b4 --- /dev/null +++ b/dss-apps/dss-dataasset-management/data-assets-server/src/main/assembly/distribution.xml @@ -0,0 +1,72 @@ + + + + data-assets-server + + zip + + true + data-assets-server + + + + + + lib + true + true + false + false + true + + + + + + ${basedir}/src/main/resources + + * + + 0777 + conf + unix + + + ${basedir}/bin + + * + + 0777 + bin + unix + + + . + + */** + + logs + + + + + diff --git a/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/atlas/AtlasClient.java b/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/atlas/AtlasClient.java new file mode 100644 index 0000000000..fbcda8ddc4 --- /dev/null +++ b/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/atlas/AtlasClient.java @@ -0,0 +1,276 @@ +package com.webank.wedatasphere.dss.data.governance.atlas; + +import com.google.common.collect.Lists; +import com.google.common.collect.Sets; +import com.google.gson.Gson; +import com.sun.jersey.core.util.MultivaluedMapImpl; +import com.webank.wedatasphere.dss.data.governance.entity.GlossaryConstant; +import org.apache.atlas.AtlasClientV2; +import org.apache.atlas.AtlasServiceException; +import org.apache.atlas.model.discovery.SearchParameters; +import org.apache.atlas.model.glossary.AtlasGlossaryTerm; +import org.apache.atlas.model.glossary.relations.AtlasGlossaryHeader; +import org.apache.atlas.model.instance.AtlasRelatedObjectId; +import org.apache.atlas.model.lineage.AtlasLineageInfo; +import org.apache.atlas.model.typedef.AtlasClassificationDef; +import org.apache.atlas.model.typedef.AtlasTypesDef; +import org.apache.commons.configuration.Configuration; + +import javax.ws.rs.HttpMethod; +import javax.ws.rs.core.MultivaluedMap; +import javax.ws.rs.core.Response; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + + +public class AtlasClient extends AtlasClientV2 { + // Entities APIs + public static final String ENTITIES_API = BASE_URI + "entities/"; + public static final String UPDATE_ENTITY_ATTR_API = ENTITY_API + "guid/"; + public static final String SET_ENTITY_LABELS_BY_GUID_TEMPLATE = ENTITY_API + "/guid/%s/labels"; + public static final String GET_ENTITY_HEADER_BY_GUID_TEMPLATE = ENTITY_API + "/guid/{guid}/header"; + private final Gson gson = new Gson(); + + public AtlasClient(Configuration configuration,String[] baseUrl, String[] basicAuthUserNamePassword) { + super(configuration,baseUrl, basicAuthUserNamePassword); + } + + /** + * 获取所有的hive db实体 + */ + public String getHiveDbs() throws AtlasServiceException { + MultivaluedMap queryParams = new MultivaluedMapImpl(); + queryParams.add("type", "hive_db"); + return callAPI(API_V3.GET_ENTITIES, String.class, queryParams); + } + + /** + * 获取所有的hive table实体 + */ + public String getHiveTables() throws AtlasServiceException { + MultivaluedMap queryParams = new MultivaluedMapImpl(); + queryParams.add("type", "hive_table"); + return callAPI(API_V3.GET_ENTITIES, String.class, queryParams); + } + + /** + * 获取表实体的血缘信息 + */ + public String getLineageInfoForString(final String guid, final AtlasLineageInfo.LineageDirection direction, final int depth) throws AtlasServiceException { + MultivaluedMap queryParams = new MultivaluedMapImpl(); + queryParams.add("direction", direction.toString()); + queryParams.add("depth", String.valueOf(depth)); + + return callAPI(API_V2.LINEAGE_INFO, String.class, queryParams, guid); + } + + /** + * 获取实体信息 + */ + public String getEntityByGuidForString(String guid) throws AtlasServiceException { + return getEntityByGuidForString(guid, false, false); + } + + /** + * 获取实体信息 + */ + public String getEntityByGuidForString(String guid, boolean minExtInfo, boolean ignoreRelationships) throws AtlasServiceException { + MultivaluedMap queryParams = new MultivaluedMapImpl(); + + queryParams.add("minExtInfo", String.valueOf(minExtInfo)); + queryParams.add("ignoreRelationships", String.valueOf(ignoreRelationships)); + + return callAPI(API_V2.GET_ENTITY_BY_GUID, String.class, queryParams, guid); + } + + public String getEntitiesByGuidsForString(List guids) throws AtlasServiceException { + return getEntitiesByGuidsForString(guids, false, false); + } + + public String getEntitiesByGuidsForString(List guids, boolean minExtInfo, boolean ignoreRelationships) throws AtlasServiceException { + MultivaluedMap queryParams = new MultivaluedMapImpl(); + + queryParams.put("guid", guids); + queryParams.add("minExtInfo", String.valueOf(minExtInfo)); + queryParams.add("ignoreRelationships", String.valueOf(ignoreRelationships)); + + return callAPI(API_V2.GET_ENTITIES_BY_GUIDS, String.class, queryParams); + } + + /** + * 获取实体基本信息 + */ + public String getHeaderByIdForString(String guid) throws AtlasServiceException { + API api = new API(String.format(GET_ENTITY_HEADER_BY_GUID_TEMPLATE, guid), HttpMethod.GET, Response.Status.OK); + return callAPI(api, String.class,null); + } + + /** + * 根据关键字检索实体 + */ + public String basicSearchForString(final String typeName, final String classification, final String query, + final boolean excludeDeletedEntities, final int limit, final int offset) throws AtlasServiceException { + MultivaluedMap queryParams = new MultivaluedMapImpl(); + queryParams.add("typeName", typeName); + queryParams.add("classification", classification); + queryParams.add(QUERY, query); + queryParams.add("excludeDeletedEntities", String.valueOf(excludeDeletedEntities)); + queryParams.add(LIMIT, String.valueOf(limit)); + queryParams.add(OFFSET, String.valueOf(offset)); + + return callAPI(API_V2.BASIC_SEARCH, String.class, queryParams); + } + + + /** + * 创建子类型 + * @param name + * @param superType + * @return + * @throws AtlasServiceException + */ + public String createSubClassification(String name, String superType) throws AtlasServiceException { + AtlasClassificationDef atlasClassificationDef = new AtlasClassificationDef(); + atlasClassificationDef.setName(name); + atlasClassificationDef.setSuperTypes(Sets.newHashSet(superType)); + AtlasTypesDef atlasTypesDef = new AtlasTypesDef(); + atlasTypesDef.setClassificationDefs(Lists.newArrayList(atlasClassificationDef)); + MultivaluedMap params = new MultivaluedMapImpl(); + params.add("type","classification"); + return callAPI(AtlasClientV2.API_V2.CREATE_TYPE_DEFS, String.class, gson.toJson(atlasTypesDef),params); + } + + // Glossary APIs + public String getAllGlossaries() throws AtlasServiceException { + MultivaluedMap queryParams = new MultivaluedMapImpl(); + + queryParams.add(LIMIT, "99999"); + queryParams.add(OFFSET, "0"); + + return callAPI(API_V2.GET_ALL_GLOSSARIES, String.class, queryParams); + } + + /** + * 新建分词 + * @param name + * @param rootGuid + * @return + * @throws AtlasServiceException + */ + public String createGlossaryTerm(String name,String rootGuid) throws AtlasServiceException { + AtlasGlossaryTerm glossaryTerm = new AtlasGlossaryTerm(); + glossaryTerm.setName(name); + AtlasGlossaryHeader anchor = new AtlasGlossaryHeader(); + anchor.setGlossaryGuid(rootGuid); + glossaryTerm.setAnchor(anchor); + return callAPI(API_V2.CREATE_GLOSSARY_TERM, String.class, gson.toJson(glossaryTerm)); + } + + /** + * 根据名称 查询分词详情 + * @param glossaryConstant + * @param name + * @return + * @throws AtlasServiceException + */ + public String attributeSearchByName(GlossaryConstant glossaryConstant, String name) throws AtlasServiceException { + return attributeSearch0(glossaryConstant.getAtlasType(),glossaryConstant.formatQuery(name), GlossaryConstant.ARR,1,0); + } + + public String attributeSearch0(String typeName, String name, String attrName, Integer limit, Integer offset) throws AtlasServiceException { + MultivaluedMap queryParams = new MultivaluedMapImpl(); + + queryParams.add("attrName", attrName); + queryParams.add("attrValuePrefix",name); + queryParams.add("typeName", typeName); + queryParams.add(LIMIT, limit+""); + queryParams.add(OFFSET, offset+""); + + return callAPI(API_V2.ATTRIBUTE_SEARCH, String.class, queryParams); + } + + + /** + * 实体绑定分词 + * @param termGuid + * @param entityGuids + * @throws AtlasServiceException + */ + public void assignTermToEntities0(String termGuid, List entityGuids) throws AtlasServiceException { + List relatedObjectIds = entityGuids.stream().map(entityGuid->{AtlasRelatedObjectId atlasRelatedObjectId = new AtlasRelatedObjectId(); + atlasRelatedObjectId.setGuid(entityGuid); + return atlasRelatedObjectId; + }).collect(Collectors.toList()); + callAPI(formatPathParameters(API_V2.ASSIGN_TERM_TO_ENTITIES, termGuid), (Class) null, gson.toJson(relatedObjectIds)); + } + + /** + * 解绑实体 + * @param termGuid + * @param + * @throws AtlasServiceException + */ + public void disassociateTermFromEntities0(String termGuid, List relatedObjectIds) throws AtlasServiceException { + callAPI(formatPathParameters(API_V2.DISASSOCIATE_TERM_FROM_ENTITIES, termGuid), (Class) null, gson.toJson(relatedObjectIds)); + } + + /** + * 根据关键字检索实体 + */ + public String basicSearchPostForString(final String typeName, final String classification, final String query,final String termName, + final boolean excludeDeletedEntities, final int limit, final int offset) throws AtlasServiceException { + SearchParameters searchParameters = new SearchParameters(); + Set returnColumnsParams = Sets.newHashSet("aliases", "parameters", "lastAccessTime","comment"); + searchParameters.setTypeName(typeName); + searchParameters.setClassification(classification); + searchParameters.setQuery(query); + searchParameters.setLimit(limit); + searchParameters.setTermName(termName); + searchParameters.setOffset(offset); + searchParameters.setIncludeSubClassifications(true); + searchParameters.setIncludeSubTypes(true); + searchParameters.setIncludeClassificationAttributes(true); + searchParameters.setExcludeDeletedEntities(excludeDeletedEntities); + searchParameters.setAttributes(returnColumnsParams); + return callAPI(AtlasClientV2.API_V2.FACETED_SEARCH, String.class, gson.toJson(searchParameters),new MultivaluedMapImpl()); + } + + /** + * 修改实体的注释 + */ + public String modifyComment(String guid,String commentStr) throws AtlasServiceException { + MultivaluedMap queryParams = new MultivaluedMapImpl(); + queryParams.add("name", "comment"); + + return callAPI(API_V3.UPDATE_ENTITY_ATTR,String.class,commentStr,queryParams,guid); + } + + /** + * 获取分词详情 + * @param termGuid + * @return + * @throws AtlasServiceException + */ + public String getGlossaryTerm0(String termGuid) throws AtlasServiceException { + + return callAPI(API_V2.GET_GLOSSARY_TERM, String.class, null, termGuid); + } + + /** + * 设置实体的标签 + */ + public void setLabels(String guid, Set labels) throws AtlasServiceException { + API api = new API(String.format(SET_ENTITY_LABELS_BY_GUID_TEMPLATE, guid), HttpMethod.POST, Response.Status.NO_CONTENT); + callAPI(api, (Class)null, labels); + } + + public static class API_V3 extends API { + public static final API_V3 GET_ENTITIES = new API_V3(ENTITIES_API, HttpMethod.GET, Response.Status.OK); + public static final API_V3 UPDATE_ENTITY_ATTR = new API_V3(UPDATE_ENTITY_ATTR_API,HttpMethod.PUT,Response.Status.OK); + + private API_V3(String path, String method, Response.Status status) { + super(path, method, status); + } + } +} diff --git a/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/atlas/AtlasService.java b/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/atlas/AtlasService.java new file mode 100644 index 0000000000..3a5046ee5f --- /dev/null +++ b/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/atlas/AtlasService.java @@ -0,0 +1,358 @@ +package com.webank.wedatasphere.dss.data.governance.atlas; + +import com.google.common.collect.Lists; +import com.google.gson.*; +import com.google.gson.reflect.TypeToken; +import com.sun.jersey.core.util.MultivaluedMapImpl; +import com.webank.wedatasphere.dss.data.governance.conf.GovernanceConf; +import com.webank.wedatasphere.dss.data.governance.entity.GlossaryConstant; +import com.webank.wedatasphere.dss.data.governance.entity.RelatedObjectId; +import com.webank.wedatasphere.dss.data.governance.exception.DataGovernanceException; +import org.apache.atlas.ApplicationProperties; +import org.apache.atlas.AtlasClientV2; +import org.apache.atlas.AtlasException; +import org.apache.atlas.AtlasServiceException; +import org.apache.atlas.model.AtlasBaseModelObject; +import org.apache.atlas.model.discovery.AtlasSearchResult; +import org.apache.atlas.model.glossary.AtlasGlossary; +import org.apache.atlas.model.glossary.AtlasGlossaryTerm; +import org.apache.atlas.model.instance.*; +import org.apache.atlas.model.lineage.AtlasLineageInfo; +import org.apache.atlas.model.typedef.AtlasClassificationDef; +import org.apache.atlas.model.typedef.AtlasTypesDef; +import org.apache.commons.configuration.Configuration; +import org.apache.commons.lang.StringUtils; +import org.springframework.stereotype.Service; +import org.springframework.util.CollectionUtils; + +import java.lang.reflect.Type; +import java.util.*; +import java.util.stream.Collectors; + + +@Service("atlasService") +public class AtlasService { + private final AtlasClient atlasClient; + private final Gson gson; + + public AtlasService() throws AtlasException { + Configuration configuration = ApplicationProperties.get("linkis.properties"); + String[] urls = new String[]{GovernanceConf.ATLAS_REST_ADDRESS.getValue()}; + String[] basicAuthUsernamePassword = new String[]{GovernanceConf.ATLAS_USERNAME.getValue(), GovernanceConf.ATLAS_PASSWORD.getValue()}; + + atlasClient = new AtlasClient(configuration, urls, basicAuthUsernamePassword); + + + GsonBuilder builder = new GsonBuilder(); + // Register an adapter to manage the date types as long values + builder.registerTypeAdapter(Date.class, new JsonDeserializer() { + @Override + public Date deserialize(JsonElement json, Type typeOfT, JsonDeserializationContext context) throws JsonParseException { + return new Date(json.getAsJsonPrimitive().getAsLong()); + } + }); + + gson = builder.create(); + } + + /** + * hive db数量 + */ + public Long getHiveDbCnt() throws AtlasServiceException { + String jsonStr = atlasClient.getHiveDbs(); + if (jsonStr != null && jsonStr.trim() != "") { + JsonObject jsonObject = gson.fromJson(jsonStr, JsonObject.class); + if (jsonObject != null) { + return jsonObject.get("count").getAsLong(); + } else { + return 0l; + } + } else { + return 0l; + } + } + + /** + * hive table数量 + */ + public long getHiveTableCnt() throws AtlasServiceException { + String jsonStr = atlasClient.getHiveTables(); + if (jsonStr != null && jsonStr.trim() != "") { + JsonObject jsonObject = gson.fromJson(jsonStr, JsonObject.class); + if (jsonObject != null) { + return jsonObject.get("count").getAsLong(); + } else { + return 0l; + } + } else { + return 0l; + } + } + + + /** + * 根据关键字搜索hive table + */ + public List searchHiveTable(String classification, String query, + boolean excludeDeletedEntities, int limit, int offset) throws AtlasServiceException { + String jsonStr = atlasClient.basicSearchForString("hive_table", classification, query, excludeDeletedEntities, limit, offset); + AtlasSearchResult atlasSearchResult = gson.fromJson(jsonStr, AtlasSearchResult.class); + + return atlasSearchResult.getEntities(); + } + + /** + * 根据关键字搜索hive table + */ + public List searchHiveTable0(String classification, String query, String termName, + boolean excludeDeletedEntities, int limit, int offset) throws AtlasServiceException { + + String jsonStr = atlasClient.basicSearchPostForString("hive_table", classification, query, termName , excludeDeletedEntities, limit, offset); + AtlasSearchResult atlasSearchResult = gson.fromJson(jsonStr, AtlasSearchResult.class);//atlasClient.facetedSearch(searchParameters); + + //实体绑定类型 + //atlasClient.addClassification(); + //创建子类型 + //atlasClient.createAtlasTypeDefs() + return atlasSearchResult.getEntities(); + } + + /** + * 创建子类型 + * + * @param name + * @param superType + * @return + * @throws AtlasServiceException + */ + public AtlasClassificationDef createSubClassification(String name, String superType) throws AtlasServiceException { + String jsonStr = atlasClient.createSubClassification(name, superType); + AtlasTypesDef atlasTypesDef = gson.fromJson(jsonStr, AtlasTypesDef.class); + return atlasTypesDef.getClassificationDefs().get(0); + } + + /** + * 删除指定模型类型 + * + * @param name + * @throws AtlasServiceException + */ + public void deleteClassification(String name) throws AtlasServiceException { + atlasClient.deleteTypeByName(name); + } + + + /** + * 绑定类型 + * + * @param typeName + * @param guid + * @throws AtlasServiceException + */ + public void addClassification(String typeName, String guid, boolean propagate) throws AtlasServiceException { + AtlasClassification atlasClassification = new AtlasClassification(); + atlasClassification.setTypeName(typeName); + atlasClassification.setPropagate(propagate); + atlasClassification.setRemovePropagationsOnEntityDelete(false); + + ClassificationAssociateRequest request = new ClassificationAssociateRequest(Lists.newArrayList(guid), atlasClassification); + atlasClient.callAPI(AtlasClientV2.API_V2.ADD_CLASSIFICATION, (Class) null, gson.toJson(request), new MultivaluedMapImpl()); + } + + /** + * 解绑类型 + * + * @param guid + * @param typeName + * @throws AtlasServiceException + */ + public void deleteClassification(String guid, String typeName) throws AtlasServiceException { + atlasClient.deleteClassification(guid, typeName); + } + + /** + * 获取hive table对象 + */ + public AtlasEntity getHiveTbl(String guid) throws AtlasServiceException { + String jsonStr = atlasClient.getEntityByGuidForString(guid, false, false); + + AtlasEntity.AtlasEntityWithExtInfo atlasEntityWithExtInfo = gson.fromJson(jsonStr, AtlasEntity.AtlasEntityWithExtInfo.class); + + return atlasEntityWithExtInfo.getEntity(); + } + + /** + * 获取hive column对象 + */ + public AtlasEntity getHiveColumn(String guid) throws AtlasServiceException { + String jsonStr = atlasClient.getEntityByGuidForString(guid, true, true); + + AtlasEntity.AtlasEntityWithExtInfo atlasEntityWithExtInfo = gson.fromJson(jsonStr, AtlasEntity.AtlasEntityWithExtInfo.class); + + return atlasEntityWithExtInfo.getEntity(); + } + + /** + * 获取多个hive column对象 + */ + public List getHiveColumnsByGuids(List guids) throws AtlasServiceException { + String jsonStr = atlasClient.getEntitiesByGuidsForString(guids, true, true); + AtlasEntity.AtlasEntitiesWithExtInfo atlasEntitiesWithExtInfo = gson.fromJson(jsonStr, AtlasEntity.AtlasEntitiesWithExtInfo.class); + + return atlasEntitiesWithExtInfo.getEntities(); + } + + + /** + * 修改实体的注释 + */ + public void modifyComment(String guid, String commentStr) throws AtlasServiceException { + atlasClient.modifyComment(guid, commentStr); + } + + /** + * 设置实体的标签 + */ + public void setLabels(String guid, Set labels) throws AtlasServiceException { + atlasClient.setLabels(guid, labels); + } + + /** + * 获取根glossary guid + * + * @return + * @throws AtlasServiceException + */ + public Optional getRootGlossaryGuid(GlossaryConstant glossaryConstant) throws AtlasServiceException { + String result = atlasClient.getAllGlossaries(); + List glossaries = gson.fromJson(result, new TypeToken>() { + }.getType()); + Optional root = glossaries.stream().filter(atlasGlossary -> StringUtils.equals(atlasGlossary.getName(), glossaryConstant.getRoot())).findFirst(); + return root.map(AtlasBaseModelObject::getGuid); + } + + /** + * 新建标签 + * + * @param labelName + * @return + * @throws AtlasServiceException + */ + public AtlasGlossaryTerm createLabel(String labelName) throws AtlasServiceException { + String result = atlasClient.createGlossaryTerm(labelName, getRootGlossaryGuid(GlossaryConstant.LABEL).get()); + return gson.fromJson(result, AtlasGlossaryTerm.class); + } + + /** + * 绑定标签 + * @param termGuid + * @param entityGuids + * @throws AtlasServiceException + */ + public void assignTermToEntities(String termGuid,List entityGuids) throws AtlasServiceException{ + atlasClient.assignTermToEntities0(termGuid,entityGuids); + } + + /** + * 解绑实体 + * @param termGuid + * @param + * @throws AtlasServiceException + */ + public void disassociateTermFromEntities(String termGuid, List relatedObjectIds) throws AtlasServiceException { + atlasClient.disassociateTermFromEntities0(termGuid,relatedObjectIds.stream().map( + relatedObjectId -> { + AtlasRelatedObjectId atlasRelatedObjectId = new AtlasRelatedObjectId(); + atlasRelatedObjectId.setGuid(relatedObjectId.getGuid()); + atlasRelatedObjectId.setRelationshipGuid(relatedObjectId.getRelationshipGuid()); + return atlasRelatedObjectId; + } + ).collect(Collectors.toList())); + } + + + + /** + * 删除标签 + * @param labelName + * @throws AtlasServiceException + */ + public void deleteLabel(String labelName) throws Exception{ + Optional optional = getTermGuid(GlossaryConstant.LABEL,labelName); + if (!optional.isPresent()){ + throw new DataGovernanceException(23000, labelName + "标签不存在"); + } + atlasClient.deleteGlossaryTermByGuid(optional.get()); + } + + /** + * 获取分词guid + * @param glossaryConstant + * @param name + * @return + * @throws AtlasServiceException + */ + public Optional getTermGuid(GlossaryConstant glossaryConstant, String name) throws AtlasServiceException { + String result = atlasClient.attributeSearchByName(glossaryConstant, name); + AtlasSearchResult atlasSearchResult = gson.fromJson(result, AtlasSearchResult.class); + return !CollectionUtils.isEmpty(atlasSearchResult.getEntities()) ? + Optional.ofNullable(atlasSearchResult.getEntities().get(0).getGuid()) : Optional.empty(); + } + + public List listLabels(String query,Integer limit,Integer offset) throws AtlasServiceException{ + String result = atlasClient.attributeSearch0(GlossaryConstant.LABEL.getAtlasType(),query,GlossaryConstant.ARR,limit,offset); + AtlasSearchResult atlasSearchResult = gson.fromJson(result, AtlasSearchResult.class); + return !CollectionUtils.isEmpty(atlasSearchResult.getEntities())?atlasSearchResult.getEntities():Lists.newArrayList(); + } + + + /** + * 血缘信息 + */ + public AtlasLineageInfo getLineageInfo(final String guid, final AtlasLineageInfo.LineageDirection direction, final int depth) throws AtlasServiceException { + String jsonStr = atlasClient.getLineageInfoForString(guid, direction, depth); + AtlasLineageInfo atlasLineageInfo = gson.fromJson(jsonStr, AtlasLineageInfo.class); + return atlasLineageInfo; + } + + /** + * 根据guid来获取hive tbl名称： db.table + */ + public String getHiveTblNameById(String guid) throws AtlasServiceException { + String jsonStr = atlasClient.getHeaderByIdForString(guid); + AtlasEntityHeader atlasEntityHeader = gson.fromJson(jsonStr, AtlasEntityHeader.class); + + return atlasEntityHeader.getAttribute("qualifiedName").toString().split("@")[0]; + } + + /** + * 根据guid来获取hive tbl名称 和 是否分区表 + */ + public Map getHiveTblNameAndIsPartById(String guid) throws AtlasServiceException { + Map result = new HashMap<>(2); + + String jsonStr = atlasClient.getEntityByGuidForString(guid, true, false); + AtlasEntity.AtlasEntityWithExtInfo atlasEntityWithExtInfo = gson.fromJson(jsonStr, AtlasEntity.AtlasEntityWithExtInfo.class); + + result.put("tblName", atlasEntityWithExtInfo.getEntity().getAttribute("qualifiedName").toString().split("@")[0]); + result.put("isPartition", ((List) atlasEntityWithExtInfo.getEntity().getAttribute("partitionKeys")).size() > 0); + + return result; + } + + /** + * 获取hive db对象 + */ + public List searchHiveDb(String classification, String query, + boolean excludeDeletedEntities, int limit, int offset) throws AtlasServiceException { + String jsonStr = atlasClient.basicSearchPostForString("hive_db", classification, query,null, excludeDeletedEntities, limit, offset); + AtlasSearchResult atlasSearchResult = gson.fromJson(jsonStr, AtlasSearchResult.class); + + return atlasSearchResult.getEntities(); + } + + public AtlasGlossaryTerm getGlossaryTermDetail(String termGuid) throws AtlasServiceException { + String detail = atlasClient.getGlossaryTerm0(termGuid); + return gson.fromJson(detail,AtlasGlossaryTerm.class); + } +} diff --git a/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/conf/ClientCommonConfig.java b/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/conf/ClientCommonConfig.java new file mode 100644 index 0000000000..2a91c38828 --- /dev/null +++ b/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/conf/ClientCommonConfig.java @@ -0,0 +1,34 @@ +package com.webank.wedatasphere.dss.data.governance.conf; + +import com.webank.wedatasphere.dss.framework.workspace.client.impl.LinkisWorkSpaceRemoteClient; +import org.apache.linkis.httpclient.authentication.AuthenticationStrategy; +import org.apache.linkis.httpclient.dws.config.DWSClientConfig; +import org.apache.linkis.httpclient.dws.config.DWSClientConfigBuilder; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +import java.util.concurrent.TimeUnit; + +@Configuration +public class ClientCommonConfig { + + @Bean + public LinkisWorkSpaceRemoteClient linkisWorkSpaceRemoteClient() throws Exception { + AuthenticationStrategy authenticationStrategy = (AuthenticationStrategy) Class.forName(DataWorkspaceRemoteConfig.AUTHENTICATION_STRATEGY.getValue()).newInstance(); + DWSClientConfig clientConfig = ((DWSClientConfigBuilder) DWSClientConfigBuilder.newBuilder() + .addServerUrl(DataWorkspaceRemoteConfig.SERVER_URL.getValue()) + .connectionTimeout(DataWorkspaceRemoteConfig.CONNECTION_TIMEOUT.getValue()) + .discoveryEnabled(DataWorkspaceRemoteConfig.DISCOVERY_ENABLED.getValue()) + .discoveryFrequency(DataWorkspaceRemoteConfig.DISCOVERY_FREQUENCY_PERIOD.getValue(), TimeUnit.MINUTES) + .loadbalancerEnabled(DataWorkspaceRemoteConfig.LOAD_BALANCER_ENABLED.getValue()) + .maxConnectionSize(DataWorkspaceRemoteConfig.MAX_CONNECTION_SIZE.getValue()) + .retryEnabled(DataWorkspaceRemoteConfig.RETRY_ENABLED.getValue()) + .readTimeout(DataWorkspaceRemoteConfig.READ_TIMEOUT.getValue()) + .setAuthenticationStrategy(authenticationStrategy) + .setAuthTokenKey(DataWorkspaceRemoteConfig.AUTHTOKEN_KEY.getValue()) + .setAuthTokenValue(DataWorkspaceRemoteConfig.AUTHTOKEN_VALUE.getValue()) + ).setDWSVersion(DataWorkspaceRemoteConfig.DWS_VERSION.getValue()) + .build(); + return new LinkisWorkSpaceRemoteClient(clientConfig); + } +} diff --git a/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/conf/ClientStrategy.java b/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/conf/ClientStrategy.java new file mode 100644 index 0000000000..6f1ab65d61 --- /dev/null +++ b/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/conf/ClientStrategy.java @@ -0,0 +1,17 @@ +package com.webank.wedatasphere.dss.data.governance.conf; + + +public enum ClientStrategy { + TOKEN("token"), + STATIC("static"); + + private final String code; + + ClientStrategy(String code) { + this.code = code; + } + + public String getCode() { + return code; + } +} diff --git a/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/conf/DataWorkspaceRemoteConfig.java b/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/conf/DataWorkspaceRemoteConfig.java new file mode 100644 index 0000000000..9eba05e76e --- /dev/null +++ b/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/conf/DataWorkspaceRemoteConfig.java @@ -0,0 +1,21 @@ +package com.webank.wedatasphere.dss.data.governance.conf; + + +import org.apache.linkis.common.conf.CommonVars; + +public class DataWorkspaceRemoteConfig { + public static final CommonVars SERVER_URL = CommonVars.apply("wds.workspace.client.serverurl", ""); + public static final CommonVars CONNECTION_TIMEOUT = CommonVars.apply("wds.workspace.client.connection.timeout", 30000L); + public static final CommonVars DISCOVERY_ENABLED = CommonVars.apply("wds.workspace.client.discovery.enabled", false); + public static final CommonVars DISCOVERY_FREQUENCY_PERIOD = CommonVars.apply("wds.workspace.client.discoveryfrequency.period", 1L); + public static final CommonVars LOAD_BALANCER_ENABLED = CommonVars.apply("wds.workspace.client.loadbalancer.enabled", true); + public static final CommonVars MAX_CONNECTION_SIZE = CommonVars.apply("wds.workspace.client.maxconnection.size", 5); + public static final CommonVars RETRY_ENABLED = CommonVars.apply("wds.workspace.client.retryenabled", false); + public static final CommonVars READ_TIMEOUT = CommonVars.apply("wds.workspace.client.readtimeout", 30000L); + public static final CommonVars AUTHENTICATION_STRATEGY = CommonVars.apply("wds.workspace.client.authenticationStrategy", ""); + + public static final CommonVars AUTHTOKEN_KEY = CommonVars.apply("wds.workspace.client.authtoken.key", ""); + public static final CommonVars AUTHTOKEN_VALUE = CommonVars.apply("wds.workspace.client.authtoken.value", ""); + public static final CommonVars DWS_VERSION = CommonVars.apply("wds.workspace.client.dws.version", ""); + +} diff --git a/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/conf/GovernanceConf.java b/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/conf/GovernanceConf.java new file mode 100644 index 0000000000..9769d05482 --- /dev/null +++ b/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/conf/GovernanceConf.java @@ -0,0 +1,44 @@ +/* + * + * * Copyright 2019 WeBank + * * + * * Licensed under the Apache License, Version 2.0 (the "License"); + * * you may not use this file except in compliance with the License. + * * You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.data.governance.conf; + +import org.apache.linkis.common.conf.CommonVars; + + +public interface GovernanceConf { + CommonVars ATLAS_REST_ADDRESS = CommonVars.apply("atlas.rest.address"); + CommonVars ATLAS_USERNAME = CommonVars.apply("atlas.username"); + CommonVars ATLAS_PASSWORD = CommonVars.apply("atlas.password"); + //CommonVars METASTORE_DATASOURCE_PASSWORD = CommonVars.apply("atlas.client.readTimeoutMSecs"); + //CommonVars METASTORE_DATASOURCE_PASSWORD = CommonVars.apply("atlas.client.connectTimeoutMSecs"); + CommonVarsATLAS_ROOT_INDICATOR = CommonVars.apply("atlas.root.indicator"); + CommonVarsATLAS_ROOT_MEASURE = CommonVars.apply("atlas.root.measure"); + CommonVarsATLAS_ROOT_DIMENSION = CommonVars.apply("atlas.root.dimension"); + CommonVarsATLAS_ROOT_LAYER = CommonVars.apply("atlas.root.layer"); + CommonVarsATLAS_ROOT_THEME = CommonVars.apply("atlas.root.theme"); + + CommonVarsATLAS_ROOT_LABEL = CommonVars.apply("atlas.root.label"); + CommonVarsATLAS_ROOT_COLLECTION = CommonVars.apply("atlas.root.collection"); + + + CommonVars METASTORE_DATASOURCE_DRIVER = CommonVars.apply("metastore.datasource.driver", "com.mysql.jdbc.Driver"); + CommonVars METASTORE_DATASOURCE_URL = CommonVars.apply("metastore.datasource.url"); + CommonVars METASTORE_DATASOURCE_USERNAME = CommonVars.apply("metastore.datasource.username"); + CommonVars METASTORE_DATASOURCE_PASSWORD = CommonVars.apply("metastore.datasource.password"); +} diff --git a/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/dao/MetaInfoMapper.java b/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/dao/MetaInfoMapper.java new file mode 100644 index 0000000000..9641cef8eb --- /dev/null +++ b/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/dao/MetaInfoMapper.java @@ -0,0 +1,19 @@ +package com.webank.wedatasphere.dss.data.governance.dao; + +import com.webank.wedatasphere.dss.data.governance.entity.PartInfo; +import com.webank.wedatasphere.dss.data.governance.entity.TableInfo; +import com.webank.wedatasphere.dss.data.governance.exception.DAOException; +import org.apache.ibatis.annotations.Mapper; +import org.apache.ibatis.annotations.Param; + +import java.sql.SQLException; +import java.util.List; + + +@Mapper +public interface MetaInfoMapper { + Long getTableStorage() throws DAOException; + List getTop10Table() throws DAOException; + int getTableInfo(@Param("dbName") String dbName,@Param("tableName") String tableName,@Param("isPartTable") Boolean isPartTable) throws DAOException; + List getPartInfo(@Param("dbName") String dbName, @Param("tableName") String tableName) throws DAOException; +} diff --git a/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/dao/TableColumnCountQueryMapper.java b/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/dao/TableColumnCountQueryMapper.java new file mode 100644 index 0000000000..c96bad9a6a --- /dev/null +++ b/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/dao/TableColumnCountQueryMapper.java @@ -0,0 +1,28 @@ +package com.webank.wedatasphere.dss.data.governance.dao; + + +import com.baomidou.mybatisplus.core.conditions.Wrapper; +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import com.baomidou.mybatisplus.core.toolkit.Constants; +import com.webank.wedatasphere.dss.data.governance.entity.TableColumnCount; +import org.apache.ibatis.annotations.Param; +import org.apache.ibatis.annotations.Select; + +import java.util.List; + +public interface TableColumnCountQueryMapper extends BaseMapper { + + String querySql = "select tbl_db.db_name as db_name,tbl_db.tbl_name as tbl_name,tbl_db.tbl_id as tbl_id , count(*) as column_count \n" + + "from COLUMNS_V2 as col\n" + + " left join CDS C on C.CD_ID = col.CD_ID\n" + + " right join (select t.TBL_NAME as tbl_name,t.TBL_ID as tbl_id, s.CD_ID as cd_id ,d.DB_ID as db_id ,d.NAME as db_name from TBLS as t left join SDS as s on t.SD_ID = s.SD_ID left join DBS d on t.DB_ID = d.DB_ID) as tbl_db\n" + + " on tbl_db.cd_id = col.CD_ID\n" + + " ${ew.customSqlSegment} " + + " group by tbl_db.tbl_id "; + + String wrapperSql = "SELECT * from ( " + querySql + " ) AS q ${ew.customSqlSegment}"; + + + @Select(querySql) + List query(@Param(Constants.WRAPPER) Wrapper queryWrapper); +} diff --git a/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/dao/TableSizeInfoMapper.java b/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/dao/TableSizeInfoMapper.java new file mode 100644 index 0000000000..2d0d526059 --- /dev/null +++ b/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/dao/TableSizeInfoMapper.java @@ -0,0 +1,22 @@ +package com.webank.wedatasphere.dss.data.governance.dao; + + +import com.baomidou.mybatisplus.core.conditions.Wrapper; +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import com.baomidou.mybatisplus.core.toolkit.Constants; +import com.webank.wedatasphere.dss.data.governance.entity.TableSizeInfo; +import org.apache.ibatis.annotations.Param; +import org.apache.ibatis.annotations.Select; + +import java.util.List; + +public interface TableSizeInfoMapper extends BaseMapper { + String querySql = "select tbls.TBL_ID as id,d.NAME as db_name,tbls.TBL_NAME as tbl_name, tp.PARAM_KEY as param_key, tp.PARAM_VALUE as param_value\n" + + "from TBLS as tbls left join TABLE_PARAMS as tp on tbls.TBL_ID = tp.TBL_ID left join DBS d on tbls.DB_ID = d.DB_ID \n";; + + String wrapperSql = "SELECT * from ( " + querySql + " ) AS q ${ew.customSqlSegment}"; + + + @Select(wrapperSql) + List query(@Param(Constants.WRAPPER) Wrapper queryWrapper); +} diff --git a/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/dao/TableSizePartitionInfoMapper.java b/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/dao/TableSizePartitionInfoMapper.java new file mode 100644 index 0000000000..8c00d70765 --- /dev/null +++ b/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/dao/TableSizePartitionInfoMapper.java @@ -0,0 +1,24 @@ +package com.webank.wedatasphere.dss.data.governance.dao; + + +import com.baomidou.mybatisplus.core.conditions.Wrapper; +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import com.baomidou.mybatisplus.core.toolkit.Constants; +import com.webank.wedatasphere.dss.data.governance.entity.TablePartitionSizeInfo; +import com.webank.wedatasphere.dss.data.governance.entity.TableSizeInfo; +import org.apache.ibatis.annotations.Param; +import org.apache.ibatis.annotations.Select; + +import java.util.List; + +public interface TableSizePartitionInfoMapper extends BaseMapper { + + String querySql = "select tbl_db.TBL_ID as id, tbl_db.tbl_name as tbl_name, tbl_db.db_name as db_name, p.PART_ID as part_id,p.LAST_ACCESS_TIME as last_access_time,p.SD_ID as sd_id,pp.PARAM_KEY as param_key,pp.PARAM_VALUE as param_value from PARTITIONS AS p left join PARTITION_PARAMS pp on pp.PART_ID = p.PART_ID left join\n" + + "(select tbls.TBL_ID as tbl_id, tbls.TBL_NAME as tbl_name, db.NAME as db_name from TBLS as tbls, DBS as db where tbls.DB_ID = db.DB_ID ) as tbl_db on tbl_db.tbl_id = p.TBL_ID\n"; + + String wrapperSql = "SELECT * from ( " + querySql + " ) AS q ${ew.customSqlSegment}"; + + + @Select(wrapperSql) + List query(@Param(Constants.WRAPPER) Wrapper queryWrapper); +} diff --git a/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/dao/WorkspaceInfoMapper.java b/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/dao/WorkspaceInfoMapper.java new file mode 100644 index 0000000000..33c9c97ff5 --- /dev/null +++ b/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/dao/WorkspaceInfoMapper.java @@ -0,0 +1,18 @@ +package com.webank.wedatasphere.dss.data.governance.dao; + +import org.apache.ibatis.annotations.Mapper; +import org.apache.ibatis.annotations.Param; +import org.apache.ibatis.annotations.Select; + +import java.util.List; + + +@Mapper +public interface WorkspaceInfoMapper { + + @Select("select username from dss_workspace_user where workspace_id= #{workspaceId} and username like #{search} ") + List getWorkspaceUsersName (@Param("workspaceId") int workspaceId,@Param("search") String search); + + + +} diff --git a/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/dao/impl/MetaInfoMapperImpl.java b/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/dao/impl/MetaInfoMapperImpl.java new file mode 100644 index 0000000000..6ec20f2b07 --- /dev/null +++ b/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/dao/impl/MetaInfoMapperImpl.java @@ -0,0 +1,140 @@ +package com.webank.wedatasphere.dss.data.governance.dao.impl; + +import com.webank.wedatasphere.dss.data.governance.dao.MetaInfoMapper; +import com.webank.wedatasphere.dss.data.governance.entity.PartInfo; +import com.webank.wedatasphere.dss.data.governance.entity.TableInfo; +import com.webank.wedatasphere.dss.data.governance.exception.DAOException; +import com.webank.wedatasphere.dss.data.governance.utils.DataSourceUtil; +import com.webank.wedatasphere.dss.data.governance.utils.DateUtil; + +import javax.sql.DataSource; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; +import java.util.stream.Collectors; + + +public class MetaInfoMapperImpl implements MetaInfoMapper { + @Override + public Long getTableStorage() throws DAOException { + DataSource dataSource = DataSourceUtil.getDataSource(); + + long num = 0; + PreparedStatement ps = null; + ResultSet rs = null; + try (Connection con = dataSource.getConnection()) { + String sql = "select SUM(PARAM_VALUE) from TABLE_PARAMS WHERE PARAM_KEY='totalSize'"; + ps = con.prepareStatement(sql); + rs = ps.executeQuery(); + while (rs.next()) { + num = rs.getLong(1); + } + String sql2 = "select SUM(PARAM_VALUE) from PARTITION_PARAMS WHERE PARAM_KEY='totalSize'"; + ps = con.prepareStatement(sql2); + rs = ps.executeQuery(); + while (rs.next()) { + num = num + rs.getLong(1); + } + } catch (SQLException e) { + throw new DAOException(23001, e.getMessage()); + } + return num; + } + + @Override + public List getTop10Table() throws DAOException { + DataSource dataSource = DataSourceUtil.getDataSource(); + + PreparedStatement ps = null; + ResultSet rs = null; + List tableInfos = new ArrayList<>(); + try (Connection con = dataSource.getConnection();) { + String sql = "select DBS.NAME ,TBLS.TBL_NAME,TABLE_PARAMS.PARAM_VALUE as totalSize from DBS, TBLS,TABLE_PARAMS where TBLS.TBL_ID=TABLE_PARAMS.TBL_ID AND TBLS.DB_ID=DBS.DB_ID AND TABLE_PARAMS.PARAM_KEY='totalSize' order by totalSize DESC limit 10"; + ps = con.prepareStatement(sql); + rs = ps.executeQuery(); + while (rs.next()) { + TableInfo tableinfo = new TableInfo(); + tableinfo.setTableName(rs.getString(1) + "." + rs.getString(2)); + tableinfo.setStorage(rs.getString(3)); + tableInfos.add(tableinfo); + } + String sql2 = "select DBS.NAME ,TBLS.TBL_NAME,SUM(PARTITION_PARAMS.PARAM_VALUE) as totalSize from DBS,TBLS,PARTITIONS ,PARTITION_PARAMS where DBS.DB_ID=TBLS.DB_ID AND TBLS.TBL_ID=PARTITIONS.TBL_ID AND PARTITIONS.PART_ID =PARTITION_PARAMS.PART_ID AND PARTITION_PARAMS.PARAM_KEY='totalSize' group by TBLS.TBL_NAME order by totalSize desc limit 10"; + ps = con.prepareStatement(sql2); + rs = ps.executeQuery(); + while (rs.next()) { + TableInfo tableinfo = new TableInfo(); + tableinfo.setTableName(rs.getString(1) + "." + rs.getString(2)); + tableinfo.setStorage(rs.getString(3)); + tableInfos.add(tableinfo); + } + + tableInfos.sort((o1, o2) -> { + long result = Long.parseLong(o2.getStorage()) - Long.parseLong(o1.getStorage()); + return Long.compare(result, 0L); + }); + } catch (SQLException e) { + throw new DAOException(23001, e.getMessage()); + } + + return tableInfos.stream().limit(10).collect(Collectors.toList()); + } + + @Override + public int getTableInfo(String dbName, String tableName, Boolean isPartTable) throws DAOException { + DataSource dataSource = DataSourceUtil.getDataSource(); + PreparedStatement ps = null; + ResultSet rs = null; + int res = 0; + try (Connection con = dataSource.getConnection()) { + String sql = null; + if (isPartTable == false) { + sql = "select TABLE_PARAMS.PARAM_VALUE as totalSize from DBS, TBLS,TABLE_PARAMS where TBLS.TBL_ID=TABLE_PARAMS.TBL_ID AND TBLS.DB_ID=DBS.DB_ID AND TABLE_PARAMS.PARAM_KEY='totalSize' AND DBS.NAME=" + "'" + dbName + "' AND TBLS.TBL_NAME=" + "'" + tableName + "'"; + } else { + + sql = "select SUM(PARTITION_PARAMS.PARAM_VALUE) as totalSize from DBS,TBLS,PARTITIONS ,PARTITION_PARAMS where DBS.DB_ID=TBLS.DB_ID AND TBLS.TBL_ID=PARTITIONS.TBL_ID AND PARTITIONS.PART_ID =PARTITION_PARAMS.PART_ID AND PARTITION_PARAMS.PARAM_KEY='totalSize' AND DBS.NAME=" + "'" + dbName + "' AND TBLS.TBL_NAME=" + "'" + tableName + "' group by TBLS.TBL_NAME"; + } + ps = con.prepareStatement(sql); + rs = ps.executeQuery(); + while (rs.next()) { + res = rs.getInt(1); + } + } catch (SQLException e) { + throw new DAOException(23001, e.getMessage()); + } + return res; + } + + @Override + public List getPartInfo(String dbName, String tableName) throws DAOException { + DataSource dataSource = DataSourceUtil.getDataSource(); + + PreparedStatement ps = null; + ResultSet rs = null; + List PartInfos = new ArrayList<>(); + try (Connection con = dataSource.getConnection()) { + String sql = "select b.PART_NAME,b.CREATE_TIME,MAX(CASE c.PARAM_KEY WHEN 'transient_lastDdlTime' THEN c.PARAM_VALUE ELSE null END) transient_lastDdlTime ,MAX(CASE c.PARAM_KEY WHEN 'numRows' THEN c.PARAM_VALUE ELSE null END) numRows,MAX(CASE c.PARAM_KEY WHEN 'totalSize' THEN c.PARAM_VALUE ELSE null END) totalSize, MAX(CASE c.PARAM_KEY WHEN 'numFiles' THEN c.PARAM_VALUE ELSE null END) numFiles from TBLS a,PARTITIONS b,PARTITION_PARAMS c,DBS d where a.TBL_NAME=" + "'" + tableName + "'" + "AND d.NAME=" + "'" + dbName + "'" + "AND a.TBL_ID=b.TBL_ID AND a.DB_ID=d.DB_ID AND b.PART_ID=c.PART_ID GROUP BY c.PART_ID"; + ps = con.prepareStatement(sql); + rs = ps.executeQuery(); + while (rs.next()) { + PartInfo part = new PartInfo(); + part.setPartName(rs.getString(1)); + part.setCreateTime(DateUtil.unixToTimeStr(Long.valueOf(rs.getInt(2)) * 1000)); + part.setLastAccessTime(DateUtil.unixToTimeStr(Long.valueOf(rs.getInt(3)) * 1000)); + part.setReordCnt(rs.getInt(4)); + part.setStore(rs.getInt(5)); + part.setFileCount(rs.getInt(6)); + PartInfos.add(part); + } + + } catch (SQLException e) { + throw new DAOException(23001, e.getMessage()); + } + + return PartInfos; + } +} diff --git a/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/dto/HiveTblStatsDTO.java b/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/dto/HiveTblStatsDTO.java new file mode 100644 index 0000000000..56f4a2abe8 --- /dev/null +++ b/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/dto/HiveTblStatsDTO.java @@ -0,0 +1,18 @@ +package com.webank.wedatasphere.dss.data.governance.dto; + + +import lombok.Data; + +@Data +public class HiveTblStatsDTO { + + private Integer columnCount = 0; + + private Long totalSize = 0L; + + private Integer numFiles = 0; + + private Integer partitionCount = 0; + + private Integer accessCount = 0; +} diff --git a/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/dto/SearchLabelDTO.java b/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/dto/SearchLabelDTO.java new file mode 100644 index 0000000000..e7cfdc6cf4 --- /dev/null +++ b/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/dto/SearchLabelDTO.java @@ -0,0 +1,18 @@ +package com.webank.wedatasphere.dss.data.governance.dto; + + +import lombok.Data; +import org.apache.atlas.model.instance.AtlasEntityHeader; + +@Data +public class SearchLabelDTO { + private String name; + private String guid; + + public static SearchLabelDTO from(AtlasEntityHeader atlasEntityHeader){ + SearchLabelDTO dto = new SearchLabelDTO(); + dto.setGuid(atlasEntityHeader.getGuid()); + dto.setName(atlasEntityHeader.getAttribute("name").toString()); + return dto; + } +} diff --git a/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/entity/ClassificationConstant.java b/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/entity/ClassificationConstant.java new file mode 100644 index 0000000000..9fdb2d5ff9 --- /dev/null +++ b/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/entity/ClassificationConstant.java @@ -0,0 +1,116 @@ +package com.webank.wedatasphere.dss.data.governance.entity; + + +import com.webank.wedatasphere.dss.data.governance.conf.GovernanceConf; +import org.apache.commons.lang.StringUtils; + +import java.util.Optional; + +public enum ClassificationConstant { + /** + * 指标 + */ + INDICATOR(1, "indicator_", GovernanceConf.ATLAS_ROOT_INDICATOR.getValue(),"indicator"), + /** + * 度量 + */ + MEASURE(2, "measure_", GovernanceConf.ATLAS_ROOT_MEASURE.getValue(),"measure"), + /** + * 维度 + */ + DIMENSION(0, "dimension_", GovernanceConf.ATLAS_ROOT_DIMENSION.getValue(),"dimension"), + /** + * 主题 + */ + THEME(3, "theme_", GovernanceConf.ATLAS_ROOT_THEME.getValue(),"theme"), + /** + * 分层 + */ + LAYER(4, "layer_", GovernanceConf.ATLAS_ROOT_LAYER.getValue(),"layer"); + + private int type; + + private String prefix; + + private String root; + + private String typeCode; + + ClassificationConstant(int type, String prefix, String root,String typeCode) { + this.type = type; + this.prefix = prefix; + this.root = root; + this.typeCode = typeCode; + } + + public static boolean isTypeScope(int type) { + return type >= 0 && type < values().length; + } + + public static boolean isTypeScope(String type) { + return getClassificationConstantByTypeCode(type).isPresent(); + } + + public static Optional getClassificationConstantByTypeCode(String typeCode) { + for (ClassificationConstant c : values()) { + if (StringUtils.equals(c.typeCode,typeCode)) { + return Optional.of(c); + } + } + return Optional.empty(); + } + + public static Optional getClassificationConstantByType(int type) { + for (ClassificationConstant c : values()) { + if (c.type == type) { + return Optional.of(c); + } + } + return Optional.empty(); + } + + public static Optional formatName(int type, String name){ + return Optional.of(getPrefix(type).get() + name); + } + + public static Optional formatName(String typeCode, String name){ + return Optional.of(getPrefix(typeCode).get() + name); + } + + public static Optional getRoot(int type) { + Optional optional = getClassificationConstantByType(type); + return optional.map(ClassificationConstant::getRoot); + } + + public static Optional getRoot(String typeCode) { + Optional optional = getClassificationConstantByTypeCode(typeCode); + return optional.map(ClassificationConstant::getRoot); + } + + public static Optional getPrefix(int type) { + Optional optional = getClassificationConstantByType(type); + return optional.map(ClassificationConstant::getPrefix); + } + + public static Optional getPrefix(String typeCode) { + Optional optional = getClassificationConstantByTypeCode(typeCode); + return optional.map(ClassificationConstant::getPrefix); + } + + + public int getType() { + return type; + } + + public String getPrefix() { + return prefix; + } + + public String getRoot() { + return root; + } + + public String getTypeCode() { + return typeCode; + } +} diff --git a/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/entity/CreateLabelInfo.java b/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/entity/CreateLabelInfo.java new file mode 100644 index 0000000000..e8aaca5adc --- /dev/null +++ b/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/entity/CreateLabelInfo.java @@ -0,0 +1,19 @@ +package com.webank.wedatasphere.dss.data.governance.entity; + + +import lombok.Data; +import org.apache.atlas.model.glossary.AtlasGlossaryTerm; + +@Data +public class CreateLabelInfo { + private String name; + + private String guid; + + public static CreateLabelInfo from(AtlasGlossaryTerm term) { + CreateLabelInfo info = new CreateLabelInfo(); + info.setName(term.getName()); + info.setGuid(term.getGuid()); + return info; + } +} diff --git a/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/entity/CreateModelTypeInfo.java b/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/entity/CreateModelTypeInfo.java new file mode 100644 index 0000000000..da26e0000c --- /dev/null +++ b/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/entity/CreateModelTypeInfo.java @@ -0,0 +1,12 @@ +package com.webank.wedatasphere.dss.data.governance.entity; + +import lombok.Data; +import org.apache.atlas.model.glossary.AtlasGlossaryTerm; + +@Data +public class CreateModelTypeInfo { + + private String name; + + private String guid; +} diff --git a/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/entity/GlossaryConstant.java b/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/entity/GlossaryConstant.java new file mode 100644 index 0000000000..280e744fd9 --- /dev/null +++ b/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/entity/GlossaryConstant.java @@ -0,0 +1,55 @@ +package com.webank.wedatasphere.dss.data.governance.entity; + + +import com.webank.wedatasphere.dss.data.governance.conf.GovernanceConf; + +public enum GlossaryConstant { + LABEL(0,"label","AtlasGlossaryTerm", GovernanceConf.ATLAS_ROOT_LABEL.getValue()), + COLLECTION(1,"collection","AtlasGlossaryTerm",GovernanceConf.ATLAS_ROOT_COLLECTION.getValue()); + + private int type; + + + private String typeCode; + + private String atlasType; + + private String root; + + public static final String SEPARATOR = "@"; + + + + public static final String ARR = "qualifiedName"; + + GlossaryConstant(int type, String typeCode,String atlasType, String root) { + this.type = type; + this.atlasType = atlasType; + this.typeCode = typeCode; + this.root = root; + } + + public int getType() { + return type; + } + + public String getTypeCode() { + return typeCode; + } + + public String getAtlasType() { + return atlasType; + } + + public String getRoot() { + return root; + } + + public String formatQuery(String query){ + return query+SEPARATOR+this.getRoot(); + } + + public String endWith(){ + return SEPARATOR+this.getRoot(); + } +} diff --git a/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/entity/HiveTblDetailInfo.java b/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/entity/HiveTblDetailInfo.java new file mode 100644 index 0000000000..97cae23c02 --- /dev/null +++ b/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/entity/HiveTblDetailInfo.java @@ -0,0 +1,33 @@ +package com.webank.wedatasphere.dss.data.governance.entity; + +import lombok.Data; + +import java.io.Serializable; +import java.util.List; +import java.util.Set; + + +@Data +public class HiveTblDetailInfo implements Serializable { + private HiveTblBasicInfo basic; + private List columns; + private List partitionKeys; + + + @Data + public static class HiveTblBasicInfo extends HiveTblSimpleInfo { + private String store; + private String comment; + private Set labels; + private Boolean isParTbl; + } + + @Data + public static class HiveColumnInfo { + private String name; + private String type; + private String guid; + private String comment; + } + +} diff --git a/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/entity/HiveTblSimpleInfo.java b/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/entity/HiveTblSimpleInfo.java new file mode 100644 index 0000000000..0a6d7afd62 --- /dev/null +++ b/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/entity/HiveTblSimpleInfo.java @@ -0,0 +1,21 @@ +package com.webank.wedatasphere.dss.data.governance.entity; + +import lombok.Data; + +import java.util.List; + + +@Data +public class HiveTblSimpleInfo { + private String guid; + private String name; + private String qualifiedName; + private String createTime; + private String owner; + private String aliases; + private String lastAccessTime; + private String comment; + private List classifications; + private String totalSize; + private String external; +} diff --git a/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/entity/PartInfo.java b/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/entity/PartInfo.java new file mode 100644 index 0000000000..1a5d85685a --- /dev/null +++ b/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/entity/PartInfo.java @@ -0,0 +1,14 @@ +package com.webank.wedatasphere.dss.data.governance.entity; + +import lombok.Data; + + +@Data +public class PartInfo { + private String partName; + private int reordCnt; + private int store; + private String createTime; + private String lastAccessTime; + private int fileCount; +} \ No newline at end of file diff --git a/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/entity/QueryType.java b/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/entity/QueryType.java new file mode 100644 index 0000000000..f54a0e6cd5 --- /dev/null +++ b/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/entity/QueryType.java @@ -0,0 +1,18 @@ +package com.webank.wedatasphere.dss.data.governance.entity; + + +public enum QueryType { + PRECISE(1), + FUZZY(0); + private int code; + + QueryType(int code) { + this.code = code; + } + + public int getCode() { + return code; + } + + +} diff --git a/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/entity/RelatedObjectId.java b/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/entity/RelatedObjectId.java new file mode 100644 index 0000000000..53437b59d8 --- /dev/null +++ b/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/entity/RelatedObjectId.java @@ -0,0 +1,17 @@ +package com.webank.wedatasphere.dss.data.governance.entity; + + +import lombok.Data; + +@Data +public class RelatedObjectId { + private String guid; + public String relationshipGuid; + + public static RelatedObjectId from(String guid,String relationshipGuid){ + RelatedObjectId relatedObjectId = new RelatedObjectId(); + relatedObjectId.setGuid(guid); + relatedObjectId.setRelationshipGuid(relationshipGuid); + return relatedObjectId; + } +} diff --git a/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/entity/TableColumnCount.java b/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/entity/TableColumnCount.java new file mode 100644 index 0000000000..e5b20bf934 --- /dev/null +++ b/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/entity/TableColumnCount.java @@ -0,0 +1,12 @@ +package com.webank.wedatasphere.dss.data.governance.entity; + + +import lombok.Data; + +@Data +public class TableColumnCount { + private String dbName; + private String tblName; + private Long tblId; + private Integer columnCount; +} diff --git a/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/entity/TableInfo.java b/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/entity/TableInfo.java new file mode 100644 index 0000000000..8c971dc158 --- /dev/null +++ b/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/entity/TableInfo.java @@ -0,0 +1,10 @@ +package com.webank.wedatasphere.dss.data.governance.entity; + +import lombok.Data; + + +@Data +public class TableInfo { + private String tableName; + private String storage; +} \ No newline at end of file diff --git a/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/entity/TablePartitionSizeInfo.java b/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/entity/TablePartitionSizeInfo.java new file mode 100644 index 0000000000..fb5cb34414 --- /dev/null +++ b/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/entity/TablePartitionSizeInfo.java @@ -0,0 +1,15 @@ +package com.webank.wedatasphere.dss.data.governance.entity; + +import lombok.Data; + +@Data +public class TablePartitionSizeInfo { + private Long id; + private Long parId; + private Long lastAccessTime; + private Long sdId; + private String dbName; + private String tblName; + private String paramKey; + private String paramValue; +} diff --git a/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/entity/TableSizeInfo.java b/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/entity/TableSizeInfo.java new file mode 100644 index 0000000000..8355cdc54d --- /dev/null +++ b/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/entity/TableSizeInfo.java @@ -0,0 +1,13 @@ +package com.webank.wedatasphere.dss.data.governance.entity; + + +import lombok.Data; + +@Data +public class TableSizeInfo { + private Long id; + private String dbName; + private String tblName; + private String paramKey; + private String paramValue; +} diff --git a/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/entity/UpdateLabelInfo.java b/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/entity/UpdateLabelInfo.java new file mode 100644 index 0000000000..8c302549de --- /dev/null +++ b/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/entity/UpdateLabelInfo.java @@ -0,0 +1,18 @@ +package com.webank.wedatasphere.dss.data.governance.entity; + +import lombok.Data; +import org.apache.atlas.model.glossary.AtlasGlossaryTerm; + +@Data +public class UpdateLabelInfo { + private String name; + + private String guid; + + public static UpdateLabelInfo from(AtlasGlossaryTerm term) { + UpdateLabelInfo info = new UpdateLabelInfo(); + info.setName(term.getName()); + info.setGuid(term.getGuid()); + return info; + } +} diff --git a/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/entity/UpdateModelTypeInfo.java b/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/entity/UpdateModelTypeInfo.java new file mode 100644 index 0000000000..8fc0ec1a0c --- /dev/null +++ b/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/entity/UpdateModelTypeInfo.java @@ -0,0 +1,12 @@ +package com.webank.wedatasphere.dss.data.governance.entity; + + +import lombok.Data; + +@Data +public class UpdateModelTypeInfo { + private String name; + + private String guid; + +} diff --git a/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/exception/DAOException.java b/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/exception/DAOException.java new file mode 100644 index 0000000000..9a5e54fe07 --- /dev/null +++ b/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/exception/DAOException.java @@ -0,0 +1,15 @@ +package com.webank.wedatasphere.dss.data.governance.exception; + + +import org.apache.linkis.common.exception.ErrorException; + +public class DAOException extends ErrorException { + + public DAOException(int errCode, String desc) { + super(errCode, desc); + } + + public DAOException(int errCode, String desc, String ip, int port, String serviceKind) { + super(errCode, desc, ip, port, serviceKind); + } +} diff --git a/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/exception/DataGovernanceException.java b/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/exception/DataGovernanceException.java new file mode 100644 index 0000000000..b36fddd91d --- /dev/null +++ b/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/exception/DataGovernanceException.java @@ -0,0 +1,16 @@ +package com.webank.wedatasphere.dss.data.governance.exception; + + +import org.apache.linkis.common.exception.ErrorException; + +public class DataGovernanceException extends ErrorException { + + public DataGovernanceException(int errCode, String desc) { + super(errCode, desc); + } + + public DataGovernanceException(int errCode, String desc, String ip, int port, String serviceKind) { + super(errCode, desc, ip, port, serviceKind); + } + +} diff --git a/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/restful/DSSDataGovernanceAssetRestful.java b/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/restful/DSSDataGovernanceAssetRestful.java new file mode 100644 index 0000000000..18c022b646 --- /dev/null +++ b/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/restful/DSSDataGovernanceAssetRestful.java @@ -0,0 +1,496 @@ +package com.webank.wedatasphere.dss.data.governance.restful; + +import com.webank.wedatasphere.dss.data.governance.entity.*; +import com.webank.wedatasphere.dss.data.governance.service.AssetService; +import com.webank.wedatasphere.dss.data.governance.service.AuthenticationClientStrategy; +import com.webank.wedatasphere.dss.data.governance.service.WorkspaceInfoService; +import com.webank.wedatasphere.dss.data.governance.vo.*; +import com.webank.wedatasphere.dss.framework.workspace.client.impl.LinkisWorkSpaceRemoteClient; +import com.webank.wedatasphere.dss.framework.workspace.client.request.GetWorkspaceUsersAction; +import com.webank.wedatasphere.dss.framework.workspace.client.response.GetWorkspaceUsersResult; +import org.apache.linkis.common.exception.ErrorException; +import org.apache.linkis.server.Message; +import org.apache.atlas.model.lineage.AtlasLineageInfo; +import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; +import org.springframework.util.CollectionUtils; +import org.springframework.web.bind.annotation.RequestBody; + +import javax.annotation.Resource; +import javax.servlet.http.HttpServletRequest; +import javax.ws.rs.*; +import javax.ws.rs.core.Context; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Set; + + +@Path("/data-assets/asset") +@Consumes(MediaType.APPLICATION_JSON) +@Produces(MediaType.APPLICATION_JSON) +@Component +public class DSSDataGovernanceAssetRestful implements AuthenticationClientStrategy { + private static final Logger logger = LoggerFactory.getLogger(DSSDataGovernanceAssetRestful.class); + + private static final String DEFAULT_DIRECTION = "BOTH"; + private static final String DEFAULT_DEPTH = "3"; + private static final String DEFAULT_LIMIT = "25"; + private static final String DEFAULT_OFFSET = "0"; + + @Autowired + private AssetService assetService; + @Autowired + private WorkspaceInfoService workspaceInfoService; + + @Resource + private LinkisWorkSpaceRemoteClient linkisWorkSpaceRemoteClient; + + /** + * 获取数据资产概要：hivedb数、hivetable数据、总存储量 + */ + @GET + @Path("/hiveSummary") + public Response getHiveSummary(@Context HttpServletRequest req) throws Exception { + + return Message.messageToResponse(Message.ok().data("result", assetService.getHiveSummary())); + } + + /** + * 搜索hive表 + */ + @GET + @Path("/hiveTbl/search") + public Response searchHiveTbl(@QueryParam("classification") String classification, + @QueryParam("query") String query, + @QueryParam("label") String label, + @QueryParam("type") String type, + @QueryParam("precise")@DefaultValue("0") int precise, + @QueryParam("owner") @DefaultValue("") String owner, + @QueryParam("limit") @DefaultValue(DEFAULT_LIMIT) int limit, + @QueryParam("offset") @DefaultValue(DEFAULT_OFFSET) int offset) throws Exception { + + //适配模型 + if (ClassificationConstant.isTypeScope(type)) { + if (StringUtils.isNotBlank(classification)) { + classification = ClassificationConstant.getPrefix(type).orElse(null) + classification; + } else { + classification = ClassificationConstant.getRoot(type).orElse(null); + } + } + //适配标签 + if (!StringUtils.isBlank(label)){ + label = GlossaryConstant.LABEL.formatQuery(label); + } + //判断是否精确查询 + if (QueryType.PRECISE.getCode()!=precise){ + query = "*" + query + "*"; + } + List hiveTblBasicList = assetService.searchHiveTable(classification, query,label,limit, offset); + if (StringUtils.isBlank(owner) || owner.equals("undefined")|| CollectionUtils.isEmpty(hiveTblBasicList)) { + return Message.messageToResponse(Message.ok().data("result", hiveTblBasicList)); + } else { + List res = new ArrayList<>(); + for (HiveTblSimpleInfo hiveTblSimpleInfo : hiveTblBasicList) { + if (hiveTblSimpleInfo.getOwner().equals(owner)) { + res.add(hiveTblSimpleInfo); + } + } + return Message.messageToResponse(Message.ok().data("result", res)); + } + } + + /** + * 搜索hive表统计信息 + */ + @GET + @Path("/hiveTbl/stats") + public Response searchHiveTblStats(@QueryParam("dbName") String dbName, + @QueryParam("tableName") String tableName, + @QueryParam("guid") String guid) throws Exception { + logger.info("searchHiveTblStats dbName : {}, tableName : {}, guid : {}", dbName, tableName, guid); + return Message.messageToResponse(Message.ok().data("result", assetService.hiveTblStats(dbName, tableName, guid))); + + } + + /** + * 搜索hive表容量 + */ + @GET + @Path("/hiveTbl/size") + public Response searchHiveTblSize(@QueryParam("dbName") String dbName, + @QueryParam("tableName") String tableName, + @QueryParam("guid") String guid) throws Exception { + logger.info("searchHiveTblSize dbName : {}, tableName : {}, guid : {}", dbName, tableName, guid); + return Message.messageToResponse(Message.ok().data("result", assetService.hiveTblSize(dbName, tableName, guid))); + + } + + + /** + * 搜索hive库 + */ + @GET + @Path("/hiveDb/search") + public Response searchHiveDb(@QueryParam("classification") String classification, + @QueryParam("query") String query, + @QueryParam("owner") @DefaultValue("") String owner, + @QueryParam("limit") @DefaultValue(DEFAULT_LIMIT) int limit, + @QueryParam("offset") @DefaultValue(DEFAULT_OFFSET) int offset) throws Exception { + + List hiveTblBasicList = assetService.searchHiveDb(classification, '*' + query + '*', limit, offset); + if (StringUtils.isBlank(owner) || owner.equals("undefined")) { + return Message.messageToResponse(Message.ok().data("result", hiveTblBasicList)); + } else { + List res = new ArrayList<>(); + for (HiveTblSimpleInfo hiveTblSimpleInfo : hiveTblBasicList) { + if (hiveTblSimpleInfo.getOwner().equals(owner)) { + res.add(hiveTblSimpleInfo); + } + } + return Message.messageToResponse(Message.ok().data("result", res)); + } + } + + /** + * 获取单个表的详细信息，包括：基本信息、字段信息 + */ + @GET + @Path("/hiveTbl/{guid}/basic") + public Response getHiveTblBasic(@PathParam("guid") String guid) throws Exception { + return Message.messageToResponse(Message.ok().data("result", assetService.getHiveTblDetail(guid))); + } + + /** + * 获取表分区信息 + */ + @GET + @Path("/hiveTbl/{guid}/partition") + public Response getHiveTblPartition(@PathParam("guid") String guid) throws Exception { + List hiveTblPartition = assetService.getHiveTblPartition(guid); + if (hiveTblPartition.size() > 0) { + return Message.messageToResponse(Message.ok().data("result", hiveTblPartition)); + } else { + return Message.messageToResponse(Message.ok().data("result", null)); + } + } + + /** + * 根据表名获取表分区信息 + */ + @GET + @Path("/hiveTbl/partition/name") + public Response getHiveTblPartitionByName(@QueryParam("dbName") String dbName,@QueryParam("tableName") String tableName) throws Exception { + logger.info("getHiveTblPartitionByName dbName : {}, tableName : {}", dbName, tableName); + List hiveTblPartition = assetService.getHiveTblPartitionByName(dbName,tableName); + return Message.messageToResponse(Message.ok().data("result", hiveTblPartition)); + } + + /** + * 获取表的血缘信息 + */ + @GET + @Path("/hiveTbl/{guid}/lineage") + public Response getHiveTblLineage(@PathParam("guid") String guid, + @QueryParam("direction") @DefaultValue(DEFAULT_DIRECTION) AtlasLineageInfo.LineageDirection direction, + @QueryParam("depth") @DefaultValue(DEFAULT_DEPTH) int depth) throws Exception { + return Message.messageToResponse(Message.ok().data("result", assetService.getHiveTblLineage(guid, direction, depth))); + } + + /** + * 获取表的select语句 + */ + @GET + @Path("/hiveTbl/{guid}/select") + public Response getHiveTblSelect(@PathParam("guid") String guid) throws Exception { + + + return Message.messageToResponse(Message.ok().data("result", assetService.getTbSelect(guid))); + + } + + /** + * 获取表的create语句 + */ + @GET + @Path("/hiveTbl/{guid}/create") + public Response getHiveTblCreate(@PathParam("guid") String guid) throws Exception { + return Message.messageToResponse(Message.ok().data("result", assetService.getTbCreate(guid))); + + } + + /** + * 获取存储量前10的表信息 + */ + @GET + @Path("/hiveTbl/topStorage") + public Response getTop10Storage(@Context HttpServletRequest req) throws Exception { + List top10Table = assetService.getTop10Table(); + return Message.messageToResponse(Message.ok().data("result", top10Table)); + } + + /** + * 修改单个表或单个列注释 + */ + @PUT + @Path("/comment/{guid}") + public Response modifyComment(@PathParam("guid") String guid, @QueryParam("comment") String comment) throws Exception { + comment = "\"" + comment + "\""; + assetService.modifyComment(guid, comment); + return Message.messageToResponse(Message.ok().data("result", "修改成功")); + } + + /** + * 批量修改多个个表或列注释 + */ + @PUT + @Path("/comment/bulk") + public Response modifyComment(@RequestBody Map commentMap) throws Exception { + for (Map.Entry stringStringEntry : commentMap.entrySet()) { + stringStringEntry.setValue("\"" + stringStringEntry.getValue() + "\""); + } + assetService.bulkModifyComment(commentMap); + + return Message.messageToResponse(Message.ok().data("result", "修改成功")); + } + + /** + * 创建模型 + * + * @param req + * @param vo + * @return + * @throws Exception + */ + @POST + @Path("/model/type") + public Response createModelType(@Context HttpServletRequest req, @RequestBody CreateModelTypeVO vo) throws Exception { + logger.info("createModelType : {}", vo); + return Message.messageToResponse(Message.ok().data("result", assetService.createModelType(vo))); + } + + /** + * 删除模型 + * + * @param req + * @param vo + * @return + * @throws Exception + */ + @POST + @Path("/model/type/delete") + public Response deleteModelType(@Context HttpServletRequest req, @RequestBody DeleteModelTypeVO vo) throws Exception { + logger.info("deleteModelTypeVO : {}", vo); + assetService.deleteModelType(vo); + return Message.messageToResponse(Message.ok().data("result", "删除成功")); + } + + + /** + * 绑定模型 + * + * @param req + * @param vo + * @return + * @throws Exception + */ + @POST + @Path("/model/bind") + public Response bindModelType(@Context HttpServletRequest req, @RequestBody BindModelVO vo) throws Exception { + logger.info("bindModelVO : {}", vo); + assetService.bindModelType(vo); + return Message.messageToResponse(Message.ok().data("result", "绑定成功")); + } + + /** + * 解绑模型 + * + * @param req + * @param vo + * @return + * @throws Exception + */ + @POST + @Path("/model/unbind") + public Response unBindModelType(@Context HttpServletRequest req, @RequestBody UnBindModelVO vo) throws Exception { + logger.info("unBindModelVO : {}", vo); + assetService.unBindModel(vo); + return Message.messageToResponse(Message.ok().data("result", "解绑成功")); + } + + + /** + * 更新模型 + * + * @param req + * @param vo + * @return + * @throws Exception + */ + @POST + @Path("/model/type/modify") + public Response updateModelType(@Context HttpServletRequest req, @RequestBody UpdateModelTypeVO vo) throws Exception { + logger.info("updateModelTypeVO : {}", vo); + return Message.messageToResponse(Message.ok().data("result", assetService.updateModelType(vo))); + } + + /** + * 创建标签 + * + * @param req + * @param vo + * @return + * @throws Exception + */ + @POST + @Path("/labels") + public Response createLabel(@Context HttpServletRequest req, @RequestBody CreateLabelVO vo) throws Exception { + logger.info("createLabel vo : {}", vo); + return Message.messageToResponse(Message.ok().data("result", assetService.createLabel(vo))); + } + + /** + * 更新标签 + * + * @param req + * @param vo + * @return + * @throws Exception + */ + @POST + @Path("/labels/modify") + public Response updateLabel(@Context HttpServletRequest req, @RequestBody UpdateLabelVO vo) throws Exception { + logger.info("updateLabel vo : {}", vo); + return Message.messageToResponse(Message.ok().data("result", assetService.updateLabel(vo))); + } + + /** + * 删除标签 + * + * @param req + * @param vo + * @return + * @throws Exception + */ + @POST + @Path("/labels/delete") + public Response deleteLabel(@Context HttpServletRequest req, @RequestBody DeleteLabelVO vo) throws Exception { + logger.info("deleteLabel vo : {}", vo); + assetService.deleteLabel(vo); + return Message.messageToResponse(Message.ok().data("result", "删除成功")); + } + + /** + * 实体绑定标签 + * + * @param req + * @param vo + * @return + * @throws Exception + */ + @POST + @Path("/labels/bind") + public Response bindLabel(@Context HttpServletRequest req, @RequestBody BindLabelVO vo) throws Exception { + logger.info("bindLabel vo : {}", vo); + assetService.bindLabel(vo); + return Message.messageToResponse(Message.ok().data("result", "绑定成功")); + } + + + /** + * 实体解绑标签 + * + * @param req + * @param vo + * @return + * @throws Exception + */ + @POST + @Path("/labels/unbind") + public Response unBindLabel(@Context HttpServletRequest req, @RequestBody UnBindLabelVO vo) throws Exception { + logger.info("unBindLabel vo : {}", vo); + assetService.unBindLabel(vo); + return Message.messageToResponse(Message.ok().data("result", "解绑成功")); + } + + + /** + * 搜索标签 + * + * @param req + * @param query + * @return + * @throws Exception + */ + @GET + @Path("/labels/search") + public Response searchLabel(@Context HttpServletRequest req + , @QueryParam("query") String query + , @QueryParam("limit") @DefaultValue(DEFAULT_LIMIT) int limit + , @QueryParam("offset") @DefaultValue(DEFAULT_OFFSET) int offset) throws Exception { + logger.info("searchLabel query : {}", query); + return Message.messageToResponse(Message.ok().data("result",assetService.listLabels(query,limit,offset))); + } + + /** + * 设置单个表或单个列的标签 + */ + @POST + @Path("/label/{guid}") + public Response setLabels(@PathParam("guid") String guid, @RequestBody Set labels) throws Exception { + logger.info("setLabels guid : {}, labels : {}", guid, labels); + assetService.setLabels(guid, labels); + + return Message.messageToResponse(Message.ok().data("result", "设置成功")); + } + + /** + * 获取工作空间下所有用户名 + */ + @GET + @Path("getWorkspaceUsers/{workspaceId}/{search}") + public Response getWorkspaceUsers(@PathParam("workspaceId") int workspaceId, @PathParam("search") String search) throws Exception { + String searchs = "%" + search + "%"; + List workspaceUsers = workspaceInfoService.getWorkspaceUsers(workspaceId, searchs); + return Message.messageToResponse(Message.ok().data("result", workspaceUsers)); + + } + + /** + * 查询用户 + * + * @param req + * @param workspaceId + * @return + */ + @GET + @Path("/users/{workspaceId}") + public Response users(@Context HttpServletRequest req, @PathParam("workspaceId") String workspaceId) throws ErrorException { + logger.info("users workspaceId : {}", workspaceId); + GetWorkspaceUsersResult result = linkisWorkSpaceRemoteClient.getWorkspaceUsers(GetWorkspaceUsersAction.builder().setUser(getStrategyUser(req)).setWorkspaceId(workspaceId).build()); + return Message.messageToResponse(Message.ok().data("users", result.getWorkspaceUserList())); + } + + + /** + * 查询角色 + * + * @param req + * @param workspaceId + * @return + */ + @GET + @Path("/roles/{workspaceId}") + public Response roles(@Context HttpServletRequest req, @PathParam("workspaceId") String workspaceId) throws ErrorException { + logger.info("roles workspaceId : {}", workspaceId); + GetWorkspaceUsersResult result = linkisWorkSpaceRemoteClient.getWorkspaceUsers(GetWorkspaceUsersAction.builder().setUser(getStrategyUser(req)).setWorkspaceId(workspaceId).build()); + return Message.messageToResponse(Message.ok().data("users", result.getWorkspaceRoleList())); + } + + +} diff --git a/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/service/AssetService.java b/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/service/AssetService.java new file mode 100644 index 0000000000..d0c0ded84b --- /dev/null +++ b/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/service/AssetService.java @@ -0,0 +1,170 @@ +package com.webank.wedatasphere.dss.data.governance.service; + + +import com.webank.wedatasphere.dss.data.governance.dto.HiveTblStatsDTO; +import com.webank.wedatasphere.dss.data.governance.dto.SearchLabelDTO; +import com.webank.wedatasphere.dss.data.governance.entity.*; +import com.webank.wedatasphere.dss.data.governance.exception.DAOException; +import com.webank.wedatasphere.dss.data.governance.exception.DataGovernanceException; +import com.webank.wedatasphere.dss.data.governance.vo.*; +import org.apache.atlas.AtlasServiceException; +import org.apache.atlas.model.lineage.AtlasLineageInfo; + +import java.sql.SQLException; +import java.util.List; +import java.util.Map; +import java.util.Set; + + +public interface AssetService { + public Map getHiveSummary() throws DataGovernanceException; + + public List searchHiveTable(String classification, String query,String termName, + int limit, int offset) throws DataGovernanceException; + + public List searchHiveDb(String classification, String query, + int limit, int offset) throws DataGovernanceException; + + public HiveTblDetailInfo getHiveTblDetail(String guid) throws DataGovernanceException ; + + public List getHiveTblPartition(String guid) throws DataGovernanceException; + + public String getTbSelect(String guid) throws DataGovernanceException; + + public String getTbCreate(String guid) throws DataGovernanceException; + + public void modifyComment(String guid, String commentStr) throws DataGovernanceException ; + + public void bulkModifyComment(Map commentMap) throws DataGovernanceException ; + + public void setLabels(String guid, Set labels) throws DataGovernanceException; + + /** + * 获取表实体的血缘信息 + */ + public AtlasLineageInfo getHiveTblLineage(final String guid, final AtlasLineageInfo.LineageDirection direction, final int depth) throws DataGovernanceException; + + public List getTop10Table () throws DataGovernanceException, SQLException, DAOException; + + + /** + * 创建模型 + * @param vo + * @return + */ + CreateModelTypeInfo createModelType(CreateModelTypeVO vo) throws DataGovernanceException, AtlasServiceException; + + + /** + * 删除模型 + * @param vo + * @throws Exception + */ + void deleteModelType(DeleteModelTypeVO vo) throws Exception; + + + + /** + * 更新模型 + * @param vo + * @return + */ + UpdateModelTypeInfo updateModelType(UpdateModelTypeVO vo) throws Exception; + + + /** + * 绑定模型 + * @param vo + * @throws Exception + */ + void bindModelType(BindModelVO vo) throws Exception; + + + /** + * 解绑模型 + * @param vo + * @throws Exception + */ + void unBindModel(UnBindModelVO vo) throws Exception; + + + + /** + * 表统计信息 + * @param dbName + * @param tableName + * @param guid + * @return + */ + HiveTblStatsDTO hiveTblStats(String dbName, String tableName, String guid) throws AtlasServiceException, Exception; + + + /** + * 表统计信息 + * @param dbName + * @param tableName + * @param guid + * @return + */ + Long hiveTblSize(String dbName, String tableName, String guid) throws Exception; + + + /** + * 新建标签 + * @param vo + * @return + * @throws Exception + */ + CreateLabelInfo createLabel(CreateLabelVO vo) throws Exception; + + + /** + * 删除标签 + * @param vo + * @return + * @throws Exception + */ + void deleteLabel(DeleteLabelVO vo) throws Exception; + + + /** + * 删除标签 + * @param vo + * @return + * @throws Exception + */ + UpdateLabelInfo updateLabel(UpdateLabelVO vo) throws Exception; + + + /** + * 绑定标签 + * @param vo + * @throws Exception + */ + void bindLabel(BindLabelVO vo) throws Exception; + + + /** + * 解绑实体 + * @param vo + * @throws Exception + */ + void unBindLabel(UnBindLabelVO vo) throws Exception; + + /** + * 标签搜索列表 + * @param query + * @param limit + * @param offset + * @return + */ + List listLabels(String query, Integer limit, Integer offset) throws AtlasServiceException, Exception; + /* + * 根据名称查询分区 + * @param name + * @return + * @throws Exception + */ + List getHiveTblPartitionByName(String dbName, String tableName) throws Exception; + +} diff --git a/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/service/AuthenticationClientStrategy.java b/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/service/AuthenticationClientStrategy.java new file mode 100644 index 0000000000..4a036cf58a --- /dev/null +++ b/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/service/AuthenticationClientStrategy.java @@ -0,0 +1,22 @@ +package com.webank.wedatasphere.dss.data.governance.service; + + + +import com.webank.wedatasphere.dss.data.governance.conf.ClientStrategy; +import org.apache.linkis.common.conf.CommonVars; +import org.apache.linkis.server.security.SecurityFilter; +import org.apache.commons.lang.StringUtils; + +import javax.servlet.http.HttpServletRequest; + +public interface AuthenticationClientStrategy { + + CommonVars CLIENT_STRATEGY = CommonVars.apply("wds.wedatasphere.data.assert.client.strategy", ""); + + default String getStrategyUser(HttpServletRequest req){ + if (StringUtils.equals(ClientStrategy.TOKEN.getCode(),CLIENT_STRATEGY.getValue())){ + return SecurityFilter.getLoginUsername(req); + } + return "hdfs"; + } +} diff --git a/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/service/WorkspaceInfoService.java b/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/service/WorkspaceInfoService.java new file mode 100644 index 0000000000..2cd9e43441 --- /dev/null +++ b/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/service/WorkspaceInfoService.java @@ -0,0 +1,31 @@ +/* + * + * * Copyright 2019 WeBank + * * + * * Licensed under the Apache License, Version 2.0 (the "License"); + * * you may not use this file except in compliance with the License. + * * You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.data.governance.service; + + +import com.webank.wedatasphere.dss.data.governance.exception.DataGovernanceException; + +import java.util.List; + + +public interface WorkspaceInfoService { + + public List getWorkspaceUsers(int workspaceId,String search) throws DataGovernanceException; + +} diff --git a/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/service/impl/AssetServiceImpl.java b/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/service/impl/AssetServiceImpl.java new file mode 100644 index 0000000000..773318a16c --- /dev/null +++ b/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/service/impl/AssetServiceImpl.java @@ -0,0 +1,841 @@ +package com.webank.wedatasphere.dss.data.governance.service.impl; + +import com.baomidou.mybatisplus.core.toolkit.Wrappers; +import com.google.common.collect.Sets; +import com.google.gson.internal.LinkedTreeMap; +import com.webank.wedatasphere.dss.data.governance.atlas.AtlasService; +import com.webank.wedatasphere.dss.data.governance.dao.*; +import com.webank.wedatasphere.dss.data.governance.dao.impl.MetaInfoMapperImpl; +import com.webank.wedatasphere.dss.data.governance.dto.HiveTblStatsDTO; +import com.webank.wedatasphere.dss.data.governance.dto.SearchLabelDTO; +import com.webank.wedatasphere.dss.data.governance.entity.*; +import com.webank.wedatasphere.dss.data.governance.exception.DAOException; +import com.webank.wedatasphere.dss.data.governance.exception.DataGovernanceException; +import com.webank.wedatasphere.dss.data.governance.restful.DSSDataGovernanceAssetRestful; +import com.webank.wedatasphere.dss.data.governance.service.AssetService; +import com.webank.wedatasphere.dss.data.governance.utils.DateUtil; +import com.webank.wedatasphere.dss.data.governance.vo.*; +import jersey.repackaged.com.google.common.collect.Lists; +import org.apache.atlas.AtlasServiceException; +import org.apache.atlas.model.glossary.AtlasGlossaryTerm; +import org.apache.atlas.model.glossary.relations.AtlasTermAssignmentHeader; +import org.apache.atlas.model.instance.AtlasEntity; +import org.apache.atlas.model.instance.AtlasEntityHeader; +import org.apache.atlas.model.instance.AtlasStruct; +import org.apache.atlas.model.lineage.AtlasLineageInfo; +import org.apache.atlas.model.typedef.AtlasClassificationDef; +import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Service; +import org.springframework.util.CollectionUtils; + +import javax.annotation.Resource; +import java.util.*; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; +import java.util.stream.Collectors; + + +@Service +public class AssetServiceImpl implements AssetService { + private static final Logger logger = LoggerFactory.getLogger(AssetServiceImpl.class); + private AtlasService atlasService; + private MetaInfoMapper metaInfoMapper; + private WorkspaceInfoMapper workspaceInfoMapper; + + @Resource + private TableColumnCountQueryMapper tableColumnCountQueryMapper; + + @Resource + private TableSizeInfoMapper tableSizeInfoMapper; + + @Resource + private TableSizePartitionInfoMapper tableSizePartitionInfoMapper; + + public AssetServiceImpl(AtlasService atlasService) { + this.atlasService = atlasService; + this.metaInfoMapper = new MetaInfoMapperImpl(); + } + + @Override + public Map getHiveSummary() throws DataGovernanceException { + try { + Map result = new HashMap<>(); + + result.put("hiveDb", atlasService.getHiveDbCnt()); + result.put("hiveTable", atlasService.getHiveTableCnt()); + result.put("hiveStore", metaInfoMapper.getTableStorage()); + + return result; + } catch (AtlasServiceException | DAOException exception) { + throw new DataGovernanceException(23000, exception.getMessage()); + } + } + + + @Override + public CreateModelTypeInfo createModelType(CreateModelTypeVO vo) throws DataGovernanceException { + if (!ClassificationConstant.isTypeScope(vo.getType())) { + throw new DataGovernanceException(23000, "不支持此类型" + vo.getType()); + } + try { + AtlasClassificationDef atlasClassificationDef = atlasService.createSubClassification(ClassificationConstant.formatName(vo.getType(), vo.getName()).get(), ClassificationConstant.getRoot(vo.getType()).get()); + CreateModelTypeInfo dto = new CreateModelTypeInfo(); + dto.setGuid(atlasClassificationDef.getGuid()); + dto.setName(atlasClassificationDef.getName()); + return dto; + } catch (AtlasServiceException ex) { + throw new DataGovernanceException(23000, ex.getMessage()); + } + } + + + @Override + public void deleteModelType(DeleteModelTypeVO vo) throws Exception { + if (!ClassificationConstant.isTypeScope(vo.getType())) { + throw new DataGovernanceException(23000, "不支持此类型" + vo.getType()); + } + try { + atlasService.deleteClassification(ClassificationConstant.formatName(vo.getType(), vo.getName()).get()); + } catch (AtlasServiceException ex) { + throw new DataGovernanceException(23000, ex.getMessage()); + } + } + + @Override + public UpdateModelTypeInfo updateModelType(UpdateModelTypeVO vo) throws Exception { + if (!ClassificationConstant.isTypeScope(vo.getType())) { + throw new DataGovernanceException(23000, "不支持此类型" + vo.getType()); + } + String newName = ClassificationConstant.formatName(vo.getType(), vo.getName()).get(); + String root = ClassificationConstant.getRoot(vo.getType()).get(); + String orgName = ClassificationConstant.formatName(vo.getType(), vo.getOrgName()).get(); + AtlasClassificationDef atlasClassificationDef = null; + //首先尝试创建 + try { + atlasClassificationDef = atlasService.createSubClassification(newName, root); + } catch (AtlasServiceException ex) { + //创建失败直接回退 + throw new DataGovernanceException(23000, ex.getMessage()); + } + + //尝试原类型 + try { + atlasService.deleteClassification(orgName); + } catch (AtlasServiceException ex) { + //回滚删除创建的新类型 + try { + atlasService.deleteClassification(newName); + } catch (AtlasServiceException ex1) { + throw new DataGovernanceException(23000, ex1.getMessage()); + } + throw new DataGovernanceException(23000, ex.getMessage()); + } + UpdateModelTypeInfo info = new UpdateModelTypeInfo(); + info.setGuid(atlasClassificationDef.getGuid()); + info.setName(atlasClassificationDef.getName()); + return info; + } + + + @Override + public void bindModelType(BindModelVO vo) throws Exception { + if (!ClassificationConstant.isTypeScope(vo.getModelType())) { + throw new DataGovernanceException(23000, "不支持此类型" + vo.getModelType()); + } + String tableGuid = vo.getGuid(); + + if (StringUtils.isBlank(tableGuid)) { + tableGuid = getGuid(vo.getTableName()); + } + + try { + atlasService.addClassification(ClassificationConstant.formatName(vo.getModelType(), vo.getModelName()).get(), tableGuid, false); + } catch (AtlasServiceException ex) { + throw new DataGovernanceException(23000, ex.getMessage()); + } + + } + + @Override + public void unBindModel(UnBindModelVO vo) throws Exception { + if (!ClassificationConstant.isTypeScope(vo.getModelType())) { + throw new DataGovernanceException(23000, "不支持此类型" + vo.getModelType()); + } + String tableGuid = vo.getGuid(); + + if (StringUtils.isBlank(tableGuid)) { + tableGuid = getGuid(vo.getTableName()); + } + + try { + atlasService.deleteClassification(tableGuid, ClassificationConstant.formatName(vo.getModelType(), vo.getModelName()).get()); + } catch (AtlasServiceException ex) { + throw new DataGovernanceException(23000, ex.getMessage()); + } + + } + + private String getGuid(String tableName) throws DataGovernanceException { + + //首先搜索指定表,查找guid + List atlasEntityHeaders = null; + try { + atlasEntityHeaders = atlasService.searchHiveTable0(null, tableName, null,true, 1, 0); + } catch (AtlasServiceException ex) { + throw new DataGovernanceException(23000, ex.getMessage()); + } + if (CollectionUtils.isEmpty(atlasEntityHeaders)) { + throw new DataGovernanceException(23000, "table " + tableName + " not find"); + } + + if (atlasEntityHeaders.size() > 1) { + throw new DataGovernanceException(23000, "table " + tableName + " duplicate " + atlasEntityHeaders); + } + return atlasEntityHeaders.get(0).getGuid(); + } + + @Override + public List searchHiveTable(String classification, String query, String termName, + int limit, int offset) throws DataGovernanceException { + List atlasEntityHeaders = null; + try { + atlasEntityHeaders = atlasService.searchHiveTable0(classification, query, termName,true, limit, offset); + } catch (AtlasServiceException ex) { + throw new DataGovernanceException(23000, ex.getMessage()); + } + + if (atlasEntityHeaders != null) { + return atlasEntityHeaders.parallelStream().map(atlasEntityHeader -> { + HiveTblSimpleInfo hiveTblBasic = new HiveTblSimpleInfo(); + hiveTblBasic.setGuid(atlasEntityHeader.getGuid()); + hiveTblBasic.setName(atlasEntityHeader.getAttribute("name").toString()); + hiveTblBasic.setQualifiedName(atlasEntityHeader.getAttribute("qualifiedName").toString()); + hiveTblBasic.setOwner(atlasEntityHeader.getAttribute("owner").toString()); + Object createTime = atlasEntityHeader.getAttribute("createTime"); + if (createTime != null) { + hiveTblBasic.setCreateTime(DateUtil.unixToTimeStr((Double) createTime)); + } + hiveTblBasic.setClassifications(atlasEntityHeader.getClassificationNames()); + + Object comment = atlasEntityHeader.getAttribute("comment"); + if (comment != null) { + hiveTblBasic.setComment(comment.toString()); + } + + Object aliases = atlasEntityHeader.getAttribute("aliases"); + if (aliases != null) { + hiveTblBasic.setAliases(aliases.toString()); + } + + Object lastAccessTime = atlasEntityHeader.getAttribute("lastAccessTime"); + if (lastAccessTime != null) { + hiveTblBasic.setLastAccessTime(DateUtil.unixToTimeStr((Double) lastAccessTime)); + } + + Object parameters = atlasEntityHeader.getAttribute("parameters"); + if (parameters != null) { + Map mapParameters = (Map) parameters; + Object totalSize = mapParameters.get("totalSize"); + if (totalSize != null) { + hiveTblBasic.setTotalSize(totalSize.toString()); + } + } + + return hiveTblBasic; + }).collect(Collectors.toList()); + } + return null; + } + + @Override + public List searchHiveDb(String classification, String query, int limit, int offset) throws DataGovernanceException { + List atlasEntityHeaders = null; + try { + atlasEntityHeaders = atlasService.searchHiveDb(classification, query, true, limit, offset); + } catch (AtlasServiceException ex) { + throw new DataGovernanceException(23000, ex.getMessage()); + } + + if (atlasEntityHeaders != null) { + return atlasEntityHeaders.parallelStream().map(atlasEntityHeader -> { + HiveTblSimpleInfo hiveTblBasic = new HiveTblSimpleInfo(); + hiveTblBasic.setGuid(atlasEntityHeader.getGuid()); + hiveTblBasic.setName(atlasEntityHeader.getAttribute("name").toString()); + hiveTblBasic.setQualifiedName(atlasEntityHeader.getAttribute("qualifiedName").toString()); + hiveTblBasic.setOwner(atlasEntityHeader.getAttribute("owner").toString()); + + + Object createTime = atlasEntityHeader.getAttribute("createTime"); + if (createTime != null) { + hiveTblBasic.setCreateTime(DateUtil.unixToTimeStr((Double) createTime)); + } + + + return hiveTblBasic; + }).collect(Collectors.toList()); + } + return null; + } + + @Override + public HiveTblDetailInfo getHiveTblDetail(String guid) throws DataGovernanceException { + try { + HiveTblDetailInfo hiveTblDetailInfo = new HiveTblDetailInfo(); + + AtlasEntity atlasEntity = atlasService.getHiveTbl(guid); + Map hiveTblNameAndIsPartById = atlasService.getHiveTblNameAndIsPartById(guid); + Boolean isPartTable = (Boolean) hiveTblNameAndIsPartById.get("isPartition"); + int storage = 0; + String db_name = String.valueOf(atlasEntity.getAttributes().get("qualifiedName")).split("@")[0]; + String tableName = db_name.split("\\.")[1]; + String dbName = db_name.split("\\.")[0]; + try { + storage = metaInfoMapper.getTableInfo(dbName, tableName, isPartTable); + } catch (DAOException e) { + e.printStackTrace(); + } + List guids = new ArrayList<>(); + List> columns = (List>) atlasEntity.getAttributes().get("columns"); + for (LinkedTreeMap column : columns) { + guids.add(column.get("guid")); + } + List hiveColumnInfos = new ArrayList<>(); + if (guids.size() > 0) { + List hiveColumnsByGuids = atlasService.getHiveColumnsByGuids(guids); + for (AtlasEntity hiveColumnsByGuid : hiveColumnsByGuids) { + HiveTblDetailInfo.HiveColumnInfo hiveColumnInfo = new HiveTblDetailInfo.HiveColumnInfo(); + hiveColumnInfo.setName(String.valueOf(hiveColumnsByGuid.getAttributes().get("name"))); + hiveColumnInfo.setType(String.valueOf(hiveColumnsByGuid.getAttributes().get("type"))); + hiveColumnInfo.setComment(String.valueOf(hiveColumnsByGuid.getAttributes().get("comment"))); + hiveColumnInfo.setGuid(hiveColumnsByGuid.getGuid()); + hiveColumnInfos.add(hiveColumnInfo); + } + } + List partguids = new ArrayList<>(); + List> partitionKeys = (List>) atlasEntity.getAttributes().get("partitionKeys"); + for (LinkedTreeMap column : partitionKeys) { + partguids.add(column.get("guid")); + } + List partitionColumns = new ArrayList<>(); + if (partguids.size() > 0) { + List hivePartColumnsByGuids = atlasService.getHiveColumnsByGuids(partguids); + for (AtlasEntity hiveColumnsByGuid : hivePartColumnsByGuids) { + HiveTblDetailInfo.HiveColumnInfo hiveColumnInfo = new HiveTblDetailInfo.HiveColumnInfo(); + hiveColumnInfo.setName(String.valueOf(hiveColumnsByGuid.getAttributes().get("name"))); + hiveColumnInfo.setType(String.valueOf(hiveColumnsByGuid.getAttributes().get("type"))); + hiveColumnInfo.setComment(String.valueOf(hiveColumnsByGuid.getAttributes().get("comment"))); + hiveColumnInfo.setGuid(hiveColumnsByGuid.getGuid()); + partitionColumns.add(hiveColumnInfo); + } + } + HiveTblDetailInfo.HiveTblBasicInfo basic = new HiveTblDetailInfo.HiveTblBasicInfo(); + basic.setName(tableName); + basic.setQualifiedName(atlasEntity.getAttribute("qualifiedName").toString()); + basic.setOwner(String.valueOf(atlasEntity.getAttributes().get("owner"))); + basic.setCreateTime(new java.text.SimpleDateFormat("yyyy MM-dd HH:mm:ss").format(atlasEntity.getCreateTime())); + basic.setStore(String.valueOf(storage)); + //设置标签 + Set labels = Sets.newHashSet(); + List> meanings =(List>) atlasEntity.getRelationshipAttributes().get("meanings"); + if (!CollectionUtils.isEmpty(meanings)) { + labels = meanings.stream() + .filter(label->isLabel(label.get("guid"))) + .map(label->(label.get("displayText"))) + .collect(Collectors.toSet()); + } + basic.setLabels(labels); + basic.setIsParTbl(isPartTable); + basic.setGuid(guid); + + if (!CollectionUtils.isEmpty(atlasEntity.getClassifications())) { + basic.setClassifications(atlasEntity.getClassifications().stream().map(AtlasStruct::getTypeName).collect(Collectors.toList())); + } + Object comment = atlasEntity.getAttribute("comment"); + if (comment != null) { + basic.setComment(comment.toString()); + } + + Object aliases = atlasEntity.getAttribute("aliases"); + if (aliases != null) { + basic.setAliases(aliases.toString()); + } + + Object lastAccessTime = atlasEntity.getAttribute("lastAccessTime"); + if (lastAccessTime != null) { + basic.setLastAccessTime(DateUtil.unixToTimeStr((Double) lastAccessTime)); + } + + Object parameters = atlasEntity.getAttribute("parameters"); + if (parameters != null) { + Map mapParameters = (Map) parameters; + Object totalSize = mapParameters.get("totalSize"); + if (totalSize != null) { + basic.setTotalSize(totalSize.toString()); + } + + } + + Object external = atlasEntity.getAttribute("tableType"); + if (external != null) { + basic.setExternal(external.toString()); + } + + + hiveTblDetailInfo.setColumns(hiveColumnInfos); + hiveTblDetailInfo.setBasic(basic); + hiveTblDetailInfo.setPartitionKeys(partitionColumns); + + + return hiveTblDetailInfo; + } catch (AtlasServiceException ex) { + throw new DataGovernanceException(23000, ex.getMessage()); + } + } + + @Override + public List getHiveTblPartition(String guid) throws DataGovernanceException { + try { + AtlasEntity atlasEntity = atlasService.getHiveTbl(guid); + String db_name = String.valueOf(atlasEntity.getAttributes().get("qualifiedName")).split("@")[0]; + return getPartInfos(db_name); + + } catch (AtlasServiceException ex) { + throw new DataGovernanceException(23000, ex.getMessage()); + } + } + + private List getPartInfos(String db_name) { + String tableName = db_name.split("\\.")[1]; + String dbName = db_name.split("\\.")[0]; + List partInfo = new ArrayList<>(); + try { + partInfo = metaInfoMapper.getPartInfo(dbName, tableName); + } catch (DAOException e) { + e.printStackTrace(); + } + return partInfo; + } + + @Override + public String getTbSelect(String guid) throws DataGovernanceException { + try { + AtlasEntity atlasEntity = atlasService.getHiveTbl(guid); + String db_name = String.valueOf(atlasEntity.getAttributes().get("qualifiedName")).split("@")[0]; + String tableName = db_name.split("\\.")[1]; + List guids = new ArrayList<>(); + List> columns = (List>) atlasEntity.getAttributes().get("columns"); + for (LinkedTreeMap column : columns) { + guids.add(column.get("guid")); + } + List fields = new ArrayList<>(); + List hiveColumnsByGuids = atlasService.getHiveColumnsByGuids(guids); + for (AtlasEntity hiveColumnsByGuid : hiveColumnsByGuids) { + fields.add((String) hiveColumnsByGuid.getAttributes().get("name")); + } + Map hiveTblNameAndIsPartById = atlasService.getHiveTblNameAndIsPartById(guid); + Boolean isPartTable = (Boolean) hiveTblNameAndIsPartById.get("isPartition"); + if (isPartTable == true) { + List partguids = new ArrayList<>(); + List> partitionKeys = (List>) atlasEntity.getAttributes().get("partitionKeys"); + for (LinkedTreeMap column : partitionKeys) { + partguids.add(column.get("guid")); + } + List hiveColumnsByGuids1 = atlasService.getHiveColumnsByGuids(partguids); + for (AtlasEntity entity : hiveColumnsByGuids1) { + fields.add((String) entity.getAttributes().get("name")); + } + } + StringBuilder sql = new StringBuilder(); + sql.append("SELECT @$ "); + for (int i = 0; i < fields.size() - 1; i++) { + sql.append(fields.get(i)).append(", @$ "); + } + sql.append(fields.get(fields.size() - 1)).append(" @$ from ").append(tableName); + return sql.toString(); + } catch (AtlasServiceException ex) { + throw new DataGovernanceException(23000, ex.getMessage()); + } + } + + @Override + public String getTbCreate(String guid) throws DataGovernanceException { + try { + StringBuilder sql = new StringBuilder(); + AtlasEntity atlasEntity = atlasService.getHiveTbl(guid); + String db_name = String.valueOf(atlasEntity.getAttributes().get("qualifiedName")).split("@")[0]; + String tableName = db_name.split("\\.")[1]; + List guids = new ArrayList<>(); + List> columns = (List>) atlasEntity.getAttributes().get("columns"); + for (LinkedTreeMap column : columns) { + guids.add(column.get("guid")); + } + sql.append("CREATE TABLE IF NOT EXISTS ").append(tableName).append(" @$ ( @$ "); + List fields = new ArrayList<>(); + List hiveColumnsByGuids = atlasService.getHiveColumnsByGuids(guids); + for (int i = 0; i < hiveColumnsByGuids.size(); i++) { + if (i < hiveColumnsByGuids.size() - 1) { + AtlasEntity hiveColumnsByGuid = hiveColumnsByGuids.get(i); + StringBuilder sb = new StringBuilder(); + sb.append((String) hiveColumnsByGuid.getAttributes().get("name")).append(" ").append((String) hiveColumnsByGuid.getAttributes().get("type")); + if (hiveColumnsByGuid.getAttributes().get("comment") == null) { + sb.append(", "); + } else { + sb.append(" COMMENT '").append(hiveColumnsByGuid.getAttributes().get("comment")).append("',"); + } + fields.add(sb.append(" @$ ").toString()); + } else { + AtlasEntity hiveColumnsByGuid = hiveColumnsByGuids.get(i); + StringBuilder sb = new StringBuilder(); + sb.append((String) hiveColumnsByGuid.getAttributes().get("name")).append(" ").append((String) hiveColumnsByGuid.getAttributes().get("type")); + if (hiveColumnsByGuid.getAttributes().get("comment") == null) { + sb.append(" "); + } else { + sb.append(" COMMENT '").append(hiveColumnsByGuid.getAttributes().get("comment")).append("' "); + } + fields.add(sb.append(" @$ ").toString()); + } + } + for (String field : fields) { + sql.append(field); + } + sql.append(") @$ "); + Map hiveTblNameAndIsPartById = atlasService.getHiveTblNameAndIsPartById(guid); + Boolean isPartTable = (Boolean) hiveTblNameAndIsPartById.get("isPartition"); + if (isPartTable == true) { + sql.append("PARTITIONED BY @$ ( @$ "); + List partguids = new ArrayList<>(); + List> partitionKeys = (List>) atlasEntity.getAttributes().get("partitionKeys"); + for (LinkedTreeMap column : partitionKeys) { + partguids.add(column.get("guid")); + } + List keyFields = new ArrayList<>(); + List hiveColumnsByGuids1 = atlasService.getHiveColumnsByGuids(partguids); + for (int i = 0; i < hiveColumnsByGuids1.size(); i++) { + if (i < hiveColumnsByGuids1.size() - 1) { + AtlasEntity entity = hiveColumnsByGuids1.get(i); + StringBuilder sb = new StringBuilder(); + sb.append((String) entity.getAttributes().get("name")).append(" ").append((String) entity.getAttributes().get("type")); + if (entity.getAttributes().get("comment") == null) { + sb.append(", "); + } else { + sb.append(" COMMENT '").append(entity.getAttributes().get("comment")).append("', "); + } + keyFields.add(sb.append(" @$ ").toString()); + } else { + AtlasEntity entity = hiveColumnsByGuids1.get(i); + StringBuilder sb = new StringBuilder(); + sb.append((String) entity.getAttributes().get("name")).append(" ").append((String) entity.getAttributes().get("type")); + if (entity.getAttributes().get("comment") == null) { + sb.append(" "); + } else { + sb.append(" COMMENT '").append(entity.getAttributes().get("comment")).append("' "); + } + keyFields.add(sb.append(" @$ ").toString()); + } + } + for (int i = 0; i < keyFields.size(); i++) { + sql.append(keyFields.get(i)); + } + sql.append(")"); + } + return sql.toString(); + } catch (AtlasServiceException ex) { + throw new DataGovernanceException(23000, ex.getMessage()); + } + } + + @Override + public void modifyComment(String guid, String commentStr) throws DataGovernanceException { + try { + atlasService.modifyComment(guid, commentStr); + } catch (AtlasServiceException ex) { + throw new DataGovernanceException(23000, ex.getMessage()); + } + } + + @Override + public void bulkModifyComment(Map commentMap) throws DataGovernanceException { + commentMap.keySet().forEach(key -> { + try { + atlasService.modifyComment(key, commentMap.get(key)); + } catch (AtlasServiceException ex) { + + } + }); + } + + @Override + public void setLabels(String guid, Set labels) throws DataGovernanceException { + try { + atlasService.setLabels(guid, labels); + } catch (AtlasServiceException ex) { + throw new DataGovernanceException(23000, ex.getMessage()); + } + } + + @Override + public AtlasLineageInfo getHiveTblLineage(final String guid, final AtlasLineageInfo.LineageDirection direction, final int depth) throws DataGovernanceException { + try { + return atlasService.getLineageInfo(guid, direction, depth); + } catch (AtlasServiceException exception) { + throw new DataGovernanceException(23000, exception.getMessage()); + } + } + + @Override + public List getTop10Table() throws DAOException { + return metaInfoMapper.getTop10Table(); + } + + + @Override + public HiveTblStatsDTO hiveTblStats(String dbName, String tableName, String guid) throws Exception { + //如果有guid则使用guid + if (StringUtils.isNotBlank(guid)) { + String qualifiedName = getQualifiedName(guid); + dbName = StringUtils.substringBefore(qualifiedName, "."); + tableName = StringUtils.substringAfter(qualifiedName, "."); + } + + + HiveTblStatsDTO dto = new HiveTblStatsDTO(); + + //字段数量 + List columnCounts = tableColumnCountQueryMapper.query(Wrappers.lambdaQuery().eq(TableColumnCount::getDbName, dbName).eq(TableColumnCount::getTblName, tableName)); + if (!CollectionUtils.isEmpty(columnCounts)) { + dto.setColumnCount(columnCounts.get(0).getColumnCount()); + } + + //表容量 + List tableSizeInfos = tableSizeInfoMapper.query(Wrappers.lambdaQuery().eq(TableSizeInfo::getDbName, dbName).eq(TableSizeInfo::getTblName, tableName)); + if (!CollectionUtils.isEmpty(tableSizeInfos)) { + tableSizeInfos.forEach(tableSizeInfo -> { + if (StringUtils.equals("totalSize", tableSizeInfo.getParamKey())) { + dto.setTotalSize(Long.parseLong(tableSizeInfo.getParamValue())); + } + if (StringUtils.equals("numFiles", tableSizeInfo.getParamKey())) { + dto.setNumFiles(Integer.parseInt(tableSizeInfo.getParamValue())); + } + }); + } + //分区信息 + List tablePartitionSizeInfos = tableSizePartitionInfoMapper.query(Wrappers.lambdaQuery().eq(TablePartitionSizeInfo::getDbName, dbName).eq(TablePartitionSizeInfo::getTblName, tableName)); + if (!CollectionUtils.isEmpty(tablePartitionSizeInfos)) { + Set partIds = Sets.newHashSet(); + tablePartitionSizeInfos.forEach(tablePartitionSizeInfo -> { + partIds.add(tablePartitionSizeInfo.getParId()); + if (StringUtils.equals("totalSize", tablePartitionSizeInfo.getParamKey())) { + dto.setTotalSize(dto.getTotalSize() + Long.parseLong(tablePartitionSizeInfo.getParamValue())); + } + if (StringUtils.equals("numFiles", tablePartitionSizeInfo.getParamKey())) { + dto.setNumFiles(dto.getNumFiles() + Integer.parseInt(tablePartitionSizeInfo.getParamValue())); + } + }); + dto.setPartitionCount(partIds.size()); + } + return dto; + } + + private String getQualifiedName(String guid) throws AtlasServiceException { + AtlasEntity atlasEntity = atlasService.getHiveTbl(guid); + return StringUtils.substringBefore(String.valueOf(atlasEntity.getAttributes().get("qualifiedName")), "@"); + } + + @Override + public Long hiveTblSize(String dbName, String tableName, String guid) throws Exception { + //如果有guid则使用guid + if (StringUtils.isNotBlank(guid)) { + String qualifiedName = getQualifiedName(guid); + dbName = StringUtils.substringBefore(qualifiedName, "."); + tableName = StringUtils.substringAfter(qualifiedName, "."); + } + + AtomicReference size = new AtomicReference<>(0L); + //表容量 + List tableSizeInfos = tableSizeInfoMapper.query(Wrappers.lambdaQuery().eq(TableSizeInfo::getDbName, dbName).eq(TableSizeInfo::getTblName, tableName)); + if (!CollectionUtils.isEmpty(tableSizeInfos)) { + tableSizeInfos.forEach(tableSizeInfo -> { + if (StringUtils.equals("totalSize", tableSizeInfo.getParamKey())) { + size.set(Long.parseLong(tableSizeInfo.getParamValue())); + } + }); + } + //分区信息 + List tablePartitionSizeInfos = tableSizePartitionInfoMapper.query(Wrappers.lambdaQuery().eq(TablePartitionSizeInfo::getDbName, dbName).eq(TablePartitionSizeInfo::getTblName, tableName)); + if (!CollectionUtils.isEmpty(tablePartitionSizeInfos)) { + Set partIds = Sets.newHashSet(); + tablePartitionSizeInfos.forEach(tablePartitionSizeInfo -> { + partIds.add(tablePartitionSizeInfo.getParId()); + if (StringUtils.equals("totalSize", tablePartitionSizeInfo.getParamKey())) { + size.set(size.get() + Long.parseLong(tablePartitionSizeInfo.getParamValue())); + } + + }); + + } + return size.get(); + } + + + @Override + public CreateLabelInfo createLabel(CreateLabelVO vo) throws Exception { + try { + return CreateLabelInfo.from(atlasService.createLabel(vo.getName())); + } catch (AtlasServiceException exception) { + throw new DataGovernanceException(23000, exception.getMessage()); + } + + } + + + @Override + public void deleteLabel(DeleteLabelVO vo) throws Exception { + try { + atlasService.deleteLabel(vo.getName()); + } catch (Exception exception) { + throw new DataGovernanceException(23000, exception.getMessage()); + } + } + + + @Override + public UpdateLabelInfo updateLabel(UpdateLabelVO vo) throws Exception { + + if (StringUtils.equals(vo.getName(),vo.getOrgName())){ + throw new DataGovernanceException(23000, "修改标签名称前后相同"); + } + + Optional termGuidOptional = getTermGuidOptional(vo.getOrgName()); + if (!termGuidOptional.isPresent()){ + throw new DataGovernanceException(23000, "标签"+vo.getOrgName()+ "不存在"); + } + + AtlasGlossaryTerm atlasGlossaryTerm = null; + //首先新建标签 + try { + atlasGlossaryTerm = atlasService.createLabel(vo.getName()); + } catch (Exception exception) { + throw new DataGovernanceException(23000, exception.getMessage()); + } + //尝试删除原标签 + try { + atlasService.deleteLabel(vo.getOrgName()); + } catch (Exception exception) { + //休眠五秒,atlas新建分词有延迟 + logger.error("wait for 5 seconds to roll back term " + vo.getName()); + TimeUnit.SECONDS.sleep(5); + //回滚删除新建标签 + try { + atlasService.deleteLabel(vo.getName()); + } catch (Exception exception1) { + throw new DataGovernanceException(23000, exception1.getMessage()); + } + throw new DataGovernanceException(23000, exception.getMessage()); + } + + return UpdateLabelInfo.from(atlasGlossaryTerm); + } + + + @Override + public void bindLabel(BindLabelVO vo) throws Exception { + + if (StringUtils.isNotBlank(vo.getLabelGuid()) && StringUtils.isNotBlank(vo.getTableGuid())) { + try { + atlasService.assignTermToEntities(vo.getLabelGuid(), Lists.newArrayList(vo.getTableGuid())); + } catch (AtlasServiceException exception1) { + throw new DataGovernanceException(23000, exception1.getMessage()); + } + } + + Optional termGuidOptional = getTermGuidOptional(vo.getLabel()); + List entityHeaders = getAtlasEntityHeaders(vo.getTableName()); + + try { + atlasService.assignTermToEntities(termGuidOptional.get(), Lists.newArrayList(entityHeaders.get(0).getGuid())); + } catch (AtlasServiceException exception1) { + throw new DataGovernanceException(23000, exception1.getMessage()); + } + + } + + private Optional getTermGuidOptional(String label) throws AtlasServiceException, DataGovernanceException { + Optional termGuidOptional = atlasService.getTermGuid(GlossaryConstant.LABEL, label); + if (!termGuidOptional.isPresent()) { + throw new DataGovernanceException(23000, label + "标签不存在"); + } + return termGuidOptional; + } + + @Override + public void unBindLabel(UnBindLabelVO vo) throws Exception { + if (StringUtils.isNotBlank(vo.getLabelGuid()) && StringUtils.isNotBlank(vo.getTableGuid()) && StringUtils.isNotBlank(vo.getRelationGuid())) { + try { + atlasService.disassociateTermFromEntities(vo.getLabelGuid(), Lists.newArrayList(RelatedObjectId.from(vo.getTableGuid(), vo.getRelationGuid()))); + } catch (AtlasServiceException exception1) { + throw new DataGovernanceException(23000, exception1.getMessage()); + } + } + + Optional termGuidOptional = getTermGuidOptional(vo.getLabel()); + List entityHeaders = getAtlasEntityHeaders(vo.getTableName()); + + String termGuid = termGuidOptional.get(); + AtlasEntityHeader atlasEntityHeader = entityHeaders.get(0); + String tableGuid = atlasEntityHeader.getGuid(); + Optional termAssignmentHeaderOptional = atlasEntityHeader.getMeanings().stream().filter(atlasTermAssignmentHeader -> StringUtils.equals(termGuid, atlasTermAssignmentHeader.getTermGuid())).findFirst(); + if (!termAssignmentHeaderOptional.isPresent()){ + throw new DataGovernanceException(23000, vo.getTableName() + "表与标签" + vo.getLabel() +"未绑定"); + } + try { + atlasService.disassociateTermFromEntities(termGuidOptional.get(), Lists.newArrayList(RelatedObjectId.from(tableGuid, termAssignmentHeaderOptional.get().getRelationGuid()))); + } catch (AtlasServiceException exception1) { + throw new DataGovernanceException(23000, exception1.getMessage()); + } + } + + private List getAtlasEntityHeaders(String tableName) throws AtlasServiceException, DataGovernanceException { + List entityHeaders = atlasService.searchHiveTable0(null, tableName, null,true, 1, 0); + if (CollectionUtils.isEmpty(entityHeaders)) { + throw new DataGovernanceException(23000, tableName + "表不存在"); + } + return entityHeaders; + } + + @Override + public List listLabels(String query, Integer limit, Integer offset) throws Exception { + List atlasEntityHeaders = atlasService.listLabels(query, limit, offset); + Optional labelOptional = atlasService.getRootGlossaryGuid(GlossaryConstant.LABEL); + if (!labelOptional.isPresent()) { + throw new DataGovernanceException(23000, "需要创建 " + GlossaryConstant.LABEL.getRoot() + " glossary "); + } + return atlasEntityHeaders.stream() + .filter(atlasEntityHeader -> + StringUtils.endsWith(atlasEntityHeader.getAttribute("qualifiedName").toString(), GlossaryConstant.LABEL.endWith())) + .map(SearchLabelDTO::from).collect(Collectors.toList()); + } + + public List getHiveTblPartitionByName(String dbName,String tableName) throws Exception { + return metaInfoMapper.getPartInfo(dbName,tableName); + + } + + + private boolean isLabel(String termGuid) { + + AtlasGlossaryTerm term; + try { + term = atlasService.getGlossaryTermDetail(termGuid); + }catch (AtlasServiceException e){ + logger.error(e.getMessage(),e); + return false; + } + + if (term == null){ + return false; + } + return term.getQualifiedName().endsWith(GlossaryConstant.LABEL.endWith()); + } +} diff --git a/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/service/impl/WorkspaceInfoServiceImpl.java b/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/service/impl/WorkspaceInfoServiceImpl.java new file mode 100644 index 0000000000..ce78735b79 --- /dev/null +++ b/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/service/impl/WorkspaceInfoServiceImpl.java @@ -0,0 +1,43 @@ +/* + * + * * Copyright 2019 WeBank + * * + * * Licensed under the Apache License, Version 2.0 (the "License"); + * * you may not use this file except in compliance with the License. + * * You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.data.governance.service.impl; + +import com.webank.wedatasphere.dss.data.governance.dao.WorkspaceInfoMapper; +import com.webank.wedatasphere.dss.data.governance.exception.DataGovernanceException; +import com.webank.wedatasphere.dss.data.governance.service.WorkspaceInfoService; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.util.List; + + +@Service +public class WorkspaceInfoServiceImpl implements WorkspaceInfoService { + + @Autowired + private WorkspaceInfoMapper workspaceInfoMapper; + + + @Override + public List getWorkspaceUsers(int workspaceId,String search) throws DataGovernanceException { + + List workspaceUsers = workspaceInfoMapper.getWorkspaceUsersName(workspaceId,search); + return workspaceUsers; + } +} diff --git a/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/utils/DataSourceUtil.java b/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/utils/DataSourceUtil.java new file mode 100644 index 0000000000..92a4a2316d --- /dev/null +++ b/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/utils/DataSourceUtil.java @@ -0,0 +1,27 @@ +package com.webank.wedatasphere.dss.data.governance.utils; + +import com.alibaba.druid.pool.DruidDataSource; +import com.webank.wedatasphere.dss.data.governance.conf.GovernanceConf; + + +public class DataSourceUtil { + private static DruidDataSource druidDataSource =null; + + private DataSourceUtil(){} + + public static DruidDataSource getDataSource(){ + if(druidDataSource ==null) { + synchronized (DataSourceUtil.class){ + if(druidDataSource ==null){ + druidDataSource = new DruidDataSource(); + druidDataSource.setDriverClassName(GovernanceConf.METASTORE_DATASOURCE_DRIVER.getValue()); + druidDataSource.setUrl(GovernanceConf.METASTORE_DATASOURCE_URL.getValue()); + druidDataSource.setUsername(GovernanceConf.METASTORE_DATASOURCE_USERNAME.getValue()); + druidDataSource.setPassword(GovernanceConf.METASTORE_DATASOURCE_PASSWORD.getValue()); + } + } + } + return druidDataSource; + } + +} diff --git a/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/utils/DateUtil.java b/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/utils/DateUtil.java new file mode 100644 index 0000000000..31a0cd07d0 --- /dev/null +++ b/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/utils/DateUtil.java @@ -0,0 +1,382 @@ +/* + * + * * Copyright 2019 WeBank + * * + * * Licensed under the Apache License, Version 2.0 (the "License"); + * * you may not use this file except in compliance with the License. + * * You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. + * + */ +package com.webank.wedatasphere.dss.data.governance.utils; + +import java.sql.Timestamp; +import java.text.DecimalFormat; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.Calendar; +import java.util.Date; + + +public class DateUtil { + + /** + * 英文简写（默认）如：2010-12-01 + */ + public static String FORMAT_MONTH = "yyyy-MM"; + /** + * 英文简写（默认）如：2010-12-01 + */ + public static String FORMAT_SHORT = "yyyy-MM-dd"; + + public static String FORMAT_SHORT_REPORT = "yyyy/MM/dd"; + /** + * 英文全称 如：2010-12-01 23:15:06 + */ + public static String FORMAT_LONG = "yyyy-MM-dd HH:mm:ss"; + public static String FORMAT_WITHOUT_SECOND = "yyyy-MM-dd HH:mm"; + + public static String FORMAT_MINUTE_REPORT = "yyyy/MM/dd HH:mm"; + public static String FORMAT_MINUTE = "yyyy-MM-dd HH:mm"; + /** + * 精确到毫秒的完整时间 如：yyyy-MM-dd HH:mm:ss.S + */ + public static String FORMAT_FULL = "yyyy-MM-dd HH:mm:ss.S"; + /** + * 中文简写 如：2010年12月01日 + */ + public static String FORMAT_SHORT_CN = "yyyy年MM月dd日"; + /** + * 中文全称 如：2010年12月01日 23时15分06秒 + */ + public static String FORMAT_LONG_CN = "yyyy年MM月dd日 HH时mm分ss秒"; + /** + * 精确到毫秒的完整中文时间 + */ + public static String FORMAT_FULL_CN = "yyyy年MM月dd日 HH时mm分ss秒SSS毫秒"; + + /** + * 获得默认的 date pattern + */ + public static String getDatePattern() { + return FORMAT_LONG; + } + + /** + * 根据预设格式返回当前日期 + * + * @return + */ + public static String getNow() { + return format(new Date()); + } + + /** + * 根据用户格式返回当前日期 + * + * @param format + * @return + */ + public static String getNow(String format) { + return format(new Date(), format); + } + + /** + * 使用预设格式格式化日期 + * + * @param date + * @return + */ + public static String format(Date date) { + return format(date, getDatePattern()); + } + + /** + * 使用用户格式格式化日期 + * + * @param date 日期 + * @param pattern 日期格式 + * @return + */ + public static String format(Date date, String pattern) { + String returnValue = ""; + if (date != null) { + SimpleDateFormat df = new SimpleDateFormat(pattern); + returnValue = df.format(date); + } + return (returnValue); + } + + /** + * 使用预设格式提取字符串日期 + * + * @param strDate 日期字符串 + * @return + */ + public static Date parse(String strDate) { + return parse(strDate, getDatePattern()); + } + + /** + * 使用用户格式提取字符串日期 + * + * @param strDate 日期字符串 + * @param pattern 日期格式 + * @return + */ + public static Date parse(String strDate, String pattern) { + SimpleDateFormat df = new SimpleDateFormat(pattern); + try { + return df.parse(strDate); + } catch (ParseException e) { + e.printStackTrace(); + return null; + } + } + + /** + * 在日期上增加数个整月 + * + * @param date 日期 + * @param n 要增加的月数 + * @return + */ + public static Date addMonth(Date date, int n) { + Calendar cal = Calendar.getInstance(); + cal.setTime(date); + cal.add(Calendar.MONTH, n); + return cal.getTime(); + } + + /** + * 在日期上增加天数 + * + * @param date 日期 + * @param n 要增加的天数 + * @return + */ + public static Date addDay(Date date, int n) { + Calendar cal = Calendar.getInstance(); + cal.setTime(date); + cal.add(Calendar.DATE, n); + return cal.getTime(); + } + + /** + * 获取时间戳 + */ + public static String getTimeString() { + SimpleDateFormat df = new SimpleDateFormat(FORMAT_FULL); + Calendar calendar = Calendar.getInstance(); + return df.format(calendar.getTime()); + } + + /** + * 获取日期年份 + * + * @param date 日期 + * @return + */ + public static String getYear(Date date) { + return format(date).substring(0, 4); + } + + /** + * 按默认格式的字符串距离今天的天数 + * + * @param date 日期字符串 + * @return + */ + public static int countDays(String date) { + long t = Calendar.getInstance().getTime().getTime(); + Calendar c = Calendar.getInstance(); + c.setTime(parse(date)); + long t1 = c.getTime().getTime(); + return (int) (t / 1000 - t1 / 1000) / 3600 / 24; + } + + /** + * 按用户格式字符串距离今天的天数 + * + * @param date 日期字符串 + * @param format 日期格式 + * @return + */ + public static int countDaysAbs(String date, String format) { + long t = Calendar.getInstance().getTime().getTime(); + Calendar c = Calendar.getInstance(); + c.setTime(parse(date, format)); + long t1 = c.getTime().getTime(); + return (int) Math.abs(t / 1000 - t1 / 1000) / 3600 / 24; + } + + /** + * 按用户格式字符串距离今天的天数 + * + * @param date 日期字符串 + * @param format 日期格式 + * @return + */ + public static int countDays(String date, String format) { + long t = Calendar.getInstance().getTime().getTime(); + Calendar c = Calendar.getInstance(); + c.setTime(parse(date, format)); + long t1 = c.getTime().getTime(); + return (int) (t / 1000 - t1 / 1000) / 3600 / 24; + } + + public static String timeStrComplete(String timeStr) { + if (null == timeStr || "".equals(timeStr)) { + return null; + } + String str = timeStr; + str = str.replaceAll("[0-9]", "0"); + + String tempStr = "0000-00-00 00:00:00"; + String tempSplitStr = "1970-01-01 00:00:00"; + int index = tempStr.indexOf(str); + if (0 == index) { + /** + * 说明符合日期格式 截取tempStr后面的部分与timeStr拼接构成时间 + */ + String lastStr = tempSplitStr.substring(timeStr.length()); + return timeStr + lastStr; + } + return null; + } + + public static Timestamp str2Time(String timeStr) { + if (null == timeStr || "".equals(timeStr)) { + return null; + } + String str = timeStr; + str = str.replaceAll("[0-9]", "0"); + + String tempStr = "0000-00-00 00:00:00"; + String tempSplitStr = "1970-01-01 00:00:00"; + int index = tempStr.indexOf(str); + if (0 == index) { + /** + * 说明符合日期格式 截取tempStr后面的部分与timeStr拼接构成时间 + */ + String lastStr = tempSplitStr.substring(timeStr.length()); + return Timestamp.valueOf(timeStr + lastStr); + } + + return null; + } + + public static Timestamp str2TimeMax(String timeStr) { + if (null == timeStr || "".equals(timeStr)) { + return null; + } + String str = timeStr; + str = str.replaceAll("[0-9]", "0"); + + String tempStr = "0000-00-00 00:00:00"; + String tempSplitStr = "1970-01-01 23:59:59"; + int index = tempStr.indexOf(str); + if (0 == index) { + /** + * 说明符合日期格式 截取tempStr后面的部分与timeStr拼接构成时间 + */ + String lastStr = tempSplitStr.substring(timeStr.length()); + return Timestamp.valueOf(timeStr + lastStr); + } + + return null; + } + + public static Date str2DateTime(String timeStr) { + if (null == timeStr || "".equals(timeStr)) { + return null; + } + String str = timeStr; + str = str.replaceAll("[0-9]", "0"); + + String tempStr = "0000-00-00 00:00:00"; + String tempSplitStr = "1970-01-01 00:00:00"; + int index = tempStr.indexOf(str); + if (0 == index) { + /** + * 说明符合日期格式 截取tempStr后面的部分与timeStr拼接构成时间 + */ + String lastStr = tempSplitStr.substring(timeStr.length()); + return parse(timeStr + lastStr); + } + + return null; + } + + public static Date str2DateTimeMax(String timeStr) { + if (null == timeStr || "".equals(timeStr)) { + return null; + } + String str = timeStr; + str = str.replaceAll("[0-9]", "0"); + + String tempStr = "0000-00-00 00:00:00"; + String tempSplitStr = "1970-01-01 23:59:59"; + int index = tempStr.indexOf(str); + if (0 == index) { + /** + * 说明符合日期格式 截取tempStr后面的部分与timeStr拼接构成时间 + */ + String lastStr = tempSplitStr.substring(timeStr.length()); + return parse(timeStr + lastStr); + } + + return null; + } + + /** + * 字符串数字转化为double + * + * @param string + * @return + */ + public static double str2double(String string) { + if (null == string || "".equals(string)) { + return 0; + } + try { + return Double.valueOf(string); + } catch (NumberFormatException e) { + return 0; + } + } + + public static String addHours(Date date,int hours) { + SimpleDateFormat df = new SimpleDateFormat(FORMAT_LONG); + + Calendar cal = Calendar.getInstance(); + cal.setTime(date); + cal.add(Calendar.HOUR, hours); + return df.format(cal.getTime()); + } + + public static Long strToTimeStamp(String date,String format) throws ParseException { + SimpleDateFormat sdf = new SimpleDateFormat(format); + return sdf.parse(date).getTime(); + } + + public static String unixToTimeStr(Long unixtime) { + return new SimpleDateFormat("yyyy MM-dd HH:mm:ss").format(new Date (unixtime)); + } + + public static String unixToTimeStr(Double unixtime){ + String strTime =""; + if(unixtime !=null) { + strTime = new DecimalFormat("#").format(unixtime); + return unixToTimeStr(Long.valueOf(strTime)); + } + return null; + } +} diff --git a/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/utils/HttpClientUtil.java b/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/utils/HttpClientUtil.java new file mode 100644 index 0000000000..762a402eea --- /dev/null +++ b/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/utils/HttpClientUtil.java @@ -0,0 +1,499 @@ +/* + * + * * Copyright 2019 WeBank + * * + * * Licensed under the Apache License, Version 2.0 (the "License"); + * * you may not use this file except in compliance with the License. + * * You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.data.governance.utils; + +import org.apache.http.Consts; +import org.apache.http.HttpEntity; +import org.apache.http.HttpResponse; +import org.apache.http.NameValuePair; +import org.apache.http.client.ClientProtocolException; +import org.apache.http.client.config.RequestConfig; +import org.apache.http.client.entity.UrlEncodedFormEntity; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.entity.ContentType; +import org.apache.http.entity.StringEntity; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClients; +import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; +import org.apache.http.message.BasicNameValuePair; +import org.apache.http.util.EntityUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.net.ssl.TrustManager; +import javax.net.ssl.X509TrustManager; +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.net.SocketTimeoutException; +import java.net.URLEncoder; +import java.security.cert.CertificateException; +import java.text.SimpleDateFormat; +import java.util.*; +import java.util.Map.Entry; + + +@SuppressWarnings("all") +public final class HttpClientUtil { + private final static Logger logger = LoggerFactory.getLogger(HttpClientUtil.class); + public final static int connectTimeout = 5000; + private static PoolingHttpClientConnectionManager connManager = null; + private static CloseableHttpClient httpclient = null; + + /** + * 重写验证方法，取消检测ssl + */ + private static TrustManager trustAllManager = new X509TrustManager() { + @Override + public void checkClientTrusted(java.security.cert.X509Certificate[] arg0, String arg1) + throws CertificateException { + } + @Override + public void checkServerTrusted(java.security.cert.X509Certificate[] arg0, String arg1) + throws CertificateException { + } + @Override + public java.security.cert.X509Certificate[] getAcceptedIssuers() { + return null; + } + + }; + + static { + httpclient = HttpClients.createDefault(); + } + + + /** + * + * @param url + * @param timeout + * @param headerMap + * @param paramsList + * @param encoding + * @return + */ + public static String postForm(String url, int timeout, Map headerMap, List paramsList, String encoding){ + HttpPost post = new HttpPost(url); + try { + if(headerMap != null){ + for(Entry entry : headerMap.entrySet()){ + post.setHeader(entry.getKey(), entry.getValue().toString()); + } + } + //post.setHeader("Content-type", "application/json"); + RequestConfig requestConfig = RequestConfig.custom() + .setSocketTimeout(timeout) + .setConnectTimeout(timeout) + .setConnectionRequestTimeout(timeout) + .setExpectContinueEnabled(false).build(); + post.setConfig(requestConfig); + + post.setEntity(new UrlEncodedFormEntity(paramsList, encoding)); + CloseableHttpResponse response = httpclient.execute(post); + try { + HttpEntity entity = response.getEntity(); + try { + if(entity != null){ + String str = EntityUtils.toString(entity, encoding); + return str; + } + } finally { + if(entity != null){ + entity.getContent().close(); + } + } + } finally { + if(response != null){ + response.close(); + } + } + } catch (Exception e) { + throw new RuntimeException("invoke http post error!",e); + } finally { + post.releaseConnection(); + } + return ""; + } + + /** + * 调用saltapi时 + * + * @author: XIEJIAN948@pingan.com.cn + */ + public static String postJsonBody(String url, int timeout, Map headerMap, + String paraData, String encoding) { + + logger.info("successfully start post Json Body url{} ", url); + HttpPost post = new HttpPost(url); + try { + if (headerMap != null) { + for (Entry entry : headerMap.entrySet()) { + post.setHeader(entry.getKey(), entry.getValue().toString()); + } + } + RequestConfig requestConfig = RequestConfig.custom().setSocketTimeout(timeout).setConnectTimeout(timeout) + .setConnectionRequestTimeout(timeout).setExpectContinueEnabled(false).build(); + StringEntity jsonEntity = new StringEntity(paraData, ContentType.APPLICATION_JSON); + post.setConfig(requestConfig); + post.setEntity(jsonEntity); + CloseableHttpResponse response = httpclient.execute(post); + try { + HttpEntity entity = response.getEntity(); + try { + if (entity != null) { + String str = EntityUtils.toString(entity, encoding); + return str; + } + } finally { + if (entity != null) { + entity.getContent().close(); + } + } + } finally { + if (response != null) { + response.close(); + } + } + } catch (UnsupportedEncodingException e) { + logger.error("UnsupportedEncodingException", e); + throw new RuntimeException("failed post json return blank!"); + } catch (Exception e) { + logger.error("Exception", e); + throw new RuntimeException("failed post json return blank!"); + } finally { + post.releaseConnection(); + } + logger.info("successfully end post Json Body url{} ", url); + return ""; + } + + @SuppressWarnings("deprecation") + public static String invokeGet(String url, Map params, String encode, int connectTimeout, + int soTimeout) { + String responseString = null; + RequestConfig requestConfig = RequestConfig.custom().setSocketTimeout(connectTimeout) + .setConnectTimeout(connectTimeout).setConnectionRequestTimeout(connectTimeout).build(); + + StringBuilder sb = new StringBuilder(); + sb.append(url); + int i = 0; + if (params != null) { + for (Entry entry : params.entrySet()) { + if (i == 0 && !url.contains("?")) { + sb.append("?"); + } else { + sb.append("&"); + } + sb.append(entry.getKey()); + sb.append("="); + String value = entry.getValue(); + try { + sb.append(URLEncoder.encode(value, "UTF-8")); + } catch (UnsupportedEncodingException e) { + logger.warn("encode http get params error, value is " + value, e); + sb.append(URLEncoder.encode(value)); + } + i++; + } + } + HttpGet get = new HttpGet(sb.toString()); + get.setConfig(requestConfig); + try { + CloseableHttpResponse response = httpclient.execute(get); + try { + HttpEntity entity = response.getEntity(); + try { + if (entity != null) { + responseString = EntityUtils.toString(entity, encode); + } + } finally { + if (entity != null) { + entity.getContent().close(); + } + } + } catch (Exception e) { + logger.error(String.format("[HttpUtils Get]get response error, url:%s", sb.toString()), e); + return responseString; + } finally { + if (response != null) { + response.close(); + } + } + // System.out.println(String.format("[HttpUtils Get]Debug url:%s , + // response string %s:", sb.toString(), responseString)); + } catch (SocketTimeoutException e) { + logger.error(String.format("[HttpUtils Get]invoke get timout error, url:%s", sb.toString()), e); + return responseString; + } catch (Exception e) { + logger.error(String.format("[HttpUtils Get]invoke get error, url:%s", sb.toString()), e); + } finally { + get.releaseConnection(); + } + return responseString; + } + + /** + * HTTPS请求，默认超时为5S + * + * @param reqURL + * @param params + * @return + */ + public static String connectPostHttps(String reqURL, Map params) { + + String responseContent = null; + HttpPost httpPost = new HttpPost(reqURL); + try { + RequestConfig requestConfig = RequestConfig.custom().setSocketTimeout(connectTimeout) + .setConnectTimeout(connectTimeout).setConnectionRequestTimeout(connectTimeout).build(); + List formParams = new ArrayList(); + httpPost.setEntity(new UrlEncodedFormEntity(formParams, Consts.UTF_8)); + httpPost.setConfig(requestConfig); + // 绑定到请求 Entry + for (Entry entry : params.entrySet()) { + formParams.add(new BasicNameValuePair(entry.getKey(), entry.getValue())); + } + CloseableHttpResponse response = httpclient.execute(httpPost); + try { + // 执行POST请求 + HttpEntity entity = response.getEntity(); // 获取响应实体 + try { + if (null != entity) { + responseContent = EntityUtils.toString(entity, Consts.UTF_8); + } + } finally { + if (entity != null) { + entity.getContent().close(); + } + } + } finally { + if (response != null) { + response.close(); + } + } + logger.info("requestURI : " + httpPost.getURI() + ", responseContent: " + responseContent); + } catch (ClientProtocolException e) { + logger.error("ClientProtocolException", e); + } catch (IOException e) { + logger.error("IOException", e); + } finally { + httpPost.releaseConnection(); + } + return responseContent; + + } + + class Test { + String v; + String k; + + public String getV() { + return v; + } + + public void setV(String v) { + this.v = v; + } + + public String getK() { + return k; + } + + public void setK(String k) { + this.k = k; + } + + } + + // 随机4位数 + public static String getRandomValue() { + String str = "0123456789"; + StringBuilder sb = new StringBuilder(4); + for (int i = 0; i < 4; i++) { + char ch = str.charAt(new Random().nextInt(str.length())); + sb.append(ch); + } + return sb.toString(); + } + + // 当前时间到秒 + public static String getTimestamp() { + + Date date = new Date(); + String timestamp = String.valueOf(date.getTime() / 1000); + return timestamp; + } + + // 当前时间到秒 + public static String getNowDate() { + Date date = new Date(); + SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMddHHmmss"); + return sdf.format(date); + } + + /** + * 调用saltapi时 + * + * @author: XIEJIAN948@pingan.com.cn + */ + public static String postJsonBody2(String url, int timeout, Map headerMap, + List paramsList, String encoding) { + logger.info("successfully start post Json Body url{} ", url); + HttpPost post = new HttpPost(url); + try { + if (headerMap != null) { + for (Entry entry : headerMap.entrySet()) { + post.setHeader(entry.getKey(), entry.getValue().toString()); + } + } + RequestConfig requestConfig = RequestConfig.custom().setSocketTimeout(timeout).setConnectTimeout(timeout) + .setConnectionRequestTimeout(timeout).setExpectContinueEnabled(false).build(); + post.setConfig(requestConfig); + if (paramsList.size() > 0) { + UrlEncodedFormEntity entity = new UrlEncodedFormEntity(paramsList, encoding); + post.setEntity(entity); + } + CloseableHttpResponse response = httpclient.execute(post); + try { + HttpEntity entity = response.getEntity(); + try { + if (entity != null) { + String str = EntityUtils.toString(entity, encoding); + return str; + } + } finally { + if (entity != null) { + entity.getContent().close(); + } + } + } finally { + if (response != null) { + response.close(); + } + } + } catch (UnsupportedEncodingException e) { + logger.error("UnsupportedEncodingException", e); + throw new RuntimeException("failed post json return blank!"); + } catch (Exception e) { + logger.error("Exception", e); + throw new RuntimeException("failed post json return blank!"); + } finally { + post.releaseConnection(); + } + logger.info("successfully end post Json Body url{} ", url); + return ""; + } + + /** + * 调用saltapi时 + * + * @author: XIEJIAN948@pingan.com.cn + */ + public static String postJsonBody3(String url, int timeout, Map headerMap, + Map paramsList, String encoding) { + HttpPost post = new HttpPost(url); + try { + if (headerMap != null) { + for (Entry entry : headerMap.entrySet()) { + post.setHeader(entry.getKey(), entry.getValue().toString()); + } + } + RequestConfig requestConfig = RequestConfig.custom().setSocketTimeout(timeout).setConnectTimeout(timeout) + .setConnectionRequestTimeout(timeout).setExpectContinueEnabled(false).build(); + post.setConfig(requestConfig); + if (paramsList.size() > 0) { + //JSONArray jsonArray = JSONArray.fromObject(paramsList); + //post.setEntity(new StringEntity(jsonArray.get(0).toString(), encoding)); + post.setEntity(new StringEntity(null, encoding)); + //logger.info("successfully start post Json Body url{},params ", url,jsonArray.get(0).toString()); + logger.info("successfully start post Json Body url{},params ", url,null); + } + CloseableHttpResponse response = httpclient.execute(post); + try { + HttpEntity entity = response.getEntity(); + try { + if (entity != null) { + String str = EntityUtils.toString(entity, encoding); + return str; + } + } finally { + if (entity != null) { + entity.getContent().close(); + } + } + } finally { + if (response != null) { + response.close(); + } + } + } catch (UnsupportedEncodingException e) { + logger.error("UnsupportedEncodingException", e); + throw new RuntimeException("failed post json return blank!"); + } catch (Exception e) { + logger.error("Exception", e); + throw new RuntimeException("failed post json return blank!"); + } finally { + post.releaseConnection(); + } + logger.info("successfully end post Json Body url{} ", url); + return ""; + } + + public static String executeGet(String url) + { + String rtnStr = ""; + HttpGet httpGet = new HttpGet(url); + try { + HttpResponse httpResponse = httpclient.execute(httpGet); + //获得返回的结果 + rtnStr = EntityUtils.toString(httpResponse.getEntity()); + } catch (IOException e) { + e.printStackTrace(); + } finally { + httpGet.releaseConnection(); + } + return rtnStr; + } + + + public static String executeGetByHeader(String url,Map headerMap) + { + String rtnStr = ""; + HttpGet httpGet = new HttpGet(url); + if (headerMap != null) { + for (Entry entry : headerMap.entrySet()) { + httpGet.setHeader(entry.getKey(), entry.getValue().toString()); + } + } + try { + HttpResponse httpResponse = httpclient.execute(httpGet); + System.out.println(httpResponse.getStatusLine().getStatusCode()); + //获得返回的结果 + rtnStr = EntityUtils.toString(httpResponse.getEntity()); + } catch (IOException e) { + e.printStackTrace(); + } finally { + httpGet.releaseConnection(); + } + return rtnStr; + } + +} diff --git a/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/utils/RestfulUtils.java b/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/utils/RestfulUtils.java new file mode 100644 index 0000000000..e2f0d966d0 --- /dev/null +++ b/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/utils/RestfulUtils.java @@ -0,0 +1,50 @@ +/* + * + * * Copyright 2019 WeBank + * * + * * Licensed under the Apache License, Version 2.0 (the "License"); + * * you may not use this file except in compliance with the License. + * * You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.data.governance.utils; + +import org.apache.linkis.server.Message; +import org.apache.commons.math3.util.Pair; + +import javax.ws.rs.core.Response; +import java.util.Arrays; + + +public class RestfulUtils { + + public static Response dealError(String reason){ + Message message = Message.error(reason); + return Message.messageToResponse(message); + } + + public static Response dealOk(String msg){ + Message message = Message.ok(msg); + return Message.messageToResponse(message); + } + + + + @SafeVarargs + public static Response dealOk(String msg, Pair... data){ + Message message = Message.ok(msg); + Arrays.stream(data).forEach(p -> message.data(p.getKey(), p.getValue())); + return Message.messageToResponse(message); + } + + +} diff --git a/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/vo/BindLabelVO.java b/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/vo/BindLabelVO.java new file mode 100644 index 0000000000..dd2d23095a --- /dev/null +++ b/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/vo/BindLabelVO.java @@ -0,0 +1,17 @@ +package com.webank.wedatasphere.dss.data.governance.vo; + + +import lombok.Data; + +@Data +public class BindLabelVO { + + private String label; + + private String tableName; + + private String labelGuid; + + private String tableGuid; + +} diff --git a/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/vo/BindModelVO.java b/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/vo/BindModelVO.java new file mode 100644 index 0000000000..572f15ca9f --- /dev/null +++ b/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/vo/BindModelVO.java @@ -0,0 +1,16 @@ +package com.webank.wedatasphere.dss.data.governance.vo; + + +import lombok.Data; + +@Data +public class BindModelVO { + + private String modelName; + + private String modelType; + + private String tableName; + + private String guid; +} diff --git a/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/vo/CreateLabelVO.java b/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/vo/CreateLabelVO.java new file mode 100644 index 0000000000..f25c96328e --- /dev/null +++ b/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/vo/CreateLabelVO.java @@ -0,0 +1,9 @@ +package com.webank.wedatasphere.dss.data.governance.vo; + + +import lombok.Data; + +@Data +public class CreateLabelVO { + private String name; +} diff --git a/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/vo/CreateModelTypeVO.java b/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/vo/CreateModelTypeVO.java new file mode 100644 index 0000000000..53b33ed390 --- /dev/null +++ b/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/vo/CreateModelTypeVO.java @@ -0,0 +1,12 @@ +package com.webank.wedatasphere.dss.data.governance.vo; + + +import lombok.Data; + +@Data +public class CreateModelTypeVO { + + private String name; + + private String type; +} diff --git a/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/vo/DeleteLabelVO.java b/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/vo/DeleteLabelVO.java new file mode 100644 index 0000000000..9488c26e5f --- /dev/null +++ b/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/vo/DeleteLabelVO.java @@ -0,0 +1,9 @@ +package com.webank.wedatasphere.dss.data.governance.vo; + + +import lombok.Data; + +@Data +public class DeleteLabelVO { + private String name; +} diff --git a/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/vo/DeleteModelTypeVO.java b/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/vo/DeleteModelTypeVO.java new file mode 100644 index 0000000000..0079ae6cb2 --- /dev/null +++ b/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/vo/DeleteModelTypeVO.java @@ -0,0 +1,11 @@ +package com.webank.wedatasphere.dss.data.governance.vo; + + +import lombok.Data; + +@Data +public class DeleteModelTypeVO { + private String name; + + private String type; +} diff --git a/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/vo/UnBindLabelVO.java b/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/vo/UnBindLabelVO.java new file mode 100644 index 0000000000..2f50ddcaaa --- /dev/null +++ b/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/vo/UnBindLabelVO.java @@ -0,0 +1,17 @@ +package com.webank.wedatasphere.dss.data.governance.vo; + + +import lombok.Data; + +@Data +public class UnBindLabelVO { + private String label; + + private String tableName; + + private String labelGuid; + + private String tableGuid; + + private String relationGuid; +} diff --git a/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/vo/UnBindModelVO.java b/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/vo/UnBindModelVO.java new file mode 100644 index 0000000000..d507df30f9 --- /dev/null +++ b/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/vo/UnBindModelVO.java @@ -0,0 +1,14 @@ +package com.webank.wedatasphere.dss.data.governance.vo; + +import lombok.Data; + +@Data +public class UnBindModelVO { + private String modelName; + + private String modelType; + + private String tableName; + + private String guid; +} diff --git a/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/vo/UpdateLabelVO.java b/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/vo/UpdateLabelVO.java new file mode 100644 index 0000000000..835ee3f843 --- /dev/null +++ b/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/vo/UpdateLabelVO.java @@ -0,0 +1,10 @@ +package com.webank.wedatasphere.dss.data.governance.vo; + +import lombok.Data; + +@Data +public class UpdateLabelVO { + private String name; + + private String orgName; +} diff --git a/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/vo/UpdateModelTypeVO.java b/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/vo/UpdateModelTypeVO.java new file mode 100644 index 0000000000..07840d41ea --- /dev/null +++ b/dss-apps/dss-dataasset-management/data-assets-server/src/main/java/com/webank/wedatasphere/dss/data/governance/vo/UpdateModelTypeVO.java @@ -0,0 +1,13 @@ +package com.webank.wedatasphere.dss.data.governance.vo; + + +import lombok.Data; + +@Data +public class UpdateModelTypeVO { + private String name; + + private String orgName; + + private String type; +} diff --git a/dss-apps/dss-dataasset-management/data-assets-server/src/main/resources/application.yml b/dss-apps/dss-dataasset-management/data-assets-server/src/main/resources/application.yml new file mode 100644 index 0000000000..257ead5789 --- /dev/null +++ b/dss-apps/dss-dataasset-management/data-assets-server/src/main/resources/application.yml @@ -0,0 +1,19 @@ +server: + port: 20082 +spring: + application: + name: data-assets-server +eureka: + client: + serviceUrl: + defaultZone: http://localhost:20303/eureka/ +management: + endpoints: + web: + exposure: + include: refresh,info +logging: + config: classpath:log4j2.xml +#mybatis-plus: +# configuration: +# log-impl: org.apache.ibatis.logging.stdout.StdOutImpl \ No newline at end of file diff --git a/dss-apps/dss-dataasset-management/data-assets-server/src/main/resources/linkis.properties b/dss-apps/dss-dataasset-management/data-assets-server/src/main/resources/linkis.properties new file mode 100644 index 0000000000..7aaada8c63 --- /dev/null +++ b/dss-apps/dss-dataasset-management/data-assets-server/src/main/resources/linkis.properties @@ -0,0 +1,82 @@ +# +# /* +# * Copyright 2019 WeBank +# * +# * Licensed under the Apache License, Version 2.0 (the "License"); +# * you may not use this file except in compliance with the License. +# * You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ +# + +wds.linkis.test.mode=true +wds.linkis.log.clear=true +wds.linkis.server.version=v1 + + +wds.linkis.server.mybatis.datasource.url=jdbc:mysql://localhost:3306/hive?useSSL=false&characterEncoding=UTF-8 + +wds.linkis.server.mybatis.datasource.username= + +wds.linkis.server.mybatis.datasource.password= + + + + + +wds.linkis.server.restful.scan.packages=com.webank.wedatasphere.dss.data.governance.restful + +##mybatis +wds.linkis.server.mybatis.mapperLocations=classpath*:com/webank/wedatasphere/dss/data/governance/dao/impl/*.xml + +wds.linkis.server.mybatis.typeAliasesPackage=com.webank.wedatasphere.dss.data.governance.entity + +wds.linkis.server.mybatis.BasePackage=com.webank.wedatasphere.dss.data.governance.dao + + +# atlas config +atlas.rest.address=http://localhost:21000 +atlas.username=admin +atlas.password=admin +atlas.client.readTimeoutMSecs=60000 +atlas.client.connectTimeoutMSecs=60000 + +# create root classification +atlas.root.indicator=datamodel_indicator +atlas.root.measure=datamodel_measure +atlas.root.dimension=datamodel_dimension +atlas.root.layer=datamodel_layer +atlas.root.theme=datamodel_theme + +# create root glossary +atlas.root.label=datamodel_label +atlas.root.collection=datamodel_collection + + +# hive metadata config +metastore.datasource.driver=com.mysql.jdbc.Driver +metastore.datasource.url=jdbc:mysql://localhost:3306/hive?characterEncoding=UTF-8 +metastore.datasource.username= +metastore.datasource.password= + + +wds.workspace.client.serverurl=http://localhost:29001 + +wds.workspace.client.authtoken.key=hdfs + +wds.workspace.client.authenticationStrategy=org.apache.linkis.httpclient.dws.authentication.TokenAuthenticationStrategy + +wds.workspace.client.discovery.enabled=false + +wds.workspace.client.authtoken.value=hdfs + +wds.workspace.client.dws.version=v1 + +wds.wedatasphere.data.assert.client.strategy=static \ No newline at end of file diff --git a/dss-apps/dss-dataasset-management/data-assets-server/src/main/resources/log4j.properties b/dss-apps/dss-dataasset-management/data-assets-server/src/main/resources/log4j.properties new file mode 100644 index 0000000000..0807e60877 --- /dev/null +++ b/dss-apps/dss-dataasset-management/data-assets-server/src/main/resources/log4j.properties @@ -0,0 +1,37 @@ +# +# Copyright 2019 WeBank +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# + +### set log levels ### + +log4j.rootCategory=INFO,console + +log4j.appender.console=org.apache.log4j.ConsoleAppender +log4j.appender.console.Threshold=INFO +log4j.appender.console.layout=org.apache.log4j.PatternLayout +#log4j.appender.console.layout.ConversionPattern= %d{ISO8601} %-5p (%t) [%F:%M(%L)] - %m%n +log4j.appender.console.layout.ConversionPattern= %d{ISO8601} %-5p (%t) %p %c{1} - %m%n + + +log4j.appender.com.webank.bdp.ide.core=org.apache.log4j.DailyRollingFileAppender +log4j.appender.com.webank.bdp.ide.core.Threshold=INFO +log4j.additivity.com.webank.bdp.ide.core=false +log4j.appender.com.webank.bdp.ide.core.layout=org.apache.log4j.PatternLayout +log4j.appender.com.webank.bdp.ide.core.Append=true +log4j.appender.com.webank.bdp.ide.core.File=logs/linkis.log +log4j.appender.com.webank.bdp.ide.core.layout.ConversionPattern= %d{ISO8601} %-5p (%t) [%F:%M(%L)] - %m%n + +log4j.logger.org.springframework=INFO diff --git a/dss-apps/dss-dataasset-management/data-assets-server/src/main/resources/log4j2.xml b/dss-apps/dss-dataasset-management/data-assets-server/src/main/resources/log4j2.xml new file mode 100644 index 0000000000..3923cd9f39 --- /dev/null +++ b/dss-apps/dss-dataasset-management/data-assets-server/src/main/resources/log4j2.xml @@ -0,0 +1,39 @@ + + + + + + + + + + + + + + + + + + + + + + + diff --git a/dss-apps/dss-dataasset-management/data-assets-server/src/main/scala/com/webank/wedatasphere/dss/data/governance/DataAssetsApplication.scala b/dss-apps/dss-dataasset-management/data-assets-server/src/main/scala/com/webank/wedatasphere/dss/data/governance/DataAssetsApplication.scala new file mode 100644 index 0000000000..d2f77b9db9 --- /dev/null +++ b/dss-apps/dss-dataasset-management/data-assets-server/src/main/scala/com/webank/wedatasphere/dss/data/governance/DataAssetsApplication.scala @@ -0,0 +1,32 @@ +/* + * + * * Copyright 2019 WeBank + * * + * * Licensed under the Apache License, Version 2.0 (the "License"); + * * you may not use this file except in compliance with the License. + * * You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.data.governance + +import org.apache.linkis.common.utils.Logging +import org.apache.linkis.{DataWorkCloudApplication, LinkisBaseServerApp} + + + +object DataAssetsApplication extends Logging { + + + def main(args: Array[String]): Unit = { + DataWorkCloudApplication.main(args) + } +} \ No newline at end of file diff --git a/dss-apps/dss-dataasset-management/pom.xml b/dss-apps/dss-dataasset-management/pom.xml new file mode 100644 index 0000000000..4dbf3f850e --- /dev/null +++ b/dss-apps/dss-dataasset-management/pom.xml @@ -0,0 +1,112 @@ + + + + dss + com.webank.wedatasphere.dss + 1.0.1 + ../../pom.xml + + 4.0.0 + + dss-dataasset-management + pom + + data-assets-server + data-assets-client + + + UTF-8 + 2.16 + 2.22.2 + 2.22.2 + + + + + org.apache.linkis + linkis-module + ${linkis.version} + + + asm + org.ow2.asm + + + hk2-api + org.glassfish.hk2 + + + jersey-common + org.glassfish.jersey.core + + + linkis-common + org.apache.linkis + + + + + org.apache.linkis + linkis-mybatis + ${linkis.version} + + + + org.modelmapper + modelmapper + 2.4.4 + + + + junit + junit + 4.12 + test + + + org.junit.platform + junit-platform-launcher + 1.5.2 + test + + + + + + + + maven-surefire-plugin + ${maven-surefire-plugin.version} + + + maven-failsafe-plugin + ${maven-failsafe-plugin.version} + + + org.apache.maven.plugins + maven-deploy-plugin + + + + net.alchim31.maven + scala-maven-plugin + + + org.apache.maven.plugins + maven-jar-plugin + + + + + src/main/java + + **/*.xml + + + + ${project.artifactId}-${project.version} + + + \ No newline at end of file diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-center-client/pom.xml b/dss-apps/dss-datamodel-center/dss-datamodel-center-client/pom.xml new file mode 100644 index 0000000000..dbf8072de1 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-center-client/pom.xml @@ -0,0 +1,57 @@ + + + + dss-datamodel-center + com.webank.wedatasphere.dss + 1.0.1 + + 4.0.0 + + dss-datamodel-center-client + + + 8 + 8 + + + + + org.apache.linkis + linkis-gateway-httpclient-support + ${linkis.version} + + + org.projectlombok + lombok + 1.18.16 + compile + + + + + + + + org.apache.maven.plugins + maven-deploy-plugin + + + + net.alchim31.maven + scala-maven-plugin + + + org.apache.maven.plugins + maven-jar-plugin + + + + + src/main/resources + + + ${project.artifactId}-${project.version} + + \ No newline at end of file diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-center-client/src/main/java/com/webank/wedatasphere/dss/datamodel/center/client/entity/ClassificationConstant.java b/dss-apps/dss-datamodel-center/dss-datamodel-center-client/src/main/java/com/webank/wedatasphere/dss/datamodel/center/client/entity/ClassificationConstant.java new file mode 100644 index 0000000000..723418e5b0 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-center-client/src/main/java/com/webank/wedatasphere/dss/datamodel/center/client/entity/ClassificationConstant.java @@ -0,0 +1,76 @@ +package com.webank.wedatasphere.dss.datamodel.center.client.entity; + + +import org.apache.commons.lang.StringUtils; + +import java.util.Optional; + +public enum ClassificationConstant { + /** + * 指标 + */ + INDICATOR(1, "indicator"), + /** + * 度量 + */ + MEASURE(2,"measure"), + /** + * 维度 + */ + DIMENSION(0, "dimension"), + /** + * 主题 + */ + THEME(3, "theme"), + /** + * 分层 + */ + LAYER(4, "layer"); + + private int type; + + + private String typeCode; + + ClassificationConstant(int type, String typeCode) { + this.type = type; + + this.typeCode = typeCode; + } + + public static boolean isTypeScope(int type) { + return type >= 0 && type < values().length; + } + + public static boolean isTypeScope(String type) { + return getClassificationConstantByTypeCode(type).isPresent(); + } + + public static Optional getClassificationConstantByTypeCode(String typeCode) { + for (ClassificationConstant c : values()) { + if (StringUtils.equals(c.typeCode,typeCode)) { + return Optional.of(c); + } + } + return Optional.empty(); + } + + public static Optional getClassificationConstantByType(int type) { + for (ClassificationConstant c : values()) { + if (c.type == type) { + return Optional.of(c); + } + } + return Optional.empty(); + } + + + + public int getType() { + return type; + } + + public String getTypeCode() { + return typeCode; + } +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-center-client/src/main/java/com/webank/wedatasphere/dss/datamodel/center/client/entity/CreateModelTypeInfo.java b/dss-apps/dss-datamodel-center/dss-datamodel-center-client/src/main/java/com/webank/wedatasphere/dss/datamodel/center/client/entity/CreateModelTypeInfo.java new file mode 100644 index 0000000000..e3a210f784 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-center-client/src/main/java/com/webank/wedatasphere/dss/datamodel/center/client/entity/CreateModelTypeInfo.java @@ -0,0 +1,12 @@ +package com.webank.wedatasphere.dss.datamodel.center.client.entity; + + +import lombok.Data; + +@Data +public class CreateModelTypeInfo { + private String name; + + private String guid; + +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-center-client/src/main/java/com/webank/wedatasphere/dss/datamodel/center/client/entity/HiveSimpleInfo.java b/dss-apps/dss-datamodel-center/dss-datamodel-center-client/src/main/java/com/webank/wedatasphere/dss/datamodel/center/client/entity/HiveSimpleInfo.java new file mode 100644 index 0000000000..36f363ac29 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-center-client/src/main/java/com/webank/wedatasphere/dss/datamodel/center/client/entity/HiveSimpleInfo.java @@ -0,0 +1,23 @@ +package com.webank.wedatasphere.dss.datamodel.center.client.entity; + +import lombok.Data; + +import java.io.Serializable; +import java.util.List; + + +@Data +public class HiveSimpleInfo implements Serializable { + private String guid; + private String name; + private String qualifiedName; + private String createTime; + private String owner; + private String aliases; + private String lastAccessTime; + private String comment; + private List classifications; + private String totalSize; + + +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-center-client/src/main/java/com/webank/wedatasphere/dss/datamodel/center/client/entity/UpdateModelTypeInfo.java b/dss-apps/dss-datamodel-center/dss-datamodel-center-client/src/main/java/com/webank/wedatasphere/dss/datamodel/center/client/entity/UpdateModelTypeInfo.java new file mode 100644 index 0000000000..a8b3ffa67e --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-center-client/src/main/java/com/webank/wedatasphere/dss/datamodel/center/client/entity/UpdateModelTypeInfo.java @@ -0,0 +1,11 @@ +package com.webank.wedatasphere.dss.datamodel.center.client.entity; + + +import lombok.Data; + +@Data +public class UpdateModelTypeInfo { + private String name; + + private String guid; +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-center-client/src/main/scala/com/webank/wedatasphere/dss/datamodel/center/client/AbstractRemoteClient.scala b/dss-apps/dss-datamodel-center/dss-datamodel-center-client/src/main/scala/com/webank/wedatasphere/dss/datamodel/center/client/AbstractRemoteClient.scala new file mode 100644 index 0000000000..90a2bc2f0d --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-center-client/src/main/scala/com/webank/wedatasphere/dss/datamodel/center/client/AbstractRemoteClient.scala @@ -0,0 +1,15 @@ +package com.webank.wedatasphere.dss.datamodel.center.client + +import org.apache.linkis.httpclient.dws.DWSHttpClient +import org.apache.linkis.httpclient.request.Action +import org.apache.linkis.httpclient.response.Result + +abstract class AbstractRemoteClient extends RemoteClient { + protected val dwsHttpClient:DWSHttpClient + + override def execute(action: Action): Result = action match { + case action: Action => dwsHttpClient.execute(action) + } + + override def close(): Unit = dwsHttpClient.close() +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-center-client/src/main/scala/com/webank/wedatasphere/dss/datamodel/center/client/DatamodelCenterRemoteClient.scala b/dss-apps/dss-datamodel-center/dss-datamodel-center-client/src/main/scala/com/webank/wedatasphere/dss/datamodel/center/client/DatamodelCenterRemoteClient.scala new file mode 100644 index 0000000000..0632b21164 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-center-client/src/main/scala/com/webank/wedatasphere/dss/datamodel/center/client/DatamodelCenterRemoteClient.scala @@ -0,0 +1,11 @@ +package com.webank.wedatasphere.dss.datamodel.center.client + +import com.webank.wedatasphere.dss.datamodel.center.client.request._ +import com.webank.wedatasphere.dss.datamodel.center.client.response._ + +trait DatamodelCenterRemoteClient extends RemoteClient { + def themesReference(action: ThemesReferenceAction) : ThemesReferenceResult + def layersReference(action: LayersReferenceAction) : LayersReferenceResult + def modifiersReference(action: ModifiersReferenceAction):ModifiersReferenceResult + def cyclesReference(action:CyclesReferenceAction) : CyclesReferenceResult +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-center-client/src/main/scala/com/webank/wedatasphere/dss/datamodel/center/client/RemoteClient.scala b/dss-apps/dss-datamodel-center/dss-datamodel-center-client/src/main/scala/com/webank/wedatasphere/dss/datamodel/center/client/RemoteClient.scala new file mode 100644 index 0000000000..614bcfa721 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-center-client/src/main/scala/com/webank/wedatasphere/dss/datamodel/center/client/RemoteClient.scala @@ -0,0 +1,11 @@ +package com.webank.wedatasphere.dss.datamodel.center.client + +import org.apache.linkis.httpclient.request.Action +import org.apache.linkis.httpclient.response.Result +import java.io.Closeable + +trait RemoteClient extends Closeable{ + protected def execute(action: Action): Result + + override def close(): Unit +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-center-client/src/main/scala/com/webank/wedatasphere/dss/datamodel/center/client/exception/DatamodelCenterClientBuilderException.scala b/dss-apps/dss-datamodel-center/dss-datamodel-center-client/src/main/scala/com/webank/wedatasphere/dss/datamodel/center/client/exception/DatamodelCenterClientBuilderException.scala new file mode 100644 index 0000000000..9ac06cd545 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-center-client/src/main/scala/com/webank/wedatasphere/dss/datamodel/center/client/exception/DatamodelCenterClientBuilderException.scala @@ -0,0 +1,5 @@ +package com.webank.wedatasphere.dss.datamodel.center.client.exception + +import org.apache.linkis.common.exception.ErrorException + +class DatamodelCenterClientBuilderException(errorDesc: String) extends ErrorException(212400, errorDesc) diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-center-client/src/main/scala/com/webank/wedatasphere/dss/datamodel/center/client/impl/LinkisDatamodelCenterRemoteClient.scala b/dss-apps/dss-datamodel-center/dss-datamodel-center-client/src/main/scala/com/webank/wedatasphere/dss/datamodel/center/client/impl/LinkisDatamodelCenterRemoteClient.scala new file mode 100644 index 0000000000..7f2a67284f --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-center-client/src/main/scala/com/webank/wedatasphere/dss/datamodel/center/client/impl/LinkisDatamodelCenterRemoteClient.scala @@ -0,0 +1,19 @@ +package com.webank.wedatasphere.dss.datamodel.center.client.impl + +import com.webank.wedatasphere.dss.datamodel.center.client.request.{CyclesReferenceAction, LayersReferenceAction, ModifiersReferenceAction, ThemesReferenceAction} +import com.webank.wedatasphere.dss.datamodel.center.client.response.{CyclesReferenceResult, LayersReferenceResult, ModifiersReferenceResult, ThemesReferenceResult} +import com.webank.wedatasphere.dss.datamodel.center.client.{AbstractRemoteClient, DatamodelCenterRemoteClient} +import org.apache.linkis.httpclient.dws.DWSHttpClient +import org.apache.linkis.httpclient.dws.config.DWSClientConfig + +class LinkisDatamodelCenterRemoteClient(clientConfig: DWSClientConfig) extends AbstractRemoteClient with DatamodelCenterRemoteClient { + override protected val dwsHttpClient: DWSHttpClient = new DWSHttpClient(clientConfig, "Datamodel-center-Client") + + override def themesReference(action: ThemesReferenceAction): ThemesReferenceResult = execute(action).asInstanceOf[ThemesReferenceResult] + + override def layersReference(action: LayersReferenceAction): LayersReferenceResult = execute(action).asInstanceOf[LayersReferenceResult] + + override def modifiersReference(action: ModifiersReferenceAction): ModifiersReferenceResult = execute(action).asInstanceOf[ModifiersReferenceResult] + + override def cyclesReference(action: CyclesReferenceAction): CyclesReferenceResult = execute(action).asInstanceOf[CyclesReferenceResult] +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-center-client/src/main/scala/com/webank/wedatasphere/dss/datamodel/center/client/request/CyclesReferenceAction.scala b/dss-apps/dss-datamodel-center/dss-datamodel-center-client/src/main/scala/com/webank/wedatasphere/dss/datamodel/center/client/request/CyclesReferenceAction.scala new file mode 100644 index 0000000000..ef267eefaf --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-center-client/src/main/scala/com/webank/wedatasphere/dss/datamodel/center/client/request/CyclesReferenceAction.scala @@ -0,0 +1,43 @@ +package com.webank.wedatasphere.dss.datamodel.center.client.request + +import com.webank.wedatasphere.dss.datamodel.center.client.exception.DatamodelCenterClientBuilderException +import org.apache.linkis.httpclient.request.GetAction + +class CyclesReferenceAction extends GetAction with DatamodelCenterAction{ + + private var name:String=_ + + private var user:String = _ + + override def setUser(user: String): Unit = this.user = user + + override def getUser: String = this.user + + override def suffixURLs: Array[String] = Array("datamodel", "cycles", "reference",name) +} +object CyclesReferenceAction{ + def builder(): Builder = new Builder + + class Builder private[CyclesReferenceAction]() { + private var name:String = _ + private var user: String = _ + + def setUser(user: String): Builder ={ + this.user = user + this + } + def setName(name:String): Builder = { + this.name = name + this + } + + def build(): CyclesReferenceAction = { + val action = new CyclesReferenceAction + if(name == null) throw new DatamodelCenterClientBuilderException("name is needed!") + action.name = name + action.setUser(user) + action + } + } +} + diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-center-client/src/main/scala/com/webank/wedatasphere/dss/datamodel/center/client/request/DatamodelCenterAction.scala b/dss-apps/dss-datamodel-center/dss-datamodel-center-client/src/main/scala/com/webank/wedatasphere/dss/datamodel/center/client/request/DatamodelCenterAction.scala new file mode 100644 index 0000000000..5d238c773b --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-center-client/src/main/scala/com/webank/wedatasphere/dss/datamodel/center/client/request/DatamodelCenterAction.scala @@ -0,0 +1,5 @@ +package com.webank.wedatasphere.dss.datamodel.center.client.request + +import org.apache.linkis.httpclient.dws.request.DWSHttpAction + +trait DatamodelCenterAction extends DWSHttpAction with org.apache.linkis.httpclient.request.UserAction diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-center-client/src/main/scala/com/webank/wedatasphere/dss/datamodel/center/client/request/LayersReferenceAction.scala b/dss-apps/dss-datamodel-center/dss-datamodel-center-client/src/main/scala/com/webank/wedatasphere/dss/datamodel/center/client/request/LayersReferenceAction.scala new file mode 100644 index 0000000000..68ee6532f2 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-center-client/src/main/scala/com/webank/wedatasphere/dss/datamodel/center/client/request/LayersReferenceAction.scala @@ -0,0 +1,43 @@ +package com.webank.wedatasphere.dss.datamodel.center.client.request + +import com.webank.wedatasphere.dss.datamodel.center.client.exception.DatamodelCenterClientBuilderException +import org.apache.linkis.httpclient.request.GetAction + +class LayersReferenceAction extends GetAction with DatamodelCenterAction{ + + private var name:String=_ + + private var user:String = _ + + override def setUser(user: String): Unit = this.user = user + + override def getUser: String = this.user + + override def suffixURLs: Array[String] = Array("datamodel", "layers", "reference",name) +} +object LayersReferenceAction{ + def builder(): Builder = new Builder + + class Builder private[LayersReferenceAction]() { + private var name:String = _ + private var user: String = _ + + def setUser(user: String): Builder ={ + this.user = user + this + } + def setName(name:String): Builder = { + this.name = name + this + } + + def build(): LayersReferenceAction = { + val action = new LayersReferenceAction + if(name == null) throw new DatamodelCenterClientBuilderException("name is needed!") + action.name = name + action.setUser(user) + action + } + } +} + diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-center-client/src/main/scala/com/webank/wedatasphere/dss/datamodel/center/client/request/ModifiersReferenceAction.scala b/dss-apps/dss-datamodel-center/dss-datamodel-center-client/src/main/scala/com/webank/wedatasphere/dss/datamodel/center/client/request/ModifiersReferenceAction.scala new file mode 100644 index 0000000000..684b792651 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-center-client/src/main/scala/com/webank/wedatasphere/dss/datamodel/center/client/request/ModifiersReferenceAction.scala @@ -0,0 +1,42 @@ +package com.webank.wedatasphere.dss.datamodel.center.client.request + +import com.webank.wedatasphere.dss.datamodel.center.client.exception.DatamodelCenterClientBuilderException +import org.apache.linkis.httpclient.request.GetAction + +class ModifiersReferenceAction extends GetAction with DatamodelCenterAction{ + + private var name:String=_ + + private var user:String = _ + + override def setUser(user: String): Unit = this.user = user + + override def getUser: String = this.user + + override def suffixURLs: Array[String] = Array("datamodel", "modifiers", "reference",name) +} +object ModifiersReferenceAction{ + def builder(): Builder = new Builder + + class Builder private[ModifiersReferenceAction]() { + private var name:String = _ + private var user: String = _ + + def setUser(user: String): Builder ={ + this.user = user + this + } + def setName(name:String): Builder = { + this.name = name + this + } + + def build(): ModifiersReferenceAction = { + val action = new ModifiersReferenceAction + if(name == null) throw new DatamodelCenterClientBuilderException("name is needed!") + action.name = name + action.setUser(user) + action + } + } +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-center-client/src/main/scala/com/webank/wedatasphere/dss/datamodel/center/client/request/ThemesReferenceAction.scala b/dss-apps/dss-datamodel-center/dss-datamodel-center-client/src/main/scala/com/webank/wedatasphere/dss/datamodel/center/client/request/ThemesReferenceAction.scala new file mode 100644 index 0000000000..484f0bb3fa --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-center-client/src/main/scala/com/webank/wedatasphere/dss/datamodel/center/client/request/ThemesReferenceAction.scala @@ -0,0 +1,42 @@ +package com.webank.wedatasphere.dss.datamodel.center.client.request + +import com.webank.wedatasphere.dss.datamodel.center.client.exception.DatamodelCenterClientBuilderException +import org.apache.linkis.httpclient.request.GetAction + +class ThemesReferenceAction extends GetAction with DatamodelCenterAction{ + + private var name:String=_ + + private var user:String = _ + + override def setUser(user: String): Unit = this.user = user + + override def getUser: String = this.user + + override def suffixURLs: Array[String] = Array("datamodel", "themes", "reference",name) +} +object ThemesReferenceAction{ + def builder(): Builder = new Builder + + class Builder private[ThemesReferenceAction]() { + private var name:String = _ + private var user: String = _ + + def setUser(user: String): Builder ={ + this.user = user + this + } + def setName(name:String): Builder = { + this.name = name + this + } + + def build(): ThemesReferenceAction = { + val action = new ThemesReferenceAction + if(name == null) throw new DatamodelCenterClientBuilderException("name is needed!") + action.name = name + action.setUser(user) + action + } + } +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-center-client/src/main/scala/com/webank/wedatasphere/dss/datamodel/center/client/response/CyclesReferenceResult.scala b/dss-apps/dss-datamodel-center/dss-datamodel-center-client/src/main/scala/com/webank/wedatasphere/dss/datamodel/center/client/response/CyclesReferenceResult.scala new file mode 100644 index 0000000000..125d0c6dcc --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-center-client/src/main/scala/com/webank/wedatasphere/dss/datamodel/center/client/response/CyclesReferenceResult.scala @@ -0,0 +1,11 @@ +package com.webank.wedatasphere.dss.datamodel.center.client.response + +import org.apache.linkis.httpclient.dws.annotation.DWSHttpMessageResult +import org.apache.linkis.httpclient.dws.response.DWSResult + +import scala.beans.BeanProperty + +@DWSHttpMessageResult("/api/rest_j/v\\d+/datamodel/cycles/reference/(\\S+)") +class CyclesReferenceResult extends DWSResult{ + @BeanProperty var result:Int = _ +} \ No newline at end of file diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-center-client/src/main/scala/com/webank/wedatasphere/dss/datamodel/center/client/response/LayersReferenceResult.scala b/dss-apps/dss-datamodel-center/dss-datamodel-center-client/src/main/scala/com/webank/wedatasphere/dss/datamodel/center/client/response/LayersReferenceResult.scala new file mode 100644 index 0000000000..916266be60 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-center-client/src/main/scala/com/webank/wedatasphere/dss/datamodel/center/client/response/LayersReferenceResult.scala @@ -0,0 +1,11 @@ +package com.webank.wedatasphere.dss.datamodel.center.client.response + +import org.apache.linkis.httpclient.dws.annotation.DWSHttpMessageResult +import org.apache.linkis.httpclient.dws.response.DWSResult + +import scala.beans.BeanProperty + +@DWSHttpMessageResult("/api/rest_j/v\\d+/datamodel/layers/reference/(\\S+)") +class LayersReferenceResult extends DWSResult{ + @BeanProperty var result:Int = _ +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-center-client/src/main/scala/com/webank/wedatasphere/dss/datamodel/center/client/response/ModifiersReferenceResult.scala b/dss-apps/dss-datamodel-center/dss-datamodel-center-client/src/main/scala/com/webank/wedatasphere/dss/datamodel/center/client/response/ModifiersReferenceResult.scala new file mode 100644 index 0000000000..fdcf81cb37 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-center-client/src/main/scala/com/webank/wedatasphere/dss/datamodel/center/client/response/ModifiersReferenceResult.scala @@ -0,0 +1,11 @@ +package com.webank.wedatasphere.dss.datamodel.center.client.response + +import org.apache.linkis.httpclient.dws.annotation.DWSHttpMessageResult +import org.apache.linkis.httpclient.dws.response.DWSResult + +import scala.beans.BeanProperty + +@DWSHttpMessageResult("/api/rest_j/v\\d+/datamodel/modifiers/reference/(\\S+)") +class ModifiersReferenceResult extends DWSResult{ + @BeanProperty var result:Int = _ +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-center-client/src/main/scala/com/webank/wedatasphere/dss/datamodel/center/client/response/ThemesReferenceResult.scala b/dss-apps/dss-datamodel-center/dss-datamodel-center-client/src/main/scala/com/webank/wedatasphere/dss/datamodel/center/client/response/ThemesReferenceResult.scala new file mode 100644 index 0000000000..a8111cee5a --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-center-client/src/main/scala/com/webank/wedatasphere/dss/datamodel/center/client/response/ThemesReferenceResult.scala @@ -0,0 +1,12 @@ +package com.webank.wedatasphere.dss.datamodel.center.client.response + +import org.apache.linkis.httpclient.dws.annotation.DWSHttpMessageResult +import org.apache.linkis.httpclient.dws.response.DWSResult + +import scala.beans.BeanProperty + +@DWSHttpMessageResult("/api/rest_j/v\\d+/datamodel/themes/reference/(\\S+)") +class ThemesReferenceResult extends DWSResult{ + @BeanProperty var result:Int = _ +} + diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-center-client/src/test/scala/com/webank/wedatasphere/dss/datamodel/center/client/TestDatamodelCenterRemoteClient.scala b/dss-apps/dss-datamodel-center/dss-datamodel-center-client/src/test/scala/com/webank/wedatasphere/dss/datamodel/center/client/TestDatamodelCenterRemoteClient.scala new file mode 100644 index 0000000000..06dd824511 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-center-client/src/test/scala/com/webank/wedatasphere/dss/datamodel/center/client/TestDatamodelCenterRemoteClient.scala @@ -0,0 +1,48 @@ +package com.webank.wedatasphere.dss.datamodel.center.client + +import com.webank.wedatasphere.dss.datamodel.center.client.impl.LinkisDatamodelCenterRemoteClient +import com.webank.wedatasphere.dss.datamodel.center.client.request.{CyclesReferenceAction, LayersReferenceAction, ModifiersReferenceAction, ThemesReferenceAction} +import org.apache.linkis.httpclient.dws.config.DWSClientConfigBuilder + +import java.util.concurrent.TimeUnit + +object TestDatamodelCenterRemoteClient { + def main(args: Array[String]): Unit = { + val clientConfig = DWSClientConfigBuilder.newBuilder() + .addServerUrl("http://localhost:9321") + .connectionTimeout(30000) + .discoveryEnabled(false) + .discoveryFrequency(1,TimeUnit.MINUTES) + .loadbalancerEnabled(true) + .maxConnectionSize(5) + .retryEnabled(false) + .readTimeout(30000) + .setAuthenticationStrategy(null) + .setAuthTokenKey("hdfs") + .setAuthTokenValue("hdfs") + .setDWSVersion("v1") + .build() + + val datamodelCenterClient = new LinkisDatamodelCenterRemoteClient(clientConfig) + + val themesResult = datamodelCenterClient + .themesReference(ThemesReferenceAction.builder().setUser("hdfs").setName("主题").build()) + .result + println(themesResult) + + val layersResult = datamodelCenterClient + .layersReference(LayersReferenceAction.builder().setUser("hdfs").setName("DWS").build()) + .result + println(layersResult) + + val cyclesResult = datamodelCenterClient + .cyclesReference(CyclesReferenceAction.builder().setUser("hdfs").setName("Once").build()) + .result + println(cyclesResult) + + val modifiersResult = datamodelCenterClient + .modifiersReference(ModifiersReferenceAction.builder().setUser("hdfs").setName("在商场").build()) + .result + println(modifiersResult) + } +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-center-common/pom.xml b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/pom.xml new file mode 100644 index 0000000000..acc086314b --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/pom.xml @@ -0,0 +1,96 @@ + + + + + + dss-datamodel-center + com.webank.wedatasphere.dss + 1.0.1 + ../pom.xml + + 4.0.0 + + dss-datamodel-center-common + + + + + org.apache.linkis + linkis-module + ${linkis.version} + + + org.apache.linkis + linkis-mybatis + ${linkis.version} + + + + com.webank.wedatasphere.dss + data-warehouse-client + ${dss.version} + + + + org.apache.linkis + linkis-computation-client + ${linkis.version} + + + + com.webank.wedatasphere.dss + data-assets-client + ${dss.version} + + + + com.webank.wedatasphere.dss + dss-framework-workspace-client + ${dss.version} + + + + + org.projectlombok + lombok + 1.18.16 + compile + + + + + + + + org.apache.maven.plugins + maven-deploy-plugin + + + net.alchim31.maven + scala-maven-plugin + + + org.apache.maven.plugins + maven-jar-plugin + + + + ${project.artifactId}-${project.version} + + diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/config/ClientStrategy.java b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/config/ClientStrategy.java new file mode 100644 index 0000000000..3f25d32623 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/config/ClientStrategy.java @@ -0,0 +1,19 @@ +package com.webank.wedatasphere.dss.datamodel.center.common.config; + +import lombok.Data; + + +public enum ClientStrategy { + TOKEN("token"), + STATIC("static"); + + private final String code; + + ClientStrategy(String code) { + this.code = code; + } + + public String getCode() { + return code; + } +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/config/CommonConfig.java b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/config/CommonConfig.java new file mode 100644 index 0000000000..2892e47712 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/config/CommonConfig.java @@ -0,0 +1,127 @@ +package com.webank.wedatasphere.dss.datamodel.center.common.config; + +import com.baomidou.mybatisplus.core.handlers.MetaObjectHandler; +import com.webank.wedatasphere.dss.data.governance.impl.LinkisDataAssetsRemoteClient; +import com.webank.wedatasphere.dss.datamodel.center.common.filter.AuthFilter; +import com.webank.wedatasphere.dss.framework.workspace.client.impl.LinkisWorkSpaceRemoteClient; +import org.apache.linkis.httpclient.authentication.AuthenticationStrategy; +import org.apache.linkis.httpclient.dws.config.DWSClientConfig; +import org.apache.linkis.httpclient.dws.config.DWSClientConfigBuilder; +import org.apache.linkis.ujes.client.UJESClient; +import org.apache.linkis.ujes.client.UJESClientImpl; +import com.webank.wedatasphere.warehouse.client.GovernanceDwRemoteClient; +import org.springframework.boot.web.servlet.FilterRegistrationBean; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +import javax.annotation.Resource; +import java.util.concurrent.TimeUnit; + + +@Configuration +public class CommonConfig { + + @Bean("myMetaObjectHandler") + public MetaObjectHandler myMetaObjectHandler() { + + //TODO 无法自动组装生效 + return new MyMetaObjectHandler(); + } + + @Bean + public GovernanceDwRemoteClient getGovernanceDwRemoteClient() throws Exception { + System.out.println("url: " + DataWarehouseGovernanceConfig.SERVER_URL.getValue()); + AuthenticationStrategy authenticationStrategy = (AuthenticationStrategy) Class.forName(DataWarehouseGovernanceConfig.AUTHENTICATION_STRATEGY.getValue()).newInstance(); + DWSClientConfig clientConfig = ((DWSClientConfigBuilder) DWSClientConfigBuilder.newBuilder() + .addServerUrl(DataWarehouseGovernanceConfig.SERVER_URL.getValue()) + .connectionTimeout(DataWarehouseGovernanceConfig.CONNECTION_TIMEOUT.getValue()) + .discoveryEnabled(DataWarehouseGovernanceConfig.DISCOVERY_ENABLED.getValue()) + .discoveryFrequency(DataWarehouseGovernanceConfig.DISCOVERY_FREQUENCY_PERIOD.getValue(), TimeUnit.MINUTES) + .loadbalancerEnabled(DataWarehouseGovernanceConfig.LOAD_BALANCER_ENABLED.getValue()) + .maxConnectionSize(DataWarehouseGovernanceConfig.MAX_CONNECTION_SIZE.getValue()) + .retryEnabled(DataWarehouseGovernanceConfig.RETRY_ENABLED.getValue()) + .readTimeout(DataWarehouseGovernanceConfig.READ_TIMEOUT.getValue()) + .setAuthenticationStrategy(authenticationStrategy) + .setAuthTokenKey(DataWarehouseGovernanceConfig.AUTHTOKEN_KEY.getValue()) + .setAuthTokenValue(DataWarehouseGovernanceConfig.AUTHTOKEN_VALUE.getValue()) + ).setDWSVersion(DataWarehouseGovernanceConfig.DWS_VERSION.getValue()) + .build(); + return new GovernanceDwRemoteClient(clientConfig); + } + + @Bean + public LinkisDataAssetsRemoteClient linkisDataAssetsRemoteClient() throws Exception { + AuthenticationStrategy authenticationStrategy = (AuthenticationStrategy) Class.forName(DataWarehouseAssetsRemoteConfig.AUTHENTICATION_STRATEGY.getValue()).newInstance(); + DWSClientConfig clientConfig = ((DWSClientConfigBuilder) DWSClientConfigBuilder.newBuilder() + .addServerUrl(DataWarehouseAssetsRemoteConfig.SERVER_URL.getValue()) + .connectionTimeout(DataWarehouseAssetsRemoteConfig.CONNECTION_TIMEOUT.getValue()) + .discoveryEnabled(DataWarehouseAssetsRemoteConfig.DISCOVERY_ENABLED.getValue()) + .discoveryFrequency(DataWarehouseAssetsRemoteConfig.DISCOVERY_FREQUENCY_PERIOD.getValue(), TimeUnit.MINUTES) + .loadbalancerEnabled(DataWarehouseAssetsRemoteConfig.LOAD_BALANCER_ENABLED.getValue()) + .maxConnectionSize(DataWarehouseAssetsRemoteConfig.MAX_CONNECTION_SIZE.getValue()) + .retryEnabled(DataWarehouseAssetsRemoteConfig.RETRY_ENABLED.getValue()) + .readTimeout(DataWarehouseAssetsRemoteConfig.READ_TIMEOUT.getValue()) + .setAuthenticationStrategy(authenticationStrategy) + .setAuthTokenKey(DataWarehouseAssetsRemoteConfig.AUTHTOKEN_KEY.getValue()) + .setAuthTokenValue(DataWarehouseAssetsRemoteConfig.AUTHTOKEN_VALUE.getValue()) + ).setDWSVersion(DataWarehouseAssetsRemoteConfig.DWS_VERSION.getValue()) + .build(); + return new LinkisDataAssetsRemoteClient(clientConfig); + } + + @Bean + public LinkisWorkSpaceRemoteClient linkisWorkSpaceRemoteClient() throws Exception { + AuthenticationStrategy authenticationStrategy = (AuthenticationStrategy) Class.forName(DataWorkspaceRemoteConfig.AUTHENTICATION_STRATEGY.getValue()).newInstance(); + DWSClientConfig clientConfig = ((DWSClientConfigBuilder) DWSClientConfigBuilder.newBuilder() + .addServerUrl(DataWorkspaceRemoteConfig.SERVER_URL.getValue()) + .connectionTimeout(DataWorkspaceRemoteConfig.CONNECTION_TIMEOUT.getValue()) + .discoveryEnabled(DataWorkspaceRemoteConfig.DISCOVERY_ENABLED.getValue()) + .discoveryFrequency(DataWorkspaceRemoteConfig.DISCOVERY_FREQUENCY_PERIOD.getValue(), TimeUnit.MINUTES) + .loadbalancerEnabled(DataWorkspaceRemoteConfig.LOAD_BALANCER_ENABLED.getValue()) + .maxConnectionSize(DataWorkspaceRemoteConfig.MAX_CONNECTION_SIZE.getValue()) + .retryEnabled(DataWorkspaceRemoteConfig.RETRY_ENABLED.getValue()) + .readTimeout(DataWorkspaceRemoteConfig.READ_TIMEOUT.getValue()) + .setAuthenticationStrategy(authenticationStrategy) + .setAuthTokenKey(DataWorkspaceRemoteConfig.AUTHTOKEN_KEY.getValue()) + .setAuthTokenValue(DataWorkspaceRemoteConfig.AUTHTOKEN_VALUE.getValue()) + ).setDWSVersion(DataWorkspaceRemoteConfig.DWS_VERSION.getValue()) + .build(); + return new LinkisWorkSpaceRemoteClient(clientConfig); + } + + + + @Bean + public UJESClient ujesClient()throws Exception { + AuthenticationStrategy authenticationStrategy = (AuthenticationStrategy) Class.forName(LinkisJobConfiguration.AUTHENTICATION_STRATEGY.getValue()).newInstance(); + DWSClientConfig clientConfig = ((DWSClientConfigBuilder) (DWSClientConfigBuilder.newBuilder().addServerUrl(LinkisJobConfiguration.LINKIS_SERVER_URL.getValue()) + .connectionTimeout(LinkisJobConfiguration.CONNECTION_TIMEOUT.getValue()) + .discoveryEnabled(LinkisJobConfiguration.DISCOVERY_ENABLED.getValue()) + .discoveryFrequency(LinkisJobConfiguration.DISCOVERY_FREQUENCY_PERIOD.getValue(), TimeUnit.MINUTES) + .loadbalancerEnabled(LinkisJobConfiguration.LOAD_BALANCER_ENABLED.getValue()) + .maxConnectionSize(LinkisJobConfiguration.MAX_CONNECTION_SIZE.getValue()) + .retryEnabled(LinkisJobConfiguration.RETRY_ENABLED.getValue()) + .readTimeout(LinkisJobConfiguration.READ_TIMEOUT.getValue()) + .setAuthenticationStrategy(authenticationStrategy) + .setAuthTokenKey(LinkisJobConfiguration.AUTHTOKEN_KEY.getValue()) + .setAuthTokenValue(LinkisJobConfiguration.AUTHTOKEN_VALUE.getValue()))) + .setDWSVersion(LinkisJobConfiguration.DWS_VERSION.getValue()).build(); + return new UJESClientImpl(clientConfig); + } + + @Resource + private AuthFilter authFilter; + + @Bean + public FilterRegistrationBean registrationBean(){ + //通过FilterRegistrationBean实例设置优先级可以生效 + //通过@WebFilter无效 + FilterRegistrationBean bean = new FilterRegistrationBean(); + bean.setFilter(authFilter);//注册自定义过滤器 + bean.setName("authFilter1");//过滤器名称 + bean.addUrlPatterns("/*");//过滤所有路径 + bean.setOrder(1);//优先级，最顶级 + return bean; + } + +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/config/DataWarehouseAssetsRemoteConfig.java b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/config/DataWarehouseAssetsRemoteConfig.java new file mode 100644 index 0000000000..310a187222 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/config/DataWarehouseAssetsRemoteConfig.java @@ -0,0 +1,21 @@ +package com.webank.wedatasphere.dss.datamodel.center.common.config; + + +import org.apache.linkis.common.conf.CommonVars; + +public class DataWarehouseAssetsRemoteConfig { + public static final CommonVars SERVER_URL = CommonVars.apply("wds.wedatasphere.assets.client.serverurl", ""); + public static final CommonVars CONNECTION_TIMEOUT = CommonVars.apply("wds.wedatasphere.assets.client.connection.timeout", 30000L); + public static final CommonVars DISCOVERY_ENABLED = CommonVars.apply("wds.wedatasphere.assets.client.discovery.enabled", false); + public static final CommonVars DISCOVERY_FREQUENCY_PERIOD = CommonVars.apply("wds.wedatasphere.assets.client.discoveryfrequency.period", 1L); + public static final CommonVars LOAD_BALANCER_ENABLED = CommonVars.apply("wds.wedatasphere.assets.client.loadbalancer.enabled", true); + public static final CommonVars MAX_CONNECTION_SIZE = CommonVars.apply("wds.wedatasphere.assets.client.maxconnection.size", 5); + public static final CommonVars RETRY_ENABLED = CommonVars.apply("wds.wedatasphere.assets.client.retryenabled", false); + public static final CommonVars READ_TIMEOUT = CommonVars.apply("wds.wedatasphere.assets.client.readtimeout", 30000L); + public static final CommonVars AUTHENTICATION_STRATEGY = CommonVars.apply("wds.wedatasphere.assets.client.authenticationStrategy", ""); + + public static final CommonVars AUTHTOKEN_KEY = CommonVars.apply("wds.wedatasphere.assets.client.authtoken.key", ""); + public static final CommonVars AUTHTOKEN_VALUE = CommonVars.apply("wds.wedatasphere.assets.client.authtoken.value", ""); + public static final CommonVars DWS_VERSION = CommonVars.apply("wds.wedatasphere.assets.client.dws.version", ""); + +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/config/DataWarehouseGovernanceConfig.java b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/config/DataWarehouseGovernanceConfig.java new file mode 100644 index 0000000000..2759099c39 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/config/DataWarehouseGovernanceConfig.java @@ -0,0 +1,21 @@ +package com.webank.wedatasphere.dss.datamodel.center.common.config; + + +import org.apache.linkis.common.conf.CommonVars; + +public class DataWarehouseGovernanceConfig { + public static final CommonVars SERVER_URL = CommonVars.apply("wds.wedatasphere.warehouse.client.serverurl", ""); + public static final CommonVars CONNECTION_TIMEOUT = CommonVars.apply("wds.wedatasphere.warehouse.client.connection.timeout", 30000L); + public static final CommonVars DISCOVERY_ENABLED = CommonVars.apply("wds.wedatasphere.warehouse.client.discovery.enabled", false); + public static final CommonVars DISCOVERY_FREQUENCY_PERIOD = CommonVars.apply("wds.wedatasphere.warehouse.client.discoveryfrequency.period", 1L); + public static final CommonVars LOAD_BALANCER_ENABLED = CommonVars.apply("wds.wedatasphere.warehouse.client.loadbalancer.enabled", true); + public static final CommonVars MAX_CONNECTION_SIZE = CommonVars.apply("wds.wedatasphere.warehouse.client.maxconnection.size", 5); + public static final CommonVars RETRY_ENABLED = CommonVars.apply("wds.wedatasphere.warehouse.client.retryenabled", false); + public static final CommonVars READ_TIMEOUT = CommonVars.apply("wds.wedatasphere.warehouse.client.readtimeout", 30000L); + public static final CommonVars AUTHENTICATION_STRATEGY = CommonVars.apply("wds.wedatasphere.warehouse.client.authenticationStrategy", "org.apache.linkis.httpclient.dws.authentication.StaticAuthenticationStrategy"); + + public static final CommonVars AUTHTOKEN_KEY = CommonVars.apply("wds.wedatasphere.warehouse.client.authtoken.key", ""); + public static final CommonVars AUTHTOKEN_VALUE = CommonVars.apply("wds.wedatasphere.warehouse.client.authtoken.value", ""); + public static final CommonVars DWS_VERSION = CommonVars.apply("wds.wedatasphere.warehouse.client.dws.version", ""); + +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/config/DataWorkspaceRemoteConfig.java b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/config/DataWorkspaceRemoteConfig.java new file mode 100644 index 0000000000..ce5fe4a6e5 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/config/DataWorkspaceRemoteConfig.java @@ -0,0 +1,21 @@ +package com.webank.wedatasphere.dss.datamodel.center.common.config; + + +import org.apache.linkis.common.conf.CommonVars; + +public class DataWorkspaceRemoteConfig { + public static final CommonVars SERVER_URL = CommonVars.apply("wds.workspace.client.serverurl", ""); + public static final CommonVars CONNECTION_TIMEOUT = CommonVars.apply("wds.workspace.client.connection.timeout", 30000L); + public static final CommonVars DISCOVERY_ENABLED = CommonVars.apply("wds.workspace.client.discovery.enabled", false); + public static final CommonVars DISCOVERY_FREQUENCY_PERIOD = CommonVars.apply("wds.workspace.client.discoveryfrequency.period", 1L); + public static final CommonVars LOAD_BALANCER_ENABLED = CommonVars.apply("wds.workspace.client.loadbalancer.enabled", true); + public static final CommonVars MAX_CONNECTION_SIZE = CommonVars.apply("wds.workspace.client.maxconnection.size", 5); + public static final CommonVars RETRY_ENABLED = CommonVars.apply("wds.workspace.client.retryenabled", false); + public static final CommonVars READ_TIMEOUT = CommonVars.apply("wds.workspace.client.readtimeout", 30000L); + public static final CommonVars AUTHENTICATION_STRATEGY = CommonVars.apply("wds.workspace.client.authenticationStrategy", ""); + + public static final CommonVars AUTHTOKEN_KEY = CommonVars.apply("wds.workspace.client.authtoken.key", ""); + public static final CommonVars AUTHTOKEN_VALUE = CommonVars.apply("wds.workspace.client.authtoken.value", ""); + public static final CommonVars DWS_VERSION = CommonVars.apply("wds.workspace.client.dws.version", ""); + +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/config/LinkisJobConfiguration.java b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/config/LinkisJobConfiguration.java new file mode 100644 index 0000000000..0f4b1719d5 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/config/LinkisJobConfiguration.java @@ -0,0 +1,20 @@ +package com.webank.wedatasphere.dss.datamodel.center.common.config; + +import org.apache.linkis.common.conf.CommonVars; + +public class LinkisJobConfiguration { + public static final CommonVars LINKIS_SERVER_URL = CommonVars.apply("wds.wedatasphere.linkis.serverurl", ""); + public static final CommonVars CONNECTION_TIMEOUT = CommonVars.apply("wds.wedatasphere.linkis.connection.timeout", 30000L); + public static final CommonVars DISCOVERY_ENABLED = CommonVars.apply("wds.wedatasphere.linkis.discovery.enabled", false); + public static final CommonVars DISCOVERY_FREQUENCY_PERIOD = CommonVars.apply("wds.wedatasphere.linkis.discoveryfrequency.period", 1L); + public static final CommonVars LOAD_BALANCER_ENABLED = CommonVars.apply("wds.wedatasphere.linkis.loadbalancer.enabled", true); + public static final CommonVars MAX_CONNECTION_SIZE = CommonVars.apply("wds.wedatasphere.linkis.maxconnection.size", 5); + public static final CommonVars RETRY_ENABLED = CommonVars.apply("wds.wedatasphere.linkis.retryenabled", false); + public static final CommonVars READ_TIMEOUT = CommonVars.apply("wds.wedatasphere.linkis.readtimeout", 30000L); + public static final CommonVars AUTHENTICATION_STRATEGY = CommonVars.apply("wds.wedatasphere.linkis.authenticationStrategy", ""); + + public static final CommonVars AUTHTOKEN_KEY = CommonVars.apply("wds.wedatasphere.linkis.authtoken.key", ""); + public static final CommonVars AUTHTOKEN_VALUE = CommonVars.apply("wds.wedatasphere.linkis.authtoken.value", ""); + public static final CommonVars DWS_VERSION = CommonVars.apply("wds.wedatasphere.linkis.dws.version", ""); + +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/config/ModelMapperConfig.java b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/config/ModelMapperConfig.java new file mode 100644 index 0000000000..7e0aa8ea7b --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/config/ModelMapperConfig.java @@ -0,0 +1,18 @@ +package com.webank.wedatasphere.dss.datamodel.center.common.config; + +import org.modelmapper.ModelMapper; +import org.modelmapper.convention.MatchingStrategies; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +@Configuration +public class ModelMapperConfig { + + @Bean + public ModelMapper getModelMapper() { + ModelMapper modelMapper = new ModelMapper(); + modelMapper.getConfiguration().setMatchingStrategy(MatchingStrategies.STRICT); + return modelMapper; + } + +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/config/MyMetaObjectHandler.java b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/config/MyMetaObjectHandler.java new file mode 100644 index 0000000000..4e5e981f5e --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/config/MyMetaObjectHandler.java @@ -0,0 +1,23 @@ +package com.webank.wedatasphere.dss.datamodel.center.common.config; + +import com.baomidou.mybatisplus.core.handlers.MetaObjectHandler; +import org.apache.ibatis.reflection.MetaObject; +import org.springframework.stereotype.Component; + +import java.util.Date; + + +public class MyMetaObjectHandler implements MetaObjectHandler { + + + @Override + public void insertFill(MetaObject metaObject) { + this.strictInsertFill(metaObject,"createTime",Date.class,new Date()); + this.strictInsertFill(metaObject,"updateTime",Date.class,new Date()); + } + + @Override + public void updateFill(MetaObject metaObject) { + this.strictUpdateFill(metaObject,"updateTime",Date.class,new Date()); + } +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/config/MybatisPlusConfig.java b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/config/MybatisPlusConfig.java new file mode 100644 index 0000000000..91b13c1de1 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/config/MybatisPlusConfig.java @@ -0,0 +1,24 @@ +package com.webank.wedatasphere.dss.datamodel.center.common.config; + +import com.baomidou.mybatisplus.annotation.DbType; +import com.baomidou.mybatisplus.extension.plugins.MybatisPlusInterceptor; +import com.baomidou.mybatisplus.extension.plugins.PaginationInterceptor; +import com.baomidou.mybatisplus.extension.plugins.inner.OptimisticLockerInnerInterceptor; +import com.baomidou.mybatisplus.extension.plugins.inner.PaginationInnerInterceptor; +import org.springframework.boot.autoconfigure.condition.ConditionalOnClass; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + + +@Configuration +public class MybatisPlusConfig { + + @Bean + public MybatisPlusInterceptor mybatisPlusInterceptor() { + System.out.println("mybatisPlusInterceptor init"); + MybatisPlusInterceptor interceptor = new MybatisPlusInterceptor(); + interceptor.addInnerInterceptor(new PaginationInnerInterceptor(DbType.MYSQL)); + //interceptor.addInnerInterceptor(new OptimisticLockerInnerInterceptor());//乐观锁插件 + return interceptor; + } +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/config/ThreadPoolTaskConfig.java b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/config/ThreadPoolTaskConfig.java new file mode 100644 index 0000000000..fd56906c34 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/config/ThreadPoolTaskConfig.java @@ -0,0 +1,34 @@ +package com.webank.wedatasphere.dss.datamodel.center.common.config; + +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.scheduling.annotation.EnableAsync; +import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; + +import java.util.concurrent.ThreadPoolExecutor; + +@Configuration +@EnableAsync +public class ThreadPoolTaskConfig { + private static final int corePoolSize = 10; // 核心线程数（默认线程数） + private static final int maxPoolSize = 100; // 最大线程数 + private static final int keepAliveTime = 10; // 允许线程空闲时间（单位：默认为秒） + private static final int queueCapacity = 200; // 缓冲队列数 + private static final String threadNamePrefix = "Async-Service-"; // 线程池名前缀 + + @Bean("taskExecutor") // bean的名称，默认为首字母小写的方法名 + public ThreadPoolTaskExecutor getAsyncExecutor(){ + ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor(); + executor.setCorePoolSize(corePoolSize); + executor.setMaxPoolSize(maxPoolSize); + executor.setQueueCapacity(queueCapacity); + executor.setKeepAliveSeconds(keepAliveTime); + executor.setThreadNamePrefix(threadNamePrefix); + + // 线程池对拒绝任务的处理策略 + executor.setRejectedExecutionHandler(new ThreadPoolExecutor.CallerRunsPolicy()); + // 初始化 + executor.initialize(); + return executor; + } +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/constant/ColumnType.java b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/constant/ColumnType.java new file mode 100644 index 0000000000..a959bc87fd --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/constant/ColumnType.java @@ -0,0 +1,16 @@ +package com.webank.wedatasphere.dss.datamodel.center.common.constant; + + +public enum ColumnType { + COLUMN(0), + PARTITION_KEY(1); + private Integer code; + + ColumnType(Integer code) { + this.code = code; + } + + public Integer getCode() { + return code; + } +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/constant/ErrorCode.java b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/constant/ErrorCode.java new file mode 100644 index 0000000000..ca55e0e03b --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/constant/ErrorCode.java @@ -0,0 +1,56 @@ +package com.webank.wedatasphere.dss.datamodel.center.common.constant; + + +public enum ErrorCode { + + INDICATOR_VERSION_ADD_ERROR(212031), + INDICATOR_VERSION_ROLL_BACK_ERROR(212032), + + MEASURE_QUERY_ERROR(212101), + MEASURE_DELETE_ERROR(212102), + MEASURE_ADD_ERROR(212103), + MEASURE_UPDATE_ERROR(212104), + + DIMENSION_QUERY_ERROR(212201), + DIMENSION_DELETE_ERROR(212202), + DIMENSION_ADD_ERROR(212203), + DIMENSION_UPDATE_ERROR(212204), + + INDICATOR_ADD_ERROR(212001), + INDICATOR_QUERY_ERROR(212002), + INDICATOR_UPDATE_ERROR(212003), + INDICATOR_DELETE_ERROR(212004), + INDICATOR_CONTENT_ADD_ERROR(212011), + + + TABLE_ADD_ERROR(212301), + TABLE_UPDATE_ERROR(212302), + TABLE_QUERY_ERROR(212303), + TABLE_VERSION_ADD_ERROR(212304), + TABLE_VERSION_ROLL_BACK_ERROR(212305), + TABLE_COLLECT_ADD_ERROR(212306), + TABLE_COLUMN_ADD_ERROR(212307), + TABLE_CREATE_ERROR(212308), + TABLE_CREATE_SQL_ERROR(212309), + TABLE_CHECK_ERROR(212310), + TABLE_BIND_ERROR(212311), + TABLE_DELETE_ERROR(212312), + + LABEL_ADD_ERROR(212401), + LABEL_UPDATE_ERROR(212402), + LABEL_QUERY_ERROR(212403), + LABEL_DELETE_ERROR(212404); + + + + + private int code; + + ErrorCode(int i) { + code = i; + } + + public int getCode() { + return code; + } +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/constant/IndicatorSourceInfoConstant.java b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/constant/IndicatorSourceInfoConstant.java new file mode 100644 index 0000000000..b587c0bb83 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/constant/IndicatorSourceInfoConstant.java @@ -0,0 +1,19 @@ +package com.webank.wedatasphere.dss.datamodel.center.common.constant; + + +public enum IndicatorSourceInfoConstant { + cycle, + cycleEn, + + modifier, + modifierEn, + + dimension, + dimensionEn, + + measure, + measureEn, + + indicatorName, + indicatorNameEn +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/constant/LabelConstant.java b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/constant/LabelConstant.java new file mode 100644 index 0000000000..70f5226f2c --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/constant/LabelConstant.java @@ -0,0 +1,7 @@ +package com.webank.wedatasphere.dss.datamodel.center.common.constant; + + +public enum LabelConstant { + LABEL; + public static final String SEPARATOR=","; +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/constant/ModeType.java b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/constant/ModeType.java new file mode 100644 index 0000000000..f5ae5e5a67 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/constant/ModeType.java @@ -0,0 +1,49 @@ +package com.webank.wedatasphere.dss.datamodel.center.common.constant; + + +import com.webank.wedatasphere.dss.data.governance.entity.ClassificationConstant; + +public enum ModeType { + + /** + * 指标 + */ + INDICATOR(ClassificationConstant.INDICATOR.getType(), ClassificationConstant.INDICATOR.getTypeCode()), + /** + * 度量 + */ + MEASURE(ClassificationConstant.MEASURE.getType(), ClassificationConstant.MEASURE.getTypeCode()), + /** + * 维度 + */ + DIMENSION(ClassificationConstant.INDICATOR.getType(), ClassificationConstant.INDICATOR.getTypeCode()); + + + private int type; + + + + private String typeCode; + + ModeType(int type, String typeCode) { + this.type = type; + + this.typeCode = typeCode; + } + + public int getType() { + return type; + } + + public void setType(int type) { + this.type = type; + } + + public String getTypeCode() { + return typeCode; + } + + public void setTypeCode(String typeCode) { + this.typeCode = typeCode; + } +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/constant/ReferenceExpression.java b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/constant/ReferenceExpression.java new file mode 100644 index 0000000000..20844d11ba --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/constant/ReferenceExpression.java @@ -0,0 +1,25 @@ +package com.webank.wedatasphere.dss.datamodel.center.common.constant; + + +public enum ReferenceExpression { + + EXPRESSION; + + private final static String oriCode ="(\"%s\":)(\"|\".*,)(%s)(\"|,)"; + + private final static String regexpCode = " REGEXP '"+oriCode+"' "; + + private String getFormatExpCode(String key, String value){ + return String.format(oriCode,key,value); + } + + public String getSqlExpCode(String key,String value){ + return String.format(regexpCode,key,value); + } + + + public static void main(String[] args) { + System.out.println(ReferenceExpression.EXPRESSION.getSqlExpCode("warehouseLayerNameEn","year")); + } +} + diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/constant/ReferenceKey.java b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/constant/ReferenceKey.java new file mode 100644 index 0000000000..9688af471f --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/constant/ReferenceKey.java @@ -0,0 +1,35 @@ +package com.webank.wedatasphere.dss.datamodel.center.common.constant; + + +public enum ReferenceKey { + label, + measure, + dimension, + indicatorName, + + //分层 + warehouseLayerName, + warehouseLayerNameEn, + + //主题 + warehouseThemeName, + warehouseThemeNameEn, + + //主题 + themeArea, + themeAreaEn, + + //分层 + layerArea, + layerAreaEn, + + //周期 + lifecycle, + lifecycleEn, + + //修饰词 + modifier, + modifierEn, + + +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/constant/TabelExternalType.java b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/constant/TabelExternalType.java new file mode 100644 index 0000000000..0952fe0209 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/constant/TabelExternalType.java @@ -0,0 +1,37 @@ +package com.webank.wedatasphere.dss.datamodel.center.common.constant; + + +import org.apache.commons.lang.StringUtils; + +import java.util.Optional; + +public enum TabelExternalType { + INNER(0,"MANAGED_TABLE"), + EXTERNAL(1,"EXTERNAL_TABLE"); + + private Integer code; + + private String type; + + TabelExternalType(Integer code, String type) { + this.code = code; + this.type = type; + } + + public Integer getCode() { + return code; + } + + public String getType() { + return type; + } + + public static Optional getByType(String type){ + for (TabelExternalType e : values()){ + if (StringUtils.equals(type,e.getType())){ + return Optional.of(e); + } + } + return Optional.empty(); + } +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/context/DataModelAuthentication.java b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/context/DataModelAuthentication.java new file mode 100644 index 0000000000..958db2041a --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/context/DataModelAuthentication.java @@ -0,0 +1,9 @@ +package com.webank.wedatasphere.dss.datamodel.center.common.context; + +import lombok.Data; + +@Data +public class DataModelAuthentication { + + private String user; +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/context/DataModelSecurityContext.java b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/context/DataModelSecurityContext.java new file mode 100644 index 0000000000..c19cfce89a --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/context/DataModelSecurityContext.java @@ -0,0 +1,9 @@ +package com.webank.wedatasphere.dss.datamodel.center.common.context; + + +import lombok.Data; + +@Data +public class DataModelSecurityContext { + private DataModelAuthentication dataModelAuthentication; +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/context/DataModelSecurityContextHolder.java b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/context/DataModelSecurityContextHolder.java new file mode 100644 index 0000000000..d4c0c17e73 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/context/DataModelSecurityContextHolder.java @@ -0,0 +1,35 @@ +package com.webank.wedatasphere.dss.datamodel.center.common.context; + + +public class DataModelSecurityContextHolder { + + private static DataModelSecurityContextHolderStrategy strategy; + + static { + initialize(); + } + + private static void initialize(){ + strategy = new ThreadLocalSecurityContextHolderStrategy(); + } + + + /** + * Explicitly clears the context value from the current thread. + */ + public static void clearContext() { + strategy.clearContext(); + } + + /** + * Obtain the current SecurityContext. + * @return the security context (never null) + */ + public static DataModelSecurityContext getContext() { + return strategy.getContext(); + } + + public static void setContext(DataModelSecurityContext context) { + strategy.setContext(context); + } +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/context/DataModelSecurityContextHolderStrategy.java b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/context/DataModelSecurityContextHolderStrategy.java new file mode 100644 index 0000000000..9c6f3b90af --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/context/DataModelSecurityContextHolderStrategy.java @@ -0,0 +1,30 @@ +package com.webank.wedatasphere.dss.datamodel.center.common.context; + + +public interface DataModelSecurityContextHolderStrategy { + + void clearContext(); + + /** + * Obtains the current context. + * @return a context (never null - create a default implementation if + * necessary) + */ + DataModelSecurityContext getContext(); + + /** + * Sets the current context. + * @param context to the new argument (should never be null, although + * implementations must check if null has been passed and throw an + * IllegalArgumentException in such cases) + */ + void setContext(DataModelSecurityContext context); + + /** + * Creates a new, empty context implementation, for use by + * SecurityContextRepository implementations, when creating a new context for + * the first time. + * @return the empty context. + */ + DataModelSecurityContext createEmptyContext(); +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/context/ThreadLocalSecurityContextHolderStrategy.java b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/context/ThreadLocalSecurityContextHolderStrategy.java new file mode 100644 index 0000000000..27769e47e8 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/context/ThreadLocalSecurityContextHolderStrategy.java @@ -0,0 +1,35 @@ +package com.webank.wedatasphere.dss.datamodel.center.common.context; + + +import org.springframework.util.Assert; + +final class ThreadLocalSecurityContextHolderStrategy implements DataModelSecurityContextHolderStrategy{ + + private static final ThreadLocal contextHolder = new ThreadLocal<>(); + + @Override + public void clearContext() { + contextHolder.remove(); + } + + @Override + public DataModelSecurityContext getContext() { + DataModelSecurityContext ctx = contextHolder.get(); + if (ctx == null) { + ctx = createEmptyContext(); + contextHolder.set(ctx); + } + return ctx; + } + + @Override + public void setContext(DataModelSecurityContext context) { + Assert.notNull(context, "Only non-null SecurityContext instances are permitted"); + contextHolder.set(context); + } + + @Override + public DataModelSecurityContext createEmptyContext() { + return new DataModelSecurityContext(); + } +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/dto/CreateTableDTO.java b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/dto/CreateTableDTO.java new file mode 100644 index 0000000000..0c053569ab --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/dto/CreateTableDTO.java @@ -0,0 +1,13 @@ +package com.webank.wedatasphere.dss.datamodel.center.common.dto; + + +import lombok.Data; + +@Data +public class CreateTableDTO { + private Integer status; + + private String taskId; + + private String sql; +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/dto/PreviewDataDTO.java b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/dto/PreviewDataDTO.java new file mode 100644 index 0000000000..9d85b6594d --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/dto/PreviewDataDTO.java @@ -0,0 +1,11 @@ +package com.webank.wedatasphere.dss.datamodel.center.common.dto; + + +import lombok.Data; + +@Data +public class PreviewDataDTO { + private Object metadata; + + private Object data; +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/event/BindLabelEvent.java b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/event/BindLabelEvent.java new file mode 100644 index 0000000000..59bcbd7f9a --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/event/BindLabelEvent.java @@ -0,0 +1,36 @@ +package com.webank.wedatasphere.dss.datamodel.center.common.event; + +import lombok.Getter; +import lombok.ToString; +import org.springframework.context.ApplicationEvent; + +@Getter +@ToString +public class BindLabelEvent extends ApplicationEvent { + + private String user; + + private String label; + + private String tableName; + + private String labelGuid; + + private String tableGuid; + + public BindLabelEvent(Object source, String user, String label, String tableName, String labelGuid, String tableGuid) { + super(source); + this.user = user; + this.label = label; + this.tableName = tableName; + this.labelGuid = labelGuid; + this.tableGuid = tableGuid; + } + + public BindLabelEvent(Object source, String user, String label, String tableName) { + super(source); + this.user = user; + this.label = label; + this.tableName = tableName; + } +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/event/BindModelEvent.java b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/event/BindModelEvent.java new file mode 100644 index 0000000000..182b041347 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/event/BindModelEvent.java @@ -0,0 +1,33 @@ +package com.webank.wedatasphere.dss.datamodel.center.common.event; + + +import com.webank.wedatasphere.dss.data.governance.entity.ClassificationConstant; +import com.webank.wedatasphere.dss.data.governance.request.BindModelTypeAction; +import lombok.Getter; +import lombok.ToString; +import org.springframework.context.ApplicationEvent; + +@Getter +@ToString +public class BindModelEvent extends ApplicationEvent { + + private String user; + + private String modelName; + + private ClassificationConstant modelType; + + private String tableName; + + private String guid; + + + public BindModelEvent(Object source,String user,String guid,String tableName,String modelName,ClassificationConstant modelType) { + super(source); + this.user = user; + this.guid = guid; + this.tableName = tableName; + this.modelName = modelName; + this.modelType = modelType; + } +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/event/CreateLabelEvent.java b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/event/CreateLabelEvent.java new file mode 100644 index 0000000000..bb301eddb4 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/event/CreateLabelEvent.java @@ -0,0 +1,22 @@ +package com.webank.wedatasphere.dss.datamodel.center.common.event; + + +import com.webank.wedatasphere.dss.data.governance.entity.ClassificationConstant; +import lombok.Getter; +import lombok.ToString; +import org.springframework.context.ApplicationEvent; + +@Getter +@ToString +public class CreateLabelEvent extends ApplicationEvent { + + private String user; + + private String name; + + public CreateLabelEvent(Object source, String user, String name) { + super(source); + this.user = user; + this.name = name; + } +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/event/CreateModelEvent.java b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/event/CreateModelEvent.java new file mode 100644 index 0000000000..8a1f52b98d --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/event/CreateModelEvent.java @@ -0,0 +1,24 @@ +package com.webank.wedatasphere.dss.datamodel.center.common.event; + + +import com.webank.wedatasphere.dss.data.governance.entity.ClassificationConstant; +import lombok.Getter; +import lombok.ToString; +import org.springframework.context.ApplicationEvent; +@Getter +@ToString +public class CreateModelEvent extends ApplicationEvent { + + private String user; + + private ClassificationConstant type; + + private String name; + + public CreateModelEvent(Object source,String user, String name, ClassificationConstant type) { + super(source); + this.type = type; + this.name = name; + this.user = user; + } +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/event/DeleteLabelEvent.java b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/event/DeleteLabelEvent.java new file mode 100644 index 0000000000..a9ff076d3b --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/event/DeleteLabelEvent.java @@ -0,0 +1,20 @@ +package com.webank.wedatasphere.dss.datamodel.center.common.event; + + +import lombok.Getter; +import lombok.ToString; +import org.springframework.context.ApplicationEvent; + +@Getter +@ToString +public class DeleteLabelEvent extends ApplicationEvent { + private String user; + + private String name; + + public DeleteLabelEvent(Object source, String user, String name) { + super(source); + this.user = user; + this.name = name; + } +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/event/DeleteModelEvent.java b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/event/DeleteModelEvent.java new file mode 100644 index 0000000000..e2690c6650 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/event/DeleteModelEvent.java @@ -0,0 +1,24 @@ +package com.webank.wedatasphere.dss.datamodel.center.common.event; + + +import com.webank.wedatasphere.dss.data.governance.entity.ClassificationConstant; +import lombok.Getter; +import lombok.ToString; +import org.springframework.context.ApplicationEvent; + +@Getter +@ToString +public class DeleteModelEvent extends ApplicationEvent { + private String user; + + private ClassificationConstant type; + + private String name; + + public DeleteModelEvent(Object source,String user, String name, ClassificationConstant type) { + super(source); + this.type = type; + this.name = name; + this.user = user; + } +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/event/UnBindLabelEvent.java b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/event/UnBindLabelEvent.java new file mode 100644 index 0000000000..5c8066aa17 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/event/UnBindLabelEvent.java @@ -0,0 +1,40 @@ +package com.webank.wedatasphere.dss.datamodel.center.common.event; + +import javafx.application.Application; +import lombok.Getter; +import lombok.ToString; +import org.springframework.context.ApplicationEvent; + +@Getter +@ToString +public class UnBindLabelEvent extends ApplicationEvent { + + private String user; + + private String label; + + private String tableName; + + private String labelGuid; + + private String tableGuid; + + private String relationGuid; + + public UnBindLabelEvent(Object source, String user, String label, String tableName, String labelGuid, String tableGuid, String relationGuid) { + super(source); + this.user = user; + this.label = label; + this.tableName = tableName; + this.labelGuid = labelGuid; + this.tableGuid = tableGuid; + this.relationGuid = relationGuid; + } + + public UnBindLabelEvent(Object source, String user, String label, String tableName) { + super(source); + this.user = user; + this.label = label; + this.tableName = tableName; + } +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/event/UnBindModelEvent.java b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/event/UnBindModelEvent.java new file mode 100644 index 0000000000..a75f305394 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/event/UnBindModelEvent.java @@ -0,0 +1,32 @@ +package com.webank.wedatasphere.dss.datamodel.center.common.event; + + +import com.webank.wedatasphere.dss.data.governance.entity.ClassificationConstant; +import lombok.Getter; +import lombok.ToString; +import org.springframework.context.ApplicationEvent; + +@Getter +@ToString +public class UnBindModelEvent extends ApplicationEvent { + + private String user; + + private String modelName; + + private ClassificationConstant modelType; + + private String tableName; + + private String guid; + + + public UnBindModelEvent(Object source,String user,String guid,String tableName,String modelName,ClassificationConstant modelType) { + super(source); + this.user = user; + this.guid = guid; + this.tableName = tableName; + this.modelName = modelName; + this.modelType = modelType; + } +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/event/UpdateLabelEvent.java b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/event/UpdateLabelEvent.java new file mode 100644 index 0000000000..8c7831fb27 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/event/UpdateLabelEvent.java @@ -0,0 +1,22 @@ +package com.webank.wedatasphere.dss.datamodel.center.common.event; + +import lombok.Getter; +import lombok.ToString; +import org.springframework.context.ApplicationEvent; + +@Getter +@ToString +public class UpdateLabelEvent extends ApplicationEvent { + private String user; + + private String name; + + private String oriName; + + public UpdateLabelEvent(Object source, String user, String name, String oriName) { + super(source); + this.user = user; + this.name = name; + this.oriName = oriName; + } +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/event/UpdateModelEvent.java b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/event/UpdateModelEvent.java new file mode 100644 index 0000000000..7e71bd08d2 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/event/UpdateModelEvent.java @@ -0,0 +1,29 @@ +package com.webank.wedatasphere.dss.datamodel.center.common.event; + + +import com.webank.wedatasphere.dss.data.governance.entity.ClassificationConstant; +import lombok.Getter; +import lombok.ToString; +import org.springframework.context.ApplicationEvent; + +@Getter +@ToString +public class UpdateModelEvent extends ApplicationEvent { + + private String name; + + private ClassificationConstant type; + + private String user; + + private String orgName; + + + public UpdateModelEvent(Object source,String user,String name,String orgName, ClassificationConstant type) { + super(source); + this.user = user; + this.type = type; + this.name = name; + this.orgName = orgName; + } +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/exception/DSSDatamodelCenterException.java b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/exception/DSSDatamodelCenterException.java new file mode 100644 index 0000000000..1e3b2464bb --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/exception/DSSDatamodelCenterException.java @@ -0,0 +1,15 @@ +package com.webank.wedatasphere.dss.datamodel.center.common.exception; + +import org.apache.linkis.common.exception.ErrorException; + + +public class DSSDatamodelCenterException extends ErrorException { + + public DSSDatamodelCenterException(int errCode, String desc) { + super(errCode, desc); + } + + public DSSDatamodelCenterException(int errCode, String desc, String ip, int port, String serviceKind) { + super(errCode, desc, ip, port, serviceKind); + } +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/filter/AuthFilter.java b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/filter/AuthFilter.java new file mode 100644 index 0000000000..3cd1687279 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/filter/AuthFilter.java @@ -0,0 +1,32 @@ +package com.webank.wedatasphere.dss.datamodel.center.common.filter; + + +import com.webank.wedatasphere.dss.datamodel.center.common.context.DataModelAuthentication; +import com.webank.wedatasphere.dss.datamodel.center.common.context.DataModelSecurityContextHolder; +import com.webank.wedatasphere.dss.datamodel.center.common.service.AuthenticationClientStrategy; +import org.springframework.stereotype.Component; + +import javax.servlet.*; +import javax.servlet.http.HttpServletRequest; +import java.io.IOException; + +@Component +public class AuthFilter implements Filter, AuthenticationClientStrategy { + @Override + public void init(FilterConfig filterConfig) throws ServletException { + + } + + @Override + public void destroy() { + + } + + @Override + public void doFilter(ServletRequest servletRequest, ServletResponse servletResponse, FilterChain filterChain) throws IOException, ServletException { + DataModelAuthentication dataModelAuthentication = new DataModelAuthentication(); + dataModelAuthentication.setUser(getStrategyUser((HttpServletRequest) servletRequest)); + DataModelSecurityContextHolder.getContext().setDataModelAuthentication(dataModelAuthentication); + filterChain.doFilter(servletRequest, servletResponse); + } +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/launcher/CommonDataModelJobLauncher.java b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/launcher/CommonDataModelJobLauncher.java new file mode 100644 index 0000000000..48593eb41d --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/launcher/CommonDataModelJobLauncher.java @@ -0,0 +1,12 @@ +package com.webank.wedatasphere.dss.datamodel.center.common.launcher; + + +import org.apache.linkis.computation.client.interactive.SubmittableInteractiveJob; + +public class CommonDataModelJobLauncher implements DataModelJobLauncher { + + @Override + public SubmittableInteractiveJob callBack(SubmittableInteractiveJob job) { + return job; + } +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/launcher/CommonExistsDataModelJobTask.java b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/launcher/CommonExistsDataModelJobTask.java new file mode 100644 index 0000000000..dbc55e8288 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/launcher/CommonExistsDataModelJobTask.java @@ -0,0 +1,10 @@ +package com.webank.wedatasphere.dss.datamodel.center.common.launcher; + + +public class CommonExistsDataModelJobTask extends DataModelJobTask { + + @Override + public void formatCode(String orgCode) { + setCode(orgCode); + } +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/launcher/DataExistsDataModelJobLauncher.java b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/launcher/DataExistsDataModelJobLauncher.java new file mode 100644 index 0000000000..7fe70468a7 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/launcher/DataExistsDataModelJobLauncher.java @@ -0,0 +1,20 @@ +package com.webank.wedatasphere.dss.datamodel.center.common.launcher; + + +import org.apache.linkis.computation.client.ResultSetIterator; +import org.apache.linkis.computation.client.interactive.SubmittableInteractiveJob; + +public class DataExistsDataModelJobLauncher implements DataModelJobLauncher { + @Override + public Integer callBack(SubmittableInteractiveJob job) { + ResultSetIterator iterator = job.getResultSetIterables()[0].iterator(); + + System.out.println(iterator.getMetadata()); + + // 如果已数据返回1 + if (iterator.hasNext()) { + return 1; + } + return 0; + } +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/launcher/DataExistsDataModelJobTask.java b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/launcher/DataExistsDataModelJobTask.java new file mode 100644 index 0000000000..a9f28eeb18 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/launcher/DataExistsDataModelJobTask.java @@ -0,0 +1,10 @@ +package com.webank.wedatasphere.dss.datamodel.center.common.launcher; + + +public class DataExistsDataModelJobTask extends DataModelJobTask { + @Override + public void formatCode(String orgCode) { + setCode(String.format("select * from %s limit 1",orgCode)); + } + +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/launcher/DataModelJobLauncher.java b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/launcher/DataModelJobLauncher.java new file mode 100644 index 0000000000..915e2700cf --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/launcher/DataModelJobLauncher.java @@ -0,0 +1,28 @@ +package com.webank.wedatasphere.dss.datamodel.center.common.launcher; + + +import com.webank.wedatasphere.dss.datamodel.center.common.config.LinkisJobConfiguration; +import org.apache.linkis.computation.client.LinkisJobClient; +import org.apache.linkis.computation.client.interactive.SubmittableInteractiveJob; + +public interface DataModelJobLauncher { + /** + * + * @param launchTask + * @return + */ + default E launch(DataModelJobTask launchTask){ + LinkisJobClient.config().setDefaultServerUrl(LinkisJobConfiguration.LINKIS_SERVER_URL.getValue()); + SubmittableInteractiveJob job = + LinkisJobClient.interactive().builder().setEngineType(launchTask.getEngineType()) + .setRunTypeStr(launchTask.getRunType()).setCreator(launchTask.getCreator()) + .setCode(launchTask.getCode()).addExecuteUser(launchTask.getExecuteUser()).build(); + // 3. Submit Job to Linkis + job.submit(); + // 4. Wait for Job completed + job.waitForCompleted(); + return callBack(job); + } + + E callBack(SubmittableInteractiveJob job); +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/launcher/DataModelJobTask.java b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/launcher/DataModelJobTask.java new file mode 100644 index 0000000000..cbc23adf1e --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/launcher/DataModelJobTask.java @@ -0,0 +1,62 @@ +package com.webank.wedatasphere.dss.datamodel.center.common.launcher; + + + +public abstract class DataModelJobTask { + + private String engineType; + + private String runType; + + private String code; + + private String creator; + + private String executeUser; + + public void setEngineType(String engineType) { + this.engineType = engineType; + } + + public void setRunType(String runType) { + this.runType = runType; + } + + public void setCode(String code) { + this.code = code; + } + + public void setCreator(String creator) { + this.creator = creator; + } + + public void setExecuteUser(String executeUser) { + this.executeUser = executeUser; + } + + public String getEngineType() { + return engineType; + } + + public String getRunType() { + return runType; + } + + public String getCode() { + return code; + } + + + public String getCreator() { + return creator; + } + + + public String getExecuteUser() { + return executeUser; + } + + + + public abstract void formatCode(String orgCode); +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/launcher/DataModelJobTaskBuilder.java b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/launcher/DataModelJobTaskBuilder.java new file mode 100644 index 0000000000..8a480d8bde --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/launcher/DataModelJobTaskBuilder.java @@ -0,0 +1,46 @@ +package com.webank.wedatasphere.dss.datamodel.center.common.launcher; + + +public class DataModelJobTaskBuilder { + + private DataModelJobTask dataModelJobTask; + + public DataModelJobTaskBuilder withDataExistsExchangisJobTask(){ + dataModelJobTask = new DataExistsDataModelJobTask(); + return this; + } + + public DataModelJobTaskBuilder withCommonExchangisJobTask(){ + dataModelJobTask = new CommonExistsDataModelJobTask(); + return this; + } + + public DataModelJobTaskBuilder creator(String creator){ + dataModelJobTask.setCreator(creator); + return this; + } + + public DataModelJobTaskBuilder engineType(String engineType){ + dataModelJobTask.setEngineType(engineType); + return this; + } + + public DataModelJobTaskBuilder runType(String runType){ + dataModelJobTask.setRunType(runType); + return this; + } + + public DataModelJobTaskBuilder executeUser(String executeUser){ + dataModelJobTask.setExecuteUser(executeUser); + return this; + } + + public DataModelJobTaskBuilder code(String code){ + dataModelJobTask.formatCode(code); + return this; + } + + public DataModelJobTask build(){ + return dataModelJobTask; + } +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/launcher/PreviewDataModelJobLauncher.java b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/launcher/PreviewDataModelJobLauncher.java new file mode 100644 index 0000000000..d637977b19 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/launcher/PreviewDataModelJobLauncher.java @@ -0,0 +1,18 @@ +package com.webank.wedatasphere.dss.datamodel.center.common.launcher; + + +import org.apache.linkis.computation.client.ResultSetIterator; +import org.apache.linkis.computation.client.interactive.SubmittableInteractiveJob; +import scala.collection.Map; + +import java.util.List; + +public class PreviewDataModelJobLauncher implements DataModelJobLauncher>>{ + + @Override + public List> callBack(SubmittableInteractiveJob job) { + ResultSetIterator iterator = job.getResultSetIterables()[0].iterator(); + + return null; + } +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/listener/AssertsListener.java b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/listener/AssertsListener.java new file mode 100644 index 0000000000..ce44875ca6 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/listener/AssertsListener.java @@ -0,0 +1,136 @@ +package com.webank.wedatasphere.dss.datamodel.center.common.listener; + + +import com.webank.wedatasphere.dss.data.governance.impl.LinkisDataAssetsRemoteClient; +import com.webank.wedatasphere.dss.data.governance.request.*; +import com.webank.wedatasphere.dss.data.governance.response.*; +import com.webank.wedatasphere.dss.datamodel.center.common.event.*; +import com.webank.wedatasphere.dss.datamodel.center.common.service.AssertsSyncService; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.context.event.EventListener; +import org.springframework.scheduling.annotation.Async; +import org.springframework.scheduling.annotation.EnableAsync; +import org.springframework.stereotype.Component; + +import javax.annotation.Resource; + +@Component +public class AssertsListener { + + private static final Logger LOGGER = LoggerFactory.getLogger(AssertsListener.class); + + + @Resource + private AssertsSyncService assertsSyncService; + + @EventListener + @Async("taskExecutor") + public void createModel(CreateModelEvent event) { + try { + CreateModelTypeResult result = assertsSyncService.syncCreateModel(event); + } catch (Exception e) { + LOGGER.error(e.getMessage(), e); + } + + } + + @EventListener + @Async("taskExecutor") + public void updateModel(UpdateModelEvent event) { + try { + UpdateModelTypeResult result = assertsSyncService.syncUpdateModel(event); + } catch (Exception e) { + LOGGER.error(e.getMessage(), e); + } + } + + @EventListener + @Async("taskExecutor") + public void deleteModel(DeleteModelEvent event) { + LOGGER.info("deleteModel event : {}", event); + try { + DeleteModelTypeResult result = assertsSyncService.syncDeleteModel(event); + LOGGER.info("deleteModel result : {}", result.getResult()); + } catch (Exception e) { + LOGGER.error(e.getMessage(), e); + } + } + + @EventListener + @Async("taskExecutor") + public void bindModel(BindModelEvent event) { + LOGGER.info("bindModel event : {}", event); + try { + BindModelTypeResult result = assertsSyncService.syncBindModel(event); + } catch (Exception e) { + LOGGER.error(e.getMessage(), e); + } + } + + @EventListener + @Async("taskExecutor") + public void unBindModel(UnBindModelEvent event) { + LOGGER.info("unBindModel event : {}", event); + try { + UnBindModelTypeResult result = assertsSyncService.syncUnBindModel(event); + } catch (Exception e) { + LOGGER.error(e.getMessage(), e); + } + } + + @EventListener + @Async("taskExecutor") + public void createLabel(CreateLabelEvent event) { + LOGGER.info("createLabel event : {}", event); + try { + CreateLabelResult result = assertsSyncService.syncCreateLabel(event); + } catch (Exception e) { + LOGGER.error(e.getMessage(), e); + } + } + + @EventListener + @Async("taskExecutor") + public void updateLabel(UpdateLabelEvent event) { + LOGGER.info("updateLabel event : {}", event); + try { + UpdateLabelResult result = assertsSyncService.syncUpdateLabel(event); + } catch (Exception e) { + LOGGER.error(e.getMessage(), e); + } + } + + @EventListener + @Async("taskExecutor") + public void deleteLabel(DeleteLabelEvent event) { + LOGGER.info("deleteLabel event : {}", event); + try { + DeleteLabelResult result = assertsSyncService.syncDeleteLabel(event); + } catch (Exception e) { + LOGGER.error(e.getMessage(), e); + } + } + + @EventListener + @Async("taskExecutor") + public void bindLabel(BindLabelEvent event) { + LOGGER.info("bindLabel event : {}", event); + try { + BindLabelResult result = assertsSyncService.syncBindLabel(event); + } catch (Exception e) { + LOGGER.error(e.getMessage(), e); + } + } + + @EventListener + @Async("taskExecutor") + public void unBindLabel(UnBindLabelEvent event) { + LOGGER.info("unBindLabel event : {}", event); + try { + UnBindLabelResult result = assertsSyncService.syncUnBindLabel(event); + } catch (Exception e) { + LOGGER.error(e.getMessage(), e); + } + } +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/service/AssertsSyncService.java b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/service/AssertsSyncService.java new file mode 100644 index 0000000000..48e55a9433 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/service/AssertsSyncService.java @@ -0,0 +1,210 @@ +package com.webank.wedatasphere.dss.datamodel.center.common.service; + +import com.webank.wedatasphere.dss.data.governance.impl.LinkisDataAssetsRemoteClient; +import com.webank.wedatasphere.dss.data.governance.request.*; +import com.webank.wedatasphere.dss.data.governance.response.*; +import com.webank.wedatasphere.dss.datamodel.center.common.event.*; +import com.webank.wedatasphere.dss.datamodel.center.common.listener.AssertsListener; +import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.context.event.EventListener; +import org.springframework.scheduling.annotation.Async; +import org.springframework.stereotype.Component; + +import javax.annotation.Resource; + +@Component +public class AssertsSyncService { + @Resource + private LinkisDataAssetsRemoteClient client; + + private static final Logger LOGGER = LoggerFactory.getLogger(AssertsListener.class); + + + public CreateModelTypeResult syncCreateModel(CreateModelEvent event){ + LOGGER.info("createModel event : {}",event); + try { + CreateModelTypeResult result = client.createModelType(CreateModelTypeAction.builder() + .setUser(event.getUser()) + .setType(event.getType()) + .setName(event.getName()) + .build()); + LOGGER.info("createModel result : {}",result.getInfo()); + return result; + }catch (Exception e){ + LOGGER.error(e.getMessage(),e); + throw e; + } + } + + public UpdateModelTypeResult syncUpdateModel(UpdateModelEvent event){ + LOGGER.info("updateModel event : {}",event); + if(StringUtils.equals(event.getName(),event.getOrgName())){ + LOGGER.info("name is same ignore"); + return null; + } + try { + UpdateModelTypeResult result = client.updateModelType(UpdateModelTypeAction.builder() + .setUser(event.getUser()) + .setName(event.getName()) + .setType(event.getType()) + .setOrgName(event.getOrgName()) + .build()); + LOGGER.info("updateModel result : {}",result.getInfo()); + return result; + }catch (Exception e){ + LOGGER.error(e.getMessage(),e); + throw e; + } + } + + + public DeleteModelTypeResult syncDeleteModel(DeleteModelEvent event){ + LOGGER.info("deleteModel event : {}",event); + try { + DeleteModelTypeResult result = client.deleteModelType(DeleteModelTypeAction.builder() + .setUser(event.getUser()) + .setType(event.getType()) + .setName(event.getName()) + .build()); + LOGGER.info("deleteModel result : {}",result.getResult()); + return result; + }catch (Exception e){ + LOGGER.error(e.getMessage(),e); + throw e; + } + } + + + public BindModelTypeResult syncBindModel(BindModelEvent event){ + LOGGER.info("bindModel event : {}",event); + try { + BindModelTypeResult result = client.bindModelType(BindModelTypeAction.builder() + .setUser(event.getUser()) + .setGuid(event.getGuid()) + .setModelName(event.getModelName()) + .setModelType(event.getModelType()) + .setTableName(event.getTableName()) + .build()); + LOGGER.info("bindModel result : {}",result.getResult()); + return result; + }catch (Exception e){ + LOGGER.error(e.getMessage(),e); + throw e; + } + } + + + public UnBindModelTypeResult syncUnBindModel(UnBindModelEvent event){ + LOGGER.info("unBindModel event : {}", event); + try { + UnBindModelTypeResult result = client.unBindModelType(UnBindModelTypeAction.builder() + .setUser(event.getUser()) + .setGuid(event.getGuid()) + .setModelName(event.getModelName()) + .setModelType(event.getModelType()) + .setTableName(event.getTableName()) + .build()); + LOGGER.info("unBindModel result : {}",result.getResult()); + return result; + }catch (Exception e){ + LOGGER.error(e.getMessage(),e); + throw e; + } + } + + + public CreateLabelResult syncCreateLabel(CreateLabelEvent event){ + LOGGER.info("createLabel event : {}", event); + try { + CreateLabelResult result = client.createLabel(CreateLabelAction.builder() + .setUser(event.getUser()) + .setName(event.getName()) + .build()); + LOGGER.info("createLabel result : {}",result.getResult()); + return result; + }catch (Exception e){ + LOGGER.error(e.getMessage(),e); + throw e; + } + } + + + public UpdateLabelResult syncUpdateLabel(UpdateLabelEvent event){ + LOGGER.info("updateLabel event : {}", event); + if(StringUtils.equals(event.getName(),event.getOriName())){ + LOGGER.info("name is same ignore"); + return null; + } + try { + UpdateLabelResult result = client.updateLabel(UpdateLabelAction.builder() + .setUser(event.getUser()) + .setOrgName(event.getOriName()) + .setName(event.getName()) + .build()); + LOGGER.info("updateLabel result : {}",result.getResult()); + return result; + }catch (Exception e){ + LOGGER.error(e.getMessage(),e); + throw e; + } + } + + public DeleteLabelResult syncDeleteLabel(DeleteLabelEvent event){ + LOGGER.info("deleteLabel event : {}", event); + try { + DeleteLabelResult result = client.deleteLabel(DeleteLabelAction.builder() + .setUser(event.getUser()) + .setName(event.getName()) + .build()); + LOGGER.info("deleteLabel result : {}",result.getResult()); + return result; + }catch (Exception e){ + LOGGER.error(e.getMessage(),e); + throw e; + } + } + + + public BindLabelResult syncBindLabel(BindLabelEvent event){ + LOGGER.info("bindLabel event : {}", event); + try { + BindLabelResult result = client.bindLabel(BindLabelAction.builder() + .setUser(event.getUser()) + .setLabel(event.getLabel()) + .setTableName(event.getTableName()) + .setLabelGuid(event.getLabelGuid()) + .setTableGuid(event.getTableGuid()) + .build()); + LOGGER.info("bindLabel result : {}",result.getResult()); + return result; + }catch (Exception e){ + LOGGER.error(e.getMessage(),e); + throw e; + } + } + + + public UnBindLabelResult syncUnBindLabel(UnBindLabelEvent event){ + LOGGER.info("unBindLabel event : {}", event); + try { + UnBindLabelResult result = client.unBindLabel(UnBindLabelAction.builder() + .setUser(event.getUser()) + .setLabel(event.getLabel()) + .setTableName(event.getTableName()) + .setLabelGuid(event.getLabelGuid()) + .setTableGuid(event.getTableGuid()) + .setRelationGuid(event.getRelationGuid()) + .build()); + LOGGER.info("unBindLabel result : {}",result.getResult()); + return result; + }catch (Exception e){ + LOGGER.error(e.getMessage(),e); + throw e; + } + } + + + +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/service/AuthenticationClientStrategy.java b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/service/AuthenticationClientStrategy.java new file mode 100644 index 0000000000..60bda13e3d --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/service/AuthenticationClientStrategy.java @@ -0,0 +1,21 @@ +package com.webank.wedatasphere.dss.datamodel.center.common.service; + + +import com.webank.wedatasphere.dss.datamodel.center.common.config.ClientStrategy; +import org.apache.linkis.common.conf.CommonVars; +import org.apache.linkis.server.security.SecurityFilter; +import org.apache.commons.lang.StringUtils; + +import javax.servlet.http.HttpServletRequest; + +public interface AuthenticationClientStrategy { + + CommonVars CLIENT_STRATEGY = CommonVars.apply("wds.wedatasphere.data.model.client.strategy", ""); + + default String getStrategyUser(HttpServletRequest req){ + if (StringUtils.equals(ClientStrategy.TOKEN.getCode(),CLIENT_STRATEGY.getValue())){ + return SecurityFilter.getLoginUsername(req); + } + return "hdfs"; + } +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/service/CycleReferenceService.java b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/service/CycleReferenceService.java new file mode 100644 index 0000000000..f0f72e4a2b --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/service/CycleReferenceService.java @@ -0,0 +1,11 @@ +package com.webank.wedatasphere.dss.datamodel.center.common.service; + + +public interface CycleReferenceService { + /** + * 周期被引用情况 + * @param name + * @return + */ + int cycleReferenceCount(String name); +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/service/DataWarehouseDimensionReferenceService.java b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/service/DataWarehouseDimensionReferenceService.java new file mode 100644 index 0000000000..c2412c343c --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/service/DataWarehouseDimensionReferenceService.java @@ -0,0 +1,5 @@ +package com.webank.wedatasphere.dss.datamodel.center.common.service; + + +public interface DataWarehouseDimensionReferenceService extends DimensionThemeReferenceService{ +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/service/DataWarehouseIndicatorReferenceService.java b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/service/DataWarehouseIndicatorReferenceService.java new file mode 100644 index 0000000000..4f70599fee --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/service/DataWarehouseIndicatorReferenceService.java @@ -0,0 +1,5 @@ +package com.webank.wedatasphere.dss.datamodel.center.common.service; + + +public interface DataWarehouseIndicatorReferenceService extends IndicatorThemeReferenceService,IndicatorModifierReferenceService,IndicatorCycleReferenceService,IndicatorLayerReferenceService{ +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/service/DataWarehouseLabelReferenceService.java b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/service/DataWarehouseLabelReferenceService.java new file mode 100644 index 0000000000..45a7696fb6 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/service/DataWarehouseLabelReferenceService.java @@ -0,0 +1,7 @@ +package com.webank.wedatasphere.dss.datamodel.center.common.service; + +/** + * 标签引用数仓情况统计 + */ +public interface DataWarehouseLabelReferenceService extends LabelThemeReferenceService{ +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/service/DataWarehouseMeasuredReferenceService.java b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/service/DataWarehouseMeasuredReferenceService.java new file mode 100644 index 0000000000..1739046795 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/service/DataWarehouseMeasuredReferenceService.java @@ -0,0 +1,6 @@ +package com.webank.wedatasphere.dss.datamodel.center.common.service; + + + +public interface DataWarehouseMeasuredReferenceService extends MeasuredThemeReferenceService{ +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/service/DataWarehouseReferenceService.java b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/service/DataWarehouseReferenceService.java new file mode 100644 index 0000000000..891383d26d --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/service/DataWarehouseReferenceService.java @@ -0,0 +1,7 @@ +package com.webank.wedatasphere.dss.datamodel.center.common.service; + +/** + * 数仓被引用情况统计 + */ +public interface DataWarehouseReferenceService extends ThemeReferenceService,LayerReferenceService,CycleReferenceService,ModifierReferenceService{ +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/service/DataWarehouseTableReferenceService.java b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/service/DataWarehouseTableReferenceService.java new file mode 100644 index 0000000000..d749b6aaa6 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/service/DataWarehouseTableReferenceService.java @@ -0,0 +1,5 @@ +package com.webank.wedatasphere.dss.datamodel.center.common.service; + + +public interface DataWarehouseTableReferenceService extends TableThemeReferenceService,TableCycleReferenceService,TableLayerReferenceService,TableModifierReferenceService{ +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/service/DatamodelIndicatorReferenceService.java b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/service/DatamodelIndicatorReferenceService.java new file mode 100644 index 0000000000..ceac232038 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/service/DatamodelIndicatorReferenceService.java @@ -0,0 +1,7 @@ +package com.webank.wedatasphere.dss.datamodel.center.common.service; + +/** + * 指标引用数据模型情况统计 + */ +public interface DatamodelIndicatorReferenceService extends IndicatorDimensionReferenceService,IndicatorMeasuredReferenceService,IndicatorIndicatorReferenceService{ +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/service/DatamodelReferencService.java b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/service/DatamodelReferencService.java new file mode 100644 index 0000000000..81d772f8c0 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/service/DatamodelReferencService.java @@ -0,0 +1,9 @@ +package com.webank.wedatasphere.dss.datamodel.center.common.service; + + +/** + * 数据模型被引用情况统计 + */ +public interface DatamodelReferencService extends IndicatorReferenceService,DimensionReferenceService,MeasureReferenceService,LabelReferenceService{ + +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/service/DatamodelTableReferenceService.java b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/service/DatamodelTableReferenceService.java new file mode 100644 index 0000000000..7ab7141d3f --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/service/DatamodelTableReferenceService.java @@ -0,0 +1,5 @@ +package com.webank.wedatasphere.dss.datamodel.center.common.service; + + +public interface DatamodelTableReferenceService extends TableIndicatorReferenceService,TableDimensionReferenceService,TableMeasuredReferenceService,TableLabelReferenceService{ +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/service/DimensionReferenceService.java b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/service/DimensionReferenceService.java new file mode 100644 index 0000000000..251c34fcb5 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/service/DimensionReferenceService.java @@ -0,0 +1,12 @@ +package com.webank.wedatasphere.dss.datamodel.center.common.service; + + +public interface DimensionReferenceService { + + /** + * 维度被引用情况 + * @param name + * @return + */ + int dimensionReferenceCount(String name); +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/service/DimensionThemeReferenceService.java b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/service/DimensionThemeReferenceService.java new file mode 100644 index 0000000000..7f0dd3b82b --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/service/DimensionThemeReferenceService.java @@ -0,0 +1,12 @@ +package com.webank.wedatasphere.dss.datamodel.center.common.service; + + +public interface DimensionThemeReferenceService { + + /** + * 主题引用情况 + * @param name + * @return + */ + int dimensionThemeReferenceCount(String name); +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/service/IndicatorCycleReferenceService.java b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/service/IndicatorCycleReferenceService.java new file mode 100644 index 0000000000..c1aa298991 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/service/IndicatorCycleReferenceService.java @@ -0,0 +1,11 @@ +package com.webank.wedatasphere.dss.datamodel.center.common.service; + + +public interface IndicatorCycleReferenceService { + /** + * 周期引用情况 + * @param name + * @return + */ + int indicatorCycleReferenceCount(String name); +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/service/IndicatorDimensionReferenceService.java b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/service/IndicatorDimensionReferenceService.java new file mode 100644 index 0000000000..8a7677a471 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/service/IndicatorDimensionReferenceService.java @@ -0,0 +1,12 @@ +package com.webank.wedatasphere.dss.datamodel.center.common.service; + + +public interface IndicatorDimensionReferenceService { + + /** + * 指标引用维度计数 + * @param name + * @return + */ + int indicatorDimensionReferenceCount(String name); +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/service/IndicatorIndicatorReferenceService.java b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/service/IndicatorIndicatorReferenceService.java new file mode 100644 index 0000000000..04603abe3d --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/service/IndicatorIndicatorReferenceService.java @@ -0,0 +1,11 @@ +package com.webank.wedatasphere.dss.datamodel.center.common.service; + + +public interface IndicatorIndicatorReferenceService { + /** + * 原子指标引用情况 + * @param name + * @return + */ + int indicatorIndicatorCount(String name); +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/service/IndicatorLayerReferenceService.java b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/service/IndicatorLayerReferenceService.java new file mode 100644 index 0000000000..ac9a7ad2a6 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/service/IndicatorLayerReferenceService.java @@ -0,0 +1,13 @@ +package com.webank.wedatasphere.dss.datamodel.center.common.service; + + +public interface IndicatorLayerReferenceService { + + /** + * 分层引用情况 + * @param name + * @return + */ + int indicatorLayerReferenceCount(String name); + +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/service/IndicatorMeasuredReferenceService.java b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/service/IndicatorMeasuredReferenceService.java new file mode 100644 index 0000000000..e9606873fe --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/service/IndicatorMeasuredReferenceService.java @@ -0,0 +1,12 @@ +package com.webank.wedatasphere.dss.datamodel.center.common.service; + + +public interface IndicatorMeasuredReferenceService { + + /** + * 指标引用度量计数 + * @param name + * @return + */ + int indicatorMeasuredReference(String name); +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/service/IndicatorModifierReferenceService.java b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/service/IndicatorModifierReferenceService.java new file mode 100644 index 0000000000..edb7ccc583 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/service/IndicatorModifierReferenceService.java @@ -0,0 +1,11 @@ +package com.webank.wedatasphere.dss.datamodel.center.common.service; + + +public interface IndicatorModifierReferenceService { + /** + * 修饰词引用情况 + * @param name + * @return + */ + int indicatorModifierReferenceCount(String name); +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/service/IndicatorReferenceService.java b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/service/IndicatorReferenceService.java new file mode 100644 index 0000000000..fea3805440 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/service/IndicatorReferenceService.java @@ -0,0 +1,12 @@ +package com.webank.wedatasphere.dss.datamodel.center.common.service; + + +public interface IndicatorReferenceService { + + /** + * 指标被引用情况 + * @param name + * @return + */ + int indicatorReferenceCount(String name); +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/service/IndicatorThemeReferenceService.java b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/service/IndicatorThemeReferenceService.java new file mode 100644 index 0000000000..d89e04b682 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/service/IndicatorThemeReferenceService.java @@ -0,0 +1,11 @@ +package com.webank.wedatasphere.dss.datamodel.center.common.service; + + +public interface IndicatorThemeReferenceService { + /** + * 主题引用情况 + * @param name + * @return + */ + int indicatorThemeReferenceCount(String name); +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/service/LabelReferenceService.java b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/service/LabelReferenceService.java new file mode 100644 index 0000000000..cbdca0dcf9 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/service/LabelReferenceService.java @@ -0,0 +1,11 @@ +package com.webank.wedatasphere.dss.datamodel.center.common.service; + + +public interface LabelReferenceService { + /** + * 标签被引用情况 + * @param name + * @return + */ + int labelReferenceCount(String name); +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/service/LabelThemeReferenceService.java b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/service/LabelThemeReferenceService.java new file mode 100644 index 0000000000..2627ee4a97 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/service/LabelThemeReferenceService.java @@ -0,0 +1,11 @@ +package com.webank.wedatasphere.dss.datamodel.center.common.service; + + +public interface LabelThemeReferenceService { + /** + * 标签引用情况 + * @param name + * @return + */ + int labelThemeReferenceCount(String name); +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/service/LayerReferenceService.java b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/service/LayerReferenceService.java new file mode 100644 index 0000000000..5d0228ab10 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/service/LayerReferenceService.java @@ -0,0 +1,12 @@ +package com.webank.wedatasphere.dss.datamodel.center.common.service; + + + +public interface LayerReferenceService { + /** + * 分层被引用情况 + * @param name + * @return + */ + int layerReferenceCount(String name); +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/service/MeasureReferenceService.java b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/service/MeasureReferenceService.java new file mode 100644 index 0000000000..289eb18794 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/service/MeasureReferenceService.java @@ -0,0 +1,11 @@ +package com.webank.wedatasphere.dss.datamodel.center.common.service; + + +public interface MeasureReferenceService { + /** + * 度量被引用情况 + * @param name + * @return + */ + int measureReferenceCount(String name); +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/service/MeasuredThemeReferenceService.java b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/service/MeasuredThemeReferenceService.java new file mode 100644 index 0000000000..251a77c28a --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/service/MeasuredThemeReferenceService.java @@ -0,0 +1,12 @@ +package com.webank.wedatasphere.dss.datamodel.center.common.service; + + + +public interface MeasuredThemeReferenceService { + /** + * 主题引用情况 + * @param name + * @return + */ + int measureThemeReferenceCount(String name); +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/service/ModifierReferenceService.java b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/service/ModifierReferenceService.java new file mode 100644 index 0000000000..2e1328ed72 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/service/ModifierReferenceService.java @@ -0,0 +1,11 @@ +package com.webank.wedatasphere.dss.datamodel.center.common.service; + + +public interface ModifierReferenceService { + /** + * 修饰词被引用情况 + * @param name + * @return + */ + int modifierReferenceCount(String name); +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/service/TableCycleReferenceService.java b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/service/TableCycleReferenceService.java new file mode 100644 index 0000000000..bdfb48b778 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/service/TableCycleReferenceService.java @@ -0,0 +1,11 @@ +package com.webank.wedatasphere.dss.datamodel.center.common.service; + + +public interface TableCycleReferenceService { + /** + * 周期引用情况 + * @param name + * @return + */ + int tableCycleReferenceCount(String name); +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/service/TableDimensionReferenceService.java b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/service/TableDimensionReferenceService.java new file mode 100644 index 0000000000..02b00eda85 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/service/TableDimensionReferenceService.java @@ -0,0 +1,12 @@ +package com.webank.wedatasphere.dss.datamodel.center.common.service; + + +public interface TableDimensionReferenceService { + + /** + * 表引用维度计数 + * @param name + * @return + */ + int tableDimensionReferenceCount(String name); +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/service/TableIndicatorReferenceService.java b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/service/TableIndicatorReferenceService.java new file mode 100644 index 0000000000..57c2541adb --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/service/TableIndicatorReferenceService.java @@ -0,0 +1,12 @@ +package com.webank.wedatasphere.dss.datamodel.center.common.service; + + +public interface TableIndicatorReferenceService { + + /** + * 表引用指标计数 + * @param name + * @return + */ + int tableIndicatorReferenceCount(String name); +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/service/TableLabelReferenceService.java b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/service/TableLabelReferenceService.java new file mode 100644 index 0000000000..58eac733a9 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/service/TableLabelReferenceService.java @@ -0,0 +1,12 @@ +package com.webank.wedatasphere.dss.datamodel.center.common.service; + + + +public interface TableLabelReferenceService { + /** + * 表引用标签计数 + * @param name + * @return + */ + int tableLabelReferenceCount(String name); +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/service/TableLayerReferenceService.java b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/service/TableLayerReferenceService.java new file mode 100644 index 0000000000..2cfb2be05e --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/service/TableLayerReferenceService.java @@ -0,0 +1,11 @@ +package com.webank.wedatasphere.dss.datamodel.center.common.service; + + +public interface TableLayerReferenceService { + /** + * 分层引用情况 + * @param name + * @return + */ + int tableLayerReferenceCount(String name); +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/service/TableMeasuredReferenceService.java b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/service/TableMeasuredReferenceService.java new file mode 100644 index 0000000000..8dcd53655a --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/service/TableMeasuredReferenceService.java @@ -0,0 +1,12 @@ +package com.webank.wedatasphere.dss.datamodel.center.common.service; + + +public interface TableMeasuredReferenceService { + + /** + * 表引用度量计数 + * @param name + * @return + */ + int tableMeasuredReferenceService(String name); +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/service/TableModifierReferenceService.java b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/service/TableModifierReferenceService.java new file mode 100644 index 0000000000..51f6534163 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/service/TableModifierReferenceService.java @@ -0,0 +1,11 @@ +package com.webank.wedatasphere.dss.datamodel.center.common.service; + + +public interface TableModifierReferenceService { + /** + * 修饰词引用情况 + * @param name + * @return + */ + int tableModifierReferenceCount(String name); +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/service/TableThemeReferenceService.java b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/service/TableThemeReferenceService.java new file mode 100644 index 0000000000..ad72534aa8 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/service/TableThemeReferenceService.java @@ -0,0 +1,11 @@ +package com.webank.wedatasphere.dss.datamodel.center.common.service; + + +public interface TableThemeReferenceService { + /** + * 主题引用情况 + * @param name + * @return + */ + int tableThemeReferenceCount(String name); +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/service/ThemeReferenceService.java b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/service/ThemeReferenceService.java new file mode 100644 index 0000000000..1d50fc5b8b --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/service/ThemeReferenceService.java @@ -0,0 +1,11 @@ +package com.webank.wedatasphere.dss.datamodel.center.common.service; + + +public interface ThemeReferenceService { + /** + * 主题被引用情况 + * @param name + * @return + */ + int themeReferenceCount(String name); +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/service/impl/DatamodelReferenceServiceImpl.java b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/service/impl/DatamodelReferenceServiceImpl.java new file mode 100644 index 0000000000..0f6f7355b5 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/service/impl/DatamodelReferenceServiceImpl.java @@ -0,0 +1,42 @@ +package com.webank.wedatasphere.dss.datamodel.center.common.service.impl; + +import com.webank.wedatasphere.dss.datamodel.center.common.service.DatamodelIndicatorReferenceService; +import com.webank.wedatasphere.dss.datamodel.center.common.service.DatamodelReferencService; +import com.webank.wedatasphere.dss.datamodel.center.common.service.DatamodelTableReferenceService; +import org.springframework.stereotype.Service; + +import javax.annotation.Resource; + +@Service +public class DatamodelReferenceServiceImpl implements DatamodelReferencService { + + @Resource + private DatamodelIndicatorReferenceService datamodelIndicatorReferenceService; + + @Resource + private DatamodelTableReferenceService datamodelTableReferenceService; + + @Override + public int dimensionReferenceCount(String name) { + return datamodelIndicatorReferenceService.indicatorDimensionReferenceCount(name) + + datamodelTableReferenceService.tableDimensionReferenceCount(name); + } + + @Override + public int indicatorReferenceCount(String name) { + return datamodelTableReferenceService.tableIndicatorReferenceCount(name) + + datamodelIndicatorReferenceService.indicatorIndicatorCount(name); + } + + @Override + public int measureReferenceCount(String name) { + return datamodelIndicatorReferenceService.indicatorMeasuredReference(name) + + datamodelTableReferenceService.tableMeasuredReferenceService(name); + } + + @Override + public int labelReferenceCount(String name){ + return datamodelTableReferenceService.tableLabelReferenceCount(name); + } + +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/ujes/DataModelUJESJobTaskBuilder.java b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/ujes/DataModelUJESJobTaskBuilder.java new file mode 100644 index 0000000000..74f390f720 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/ujes/DataModelUJESJobTaskBuilder.java @@ -0,0 +1,32 @@ +package com.webank.wedatasphere.dss.datamodel.center.common.ujes; + + +import com.webank.wedatasphere.dss.datamodel.center.common.ujes.task.DataModelUJESJobTask; + +public class DataModelUJESJobTaskBuilder { + + private DataModelUJESJobTask dataModelUJESJobTask; + + public DataModelUJESJobTaskBuilder(DataModelUJESJobTask dataModelUJESJobTask){ + this.dataModelUJESJobTask = dataModelUJESJobTask; + } + + public DataModelUJESJobTaskBuilder code(String code){ + dataModelUJESJobTask.formatCode(code); + return this; + } + + public DataModelUJESJobTaskBuilder count(Integer count){ + dataModelUJESJobTask.setCount(count); + return this; + } + + public DataModelUJESJobTaskBuilder user(String user){ + dataModelUJESJobTask.setUser(user); + return this; + } + + public DataModelUJESJobTask build(){ + return dataModelUJESJobTask; + } +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/ujes/launcher/AbstractDataModelUJESJobLauncher.java b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/ujes/launcher/AbstractDataModelUJESJobLauncher.java new file mode 100644 index 0000000000..ea9417b1cb --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/ujes/launcher/AbstractDataModelUJESJobLauncher.java @@ -0,0 +1,56 @@ +package com.webank.wedatasphere.dss.datamodel.center.common.ujes.launcher; + + +import com.webank.wedatasphere.dss.datamodel.center.common.ujes.task.DataModelUJESJobTask; +import org.apache.linkis.common.utils.Utils; +import org.apache.linkis.ujes.client.UJESClient; +import org.apache.linkis.ujes.client.request.JobExecuteAction; +import org.apache.linkis.ujes.client.response.*; +import org.apache.commons.io.IOUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.annotation.Resource; + +public abstract class AbstractDataModelUJESJobLauncher implements DataModelUJESJobLauncher{ + + private static final Logger LOGGER = LoggerFactory.getLogger(AbstractDataModelUJESJobLauncher.class); + + @Resource + protected UJESClient client; + + + protected void launch0(DataModelUJESJobTask task, JobExecuteResult jobExecuteResult){ + LOGGER.info("exec code : {}", task.getCode()); + //System.out.println("execId: " + jobExecuteResult.getExecID() + ", taskId: " + jobExecuteResult.taskID()); + LOGGER.info("execId : {}, taskId : {}",jobExecuteResult.getExecID(),jobExecuteResult.taskID()); + JobStatusResult status = client.status(jobExecuteResult); + while (!status.isCompleted()) { + JobProgressResult progress = client.progress(jobExecuteResult); + //System.out.println("progress: " + progress.getProgress()); + LOGGER.info("progress : {}",progress.getProgress()); + Utils.sleepQuietly(500); + status = client.status(jobExecuteResult); + } + + } + + @Override + public E launch(DataModelUJESJobTask task) { + try { + JobExecuteResult jobExecuteResult = client.execute(JobExecuteAction.builder().setCreator("hdfs") + .addExecuteCode(task.getCode()) + .setEngineType((JobExecuteAction.EngineType) JobExecuteAction.EngineType$.MODULE$.HIVE()).setEngineTypeStr("hql") + .setUser(task.getUser()).build()); + launch0(task,jobExecuteResult); + return callBack(jobExecuteResult); + }catch (Exception e){ + LOGGER.error(e.getMessage(),e); + throw e; + }finally { + //IOUtils.closeQuietly(client); + } + } + + abstract E callBack(JobExecuteResult jobExecuteResult); +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/ujes/launcher/CreateTableDataModelUJESJobLauncher.java b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/ujes/launcher/CreateTableDataModelUJESJobLauncher.java new file mode 100644 index 0000000000..54d0e2e904 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/ujes/launcher/CreateTableDataModelUJESJobLauncher.java @@ -0,0 +1,24 @@ +package com.webank.wedatasphere.dss.datamodel.center.common.ujes.launcher; + +import com.webank.wedatasphere.dss.datamodel.center.common.dto.CreateTableDTO; +import org.apache.linkis.ujes.client.response.JobExecuteResult; +import org.apache.linkis.ujes.client.response.JobInfoResult; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Component; + +@Component +public class CreateTableDataModelUJESJobLauncher extends AbstractDataModelUJESJobLauncher{ + private static final Logger LOGGER = LoggerFactory.getLogger(CreateTableDataModelUJESJobLauncher.class); + + @Override + CreateTableDTO callBack(JobExecuteResult jobExecuteResult) { + JobInfoResult jobInfo = client.getJobInfo(jobExecuteResult); + + CreateTableDTO createTableDTO = new CreateTableDTO(); + createTableDTO.setStatus(jobInfo.getStatus()); + createTableDTO.setTaskId(jobExecuteResult.getTaskID()); + LOGGER.info("create table status : {}, info : {}",createTableDTO,jobInfo); + return createTableDTO; + } +} \ No newline at end of file diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/ujes/launcher/DataExistsDataModelUJESJobLauncher.java b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/ujes/launcher/DataExistsDataModelUJESJobLauncher.java new file mode 100644 index 0000000000..0c786175af --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/ujes/launcher/DataExistsDataModelUJESJobLauncher.java @@ -0,0 +1,25 @@ +package com.webank.wedatasphere.dss.datamodel.center.common.ujes.launcher; + + +import org.apache.linkis.ujes.client.request.ResultSetAction; +import org.apache.linkis.ujes.client.response.JobExecuteResult; +import org.apache.linkis.ujes.client.response.JobInfoResult; +import org.apache.linkis.ujes.client.response.ResultSetResult; +import org.apache.commons.collections.CollectionUtils; +import org.springframework.stereotype.Component; + +import java.util.List; + +@Component +public class DataExistsDataModelUJESJobLauncher extends AbstractDataModelUJESJobLauncher{ + + @Override + Boolean callBack( JobExecuteResult jobExecuteResult) { + JobInfoResult jobInfo = client.getJobInfo(jobExecuteResult); + String resultSet = jobInfo.getResultSetList(client)[0]; + ResultSetResult resultSetResult = client.resultSet(ResultSetAction.builder().setPath(resultSet).setUser(jobExecuteResult.getUser()).build()); + + Object fileContents = resultSetResult.getFileContent(); + return !CollectionUtils.isEmpty((List)fileContents); + } +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/ujes/launcher/DataModelUJESJobLauncher.java b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/ujes/launcher/DataModelUJESJobLauncher.java new file mode 100644 index 0000000000..9646b3b45a --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/ujes/launcher/DataModelUJESJobLauncher.java @@ -0,0 +1,16 @@ +package com.webank.wedatasphere.dss.datamodel.center.common.ujes.launcher; + + +import com.webank.wedatasphere.dss.datamodel.center.common.ujes.task.DataModelUJESJobTask; + +public interface DataModelUJESJobLauncher { + + + /** + * + * @param task + * @return + */ + E launch(DataModelUJESJobTask task); + +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/ujes/launcher/DropTableDataModelUJESJobLauncher.java b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/ujes/launcher/DropTableDataModelUJESJobLauncher.java new file mode 100644 index 0000000000..eff067f672 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/ujes/launcher/DropTableDataModelUJESJobLauncher.java @@ -0,0 +1,16 @@ +package com.webank.wedatasphere.dss.datamodel.center.common.ujes.launcher; + + +import org.apache.linkis.ujes.client.response.JobExecuteResult; +import org.apache.linkis.ujes.client.response.JobInfoResult; +import org.springframework.stereotype.Component; + +@Component +public class DropTableDataModelUJESJobLauncher extends AbstractDataModelUJESJobLauncher{ + + @Override + Boolean callBack(JobExecuteResult jobExecuteResult) { + JobInfoResult jobInfo = client.getJobInfo(jobExecuteResult); + return jobInfo.getStatus()==0; + } +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/ujes/launcher/PreviewDataModelUJESJobLauncher.java b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/ujes/launcher/PreviewDataModelUJESJobLauncher.java new file mode 100644 index 0000000000..63784d1873 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/ujes/launcher/PreviewDataModelUJESJobLauncher.java @@ -0,0 +1,28 @@ +package com.webank.wedatasphere.dss.datamodel.center.common.ujes.launcher; + + + +import com.webank.wedatasphere.dss.datamodel.center.common.dto.PreviewDataDTO; +import org.apache.linkis.ujes.client.request.ResultSetAction; +import org.apache.linkis.ujes.client.response.JobExecuteResult; +import org.apache.linkis.ujes.client.response.JobInfoResult; +import org.apache.linkis.ujes.client.response.ResultSetResult; +import org.springframework.stereotype.Component; + +@Component +public class PreviewDataModelUJESJobLauncher extends AbstractDataModelUJESJobLauncher { + + @Override + PreviewDataDTO callBack(JobExecuteResult jobExecuteResult) { + JobInfoResult jobInfo = client.getJobInfo(jobExecuteResult); + String resultSet = jobInfo.getResultSetList(client)[0]; + ResultSetResult resultSetResult = client.resultSet(ResultSetAction.builder().setPath(resultSet).setUser(jobExecuteResult.getUser()).build()); + + Object fileContents = resultSetResult.getFileContent(); + Object metadata = resultSetResult.getMetadata(); + PreviewDataDTO preview = new PreviewDataDTO(); + preview.setMetadata(metadata); + preview.setData(fileContents); + return preview; + } +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/ujes/task/CreateTableDataModelUJESJobTask.java b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/ujes/task/CreateTableDataModelUJESJobTask.java new file mode 100644 index 0000000000..8c18728a5e --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/ujes/task/CreateTableDataModelUJESJobTask.java @@ -0,0 +1,20 @@ +package com.webank.wedatasphere.dss.datamodel.center.common.ujes.task; + + +import com.webank.wedatasphere.dss.datamodel.center.common.ujes.DataModelUJESJobTaskBuilder; + +public class CreateTableDataModelUJESJobTask extends DataModelUJESJobTask { + + public static DataModelUJESJobTaskBuilder newBuilder() { + return new DataModelUJESJobTaskBuilder(new CreateTableDataModelUJESJobTask()); + } + + CreateTableDataModelUJESJobTask() { + } + + + @Override + public void formatCode(String code) { + setCode(code); + } +} \ No newline at end of file diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/ujes/task/DataExistsDataModelUJESJobTask.java b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/ujes/task/DataExistsDataModelUJESJobTask.java new file mode 100644 index 0000000000..b39415e706 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/ujes/task/DataExistsDataModelUJESJobTask.java @@ -0,0 +1,18 @@ +package com.webank.wedatasphere.dss.datamodel.center.common.ujes.task; + + +import com.webank.wedatasphere.dss.datamodel.center.common.ujes.DataModelUJESJobTaskBuilder; + +public class DataExistsDataModelUJESJobTask extends DataModelUJESJobTask{ + + public static DataModelUJESJobTaskBuilder newBuilder(){ + return new DataModelUJESJobTaskBuilder(new DataExistsDataModelUJESJobTask()); + } + + DataExistsDataModelUJESJobTask(){}; + + @Override + public void formatCode(String code) { + setCode( String.format("select * from %s limit 1",code)); + } +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/ujes/task/DataModelUJESJobTask.java b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/ujes/task/DataModelUJESJobTask.java new file mode 100644 index 0000000000..f417f50011 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/ujes/task/DataModelUJESJobTask.java @@ -0,0 +1,16 @@ +package com.webank.wedatasphere.dss.datamodel.center.common.ujes.task; + + +import lombok.Data; + +@Data +public abstract class DataModelUJESJobTask { + + private String code; + + private Integer count = 10; + + private String user; + + public abstract void formatCode(String code); +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/ujes/task/DropTableDataModelUJESJobTask.java b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/ujes/task/DropTableDataModelUJESJobTask.java new file mode 100644 index 0000000000..ca78c06837 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/ujes/task/DropTableDataModelUJESJobTask.java @@ -0,0 +1,21 @@ +package com.webank.wedatasphere.dss.datamodel.center.common.ujes.task; + + +import com.webank.wedatasphere.dss.datamodel.center.common.ujes.DataModelUJESJobTaskBuilder; + +public class DropTableDataModelUJESJobTask extends DataModelUJESJobTask { + + public static DataModelUJESJobTaskBuilder newBuilder() { + return new DataModelUJESJobTaskBuilder(new DropTableDataModelUJESJobTask()); + } + + DropTableDataModelUJESJobTask() { + } + + + @Override + public void formatCode(String code) { + setCode(String.format("drop table if exists %s",code)); + } + +} \ No newline at end of file diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/ujes/task/PreviewDataModelUJESJobTask.java b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/ujes/task/PreviewDataModelUJESJobTask.java new file mode 100644 index 0000000000..4188e53832 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/ujes/task/PreviewDataModelUJESJobTask.java @@ -0,0 +1,21 @@ +package com.webank.wedatasphere.dss.datamodel.center.common.ujes.task; + + +import com.webank.wedatasphere.dss.datamodel.center.common.ujes.DataModelUJESJobTaskBuilder; + +public class PreviewDataModelUJESJobTask extends DataModelUJESJobTask { + + public static DataModelUJESJobTaskBuilder newBuilder() { + return new DataModelUJESJobTaskBuilder(new PreviewDataModelUJESJobTask()); + } + + PreviewDataModelUJESJobTask() { + } + + + @Override + public void formatCode(String code) { + setCode(String.format("select * from %s limit %s", code,getCount()+"")); + } + +} \ No newline at end of file diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/vo/PageVO.java b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/vo/PageVO.java new file mode 100644 index 0000000000..ec3e6a80c6 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/main/java/com/webank/wedatasphere/dss/datamodel/center/common/vo/PageVO.java @@ -0,0 +1,13 @@ +package com.webank.wedatasphere.dss.datamodel.center.common.vo; + + +import lombok.Data; +import lombok.ToString; + +@Data +@ToString +public class PageVO { + private Integer pageSize = 50; + + private Integer pageNum = 1; +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/test/java/com/webank/wedatasphere/dss/datamodel/TestJobTask.java b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/test/java/com/webank/wedatasphere/dss/datamodel/TestJobTask.java new file mode 100644 index 0000000000..77819be5c0 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/test/java/com/webank/wedatasphere/dss/datamodel/TestJobTask.java @@ -0,0 +1,35 @@ +package com.webank.wedatasphere.dss.datamodel; + + +import org.apache.linkis.computation.client.LinkisJobClient; +import org.apache.linkis.computation.client.ResultSetIterator; +import org.apache.linkis.computation.client.interactive.SubmittableInteractiveJob; + +public class TestJobTask { + public static void main(String[] args) { + // TODO First, set the right gateway url. + LinkisJobClient.config().setDefaultServerUrl("http://121.36.12.247:8088"); + //TODO Secondly, please modify the executeUser + SubmittableInteractiveJob job = LinkisJobClient.interactive().builder() + .setEngineType("hive-2.3.3").setRunTypeStr("hql") + .setCode("select * from linkis_db.linkis_partitions limit 1") + .addExecuteUser("hdfs") + .setCreator("hdfs") + .build(); + // 3. Submit Job to Linkis + job.submit(); +// System.out.println("execId: " + job.getJobSubmitResult().getExecID() + ", taskId: " + job.getJobSubmitResult().taskID()); +// System.out.println("ResponseBody:"+job.getJobSubmitResult().getResponseBody()); + +// // 4. Wait for Job completed + job.waitForCompleted(); + +// // 5. Get results from iterators. + ResultSetIterator iterator = job.getResultSetIterables()[0].iterator(); + System.out.println(iterator.getMetadata()); + while(iterator.hasNext()){ + System.out.println(iterator.next()); + } + + } +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/test/java/com/webank/wedatasphere/dss/datamodel/UJESClientImplTestJ.java b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/test/java/com/webank/wedatasphere/dss/datamodel/UJESClientImplTestJ.java new file mode 100644 index 0000000000..439fd817e7 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-center-common/src/test/java/com/webank/wedatasphere/dss/datamodel/UJESClientImplTestJ.java @@ -0,0 +1,97 @@ +/* + * Copyright 2019 WeBank + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.webank.wedatasphere.dss.datamodel; + +import com.google.gson.Gson; +import org.apache.linkis.common.utils.Utils; +import org.apache.linkis.httpclient.dws.authentication.StaticAuthenticationStrategy; +import org.apache.linkis.httpclient.dws.config.DWSClientConfig; +import org.apache.linkis.httpclient.dws.config.DWSClientConfigBuilder; +import org.apache.linkis.ujes.client.UJESClient; +import org.apache.linkis.ujes.client.UJESClientImpl; +import org.apache.linkis.ujes.client.request.JobExecuteAction; +import org.apache.linkis.ujes.client.request.ResultSetAction; +import org.apache.linkis.ujes.client.response.*; +import lombok.Data; +import org.apache.commons.io.IOUtils; + +import java.util.List; +import java.util.concurrent.TimeUnit; + +@Deprecated +public class UJESClientImplTestJ{ + public static void main(String[] args){ + // Suggest to use LinkisJobClient to submit job to Linkis. + DWSClientConfig clientConfig = ((DWSClientConfigBuilder) (DWSClientConfigBuilder.newBuilder().addServerUrl("http://121.36.12.247:8088") + .connectionTimeout(30000).discoveryEnabled(true) + .discoveryFrequency(1, TimeUnit.MINUTES) + .loadbalancerEnabled(true).maxConnectionSize(5) + .retryEnabled(false).readTimeout(30000) + .setAuthenticationStrategy(new StaticAuthenticationStrategy()).setAuthTokenKey("hdfs") + .setAuthTokenValue("hdfs"))).setDWSVersion("v1").build(); + UJESClient client = new UJESClientImpl(clientConfig); + + JobExecuteResult jobExecuteResult = client.execute(JobExecuteAction.builder().setCreator("hdfs") + .addExecuteCode("select * from default.testTable3 limit 1") + //.addExecuteCode("show tables") + //.addExecuteCode(String.format("drop table if exists %s","linkis_db.linkis_test01")) + .setEngineType((JobExecuteAction.EngineType)JobExecuteAction.EngineType$.MODULE$.HIVE()).setEngineTypeStr("hql") + .setUser("hdfs").build()); + System.out.println("execId: " + jobExecuteResult.getExecID() + ", taskId: " + jobExecuteResult.taskID()); + JobStatusResult status = client.status(jobExecuteResult); + while(!status.isCompleted()) { + JobProgressResult progress = client.progress(jobExecuteResult); + System.out.println("progress: " + progress.getProgress()); + Utils.sleepQuietly(500); + status = client.status(jobExecuteResult); + } + JobInfoResult jobInfo = client.getJobInfo(jobExecuteResult); + System.out.println(jobInfo.getStatus()); + String resultSet = jobInfo.getResultSetList(client)[0]; + ResultSetResult resultSetResult =client.resultSet(ResultSetAction.builder().setPath(resultSet).setUser(jobExecuteResult.getUser()).build()); + Object fileContents = resultSetResult.getFileContent(); + Object metadata = resultSetResult.getMetadata(); + System.out.println("metadata: " + metadata); + System.out.println("fileContents: " + fileContents); + System.out.println("fileContents size: " + ((List)fileContents).size()); + Preview preview = new Preview(); + preview.setMetadata(metadata); + preview.setData(fileContents); + Gson gson = new Gson(); + System.out.println(gson.toJson(preview)); + IOUtils.closeQuietly(client); + } + + @Data + public static class Preview{ + private Object metadata; + + private Object data; + } + + @Data + public static class Metadata{ + private String columnName; + + private String comment; + + private String dataType; + } + + + +} \ No newline at end of file diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-center-server/bin/start-datamodel-center.sh b/dss-apps/dss-datamodel-center/dss-datamodel-center-server/bin/start-datamodel-center.sh new file mode 100755 index 0000000000..af94bd2835 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-center-server/bin/start-datamodel-center.sh @@ -0,0 +1,38 @@ +#!/bin/bash + +cd `dirname $0` +cd .. +HOME=`pwd` + +export SERVER_PID=$HOME/bin/linkis.pid +export SERVER_LOG_PATH=$HOME/logs +export SERVER_CLASS=com.webank.wedatasphere.dss.datamodel.DataModelApplication + +if test -z "$SERVER_HEAP_SIZE" +then + export SERVER_HEAP_SIZE="512M" +fi + +if test -z "$SERVER_JAVA_OPTS" +then + export SERVER_JAVA_OPTS=" -Xmx$SERVER_HEAP_SIZE -XX:+UseG1GC -Xloggc:$HOME/logs/linkis-gc.log" +fi + +if [[ -f "${SERVER_PID}" ]]; then + pid=$(cat ${SERVER_PID}) + if kill -0 ${pid} >/dev/null 2>&1; then + echo "Server is already running." + exit 1 + fi +fi + +nohup java $SERVER_JAVA_OPTS -cp $HOME/conf:$HOME/lib/* $SERVER_CLASS 2>&1 > $SERVER_LOG_PATH/linkis.out & +pid=$! +if [[ -z "${pid}" ]]; then + echo "server $SERVER_NAME start failed!" + exit 1 +else + echo "server $SERVER_NAME start succeeded!" + echo $pid > $SERVER_PID + sleep 1 +fi \ No newline at end of file diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-center-server/bin/stop-datamodel-center.sh b/dss-apps/dss-datamodel-center/dss-datamodel-center-server/bin/stop-datamodel-center.sh new file mode 100755 index 0000000000..f032887111 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-center-server/bin/stop-datamodel-center.sh @@ -0,0 +1,47 @@ +#!/bin/bash + +cd `dirname $0` +cd .. +HOME=`pwd` + +export SERVER_PID=$HOME/bin/linkis.pid + +function wait_for_server_to_die() { + local pid + local count + pid=$1 + timeout=$2 + count=0 + timeoutTime=$(date "+%s") + let "timeoutTime+=$timeout" + currentTime=$(date "+%s") + forceKill=1 + + while [[ $currentTime -lt $timeoutTime ]]; do + $(kill ${pid} > /dev/null 2> /dev/null) + if kill -0 ${pid} > /dev/null 2>&1; then + sleep 3 + else + forceKill=0 + break + fi + currentTime=$(date "+%s") + done + + if [[ forceKill -ne 0 ]]; then + $(kill -9 ${pid} > /dev/null 2> /dev/null) + fi +} + +if [[ ! -f "${SERVER_PID}" ]]; then + echo "server $SERVER_NAME is not running" +else + pid=$(cat ${SERVER_PID}) + if [[ -z "${pid}" ]]; then + echo "server $SERVER_NAME is not running" + else + wait_for_server_to_die $pid 40 + $(rm -f ${SERVER_PID}) + echo "server $SERVER_NAME is stopped." + fi +fi \ No newline at end of file diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-center-server/db/datamodel_ddl.sql b/dss-apps/dss-datamodel-center/dss-datamodel-center-server/db/datamodel_ddl.sql new file mode 100644 index 0000000000..ddf7202abd --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-center-server/db/datamodel_ddl.sql @@ -0,0 +1,320 @@ + +SET NAMES utf8mb4; +SET FOREIGN_KEY_CHECKS = 0; + + +-- ---------------------------- +-- Table structure for dss_datamodel_dimension +-- ---------------------------- +DROP TABLE IF EXISTS `dss_datamodel_dimension`; +CREATE TABLE `dss_datamodel_dimension` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `name` varchar(255) COLLATE utf8_bin NOT NULL, + `field_identifier` varchar(255) COLLATE utf8_bin DEFAULT NULL, + `formula` varchar(255) COLLATE utf8_bin NOT NULL, + `comment` varchar(255) COLLATE utf8_bin DEFAULT NULL, + `warehouse_theme_name` varchar(512) COLLATE utf8_bin DEFAULT NULL COMMENT '数仓主题格式为： theme_domain_name.theme_name', + `owner` varchar(64) COLLATE utf8_bin NOT NULL, + `principal_name` varchar(128) COLLATE utf8_bin DEFAULT NULL COMMENT '授权的名字：userName、roleName', + `is_available` tinyint(1) NOT NULL, + `create_time` datetime NOT NULL, + `update_time` datetime NOT NULL, + `warehouse_theme_name_en` varchar(512) COLLATE utf8_bin DEFAULT NULL COMMENT '英文', + PRIMARY KEY (`id`) USING BTREE, + UNIQUE KEY `name` (`name`) USING BTREE, + UNIQUE KEY `field_identifier` (`field_identifier`) USING BTREE +) ENGINE=InnoDB AUTO_INCREMENT=27 DEFAULT CHARSET=utf8 COLLATE=utf8_bin; + +-- ---------------------------- +-- Table structure for dss_datamodel_indicator +-- ---------------------------- +DROP TABLE IF EXISTS `dss_datamodel_indicator`; +CREATE TABLE `dss_datamodel_indicator` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `name` varchar(255) COLLATE utf8_bin NOT NULL, + `field_identifier` varchar(255) COLLATE utf8_bin DEFAULT NULL, + `comment` varchar(255) COLLATE utf8_bin DEFAULT NULL, + `warehouse_theme_name` varchar(512) COLLATE utf8_bin DEFAULT NULL COMMENT '数仓主题格式为： theme_domain_name.theme_name', + `owner` varchar(64) COLLATE utf8_bin NOT NULL, + `principal_name` varchar(128) COLLATE utf8_bin DEFAULT NULL COMMENT '授权的名字：userName、roleName', + `is_available` tinyint(1) NOT NULL, + `is_core_indicator` tinyint(1) NOT NULL, + `theme_area` varchar(1000) COLLATE utf8_bin NOT NULL COMMENT '空：代表所有，如果是逗号分隔的字符串则代表对应的theme的names', + `layer_area` varchar(1000) COLLATE utf8_bin NOT NULL COMMENT '空：代表所有，如果是逗号分隔的字符串则代表对应的layer的names', + `create_time` datetime NOT NULL, + `update_time` datetime NOT NULL, + `version` varchar(255) COLLATE utf8_bin NOT NULL, + `warehouse_theme_name_en` varchar(512) COLLATE utf8_bin DEFAULT NULL COMMENT '数仓主题英文', + `theme_area_en` varchar(1000) COLLATE utf8_bin NOT NULL COMMENT '英文名称', + `layer_area_en` varchar(1000) COLLATE utf8_bin NOT NULL COMMENT '英文名称', + PRIMARY KEY (`id`) USING BTREE, + UNIQUE KEY `name` (`name`) USING BTREE, + KEY `field_identifier` (`field_identifier`) USING BTREE +) ENGINE=InnoDB AUTO_INCREMENT=82 DEFAULT CHARSET=utf8 COLLATE=utf8_bin; + +-- ---------------------------- +-- Table structure for dss_datamodel_indicator_content +-- ---------------------------- +DROP TABLE IF EXISTS `dss_datamodel_indicator_content`; +CREATE TABLE `dss_datamodel_indicator_content` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `indicator_id` int(11) NOT NULL, + `version` varchar(255) COLLATE utf8_bin NOT NULL, + `indicator_type` int(4) NOT NULL COMMENT '0 原子 1 衍生 2 派生 3 复杂 4 自定义', + `measure_id` int(11) DEFAULT NULL, + `indicator_source_info` text COLLATE utf8_bin NOT NULL COMMENT '指标来源信息', + `formula` varchar(255) COLLATE utf8_bin NOT NULL, + `business` varchar(255) COLLATE utf8_bin NOT NULL, + `business_owner` varchar(255) COLLATE utf8_bin NOT NULL, + `calculation` varchar(255) COLLATE utf8_bin NOT NULL, + `calculation_owner` varchar(255) COLLATE utf8_bin NOT NULL, + `create_time` datetime NOT NULL, + `update_time` datetime NOT NULL, + PRIMARY KEY (`id`) USING BTREE +) ENGINE=InnoDB AUTO_INCREMENT=158 DEFAULT CHARSET=utf8 COLLATE=utf8_bin; + +-- ---------------------------- +-- Table structure for dss_datamodel_indicator_version +-- ---------------------------- +DROP TABLE IF EXISTS `dss_datamodel_indicator_version`; +CREATE TABLE `dss_datamodel_indicator_version` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `name` varchar(255) COLLATE utf8_bin NOT NULL, + `owner` varchar(64) COLLATE utf8_bin NOT NULL, + `principal_name` varchar(128) COLLATE utf8_bin DEFAULT NULL COMMENT '授权的名字：userName、roleName', + `version` varchar(255) COLLATE utf8_bin NOT NULL, + `version_context` text COLLATE utf8_bin NOT NULL COMMENT '历史版本详细信息快照', + `create_time` datetime NOT NULL, + `update_time` datetime NOT NULL, + `comment` varchar(255) COLLATE utf8_bin DEFAULT NULL, + PRIMARY KEY (`id`) USING BTREE, + UNIQUE KEY `name` (`name`,`version`) USING BTREE +) ENGINE=InnoDB AUTO_INCREMENT=50 DEFAULT CHARSET=utf8 COLLATE=utf8_bin; + +-- ---------------------------- +-- Table structure for dss_datamodel_label +-- ---------------------------- +DROP TABLE IF EXISTS `dss_datamodel_label`; +CREATE TABLE `dss_datamodel_label` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `name` varchar(255) COLLATE utf8_bin NOT NULL, + `field_identifier` varchar(255) COLLATE utf8_bin DEFAULT NULL, + `params` text COLLATE utf8_bin COMMENT '标签键值对 json', + `comment` varchar(255) COLLATE utf8_bin DEFAULT NULL, + `warehouse_theme_name` varchar(512) COLLATE utf8_bin DEFAULT NULL COMMENT '数仓主题格式为： theme_domain_name.theme_name', + `owner` varchar(64) COLLATE utf8_bin NOT NULL, + `principal_name` varchar(128) COLLATE utf8_bin DEFAULT NULL COMMENT '授权的名字：userName、roleName', + `is_available` tinyint(1) NOT NULL, + `create_time` datetime NOT NULL, + `update_time` datetime NOT NULL, + `warehouse_theme_name_en` varchar(512) COLLATE utf8_bin DEFAULT NULL COMMENT '英文', + PRIMARY KEY (`id`) USING BTREE, + UNIQUE KEY `name` (`name`) USING BTREE, + UNIQUE KEY `field_identifier` (`field_identifier`) USING BTREE +) ENGINE=InnoDB AUTO_INCREMENT=34 DEFAULT CHARSET=utf8 COLLATE=utf8_bin; + +-- ---------------------------- +-- Table structure for dss_datamodel_measure +-- ---------------------------- +DROP TABLE IF EXISTS `dss_datamodel_measure`; +CREATE TABLE `dss_datamodel_measure` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `name` varchar(255) COLLATE utf8_bin NOT NULL, + `field_identifier` varchar(255) COLLATE utf8_bin DEFAULT NULL, + `formula` varchar(255) COLLATE utf8_bin NOT NULL, + `comment` varchar(255) COLLATE utf8_bin DEFAULT NULL, + `warehouse_theme_name` varchar(512) COLLATE utf8_bin DEFAULT NULL COMMENT '数仓主题格式为： theme_domain_name.theme_name', + `owner` varchar(64) COLLATE utf8_bin NOT NULL, + `principal_name` varchar(128) COLLATE utf8_bin DEFAULT NULL COMMENT '授权的名字：userName、roleName', + `is_available` tinyint(1) NOT NULL, + `create_time` datetime NOT NULL, + `update_time` datetime NOT NULL, + `warehouse_theme_name_en` varchar(512) COLLATE utf8_bin DEFAULT NULL COMMENT '英文', + PRIMARY KEY (`id`) USING BTREE, + UNIQUE KEY `name` (`name`) USING BTREE, + UNIQUE KEY `field_identifier` (`field_identifier`) USING BTREE +) ENGINE=InnoDB AUTO_INCREMENT=13 DEFAULT CHARSET=utf8 COLLATE=utf8_bin; + +-- ---------------------------- +-- Table structure for dss_datamodel_table +-- ---------------------------- +DROP TABLE IF EXISTS `dss_datamodel_table`; +CREATE TABLE `dss_datamodel_table` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `data_base` varchar(255) COLLATE utf8_bin NOT NULL, + `name` varchar(255) COLLATE utf8_bin NOT NULL, + `alias` varchar(255) COLLATE utf8_bin DEFAULT NULL, + `creator` varchar(255) COLLATE utf8_bin NOT NULL, + `comment` varchar(255) COLLATE utf8_bin DEFAULT NULL, + `create_time` datetime NOT NULL, + `update_time` datetime NOT NULL, + `warehouse_layer_name` varchar(512) COLLATE utf8_bin DEFAULT NULL COMMENT '数仓层级', + `warehouse_layer_name_en` varchar(512) COLLATE utf8_bin DEFAULT NULL COMMENT '数仓层级英文', + `warehouse_theme_name` varchar(512) COLLATE utf8_bin DEFAULT NULL COMMENT '数仓主题格式为： theme_domain_name.theme_name', + `warehouse_theme_name_en` varchar(512) COLLATE utf8_bin DEFAULT NULL COMMENT '数仓主题英文', + `lifecycle` varchar(512) COLLATE utf8_bin DEFAULT NULL COMMENT '生命周期', + `is_partition_table` tinyint(1) NOT NULL, + `is_available` tinyint(1) NOT NULL, + `storage_type` varchar(255) COLLATE utf8_bin NOT NULL COMMENT '存储类型：hive/mysql', + `principal_name` varchar(128) COLLATE utf8_bin DEFAULT NULL COMMENT '授权的名字：userName、roleName', + `compress` varchar(128) COLLATE utf8_bin NOT NULL COMMENT '压缩格式', + `file_type` varchar(128) COLLATE utf8_bin NOT NULL COMMENT '文件格式', + `version` varchar(128) COLLATE utf8_bin NOT NULL COMMENT '版本信息：默认1', + `is_external` tinyint(1) NOT NULL COMMENT '是否外部表 0 内部表 1外部表', + `location` varchar(255) COLLATE utf8_bin DEFAULT NULL COMMENT '外部表时 location', + `label` varchar(255) COLLATE utf8_bin DEFAULT NULL COMMENT '标签', + `lifecycle_en` varchar(512) COLLATE utf8_bin DEFAULT NULL COMMENT '生命周期英文', + PRIMARY KEY (`id`) USING BTREE, + UNIQUE KEY `database` (`data_base`,`name`) USING BTREE, + UNIQUE KEY `name` (`name`) USING BTREE +) ENGINE=InnoDB AUTO_INCREMENT=92 DEFAULT CHARSET=utf8 COLLATE=utf8_bin; + +-- ---------------------------- +-- Table structure for dss_datamodel_table_collcetion +-- ---------------------------- +DROP TABLE IF EXISTS `dss_datamodel_table_collcetion`; +CREATE TABLE `dss_datamodel_table_collcetion` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `data_base` varchar(255) COLLATE utf8_bin DEFAULT NULL, + `name` varchar(255) COLLATE utf8_bin NOT NULL, + `alias` varchar(255) COLLATE utf8_bin DEFAULT NULL, + `creator` varchar(255) COLLATE utf8_bin DEFAULT NULL, + `comment` varchar(255) COLLATE utf8_bin DEFAULT NULL, + `create_time` datetime NOT NULL, + `update_time` datetime NOT NULL, + `warehouse_layer_name` varchar(512) COLLATE utf8_bin DEFAULT NULL COMMENT '数仓层级', + `warehouse_theme_name` varchar(512) COLLATE utf8_bin DEFAULT NULL COMMENT '数仓主题格式为： theme_domain_name.theme_name', + `lifecycle` varchar(512) COLLATE utf8_bin DEFAULT NULL COMMENT '生命周期', + `is_partition_table` tinyint(1) DEFAULT NULL, + `is_available` tinyint(1) DEFAULT NULL, + `storage_type` varchar(255) COLLATE utf8_bin DEFAULT NULL COMMENT '存储类型：hive/mysql', + `principal_name` varchar(128) COLLATE utf8_bin DEFAULT NULL COMMENT '授权的名字：userName、roleName', + `compress` varchar(128) COLLATE utf8_bin DEFAULT NULL COMMENT '压缩格式', + `file_type` varchar(128) COLLATE utf8_bin DEFAULT NULL COMMENT '文件格式', + `user` varchar(255) COLLATE utf8_bin NOT NULL COMMENT '收藏人', + `version` varchar(128) COLLATE utf8_bin DEFAULT NULL COMMENT '版本信息：默认1', + `is_external` tinyint(1) DEFAULT NULL COMMENT '是否外部表 0 内部表 1外部表', + `location` varchar(255) COLLATE utf8_bin DEFAULT NULL COMMENT '外部表时 location', + `label` varchar(255) COLLATE utf8_bin DEFAULT NULL COMMENT '标签', + `guid` varchar(255) COLLATE utf8_bin DEFAULT NULL COMMENT 'atlas标识', + `warehouse_layer_name_en` varchar(512) COLLATE utf8_bin DEFAULT NULL COMMENT '数仓层级英文', + `warehouse_theme_name_en` varchar(512) COLLATE utf8_bin DEFAULT NULL COMMENT '数仓主题英文', + `lifecycle_en` varchar(512) COLLATE utf8_bin DEFAULT NULL COMMENT '生命周期英文', + PRIMARY KEY (`id`) USING BTREE +) ENGINE=InnoDB AUTO_INCREMENT=26 DEFAULT CHARSET=utf8 COLLATE=utf8_bin; + +-- ---------------------------- +-- Table structure for dss_datamodel_table_columns +-- ---------------------------- +DROP TABLE IF EXISTS `dss_datamodel_table_columns`; +CREATE TABLE `dss_datamodel_table_columns` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `table_id` int(11) NOT NULL, + `name` varchar(255) COLLATE utf8_bin NOT NULL, + `alias` varchar(255) COLLATE utf8_bin DEFAULT NULL, + `type` varchar(64) COLLATE utf8_bin NOT NULL, + `comment` varchar(255) COLLATE utf8_bin DEFAULT NULL, + `is_partition_field` tinyint(1) NOT NULL, + `is_primary` tinyint(1) NOT NULL, + `length` int(11) DEFAULT NULL, + `rule` varchar(128) COLLATE utf8_bin DEFAULT NULL, + `model_type` int(4) DEFAULT NULL COMMENT '0 维度，1 指标 2 度量', + `model_name_en` varchar(255) COLLATE utf8_bin DEFAULT NULL COMMENT '模型信息英文名称', + `model_id` int(11) DEFAULT NULL COMMENT '关联具体模型id信息（因为有版本数据表id不可靠，暂时不用）', + `model_name` varchar(255) COLLATE utf8_bin DEFAULT NULL COMMENT '模型信息名称', + `create_time` datetime NOT NULL, + `update_time` datetime NOT NULL, + PRIMARY KEY (`id`) USING BTREE +) ENGINE=InnoDB AUTO_INCREMENT=226 DEFAULT CHARSET=utf8 COLLATE=utf8_bin; + +-- ---------------------------- +-- Table structure for dss_datamodel_table_materialized_history +-- ---------------------------- +DROP TABLE IF EXISTS `dss_datamodel_table_materialized_history`; +CREATE TABLE `dss_datamodel_table_materialized_history` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `materialized_code` mediumtext COLLATE utf8_bin COMMENT '物化sql', + `reason` varchar(255) COLLATE utf8_bin DEFAULT NULL COMMENT '物化原因', + `creator` varchar(255) COLLATE utf8_bin DEFAULT NULL COMMENT '物化者', + `status` int(4) NOT NULL COMMENT 'succeed,failed,in progess', + `create_time` datetime NOT NULL, + `last_update_time` datetime NOT NULL, + `task_id` varchar(512) COLLATE utf8_bin NOT NULL, + `error_msg` varchar(255) COLLATE utf8_bin DEFAULT NULL, + `tableName` varchar(255) COLLATE utf8_bin NOT NULL COMMENT '表名', + `data_base` varchar(255) COLLATE utf8_bin NOT NULL, + `version` varchar(128) COLLATE utf8_bin NOT NULL COMMENT '版本信息：默认1', + PRIMARY KEY (`id`) USING BTREE +) ENGINE=InnoDB AUTO_INCREMENT=66 DEFAULT CHARSET=utf8 COLLATE=utf8_bin; + +-- ---------------------------- +-- Table structure for dss_datamodel_table_params +-- ---------------------------- +DROP TABLE IF EXISTS `dss_datamodel_table_params`; +CREATE TABLE `dss_datamodel_table_params` ( + `tbl_id` int(11) NOT NULL, + `param_key` varchar(256) COLLATE utf8_bin NOT NULL COMMENT '创建表参数：压缩/orc等', + `param_value` mediumtext COLLATE utf8_bin, + PRIMARY KEY (`tbl_id`,`param_key`) USING BTREE, + KEY `table_params_n49` (`tbl_id`) USING BTREE +) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin; + +-- ---------------------------- +-- Table structure for dss_datamodel_table_statics +-- ---------------------------- +DROP TABLE IF EXISTS `dss_datamodel_table_statics`; +CREATE TABLE `dss_datamodel_table_statics` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `origin_tables` varchar(4000) COLLATE utf8_bin DEFAULT NULL, + `table_id` int(11) NOT NULL, + `access_count` int(11) NOT NULL, + `last_access_time` int(11) NOT NULL, + `sample_data_path` varchar(256) COLLATE utf8_bin DEFAULT NULL COMMENT '存储10行用例数据', + `sample_update_time` int(11) NOT NULL, + PRIMARY KEY (`id`) USING BTREE +) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin; + +-- ---------------------------- +-- Table structure for dss_datamodel_table_stats +-- ---------------------------- +DROP TABLE IF EXISTS `dss_datamodel_table_stats`; +CREATE TABLE `dss_datamodel_table_stats` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `data_base` varchar(255) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL, + `name` varchar(255) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL, + `create_time` datetime NOT NULL, + `update_time` datetime NOT NULL, + `column_count` int(11) NOT NULL COMMENT '字段数', + `storage_size` int(11) NOT NULL COMMENT '存储大小', + `file_count` int(11) NOT NULL COMMENT '文件数', + `partition_count` int(11) NOT NULL COMMENT '分区数', + `access_count` int(11) NOT NULL COMMENT '访问次数', + `collect_count` int(11) NOT NULL COMMENT '收藏次数', + `ref_count` int(11) NOT NULL COMMENT '引用次数', + `version` varchar(128) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT '版本信息：默认1', + PRIMARY KEY (`id`) USING BTREE +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +-- ---------------------------- +-- Table structure for dss_datamodel_table_version +-- ---------------------------- +DROP TABLE IF EXISTS `dss_datamodel_table_version`; +CREATE TABLE `dss_datamodel_table_version` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `tbl_id` int(11) NOT NULL, + `name` varchar(255) COLLATE utf8_bin NOT NULL, + `is_materialized` tinyint(1) NOT NULL COMMENT '是否物化', + `table_code` mediumtext COLLATE utf8_bin COMMENT '创建table的sql', + `comment` varchar(255) COLLATE utf8_bin DEFAULT NULL COMMENT '版本注释', + `version` varchar(128) COLLATE utf8_bin NOT NULL COMMENT '版本信息：默认version0002', + `table_params` mediumtext COLLATE utf8_bin, + `columns` varchar(4000) COLLATE utf8_bin DEFAULT NULL, + `source_type` varchar(128) COLLATE utf8_bin DEFAULT 'add' COMMENT 'rollback,update,add', + `create_time` datetime NOT NULL, + `update_time` datetime NOT NULL, + `creator` varchar(255) COLLATE utf8_bin DEFAULT NULL, + PRIMARY KEY (`id`) USING BTREE, + UNIQUE KEY `table_version` (`version`,`name`) USING BTREE +) ENGINE=InnoDB AUTO_INCREMENT=40 DEFAULT CHARSET=utf8 COLLATE=utf8_bin; + +SET FOREIGN_KEY_CHECKS = 1; diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-center-server/db/datamodel_dml.sql b/dss-apps/dss-datamodel-center/dss-datamodel-center-server/db/datamodel_dml.sql new file mode 100644 index 0000000000..5a7024fe83 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-center-server/db/datamodel_dml.sql @@ -0,0 +1,40 @@ + +SET NAMES utf8mb4; +SET FOREIGN_KEY_CHECKS = 0; + +-- ---------------------------- +-- Table structure for dss_datamodel_dictionary +-- ---------------------------- +DROP TABLE IF EXISTS `dss_datamodel_dictionary`; +CREATE TABLE `dss_datamodel_dictionary` ( + `id` int(50) NOT NULL COMMENT '主键id', + `code` varchar(128) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL COMMENT '类型code', + `type` varchar(128) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL COMMENT '字典类型', + `description` varchar(255) CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL COMMENT '描述', + `created_time` datetime(0) NOT NULL DEFAULT CURRENT_TIMESTAMP(0) COMMENT '创建时间', + `update_time` datetime(0) NOT NULL DEFAULT CURRENT_TIMESTAMP(0) ON UPDATE CURRENT_TIMESTAMP(0) COMMENT '更新时间', + `sort` int(11) NULL DEFAULT NULL, + PRIMARY KEY (`id`) USING BTREE +) ENGINE = InnoDB CHARACTER SET = utf8 COLLATE = utf8_general_ci COMMENT = '数模字典' ROW_FORMAT = Dynamic; + +-- ---------------------------- +-- Records of dss_datamodel_dictionary +-- ---------------------------- +INSERT INTO `dss_datamodel_dictionary` VALUES (10000001, 'Snappy', 'COMPRESS', 'Snappy', '2021-10-09 15:41:23', '2021-10-09 15:55:44', 1); +INSERT INTO `dss_datamodel_dictionary` VALUES (10000002, 'None', 'COMPRESS', '无', '2021-10-09 15:42:17', '2021-10-20 14:48:59', 2); +INSERT INTO `dss_datamodel_dictionary` VALUES (10000003, 'ORC', 'FILE_STORAGE', 'orc', '2021-10-09 15:55:32', '2021-10-20 14:49:00', 1); +INSERT INTO `dss_datamodel_dictionary` VALUES (10000004, 'Parquet', 'FILE_STORAGE', 'Parquet', '2021-10-09 15:55:32', '2021-10-20 14:49:02', 2); +INSERT INTO `dss_datamodel_dictionary` VALUES (10000005, 'None', 'FILE_STORAGE', '无', '2021-10-09 15:55:32', '2021-10-20 14:49:04', 3); +INSERT INTO `dss_datamodel_dictionary` VALUES (10000006, 'Once', 'LIFECYCLE', '一次', '2021-10-09 15:55:32', '2021-10-20 14:49:05', 1); +INSERT INTO `dss_datamodel_dictionary` VALUES (10000007, 'OneDay', 'LIFECYCLE', '一天', '2021-10-09 15:55:32', '2021-10-20 14:49:06', 2); +INSERT INTO `dss_datamodel_dictionary` VALUES (10000008, 'HalfMonth', 'LIFECYCLE', '半月', '2021-10-09 15:55:32', '2021-10-20 14:49:07', 3); +INSERT INTO `dss_datamodel_dictionary` VALUES (10000009, 'OneMonth', 'LIFECYCLE', '一个月', '2021-10-09 15:55:32', '2021-10-20 14:49:08', 4); +INSERT INTO `dss_datamodel_dictionary` VALUES (10000010, 'ThreeMonths', 'LIFECYCLE', '三个月', '2021-10-09 15:55:32', '2021-10-20 14:49:09', 5); +INSERT INTO `dss_datamodel_dictionary` VALUES (10000011, 'SixMonths', 'LIFECYCLE', '六个月', '2021-10-09 15:55:32', '2021-10-20 14:49:10', 6); +INSERT INTO `dss_datamodel_dictionary` VALUES (10000012, 'OneYear', 'LIFECYCLE', '一年', '2021-10-09 15:55:32', '2021-10-20 14:49:11', 7); +INSERT INTO `dss_datamodel_dictionary` VALUES (10000013, 'ThreeYears', 'LIFECYCLE', '三年', '2021-10-09 15:55:32', '2021-10-20 14:49:12', 8); +INSERT INTO `dss_datamodel_dictionary` VALUES (10000014, 'Hive', 'STORAGE_ENGINE', 'hive', '2021-10-09 15:55:32', '2021-10-20 14:49:19', 1); +INSERT INTO `dss_datamodel_dictionary` VALUES (10000015, 'Mysql', 'STORAGE_ENGINE', 'mysql', '2021-10-09 15:55:32', '2021-10-20 14:49:20', 2); +INSERT INTO `dss_datamodel_dictionary` VALUES (10000016, 'ES', 'STORAGE_ENGINE', 'es', '2021-10-09 15:55:32', '2021-10-20 14:49:25', 3); + +SET FOREIGN_KEY_CHECKS = 1; diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-center-server/pom.xml b/dss-apps/dss-datamodel-center/dss-datamodel-center-server/pom.xml new file mode 100644 index 0000000000..77fa639cc3 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-center-server/pom.xml @@ -0,0 +1,113 @@ + + + + dss-datamodel-center + com.webank.wedatasphere.dss + 1.0.1 + ../pom.xml + + 4.0.0 + + dss-datamodel-center-server + + + 8 + 8 + + + + + com.webank.wedatasphere.dss + dss-datamodel-dimension-management + ${dss.version} + + + com.webank.wedatasphere.dss + dss-datamodel-measure-management + ${dss.version} + + + com.webank.wedatasphere.dss + dss-datamodel-indicator-management + ${dss.version} + + + com.webank.wedatasphere.dss + dss-datamodel-table-management + ${dss.version} + + + + + + + org.apache.maven.plugins + maven-deploy-plugin + + + + net.alchim31.maven + scala-maven-plugin + + + org.apache.maven.plugins + maven-jar-plugin + + + org.apache.maven.plugins + maven-assembly-plugin + 2.3 + false + + + make-assembly + package + + single + + + + src/main/assembly/distribution.xml + + + + + + false + datamodel-center + false + false + + src/main/assembly/distribution.xml + + + + + org.apache.maven.plugins + maven-compiler-plugin + + 8 + 8 + + + + + + src/main/java + + **/*.xml + + + + src/main/resources + + **/*.xml + **/*.properties + **/*.yml + + + + + \ No newline at end of file diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-center-server/src/main/assembly/distribution.xml b/dss-apps/dss-datamodel-center/dss-datamodel-center-server/src/main/assembly/distribution.xml new file mode 100644 index 0000000000..40536239b6 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-center-server/src/main/assembly/distribution.xml @@ -0,0 +1,71 @@ + + + + datamodel-center + + zip + + true + datamodel-center + + + + + + lib + true + true + false + false + true + + + + + + ${basedir}/src/main/resources + + * + + 0777 + conf + unix + + + ${basedir}/bin + + * + + 0777 + bin + unix + + + . + + */** + + logs + + + + + diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-center-server/src/main/resources/application.yml b/dss-apps/dss-datamodel-center/dss-datamodel-center-server/src/main/resources/application.yml new file mode 100644 index 0000000000..bddac1e886 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-center-server/src/main/resources/application.yml @@ -0,0 +1,20 @@ +server: + port: 9321 +spring: + application: + name: datamodel-server +eureka: + client: + serviceUrl: + defaultZone: http://localhost:20303/eureka/ + instance: + metadata-map: + test: wedatasphere + +management: + endpoints: + web: + exposure: + include: refresh,info +logging: + config: classpath:log4j2.xml diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-center-server/src/main/resources/linkis.properties b/dss-apps/dss-datamodel-center/dss-datamodel-center-server/src/main/resources/linkis.properties new file mode 100644 index 0000000000..4dcb6b7d90 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-center-server/src/main/resources/linkis.properties @@ -0,0 +1,104 @@ +# +# Copyright 2019 WeBank +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# + +wds.linkis.test.mode=true + + + +wds.linkis.server.mybatis.datasource.url=jdbc:mysql://localhost:3306/datamodel?useSSL=false&characterEncoding=UTF-8 + +wds.linkis.server.mybatis.datasource.username= + +wds.linkis.server.mybatis.datasource.password= + + +wds.linkis.log.clear=true + +wds.linkis.server.version=v1 + + +wds.wedatasphere.linkis.serverurl=http://localhost:8088 + +wds.wedatasphere.linkis.authtoken.key=hdfs + +wds.wedatasphere.linkis.discovery.enabled=false + +wds.wedatasphere.linkis.authenticationStrategy=org.apache.linkis.httpclient.dws.authentication.TokenAuthenticationStrategy + +wds.wedatasphere.linkis.authtoken.value=hdfs + +wds.wedatasphere.linkis.dws.version=v1 + + +##restful +wds.linkis.server.restful.scan.packages=com.webank.wedatasphere.dss.datamodel.indicator.restful,\ + com.webank.wedatasphere.dss.datamodel.table.restful,\ + com.webank.wedatasphere.dss.datamodel.dimension.restful,\ + com.webank.wedatasphere.dss.datamodel.measure.restful + +wds.linkis.server.mybatis.mapperLocations=classpath*:com/webank/wedatasphere/dss/datamodel/dimension/dao/impl/*.xml,\ + classpath*:com/webank/wedatasphere/dss/datamodel/table/dao/impl/*.xml,\ + classpath*:com/webank/wedatasphere/dss/datamodel/measure/dao/impl/*.xml,\ + classpath*:com/webank/wedatasphere/dss/datamodel/indicator/dao/impl/*.xml + +wds.linkis.server.mybatis.BasePackage=com.webank.wedatasphere.dss.datamodel.dimension.dao,\ + com.webank.wedatasphere.dss.datamodel.table.dao,\ + com.webank.wedatasphere.dss.datamodel.measure.dao,\ + com.webank.wedatasphere.dss.datamodel.indicator.dao + +wds.wedatasphere.warehouse.client.serverurl=http://localhost:29001 + +wds.wedatasphere.warehouse.client.authtoken.key=hdfs + +wds.wedatasphere.warehouse.client.authenticationStrategy=org.apache.linkis.httpclient.dws.authentication.TokenAuthenticationStrategy + +wds.wedatasphere.warehouse.client.authtoken.value=hdfs + +wds.wedatasphere.warehouse.client.discovery.enabled=false + +wds.wedatasphere.warehouse.client.dws.version=v1 + + +wds.wedatasphere.assets.client.serverurl=http://localhost:29001 + +wds.wedatasphere.assets.client.authtoken.key=hdfs + +wds.wedatasphere.assets.client.authenticationStrategy=org.apache.linkis.httpclient.dws.authentication.TokenAuthenticationStrategy + +wds.wedatasphere.assets.client.discovery.enabled=false + +wds.wedatasphere.assets.client.authtoken.value=hdfs + +wds.wedatasphere.assets.client.dws.version=v1 + + + +wds.workspace.client.serverurl=http://localhost:29001 + +wds.workspace.client.authtoken.key=hdfs + +wds.workspace.client.authenticationStrategy=org.apache.linkis.httpclient.dws.authentication.TokenAuthenticationStrategy + +wds.workspace.client.discovery.enabled=false + +wds.workspace.client.authtoken.value=hdfs + +wds.workspace.client.dws.version=v1 + + + +wds.wedatasphere.data.model.client.strategy=static diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-center-server/src/main/resources/log4j.properties b/dss-apps/dss-datamodel-center/dss-datamodel-center-server/src/main/resources/log4j.properties new file mode 100644 index 0000000000..0807e60877 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-center-server/src/main/resources/log4j.properties @@ -0,0 +1,37 @@ +# +# Copyright 2019 WeBank +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# + +### set log levels ### + +log4j.rootCategory=INFO,console + +log4j.appender.console=org.apache.log4j.ConsoleAppender +log4j.appender.console.Threshold=INFO +log4j.appender.console.layout=org.apache.log4j.PatternLayout +#log4j.appender.console.layout.ConversionPattern= %d{ISO8601} %-5p (%t) [%F:%M(%L)] - %m%n +log4j.appender.console.layout.ConversionPattern= %d{ISO8601} %-5p (%t) %p %c{1} - %m%n + + +log4j.appender.com.webank.bdp.ide.core=org.apache.log4j.DailyRollingFileAppender +log4j.appender.com.webank.bdp.ide.core.Threshold=INFO +log4j.additivity.com.webank.bdp.ide.core=false +log4j.appender.com.webank.bdp.ide.core.layout=org.apache.log4j.PatternLayout +log4j.appender.com.webank.bdp.ide.core.Append=true +log4j.appender.com.webank.bdp.ide.core.File=logs/linkis.log +log4j.appender.com.webank.bdp.ide.core.layout.ConversionPattern= %d{ISO8601} %-5p (%t) [%F:%M(%L)] - %m%n + +log4j.logger.org.springframework=INFO diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-center-server/src/main/resources/log4j2.xml b/dss-apps/dss-datamodel-center/dss-datamodel-center-server/src/main/resources/log4j2.xml new file mode 100644 index 0000000000..5ae60f1441 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-center-server/src/main/resources/log4j2.xml @@ -0,0 +1,39 @@ + + + + + + + + + + + + + + + + + + + + + + + diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-center-server/src/main/scala/com/webank/wedatasphere/dss/datamodel/DataModelApplication.scala b/dss-apps/dss-datamodel-center/dss-datamodel-center-server/src/main/scala/com/webank/wedatasphere/dss/datamodel/DataModelApplication.scala new file mode 100644 index 0000000000..1218c094ff --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-center-server/src/main/scala/com/webank/wedatasphere/dss/datamodel/DataModelApplication.scala @@ -0,0 +1,9 @@ +package com.webank.wedatasphere.dss.datamodel + +import org.apache.linkis.DataWorkCloudApplication + +object DataModelApplication { + def main(args: Array[String]): Unit = { + DataWorkCloudApplication.main(args) + } +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-dimension-management/pom.xml b/dss-apps/dss-datamodel-center/dss-datamodel-dimension-management/pom.xml new file mode 100644 index 0000000000..69d330ed7f --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-dimension-management/pom.xml @@ -0,0 +1,59 @@ + + + + + + dss-datamodel-center + com.webank.wedatasphere.dss + 1.0.1 + ../pom.xml + + 4.0.0 + + dss-datamodel-dimension-management + + + + + com.webank.wedatasphere.dss + dss-datamodel-center-common + ${dss.version} + + + + + + + + org.apache.maven.plugins + maven-deploy-plugin + + + net.alchim31.maven + scala-maven-plugin + + + org.apache.maven.plugins + maven-jar-plugin + + + + ${project.artifactId}-${project.version} + + diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-dimension-management/src/main/java/com/webank/wedatasphere/dss/datamodel/dimension/dao/DssDatamodelDimensionMapper.java b/dss-apps/dss-datamodel-center/dss-datamodel-dimension-management/src/main/java/com/webank/wedatasphere/dss/datamodel/dimension/dao/DssDatamodelDimensionMapper.java new file mode 100644 index 0000000000..34ccc92025 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-dimension-management/src/main/java/com/webank/wedatasphere/dss/datamodel/dimension/dao/DssDatamodelDimensionMapper.java @@ -0,0 +1,22 @@ +package com.webank.wedatasphere.dss.datamodel.dimension.dao; +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import org.apache.ibatis.annotations.Param; + +import com.webank.wedatasphere.dss.datamodel.dimension.entity.DssDatamodelDimension; + +public interface DssDatamodelDimensionMapper extends BaseMapper { + int deleteByPrimaryKey(Long id); + + int insert(DssDatamodelDimension record); + + int insertSelective(DssDatamodelDimension record); + + DssDatamodelDimension selectByPrimaryKey(Long id); + + int updateByPrimaryKeySelective(DssDatamodelDimension record); + + int updateByPrimaryKey(DssDatamodelDimension record); + + int updateIsAvailableById(@Param("updatedIsAvailable")Integer updatedIsAvailable,@Param("id")Long id); + +} \ No newline at end of file diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-dimension-management/src/main/java/com/webank/wedatasphere/dss/datamodel/dimension/dao/impl/DssDatamodelDimensionMapper.xml b/dss-apps/dss-datamodel-center/dss-datamodel-dimension-management/src/main/java/com/webank/wedatasphere/dss/datamodel/dimension/dao/impl/DssDatamodelDimensionMapper.xml new file mode 100644 index 0000000000..2e71b2525e --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-dimension-management/src/main/java/com/webank/wedatasphere/dss/datamodel/dimension/dao/impl/DssDatamodelDimensionMapper.xml @@ -0,0 +1,174 @@ + + + + + + + + + + + + + + + + + + + + + id, `name`, field_identifier, formula, `comment`, warehouse_theme_name, `owner`, + principal_name, is_available, create_time, update_time + + + + + delete from dss_datamodel_dimension + where id = #{id,jdbcType=BIGINT} + + + + insert into dss_datamodel_dimension (`name`, field_identifier, formula, + `comment`, warehouse_theme_name, `owner`, + principal_name, is_available, create_time, + update_time) + values (#{name,jdbcType=VARCHAR}, #{fieldIdentifier,jdbcType=VARCHAR}, #{formula,jdbcType=VARCHAR}, + #{comment,jdbcType=VARCHAR}, #{warehouseThemeName,jdbcType=VARCHAR}, #{owner,jdbcType=VARCHAR}, + #{principalName,jdbcType=VARCHAR}, #{isAvailable,jdbcType=BOOLEAN}, #{createTime,jdbcType=TIMESTAMP}, + #{updateTime,jdbcType=TIMESTAMP}) + + + + insert into dss_datamodel_dimension + + + `name`, + + + field_identifier, + + + formula, + + + `comment`, + + + warehouse_theme_name, + + + `owner`, + + + principal_name, + + + is_available, + + + create_time, + + + update_time, + + + + + #{name,jdbcType=VARCHAR}, + + + #{fieldIdentifier,jdbcType=VARCHAR}, + + + #{formula,jdbcType=VARCHAR}, + + + #{comment,jdbcType=VARCHAR}, + + + #{warehouseThemeName,jdbcType=VARCHAR}, + + + #{owner,jdbcType=VARCHAR}, + + + #{principalName,jdbcType=VARCHAR}, + + + #{isAvailable,jdbcType=BOOLEAN}, + + + #{createTime,jdbcType=TIMESTAMP}, + + + #{updateTime,jdbcType=TIMESTAMP}, + + + + + + update dss_datamodel_dimension + + + `name` = #{name,jdbcType=VARCHAR}, + + + field_identifier = #{fieldIdentifier,jdbcType=VARCHAR}, + + + formula = #{formula,jdbcType=VARCHAR}, + + + `comment` = #{comment,jdbcType=VARCHAR}, + + + warehouse_theme_name = #{warehouseThemeName,jdbcType=VARCHAR}, + + + `owner` = #{owner,jdbcType=VARCHAR}, + + + principal_name = #{principalName,jdbcType=VARCHAR}, + + + is_available = #{isAvailable,jdbcType=BOOLEAN}, + + + create_time = #{createTime,jdbcType=TIMESTAMP}, + + + update_time = #{updateTime,jdbcType=TIMESTAMP}, + + + where id = #{id,jdbcType=BIGINT} + + + + update dss_datamodel_dimension + set `name` = #{name,jdbcType=VARCHAR}, + field_identifier = #{fieldIdentifier,jdbcType=VARCHAR}, + formula = #{formula,jdbcType=VARCHAR}, + `comment` = #{comment,jdbcType=VARCHAR}, + warehouse_theme_name = #{warehouseThemeName,jdbcType=VARCHAR}, + `owner` = #{owner,jdbcType=VARCHAR}, + principal_name = #{principalName,jdbcType=VARCHAR}, + is_available = #{isAvailable,jdbcType=BOOLEAN}, + create_time = #{createTime,jdbcType=TIMESTAMP}, + update_time = #{updateTime,jdbcType=TIMESTAMP} + where id = #{id,jdbcType=BIGINT} + + + + + update dss_datamodel_dimension + set is_available=#{updatedIsAvailable,jdbcType=BOOLEAN} + where id=#{id,jdbcType=BIGINT} + + \ No newline at end of file diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-dimension-management/src/main/java/com/webank/wedatasphere/dss/datamodel/dimension/dto/DimensionQueryDTO.java b/dss-apps/dss-datamodel-center/dss-datamodel-dimension-management/src/main/java/com/webank/wedatasphere/dss/datamodel/dimension/dto/DimensionQueryDTO.java new file mode 100644 index 0000000000..78b83523e0 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-dimension-management/src/main/java/com/webank/wedatasphere/dss/datamodel/dimension/dto/DimensionQueryDTO.java @@ -0,0 +1,47 @@ +package com.webank.wedatasphere.dss.datamodel.dimension.dto; + + +import lombok.Data; + +import java.util.Date; + +@Data +public class DimensionQueryDTO { + private Long id; + + private String name; + + private String fieldIdentifier; + + private String formula; + + private String comment; + + /** + * 数仓主题格式为： theme_domain_name.theme_name + */ + private String warehouseThemeName; + + + private String warehouseThemeNameEn; + + + private String owner; + + /** + * 授权的名字：userName、roleName + */ + private String principalName; + + private Integer isAvailable; + + private Date createTime; + + private Date updateTime; + + /** + * 被引用次数 + */ + private Integer refCount =0; + +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-dimension-management/src/main/java/com/webank/wedatasphere/dss/datamodel/dimension/entity/DssDatamodelDimension.java b/dss-apps/dss-datamodel-center/dss-datamodel-dimension-management/src/main/java/com/webank/wedatasphere/dss/datamodel/dimension/entity/DssDatamodelDimension.java new file mode 100644 index 0000000000..5db862c15d --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-dimension-management/src/main/java/com/webank/wedatasphere/dss/datamodel/dimension/entity/DssDatamodelDimension.java @@ -0,0 +1,47 @@ +package com.webank.wedatasphere.dss.datamodel.dimension.entity; + +import com.baomidou.mybatisplus.annotation.FieldFill; +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableField; +import com.baomidou.mybatisplus.annotation.TableId; +import lombok.Data; + +import java.util.Date; + +@Data +public class DssDatamodelDimension { + + @TableId(type = IdType.AUTO) + private Long id; + + private String name; + + private String fieldIdentifier; + + private String formula; + + private String comment; + + /** + * 数仓主题格式为： theme_domain_name.theme_name + */ + private String warehouseThemeName; + + private String warehouseThemeNameEn; + + private String owner; + + /** + * 授权的名字：userName、roleName + */ + private String principalName; + + private Integer isAvailable; + + @TableField(fill = FieldFill.INSERT) + private Date createTime; + + @TableField(fill = FieldFill.INSERT_UPDATE) + private Date updateTime; + + } \ No newline at end of file diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-dimension-management/src/main/java/com/webank/wedatasphere/dss/datamodel/dimension/restful/DimensionRestfulApi.java b/dss-apps/dss-datamodel-center/dss-datamodel-dimension-management/src/main/java/com/webank/wedatasphere/dss/datamodel/dimension/restful/DimensionRestfulApi.java new file mode 100644 index 0000000000..54f2f9ec25 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-dimension-management/src/main/java/com/webank/wedatasphere/dss/datamodel/dimension/restful/DimensionRestfulApi.java @@ -0,0 +1,114 @@ +package com.webank.wedatasphere.dss.datamodel.dimension.restful; + +import com.webank.wedatasphere.dss.datamodel.dimension.service.DimensionService; +import com.webank.wedatasphere.dss.datamodel.dimension.vo.DimensionAddVO; +import com.webank.wedatasphere.dss.datamodel.dimension.vo.DimensionEnableVO; +import com.webank.wedatasphere.dss.datamodel.dimension.vo.DimensionQueryVO; +import com.webank.wedatasphere.dss.datamodel.dimension.vo.DimensionUpdateVO; +import org.apache.linkis.common.exception.ErrorException; +import org.apache.linkis.server.Message; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; +import org.springframework.web.bind.annotation.RequestBody; + +import javax.servlet.http.HttpServletRequest; +import javax.ws.rs.*; +import javax.ws.rs.core.Context; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; +import java.io.IOException; + +@Component +@Path("/datamodel/") +@Produces(MediaType.APPLICATION_JSON) +@Consumes(MediaType.APPLICATION_JSON) +public class DimensionRestfulApi { + private static final Logger LOGGER = LoggerFactory.getLogger(DimensionRestfulApi.class); + + @Autowired + private DimensionService dimensionService; + + /** + * 新增 + * + * @param req + * @param vo + * @return + * @throws IOException + */ + @POST + @Path("/dimensions") + public Response add(@Context HttpServletRequest req, @RequestBody DimensionAddVO vo) throws ErrorException { + LOGGER.info("dimensionAddVO : {}", vo); + return Message.messageToResponse(Message.ok().data("id", dimensionService.addDimension(vo))); + } + + /** + * 启用/禁用 + * @param req + * @param id + * @param vo + * @return + */ + @PUT + @Path("/dimensions/enable/{id}") + public Response enable(@Context HttpServletRequest req, @PathParam("id") Long id, @RequestBody DimensionEnableVO vo) { + LOGGER.info("enable id : {}, vo : {}", id, vo); + return Message.messageToResponse(Message.ok().data("count", dimensionService.enableDimension(id, vo))); + } + + /** + * 修改 + * @param req + * @param id + * @param vo + * @return + */ + @PUT + @Path("/dimensions/{id}") + public Response update(@Context HttpServletRequest req, @PathParam("id") Long id, @RequestBody DimensionUpdateVO vo) throws ErrorException{ + LOGGER.info("update id : {}, vo : {}", id, vo); + return Message.messageToResponse(Message.ok().data("count",dimensionService.updateDimension(id,vo))); + } + + /** + * 删除 + * @param req + * @param id + * @return + */ + @DELETE + @Path("/dimensions/{id}") + public Response delete(@Context HttpServletRequest req, @PathParam("id") Long id) throws ErrorException { + LOGGER.info("delete id : {}", id); + return Message.messageToResponse(Message.ok().data("count",dimensionService.deleteDimension(id))); + } + + /** + * 分页搜索 + * @param req + * @return + */ + @POST + @Path("/dimensions/list") + public Response list(@Context HttpServletRequest req,@RequestBody DimensionQueryVO vo){ + LOGGER.info("list vo : {}",vo); + return Message.messageToResponse(dimensionService.listDimensions(vo)); + } + + + /** + * 查看 + * @param req + * @param id + * @return + */ + @GET + @Path("/dimensions/{id}") + public Response query(@Context HttpServletRequest req, @PathParam("id") Long id) throws ErrorException { + LOGGER.info("query id : {}", id); + return Message.messageToResponse(Message.ok().data("detail",dimensionService.queryById(id))); + } +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-dimension-management/src/main/java/com/webank/wedatasphere/dss/datamodel/dimension/service/DimensionService.java b/dss-apps/dss-datamodel-center/dss-datamodel-dimension-management/src/main/java/com/webank/wedatasphere/dss/datamodel/dimension/service/DimensionService.java new file mode 100644 index 0000000000..fa9c089bc2 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-dimension-management/src/main/java/com/webank/wedatasphere/dss/datamodel/dimension/service/DimensionService.java @@ -0,0 +1,70 @@ +package com.webank.wedatasphere.dss.datamodel.dimension.service; + +import com.baomidou.mybatisplus.extension.service.IService; +import com.webank.wedatasphere.dss.datamodel.dimension.dto.DimensionQueryDTO; +import com.webank.wedatasphere.dss.datamodel.dimension.entity.DssDatamodelDimension; +import com.webank.wedatasphere.dss.datamodel.dimension.vo.DimensionAddVO; +import com.webank.wedatasphere.dss.datamodel.dimension.vo.DimensionEnableVO; +import com.webank.wedatasphere.dss.datamodel.dimension.vo.DimensionQueryVO; +import com.webank.wedatasphere.dss.datamodel.dimension.vo.DimensionUpdateVO; +import org.apache.linkis.common.exception.ErrorException; +import org.apache.linkis.server.Message; + + +public interface DimensionService extends IService { + + /** + * 新增维度 + * @param vo + * @return + */ + long addDimension(DimensionAddVO vo)throws ErrorException; + + /** + * 启用/禁用 + * @param vo + * @return + */ + int enableDimension(Long id, DimensionEnableVO vo); + + + /** + * 更新 + * @param id + * @param vo + * @return + */ + int updateDimension(Long id, DimensionUpdateVO vo) throws ErrorException; + + + /** + * 删除 + * @param id + * @return + */ + int deleteDimension(Long id) throws ErrorException; + + + /** + * 查看 + * @param id + * @return + */ + DimensionQueryDTO queryById(Long id) throws ErrorException; + + + /** + * 查询列表 + * @param vo + * @return + */ + Message listDimensions(DimensionQueryVO vo); + + + /** + * 主题引用情况 + * @param name + * @return + */ + int dimensionThemeReferenceCount(String name); +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-dimension-management/src/main/java/com/webank/wedatasphere/dss/datamodel/dimension/service/impl/DataWarehouseDimensionReferenceServiceImpl.java b/dss-apps/dss-datamodel-center/dss-datamodel-dimension-management/src/main/java/com/webank/wedatasphere/dss/datamodel/dimension/service/impl/DataWarehouseDimensionReferenceServiceImpl.java new file mode 100644 index 0000000000..3b4da459f5 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-dimension-management/src/main/java/com/webank/wedatasphere/dss/datamodel/dimension/service/impl/DataWarehouseDimensionReferenceServiceImpl.java @@ -0,0 +1,20 @@ +package com.webank.wedatasphere.dss.datamodel.dimension.service.impl; + + +import com.webank.wedatasphere.dss.datamodel.center.common.service.DataWarehouseDimensionReferenceService; +import com.webank.wedatasphere.dss.datamodel.dimension.service.DimensionService; +import org.springframework.stereotype.Service; + +import javax.annotation.Resource; + +@Service +public class DataWarehouseDimensionReferenceServiceImpl implements DataWarehouseDimensionReferenceService { + + @Resource + private DimensionService dimensionService; + + @Override + public int dimensionThemeReferenceCount(String name) { + return dimensionService.dimensionThemeReferenceCount(name); + } +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-dimension-management/src/main/java/com/webank/wedatasphere/dss/datamodel/dimension/service/impl/DimensionServiceImpl.java b/dss-apps/dss-datamodel-center/dss-datamodel-dimension-management/src/main/java/com/webank/wedatasphere/dss/datamodel/dimension/service/impl/DimensionServiceImpl.java new file mode 100644 index 0000000000..3664b02d0e --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-dimension-management/src/main/java/com/webank/wedatasphere/dss/datamodel/dimension/service/impl/DimensionServiceImpl.java @@ -0,0 +1,214 @@ +package com.webank.wedatasphere.dss.datamodel.dimension.service.impl; + +import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; +import com.baomidou.mybatisplus.core.toolkit.Wrappers; +import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl; +import com.github.pagehelper.PageHelper; +import com.github.pagehelper.PageInfo; +import com.webank.wedatasphere.dss.data.governance.entity.ClassificationConstant; +import com.webank.wedatasphere.dss.data.governance.response.CreateModelTypeResult; +import com.webank.wedatasphere.dss.data.governance.response.UpdateModelTypeResult; +import com.webank.wedatasphere.dss.datamodel.center.common.constant.ErrorCode; +import com.webank.wedatasphere.dss.datamodel.center.common.context.DataModelSecurityContextHolder; +import com.webank.wedatasphere.dss.datamodel.center.common.event.CreateModelEvent; +import com.webank.wedatasphere.dss.datamodel.center.common.event.DeleteModelEvent; +import com.webank.wedatasphere.dss.datamodel.center.common.event.UpdateModelEvent; +import com.webank.wedatasphere.dss.datamodel.center.common.exception.DSSDatamodelCenterException; +import com.webank.wedatasphere.dss.datamodel.center.common.service.AssertsSyncService; +import com.webank.wedatasphere.dss.datamodel.center.common.service.DatamodelReferencService; +import com.webank.wedatasphere.dss.datamodel.dimension.dao.DssDatamodelDimensionMapper; +import com.webank.wedatasphere.dss.datamodel.dimension.dto.DimensionQueryDTO; +import com.webank.wedatasphere.dss.datamodel.dimension.entity.DssDatamodelDimension; +import com.webank.wedatasphere.dss.datamodel.dimension.service.DimensionService; +import com.webank.wedatasphere.dss.datamodel.dimension.vo.DimensionAddVO; +import com.webank.wedatasphere.dss.datamodel.dimension.vo.DimensionEnableVO; +import com.webank.wedatasphere.dss.datamodel.dimension.vo.DimensionQueryVO; +import com.webank.wedatasphere.dss.datamodel.dimension.vo.DimensionUpdateVO; +import org.apache.linkis.common.exception.ErrorException; +import org.apache.linkis.server.Message; +import org.apache.commons.lang.StringUtils; +import org.modelmapper.ModelMapper; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +import javax.annotation.Resource; +import java.util.Date; +import java.util.stream.Collectors; + + + +@Service +public class DimensionServiceImpl extends ServiceImpl implements DimensionService { + + private static final Logger LOGGER = LoggerFactory.getLogger(DimensionServiceImpl.class); + + private final ModelMapper modelMapper = new ModelMapper(); + + + @Resource + private AssertsSyncService assertsSyncService; + + @Resource + private DatamodelReferencService datamodelReferencService; + + @Override + @Transactional(rollbackFor = Exception.class) + public long addDimension(DimensionAddVO vo) throws DSSDatamodelCenterException{ + if (getBaseMapper().selectCount(Wrappers.lambdaQuery().eq(DssDatamodelDimension::getName, vo.getName()) + .or().eq(DssDatamodelDimension::getFieldIdentifier,vo.getFieldIdentifier())) > 0) { + LOGGER.error("errorCode : {}, dimension name or field identifier can not repeat, name : {}", ErrorCode.INDICATOR_ADD_ERROR.getCode(), vo.getName()); + throw new DSSDatamodelCenterException(ErrorCode.DIMENSION_ADD_ERROR.getCode(), "dimension name or field identifier can not repeat"); + } + + if (datamodelReferencService.dimensionReferenceCount(vo.getName())>0 + ||datamodelReferencService.dimensionReferenceCount(vo.getFieldIdentifier())>0){ + LOGGER.error("errorCode : {}, dimension name can not referenced", ErrorCode.DIMENSION_ADD_ERROR.getCode()); + throw new DSSDatamodelCenterException(ErrorCode.DIMENSION_ADD_ERROR.getCode(), "dimension name can not referenced"); + } + + DssDatamodelDimension newOne = modelMapper.map(vo,DssDatamodelDimension.class); + newOne.setCreateTime(new Date()); + newOne.setUpdateTime(new Date()); + + getBaseMapper().insert(newOne); + + //同步atlas + CreateModelTypeResult result = assertsSyncService.syncCreateModel( + new CreateModelEvent(this + , DataModelSecurityContextHolder.getContext().getDataModelAuthentication().getUser() + , vo.getFieldIdentifier() + , ClassificationConstant.DIMENSION)); + return newOne.getId(); + } + + + @Override + @Transactional(rollbackFor = Exception.class) + public int enableDimension(Long id ,DimensionEnableVO vo) { + DssDatamodelDimension enableOne = new DssDatamodelDimension(); + enableOne.setIsAvailable(vo.getIsAvailable()); + enableOne.setUpdateTime(new Date()); + return getBaseMapper().update(enableOne, Wrappers.lambdaUpdate().eq(DssDatamodelDimension::getId,id)); + } + + + @Override + @Transactional(rollbackFor = Exception.class) + public int updateDimension(Long id, DimensionUpdateVO vo) throws DSSDatamodelCenterException{ + DssDatamodelDimension org = getBaseMapper().selectById(id); + if (org == null) { + LOGGER.error("errorCode : {}, update dimension error not exists", ErrorCode.DIMENSION_UPDATE_ERROR.getCode()); + throw new DSSDatamodelCenterException(ErrorCode.DIMENSION_UPDATE_ERROR.getCode(), "update dimension error not exists"); + } + + if (datamodelReferencService.dimensionReferenceCount(org.getName())>0 + ||datamodelReferencService.dimensionReferenceCount(org.getFieldIdentifier())>0 + ||datamodelReferencService.dimensionReferenceCount(vo.getName())>0 + ||datamodelReferencService.dimensionReferenceCount(vo.getFieldIdentifier())>0){ + LOGGER.error("errorCode : {}, dimension name can not referenced", ErrorCode.DIMENSION_UPDATE_ERROR.getCode()); + throw new DSSDatamodelCenterException(ErrorCode.DIMENSION_UPDATE_ERROR.getCode(), "dimension name can not referenced"); + } + + + //当更新名称时 + if (!StringUtils.equals(vo.getName(), org.getName())) { + int repeat = getBaseMapper().selectCount(Wrappers.lambdaQuery().eq(DssDatamodelDimension::getName, vo.getName())); + if (repeat > 0) { + LOGGER.error("errorCode : {}, dimension name can not repeat ", ErrorCode.DIMENSION_UPDATE_ERROR.getCode()); + throw new DSSDatamodelCenterException(ErrorCode.DIMENSION_UPDATE_ERROR.getCode(), "dimension name can not repeat "); + } + } + + String orgFieldIdentifier = org.getFieldIdentifier(); + //当更新标识时 + if (!StringUtils.equals(vo.getFieldIdentifier(), orgFieldIdentifier)) { + int repeat = getBaseMapper().selectCount(Wrappers.lambdaQuery().eq(DssDatamodelDimension::getFieldIdentifier, vo.getFieldIdentifier())); + if (repeat > 0) { + LOGGER.error("errorCode : {}, dimension field identifier can not repeat", ErrorCode.DIMENSION_UPDATE_ERROR.getCode()); + throw new DSSDatamodelCenterException(ErrorCode.DIMENSION_UPDATE_ERROR.getCode(), "dimension field identifier can not repeat "); + } + } + DssDatamodelDimension updateOne = modelMapper.map(vo,DssDatamodelDimension.class); + updateOne.setUpdateTime(new Date()); + getBaseMapper().update(updateOne, Wrappers.lambdaUpdate().eq(DssDatamodelDimension::getId,id)); + + //同步atlas + UpdateModelTypeResult updateModelTypeResult = assertsSyncService.syncUpdateModel( + new UpdateModelEvent(this + ,DataModelSecurityContextHolder.getContext().getDataModelAuthentication().getUser() + ,vo.getFieldIdentifier() + ,orgFieldIdentifier + ,ClassificationConstant.DIMENSION)); + return 1; + } + + + @Override + @Transactional(rollbackFor = Exception.class) + public int deleteDimension(Long id) throws ErrorException{ + + DssDatamodelDimension dssDatamodelDimension = getBaseMapper().selectById(id); + if (dssDatamodelDimension == null){ + throw new DSSDatamodelCenterException(ErrorCode.DIMENSION_DELETE_ERROR.getCode(), "dimension id " +id +" not exists"); + } + //校验引用情况 + if(datamodelReferencService.dimensionReferenceCount(dssDatamodelDimension.getName())>0||datamodelReferencService.dimensionReferenceCount(dssDatamodelDimension.getFieldIdentifier())>0){ + throw new DSSDatamodelCenterException(ErrorCode.DIMENSION_DELETE_ERROR.getCode(), "dimension id " +id +" has referenced"); + } + getBaseMapper().deleteById(id); + //同步资产 + assertsSyncService.syncDeleteModel(new DeleteModelEvent(this + ,DataModelSecurityContextHolder.getContext().getDataModelAuthentication().getUser() + ,dssDatamodelDimension.getFieldIdentifier() + ,ClassificationConstant.DIMENSION)); + return 1; + } + + + @Override + public Message listDimensions(DimensionQueryVO vo) { + QueryWrapper queryWrapper = new QueryWrapper() + .like(StringUtils.isNotBlank(vo.getName()),"name",vo.getName()) + .eq(vo.getIsAvailable()!=null,"is_available",vo.getIsAvailable()) + .eq(StringUtils.isNotBlank(vo.getWarehouseThemeName()),"warehouse_theme_name",vo.getWarehouseThemeName()) + .like(StringUtils.isNotBlank(vo.getOwner()),"owner",vo.getOwner()); + PageHelper.clearPage(); + PageHelper.startPage(vo.getPageNum(),vo.getPageSize()); + PageInfo pageInfo = new PageInfo<>(getBaseMapper().selectList(queryWrapper)); + //IPage iPage = page(new Page<>(vo.getPageNum(),vo.getPageSize()),queryWrapper); + + return Message.ok() + .data("list",pageInfo + .getList() + .stream() + .map(dssDatamodelDimension ->{ + DimensionQueryDTO dto = modelMapper.map(dssDatamodelDimension, DimensionQueryDTO.class); + dto.setRefCount(datamodelReferencService.dimensionReferenceCount(dssDatamodelDimension.getName())); + return dto; + }) + .collect(Collectors.toList())) + .data("total",pageInfo.getTotal()); + } + + + @Override + public DimensionQueryDTO queryById(Long id) throws ErrorException { + DssDatamodelDimension dssDatamodelDimension = getBaseMapper().selectById(id); + if (dssDatamodelDimension == null){ + throw new DSSDatamodelCenterException(ErrorCode.DIMENSION_QUERY_ERROR.getCode(), "dimension id " +id +" not exists"); + } + DimensionQueryDTO dto = modelMapper.map(dssDatamodelDimension,DimensionQueryDTO.class); + dto.setRefCount(datamodelReferencService.dimensionReferenceCount(dssDatamodelDimension.getName())); + return dto; + } + + + @Override + public int dimensionThemeReferenceCount(String name) { + int count = getBaseMapper().selectCount(Wrappers.lambdaQuery().eq(DssDatamodelDimension::getWarehouseThemeName,name)); + int countEn = getBaseMapper().selectCount(Wrappers.lambdaQuery().eq(DssDatamodelDimension::getWarehouseThemeNameEn,name)); + return count + countEn; + } +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-dimension-management/src/main/java/com/webank/wedatasphere/dss/datamodel/dimension/vo/DimensionAddVO.java b/dss-apps/dss-datamodel-center/dss-datamodel-dimension-management/src/main/java/com/webank/wedatasphere/dss/datamodel/dimension/vo/DimensionAddVO.java new file mode 100644 index 0000000000..cd5e1398a1 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-dimension-management/src/main/java/com/webank/wedatasphere/dss/datamodel/dimension/vo/DimensionAddVO.java @@ -0,0 +1,35 @@ +package com.webank.wedatasphere.dss.datamodel.dimension.vo; + +import com.webank.wedatasphere.dss.datamodel.dimension.entity.DssDatamodelDimension; +import lombok.Data; +import lombok.ToString; + +@Data +@ToString +public class DimensionAddVO { + + private String name; + + private String fieldIdentifier; + + private String formula; + + private String comment; + + /** + * 数仓主题格式为： theme_domain_name.theme_name + */ + private String warehouseThemeName; + + private String warehouseThemeNameEn; + + private String owner; + + /** + * 授权的名字：userName、roleName + */ + private String principalName; + + private Integer isAvailable; + +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-dimension-management/src/main/java/com/webank/wedatasphere/dss/datamodel/dimension/vo/DimensionEnableVO.java b/dss-apps/dss-datamodel-center/dss-datamodel-dimension-management/src/main/java/com/webank/wedatasphere/dss/datamodel/dimension/vo/DimensionEnableVO.java new file mode 100644 index 0000000000..b64683c7e5 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-dimension-management/src/main/java/com/webank/wedatasphere/dss/datamodel/dimension/vo/DimensionEnableVO.java @@ -0,0 +1,24 @@ +package com.webank.wedatasphere.dss.datamodel.dimension.vo; + + +public class DimensionEnableVO { + + + private Integer isAvailable; + + + public Integer getIsAvailable() { + return isAvailable; + } + + public void setIsAvailable(Integer isAvailable) { + this.isAvailable = isAvailable; + } + + @Override + public String toString() { + return "DimensionEnableVO{" + + "isAvailable=" + isAvailable + + '}'; + } +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-dimension-management/src/main/java/com/webank/wedatasphere/dss/datamodel/dimension/vo/DimensionQueryVO.java b/dss-apps/dss-datamodel-center/dss-datamodel-dimension-management/src/main/java/com/webank/wedatasphere/dss/datamodel/dimension/vo/DimensionQueryVO.java new file mode 100644 index 0000000000..2e46a9c3de --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-dimension-management/src/main/java/com/webank/wedatasphere/dss/datamodel/dimension/vo/DimensionQueryVO.java @@ -0,0 +1,19 @@ +package com.webank.wedatasphere.dss.datamodel.dimension.vo; + +import com.webank.wedatasphere.dss.datamodel.center.common.vo.PageVO; +import lombok.Data; +import lombok.ToString; + +@Data +@ToString +public class DimensionQueryVO extends PageVO { + + private String name; + + private Integer isAvailable; + + private String owner; + + private String warehouseThemeName; + +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-dimension-management/src/main/java/com/webank/wedatasphere/dss/datamodel/dimension/vo/DimensionUpdateVO.java b/dss-apps/dss-datamodel-center/dss-datamodel-dimension-management/src/main/java/com/webank/wedatasphere/dss/datamodel/dimension/vo/DimensionUpdateVO.java new file mode 100644 index 0000000000..74b2456d90 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-dimension-management/src/main/java/com/webank/wedatasphere/dss/datamodel/dimension/vo/DimensionUpdateVO.java @@ -0,0 +1,35 @@ +package com.webank.wedatasphere.dss.datamodel.dimension.vo; + +import lombok.Data; +import lombok.ToString; + +@Data +@ToString +public class DimensionUpdateVO { + + private String name; + + private String fieldIdentifier; + + private String formula; + + private String comment; + + /** + * 数仓主题格式为： theme_domain_name.theme_name + */ + private String warehouseThemeName; + + private String warehouseThemeNameEn; + + private String owner; + + /** + * 授权的名字：userName、roleName + */ + private String principalName; + + private Integer isAvailable; + + +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-indicator-management/pom.xml b/dss-apps/dss-datamodel-center/dss-datamodel-indicator-management/pom.xml new file mode 100644 index 0000000000..836320ee2a --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-indicator-management/pom.xml @@ -0,0 +1,60 @@ + + + + + + dss-datamodel-center + com.webank.wedatasphere.dss + 1.0.1 + ../pom.xml + + 4.0.0 + + dss-datamodel-indicator-management + + + + + + com.webank.wedatasphere.dss + dss-datamodel-center-common + ${dss.version} + + + + + + + + org.apache.maven.plugins + maven-deploy-plugin + + + net.alchim31.maven + scala-maven-plugin + + + org.apache.maven.plugins + maven-jar-plugin + + + + ${project.artifactId}-${project.version} + + diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-indicator-management/src/main/java/com/webank/wedatasphere/dss/datamodel/indicator/dao/DssDatamodelIndicatorContentMapper.java b/dss-apps/dss-datamodel-center/dss-datamodel-indicator-management/src/main/java/com/webank/wedatasphere/dss/datamodel/indicator/dao/DssDatamodelIndicatorContentMapper.java new file mode 100644 index 0000000000..fc78a2d5c4 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-indicator-management/src/main/java/com/webank/wedatasphere/dss/datamodel/indicator/dao/DssDatamodelIndicatorContentMapper.java @@ -0,0 +1,7 @@ +package com.webank.wedatasphere.dss.datamodel.indicator.dao; + +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import com.webank.wedatasphere.dss.datamodel.indicator.entity.DssDatamodelIndicatorContent; + +public interface DssDatamodelIndicatorContentMapper extends BaseMapper { +} \ No newline at end of file diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-indicator-management/src/main/java/com/webank/wedatasphere/dss/datamodel/indicator/dao/DssDatamodelIndicatorMapper.java b/dss-apps/dss-datamodel-center/dss-datamodel-indicator-management/src/main/java/com/webank/wedatasphere/dss/datamodel/indicator/dao/DssDatamodelIndicatorMapper.java new file mode 100644 index 0000000000..e75fbdec1e --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-indicator-management/src/main/java/com/webank/wedatasphere/dss/datamodel/indicator/dao/DssDatamodelIndicatorMapper.java @@ -0,0 +1,7 @@ +package com.webank.wedatasphere.dss.datamodel.indicator.dao; + +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import com.webank.wedatasphere.dss.datamodel.indicator.entity.DssDatamodelIndicator; + +public interface DssDatamodelIndicatorMapper extends BaseMapper { +} \ No newline at end of file diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-indicator-management/src/main/java/com/webank/wedatasphere/dss/datamodel/indicator/dao/DssDatamodelIndicatorVersionMapper.java b/dss-apps/dss-datamodel-center/dss-datamodel-indicator-management/src/main/java/com/webank/wedatasphere/dss/datamodel/indicator/dao/DssDatamodelIndicatorVersionMapper.java new file mode 100644 index 0000000000..6846fd04ac --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-indicator-management/src/main/java/com/webank/wedatasphere/dss/datamodel/indicator/dao/DssDatamodelIndicatorVersionMapper.java @@ -0,0 +1,7 @@ +package com.webank.wedatasphere.dss.datamodel.indicator.dao; + +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import com.webank.wedatasphere.dss.datamodel.indicator.entity.DssDatamodelIndicatorVersion; + +public interface DssDatamodelIndicatorVersionMapper extends BaseMapper { +} \ No newline at end of file diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-indicator-management/src/main/java/com/webank/wedatasphere/dss/datamodel/indicator/dao/IndicatorQueryMapper.java b/dss-apps/dss-datamodel-center/dss-datamodel-indicator-management/src/main/java/com/webank/wedatasphere/dss/datamodel/indicator/dao/IndicatorQueryMapper.java new file mode 100644 index 0000000000..32778367a1 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-indicator-management/src/main/java/com/webank/wedatasphere/dss/datamodel/indicator/dao/IndicatorQueryMapper.java @@ -0,0 +1,22 @@ +package com.webank.wedatasphere.dss.datamodel.indicator.dao; + +import com.baomidou.mybatisplus.core.conditions.Wrapper; +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import com.baomidou.mybatisplus.core.toolkit.Constants; +import com.baomidou.mybatisplus.extension.plugins.pagination.Page; +import com.webank.wedatasphere.dss.datamodel.indicator.entity.DssDatamodelIndicatorQuery; +import org.apache.ibatis.annotations.Param; +import org.apache.ibatis.annotations.Select; + +import java.util.List; + + +public interface IndicatorQueryMapper extends BaseMapper { + + String querySql = "SELECT a.*, b.indicator_type, b.measure_id, b.indicator_source_info, b.formula, b.business, b.business_owner, b.calculation, b.calculation_owner " + + " FROM dss_datamodel_indicator AS a LEFT JOIN dss_datamodel_indicator_content AS b ON b.indicator_id = a.id ${ew.customSqlSegment}"; + String wrapperSql = "SELECT * from ( " + querySql + " ) AS q ${ew.customSqlSegment}"; + + @Select(querySql) + List page(@Param(Constants.WRAPPER) Wrapper queryWrapper); +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-indicator-management/src/main/java/com/webank/wedatasphere/dss/datamodel/indicator/dao/impl/DssDatamodelIndicatorContentMapper.xml b/dss-apps/dss-datamodel-center/dss-datamodel-indicator-management/src/main/java/com/webank/wedatasphere/dss/datamodel/indicator/dao/impl/DssDatamodelIndicatorContentMapper.xml new file mode 100644 index 0000000000..4dd3eebf85 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-indicator-management/src/main/java/com/webank/wedatasphere/dss/datamodel/indicator/dao/impl/DssDatamodelIndicatorContentMapper.xml @@ -0,0 +1,26 @@ + + + + + + + + + + + + + + + + + + + + + + + id, indicator_id, version, indicator_type, measure_id, indicator_source_info, formula, + business, business_owner, calculation, calculation_owner, create_time, update_time + + \ No newline at end of file diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-indicator-management/src/main/java/com/webank/wedatasphere/dss/datamodel/indicator/dao/impl/DssDatamodelIndicatorMapper.xml b/dss-apps/dss-datamodel-center/dss-datamodel-indicator-management/src/main/java/com/webank/wedatasphere/dss/datamodel/indicator/dao/impl/DssDatamodelIndicatorMapper.xml new file mode 100644 index 0000000000..a0cd76e2be --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-indicator-management/src/main/java/com/webank/wedatasphere/dss/datamodel/indicator/dao/impl/DssDatamodelIndicatorMapper.xml @@ -0,0 +1,29 @@ + + + + + + + + + + + + + + + + + + + + + + + + + id, `name`, field_identifier, `comment`, warehouse_theme_name, `owner`, principal_name, + is_available, is_core_indicator, theme_area, layer_area, create_time, update_time, + version, warehouse_theme_name_en + + \ No newline at end of file diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-indicator-management/src/main/java/com/webank/wedatasphere/dss/datamodel/indicator/dao/impl/DssDatamodelIndicatorVersionMapper.xml b/dss-apps/dss-datamodel-center/dss-datamodel-indicator-management/src/main/java/com/webank/wedatasphere/dss/datamodel/indicator/dao/impl/DssDatamodelIndicatorVersionMapper.xml new file mode 100644 index 0000000000..30981471b0 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-indicator-management/src/main/java/com/webank/wedatasphere/dss/datamodel/indicator/dao/impl/DssDatamodelIndicatorVersionMapper.xml @@ -0,0 +1,22 @@ + + + + + + + + + + + + + + + + + + + id, `name`, `owner`, principal_name, version, version_context, create_time, update_time, + `comment` + + \ No newline at end of file diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-indicator-management/src/main/java/com/webank/wedatasphere/dss/datamodel/indicator/dto/IndicatorContentQueryDTO.java b/dss-apps/dss-datamodel-center/dss-datamodel-indicator-management/src/main/java/com/webank/wedatasphere/dss/datamodel/indicator/dto/IndicatorContentQueryDTO.java new file mode 100644 index 0000000000..cde5742c38 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-indicator-management/src/main/java/com/webank/wedatasphere/dss/datamodel/indicator/dto/IndicatorContentQueryDTO.java @@ -0,0 +1,143 @@ +package com.webank.wedatasphere.dss.datamodel.indicator.dto; + +import java.util.Date; + + +public class IndicatorContentQueryDTO { + + private Long id; + + private Long indicatorId; + + private String version; + + /** + * 0 原子 1 衍生 2 派生 3 复杂 4 自定义 + */ + private Integer indicatorType; + + private Long measureId; + + /** + * 指标来源信息 + */ + private String indicatorSourceInfo; + + private String formula; + + private String business; + + private String businessOwner; + + private String calculation; + + private String calculationOwner; + + private Date createTime; + + private Date updateTime; + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public Long getIndicatorId() { + return indicatorId; + } + + public void setIndicatorId(Long indicatorId) { + this.indicatorId = indicatorId; + } + + public String getVersion() { + return version; + } + + public void setVersion(String version) { + this.version = version; + } + + public Integer getIndicatorType() { + return indicatorType; + } + + public void setIndicatorType(Integer indicatorType) { + this.indicatorType = indicatorType; + } + + public Long getMeasureId() { + return measureId; + } + + public void setMeasureId(Long measureId) { + this.measureId = measureId; + } + + public String getIndicatorSourceInfo() { + return indicatorSourceInfo; + } + + public void setIndicatorSourceInfo(String indicatorSourceInfo) { + this.indicatorSourceInfo = indicatorSourceInfo; + } + + public String getFormula() { + return formula; + } + + public void setFormula(String formula) { + this.formula = formula; + } + + public String getBusiness() { + return business; + } + + public void setBusiness(String business) { + this.business = business; + } + + public String getBusinessOwner() { + return businessOwner; + } + + public void setBusinessOwner(String businessOwner) { + this.businessOwner = businessOwner; + } + + public String getCalculation() { + return calculation; + } + + public void setCalculation(String calculation) { + this.calculation = calculation; + } + + public String getCalculationOwner() { + return calculationOwner; + } + + public void setCalculationOwner(String calculationOwner) { + this.calculationOwner = calculationOwner; + } + + public Date getCreateTime() { + return createTime; + } + + public void setCreateTime(Date createTime) { + this.createTime = createTime; + } + + public Date getUpdateTime() { + return updateTime; + } + + public void setUpdateTime(Date updateTime) { + this.updateTime = updateTime; + } +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-indicator-management/src/main/java/com/webank/wedatasphere/dss/datamodel/indicator/dto/IndicatorQueryDTO.java b/dss-apps/dss-datamodel-center/dss-datamodel-indicator-management/src/main/java/com/webank/wedatasphere/dss/datamodel/indicator/dto/IndicatorQueryDTO.java new file mode 100644 index 0000000000..b6ea24bba6 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-indicator-management/src/main/java/com/webank/wedatasphere/dss/datamodel/indicator/dto/IndicatorQueryDTO.java @@ -0,0 +1,65 @@ +package com.webank.wedatasphere.dss.datamodel.indicator.dto; + +import lombok.Data; +import lombok.ToString; + +import java.util.Date; + +@Data +@ToString +public class IndicatorQueryDTO { + private Long id; + + private String name; + + private String fieldIdentifier; + + private String comment; + + /** + * 数仓主题格式为： theme_domain_name.theme_name + */ + private String warehouseThemeName; + + private String warehouseThemeNameEn; + + private String themeAreaEn; + + private String layerAreaEn; + + private String owner; + + /** + * 授权的名字：userName、roleName + */ + private String principalName; + + private Integer isAvailable; + + private Integer isCoreIndicator; + + /** + * 空：代表所有，如果是逗号分隔的字符串则代表对应的theme的names + */ + private String themeArea; + + /** + * 空：代表所有，如果是逗号分隔的字符串则代表对应的layer的names + */ + private String layerArea; + + private Date createTime; + + private Date updateTime; + + private String version; + + /** + * 引用次数 + */ + private Integer refCount = 0; + + private IndicatorContentQueryDTO content; + + +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-indicator-management/src/main/java/com/webank/wedatasphere/dss/datamodel/indicator/dto/IndicatorVersionDTO.java b/dss-apps/dss-datamodel-center/dss-datamodel-indicator-management/src/main/java/com/webank/wedatasphere/dss/datamodel/indicator/dto/IndicatorVersionDTO.java new file mode 100644 index 0000000000..d2ba5f11ba --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-indicator-management/src/main/java/com/webank/wedatasphere/dss/datamodel/indicator/dto/IndicatorVersionDTO.java @@ -0,0 +1,43 @@ +package com.webank.wedatasphere.dss.datamodel.indicator.dto; + +import com.webank.wedatasphere.dss.datamodel.indicator.entity.DssDatamodelIndicator; +import com.webank.wedatasphere.dss.datamodel.indicator.entity.DssDatamodelIndicatorContent; + + +public class IndicatorVersionDTO { + + private DssDatamodelIndicator essential; + + private DssDatamodelIndicatorContent content; + + public IndicatorVersionDTO() { + } + + public IndicatorVersionDTO(DssDatamodelIndicator essential, DssDatamodelIndicatorContent content) { + this.essential = essential; + this.content = content; + } + + public DssDatamodelIndicator getEssential() { + return essential; + } + + public void setEssential(DssDatamodelIndicator essential) { + this.essential = essential; + } + + public DssDatamodelIndicatorContent getContent() { + return content; + } + + public void setContent(DssDatamodelIndicatorContent content) { + this.content = content; + } + + public static IndicatorVersionDTO of(DssDatamodelIndicator essential, DssDatamodelIndicatorContent content){ + IndicatorVersionDTO dto = new IndicatorVersionDTO(); + dto.setContent(content); + dto.setEssential(essential); + return dto; + } +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-indicator-management/src/main/java/com/webank/wedatasphere/dss/datamodel/indicator/dto/IndicatorVersionQueryDTO.java b/dss-apps/dss-datamodel-center/dss-datamodel-indicator-management/src/main/java/com/webank/wedatasphere/dss/datamodel/indicator/dto/IndicatorVersionQueryDTO.java new file mode 100644 index 0000000000..4a1ae6cf6e --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-indicator-management/src/main/java/com/webank/wedatasphere/dss/datamodel/indicator/dto/IndicatorVersionQueryDTO.java @@ -0,0 +1,106 @@ +package com.webank.wedatasphere.dss.datamodel.indicator.dto; + +import com.baomidou.mybatisplus.annotation.FieldFill; +import com.baomidou.mybatisplus.annotation.TableField; + +import java.util.Date; + + +public class IndicatorVersionQueryDTO { + + private Long id; + + private String name; + + private String owner; + + /** + * 授权的名字：userName、roleName + */ + private String principalName; + + private String version; + + /** + * 历史版本详细信息快照 + */ + private String versionContext; + + private Date createTime; + + private Date updateTime; + + private String comment; + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getOwner() { + return owner; + } + + public void setOwner(String owner) { + this.owner = owner; + } + + public String getPrincipalName() { + return principalName; + } + + public void setPrincipalName(String principalName) { + this.principalName = principalName; + } + + public String getVersion() { + return version; + } + + public void setVersion(String version) { + this.version = version; + } + + public String getVersionContext() { + return versionContext; + } + + public void setVersionContext(String versionContext) { + this.versionContext = versionContext; + } + + public Date getCreateTime() { + return createTime; + } + + public void setCreateTime(Date createTime) { + this.createTime = createTime; + } + + public Date getUpdateTime() { + return updateTime; + } + + public void setUpdateTime(Date updateTime) { + this.updateTime = updateTime; + } + + public String getComment() { + return comment; + } + + public void setComment(String comment) { + this.comment = comment; + } +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-indicator-management/src/main/java/com/webank/wedatasphere/dss/datamodel/indicator/entity/DssDatamodelIndicator.java b/dss-apps/dss-datamodel-center/dss-datamodel-indicator-management/src/main/java/com/webank/wedatasphere/dss/datamodel/indicator/entity/DssDatamodelIndicator.java new file mode 100644 index 0000000000..aa6aa72e47 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-indicator-management/src/main/java/com/webank/wedatasphere/dss/datamodel/indicator/entity/DssDatamodelIndicator.java @@ -0,0 +1,61 @@ +package com.webank.wedatasphere.dss.datamodel.indicator.entity; + +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableId; +import lombok.Data; + +import java.util.Date; + +@Data +public class DssDatamodelIndicator { + @TableId(type = IdType.AUTO) + private Long id; + + private String name; + + private String fieldIdentifier; + + private String comment; + + /** + * 数仓主题格式为： theme_domain_name.theme_name + */ + private String warehouseThemeName; + + private String owner; + + /** + * 授权的名字：userName、roleName + */ + private String principalName; + + private Integer isAvailable; + + private Integer isCoreIndicator; + + /** + * 空：代表所有，如果是逗号分隔的字符串则代表对应的theme的names + */ + private String themeArea; + + private String themeAreaEn; + + /** + * 空：代表所有，如果是逗号分隔的字符串则代表对应的layer的names + */ + private String layerArea; + + private String layerAreaEn; + + private Date createTime; + + private Date updateTime; + + private String version; + + /** + * 数仓主题英文 + */ + private String warehouseThemeNameEn; + +} \ No newline at end of file diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-indicator-management/src/main/java/com/webank/wedatasphere/dss/datamodel/indicator/entity/DssDatamodelIndicatorContent.java b/dss-apps/dss-datamodel-center/dss-datamodel-indicator-management/src/main/java/com/webank/wedatasphere/dss/datamodel/indicator/entity/DssDatamodelIndicatorContent.java new file mode 100644 index 0000000000..517acbe728 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-indicator-management/src/main/java/com/webank/wedatasphere/dss/datamodel/indicator/entity/DssDatamodelIndicatorContent.java @@ -0,0 +1,150 @@ +package com.webank.wedatasphere.dss.datamodel.indicator.entity; + +import com.baomidou.mybatisplus.annotation.FieldFill; +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableField; +import com.baomidou.mybatisplus.annotation.TableId; + +import java.util.Date; + +public class DssDatamodelIndicatorContent { + + @TableId(type = IdType.AUTO) + private Long id; + + private Long indicatorId; + + private String version; + + /** + * 0 原子 1 衍生 2 派生 3 复杂 4 自定义 + */ + private Integer indicatorType; + + private Long measureId; + + /** + * 指标来源信息 + */ + private String indicatorSourceInfo; + + private String formula; + + private String business; + + private String businessOwner; + + private String calculation; + + private String calculationOwner; + + @TableField(fill = FieldFill.INSERT) + private Date createTime; + + @TableField(fill = FieldFill.INSERT_UPDATE) + private Date updateTime; + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public Long getIndicatorId() { + return indicatorId; + } + + public void setIndicatorId(Long indicatorId) { + this.indicatorId = indicatorId; + } + + public String getVersion() { + return version; + } + + public void setVersion(String version) { + this.version = version; + } + + public Integer getIndicatorType() { + return indicatorType; + } + + public void setIndicatorType(Integer indicatorType) { + this.indicatorType = indicatorType; + } + + public Long getMeasureId() { + return measureId; + } + + public void setMeasureId(Long measureId) { + this.measureId = measureId; + } + + public String getIndicatorSourceInfo() { + return indicatorSourceInfo; + } + + public void setIndicatorSourceInfo(String indicatorSourceInfo) { + this.indicatorSourceInfo = indicatorSourceInfo; + } + + public String getFormula() { + return formula; + } + + public void setFormula(String formula) { + this.formula = formula; + } + + public String getBusiness() { + return business; + } + + public void setBusiness(String business) { + this.business = business; + } + + public String getBusinessOwner() { + return businessOwner; + } + + public void setBusinessOwner(String businessOwner) { + this.businessOwner = businessOwner; + } + + public String getCalculation() { + return calculation; + } + + public void setCalculation(String calculation) { + this.calculation = calculation; + } + + public String getCalculationOwner() { + return calculationOwner; + } + + public void setCalculationOwner(String calculationOwner) { + this.calculationOwner = calculationOwner; + } + + public Date getCreateTime() { + return createTime; + } + + public void setCreateTime(Date createTime) { + this.createTime = createTime; + } + + public Date getUpdateTime() { + return updateTime; + } + + public void setUpdateTime(Date updateTime) { + this.updateTime = updateTime; + } +} \ No newline at end of file diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-indicator-management/src/main/java/com/webank/wedatasphere/dss/datamodel/indicator/entity/DssDatamodelIndicatorQuery.java b/dss-apps/dss-datamodel-center/dss-datamodel-indicator-management/src/main/java/com/webank/wedatasphere/dss/datamodel/indicator/entity/DssDatamodelIndicatorQuery.java new file mode 100644 index 0000000000..3932fd9a72 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-indicator-management/src/main/java/com/webank/wedatasphere/dss/datamodel/indicator/entity/DssDatamodelIndicatorQuery.java @@ -0,0 +1,258 @@ +package com.webank.wedatasphere.dss.datamodel.indicator.entity; + +import java.util.Date; + + +public class DssDatamodelIndicatorQuery { + + private Long id; + + private String name; + + private String fieldIdentifier; + + private String comment; + + /** + * 数仓主题格式为： theme_domain_name.theme_name + */ + private String warehouseThemeName; + + private String owner; + + /** + * 授权的名字：userName、roleName + */ + private String principalName; + + private Integer isAvailable; + + private Integer isCoreIndicator; + + /** + * 空：代表所有，如果是逗号分隔的字符串则代表对应的theme的names + */ + private String themeArea; + + /** + * 空：代表所有，如果是逗号分隔的字符串则代表对应的layer的names + */ + private String layerArea; + + private Date createTime; + + private Date updateTime; + + private String version; + + /** + * 0 原子 1 衍生 2 派生 3 复杂 4 自定义 + */ + private Integer indicatorType; + + private Long measureId; + + /** + * 指标来源信息 + */ + private String indicatorSourceInfo; + + private String formula; + + private String business; + + private String businessOwner; + + private String calculation; + + private String calculationOwner; + + /** + * 引用次数 + */ + private Integer refCount = 0; + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getFieldIdentifier() { + return fieldIdentifier; + } + + public void setFieldIdentifier(String fieldIdentifier) { + this.fieldIdentifier = fieldIdentifier; + } + + public String getComment() { + return comment; + } + + public void setComment(String comment) { + this.comment = comment; + } + + public String getWarehouseThemeName() { + return warehouseThemeName; + } + + public void setWarehouseThemeName(String warehouseThemeName) { + this.warehouseThemeName = warehouseThemeName; + } + + public String getOwner() { + return owner; + } + + public void setOwner(String owner) { + this.owner = owner; + } + + public String getPrincipalName() { + return principalName; + } + + public void setPrincipalName(String principalName) { + this.principalName = principalName; + } + + public Integer getIsAvailable() { + return isAvailable; + } + + public void setIsAvailable(Integer isAvailable) { + this.isAvailable = isAvailable; + } + + public Integer getIsCoreIndicator() { + return isCoreIndicator; + } + + public void setIsCoreIndicator(Integer isCoreIndicator) { + this.isCoreIndicator = isCoreIndicator; + } + + public String getThemeArea() { + return themeArea; + } + + public void setThemeArea(String themeArea) { + this.themeArea = themeArea; + } + + public String getLayerArea() { + return layerArea; + } + + public void setLayerArea(String layerArea) { + this.layerArea = layerArea; + } + + public Date getCreateTime() { + return createTime; + } + + public void setCreateTime(Date createTime) { + this.createTime = createTime; + } + + public Date getUpdateTime() { + return updateTime; + } + + public void setUpdateTime(Date updateTime) { + this.updateTime = updateTime; + } + + public String getVersion() { + return version; + } + + public void setVersion(String version) { + this.version = version; + } + + public Integer getIndicatorType() { + return indicatorType; + } + + public void setIndicatorType(Integer indicatorType) { + this.indicatorType = indicatorType; + } + + public Long getMeasureId() { + return measureId; + } + + public void setMeasureId(Long measureId) { + this.measureId = measureId; + } + + public String getIndicatorSourceInfo() { + return indicatorSourceInfo; + } + + public void setIndicatorSourceInfo(String indicatorSourceInfo) { + this.indicatorSourceInfo = indicatorSourceInfo; + } + + public String getFormula() { + return formula; + } + + public void setFormula(String formula) { + this.formula = formula; + } + + public String getBusiness() { + return business; + } + + public void setBusiness(String business) { + this.business = business; + } + + public String getBusinessOwner() { + return businessOwner; + } + + public void setBusinessOwner(String businessOwner) { + this.businessOwner = businessOwner; + } + + public String getCalculation() { + return calculation; + } + + public void setCalculation(String calculation) { + this.calculation = calculation; + } + + public String getCalculationOwner() { + return calculationOwner; + } + + public void setCalculationOwner(String calculationOwner) { + this.calculationOwner = calculationOwner; + } + + public Integer getRefCount() { + return refCount; + } + + public void setRefCount(Integer refCount) { + this.refCount = refCount; + } +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-indicator-management/src/main/java/com/webank/wedatasphere/dss/datamodel/indicator/entity/DssDatamodelIndicatorVersion.java b/dss-apps/dss-datamodel-center/dss-datamodel-indicator-management/src/main/java/com/webank/wedatasphere/dss/datamodel/indicator/entity/DssDatamodelIndicatorVersion.java new file mode 100644 index 0000000000..0acff1f3ee --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-indicator-management/src/main/java/com/webank/wedatasphere/dss/datamodel/indicator/entity/DssDatamodelIndicatorVersion.java @@ -0,0 +1,110 @@ +package com.webank.wedatasphere.dss.datamodel.indicator.entity; + +import com.baomidou.mybatisplus.annotation.FieldFill; +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableField; +import com.baomidou.mybatisplus.annotation.TableId; + +import java.util.Date; + +public class DssDatamodelIndicatorVersion { + + @TableId(type = IdType.AUTO) + private Long id; + + private String name; + + private String owner; + + /** + * 授权的名字：userName、roleName + */ + private String principalName; + + private String version; + + /** + * 历史版本详细信息快照 + */ + private String versionContext; + + @TableField(fill = FieldFill.INSERT) + private Date createTime; + + @TableField(fill = FieldFill.INSERT_UPDATE) + private Date updateTime; + + private String comment; + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getOwner() { + return owner; + } + + public void setOwner(String owner) { + this.owner = owner; + } + + public String getPrincipalName() { + return principalName; + } + + public void setPrincipalName(String principalName) { + this.principalName = principalName; + } + + public String getVersion() { + return version; + } + + public void setVersion(String version) { + this.version = version; + } + + public String getVersionContext() { + return versionContext; + } + + public void setVersionContext(String versionContext) { + this.versionContext = versionContext; + } + + public Date getCreateTime() { + return createTime; + } + + public void setCreateTime(Date createTime) { + this.createTime = createTime; + } + + public Date getUpdateTime() { + return updateTime; + } + + public void setUpdateTime(Date updateTime) { + this.updateTime = updateTime; + } + + public String getComment() { + return comment; + } + + public void setComment(String comment) { + this.comment = comment; + } +} \ No newline at end of file diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-indicator-management/src/main/java/com/webank/wedatasphere/dss/datamodel/indicator/restful/IndicatorRestfulApi.java b/dss-apps/dss-datamodel-center/dss-datamodel-indicator-management/src/main/java/com/webank/wedatasphere/dss/datamodel/indicator/restful/IndicatorRestfulApi.java new file mode 100644 index 0000000000..ef93875f64 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-indicator-management/src/main/java/com/webank/wedatasphere/dss/datamodel/indicator/restful/IndicatorRestfulApi.java @@ -0,0 +1,234 @@ +package com.webank.wedatasphere.dss.datamodel.indicator.restful; + +import com.webank.wedatasphere.dss.datamodel.center.common.service.AuthenticationClientStrategy; +import com.webank.wedatasphere.dss.datamodel.indicator.service.IndicatorService; +import com.webank.wedatasphere.dss.datamodel.indicator.vo.*; +import org.apache.linkis.common.exception.ErrorException; +import org.apache.linkis.server.Message; +import com.webank.wedatasphere.warehouse.client.GovernanceDwRemoteClient; +import com.webank.wedatasphere.warehouse.client.action.ListDwLayerAction; +import com.webank.wedatasphere.warehouse.client.action.ListDwModifierAction; +import com.webank.wedatasphere.warehouse.client.action.ListDwStatisticalPeriodAction; +import com.webank.wedatasphere.warehouse.client.action.ListDwThemeDomainAction; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; +import org.springframework.web.bind.annotation.RequestBody; + +import javax.servlet.http.HttpServletRequest; +import javax.ws.rs.*; +import javax.ws.rs.core.Context; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; +import java.io.IOException; + + +@Component +@Path("/datamodel") +@Produces(MediaType.APPLICATION_JSON) +@Consumes(MediaType.APPLICATION_JSON) +public class IndicatorRestfulApi implements AuthenticationClientStrategy { + + + private static final Logger LOGGER = LoggerFactory.getLogger(IndicatorRestfulApi.class); + + @Autowired + private IndicatorService indicatorService; + + @Autowired + private GovernanceDwRemoteClient governanceDwRemoteClient; + + /** + * 新增 + * + * @param req + * @param vo + * @return + * @throws IOException + */ + @POST + @Path("/indicators") + public Response add(@Context HttpServletRequest req, @RequestBody IndicatorAddVO vo) throws Exception { + LOGGER.info("indicatorAddVO : {}", vo); + return Message.messageToResponse(Message.ok().data("count",indicatorService.addIndicator(vo,"1" ))); + } + + + /** + * 修改 + * @param req + * @param id + * @param vo + * @return + * @throws Exception + */ + @PUT + @Path("/indicators/{id}") + public Response update(@Context HttpServletRequest req, @PathParam("id") Long id , @RequestBody IndicatorUpdateVO vo) throws Exception { + LOGGER.info("update id : {}, indicatorUpdateVO : {}", id, vo); + return Message.messageToResponse(Message.ok().data("count",indicatorService.updateIndicator(id,vo))); + } + + + /** + * 启用/禁用 + * @param req + * @param id + * @param vo + * @return + */ + @PUT + @Path("/indicators/enable/{id}") + public Response enable(@Context HttpServletRequest req, @PathParam("id") Long id, @RequestBody IndicatorEnableVO vo) { + LOGGER.info("enable id : {}, vo : {}", id, vo); + return Message.messageToResponse(Message.ok().data("count", indicatorService.enableIndicator(id, vo))); + } + + + /** + * 分页搜索 + * @param req + * @return + */ + @POST + @Path("/indicators/list") + public Response list(@Context HttpServletRequest req, @RequestBody IndicatorQueryVO vo){ + LOGGER.info("list vo : {}",vo); + return Message.messageToResponse(indicatorService.listIndicators(vo)); + } + + + + /** + * 查看 + * @param req + * @param id + * @return + */ + @GET + @Path("/indicators/{id}") + public Response query(@Context HttpServletRequest req, @PathParam("id") Long id) throws ErrorException { + LOGGER.info("query id : {}", id); + return Message.messageToResponse(indicatorService.queryById(id)); + } + + /** + * 删除 + * @param req + * @param id + * @return + */ + @DELETE + @Path("/indicators/{id}") + public Response delete(@Context HttpServletRequest req, @PathParam("id") Long id) throws ErrorException { + LOGGER.info("delete id : {}", id); + return Message.messageToResponse(Message.ok().data("count",indicatorService.deleteIndicator(id))); + } + + + /** + * 新增版本 + * + * @param req + * @param vo + * @return + * @throws IOException + */ + @POST + @Path("/indicators/versions/{id}") + public Response addVersion(@Context HttpServletRequest req, @PathParam("id") Long id,@RequestBody IndicatorVersionAddVO vo) throws Exception { + LOGGER.info("indicatorVersionAddVO : {}", vo); + return Message.messageToResponse(Message.ok().data("count",indicatorService.addIndicatorVersion(id,vo))); + } + + + + /** + * + * 回退指定版本 + * @param req + * @param vo + * @return + * @throws IOException + */ + @POST + @Path("/indicators/versions/rollback") + public Response versionRollBack(@Context HttpServletRequest req,IndicatorVersionRollBackVO vo) throws Exception { + LOGGER.info("indicatorVersionRollBackVO : {}", vo); + return Message.messageToResponse(Message.ok().data("count",indicatorService.versionRollBack(vo))); + } + + + + /** + * 搜索指标版本 + * @param req + * @return + */ + @POST + @Path("/indicators/versions/list") + public Response indicatorVersionsList(@Context HttpServletRequest req, @RequestBody IndicatorVersionQueryVO vo){ + LOGGER.info("version list vo : {}",vo); + return Message.messageToResponse(indicatorService.listIndicatorVersions(vo)); + } + + + /** + * 指标相关主题可选列表 + * @param req + * @return + */ + @POST + @Path("/themes/list") + public Response themesList(@Context HttpServletRequest req){ + ListDwThemeDomainAction action = ListDwThemeDomainAction.builder().setUser(getStrategyUser(req)).setIsAvailable(true).build(); + return Message.messageToResponse(Message.ok().data("list",governanceDwRemoteClient.listThemeDomains(action).getAll())); + } + + /** + * 指标相关分层 + * @param req + * @return + */ + @POST + @Path("/layers/list") + public Response layerList(@Context HttpServletRequest req,@RequestBody LayerVO vo){ + LOGGER.info("layerList vo : {}",vo); + ListDwLayerAction action = ListDwLayerAction.builder().setIsAvailable(true).setDb(vo.getDbName()).setUser(getStrategyUser(req)).build(); + return Message.messageToResponse(Message.ok().data("list",governanceDwRemoteClient.listLayers(action).getAll())); + } + + /** + * 指标相关周期列表 + * @param req + * @return + */ + @POST + @Path("/cycles/list") + public Response cycleList(@Context HttpServletRequest req,@RequestBody CycleVO vo){ + LOGGER.info("cycleList vo : {}",vo); + ListDwStatisticalPeriodAction action =ListDwStatisticalPeriodAction.builder().setUser(getStrategyUser(req)).setLayer(vo.getLayer()).setIsAvailable(true).setTheme(vo.getTheme()).build(); + return Message.messageToResponse(Message.ok().data("list",governanceDwRemoteClient.listStatisticalPeriods(action).getAll())); + } + + /** + * 指标相关修饰词列表 + * @param req + * @return + */ + @POST + @Path("/modifiers/list") + public Response modifierList(@Context HttpServletRequest req,@RequestBody ModifierVO vo){ + LOGGER.info("modifierList vo : {}",vo); + ListDwModifierAction action = ListDwModifierAction.builder().setUser(getStrategyUser(req)).setLayer(vo.getLayer()).setIsAvailable(true).setTheme(vo.getTheme()).build(); + return Message.messageToResponse(Message.ok().data("list",governanceDwRemoteClient.listModifiers(action).getAll())); + } + + + @POST + @Path("/current/user") + public Response currentUser(@Context HttpServletRequest req){ + return Message.messageToResponse(Message.ok().data("user",getStrategyUser(req))); + } +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-indicator-management/src/main/java/com/webank/wedatasphere/dss/datamodel/indicator/service/IndicatorContentService.java b/dss-apps/dss-datamodel-center/dss-datamodel-indicator-management/src/main/java/com/webank/wedatasphere/dss/datamodel/indicator/service/IndicatorContentService.java new file mode 100644 index 0000000000..0222ef4060 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-indicator-management/src/main/java/com/webank/wedatasphere/dss/datamodel/indicator/service/IndicatorContentService.java @@ -0,0 +1,59 @@ +package com.webank.wedatasphere.dss.datamodel.indicator.service; + +import com.baomidou.mybatisplus.extension.service.IService; +import com.webank.wedatasphere.dss.datamodel.indicator.entity.DssDatamodelIndicatorContent; +import com.webank.wedatasphere.dss.datamodel.indicator.vo.IndicatorContentVO; +import org.apache.linkis.common.exception.ErrorException; + +import java.util.List; + + +public interface IndicatorContentService extends IService { + + /** + * 新增 + * + * @param vo + * @return + */ + int addIndicatorContent(Long indicateId, String version, IndicatorContentVO vo) throws ErrorException; + + + /** + * 修改 + * @param indicateId + * @param version + * @param vo + * @return + * @throws ErrorException + */ + int updateIndicatorContent(Long indicateId, String version, IndicatorContentVO vo) throws ErrorException; + + + /** + * 查看 + * @param indicateId + * @return + */ + DssDatamodelIndicatorContent queryByIndicateId(Long indicateId); + + + /** + * 根据indicator 删除指标详情 + * @param id + * @return + * @throws ErrorException + */ + int deleteByIndicatorId(Long id)throws ErrorException; + + + /** + * 校验indicator引用情况 + * @param name + * @return + * @throws ErrorException + */ + List sourceInfoReference(String name); + + +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-indicator-management/src/main/java/com/webank/wedatasphere/dss/datamodel/indicator/service/IndicatorService.java b/dss-apps/dss-datamodel-center/dss-datamodel-indicator-management/src/main/java/com/webank/wedatasphere/dss/datamodel/indicator/service/IndicatorService.java new file mode 100644 index 0000000000..e7449a8eb2 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-indicator-management/src/main/java/com/webank/wedatasphere/dss/datamodel/indicator/service/IndicatorService.java @@ -0,0 +1,146 @@ +package com.webank.wedatasphere.dss.datamodel.indicator.service; + +import com.baomidou.mybatisplus.extension.service.IService; +import com.webank.wedatasphere.dss.datamodel.center.common.exception.DSSDatamodelCenterException; +import com.webank.wedatasphere.dss.datamodel.indicator.entity.DssDatamodelIndicator; +import com.webank.wedatasphere.dss.datamodel.indicator.vo.*; +import org.apache.linkis.common.exception.ErrorException; +import org.apache.linkis.server.Message; + + +public interface IndicatorService extends IService { + + /** + * 新增 + * @param vo + * @param version + * @return + */ + Long addIndicator(IndicatorAddVO vo, String version) throws ErrorException; + + + + /** + * 更新 + * @param vo + * @return + * @throws ErrorException + */ + int updateIndicator(Long id, IndicatorUpdateVO vo) throws ErrorException; + + + + /** + * 启用/禁用 + * @param vo + * @return + */ + int enableIndicator(Long id, IndicatorEnableVO vo); + + + + /** + * 删除 + * @param id + * @return + * @throws ErrorException + */ + int deleteIndicator(Long id) throws ErrorException; + + + + /** + * 查询列表 + * @param vo + * @return + */ + Message listIndicators(IndicatorQueryVO vo); + + + /** + * 查看 + * @param id + * @return + */ + Message queryById(Long id) throws DSSDatamodelCenterException; + + + /** + * 新增版本 + * @param id + * @return + */ + int addIndicatorVersion(Long id, IndicatorVersionAddVO vo) throws ErrorException; + + + /** + * 回退到某一版本 + * @param vo + * @return + * @throws ErrorException + */ + int versionRollBack(IndicatorVersionRollBackVO vo) throws ErrorException; + + + /** + * 查询历史版本列表 + * @param vo + * @return + */ + Message listIndicatorVersions(IndicatorVersionQueryVO vo); + + + /** + * 主题引用情况 + * @param name + * @return + */ + int indicatorThemeReferenceCount(String name); + + /** + * 分层引用情况 + * @param name + * @return + */ + int indicatorLayerReferenceCount(String name); + + + /** + * 周期引用情况 + * @param name + * @return + */ + int indicatorCycleReferenceCount(String name); + + + /** + * 修饰词引用情况 + * @param name + * @return + */ + int indicatorModifierReferenceCount(String name); + + + /** + * 维度引用情况 + * @param name + * @return + */ + int indicatorDimensionReferenceCount(String name); + + + /** + * 度量引用情况 + * @param name + * @return + */ + int indicatorMeasureReferenceCount(String name); + + + /** + * 指标引用指标情况 + * @param name + * @return + */ + int indicatorIndicatorReferenceCount(String name); +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-indicator-management/src/main/java/com/webank/wedatasphere/dss/datamodel/indicator/service/IndicatorVersionService.java b/dss-apps/dss-datamodel-center/dss-datamodel-indicator-management/src/main/java/com/webank/wedatasphere/dss/datamodel/indicator/service/IndicatorVersionService.java new file mode 100644 index 0000000000..04f77dc1ba --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-indicator-management/src/main/java/com/webank/wedatasphere/dss/datamodel/indicator/service/IndicatorVersionService.java @@ -0,0 +1,57 @@ +package com.webank.wedatasphere.dss.datamodel.indicator.service; + +import com.baomidou.mybatisplus.extension.service.IService; +import com.webank.wedatasphere.dss.datamodel.indicator.dto.IndicatorVersionDTO; +import com.webank.wedatasphere.dss.datamodel.indicator.entity.DssDatamodelIndicatorVersion; +import org.apache.linkis.common.exception.ErrorException; + +import java.util.List; + + +public interface IndicatorVersionService extends IService { + + + /** + * 保存旧版本 + * + * @param name + * @param owner + * @param principalName + * @param version + * @param versionContext + * @return + */ + int addOlderVersion(String name, String owner, String principalName, String version, String comment, IndicatorVersionDTO versionContext) throws ErrorException; + + /** + * 根据指标名称查找最大版本 + * @param name + * @return + */ + String findLastVersion(String name); + + + /** + * 查询指定版本指标 + * @param name + * @param version + * @return + */ + DssDatamodelIndicatorVersion findBackup(String name,String version); + + + /** + * 引用情况 + * @param context + * @return + */ + List contentReferenceCount(String context); + + + /** + * 原子指标引用 + * @param indicatorName + * @return + */ + int sourceAtomicIndicatorReference(String indicatorName); +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-indicator-management/src/main/java/com/webank/wedatasphere/dss/datamodel/indicator/service/impl/DataWarehouseIndicatorReferenceServiceImpl.java b/dss-apps/dss-datamodel-center/dss-datamodel-indicator-management/src/main/java/com/webank/wedatasphere/dss/datamodel/indicator/service/impl/DataWarehouseIndicatorReferenceServiceImpl.java new file mode 100644 index 0000000000..e762d51621 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-indicator-management/src/main/java/com/webank/wedatasphere/dss/datamodel/indicator/service/impl/DataWarehouseIndicatorReferenceServiceImpl.java @@ -0,0 +1,35 @@ +package com.webank.wedatasphere.dss.datamodel.indicator.service.impl; + +import com.webank.wedatasphere.dss.datamodel.center.common.service.DataWarehouseIndicatorReferenceService; +import com.webank.wedatasphere.dss.datamodel.indicator.service.IndicatorService; +import org.springframework.stereotype.Service; + +import javax.annotation.Resource; + +@Service +public class DataWarehouseIndicatorReferenceServiceImpl implements DataWarehouseIndicatorReferenceService { + + @Resource + private IndicatorService indicatorService; + + + @Override + public int indicatorCycleReferenceCount(String name) { + return indicatorService.indicatorCycleReferenceCount(name); + } + + @Override + public int indicatorLayerReferenceCount(String name) { + return indicatorService.indicatorLayerReferenceCount(name); + } + + @Override + public int indicatorModifierReferenceCount(String name) { + return indicatorService.indicatorModifierReferenceCount(name); + } + + @Override + public int indicatorThemeReferenceCount(String name) { + return indicatorService.indicatorThemeReferenceCount(name); + } +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-indicator-management/src/main/java/com/webank/wedatasphere/dss/datamodel/indicator/service/impl/DatamodelIndicatorReferenceServiceImpl.java b/dss-apps/dss-datamodel-center/dss-datamodel-indicator-management/src/main/java/com/webank/wedatasphere/dss/datamodel/indicator/service/impl/DatamodelIndicatorReferenceServiceImpl.java new file mode 100644 index 0000000000..e847a7ea57 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-indicator-management/src/main/java/com/webank/wedatasphere/dss/datamodel/indicator/service/impl/DatamodelIndicatorReferenceServiceImpl.java @@ -0,0 +1,40 @@ +package com.webank.wedatasphere.dss.datamodel.indicator.service.impl; + + +import com.baomidou.mybatisplus.core.toolkit.Wrappers; +import com.webank.wedatasphere.dss.datamodel.center.common.service.DatamodelIndicatorReferenceService; +import com.webank.wedatasphere.dss.datamodel.indicator.entity.DssDatamodelIndicatorContent; +import com.webank.wedatasphere.dss.datamodel.indicator.service.IndicatorContentService; +import com.webank.wedatasphere.dss.datamodel.indicator.service.IndicatorService; +import com.webank.wedatasphere.dss.datamodel.indicator.service.IndicatorVersionService; +import org.springframework.stereotype.Service; + +import javax.annotation.Resource; + +@Service +public class DatamodelIndicatorReferenceServiceImpl implements DatamodelIndicatorReferenceService { + + @Resource + public IndicatorService indicatorService; + + @Resource + public IndicatorContentService indicatorContentService; + + @Resource + public IndicatorVersionService indicatorVersionService; + + @Override + public int indicatorDimensionReferenceCount(String name) { + return indicatorService.indicatorDimensionReferenceCount(name); + } + + @Override + public int indicatorMeasuredReference(String name) { + return indicatorService.indicatorMeasureReferenceCount(name); + } + + @Override + public int indicatorIndicatorCount(String name) { + return indicatorService.indicatorIndicatorReferenceCount(name); + } +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-indicator-management/src/main/java/com/webank/wedatasphere/dss/datamodel/indicator/service/impl/IndicatorContentServiceImpl.java b/dss-apps/dss-datamodel-center/dss-datamodel-indicator-management/src/main/java/com/webank/wedatasphere/dss/datamodel/indicator/service/impl/IndicatorContentServiceImpl.java new file mode 100644 index 0000000000..2c07e78f8f --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-indicator-management/src/main/java/com/webank/wedatasphere/dss/datamodel/indicator/service/impl/IndicatorContentServiceImpl.java @@ -0,0 +1,88 @@ +package com.webank.wedatasphere.dss.datamodel.indicator.service.impl; + +import com.baomidou.mybatisplus.core.toolkit.Wrappers; +import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl; +import com.google.gson.Gson; +import com.webank.wedatasphere.dss.datamodel.center.common.constant.ErrorCode; +import com.webank.wedatasphere.dss.datamodel.center.common.exception.DSSDatamodelCenterException; +import com.webank.wedatasphere.dss.datamodel.indicator.dao.DssDatamodelIndicatorContentMapper; +import com.webank.wedatasphere.dss.datamodel.indicator.entity.DssDatamodelIndicatorContent; +import com.webank.wedatasphere.dss.datamodel.indicator.service.IndicatorContentService; +import com.webank.wedatasphere.dss.datamodel.indicator.vo.IndicatorContentVO; +import org.apache.linkis.common.exception.ErrorException; +import org.modelmapper.ModelMapper; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +import javax.annotation.Resource; +import java.util.Date; +import java.util.List; + + +@Service +public class IndicatorContentServiceImpl extends ServiceImpl implements IndicatorContentService { + + private final Gson gson = new Gson(); + + @Resource + private ModelMapper modelMapper; + + + @Override + @Transactional(rollbackFor = Exception.class) + public int addIndicatorContent(Long indicatorId, String version, IndicatorContentVO vo) throws ErrorException { + + DssDatamodelIndicatorContent newOne = modelMapper.map(vo, DssDatamodelIndicatorContent.class); + newOne.setVersion(version); + newOne.setIndicatorId(indicatorId); + //todo 检查内容 + newOne.setIndicatorSourceInfo(gson.toJson(vo.getSourceInfo())); + newOne.setCreateTime(new Date()); + newOne.setUpdateTime(new Date()); + if (!save(newOne)) { + throw new DSSDatamodelCenterException(ErrorCode.INDICATOR_CONTENT_ADD_ERROR.getCode(), "add indicator content error"); + } + return 1; + } + + + @Override + @Transactional(rollbackFor = Exception.class) + public int updateIndicatorContent(Long indicateId, String version, IndicatorContentVO vo) throws ErrorException { + //删除原有详细信息 + getBaseMapper().delete(Wrappers.lambdaQuery().eq(DssDatamodelIndicatorContent::getIndicatorId, indicateId)); + //新增一条 + return addIndicatorContent(indicateId, version, vo); + } + + + @Override + public DssDatamodelIndicatorContent queryByIndicateId(Long indicateId) { + return getBaseMapper() + .selectOne(Wrappers.lambdaQuery().eq(DssDatamodelIndicatorContent::getIndicatorId, indicateId)); + } + + + @Override + public int deleteByIndicatorId(Long id) throws ErrorException { + return getBaseMapper().delete(Wrappers.lambdaQuery().eq(DssDatamodelIndicatorContent::getIndicatorId, id)); + } + + + @Override + public List sourceInfoReference(String name) { + + return getBaseMapper().selectList( + Wrappers.lambdaQuery().like(DssDatamodelIndicatorContent::getIndicatorSourceInfo, name)); +// .like(DssDatamodelIndicatorContent::getIndicatorSourceInfo,"\""+ name + "\"") +// .or() +// .like(DssDatamodelIndicatorContent::getIndicatorSourceInfo,"\""+ name + ",") +// .or() +// .like(DssDatamodelIndicatorContent::getIndicatorSourceInfo,","+ name + ",") +// .or() +// .like(DssDatamodelIndicatorContent::getIndicatorSourceInfo,","+ name + "\"")); + } + + + +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-indicator-management/src/main/java/com/webank/wedatasphere/dss/datamodel/indicator/service/impl/IndicatorServiceImpl.java b/dss-apps/dss-datamodel-center/dss-datamodel-indicator-management/src/main/java/com/webank/wedatasphere/dss/datamodel/indicator/service/impl/IndicatorServiceImpl.java new file mode 100644 index 0000000000..d148ea619b --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-indicator-management/src/main/java/com/webank/wedatasphere/dss/datamodel/indicator/service/impl/IndicatorServiceImpl.java @@ -0,0 +1,539 @@ +package com.webank.wedatasphere.dss.datamodel.indicator.service.impl; + +import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; +import com.baomidou.mybatisplus.core.toolkit.Wrappers; +import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl; +import com.github.pagehelper.PageHelper; +import com.github.pagehelper.PageInfo; +import com.google.common.collect.Sets; +import com.google.gson.Gson; +import com.webank.wedatasphere.dss.data.governance.entity.ClassificationConstant; +import com.webank.wedatasphere.dss.data.governance.impl.LinkisDataAssetsRemoteClient; +import com.webank.wedatasphere.dss.data.governance.response.CreateModelTypeResult; +import com.webank.wedatasphere.dss.data.governance.response.UpdateModelTypeResult; +import com.webank.wedatasphere.dss.datamodel.center.common.constant.ErrorCode; +import com.webank.wedatasphere.dss.datamodel.center.common.constant.IndicatorSourceInfoConstant; +import com.webank.wedatasphere.dss.datamodel.center.common.context.DataModelSecurityContextHolder; +import com.webank.wedatasphere.dss.datamodel.center.common.event.CreateModelEvent; +import com.webank.wedatasphere.dss.datamodel.center.common.event.DeleteModelEvent; +import com.webank.wedatasphere.dss.datamodel.center.common.event.UpdateModelEvent; +import com.webank.wedatasphere.dss.datamodel.center.common.exception.DSSDatamodelCenterException; +import com.webank.wedatasphere.dss.datamodel.center.common.service.AssertsSyncService; +import com.webank.wedatasphere.dss.datamodel.center.common.service.DatamodelReferencService; +import com.webank.wedatasphere.dss.datamodel.indicator.dao.DssDatamodelIndicatorMapper; +import com.webank.wedatasphere.dss.datamodel.indicator.dao.IndicatorQueryMapper; +import com.webank.wedatasphere.dss.datamodel.indicator.dto.IndicatorContentQueryDTO; +import com.webank.wedatasphere.dss.datamodel.indicator.dto.IndicatorQueryDTO; +import com.webank.wedatasphere.dss.datamodel.indicator.dto.IndicatorVersionDTO; +import com.webank.wedatasphere.dss.datamodel.indicator.dto.IndicatorVersionQueryDTO; +import com.webank.wedatasphere.dss.datamodel.indicator.entity.DssDatamodelIndicator; +import com.webank.wedatasphere.dss.datamodel.indicator.entity.DssDatamodelIndicatorContent; +import com.webank.wedatasphere.dss.datamodel.indicator.entity.DssDatamodelIndicatorQuery; +import com.webank.wedatasphere.dss.datamodel.indicator.entity.DssDatamodelIndicatorVersion; +import com.webank.wedatasphere.dss.datamodel.indicator.service.IndicatorContentService; +import com.webank.wedatasphere.dss.datamodel.indicator.service.IndicatorService; +import com.webank.wedatasphere.dss.datamodel.indicator.service.IndicatorVersionService; +import com.webank.wedatasphere.dss.datamodel.indicator.vo.*; +import org.apache.linkis.common.exception.ErrorException; +import org.apache.linkis.server.Message; +import org.apache.commons.lang.StringUtils; +import org.modelmapper.ModelMapper; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +import javax.annotation.Resource; +import java.util.Date; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + + +@Service +public class IndicatorServiceImpl extends ServiceImpl implements IndicatorService { + + private static final Logger LOGGER = LoggerFactory.getLogger(IndicatorServiceImpl.class); + + + private final Gson gson = new Gson(); + + @Resource + private ModelMapper modelMapper; + + @Resource + private IndicatorContentService indicatorContentService; + + @Resource + private IndicatorVersionService indicatorVersionService; + + @Resource + private IndicatorQueryMapper indicatorQueryMapper; + + + @Resource + private LinkisDataAssetsRemoteClient linkisDataAssetsRemoteClient; + + @Resource + private AssertsSyncService assertsSyncService; + + + @Resource + private DatamodelReferencService datamodelReferencService; + + @Override + @Transactional(rollbackFor = Exception.class) + public Long addIndicator(IndicatorAddVO vo, String version) throws ErrorException { + if (getBaseMapper().selectCount(Wrappers.lambdaQuery().eq(DssDatamodelIndicator::getName, vo.getName()) + .or().eq(DssDatamodelIndicator::getFieldIdentifier, vo.getFieldIdentifier())) > 0) { + LOGGER.error("errorCode : {}, indicator name or field identifier can not repeat, name : {}", ErrorCode.INDICATOR_ADD_ERROR.getCode(), vo.getName()); + throw new DSSDatamodelCenterException(ErrorCode.INDICATOR_ADD_ERROR.getCode(), "indicator name or field identifier can not repeat"); + } + //校验引用情况 + if (datamodelReferencService.indicatorReferenceCount(vo.getName())>0 + ||datamodelReferencService.indicatorReferenceCount(vo.getFieldIdentifier())>0){ + LOGGER.error("errorCode : {}, indicator name can not referenced", ErrorCode.INDICATOR_ADD_ERROR.getCode()); + throw new DSSDatamodelCenterException(ErrorCode.INDICATOR_ADD_ERROR.getCode(), "indicator name can not referenced"); + } + + + DssDatamodelIndicator newOne = modelMapper.map(vo, DssDatamodelIndicator.class); + //基本信息中设置版本号 + newOne.setVersion(version); + newOne.setCreateTime(new Date()); + newOne.setUpdateTime(new Date()); + if (!save(newOne)) { + LOGGER.error("errorCode : {}, add indicator error", ErrorCode.INDICATOR_ADD_ERROR.getCode()); + throw new DSSDatamodelCenterException(ErrorCode.INDICATOR_ADD_ERROR.getCode(), "add indicator error"); + } + + indicatorContentService.addIndicatorContent(newOne.getId(), version, vo.getContent()); + + //同步atlas + CreateModelTypeResult result = assertsSyncService.syncCreateModel( + new CreateModelEvent(this + , DataModelSecurityContextHolder.getContext().getDataModelAuthentication().getUser() + , vo.getFieldIdentifier() + , ClassificationConstant.INDICATOR)); + return newOne.getId(); + } + + + @Override + @Transactional(rollbackFor = Exception.class) + public int updateIndicator(Long id, IndicatorUpdateVO vo) throws ErrorException { + DssDatamodelIndicator org = getBaseMapper().selectById(id); + if (org == null) { + LOGGER.error("errorCode : {}, update indicator error not exists", ErrorCode.INDICATOR_UPDATE_ERROR.getCode()); + throw new DSSDatamodelCenterException(ErrorCode.INDICATOR_UPDATE_ERROR.getCode(), "update indicator error not exists"); + } + + //校验引用情况 + if (datamodelReferencService.indicatorReferenceCount(org.getName())>0 + ||datamodelReferencService.indicatorReferenceCount(vo.getName())>0 + ||datamodelReferencService.indicatorReferenceCount(org.getFieldIdentifier())>0 + ||datamodelReferencService.indicatorReferenceCount(vo.getFieldIdentifier())>0){ + LOGGER.error("errorCode : {}, indicator name can not referenced", ErrorCode.INDICATOR_UPDATE_ERROR.getCode()); + throw new DSSDatamodelCenterException(ErrorCode.INDICATOR_UPDATE_ERROR.getCode(), "indicator name can not referenced"); + } + + + DssDatamodelIndicatorContent orgContent = indicatorContentService.queryByIndicateId(id); + + + //当更新指标名称时,存在其他指标名称同名或者当前指标名称已经存在版本信息，则不允许修改指标名称 + if (!StringUtils.equals(vo.getName(), org.getName())) { + int repeat = getBaseMapper().selectCount(Wrappers.lambdaQuery().eq(DssDatamodelIndicator::getName, vo.getName())); + String lastVersion = indicatorVersionService.findLastVersion(org.getName()); + if (repeat > 0 || StringUtils.isNotBlank(lastVersion)) { + LOGGER.error("errorCode : {}, indicator name can not repeat or has version", ErrorCode.INDICATOR_UPDATE_ERROR.getCode()); + throw new DSSDatamodelCenterException(ErrorCode.INDICATOR_UPDATE_ERROR.getCode(), "indicator name can not repeat or has version"); + } + } + + String orgFieldIdentifier = org.getFieldIdentifier(); + //当更新指标标识时,存在其他指标名称同名或者当前指标名称已经存在版本信息，则不允许修改指标名称 + if (!StringUtils.equals(vo.getFieldIdentifier(), orgFieldIdentifier)) { + int repeat = getBaseMapper().selectCount(Wrappers.lambdaQuery().eq(DssDatamodelIndicator::getFieldIdentifier, vo.getFieldIdentifier())); + if (repeat > 0) { + LOGGER.error("errorCode : {}, indicator field identifier can not repeat or has version", ErrorCode.INDICATOR_UPDATE_ERROR.getCode()); + throw new DSSDatamodelCenterException(ErrorCode.INDICATOR_UPDATE_ERROR.getCode(), "indicator field identifier can not repeat or has version"); + } + } + + + Long orgId = org.getId(); + String version = org.getVersion(); + + //更新原有指标 + DssDatamodelIndicator updateOne = modelMapper.map(vo, DssDatamodelIndicator.class); + updateOne.setUpdateTime(new Date()); + int result = getBaseMapper().update(updateOne, Wrappers.lambdaUpdate().eq(DssDatamodelIndicator::getId, id)); + if (result != 1) { + LOGGER.error("errorCode : {}, update indicator error not exists", ErrorCode.INDICATOR_UPDATE_ERROR.getCode()); + throw new DSSDatamodelCenterException(ErrorCode.INDICATOR_UPDATE_ERROR.getCode(), "update indicator error not exists"); + } + + indicatorContentService.updateIndicatorContent(orgId, version, vo.getContent()); + + + + //同步atlas + UpdateModelTypeResult updateModelTypeResult = assertsSyncService.syncUpdateModel( + new UpdateModelEvent(this + ,DataModelSecurityContextHolder.getContext().getDataModelAuthentication().getUser() + ,vo.getFieldIdentifier() + ,orgFieldIdentifier + ,ClassificationConstant.INDICATOR) + ); + + + return 1; + } + + + @Override + public int deleteIndicator(Long id) throws ErrorException { + DssDatamodelIndicator dssDatamodelIndicator = getBaseMapper().selectById(id); + if (dssDatamodelIndicator == null) { + throw new DSSDatamodelCenterException(ErrorCode.INDICATOR_DELETE_ERROR.getCode(), "indicator id " + id + " not exists"); + } + + //校验引用情况 + if (datamodelReferencService.indicatorReferenceCount(dssDatamodelIndicator.getName())>0 + ||datamodelReferencService.indicatorReferenceCount(dssDatamodelIndicator.getFieldIdentifier())>0){ + throw new DSSDatamodelCenterException(ErrorCode.INDICATOR_DELETE_ERROR.getCode(), "indicator id " + id + " has referenced"); + } + + +// //有版本不能删除 +// if (indicatorVersionService.getBaseMapper().selectCount(Wrappers.lambdaQuery().eq(DssDatamodelIndicatorVersion::getName, dssDatamodelIndicator.getName())) > 0) { +// throw new DSSDatamodelCenterException(ErrorCode.INDICATOR_DELETE_ERROR.getCode(), "indicator id " + id + " has version"); +// } + //同时删除版本 + indicatorVersionService.getBaseMapper() + .delete(Wrappers.lambdaQuery().eq(DssDatamodelIndicatorVersion::getName, dssDatamodelIndicator.getName())); + //删除指标 + getBaseMapper().deleteById(id); + indicatorContentService.getBaseMapper().delete(Wrappers.lambdaQuery().eq(DssDatamodelIndicatorContent::getIndicatorId, id)); + + + //同步资产 + assertsSyncService.syncDeleteModel(new DeleteModelEvent(this + ,DataModelSecurityContextHolder.getContext().getDataModelAuthentication().getUser() + ,dssDatamodelIndicator.getFieldIdentifier() + ,ClassificationConstant.INDICATOR)); + + return 1; + } + + + @Override + public Message listIndicators(IndicatorQueryVO vo) { + QueryWrapper queryWrapper = new QueryWrapper() + .like(StringUtils.isNotBlank(vo.getName()), "name", vo.getName()) + .eq(vo.getIsAvailable() != null, "is_available", vo.getIsAvailable()) + .eq(vo.getIndicatorType() != null, "indicator_type", vo.getIndicatorType()) + .eq(StringUtils.isNotBlank(vo.getWarehouseThemeName()), "theme_area", vo.getWarehouseThemeName()) + .like(StringUtils.isNotBlank(vo.getOwner()), "owner", vo.getOwner()); + PageHelper.clearPage(); + PageHelper.startPage(vo.getPageNum(), vo.getPageSize()); + PageInfo pageInfo = new PageInfo<>(indicatorQueryMapper.page(queryWrapper)); + //IPage iPage = indicatorQueryMapper.page(new Page<>(vo.getPageNum(), vo.getPageSize()), queryWrapper); + + return Message.ok() + .data("list", pageInfo.getList().stream().peek(e-> e.setRefCount(datamodelReferencService.indicatorReferenceCount(e.getName()))).collect(Collectors.toList())) + .data("total", pageInfo.getTotal()); + } + + + @Override + public Message queryById(Long id) throws DSSDatamodelCenterException { + DssDatamodelIndicator indicator = baseMapper.selectById(id); + if (indicator == null) { + LOGGER.error("errorCode : {}, indicator id : {} not exists", ErrorCode.INDICATOR_QUERY_ERROR.getCode(), id); + throw new DSSDatamodelCenterException(ErrorCode.INDICATOR_QUERY_ERROR.getCode(), "indicator id " + id + " not exists"); + } + + DssDatamodelIndicatorContent content = indicatorContentService.queryByIndicateId(id); + if (content == null) { + LOGGER.error("errorCode : {}, indicator content indicatorId : {} not exists", ErrorCode.INDICATOR_QUERY_ERROR.getCode(), id); + throw new DSSDatamodelCenterException(ErrorCode.INDICATOR_QUERY_ERROR.getCode(), "indicator content id " + id + " not exists"); + } + + IndicatorQueryDTO indicatorQueryDTO = modelMapper.map(indicator, IndicatorQueryDTO.class); + IndicatorContentQueryDTO indicatorContentQueryDTO = modelMapper.map(content, IndicatorContentQueryDTO.class); + indicatorQueryDTO.setContent(indicatorContentQueryDTO); + indicatorQueryDTO.setRefCount(datamodelReferencService.indicatorReferenceCount(indicator.getName())); + return Message.ok().data("detail", indicatorQueryDTO); + } + + + @Override + @Transactional(rollbackFor = Exception.class) + public int enableIndicator(Long id, IndicatorEnableVO vo) { + DssDatamodelIndicator updateOne = modelMapper.map(vo, DssDatamodelIndicator.class); + updateOne.setUpdateTime(new Date()); + return getBaseMapper().update(updateOne, Wrappers.lambdaUpdate().eq(DssDatamodelIndicator::getId, id)); + } + + + @Override + @Transactional(rollbackFor = Exception.class) + public int addIndicatorVersion(Long id, IndicatorVersionAddVO vo) throws ErrorException { + //todo 生成新版本 则旧版本是否判断存在 关联其他表信息 + + //判断旧版本指标是否存在 + DssDatamodelIndicator orgVersion = getBaseMapper().selectById(id); + if (orgVersion == null) { + LOGGER.error("errorCode : {}, indicator id : {} not exists", ErrorCode.INDICATOR_VERSION_ADD_ERROR.getCode(), id); + throw new DSSDatamodelCenterException(ErrorCode.INDICATOR_VERSION_ADD_ERROR.getCode(), "indicator id " + id + " not exists"); + } + + //校验引用情况 + if (datamodelReferencService.indicatorReferenceCount(orgVersion.getName())>0 + ||datamodelReferencService.indicatorReferenceCount(vo.getName())>0 + ||datamodelReferencService.indicatorReferenceCount(orgVersion.getFieldIdentifier())>0 + ||datamodelReferencService.indicatorReferenceCount(vo.getFieldIdentifier())>0){ + LOGGER.error("errorCode : {}, indicator name can not referenced", ErrorCode.INDICATOR_VERSION_ADD_ERROR.getCode()); + throw new DSSDatamodelCenterException(ErrorCode.INDICATOR_VERSION_ADD_ERROR.getCode(), "indicator name can not referenced"); + } + + + + + String orgName = orgVersion.getName(); + String orgNameEn = orgVersion.getFieldIdentifier(); + String orgV = orgVersion.getVersion(); + String assignVersion = newVersion(orgName, orgV); + + //指标名称不能改变 + if (!StringUtils.equals(orgName, vo.getName()) || !StringUtils.equals(orgNameEn, vo.getFieldIdentifier())) { + LOGGER.error("errorCode : {}, indicator name or fieldIdentifier must be same", ErrorCode.INDICATOR_VERSION_ADD_ERROR.getCode()); + throw new DSSDatamodelCenterException(ErrorCode.INDICATOR_VERSION_ADD_ERROR.getCode(), "indicator name or fieldIdentifier must be same"); + } + + //查询指标详细信息 + DssDatamodelIndicatorContent orgContent = indicatorContentService.queryByIndicateId(id); + if (orgContent == null) { + LOGGER.error("errorCode : {}, indicator content indicatorId : {} not exists", ErrorCode.INDICATOR_VERSION_ADD_ERROR.getCode(), id); + throw new DSSDatamodelCenterException(ErrorCode.INDICATOR_VERSION_ADD_ERROR.getCode(), "indicator content id " + id + " not exists"); + } + + //旧版本备份 + indicatorVersionService.addOlderVersion(orgName, orgVersion.getOwner(), orgVersion.getPrincipalName(), orgV, orgVersion.getComment(), IndicatorVersionDTO.of(orgVersion, orgContent)); + + + //删除旧版本 + getBaseMapper().deleteById(id); + indicatorContentService.deleteByIndicatorId(id); + + //填充新增版本 + DssDatamodelIndicator newOne = modelMapper.map(vo, DssDatamodelIndicator.class); + //基本信息中设置版本号,如果新增时未指定版本号则生成新版本号 + String newVersion = assignVersion; + newOne.setVersion(newVersion); + newOne.setCreateTime(new Date()); + newOne.setUpdateTime(new Date()); + //保存新版本 + if (!save(newOne)) { + throw new DSSDatamodelCenterException(ErrorCode.INDICATOR_VERSION_ADD_ERROR.getCode(), "save new version indicator error"); + + } + //更新指标详细信息 + indicatorContentService.updateIndicatorContent(newOne.getId(), newVersion, vo.getContent()); + + return 1; + } + + + private String newVersion(String orgName, String orgVersion) { + String lastVersion = indicatorVersionService.findLastVersion(orgName); + + return StringUtils.isBlank(lastVersion) ? + (Integer.parseInt(orgVersion) + 1 + "") : + (Integer.parseInt(lastVersion) > Integer.parseInt(orgVersion)) ? (Integer.parseInt(lastVersion) + 1 + "") : + (Integer.parseInt(orgVersion) + 1 + ""); + } + + + @Override + @Transactional(rollbackFor = Exception.class) + public int versionRollBack(IndicatorVersionRollBackVO vo) throws ErrorException { + + String name = vo.getName(); + String version = vo.getVersion(); + + //获取当前使用版本指标 + DssDatamodelIndicator current = + getBaseMapper().selectOne(Wrappers.lambdaQuery().eq(DssDatamodelIndicator::getName, name)); + //判断当前使用指标 + if (current == null) { + LOGGER.error("errorCode : {}, current indicator not exists, name : {} version : {}", ErrorCode.INDICATOR_VERSION_ROLL_BACK_ERROR.getCode(), name, version); + throw new DSSDatamodelCenterException(ErrorCode.INDICATOR_VERSION_ROLL_BACK_ERROR.getCode(), "current indicator not exists, name : " + name + " version : " + version); + } + + //校验引用情况 + if (datamodelReferencService.indicatorReferenceCount(current.getName())>0 + ||datamodelReferencService.indicatorReferenceCount(current.getFieldIdentifier())>0){ + throw new DSSDatamodelCenterException(ErrorCode.INDICATOR_VERSION_ROLL_BACK_ERROR.getCode(), "indicator id " + current.getId() + " has referenced"); + } + + + //查询当前详细信息 + DssDatamodelIndicatorContent currentContent = indicatorContentService.queryByIndicateId(current.getId()); + if (currentContent == null) { + LOGGER.error("errorCode : {}, current indicator content not exists, name : {} version : {}", ErrorCode.INDICATOR_VERSION_ROLL_BACK_ERROR.getCode(), name, version); + throw new DSSDatamodelCenterException(ErrorCode.INDICATOR_VERSION_ROLL_BACK_ERROR.getCode(), "current indicator content not exists, name : " + name + " version : " + version); + } + //当前版本备份 + indicatorVersionService.addOlderVersion(current.getName(), current.getOwner(), current.getPrincipalName(), current.getVersion(), current.getComment(), IndicatorVersionDTO.of(current, currentContent)); + //删除当前版本 + getBaseMapper().deleteById(current.getId()); + //删除当前版本详情 + indicatorContentService.deleteByIndicatorId(current.getId()); + + //获取需要回退指标版本信息 + DssDatamodelIndicatorVersion rollBackVersion = indicatorVersionService.findBackup(name, version); + if (rollBackVersion == null) { + LOGGER.error("errorCode : {}, indicator name : {} version : {}", ErrorCode.INDICATOR_VERSION_ROLL_BACK_ERROR.getCode(), name, version); + throw new DSSDatamodelCenterException(ErrorCode.INDICATOR_VERSION_ROLL_BACK_ERROR.getCode(), "indicator name : " + name + " version : " + version + " not exist"); + } + Long rollBackId = rollBackVersion.getId(); + + IndicatorVersionDTO rollBackDTO = gson.fromJson(rollBackVersion.getVersionContext(), IndicatorVersionDTO.class); + DssDatamodelIndicator rollBackIndicator = rollBackDTO.getEssential(); + //重置id + rollBackIndicator.setId(null); + save(rollBackIndicator); + + + DssDatamodelIndicatorContent rollBackContent = rollBackDTO.getContent(); + //重置id + rollBackContent.setId(null); + //设置新指标id + rollBackContent.setIndicatorId(rollBackIndicator.getId()); + indicatorContentService.save(rollBackContent); + indicatorVersionService.getBaseMapper().deleteById(rollBackId); + return 1; + } + + + @Override + public Message listIndicatorVersions(IndicatorVersionQueryVO vo) { + List list = indicatorVersionService.getBaseMapper() + .selectList(Wrappers.lambdaQuery().eq(DssDatamodelIndicatorVersion::getName, vo.getName())) + .stream() + .map(dssDatamodelIndicatorVersion -> modelMapper.map(dssDatamodelIndicatorVersion, IndicatorVersionQueryDTO.class)) + .collect(Collectors.toList()); + return Message.ok().data("list", list); + } + + + @Override + public int indicatorThemeReferenceCount(String name) { + int currentCount = getBaseMapper().selectCount(Wrappers.lambdaQuery().eq(DssDatamodelIndicator::getThemeArea, name)); + int currentCountEn = getBaseMapper().selectCount(Wrappers.lambdaQuery().eq(DssDatamodelIndicator::getThemeAreaEn, name)); + List preReferences = indicatorVersionService.contentReferenceCount(name); + int versionCount = (int) preReferences.stream().filter(e->{ + IndicatorVersionDTO rollBackDTO = gson.fromJson(e.getVersionContext(), IndicatorVersionDTO.class); + DssDatamodelIndicator rollBackIndicator = rollBackDTO.getEssential(); + return StringUtils.equals(rollBackIndicator.getThemeArea(),name)||StringUtils.equals(rollBackIndicator.getThemeAreaEn(),name); + }).count(); + return currentCount + versionCount + currentCountEn; + } + + @Override + public int indicatorLayerReferenceCount(String name) { + int currentCount = getBaseMapper().selectCount(Wrappers.lambdaQuery().eq(DssDatamodelIndicator::getLayerArea, name)); + int currentCountEn = getBaseMapper().selectCount(Wrappers.lambdaQuery().eq(DssDatamodelIndicator::getLayerAreaEn, name)); + List preReferences = indicatorVersionService.contentReferenceCount(name); + int versionCount = (int) preReferences.stream().filter(e->{ + IndicatorVersionDTO rollBackDTO = gson.fromJson(e.getVersionContext(), IndicatorVersionDTO.class); + DssDatamodelIndicator rollBackIndicator = rollBackDTO.getEssential(); + return StringUtils.equals(rollBackIndicator.getLayerArea(),name)||StringUtils.equals(rollBackIndicator.getLayerAreaEn(),name); + }).count(); + return currentCount + versionCount + currentCountEn; + } + + @Override + public int indicatorCycleReferenceCount(String name) { + List currentContents = indicatorContentService.sourceInfoReference(name); + int currentCount = currentCountFromIndicatorContentSourceInfo(IndicatorSourceInfoConstant.cycle,name,currentContents); + int currentCountEn = currentCountFromIndicatorContentSourceInfo(IndicatorSourceInfoConstant.cycleEn,name,currentContents); + List preReferences = indicatorVersionService.contentReferenceCount(name); + int versionCount = versionCountFromIndicatorContentSourceInfo(preReferences,IndicatorSourceInfoConstant.cycle.name(),name); + int versionCountEn = versionCountFromIndicatorContentSourceInfo(preReferences,IndicatorSourceInfoConstant.cycleEn.name(),name); + return currentCount + currentCountEn + versionCount + versionCountEn; + } + + private int versionCountFromIndicatorContentSourceInfo(List preReferences,String type,String name){ + return (int) preReferences.stream().filter(e->{ + IndicatorVersionDTO rollBackDTO = gson.fromJson(e.getVersionContext(), IndicatorVersionDTO.class); + DssDatamodelIndicatorContent rollBackContent = rollBackDTO.getContent(); + return filterFromIndicatorContentSourceInfo(type, name, rollBackContent); + }).count(); + } + + private int currentCountFromIndicatorContentSourceInfo(IndicatorSourceInfoConstant type, String name,List contents){ + return (int) contents.stream().filter(e-> filterFromIndicatorContentSourceInfo(type.name(),name,e)).count(); + } + + private boolean filterFromIndicatorContentSourceInfo(String type, String name, DssDatamodelIndicatorContent rollBackContent) { + Map content = gson.fromJson(rollBackContent.getIndicatorSourceInfo(),Map.class); + if (!content.containsKey(type)){ + return false; + } + return Sets.newHashSet(StringUtils.split(content.get(type), ",")).contains(name); + } + + @Override + public int indicatorModifierReferenceCount(String name) { + List currentContents = indicatorContentService.sourceInfoReference(name); + int currentCount = currentCountFromIndicatorContentSourceInfo(IndicatorSourceInfoConstant.modifier,name,currentContents); + int currentCountEn = currentCountFromIndicatorContentSourceInfo(IndicatorSourceInfoConstant.modifierEn,name,currentContents); + List preReferences = indicatorVersionService.contentReferenceCount(name); + int versionCount = versionCountFromIndicatorContentSourceInfo(preReferences,IndicatorSourceInfoConstant.modifier.name(),name); + int versionCountEn = versionCountFromIndicatorContentSourceInfo(preReferences,IndicatorSourceInfoConstant.modifierEn.name(),name); + return currentCount + currentCountEn + versionCount + versionCountEn; + } + + + @Override + public int indicatorDimensionReferenceCount(String name) { + List currentContents = indicatorContentService.sourceInfoReference(name); + int currentCount = currentCountFromIndicatorContentSourceInfo(IndicatorSourceInfoConstant.dimension,name,currentContents); + int currentCountEn = currentCountFromIndicatorContentSourceInfo(IndicatorSourceInfoConstant.dimensionEn,name,currentContents); + + List preReferences = indicatorVersionService.contentReferenceCount(name); + int versionCount = versionCountFromIndicatorContentSourceInfo(preReferences,IndicatorSourceInfoConstant.dimension.name(),name); + int versionCountEn = versionCountFromIndicatorContentSourceInfo(preReferences,IndicatorSourceInfoConstant.dimensionEn.name(),name); + return currentCount + currentCountEn + versionCount + versionCountEn; + } + + @Override + public int indicatorMeasureReferenceCount(String name) { + List currentContents = indicatorContentService.sourceInfoReference(name); + int currentCount = currentCountFromIndicatorContentSourceInfo(IndicatorSourceInfoConstant.measure,name,currentContents); + int currentCountEn = currentCountFromIndicatorContentSourceInfo(IndicatorSourceInfoConstant.measureEn,name,currentContents); + + List preReferences = indicatorVersionService.contentReferenceCount(name); + int versionCount = versionCountFromIndicatorContentSourceInfo(preReferences,IndicatorSourceInfoConstant.measure.name(),name); + int versionCountEn = versionCountFromIndicatorContentSourceInfo(preReferences,IndicatorSourceInfoConstant.measureEn.name(), name); + return currentCount + currentCountEn + versionCount + versionCountEn; + } + + @Override + public int indicatorIndicatorReferenceCount(String name) { + List currentContents = indicatorContentService.sourceInfoReference(name); + int currentCount = currentCountFromIndicatorContentSourceInfo(IndicatorSourceInfoConstant.indicatorName,name,currentContents); + int currentCountEn = currentCountFromIndicatorContentSourceInfo(IndicatorSourceInfoConstant.indicatorNameEn,name,currentContents); + + List preReferences = indicatorVersionService.contentReferenceCount(name); + int versionCount = versionCountFromIndicatorContentSourceInfo(preReferences,IndicatorSourceInfoConstant.indicatorName.name(),name); + int versionCountEn = versionCountFromIndicatorContentSourceInfo(preReferences,IndicatorSourceInfoConstant.indicatorNameEn.name(),name); + return currentCount + currentCountEn + versionCount + versionCountEn; + } + + +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-indicator-management/src/main/java/com/webank/wedatasphere/dss/datamodel/indicator/service/impl/IndicatorVersionServiceImpl.java b/dss-apps/dss-datamodel-center/dss-datamodel-indicator-management/src/main/java/com/webank/wedatasphere/dss/datamodel/indicator/service/impl/IndicatorVersionServiceImpl.java new file mode 100644 index 0000000000..c6d6ee2853 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-indicator-management/src/main/java/com/webank/wedatasphere/dss/datamodel/indicator/service/impl/IndicatorVersionServiceImpl.java @@ -0,0 +1,97 @@ +package com.webank.wedatasphere.dss.datamodel.indicator.service.impl; + +import com.baomidou.mybatisplus.core.toolkit.Wrappers; +import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl; +import com.github.pagehelper.PageHelper; +import com.github.pagehelper.PageInfo; +import com.google.gson.Gson; +import com.webank.wedatasphere.dss.datamodel.center.common.constant.ErrorCode; +import com.webank.wedatasphere.dss.datamodel.center.common.exception.DSSDatamodelCenterException; +import com.webank.wedatasphere.dss.datamodel.indicator.dao.DssDatamodelIndicatorVersionMapper; +import com.webank.wedatasphere.dss.datamodel.indicator.dto.IndicatorVersionDTO; +import com.webank.wedatasphere.dss.datamodel.indicator.entity.DssDatamodelIndicatorContent; +import com.webank.wedatasphere.dss.datamodel.indicator.entity.DssDatamodelIndicatorVersion; +import com.webank.wedatasphere.dss.datamodel.indicator.service.IndicatorVersionService; +import org.apache.linkis.common.exception.ErrorException; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; +import org.springframework.util.CollectionUtils; + +import java.util.Date; +import java.util.List; + + +@Service +public class IndicatorVersionServiceImpl extends ServiceImpl implements IndicatorVersionService { + + private final Gson gson = new Gson(); + + @Override + @Transactional + public int addOlderVersion(String name, String owner, String principalName, String version, String comment, IndicatorVersionDTO versionContext) throws ErrorException { + DssDatamodelIndicatorVersion oldVersion = new DssDatamodelIndicatorVersion(); + oldVersion.setName(name); + oldVersion.setPrincipalName(principalName); + oldVersion.setOwner(owner); + oldVersion.setVersion(version); + oldVersion.setVersionContext(gson.toJson(versionContext)); + oldVersion.setComment(comment); + oldVersion.setCreateTime(new Date()); + oldVersion.setUpdateTime(new Date()); + if (!save(oldVersion)) { + throw new DSSDatamodelCenterException(ErrorCode.INDICATOR_VERSION_ADD_ERROR.getCode(), "old version save error"); + + } + return 1; + } + + + @Override + public String findLastVersion(String name) { + PageHelper.clearPage(); + PageHelper.startPage(1,1); + //查询当前指标名称最大版本 + PageInfo pageInfo = new PageInfo<>(getBaseMapper().selectList( + Wrappers.lambdaQuery() + .eq(DssDatamodelIndicatorVersion::getName, name) + .orderByDesc(DssDatamodelIndicatorVersion::getVersion))); + + List list = pageInfo.getList(); + return !CollectionUtils.isEmpty(list) ? list.get(0).getVersion() : null; + } + + + @Override + public DssDatamodelIndicatorVersion findBackup(String name, String version) { + return getBaseMapper().selectOne( + Wrappers.lambdaQuery() + .eq(DssDatamodelIndicatorVersion::getName, name) + .eq(DssDatamodelIndicatorVersion::getVersion, version)); + } + + + @Override + public List contentReferenceCount(String context) { + return getBaseMapper().selectList( + Wrappers.lambdaQuery() + .like(DssDatamodelIndicatorVersion::getVersionContext,context )); +// .like(DssDatamodelIndicatorVersion::getVersionContext,"\""+ context + "\"") +// .or() +// .like(DssDatamodelIndicatorVersion::getVersionContext,"\""+ context + "\"") +// .or() +// .like(DssDatamodelIndicatorVersion::getVersionContext,"\""+ context + ",") +// .or() +// .like(DssDatamodelIndicatorVersion::getVersionContext,","+ context + ",") +// .or() +// .like(DssDatamodelIndicatorVersion::getVersionContext,","+ context + "\"")); + } + + + @Override + public int sourceAtomicIndicatorReference(String indicatorName) { + return getBaseMapper().selectCount( + Wrappers.lambdaQuery() + .like(DssDatamodelIndicatorVersion::getVersionContext,"\"indicatorName\":\""+ indicatorName + "\"")); + + } +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-indicator-management/src/main/java/com/webank/wedatasphere/dss/datamodel/indicator/vo/CycleVO.java b/dss-apps/dss-datamodel-center/dss-datamodel-indicator-management/src/main/java/com/webank/wedatasphere/dss/datamodel/indicator/vo/CycleVO.java new file mode 100644 index 0000000000..38c6551a93 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-indicator-management/src/main/java/com/webank/wedatasphere/dss/datamodel/indicator/vo/CycleVO.java @@ -0,0 +1,10 @@ +package com.webank.wedatasphere.dss.datamodel.indicator.vo; + +import lombok.Data; + +@Data +public class CycleVO { + private String theme; + + private String layer; +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-indicator-management/src/main/java/com/webank/wedatasphere/dss/datamodel/indicator/vo/IndicatorAddVO.java b/dss-apps/dss-datamodel-center/dss-datamodel-indicator-management/src/main/java/com/webank/wedatasphere/dss/datamodel/indicator/vo/IndicatorAddVO.java new file mode 100644 index 0000000000..9fe86ce72b --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-indicator-management/src/main/java/com/webank/wedatasphere/dss/datamodel/indicator/vo/IndicatorAddVO.java @@ -0,0 +1,54 @@ +package com.webank.wedatasphere.dss.datamodel.indicator.vo; + + +import lombok.Data; +import lombok.ToString; + +@Data +@ToString +public class IndicatorAddVO { + + private String name; + + private String fieldIdentifier; + + private String comment; + + /** + * 数仓主题格式为： theme_domain_name.theme_name + */ + private String warehouseThemeName; + + private String owner; + + /** + * 授权的名字：userName、roleName + */ + private String principalName; + + private Integer isAvailable; + + private Integer isCoreIndicator; + + /** + * 空：代表所有，如果是逗号分隔的字符串则代表对应的theme的names + */ + private String themeArea; + + /** + * 空：代表所有，如果是逗号分隔的字符串则代表对应的layer的names + */ + private String layerArea; + + /** + * 数仓主题英文 + */ + private String warehouseThemeNameEn; + + private String themeAreaEn; + + private String layerAreaEn; + + private IndicatorContentVO content; + +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-indicator-management/src/main/java/com/webank/wedatasphere/dss/datamodel/indicator/vo/IndicatorContentVO.java b/dss-apps/dss-datamodel-center/dss-datamodel-indicator-management/src/main/java/com/webank/wedatasphere/dss/datamodel/indicator/vo/IndicatorContentVO.java new file mode 100644 index 0000000000..ada0eecf89 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-indicator-management/src/main/java/com/webank/wedatasphere/dss/datamodel/indicator/vo/IndicatorContentVO.java @@ -0,0 +1,119 @@ +package com.webank.wedatasphere.dss.datamodel.indicator.vo; + +import java.util.Map; + +public class IndicatorContentVO { + + private Long indicatorId; + + /** + * 0 原子 1 衍生 2 派生 3 复杂 4 自定义 + */ + private Integer indicatorType; + + private Long measureId; + + /** + * 指标来源信息 + */ + private Map sourceInfo; + + private String formula; + + private String business; + + private String businessOwner; + + private String calculation; + + private String calculationOwner; + + public Long getIndicatorId() { + return indicatorId; + } + + public void setIndicatorId(Long indicatorId) { + this.indicatorId = indicatorId; + } + + + public Integer getIndicatorType() { + return indicatorType; + } + + public void setIndicatorType(Integer indicatorType) { + this.indicatorType = indicatorType; + } + + public Long getMeasureId() { + return measureId; + } + + public void setMeasureId(Long measureId) { + this.measureId = measureId; + } + + + public Map getSourceInfo() { + return sourceInfo; + } + + public void setSourceInfo(Map sourceInfo) { + this.sourceInfo = sourceInfo; + } + + public String getFormula() { + return formula; + } + + public void setFormula(String formula) { + this.formula = formula; + } + + public String getBusiness() { + return business; + } + + public void setBusiness(String business) { + this.business = business; + } + + public String getBusinessOwner() { + return businessOwner; + } + + public void setBusinessOwner(String businessOwner) { + this.businessOwner = businessOwner; + } + + public String getCalculation() { + return calculation; + } + + public void setCalculation(String calculation) { + this.calculation = calculation; + } + + public String getCalculationOwner() { + return calculationOwner; + } + + public void setCalculationOwner(String calculationOwner) { + this.calculationOwner = calculationOwner; + } + + @Override + public String toString() { + return "IndicatorContentVO{" + + "indicatorId=" + indicatorId + + ", indicatorType=" + indicatorType + + ", measureId=" + measureId + + ", sourceInfo=" + sourceInfo + + ", formula='" + formula + '\'' + + ", business='" + business + '\'' + + ", businessOwner='" + businessOwner + '\'' + + ", calculation='" + calculation + '\'' + + ", calculationOwner='" + calculationOwner + '\'' + + '}'; + } +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-indicator-management/src/main/java/com/webank/wedatasphere/dss/datamodel/indicator/vo/IndicatorEnableVO.java b/dss-apps/dss-datamodel-center/dss-datamodel-indicator-management/src/main/java/com/webank/wedatasphere/dss/datamodel/indicator/vo/IndicatorEnableVO.java new file mode 100644 index 0000000000..b5311eea34 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-indicator-management/src/main/java/com/webank/wedatasphere/dss/datamodel/indicator/vo/IndicatorEnableVO.java @@ -0,0 +1,23 @@ +package com.webank.wedatasphere.dss.datamodel.indicator.vo; + + +public class IndicatorEnableVO { + + private Integer isAvailable; + + + public Integer getIsAvailable() { + return isAvailable; + } + + public void setIsAvailable(Integer isAvailable) { + this.isAvailable = isAvailable; + } + + @Override + public String toString() { + return "IndicatorEnableVO{" + + "isAvailable=" + isAvailable + + '}'; + } +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-indicator-management/src/main/java/com/webank/wedatasphere/dss/datamodel/indicator/vo/IndicatorQueryVO.java b/dss-apps/dss-datamodel-center/dss-datamodel-indicator-management/src/main/java/com/webank/wedatasphere/dss/datamodel/indicator/vo/IndicatorQueryVO.java new file mode 100644 index 0000000000..c746baa157 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-indicator-management/src/main/java/com/webank/wedatasphere/dss/datamodel/indicator/vo/IndicatorQueryVO.java @@ -0,0 +1,21 @@ +package com.webank.wedatasphere.dss.datamodel.indicator.vo; + +import com.webank.wedatasphere.dss.datamodel.center.common.vo.PageVO; +import lombok.Data; +import lombok.ToString; + +@Data +@ToString +public class IndicatorQueryVO extends PageVO { + + private String name; + + private Integer indicatorType; + + private Integer isAvailable; + + private String owner; + + private String warehouseThemeName; + +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-indicator-management/src/main/java/com/webank/wedatasphere/dss/datamodel/indicator/vo/IndicatorUpdateVO.java b/dss-apps/dss-datamodel-center/dss-datamodel-indicator-management/src/main/java/com/webank/wedatasphere/dss/datamodel/indicator/vo/IndicatorUpdateVO.java new file mode 100644 index 0000000000..c9268a5304 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-indicator-management/src/main/java/com/webank/wedatasphere/dss/datamodel/indicator/vo/IndicatorUpdateVO.java @@ -0,0 +1,50 @@ +package com.webank.wedatasphere.dss.datamodel.indicator.vo; + +import lombok.Data; +import lombok.ToString; + +@Data +@ToString +public class IndicatorUpdateVO { + + private String name; + + private String fieldIdentifier; + + private String comment; + + /** + * 数仓主题格式为： theme_domain_name.theme_name + */ + private String warehouseThemeName; + + private String owner; + + /** + * 授权的名字：userName、roleName + */ + private String principalName; + + private Integer isAvailable; + + private Integer isCoreIndicator; + + /** + * 空：代表所有，如果是逗号分隔的字符串则代表对应的theme的names + */ + private String themeArea; + + /** + * 空：代表所有，如果是逗号分隔的字符串则代表对应的layer的names + */ + private String layerArea; + + private String warehouseThemeNameEn; + + private String themeAreaEn; + + private String layerAreaEn; + + private IndicatorContentVO content; + +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-indicator-management/src/main/java/com/webank/wedatasphere/dss/datamodel/indicator/vo/IndicatorVersionAddVO.java b/dss-apps/dss-datamodel-center/dss-datamodel-indicator-management/src/main/java/com/webank/wedatasphere/dss/datamodel/indicator/vo/IndicatorVersionAddVO.java new file mode 100644 index 0000000000..a57918986b --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-indicator-management/src/main/java/com/webank/wedatasphere/dss/datamodel/indicator/vo/IndicatorVersionAddVO.java @@ -0,0 +1,51 @@ +package com.webank.wedatasphere.dss.datamodel.indicator.vo; + +import lombok.Data; +import lombok.ToString; + +@Data +@ToString +public class IndicatorVersionAddVO { + + + private String name; + + private String fieldIdentifier; + + private String comment; + + /** + * 数仓主题格式为： theme_domain_name.theme_name + */ + private String warehouseThemeName; + + private String owner; + + /** + * 授权的名字：userName、roleName + */ + private String principalName; + + private Integer isAvailable; + + private Integer isCoreIndicator; + + /** + * 空：代表所有，如果是逗号分隔的字符串则代表对应的theme的names + */ + private String themeArea; + + /** + * 空：代表所有，如果是逗号分隔的字符串则代表对应的layer的names + */ + private String layerArea; + + private String warehouseThemeNameEn; + + private String themeAreaEn; + + private String layerAreaEn; + + private IndicatorContentVO content; + +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-indicator-management/src/main/java/com/webank/wedatasphere/dss/datamodel/indicator/vo/IndicatorVersionQueryVO.java b/dss-apps/dss-datamodel-center/dss-datamodel-indicator-management/src/main/java/com/webank/wedatasphere/dss/datamodel/indicator/vo/IndicatorVersionQueryVO.java new file mode 100644 index 0000000000..9d806b46e9 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-indicator-management/src/main/java/com/webank/wedatasphere/dss/datamodel/indicator/vo/IndicatorVersionQueryVO.java @@ -0,0 +1,22 @@ +package com.webank.wedatasphere.dss.datamodel.indicator.vo; + + +public class IndicatorVersionQueryVO { + + private String name; + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + @Override + public String toString() { + return "IndicatorVersionQueryVO{" + + "name='" + name + '\'' + + '}'; + } +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-indicator-management/src/main/java/com/webank/wedatasphere/dss/datamodel/indicator/vo/IndicatorVersionRollBackVO.java b/dss-apps/dss-datamodel-center/dss-datamodel-indicator-management/src/main/java/com/webank/wedatasphere/dss/datamodel/indicator/vo/IndicatorVersionRollBackVO.java new file mode 100644 index 0000000000..fdf2e89db5 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-indicator-management/src/main/java/com/webank/wedatasphere/dss/datamodel/indicator/vo/IndicatorVersionRollBackVO.java @@ -0,0 +1,33 @@ +package com.webank.wedatasphere.dss.datamodel.indicator.vo; + + +public class IndicatorVersionRollBackVO { + + private String name; + + private String version; + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getVersion() { + return version; + } + + public void setVersion(String version) { + this.version = version; + } + + @Override + public String toString() { + return "VersionRollBackVO{" + + "name='" + name + '\'' + + ", version='" + version + '\'' + + '}'; + } +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-indicator-management/src/main/java/com/webank/wedatasphere/dss/datamodel/indicator/vo/LayerVO.java b/dss-apps/dss-datamodel-center/dss-datamodel-indicator-management/src/main/java/com/webank/wedatasphere/dss/datamodel/indicator/vo/LayerVO.java new file mode 100644 index 0000000000..f89c7a0926 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-indicator-management/src/main/java/com/webank/wedatasphere/dss/datamodel/indicator/vo/LayerVO.java @@ -0,0 +1,8 @@ +package com.webank.wedatasphere.dss.datamodel.indicator.vo; + +import lombok.Data; + +@Data +public class LayerVO { + private String dbName; +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-indicator-management/src/main/java/com/webank/wedatasphere/dss/datamodel/indicator/vo/ModifierVO.java b/dss-apps/dss-datamodel-center/dss-datamodel-indicator-management/src/main/java/com/webank/wedatasphere/dss/datamodel/indicator/vo/ModifierVO.java new file mode 100644 index 0000000000..62f89df6cc --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-indicator-management/src/main/java/com/webank/wedatasphere/dss/datamodel/indicator/vo/ModifierVO.java @@ -0,0 +1,10 @@ +package com.webank.wedatasphere.dss.datamodel.indicator.vo; + +import lombok.Data; + +@Data +public class ModifierVO { + private String theme; + + private String layer; +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-measure-management/pom.xml b/dss-apps/dss-datamodel-center/dss-datamodel-measure-management/pom.xml new file mode 100644 index 0000000000..413a133295 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-measure-management/pom.xml @@ -0,0 +1,60 @@ + + + + + + dss-datamodel-center + com.webank.wedatasphere.dss + 1.0.1 + ../pom.xml + + 4.0.0 + + dss-datamodel-measure-management + + + + + + com.webank.wedatasphere.dss + dss-datamodel-center-common + ${dss.version} + + + + + + + + org.apache.maven.plugins + maven-deploy-plugin + + + net.alchim31.maven + scala-maven-plugin + + + org.apache.maven.plugins + maven-jar-plugin + + + + ${project.artifactId}-${project.version} + + diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-measure-management/src/main/java/com/webank/wedatasphere/dss/datamodel/measure/dao/DssDatamodelMeasureMapper.java b/dss-apps/dss-datamodel-center/dss-datamodel-measure-management/src/main/java/com/webank/wedatasphere/dss/datamodel/measure/dao/DssDatamodelMeasureMapper.java new file mode 100644 index 0000000000..f68df7ce3d --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-measure-management/src/main/java/com/webank/wedatasphere/dss/datamodel/measure/dao/DssDatamodelMeasureMapper.java @@ -0,0 +1,9 @@ +package com.webank.wedatasphere.dss.datamodel.measure.dao; + +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import com.webank.wedatasphere.dss.datamodel.measure.entity.DssDatamodelMeasure; +import org.apache.ibatis.annotations.Param; + +public interface DssDatamodelMeasureMapper extends BaseMapper { + +} \ No newline at end of file diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-measure-management/src/main/java/com/webank/wedatasphere/dss/datamodel/measure/dao/impl/DssDatamodelMeasureMapper.xml b/dss-apps/dss-datamodel-center/dss-datamodel-measure-management/src/main/java/com/webank/wedatasphere/dss/datamodel/measure/dao/impl/DssDatamodelMeasureMapper.xml new file mode 100644 index 0000000000..12c13ba566 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-measure-management/src/main/java/com/webank/wedatasphere/dss/datamodel/measure/dao/impl/DssDatamodelMeasureMapper.xml @@ -0,0 +1,24 @@ + + + + + + + + + + + + + + + + + + + + + id, `name`, field_identifier, formula, `comment`, warehouse_theme_name, `owner`, + principal_name, is_available, create_time, update_time + + \ No newline at end of file diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-measure-management/src/main/java/com/webank/wedatasphere/dss/datamodel/measure/dto/MeasureQueryDTO.java b/dss-apps/dss-datamodel-center/dss-datamodel-measure-management/src/main/java/com/webank/wedatasphere/dss/datamodel/measure/dto/MeasureQueryDTO.java new file mode 100644 index 0000000000..d543d3f1df --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-measure-management/src/main/java/com/webank/wedatasphere/dss/datamodel/measure/dto/MeasureQueryDTO.java @@ -0,0 +1,44 @@ +package com.webank.wedatasphere.dss.datamodel.measure.dto; + +import lombok.Data; + +import java.util.Date; + +@Data +public class MeasureQueryDTO { + private Long id; + + private String name; + + private String fieldIdentifier; + + private String formula; + + private String comment; + + /** + * 数仓主题格式为： theme_domain_name.theme_name + */ + private String warehouseThemeName; + + private String warehouseThemeNameEn; + + private String owner; + + /** + * 授权的名字：userName、roleName + */ + private String principalName; + + private Integer isAvailable; + + private Date createTime; + + private Date updateTime; + + /** + * 引用次数 + */ + private Integer refCount = 0; + + } diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-measure-management/src/main/java/com/webank/wedatasphere/dss/datamodel/measure/entity/DssDatamodelMeasure.java b/dss-apps/dss-datamodel-center/dss-datamodel-measure-management/src/main/java/com/webank/wedatasphere/dss/datamodel/measure/entity/DssDatamodelMeasure.java new file mode 100644 index 0000000000..1d5e0a65a5 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-measure-management/src/main/java/com/webank/wedatasphere/dss/datamodel/measure/entity/DssDatamodelMeasure.java @@ -0,0 +1,43 @@ +package com.webank.wedatasphere.dss.datamodel.measure.entity; + +import com.baomidou.mybatisplus.annotation.*; +import lombok.Data; + +import java.util.Date; +@TableName("dss_datamodel_measure") +@Data +public class DssDatamodelMeasure { + + @TableId(type = IdType.AUTO) + private Long id; + + private String name; + + private String fieldIdentifier; + + private String formula; + + private String comment; + + /** + * 数仓主题格式为： theme_domain_name.theme_name + */ + private String warehouseThemeName; + + private String warehouseThemeNameEn; + + private String owner; + + /** + * 授权的名字：userName、roleName + */ + private String principalName; + + private Integer isAvailable; + + @TableField(fill = FieldFill.INSERT) + private Date createTime; + + @TableField(fill = FieldFill.INSERT_UPDATE) + private Date updateTime; +} \ No newline at end of file diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-measure-management/src/main/java/com/webank/wedatasphere/dss/datamodel/measure/restful/MeasureRestfulApi.java b/dss-apps/dss-datamodel-center/dss-datamodel-measure-management/src/main/java/com/webank/wedatasphere/dss/datamodel/measure/restful/MeasureRestfulApi.java new file mode 100644 index 0000000000..61351fd339 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-measure-management/src/main/java/com/webank/wedatasphere/dss/datamodel/measure/restful/MeasureRestfulApi.java @@ -0,0 +1,121 @@ +package com.webank.wedatasphere.dss.datamodel.measure.restful; + +import com.google.common.collect.Lists; +import com.webank.wedatasphere.dss.datamodel.center.common.exception.DSSDatamodelCenterException; +import com.webank.wedatasphere.dss.datamodel.measure.service.MeasureService; +import com.webank.wedatasphere.dss.datamodel.measure.vo.MeasureAddVO; +import com.webank.wedatasphere.dss.datamodel.measure.vo.MeasureEnableVO; +import com.webank.wedatasphere.dss.datamodel.measure.vo.MeasureQueryVO; +import com.webank.wedatasphere.dss.datamodel.measure.vo.MeasureUpdateVO; +import org.apache.linkis.common.exception.ErrorException; +import org.apache.linkis.server.Message; +import org.apache.linkis.server.security.SecurityFilter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; +import org.springframework.web.bind.annotation.RequestBody; + +import javax.servlet.http.HttpServletRequest; +import javax.ws.rs.*; +import javax.ws.rs.core.Context; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; +import java.io.IOException; + + +@Component +@Path("/datamodel/") +@Produces(MediaType.APPLICATION_JSON) +@Consumes(MediaType.APPLICATION_JSON) +public class MeasureRestfulApi { + + + private static final Logger LOGGER = LoggerFactory.getLogger(MeasureRestfulApi.class); + + @Autowired + private MeasureService measureService; + + /** + * 新增 + * + * @param req + * @param vo + * @return + * @throws IOException + */ + @POST + @Path("/measures") + public Response add(@Context HttpServletRequest req, @RequestBody MeasureAddVO vo) throws ErrorException{ + LOGGER.info("measureAddVO : {}", vo); + //String userName = SecurityFilter.getLoginUsername(req); + return Message.messageToResponse(Message.ok().data("id", measureService.addMeasure(vo))); + } + + /** + * 启用/禁用 + * @param req + * @param id + * @param vo + * @return + */ + @PUT + @Path("/measures/enable/{id}") + public Response enable(@Context HttpServletRequest req, @PathParam("id") Long id, @RequestBody MeasureEnableVO vo) { + LOGGER.info("enable id : {}, vo : {}", id, vo); + return Message.messageToResponse(Message.ok().data("count", measureService.enableMeasure(id, vo))); + } + + /** + * 修改 + * @param req + * @param id + * @param vo + * @return + */ + @PUT + @Path("/measures/{id}") + public Response update(@Context HttpServletRequest req, @PathParam("id") Long id, @RequestBody MeasureUpdateVO vo) throws ErrorException{ + LOGGER.info("update id : {}, vo : {}", id, vo); + return Message.messageToResponse(Message.ok().data("count",measureService.updateMeasure(id,vo))); + } + + + /** + * 查看 + * @param req + * @param id + * @return + */ + @GET + @Path("/measures/{id}") + public Response query(@Context HttpServletRequest req, @PathParam("id") Long id) throws ErrorException { + LOGGER.info("query id : {}", id); + return Message.messageToResponse(Message.ok().data("detail",measureService.queryById(id))); + } + + /** + * 删除 + * @param req + * @param id + * @return + */ + @DELETE + @Path("/measures/{id}") + public Response delete(@Context HttpServletRequest req, @PathParam("id") Long id) throws ErrorException { + LOGGER.info("delete id : {}", id); + return Message.messageToResponse(Message.ok().data("count",measureService.deleteMeasure(id))); + } + + /** + * 分页搜索 + * @param req + * @return + */ + @POST + @Path("/measures/list") + public Response list(@Context HttpServletRequest req,@RequestBody MeasureQueryVO vo){ + LOGGER.info("list vo : {}",vo); + return Message.messageToResponse(measureService.listMeasures(vo)); + } +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-measure-management/src/main/java/com/webank/wedatasphere/dss/datamodel/measure/service/MeasureService.java b/dss-apps/dss-datamodel-center/dss-datamodel-measure-management/src/main/java/com/webank/wedatasphere/dss/datamodel/measure/service/MeasureService.java new file mode 100644 index 0000000000..1864413e41 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-measure-management/src/main/java/com/webank/wedatasphere/dss/datamodel/measure/service/MeasureService.java @@ -0,0 +1,70 @@ +package com.webank.wedatasphere.dss.datamodel.measure.service; + +import com.baomidou.mybatisplus.extension.service.IService; +import com.webank.wedatasphere.dss.datamodel.measure.dto.MeasureQueryDTO; +import com.webank.wedatasphere.dss.datamodel.measure.entity.DssDatamodelMeasure; +import com.webank.wedatasphere.dss.datamodel.measure.vo.MeasureAddVO; +import com.webank.wedatasphere.dss.datamodel.measure.vo.MeasureEnableVO; +import com.webank.wedatasphere.dss.datamodel.measure.vo.MeasureQueryVO; +import com.webank.wedatasphere.dss.datamodel.measure.vo.MeasureUpdateVO; +import org.apache.linkis.common.exception.ErrorException; +import org.apache.linkis.server.Message; + + +public interface MeasureService extends IService { + + /** + * 新增维度 + * @param vo + * @return + */ + long addMeasure(MeasureAddVO vo)throws ErrorException; + + /** + * 启用/禁用 + * @param vo + * @return + */ + int enableMeasure(Long id, MeasureEnableVO vo); + + + /** + * 更新 + * @param id + * @param vo + * @return + */ + int updateMeasure(Long id, MeasureUpdateVO vo)throws ErrorException; + + + /** + * 查看 + * @param id + * @return + */ + MeasureQueryDTO queryById(Long id) throws ErrorException; + + + /** + * 删除 + * @param id + * @return + */ + int deleteMeasure(Long id) throws ErrorException; + + + /** + * 查询列表 + * @param vo + * @return + */ + Message listMeasures(MeasureQueryVO vo); + + + /** + * 主题引用情况 + * @param name + * @return + */ + int measureThemeReferenceCount(String name); +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-measure-management/src/main/java/com/webank/wedatasphere/dss/datamodel/measure/service/impl/DataWarehouseMeasuredReferenceServiceImpl.java b/dss-apps/dss-datamodel-center/dss-datamodel-measure-management/src/main/java/com/webank/wedatasphere/dss/datamodel/measure/service/impl/DataWarehouseMeasuredReferenceServiceImpl.java new file mode 100644 index 0000000000..959d668f75 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-measure-management/src/main/java/com/webank/wedatasphere/dss/datamodel/measure/service/impl/DataWarehouseMeasuredReferenceServiceImpl.java @@ -0,0 +1,20 @@ +package com.webank.wedatasphere.dss.datamodel.measure.service.impl; + + +import com.webank.wedatasphere.dss.datamodel.center.common.service.DataWarehouseMeasuredReferenceService; +import com.webank.wedatasphere.dss.datamodel.measure.service.MeasureService; +import org.springframework.stereotype.Service; + +import javax.annotation.Resource; + +@Service +public class DataWarehouseMeasuredReferenceServiceImpl implements DataWarehouseMeasuredReferenceService { + + @Resource + private MeasureService measureService; + + @Override + public int measureThemeReferenceCount(String name) { + return measureService.measureThemeReferenceCount(name); + } +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-measure-management/src/main/java/com/webank/wedatasphere/dss/datamodel/measure/service/impl/MeasureServiceImpl.java b/dss-apps/dss-datamodel-center/dss-datamodel-measure-management/src/main/java/com/webank/wedatasphere/dss/datamodel/measure/service/impl/MeasureServiceImpl.java new file mode 100644 index 0000000000..51f0eccaf7 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-measure-management/src/main/java/com/webank/wedatasphere/dss/datamodel/measure/service/impl/MeasureServiceImpl.java @@ -0,0 +1,214 @@ +package com.webank.wedatasphere.dss.datamodel.measure.service.impl; + +import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; +import com.baomidou.mybatisplus.core.toolkit.Wrappers; +import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl; +import com.github.pagehelper.PageHelper; +import com.github.pagehelper.PageInfo; +import com.webank.wedatasphere.dss.data.governance.entity.ClassificationConstant; +import com.webank.wedatasphere.dss.data.governance.response.CreateModelTypeResult; +import com.webank.wedatasphere.dss.data.governance.response.UpdateModelTypeResult; +import com.webank.wedatasphere.dss.datamodel.center.common.constant.ErrorCode; +import com.webank.wedatasphere.dss.datamodel.center.common.context.DataModelSecurityContextHolder; +import com.webank.wedatasphere.dss.datamodel.center.common.event.CreateModelEvent; +import com.webank.wedatasphere.dss.datamodel.center.common.event.DeleteModelEvent; +import com.webank.wedatasphere.dss.datamodel.center.common.event.UpdateModelEvent; +import com.webank.wedatasphere.dss.datamodel.center.common.exception.DSSDatamodelCenterException; +import com.webank.wedatasphere.dss.datamodel.center.common.service.AssertsSyncService; +import com.webank.wedatasphere.dss.datamodel.center.common.service.DatamodelReferencService; +import com.webank.wedatasphere.dss.datamodel.measure.dao.DssDatamodelMeasureMapper; +import com.webank.wedatasphere.dss.datamodel.measure.dto.MeasureQueryDTO; +import com.webank.wedatasphere.dss.datamodel.measure.entity.DssDatamodelMeasure; +import com.webank.wedatasphere.dss.datamodel.measure.service.MeasureService; +import com.webank.wedatasphere.dss.datamodel.measure.vo.MeasureAddVO; +import com.webank.wedatasphere.dss.datamodel.measure.vo.MeasureEnableVO; +import com.webank.wedatasphere.dss.datamodel.measure.vo.MeasureQueryVO; +import com.webank.wedatasphere.dss.datamodel.measure.vo.MeasureUpdateVO; +import org.apache.linkis.common.exception.ErrorException; +import org.apache.linkis.server.Message; +import org.apache.commons.lang.StringUtils; +import org.modelmapper.ModelMapper; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +import javax.annotation.Resource; +import java.util.Date; +import java.util.stream.Collectors; + + +@Service +public class MeasureServiceImpl extends ServiceImpl implements MeasureService { + + private static final Logger LOGGER = LoggerFactory.getLogger(MeasureServiceImpl.class); + + private final ModelMapper modelMapper = new ModelMapper(); + + + + @Resource + private AssertsSyncService assertsSyncService; + + @Resource + private DatamodelReferencService datamodelReferencService; + + @Override + @Transactional(rollbackFor = Exception.class) + public long addMeasure(MeasureAddVO vo) throws ErrorException { + if (getBaseMapper().selectCount(Wrappers.lambdaQuery().eq(DssDatamodelMeasure::getName, vo.getName()) + .or().eq(DssDatamodelMeasure::getFieldIdentifier, vo.getFieldIdentifier())) > 0) { + LOGGER.error("errorCode : {}, measure name or field identifier can not repeat, name : {}", ErrorCode.INDICATOR_ADD_ERROR.getCode(), vo.getName()); + throw new DSSDatamodelCenterException(ErrorCode.MEASURE_ADD_ERROR.getCode(), "measure name or field identifier can not repeat"); + } + + + //校验引用情况 + if(datamodelReferencService.measureReferenceCount(vo.getName())>0 + ||datamodelReferencService.measureReferenceCount(vo.getFieldIdentifier())>0){ + LOGGER.error("errorCode : {}, measure name can not be referenced ", ErrorCode.MEASURE_UPDATE_ERROR.getCode()); + throw new DSSDatamodelCenterException(ErrorCode.MEASURE_UPDATE_ERROR.getCode(), "measure name can not be referenced"); + } + + DssDatamodelMeasure newOne = modelMapper.map(vo, DssDatamodelMeasure.class); + newOne.setCreateTime(new Date()); + newOne.setUpdateTime(new Date()); + getBaseMapper().insert(newOne); + //同步atlas + CreateModelTypeResult result = assertsSyncService.syncCreateModel( + new CreateModelEvent(this + , DataModelSecurityContextHolder.getContext().getDataModelAuthentication().getUser() + , vo.getFieldIdentifier() + , ClassificationConstant.MEASURE)); + return newOne.getId(); + } + + + @Override + @Transactional(rollbackFor = Exception.class) + public int enableMeasure(Long id, MeasureEnableVO vo) { + DssDatamodelMeasure enableOne = new DssDatamodelMeasure(); + enableOne.setIsAvailable(vo.getIsAvailable()); + enableOne.setUpdateTime(new Date()); + return getBaseMapper().update(enableOne, Wrappers.lambdaUpdate().eq(DssDatamodelMeasure::getId, id)); + } + + + @Override + @Transactional(rollbackFor = Exception.class) + public int updateMeasure(Long id, MeasureUpdateVO vo) throws ErrorException { + DssDatamodelMeasure org = getBaseMapper().selectById(id); + if (org == null) { + LOGGER.error("errorCode : {}, update measure error not exists", ErrorCode.MEASURE_UPDATE_ERROR.getCode()); + throw new DSSDatamodelCenterException(ErrorCode.MEASURE_UPDATE_ERROR.getCode(), "update measure error not exists"); + } + + //校验引用情况 + if(datamodelReferencService.measureReferenceCount(vo.getName())>0 + ||datamodelReferencService.measureReferenceCount(vo.getFieldIdentifier())>0 + ||datamodelReferencService.measureReferenceCount(org.getName())>0 + ||datamodelReferencService.measureReferenceCount(org.getFieldIdentifier())>0){ + LOGGER.error("errorCode : {}, measure name can not be referenced ", ErrorCode.MEASURE_UPDATE_ERROR.getCode()); + throw new DSSDatamodelCenterException(ErrorCode.MEASURE_UPDATE_ERROR.getCode(), "measure name can not be referenced"); + } + + //当更新名称时 + if (!StringUtils.equals(vo.getName(), org.getName())) { + int repeat = getBaseMapper().selectCount(Wrappers.lambdaQuery().eq(DssDatamodelMeasure::getName, vo.getName())); + if (repeat > 0 ) { + LOGGER.error("errorCode : {}, measure name can not repeat ", ErrorCode.MEASURE_UPDATE_ERROR.getCode()); + throw new DSSDatamodelCenterException(ErrorCode.MEASURE_UPDATE_ERROR.getCode(), "measure name can not repeat "); + } + } + + String orgFieldIdentifier = org.getFieldIdentifier(); + //当更新标识时 + if (!StringUtils.equals(vo.getFieldIdentifier(), orgFieldIdentifier)) { + int repeat = getBaseMapper().selectCount(Wrappers.lambdaQuery().eq(DssDatamodelMeasure::getFieldIdentifier, vo.getFieldIdentifier())); + + if (repeat > 0 || datamodelReferencService.measureReferenceCount(orgFieldIdentifier)>0) { + LOGGER.error("errorCode : {}, measure field identifier can not repeat or referenced", ErrorCode.MEASURE_UPDATE_ERROR.getCode()); + throw new DSSDatamodelCenterException(ErrorCode.MEASURE_UPDATE_ERROR.getCode(), "measure field identifier can not repeat or referenced"); + } + } + DssDatamodelMeasure updateOne = modelMapper.map(vo, DssDatamodelMeasure.class); + updateOne.setUpdateTime(new Date()); + getBaseMapper().update(updateOne, Wrappers.lambdaUpdate().eq(DssDatamodelMeasure::getId, id)); + //同步atlas + UpdateModelTypeResult updateModelTypeResult = assertsSyncService.syncUpdateModel( + new UpdateModelEvent(this + , DataModelSecurityContextHolder.getContext().getDataModelAuthentication().getUser() + , vo.getFieldIdentifier() + , orgFieldIdentifier + , ClassificationConstant.MEASURE)); + return 1; + } + + + @Override + @Transactional(rollbackFor = Exception.class) + public int deleteMeasure(Long id) throws ErrorException { + DssDatamodelMeasure dssDatamodelMeasure = getBaseMapper().selectById(id); + if (dssDatamodelMeasure == null) { + throw new DSSDatamodelCenterException(ErrorCode.MEASURE_DELETE_ERROR.getCode(), "measure id " + id + " not exists"); + } + //校验引用情况 + if(datamodelReferencService.measureReferenceCount(dssDatamodelMeasure.getName())>0 + ||datamodelReferencService.measureReferenceCount(dssDatamodelMeasure.getFieldIdentifier())>0){ + throw new DSSDatamodelCenterException(ErrorCode.MEASURE_DELETE_ERROR.getCode(), "measure id " + id + " has referenced"); + } + + getBaseMapper().deleteById(id); + //同步资产 + assertsSyncService.syncDeleteModel(new DeleteModelEvent(this + , DataModelSecurityContextHolder.getContext().getDataModelAuthentication().getUser() + , dssDatamodelMeasure.getFieldIdentifier() + , ClassificationConstant.MEASURE)); + return 1; + } + + + @Override + public Message listMeasures(MeasureQueryVO vo) { + QueryWrapper queryWrapper = new QueryWrapper() + .like(StringUtils.isNotBlank(vo.getName()), "name", vo.getName()) + .eq(vo.getIsAvailable() != null, "is_available", vo.getIsAvailable()) + .eq(StringUtils.isNotBlank(vo.getWarehouseThemeName()), "warehouse_theme_name", vo.getWarehouseThemeName()) + .like(StringUtils.isNotBlank(vo.getOwner()), "owner", vo.getOwner()); + PageHelper.clearPage(); + PageHelper.startPage(vo.getPageNum(), vo.getPageSize()); + PageInfo pageInfo = new PageInfo<>(getBaseMapper().selectList(queryWrapper)); + //IPage iPage = page(new Page<>(vo.getPageNum(),vo.getPageSize()),queryWrapper); + return Message.ok() + .data("list", pageInfo + .getList() + .stream() + .map(dssDatamodelMeasure -> { + MeasureQueryDTO dto = modelMapper.map(dssDatamodelMeasure, MeasureQueryDTO.class); + dto.setRefCount(datamodelReferencService.measureReferenceCount(dssDatamodelMeasure.getName())); + return dto; + }) + .collect(Collectors.toList())) + .data("total", pageInfo.getTotal()); + } + + + @Override + public MeasureQueryDTO queryById(Long id) throws DSSDatamodelCenterException { + DssDatamodelMeasure dssDatamodelMeasure = getBaseMapper().selectById(id); + if (dssDatamodelMeasure == null) { + throw new DSSDatamodelCenterException(ErrorCode.MEASURE_QUERY_ERROR.getCode(), "measure id " + id + " not exists"); + } + MeasureQueryDTO dto = modelMapper.map(dssDatamodelMeasure, MeasureQueryDTO.class); + dto.setRefCount(datamodelReferencService.measureReferenceCount(dssDatamodelMeasure.getName())); + return dto; + } + + + @Override + public int measureThemeReferenceCount(String name) { + int count = getBaseMapper().selectCount(Wrappers.lambdaQuery().eq(DssDatamodelMeasure::getWarehouseThemeName, name)); + int countEn = getBaseMapper().selectCount(Wrappers.lambdaQuery().eq(DssDatamodelMeasure::getWarehouseThemeNameEn, name)); + return count + countEn; + } +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-measure-management/src/main/java/com/webank/wedatasphere/dss/datamodel/measure/vo/MeasureAddVO.java b/dss-apps/dss-datamodel-center/dss-datamodel-measure-management/src/main/java/com/webank/wedatasphere/dss/datamodel/measure/vo/MeasureAddVO.java new file mode 100644 index 0000000000..dc15c333e0 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-measure-management/src/main/java/com/webank/wedatasphere/dss/datamodel/measure/vo/MeasureAddVO.java @@ -0,0 +1,35 @@ +package com.webank.wedatasphere.dss.datamodel.measure.vo; + +import lombok.Data; +import lombok.ToString; + +@Data +@ToString +public class MeasureAddVO { + + private String name; + + private String fieldIdentifier; + + private String formula; + + private String comment; + + /** + * 数仓主题格式为： theme_domain_name.theme_name + */ + private String warehouseThemeName; + + private String warehouseThemeNameEn; + + private String owner; + + /** + * 授权的名字：userName、roleName + */ + private String principalName; + + private Integer isAvailable; + + +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-measure-management/src/main/java/com/webank/wedatasphere/dss/datamodel/measure/vo/MeasureEnableVO.java b/dss-apps/dss-datamodel-center/dss-datamodel-measure-management/src/main/java/com/webank/wedatasphere/dss/datamodel/measure/vo/MeasureEnableVO.java new file mode 100644 index 0000000000..bb2f09fed3 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-measure-management/src/main/java/com/webank/wedatasphere/dss/datamodel/measure/vo/MeasureEnableVO.java @@ -0,0 +1,23 @@ +package com.webank.wedatasphere.dss.datamodel.measure.vo; + + +public class MeasureEnableVO { + + + private Integer isAvailable; + + public Integer getIsAvailable() { + return isAvailable; + } + + public void setIsAvailable(Integer isAvailable) { + this.isAvailable = isAvailable; + } + + @Override + public String toString() { + return "MeasureEnableVO{" + + "isAvailable=" + isAvailable + + '}'; + } +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-measure-management/src/main/java/com/webank/wedatasphere/dss/datamodel/measure/vo/MeasureQueryVO.java b/dss-apps/dss-datamodel-center/dss-datamodel-measure-management/src/main/java/com/webank/wedatasphere/dss/datamodel/measure/vo/MeasureQueryVO.java new file mode 100644 index 0000000000..75837a7027 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-measure-management/src/main/java/com/webank/wedatasphere/dss/datamodel/measure/vo/MeasureQueryVO.java @@ -0,0 +1,19 @@ +package com.webank.wedatasphere.dss.datamodel.measure.vo; + +import com.webank.wedatasphere.dss.datamodel.center.common.vo.PageVO; +import lombok.Data; +import lombok.ToString; + +@Data +@ToString +public class MeasureQueryVO extends PageVO { + + private String name; + + private Integer isAvailable; + + private String owner; + + private String warehouseThemeName; + +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-measure-management/src/main/java/com/webank/wedatasphere/dss/datamodel/measure/vo/MeasureUpdateVO.java b/dss-apps/dss-datamodel-center/dss-datamodel-measure-management/src/main/java/com/webank/wedatasphere/dss/datamodel/measure/vo/MeasureUpdateVO.java new file mode 100644 index 0000000000..59ed05435f --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-measure-management/src/main/java/com/webank/wedatasphere/dss/datamodel/measure/vo/MeasureUpdateVO.java @@ -0,0 +1,34 @@ +package com.webank.wedatasphere.dss.datamodel.measure.vo; + +import lombok.Data; +import lombok.ToString; + +@Data +@ToString +public class MeasureUpdateVO { + + private String name; + + private String fieldIdentifier; + + private String formula; + + private String comment; + + /** + * 数仓主题格式为： theme_domain_name.theme_name + */ + private String warehouseThemeName; + + private String warehouseThemeNameEn; + + private String owner; + + /** + * 授权的名字：userName、roleName + */ + private String principalName; + + private Integer isAvailable; + +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-table-management/pom.xml b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/pom.xml new file mode 100644 index 0000000000..d520b79ad3 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/pom.xml @@ -0,0 +1,59 @@ + + + + + + dss-datamodel-center + com.webank.wedatasphere.dss + 1.0.1 + ../pom.xml + + 4.0.0 + + dss-datamodel-table-management + + + + + com.webank.wedatasphere.dss + dss-datamodel-center-common + ${dss.version} + + + + + + + + org.apache.maven.plugins + maven-deploy-plugin + + + net.alchim31.maven + scala-maven-plugin + + + org.apache.maven.plugins + maven-jar-plugin + + + + ${project.artifactId}-${project.version} + + diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/dao/DssDatamodelDictionaryMapper.java b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/dao/DssDatamodelDictionaryMapper.java new file mode 100644 index 0000000000..71cd712559 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/dao/DssDatamodelDictionaryMapper.java @@ -0,0 +1,7 @@ +package com.webank.wedatasphere.dss.datamodel.table.dao; + +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import com.webank.wedatasphere.dss.datamodel.table.entity.DssDatamodelDictionary; + +public interface DssDatamodelDictionaryMapper extends BaseMapper { +} \ No newline at end of file diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/dao/DssDatamodelLabelMapper.java b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/dao/DssDatamodelLabelMapper.java new file mode 100644 index 0000000000..242aab574b --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/dao/DssDatamodelLabelMapper.java @@ -0,0 +1,8 @@ +package com.webank.wedatasphere.dss.datamodel.table.dao; + +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import com.webank.wedatasphere.dss.datamodel.table.entity.DssDatamodelLabel; + +public interface DssDatamodelLabelMapper extends BaseMapper { + +} \ No newline at end of file diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/dao/DssDatamodelTableCollcetionMapper.java b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/dao/DssDatamodelTableCollcetionMapper.java new file mode 100644 index 0000000000..714abfa68c --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/dao/DssDatamodelTableCollcetionMapper.java @@ -0,0 +1,7 @@ +package com.webank.wedatasphere.dss.datamodel.table.dao; + +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import com.webank.wedatasphere.dss.datamodel.table.entity.DssDatamodelTableCollcetion; + +public interface DssDatamodelTableCollcetionMapper extends BaseMapper { +} \ No newline at end of file diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/dao/DssDatamodelTableColumnsMapper.java b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/dao/DssDatamodelTableColumnsMapper.java new file mode 100644 index 0000000000..8444d4a805 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/dao/DssDatamodelTableColumnsMapper.java @@ -0,0 +1,7 @@ +package com.webank.wedatasphere.dss.datamodel.table.dao; + +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import com.webank.wedatasphere.dss.datamodel.table.entity.DssDatamodelTableColumns; + +public interface DssDatamodelTableColumnsMapper extends BaseMapper { +} \ No newline at end of file diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/dao/DssDatamodelTableMapper.java b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/dao/DssDatamodelTableMapper.java new file mode 100644 index 0000000000..31b1328059 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/dao/DssDatamodelTableMapper.java @@ -0,0 +1,7 @@ +package com.webank.wedatasphere.dss.datamodel.table.dao; + +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import com.webank.wedatasphere.dss.datamodel.table.entity.DssDatamodelTable; + +public interface DssDatamodelTableMapper extends BaseMapper { +} \ No newline at end of file diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/dao/DssDatamodelTableMaterializedHistoryMapper.java b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/dao/DssDatamodelTableMaterializedHistoryMapper.java new file mode 100644 index 0000000000..e0ac1b87a6 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/dao/DssDatamodelTableMaterializedHistoryMapper.java @@ -0,0 +1,7 @@ +package com.webank.wedatasphere.dss.datamodel.table.dao; + +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import com.webank.wedatasphere.dss.datamodel.table.entity.DssDatamodelTableMaterializedHistory; + +public interface DssDatamodelTableMaterializedHistoryMapper extends BaseMapper { +} \ No newline at end of file diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/dao/DssDatamodelTableStaticsMapper.java b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/dao/DssDatamodelTableStaticsMapper.java new file mode 100644 index 0000000000..a1c5276eb0 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/dao/DssDatamodelTableStaticsMapper.java @@ -0,0 +1,4 @@ +package com.webank.wedatasphere.dss.datamodel.table.dao; + +public interface DssDatamodelTableStaticsMapper { +} \ No newline at end of file diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/dao/DssDatamodelTableStatsMapper.java b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/dao/DssDatamodelTableStatsMapper.java new file mode 100644 index 0000000000..0fde86e310 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/dao/DssDatamodelTableStatsMapper.java @@ -0,0 +1,7 @@ +package com.webank.wedatasphere.dss.datamodel.table.dao; + +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import com.webank.wedatasphere.dss.datamodel.table.entity.DssDatamodelTableStats; + +public interface DssDatamodelTableStatsMapper extends BaseMapper { +} \ No newline at end of file diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/dao/DssDatamodelTableVersionMapper.java b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/dao/DssDatamodelTableVersionMapper.java new file mode 100644 index 0000000000..052c8a234a --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/dao/DssDatamodelTableVersionMapper.java @@ -0,0 +1,7 @@ +package com.webank.wedatasphere.dss.datamodel.table.dao; + +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import com.webank.wedatasphere.dss.datamodel.table.entity.DssDatamodelTableVersion; + +public interface DssDatamodelTableVersionMapper extends BaseMapper { +} \ No newline at end of file diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/dao/TableQueryMapper.java b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/dao/TableQueryMapper.java new file mode 100644 index 0000000000..d703a816de --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/dao/TableQueryMapper.java @@ -0,0 +1,22 @@ +package com.webank.wedatasphere.dss.datamodel.table.dao; + + +import com.baomidou.mybatisplus.core.conditions.Wrapper; +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import com.baomidou.mybatisplus.core.toolkit.Constants; +import com.webank.wedatasphere.dss.datamodel.table.entity.DssDatamodelTabelQuery; +import org.apache.ibatis.annotations.Param; +import org.apache.ibatis.annotations.Select; + +import java.util.List; + +public interface TableQueryMapper extends BaseMapper { + String querySql = "SELECT a.*, b.model_type,b.model_name FROM dss_datamodel_table AS a LEFT JOIN dss_datamodel_table_columns AS b ON b.table_id = a.id " + + " ${ew.customSqlSegment}"; + String wrapperSql = "SELECT * from ( " + querySql + " ) AS q ${ew.customSqlSegment}"; + + + + @Select(querySql) + List page(@Param(Constants.WRAPPER) Wrapper queryWrapper); +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/dao/impl/DssDatamodelDictionaryMapper.xml b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/dao/impl/DssDatamodelDictionaryMapper.xml new file mode 100644 index 0000000000..6aa69080d9 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/dao/impl/DssDatamodelDictionaryMapper.xml @@ -0,0 +1,19 @@ + + + + + + + + + + + + + + + + + id, code, `type`, description, created_time, update_time, sort + + \ No newline at end of file diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/dao/impl/DssDatamodelLabelMapper.xml b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/dao/impl/DssDatamodelLabelMapper.xml new file mode 100644 index 0000000000..394fc1f040 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/dao/impl/DssDatamodelLabelMapper.xml @@ -0,0 +1,25 @@ + + + + + + + + + + + + + + + + + + + + + + id, `name`, field_identifier, params, `comment`, warehouse_theme_name, `owner`, principal_name, + is_available, create_time, update_time, warehouse_theme_name_en + + \ No newline at end of file diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/dao/impl/DssDatamodelTableCollcetionMapper.xml b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/dao/impl/DssDatamodelTableCollcetionMapper.xml new file mode 100644 index 0000000000..9a1e80df6f --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/dao/impl/DssDatamodelTableCollcetionMapper.xml @@ -0,0 +1,40 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + id, data_base, `name`, `alias`, creator, `comment`, create_time, update_time, warehouse_layer_name, + warehouse_theme_name, lifecycle, is_partition_table, is_available, storage_type, + principal_name, `compress`, file_type, `user`, version, is_external, `location`, + `label`, guid, warehouse_layer_name_en, warehouse_theme_name_en + + \ No newline at end of file diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/dao/impl/DssDatamodelTableColumnsMapper.xml b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/dao/impl/DssDatamodelTableColumnsMapper.xml new file mode 100644 index 0000000000..dcbf7e32f1 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/dao/impl/DssDatamodelTableColumnsMapper.xml @@ -0,0 +1,29 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + id, table_id, `name`, `alias`, `type`, `comment`, is_partition_field, is_primary, + `length`, `rule`, model_type, model_name_en, model_id, model_name, create_time, update_time + + \ No newline at end of file diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/dao/impl/DssDatamodelTableMapper.xml b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/dao/impl/DssDatamodelTableMapper.xml new file mode 100644 index 0000000000..57d36a49da --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/dao/impl/DssDatamodelTableMapper.xml @@ -0,0 +1,38 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + id, data_base, `name`, `alias`, creator, `comment`, create_time, update_time, warehouse_layer_name, + warehouse_layer_name_en, warehouse_theme_name, warehouse_theme_name_en, lifecycle, + is_partition_table, is_available, storage_type, principal_name, `compress`, file_type, + version, is_external, `location`, `label` + + \ No newline at end of file diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/dao/impl/DssDatamodelTableMaterializedHistoryMapper.xml b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/dao/impl/DssDatamodelTableMaterializedHistoryMapper.xml new file mode 100644 index 0000000000..8435cbca06 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/dao/impl/DssDatamodelTableMaterializedHistoryMapper.xml @@ -0,0 +1,25 @@ + + + + + + + + + + + + + + + + + + + + + + id, materialized_code, reason, creator, `status`, create_time, last_update_time, + task_id, error_msg, tableName, `database`, version + + \ No newline at end of file diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/dao/impl/DssDatamodelTableStaticsMapper.xml b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/dao/impl/DssDatamodelTableStaticsMapper.xml new file mode 100644 index 0000000000..2e1b1d14b9 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/dao/impl/DssDatamodelTableStaticsMapper.xml @@ -0,0 +1,19 @@ + + + + + + + + + + + + + + + + + id, origin_tables, table_id, access_count, last_access_time, sample_data_path, sample_update_time + + \ No newline at end of file diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/dao/impl/DssDatamodelTableStatsMapper.xml b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/dao/impl/DssDatamodelTableStatsMapper.xml new file mode 100644 index 0000000000..c77d513351 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/dao/impl/DssDatamodelTableStatsMapper.xml @@ -0,0 +1,26 @@ + + + + + + + + + + + + + + + + + + + + + + + id, data_base, `name`, create_time, update_time, column_count, storage_size, file_count, + partition_count, access_count, collect_count, ref_count, version + + \ No newline at end of file diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/dao/impl/DssDatamodelTableVersionMapper.xml b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/dao/impl/DssDatamodelTableVersionMapper.xml new file mode 100644 index 0000000000..4131b9e4d9 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/dao/impl/DssDatamodelTableVersionMapper.xml @@ -0,0 +1,26 @@ + + + + + + + + + + + + + + + + + + + + + + + id, tbl_id, `name`, is_materialized, table_code, `comment`, version, table_params, + `columns`, source_type, create_time, update_time, creator + + \ No newline at end of file diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/dto/HiveTblDetailInfoDTO.java b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/dto/HiveTblDetailInfoDTO.java new file mode 100644 index 0000000000..e3c10a8dea --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/dto/HiveTblDetailInfoDTO.java @@ -0,0 +1,33 @@ +package com.webank.wedatasphere.dss.datamodel.table.dto; + +import lombok.Data; + +import java.io.Serializable; +import java.util.List; +import java.util.Set; + + +@Data +public class HiveTblDetailInfoDTO implements Serializable { + private HiveTblBasicInfo basic; + private List columns; + private List partitionKeys; + + + @Data + public static class HiveTblBasicInfo extends HiveTblSimpleInfo { + private String store; + private String comment; + private Set labels; + private Boolean isParTbl; + } + + @Data + public static class HiveColumnInfoDTO { + private String name; + private String type; + private String guid; + private String comment; + } + +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/dto/HiveTblSimpleInfo.java b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/dto/HiveTblSimpleInfo.java new file mode 100644 index 0000000000..146a133b3a --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/dto/HiveTblSimpleInfo.java @@ -0,0 +1,17 @@ +package com.webank.wedatasphere.dss.datamodel.table.dto; + +import lombok.Data; + +import java.util.Date; + + +@Data +public class HiveTblSimpleInfo { + private String guid; + private String name; + private String qualifiedName; + private Date createTime; + private String owner; + private Date lastAccessTime; + private String external; +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/dto/HiveTblSimpleInfoDTO.java b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/dto/HiveTblSimpleInfoDTO.java new file mode 100644 index 0000000000..342c303d30 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/dto/HiveTblSimpleInfoDTO.java @@ -0,0 +1,21 @@ +package com.webank.wedatasphere.dss.datamodel.table.dto; + + +import lombok.Data; + +import java.util.Date; +import java.util.List; + +@Data +public class HiveTblSimpleInfoDTO { + private String guid; + private String name; + private String qualifiedName; + private Date createTime; + private String owner; + private Date lastAccessTime; + private String aliases; + private String comment; + private List classifications; + private String totalSize; +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/dto/LabelQueryDTO.java b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/dto/LabelQueryDTO.java new file mode 100644 index 0000000000..4c70def69a --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/dto/LabelQueryDTO.java @@ -0,0 +1,52 @@ +package com.webank.wedatasphere.dss.datamodel.table.dto; + +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableId; +import lombok.Data; + +import java.util.Date; + +@Data +public class LabelQueryDTO { + + private Long id; + + private String name; + + private String fieldIdentifier; + + /** + * 标签键值对 json + */ + private String params; + + private String comment; + + /** + * 数仓主题格式为： theme_domain_name.theme_name + */ + private String warehouseThemeName; + + private String owner; + + /** + * 授权的名字：userName、roleName + */ + private String principalName; + + private Integer isAvailable; + + private Date createTime; + + private Date updateTime; + + /** + * 英文 + */ + private String warehouseThemeNameEn; + + /** + * 被引用次数 + */ + private Integer refCount =0; +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/dto/ListDataBasesVO.java b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/dto/ListDataBasesVO.java new file mode 100644 index 0000000000..4d9673a640 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/dto/ListDataBasesVO.java @@ -0,0 +1,11 @@ +package com.webank.wedatasphere.dss.datamodel.table.dto; + + +import lombok.Data; +import lombok.ToString; + +@Data +@ToString +public class ListDataBasesVO { + private String name; +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/dto/ModelTypeDTO.java b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/dto/ModelTypeDTO.java new file mode 100644 index 0000000000..a581ee5fd0 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/dto/ModelTypeDTO.java @@ -0,0 +1,28 @@ +package com.webank.wedatasphere.dss.datamodel.table.dto; + + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.EqualsAndHashCode; + +@Data +@EqualsAndHashCode +@AllArgsConstructor +public class ModelTypeDTO { + + /** + * 0 维度，1 指标 2 度量 + */ + private Integer modelType; + + /** + * 模型信息英文名称 + */ + private String modelNameEn; + + + /** + * 模型信息名称 + */ + private String modelName; +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/dto/PartInfoDTO.java b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/dto/PartInfoDTO.java new file mode 100644 index 0000000000..72ed291cc6 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/dto/PartInfoDTO.java @@ -0,0 +1,16 @@ +package com.webank.wedatasphere.dss.datamodel.table.dto; + +import lombok.Data; + +import java.util.Date; + + +@Data +public class PartInfoDTO { + private String partName; + private int reordCnt; + private int store; + private int fileCount; + private Date createTime; + private Date lastAccessTime; +} \ No newline at end of file diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/dto/TableCollectionDTO.java b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/dto/TableCollectionDTO.java new file mode 100644 index 0000000000..239229cae0 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/dto/TableCollectionDTO.java @@ -0,0 +1,105 @@ +package com.webank.wedatasphere.dss.datamodel.table.dto; + + +import lombok.Data; +import lombok.ToString; + +import java.util.Date; + +@Data +@ToString +public class TableCollectionDTO { + + private Long id; + + private String dataBase; + + private String name; + + private String alias; + + private String creator; + + private String comment; + + private Date createTime; + + private Date updateTime; + + /** + * 数仓层级 + */ + private String warehouseLayerName; + + /** + * 数仓层级英文 + */ + private String warehouseLayerNameEn; + + /** + * 数仓主题格式为： theme_domain_name.theme_name + */ + private String warehouseThemeName; + + /** + * 数仓主题英文 + */ + private String warehouseThemeNameEn; + /** + * 生命周期 + */ + private String lifecycle; + + private String lifecycleEn; + + private Integer isPartitionTable; + + private Integer isAvailable; + + /** + * 存储类型：hive/mysql + */ + private String storageType; + + /** + * 授权的名字：userName、roleName + */ + private String principalName; + + /** + * 压缩格式 + */ + private String compress; + + /** + * 文件格式 + */ + private String fileType; + + /** + * 收藏人 + */ + private String user; + + + /** + * 是否外部表 0 内部表 1外部表 + */ + private Integer isExternal; + + /** + * 外部表时 location + */ + private String location; + + /** + * 标签 + */ + private String label; + + /** + * atlas标识 + */ + private String guid; + +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/dto/TableColumnQueryDTO.java b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/dto/TableColumnQueryDTO.java new file mode 100644 index 0000000000..7dcd94bec0 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/dto/TableColumnQueryDTO.java @@ -0,0 +1,50 @@ +package com.webank.wedatasphere.dss.datamodel.table.dto; + +import lombok.Data; + +import java.util.Date; + +@Data +public class TableColumnQueryDTO { + + private Long id; + + private Long tableId; + + private String name; + + private String alias; + + private String type; + + private String comment; + + private Integer isPartitionField; + + private Integer isPrimary; + + private Integer length; + + private String rule; + + /** + * 0 维度，1 指标 2 度量 + */ + private Integer modelType; + + /** + * 模型信息名称 + */ + private String modelName; + + /** + * 模型信息英文名称 + */ + private String modelNameEn; + + private Date createTime; + + private Date updateTime; + + +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/dto/TableHeadlineDTO.java b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/dto/TableHeadlineDTO.java new file mode 100644 index 0000000000..edea9a8c9e --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/dto/TableHeadlineDTO.java @@ -0,0 +1,25 @@ +package com.webank.wedatasphere.dss.datamodel.table.dto; + +import lombok.Data; +import lombok.ToString; + +@Data +@ToString +public class TableHeadlineDTO { + + /** + * 0 表示 Hive + */ + private Integer storageType; + + /** + * 0 表示离线表 + */ + private Integer tableType; + + + /** + * 0 逻辑表 1 物理表 + */ + private Integer entityType; +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/dto/TableListDTO.java b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/dto/TableListDTO.java new file mode 100644 index 0000000000..e3476a1856 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/dto/TableListDTO.java @@ -0,0 +1,102 @@ +package com.webank.wedatasphere.dss.datamodel.table.dto; + + +import lombok.Data; +import lombok.ToString; + +import java.util.Date; + +@Data +@ToString +public class TableListDTO { + + private Long id; + + private String dataBase; + + private String name; + + private String alias; + + private String creator; + + private String comment; + + private Date createTime; + + private Date updateTime; + + /** + * 数仓层级 + */ + private String warehouseLayerName; + + /** + * 数仓层级英文 + */ + private String warehouseLayerNameEn; + + /** + * 数仓主题格式为： theme_domain_name.theme_name + */ + private String warehouseThemeName; + + /** + * 数仓主题英文 + */ + private String warehouseThemeNameEn; + /** + * 生命周期 + */ + private String lifecycle; + + + private String lifecycleEn; + + + private Integer isPartitionTable; + + private Integer isAvailable; + + /** + * 存储类型：hive/mysql + */ + private String storageType; + + /** + * 授权的名字：userName、roleName + */ + private String principalName; + + /** + * 压缩格式 + */ + private String compress; + + /** + * 文件格式 + */ + private String fileType; + + /** + * 版本信息：默认1 + */ + private String version; + + /** + * 是否外部表 0 内部表 1外部表 + */ + private Integer isExternal; + + /** + * 外部表时 location + */ + private String location; + + + private Long size; + + + private String guid; + +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/dto/TableQueryDTO.java b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/dto/TableQueryDTO.java new file mode 100644 index 0000000000..dbe40067cd --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/dto/TableQueryDTO.java @@ -0,0 +1,170 @@ +package com.webank.wedatasphere.dss.datamodel.table.dto; + + +import com.google.common.collect.Lists; +import com.webank.wedatasphere.dss.data.governance.entity.HiveTblStatsDTO; +import com.webank.wedatasphere.dss.datamodel.center.common.constant.ColumnType; +import com.webank.wedatasphere.dss.datamodel.center.common.constant.LabelConstant; +import com.webank.wedatasphere.dss.datamodel.center.common.constant.TabelExternalType; +import lombok.Data; +import lombok.ToString; +import org.apache.commons.lang.StringUtils; +import org.springframework.util.CollectionUtils; + +import java.util.Date; +import java.util.List; +import java.util.Optional; + +@Data +@ToString +public class TableQueryDTO { + + private Long id; + + private String dataBase; + + private String name; + + private String alias; + + private String creator; + + private String comment; + + private Date createTime; + + private Date updateTime; + + private Date lastAccessTime; + + /** + * 数仓层级 + */ + private String warehouseLayerName; + + /** + * 数仓层级英文 + */ + private String warehouseLayerNameEn; + + /** + * 数仓主题格式为： theme_domain_name.theme_name + */ + private String warehouseThemeName; + + /** + * 数仓主题英文 + */ + private String warehouseThemeNameEn; + + /** + * 生命周期 + */ + private String lifecycle; + + private String lifecycleEn; + + private Integer isPartitionTable; + + private Integer isAvailable; + + /** + * 存储类型：hive/mysql + */ + private String storageType; + + /** + * 授权的名字：userName、roleName + */ + private String principalName; + + /** + * 压缩格式 + */ + private String compress; + + /** + * 文件格式 + */ + private String fileType; + + /** + * 版本信息：默认1 + */ + private String version; + + + /** + * 是否外部表 0 内部表 1外部表 + */ + private Integer isExternal; + + /** + * 外部表时 location + */ + private String location; + + private List columns = Lists.newArrayList(); + + /** + * 标签 + */ + private String label; + + + private TableStatsDTO stats = new TableStatsDTO(); + + + private TableHeadlineDTO headline; + + public static TableQueryDTO toTableStatsDTO(HiveTblDetailInfoDTO hiveTblDTO, HiveTblStatsDTO hiveTblStatsDTO, String name){ + TableQueryDTO dto = new TableQueryDTO(); + dto.setName(name); + dto.setCreateTime(hiveTblDTO.getBasic().getCreateTime()); + dto.setLastAccessTime(hiveTblDTO.getBasic().getLastAccessTime()); + dto.setCreator(hiveTblDTO.getBasic().getOwner()); + dto.setComment(hiveTblDTO.getBasic().getComment()); + + Optional tableExternalType = TabelExternalType.getByType(hiveTblDTO.getBasic().getExternal()); + tableExternalType.ifPresent(e -> dto.setIsExternal(e.getCode())); + + if (!CollectionUtils.isEmpty(hiveTblDTO.getBasic().getLabels())){ + StringBuilder sb = new StringBuilder(); + for (String label:hiveTblDTO.getBasic().getLabels()){ + sb.append(label).append(LabelConstant.SEPARATOR); + } + dto.setLabel(StringUtils.substringBeforeLast(sb.toString(),LabelConstant.SEPARATOR)); + } + dto.setIsPartitionTable(hiveTblDTO.getBasic().getIsParTbl()?ColumnType.PARTITION_KEY.getCode():ColumnType.COLUMN.getCode()); + + if (!CollectionUtils.isEmpty(hiveTblDTO.getColumns())){ + hiveTblDTO.getColumns().forEach(hiveColumn->{ + TableColumnQueryDTO columnQueryDTO = new TableColumnQueryDTO(); + columnQueryDTO.setComment(hiveColumn.getComment()); + columnQueryDTO.setName(hiveColumn.getName()); + columnQueryDTO.setType(hiveColumn.getType()); + columnQueryDTO.setIsPartitionField(ColumnType.COLUMN.getCode()); + dto.columns.add(columnQueryDTO); + }); + } + + if (!CollectionUtils.isEmpty(hiveTblDTO.getPartitionKeys())){ + hiveTblDTO.getPartitionKeys().forEach(hiveColumn->{ + TableColumnQueryDTO columnQueryDTO = new TableColumnQueryDTO(); + columnQueryDTO.setComment(hiveColumn.getComment()); + columnQueryDTO.setName(hiveColumn.getName()); + columnQueryDTO.setType(hiveColumn.getType()); + columnQueryDTO.setIsPartitionField(ColumnType.PARTITION_KEY.getCode()); + dto.columns.add(columnQueryDTO); + }); + } + TableHeadlineDTO headlineDTO = new TableHeadlineDTO(); + headlineDTO.setStorageType(0); + headlineDTO.setTableType(0); + headlineDTO.setEntityType(1); + dto.setHeadline(headlineDTO); + + dto.setStats(TableStatsDTO.from(hiveTblStatsDTO,0)); + return dto; + } +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/dto/TableStatsDTO.java b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/dto/TableStatsDTO.java new file mode 100644 index 0000000000..4289a17bf3 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/dto/TableStatsDTO.java @@ -0,0 +1,73 @@ +package com.webank.wedatasphere.dss.datamodel.table.dto; + +import com.webank.wedatasphere.dss.data.governance.entity.HiveTblStatsDTO; +import lombok.Data; +import lombok.ToString; + +import java.util.Date; + +@Data +@ToString +public class TableStatsDTO { + + private Long id; + + private String dataBase; + + private String name; + + private Date createTime; + + private Date updateTime; + + /** + * 字段数 + */ + private Integer columnCount = 0; + + /** + * 存储大小 + */ + private Long storageSize = 0L; + + /** + * 文件数 + */ + private Integer fileCount = 0; + + /** + * 分区数 + */ + private Integer partitionCount = 0; + + /** + * 访问次数 + */ + private Integer accessCount = 0; + + /** + * 收藏次数 + */ + private Integer collectCount = 0; + + /** + * 引用次数 + */ + private Integer refCount = 0; + + /** + * 版本信息：默认1 + */ + private String version; + + + public static TableStatsDTO from(HiveTblStatsDTO hiveTblStatsDTO,Integer collectCount){ + TableStatsDTO dto = new TableStatsDTO(); + dto.setColumnCount(hiveTblStatsDTO.getColumnCount()); + dto.setFileCount(hiveTblStatsDTO.getNumFiles()); + dto.setPartitionCount(hiveTblStatsDTO.getPartitionCount()); + dto.setStorageSize(hiveTblStatsDTO.getTotalSize()); + dto.setCollectCount(collectCount); + return dto; + } +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/dto/TableVersionQueryDTO.java b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/dto/TableVersionQueryDTO.java new file mode 100644 index 0000000000..a60a68497d --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/dto/TableVersionQueryDTO.java @@ -0,0 +1,51 @@ +package com.webank.wedatasphere.dss.datamodel.table.dto; + + +import lombok.Data; + +import java.util.Date; + +@Data +public class TableVersionQueryDTO { + + private Long id; + + private Long tblId; + + private String name; + + /** + * 是否物化 + */ + private Integer isMaterialized; + + /** + * 创建table的sql + */ + private String tableCode; + + /** + * 版本注释 + */ + private String comment; + + /** + * 版本信息：默认 1 + */ + private String version; + + private String tableParams; + + private String columns; + + /** + * rollback,update,add + */ + private String sourceType; + + private Date createTime; + + private Date updateTime; + + private String creator; +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/entity/DssDatamodelDictionary.java b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/entity/DssDatamodelDictionary.java new file mode 100644 index 0000000000..a8e50abc00 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/entity/DssDatamodelDictionary.java @@ -0,0 +1,100 @@ +package com.webank.wedatasphere.dss.datamodel.table.entity; + +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableId; + +import java.util.Date; + +/** + * 数模字典 + */ +public class DssDatamodelDictionary { + /** + * 主键id + */ + @TableId(type = IdType.AUTO) + private Integer id; + + /** + * 类型code + */ + private String code; + + /** + * 字典类型 + */ + private String type; + + /** + * 描述 + */ + private String description; + + /** + * 创建时间 + */ + private Date createdTime; + + /** + * 更新时间 + */ + private Date updateTime; + + private Integer sort; + + public Integer getId() { + return id; + } + + public void setId(Integer id) { + this.id = id; + } + + public String getCode() { + return code; + } + + public void setCode(String code) { + this.code = code; + } + + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + + public Date getCreatedTime() { + return createdTime; + } + + public void setCreatedTime(Date createdTime) { + this.createdTime = createdTime; + } + + public Date getUpdateTime() { + return updateTime; + } + + public void setUpdateTime(Date updateTime) { + this.updateTime = updateTime; + } + + public Integer getSort() { + return sort; + } + + public void setSort(Integer sort) { + this.sort = sort; + } +} \ No newline at end of file diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/entity/DssDatamodelLabel.java b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/entity/DssDatamodelLabel.java new file mode 100644 index 0000000000..4bcbd1da5f --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/entity/DssDatamodelLabel.java @@ -0,0 +1,47 @@ +package com.webank.wedatasphere.dss.datamodel.table.entity; + +import java.util.Date; + +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableId; +import lombok.Data; + +@Data +public class DssDatamodelLabel { + @TableId(type = IdType.AUTO) + private Long id; + + private String name; + + private String fieldIdentifier; + + /** + * 标签键值对 json + */ + private String params; + + private String comment; + + /** + * 数仓主题格式为： theme_domain_name.theme_name + */ + private String warehouseThemeName; + + private String owner; + + /** + * 授权的名字：userName、roleName + */ + private String principalName; + + private Integer isAvailable; + + private Date createTime; + + private Date updateTime; + + /** + * 英文 + */ + private String warehouseThemeNameEn; +} \ No newline at end of file diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/entity/DssDatamodelTabelQuery.java b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/entity/DssDatamodelTabelQuery.java new file mode 100644 index 0000000000..412256ea58 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/entity/DssDatamodelTabelQuery.java @@ -0,0 +1,83 @@ +package com.webank.wedatasphere.dss.datamodel.table.entity; + + +import lombok.Data; + +import java.util.Date; + +@Data +public class DssDatamodelTabelQuery { + private Long id; + + private String dataBase; + + private String name; + + private String alias; + + private String creator; + + private String comment; + + private Date createTime; + + private Date updateTime; + + /** + * 数仓层级 + */ + private String warehouseLayerName; + + /** + * 数仓主题格式为： theme_domain_name.theme_name + */ + private String warehouseThemeName; + + /** + * 生命周期 + */ + private String lifecycle; + + private Integer isPartitionTable; + + private Integer isAvailable; + + /** + * 存储类型：hive/mysql + */ + private String storageType; + + /** + * 授权的名字：userName、roleName + */ + private String principalName; + + /** + * 压缩格式 + */ + private String compress; + + /** + * 文件格式 + */ + private String fileType; + + /** + * 版本信息：默认1 + */ + private String version; + + /** + * 是否外部表 0 内部表 1外部表 + */ + private Integer isExternal; + + /** + * 外部表时 location + */ + private String location; + + private Integer modelType; + + private String modelName; +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/entity/DssDatamodelTable.java b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/entity/DssDatamodelTable.java new file mode 100644 index 0000000000..8bd88066cd --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/entity/DssDatamodelTable.java @@ -0,0 +1,275 @@ +package com.webank.wedatasphere.dss.datamodel.table.entity; + +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableId; +import lombok.Data; + +import java.util.Date; + +@Data +public class DssDatamodelTable { + @TableId(type = IdType.AUTO) + private Long id; + + private String dataBase; + + private String name; + + private String alias; + + private String creator; + + private String comment; + + private Date createTime; + + private Date updateTime; + + /** + * 数仓层级 + */ + private String warehouseLayerName; + + /** + * 数仓层级英文 + */ + private String warehouseLayerNameEn; + + /** + * 数仓主题格式为： theme_domain_name.theme_name + */ + private String warehouseThemeName; + + /** + * 数仓主题英文 + */ + private String warehouseThemeNameEn; + + /** + * 生命周期 + */ + private String lifecycle; + + private String lifecycleEn; + + private Integer isPartitionTable; + + private Integer isAvailable; + + /** + * 存储类型：hive/mysql + */ + private String storageType; + + /** + * 授权的名字：userName、roleName + */ + private String principalName; + + /** + * 压缩格式 + */ + private String compress; + + /** + * 文件格式 + */ + private String fileType; + + /** + * 版本信息：默认1 + */ + private String version; + + /** + * 是否外部表 0 内部表 1外部表 + */ + private Integer isExternal; + + /** + * 外部表时 location + */ + private String location; + + /** + * 标签 + */ + private String label; + + + public String getDataBase() { + return dataBase; + } + + public void setDataBase(String dataBase) { + this.dataBase = dataBase; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getAlias() { + return alias; + } + + public void setAlias(String alias) { + this.alias = alias; + } + + public String getCreator() { + return creator; + } + + public void setCreator(String creator) { + this.creator = creator; + } + + public String getComment() { + return comment; + } + + public void setComment(String comment) { + this.comment = comment; + } + + public Date getCreateTime() { + return createTime; + } + + public void setCreateTime(Date createTime) { + this.createTime = createTime; + } + + public Date getUpdateTime() { + return updateTime; + } + + public void setUpdateTime(Date updateTime) { + this.updateTime = updateTime; + } + + public String getWarehouseLayerName() { + return warehouseLayerName; + } + + public void setWarehouseLayerName(String warehouseLayerName) { + this.warehouseLayerName = warehouseLayerName; + } + + public String getWarehouseLayerNameEn() { + return warehouseLayerNameEn; + } + + public void setWarehouseLayerNameEn(String warehouseLayerNameEn) { + this.warehouseLayerNameEn = warehouseLayerNameEn; + } + + public String getWarehouseThemeName() { + return warehouseThemeName; + } + + public void setWarehouseThemeName(String warehouseThemeName) { + this.warehouseThemeName = warehouseThemeName; + } + + public String getWarehouseThemeNameEn() { + return warehouseThemeNameEn; + } + + public void setWarehouseThemeNameEn(String warehouseThemeNameEn) { + this.warehouseThemeNameEn = warehouseThemeNameEn; + } + + public String getLifecycle() { + return lifecycle; + } + + public void setLifecycle(String lifecycle) { + this.lifecycle = lifecycle; + } + + public Integer getIsPartitionTable() { + return isPartitionTable; + } + + public void setIsPartitionTable(Integer isPartitionTable) { + this.isPartitionTable = isPartitionTable; + } + + public Integer getIsAvailable() { + return isAvailable; + } + + public void setIsAvailable(Integer isAvailable) { + this.isAvailable = isAvailable; + } + + public String getStorageType() { + return storageType; + } + + public void setStorageType(String storageType) { + this.storageType = storageType; + } + + public String getPrincipalName() { + return principalName; + } + + public void setPrincipalName(String principalName) { + this.principalName = principalName; + } + + public String getCompress() { + return compress; + } + + public void setCompress(String compress) { + this.compress = compress; + } + + public String getFileType() { + return fileType; + } + + public void setFileType(String fileType) { + this.fileType = fileType; + } + + public String getVersion() { + return version; + } + + public void setVersion(String version) { + this.version = version; + } + + public Integer getIsExternal() { + return isExternal; + } + + public void setIsExternal(Integer isExternal) { + this.isExternal = isExternal; + } + + public String getLocation() { + return location; + } + + public void setLocation(String location) { + this.location = location; + } + + public String getLabel() { + return label; + } + + public void setLabel(String label) { + this.label = label; + } +} \ No newline at end of file diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/entity/DssDatamodelTableCollcetion.java b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/entity/DssDatamodelTableCollcetion.java new file mode 100644 index 0000000000..90e204e3b3 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/entity/DssDatamodelTableCollcetion.java @@ -0,0 +1,110 @@ +package com.webank.wedatasphere.dss.datamodel.table.entity; + +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableId; +import lombok.Data; + +import java.util.Date; + +@Data +public class DssDatamodelTableCollcetion { + @TableId(type = IdType.AUTO) + private Long id; + + private String dataBase; + + private String name; + + private String alias; + + private String creator; + + private String comment; + + private Date createTime; + + private Date updateTime; + + /** + * 数仓层级 + */ + private String warehouseLayerName; + + /** + * 数仓主题格式为： theme_domain_name.theme_name + */ + private String warehouseThemeName; + + /** + * 生命周期 + */ + private String lifecycle; + + private String lifecycleEn; + + private Integer isPartitionTable; + + private Integer isAvailable; + + /** + * 存储类型：hive/mysql + */ + private String storageType; + + /** + * 授权的名字：userName、roleName + */ + private String principalName; + + /** + * 压缩格式 + */ + private String compress; + + /** + * 文件格式 + */ + private String fileType; + + /** + * 收藏人 + */ + private String user; + + /** + * 版本信息：默认1 + */ + private String version; + + /** + * 是否外部表 0 内部表 1外部表 + */ + private Integer isExternal; + + /** + * 外部表时 location + */ + private String location; + + /** + * 标签 + */ + private String label; + + /** + * atlas标识 + */ + private String guid; + + /** + * 数仓层级英文 + */ + private String warehouseLayerNameEn; + + /** + * 数仓主题英文 + */ + private String warehouseThemeNameEn; + + +} \ No newline at end of file diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/entity/DssDatamodelTableColumns.java b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/entity/DssDatamodelTableColumns.java new file mode 100644 index 0000000000..464a4e1e6c --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/entity/DssDatamodelTableColumns.java @@ -0,0 +1,169 @@ +package com.webank.wedatasphere.dss.datamodel.table.entity; + +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableId; +import lombok.Data; + +import java.util.Date; + +@Data +public class DssDatamodelTableColumns { + @TableId(type = IdType.AUTO) + private Long id; + + private Long tableId; + + private String name; + + private String alias; + + private String type; + + private String comment; + + private Integer isPartitionField; + + private Integer isPrimary; + + private Integer length; + + private String rule; + + /** + * 0 维度，1 指标 2 度量 + */ + private Integer modelType; + + /** + * 模型信息英文名称 + */ + private String modelNameEn; + + /** + * 关联具体模型id信息（因为有版本数据表id不可靠，暂时不用） + */ + private Integer modelId; + + /** + * 模型信息名称 + */ + private String modelName; + + private Date createTime; + + private Date updateTime; + + + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getAlias() { + return alias; + } + + public void setAlias(String alias) { + this.alias = alias; + } + + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } + + public String getComment() { + return comment; + } + + public void setComment(String comment) { + this.comment = comment; + } + + public Integer getIsPartitionField() { + return isPartitionField; + } + + public void setIsPartitionField(Integer isPartitionField) { + this.isPartitionField = isPartitionField; + } + + public Integer getIsPrimary() { + return isPrimary; + } + + public void setIsPrimary(Integer isPrimary) { + this.isPrimary = isPrimary; + } + + public Integer getLength() { + return length; + } + + public void setLength(Integer length) { + this.length = length; + } + + public String getRule() { + return rule; + } + + public void setRule(String rule) { + this.rule = rule; + } + + public Integer getModelType() { + return modelType; + } + + public void setModelType(Integer modelType) { + this.modelType = modelType; + } + + public String getModelNameEn() { + return modelNameEn; + } + + public void setModelNameEn(String modelNameEn) { + this.modelNameEn = modelNameEn; + } + + public Integer getModelId() { + return modelId; + } + + public void setModelId(Integer modelId) { + this.modelId = modelId; + } + + public String getModelName() { + return modelName; + } + + public void setModelName(String modelName) { + this.modelName = modelName; + } + + public Date getCreateTime() { + return createTime; + } + + public void setCreateTime(Date createTime) { + this.createTime = createTime; + } + + public Date getUpdateTime() { + return updateTime; + } + + public void setUpdateTime(Date updateTime) { + this.updateTime = updateTime; + } +} \ No newline at end of file diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/entity/DssDatamodelTableMaterializedHistory.java b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/entity/DssDatamodelTableMaterializedHistory.java new file mode 100644 index 0000000000..c5b942a81f --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/entity/DssDatamodelTableMaterializedHistory.java @@ -0,0 +1,57 @@ +package com.webank.wedatasphere.dss.datamodel.table.entity; + +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableId; +import lombok.Data; +import lombok.ToString; + +import java.util.Date; + +@Data +@ToString +public class DssDatamodelTableMaterializedHistory { + @TableId(type = IdType.AUTO) + private Long id; + + /** + * 物化sql + */ + private String materializedCode; + + /** + * 物化原因 + */ + private String reason; + + /** + * 物化者 + */ + private String creator; + + /** + * succeed,failed,in progess + */ + private Integer status; + + private Date createTime; + + private Date lastUpdateTime; + + private String taskId; + + private String errorMsg; + + /** + * 表名 + */ + private String tablename; + + private String dataBase; + + /** + * 版本信息：默认1 + */ + private String version; + + +} \ No newline at end of file diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/entity/DssDatamodelTableStatics.java b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/entity/DssDatamodelTableStatics.java new file mode 100644 index 0000000000..16ca259b7a --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/entity/DssDatamodelTableStatics.java @@ -0,0 +1,80 @@ +package com.webank.wedatasphere.dss.datamodel.table.entity; + +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableId; + +public class DssDatamodelTableStatics { + @TableId(type = IdType.AUTO) + private Long id; + + private String originTables; + + private Long tableId; + + private Integer accessCount; + + private Integer lastAccessTime; + + /** + * 存储10行用例数据 + */ + private String sampleDataPath; + + private Integer sampleUpdateTime; + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public String getOriginTables() { + return originTables; + } + + public void setOriginTables(String originTables) { + this.originTables = originTables; + } + + public Long getTableId() { + return tableId; + } + + public void setTableId(Long tableId) { + this.tableId = tableId; + } + + public Integer getAccessCount() { + return accessCount; + } + + public void setAccessCount(Integer accessCount) { + this.accessCount = accessCount; + } + + public Integer getLastAccessTime() { + return lastAccessTime; + } + + public void setLastAccessTime(Integer lastAccessTime) { + this.lastAccessTime = lastAccessTime; + } + + public String getSampleDataPath() { + return sampleDataPath; + } + + public void setSampleDataPath(String sampleDataPath) { + this.sampleDataPath = sampleDataPath; + } + + public Integer getSampleUpdateTime() { + return sampleUpdateTime; + } + + public void setSampleUpdateTime(Integer sampleUpdateTime) { + this.sampleUpdateTime = sampleUpdateTime; + } +} \ No newline at end of file diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/entity/DssDatamodelTableStats.java b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/entity/DssDatamodelTableStats.java new file mode 100644 index 0000000000..3ece606c5e --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/entity/DssDatamodelTableStats.java @@ -0,0 +1,163 @@ +package com.webank.wedatasphere.dss.datamodel.table.entity; + +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableId; + +import java.util.Date; + +public class DssDatamodelTableStats { + @TableId(type = IdType.AUTO) + private Long id; + + private String dataBase; + + private String name; + + private Date createTime; + + private Date updateTime; + + /** + * 字段数 + */ + private Integer columnCount; + + /** + * 存储大小 + */ + private Integer storageSize; + + /** + * 文件数 + */ + private Integer fileCount; + + /** + * 分区数 + */ + private Integer partitionCount; + + /** + * 访问次数 + */ + private Integer accessCount; + + /** + * 收藏次数 + */ + private Integer collectCount; + + /** + * 引用次数 + */ + private Integer refCount; + + /** + * 版本信息：默认1 + */ + private String version; + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public String getDataBase() { + return dataBase; + } + + public void setDataBase(String dataBase) { + this.dataBase = dataBase; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public Date getCreateTime() { + return createTime; + } + + public void setCreateTime(Date createTime) { + this.createTime = createTime; + } + + public Date getUpdateTime() { + return updateTime; + } + + public void setUpdateTime(Date updateTime) { + this.updateTime = updateTime; + } + + public Integer getColumnCount() { + return columnCount; + } + + public void setColumnCount(Integer columnCount) { + this.columnCount = columnCount; + } + + public Integer getStorageSize() { + return storageSize; + } + + public void setStorageSize(Integer storageSize) { + this.storageSize = storageSize; + } + + public Integer getFileCount() { + return fileCount; + } + + public void setFileCount(Integer fileCount) { + this.fileCount = fileCount; + } + + public Integer getPartitionCount() { + return partitionCount; + } + + public void setPartitionCount(Integer partitionCount) { + this.partitionCount = partitionCount; + } + + public Integer getAccessCount() { + return accessCount; + } + + public void setAccessCount(Integer accessCount) { + this.accessCount = accessCount; + } + + public Integer getCollectCount() { + return collectCount; + } + + public void setCollectCount(Integer collectCount) { + this.collectCount = collectCount; + } + + public Integer getRefCount() { + return refCount; + } + + public void setRefCount(Integer refCount) { + this.refCount = refCount; + } + + public String getVersion() { + return version; + } + + public void setVersion(String version) { + this.version = version; + } +} \ No newline at end of file diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/entity/DssDatamodelTableVersion.java b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/entity/DssDatamodelTableVersion.java new file mode 100644 index 0000000000..83689f366e --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/entity/DssDatamodelTableVersion.java @@ -0,0 +1,154 @@ +package com.webank.wedatasphere.dss.datamodel.table.entity; + +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableId; + +import java.util.Date; + +public class DssDatamodelTableVersion { + @TableId(type = IdType.AUTO) + private Long id; + + private Long tblId; + + private String name; + + /** + * 是否物化 + */ + private Integer isMaterialized; + + /** + * 创建table的sql + */ + private String tableCode; + + /** + * 版本注释 + */ + private String comment; + + /** + * 版本信息：默认version0002 + */ + private String version; + + private String tableParams; + + private String columns; + + /** + * rollback,update,add + */ + private String sourceType; + + private Date createTime; + + private Date updateTime; + + private String creator; + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public Long getTblId() { + return tblId; + } + + public void setTblId(Long tblId) { + this.tblId = tblId; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public Integer getIsMaterialized() { + return isMaterialized; + } + + public void setIsMaterialized(Integer isMaterialized) { + this.isMaterialized = isMaterialized; + } + + public String getTableCode() { + return tableCode; + } + + public void setTableCode(String tableCode) { + this.tableCode = tableCode; + } + + public String getComment() { + return comment; + } + + public void setComment(String comment) { + this.comment = comment; + } + + public String getVersion() { + return version; + } + + public void setVersion(String version) { + this.version = version; + } + + public String getTableParams() { + return tableParams; + } + + public void setTableParams(String tableParams) { + this.tableParams = tableParams; + } + + public String getColumns() { + return columns; + } + + public void setColumns(String columns) { + this.columns = columns; + } + + public String getSourceType() { + return sourceType; + } + + public void setSourceType(String sourceType) { + this.sourceType = sourceType; + } + + public Date getCreateTime() { + return createTime; + } + + public void setCreateTime(Date createTime) { + this.createTime = createTime; + } + + public Date getUpdateTime() { + return updateTime; + } + + public void setUpdateTime(Date updateTime) { + this.updateTime = updateTime; + } + + public String getCreator() { + return creator; + } + + public void setCreator(String creator) { + this.creator = creator; + } +} \ No newline at end of file diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/event/BindLabelByTableEvent.java b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/event/BindLabelByTableEvent.java new file mode 100644 index 0000000000..6c0630353c --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/event/BindLabelByTableEvent.java @@ -0,0 +1,22 @@ +package com.webank.wedatasphere.dss.datamodel.table.event; + + +import com.webank.wedatasphere.dss.datamodel.table.entity.DssDatamodelTable; +import lombok.Getter; +import lombok.ToString; +import org.springframework.context.ApplicationEvent; + +@Getter +@ToString +public class BindLabelByTableEvent extends ApplicationEvent { + + private String user; + + private DssDatamodelTable table; + + public BindLabelByTableEvent(Object source,String user,DssDatamodelTable table) { + super(source); + this.user = user; + this.table = table; + } +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/event/BindModelByColumnsEvent.java b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/event/BindModelByColumnsEvent.java new file mode 100644 index 0000000000..27f16ed691 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/event/BindModelByColumnsEvent.java @@ -0,0 +1,25 @@ +package com.webank.wedatasphere.dss.datamodel.table.event; + + +import com.webank.wedatasphere.dss.datamodel.table.entity.DssDatamodelTableColumns; +import lombok.Getter; +import org.springframework.context.ApplicationEvent; + +import java.util.List; + +@Getter +public class BindModelByColumnsEvent extends ApplicationEvent { + + private String user; + + private String tableName; + + List columns; + + public BindModelByColumnsEvent(Object source, String user, String tableName, List columns) { + super(source); + this.user = user; + this.tableName = tableName; + this.columns = columns; + } +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/event/BindModelByTableEvent.java b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/event/BindModelByTableEvent.java new file mode 100644 index 0000000000..55b30e7ded --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/event/BindModelByTableEvent.java @@ -0,0 +1,20 @@ +package com.webank.wedatasphere.dss.datamodel.table.event; + + +import com.webank.wedatasphere.dss.datamodel.table.entity.DssDatamodelTable; +import lombok.Getter; +import org.springframework.context.ApplicationEvent; + +@Getter +public class BindModelByTableEvent extends ApplicationEvent { + + private String user; + + private DssDatamodelTable table; + + public BindModelByTableEvent(Object source,String user,DssDatamodelTable table) { + super(source); + this.user = user; + this.table = table; + } +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/event/TableFirstBindEvent.java b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/event/TableFirstBindEvent.java new file mode 100644 index 0000000000..aae0783449 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/event/TableFirstBindEvent.java @@ -0,0 +1,18 @@ +package com.webank.wedatasphere.dss.datamodel.table.event; + +import lombok.Getter; +import org.springframework.context.ApplicationEvent; + +@Getter +public class TableFirstBindEvent extends ApplicationEvent { + private String user; + private Long tableId; + private String tableName; + + public TableFirstBindEvent(Object source, String user, Long tableId, String tableName) { + super(source); + this.user = user; + this.tableId = tableId; + this.tableName = tableName; + } +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/event/UnBindLabelByTableEvent.java b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/event/UnBindLabelByTableEvent.java new file mode 100644 index 0000000000..06f7302c55 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/event/UnBindLabelByTableEvent.java @@ -0,0 +1,22 @@ +package com.webank.wedatasphere.dss.datamodel.table.event; + +import com.webank.wedatasphere.dss.datamodel.table.entity.DssDatamodelTable; +import lombok.Getter; +import lombok.ToString; +import org.springframework.context.ApplicationEvent; +import org.springframework.stereotype.Component; + +@ToString +@Getter +public class UnBindLabelByTableEvent extends ApplicationEvent { + + private String user; + + private DssDatamodelTable table; + + public UnBindLabelByTableEvent(Object source,String user,DssDatamodelTable table) { + super(source); + this.user = user; + this.table = table; + } +} \ No newline at end of file diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/event/UnBindModelByColumnsEvent.java b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/event/UnBindModelByColumnsEvent.java new file mode 100644 index 0000000000..7d8c8a067f --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/event/UnBindModelByColumnsEvent.java @@ -0,0 +1,24 @@ +package com.webank.wedatasphere.dss.datamodel.table.event; + +import com.webank.wedatasphere.dss.datamodel.table.entity.DssDatamodelTableColumns; +import lombok.Getter; +import org.springframework.context.ApplicationEvent; + +import java.util.List; + +@Getter +public class UnBindModelByColumnsEvent extends ApplicationEvent { + + private String user; + + private String tableName; + + List columns; + + public UnBindModelByColumnsEvent(Object source, String user, String tableName, List columns) { + super(source); + this.user = user; + this.tableName = tableName; + this.columns = columns; + } +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/event/UnBindModelByTableEvent.java b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/event/UnBindModelByTableEvent.java new file mode 100644 index 0000000000..5a0b7d6d83 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/event/UnBindModelByTableEvent.java @@ -0,0 +1,19 @@ +package com.webank.wedatasphere.dss.datamodel.table.event; + + +import com.webank.wedatasphere.dss.datamodel.table.entity.DssDatamodelTable; +import lombok.Getter; +import org.springframework.context.ApplicationEvent; +@Getter +public class UnBindModelByTableEvent extends ApplicationEvent { + + private String user; + + private DssDatamodelTable table; + + public UnBindModelByTableEvent(Object source,String user,DssDatamodelTable table) { + super(source); + this.user = user; + this.table = table; + } +} \ No newline at end of file diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/event/UpdateBindLabelByTableEvent.java b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/event/UpdateBindLabelByTableEvent.java new file mode 100644 index 0000000000..679bc4e8be --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/event/UpdateBindLabelByTableEvent.java @@ -0,0 +1,23 @@ +package com.webank.wedatasphere.dss.datamodel.table.event; + +import com.webank.wedatasphere.dss.datamodel.table.entity.DssDatamodelTable; +import lombok.Getter; +import lombok.ToString; +import org.springframework.context.ApplicationEvent; + +@ToString +@Getter +public class UpdateBindLabelByTableEvent extends ApplicationEvent { + private String user; + + private DssDatamodelTable ori; + + private DssDatamodelTable updateOne; + + public UpdateBindLabelByTableEvent(Object source, String user, DssDatamodelTable ori, DssDatamodelTable updateOne) { + super(source); + this.user = user; + this.ori = ori; + this.updateOne = updateOne; + } +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/event/UpdateBindModelByColumnsEvent.java b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/event/UpdateBindModelByColumnsEvent.java new file mode 100644 index 0000000000..3b869b36fd --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/event/UpdateBindModelByColumnsEvent.java @@ -0,0 +1,27 @@ +package com.webank.wedatasphere.dss.datamodel.table.event; + + +import com.webank.wedatasphere.dss.datamodel.table.entity.DssDatamodelTableColumns; +import lombok.Getter; +import org.springframework.context.ApplicationEvent; + +import java.util.List; + +@Getter +public class UpdateBindModelByColumnsEvent extends ApplicationEvent { + private String user; + + private String tableName; + + private List orgColumns; + + private List updateColumns; + + public UpdateBindModelByColumnsEvent(Object source, String user, String tableName, List orgColumns, List updateColumns) { + super(source); + this.user = user; + this.tableName = tableName; + this.orgColumns = orgColumns; + this.updateColumns = updateColumns; + } +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/event/UpdateBindModelByTableEvent.java b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/event/UpdateBindModelByTableEvent.java new file mode 100644 index 0000000000..65629a87ad --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/event/UpdateBindModelByTableEvent.java @@ -0,0 +1,22 @@ +package com.webank.wedatasphere.dss.datamodel.table.event; + + +import com.webank.wedatasphere.dss.datamodel.table.entity.DssDatamodelTable; +import lombok.Getter; +import org.springframework.context.ApplicationEvent; + +@Getter +public class UpdateBindModelByTableEvent extends ApplicationEvent { + private String user; + + private DssDatamodelTable org; + + private DssDatamodelTable updateOne; + + public UpdateBindModelByTableEvent(Object source, String user, DssDatamodelTable org, DssDatamodelTable updateOne) { + super(source); + this.user = user; + this.org = org; + this.updateOne = updateOne; + } +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/listener/TableFirstBindListener.java b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/listener/TableFirstBindListener.java new file mode 100644 index 0000000000..2f20ff6ec9 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/listener/TableFirstBindListener.java @@ -0,0 +1,47 @@ +package com.webank.wedatasphere.dss.datamodel.table.listener; + +import com.webank.wedatasphere.dss.datamodel.table.event.BindLabelByTableEvent; +import com.webank.wedatasphere.dss.datamodel.table.event.TableFirstBindEvent; +import com.webank.wedatasphere.dss.datamodel.table.service.TableMaterializedHistoryService; +import com.webank.wedatasphere.dss.datamodel.table.service.TableService; +import org.apache.linkis.common.exception.ErrorException; +import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.context.event.EventListener; +import org.springframework.scheduling.annotation.Async; +import org.springframework.stereotype.Component; + +import javax.annotation.Resource; +import java.util.Arrays; +import java.util.concurrent.TimeUnit; + +@Component +public class TableFirstBindListener { + private static final Logger LOGGER = LoggerFactory.getLogger(TableFirstBindListener.class); + + @Resource + private TableService tableService; + + @Resource + private TableMaterializedHistoryService tableMaterializedHistoryService; + + + @EventListener + @Async("taskExecutor") + public void firstBindTable(TableFirstBindEvent event) throws ErrorException, InterruptedException { + int count = 0; + do { + LOGGER.info("table id : {}, tableName : {}, user : {} wait 5s to bind ",event.getTableId(),event.getTableName(),event.getUser()); + TimeUnit.SECONDS.sleep(5); + if (tableMaterializedHistoryService.tableExists(event.getTableName(), event.getUser())) { + LOGGER.info("table id : {}, tableName : {}, user : {} exists bind",event.getTableId(),event.getTableName(),event.getUser()); + tableService.bind(event.getTableId(), event.getUser()); + count++; + break; + } + count++; + } while (count < 5); + LOGGER.info("table id : {}, tableName : {}, user : {} try bind {} times ",event.getTableId(),event.getTableName(),event.getUser(),count); + } +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/listener/TableLabelListener.java b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/listener/TableLabelListener.java new file mode 100644 index 0000000000..8c23c97003 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/listener/TableLabelListener.java @@ -0,0 +1,91 @@ +package com.webank.wedatasphere.dss.datamodel.table.listener; + +import com.webank.wedatasphere.dss.datamodel.center.common.constant.LabelConstant; +import com.webank.wedatasphere.dss.datamodel.center.common.event.BindLabelEvent; +import com.webank.wedatasphere.dss.datamodel.center.common.event.UnBindLabelEvent; +import com.webank.wedatasphere.dss.datamodel.table.entity.DssDatamodelTable; +import com.webank.wedatasphere.dss.datamodel.table.event.BindLabelByTableEvent; +import com.webank.wedatasphere.dss.datamodel.table.event.UnBindLabelByTableEvent; +import com.webank.wedatasphere.dss.datamodel.table.event.UpdateBindLabelByTableEvent; +import jersey.repackaged.com.google.common.collect.Sets; +import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.context.ApplicationEventPublisher; +import org.springframework.context.event.EventListener; +import org.springframework.scheduling.annotation.Async; +import org.springframework.stereotype.Component; + +import javax.annotation.Resource; +import java.util.Arrays; +import java.util.Set; +import java.util.stream.Collectors; + +@Component +public class TableLabelListener { + private static final Logger LOGGER = LoggerFactory.getLogger(TableLabelListener.class); + + public static final String LABEL_SEPARATOR = LabelConstant.SEPARATOR; + + @Resource + private ApplicationEventPublisher publisher; + + @EventListener + @Async("taskExecutor") + public void bindLabelByTable(BindLabelByTableEvent event){ + if (preEvent(event.getTable())) return; + String labels = event.getTable().getLabel(); + String[] binds = StringUtils.split(labels,LABEL_SEPARATOR); + + Arrays.stream(binds).forEach(bind->{ + publishBind(event.getUser(),bind,event.getTable().getName()); + }); + } + + private boolean preEvent(DssDatamodelTable table) { + return StringUtils.isBlank(table.getLabel()); + } + + private void publishBind(String user,String labelName,String tableName){ + publisher.publishEvent(new BindLabelEvent(this,user,labelName,tableName)); + } + + private void publishUnBind(String user,String labelName,String tableName){ + publisher.publishEvent(new UnBindLabelEvent(this,user,labelName,tableName)); + } + + @EventListener + @Async("taskExecutor") + public void unBindLabelByTable(UnBindLabelByTableEvent event){ + if (preEvent(event.getTable())) return; + String labels = event.getTable().getLabel(); + String[] unBinds = StringUtils.split(labels,LABEL_SEPARATOR); + + Arrays.stream(unBinds).forEach(bind->{ + publishBind(event.getUser(),bind,event.getTable().getName()); + }); + } + + @EventListener + @Async("taskExecutor") + public void updateBindLabelByTable(UpdateBindLabelByTableEvent event){ + String tableName = event.getOri().getName(); + String bindLabels = event.getUpdateOne().getLabel(); + String unBindLabels = event.getOri().getLabel(); + Set bindSets = Sets.newHashSet(); + Set unBindSets = Sets.newHashSet(); + if (StringUtils.isNotBlank(bindLabels)){ + bindSets.addAll(Arrays.stream(StringUtils.split(bindLabels,LABEL_SEPARATOR)).collect(Collectors.toSet())); + } + if (StringUtils.isNotBlank(unBindLabels)){ + unBindSets.addAll(Arrays.stream(StringUtils.split(unBindLabels,LABEL_SEPARATOR)).collect(Collectors.toSet())); + } + + Set willBinds = bindSets.stream().filter(bind->!unBindSets.contains(bind)).collect(Collectors.toSet()); + Set willUnBinds = unBindSets.stream().filter(unBind->!bindLabels.contains(unBind)).collect(Collectors.toSet()); + LOGGER.info("bind labels : {}",willBinds); + willBinds.forEach(bind->publishBind(event.getUser(),bind,tableName)); + LOGGER.info("unBind labels : {}",willUnBinds); + willUnBinds.forEach(unBind->publishUnBind(event.getUser(),unBind,tableName)); + } +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/listener/TableModelListener.java b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/listener/TableModelListener.java new file mode 100644 index 0000000000..7683ce74b0 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/listener/TableModelListener.java @@ -0,0 +1,147 @@ +package com.webank.wedatasphere.dss.datamodel.table.listener; + + +import com.webank.wedatasphere.dss.data.governance.entity.ClassificationConstant; +import com.webank.wedatasphere.dss.datamodel.center.common.event.BindModelEvent; +import com.webank.wedatasphere.dss.datamodel.center.common.event.UnBindModelEvent; +import com.webank.wedatasphere.dss.datamodel.table.dto.ModelTypeDTO; +import com.webank.wedatasphere.dss.datamodel.table.entity.DssDatamodelTable; +import com.webank.wedatasphere.dss.datamodel.table.entity.DssDatamodelTableColumns; +import com.webank.wedatasphere.dss.datamodel.table.event.*; +import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.context.ApplicationEventPublisher; +import org.springframework.context.event.EventListener; +import org.springframework.scheduling.annotation.Async; +import org.springframework.stereotype.Component; + +import javax.annotation.Resource; +import java.util.Optional; +import java.util.Set; +import java.util.function.Predicate; +import java.util.stream.Collectors; + +@Component +public class TableModelListener { + + private static final Logger LOGGER = LoggerFactory.getLogger(TableModelListener.class); + + + @Resource + private ApplicationEventPublisher publisher; + + @EventListener + @Async("taskExecutor") + public void bindByTable(BindModelByTableEvent event){ + bindByTable(event.getUser(),event.getTable()); + } + + @EventListener + @Async("taskExecutor") + public void unBindByTable(UnBindModelByTableEvent event){ + unBindByTable(event.getUser(),event.getTable()); + } + + @Async("taskExecutor") + @EventListener + public void updateBindByTable(UpdateBindModelByTableEvent event){ + if (StringUtils.equals(event.getOrg().getName(),event.getUpdateOne().getName())){ + return; + } + String user = event.getUser(); + String tableName = event.getOrg().getName(); + + String orgLayerNameEn = event.getOrg().getWarehouseLayerNameEn(); + String updateLayerNameEn = event.getUpdateOne().getWarehouseLayerNameEn(); + //分层名称不同 则换绑 + if (!StringUtils.equals(orgLayerNameEn,updateLayerNameEn)){ + publishBind(user,null, tableName,updateLayerNameEn,ClassificationConstant.LAYER); + publishUnBind(user,null, tableName,orgLayerNameEn,ClassificationConstant.LAYER); + } + + + String orgThemeNameEn = event.getOrg().getWarehouseThemeNameEn(); + String updateThemeNameEn = event.getUpdateOne().getWarehouseThemeNameEn(); + //主题名称不同 则换绑 + if (!StringUtils.equals(orgThemeNameEn,updateThemeNameEn)){ + publishBind(user,null, tableName,updateThemeNameEn,ClassificationConstant.THEME); + publishUnBind(user,null, tableName,orgThemeNameEn,ClassificationConstant.THEME); + } + } + + + + + @Async("taskExecutor") + @EventListener + public void updateBindByColumns(UpdateBindModelByColumnsEvent event){ + Set orgModels = event.getUpdateColumns().stream().filter(filterNull()).map(c->new ModelTypeDTO(c.getModelType(),c.getModelNameEn(),c.getModelName())).collect(Collectors.toSet()); + Set updateModels = event.getOrgColumns().stream().filter(filterNull()).map(c->new ModelTypeDTO(c.getModelType(),c.getModelNameEn(),c.getModelName())).collect(Collectors.toSet()); + + //去重 获取绑定列表 + Set bindModels = updateModels.stream().filter(bindOne->!orgModels.contains(bindOne)).collect(Collectors.toSet()); + LOGGER.info("bindModels : {}",bindModels); + //去重 获取解绑列表 + Set unBindModels = orgModels.stream().filter(unBindOne->!updateModels.contains(unBindOne)).collect(Collectors.toSet()); + LOGGER.info("unBindModels : {}",unBindModels); + + bindByColumnsModel(event.getUser(),event.getTableName(),bindModels); + unBindByColumnsModel(event.getUser(), event.getTableName(), unBindModels); + } + + private void unBindByTable(String user,DssDatamodelTable deleteOne) { + publishUnBind(user,null, deleteOne.getName(), deleteOne.getWarehouseThemeNameEn(),ClassificationConstant.THEME); + publishUnBind(user,null, deleteOne.getName(), deleteOne.getWarehouseLayerNameEn(),ClassificationConstant.LAYER); + } + + private void bindByTable(String user,DssDatamodelTable newOne) { + publishBind(user,null, newOne.getName(), newOne.getWarehouseThemeNameEn(),ClassificationConstant.THEME); + publishBind(user,null, newOne.getName(), newOne.getWarehouseLayerNameEn(),ClassificationConstant.LAYER); + } + + private void publishBind(String user, String guid, String tableName, String modelName, ClassificationConstant modelType){ + publisher.publishEvent(new BindModelEvent(this,user,guid,tableName,modelName,modelType)); + } + + private void publishUnBind(String user, String guid, String tableName, String modelName, ClassificationConstant modelType){ + publisher.publishEvent(new UnBindModelEvent(this,user,guid,tableName,modelName,modelType)); + } + + @EventListener + @Async("taskExecutor") + public void bindByColumnsModel(BindModelByColumnsEvent event){ + bindByColumnsModel(event.getUser(),event.getTableName(),event.getColumns().stream().filter(filterNull()).map(c->new ModelTypeDTO(c.getModelType(),c.getModelNameEn(),c.getModelName())).collect(Collectors.toSet())); + } + + //过滤所有空值 + private Predicate filterNull() { + return c -> c.getModelType() != null && c.getModelName() != null && c.getModelNameEn() != null; + } + + @EventListener + @Async("taskExecutor") + public void unBindByColumnsModel(UnBindModelByColumnsEvent event){ + unBindByColumnsModel(event.getUser(),event.getTableName(),event.getColumns().stream().filter(filterNull()).map(c->new ModelTypeDTO(c.getModelType(),c.getModelNameEn(),c.getModelName())).collect(Collectors.toSet())); + } + + private void bindByColumnsModel(String user, String tableName, Set modelTypeDTOS){ + for (ModelTypeDTO model : modelTypeDTOS){ + Optional optional = ClassificationConstant.getClassificationConstantByType(model.getModelType()); + if (!optional.isPresent()){ + continue; + } + publishBind(user,null, tableName, model.getModelNameEn(),optional.get()); + } + } + + private void unBindByColumnsModel(String user, String tableName, Set modelTypeDTOS){ + for (ModelTypeDTO column : modelTypeDTOS){ + Optional optional = ClassificationConstant.getClassificationConstantByType(column.getModelType()); + if (!optional.isPresent()){ + continue; + } + publishUnBind(user,null, tableName, column.getModelNameEn(),optional.get()); + } + } +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/materialized/HiveSchema.java b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/materialized/HiveSchema.java new file mode 100644 index 0000000000..3aa6a73426 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/materialized/HiveSchema.java @@ -0,0 +1,180 @@ +package com.webank.wedatasphere.dss.datamodel.table.materialized; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + + +public class HiveSchema { + + private Map columnMap = new HashMap(); + private List columnList = new ArrayList(); + private String tableName; + private String dataBase; + private String originalDataFormat; + private String storedType; + private boolean isExternal =false; + private String location; + private String comment; + public List partitionList = new ArrayList(); + + public String getComment() { + return comment; + } + + public void setComment(String comment) { + this.comment = comment; + } + + public String getLocation() { + return location; + } + + public void setLocation(String location) { + this.location = location; + } + + public void setExternal(boolean external) { + isExternal = external; + } + + public void setStoredType(String storedType) { + this.storedType = storedType; + } + + public String getStoredType() { + return storedType; + } + + public boolean isExternal() { + return isExternal; + } + + public String getDataBase() { + return dataBase; + } + + public void setDataBase(String dataBase) { + this.dataBase = dataBase; + } + + public List getPartitionColumns() { + return partitionList; + } + + public List getColumns() { + return columnList; + } + + public Column getColumn(String name) { + return columnMap.get(name); + } + + + public void setTableName(String tableName) { + this.tableName = tableName; + } + + public String getTableName() { + return tableName; + } + + + public String getOriginalDataFormat() { + return originalDataFormat; + } + + public void setOriginalDataFormat(String originalDataFormat) { + this.originalDataFormat = originalDataFormat; + } + + public void addColumn(String name, String type, int length, int percision, boolean isPartition, boolean isPrimaryKey,String comment) { + Column column = new Column(name, type, length, percision, isPartition, isPrimaryKey,comment); + columnList.add(column); + columnMap.put(name, column); + if (isPartition) { + partitionList.add(column); + } + } + + + public static class Column { + String name; + String type; + int length; + int percision; + private String comment; + boolean isPartition; + boolean isPrimaryKey; + + public Column(String name, String type, int length, int percision, boolean isPartition, boolean isPrimaryKey,String comment) { + super(); + this.name = name; + this.type = type; + this.length = length; + this.percision = percision; + this.isPartition = isPartition; + this.isPrimaryKey = isPrimaryKey; + this.comment = comment; + } + + public String getComment() { + return comment; + } + + public void setComment(String comment) { + this.comment = comment; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } + + public int getLength() { + return length; + } + + public void setLength(int length) { + this.length = length; + } + + public int getPercision() { + return percision; + } + + public void setPercision(int percision) { + this.percision = percision; + } + + public boolean isPartition() { + return isPartition; + } + + public void setPartition(boolean isPartition) { + this.isPartition = isPartition; + } + + public boolean isPrimaryKey() { + return isPrimaryKey; + } + + public void setPrimaryKey(boolean isPrimaryKey) { + this.isPrimaryKey = isPrimaryKey; + } + + + } +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/materialized/HiveSchemaDdlBuilder.java b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/materialized/HiveSchemaDdlBuilder.java new file mode 100644 index 0000000000..6ae6d0f646 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/materialized/HiveSchemaDdlBuilder.java @@ -0,0 +1,182 @@ +package com.webank.wedatasphere.dss.datamodel.table.materialized; + +import com.webank.wedatasphere.dss.datamodel.table.materialized.HiveSchema.Column; +import org.apache.commons.lang.StringUtils; + +import java.util.List; + + +public class HiveSchemaDdlBuilder { + public static String lineSeparator = System.getProperty("line.separator"); + + private String fieldsTerminated = "\'\\t\'"; + + private String linesTerminated = "\'\\n\'"; + + private HiveSchema hiveSchema; + + private boolean isExternal = false; + + public HiveSchemaDdlBuilder() { + hiveSchema = new HiveSchema(); + } + + + public HiveSchemaDdlBuilder tableName(String tableName) { + hiveSchema.setTableName(tableName); + return this; + } + + public HiveSchemaDdlBuilder storedType(String storedType) { + hiveSchema.setStoredType(storedType); + return this; + } + + public HiveSchemaDdlBuilder location(String location) { + hiveSchema.setLocation(location); + return this; + } + + public HiveSchemaDdlBuilder comment(String comment) { + hiveSchema.setComment(comment); + return this; + } + + + public HiveSchemaDdlBuilder withExternal() { + hiveSchema.setExternal(true); + return this; + } + + public HiveSchemaDdlBuilder fieldsTerminated(String fieldsTerminated) { + this.fieldsTerminated = fieldsTerminated; + return this; + } + + public HiveSchemaDdlBuilder linesTerminated(String linesTerminated) { + this.linesTerminated = linesTerminated; + return this; + } + + public HiveSchemaDdlBuilder dataBase(String dataBase) { + hiveSchema.setDataBase(dataBase); + return this; + } + + public HiveSchemaDdlBuilder addColumn(String name, String type, int length, int percision, boolean isPartition, boolean isPrimaryKey, String comment) { + hiveSchema.addColumn(name, type, length, percision, isPartition, isPrimaryKey, comment); + return this; + } + + public void addColumn(String name, String type, boolean isPartition, String comment) { + addColumn(name,type,0,0,isPartition,false,comment); + } + + public String createTableString() { + StringBuilder builder = new StringBuilder(); + + builder.append("CREATE "); + + if (hiveSchema.isExternal()) { + builder.append("EXTERNAL "); + } + +// builder.append("TABLE " + hiveSchema.getDataBase() + "." + hiveSchema.getTableName() +// + lineSeparator); + builder.append("TABLE IF NOT EXISTS ").append(hiveSchema.getTableName()).append(lineSeparator); + + + builder.append("( "); + boolean isFirstColumn = true; + for (HiveSchema.Column c : hiveSchema.getColumns()) { + if (!c.isPartition()) { + if (isFirstColumn) { + isFirstColumn = false; + } else { + builder.append(","); + } + builder.append(lineSeparator); + builder.append(" ").append(c.getName()).append(" ").append(c.getType()); + if (c.getComment() != null && !c.getComment().equals("")) { + builder.append(" ").append("COMMENT '").append(c.getComment()).append("'"); + } + } + } + + builder.append(") ").append(lineSeparator); + + if (hiveSchema.getComment() != null && !hiveSchema.getComment().equals("")) { + builder.append("COMMENT '").append(hiveSchema.getComment()).append("'").append(lineSeparator); + } + + List partitionColumns = hiveSchema.getPartitionColumns(); + if (partitionColumns.size() > 0) { + builder.append("PARTITIONED BY ("); + + isFirstColumn = true; + for (Column c : partitionColumns) { + if (isFirstColumn) { + isFirstColumn = false; + } else { + builder.append(", "); + } + builder.append(c.getName()).append(" ").append(c.getType()); + } + + builder.append(")").append(lineSeparator); + } + + if (hiveSchema.getStoredType() != null) { + builder.append("STORED AS ").append(hiveSchema.getStoredType()).append(lineSeparator); + } + + if (hiveSchema.isExternal()&&StringUtils.isNotBlank(hiveSchema.getLocation())) { + builder.append("LOCATION ").append("'").append(hiveSchema.getLocation()).append("'"); + } + + + builder.append(";"); + + return builder.toString(); + } + + private static String convertColumnType(Column column) { + String dbType = column.getType().toUpperCase(); + + if (dbType.equals("STRING") || dbType.equals("VARCHAR2") + || dbType.equals("CHAR") || dbType.equals("VARCHAR") + || dbType.equals("CLOB")) { + return "STRING"; + } else if (dbType.equals("NUMBER") || dbType.equals("DECIMAL") + || dbType.equals("BYTEINT") || dbType.equals("SMALLINT") + || dbType.equals("INTEGER") || dbType.equals("BIGINT") + || dbType.equals("INT") || dbType.equals("FLOAT")) { + if (column.getPercision() == 0) { + if (column.getLength() > 18) { + return "STRING"; + } else if (column.getLength() > 9 || dbType.equals("BIGINT")) { + return "BIGINT"; + } else if (column.getLength() > 4 || dbType.equals("INTEGER")) { + return "INT"; + } else if (column.getLength() > 2 || dbType.equals("SMALLINT")) { + return "SMALLINT"; + } else { + return "TINYINT"; + } + } else { + if (column.getLength() > 17) { + return "STRING"; + } else { + return "DOUBLE"; + } + } + + } else if (dbType.equals("OTHERDATE") || dbType.equals("DATE") + || dbType.equals("DATETIME") || dbType.equals("TIMESTAMP")) { + return "TIMESTAMP"; + } else { + throw new RuntimeException("Currently doesn't support " + dbType + " for column " + column.getName()); + } + } + +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/restful/TableRestfulApi.java b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/restful/TableRestfulApi.java new file mode 100644 index 0000000000..b019157044 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/restful/TableRestfulApi.java @@ -0,0 +1,605 @@ +package com.webank.wedatasphere.dss.datamodel.table.restful; + +import com.webank.wedatasphere.dss.datamodel.center.common.service.AuthenticationClientStrategy; +import com.webank.wedatasphere.dss.datamodel.center.common.service.DataWarehouseReferenceService; +import com.webank.wedatasphere.dss.datamodel.table.service.LabelService; +import com.webank.wedatasphere.dss.datamodel.table.service.TableService; +import com.webank.wedatasphere.dss.datamodel.table.vo.*; +import com.webank.wedatasphere.dss.framework.workspace.client.impl.LinkisWorkSpaceRemoteClient; +import com.webank.wedatasphere.dss.framework.workspace.client.request.GetWorkspaceUsersAction; +import com.webank.wedatasphere.dss.framework.workspace.client.response.GetWorkspaceUsersResult; +import org.apache.linkis.common.exception.ErrorException; +import org.apache.linkis.server.Message; +import com.webank.wedatasphere.warehouse.client.GovernanceDwRemoteClient; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; +import org.springframework.web.bind.annotation.RequestBody; + +import javax.annotation.Resource; +import javax.servlet.http.HttpServletRequest; +import javax.ws.rs.*; +import javax.ws.rs.core.Context; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; +import java.io.IOException; + + +@Component +@Path("/datamodel/") +@Produces(MediaType.APPLICATION_JSON) +@Consumes(MediaType.APPLICATION_JSON) +public class TableRestfulApi implements AuthenticationClientStrategy { + + + private static final Logger LOGGER = LoggerFactory.getLogger(TableRestfulApi.class); + + + @Autowired + private TableService tableService; + + + @Resource + private LabelService labelService; + + + @Autowired + private GovernanceDwRemoteClient governanceDwRemoteClient; + + + @Resource + private DataWarehouseReferenceService dataWarehouseReferenceService; + + @Resource + private LinkisWorkSpaceRemoteClient linkisWorkSpaceRemoteClient; + + /** + * 新增 + * + * @param req + * @param vo + * @return + * @throws IOException + */ + @POST + @Path("/tables") + public Response add(@Context HttpServletRequest req, @RequestBody TableAddVO vo) throws ErrorException { + vo.setCreator(getStrategyUser(req)); + LOGGER.info("tablesAddVO : {}", vo); + return Message.messageToResponse(Message.ok().data("id", tableService.addTable(vo))); + } + + + /** + * 更新 + * + * @param req + * @param vo + * @return + * @throws IOException + */ + @PUT + @Path("/tables/{id}") + public Response update(@Context HttpServletRequest req, @PathParam("id") Long id, @RequestBody TableUpdateVO vo) throws ErrorException { + vo.setCreator(getStrategyUser(req)); + LOGGER.info("update id : {}, tableUpdateVO : {}", id, vo); + return Message.messageToResponse(Message.ok().data("count", tableService.updateTable(id, vo))); + } + + /** + * 删除 + * + * @param req + * @return + * @throws IOException + */ + @DELETE + @Path("/tables/{id}") + public Response delete(@Context HttpServletRequest req, @PathParam("id") Long id) throws ErrorException { + LOGGER.info("delete id : {}", id); + return Message.messageToResponse(Message.ok().data("count", tableService.deleteTable(id))); + } + + + /** + * 主动绑定 + * + * @param req + * @return + * @throws IOException + */ + @PUT + @Path("/tables/bind/{id}") + public Response bind(@Context HttpServletRequest req, @PathParam("id") Long id) throws ErrorException { + LOGGER.info("bind id : {}", id); + tableService.tryBind(id); + return Message.messageToResponse(Message.ok()); + } + + + /** + * 查看 + * + * @param req + * @param id + * @return + */ + @GET + @Path("/tables/{id}") + public Response query(@Context HttpServletRequest req, @PathParam("id") Long id) throws ErrorException { + LOGGER.info("query id : {}", id); + return Message.messageToResponse(Message.ok().data("detail", tableService.queryById(id))); + } + + + /** + * 按表名搜索查看 + * + * @param req + * @param vo + * @return + */ + @POST + @Path("/tables/name") + public Response queryName(@Context HttpServletRequest req, @RequestBody TableQueryOneVO vo) throws ErrorException { + vo.setUser(getStrategyUser(req)); + LOGGER.info("query vo : {}", vo); + return Message.messageToResponse(Message.ok().data("detail", tableService.queryByName(vo))); + } + + + /** + * 分页搜索 + * + * @param req + * @return + */ + @POST + @Path("/tables/list") + public Response list(@Context HttpServletRequest req, @RequestBody TableListVO vo) { + vo.setUser(getStrategyUser(req)); + LOGGER.info("list vo : {}", vo); + return Message.messageToResponse(tableService.list(vo)); + } + + + /** + * 新增版本 + * + * @param req + * @param vo + * @return + * @throws IOException + */ + @POST + @Path("/tables/versions/{id}") + public Response addVersion(@Context HttpServletRequest req, @PathParam("id") Long id, @RequestBody TableVersionAddVO vo) throws Exception { + vo.setCreator(getStrategyUser(req)); + LOGGER.info("tableVersionAddVO : {}", vo); + return Message.messageToResponse(Message.ok().data("count", tableService.addTableVersion(id, vo))); + } + + + /** + * 回退指定版本 + * + * @param req + * @param vo + * @return + * @throws IOException + */ + @POST + @Path("/tables/versions/rollback") + public Response versionRollBack(@Context HttpServletRequest req, @RequestBody TableVersionRollBackVO vo) throws Exception { + vo.setUser(getStrategyUser(req)); + LOGGER.info("tableVersionRollBackVO : {}", vo); + return Message.messageToResponse(Message.ok().data("count", tableService.versionRollBack(vo))); + } + + + /** + * 搜索表版本历史 + * + * @param req + * @return + */ + @POST + @Path("/tables/versions/list") + public Response tableVersionsList(@Context HttpServletRequest req, @RequestBody TableVersionQueryVO vo) { + LOGGER.info("version list vo : {}", vo); + return Message.messageToResponse(tableService.listTableVersions(vo)); + } + + +// /** +// * 相关主题可选列表 +// * +// * @param req +// * @return +// */ +// @POST +// @Path("/tables/themes/list") +// public Response tableThemesList(@Context HttpServletRequest req) { +// ListDwThemeDomainAction action = ListDwThemeDomainAction.builder().setUser(getStrategyUser(req)).setIsAvailable(true).build(); +// return Message.messageToResponse(Message.ok().data("list", governanceDwRemoteClient.listThemeDomains(action).getAll())); +// } + + + /** + * 主题引用情况 + * + * @param req + * @return + */ + @GET + @Path("/themes/reference/{name}") + public Response themesReference(@Context HttpServletRequest req, @PathParam("name") String name) { + LOGGER.info("themes reference name : {}", name); + return Message.messageToResponse(Message.ok().data("result", dataWarehouseReferenceService.themeReferenceCount(name))); + } + +// /** +// * 相关分层 +// * +// * @param req +// * @return +// */ +// @POST +// @Path("/tables/layers/list") +// public Response tableLayerList(@Context HttpServletRequest req) { +// ListDwLayerAction action = ListDwLayerAction.builder().setIsAvailable(true).setUser(getStrategyUser(req)).build(); +// return Message.messageToResponse(Message.ok().data("list", governanceDwRemoteClient.listLayers(action).getAll())); +// } + + /** + * 分层引用情况 + * + * @param req + * @return + */ + @GET + @Path("/layers/reference/{name}") + public Response layersReference(@Context HttpServletRequest req, @PathParam("name") String name) { + LOGGER.info("layers reference name : {}", name); + return Message.messageToResponse(Message.ok().data("result", dataWarehouseReferenceService.layerReferenceCount(name))); + } + + /** + * 周期引用情况 + * + * @param req + * @return + */ + @GET + @Path("/cycles/reference/{name}") + public Response cycleReference(@Context HttpServletRequest req, @PathParam("name") String name) { + LOGGER.info("cycles reference name : {}", name); + return Message.messageToResponse(Message.ok().data("result", dataWarehouseReferenceService.cycleReferenceCount(name))); + } + + /** + * 修饰词引用情况 + * + * @param req + * @return + */ + @GET + @Path("/modifiers/reference/{name}") + public Response modifiersReference(@Context HttpServletRequest req, @PathParam("name") String name) { + LOGGER.info("modifiers reference name : {}", name); + return Message.messageToResponse(Message.ok().data("result", dataWarehouseReferenceService.modifierReferenceCount(name))); + } + + /** + * 数据库列表 + * + * @param req + * @return + */ + @POST + @Path("/tables/databases/list") + public Response tableDataBasesList(@Context HttpServletRequest req, TableDatabasesQueryVO vo) { + vo.setUser(getStrategyUser(req)); + LOGGER.info("table databases vo : {}", vo); + return Message.messageToResponse(tableService.listDataBases(vo)); + } + + + /** + * 收藏 + * + * @param req + * @return + */ + @POST + @Path("/tables/collect/") + public Response tableCollect(@Context HttpServletRequest req, @RequestBody TableCollectVO vo) throws ErrorException { + vo.setUser(getStrategyUser(req)); + LOGGER.info("table collection vo : {}", vo); + return Message.messageToResponse(Message.ok().data("count", tableService.tableCollect(vo))); + } + + /** + * 取消收藏 + * + * @param req + * @return + */ + @POST + @Path("/tables/collect/cancel") + public Response tableCancelCollect(@Context HttpServletRequest req, @RequestBody TableCollectCancelVO vo) throws ErrorException { + vo.setUser(getStrategyUser(req)); + LOGGER.info("table collection cancel vo : {}", vo); + return Message.messageToResponse(Message.ok().data("count", tableService.tableCancel(vo))); + } + + + /** + * 我的收藏列表 + * + * @param req + * @return + */ + @POST + @Path("/tables/collect/list") + public Response tableCollectList(@Context HttpServletRequest req, @RequestBody TableCollectQueryVO vo) throws ErrorException { + vo.setUser(getStrategyUser(req)); + LOGGER.info("table collection list vo : {}", vo); + return Message.messageToResponse(tableService.tableCollections(vo)); + } + + /** + * 表数据预览 + * + * @param req + * @param vo + * @return + */ + @POST + @Path("/tables/data/preview/") + public Response tableDataPreview(@Context HttpServletRequest req, @RequestBody TableDataPreviewVO vo) throws ErrorException { + vo.setUser(getStrategyUser(req)); + LOGGER.info("table data preview : {}", vo); + return Message.messageToResponse(tableService.previewData(vo)); + } + + + /** + * 执行建表 + * + * @param req + * @param vo + * @return + */ + @POST + @Path("/tables/create/") + public Response tableCreate(@Context HttpServletRequest req, @RequestBody TableCreateVO vo) throws ErrorException { + vo.setUser(getStrategyUser(req)); + LOGGER.info("table create vo : {}", vo); + return Message.messageToResponse(Message.ok().data("count", tableService.tableCreate(vo))); + } + + /** + * 生成建表语句 + * + * @param req + * @param vo + * @return + */ + @POST + @Path("/tables/create/sql") + public Response tableCreateSql(@Context HttpServletRequest req, @RequestBody TableCreateSqlVO vo) throws ErrorException { + vo.setUser(getStrategyUser(req)); + LOGGER.info("table create sql vo : {}", vo); + return Message.messageToResponse(Message.ok().data("detail", tableService.tableCreateSql(vo))); + } + + + /** + * 字典列表 + * + * @param req + * @param vo + * @return + */ + @POST + @Path("/tables/dictionaries/list") + public Response tableDictionaryList(@Context HttpServletRequest req, @RequestBody TableDictionaryListVO vo) { + LOGGER.info("table dictionaries list vo : {}", vo); + return Message.messageToResponse(tableService.dictionaryList(vo)); + } + + + /** + * 新增字段 + * + * @param req + * @param vo + * @return + */ + @POST + @Path("/tables/columns/add") + public Response tableColumnsAdd(@Context HttpServletRequest req, @RequestBody TableColumnsAddVO vo) throws ErrorException { + LOGGER.info("table column add vo : {}", vo); + return Message.messageToResponse(Message.ok().data("count", tableService.addTableColumn(vo))); + } + + /** + * 字段绑定模型 + * + * @param req + * @param vo + * @return + */ + @POST + @Path("/tables/columns/bind/{columnId}") + public Response tableColumnBind(@Context HttpServletRequest req, @PathParam("columnId") Long columnId, @RequestBody TableColumnBindVO vo) throws ErrorException { + LOGGER.info("table column bind model columnId : {}, vo : {}", columnId, vo); + return Message.messageToResponse(Message.ok().data("count", tableService.tableColumnBind(columnId, vo))); + } + + /** + * 分区统计信息 + * + * @param vo + * @return + */ + @POST + @Path("/tables/partition/stats/") + public Response tblPartitionStats(@Context HttpServletRequest req, TblPartitionStatsVO vo) { + vo.setUser(getStrategyUser(req)); + LOGGER.info("table partition stats vo : {}", vo); + return Message.messageToResponse(tableService.listTablePartitionStats(vo)); + } + + + /** + * 获取当前用户 + * + * @param req + * @return + */ + @GET + @Path("/current/user") + public Response currentUser(@Context HttpServletRequest req) { + return Message.messageToResponse(Message.ok().data("user", getStrategyUser(req))); + } + + + /** + * 检测表是否有数据 + * + * @param req + * @return + */ + @POST + @Path("/tables/check/data") + public Response tableCheckData(@Context HttpServletRequest req, @RequestBody TableCheckDataVO vo) throws ErrorException { + vo.setUser(getStrategyUser(req)); + LOGGER.info("table partition stats vo : {}", vo); + return Message.messageToResponse(Message.ok().data("status", tableService.tableCheckData(vo))); + } + + + /** + * 新增标签 + * + * @param req + * @param vo + * @return + * @throws IOException + */ + @POST + @Path("/labels") + public Response addLabels(@Context HttpServletRequest req, @RequestBody LabelAddVO vo) throws ErrorException { + LOGGER.info("addLabels vo : {}", vo); + return Message.messageToResponse(Message.ok().data("id", labelService.add(vo))); + } + + + /** + * 更新标签 + * + * @param req + * @param vo + * @return + * @throws IOException + */ + @PUT + @Path("/labels/{id}") + public Response updateLabels(@Context HttpServletRequest req, @PathParam("id") Long id, @RequestBody LabelUpdateVO vo) throws ErrorException { + LOGGER.info("updateLabels id : {}, vo : {}", id, vo); + return Message.messageToResponse(Message.ok().data("count", labelService.update(id, vo))); + } + + /** + * 删除标签 + * + * @param req + * @return + * @throws IOException + */ + @DELETE + @Path("/labels/{id}") + public Response deleteLabels(@Context HttpServletRequest req, @PathParam("id") Long id) throws ErrorException { + LOGGER.info("deleteLabels id : {}", id); + return Message.messageToResponse(Message.ok().data("count", labelService.delete(id))); + } + + /** + * 查看标签 + * + * @param req + * @param id + * @return + */ + @GET + @Path("/labels/{id}") + public Response queryLabels(@Context HttpServletRequest req, @PathParam("id") Long id) throws ErrorException { + LOGGER.info("queryLabels id : {}", id); + return Message.messageToResponse(Message.ok().data("detail", labelService.query(id))); + } + + + /** + * 标签搜索 + * + * @param req + * @return + */ + @POST + @Path("/labels/list") + public Response listLabels(@Context HttpServletRequest req, @RequestBody LabelsQueryVO vo) throws ErrorException { + LOGGER.info("listLabels vo : {}", vo); + return Message.messageToResponse(labelService.list(vo)); + } + + /** + * 启用/禁用 + * + * @param req + * @param id + * @param vo + * @return + */ + @PUT + @Path("/labels/enable/{id}") + public Response enableLabel(@Context HttpServletRequest req, @PathParam("id") Long id, @RequestBody LabelEnableVO vo) throws ErrorException { + LOGGER.info("enableLabel id : {}, vo : {}", id, vo); + return Message.messageToResponse(Message.ok().data("count", labelService.enable(id, vo))); + } + + + /** + * 查询用户 + * + * @param req + * @param workspaceId + * @return + */ + @GET + @Path("/users/{workspaceId}") + public Response users(@Context HttpServletRequest req, @PathParam("workspaceId") String workspaceId) throws ErrorException { + LOGGER.info("users workspaceId : {}", workspaceId); + GetWorkspaceUsersResult result = linkisWorkSpaceRemoteClient.getWorkspaceUsers(GetWorkspaceUsersAction.builder().setUser(getStrategyUser(req)).setWorkspaceId(workspaceId).build()); + return Message.messageToResponse(Message.ok().data("users", result.getWorkspaceUserList())); + } + + + /** + * 查询角色 + * + * @param req + * @param workspaceId + * @return + */ + @GET + @Path("/roles/{workspaceId}") + public Response roles(@Context HttpServletRequest req, @PathParam("workspaceId") String workspaceId) throws ErrorException { + LOGGER.info("roles workspaceId : {}", workspaceId); + GetWorkspaceUsersResult result = linkisWorkSpaceRemoteClient.getWorkspaceUsers(GetWorkspaceUsersAction.builder().setUser(getStrategyUser(req)).setWorkspaceId(workspaceId).build()); + return Message.messageToResponse(Message.ok().data("users", result.getWorkspaceRoleList())); + } + +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/service/LabelService.java b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/service/LabelService.java new file mode 100644 index 0000000000..1e2cd63950 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/service/LabelService.java @@ -0,0 +1,77 @@ +package com.webank.wedatasphere.dss.datamodel.table.service; + + +import com.baomidou.mybatisplus.extension.service.IService; +import com.webank.wedatasphere.dss.datamodel.table.dto.LabelQueryDTO; +import com.webank.wedatasphere.dss.datamodel.table.entity.DssDatamodelLabel; +import com.webank.wedatasphere.dss.datamodel.table.vo.LabelAddVO; +import com.webank.wedatasphere.dss.datamodel.table.vo.LabelEnableVO; +import com.webank.wedatasphere.dss.datamodel.table.vo.LabelUpdateVO; +import com.webank.wedatasphere.dss.datamodel.table.vo.LabelsQueryVO; +import org.apache.linkis.common.exception.ErrorException; +import org.apache.linkis.server.Message; + +public interface LabelService extends IService { + + /** + * 新增标签 + * @param vo + * @return + * @throws ErrorException + */ + Long add(LabelAddVO vo) throws ErrorException; + + + /** + * 更新标签 + * @return + * @throws ErrorException + */ + int update(Long id, LabelUpdateVO vo) throws ErrorException; + + + /** + * 查看标签 + * @param id + * @return + * @throws ErrorException + */ + LabelQueryDTO query(Long id) throws ErrorException; + + + /** + * 启用/禁用 + * @param id + * @param vo + * @return + * @throws ErrorException + */ + int enable(Long id, LabelEnableVO vo) throws ErrorException; + + + /** + * 删除标签 + * @param id + * @return + * @throws ErrorException + */ + int delete(Long id) throws ErrorException; + + + /** + * 查询标签 + * @param vo + * @return + * @throws ErrorException + */ + Message list(LabelsQueryVO vo) throws ErrorException; + + + /** + * 标签引用主题统计 + * @param themeName + * @return + * @throws ErrorException + */ + int labelThemeReferenceCount(String themeName); +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/service/TableCollectService.java b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/service/TableCollectService.java new file mode 100644 index 0000000000..3ee1cfb4af --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/service/TableCollectService.java @@ -0,0 +1,38 @@ +package com.webank.wedatasphere.dss.datamodel.table.service; + + +import com.baomidou.mybatisplus.extension.service.IService; +import com.webank.wedatasphere.dss.datamodel.table.entity.DssDatamodelTable; +import com.webank.wedatasphere.dss.datamodel.table.entity.DssDatamodelTableCollcetion; +import com.webank.wedatasphere.dss.datamodel.table.vo.TableCollectVO; + +public interface TableCollectService extends IService { + + /** + * 收藏目标表 + * + * @param user + * @param targetTable + * @return + */ + Integer tableCollect(String user, DssDatamodelTable targetTable); + + + /** + * 收藏上传表信息 + * @param vo + * @return + */ + Integer tableCollect(TableCollectVO vo); + + + /** + * 取消收藏 + * @param user + * @param tableName + * @return + */ + Integer tableCollectCancel(String user, String tableName); + + +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/service/TableColumnsService.java b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/service/TableColumnsService.java new file mode 100644 index 0000000000..a875f11256 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/service/TableColumnsService.java @@ -0,0 +1,77 @@ +package com.webank.wedatasphere.dss.datamodel.table.service; + + +import com.baomidou.mybatisplus.extension.service.IService; +import com.webank.wedatasphere.dss.data.governance.entity.ClassificationConstant; +import com.webank.wedatasphere.dss.datamodel.center.common.constant.ModeType; +import com.webank.wedatasphere.dss.datamodel.table.entity.DssDatamodelTableColumns; +import org.apache.linkis.common.exception.ErrorException; + +import java.util.List; + +public interface TableColumnsService extends IService { + + /** + * 批量新增表字段 + * @param columnsList + * @return + */ + int batchInsert(Long tableId,List columnsList) throws ErrorException; + + + /** + * 批量更新表字段 + * @param tableId + * @param columnsList + * @return + * @throws ErrorException + */ + int batchUpdate(Long tableId,List columnsList) throws ErrorException; + + + /** + * 表字段列表 + * @param tableId + * @return + * @throws ErrorException + */ + List listByTableId(Long tableId); + + + /** + * 根据表id删除字段 + * @param tableId + * @return + * @throws ErrorException + */ + Integer deleteByTableId(Long tableId)throws ErrorException; + + + /** + * 新增数据表字段 + * @param column + * @return + */ + Integer addColumn(DssDatamodelTableColumns column); + + + + Integer tableColumnBind(Long id, Integer modelType, String modelName,String modelNameEn) throws ErrorException; + + + /** + * 模型引用计数 + * @param modeType + * @param name + * @return + */ + int modelReferenceCount(ModeType modeType, String name); + + /** + * 模型英文引用计数 + * @param modeType + * @param name + * @return + */ + int modelReferenceCountEn(ModeType modeType, String name); +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/service/TableDictionaryService.java b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/service/TableDictionaryService.java new file mode 100644 index 0000000000..cd368bd398 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/service/TableDictionaryService.java @@ -0,0 +1,17 @@ +package com.webank.wedatasphere.dss.datamodel.table.service; + + +import com.baomidou.mybatisplus.extension.service.IService; +import com.webank.wedatasphere.dss.datamodel.table.entity.DssDatamodelDictionary; + +import java.util.List; + +public interface TableDictionaryService extends IService { + + /** + * 根据类型搜索字典 + * @param type + * @return + */ + List listByType(String type); +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/service/TableMaterializedHistoryService.java b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/service/TableMaterializedHistoryService.java new file mode 100644 index 0000000000..5541713e5f --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/service/TableMaterializedHistoryService.java @@ -0,0 +1,97 @@ +package com.webank.wedatasphere.dss.datamodel.table.service; + + +import com.baomidou.mybatisplus.extension.service.IService; +import com.webank.wedatasphere.dss.datamodel.center.common.dto.CreateTableDTO; +import com.webank.wedatasphere.dss.datamodel.table.dto.HiveTblSimpleInfoDTO; +import com.webank.wedatasphere.dss.datamodel.table.entity.DssDatamodelTable; +import com.webank.wedatasphere.dss.datamodel.table.entity.DssDatamodelTableMaterializedHistory; +import org.apache.linkis.common.exception.ErrorException; + +import java.util.Optional; + +public interface TableMaterializedHistoryService extends IService { + + /** + * 物化表结构 + * @param current + * @param user + * @return + */ + Integer materializedTable(DssDatamodelTable current, String user) throws ErrorException; + + + /** + * 生成建表语句 + * @param current + * @return + */ + String generateSql(DssDatamodelTable current); + + + /** + * 检查是否有数据 + * @param current + * @param user + * @throws ErrorException + */ + void checkData(DssDatamodelTable current, String user) throws ErrorException; + + + /** + * 判断是否有数据 + * + * @param tableName @return + * @param user + */ + boolean hasData(String tableName, String user); + + + /** + * + * @param tableName + * @param user + * @throws ErrorException + * @return + */ + boolean tableExists(String tableName, String user) throws ErrorException; + + + /** + * 删除表 + * @param tableName + * @param user + * @return + * @throws ErrorException + */ + boolean dropTable(String tableName, String user) throws ErrorException; + + + /** + * 创建表 + * @param current + * @param user + * @return + * @throws ErrorException + */ + CreateTableDTO createTable(DssDatamodelTable current, String user) throws ErrorException; + + + /** + * 查看是否物化 + * @param tableName + * @param version + * @return + * @throws ErrorException + */ + boolean isMaterialized(String tableName, String version) throws ErrorException; + + + /** + * 根据表名查询查询资产 + * @param tableName + * @return + * @throws ErrorException + */ + Optional getHiveTblSimpleInfoByName(String tableName, String user)throws ErrorException; +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/service/TableService.java b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/service/TableService.java new file mode 100644 index 0000000000..ef3b0235cd --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/service/TableService.java @@ -0,0 +1,267 @@ +package com.webank.wedatasphere.dss.datamodel.table.service; + + +import com.baomidou.mybatisplus.extension.service.IService; +import com.webank.wedatasphere.dss.datamodel.table.dto.TableQueryDTO; +import com.webank.wedatasphere.dss.datamodel.table.entity.DssDatamodelTable; +import com.webank.wedatasphere.dss.datamodel.table.vo.*; +import org.apache.linkis.common.exception.ErrorException; +import org.apache.linkis.server.Message; + +public interface TableService extends IService { + + /** + * 新增表 + * @param vo + * @return + */ + Long addTable(TableAddVO vo) throws ErrorException; + + + /** + * 更新 + * + * @param id + * @param vo + * @return + * @throws ErrorException + */ + int updateTable(Long id, TableUpdateVO vo) throws ErrorException; + + + /** + * 按表名搜索 + * @param vo + * @return + */ + TableQueryDTO queryByName(TableQueryOneVO vo)throws ErrorException; + + + /** + * 按id搜索 + * @param id + * @return + */ + TableQueryDTO queryById(Long id)throws ErrorException; + + + /** + * 新增版本 + * @param id + * @return + * @throws ErrorException + */ + Integer addTableVersion(Long id, TableVersionAddVO vo) throws ErrorException; + + /** + * 版本回退 + * @param vo + * @return + * @throws ErrorException + */ + Integer versionRollBack(TableVersionRollBackVO vo) throws ErrorException; + + + /** + * 查询历史版本列表 + * @param vo + * @return + */ + Message listTableVersions(TableVersionQueryVO vo); + + + /** + * 添加我的收藏 + * @param vo + * @return + */ + Integer tableCollect(TableCollectVO vo) throws ErrorException; + + /** + * 取消收藏 + * @param vo + * @return + * @throws ErrorException + */ + Integer tableCancel(TableCollectCancelVO vo) throws ErrorException; + + /** + * 基于收藏搜索 + * @param vo + * @return + * @throws ErrorException + */ + Message tableCollections(TableCollectQueryVO vo) throws ErrorException; + + /** + * + * @param vo + * @return + * @throws ErrorException + */ + Message dictionaryList(TableDictionaryListVO vo); + + + /** + * 增加字段 + * @param vo + * @return + */ + Integer addTableColumn(TableColumnsAddVO vo) throws ErrorException; + + + /** + * 字段绑定模型信息 + * @param columnId + * @param vo + * @return + */ + Integer tableColumnBind(Long columnId, TableColumnBindVO vo)throws ErrorException; + + + /** + * 执行建表 + * @param vo + * @return + * @throws ErrorException + */ + Integer tableCreate(TableCreateVO vo) throws ErrorException; + + + + /** + * 生成建表语句 + * @param vo + * @return + * @throws ErrorException + */ + String tableCreateSql(TableCreateSqlVO vo) throws ErrorException; + + + /** + * table 列表 + * @param vo + * @return + * @throws ErrorException + */ + Message list(TableListVO vo); + + + /** + * 分区统计信息 + * @param vo + * @return + */ + Message listTablePartitionStats(TblPartitionStatsVO vo); + + + /** + * 库列表 + * @return + * @param vo + */ + Message listDataBases(TableDatabasesQueryVO vo); + + + /** + * 数据预览 + * @param vo + * @return + */ + Message previewData(TableDataPreviewVO vo) throws ErrorException; + + + /** + * 检测表状态 + * @param vo + * @return + */ + Integer tableCheckData(TableCheckDataVO vo) throws ErrorException; + + + /** + * 主题引用情况 + * @param name + * @return + */ + int tableThemeReferenceCount(String name); + + /** + * 分层引用情况 + * @param name + * @return + */ + int tableLayerReferenceCount(String name); + + + /** + * 周期引用情况 + * @param name + * @return + */ + int tableCycleReferenceCount(String name); + + + /** + * 修饰词引用情况 + * @param name + * @return + */ + int tableModifierReferenceCount(String name); + + + /** + * 维度引用情况 + * @param name + * @return + */ + int tableDimensionReferenceCount(String name); + + + /** + * 度量引用情况 + * @param name + * @return + */ + int tableMeasureReferenceCount(String name); + + + /** + * 指标引用情况 + * @param name + * @return + */ + int tableIndicatorReferenceCount(String name); + + /** + * 主动绑定模型 + * @param id + * @param user + */ + void bind(long id, String user) throws ErrorException; + + + /** + * 主动绑定模型 + * @param id + * + */ + void tryBind(long id) throws ErrorException; + + + /** + * 删除逻辑表 + * @param id + * @return + * @throws ErrorException + */ + int deleteTable(Long id) throws ErrorException; + + + /** + * 标签引用情况 + * @param name + * @return + */ + int tableLabelReferenceCount(String name); +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/service/TableStatsService.java b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/service/TableStatsService.java new file mode 100644 index 0000000000..9c211b3d5e --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/service/TableStatsService.java @@ -0,0 +1,15 @@ +package com.webank.wedatasphere.dss.datamodel.table.service; + + +import com.baomidou.mybatisplus.extension.service.IService; +import com.webank.wedatasphere.dss.datamodel.table.entity.DssDatamodelTableStats; + +public interface TableStatsService extends IService { + + /** + * 搜索指定版本表指定版本统计信息 + * @param tableName + * @return + */ + DssDatamodelTableStats queryByTableName(String tableName); +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/service/TableVersionService.java b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/service/TableVersionService.java new file mode 100644 index 0000000000..c30dc28afd --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/service/TableVersionService.java @@ -0,0 +1,61 @@ +package com.webank.wedatasphere.dss.datamodel.table.service; + +import com.baomidou.mybatisplus.extension.service.IService; +import com.webank.wedatasphere.dss.datamodel.table.entity.DssDatamodelTable; +import com.webank.wedatasphere.dss.datamodel.table.entity.DssDatamodelTableColumns; +import com.webank.wedatasphere.dss.datamodel.table.entity.DssDatamodelTableVersion; + +import java.util.List; + + +public interface TableVersionService extends IService { + + /** + * 根据表名称查找最大版本 + * @param name + * @return + */ + String findLastVersion(String name); + + + /** + * 增加新版本 + * @param orgVersion + * @param orgColumns + * @return + */ + Long addOlderVersion(DssDatamodelTable orgVersion, List orgColumns); + + + + /** + * 查询指定版本 + * @param name + * @param version + * @return + */ + DssDatamodelTableVersion findBackup(String name,String version); + + + /** + * 表内容引用情况 + * @param content + * @return + */ + List tableContentReference(String content); + + + /** + * 字段内容引用情况 + * @param content + * @return + */ + List tableColumnsReference(String content); + + + /** + * 表多重引用 + * @return + */ + List tableContentMultipleReference(String content); +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/service/impl/DataWarehouseLabelReferenceServiceImpl.java b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/service/impl/DataWarehouseLabelReferenceServiceImpl.java new file mode 100644 index 0000000000..0f6eb76cdc --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/service/impl/DataWarehouseLabelReferenceServiceImpl.java @@ -0,0 +1,19 @@ +package com.webank.wedatasphere.dss.datamodel.table.service.impl; + +import com.webank.wedatasphere.dss.datamodel.center.common.service.DataWarehouseLabelReferenceService; +import com.webank.wedatasphere.dss.datamodel.table.service.LabelService; +import org.springframework.stereotype.Service; + +import javax.annotation.Resource; + +@Service +public class DataWarehouseLabelReferenceServiceImpl implements DataWarehouseLabelReferenceService { + + @Resource + private LabelService labelService; + + @Override + public int labelThemeReferenceCount(String name) { + return labelService.labelThemeReferenceCount(name); + } +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/service/impl/DataWarehouseReferenceServiceImpl.java b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/service/impl/DataWarehouseReferenceServiceImpl.java new file mode 100644 index 0000000000..ada8079393 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/service/impl/DataWarehouseReferenceServiceImpl.java @@ -0,0 +1,50 @@ +package com.webank.wedatasphere.dss.datamodel.table.service.impl; + + +import com.webank.wedatasphere.dss.datamodel.center.common.service.*; +import org.springframework.stereotype.Service; + +import javax.annotation.Resource; + +@Service +public class DataWarehouseReferenceServiceImpl implements DataWarehouseReferenceService { + + @Resource + private DataWarehouseTableReferenceService tableReferenceService; + + @Resource + private DataWarehouseIndicatorReferenceService indicatorReferenceService; + + @Resource + private DataWarehouseMeasuredReferenceService measuredReferenceService; + + @Resource + private DataWarehouseDimensionReferenceService dimensionReferenceService; + + @Resource + private DataWarehouseLabelReferenceService labelReferenceService; + + @Override + public int cycleReferenceCount(String name) { + return tableReferenceService.tableCycleReferenceCount(name) + indicatorReferenceService.indicatorCycleReferenceCount(name); + } + + @Override + public int layerReferenceCount(String name) { + return tableReferenceService.tableLayerReferenceCount(name) + indicatorReferenceService.indicatorLayerReferenceCount(name); + } + + @Override + public int modifierReferenceCount(String name) { + return tableReferenceService.tableModifierReferenceCount(name) + indicatorReferenceService.indicatorModifierReferenceCount(name); + } + + @Override + public int themeReferenceCount(String name) { + return tableReferenceService.tableThemeReferenceCount(name) + + indicatorReferenceService.indicatorThemeReferenceCount(name) + + dimensionReferenceService.dimensionThemeReferenceCount(name) + + measuredReferenceService.measureThemeReferenceCount(name) + + labelReferenceService.labelThemeReferenceCount(name); + } +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/service/impl/DataWarehouseTableReferenceServiceImpl.java b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/service/impl/DataWarehouseTableReferenceServiceImpl.java new file mode 100644 index 0000000000..2d6bd5d183 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/service/impl/DataWarehouseTableReferenceServiceImpl.java @@ -0,0 +1,34 @@ +package com.webank.wedatasphere.dss.datamodel.table.service.impl; + +import com.webank.wedatasphere.dss.datamodel.center.common.service.DataWarehouseTableReferenceService; +import com.webank.wedatasphere.dss.datamodel.table.service.TableService; +import org.springframework.stereotype.Service; + +import javax.annotation.Resource; + +@Service +public class DataWarehouseTableReferenceServiceImpl implements DataWarehouseTableReferenceService { + + @Resource + private TableService tableService; + + @Override + public int tableLayerReferenceCount(String name) { + return tableService.tableLayerReferenceCount(name); + } + + @Override + public int tableModifierReferenceCount(String name) { + return tableService.tableModifierReferenceCount(name); + } + + @Override + public int tableThemeReferenceCount(String name) { + return tableService.tableThemeReferenceCount(name); + } + + @Override + public int tableCycleReferenceCount(String name) { + return tableService.tableCycleReferenceCount(name); + } +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/service/impl/DatamodelTableReferenceServiceImpl.java b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/service/impl/DatamodelTableReferenceServiceImpl.java new file mode 100644 index 0000000000..6ae88cb741 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/service/impl/DatamodelTableReferenceServiceImpl.java @@ -0,0 +1,35 @@ +package com.webank.wedatasphere.dss.datamodel.table.service.impl; + + +import com.webank.wedatasphere.dss.datamodel.center.common.service.DatamodelTableReferenceService; +import com.webank.wedatasphere.dss.datamodel.table.service.TableService; +import org.springframework.stereotype.Service; + +import javax.annotation.Resource; + +@Service +public class DatamodelTableReferenceServiceImpl implements DatamodelTableReferenceService { + + @Resource + private TableService tableService; + + @Override + public int tableDimensionReferenceCount(String name) { + return tableService.tableDimensionReferenceCount(name); + } + + @Override + public int tableIndicatorReferenceCount(String name) { + return tableService.tableIndicatorReferenceCount(name); + } + + @Override + public int tableMeasuredReferenceService(String name) { + return tableService.tableMeasureReferenceCount(name); + } + + @Override + public int tableLabelReferenceCount(String name) { + return tableService.tableLabelReferenceCount(name); + } +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/service/impl/LabelServiceImpl.java b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/service/impl/LabelServiceImpl.java new file mode 100644 index 0000000000..8862543ad1 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/service/impl/LabelServiceImpl.java @@ -0,0 +1,203 @@ +package com.webank.wedatasphere.dss.datamodel.table.service.impl; + +import com.baomidou.mybatisplus.core.toolkit.Wrappers; +import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl; +import com.github.pagehelper.PageHelper; +import com.github.pagehelper.PageInfo; +import com.google.gson.Gson; +import com.webank.wedatasphere.dss.datamodel.center.common.constant.ErrorCode; +import com.webank.wedatasphere.dss.datamodel.center.common.context.DataModelSecurityContextHolder; +import com.webank.wedatasphere.dss.datamodel.center.common.event.CreateLabelEvent; +import com.webank.wedatasphere.dss.datamodel.center.common.event.DeleteLabelEvent; +import com.webank.wedatasphere.dss.datamodel.center.common.event.UpdateLabelEvent; +import com.webank.wedatasphere.dss.datamodel.center.common.exception.DSSDatamodelCenterException; +import com.webank.wedatasphere.dss.datamodel.center.common.service.AssertsSyncService; +import com.webank.wedatasphere.dss.datamodel.center.common.service.DatamodelReferencService; +import com.webank.wedatasphere.dss.datamodel.table.dao.DssDatamodelLabelMapper; +import com.webank.wedatasphere.dss.datamodel.table.dto.LabelQueryDTO; +import com.webank.wedatasphere.dss.datamodel.table.entity.DssDatamodelLabel; +import com.webank.wedatasphere.dss.datamodel.table.entity.DssDatamodelTable; +import com.webank.wedatasphere.dss.datamodel.table.service.LabelService; +import com.webank.wedatasphere.dss.datamodel.table.vo.LabelAddVO; +import com.webank.wedatasphere.dss.datamodel.table.vo.LabelEnableVO; +import com.webank.wedatasphere.dss.datamodel.table.vo.LabelUpdateVO; +import com.webank.wedatasphere.dss.datamodel.table.vo.LabelsQueryVO; +import org.apache.linkis.common.exception.ErrorException; +import org.apache.linkis.server.Message; +import org.apache.commons.lang.StringUtils; +import org.modelmapper.ModelMapper; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +import javax.annotation.Resource; +import java.util.Date; +import java.util.stream.Collectors; + +@Service +public class LabelServiceImpl extends ServiceImpl implements LabelService { + + private static final Logger LOGGER = LoggerFactory.getLogger(LabelServiceImpl.class); + + private final Gson gson = new Gson(); + + @Resource + private DatamodelReferencService datamodelReferencService; + + @Resource + private ModelMapper modelMapper; + + @Resource + private AssertsSyncService assertsSyncService; + + @Override + @Transactional(rollbackFor = Exception.class) + public Long add(LabelAddVO vo) throws ErrorException { + if (getBaseMapper().selectCount(Wrappers.lambdaQuery().eq(DssDatamodelLabel::getName, vo.getName()) + .or().eq(DssDatamodelLabel::getFieldIdentifier,vo.getFieldIdentifier())) > 0) { + LOGGER.error("errorCode : {}, label name or field identifier can not repeat, name : {}", ErrorCode.LABEL_ADD_ERROR.getCode(), vo.getName()); + throw new DSSDatamodelCenterException(ErrorCode.LABEL_ADD_ERROR.getCode(), "label name or field identifier can not repeat"); + } + + if (datamodelReferencService.labelReferenceCount(vo.getName())>0 + ||datamodelReferencService.labelReferenceCount(vo.getFieldIdentifier())>0){ + LOGGER.error("errorCode : {}, label name can not be referenced", ErrorCode.DIMENSION_ADD_ERROR.getCode()); + throw new DSSDatamodelCenterException(ErrorCode.DIMENSION_ADD_ERROR.getCode(), "label name can not be referenced"); + } + + + DssDatamodelLabel newOne = modelMapper.map(vo,DssDatamodelLabel.class); + newOne.setParams(gson.toJson(vo.getParamMap())); + newOne.setCreateTime(new Date()); + newOne.setUpdateTime(new Date()); + + getBaseMapper().insert(newOne); + + //todo 同步资产创建标签 + assertsSyncService.syncCreateLabel(new CreateLabelEvent(this + , DataModelSecurityContextHolder.getContext().getDataModelAuthentication().getUser() + , vo.getName())); + + return newOne.getId(); + } + + @Override + @Transactional(rollbackFor = Exception.class) + public int enable(Long id, LabelEnableVO vo) throws ErrorException { + DssDatamodelLabel enableOne = new DssDatamodelLabel(); + enableOne.setIsAvailable(vo.getIsAvailable()); + enableOne.setUpdateTime(new Date()); + return getBaseMapper().update(enableOne, Wrappers.lambdaUpdate().eq(DssDatamodelLabel::getId,id)); + } + + @Override + @Transactional(rollbackFor = Exception.class) + public int update(Long id , LabelUpdateVO vo) throws ErrorException { + DssDatamodelLabel ori = getBaseMapper().selectById(id); + if (ori == null) { + LOGGER.error("errorCode : {}, update label error not exists", ErrorCode.LABEL_UPDATE_ERROR.getCode()); + throw new DSSDatamodelCenterException(ErrorCode.LABEL_UPDATE_ERROR.getCode(), "update label error not exists"); + } + + if (datamodelReferencService.labelReferenceCount(ori.getName())>0 + ||datamodelReferencService.labelReferenceCount(ori.getFieldIdentifier())>0 + ||datamodelReferencService.labelReferenceCount(vo.getName())>0 + ||datamodelReferencService.labelReferenceCount(vo.getFieldIdentifier())>0){ + LOGGER.error("errorCode : {}, label name can not referenced", ErrorCode.DIMENSION_UPDATE_ERROR.getCode()); + throw new DSSDatamodelCenterException(ErrorCode.DIMENSION_UPDATE_ERROR.getCode(), "label name can not referenced"); + } + + //当更新名称时 + if (!StringUtils.equals(vo.getName(), ori.getName())) { + int repeat = getBaseMapper().selectCount(Wrappers.lambdaQuery().eq(DssDatamodelLabel::getName, vo.getName())); + if (repeat > 0) { + LOGGER.error("errorCode : {}, label name can not repeat ", ErrorCode.DIMENSION_UPDATE_ERROR.getCode()); + throw new DSSDatamodelCenterException(ErrorCode.DIMENSION_UPDATE_ERROR.getCode(), "label name can not repeat "); + } + } + + String orgFieldIdentifier = ori.getFieldIdentifier(); + //当更新标识时 + if (!StringUtils.equals(vo.getFieldIdentifier(), orgFieldIdentifier)) { + int repeat = getBaseMapper().selectCount(Wrappers.lambdaQuery().eq(DssDatamodelLabel::getFieldIdentifier, vo.getFieldIdentifier())); + if (repeat > 0) { + LOGGER.error("errorCode : {}, label field identifier can not repeat", ErrorCode.DIMENSION_UPDATE_ERROR.getCode()); + throw new DSSDatamodelCenterException(ErrorCode.DIMENSION_UPDATE_ERROR.getCode(), "label field identifier can not repeat "); + } + } + + DssDatamodelLabel updateOne = modelMapper.map(vo,DssDatamodelLabel.class); + updateOne.setParams(gson.toJson(vo.getParamMap())); + updateOne.setUpdateTime(new Date()); + getBaseMapper().update(updateOne, Wrappers.lambdaUpdate().eq(DssDatamodelLabel::getId,id)); + + //todo 同步资产 + assertsSyncService.syncUpdateLabel(new UpdateLabelEvent(this + , DataModelSecurityContextHolder.getContext().getDataModelAuthentication().getUser() + , vo.getName(),ori.getName())); + return 1; + } + + @Override + public LabelQueryDTO query(Long id) throws ErrorException { + DssDatamodelLabel dssDatamodelLabel = getBaseMapper().selectById(id); + if (dssDatamodelLabel == null){ + throw new DSSDatamodelCenterException(ErrorCode.LABEL_QUERY_ERROR.getCode(), "label id " +id +" not exists"); + } + LabelQueryDTO dto = modelMapper.map(dssDatamodelLabel,LabelQueryDTO.class); + dto.setRefCount(datamodelReferencService.labelReferenceCount(dssDatamodelLabel.getName())); + return dto; + } + + @Override + @Transactional(rollbackFor = Exception.class) + public int delete(Long id) throws ErrorException { + DssDatamodelLabel dssDatamodelLabel = getBaseMapper().selectById(id); + if (dssDatamodelLabel == null){ + throw new DSSDatamodelCenterException(ErrorCode.LABEL_DELETE_ERROR.getCode(), "label id " +id +" not exists"); + } + + //校验引用情况 + if(datamodelReferencService.labelReferenceCount(dssDatamodelLabel.getName())>0||datamodelReferencService.labelReferenceCount(dssDatamodelLabel.getFieldIdentifier())>0){ + throw new DSSDatamodelCenterException(ErrorCode.LABEL_DELETE_ERROR.getCode(), "label id " +id +" has referenced"); + } + + getBaseMapper().deleteById(id); + + //todo 同步资产 + assertsSyncService.syncDeleteLabel(new DeleteLabelEvent(this + , DataModelSecurityContextHolder.getContext().getDataModelAuthentication().getUser() + , dssDatamodelLabel.getName())); + return 1; + } + + @Override + public Message list(LabelsQueryVO vo) throws ErrorException { + PageHelper.clearPage(); + PageHelper.startPage(vo.getPageNum(), vo.getPageSize()); + PageInfo pageInfo = new PageInfo<>(getBaseMapper().selectList(Wrappers.lambdaQuery() + .eq(StringUtils.isNotBlank(vo.getWarehouseThemeName()), DssDatamodelLabel::getWarehouseThemeName, vo.getWarehouseThemeName()) + .eq(vo.getIsAvailable()!=null,DssDatamodelLabel::getIsAvailable,vo.getIsAvailable()) + .eq(StringUtils.isNotBlank(vo.getOwner()),DssDatamodelLabel::getOwner,vo.getOwner()) + .like(StringUtils.isNotBlank(vo.getName()), DssDatamodelLabel::getName, vo.getName()))); + return Message.ok() + .data("list",pageInfo + .getList() + .stream() + .map(dssDatamodelLabel ->{ + LabelQueryDTO dto = modelMapper.map(dssDatamodelLabel, LabelQueryDTO.class); + dto.setRefCount(datamodelReferencService.labelReferenceCount(dssDatamodelLabel.getName())); + return dto; + }) + .collect(Collectors.toList())) + .data("total",pageInfo.getTotal()); + } + + @Override + public int labelThemeReferenceCount(String themeName) { + int count = getBaseMapper().selectCount(Wrappers.lambdaQuery().eq(DssDatamodelLabel::getWarehouseThemeName,themeName)); + int countEn = getBaseMapper().selectCount(Wrappers.lambdaQuery().eq(DssDatamodelLabel::getWarehouseThemeNameEn,themeName)); + return count + countEn; + } +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/service/impl/TableCollectServiceImpl.java b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/service/impl/TableCollectServiceImpl.java new file mode 100644 index 0000000000..5f94f4b61f --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/service/impl/TableCollectServiceImpl.java @@ -0,0 +1,70 @@ +package com.webank.wedatasphere.dss.datamodel.table.service.impl; + + +import com.baomidou.mybatisplus.core.toolkit.Wrappers; +import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl; +import com.webank.wedatasphere.dss.datamodel.table.dao.DssDatamodelTableCollcetionMapper; +import com.webank.wedatasphere.dss.datamodel.table.entity.DssDatamodelTable; +import com.webank.wedatasphere.dss.datamodel.table.entity.DssDatamodelTableCollcetion; +import com.webank.wedatasphere.dss.datamodel.table.service.TableCollectService; +import com.webank.wedatasphere.dss.datamodel.table.vo.TableCollectVO; +import org.modelmapper.ModelMapper; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +import javax.annotation.Resource; +import java.util.Date; + +@Service +public class TableCollectServiceImpl extends ServiceImpl implements TableCollectService { + + private static final Logger LOGGER = LoggerFactory.getLogger(TableCollectServiceImpl.class); + + @Resource + private ModelMapper modelMapper; + + @Override + @Transactional(rollbackFor = Exception.class) + public Integer tableCollect(String user, DssDatamodelTable targetTable) { + cancelOlder(user,targetTable.getName()); + DssDatamodelTableCollcetion newCollect = modelMapper.map(targetTable, DssDatamodelTableCollcetion.class); + newCollect.setCreateTime(new Date()); + newCollect.setUpdateTime(new Date()); + newCollect.setUser(user); + getBaseMapper().insert(newCollect); + return 1; + } + + + @Override + @Transactional(rollbackFor = Exception.class) + public Integer tableCollect(TableCollectVO vo) { + cancelOlder(vo.getUser(),vo.getName()); + DssDatamodelTableCollcetion newOne = modelMapper.map(vo, DssDatamodelTableCollcetion.class); + newOne.setCreateTime(new Date()); + newOne.setUpdateTime(new Date()); + getBaseMapper().insert(newOne); + return 1; + } + + private Integer cancelOlder(String user, String tableName) { + DssDatamodelTableCollcetion olderCollection = getBaseMapper().selectOne(Wrappers.lambdaQuery() + .eq(DssDatamodelTableCollcetion::getName, tableName) + .eq(DssDatamodelTableCollcetion::getUser, user)); + + //如果已经收藏过该表则删除原有收藏,方便以后新增收藏 + if (olderCollection != null&&olderCollection.getId()!=null) { + return getBaseMapper().deleteById(olderCollection.getId()); + } + return 0; + } + + + @Override + @Transactional(rollbackFor = Exception.class) + public Integer tableCollectCancel(String user, String tableName) { + return cancelOlder(user, tableName); + } +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/service/impl/TableColumnsServiceImpl.java b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/service/impl/TableColumnsServiceImpl.java new file mode 100644 index 0000000000..e37b4b690f --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/service/impl/TableColumnsServiceImpl.java @@ -0,0 +1,91 @@ +package com.webank.wedatasphere.dss.datamodel.table.service.impl; + +import com.baomidou.mybatisplus.core.toolkit.Wrappers; +import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl; +import com.webank.wedatasphere.dss.datamodel.center.common.constant.ModeType; +import com.webank.wedatasphere.dss.datamodel.table.dao.DssDatamodelTableColumnsMapper; +import com.webank.wedatasphere.dss.datamodel.table.entity.DssDatamodelTableColumns; +import com.webank.wedatasphere.dss.datamodel.table.service.TableColumnsService; +import org.apache.linkis.common.exception.ErrorException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +import java.util.Date; +import java.util.List; + + +@Service +public class TableColumnsServiceImpl extends ServiceImpl implements TableColumnsService { + + private static final Logger LOGGER = LoggerFactory.getLogger(TableColumnsServiceImpl.class); + + @Override + @Transactional(rollbackFor = Exception.class) + public int batchInsert(Long tableId,List columnsList) throws ErrorException { + for (DssDatamodelTableColumns newOne : columnsList){ + newOne.setTableId(tableId); + newOne.setCreateTime(new Date()); + newOne.setUpdateTime(new Date()); + getBaseMapper().insert(newOne); + } + return 1; + } + + + @Override + @Transactional(rollbackFor = Exception.class) + public int batchUpdate(Long tableId, List columnsList) throws ErrorException { + getBaseMapper().delete(Wrappers.lambdaQuery().eq(DssDatamodelTableColumns::getTableId, tableId)); + return batchInsert(tableId,columnsList); + } + + + @Override + public List listByTableId(Long tableId) { + return getBaseMapper().selectList(Wrappers.lambdaQuery().eq(DssDatamodelTableColumns::getTableId,tableId)); + } + + + @Override + public Integer deleteByTableId(Long tableId) throws ErrorException { + return getBaseMapper().delete(Wrappers.lambdaQuery().eq(DssDatamodelTableColumns::getTableId,tableId)); + } + + @Override + @Transactional(rollbackFor = Exception.class) + public Integer addColumn(DssDatamodelTableColumns column) { + column.setCreateTime(new Date()); + column.setUpdateTime(new Date()); + return getBaseMapper().insert(column); + } + + + @Override + public Integer tableColumnBind(Long id, Integer modelType, String modelName,String modelNameEn) throws ErrorException{ + DssDatamodelTableColumns updateOne = new DssDatamodelTableColumns(); + updateOne.setModelType(modelType); + updateOne.setModelName(modelName); + updateOne.setModelNameEn(modelNameEn); + updateOne.setUpdateTime(new Date()); + return getBaseMapper().update(updateOne, Wrappers.lambdaUpdate().eq(DssDatamodelTableColumns::getId,id)); + } + + + @Override + public int modelReferenceCount(ModeType modeType, String name) { + return getBaseMapper().selectCount( + Wrappers.lambdaQuery() + .eq(DssDatamodelTableColumns::getModelType, modeType.getType()) + .eq(DssDatamodelTableColumns::getModelName, name)); + } + + @Override + public int modelReferenceCountEn(ModeType modeType, String name) { + return getBaseMapper().selectCount( + Wrappers.lambdaQuery() + .eq(DssDatamodelTableColumns::getModelType, modeType.getType()) + .eq(DssDatamodelTableColumns::getModelNameEn, name)); + } +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/service/impl/TableDictionaryServiceImpl.java b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/service/impl/TableDictionaryServiceImpl.java new file mode 100644 index 0000000000..b1b05524af --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/service/impl/TableDictionaryServiceImpl.java @@ -0,0 +1,27 @@ +package com.webank.wedatasphere.dss.datamodel.table.service.impl; + + +import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; +import com.baomidou.mybatisplus.core.toolkit.Wrappers; +import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl; +import com.webank.wedatasphere.dss.datamodel.table.dao.DssDatamodelDictionaryMapper; +import com.webank.wedatasphere.dss.datamodel.table.entity.DssDatamodelDictionary; +import com.webank.wedatasphere.dss.datamodel.table.entity.DssDatamodelTableCollcetion; +import com.webank.wedatasphere.dss.datamodel.table.service.TableDictionaryService; +import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Service; + +import java.util.List; + +@Service +public class TableDictionaryServiceImpl extends ServiceImpl implements TableDictionaryService { + + private static final Logger LOGGER = LoggerFactory.getLogger(TableColumnsServiceImpl.class); + + @Override + public List listByType(String type) { + return getBaseMapper().selectList(Wrappers.lambdaQuery().eq(StringUtils.isNotBlank(type),DssDatamodelDictionary::getType,type)); + } +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/service/impl/TableMaterializedHistoryServiceImpl.java b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/service/impl/TableMaterializedHistoryServiceImpl.java new file mode 100644 index 0000000000..985850bb25 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/service/impl/TableMaterializedHistoryServiceImpl.java @@ -0,0 +1,261 @@ +package com.webank.wedatasphere.dss.datamodel.table.service.impl; + +import com.baomidou.mybatisplus.core.toolkit.Wrappers; +import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.reflect.TypeToken; +import com.webank.wedatasphere.dss.data.governance.entity.QueryType; +import com.webank.wedatasphere.dss.data.governance.impl.LinkisDataAssetsRemoteClient; +import com.webank.wedatasphere.dss.data.governance.request.HiveTblSizeAction; +import com.webank.wedatasphere.dss.data.governance.request.SearchHiveTblAction; +import com.webank.wedatasphere.dss.data.governance.response.SearchHiveTblResult; +import com.webank.wedatasphere.dss.datamodel.center.common.constant.ColumnType; +import com.webank.wedatasphere.dss.datamodel.center.common.constant.ErrorCode; +import com.webank.wedatasphere.dss.datamodel.center.common.constant.TabelExternalType; +import com.webank.wedatasphere.dss.datamodel.center.common.context.DataModelSecurityContextHolder; +import com.webank.wedatasphere.dss.datamodel.center.common.dto.CreateTableDTO; +import com.webank.wedatasphere.dss.datamodel.center.common.exception.DSSDatamodelCenterException; +import com.webank.wedatasphere.dss.datamodel.center.common.launcher.*; +import com.webank.wedatasphere.dss.datamodel.center.common.ujes.launcher.CreateTableDataModelUJESJobLauncher; +import com.webank.wedatasphere.dss.datamodel.center.common.ujes.launcher.DataExistsDataModelUJESJobLauncher; +import com.webank.wedatasphere.dss.datamodel.center.common.ujes.launcher.DropTableDataModelUJESJobLauncher; +import com.webank.wedatasphere.dss.datamodel.center.common.ujes.task.CreateTableDataModelUJESJobTask; +import com.webank.wedatasphere.dss.datamodel.center.common.ujes.task.DataExistsDataModelUJESJobTask; +import com.webank.wedatasphere.dss.datamodel.center.common.ujes.task.DataModelUJESJobTask; +import com.webank.wedatasphere.dss.datamodel.center.common.ujes.task.DropTableDataModelUJESJobTask; +import com.webank.wedatasphere.dss.datamodel.table.dao.DssDatamodelTableMaterializedHistoryMapper; +import com.webank.wedatasphere.dss.datamodel.table.dto.HiveTblSimpleInfoDTO; +import com.webank.wedatasphere.dss.datamodel.table.entity.DssDatamodelTable; +import com.webank.wedatasphere.dss.datamodel.table.entity.DssDatamodelTableColumns; +import com.webank.wedatasphere.dss.datamodel.table.entity.DssDatamodelTableMaterializedHistory; +import com.webank.wedatasphere.dss.datamodel.table.materialized.HiveSchemaDdlBuilder; +import com.webank.wedatasphere.dss.datamodel.table.service.TableColumnsService; +import com.webank.wedatasphere.dss.datamodel.table.service.TableMaterializedHistoryService; +import org.apache.linkis.common.exception.ErrorException; +import org.apache.linkis.computation.client.interactive.SubmittableInteractiveJob; +import org.apache.commons.lang.StringUtils; +import org.modelmapper.ModelMapper; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; +import org.springframework.util.CollectionUtils; + +import javax.annotation.Resource; +import java.util.Date; +import java.util.List; +import java.util.Objects; +import java.util.Optional; + +@Service +public class TableMaterializedHistoryServiceImpl extends ServiceImpl implements TableMaterializedHistoryService{ + + private static final Logger LOGGER = LoggerFactory.getLogger(TableMaterializedHistoryServiceImpl.class); + + @Resource + private ModelMapper modelMapper; + + @Resource + private TableColumnsService tableColumnsService; + + @Resource + private LinkisDataAssetsRemoteClient linkisDataAssetsRemoteClient; + + @Resource + private DataExistsDataModelUJESJobLauncher dataExistsDataModelUJESJobLauncher; + + @Resource + private DropTableDataModelUJESJobLauncher dropTableDataModelUJESJobLauncher; + + @Resource + private CreateTableDataModelUJESJobLauncher createTableDataModelUJESJobLauncher; + + private final Gson assertsGson = new GsonBuilder().setDateFormat("yyyy MM-dd HH:mm:ss").create(); + + @Override + @Transactional(rollbackFor = Exception.class) + public Integer materializedTable(DssDatamodelTable current, String user) throws ErrorException { + + //判断是否有数据 + checkData(current, user); + //不存在数据删除表 + dropTable(current.getName(),user); + + CreateTableDTO dto = createTable(current, user); + + //创建表 +// SubmittableInteractiveJob job = linkisCreateTable(current, sql); +// +// if (!job.isSucceed()) { +// LOGGER.error("errorCode : {}, table id : {} create table error job : {}, sql : {}", ErrorCode.TABLE_CREATE_ERROR.getCode(), current.getId(), job.getId(), sql); +// //throw new DSSDatamodelCenterException(ErrorCode.TABLE_CREATE_ERROR.getCode(), " table id : " + current.getId() + " create table error job : " + job.getId() + ", sql : " + sql); +// } + + + DssDatamodelTableMaterializedHistory newOne = modelMapper.map(current, DssDatamodelTableMaterializedHistory.class); + newOne.setId(null); + newOne.setMaterializedCode(dto.getSql()); + newOne.setCreateTime(new Date()); + newOne.setStatus(dto.getStatus() == 0 ? 0 : 1);// + newOne.setLastUpdateTime(new Date()); + newOne.setReason("reason"); + newOne.setTaskId(dto.getTaskId()); + newOne.setTablename(current.getName()); + getBaseMapper().insert(newOne); + return 1; + } + + public void checkData(DssDatamodelTable current, String user) throws ErrorException { + //linkisJobCheck(current); + //如果表存在且存在数据 + if (hasData(current.getName(), user)) { + LOGGER.error("errorCode : {}, table id : {} has data", ErrorCode.TABLE_CHECK_ERROR.getCode(), current.getId()); + throw new DSSDatamodelCenterException(ErrorCode.TABLE_CHECK_ERROR.getCode(), " table id : " + current.getId() + " has data"); + } + } + + @Override + public boolean hasData(String tableName, String user) { +// DataModelUJESJobTask dataModelUJESJobTask = DataExistsDataModelUJESJobTask.newBuilder().user(user).code(tableName).build(); +// return dataExistsDataModelUJESJobLauncher.launch(dataModelUJESJobTask); + String tblName = StringUtils.substringAfter(tableName,"."); + String dbName = StringUtils.substringBefore(tableName,"."); + return linkisDataAssetsRemoteClient. + searchHiveTblSize(HiveTblSizeAction.builder().setUser(user).setDbName(dbName).setTableName(tblName).build()) + .getResult() >0 ; + } + + @Override + public boolean tableExists(String tableName, String user) throws ErrorException { + SearchHiveTblResult result = linkisDataAssetsRemoteClient.searchHiveTbl(SearchHiveTblAction.builder().setUser(user).setQuery(tableName).setPrecise(QueryType.PRECISE).setOffset(0).setLimit(30).build()); + List dtos = assertsGson.fromJson(assertsGson.toJson(result.getResult()), new TypeToken>() { + }.getType()); + if (CollectionUtils.isEmpty(dtos)){ + return false; + } + for(HiveTblSimpleInfoDTO dto : dtos){ + if (StringUtils.equals(StringUtils.substringBeforeLast(dto.getQualifiedName(),"@"),tableName)){ + return true; + } + } + return false; + } + + + @Override + public boolean dropTable(String tableName, String user) throws ErrorException { +// if(!tableExists(tableName)){ +// return true; +// } + DataModelUJESJobTask dataModelUJESJobTask = DropTableDataModelUJESJobTask.newBuilder().user(user).code(tableName).build(); + return dropTableDataModelUJESJobLauncher.launch(dataModelUJESJobTask); + } + + @Override + public CreateTableDTO createTable(DssDatamodelTable current, String user) throws ErrorException { + String sql = buildSql(current); + LOGGER.info("table id : {}, sql : {}", current.getId(), sql); + DataModelUJESJobTask dataModelUJESJobTask = CreateTableDataModelUJESJobTask.newBuilder().user(user).code(sql).build(); + CreateTableDTO createTableDTO = createTableDataModelUJESJobLauncher.launch(dataModelUJESJobTask); + createTableDTO.setSql(sql); + return createTableDTO; + } + + private void linkisJobCheck(DssDatamodelTable current) throws DSSDatamodelCenterException { + //校验是否已数据 + DataModelJobTaskBuilder dataModELJobTaskBuilder = new DataModelJobTaskBuilder(); + DataModelJobTask task = dataModELJobTaskBuilder.withDataExistsExchangisJobTask() + .code(current.getName()) + .creator("hdfs") + .executeUser("hdfs") + .engineType("hive-2.3.3") + .runType("hql") + .build(); + DataExistsDataModelJobLauncher launcher = new DataExistsDataModelJobLauncher(); + Integer result = launcher.launch(task); + if (result > 0) { + LOGGER.error("errorCode : {}, table id : {} has data", ErrorCode.TABLE_CREATE_ERROR.getCode(), current.getId()); + throw new DSSDatamodelCenterException(ErrorCode.TABLE_CREATE_ERROR.getCode(), " table id : " + current.getId() + " has data"); + } + } + + private SubmittableInteractiveJob linkisCreateTable(DssDatamodelTable current, String sql) { + DataModelJobTaskBuilder dataModELJobTaskBuilder = new DataModelJobTaskBuilder(); + DataModelJobTask commonTask = dataModELJobTaskBuilder.withCommonExchangisJobTask() + .code(sql) + .creator("hdfs") + .executeUser("hdfs") + .engineType("hive-2.3.3") + .runType("hql") + .build(); + CommonDataModelJobLauncher commonExchangisJobLauncher = new CommonDataModelJobLauncher(); + return commonExchangisJobLauncher.launch(commonTask); + } + + private void linkisDropTable(DssDatamodelTable current) { + DataModelJobTaskBuilder dataModELJobTaskBuilder = new DataModelJobTaskBuilder(); + DataModelJobTask commonTask = dataModELJobTaskBuilder.withCommonExchangisJobTask() + .code(String.format("drop table if exists %s", current.getName())) + .creator("hdfs") + .executeUser("hdfs") + .engineType("hive-2.3.3") + .runType("hql") + .build(); + CommonDataModelJobLauncher commonExchangisJobLauncher = new CommonDataModelJobLauncher(); + commonExchangisJobLauncher.launch(commonTask); + } + + private String buildSql(DssDatamodelTable current) { + List columns = tableColumnsService.list(Wrappers.lambdaQuery().eq(DssDatamodelTableColumns::getTableId, current.getId())); + HiveSchemaDdlBuilder builder = new HiveSchemaDdlBuilder(); + builder.tableName(current.getName()); + if (StringUtils.isNotBlank(current.getComment())) { + builder.comment(current.getComment()); + } + if (Objects.equals(current.getIsExternal(), TabelExternalType.EXTERNAL.getCode())) { + builder.withExternal(); + } + if (StringUtils.isNotBlank(current.getLocation())) { + builder.location(current.getLocation()); + } + columns.forEach(column -> { + builder.addColumn(column.getName(), column.getType(), ColumnType.PARTITION_KEY.getCode().equals(column.getIsPartitionField()), column.getComment()); + }); + builder.storedType(current.getFileType()); + return builder.createTableString(); + } + + @Override + public String generateSql(DssDatamodelTable current) { + return buildSql(current); + } + + + @Override + public boolean isMaterialized(String tableName, String version) throws ErrorException { + return tableExists(tableName, DataModelSecurityContextHolder.getContext().getDataModelAuthentication().getUser()); + +// return getBaseMapper().selectCount(Wrappers.lambdaQuery() +// .eq(DssDatamodelTableMaterializedHistory::getTablename, tableName) +// .eq(DssDatamodelTableMaterializedHistory::getVersion, version) +// .eq(DssDatamodelTableMaterializedHistory::getStatus, 0)) > 0 && tableExists(tableName, DataModelSecurityContextHolder.getContext().getDataModelAuthentication().getUser()); + } + + @Override + public Optional getHiveTblSimpleInfoByName(String tableName, String user) throws ErrorException { + + SearchHiveTblResult result = linkisDataAssetsRemoteClient.searchHiveTbl(SearchHiveTblAction.builder().setPrecise(QueryType.PRECISE).setUser(user).setQuery(tableName).setOffset(0).setLimit(1).build()); + List dtos = assertsGson.fromJson(assertsGson.toJson(result.getResult()), new TypeToken>() { + }.getType()); + if (CollectionUtils.isEmpty(dtos)){ + return Optional.empty(); + } + for(HiveTblSimpleInfoDTO dto : dtos){ + if (StringUtils.equals(StringUtils.substringBeforeLast(dto.getQualifiedName(),"@"),tableName)){ + return Optional.of(dto); + } + } + return Optional.empty(); + } +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/service/impl/TableServiceImpl.java b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/service/impl/TableServiceImpl.java new file mode 100644 index 0000000000..8c7a3f33f1 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/service/impl/TableServiceImpl.java @@ -0,0 +1,833 @@ +package com.webank.wedatasphere.dss.datamodel.table.service.impl; + +import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; +import com.baomidou.mybatisplus.core.toolkit.Wrappers; +import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl; +import com.github.pagehelper.PageHelper; +import com.github.pagehelper.PageInfo; +import com.google.common.collect.Lists; +import com.google.common.collect.Sets; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.reflect.TypeToken; +import com.webank.wedatasphere.dss.data.governance.entity.ClassificationConstant; +import com.webank.wedatasphere.dss.data.governance.entity.QueryType; +import com.webank.wedatasphere.dss.data.governance.impl.LinkisDataAssetsRemoteClient; +import com.webank.wedatasphere.dss.data.governance.request.*; +import com.webank.wedatasphere.dss.data.governance.response.*; +import com.webank.wedatasphere.dss.datamodel.center.common.constant.ErrorCode; +import com.webank.wedatasphere.dss.datamodel.center.common.constant.LabelConstant; +import com.webank.wedatasphere.dss.datamodel.center.common.constant.ModeType; +import com.webank.wedatasphere.dss.datamodel.center.common.context.DataModelSecurityContextHolder; +import com.webank.wedatasphere.dss.datamodel.center.common.dto.PreviewDataDTO; +import com.webank.wedatasphere.dss.datamodel.center.common.exception.DSSDatamodelCenterException; +import com.webank.wedatasphere.dss.datamodel.center.common.ujes.task.DataModelUJESJobTask; +import com.webank.wedatasphere.dss.datamodel.center.common.ujes.launcher.PreviewDataModelUJESJobLauncher; +import com.webank.wedatasphere.dss.datamodel.center.common.ujes.task.PreviewDataModelUJESJobTask; +import com.webank.wedatasphere.dss.datamodel.table.dao.DssDatamodelTableMapper; +import com.webank.wedatasphere.dss.datamodel.table.dao.TableQueryMapper; +import com.webank.wedatasphere.dss.datamodel.table.dto.*; +import com.webank.wedatasphere.dss.datamodel.table.entity.*; +import com.webank.wedatasphere.dss.datamodel.table.event.*; +import com.webank.wedatasphere.dss.datamodel.table.service.*; +import com.webank.wedatasphere.dss.datamodel.table.vo.*; +import org.apache.linkis.common.exception.ErrorException; +import org.apache.linkis.server.Message; +import org.apache.linkis.ujes.client.exception.UJESJobException; +import org.apache.commons.lang.StringUtils; +import org.modelmapper.ModelMapper; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.context.ApplicationEventPublisher; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; +import org.springframework.util.CollectionUtils; + +import javax.annotation.Resource; +import java.util.Date; +import java.util.List; +import java.util.stream.Collectors; + + +@Service +public class TableServiceImpl extends ServiceImpl implements TableService { + + private static final Logger LOGGER = LoggerFactory.getLogger(TableServiceImpl.class); + + @Resource + private TableColumnsService tableColumnsService; + + @Resource + private TableVersionService tableVersionService; + + @Resource + private TableStatsService tableStatsService; + + @Resource + private ModelMapper modelMapper; + + @Resource + private TableCollectService tableCollectService; + + @Resource + private TableDictionaryService tableDictionaryService; + + @Resource + private TableMaterializedHistoryService tableMaterializedHistoryService; + + @Resource + private LinkisDataAssetsRemoteClient linkisDataAssetsRemoteClient; + + @Resource + private PreviewDataModelUJESJobLauncher previewDataModelUJESJobLauncher; + + @Resource + private TableQueryMapper tableQueryMapper; + + private Gson gson = new Gson(); + + @Resource + private ApplicationEventPublisher publisher; + + private final Gson assertsGson = new GsonBuilder().setDateFormat("yyyy MM-dd HH:mm:ss").create(); + + @Override + @Transactional(rollbackFor = Exception.class) + public Long addTable(TableAddVO vo) throws ErrorException { + + int sameCount = getBaseMapper().selectCount(Wrappers.lambdaQuery().eq(DssDatamodelTable::getName, vo.getName())); + if (sameCount > 0) { + LOGGER.error("errorCode : {}, table name can not repeat, name : {}", ErrorCode.TABLE_ADD_ERROR.getCode(), vo.getName()); + throw new DSSDatamodelCenterException(ErrorCode.TABLE_ADD_ERROR.getCode(), "table name can not repeat"); + } + + DssDatamodelTable newOne = modelMapper.map(vo, DssDatamodelTable.class); + newOne.setCreateTime(new Date()); + newOne.setUpdateTime(new Date()); + newOne.setVersion("1"); + getBaseMapper().insert(newOne); + + Long tableId = newOne.getId(); + List columns = addColumns(tableId, vo.getColumns()); + + String user = DataModelSecurityContextHolder.getContext().getDataModelAuthentication().getUser(); + + //发布表绑定模型事件 + publisher.publishEvent(new BindModelByTableEvent(this, user, newOne)); + publisher.publishEvent(new BindModelByColumnsEvent(this, user, newOne.getName(), columns)); + + //发布绑定标签事件 + publisher.publishEvent(new BindLabelByTableEvent(this, user, newOne)); + + return newOne.getId(); + } + + + private List addColumns(Long tableId, List columnVOs) throws ErrorException { + List columns = Lists.newArrayList(); + for (TableColumnVO columnVO : columnVOs) { + DssDatamodelTableColumns newColumn = modelMapper.map(columnVO, DssDatamodelTableColumns.class); + columns.add(newColumn); + } + + tableColumnsService.batchInsert(tableId, columns); + return columns; + } + + @Override + @Transactional(rollbackFor = Exception.class) + public int updateTable(Long id, TableUpdateVO vo) throws ErrorException { + + DssDatamodelTable org = getBaseMapper().selectById(id); + if (org == null) { + LOGGER.error("errorCode : {}, update table error not exists", ErrorCode.TABLE_UPDATE_ERROR.getCode()); + throw new DSSDatamodelCenterException(ErrorCode.TABLE_UPDATE_ERROR.getCode(), "update table error not exists"); + } + + //判断数据表是否有数据 + tableMaterializedHistoryService.checkData(org, vo.getCreator()); + + String user = DataModelSecurityContextHolder.getContext().getDataModelAuthentication().getUser(); + + //当更新表名称时,存在其他名称同名或者当前名称已经存在版本信息，则不允许修改指标名称 + if (!StringUtils.equals(vo.getName(), org.getName())) { + //表如果已经物化不能修改名称 + if (tableMaterializedHistoryService.tableExists(org.getName(), user)) { + LOGGER.error("errorCode : {}, table name can not change as table has materialized", ErrorCode.TABLE_UPDATE_ERROR.getCode()); + throw new DSSDatamodelCenterException(ErrorCode.TABLE_UPDATE_ERROR.getCode(), "table name can not change as table has materialized"); + } + int repeat = getBaseMapper().selectCount(Wrappers.lambdaQuery().eq(DssDatamodelTable::getName, vo.getName())); + String lastVersion = tableVersionService.findLastVersion(org.getName()); + if (repeat > 0 || StringUtils.isNotBlank(lastVersion)) { + LOGGER.error("errorCode : {}, table name can not repeat or has version", ErrorCode.TABLE_UPDATE_ERROR.getCode()); + throw new DSSDatamodelCenterException(ErrorCode.TABLE_UPDATE_ERROR.getCode(), "table name can not repeat or has version"); + } + } + + Long orgId = org.getId(); + String version = org.getVersion(); + List orgColumns = tableColumnsService.listByTableId(id); + + DssDatamodelTable updateOne = modelMapper.map(vo, DssDatamodelTable.class); + updateOne.setUpdateTime(new Date()); + int result = getBaseMapper().update(updateOne, Wrappers.lambdaUpdate().eq(DssDatamodelTable::getId, id)); + if (result != 1) { + LOGGER.error("errorCode : {}, update table error not exists", ErrorCode.TABLE_UPDATE_ERROR.getCode()); + throw new DSSDatamodelCenterException(ErrorCode.TABLE_UPDATE_ERROR.getCode(), "update table error not exists"); + } + List newColumns = Lists.newArrayList(); + for (TableColumnVO columnVO : vo.getColumns()) { + DssDatamodelTableColumns newColumn = modelMapper.map(columnVO, DssDatamodelTableColumns.class); + newColumns.add(newColumn); + } + + tableColumnsService.batchUpdate(orgId, newColumns); + + + publisher.publishEvent(new UpdateBindModelByTableEvent(this, user, org, updateOne)); + publisher.publishEvent(new UpdateBindModelByColumnsEvent(this, user, org.getName(), orgColumns, newColumns)); + publisher.publishEvent(new UpdateBindLabelByTableEvent(this, user, org, updateOne)); + + return 1; + } + + + @Override + public TableQueryDTO queryByName(TableQueryOneVO vo) throws ErrorException { + DssDatamodelTable table = getBaseMapper().selectOne(Wrappers.lambdaQuery().eq(DssDatamodelTable::getName, vo.getName())); + //如果查询不到则查询资产 + if (table == null) { + //查资产 guid 086c5785-8bda-4756-8ba6-46f9c3d597f1 a3be4a97-6465-4c3d-adee-76dfa662e531 ef09c10a-e156-4d09-96af-af30eb3af26a + GetHiveTblBasicResult result = linkisDataAssetsRemoteClient.getHiveTblBasic(GetHiveTblBasicAction.builder().setUser(vo.getUser()).setGuid(vo.getGuid()).build()); + HiveTblDetailInfoDTO dto = assertsGson.fromJson(assertsGson.toJson(result.getResult()), HiveTblDetailInfoDTO.class); + HiveTblStatsResult hiveTblStatsResult = linkisDataAssetsRemoteClient.searchHiveTblStats(HiveTblStatsAction.builder().setUser(vo.getUser()).setGuid(vo.getGuid()).build()); + return TableQueryDTO.toTableStatsDTO(dto, hiveTblStatsResult.getInfo(), StringUtils.substringBefore(dto.getBasic().getQualifiedName(),"@")); + } + return queryTable(table); + } + + private TableQueryDTO queryTable(DssDatamodelTable table) throws ErrorException { + TableQueryDTO tableQueryDTO = modelMapper.map(table, TableQueryDTO.class); + + List columnQueryDTOS = tableColumnsService.listByTableId(table.getId()) + .stream().map(column -> modelMapper.map(column, TableColumnQueryDTO.class)).collect(Collectors.toList()); + tableQueryDTO.setColumns(columnQueryDTOS); + +// DssDatamodelTableStats tableStats = tableStatsService.queryByTableName(table.getName()); +// if (tableStats != null) { +// tableQueryDTO.setStats(modelMapper.map(tableStats, TableStatsDTO.class)); +// } + TableHeadlineDTO headlineDTO = new TableHeadlineDTO(); + headlineDTO.setStorageType(0); + headlineDTO.setTableType(0); + headlineDTO.setEntityType( + tableMaterializedHistoryService.isMaterialized(table.getName(), table.getVersion()) ? 1 : 0); + tableQueryDTO.setHeadline(headlineDTO); + + String user = DataModelSecurityContextHolder.getContext().getDataModelAuthentication().getUser(); + Integer collectionCount = tableCollectService.getBaseMapper().selectCount(Wrappers.lambdaQuery().eq(DssDatamodelTableCollcetion::getName, table.getName())); + String dbName = StringUtils.substringBefore(table.getName(), "."); + String tblName = StringUtils.substringAfter(table.getName(), "."); + HiveTblStatsResult hiveTblStatsResult = linkisDataAssetsRemoteClient.searchHiveTblStats(HiveTblStatsAction.builder().setUser(user) + .setTableName(tblName) + .setDbName(dbName) + .build()); + tableQueryDTO.setStats(TableStatsDTO.from(hiveTblStatsResult.getInfo(), collectionCount)); + + tableQueryDTO.setLastAccessTime(tableMaterializedHistoryService.getHiveTblSimpleInfoByName(table.getName(),user).map(HiveTblSimpleInfoDTO::getLastAccessTime).orElse(null)); + + return tableQueryDTO; + } + + + @Override + public TableQueryDTO queryById(Long id) throws ErrorException { + DssDatamodelTable table = getBaseMapper().selectOne(Wrappers.lambdaQuery().eq(DssDatamodelTable::getId, id)); + if (table == null) { + LOGGER.error("errorCode : {}, table id : {} not exists", ErrorCode.TABLE_QUERY_ERROR.getCode(), id); + throw new DSSDatamodelCenterException(ErrorCode.TABLE_QUERY_ERROR.getCode(), "table id " + id + " not exists"); + } + return queryTable(table); + } + + + @Override + @Transactional(rollbackFor = Exception.class) + public Integer addTableVersion(Long id, TableVersionAddVO vo) throws ErrorException { + + //判断旧版本指标是否存在 + DssDatamodelTable orgVersion = getBaseMapper().selectById(id); + if (orgVersion == null) { + LOGGER.error("errorCode : {}, table id : {} not exists", ErrorCode.TABLE_VERSION_ADD_ERROR.getCode(), id); + throw new DSSDatamodelCenterException(ErrorCode.TABLE_VERSION_ADD_ERROR.getCode(), "table id " + id + " not exists"); + } + + //判断旧版本是否有数据 + tableMaterializedHistoryService.checkData(orgVersion, vo.getCreator()); + //没有数据删除表 + //tableMaterializedHistoryService.dropTable(orgVersion.getName(),vo.getCreator()); + + String orgName = orgVersion.getName(); + String orgDatabase = orgVersion.getDataBase(); + String orgV = orgVersion.getVersion(); + String assignVersion = newVersion(orgName, orgV); + + //表名称和数据库不能改变 + if (!StringUtils.equals(orgName, vo.getName()) || !StringUtils.equals(orgDatabase, vo.getDataBase())) { + LOGGER.error("errorCode : {}, table name must be same", ErrorCode.TABLE_VERSION_ADD_ERROR.getCode()); + throw new DSSDatamodelCenterException(ErrorCode.TABLE_VERSION_ADD_ERROR.getCode(), "table name must be same"); + } + + //查询字段信息 + List orgColumns = tableColumnsService.listByTableId(id); + + //备份旧版本 + tableVersionService.addOlderVersion(orgVersion, orgColumns); + + //删除旧版本 + getBaseMapper().deleteById(id); + tableColumnsService.deleteByTableId(id); + + DssDatamodelTable newOne = modelMapper.map(vo, DssDatamodelTable.class); + newOne.setVersion(assignVersion); + newOne.setCreateTime(new Date()); + newOne.setUpdateTime(new Date()); + getBaseMapper().insert(newOne); + + Long tableId = newOne.getId(); + List newColumns = addColumns(tableId, vo.getColumns()); + + //发布更新模型绑定事件 + String user = DataModelSecurityContextHolder.getContext().getDataModelAuthentication().getUser(); + publisher.publishEvent(new UpdateBindModelByTableEvent(this, user, orgVersion, newOne)); + publisher.publishEvent(new UpdateBindModelByColumnsEvent(this, user, orgVersion.getName(), orgColumns, newColumns)); + publisher.publishEvent(new UpdateBindLabelByTableEvent(this, user, orgVersion, newOne)); + return 1; + } + + private String newVersion(String orgName, String orgVersion) { + String lastVersion = tableVersionService.findLastVersion(orgName); + + return StringUtils.isBlank(lastVersion) ? + (Integer.parseInt(orgVersion) + 1 + "") : + (Integer.parseInt(lastVersion) > Integer.parseInt(orgVersion)) ? (Integer.parseInt(lastVersion) + 1 + "") : + (Integer.parseInt(orgVersion) + 1 + ""); + } + + + @Override + @Transactional(rollbackFor = Exception.class) + public Integer versionRollBack(TableVersionRollBackVO vo) throws ErrorException { + + String name = vo.getName(); + String version = vo.getVersion(); + + //获取当前使用版本表 + DssDatamodelTable current = + getBaseMapper().selectOne(Wrappers.lambdaQuery().eq(DssDatamodelTable::getName, name)); + + //判断当前使用表 + if (current == null) { + LOGGER.error("errorCode : {}, current table not exists, name : {} version : {}", ErrorCode.TABLE_VERSION_ROLL_BACK_ERROR.getCode(), name, version); + throw new DSSDatamodelCenterException(ErrorCode.TABLE_VERSION_ROLL_BACK_ERROR.getCode(), "current table not exists, name : " + name + " version : " + version); + } + + //判断当前版本是否有数据 + tableMaterializedHistoryService.checkData(current, vo.getUser()); + //没有数据删除表 + //tableMaterializedHistoryService.dropTable(current.getName(),vo.getUser()); + + //查询字段信息 + List currentColumns = tableColumnsService.listByTableId(current.getId()); + + //当前版本备份 + tableVersionService.addOlderVersion(current, currentColumns); + + //删除当前版本 + getBaseMapper().deleteById(current.getId()); + tableColumnsService.deleteByTableId(current.getId()); + + + //获取需要回退指标版本信息 + DssDatamodelTableVersion rollBackVersion = tableVersionService.findBackup(name, version); + if (rollBackVersion == null) { + LOGGER.error("errorCode : {}, table name : {} version : {}", ErrorCode.TABLE_VERSION_ROLL_BACK_ERROR.getCode(), name, version); + throw new DSSDatamodelCenterException(ErrorCode.TABLE_VERSION_ROLL_BACK_ERROR.getCode(), "table name : " + name + " version : " + version + " not exist"); + } + + Long rollbackId = rollBackVersion.getId(); + + DssDatamodelTable rollbackOne = gson.fromJson(rollBackVersion.getTableParams(), DssDatamodelTable.class); + rollbackOne.setId(null); + getBaseMapper().insert(rollbackOne); + + List rollbackColumns = gson.fromJson(rollBackVersion.getColumns(), new TypeToken>() { + }.getType()); + rollbackColumns.forEach(columns -> columns.setId(null)); + tableColumnsService.batchInsert(rollbackOne.getId(), rollbackColumns); + tableVersionService.getBaseMapper().deleteById(rollbackId); + + + //发布更新模型绑定事件 + String user = DataModelSecurityContextHolder.getContext().getDataModelAuthentication().getUser(); + publisher.publishEvent(new UpdateBindModelByTableEvent(this, user, current, rollbackOne)); + publisher.publishEvent(new UpdateBindModelByColumnsEvent(this, user, current.getName(), currentColumns, rollbackColumns)); + publisher.publishEvent(new UpdateBindLabelByTableEvent(this, user, current, rollbackOne)); + + return 1; + } + + + @Override + public Message listTableVersions(TableVersionQueryVO vo) { + List list = tableVersionService.getBaseMapper() + .selectList(Wrappers.lambdaQuery().eq(DssDatamodelTableVersion::getName, vo.getName())) + .stream() + .map(dssDatamodelTableVersion -> modelMapper.map(dssDatamodelTableVersion, TableVersionQueryDTO.class)) + .collect(Collectors.toList()); + return Message.ok().data("list", list); + } + + + @Override + @Transactional(rollbackFor = Exception.class) + public Integer tableCollect(TableCollectVO vo) throws ErrorException { + if (vo.getTableId() != null) { + DssDatamodelTable collection = getBaseMapper().selectById(vo.getTableId()); + if (collection == null) { + LOGGER.error("errorCode : {}, collect table not exists id : {} ", ErrorCode.TABLE_COLLECT_ADD_ERROR.getCode(), vo.getTableId()); + throw new DSSDatamodelCenterException(ErrorCode.TABLE_COLLECT_ADD_ERROR.getCode(), " collect table not exist id : " + vo.getTableId()); + } + return tableCollectService.tableCollect(vo.getUser(), collection); + } + + return tableCollectService.tableCollect(vo); + } + + + @Override + @Transactional(rollbackFor = Exception.class) + public Integer tableCancel(TableCollectCancelVO vo) throws ErrorException { + return tableCollectService.tableCollectCancel(vo.getUser(), vo.getTableName()); + } + + + @Override + public Message tableCollections(TableCollectQueryVO vo) throws ErrorException { + QueryWrapper queryWrapper = new QueryWrapper() + .like(StringUtils.isNotBlank(vo.getName()), "name", vo.getName()) + .like(StringUtils.isNotBlank(vo.getDataBase()), "database", vo.getName()) + .eq(StringUtils.isNotBlank(vo.getUser()), "user", vo.getUser()) + .eq(StringUtils.isNotBlank(vo.getWarehouseLayerName()), "warehouse_layer_name", vo.getWarehouseLayerName()) + .eq(StringUtils.isNotBlank(vo.getWarehouseThemeName()), "warehouse_theme_name", vo.getWarehouseThemeName()); + PageHelper.clearPage(); + PageHelper.startPage(vo.getPageNum(), vo.getPageSize()); + PageInfo pageInfo = new PageInfo<>(tableCollectService.getBaseMapper().selectList(queryWrapper)); + + return Message.ok() + .data("list", pageInfo.getList().stream().map(entity -> { + TableCollectionDTO dto = modelMapper.map(entity, TableCollectionDTO.class); + //收藏表的Id 不返回前端 + dto.setId(null); + return dto; + }).collect(Collectors.toList())) + .data("total", pageInfo.getTotal()); + } + + + @Override + public Message dictionaryList(TableDictionaryListVO vo) { + return Message.ok().data("list", tableDictionaryService.listByType(vo.getType())); + } + + + @Override + public Integer addTableColumn(TableColumnsAddVO vo) throws ErrorException { + int count = getBaseMapper().selectCount(Wrappers.lambdaQuery().eq(DssDatamodelTable::getId, vo.getTableId())); + if (count < 1) { + LOGGER.error("errorCode : {}, table not exists id : {} ", ErrorCode.TABLE_COLUMN_ADD_ERROR.getCode(), vo.getTableId()); + throw new DSSDatamodelCenterException(ErrorCode.TABLE_COLUMN_ADD_ERROR.getCode(), " table not exists id : " + vo.getTableId()); + } + DssDatamodelTableColumns newColumn = modelMapper.map(vo, DssDatamodelTableColumns.class); + return tableColumnsService.addColumn(newColumn); + } + + + @Override + @Transactional(rollbackFor = Exception.class) + public Integer tableColumnBind(Long columnId, TableColumnBindVO vo) throws ErrorException { + return tableColumnsService.tableColumnBind(columnId, vo.getModelType(), vo.getModelName(), vo.getModelNameEn()); + } + + + @Override + @Transactional(rollbackFor = Exception.class) + public Integer tableCreate(TableCreateVO vo) throws ErrorException { + DssDatamodelTable current = getBaseMapper().selectById(vo.getTableId()); + if (current == null) { + LOGGER.error("errorCode : {}, table not exists id : {} ", ErrorCode.TABLE_CREATE_ERROR.getCode(), vo.getTableId()); + throw new DSSDatamodelCenterException(ErrorCode.TABLE_CREATE_ERROR.getCode(), " table not exists id : " + vo.getTableId()); + } + tableMaterializedHistoryService.materializedTable(current, vo.getUser()); + //发布尝试表绑定模型事件 + publisher.publishEvent(new TableFirstBindEvent(this + ,DataModelSecurityContextHolder.getContext().getDataModelAuthentication().getUser() + ,current.getId() + ,current.getName())); + return 1; + } + + + + + @Override + public String tableCreateSql(TableCreateSqlVO vo) throws ErrorException { + + //先查询hdfs + if (StringUtils.isNotBlank(vo.getGuid())) { + GetHiveTblCreateResult result = linkisDataAssetsRemoteClient.getHiveTblCreate(GetHiveTblCreateAction.builder().setUser(vo.getUser()).setGuid(vo.getGuid()).build()); + if (StringUtils.isNotBlank(result.getResult())) { + LOGGER.info("sql : {}", result.getResult()); + return result.getResult(); + } + } + + DssDatamodelTable current = getBaseMapper().selectById(vo.getTableId()); + if (current == null) { + LOGGER.error("errorCode : {}, table not exists id : {} ", ErrorCode.TABLE_CREATE_SQL_ERROR.getCode(), vo.getTableId()); + throw new DSSDatamodelCenterException(ErrorCode.TABLE_CREATE_SQL_ERROR.getCode(), " table not exists id : " + vo.getTableId()); + } + String sql = tableMaterializedHistoryService.generateSql(current); + LOGGER.info("sql : {}", sql); + return sql; + } + + + @Override + public Message list(TableListVO vo) { + if (vo.getModelType() > 0) { + return listByIndicator(vo); + } + + if (vo.getTableType() < 0 || StringUtils.isNotBlank(vo.getWarehouseLayerName()) || StringUtils.isNotBlank(vo.getWarehouseThemeName())) { + return listDataModel(vo); + } + return listAssets(vo); + } + + private Message listByIndicator(TableListVO vo) { + PageHelper.clearPage(); + PageHelper.startPage(vo.getPageNum(), vo.getPageSize()); + + PageInfo pageInfo = new PageInfo<>(tableQueryMapper.page(Wrappers.lambdaQuery() + .eq(DssDatamodelTabelQuery::getModelType, vo.getModelType()) + .like(StringUtils.isNotBlank(vo.getModelName()), DssDatamodelTabelQuery::getModelName, vo.getModelName()) + .eq(StringUtils.isNotBlank(vo.getWarehouseLayerName()), DssDatamodelTabelQuery::getWarehouseLayerName, vo.getWarehouseLayerName()) + .eq(StringUtils.isNotBlank(vo.getWarehouseThemeName()), DssDatamodelTabelQuery::getWarehouseThemeName, vo.getWarehouseThemeName()) + .groupBy(DssDatamodelTabelQuery::getId))); + return Message.ok().data("list", pageInfo.getList().stream().map(entity -> modelMapper.map(entity, TableListDTO.class)).collect(Collectors.toList())) + .data("total", pageInfo.getTotal()); + } + + private Message listAssets(TableListVO vo) { + SearchHiveTblResult result = linkisDataAssetsRemoteClient.searchHiveTbl(SearchHiveTblAction.builder() + .setUser(vo.getUser()) + .setQuery(vo.getName()) + .setOffset((vo.getPageNum() - 1) * vo.getPageSize()) + .setLimit(vo.getPageSize()).build()); + List dtos = assertsGson.fromJson(assertsGson.toJson(result.getResult()), new TypeToken>() { + }.getType()); + if (CollectionUtils.isEmpty(dtos)) { + return Message.ok().data("list", Lists.newArrayList()); + } + List tableListDTOS = Lists.newArrayList(); + dtos.forEach(hiveTblSimpleInfoDTO -> { + TableListDTO tableListDTO = new TableListDTO(); + tableListDTO.setGuid(hiveTblSimpleInfoDTO.getGuid()); + tableListDTO.setCreator(hiveTblSimpleInfoDTO.getOwner()); + tableListDTO.setCreateTime(hiveTblSimpleInfoDTO.getCreateTime()); + tableListDTO.setName(StringUtils.substringBefore(hiveTblSimpleInfoDTO.getQualifiedName(), "@")); + tableListDTO.setDataBase(StringUtils.substringBefore(hiveTblSimpleInfoDTO.getQualifiedName(), ".")); + + tableListDTO.setUpdateTime(hiveTblSimpleInfoDTO.getLastAccessTime()); + tableListDTO.setAlias(hiveTblSimpleInfoDTO.getAliases()); + tableListDTO.setComment(hiveTblSimpleInfoDTO.getComment()); + tableListDTO.setSize(getSize(hiveTblSimpleInfoDTO)); + String theme = getThemes(hiveTblSimpleInfoDTO.getClassifications()); + tableListDTO.setWarehouseThemeName(theme); + tableListDTO.setWarehouseThemeNameEn(theme); + String layer = getLayer(hiveTblSimpleInfoDTO.getClassifications()); + tableListDTO.setWarehouseLayerName(layer); + tableListDTO.setWarehouseLayerNameEn(layer); + + tableListDTOS.add(tableListDTO); + }); + + return Message.ok().data("list", tableListDTOS) + .data("total", tableListDTOS.size()); + } + + private String getThemes(List classifications) { + return getModel(ClassificationConstant.THEME,classifications); + } + private String getLayer(List classifications) { + return getModel(ClassificationConstant.LAYER,classifications); + } + + private String getModel(ClassificationConstant classificationConstant,List classifications) { + if (CollectionUtils.isEmpty(classifications)) { + return null; + } + for (String classification : classifications) { + String prefix = StringUtils.substringBefore(classification, ClassificationConstant.SEPARATOR); + if (classificationConstant.getTypeCode().equals(prefix)) { + return StringUtils.substringAfter(classification, ClassificationConstant.SEPARATOR); + } + } + return null; + } + + private long getSize(HiveTblSimpleInfoDTO hiveTblSimpleInfoDTO) { + long size = 0L; + if (StringUtils.isBlank(hiveTblSimpleInfoDTO.getTotalSize())) { + return size; + } + try { + size = Long.parseLong(hiveTblSimpleInfoDTO.getTotalSize()); + } catch (Exception e) { + //ignore + } + return size; + } + + private Message listDataModel(TableListVO vo) { + PageHelper.clearPage(); + PageHelper.startPage(vo.getPageNum(), vo.getPageSize()); + PageInfo pageInfo = new PageInfo<>(getBaseMapper().selectList(Wrappers.lambdaQuery() + .eq(StringUtils.isNotBlank(vo.getWarehouseThemeName()), DssDatamodelTable::getWarehouseThemeName, vo.getWarehouseThemeName()) + .eq(StringUtils.isNotBlank(vo.getWarehouseLayerName()), DssDatamodelTable::getWarehouseLayerName, vo.getWarehouseLayerName()) + .like(StringUtils.isNotBlank(vo.getName()), DssDatamodelTable::getName, vo.getName()))); + return Message.ok().data("list", pageInfo.getList().stream().map(entity -> modelMapper.map(entity, TableListDTO.class)).collect(Collectors.toList())) + .data("total", pageInfo.getTotal()); + } + + @Override + public Message listTablePartitionStats(TblPartitionStatsVO vo) { + if (StringUtils.isNotBlank(vo.getGuid())) { + return queryByGuid(vo.getGuid(), vo.getUser()); + } + SearchHiveTblResult result = linkisDataAssetsRemoteClient.searchHiveTbl(SearchHiveTblAction.builder().setUser(vo.getUser()).setPrecise(QueryType.PRECISE).setQuery(vo.getName()).setOffset(0).setLimit(1).build()); + List dtos = assertsGson.fromJson(assertsGson.toJson(result.getResult()), new TypeToken>() { + }.getType()); + if (CollectionUtils.isEmpty(dtos)) { + return Message.ok().data("list", Lists.newArrayList()); + } + return queryByGuid(dtos.get(0).getGuid(), vo.getUser()); + } + + @Override + public Message listDataBases(TableDatabasesQueryVO vo) { + SearchHiveDbResult result = linkisDataAssetsRemoteClient.searchHiveDb( + SearchHiveDbAction.builder().setQuery(vo.getName()).setUser(vo.getUser()).setOffset(vo.getPageNum() - 1).setLimit(vo.getPageSize() * (vo.getPageNum() - 1)).build()); + return Message.ok().data("list", result.getResult()); + } + + + @Override + public Message previewData(TableDataPreviewVO vo) throws ErrorException { + if (!tableMaterializedHistoryService.tableExists(vo.getTableName(), vo.getUser())) { + return Message.ok(); + } + DataModelUJESJobTask dataModelUJESJobTask = PreviewDataModelUJESJobTask.newBuilder().user(vo.getUser()).code(vo.getTableName()).count(10).build(); + PreviewDataDTO previewDataDTO = null; + try { + previewDataDTO = previewDataModelUJESJobLauncher.launch(dataModelUJESJobTask); + } catch (Exception e) { + if (e instanceof UJESJobException) { + UJESJobException ujesJobException = (UJESJobException) e; + //表不存在错误忽略 + if (ujesJobException.getErrCode() == 40002) { + LOGGER.error(e.getMessage(), e); + return Message.ok(); + } + } + throw e; + } + return Message.ok().data("detail", previewDataDTO); + } + + + @Override + public Integer tableCheckData(TableCheckDataVO vo) throws ErrorException { + return tableMaterializedHistoryService.hasData(vo.getTableName(), vo.getUser()) ? 1 : 0; + } + + private Message queryByGuid(String guid, String user) { + GetHiveTblPartitionResult result = linkisDataAssetsRemoteClient.getHiveTblPartition(GetHiveTblPartitionAction.builder().setUser(user).setGuid(guid).build()); + if (result.getResult() == null) { + return Message.ok().data("list", Lists.newArrayList()); + } + List partInfoDTOS = assertsGson.fromJson(assertsGson.toJson(result.getResult()), new TypeToken>() { + }.getType()); + return Message.ok().data("list", partInfoDTOS); + } + + + @Override + public int tableThemeReferenceCount(String name) { + int currentCount = getBaseMapper().selectCount(Wrappers.lambdaQuery().eq(DssDatamodelTable::getWarehouseThemeName, name)); + int currentCountEn = getBaseMapper().selectCount(Wrappers.lambdaQuery().eq(DssDatamodelTable::getWarehouseThemeNameEn, name)); + // int versionCount = tableVersionService.tableContentReference(name); + + List preReferences = tableVersionService.tableContentMultipleReference(name); + int versionCount = (int) preReferences.stream().filter(e->{ + DssDatamodelTable temp = gson.fromJson(e.getTableParams(),DssDatamodelTable.class); + return StringUtils.equals(temp.getWarehouseThemeName(),name)||StringUtils.equals(temp.getWarehouseThemeNameEn(),name); + }).count(); + + return currentCount + versionCount + currentCountEn; + } + + @Override + public int tableLayerReferenceCount(String name) { + int currentCount = getBaseMapper().selectCount(Wrappers.lambdaQuery().eq(DssDatamodelTable::getWarehouseLayerName, name)); + int currentCountEn = getBaseMapper().selectCount(Wrappers.lambdaQuery().eq(DssDatamodelTable::getWarehouseLayerNameEn, name)); + //int versionCount = tableVersionService.tableContentReference(name); + List preReferences = tableVersionService.tableContentMultipleReference(name); + int versionCount = (int) preReferences.stream().filter(e->{ + DssDatamodelTable temp = gson.fromJson(e.getTableParams(),DssDatamodelTable.class); + return StringUtils.equals(temp.getWarehouseLayerName(),name)||StringUtils.equals(temp.getWarehouseLayerNameEn(),name); + }).count(); + return currentCount + versionCount + currentCountEn; + } + + @Override + public int tableCycleReferenceCount(String name) { + int currentCount = getBaseMapper().selectCount(Wrappers.lambdaQuery().eq(DssDatamodelTable::getLifecycle, name)); + int currentCountEn = getBaseMapper().selectCount(Wrappers.lambdaQuery().eq(DssDatamodelTable::getLifecycleEn, name)); + List preReferences = tableVersionService.tableContentMultipleReference(name); + int versionCount = (int) preReferences.stream().filter(e->{ + DssDatamodelTable temp = gson.fromJson(e.getTableParams(),DssDatamodelTable.class); + return StringUtils.equals(temp.getLifecycle(),name)||StringUtils.equals(temp.getLifecycleEn(),name); + }).count(); + return currentCount + versionCount + currentCountEn; + } + + @Override + public int tableDimensionReferenceCount(String name) { + return referenceCount(name, ModeType.DIMENSION); + } + + private int referenceCount(String name, ModeType modeType) { + int currentCount = tableColumnsService.modelReferenceCount(modeType, name); + int currentCountEn = tableColumnsService.modelReferenceCountEn(modeType, name); + //int versionCount = tableVersionService.tableColumnsReference(name); + + List preReferences = tableVersionService.tableColumnsReference(name); + int versionCount = (int) preReferences.stream().filter(e->{ + DssDatamodelTableColumns temp = gson.fromJson(e.getTableParams(),DssDatamodelTableColumns.class); + //名称相同且类型相同 + return modeType.getType() == temp.getModelType()&&StringUtils.equals(name,temp.getModelName()); + }).count(); + return currentCount + versionCount + currentCountEn; + } + + @Override + public int tableMeasureReferenceCount(String name) { + return referenceCount(name, ModeType.MEASURE); + } + + @Override + public int tableIndicatorReferenceCount(String name) { + return referenceCount(name, ModeType.INDICATOR); + } + + @Override + public int tableModifierReferenceCount(String name) { + return 0; + } + + + @Override + public void bind(long id, String user) throws ErrorException { + DssDatamodelTable current = getBaseMapper().selectById(id); + if (current == null) { + LOGGER.error("errorCode : {}, bind table error not exists", ErrorCode.TABLE_BIND_ERROR.getCode()); + throw new DSSDatamodelCenterException(ErrorCode.TABLE_BIND_ERROR.getCode(), "bind table error not exists"); + } + + List currentColumns = tableColumnsService.listByTableId(id); + publisher.publishEvent(new BindModelByTableEvent(this, user, current)); + publisher.publishEvent(new BindModelByColumnsEvent(this, user, current.getName(), currentColumns)); + publisher.publishEvent(new BindLabelByTableEvent(this, user, current)); + } + + @Override + public void tryBind(long id) throws ErrorException { + DssDatamodelTable current = getBaseMapper().selectById(id); + if (current == null) { + LOGGER.error("errorCode : {}, table not exists id : {} ", ErrorCode.TABLE_BIND_ERROR.getCode(), id); + throw new DSSDatamodelCenterException(ErrorCode.TABLE_BIND_ERROR.getCode(), " table not exists id : " + id); + } + + //发布尝试表绑定模型事件 + publisher.publishEvent(new TableFirstBindEvent(this + ,DataModelSecurityContextHolder.getContext().getDataModelAuthentication().getUser() + ,current.getId() + ,current.getName())); + } + + + @Override + @Transactional(rollbackFor = Exception.class) + public int deleteTable(Long id) throws ErrorException { + DssDatamodelTable current = getBaseMapper().selectById(id); + if (current == null) { + return 0; + } + //有数据则不能删除 + if (tableMaterializedHistoryService.hasData(current.getName(), DataModelSecurityContextHolder.getContext().getDataModelAuthentication().getUser())) { + LOGGER.error("errorCode : {}, table id : {} has data", ErrorCode.TABLE_DELETE_ERROR.getCode(), id); + throw new DSSDatamodelCenterException(ErrorCode.TABLE_DELETE_ERROR.getCode(), "bind table id " + id + "has data"); + } + List columns = tableColumnsService.listByTableId(id); + + String user = DataModelSecurityContextHolder.getContext().getDataModelAuthentication().getUser(); + + //删除表相关内容 + getBaseMapper().deleteById(id); + //删除字段 + tableColumnsService.deleteByTableId(id); + //删除版本 + tableVersionService.getBaseMapper().delete(Wrappers.lambdaQuery().eq(DssDatamodelTableVersion::getName, current.getName())); + //删除收藏信息 + tableCollectService.getBaseMapper().delete(Wrappers.lambdaQuery().eq(DssDatamodelTableCollcetion::getName, current.getName())); + //删除物化信息 + tableMaterializedHistoryService.getBaseMapper().delete(Wrappers.lambdaQuery().eq(DssDatamodelTableMaterializedHistory::getTablename, current.getName())); + //发布表解绑模型事件 + publisher.publishEvent(new UnBindModelByTableEvent(this, user, current)); + publisher.publishEvent(new UnBindModelByColumnsEvent(this, user, current.getName(), columns)); + + return 1; + } + + + @Override + public int tableLabelReferenceCount(String name) { + int currentCount = getBaseMapper().selectCount(Wrappers.lambdaQuery() + .eq(DssDatamodelTable::getLabel, name) + .or() + .like(DssDatamodelTable::getLabel, name + ",") + .or() + .like(DssDatamodelTable::getLabel, "," + name + ",") + .or() + .like(DssDatamodelTable::getLabel, "," + name)); + + List preReferences = tableVersionService.tableContentMultipleReference(name); + int versionCount = (int) preReferences.stream().filter(e->{ + DssDatamodelTable temp = gson.fromJson(e.getTableParams(),DssDatamodelTable.class); + return Sets.newHashSet(StringUtils.split(temp.getLabel(), LabelConstant.SEPARATOR)).contains(name); + }).count(); + return currentCount + versionCount; + } +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/service/impl/TableStatsServiceImpl.java b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/service/impl/TableStatsServiceImpl.java new file mode 100644 index 0000000000..270ed5474e --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/service/impl/TableStatsServiceImpl.java @@ -0,0 +1,23 @@ +package com.webank.wedatasphere.dss.datamodel.table.service.impl; + + +import com.baomidou.mybatisplus.core.toolkit.Wrappers; +import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl; +import com.webank.wedatasphere.dss.datamodel.table.dao.DssDatamodelTableStatsMapper; +import com.webank.wedatasphere.dss.datamodel.table.entity.DssDatamodelTableStats; +import com.webank.wedatasphere.dss.datamodel.table.service.TableStatsService; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Service; + +@Service +public class TableStatsServiceImpl extends ServiceImpl implements TableStatsService { + + private static final Logger LOGGER = LoggerFactory.getLogger(TableServiceImpl.class); + + + @Override + public DssDatamodelTableStats queryByTableName(String tableName) { + return getBaseMapper().selectOne(Wrappers.lambdaQuery().eq(DssDatamodelTableStats::getName,tableName)); + } +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/service/impl/TableThemeReferenceServiceImpl.java b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/service/impl/TableThemeReferenceServiceImpl.java new file mode 100644 index 0000000000..388d059f8d --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/service/impl/TableThemeReferenceServiceImpl.java @@ -0,0 +1,19 @@ +package com.webank.wedatasphere.dss.datamodel.table.service.impl; + +import com.webank.wedatasphere.dss.datamodel.center.common.service.TableThemeReferenceService; +import com.webank.wedatasphere.dss.datamodel.table.service.TableService; +import org.springframework.stereotype.Service; + +import javax.annotation.Resource; + +@Service +public class TableThemeReferenceServiceImpl implements TableThemeReferenceService { + + @Resource + private TableService tableService; + + @Override + public int tableThemeReferenceCount(String name) { + return tableService.tableThemeReferenceCount(name); + } +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/service/impl/TableVersionServiceImpl.java b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/service/impl/TableVersionServiceImpl.java new file mode 100644 index 0000000000..bb4a6c1476 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/service/impl/TableVersionServiceImpl.java @@ -0,0 +1,109 @@ +package com.webank.wedatasphere.dss.datamodel.table.service.impl; + + +import com.baomidou.mybatisplus.core.toolkit.Wrappers; +import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl; +import com.github.pagehelper.PageHelper; +import com.github.pagehelper.PageInfo; +import com.google.gson.Gson; +import com.webank.wedatasphere.dss.datamodel.table.dao.DssDatamodelTableVersionMapper; +import com.webank.wedatasphere.dss.datamodel.table.entity.DssDatamodelTable; +import com.webank.wedatasphere.dss.datamodel.table.entity.DssDatamodelTableCollcetion; +import com.webank.wedatasphere.dss.datamodel.table.entity.DssDatamodelTableColumns; +import com.webank.wedatasphere.dss.datamodel.table.entity.DssDatamodelTableVersion; +import com.webank.wedatasphere.dss.datamodel.table.service.TableMaterializedHistoryService; +import com.webank.wedatasphere.dss.datamodel.table.service.TableVersionService; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; +import org.springframework.util.CollectionUtils; + +import javax.annotation.Resource; +import java.util.Date; +import java.util.List; + +@Service +public class TableVersionServiceImpl extends ServiceImpl implements TableVersionService { + + private static final Logger LOGGER = LoggerFactory.getLogger(TableVersionServiceImpl.class); + + private Gson gson = new Gson(); + + @Resource + private TableMaterializedHistoryService tableMaterializedHistoryService; + + + @Override + public String findLastVersion(String name) { + PageHelper.clearPage(); + PageHelper.startPage(1,1); + //查询当前表名称最大版本 + PageInfo pageInfo = new PageInfo<>(getBaseMapper().selectList( + Wrappers.lambdaQuery() + .eq(DssDatamodelTableVersion::getName, name) + .orderByDesc(DssDatamodelTableVersion::getVersion))); + List list = pageInfo.getList(); + return !CollectionUtils.isEmpty(list) ? list.get(0).getVersion() : null; + } + + + + @Override + @Transactional(rollbackFor = Exception.class) + public Long addOlderVersion(DssDatamodelTable orgTable, List orgColumns) { + DssDatamodelTableVersion version = new DssDatamodelTableVersion(); + version.setTblId(orgTable.getId()); + version.setName(orgTable.getName()); + version.setComment(orgTable.getComment()); + version.setTableCode(tableMaterializedHistoryService.generateSql(orgTable));//todo 生成建表脚本 + version.setIsMaterialized(1);//todo 判断是否物化 + version.setColumns(gson.toJson(orgColumns)); + version.setTableParams(gson.toJson(orgTable)); + version.setSourceType("add");//todo 此字段的意义 + version.setCreateTime(new Date()); + version.setUpdateTime(new Date()); + version.setVersion(orgTable.getVersion()); + version.setCreator(orgTable.getCreator()); + getBaseMapper().insert(version); + return version.getId(); + } + + + @Override + public DssDatamodelTableVersion findBackup(String name, String version) { + return getBaseMapper().selectOne( + Wrappers.lambdaQuery() + .eq(DssDatamodelTableVersion::getName, name) + .eq(DssDatamodelTableVersion::getVersion, version)); + } + + + @Override + public List tableContentReference(String content) { + return getBaseMapper().selectList( + Wrappers.lambdaQuery() + .like(DssDatamodelTableVersion::getTableParams,"\""+content + "\"")); + } + + @Override + public List tableColumnsReference(String content) { + return getBaseMapper().selectList( + Wrappers.lambdaQuery() + .like(DssDatamodelTableVersion::getColumns,"\""+content + "\"")); + } + + + @Override + public List tableContentMultipleReference(String context) { + return getBaseMapper().selectList( + Wrappers.lambdaQuery() + .like(DssDatamodelTableVersion::getTableParams,"\""+ context + "\"") + .or() + .like(DssDatamodelTableVersion::getTableParams,"\""+ context + ",") + .or() + .like(DssDatamodelTableVersion::getTableParams,","+ context + ",") + .or() + .like(DssDatamodelTableVersion::getTableParams,","+ context + "\"")); + } +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/vo/LabelAddVO.java b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/vo/LabelAddVO.java new file mode 100644 index 0000000000..95501a37f7 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/vo/LabelAddVO.java @@ -0,0 +1,37 @@ +package com.webank.wedatasphere.dss.datamodel.table.vo; + +import lombok.Data; +import java.util.Map; +@Data +public class LabelAddVO { + + private String name; + + private String fieldIdentifier; + + /** + * 标签键值对 json + */ + private Map paramMap; + + private String comment; + + /** + * 数仓主题格式为： theme_domain_name.theme_name + */ + private String warehouseThemeName; + + private String owner; + + /** + * 授权的名字：userName、roleName + */ + private String principalName; + + private Integer isAvailable; + + /** + * 英文 + */ + private String warehouseThemeNameEn; +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/vo/LabelEnableVO.java b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/vo/LabelEnableVO.java new file mode 100644 index 0000000000..2594f5b53a --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/vo/LabelEnableVO.java @@ -0,0 +1,8 @@ +package com.webank.wedatasphere.dss.datamodel.table.vo; + +import lombok.Data; + +@Data +public class LabelEnableVO { + private Integer isAvailable; +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/vo/LabelUpdateVO.java b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/vo/LabelUpdateVO.java new file mode 100644 index 0000000000..ac215667ba --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/vo/LabelUpdateVO.java @@ -0,0 +1,40 @@ +package com.webank.wedatasphere.dss.datamodel.table.vo; + +import lombok.Data; + +import java.util.Map; + +@Data +public class LabelUpdateVO { + + private String name; + + private String fieldIdentifier; + + /** + * 标签键值对 json + */ + private Map paramMap; + + + private String comment; + + /** + * 数仓主题格式为： theme_domain_name.theme_name + */ + private String warehouseThemeName; + + private String owner; + + /** + * 授权的名字：userName、roleName + */ + private String principalName; + + private Integer isAvailable; + + /** + * 英文 + */ + private String warehouseThemeNameEn; +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/vo/LabelsQueryVO.java b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/vo/LabelsQueryVO.java new file mode 100644 index 0000000000..ddfc6fb3f8 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/vo/LabelsQueryVO.java @@ -0,0 +1,15 @@ +package com.webank.wedatasphere.dss.datamodel.table.vo; + +import com.webank.wedatasphere.dss.datamodel.center.common.vo.PageVO; +import lombok.Data; + +@Data +public class LabelsQueryVO extends PageVO { + private String name; + + private Integer isAvailable; + + private String owner; + + private String warehouseThemeName; +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/vo/TableAddVO.java b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/vo/TableAddVO.java new file mode 100644 index 0000000000..d40885da1d --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/vo/TableAddVO.java @@ -0,0 +1,229 @@ +package com.webank.wedatasphere.dss.datamodel.table.vo; + + +import lombok.Data; +import lombok.ToString; + +import java.util.List; +@Data +@ToString +public class TableAddVO { + + private String dataBase; + + private String name; + + private String alias; + + private String creator; + + private String comment; + + + /** + * 是否外部表 0 内部表 1外部表 + */ + private Integer isExternal; + + /** + * 数仓层级 + */ + private String warehouseLayerName; + + /** + * 数仓层级英文 + */ + private String warehouseLayerNameEn; + + /** + * 数仓主题格式为： theme_domain_name.theme_name + */ + private String warehouseThemeName; + + /** + * 数仓主题英文 + */ + private String warehouseThemeNameEn; + + /** + * 生命周期 + */ + private String lifecycle; + + private String lifecycleEn; + + private Integer isPartitionTable; + + private Integer isAvailable; + + /** + * 存储类型：hive/mysql + */ + private String storageType; + + /** + * 授权的名字：userName、roleName + */ + private String principalName; + + /** + * 压缩格式 + */ + private String compress; + + /** + * 文件格式 + */ + private String fileType; + + /** + * 外部表时 location + */ + private String location; + + /** + * 标签 + */ + private String label; + + private List columns; + + public String getLocation() { + return location; + } + + public void setLocation(String location) { + this.location = location; + } + + public Integer getIsExternal() { + return isExternal; + } + + public void setIsExternal(Integer isExternal) { + this.isExternal = isExternal; + } + + public String getDataBase() { + return dataBase; + } + + public void setDataBase(String dataBase) { + this.dataBase = dataBase; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getAlias() { + return alias; + } + + public void setAlias(String alias) { + this.alias = alias; + } + + public String getCreator() { + return creator; + } + + public void setCreator(String creator) { + this.creator = creator; + } + + public String getComment() { + return comment; + } + + public void setComment(String comment) { + this.comment = comment; + } + + + + public String getWarehouseLayerName() { + return warehouseLayerName; + } + + public void setWarehouseLayerName(String warehouseLayerName) { + this.warehouseLayerName = warehouseLayerName; + } + + public String getWarehouseThemeName() { + return warehouseThemeName; + } + + public void setWarehouseThemeName(String warehouseThemeName) { + this.warehouseThemeName = warehouseThemeName; + } + + public String getLifecycle() { + return lifecycle; + } + + public void setLifecycle(String lifecycle) { + this.lifecycle = lifecycle; + } + + public Integer getIsPartitionTable() { + return isPartitionTable; + } + + public void setIsPartitionTable(Integer isPartitionTable) { + this.isPartitionTable = isPartitionTable; + } + + public Integer getIsAvailable() { + return isAvailable; + } + + public void setIsAvailable(Integer isAvailable) { + this.isAvailable = isAvailable; + } + + public String getStorageType() { + return storageType; + } + + public void setStorageType(String storageType) { + this.storageType = storageType; + } + + public String getPrincipalName() { + return principalName; + } + + public void setPrincipalName(String principalName) { + this.principalName = principalName; + } + + public String getCompress() { + return compress; + } + + public void setCompress(String compress) { + this.compress = compress; + } + + public String getFileType() { + return fileType; + } + + public void setFileType(String fileType) { + this.fileType = fileType; + } + + public List getColumns() { + return columns; + } + + public void setColumns(List columns) { + this.columns = columns; + } + +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/vo/TableCheckDataVO.java b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/vo/TableCheckDataVO.java new file mode 100644 index 0000000000..b1eaa66639 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/vo/TableCheckDataVO.java @@ -0,0 +1,12 @@ +package com.webank.wedatasphere.dss.datamodel.table.vo; + + +import lombok.Data; +import lombok.ToString; + +@Data +@ToString +public class TableCheckDataVO { + String tableName; + String user; +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/vo/TableCollectCancelVO.java b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/vo/TableCollectCancelVO.java new file mode 100644 index 0000000000..561a64fd5a --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/vo/TableCollectCancelVO.java @@ -0,0 +1,33 @@ +package com.webank.wedatasphere.dss.datamodel.table.vo; + + +public class TableCollectCancelVO { + + private String user; + + private String tableName; + + public String getUser() { + return user; + } + + public void setUser(String user) { + this.user = user; + } + + public String getTableName() { + return tableName; + } + + public void setTableName(String tableName) { + this.tableName = tableName; + } + + @Override + public String toString() { + return "TableCollectCancelVO{" + + "user='" + user + '\'' + + ", tableName='" + tableName + '\'' + + '}'; + } +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/vo/TableCollectQueryVO.java b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/vo/TableCollectQueryVO.java new file mode 100644 index 0000000000..7adedc957c --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/vo/TableCollectQueryVO.java @@ -0,0 +1,99 @@ +package com.webank.wedatasphere.dss.datamodel.table.vo; + + + +public class TableCollectQueryVO { + + private Integer pageSize = 20; + + private Integer pageNum = 1; + + + private String dataBase; + + + private String name; + + /** + * 数仓层级 + */ + private String warehouseLayerName; + + /** + * 数仓主题格式为： theme_domain_name.theme_name + */ + private String warehouseThemeName; + + + private String user; + + + public String getDataBase() { + return dataBase; + } + + public void setDataBase(String dataBase) { + this.dataBase = dataBase; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getWarehouseLayerName() { + return warehouseLayerName; + } + + public void setWarehouseLayerName(String warehouseLayerName) { + this.warehouseLayerName = warehouseLayerName; + } + + public String getWarehouseThemeName() { + return warehouseThemeName; + } + + public void setWarehouseThemeName(String warehouseThemeName) { + this.warehouseThemeName = warehouseThemeName; + } + + public String getUser() { + return user; + } + + public void setUser(String user) { + this.user = user; + } + + public Integer getPageSize() { + return pageSize; + } + + public void setPageSize(Integer pageSize) { + this.pageSize = pageSize; + } + + public Integer getPageNum() { + return pageNum; + } + + public void setPageNum(Integer pageNum) { + this.pageNum = pageNum; + } + + @Override + public String toString() { + return "TableCollectQueryVO{" + + "pageSize=" + pageSize + + ", pageNum=" + pageNum + + ", dataBase='" + dataBase + '\'' + + ", name='" + name + '\'' + + ", warehouseLayerName='" + warehouseLayerName + '\'' + + ", warehouseThemeName='" + warehouseThemeName + '\'' + + ", user='" + user + '\'' + + '}'; + } +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/vo/TableCollectVO.java b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/vo/TableCollectVO.java new file mode 100644 index 0000000000..425da6431d --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/vo/TableCollectVO.java @@ -0,0 +1,94 @@ +package com.webank.wedatasphere.dss.datamodel.table.vo; + + +import lombok.Data; +import lombok.ToString; + +import java.util.Date; + +@Data +@ToString +public class TableCollectVO { + + private Long tableId; + + private String user; + + private String dataBase; + + private String name; + + private String alias; + + private String creator; + + private String comment; + + + /** + * 数仓层级 + */ + private String warehouseLayerName; + + /** + * 数仓层级英文 + */ + private String warehouseLayerNameEn; + + /** + * 数仓主题格式为： theme_domain_name.theme_name + */ + private String warehouseThemeName; + + /** + * 数仓主题英文 + */ + private String warehouseThemeNameEn; + /** + * 生命周期 + */ + private String lifecycle; + + private String lifecycleEn; + + private Integer isPartitionTable; + + private Integer isAvailable; + + /** + * 存储类型：hive/mysql + */ + private String storageType; + + /** + * 授权的名字：userName、roleName + */ + private String principalName; + + /** + * 压缩格式 + */ + private String compress; + + /** + * 文件格式 + */ + private String fileType; + + + /** + * 是否外部表 0 内部表 1外部表 + */ + private Integer isExternal; + + /** + * 标签 + */ + private String label; + + + /** + * atlas标识 + */ + private String guid; +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/vo/TableColumnBindVO.java b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/vo/TableColumnBindVO.java new file mode 100644 index 0000000000..99f5be4555 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/vo/TableColumnBindVO.java @@ -0,0 +1,23 @@ +package com.webank.wedatasphere.dss.datamodel.table.vo; + +import lombok.Data; + +@Data +public class TableColumnBindVO { + /** + * 0 维度，1 指标 2 度量 + */ + private Integer modelType; + + /** + * 模型信息名称 + */ + private String modelName; + + /** + * 模型信息英文名称 + */ + private String modelNameEn; + + +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/vo/TableColumnVO.java b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/vo/TableColumnVO.java new file mode 100644 index 0000000000..adb2ae9433 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/vo/TableColumnVO.java @@ -0,0 +1,44 @@ +package com.webank.wedatasphere.dss.datamodel.table.vo; + +import lombok.Data; +import lombok.ToString; + +@Data +@ToString +public class TableColumnVO { + + private String name; + + private String alias; + + private String type; + + private String comment; + + private Integer isPartitionField; + + private Integer isPrimary; + + private Integer length; + + private String rule; + + /** + * 0 维度，1 指标 2 度量 + */ + private Integer modelType; + + /** + * 关联具体模型id信息 + */ + private Long modelId; + + + private String modelName; + + /** + * 模型信息英文名称 + */ + private String modelNameEn; + +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/vo/TableColumnsAddVO.java b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/vo/TableColumnsAddVO.java new file mode 100644 index 0000000000..987489dcaf --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/vo/TableColumnsAddVO.java @@ -0,0 +1,42 @@ +package com.webank.wedatasphere.dss.datamodel.table.vo; + +import lombok.Data; + +@Data +public class TableColumnsAddVO { + + private Long tableId; + + private String name; + + private String alias; + + private String type; + + private String comment; + + private Integer isPartitionField; + + private Integer isPrimary; + + private Integer length; + + private String rule; + + /** + * 0 维度，1 指标 2 度量 + */ + private Integer modelType; + + /** + * 模型信息名称 + */ + private String modelName; + + /** + * 模型信息英文名称 + */ + private String modelNameEn; + + +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/vo/TableCreateSqlVO.java b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/vo/TableCreateSqlVO.java new file mode 100644 index 0000000000..d7623760d9 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/vo/TableCreateSqlVO.java @@ -0,0 +1,15 @@ +package com.webank.wedatasphere.dss.datamodel.table.vo; + + +import lombok.Data; +import lombok.ToString; + +@Data +@ToString +public class TableCreateSqlVO { + private Long tableId; + + private String guid; + + private String user; +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/vo/TableCreateVO.java b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/vo/TableCreateVO.java new file mode 100644 index 0000000000..01077cb4d1 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/vo/TableCreateVO.java @@ -0,0 +1,13 @@ +package com.webank.wedatasphere.dss.datamodel.table.vo; + + +import lombok.Data; +import lombok.ToString; + +@Data +@ToString +public class TableCreateVO { + private Long tableId; + + private String user; +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/vo/TableDataPreviewVO.java b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/vo/TableDataPreviewVO.java new file mode 100644 index 0000000000..f203ddac8d --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/vo/TableDataPreviewVO.java @@ -0,0 +1,11 @@ +package com.webank.wedatasphere.dss.datamodel.table.vo; + + +import lombok.Data; + +@Data +public class TableDataPreviewVO { + private String tableName; + + private String user; +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/vo/TableDatabasesQueryVO.java b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/vo/TableDatabasesQueryVO.java new file mode 100644 index 0000000000..13f8a455bc --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/vo/TableDatabasesQueryVO.java @@ -0,0 +1,17 @@ +package com.webank.wedatasphere.dss.datamodel.table.vo; + + +import lombok.Data; +import lombok.ToString; + +@Data +@ToString +public class TableDatabasesQueryVO { + private String name; + + private Integer pageSize = 20; + + private Integer pageNum = 1; + + private String user; +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/vo/TableDictionaryListVO.java b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/vo/TableDictionaryListVO.java new file mode 100644 index 0000000000..00e09209b2 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/vo/TableDictionaryListVO.java @@ -0,0 +1,22 @@ +package com.webank.wedatasphere.dss.datamodel.table.vo; + + +public class TableDictionaryListVO { + + private String type; + + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } + + @Override + public String toString() { + return "TableDictionaryListVO{" + + "type='" + type + '\'' + + '}'; + } +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/vo/TableListVO.java b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/vo/TableListVO.java new file mode 100644 index 0000000000..9c3e4e41f7 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/vo/TableListVO.java @@ -0,0 +1,40 @@ +package com.webank.wedatasphere.dss.datamodel.table.vo; + + +import com.webank.wedatasphere.dss.datamodel.center.common.vo.PageVO; +import lombok.Data; +import lombok.ToString; + +@Data +@ToString +public class TableListVO extends PageVO { + + private String name; + + /** + * 数仓层级 + */ + private String warehouseLayerName; + + /** + * 数仓主题格式为： theme_domain_name.theme_name + */ + private String warehouseThemeName; + + + private String user; + + /** + * 0 维度 1 指标 2 度量 -1 按照表名查询 + */ + private Integer modelType = -1; + + + private String modelName; + + /** + * -1 表示查询逻辑表 0 表示查询asserts + */ + private Integer tableType = -1; + +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/vo/TablePartitionStatsVO.java b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/vo/TablePartitionStatsVO.java new file mode 100644 index 0000000000..62a027c9fa --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/vo/TablePartitionStatsVO.java @@ -0,0 +1,6 @@ +package com.webank.wedatasphere.dss.datamodel.table.vo; + + +public class TablePartitionStatsVO { + +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/vo/TableQueryOneVO.java b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/vo/TableQueryOneVO.java new file mode 100644 index 0000000000..82904d072f --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/vo/TableQueryOneVO.java @@ -0,0 +1,21 @@ +package com.webank.wedatasphere.dss.datamodel.table.vo; + + +import lombok.Data; +import lombok.ToString; + +@Data +@ToString +public class TableQueryOneVO { + + private String name; + + /** + * atlas标识 + */ + private String guid; + + + private String user; + +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/vo/TableUpdateVO.java b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/vo/TableUpdateVO.java new file mode 100644 index 0000000000..abd9e6bfa1 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/vo/TableUpdateVO.java @@ -0,0 +1,99 @@ +package com.webank.wedatasphere.dss.datamodel.table.vo; + +import lombok.Data; +import lombok.ToString; + +import java.util.Date; +import java.util.List; + +@Data +@ToString +public class TableUpdateVO { + + + private String dataBase; + + private String name; + + private String alias; + + private String creator; + + private String comment; + + private Date createTime; + + private Date updateTime; + + /** + * 数仓层级 + */ + private String warehouseLayerName; + + /** + * 数仓层级英文 + */ + private String warehouseLayerNameEn; + + /** + * 数仓主题格式为： theme_domain_name.theme_name + */ + private String warehouseThemeName; + + /** + * 数仓主题英文 + */ + private String warehouseThemeNameEn; + + /** + * 生命周期 + */ + private String lifecycle; + + private String lifecycleEn; + + private Integer isPartitionTable; + + private Integer isAvailable; + + /** + * 存储类型：hive/mysql + */ + private String storageType; + + /** + * 授权的名字：userName、roleName + */ + private String principalName; + + /** + * 压缩格式 + */ + private String compress; + + /** + * 文件格式 + */ + private String fileType; + + + /** + * 是否外部表 0 内部表 1外部表 + */ + private Integer isExternal; + + + /** + * 外部表时 location + */ + private String location; + + /** + * 标签 + */ + private String label; + + + private List columns; + +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/vo/TableVersionAddVO.java b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/vo/TableVersionAddVO.java new file mode 100644 index 0000000000..397892ca52 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/vo/TableVersionAddVO.java @@ -0,0 +1,90 @@ +package com.webank.wedatasphere.dss.datamodel.table.vo; + + +import lombok.Data; +import lombok.ToString; + +import java.util.Date; +import java.util.List; + +@Data +@ToString +public class TableVersionAddVO { + private String dataBase; + + private String name; + + private String alias; + + private String creator; + + private String comment; + + /** + * 数仓层级 + */ + private String warehouseLayerName; + + /** + * 数仓层级英文 + */ + private String warehouseLayerNameEn; + + /** + * 数仓主题格式为： theme_domain_name.theme_name + */ + private String warehouseThemeName; + + /** + * 数仓主题英文 + */ + private String warehouseThemeNameEn; + + /** + * 生命周期 + */ + private String lifecycle; + + private String lifecycleEn; + + private Integer isPartitionTable; + + private Integer isAvailable; + + /** + * 存储类型：hive/mysql + */ + private String storageType; + + /** + * 授权的名字：userName、roleName + */ + private String principalName; + + /** + * 压缩格式 + */ + private String compress; + + /** + * 文件格式 + */ + private String fileType; + + + /** + * 是否外部表 0 内部表 1外部表 + */ + private Integer isExternal; + + /** + * 外部表时 location + */ + private String location; + + private String label; + + private List columns; + + +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/vo/TableVersionQueryVO.java b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/vo/TableVersionQueryVO.java new file mode 100644 index 0000000000..c6ae4e44cf --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/vo/TableVersionQueryVO.java @@ -0,0 +1,11 @@ +package com.webank.wedatasphere.dss.datamodel.table.vo; + +import lombok.Data; +import lombok.ToString; + +@Data +@ToString +public class TableVersionQueryVO { + + private String name; +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/vo/TableVersionRollBackVO.java b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/vo/TableVersionRollBackVO.java new file mode 100644 index 0000000000..795a031979 --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/vo/TableVersionRollBackVO.java @@ -0,0 +1,14 @@ +package com.webank.wedatasphere.dss.datamodel.table.vo; + + +import lombok.Data; + +@Data +public class TableVersionRollBackVO { + + private String name; + + private String version; + + private String user; +} diff --git a/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/vo/TblPartitionStatsVO.java b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/vo/TblPartitionStatsVO.java new file mode 100644 index 0000000000..a8c7b00aad --- /dev/null +++ b/dss-apps/dss-datamodel-center/dss-datamodel-table-management/src/main/java/com/webank/wedatasphere/dss/datamodel/table/vo/TblPartitionStatsVO.java @@ -0,0 +1,15 @@ +package com.webank.wedatasphere.dss.datamodel.table.vo; + +import lombok.Data; +import lombok.ToString; + +@Data +@ToString +public class TblPartitionStatsVO { + + private String name; + + private String guid; + + private String user; +} diff --git a/dss-apps/dss-datamodel-center/pom.xml b/dss-apps/dss-datamodel-center/pom.xml new file mode 100644 index 0000000000..9afdcd866c --- /dev/null +++ b/dss-apps/dss-datamodel-center/pom.xml @@ -0,0 +1,55 @@ + + + + dss + com.webank.wedatasphere.dss + 1.0.1 + ../../pom.xml + + 4.0.0 + + dss-datamodel-center + pom + + dss-datamodel-center-server + dss-datamodel-dimension-management + dss-datamodel-indicator-management + dss-datamodel-measure-management + dss-datamodel-table-management + dss-datamodel-center-common + dss-datamodel-center-client + + + + + UTF-8 + 2.16 + 2.22.2 + 2.22.2 + + + + + org.modelmapper + modelmapper + 2.4.4 + + + + junit + junit + 4.12 + test + + + org.junit.platform + junit-platform-launcher + 1.5.2 + test + + + + + \ No newline at end of file diff --git a/dss-apps/dss-datawarehouse-design/data-warehouse-client/pom.xml b/dss-apps/dss-datawarehouse-design/data-warehouse-client/pom.xml new file mode 100644 index 0000000000..efb0a0075e --- /dev/null +++ b/dss-apps/dss-datawarehouse-design/data-warehouse-client/pom.xml @@ -0,0 +1,62 @@ + + + 4.0.0 + + dss-datawarehouse-design + com.webank.wedatasphere.dss + 1.0.1 + + + data-warehouse-client + jar + + + 8 + 8 + + + + + + org.apache.linkis + linkis-module + + + + org.apache.linkis + linkis-httpclient + + + + org.apache.linkis + linkis-gateway-httpclient-support + + + + org.projectlombok + lombok + provided + + + + + + + org.apache.maven.plugins + maven-deploy-plugin + + + net.alchim31.maven + scala-maven-plugin + + + org.apache.maven.plugins + maven-jar-plugin + + + + ${project.artifactId}-${project.version} + + \ No newline at end of file diff --git a/dss-apps/dss-datawarehouse-design/data-warehouse-client/src/main/java/com/webank/wedatasphere/warehouse/domain/DwLayerVO.java b/dss-apps/dss-datawarehouse-design/data-warehouse-client/src/main/java/com/webank/wedatasphere/warehouse/domain/DwLayerVO.java new file mode 100644 index 0000000000..6d222d3a45 --- /dev/null +++ b/dss-apps/dss-datawarehouse-design/data-warehouse-client/src/main/java/com/webank/wedatasphere/warehouse/domain/DwLayerVO.java @@ -0,0 +1,25 @@ +package com.webank.wedatasphere.warehouse.domain; + +import lombok.Getter; +import lombok.Setter; +import lombok.ToString; +import org.codehaus.jackson.annotate.JsonIgnoreProperties; + +@Setter +@Getter +@ToString +@JsonIgnoreProperties(ignoreUnknown = true) +public class DwLayerVO { + private String id; + private String name; + private String enName; + private Boolean preset; + private String description; + private String principalName; + private String dbs; + private Integer sort; + private Boolean isAvailable; + private Long createTime; + private Long updateTime; + private int referenceCount; +} diff --git a/dss-apps/dss-datawarehouse-design/data-warehouse-client/src/main/java/com/webank/wedatasphere/warehouse/domain/DwModifierVO.java b/dss-apps/dss-datawarehouse-design/data-warehouse-client/src/main/java/com/webank/wedatasphere/warehouse/domain/DwModifierVO.java new file mode 100644 index 0000000000..b9ee6cc9e5 --- /dev/null +++ b/dss-apps/dss-datawarehouse-design/data-warehouse-client/src/main/java/com/webank/wedatasphere/warehouse/domain/DwModifierVO.java @@ -0,0 +1,27 @@ +package com.webank.wedatasphere.warehouse.domain; + +import lombok.Getter; +import lombok.Setter; +import lombok.ToString; +import org.codehaus.jackson.annotate.JsonIgnoreProperties; + +import java.util.Date; + +@Setter +@Getter +@ToString +@JsonIgnoreProperties(ignoreUnknown = true) +public class DwModifierVO { + private Long id; + private String modifierType; + private String modifierTypeEn; + private String layerArea; + private String layerAreaEn; + private String themeArea; + private String themeAreaEn; + private String description; + private Boolean isAvailable; + private Date createTime; + private Date updateTime; + private int referenceCount; +} diff --git a/dss-apps/dss-datawarehouse-design/data-warehouse-client/src/main/java/com/webank/wedatasphere/warehouse/domain/DwStatisticalPeriodVO.java b/dss-apps/dss-datawarehouse-design/data-warehouse-client/src/main/java/com/webank/wedatasphere/warehouse/domain/DwStatisticalPeriodVO.java new file mode 100644 index 0000000000..77aeeee42f --- /dev/null +++ b/dss-apps/dss-datawarehouse-design/data-warehouse-client/src/main/java/com/webank/wedatasphere/warehouse/domain/DwStatisticalPeriodVO.java @@ -0,0 +1,54 @@ +package com.webank.wedatasphere.warehouse.domain; + +import lombok.Getter; +import lombok.Setter; +import lombok.ToString; +import org.codehaus.jackson.annotate.JsonIgnoreProperties; + +import java.util.Date; + +@Setter +@Getter +@ToString +@JsonIgnoreProperties(ignoreUnknown = true) +public class DwStatisticalPeriodVO { + private Long id; + + private Long themeDomainId; + + private Long layerId; + + private String name; + + private String enName; + + private String description; + + private String startTimeFormula; + + private String endTimeFormula; + + private String principalName; + + private Boolean isAvailable; + + private String owner; + + private Date createTime; + + private Date updateTime; + + private String themeArea; + + private String themeAreaEn; + + private String layerArea; + + private String layerAreaEn; + + transient private Boolean status; + + private Boolean referenced; + + private int referenceCount; +} diff --git a/dss-apps/dss-datawarehouse-design/data-warehouse-client/src/main/java/com/webank/wedatasphere/warehouse/domain/DwThemeDomainVO.java b/dss-apps/dss-datawarehouse-design/data-warehouse-client/src/main/java/com/webank/wedatasphere/warehouse/domain/DwThemeDomainVO.java new file mode 100644 index 0000000000..b3b3051b9d --- /dev/null +++ b/dss-apps/dss-datawarehouse-design/data-warehouse-client/src/main/java/com/webank/wedatasphere/warehouse/domain/DwThemeDomainVO.java @@ -0,0 +1,28 @@ +package com.webank.wedatasphere.warehouse.domain; + +import lombok.Getter; +import lombok.Setter; +import lombok.ToString; +import org.codehaus.jackson.annotate.JsonIgnoreProperties; + +import java.util.Date; + +@Setter +@Getter +@ToString +@JsonIgnoreProperties(ignoreUnknown = true) +public class DwThemeDomainVO { + private Long id; + private String name; + private String enName; + private String owner; + private String principalName; + private Integer sort; + private String description; + private Date createTime; + private Date updateTime; + private Boolean preset; + private Boolean isAvailable; + + private int referenceCount; +} diff --git a/dss-apps/dss-datawarehouse-design/data-warehouse-client/src/main/scala/com/webank/wedatasphere/warehouse/client/AbstractDwRemoteClient.scala b/dss-apps/dss-datawarehouse-design/data-warehouse-client/src/main/scala/com/webank/wedatasphere/warehouse/client/AbstractDwRemoteClient.scala new file mode 100644 index 0000000000..b4259e2b1f --- /dev/null +++ b/dss-apps/dss-datawarehouse-design/data-warehouse-client/src/main/scala/com/webank/wedatasphere/warehouse/client/AbstractDwRemoteClient.scala @@ -0,0 +1,13 @@ +package com.webank.wedatasphere.warehouse.client + +import org.apache.linkis.httpclient.dws.DWSHttpClient +import org.apache.linkis.httpclient.request.Action +import org.apache.linkis.httpclient.response.Result + +abstract class AbstractDwRemoteClient extends RemoteClient { + protected val dwsHttpClient: DWSHttpClient + override def execute(action: Action) : Result = action match { + case action: Action => dwsHttpClient.execute(action) + } + override def close() : scala.Unit = dwsHttpClient.close() +} \ No newline at end of file diff --git a/dss-apps/dss-datawarehouse-design/data-warehouse-client/src/main/scala/com/webank/wedatasphere/warehouse/client/DwAction.scala b/dss-apps/dss-datawarehouse-design/data-warehouse-client/src/main/scala/com/webank/wedatasphere/warehouse/client/DwAction.scala new file mode 100644 index 0000000000..0415687d78 --- /dev/null +++ b/dss-apps/dss-datawarehouse-design/data-warehouse-client/src/main/scala/com/webank/wedatasphere/warehouse/client/DwAction.scala @@ -0,0 +1,7 @@ +package com.webank.wedatasphere.warehouse.client + +import org.apache.linkis.httpclient.dws.request.DWSHttpAction +import org.apache.linkis.httpclient.request.UserAction + +trait DwAction extends DWSHttpAction with UserAction { +} diff --git a/dss-apps/dss-datawarehouse-design/data-warehouse-client/src/main/scala/com/webank/wedatasphere/warehouse/client/DwRemoteClient.scala b/dss-apps/dss-datawarehouse-design/data-warehouse-client/src/main/scala/com/webank/wedatasphere/warehouse/client/DwRemoteClient.scala new file mode 100644 index 0000000000..5fb3fe76f5 --- /dev/null +++ b/dss-apps/dss-datawarehouse-design/data-warehouse-client/src/main/scala/com/webank/wedatasphere/warehouse/client/DwRemoteClient.scala @@ -0,0 +1,14 @@ +package com.webank.wedatasphere.warehouse.client + +import com.webank.wedatasphere.warehouse.client.action.{ListDwLayerAction, ListDwModifierAction, ListDwStatisticalPeriodAction, ListDwThemeDomainAction} +import com.webank.wedatasphere.warehouse.client.result.{ListLayersResult, ListModifiersResult, ListStatisticalPeriodsResult, ListThemeDomainsResult} + +trait DwRemoteClient extends RemoteClient { + def listLayers(action : ListDwLayerAction) : ListLayersResult + + def listThemeDomains(action : ListDwThemeDomainAction) : ListThemeDomainsResult + + def listModifiers(action : ListDwModifierAction) : ListModifiersResult + + def listStatisticalPeriods(action : ListDwStatisticalPeriodAction) : ListStatisticalPeriodsResult +} diff --git a/dss-apps/dss-datawarehouse-design/data-warehouse-client/src/main/scala/com/webank/wedatasphere/warehouse/client/GovernanceDwRemoteClient.scala b/dss-apps/dss-datawarehouse-design/data-warehouse-client/src/main/scala/com/webank/wedatasphere/warehouse/client/GovernanceDwRemoteClient.scala new file mode 100644 index 0000000000..b0996493d0 --- /dev/null +++ b/dss-apps/dss-datawarehouse-design/data-warehouse-client/src/main/scala/com/webank/wedatasphere/warehouse/client/GovernanceDwRemoteClient.scala @@ -0,0 +1,25 @@ +package com.webank.wedatasphere.warehouse.client +import org.apache.linkis.httpclient.dws.DWSHttpClient +import org.apache.linkis.httpclient.dws.config.DWSClientConfig +import com.webank.wedatasphere.warehouse.client.action.{ListDwLayerAction, ListDwModifierAction, ListDwStatisticalPeriodAction, ListDwThemeDomainAction} +import com.webank.wedatasphere.warehouse.client.result.{ListLayersResult, ListModifiersResult, ListStatisticalPeriodsResult, ListThemeDomainsResult} + +class GovernanceDwRemoteClient(clientConfig: DWSClientConfig) extends AbstractDwRemoteClient with DwRemoteClient { + protected override val dwsHttpClient = new DWSHttpClient(clientConfig, "Governance-DataWarehouse-Client") + + override def listLayers(action: ListDwLayerAction): ListLayersResult = { + execute(action).asInstanceOf[ListLayersResult] + } + + override def listThemeDomains(action: ListDwThemeDomainAction): ListThemeDomainsResult = { + execute(action).asInstanceOf[ListThemeDomainsResult] + } + + override def listModifiers(action: ListDwModifierAction): ListModifiersResult = { + execute(action).asInstanceOf[ListModifiersResult] + } + + override def listStatisticalPeriods(action: ListDwStatisticalPeriodAction): ListStatisticalPeriodsResult = { + execute(action).asInstanceOf[ListStatisticalPeriodsResult] + } +} \ No newline at end of file diff --git a/dss-apps/dss-datawarehouse-design/data-warehouse-client/src/main/scala/com/webank/wedatasphere/warehouse/client/RemoteClient.scala b/dss-apps/dss-datawarehouse-design/data-warehouse-client/src/main/scala/com/webank/wedatasphere/warehouse/client/RemoteClient.scala new file mode 100644 index 0000000000..150331fa5c --- /dev/null +++ b/dss-apps/dss-datawarehouse-design/data-warehouse-client/src/main/scala/com/webank/wedatasphere/warehouse/client/RemoteClient.scala @@ -0,0 +1,10 @@ +package com.webank.wedatasphere.warehouse.client + +import org.apache.linkis.httpclient.request.Action +import org.apache.linkis.httpclient.response.Result + +trait RemoteClient extends java.io.Closeable { + protected def execute(action: Action): Result + + override def close(): scala.Unit +} diff --git a/dss-apps/dss-datawarehouse-design/data-warehouse-client/src/main/scala/com/webank/wedatasphere/warehouse/client/action/ListDwLayerAction.scala b/dss-apps/dss-datawarehouse-design/data-warehouse-client/src/main/scala/com/webank/wedatasphere/warehouse/client/action/ListDwLayerAction.scala new file mode 100644 index 0000000000..bfd721fb92 --- /dev/null +++ b/dss-apps/dss-datawarehouse-design/data-warehouse-client/src/main/scala/com/webank/wedatasphere/warehouse/client/action/ListDwLayerAction.scala @@ -0,0 +1,55 @@ +package com.webank.wedatasphere.warehouse.client.action + +import org.apache.linkis.httpclient.request.GetAction +import com.webank.wedatasphere.warehouse.client.DwAction + +class ListDwLayerAction() extends GetAction with DwAction { + private var user: String = _ +// private var isAvailable: Boolean = true +// def setIsAvailable(isAvailable: Boolean): Unit = this.isAvailable = isAvailable +// def getIsAvailable: Boolean = this.isAvailable + + override def setUser(user: String): Unit = this.user = user + + override def getUser: String = this.user + + override def suffixURLs: Array[String] = Array("data-warehouse", "layers", "all") +} + +object ListDwLayerAction { + def builder(): Builder = new Builder + + class Builder private[ListDwLayerAction]() { + private var user: String = _ + private var db: String = _ + private var isAvailable: Boolean = true + + def setUser(user: String): Builder = { + this.user = user + this + } + + def setIsAvailable(isAvailable: Boolean): Builder = { + this.isAvailable = isAvailable + this + } + + def setDb(db: String): Builder = { + this.db = db + this + } + + def build(): ListDwLayerAction = { + val action = new ListDwLayerAction + if (null != user) { + action.setUser(user) + } + if (null != db) { + action.setParameter("db", db); + } + action.setParameter("isAvailable", isAvailable) + action + } + } + +} \ No newline at end of file diff --git a/dss-apps/dss-datawarehouse-design/data-warehouse-client/src/main/scala/com/webank/wedatasphere/warehouse/client/action/ListDwModifierAction.scala b/dss-apps/dss-datawarehouse-design/data-warehouse-client/src/main/scala/com/webank/wedatasphere/warehouse/client/action/ListDwModifierAction.scala new file mode 100644 index 0000000000..a9b495c5c2 --- /dev/null +++ b/dss-apps/dss-datawarehouse-design/data-warehouse-client/src/main/scala/com/webank/wedatasphere/warehouse/client/action/ListDwModifierAction.scala @@ -0,0 +1,90 @@ +package com.webank.wedatasphere.warehouse.client.action + +import org.apache.linkis.httpclient.request.GetAction +import com.webank.wedatasphere.warehouse.client.DwAction + +class ListDwModifierAction extends GetAction with DwAction { + private var user: String = _ +// private var isAvailable: Boolean = _ +// private var theme: String = _ +// private var layer: String = _ + +// def setIsAvailable(isAvailable: Boolean): Unit = this.isAvailable = isAvailable +// +// def getIsAvailable: Boolean = this.isAvailable +// +// def setTheme(theme: String): Unit = this.theme = theme +// +// def getTheme: String = this.theme +// +// def setLayer(layer: String): Unit = this.layer = layer +// +// def getLayer: String = this.layer + + override def setUser(user: String): Unit = this.user = user + + override def getUser: String = this.user + + override def suffixURLs: Array[String] = Array("data-warehouse", "modifiers", "all") +} + +object ListDwModifierAction { + def builder(): Builder = new Builder + + class Builder private[ListDwModifierAction]() { + private var user: String = _ + private var name: String = "" + private var layer: String = "" + private var theme: String = "" + private var isAvailable: Boolean = true + + def setUser(user: String): Builder = { + this.user = user + this + } + + def setName(name: String): Builder = { + this.name = name + this + } + + def setLayer(layer: String): Builder = { + this.layer = layer + this + } + + def setTheme(theme: String): Builder = { + this.theme = theme + this + } + + def setIsAvailable(isAvailable: Boolean): Builder = { + this.isAvailable = isAvailable + this + } + + def build(): ListDwModifierAction = { + val action = new ListDwModifierAction +// action.setUser(user) +// action.setParameter("isAvailable", isAvailable) +// action.setParameter("typeName", name) +// action.setParameter("layer", layer) +// action.setParameter("theme", theme) + if (null != user) { + action.setUser(user) + } + action.setParameter("isAvailable", isAvailable) + if (null != name) { + action.setParameter("typeName", name) + } + if (null != layer) { + action.setParameter("layer", layer) + } + if (null != theme) { + action.setParameter("theme", theme) + } + action + } + } + +} \ No newline at end of file diff --git a/dss-apps/dss-datawarehouse-design/data-warehouse-client/src/main/scala/com/webank/wedatasphere/warehouse/client/action/ListDwStatisticalPeriodAction.scala b/dss-apps/dss-datawarehouse-design/data-warehouse-client/src/main/scala/com/webank/wedatasphere/warehouse/client/action/ListDwStatisticalPeriodAction.scala new file mode 100644 index 0000000000..db69c32bd3 --- /dev/null +++ b/dss-apps/dss-datawarehouse-design/data-warehouse-client/src/main/scala/com/webank/wedatasphere/warehouse/client/action/ListDwStatisticalPeriodAction.scala @@ -0,0 +1,73 @@ +package com.webank.wedatasphere.warehouse.client.action + +import org.apache.linkis.httpclient.request.GetAction +import com.webank.wedatasphere.warehouse.client.DwAction + +class ListDwStatisticalPeriodAction extends GetAction with DwAction { + private var user: String = _ +// private var isAvailable: Boolean = true +// def setIsAvailable(isAvailable: Boolean): Unit = this.isAvailable = isAvailable +// def getIsAvailable: Boolean = this.isAvailable + + override def setUser(user: String): Unit = this.user = user + + override def getUser: String = this.user + + override def suffixURLs: Array[String] = Array("data-warehouse", "statistical_periods", "all") +} + +object ListDwStatisticalPeriodAction { + def builder(): Builder = new Builder + + class Builder private[ListDwStatisticalPeriodAction]() { + private var user: String = _ + private var name: String = "" + private var layer: String = "" + private var theme: String = "" + private var isAvailable: Boolean = true + + def setUser(user: String): Builder = { + this.user = user + this + } + + def setName(name: String): Builder = { + this.name = name + this + } + + def setLayer(layer: String): Builder = { + this.layer = layer + this + } + + def setTheme(theme: String): Builder = { + this.theme = theme + this + } + + def setIsAvailable(isAvailable: Boolean): Builder = { + this.isAvailable = isAvailable + this + } + + def build(): ListDwStatisticalPeriodAction = { + val action = new ListDwStatisticalPeriodAction + if (null != user) { + action.setUser(user) + } + action.setParameter("isAvailable", isAvailable) + if (null != name) { + action.setParameter("name", name) + } + if (null != layer) { + action.setParameter("layer", layer) + } + if (null != theme) { + action.setParameter("theme", theme) + } + action + } + } + +} \ No newline at end of file diff --git a/dss-apps/dss-datawarehouse-design/data-warehouse-client/src/main/scala/com/webank/wedatasphere/warehouse/client/action/ListDwThemeDomainAction.scala b/dss-apps/dss-datawarehouse-design/data-warehouse-client/src/main/scala/com/webank/wedatasphere/warehouse/client/action/ListDwThemeDomainAction.scala new file mode 100644 index 0000000000..d040284fb0 --- /dev/null +++ b/dss-apps/dss-datawarehouse-design/data-warehouse-client/src/main/scala/com/webank/wedatasphere/warehouse/client/action/ListDwThemeDomainAction.scala @@ -0,0 +1,46 @@ +package com.webank.wedatasphere.warehouse.client.action + +import org.apache.linkis.httpclient.request.GetAction +import com.webank.wedatasphere.warehouse.client.DwAction + +class ListDwThemeDomainAction extends GetAction with DwAction { + private var user: String = _ +// private var isAvailable: Boolean = true +// def setIsAvailable(isAvailable: Boolean): Unit = this.isAvailable = isAvailable +// def getIsAvailable: Boolean = this.isAvailable + + override def setUser(user: String): Unit = this.user = user + + override def getUser: String = this.user + + override def suffixURLs: Array[String] = Array("data-warehouse", "themedomains", "all") +} + +object ListDwThemeDomainAction { + def builder(): Builder = new Builder + + class Builder private[ListDwThemeDomainAction]() { + private var user: String = _ + private var isAvailable: Boolean = true + + def setUser(user: String): Builder = { + this.user = user + this + } + + def setIsAvailable(isAvailable: Boolean): Builder = { + this.isAvailable = isAvailable + this + } + + def build(): ListDwThemeDomainAction = { + val action = new ListDwThemeDomainAction + if (null != user) { + action.setUser(user) + } + action.setParameter("isAvailable", isAvailable) + action + } + } + +} \ No newline at end of file diff --git a/dss-apps/dss-datawarehouse-design/data-warehouse-client/src/main/scala/com/webank/wedatasphere/warehouse/client/result/ListLayersResult.scala b/dss-apps/dss-datawarehouse-design/data-warehouse-client/src/main/scala/com/webank/wedatasphere/warehouse/client/result/ListLayersResult.scala new file mode 100644 index 0000000000..e66892d388 --- /dev/null +++ b/dss-apps/dss-datawarehouse-design/data-warehouse-client/src/main/scala/com/webank/wedatasphere/warehouse/client/result/ListLayersResult.scala @@ -0,0 +1,23 @@ +package com.webank.wedatasphere.warehouse.client.result + +import org.apache.linkis.httpclient.dws.DWSHttpClient +import org.apache.linkis.httpclient.dws.response.DWSResult +import com.webank.wedatasphere.warehouse.domain.DwLayerVO +import org.apache.linkis.httpclient.dws.annotation.DWSHttpMessageResult + +import java.util +import scala.beans.BeanProperty + +@DWSHttpMessageResult(value = "/api/rest_j/v\\d+/data-warehouse/layers/all") +class ListLayersResult extends DWSResult { + @BeanProperty var list: java.util.List[java.util.Map[String, Any]] = _ + + def getAll: util.List[DwLayerVO] = { + import scala.collection.JavaConverters._ + list.asScala.map(x=>{ + val str = DWSHttpClient.jacksonJson.writeValueAsString(x) + DWSHttpClient.jacksonJson.readValue(str, classOf[DwLayerVO]) + }).asJava + } + +} diff --git a/dss-apps/dss-datawarehouse-design/data-warehouse-client/src/main/scala/com/webank/wedatasphere/warehouse/client/result/ListModifiersResult.scala b/dss-apps/dss-datawarehouse-design/data-warehouse-client/src/main/scala/com/webank/wedatasphere/warehouse/client/result/ListModifiersResult.scala new file mode 100644 index 0000000000..aefeb646ab --- /dev/null +++ b/dss-apps/dss-datawarehouse-design/data-warehouse-client/src/main/scala/com/webank/wedatasphere/warehouse/client/result/ListModifiersResult.scala @@ -0,0 +1,22 @@ +package com.webank.wedatasphere.warehouse.client.result + +import org.apache.linkis.httpclient.dws.DWSHttpClient +import org.apache.linkis.httpclient.dws.annotation.DWSHttpMessageResult +import org.apache.linkis.httpclient.dws.response.DWSResult +import com.webank.wedatasphere.warehouse.domain.DwModifierVO + +import java.util +import scala.beans.BeanProperty + +@DWSHttpMessageResult(value = "/api/rest_j/v\\d+/data-warehouse/modifiers/all") +class ListModifiersResult extends DWSResult { + @BeanProperty var list: java.util.List[java.util.Map[String, Any]] = _ + + def getAll: util.List[DwModifierVO] = { + import scala.collection.JavaConverters._ + list.asScala.map(x=>{ + val str = DWSHttpClient.jacksonJson.writeValueAsString(x) + DWSHttpClient.jacksonJson.readValue(str, classOf[DwModifierVO]) + }).asJava + } +} diff --git a/dss-apps/dss-datawarehouse-design/data-warehouse-client/src/main/scala/com/webank/wedatasphere/warehouse/client/result/ListStatisticalPeriodsResult.scala b/dss-apps/dss-datawarehouse-design/data-warehouse-client/src/main/scala/com/webank/wedatasphere/warehouse/client/result/ListStatisticalPeriodsResult.scala new file mode 100644 index 0000000000..1d0f5df0f2 --- /dev/null +++ b/dss-apps/dss-datawarehouse-design/data-warehouse-client/src/main/scala/com/webank/wedatasphere/warehouse/client/result/ListStatisticalPeriodsResult.scala @@ -0,0 +1,22 @@ +package com.webank.wedatasphere.warehouse.client.result + +import org.apache.linkis.httpclient.dws.DWSHttpClient +import org.apache.linkis.httpclient.dws.annotation.DWSHttpMessageResult +import org.apache.linkis.httpclient.dws.response.DWSResult +import com.webank.wedatasphere.warehouse.domain.DwStatisticalPeriodVO + +import java.util +import scala.beans.BeanProperty + +@DWSHttpMessageResult(value = "/api/rest_j/v\\d+/data-warehouse/statistical_periods/all") +class ListStatisticalPeriodsResult extends DWSResult { + @BeanProperty var list: java.util.List[java.util.Map[String, Any]] = _ + + def getAll: util.List[DwStatisticalPeriodVO] = { + import scala.collection.JavaConverters._ + list.asScala.map(x=>{ + val str = DWSHttpClient.jacksonJson.writeValueAsString(x) + DWSHttpClient.jacksonJson.readValue(str, classOf[DwStatisticalPeriodVO]) + }).asJava + } +} diff --git a/dss-apps/dss-datawarehouse-design/data-warehouse-client/src/main/scala/com/webank/wedatasphere/warehouse/client/result/ListThemeDomainsResult.scala b/dss-apps/dss-datawarehouse-design/data-warehouse-client/src/main/scala/com/webank/wedatasphere/warehouse/client/result/ListThemeDomainsResult.scala new file mode 100644 index 0000000000..8424265d04 --- /dev/null +++ b/dss-apps/dss-datawarehouse-design/data-warehouse-client/src/main/scala/com/webank/wedatasphere/warehouse/client/result/ListThemeDomainsResult.scala @@ -0,0 +1,23 @@ +package com.webank.wedatasphere.warehouse.client.result + +import org.apache.linkis.httpclient.dws.DWSHttpClient +import org.apache.linkis.httpclient.dws.annotation.DWSHttpMessageResult +import org.apache.linkis.httpclient.dws.response.DWSResult +import com.webank.wedatasphere.warehouse.domain.DwThemeDomainVO + +import java.util +import scala.beans.BeanProperty + +@DWSHttpMessageResult(value = "/api/rest_j/v\\d+/data-warehouse/themedomains/all") +class ListThemeDomainsResult extends DWSResult { + @BeanProperty var list: java.util.List[java.util.Map[String, Any]] = _ + + def getAll: util.List[DwThemeDomainVO] = { + import scala.collection.JavaConverters._ + list.asScala.map(x=>{ + val str = DWSHttpClient.jacksonJson.writeValueAsString(x) + DWSHttpClient.jacksonJson.readValue(str, classOf[DwThemeDomainVO]) + }).asJava + } + +} \ No newline at end of file diff --git a/dss-apps/dss-datawarehouse-design/data-warehouse-client/src/test/java/com/webank/wedatasphere/warehouse/client/RemoteClientTest.java b/dss-apps/dss-datawarehouse-design/data-warehouse-client/src/test/java/com/webank/wedatasphere/warehouse/client/RemoteClientTest.java new file mode 100644 index 0000000000..810bb4f1f9 --- /dev/null +++ b/dss-apps/dss-datawarehouse-design/data-warehouse-client/src/test/java/com/webank/wedatasphere/warehouse/client/RemoteClientTest.java @@ -0,0 +1,56 @@ +package com.webank.wedatasphere.warehouse.client; + +import org.apache.linkis.httpclient.dws.authentication.StaticAuthenticationStrategy; +import org.apache.linkis.httpclient.dws.config.DWSClientConfig; +import org.apache.linkis.httpclient.dws.config.DWSClientConfigBuilder; +import com.webank.wedatasphere.warehouse.client.action.ListDwLayerAction; +import com.webank.wedatasphere.warehouse.client.action.ListDwModifierAction; +import com.webank.wedatasphere.warehouse.client.action.ListDwStatisticalPeriodAction; +import com.webank.wedatasphere.warehouse.client.action.ListDwThemeDomainAction; +import com.webank.wedatasphere.warehouse.client.result.ListLayersResult; +import com.webank.wedatasphere.warehouse.client.result.ListModifiersResult; +import com.webank.wedatasphere.warehouse.client.result.ListStatisticalPeriodsResult; +import com.webank.wedatasphere.warehouse.client.result.ListThemeDomainsResult; + +import java.util.List; +import java.util.concurrent.TimeUnit; + +public class RemoteClientTest { + + public static void main(String[] args) { +// LinkisMetadataSourceRemoteClient client = getMetadataSourceRemoteClient(); +// GetMetadataSourceAllDatabasesAction action = GetMetadataSourceAllDatabasesAction.builder().setUser("hdfs").build(); +// GetMetadataSourceAllDatabasesResult result = client.getAllDBMetaDataSource(action); +// List dbs = result.getDbs(); +// System.out.println(dbs); + DWSClientConfig clientConfig = ((DWSClientConfigBuilder) (DWSClientConfigBuilder.newBuilder().addServerUrl("http://dss.shineweng.com:8088") + .connectionTimeout(30000).discoveryEnabled(true) + .discoveryFrequency(1, TimeUnit.MINUTES) + .loadbalancerEnabled(true).maxConnectionSize(5) + .retryEnabled(false).readTimeout(30000) + .setAuthenticationStrategy(new StaticAuthenticationStrategy()).setAuthTokenKey("hdfs") + .setAuthTokenValue("hdfs"))).setDWSVersion("v1").build(); + GovernanceDwRemoteClient governanceDwRemoteClient = new GovernanceDwRemoteClient(clientConfig); + + ListDwStatisticalPeriodAction action = new ListDwStatisticalPeriodAction.Builder().setLayer(null).setTheme(null).setUser("hdfs").setIsAvailable(false).build(); + ListStatisticalPeriodsResult listStatisticalPeriodsResult = governanceDwRemoteClient.listStatisticalPeriods(action); + System.out.println(listStatisticalPeriodsResult.getAll().size()); + + ListDwLayerAction listDwLayerAction = new ListDwLayerAction.Builder().setUser("hdfs").setIsAvailable(false).build(); + String url = listDwLayerAction.getURL(); + System.out.println(url); + ListLayersResult listLayersResult = governanceDwRemoteClient.listLayers(listDwLayerAction); + System.out.println(listLayersResult.getAll().size()); + + ListDwThemeDomainAction listDwThemeDomainAction = new ListDwThemeDomainAction.Builder().setUser("hdfs").setIsAvailable(false).build(); + ListThemeDomainsResult listThemeDomainsResult = governanceDwRemoteClient.listThemeDomains(listDwThemeDomainAction); + System.out.println(listThemeDomainsResult.getList().size()); + + ListDwModifierAction listDwModifierAction = new ListDwModifierAction.Builder().setUser("hdfs").setIsAvailable(true).build(); + ListModifiersResult listModifiersResult = governanceDwRemoteClient.listModifiers(listDwModifierAction); + System.out.println(listModifiersResult.getAll().size()); + + } + + +} diff --git a/dss-apps/dss-datawarehouse-design/data-warehouse-client/src/test/scala/com/webank/wedatasphere/warehouse/client/DwLayerRemoteClientTest.scala b/dss-apps/dss-datawarehouse-design/data-warehouse-client/src/test/scala/com/webank/wedatasphere/warehouse/client/DwLayerRemoteClientTest.scala new file mode 100644 index 0000000000..67e2ace31b --- /dev/null +++ b/dss-apps/dss-datawarehouse-design/data-warehouse-client/src/test/scala/com/webank/wedatasphere/warehouse/client/DwLayerRemoteClientTest.scala @@ -0,0 +1,46 @@ +package com.webank.wedatasphere.warehouse.client + +import org.apache.linkis.httpclient.dws.authentication.StaticAuthenticationStrategy +import org.apache.linkis.httpclient.dws.config.{DWSClientConfig, DWSClientConfigBuilder} +import com.webank.wedatasphere.warehouse.client.action.{ListDwLayerAction, ListDwModifierAction, ListDwThemeDomainAction} + +import java.util.concurrent.TimeUnit + +object DwLayerRemoteClientTest { + val serverUrl: String = s"http://gateway:29001" + val clientConfig: DWSClientConfig = DWSClientConfigBuilder.newBuilder() + .addServerUrl(serverUrl) + .connectionTimeout(30000L) + .discoveryEnabled(true) + .discoveryFrequency(1L, TimeUnit.MINUTES) + .loadbalancerEnabled(true) + .maxConnectionSize(5) + .retryEnabled(false) + .readTimeout(30000L) + .setAuthenticationStrategy(new StaticAuthenticationStrategy()) + .setAuthTokenKey("hdfs") + .setAuthTokenValue("hdfs") + .setDWSVersion("v1") + .build() + + val client = new GovernanceDwRemoteClient(clientConfig) + + def main(args: Array[String]): Unit = { + val action = new ListDwLayerAction + val result = client.listLayers(action) + println(result.getAll) + + val action1 = new ListDwThemeDomainAction + val result1 = client.listThemeDomains(action1) + println(result1.getAll) + + val action2 = new ListDwModifierAction + val result2 = client.listModifiers(action2) + println(result2.getAll) + + val action3 = new ListDwThemeDomainAction + val result3 = client.listThemeDomains(action3) + println(result3.getAll) + } + +} diff --git a/dss-apps/dss-datawarehouse-design/data-warehouse-mybatis/pom.xml b/dss-apps/dss-datawarehouse-design/data-warehouse-mybatis/pom.xml new file mode 100644 index 0000000000..d2def4927c --- /dev/null +++ b/dss-apps/dss-datawarehouse-design/data-warehouse-mybatis/pom.xml @@ -0,0 +1,62 @@ + + + 4.0.0 + + dss-datawarehouse-design + com.webank.wedatasphere.dss + 1.0.1 + + + data-warehouse-mybatis + jar + + + 8 + 8 + 3.4.1 + + + + + org.apache.linkis + linkis-module + provided + + + spring-jdbc + org.springframework + ${spring.version} + + + com.baomidou + mybatis-plus-boot-starter + ${mybatis-plus.boot.starter.version} + + + spring-boot-starter + org.springframework.boot + + + spring-boot-autoconfigure + org.springframework.boot + + + spring-beans + org.springframework + + + spring-jdbc + org.springframework + + + + + org.projectlombok + lombok + provided + + + + \ No newline at end of file diff --git a/dss-apps/dss-datawarehouse-design/data-warehouse-mybatis/src/main/java/com/webank/wedatasphere/linkis/mybatis/DataSourceConfig.java b/dss-apps/dss-datawarehouse-design/data-warehouse-mybatis/src/main/java/com/webank/wedatasphere/linkis/mybatis/DataSourceConfig.java new file mode 100644 index 0000000000..033fb8cff1 --- /dev/null +++ b/dss-apps/dss-datawarehouse-design/data-warehouse-mybatis/src/main/java/com/webank/wedatasphere/linkis/mybatis/DataSourceConfig.java @@ -0,0 +1,76 @@ +/* + * Copyright 2019 WeBank + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.mybatis; + +import org.apache.linkis.common.utils.JavaLog; +import org.apache.linkis.mybatis.conf.MybatisConfiguration; +import org.apache.commons.dbcp.BasicDataSource; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +import javax.sql.DataSource; + +@Configuration +@ConfigurationProperties +public class DataSourceConfig extends JavaLog { + + private static final Logger LOGGER = LoggerFactory.getLogger(DataSourceConfig.class); + + @Bean(name="dataSource", destroyMethod = "close") + @ConditionalOnMissingBean + public DataSource dataSource(){ + String dbUrl = MybatisConfiguration.BDP_SERVER_MYBATIS_DATASOURCE_URL.getValue(); + String username = MybatisConfiguration.BDP_SERVER_MYBATIS_DATASOURCE_USERNAME.getValue(); + String password = MybatisConfiguration.BDP_SERVER_MYBATIS_DATASOURCE_PASSWORD.getValue(); + String driverClassName = MybatisConfiguration.BDP_SERVER_MYBATIS_DATASOURCE_DRIVER_CLASS_NAME.getValue(); + int initialSize = MybatisConfiguration.BDP_SERVER_MYBATIS_DATASOURCE_INITIALSIZE.getValue(); + int minIdle = MybatisConfiguration.BDP_SERVER_MYBATIS_DATASOURCE_MINIDLE.getValue(); + int maxActive = MybatisConfiguration.BDP_SERVER_MYBATIS_DATASOURCE_MAXACTIVE.getValue(); + int maxWait = MybatisConfiguration.BDP_SERVER_MYBATIS_DATASOURCE_MAXWAIT.getValue(); + int timeBetweenEvictionRunsMillis = MybatisConfiguration.BDP_SERVER_MYBATIS_DATASOURCE_TBERM.getValue(); + int minEvictableIdleTimeMillis = MybatisConfiguration.BDP_SERVER_MYBATIS_DATASOURCE_MEITM.getValue(); + String validationQuery = MybatisConfiguration.BDP_SERVER_MYBATIS_DATASOURCE_VALIDATIONQUERY.getValue(); + boolean testWhileIdle = MybatisConfiguration.BDP_SERVER_MYBATIS_DATASOURCE_TESTWHILEIDLE.getValue(); + boolean testOnBorrow = MybatisConfiguration.BDP_SERVER_MYBATIS_DATASOURCE_TESTONBORROW.getValue(); + boolean testOnReturn = MybatisConfiguration.BDP_SERVER_MYBATIS_DATASOURCE_TESTONRETURN.getValue(); + boolean poolPreparedStatements = MybatisConfiguration.BDP_SERVER_MYBATIS_DATASOURCE_POOLPREPAREDSTATEMENTS.getValue(); + BasicDataSource datasource = new BasicDataSource(); + info("Database connection address information(数据库连接地址信息)=" + dbUrl); + datasource.setUrl(dbUrl); + datasource.setUsername(username); + datasource.setPassword(password); + datasource.setDriverClassName(driverClassName); + datasource.setInitialSize(initialSize); + datasource.setMinIdle(minIdle); + datasource.setMaxActive(maxActive); + datasource.setMaxWait(maxWait); + datasource.setTimeBetweenEvictionRunsMillis(timeBetweenEvictionRunsMillis); + datasource.setMinEvictableIdleTimeMillis(minEvictableIdleTimeMillis); + datasource.setValidationQuery(validationQuery); + datasource.setTestWhileIdle(testWhileIdle); + datasource.setTestOnBorrow(testOnBorrow); + datasource.setTestOnReturn(testOnReturn); + datasource.setPoolPreparedStatements(poolPreparedStatements); + datasource.setDefaultAutoCommit(false); + return datasource; + } + +} diff --git a/dss-apps/dss-datawarehouse-design/data-warehouse-mybatis/src/main/java/com/webank/wedatasphere/linkis/mybatis/MyBatisMapperScannerConfig.java b/dss-apps/dss-datawarehouse-design/data-warehouse-mybatis/src/main/java/com/webank/wedatasphere/linkis/mybatis/MyBatisMapperScannerConfig.java new file mode 100644 index 0000000000..804c562b61 --- /dev/null +++ b/dss-apps/dss-datawarehouse-design/data-warehouse-mybatis/src/main/java/com/webank/wedatasphere/linkis/mybatis/MyBatisMapperScannerConfig.java @@ -0,0 +1,39 @@ +/* + * Copyright 2019 WeBank + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.mybatis; + +import org.apache.linkis.mybatis.conf.MybatisConfiguration; +import org.mybatis.spring.mapper.MapperScannerConfigurer; +import org.springframework.boot.autoconfigure.AutoConfigureAfter; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + + +@Configuration +@AutoConfigureAfter(MybatisConfigurationFactory.class) +public class MyBatisMapperScannerConfig { + @Bean + public MapperScannerConfigurer mapperScannerConfigurer() { + MapperScannerConfigurer mapperScannerConfigurer = new MapperScannerConfigurer(); + mapperScannerConfigurer.setSqlSessionFactoryBeanName("sqlSessionFactory"); + //Each table corresponds to the XXMapper.java interface type Java file + //每张表对应的XXMapper.java interface类型的Java文件 + mapperScannerConfigurer.setBasePackage(MybatisConfiguration.BDP_SERVER_MYBATIS_BASEPACKAGE.getValue()); + return mapperScannerConfigurer; + } + +} diff --git a/dss-apps/dss-datawarehouse-design/data-warehouse-mybatis/src/main/java/com/webank/wedatasphere/linkis/mybatis/MybatisConfigurationFactory.java b/dss-apps/dss-datawarehouse-design/data-warehouse-mybatis/src/main/java/com/webank/wedatasphere/linkis/mybatis/MybatisConfigurationFactory.java new file mode 100644 index 0000000000..be56464746 --- /dev/null +++ b/dss-apps/dss-datawarehouse-design/data-warehouse-mybatis/src/main/java/com/webank/wedatasphere/linkis/mybatis/MybatisConfigurationFactory.java @@ -0,0 +1,130 @@ +/* + * Copyright 2019 WeBank + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.mybatis; + +import com.baomidou.mybatisplus.annotation.DbType; +import com.baomidou.mybatisplus.extension.plugins.MybatisPlusInterceptor; +import com.baomidou.mybatisplus.extension.plugins.inner.PaginationInnerInterceptor; +import com.baomidou.mybatisplus.extension.spring.MybatisSqlSessionFactoryBean; +import org.apache.linkis.common.utils.JavaLog; +import org.apache.linkis.mybatis.conf.MybatisConfiguration; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.ibatis.plugin.Interceptor; +import org.apache.ibatis.session.SqlSessionFactory; +import org.mybatis.spring.SqlSessionTemplate; +import org.springframework.beans.factory.ObjectProvider; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.AutoConfigureAfter; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Primary; +import org.springframework.core.io.DefaultResourceLoader; +import org.springframework.core.io.Resource; +import org.springframework.core.io.support.PathMatchingResourcePatternResolver; +import org.springframework.jdbc.datasource.DataSourceTransactionManager; +import org.springframework.transaction.PlatformTransactionManager; +import org.springframework.transaction.annotation.EnableTransactionManagement; + +import javax.sql.DataSource; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + + +@Configuration +@ConfigurationProperties +@AutoConfigureAfter(DataSourceConfig.class) +@EnableTransactionManagement +public class MybatisConfigurationFactory extends JavaLog { + + @Autowired + private DataSource dataSource; + // Provide SqlSeesion(提供SqlSeesion) + @Bean(name = "sqlSessionFactory") + @Primary + public MybatisSqlSessionFactoryBean sqlSessionFactory(ObjectProvider interceptorsProvider) { + String typeAliasesPackage = MybatisConfiguration.BDP_SERVER_MYBATIS_TYPEALIASESPACKAGE.getValue(); + //Configure the mapper scan to find all mapper.xml mapping files(配置mapper的扫描，找到所有的mapper.xml映射文件) + String mapperLocations = MybatisConfiguration.BDP_SERVER_MYBATIS_MAPPER_LOCATIONS.getValue(); + //Load the global configuration file(加载全局的配置文件) + String configLocation = MybatisConfiguration.BDP_SERVER_MYBATIS_CONFIGLOCATION.getValue(); + try { + MybatisSqlSessionFactoryBean sessionFactoryBean = new MybatisSqlSessionFactoryBean(); + sessionFactoryBean.setDataSource(dataSource); + + info("Mybatis typeAliasesPackage=" + typeAliasesPackage); + info("Mybatis mapperLocations=" + mapperLocations); + info("Mybatis configLocation=" + configLocation); + // Read configuration(读取配置) + sessionFactoryBean.setTypeAliasesPackage(typeAliasesPackage); + + //Set the location of the mapper.xml file(设置mapper.xml文件所在位置) + if(StringUtils.isNotBlank(mapperLocations)) { + String[] mapperArray = mapperLocations.split(","); + List resources = new ArrayList<>(); + for(String mapperLocation : mapperArray){ + CollectionUtils.addAll(resources,new PathMatchingResourcePatternResolver().getResources(mapperLocation)); + } + sessionFactoryBean.setMapperLocations(resources.toArray(new Resource[0])); + } + /* Resource[] resources = new PathMatchingResourcePatternResolver().getResources(mapperLocations); + sessionFactoryBean.setMapperLocations(resources);*/ +// Set the location of the mybatis-config.xml configuration file(设置mybatis-config.xml配置文件位置) + sessionFactoryBean.setConfigLocation(new DefaultResourceLoader().getResource(configLocation)); + + sessionFactoryBean.setPlugins(interceptorsProvider.getIfAvailable()); + + return sessionFactoryBean; + } catch (IOException e) { + error("mybatis resolver mapper*xml is error",e); + return null; + } catch (Exception e) { + error("mybatis sqlSessionFactoryBean create error",e); + return null; + } + } + + @Bean + @Primary + public SqlSessionTemplate sqlSessionTemplate(SqlSessionFactory sqlSessionFactory) { + return new SqlSessionTemplate(sqlSessionFactory); + } + + //Transaction management(事务管理) + @Bean + @Primary + public PlatformTransactionManager annotationDrivenTransactionManager() { + return new DataSourceTransactionManager(dataSource); + } +// Log the log to be executed (if you don't want to intercept it, comment out this method) +// 将要执行的sql进行日志打印(不想拦截，就把这方法注释掉) +// @Bean +// public SqlPrintInterceptor sqlPrintInterceptor(){ +// return new SqlPrintInterceptor(); +// } + + // 3.4.1 + @Bean + public MybatisPlusInterceptor mybatisPlusInterceptor() { + MybatisPlusInterceptor interceptor = new MybatisPlusInterceptor(); + interceptor.addInnerInterceptor(new PaginationInnerInterceptor(DbType.MYSQL)); + return interceptor; + } + +} diff --git a/dss-apps/dss-datawarehouse-design/data-warehouse-mybatis/src/main/java/com/webank/wedatasphere/linkis/mybatis/conf/MybatisConfiguration.java b/dss-apps/dss-datawarehouse-design/data-warehouse-mybatis/src/main/java/com/webank/wedatasphere/linkis/mybatis/conf/MybatisConfiguration.java new file mode 100644 index 0000000000..43b91c5d80 --- /dev/null +++ b/dss-apps/dss-datawarehouse-design/data-warehouse-mybatis/src/main/java/com/webank/wedatasphere/linkis/mybatis/conf/MybatisConfiguration.java @@ -0,0 +1,43 @@ +/* + * Copyright 2019 WeBank + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.mybatis.conf; + +import org.apache.linkis.common.conf.CommonVars; + +public class MybatisConfiguration { + //Mybatis configuration + public static final CommonVars BDP_SERVER_MYBATIS_MAPPER_LOCATIONS = CommonVars.apply("wds.linkis.server.mybatis.mapperLocations", ""); + public static final CommonVars BDP_SERVER_MYBATIS_TYPEALIASESPACKAGE = CommonVars.apply("wds.linkis.server.mybatis.typeAliasesPackage", ""); + public static final CommonVars BDP_SERVER_MYBATIS_CONFIGLOCATION = CommonVars.apply("wds.linkis.server.mybatis.configLocation", "classpath:/mybatis-config.xml"); + public static final CommonVars BDP_SERVER_MYBATIS_BASEPACKAGE = CommonVars.apply("wds.linkis.server.mybatis.BasePackage", ""); + public static final CommonVars BDP_SERVER_MYBATIS_DATASOURCE_URL = CommonVars.apply("wds.linkis.server.mybatis.datasource.url", ""); + public static final CommonVars BDP_SERVER_MYBATIS_DATASOURCE_USERNAME = CommonVars.apply("wds.linkis.server.mybatis.datasource.username", ""); + public static final CommonVars BDP_SERVER_MYBATIS_DATASOURCE_PASSWORD = CommonVars.apply("wds.linkis.server.mybatis.datasource.password", ""); + public static final CommonVars BDP_SERVER_MYBATIS_DATASOURCE_DRIVER_CLASS_NAME = CommonVars.apply("wds.linkis.server.mybatis.datasource.driver-class-name", "com.mysql.jdbc.Driver"); + public static final CommonVars BDP_SERVER_MYBATIS_DATASOURCE_INITIALSIZE = CommonVars.apply("wds.linkis.server.mybatis.datasource.initialSize", new Integer(1)); + public static final CommonVars BDP_SERVER_MYBATIS_DATASOURCE_MINIDLE = CommonVars.apply("wds.linkis.server.mybatis.datasource.minIdle", new Integer(1)); + public static final CommonVars BDP_SERVER_MYBATIS_DATASOURCE_MAXACTIVE = CommonVars.apply("wds.linkis.server.mybatis.datasource.maxActive", new Integer(20)); + public static final CommonVars BDP_SERVER_MYBATIS_DATASOURCE_MAXWAIT = CommonVars.apply("wds.linkis.server.mybatis.datasource.maxWait", new Integer(6000)); + public static final CommonVars BDP_SERVER_MYBATIS_DATASOURCE_TBERM = CommonVars.apply("wds.linkis.server.mybatis.datasource.timeBetweenEvictionRunsMillis", new Integer(60000)); + public static final CommonVars BDP_SERVER_MYBATIS_DATASOURCE_MEITM = CommonVars.apply("wds.linkis.server.mybatis.datasource.minEvictableIdleTimeMillis", new Integer(300000)); + public static final CommonVars BDP_SERVER_MYBATIS_DATASOURCE_VALIDATIONQUERY = CommonVars.apply("wds.linkis.server.mybatis.datasource.validationQuery", "SELECT 1"); + + public static final CommonVars BDP_SERVER_MYBATIS_DATASOURCE_TESTWHILEIDLE = CommonVars.apply("wds.linkis.server.mybatis.datasource.testWhileIdle", new Boolean(true)); + public static final CommonVars BDP_SERVER_MYBATIS_DATASOURCE_TESTONBORROW = CommonVars.apply("wds.linkis.server.mybatis.datasource.testOnBorrow", new Boolean(false)); + public static final CommonVars BDP_SERVER_MYBATIS_DATASOURCE_TESTONRETURN = CommonVars.apply("wds.linkis.server.mybatis.datasource.testOnReturn", new Boolean(false)); + public static final CommonVars BDP_SERVER_MYBATIS_DATASOURCE_POOLPREPAREDSTATEMENTS = CommonVars.apply("wds.linkis.server.mybatis.datasource.poolPreparedStatements", new Boolean(true)); +} diff --git a/dss-apps/dss-datawarehouse-design/data-warehouse-server/bin/start-warehous.sh b/dss-apps/dss-datawarehouse-design/data-warehouse-server/bin/start-warehous.sh new file mode 100644 index 0000000000..ce0e80b0a1 --- /dev/null +++ b/dss-apps/dss-datawarehouse-design/data-warehouse-server/bin/start-warehous.sh @@ -0,0 +1,38 @@ +#!/bin/bash + +cd `dirname $0` +cd .. +HOME=`pwd` + +export SERVER_PID=$HOME/bin/linkis.pid +export SERVER_LOG_PATH=$HOME/logs +export SERVER_CLASS=com.webank.wedatasphere.warehouse.DataWarehouseApplication + +if test -z "$SERVER_HEAP_SIZE" +then + export SERVER_HEAP_SIZE="512M" +fi + +if test -z "$SERVER_JAVA_OPTS" +then + export SERVER_JAVA_OPTS=" -Xmx$SERVER_HEAP_SIZE -XX:+UseG1GC -Xloggc:$HOME/logs/linkis-gc.log" +fi + +if [[ -f "${SERVER_PID}" ]]; then + pid=$(cat ${SERVER_PID}) + if kill -0 ${pid} >/dev/null 2>&1; then + echo "Server is already running." + exit 1 + fi +fi + +nohup java $SERVER_JAVA_OPTS -cp $HOME/conf:$HOME/lib/* $SERVER_CLASS 2>&1 > $SERVER_LOG_PATH/linkis.out & +pid=$! +if [[ -z "${pid}" ]]; then + echo "server $SERVER_NAME start failed!" + exit 1 +else + echo "server $SERVER_NAME start succeeded!" + echo $pid > $SERVER_PID + sleep 1 +fi \ No newline at end of file diff --git a/dss-apps/dss-datawarehouse-design/data-warehouse-server/bin/stop-warehous.sh b/dss-apps/dss-datawarehouse-design/data-warehouse-server/bin/stop-warehous.sh new file mode 100644 index 0000000000..f032887111 --- /dev/null +++ b/dss-apps/dss-datawarehouse-design/data-warehouse-server/bin/stop-warehous.sh @@ -0,0 +1,47 @@ +#!/bin/bash + +cd `dirname $0` +cd .. +HOME=`pwd` + +export SERVER_PID=$HOME/bin/linkis.pid + +function wait_for_server_to_die() { + local pid + local count + pid=$1 + timeout=$2 + count=0 + timeoutTime=$(date "+%s") + let "timeoutTime+=$timeout" + currentTime=$(date "+%s") + forceKill=1 + + while [[ $currentTime -lt $timeoutTime ]]; do + $(kill ${pid} > /dev/null 2> /dev/null) + if kill -0 ${pid} > /dev/null 2>&1; then + sleep 3 + else + forceKill=0 + break + fi + currentTime=$(date "+%s") + done + + if [[ forceKill -ne 0 ]]; then + $(kill -9 ${pid} > /dev/null 2> /dev/null) + fi +} + +if [[ ! -f "${SERVER_PID}" ]]; then + echo "server $SERVER_NAME is not running" +else + pid=$(cat ${SERVER_PID}) + if [[ -z "${pid}" ]]; then + echo "server $SERVER_NAME is not running" + else + wait_for_server_to_die $pid 40 + $(rm -f ${SERVER_PID}) + echo "server $SERVER_NAME is stopped." + fi +fi \ No newline at end of file diff --git a/dss-apps/dss-datawarehouse-design/data-warehouse-server/db/datawarehouse.sql b/dss-apps/dss-datawarehouse-design/data-warehouse-server/db/datawarehouse.sql new file mode 100644 index 0000000000..1818a6371c --- /dev/null +++ b/dss-apps/dss-datawarehouse-design/data-warehouse-server/db/datawarehouse.sql @@ -0,0 +1,224 @@ + + +/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */; +/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */; +/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */; +SET NAMES utf8mb4; +/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */; +/*!40101 SET @OLD_SQL_MODE='NO_AUTO_VALUE_ON_ZERO', SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */; +/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */; + + +# 转储表 dss_datawarehouse_layer +# ------------------------------------------------------------ + +DROP TABLE IF EXISTS `dss_datawarehouse_layer`; + +CREATE TABLE `dss_datawarehouse_layer` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `name` varchar(255) COLLATE utf8_bin NOT NULL, + `en_name` varchar(255) COLLATE utf8_bin NOT NULL, + `owner` varchar(255) COLLATE utf8_bin DEFAULT NULL, + `principal_name` varchar(128) COLLATE utf8_bin DEFAULT NULL COMMENT '授权的名字：userName、roleName', + `is_available` bit(1) NOT NULL, + `preset` bit(1) NOT NULL DEFAULT b'0', + `sort` int(4) NOT NULL DEFAULT '1', + `description` varchar(255) COLLATE utf8_bin DEFAULT NULL, + `dbs` varchar(255) COLLATE utf8_bin NOT NULL COMMENT '如果为空代表所有的库', + `create_time` datetime NOT NULL, + `update_time` datetime NOT NULL, + `status` bit(1) NOT NULL DEFAULT b'1', + `lock_version` bigint(20) NOT NULL DEFAULT '1', + PRIMARY KEY (`id`), + UNIQUE KEY `name` (`name`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin; + +LOCK TABLES `dss_datawarehouse_layer` WRITE; +/*!40000 ALTER TABLE `dss_datawarehouse_layer` DISABLE KEYS */; + +INSERT INTO `dss_datawarehouse_layer` (`id`, `name`, `en_name`, `owner`, `principal_name`, `is_available`, `preset`, `sort`, `description`, `dbs`, `create_time`, `update_time`, `status`, `lock_version`) +VALUES + (1,'原数据层（ODS）','ods','admin','所有角色',b'1',b'1',10,'由业务系统同步到数据仓库的原始数据，一般不经过加工','ALL','2021-09-01 00:00:00','2021-09-01 00:00:00',b'1',1), + (2,'明细层（DWD）','dwd','admin','所有角色',b'1',b'1',20,'从ods层经过ETL得到的明细数据，表示具体的事实','ALL','2021-09-01 00:00:00','2021-09-01 00:00:00',b'1',1), + (3,'汇总层（DWS）','dws','admin','所有角色',b'1',b'1',30,'由明细数据经过汇总得到的数据，主要由统计维度和指标构成','ALL','2021-09-01 00:00:00','2021-09-01 00:00:00',b'1',1); + +/*!40000 ALTER TABLE `dss_datawarehouse_layer` ENABLE KEYS */; +UNLOCK TABLES; + + +# 转储表 dss_datawarehouse_layer_generalize_rule +# ------------------------------------------------------------ + +DROP TABLE IF EXISTS `dss_datawarehouse_layer_generalize_rule`; + +CREATE TABLE `dss_datawarehouse_layer_generalize_rule` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `layer_id` bigint(20) NOT NULL, + `regex` varchar(255) COLLATE utf8_bin NOT NULL COMMENT '自动归纳表达式', + `identifier` varchar(255) COLLATE utf8_bin NOT NULL, + `en_identifier` varchar(255) COLLATE utf8_bin NOT NULL, + `description` varchar(255) COLLATE utf8_bin DEFAULT NULL, + `create_time` datetime NOT NULL, + `update_time` datetime NOT NULL, + `status` bit(1) NOT NULL DEFAULT b'1', + `lock_version` bigint(20) NOT NULL DEFAULT '1', + PRIMARY KEY (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin; + + + +# 转储表 dss_datawarehouse_modifier +# ------------------------------------------------------------ + +DROP TABLE IF EXISTS `dss_datawarehouse_modifier`; + +CREATE TABLE `dss_datawarehouse_modifier` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `modifier_type` varchar(64) COLLATE utf8_bin NOT NULL, + `theme_domain_id` bigint(20) DEFAULT NULL, + `layer_id` bigint(20) DEFAULT NULL, + `theme_area` varchar(1000) COLLATE utf8_bin NOT NULL COMMENT '空：代表所有，如果是逗号分隔的字符串则代表对应的theme的names', + `layer_area` varchar(1000) COLLATE utf8_bin NOT NULL COMMENT '空：代表所有，如果是逗号分隔的字符串则代表对应的layer的names', + `description` varchar(255) COLLATE utf8_bin DEFAULT NULL, + `is_available` bit(1) NOT NULL, + `create_time` datetime NOT NULL, + `update_time` datetime NOT NULL, + `status` bit(1) NOT NULL DEFAULT b'1', + `lock_version` bigint(20) NOT NULL DEFAULT '1', + PRIMARY KEY (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin; + + + +# 转储表 dss_datawarehouse_modifier_list +# ------------------------------------------------------------ + +DROP TABLE IF EXISTS `dss_datawarehouse_modifier_list`; + +CREATE TABLE `dss_datawarehouse_modifier_list` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `modifier_id` bigint(20) NOT NULL, + `name` varchar(255) COLLATE utf8_bin NOT NULL, + `identifier` varchar(255) COLLATE utf8_bin NOT NULL, + `formula` varchar(255) COLLATE utf8_bin NOT NULL, + `create_time` datetime NOT NULL, + `update_time` datetime NOT NULL, + PRIMARY KEY (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin; + + + +# 转储表 dss_datawarehouse_statistical_period +# ------------------------------------------------------------ + +DROP TABLE IF EXISTS `dss_datawarehouse_statistical_period`; + +CREATE TABLE `dss_datawarehouse_statistical_period` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `theme_domain_id` bigint(20) NOT NULL, + `layer_id` bigint(20) NOT NULL, + `name` varchar(255) COLLATE utf8_bin NOT NULL, + `en_name` varchar(255) COLLATE utf8_bin NOT NULL, + `start_time_formula` varchar(255) COLLATE utf8_bin DEFAULT NULL, + `end_time_formula` varchar(255) COLLATE utf8_bin DEFAULT NULL, + `owner` varchar(255) COLLATE utf8_bin DEFAULT NULL, + `principal_name` varchar(128) COLLATE utf8_bin DEFAULT NULL COMMENT '授权的名字：userName、roleName', + `description` varchar(255) COLLATE utf8_bin DEFAULT NULL, + `is_available` bit(1) NOT NULL, + `create_time` datetime NOT NULL, + `update_time` datetime NOT NULL, + `status` bit(1) NOT NULL DEFAULT b'1', + `lock_version` bigint(20) NOT NULL DEFAULT '1', + PRIMARY KEY (`id`), + UNIQUE KEY `name` (`name`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin; + + + +# 转储表 dss_datawarehouse_table_rule +# ------------------------------------------------------------ + +DROP TABLE IF EXISTS `dss_datawarehouse_table_rule`; + +CREATE TABLE `dss_datawarehouse_table_rule` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `name` bigint(20) NOT NULL, + `theme_area` varchar(1000) COLLATE utf8_bin NOT NULL COMMENT '空：代表所有，如果是逗号分隔的字符串则代表对应的theme的names', + `layer_area` varchar(1000) COLLATE utf8_bin NOT NULL COMMENT '空：代表所有，如果是逗号分隔的字符串则代表对应的layer的names', + `table_name_rule` varchar(1000) COLLATE utf8_bin NOT NULL, + `table_props_rule` varchar(1000) COLLATE utf8_bin NOT NULL, + `partation_rule` varchar(1000) COLLATE utf8_bin NOT NULL, + `column_rule` varchar(1000) COLLATE utf8_bin NOT NULL, + `description` varchar(255) COLLATE utf8_bin DEFAULT NULL, + `is_available` bit(1) NOT NULL, + `create_time` datetime NOT NULL, + `update_time` datetime NOT NULL, + PRIMARY KEY (`id`), + UNIQUE KEY `name` (`name`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin; + + + +# 转储表 dss_datawarehouse_theme +# ------------------------------------------------------------ + +DROP TABLE IF EXISTS `dss_datawarehouse_theme`; + +CREATE TABLE `dss_datawarehouse_theme` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `theme_domain_id` bigint(20) NOT NULL, + `name` varchar(255) COLLATE utf8_bin NOT NULL, + `en_name` varchar(255) COLLATE utf8_bin NOT NULL, + `parent_theme_name` varchar(255) COLLATE utf8_bin DEFAULT NULL COMMENT '默认为空，如果不为空则指向父主题', + `owner` varchar(255) COLLATE utf8_bin DEFAULT NULL, + `is_available` bit(1) NOT NULL, + `sort` int(4) NOT NULL, + `description` varchar(255) COLLATE utf8_bin DEFAULT NULL, + `create_time` datetime NOT NULL, + `update_time` datetime NOT NULL, + PRIMARY KEY (`id`), + UNIQUE KEY `name` (`name`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin; + + + +# 转储表 dss_datawarehouse_theme_domain +# ------------------------------------------------------------ + +DROP TABLE IF EXISTS `dss_datawarehouse_theme_domain`; + +CREATE TABLE `dss_datawarehouse_theme_domain` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `name` varchar(255) COLLATE utf8_bin NOT NULL, + `en_name` varchar(255) COLLATE utf8_bin NOT NULL, + `owner` varchar(255) COLLATE utf8_bin DEFAULT NULL, + `principal_name` varchar(128) COLLATE utf8_bin DEFAULT NULL COMMENT '授权的名字：userName、roleName', + `is_available` bit(1) NOT NULL, + `sort` int(4) NOT NULL, + `description` varchar(255) COLLATE utf8_bin DEFAULT NULL, + `create_time` datetime NOT NULL, + `update_time` datetime NOT NULL, + `status` bit(1) NOT NULL DEFAULT b'1', + `lock_version` bigint(20) NOT NULL DEFAULT '1', + PRIMARY KEY (`id`), + UNIQUE KEY `name` (`name`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin; + +LOCK TABLES `dss_datawarehouse_theme_domain` WRITE; +/*!40000 ALTER TABLE `dss_datawarehouse_theme_domain` DISABLE KEYS */; + +INSERT INTO `dss_datawarehouse_theme_domain` (`id`, `name`, `en_name`, `owner`, `principal_name`, `is_available`, `sort`, `description`, `create_time`, `update_time`, `status`, `lock_version`) +VALUES + (1,'主题','英文名','负责人','New York',b'1',1,'描述','2021-09-28 13:18:48','2021-09-28 13:18:48',b'1',1); + +/*!40000 ALTER TABLE `dss_datawarehouse_theme_domain` ENABLE KEYS */; +UNLOCK TABLES; + + + +/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */; +/*!40101 SET SQL_MODE=@OLD_SQL_MODE */; +/*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */; +/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */; +/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */; +/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */; diff --git a/dss-apps/dss-datawarehouse-design/data-warehouse-server/db/ddl.sql b/dss-apps/dss-datawarehouse-design/data-warehouse-server/db/ddl.sql new file mode 100644 index 0000000000..1cfb217f99 --- /dev/null +++ b/dss-apps/dss-datawarehouse-design/data-warehouse-server/db/ddl.sql @@ -0,0 +1,180 @@ +-- governance.dss_datawarehouse_layer definition + +DROP TABLE IF EXISTS `dss_datawarehouse_layer`; +CREATE TABLE `dss_datawarehouse_layer` +( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `name` varchar(255) COLLATE utf8_bin NOT NULL, + `en_name` varchar(255) COLLATE utf8_bin NOT NULL, + `owner` varchar(255) COLLATE utf8_bin DEFAULT NULL, + `principal_name` varchar(128) COLLATE utf8_bin DEFAULT NULL COMMENT '授权的名字：userName、roleName', + `is_available` bit(1) NOT NULL, + `preset` bit(1) NOT NULL DEFAULT b'0', + `sort` int(4) NOT NULL DEFAULT '1', + `description` varchar(255) COLLATE utf8_bin DEFAULT NULL, + `dbs` varchar(255) COLLATE utf8_bin NOT NULL COMMENT '如果为空代表所有的库', + `create_time` datetime NOT NULL, + `update_time` datetime NOT NULL, + `status` bit(1) NOT NULL DEFAULT b'1', + `lock_version` bigint(20) NOT NULL DEFAULT '1', + PRIMARY KEY (`id`), + UNIQUE KEY `name` (`name`) +) ENGINE = InnoDB + AUTO_INCREMENT = 44 + DEFAULT CHARSET = utf8 + COLLATE = utf8_bin; + + +-- governance.dss_datawarehouse_layer_generalize_rule definition +DROP TABLE IF EXISTS `dss_datawarehouse_layer_generalize_rule`; +CREATE TABLE `dss_datawarehouse_layer_generalize_rule` +( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `layer_id` bigint(20) NOT NULL, + `regex` varchar(255) COLLATE utf8_bin NOT NULL COMMENT '自动归纳表达式', + `identifier` varchar(255) COLLATE utf8_bin NOT NULL, + `en_identifier` varchar(255) COLLATE utf8_bin NOT NULL, + `description` varchar(255) COLLATE utf8_bin DEFAULT NULL, + `create_time` datetime NOT NULL, + `update_time` datetime NOT NULL, + `status` bit(1) NOT NULL DEFAULT b'1', + `lock_version` bigint(20) NOT NULL DEFAULT '1', + PRIMARY KEY (`id`) +) ENGINE = InnoDB + DEFAULT CHARSET = utf8 + COLLATE = utf8_bin; + +-- governance.dss_datawarehouse_modifier definition +DROP TABLE IF EXISTS `dss_datawarehouse_modifier`; +CREATE TABLE `dss_datawarehouse_modifier` +( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `modifier_type` varchar(64) COLLATE utf8_bin NOT NULL, + `modifier_type_en` varchar(128) COLLATE utf8_bin DEFAULT NULL, + `theme_domain_id` bigint(20) DEFAULT NULL, + `layer_id` bigint(20) DEFAULT NULL, + `theme_area` varchar(1000) COLLATE utf8_bin NOT NULL COMMENT '空：代表所有，如果是逗号分隔的字符串则代表对应的theme的names', + `layer_area` varchar(1000) COLLATE utf8_bin NOT NULL COMMENT '空：代表所有，如果是逗号分隔的字符串则代表对应的layer的names', + `description` varchar(255) COLLATE utf8_bin DEFAULT NULL, + `theme_area_en` varchar(100) COLLATE utf8_bin DEFAULT NULL, + `layer_area_en` varchar(100) COLLATE utf8_bin DEFAULT NULL, + `is_available` bit(1) NOT NULL, + `create_time` datetime NOT NULL, + `update_time` datetime NOT NULL, + `status` bit(1) NOT NULL DEFAULT b'1', + `lock_version` bigint(20) NOT NULL DEFAULT '1', + PRIMARY KEY (`id`) +) ENGINE = InnoDB + AUTO_INCREMENT = 25 + DEFAULT CHARSET = utf8 + COLLATE = utf8_bin; + +-- governance.dss_datawarehouse_modifier_list definition +DROP TABLE IF EXISTS `dss_datawarehouse_modifier_list`; +CREATE TABLE `dss_datawarehouse_modifier_list` +( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `modifier_id` bigint(20) NOT NULL, + `name` varchar(255) COLLATE utf8_bin NOT NULL, + `identifier` varchar(255) COLLATE utf8_bin NOT NULL, + `formula` varchar(255) COLLATE utf8_bin NOT NULL, + `create_time` datetime NOT NULL, + `update_time` datetime NOT NULL, + PRIMARY KEY (`id`) +) ENGINE = InnoDB + AUTO_INCREMENT = 37 + DEFAULT CHARSET = utf8 + COLLATE = utf8_bin; + +-- governance.dss_datawarehouse_statistical_period definition +DROP TABLE IF EXISTS `dss_datawarehouse_statistical_period`; +CREATE TABLE `dss_datawarehouse_statistical_period` +( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `theme_domain_id` bigint(20) NOT NULL, + `layer_id` bigint(20) NOT NULL, + `name` varchar(255) COLLATE utf8_bin NOT NULL, + `en_name` varchar(255) COLLATE utf8_bin NOT NULL, + `start_time_formula` varchar(255) COLLATE utf8_bin DEFAULT NULL, + `end_time_formula` varchar(255) COLLATE utf8_bin DEFAULT NULL, + `owner` varchar(255) COLLATE utf8_bin DEFAULT NULL, + `principal_name` varchar(128) COLLATE utf8_bin DEFAULT NULL COMMENT '授权的名字：userName、roleName', + `description` varchar(255) COLLATE utf8_bin DEFAULT NULL, + `is_available` bit(1) NOT NULL, + `create_time` datetime NOT NULL, + `update_time` datetime NOT NULL, + `status` bit(1) NOT NULL DEFAULT b'1', + `lock_version` bigint(20) NOT NULL DEFAULT '1', + PRIMARY KEY (`id`), + UNIQUE KEY `name` (`name`) +) ENGINE = InnoDB + AUTO_INCREMENT = 23 + DEFAULT CHARSET = utf8 + COLLATE = utf8_bin; + +-- governance.dss_datawarehouse_table_rule definition +DROP TABLE IF EXISTS `dss_datawarehouse_table_rule`; +CREATE TABLE `dss_datawarehouse_table_rule` +( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `name` bigint(20) NOT NULL, + `theme_area` varchar(1000) COLLATE utf8_bin NOT NULL COMMENT '空：代表所有，如果是逗号分隔的字符串则代表对应的theme的names', + `layer_area` varchar(1000) COLLATE utf8_bin NOT NULL COMMENT '空：代表所有，如果是逗号分隔的字符串则代表对应的layer的names', + `table_name_rule` varchar(1000) COLLATE utf8_bin NOT NULL, + `table_props_rule` varchar(1000) COLLATE utf8_bin NOT NULL, + `partation_rule` varchar(1000) COLLATE utf8_bin NOT NULL, + `column_rule` varchar(1000) COLLATE utf8_bin NOT NULL, + `description` varchar(255) COLLATE utf8_bin DEFAULT NULL, + `is_available` bit(1) NOT NULL, + `create_time` datetime NOT NULL, + `update_time` datetime NOT NULL, + PRIMARY KEY (`id`), + UNIQUE KEY `name` (`name`) +) ENGINE = InnoDB + DEFAULT CHARSET = utf8 + COLLATE = utf8_bin; + + +-- governance.dss_datawarehouse_theme definition +DROP TABLE IF EXISTS `dss_datawarehouse_theme`; +CREATE TABLE `dss_datawarehouse_theme` +( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `theme_domain_id` bigint(20) NOT NULL, + `name` varchar(255) COLLATE utf8_bin NOT NULL, + `en_name` varchar(255) COLLATE utf8_bin NOT NULL, + `parent_theme_name` varchar(255) COLLATE utf8_bin DEFAULT NULL COMMENT '默认为空，如果不为空则指向父主题', + `owner` varchar(255) COLLATE utf8_bin DEFAULT NULL, + `is_available` bit(1) NOT NULL, + `sort` int(4) NOT NULL, + `description` varchar(255) COLLATE utf8_bin DEFAULT NULL, + `create_time` datetime NOT NULL, + `update_time` datetime NOT NULL, + PRIMARY KEY (`id`), + UNIQUE KEY `name` (`name`) +) ENGINE = InnoDB + DEFAULT CHARSET = utf8 + COLLATE = utf8_bin; + +-- governance.dss_datawarehouse_theme_domain definition +DROP TABLE IF EXISTS `dss_datawarehouse_theme_domain`; +CREATE TABLE `dss_datawarehouse_theme_domain` +( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `name` varchar(255) COLLATE utf8_bin NOT NULL, + `en_name` varchar(255) COLLATE utf8_bin NOT NULL, + `owner` varchar(255) COLLATE utf8_bin DEFAULT NULL, + `principal_name` varchar(128) COLLATE utf8_bin DEFAULT NULL COMMENT '授权的名字：userName、roleName', + `is_available` bit(1) NOT NULL, + `sort` int(4) NOT NULL, + `description` varchar(255) COLLATE utf8_bin DEFAULT NULL, + `create_time` datetime NOT NULL, + `update_time` datetime NOT NULL, + `status` bit(1) NOT NULL DEFAULT b'1', + `lock_version` bigint(20) NOT NULL DEFAULT '1', + PRIMARY KEY (`id`), + UNIQUE KEY `name` (`name`) +) ENGINE = InnoDB + AUTO_INCREMENT = 56 + DEFAULT CHARSET = utf8 + COLLATE = utf8_bin; \ No newline at end of file diff --git a/dss-apps/dss-datawarehouse-design/data-warehouse-server/db/dml.sql b/dss-apps/dss-datawarehouse-design/data-warehouse-server/db/dml.sql new file mode 100644 index 0000000000..65493a39d3 --- /dev/null +++ b/dss-apps/dss-datawarehouse-design/data-warehouse-server/db/dml.sql @@ -0,0 +1,4 @@ +INSERT INTO `dss_datawarehouse_layer` (name,en_name,owner,principal_name,is_available,preset,sort,description,dbs,create_time,update_time,status,lock_version) VALUES +('原数据层（ODS）','ods','admin','ALL',1,1,10,'由业务系统同步到数据仓库的原始数据，一般不经过加工','ALL','2021-09-01 00:00:00','2021-12-08 11:04:38',1,23), +('明细层（DWD）','dwd','admin','ALL',1,1,20,'从ods层经过ETL得到的明细数据，表示具体的事实','ALL','2021-09-01 00:00:00','2021-11-09 15:44:47',1,8), +('汇总层（DWS）','dws','admin','ALL',1,1,30,'由明细数据经过汇总得到的数据，主要由统计维度和指标构成','ALL','2021-09-01 00:00:00','2021-11-09 15:44:47',1,9); \ No newline at end of file diff --git a/dss-apps/dss-datawarehouse-design/data-warehouse-server/pom.xml b/dss-apps/dss-datawarehouse-design/data-warehouse-server/pom.xml new file mode 100644 index 0000000000..a0489c8b1e --- /dev/null +++ b/dss-apps/dss-datawarehouse-design/data-warehouse-server/pom.xml @@ -0,0 +1,109 @@ + + + 4.0.0 + + dss-datawarehouse-design + com.webank.wedatasphere.dss + 1.0.1 + + + data-warehouse-server + jar + + + 8 + 8 + + + + + + com.webank.wedatasphere.dss + data-warehouse-service + + + + org.apache.linkis + linkis-module + + + + org.projectlombok + lombok + provided + + + + + + + org.apache.maven.plugins + maven-deploy-plugin + + + + net.alchim31.maven + scala-maven-plugin + + + org.apache.maven.plugins + maven-jar-plugin + + + org.apache.maven.plugins + maven-assembly-plugin + 2.3 + false + + + make-assembly + package + + single + + + + src/main/assembly/distribution.xml + + + + + + false + data-warehouse-server + false + false + + src/main/assembly/distribution.xml + + + + + org.apache.maven.plugins + maven-compiler-plugin + + 8 + 8 + + + + + + src/main/java + + **/*.xml + + + + src/main/resources + + **/*.xml + **/*.properties + **/*.yml + + + + + \ No newline at end of file diff --git a/dss-apps/dss-datawarehouse-design/data-warehouse-server/src/main/assembly/distribution.xml b/dss-apps/dss-datawarehouse-design/data-warehouse-server/src/main/assembly/distribution.xml new file mode 100644 index 0000000000..bdfa90d428 --- /dev/null +++ b/dss-apps/dss-datawarehouse-design/data-warehouse-server/src/main/assembly/distribution.xml @@ -0,0 +1,71 @@ + + + + data-warehouse-server + + zip + + true + data-warehouse-server + + + + + + lib + true + true + false + false + true + + + + + + ${basedir}/src/main/resources + + * + + 0777 + conf + unix + + + ${basedir}/bin + + * + + 0777 + bin + unix + + + . + + */** + + logs + + + + + diff --git a/dss-apps/dss-datawarehouse-design/data-warehouse-server/src/main/java/com/webank/wedatasphere/warehouse/configuration/MybatisPlusConfig.java b/dss-apps/dss-datawarehouse-design/data-warehouse-server/src/main/java/com/webank/wedatasphere/warehouse/configuration/MybatisPlusConfig.java new file mode 100644 index 0000000000..61dd65abb5 --- /dev/null +++ b/dss-apps/dss-datawarehouse-design/data-warehouse-server/src/main/java/com/webank/wedatasphere/warehouse/configuration/MybatisPlusConfig.java @@ -0,0 +1,27 @@ +package com.webank.wedatasphere.warehouse.configuration; + +import com.webank.wedatasphere.warehouse.dao.interceptor.DssWorkspaceNameAutoExtractQueryInterceptor; +import com.webank.wedatasphere.warehouse.dao.interceptor.DssWorkspaceNameAutoTransformUpdateInteceptor; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +import javax.sql.DataSource; + +@Configuration +public class MybatisPlusConfig { + + @Bean + public DssWorkspaceNameAutoTransformUpdateInteceptor dssWorkspaceNameAutoTransformUpdateInteceptor( + DataSource dataSource + ) { + return new DssWorkspaceNameAutoTransformUpdateInteceptor(dataSource); + } + + @Bean + public DssWorkspaceNameAutoExtractQueryInterceptor dssWorkspaceNameAutoExtractQueryInterceptor( + DataSource dataSource + ) { + return new DssWorkspaceNameAutoExtractQueryInterceptor(dataSource); + } + +} diff --git a/dss-apps/dss-datawarehouse-design/data-warehouse-server/src/main/java/com/webank/wedatasphere/warehouse/configuration/WebMvcConfig.java b/dss-apps/dss-datawarehouse-design/data-warehouse-server/src/main/java/com/webank/wedatasphere/warehouse/configuration/WebMvcConfig.java new file mode 100644 index 0000000000..e53eb72400 --- /dev/null +++ b/dss-apps/dss-datawarehouse-design/data-warehouse-server/src/main/java/com/webank/wedatasphere/warehouse/configuration/WebMvcConfig.java @@ -0,0 +1,31 @@ +package com.webank.wedatasphere.warehouse.configuration; + +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.support.ResourceBundleMessageSource; +import org.springframework.web.servlet.LocaleResolver; +import org.springframework.web.servlet.config.annotation.EnableWebMvc; +import org.springframework.web.servlet.config.annotation.WebMvcConfigurer; +import org.springframework.web.servlet.i18n.AcceptHeaderLocaleResolver; + +import java.util.Locale; + +@Configuration +@EnableWebMvc +public class WebMvcConfig implements WebMvcConfigurer { + @Bean(name = "messageSource") + public ResourceBundleMessageSource resourceBundleMessageSource() { + ResourceBundleMessageSource resourceBundleMessageSource = new ResourceBundleMessageSource(); + resourceBundleMessageSource.setBasename("i18n/messages"); + resourceBundleMessageSource.setDefaultEncoding("UTF-8"); + resourceBundleMessageSource.setDefaultLocale(Locale.SIMPLIFIED_CHINESE); + return resourceBundleMessageSource; + } + + @Bean + public LocaleResolver localeResolver() { + AcceptHeaderLocaleResolver localeResolver = new AcceptHeaderLocaleResolver(); + localeResolver.setDefaultLocale(Locale.SIMPLIFIED_CHINESE); + return localeResolver; + } +} diff --git a/dss-apps/dss-datawarehouse-design/data-warehouse-server/src/main/java/com/webank/wedatasphere/warehouse/restful/DwDsRestfulApi.java b/dss-apps/dss-datawarehouse-design/data-warehouse-server/src/main/java/com/webank/wedatasphere/warehouse/restful/DwDsRestfulApi.java new file mode 100644 index 0000000000..1804ffa1be --- /dev/null +++ b/dss-apps/dss-datawarehouse-design/data-warehouse-server/src/main/java/com/webank/wedatasphere/warehouse/restful/DwDsRestfulApi.java @@ -0,0 +1,49 @@ +package com.webank.wedatasphere.warehouse.restful; + +import org.apache.linkis.server.Message; +import com.webank.wedatasphere.warehouse.service.DwDsService; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import javax.servlet.http.HttpServletRequest; +import javax.ws.rs.*; +import javax.ws.rs.core.Context; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; + +@Component +@Produces(MediaType.APPLICATION_JSON) +@Consumes(MediaType.APPLICATION_JSON) +@Path("/data-warehouse") +public class DwDsRestfulApi { + + private final DwDsService dwDsService; + + @Autowired + public DwDsRestfulApi(DwDsService dwDsService) { + this.dwDsService = dwDsService; + } + + // list all hive dbs + @GET + @Path("/dbs/hive") + public Response getAllHiveDbs(@Context HttpServletRequest request) throws Exception { + Message message = this.dwDsService.getAllHiveDbs(request); + return Message.messageToResponse(message); + } + + @GET + @Path("/workspace/{id}/principal_users") + public Response getAllAvailableUsers(@Context HttpServletRequest request, @PathParam(value = "id") String id) throws Exception { + Message message = this.dwDsService.getPrincipalUsers(request, id); + return Message.messageToResponse(message); + } + + @GET + @Path("/workspace/{id}/principal_roles") + public Response getAllAvailableRoles(@Context HttpServletRequest request, @PathParam(value = "id") String id) throws Exception { + Message message = this.dwDsService.getPrincipalRoles(request, id); + return Message.messageToResponse(message); + } + +} diff --git a/dss-apps/dss-datawarehouse-design/data-warehouse-server/src/main/java/com/webank/wedatasphere/warehouse/restful/DwLayerRestfulApi.java b/dss-apps/dss-datawarehouse-design/data-warehouse-server/src/main/java/com/webank/wedatasphere/warehouse/restful/DwLayerRestfulApi.java new file mode 100644 index 0000000000..ceba2655ee --- /dev/null +++ b/dss-apps/dss-datawarehouse-design/data-warehouse-server/src/main/java/com/webank/wedatasphere/warehouse/restful/DwLayerRestfulApi.java @@ -0,0 +1,129 @@ +package com.webank.wedatasphere.warehouse.restful; + +import org.apache.linkis.server.Message; +import com.webank.wedatasphere.warehouse.cqe.DwLayerCreateCommand; +import com.webank.wedatasphere.warehouse.cqe.DwLayerQueryCommand; +import com.webank.wedatasphere.warehouse.cqe.DwLayerUpdateCommand; +import com.webank.wedatasphere.warehouse.exception.DwException; +import com.webank.wedatasphere.warehouse.service.DwLayerService; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import javax.servlet.http.HttpServletRequest; +import javax.ws.rs.*; +import javax.ws.rs.core.Context; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; + +@Component +@Produces(MediaType.APPLICATION_JSON) +@Consumes(MediaType.APPLICATION_JSON) +@Path("/data-warehouse") +public class DwLayerRestfulApi { + + private final DwLayerService dwLayerService; + + @Autowired + public DwLayerRestfulApi(DwLayerService dwLayerService) { + this.dwLayerService = dwLayerService; + } + + // list all preset layers + @GET + @Path("/layers/preset") + public Response getAllPresetLayers(@Context HttpServletRequest request) throws DwException { + Message message = this.dwLayerService.getAllPresetLayers(request); + return Message.messageToResponse(message); + } + + @GET + @Path("/layers/all") + public Response getAllLayers(@Context HttpServletRequest request, @QueryParam(value = "isAvailable") Boolean isAvailable, @QueryParam(value = "db") String db) throws DwException { + Message message = this.dwLayerService.getAllLayers(request, isAvailable, db); + return Message.messageToResponse(message); + } + + // query paged custom layers + @GET + @Path("/layers/custom") + public Response queryPagedCustomLayers( + @Context HttpServletRequest request, + @QueryParam("page") Integer page, + @QueryParam("size") Integer size, + @QueryParam("name") String name, + @QueryParam("enabled") Boolean enabled + )throws DwException { + final DwLayerQueryCommand command = new DwLayerQueryCommand(); + command.setName(name); + command.setEnabled(enabled); + command.setPage(page); + command.setSize(size); + Message message = this.dwLayerService.queryPagedCustomLayers(request, command); + return Message.messageToResponse(message); + } + + // create custom layer + @POST + @Path("/layers/custom") + public Response createDwCustomLayer(@Context HttpServletRequest request, DwLayerCreateCommand command) throws DwException { + Message message = this.dwLayerService.createDwCustomLayer(request, command); + return Message.messageToResponse(message); + } + + // get layer by id + @GET + @Path("/layers/{id}") + public Response getLayerById( + @Context HttpServletRequest request, + @PathParam("id") Long id + ) throws DwException { + Message message = this.dwLayerService.getLayerById(request, id); + return Message.messageToResponse(message); + } + + // delete layer + @DELETE + @Path("/layers/{id}") + public Response deleteById( + @Context HttpServletRequest request, + @PathParam("id") Long id + ) throws DwException { + Message message = this.dwLayerService.deleteById(request, id); + return Message.messageToResponse(message); + } + + // update layer + @PUT + @Path("/layers/{id}") + public Response update( + @Context HttpServletRequest request, + @PathParam("id") Long id, + DwLayerUpdateCommand command + ) throws DwException { + command.setId(id); + Message message = this.dwLayerService.update(request, command); + return Message.messageToResponse(message); + } + + // enable layer + @PUT + @Path("/layers/{id}/enable") + public Response enable( + @Context HttpServletRequest request, + @PathParam("id") Long id + ) throws DwException { + Message message = this.dwLayerService.enable(request, id); + return Message.messageToResponse(message); + } + + // disable layer + @PUT + @Path("/layers/{id}/disable") + public Response disable( + @Context HttpServletRequest request, + @PathParam("id") Long id + ) throws DwException { + Message message = this.dwLayerService.disable(request, id); + return Message.messageToResponse(message); + } +} diff --git a/dss-apps/dss-datawarehouse-design/data-warehouse-server/src/main/java/com/webank/wedatasphere/warehouse/restful/DwModifierRestfulApi.java b/dss-apps/dss-datawarehouse-design/data-warehouse-server/src/main/java/com/webank/wedatasphere/warehouse/restful/DwModifierRestfulApi.java new file mode 100644 index 0000000000..31ac57af1e --- /dev/null +++ b/dss-apps/dss-datawarehouse-design/data-warehouse-server/src/main/java/com/webank/wedatasphere/warehouse/restful/DwModifierRestfulApi.java @@ -0,0 +1,134 @@ +package com.webank.wedatasphere.warehouse.restful; + +import com.google.common.base.Strings; +import org.apache.linkis.server.Message; +import com.webank.wedatasphere.warehouse.cqe.*; +import com.webank.wedatasphere.warehouse.exception.DwException; +import com.webank.wedatasphere.warehouse.service.DwModifierService; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import javax.servlet.http.HttpServletRequest; +import javax.ws.rs.*; +import javax.ws.rs.core.Context; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; +import java.util.Optional; + +@Component +@Produces(MediaType.APPLICATION_JSON) +@Consumes(MediaType.APPLICATION_JSON) +@Path("/data-warehouse") +public class DwModifierRestfulApi { + + private final DwModifierService dwModifierService; + + @Autowired + public DwModifierRestfulApi(DwModifierService dwModifierService) { + this.dwModifierService = dwModifierService; + } + + @GET + @Path("/modifiers/all") + public Response queryAllModifiers( + @Context HttpServletRequest request, + @QueryParam("typeName") String typeName, + @QueryParam(value = "isAvailable") Boolean isAvailable, + @QueryParam(value = "theme") String theme, + @QueryParam(value = "layer") String layer + )throws DwException { + final DwModifierQueryCommand command = new DwModifierQueryCommand(); + command.setName(typeName); + command.setEnabled(isAvailable); + command.setTheme(theme); + command.setLayer(layer); + Message message = this.dwModifierService.queryAllModifiers(request, command); + return Message.messageToResponse(message); + } + + // query paged modifiers + @GET + @Path("/modifiers") + public Response queryPagedDecorations( + @Context HttpServletRequest request, + @QueryParam("page") Integer page, + @QueryParam("size") Integer size, + @QueryParam("name") String typeName, + @QueryParam("enabled") String enabled + )throws DwException { + final DwModifierQueryCommand command = new DwModifierQueryCommand(); + command.setName(typeName); + command.setPage(page); + command.setSize(size); + if (!Strings.isNullOrEmpty(enabled)) { + command.setEnabled(Boolean.parseBoolean(enabled)); + } + Message message = this.dwModifierService.queryPage(request, command); + return Message.messageToResponse(message); + } + + // create dw decoration word + @POST + @Path("/modifiers") + public Response create(@Context HttpServletRequest request, DwModifierCreateCommand command) throws DwException { + Message message = this.dwModifierService.create(request, command); + return Message.messageToResponse(message); + } + + // fetch one decoration details by id + @GET + @Path("/modifiers/{id}") + public Response getById( + @Context HttpServletRequest request, + @PathParam("id") Long id + ) throws DwException { + Message message = this.dwModifierService.getById(request, id); + return Message.messageToResponse(message); + } + + // remove decoration logic + @DELETE + @Path("/modifiers/{id}") + public Response deleteById( + @Context HttpServletRequest request, + @PathParam("id") Long id + ) throws DwException { + Message message = this.dwModifierService.deleteById(request, id); + return Message.messageToResponse(message); + } + + // update + @PUT + @Path("/modifiers/{id}") + public Response update( + @Context HttpServletRequest request, + @PathParam("id") Long id, + DwModifierUpdateCommand command + ) throws DwException { + command.setId(id); + Message message = this.dwModifierService.update(request, command); + return Message.messageToResponse(message); + } + + // enable modifier + @PUT + @Path("/modifiers/{id}/enable") + public Response enable( + @Context HttpServletRequest request, + @PathParam("id") Long id + ) throws DwException { + Message message = this.dwModifierService.enable(request, id); + return Message.messageToResponse(message); + } + + // disable modifier + @PUT + @Path("/modifiers/{id}/disable") + public Response disable( + @Context HttpServletRequest request, + @PathParam("id") Long id + ) throws DwException { + Message message = this.dwModifierService.disable(request, id); + return Message.messageToResponse(message); + } +} diff --git a/dss-apps/dss-datawarehouse-design/data-warehouse-server/src/main/java/com/webank/wedatasphere/warehouse/restful/DwStatisticalPeriodRestfulApi.java b/dss-apps/dss-datawarehouse-design/data-warehouse-server/src/main/java/com/webank/wedatasphere/warehouse/restful/DwStatisticalPeriodRestfulApi.java new file mode 100644 index 0000000000..9948a9cabc --- /dev/null +++ b/dss-apps/dss-datawarehouse-design/data-warehouse-server/src/main/java/com/webank/wedatasphere/warehouse/restful/DwStatisticalPeriodRestfulApi.java @@ -0,0 +1,135 @@ +package com.webank.wedatasphere.warehouse.restful; + +import com.google.common.base.Strings; +import org.apache.linkis.server.Message; +import com.webank.wedatasphere.warehouse.cqe.*; +import com.webank.wedatasphere.warehouse.exception.DwException; +import com.webank.wedatasphere.warehouse.service.DwStatisticalPeriodService; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import javax.servlet.http.HttpServletRequest; +import javax.ws.rs.*; +import javax.ws.rs.core.Context; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; +import java.util.Optional; + +@Component +@Produces(MediaType.APPLICATION_JSON) +@Consumes(MediaType.APPLICATION_JSON) +@Path("/data-warehouse") +public class DwStatisticalPeriodRestfulApi { + + private final DwStatisticalPeriodService dwStatisticalPeriodService; + + @Autowired + public DwStatisticalPeriodRestfulApi(final DwStatisticalPeriodService dwStatisticalPeriodService) { + this.dwStatisticalPeriodService = dwStatisticalPeriodService; + } + + @GET + @Path("/statistical_periods/all") + public Response queryAll( + @Context HttpServletRequest request, + @QueryParam("name") String name, + @QueryParam(value = "isAvailable") Boolean isAvailable, + @QueryParam(value = "theme") String theme, + @QueryParam(value = "layer") String layer + )throws DwException { + final DwStatisticalPeriodQueryCommand command = new DwStatisticalPeriodQueryCommand(); + command.setName(name); + command.setEnabled(isAvailable); + command.setTheme(theme); + command.setLayer(layer); + Message message = this.dwStatisticalPeriodService.queryAll(request, command); + return Message.messageToResponse(message); + } + + // query paged statistical periods + @GET + @Path("/statistical_periods") + public Response queryPagedDecorations( + @Context HttpServletRequest request, + @QueryParam("page") Integer page, + @QueryParam("size") Integer size, + @QueryParam("name") String name, + @QueryParam(value = "enabled") String enabled + )throws DwException { + final DwStatisticalPeriodQueryCommand command = new DwStatisticalPeriodQueryCommand(); + command.setName(name); + command.setPage(page); + command.setSize(size); + if (!Strings.isNullOrEmpty(enabled)) { + command.setEnabled(Boolean.parseBoolean(enabled)); + } + Message message = this.dwStatisticalPeriodService.queryPage(request, command); + return Message.messageToResponse(message); + } + + // create dw statistical_periods + @POST + @Path("/statistical_periods") + public Response create(@Context HttpServletRequest request, DwStatisticalPeriodCreateCommand command) throws DwException { + Message message = this.dwStatisticalPeriodService.create(request, command); + return Message.messageToResponse(message); + } + + // fetch one statistical_periods details by id + @GET + @Path("/statistical_periods/{id}") + public Response getById( + @Context HttpServletRequest request, + @PathParam("id") Long id + ) throws DwException { + Message message = this.dwStatisticalPeriodService.getById(request, id); + return Message.messageToResponse(message); + } + + // remove statistical_periods logic + @DELETE + @Path("/statistical_periods/{id}") + public Response deleteById( + @Context HttpServletRequest request, + @PathParam("id") Long id + ) throws DwException { + Message message = this.dwStatisticalPeriodService.deleteById(request, id); + return Message.messageToResponse(message); + } + + // update statistical_periods + @PUT + @Path("/statistical_periods/{id}") + public Response update( + @Context HttpServletRequest request, + @PathParam("id") Long id, + DwStatisticalPeriodUpdateCommand command + ) throws DwException { + command.setId(id); + Message message = this.dwStatisticalPeriodService.update(request, command); + return Message.messageToResponse(message); + } + + + // enable statistical_periods + @PUT + @Path("/statistical_periods/{id}/enable") + public Response enable( + @Context HttpServletRequest request, + @PathParam("id") Long id + ) throws DwException { + Message message = this.dwStatisticalPeriodService.enable(request, id); + return Message.messageToResponse(message); + } + + // disable statistical_periods + @PUT + @Path("/statistical_periods/{id}/disable") + public Response disable( + @Context HttpServletRequest request, + @PathParam("id") Long id + ) throws DwException { + Message message = this.dwStatisticalPeriodService.disable(request, id); + return Message.messageToResponse(message); + } +} diff --git a/dss-apps/dss-datawarehouse-design/data-warehouse-server/src/main/java/com/webank/wedatasphere/warehouse/restful/DwThemeDomainRestfulApi.java b/dss-apps/dss-datawarehouse-design/data-warehouse-server/src/main/java/com/webank/wedatasphere/warehouse/restful/DwThemeDomainRestfulApi.java new file mode 100644 index 0000000000..71bc18573e --- /dev/null +++ b/dss-apps/dss-datawarehouse-design/data-warehouse-server/src/main/java/com/webank/wedatasphere/warehouse/restful/DwThemeDomainRestfulApi.java @@ -0,0 +1,127 @@ +package com.webank.wedatasphere.warehouse.restful; + +import org.apache.linkis.server.Message; +import com.webank.wedatasphere.warehouse.cqe.*; +import com.webank.wedatasphere.warehouse.exception.DwException; +import com.webank.wedatasphere.warehouse.service.DwThemeDomainService; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import javax.servlet.http.HttpServletRequest; +import javax.ws.rs.*; +import javax.ws.rs.core.Context; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; +import java.util.Optional; + +@Component +@Produces(MediaType.APPLICATION_JSON) +@Consumes(MediaType.APPLICATION_JSON) +@Path("/data-warehouse") +public class DwThemeDomainRestfulApi { + + private final DwThemeDomainService dwThemeDomainService; + + @Autowired + public DwThemeDomainRestfulApi(DwThemeDomainService dwThemeDomainService) { + this.dwThemeDomainService = dwThemeDomainService; + } + + // query paged theme domains + @GET + @Path("/themedomains") + public Response queryPagedCustomLayers( + @Context HttpServletRequest request, + @QueryParam("page") Integer page, + @QueryParam("size") Integer size, + @QueryParam("name") String name, + @QueryParam("enabled") Boolean enabled + )throws DwException { + final DwThemeDomainQueryCommand command = new DwThemeDomainQueryCommand(); + command.setName(name); + command.setPage(page); + command.setSize(size); + command.setEnabled(enabled); + Message message = this.dwThemeDomainService.queryPage(request, command); + return Message.messageToResponse(message); + } + + @GET + @Path("/themedomains/all") + public Response queryAllThemeDomains( + @Context HttpServletRequest request, + @QueryParam("name") String name, + @QueryParam(value = "isAvailable") Boolean isAvailable + )throws DwException { + final DwThemeDomainQueryCommand command = new DwThemeDomainQueryCommand(); + command.setName(name); + command.setEnabled(isAvailable); + Message message = this.dwThemeDomainService.queryAllThemeDomains(request, command); + return Message.messageToResponse(message); + } + + // create theme domain + @POST + @Path("/themedomains") + public Response createDwCustomLayer(@Context HttpServletRequest request, DwThemeDomainCreateCommand command) throws DwException { + Message message = this.dwThemeDomainService.create(request, command); + return Message.messageToResponse(message); + } + + // get theme domain by id + @GET + @Path("/themedomains/{id}") + public Response getLayerById( + @Context HttpServletRequest request, + @PathParam("id") Long id + ) throws DwException { + Message message = this.dwThemeDomainService.getById(request, id); + return Message.messageToResponse(message); + } + + // delete theme domain + @DELETE + @Path("/themedomains/{id}") + public Response deleteById( + @Context HttpServletRequest request, + @PathParam("id") Long id + ) throws DwException { + Message message = this.dwThemeDomainService.deleteById(request, id); + return Message.messageToResponse(message); + } + + // update theme domain + @PUT + @Path("/themedomains/{id}") + public Response update( + @Context HttpServletRequest request, + @PathParam("id") Long id, + DwThemeDomainUpdateCommand command + ) throws DwException { + command.setId(id); + Message message = this.dwThemeDomainService.update(request, command); + return Message.messageToResponse(message); + } + + // enable theme domain + @PUT + @Path("/themedomains/{id}/enable") + public Response enable( + @Context HttpServletRequest request, + @PathParam("id") Long id + ) throws DwException { + Message message = this.dwThemeDomainService.enable(request, id); + return Message.messageToResponse(message); + } + + // disable theme domain + @PUT + @Path("/themedomains/{id}/disable") + public Response disable( + @Context HttpServletRequest request, + @PathParam("id") Long id + ) throws DwException { + Message message = this.dwThemeDomainService.disable(request, id); + return Message.messageToResponse(message); + } +} diff --git a/dss-apps/dss-datawarehouse-design/data-warehouse-server/src/main/resources/application.yml b/dss-apps/dss-datawarehouse-design/data-warehouse-server/src/main/resources/application.yml new file mode 100644 index 0000000000..191013fa28 --- /dev/null +++ b/dss-apps/dss-datawarehouse-design/data-warehouse-server/src/main/resources/application.yml @@ -0,0 +1,20 @@ +server: + port: 20080 +spring: + application: + name: data-warehouse-server +eureka: + client: + serviceUrl: + defaultZone: http://localhost:20303/eureka/ + instance: + metadata-map: + test: wedatasphere + +management: + endpoints: + web: + exposure: + include: refresh,info +logging: + config: classpath:log4j2.xml \ No newline at end of file diff --git a/dss-apps/dss-datawarehouse-design/data-warehouse-server/src/main/resources/linkis.properties b/dss-apps/dss-datawarehouse-design/data-warehouse-server/src/main/resources/linkis.properties new file mode 100644 index 0000000000..0c5ead7d62 --- /dev/null +++ b/dss-apps/dss-datawarehouse-design/data-warehouse-server/src/main/resources/linkis.properties @@ -0,0 +1,46 @@ +# +# Copyright 2019 WeBank +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# + +wds.linkis.test.mode=true + +wds.linkis.server.mybatis.datasource.url=jdbc:mysql://localhost:3306/governance?useSSL=false&characterEncoding=UTF-8 + +wds.linkis.server.mybatis.datasource.username=root + +wds.linkis.server.mybatis.datasource.password=123456 + +wds.linkis.log.clear=true + +wds.linkis.server.version=v1 + + +## datasource client +wds.datawarehouse.datasource.client.serverurl=http://dss.shineweng.com:8088 +wds.datawarehouse.datasource.client.authtoken.key=hdfs +wds.datawarehouse.datasource.client.authtoken.value=hdfs +wds.datawarehouse.datasource.client.dws.version=v1 + +##restful +wds.linkis.server.restful.scan.packages=com.webank.wedatasphere.warehouse.restful + +wds.linkis.server.mybatis.mapperLocations=classpath*:com/webank/wedatasphere/warehouse/dao/mapper/impl/*.xml + +wds.linkis.server.mybatis.BasePackage=com.webank.wedatasphere.warehouse.dao.mapper + +wds.entity.workspace.name.auto.transform=false + + diff --git a/dss-apps/dss-datawarehouse-design/data-warehouse-server/src/main/resources/log4j.properties b/dss-apps/dss-datawarehouse-design/data-warehouse-server/src/main/resources/log4j.properties new file mode 100644 index 0000000000..0807e60877 --- /dev/null +++ b/dss-apps/dss-datawarehouse-design/data-warehouse-server/src/main/resources/log4j.properties @@ -0,0 +1,37 @@ +# +# Copyright 2019 WeBank +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# + +### set log levels ### + +log4j.rootCategory=INFO,console + +log4j.appender.console=org.apache.log4j.ConsoleAppender +log4j.appender.console.Threshold=INFO +log4j.appender.console.layout=org.apache.log4j.PatternLayout +#log4j.appender.console.layout.ConversionPattern= %d{ISO8601} %-5p (%t) [%F:%M(%L)] - %m%n +log4j.appender.console.layout.ConversionPattern= %d{ISO8601} %-5p (%t) %p %c{1} - %m%n + + +log4j.appender.com.webank.bdp.ide.core=org.apache.log4j.DailyRollingFileAppender +log4j.appender.com.webank.bdp.ide.core.Threshold=INFO +log4j.additivity.com.webank.bdp.ide.core=false +log4j.appender.com.webank.bdp.ide.core.layout=org.apache.log4j.PatternLayout +log4j.appender.com.webank.bdp.ide.core.Append=true +log4j.appender.com.webank.bdp.ide.core.File=logs/linkis.log +log4j.appender.com.webank.bdp.ide.core.layout.ConversionPattern= %d{ISO8601} %-5p (%t) [%F:%M(%L)] - %m%n + +log4j.logger.org.springframework=INFO diff --git a/dss-apps/dss-datawarehouse-design/data-warehouse-server/src/main/resources/log4j2.xml b/dss-apps/dss-datawarehouse-design/data-warehouse-server/src/main/resources/log4j2.xml new file mode 100644 index 0000000000..fe03a3b7cf --- /dev/null +++ b/dss-apps/dss-datawarehouse-design/data-warehouse-server/src/main/resources/log4j2.xml @@ -0,0 +1,39 @@ + + + + + + + + + + + + + + + + + + + + + + + diff --git a/dss-apps/dss-datawarehouse-design/data-warehouse-server/src/main/resources/mybatis-config.xml b/dss-apps/dss-datawarehouse-design/data-warehouse-server/src/main/resources/mybatis-config.xml new file mode 100644 index 0000000000..0389363102 --- /dev/null +++ b/dss-apps/dss-datawarehouse-design/data-warehouse-server/src/main/resources/mybatis-config.xml @@ -0,0 +1,7 @@ + + + + + \ No newline at end of file diff --git a/dss-apps/dss-datawarehouse-design/data-warehouse-server/src/main/scala/com/webank/wedatasphere/warehouse/DataWarehouseApplication.scala b/dss-apps/dss-datawarehouse-design/data-warehouse-server/src/main/scala/com/webank/wedatasphere/warehouse/DataWarehouseApplication.scala new file mode 100644 index 0000000000..ab85b9ddea --- /dev/null +++ b/dss-apps/dss-datawarehouse-design/data-warehouse-server/src/main/scala/com/webank/wedatasphere/warehouse/DataWarehouseApplication.scala @@ -0,0 +1,11 @@ +package com.webank.wedatasphere.warehouse + +import org.apache.linkis.DataWorkCloudApplication + +object DataWarehouseApplication { + + def main(args: Array[String]): Unit = { + DataWorkCloudApplication.main(args) + } + +} diff --git a/dss-apps/dss-datawarehouse-design/data-warehouse-service/pom.xml b/dss-apps/dss-datawarehouse-design/data-warehouse-service/pom.xml new file mode 100644 index 0000000000..895604f8db --- /dev/null +++ b/dss-apps/dss-datawarehouse-design/data-warehouse-service/pom.xml @@ -0,0 +1,143 @@ + + + 4.0.0 + + dss-datawarehouse-design + com.webank.wedatasphere.dss + 1.0.1 + + + data-warehouse-service + jar + + + 8 + 8 + + + + + com.webank.wedatasphere.dss + governance-dao + + + org.apache.linkis + linkis-mybatis + + + + + + org.apache.linkis + linkis-module + + + + org.apache.linkis + linkis-datasource-client + ${linkis.version} + + + org.apache.linkis + linkis-metadata + + + org.apache.linkis + linkis-publicservice + + + org.apache.linkis + linkis-bml + + + + + + com.webank.wedatasphere.dss + dss-datamodel-center-client + ${dss.version} + + + org.modelmapper + modelmapper + + + org.apache.linkis + linkis-gateway-httpclient-support + + + org.projectlombok + lombok + + + + + + com.webank.wedatasphere.dss + data-assets-client + ${dss.version} + + + org.apache.linkis + linkis-mybatis + + + org.modelmapper + modelmapper + + + org.apache.linkis + linkis-module + + + org.apache.linkis + linkis-gateway-httpclient-support + + + + + + com.webank.wedatasphere.dss + dss-framework-workspace-client + ${dss.version} + + + org.apache.linkis + linkis-gateway-httpclient-support + + + org.projectlombok + lombok + + + + + + org.projectlombok + lombok + provided + + + + + + + org.apache.maven.plugins + maven-deploy-plugin + + + net.alchim31.maven + scala-maven-plugin + + + org.apache.maven.plugins + maven-jar-plugin + + + + ${project.artifactId}-${project.version} + + + \ No newline at end of file diff --git a/dss-apps/dss-datawarehouse-design/data-warehouse-service/src/main/java/com/webank/wedatasphere/warehouse/client/DataSourceDTO.java b/dss-apps/dss-datawarehouse-design/data-warehouse-service/src/main/java/com/webank/wedatasphere/warehouse/client/DataSourceDTO.java new file mode 100644 index 0000000000..305803ddaa --- /dev/null +++ b/dss-apps/dss-datawarehouse-design/data-warehouse-service/src/main/java/com/webank/wedatasphere/warehouse/client/DataSourceDTO.java @@ -0,0 +1,115 @@ +package com.webank.wedatasphere.warehouse.client; + +import java.util.Date; + +public class DataSourceDTO { + + private Long id; + private String name; + private String type; + private Long dataSourceTypeId; + private String createIdentify; + private String desc; + private String createUser; + private String labels; + private Long versionId; + private String modifyUser; + private Date modifyTime; + private boolean expire; + + public boolean isExpire() { + return expire; + } + + public void setExpire(boolean expire) { + this.expire = expire; + } + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } + + public String getCreateIdentify() { + return createIdentify; + } + + public void setCreateIdentify(String createIdentify) { + this.createIdentify = createIdentify; + } + + public Long getDataSourceTypeId() { + return dataSourceTypeId; + } + + public void setDataSourceTypeId(Long dataSourceTypeId) { + this.dataSourceTypeId = dataSourceTypeId; + } + + public String getDesc() { + return desc; + } + + public void setDesc(String desc) { + this.desc = desc; + } + + public String getCreateUser() { + return createUser; + } + + public void setCreateUser(String createUser) { + this.createUser = createUser; + } + + public String getLabels() { + return labels; + } + + public void setLabels(String labels) { + this.labels = labels; + } + + public Long getVersionId() { + return versionId; + } + + public void setVersionId(Long versionId) { + this.versionId = versionId; + } + + public String getModifyUser() { + return modifyUser; + } + + public void setModifyUser(String modifyUser) { + this.modifyUser = modifyUser; + } + + public Date getModifyTime() { + return modifyTime; + } + + public void setModifyTime(Date modifyTime) { + this.modifyTime = modifyTime; + } +} diff --git a/dss-apps/dss-datawarehouse-design/data-warehouse-service/src/main/java/com/webank/wedatasphere/warehouse/client/DwDataSourceConfiguration.java b/dss-apps/dss-datawarehouse-design/data-warehouse-service/src/main/java/com/webank/wedatasphere/warehouse/client/DwDataSourceConfiguration.java new file mode 100644 index 0000000000..eca2f3985c --- /dev/null +++ b/dss-apps/dss-datawarehouse-design/data-warehouse-service/src/main/java/com/webank/wedatasphere/warehouse/client/DwDataSourceConfiguration.java @@ -0,0 +1,17 @@ +package com.webank.wedatasphere.warehouse.client; + +import org.apache.linkis.common.conf.CommonVars; + +public class DwDataSourceConfiguration { + public static final CommonVars SERVER_URL = CommonVars.apply("wds.datawarehouse.datasource.client.serverurl", ""); + public static final CommonVars CONNECTION_TIMEOUT = CommonVars.apply("wds.datawarehouse.datasource.client.connection.timeout", 30000L); + public static final CommonVars DISCOVERY_ENABLED = CommonVars.apply("wds.datawarehouse.datasource.client.discovery.enabled", false); + public static final CommonVars DISCOVERY_FREQUENCY_PERIOD = CommonVars.apply("wds.datawarehouse.datasource.client.discoveryfrequency.period", 1L); + public static final CommonVars LOAD_BALANCER_ENABLED = CommonVars.apply("wds.datawarehouse.datasource.client.loadbalancer.enabled", true); + public static final CommonVars MAX_CONNECTION_SIZE = CommonVars.apply("wds.datawarehouse.datasource.client.maxconnection.size", 5); + public static final CommonVars RETRY_ENABLED = CommonVars.apply("wds.datawarehouse.datasource.client.retryenabled", false); + public static final CommonVars READ_TIMEOUT = CommonVars.apply("wds.datawarehouse.datasource.client.readtimeout", 30000L); + public static final CommonVars AUTHTOKEN_KEY = CommonVars.apply("wds.datawarehouse.datasource.client.authtoken.key", ""); + public static final CommonVars AUTHTOKEN_VALUE = CommonVars.apply("wds.datawarehouse.datasource.client.authtoken.value", ""); + public static final CommonVars DWS_VERSION = CommonVars.apply("wds.datawarehouse.datasource.client.dws.version", ""); +} diff --git a/dss-apps/dss-datawarehouse-design/data-warehouse-service/src/main/java/com/webank/wedatasphere/warehouse/cqe/DwLayerCreateCommand.java b/dss-apps/dss-datawarehouse-design/data-warehouse-service/src/main/java/com/webank/wedatasphere/warehouse/cqe/DwLayerCreateCommand.java new file mode 100644 index 0000000000..38e481f9a2 --- /dev/null +++ b/dss-apps/dss-datawarehouse-design/data-warehouse-service/src/main/java/com/webank/wedatasphere/warehouse/cqe/DwLayerCreateCommand.java @@ -0,0 +1,19 @@ +package com.webank.wedatasphere.warehouse.cqe; + +import lombok.Getter; +import lombok.Setter; +import lombok.ToString; + +@Setter +@Getter +@ToString +public class DwLayerCreateCommand { + private String name; + private String enName; + private String owner; + private String principalName; + private String databases; + private String description; + private Integer order; +// private String autoCollectStrategy; +} diff --git a/dss-apps/dss-datawarehouse-design/data-warehouse-service/src/main/java/com/webank/wedatasphere/warehouse/cqe/DwLayerQueryCommand.java b/dss-apps/dss-datawarehouse-design/data-warehouse-service/src/main/java/com/webank/wedatasphere/warehouse/cqe/DwLayerQueryCommand.java new file mode 100644 index 0000000000..4e5a18e950 --- /dev/null +++ b/dss-apps/dss-datawarehouse-design/data-warehouse-service/src/main/java/com/webank/wedatasphere/warehouse/cqe/DwLayerQueryCommand.java @@ -0,0 +1,14 @@ +package com.webank.wedatasphere.warehouse.cqe; + +import com.webank.wedatasphere.warehouse.cqe.common.PageCommand; +import lombok.*; + +@Setter +@Getter +@ToString +//@Builder +//@Accessors(chain = true) +public class DwLayerQueryCommand extends PageCommand { + private String name; + private Boolean enabled; +} diff --git a/dss-apps/dss-datawarehouse-design/data-warehouse-service/src/main/java/com/webank/wedatasphere/warehouse/cqe/DwLayerUpdateCommand.java b/dss-apps/dss-datawarehouse-design/data-warehouse-service/src/main/java/com/webank/wedatasphere/warehouse/cqe/DwLayerUpdateCommand.java new file mode 100644 index 0000000000..bda4fb73a1 --- /dev/null +++ b/dss-apps/dss-datawarehouse-design/data-warehouse-service/src/main/java/com/webank/wedatasphere/warehouse/cqe/DwLayerUpdateCommand.java @@ -0,0 +1,20 @@ +package com.webank.wedatasphere.warehouse.cqe; + +import lombok.Getter; +import lombok.Setter; +import lombok.ToString; + +@Setter +@Getter +@ToString +public class DwLayerUpdateCommand { + private Long id; + private String name; + private String enName; + private String databases; + private Integer order; +// private String autoCollectStrategy; + private String owner; + private String description; + private String principalName; +} diff --git a/dss-apps/dss-datawarehouse-design/data-warehouse-service/src/main/java/com/webank/wedatasphere/warehouse/cqe/DwModifierCreateCommand.java b/dss-apps/dss-datawarehouse-design/data-warehouse-service/src/main/java/com/webank/wedatasphere/warehouse/cqe/DwModifierCreateCommand.java new file mode 100644 index 0000000000..4631161649 --- /dev/null +++ b/dss-apps/dss-datawarehouse-design/data-warehouse-service/src/main/java/com/webank/wedatasphere/warehouse/cqe/DwModifierCreateCommand.java @@ -0,0 +1,37 @@ +package com.webank.wedatasphere.warehouse.cqe; + +import lombok.Getter; +import lombok.Setter; +import lombok.ToString; + +import java.util.List; + +@Setter +@Getter +@ToString +public class DwModifierCreateCommand { + private Long themeDomainId; + + private String themeArea; + + private Long layerId; + + private String layerArea; + + private String typeName; + + private String typeEnName; + + private String description; + + private List list; + + @Setter + @Getter + @ToString + public static class DwModifierCreateListItem { + private String name; + private String identifier; + private String formula; + } +} diff --git a/dss-apps/dss-datawarehouse-design/data-warehouse-service/src/main/java/com/webank/wedatasphere/warehouse/cqe/DwModifierQueryCommand.java b/dss-apps/dss-datawarehouse-design/data-warehouse-service/src/main/java/com/webank/wedatasphere/warehouse/cqe/DwModifierQueryCommand.java new file mode 100644 index 0000000000..7e15f3a649 --- /dev/null +++ b/dss-apps/dss-datawarehouse-design/data-warehouse-service/src/main/java/com/webank/wedatasphere/warehouse/cqe/DwModifierQueryCommand.java @@ -0,0 +1,16 @@ +package com.webank.wedatasphere.warehouse.cqe; + +import com.webank.wedatasphere.warehouse.cqe.common.PageCommand; +import lombok.Getter; +import lombok.Setter; +import lombok.ToString; + +@Setter +@Getter +@ToString +public class DwModifierQueryCommand extends PageCommand { + private String name; + private Boolean enabled; + private String theme; + private String layer; +} \ No newline at end of file diff --git a/dss-apps/dss-datawarehouse-design/data-warehouse-service/src/main/java/com/webank/wedatasphere/warehouse/cqe/DwModifierUpdateCommand.java b/dss-apps/dss-datawarehouse-design/data-warehouse-service/src/main/java/com/webank/wedatasphere/warehouse/cqe/DwModifierUpdateCommand.java new file mode 100644 index 0000000000..03b5fd4708 --- /dev/null +++ b/dss-apps/dss-datawarehouse-design/data-warehouse-service/src/main/java/com/webank/wedatasphere/warehouse/cqe/DwModifierUpdateCommand.java @@ -0,0 +1,39 @@ +package com.webank.wedatasphere.warehouse.cqe; + +import lombok.Getter; +import lombok.Setter; +import lombok.ToString; + +import java.util.List; + +@Setter +@Getter +@ToString +public class DwModifierUpdateCommand { + private Long id; + + private Long themeDomainId; + + private String themeArea; + + private Long layerId; + + private String layerArea; + + private String typeName; + + private String typeEnName; + + private String description; + + private List list; + + @Setter + @Getter + @ToString + public static class DwModifierUpdateListItem { + private String name; + private String identifier; + private String formula; + } +} diff --git a/dss-apps/dss-datawarehouse-design/data-warehouse-service/src/main/java/com/webank/wedatasphere/warehouse/cqe/DwStatisticalPeriodCreateCommand.java b/dss-apps/dss-datawarehouse-design/data-warehouse-service/src/main/java/com/webank/wedatasphere/warehouse/cqe/DwStatisticalPeriodCreateCommand.java new file mode 100644 index 0000000000..82ce869e8c --- /dev/null +++ b/dss-apps/dss-datawarehouse-design/data-warehouse-service/src/main/java/com/webank/wedatasphere/warehouse/cqe/DwStatisticalPeriodCreateCommand.java @@ -0,0 +1,20 @@ +package com.webank.wedatasphere.warehouse.cqe; + +import lombok.Getter; +import lombok.Setter; +import lombok.ToString; + +@Setter +@Getter +@ToString +public class DwStatisticalPeriodCreateCommand { + private Long themeDomainId; + private Long layerId; + private String name; + private String enName; + private String description; + private String statStartFormula; + private String statEndFormula; + private String principalName; + private String owner; +} diff --git a/dss-apps/dss-datawarehouse-design/data-warehouse-service/src/main/java/com/webank/wedatasphere/warehouse/cqe/DwStatisticalPeriodQueryCommand.java b/dss-apps/dss-datawarehouse-design/data-warehouse-service/src/main/java/com/webank/wedatasphere/warehouse/cqe/DwStatisticalPeriodQueryCommand.java new file mode 100644 index 0000000000..1a83859c1b --- /dev/null +++ b/dss-apps/dss-datawarehouse-design/data-warehouse-service/src/main/java/com/webank/wedatasphere/warehouse/cqe/DwStatisticalPeriodQueryCommand.java @@ -0,0 +1,16 @@ +package com.webank.wedatasphere.warehouse.cqe; + +import com.webank.wedatasphere.warehouse.cqe.common.PageCommand; +import lombok.Getter; +import lombok.Setter; +import lombok.ToString; + +@Setter +@Getter +@ToString +public class DwStatisticalPeriodQueryCommand extends PageCommand { + private String name; + private Boolean enabled; + private String theme; + private String layer; +} \ No newline at end of file diff --git a/dss-apps/dss-datawarehouse-design/data-warehouse-service/src/main/java/com/webank/wedatasphere/warehouse/cqe/DwStatisticalPeriodUpdateCommand.java b/dss-apps/dss-datawarehouse-design/data-warehouse-service/src/main/java/com/webank/wedatasphere/warehouse/cqe/DwStatisticalPeriodUpdateCommand.java new file mode 100644 index 0000000000..4a08404746 --- /dev/null +++ b/dss-apps/dss-datawarehouse-design/data-warehouse-service/src/main/java/com/webank/wedatasphere/warehouse/cqe/DwStatisticalPeriodUpdateCommand.java @@ -0,0 +1,21 @@ +package com.webank.wedatasphere.warehouse.cqe; + +import lombok.Getter; +import lombok.Setter; +import lombok.ToString; + +@Setter +@Getter +@ToString +public class DwStatisticalPeriodUpdateCommand { + private Long id; + private Long themeDomainId; + private Long layerId; + private String name; + private String enName; + private String description; + private String statStartFormula; + private String statEndFormula; + private String principalName; + private String owner; +} diff --git a/dss-apps/dss-datawarehouse-design/data-warehouse-service/src/main/java/com/webank/wedatasphere/warehouse/cqe/DwThemeDomainCreateCommand.java b/dss-apps/dss-datawarehouse-design/data-warehouse-service/src/main/java/com/webank/wedatasphere/warehouse/cqe/DwThemeDomainCreateCommand.java new file mode 100644 index 0000000000..244cadb553 --- /dev/null +++ b/dss-apps/dss-datawarehouse-design/data-warehouse-service/src/main/java/com/webank/wedatasphere/warehouse/cqe/DwThemeDomainCreateCommand.java @@ -0,0 +1,17 @@ +package com.webank.wedatasphere.warehouse.cqe; + +import lombok.Getter; +import lombok.Setter; +import lombok.ToString; + +@Setter +@Getter +@ToString +public class DwThemeDomainCreateCommand { + private String name; + private String enName; + private String owner; + private String principalName; + private String description; + private Integer sort; +} diff --git a/dss-apps/dss-datawarehouse-design/data-warehouse-service/src/main/java/com/webank/wedatasphere/warehouse/cqe/DwThemeDomainQueryCommand.java b/dss-apps/dss-datawarehouse-design/data-warehouse-service/src/main/java/com/webank/wedatasphere/warehouse/cqe/DwThemeDomainQueryCommand.java new file mode 100644 index 0000000000..810823f8c4 --- /dev/null +++ b/dss-apps/dss-datawarehouse-design/data-warehouse-service/src/main/java/com/webank/wedatasphere/warehouse/cqe/DwThemeDomainQueryCommand.java @@ -0,0 +1,14 @@ +package com.webank.wedatasphere.warehouse.cqe; + +import com.webank.wedatasphere.warehouse.cqe.common.PageCommand; +import lombok.Getter; +import lombok.Setter; +import lombok.ToString; + +@Setter +@Getter +@ToString +public class DwThemeDomainQueryCommand extends PageCommand { + private String name; + private Boolean enabled; +} diff --git a/dss-apps/dss-datawarehouse-design/data-warehouse-service/src/main/java/com/webank/wedatasphere/warehouse/cqe/DwThemeDomainUpdateCommand.java b/dss-apps/dss-datawarehouse-design/data-warehouse-service/src/main/java/com/webank/wedatasphere/warehouse/cqe/DwThemeDomainUpdateCommand.java new file mode 100644 index 0000000000..a065e4ad1f --- /dev/null +++ b/dss-apps/dss-datawarehouse-design/data-warehouse-service/src/main/java/com/webank/wedatasphere/warehouse/cqe/DwThemeDomainUpdateCommand.java @@ -0,0 +1,19 @@ +package com.webank.wedatasphere.warehouse.cqe; + +import lombok.Getter; +import lombok.Setter; +import lombok.ToString; + +@Setter +@Getter +@ToString +public class DwThemeDomainUpdateCommand { + private Long id; + private String name; + private String enName; + private String owner; + private String principalName; + private String description; + private Integer sort; +// private String authority; +} diff --git a/dss-apps/dss-datawarehouse-design/data-warehouse-service/src/main/java/com/webank/wedatasphere/warehouse/cqe/common/PageCommand.java b/dss-apps/dss-datawarehouse-design/data-warehouse-service/src/main/java/com/webank/wedatasphere/warehouse/cqe/common/PageCommand.java new file mode 100644 index 0000000000..47b1a5a3bf --- /dev/null +++ b/dss-apps/dss-datawarehouse-design/data-warehouse-service/src/main/java/com/webank/wedatasphere/warehouse/cqe/common/PageCommand.java @@ -0,0 +1,17 @@ +package com.webank.wedatasphere.warehouse.cqe.common; + + + +import lombok.*; + +@Setter +@Getter +@ToString +@AllArgsConstructor +@NoArgsConstructor +//@Builder +//@Accessors(chain = true) +public class PageCommand { + private Integer page; + private Integer size; +} diff --git a/dss-apps/dss-datawarehouse-design/data-warehouse-service/src/main/java/com/webank/wedatasphere/warehouse/dto/DwLayerDTO.java b/dss-apps/dss-datawarehouse-design/data-warehouse-service/src/main/java/com/webank/wedatasphere/warehouse/dto/DwLayerDTO.java new file mode 100644 index 0000000000..58ff72c5ad --- /dev/null +++ b/dss-apps/dss-datawarehouse-design/data-warehouse-service/src/main/java/com/webank/wedatasphere/warehouse/dto/DwLayerDTO.java @@ -0,0 +1,23 @@ +package com.webank.wedatasphere.warehouse.dto; + +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import lombok.ToString; + +@Setter +@Getter +@ToString +@NoArgsConstructor +public class DwLayerDTO { + private Long id; + private String name; + private String enName; + private Boolean preset; + private String description; + private String principalName; + private String dbs; + private String owner; + private Integer sort; + private boolean referenced; +} diff --git a/dss-apps/dss-datawarehouse-design/data-warehouse-service/src/main/java/com/webank/wedatasphere/warehouse/dto/DwLayerListItemDTO.java b/dss-apps/dss-datawarehouse-design/data-warehouse-service/src/main/java/com/webank/wedatasphere/warehouse/dto/DwLayerListItemDTO.java new file mode 100644 index 0000000000..49d7ba0625 --- /dev/null +++ b/dss-apps/dss-datawarehouse-design/data-warehouse-service/src/main/java/com/webank/wedatasphere/warehouse/dto/DwLayerListItemDTO.java @@ -0,0 +1,21 @@ +package com.webank.wedatasphere.warehouse.dto; + +import lombok.*; + +import java.util.Date; + +@Data +public class DwLayerListItemDTO { + private Long id; + private String name; + private String enName; + private Boolean preset; + private String description; + private String principalName; + private String dbs; + private Integer sort; + private Boolean isAvailable; + private Date createTime; + private Date updateTime; + private int referenceCount; +} diff --git a/dss-apps/dss-datawarehouse-design/data-warehouse-service/src/main/java/com/webank/wedatasphere/warehouse/dto/DwModifierDTO.java b/dss-apps/dss-datawarehouse-design/data-warehouse-service/src/main/java/com/webank/wedatasphere/warehouse/dto/DwModifierDTO.java new file mode 100644 index 0000000000..c537c6f51c --- /dev/null +++ b/dss-apps/dss-datawarehouse-design/data-warehouse-service/src/main/java/com/webank/wedatasphere/warehouse/dto/DwModifierDTO.java @@ -0,0 +1,25 @@ +package com.webank.wedatasphere.warehouse.dto; + +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import lombok.ToString; + +import java.util.List; + +@Setter +@Getter +@ToString +@NoArgsConstructor +public class DwModifierDTO { + private Long id; + private String modifierType; + private String modifierTypeEn; + private Long layerId; + private String layerArea; + private Long themeDomainId; + private String themeArea; + private String description; + private List list; + private boolean referenced; +} diff --git a/dss-apps/dss-datawarehouse-design/data-warehouse-service/src/main/java/com/webank/wedatasphere/warehouse/dto/DwModifierListDTO.java b/dss-apps/dss-datawarehouse-design/data-warehouse-service/src/main/java/com/webank/wedatasphere/warehouse/dto/DwModifierListDTO.java new file mode 100644 index 0000000000..a9b1f3fb45 --- /dev/null +++ b/dss-apps/dss-datawarehouse-design/data-warehouse-service/src/main/java/com/webank/wedatasphere/warehouse/dto/DwModifierListDTO.java @@ -0,0 +1,18 @@ +package com.webank.wedatasphere.warehouse.dto; + +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import lombok.ToString; + +@Setter +@Getter +@ToString +@NoArgsConstructor +public class DwModifierListDTO { + private Long id; + private Long modifierId; + private String name; + private String identifier; + private String formula; +} diff --git a/dss-apps/dss-datawarehouse-design/data-warehouse-service/src/main/java/com/webank/wedatasphere/warehouse/dto/DwModifierListItemDTO.java b/dss-apps/dss-datawarehouse-design/data-warehouse-service/src/main/java/com/webank/wedatasphere/warehouse/dto/DwModifierListItemDTO.java new file mode 100644 index 0000000000..805fa84a2f --- /dev/null +++ b/dss-apps/dss-datawarehouse-design/data-warehouse-service/src/main/java/com/webank/wedatasphere/warehouse/dto/DwModifierListItemDTO.java @@ -0,0 +1,22 @@ +package com.webank.wedatasphere.warehouse.dto; + +import lombok.*; + +import java.util.Date; +import java.util.List; + +@Data +public class DwModifierListItemDTO { + private Long id; + private String modifierType; + private String modifierTypeEn; + private String layerArea; + private String layerAreaEn; + private String themeArea; + private String themeAreaEn; + private String description; + private Boolean isAvailable; + private Date createTime; + private Date updateTime; + private int referenceCount; +} diff --git a/dss-apps/dss-datawarehouse-design/data-warehouse-service/src/main/java/com/webank/wedatasphere/warehouse/dto/DwStatisticalPeriodListItemDTO.java b/dss-apps/dss-datawarehouse-design/data-warehouse-service/src/main/java/com/webank/wedatasphere/warehouse/dto/DwStatisticalPeriodListItemDTO.java new file mode 100644 index 0000000000..766e15a6ff --- /dev/null +++ b/dss-apps/dss-datawarehouse-design/data-warehouse-service/src/main/java/com/webank/wedatasphere/warehouse/dto/DwStatisticalPeriodListItemDTO.java @@ -0,0 +1,22 @@ +package com.webank.wedatasphere.warehouse.dto; + +import lombok.Data; + +import java.util.Date; + +@Data +public class DwStatisticalPeriodListItemDTO { + private Long id; + private Long themeDomainId; + private Long layerId; + private String name; + private String enName; + private Boolean isAvailable; + private String description; + private String owner; + private String statStartFormula; + private String statEndFormula; + private String principalName; + private Date createTime; + private Date updateTime; +} diff --git a/dss-apps/dss-datawarehouse-design/data-warehouse-service/src/main/java/com/webank/wedatasphere/warehouse/dto/DwThemeDomainDTO.java b/dss-apps/dss-datawarehouse-design/data-warehouse-service/src/main/java/com/webank/wedatasphere/warehouse/dto/DwThemeDomainDTO.java new file mode 100644 index 0000000000..fbbf9e972c --- /dev/null +++ b/dss-apps/dss-datawarehouse-design/data-warehouse-service/src/main/java/com/webank/wedatasphere/warehouse/dto/DwThemeDomainDTO.java @@ -0,0 +1,21 @@ +package com.webank.wedatasphere.warehouse.dto; + +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import lombok.ToString; + +@Setter +@Getter +@ToString +@NoArgsConstructor +public class DwThemeDomainDTO { + private Long id; + private String name; + private String enName; + private String owner; + private String principalName; + private Integer sort; + private String description; + private boolean referenced; +} diff --git a/dss-apps/dss-datawarehouse-design/data-warehouse-service/src/main/java/com/webank/wedatasphere/warehouse/dto/DwThemeDomainListItemDTO.java b/dss-apps/dss-datawarehouse-design/data-warehouse-service/src/main/java/com/webank/wedatasphere/warehouse/dto/DwThemeDomainListItemDTO.java new file mode 100644 index 0000000000..c5318735e2 --- /dev/null +++ b/dss-apps/dss-datawarehouse-design/data-warehouse-service/src/main/java/com/webank/wedatasphere/warehouse/dto/DwThemeDomainListItemDTO.java @@ -0,0 +1,21 @@ +package com.webank.wedatasphere.warehouse.dto; + +import lombok.*; + +import java.util.Date; + +@Data +public class DwThemeDomainListItemDTO { + private Long id; + private String name; + private String enName; + private String owner; + private String principalName; + private Integer sort; + private String description; + private Date createTime; + private Date updateTime; + private Boolean preset; + private Boolean isAvailable; + private int referenceCount; +} diff --git a/dss-apps/dss-datawarehouse-design/data-warehouse-service/src/main/java/com/webank/wedatasphere/warehouse/dto/PageInfo.java b/dss-apps/dss-datawarehouse-design/data-warehouse-service/src/main/java/com/webank/wedatasphere/warehouse/dto/PageInfo.java new file mode 100644 index 0000000000..9e3fb9a5e3 --- /dev/null +++ b/dss-apps/dss-datawarehouse-design/data-warehouse-service/src/main/java/com/webank/wedatasphere/warehouse/dto/PageInfo.java @@ -0,0 +1,17 @@ +package com.webank.wedatasphere.warehouse.dto; + +import lombok.*; + +import java.util.List; + +@Setter +@Getter +@AllArgsConstructor +@NoArgsConstructor +@ToString +public class PageInfo { + private List items; + private Long current; + private Long pageSize; + private Long total; +} diff --git a/dss-apps/dss-datawarehouse-design/data-warehouse-service/src/main/java/com/webank/wedatasphere/warehouse/exception/DwException.java b/dss-apps/dss-datawarehouse-design/data-warehouse-service/src/main/java/com/webank/wedatasphere/warehouse/exception/DwException.java new file mode 100644 index 0000000000..39520dac83 --- /dev/null +++ b/dss-apps/dss-datawarehouse-design/data-warehouse-service/src/main/java/com/webank/wedatasphere/warehouse/exception/DwException.java @@ -0,0 +1,56 @@ +package com.webank.wedatasphere.warehouse.exception; + +import org.apache.linkis.common.exception.ErrorException; + +public class DwException extends ErrorException { + + public static final Integer ARGUMENT_ERROR = 21101; + public static final Integer BUSINESS_ERROR = 22101; + + private int errorCode; + private Object[] parameters; + + public int getErrorCode() { + return errorCode; + } + + public void setErrorCode(int errorCode) { + this.errorCode = errorCode; + } + + public Object[] getParameters() { + return parameters; + } + + public static DwException argumentReject(String errorMessage, Object... parameters) { + return new DwException(ARGUMENT_ERROR, errorMessage, parameters); + } + + public static DwException argumentReject(String errorMessage) { + return new DwException(ARGUMENT_ERROR, errorMessage); + } + + public static DwException stateReject(String errorMessage) { + return new DwException(BUSINESS_ERROR, errorMessage); + } + + public static DwException stateReject(String errorMessage, Object... parameters) { + return new DwException(BUSINESS_ERROR, errorMessage, parameters); + } + + public DwException(int errorCode, String message) { + super(errorCode, message); + this.errorCode = errorCode; + } + + public DwException(int errorCode, String message, Object... parameters) { + super(errorCode, message); + this.errorCode = errorCode; + this.parameters = parameters; + } + + public DwException(int errorCode, String message, Throwable cause) { + super(errorCode, message); + this.errorCode = errorCode; + } +} diff --git a/dss-apps/dss-datawarehouse-design/data-warehouse-service/src/main/java/com/webank/wedatasphere/warehouse/exception/DwExceptionCode.java b/dss-apps/dss-datawarehouse-design/data-warehouse-service/src/main/java/com/webank/wedatasphere/warehouse/exception/DwExceptionCode.java new file mode 100644 index 0000000000..569ba6e6ae --- /dev/null +++ b/dss-apps/dss-datawarehouse-design/data-warehouse-service/src/main/java/com/webank/wedatasphere/warehouse/exception/DwExceptionCode.java @@ -0,0 +1,27 @@ +package com.webank.wedatasphere.warehouse.exception; + +// 41000 ~ 41999 +public enum DwExceptionCode { + + CREATE_MODEL_TYPE_ERROR(41001), + DELETE_MODEL_TYPE_ERROR(41002), + UPDATE_MODEL_TYPE_ERROR(41003), + GET_AVAILABLE_DBS_ERROR(41004), + GET_PRINCIPAL_USERS_ERROR(41005), + GET_PRINCIPAL_ROLES_ERROR(41006), + ; + + private int code; + + DwExceptionCode(int code) { + this.code = code; + } + + public int getCode() { + return code; + } + + public void setCode(int code) { + this.code = code; + } +} diff --git a/dss-apps/dss-datawarehouse-design/data-warehouse-service/src/main/java/com/webank/wedatasphere/warehouse/service/DwDomainReferenceAdapter.java b/dss-apps/dss-datawarehouse-design/data-warehouse-service/src/main/java/com/webank/wedatasphere/warehouse/service/DwDomainReferenceAdapter.java new file mode 100644 index 0000000000..e9339051e3 --- /dev/null +++ b/dss-apps/dss-datawarehouse-design/data-warehouse-service/src/main/java/com/webank/wedatasphere/warehouse/service/DwDomainReferenceAdapter.java @@ -0,0 +1,237 @@ +package com.webank.wedatasphere.warehouse.service; + +import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; +import com.google.common.base.Strings; +import com.webank.wedatasphere.dss.datamodel.center.client.impl.LinkisDatamodelCenterRemoteClient; +import com.webank.wedatasphere.dss.datamodel.center.client.request.CyclesReferenceAction; +import com.webank.wedatasphere.dss.datamodel.center.client.request.LayersReferenceAction; +import com.webank.wedatasphere.dss.datamodel.center.client.request.ModifiersReferenceAction; +import com.webank.wedatasphere.dss.datamodel.center.client.request.ThemesReferenceAction; +import com.webank.wedatasphere.dss.datamodel.center.client.response.CyclesReferenceResult; +import com.webank.wedatasphere.dss.datamodel.center.client.response.LayersReferenceResult; +import com.webank.wedatasphere.dss.datamodel.center.client.response.ModifiersReferenceResult; +import com.webank.wedatasphere.dss.datamodel.center.client.response.ThemesReferenceResult; +import com.webank.wedatasphere.warehouse.LinkisRemoteClientHolder; +import com.webank.wedatasphere.warehouse.dao.domain.DwLayer; +import com.webank.wedatasphere.warehouse.dao.domain.DwModifier; +import com.webank.wedatasphere.warehouse.dao.domain.DwStatisticalPeriod; +import com.webank.wedatasphere.warehouse.dao.domain.DwThemeDomain; +import com.webank.wedatasphere.warehouse.dao.mapper.DwLayerMapper; +import com.webank.wedatasphere.warehouse.dao.mapper.DwModifierMapper; +import com.webank.wedatasphere.warehouse.dao.mapper.DwStatisticalPeriodMapper; +import com.webank.wedatasphere.warehouse.dao.mapper.DwThemeDomainMapper; +import com.webank.wedatasphere.warehouse.exception.DwException; +import com.webank.wedatasphere.warehouse.utils.PreconditionUtil; + +import java.util.Optional; + +public interface DwDomainReferenceAdapter { + + DwStatisticalPeriodMapper getDwStatisticalPeriodMapper(); + + DwModifierMapper getDwModifierMapper(); + + DwLayerMapper getDwLayerMapper(); + + DwThemeDomainMapper getDwThemeDomainMapper(); + + default void checkMapper() throws DwException { +// DwLayerMapper dwLayerMapper = getDwLayerMapper(); +// PreconditionUtil.checkState(Optional.ofNullable(dwLayerMapper).isPresent(), DwException.stateReject("dw layer mapper should not be null")); +// +// DwThemeDomainMapper dwThemeDomainMapper = getDwThemeDomainMapper(); +// PreconditionUtil.checkState(Optional.ofNullable(dwThemeDomainMapper).isPresent(), DwException.stateReject("dw theme domain mapper should not be null")); +// +// DwModifierMapper dwModifierMapper = getDwModifierMapper(); +// PreconditionUtil.checkState(Optional.ofNullable(dwModifierMapper).isPresent(), DwException.stateReject("dw modifier mapper should not be null")); +// +// DwStatisticalPeriodMapper dwStatisticalPeriodMapper = getDwStatisticalPeriodMapper(); +// PreconditionUtil.checkState(Optional.ofNullable(dwStatisticalPeriodMapper).isPresent(), DwException.stateReject("dw statistical period mapper should not be null")); + checkLayerMapper(); + + checkThemeDomainMapper(); + + checkModifierMapper(); + + checkStatisticalPeriodMapper(); + } + + default void checkModifierMapper() throws DwException { + DwModifierMapper dwModifierMapper = getDwModifierMapper(); + PreconditionUtil.checkState(Optional.ofNullable(dwModifierMapper).isPresent(), DwException.stateReject("dw modifier mapper should not be null")); + } + + default void checkLayerMapper() throws DwException { + DwLayerMapper dwLayerMapper = getDwLayerMapper(); + PreconditionUtil.checkState(Optional.ofNullable(dwLayerMapper).isPresent(), DwException.stateReject("dw layer mapper should not be null")); + } + + default void checkThemeDomainMapper() throws DwException { + DwThemeDomainMapper dwThemeDomainMapper = getDwThemeDomainMapper(); + PreconditionUtil.checkState(Optional.ofNullable(dwThemeDomainMapper).isPresent(), DwException.stateReject("dw theme domain mapper should not be null")); + } + + default void checkStatisticalPeriodMapper() throws DwException { + DwStatisticalPeriodMapper dwStatisticalPeriodMapper = getDwStatisticalPeriodMapper(); + PreconditionUtil.checkState(Optional.ofNullable(dwStatisticalPeriodMapper).isPresent(), DwException.stateReject("dw statistical period mapper should not be null")); + } + + // get theme reference count + default int getThemeDomainReferenceCount(Long id, String username) throws DwException { + checkThemeDomainMapper(); + + DwThemeDomain themeDomain = getDwThemeDomainMapper().selectById(id); + PreconditionUtil.checkState(Optional.ofNullable(themeDomain).isPresent(), DwException.stateReject("dw theme domain should not be null, id = {}", id)); + + LinkisDatamodelCenterRemoteClient dataModelRemoteClient = LinkisRemoteClientHolder.getDataModelRemoteClient(); + ThemesReferenceAction themesReferenceAction = new ThemesReferenceAction.Builder().setName(themeDomain.getEnName()).setUser(username).build(); + ThemesReferenceResult themesReferenceResult = dataModelRemoteClient.themesReference(themesReferenceAction); + return themesReferenceResult.getResult(); + } + + // get modifier reference count + default int getModifierReferenceCount(Long id, String username) throws DwException { + checkModifierMapper(); + + DwModifier modifier = getDwModifierMapper().selectById(id); + PreconditionUtil.checkState(Optional.ofNullable(modifier).isPresent(), DwException.stateReject("dw modifier should not be null, id = {}", id)); + + if (Strings.isNullOrEmpty(modifier.getModifierTypeEn())) { + return 0; + } + LinkisDatamodelCenterRemoteClient dataModelRemoteClient = LinkisRemoteClientHolder.getDataModelRemoteClient(); + ModifiersReferenceAction modifiersReferenceAction = new ModifiersReferenceAction.Builder().setName(modifier.getModifierTypeEn()).setUser(username).build(); + ModifiersReferenceResult modifiersReferenceResult = dataModelRemoteClient.modifiersReference(modifiersReferenceAction); + return modifiersReferenceResult.getResult(); + } + + default int getStatisticalPeriodReferenceCount(Long id, String username) throws DwException { + checkStatisticalPeriodMapper(); + + DwStatisticalPeriod period = getDwStatisticalPeriodMapper().selectById(id); + PreconditionUtil.checkState(Optional.ofNullable(period).isPresent(), DwException.stateReject("dw statistical period should not be null, id = {}", id)); + + // check datamodel + if (Strings.isNullOrEmpty(period.getEnName())) { + return 0; + } + LinkisDatamodelCenterRemoteClient dataModelRemoteClient = LinkisRemoteClientHolder.getDataModelRemoteClient(); + CyclesReferenceAction cyclesReferenceAction = new CyclesReferenceAction.Builder().setName(period.getEnName()).setUser(username).build(); + CyclesReferenceResult cyclesReferenceResult = dataModelRemoteClient.cyclesReference(cyclesReferenceAction); + return cyclesReferenceResult.getResult(); + } + + default int getLayerReferenceCount(Long id, String username) throws DwException { + checkLayerMapper(); + + DwLayer dwLayer = getDwLayerMapper().selectById(id); + PreconditionUtil.checkState(Optional.ofNullable(dwLayer).isPresent(), DwException.stateReject("dw layer should not be null, id = {}", id)); + + LinkisDatamodelCenterRemoteClient dataModelRemoteClient = LinkisRemoteClientHolder.getDataModelRemoteClient(); + LayersReferenceAction action = new LayersReferenceAction.Builder().setUser(username).setName(dwLayer.getEnName()).build(); + LayersReferenceResult layersReferenceResult = dataModelRemoteClient.layersReference(action); + return layersReferenceResult.getResult(); + } + + default boolean isModifierInUse(Long id, String username) throws DwException { + checkMapper(); + + DwModifier modifier = getDwModifierMapper().selectById(id); + PreconditionUtil.checkState(Optional.ofNullable(modifier).isPresent(), DwException.stateReject("dw modifier should not be null, id = {}", id)); + + // check datamodel + if (!Strings.isNullOrEmpty(modifier.getModifierTypeEn())) { + LinkisDatamodelCenterRemoteClient dataModelRemoteClient = LinkisRemoteClientHolder.getDataModelRemoteClient(); + ModifiersReferenceAction modifiersReferenceAction = new ModifiersReferenceAction.Builder().setName(modifier.getModifierTypeEn()).setUser(username).build(); + ModifiersReferenceResult modifiersReferenceResult = dataModelRemoteClient.modifiersReference(modifiersReferenceAction); + int referenceCount = modifiersReferenceResult.getResult(); + return referenceCount > 0; + } + + return false; + } + + default boolean isStatisticalPeriodInUse(Long id, String username) throws DwException { + checkMapper(); + + DwStatisticalPeriod period = getDwStatisticalPeriodMapper().selectById(id); + PreconditionUtil.checkState(Optional.ofNullable(period).isPresent(), DwException.stateReject("dw statistical period should not be null, id = {}", id)); + + // check datamodel + if (!Strings.isNullOrEmpty(period.getEnName())) { + LinkisDatamodelCenterRemoteClient dataModelRemoteClient = LinkisRemoteClientHolder.getDataModelRemoteClient(); + CyclesReferenceAction cyclesReferenceAction = new CyclesReferenceAction.Builder().setName(period.getEnName()).setUser(username).build(); + CyclesReferenceResult cyclesReferenceResult = dataModelRemoteClient.cyclesReference(cyclesReferenceAction); + int referenceCount = cyclesReferenceResult.getResult(); + return referenceCount > 0; + } + + return false; + } + + default boolean isThemeDomainInUse(Long id, String username) throws DwException { + checkMapper(); + + // 检查修饰词关联数量 + QueryWrapper queryModifierWrapper = new QueryWrapper<>(); + queryModifierWrapper.eq("theme_domain_id", id); + queryModifierWrapper.eq("status", Boolean.TRUE); + queryModifierWrapper.eq("is_available", Boolean.TRUE); + Integer count1 = getDwModifierMapper().selectCount(queryModifierWrapper); + boolean isRelModifier = Optional.ofNullable(count1).orElse(0) > 0; + + // 检查统计周期关联数量 + QueryWrapper statisticalPeriodQueryWrapper = new QueryWrapper<>(); + statisticalPeriodQueryWrapper.eq("theme_domain_id", id); + statisticalPeriodQueryWrapper.eq("status", Boolean.TRUE); + statisticalPeriodQueryWrapper.eq("is_available", Boolean.TRUE); + Integer count2 = getDwStatisticalPeriodMapper().selectCount(statisticalPeriodQueryWrapper); + boolean isRelStatisticalPeriod = Optional.ofNullable(count2).orElse(0) > 0; + + DwThemeDomain themeDomain = getDwThemeDomainMapper().selectById(id); + PreconditionUtil.checkState(Optional.ofNullable(themeDomain).isPresent(), DwException.stateReject("dw theme domain should not be null, id = {}", id)); + + // check datamodel + LinkisDatamodelCenterRemoteClient dataModelRemoteClient = LinkisRemoteClientHolder.getDataModelRemoteClient(); + ThemesReferenceAction themesReferenceAction = new ThemesReferenceAction.Builder().setName(themeDomain.getEnName()).setUser(username).build(); + ThemesReferenceResult themesReferenceResult = dataModelRemoteClient.themesReference(themesReferenceAction); + int referenceCount = themesReferenceResult.getResult(); + boolean isDataModelReferenced = referenceCount > 0; + +// LinkisDataAssetsRemoteClient dataAssetsRemoteClient = LinkisRemoteClientHolder.getDataAssetsRemoteClient(); + + return isRelModifier || isRelStatisticalPeriod || isDataModelReferenced; + } + + default boolean isLayerInUse(Long id, String username) throws DwException { + checkMapper(); + + // 检查修饰词关联数量 + QueryWrapper queryModifierWrapper = new QueryWrapper<>(); + queryModifierWrapper.eq("layer_id", id); + queryModifierWrapper.eq("status", Boolean.TRUE); + queryModifierWrapper.eq("is_available", Boolean.TRUE); + Integer count1 = getDwModifierMapper().selectCount(queryModifierWrapper); + boolean isRelModifier = Optional.ofNullable(count1).orElse(0) > 0; + + // 检查统计周期关联数量 + QueryWrapper statisticalPeriodQueryWrapper = new QueryWrapper<>(); + statisticalPeriodQueryWrapper.eq("layer_id", id); + statisticalPeriodQueryWrapper.eq("status", Boolean.TRUE); + statisticalPeriodQueryWrapper.eq("is_available", Boolean.TRUE); + Integer count2 = getDwStatisticalPeriodMapper().selectCount(statisticalPeriodQueryWrapper); + boolean isRelStatisticalPeriod = Optional.ofNullable(count2).orElse(0) > 0; + + DwLayer dwLayer = getDwLayerMapper().selectById(id); + PreconditionUtil.checkState(Optional.ofNullable(dwLayer).isPresent(), DwException.stateReject("dw layer should not be null, id = {}", id)); + + LinkisDatamodelCenterRemoteClient dataModelRemoteClient = LinkisRemoteClientHolder.getDataModelRemoteClient(); + LayersReferenceAction action = new LayersReferenceAction.Builder().setUser(username).setName(dwLayer.getEnName()).build(); + LayersReferenceResult layersReferenceResult = dataModelRemoteClient.layersReference(action); + int referenceCount = layersReferenceResult.getResult(); + boolean isDataModelReferenced = referenceCount > 0; + + return isRelModifier || isRelStatisticalPeriod || isDataModelReferenced; + } + +} diff --git a/dss-apps/dss-datawarehouse-design/data-warehouse-service/src/main/java/com/webank/wedatasphere/warehouse/service/DwDsService.java b/dss-apps/dss-datawarehouse-design/data-warehouse-service/src/main/java/com/webank/wedatasphere/warehouse/service/DwDsService.java new file mode 100644 index 0000000000..ddc33ff0cb --- /dev/null +++ b/dss-apps/dss-datawarehouse-design/data-warehouse-service/src/main/java/com/webank/wedatasphere/warehouse/service/DwDsService.java @@ -0,0 +1,13 @@ +package com.webank.wedatasphere.warehouse.service; + +import org.apache.linkis.server.Message; + +import javax.servlet.http.HttpServletRequest; + +public interface DwDsService { + Message getAllHiveDbs(HttpServletRequest request) throws Exception; + + Message getPrincipalUsers(HttpServletRequest request, String id) throws Exception; + + Message getPrincipalRoles(HttpServletRequest request, String id) throws Exception; +} diff --git a/dss-apps/dss-datawarehouse-design/data-warehouse-service/src/main/java/com/webank/wedatasphere/warehouse/service/DwLayerService.java b/dss-apps/dss-datawarehouse-design/data-warehouse-service/src/main/java/com/webank/wedatasphere/warehouse/service/DwLayerService.java new file mode 100644 index 0000000000..226ab3429d --- /dev/null +++ b/dss-apps/dss-datawarehouse-design/data-warehouse-service/src/main/java/com/webank/wedatasphere/warehouse/service/DwLayerService.java @@ -0,0 +1,29 @@ +package com.webank.wedatasphere.warehouse.service; + +import org.apache.linkis.server.Message; +import com.webank.wedatasphere.warehouse.cqe.DwLayerCreateCommand; +import com.webank.wedatasphere.warehouse.cqe.DwLayerQueryCommand; +import com.webank.wedatasphere.warehouse.cqe.DwLayerUpdateCommand; +import com.webank.wedatasphere.warehouse.exception.DwException; + +import javax.servlet.http.HttpServletRequest; + +public interface DwLayerService { + Message createDwCustomLayer(HttpServletRequest request, final DwLayerCreateCommand command) throws DwException; + + Message getLayerById(HttpServletRequest request, final Long id) throws DwException; + + Message getAllPresetLayers(HttpServletRequest request) throws DwException; + + Message queryPagedCustomLayers(HttpServletRequest request, final DwLayerQueryCommand command) throws DwException; + + Message deleteById(HttpServletRequest request, Long id) throws DwException; + + Message update(HttpServletRequest request, DwLayerUpdateCommand command) throws DwException; + + Message enable(HttpServletRequest request, Long id) throws DwException; + + Message disable(HttpServletRequest request, Long id) throws DwException; + + Message getAllLayers(HttpServletRequest request, Boolean isAvailable, String db) throws DwException; +} diff --git a/dss-apps/dss-datawarehouse-design/data-warehouse-service/src/main/java/com/webank/wedatasphere/warehouse/service/DwModifierService.java b/dss-apps/dss-datawarehouse-design/data-warehouse-service/src/main/java/com/webank/wedatasphere/warehouse/service/DwModifierService.java new file mode 100644 index 0000000000..849444e99d --- /dev/null +++ b/dss-apps/dss-datawarehouse-design/data-warehouse-service/src/main/java/com/webank/wedatasphere/warehouse/service/DwModifierService.java @@ -0,0 +1,27 @@ +package com.webank.wedatasphere.warehouse.service; + +import org.apache.linkis.server.Message; +import com.webank.wedatasphere.warehouse.cqe.DwModifierCreateCommand; +import com.webank.wedatasphere.warehouse.cqe.DwModifierQueryCommand; +import com.webank.wedatasphere.warehouse.cqe.DwModifierUpdateCommand; +import com.webank.wedatasphere.warehouse.exception.DwException; + +import javax.servlet.http.HttpServletRequest; + +public interface DwModifierService { + Message queryAllModifiers(HttpServletRequest request, DwModifierQueryCommand command) throws DwException; + + Message queryPage(HttpServletRequest request, DwModifierQueryCommand command) throws DwException; + + Message create(HttpServletRequest request, DwModifierCreateCommand command) throws DwException; + + Message getById(HttpServletRequest request, Long id) throws DwException; + + Message deleteById(HttpServletRequest request, Long id) throws DwException; + + Message update(HttpServletRequest request, DwModifierUpdateCommand command) throws DwException; + + Message enable(HttpServletRequest request, Long id) throws DwException; + + Message disable(HttpServletRequest request, Long id) throws DwException; +} diff --git a/dss-apps/dss-datawarehouse-design/data-warehouse-service/src/main/java/com/webank/wedatasphere/warehouse/service/DwStatisticalPeriodService.java b/dss-apps/dss-datawarehouse-design/data-warehouse-service/src/main/java/com/webank/wedatasphere/warehouse/service/DwStatisticalPeriodService.java new file mode 100644 index 0000000000..e6cacb86a1 --- /dev/null +++ b/dss-apps/dss-datawarehouse-design/data-warehouse-service/src/main/java/com/webank/wedatasphere/warehouse/service/DwStatisticalPeriodService.java @@ -0,0 +1,27 @@ +package com.webank.wedatasphere.warehouse.service; + +import org.apache.linkis.server.Message; +import com.webank.wedatasphere.warehouse.cqe.DwStatisticalPeriodCreateCommand; +import com.webank.wedatasphere.warehouse.cqe.DwStatisticalPeriodQueryCommand; +import com.webank.wedatasphere.warehouse.cqe.DwStatisticalPeriodUpdateCommand; +import com.webank.wedatasphere.warehouse.exception.DwException; + +import javax.servlet.http.HttpServletRequest; + +public interface DwStatisticalPeriodService { + Message queryAll(HttpServletRequest request, DwStatisticalPeriodQueryCommand command) throws DwException; + + Message queryPage(HttpServletRequest request, DwStatisticalPeriodQueryCommand command) throws DwException; + + Message create(HttpServletRequest request, DwStatisticalPeriodCreateCommand command) throws DwException; + + Message getById(HttpServletRequest request, Long id) throws DwException; + + Message deleteById(HttpServletRequest request, Long id) throws DwException; + + Message update(HttpServletRequest request, DwStatisticalPeriodUpdateCommand command) throws DwException; + + Message enable(HttpServletRequest request, Long id) throws DwException; + + Message disable(HttpServletRequest request, Long id) throws DwException; +} diff --git a/dss-apps/dss-datawarehouse-design/data-warehouse-service/src/main/java/com/webank/wedatasphere/warehouse/service/DwThemeDomainService.java b/dss-apps/dss-datawarehouse-design/data-warehouse-service/src/main/java/com/webank/wedatasphere/warehouse/service/DwThemeDomainService.java new file mode 100644 index 0000000000..772e8348e4 --- /dev/null +++ b/dss-apps/dss-datawarehouse-design/data-warehouse-service/src/main/java/com/webank/wedatasphere/warehouse/service/DwThemeDomainService.java @@ -0,0 +1,25 @@ +package com.webank.wedatasphere.warehouse.service; + +import org.apache.linkis.server.Message; +import com.webank.wedatasphere.warehouse.cqe.*; +import com.webank.wedatasphere.warehouse.exception.DwException; + +import javax.servlet.http.HttpServletRequest; + +public interface DwThemeDomainService { + Message queryAllThemeDomains(HttpServletRequest request, DwThemeDomainQueryCommand command) throws DwException; + + Message queryPage(HttpServletRequest request, DwThemeDomainQueryCommand command) throws DwException; + + Message create(HttpServletRequest request, DwThemeDomainCreateCommand command) throws DwException; + + Message getById(HttpServletRequest request, Long id) throws DwException; + + Message deleteById(HttpServletRequest request, Long id) throws DwException; + + Message update(HttpServletRequest request, DwThemeDomainUpdateCommand command) throws DwException; + + Message enable(HttpServletRequest request, Long id) throws DwException; + + Message disable(HttpServletRequest request, Long id) throws DwException; +} diff --git a/dss-apps/dss-datawarehouse-design/data-warehouse-service/src/main/java/com/webank/wedatasphere/warehouse/service/impl/DwDsServiceImpl.java b/dss-apps/dss-datawarehouse-design/data-warehouse-service/src/main/java/com/webank/wedatasphere/warehouse/service/impl/DwDsServiceImpl.java new file mode 100644 index 0000000000..7a4fc07eeb --- /dev/null +++ b/dss-apps/dss-datawarehouse-design/data-warehouse-service/src/main/java/com/webank/wedatasphere/warehouse/service/impl/DwDsServiceImpl.java @@ -0,0 +1,122 @@ +package com.webank.wedatasphere.warehouse.service.impl; + +import com.webank.wedatasphere.dss.framework.workspace.client.WorkSpaceRemoteClient; +import com.webank.wedatasphere.dss.framework.workspace.client.request.GetWorkspaceRolesAction; +import com.webank.wedatasphere.dss.framework.workspace.client.request.GetWorkspaceUsersAction; +import com.webank.wedatasphere.dss.framework.workspace.client.response.GetWorkspaceRolesResult; +import com.webank.wedatasphere.dss.framework.workspace.client.response.GetWorkspaceUsersResult; +import org.apache.linkis.common.exception.ErrorException; +import org.apache.linkis.datasource.client.impl.LinkisMetadataSourceRemoteClient; +import org.apache.linkis.datasource.client.request.GetMetadataSourceAllDatabasesAction; +import org.apache.linkis.datasource.client.response.GetMetadataSourceAllDatabasesResult; +import org.apache.linkis.server.Message; +import org.apache.linkis.server.security.SecurityFilter; +import com.webank.wedatasphere.warehouse.LinkisRemoteClientHolder; +import com.webank.wedatasphere.warehouse.exception.DwException; +import com.webank.wedatasphere.warehouse.exception.DwExceptionCode; +import com.webank.wedatasphere.warehouse.service.DwDsService; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Service; + +import javax.servlet.http.HttpServletRequest; +import java.util.List; +import java.util.Map; + +@Service +public class DwDsServiceImpl implements DwDsService { + private final static Logger LOGGER = LoggerFactory.getLogger(DwDsServiceImpl.class); + + @Override + public Message getAllHiveDbs(HttpServletRequest request) throws Exception { + String userName = SecurityFilter.getLoginUsername(request); + LOGGER.info("listAllDataSources userName:" + userName); + try { + LinkisMetadataSourceRemoteClient client = LinkisRemoteClientHolder.getMetadataSourceRemoteClient(); + + GetMetadataSourceAllDatabasesAction action = GetMetadataSourceAllDatabasesAction.builder().setUser(userName).build(); + GetMetadataSourceAllDatabasesResult result = client.getAllDBMetaDataSource(action); + List dbs = result.getDbs(); + return Message.ok().data("list", dbs); + } catch (Exception e) { + if (e instanceof ErrorException) { + ErrorException ee = (ErrorException) e; + throw new DwException(DwExceptionCode.GET_AVAILABLE_DBS_ERROR.getCode(), e.getMessage(), ee.getIp(), ee.getPort(), ee.getServiceKind()); + } else { + throw new DwException(DwExceptionCode.GET_AVAILABLE_DBS_ERROR.getCode(), e.getMessage()); + } + } + +// LinkisDataSourceRemoteClient linkisDataSourceRemoteClient = LinkisRemoteClientHolder.getLinkisDataSourceRemoteClient(); +// QueryDataSourceAction.Builder builder = QueryDataSourceAction.builder() +// .setSystem("system") +// .setIdentifies("") +// .setName("hive") +// .setCurrentPage(1) +// .setPageSize(500) +// .setUser(userName) +// ; +// +// List allDataSource; +// try { +// QueryDataSourceResult result = linkisDataSourceRemoteClient.queryDataSource(builder.build()); +// allDataSource = result.getAllDataSource(); +// } catch (Exception e) { +// LOGGER.error(e.getMessage(), e.getCause()); +// throw e; +// } +// List dataSources = new ArrayList<>(); +// if (!Objects.isNull(allDataSource)) { +// allDataSource.forEach(ds -> { +// DataSourceDTO item = new DataSourceDTO(); +// item.setId(ds.getId()); +// item.setCreateIdentify(ds.getCreateIdentify()); +// item.setName(ds.getDataSourceName()); +// item.setType(ds.getDataSourceType().getName()); +// item.setDataSourceTypeId(ds.getDataSourceTypeId()); +// dataSources.add(item); +// }); +// } +// return Message.ok().data("list", dataSources); + } + + @Override + public Message getPrincipalUsers(HttpServletRequest request, String workspaceId) throws Exception { + String userName = SecurityFilter.getLoginUsername(request); + LOGGER.info("get Principal users, userName:" + userName); + try { + WorkSpaceRemoteClient client = LinkisRemoteClientHolder.getWorkspaceRemoteClient(); + GetWorkspaceUsersAction action = GetWorkspaceUsersAction.builder().setUser(userName).setWorkspaceId(workspaceId).setPageNow(1).setPageSize(500).build(); + GetWorkspaceUsersResult result = client.getWorkspaceUsers(action); + List> users = result.getWorkspaceUsers(); + return Message.ok().data("list", users); + } catch (Exception e) { + if (e instanceof ErrorException) { + ErrorException ee = (ErrorException) e; + throw new DwException(DwExceptionCode.GET_PRINCIPAL_USERS_ERROR.getCode(), e.getMessage(), ee.getIp(), ee.getPort(), ee.getServiceKind()); + } else { + throw new DwException(DwExceptionCode.GET_PRINCIPAL_USERS_ERROR.getCode(), e.getMessage()); + } + } + } + + @Override + public Message getPrincipalRoles(HttpServletRequest request, String workspaceId) throws Exception { + String userName = SecurityFilter.getLoginUsername(request); + LOGGER.info("get Principal roles, userName:" + userName); + try { + WorkSpaceRemoteClient client = LinkisRemoteClientHolder.getWorkspaceRemoteClient(); + GetWorkspaceRolesAction action = GetWorkspaceRolesAction.builder().setUser(userName).setWorkspaceId(workspaceId).build(); + GetWorkspaceRolesResult result = client.getWorkspaceRoles(action); + List> roles = result.getWorkspaceRoles(); + return Message.ok().data("list", roles); + } catch (Exception e) { + if (e instanceof ErrorException) { + ErrorException ee = (ErrorException) e; + throw new DwException(DwExceptionCode.GET_PRINCIPAL_ROLES_ERROR.getCode(), e.getMessage(), ee.getIp(), ee.getPort(), ee.getServiceKind()); + } else { + throw new DwException(DwExceptionCode.GET_PRINCIPAL_ROLES_ERROR.getCode(), e.getMessage()); + } + } + } +} diff --git a/dss-apps/dss-datawarehouse-design/data-warehouse-service/src/main/java/com/webank/wedatasphere/warehouse/service/impl/DwLayerServiceImpl.java b/dss-apps/dss-datawarehouse-design/data-warehouse-service/src/main/java/com/webank/wedatasphere/warehouse/service/impl/DwLayerServiceImpl.java new file mode 100644 index 0000000000..baaaf8f272 --- /dev/null +++ b/dss-apps/dss-datawarehouse-design/data-warehouse-service/src/main/java/com/webank/wedatasphere/warehouse/service/impl/DwLayerServiceImpl.java @@ -0,0 +1,523 @@ +package com.webank.wedatasphere.warehouse.service.impl; + +import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; +import com.baomidou.mybatisplus.core.conditions.update.UpdateWrapper; +import com.baomidou.mybatisplus.core.metadata.IPage; +import com.baomidou.mybatisplus.extension.plugins.pagination.Page; +import com.webank.wedatasphere.dss.data.governance.entity.ClassificationConstant; +import com.webank.wedatasphere.dss.data.governance.impl.LinkisDataAssetsRemoteClient; +import com.webank.wedatasphere.dss.data.governance.request.CreateModelTypeAction; +import com.webank.wedatasphere.dss.data.governance.request.DeleteModelTypeAction; +import com.webank.wedatasphere.dss.data.governance.request.UpdateModelTypeAction; +import com.webank.wedatasphere.dss.data.governance.response.CreateModelTypeResult; +import com.webank.wedatasphere.dss.data.governance.response.DeleteModelTypeResult; +import com.webank.wedatasphere.dss.data.governance.response.UpdateModelTypeResult; +import org.apache.linkis.common.exception.ErrorException; +import org.apache.linkis.server.Message; +import org.apache.linkis.server.security.SecurityFilter; +import com.webank.wedatasphere.warehouse.LinkisRemoteClientHolder; +import com.webank.wedatasphere.warehouse.cqe.DwLayerCreateCommand; +import com.webank.wedatasphere.warehouse.cqe.DwLayerQueryCommand; +import com.webank.wedatasphere.warehouse.cqe.DwLayerUpdateCommand; +import com.webank.wedatasphere.warehouse.dao.domain.DwLayer; +import com.webank.wedatasphere.warehouse.dao.mapper.DwModifierMapper; +import com.webank.wedatasphere.warehouse.dao.mapper.DwStatisticalPeriodMapper; +import com.webank.wedatasphere.warehouse.dao.mapper.DwThemeDomainMapper; +import com.webank.wedatasphere.warehouse.dto.DwLayerDTO; +import com.webank.wedatasphere.warehouse.dto.DwLayerListItemDTO; +import com.webank.wedatasphere.warehouse.dto.PageInfo; +import com.webank.wedatasphere.warehouse.exception.DwException; +import com.webank.wedatasphere.warehouse.dao.mapper.DwLayerMapper; +//import com.webank.wedatasphere.warehouse.mapper.DwLayerModelMapper; +import com.webank.wedatasphere.warehouse.exception.DwExceptionCode; +import com.webank.wedatasphere.warehouse.service.DwDomainReferenceAdapter; +import com.webank.wedatasphere.warehouse.service.DwLayerService; +import com.webank.wedatasphere.warehouse.utils.PreconditionUtil; +import com.webank.wedatasphere.warehouse.utils.RegexUtil; +import org.apache.logging.log4j.util.Strings; +import org.springframework.beans.BeanUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +import javax.servlet.http.HttpServletRequest; +import java.util.*; + +@Service +public class DwLayerServiceImpl implements DwLayerService, DwDomainReferenceAdapter { + + private final DwLayerMapper dwLayerMapper; + private final DwThemeDomainMapper dwThemeDomainMapper; + private final DwModifierMapper dwModifierMapper; + private final DwStatisticalPeriodMapper dwStatisticalPeriodMapper; + + @Autowired + public DwLayerServiceImpl(final DwLayerMapper dwLayerMapper, final DwThemeDomainMapper dwThemeDomainMapper, final DwModifierMapper dwModifierMapper, final DwStatisticalPeriodMapper dwStatisticalPeriodMapper) { + this.dwLayerMapper = dwLayerMapper; + this.dwThemeDomainMapper = dwThemeDomainMapper; + this.dwModifierMapper = dwModifierMapper; + this.dwStatisticalPeriodMapper = dwStatisticalPeriodMapper; + } + + @Override + public DwModifierMapper getDwModifierMapper() { + return dwModifierMapper; + } + + @Override + public DwStatisticalPeriodMapper getDwStatisticalPeriodMapper() { + return dwStatisticalPeriodMapper; + } + + @Override + public DwLayerMapper getDwLayerMapper() { + return dwLayerMapper; + } + + @Override + public DwThemeDomainMapper getDwThemeDomainMapper() { + return dwThemeDomainMapper; + } + + @Transactional(rollbackFor = Exception.class) + @Override + public Message createDwCustomLayer(HttpServletRequest request, DwLayerCreateCommand command) throws DwException { + String username = SecurityFilter.getLoginUsername(request); +// String username = "hdfs"; + String name = command.getName(); + String enName = command.getEnName(); + String databases = command.getDatabases(); + Integer order = command.getOrder(); +// String autoCollectStrategy = command.getAutoCollectStrategy(); + String owner = command.getOwner(); + String description = command.getDescription(); + String principalName = command.getPrincipalName(); + + name = PreconditionUtil.checkStringArgumentNotBlankTrim(name, DwException.argumentReject("name should not empty")); + PreconditionUtil.checkArgument(RegexUtil.checkCnName(name), DwException.argumentReject("name must be digitg, chinese and underline")); + enName = PreconditionUtil.checkStringArgumentNotBlankTrim(enName, DwException.argumentReject("name alias should not empty")); + PreconditionUtil.checkArgument(RegexUtil.checkEnName(enName), DwException.argumentReject("name must be digit, alpha and underline")); + owner = PreconditionUtil.checkStringArgumentNotBlankTrim(owner, DwException.argumentReject("charge user should not empty")); +// authority = PreconditionUtil.checkStringArgumentNotBlankTrim(authority, DwException.argumentReject("authority should not empty")); + QueryWrapper nameUniqueCheckQuery = new QueryWrapper<>(); + nameUniqueCheckQuery.eq("name", name); + nameUniqueCheckQuery.eq("status", Boolean.TRUE); + DwLayer exist = this.dwLayerMapper.selectOne(nameUniqueCheckQuery); + PreconditionUtil.checkState(Objects.isNull(exist), DwException.stateReject("layer name aleardy exists")); + + if (Strings.isBlank(databases)) { + databases = "ALL"; + } + if (Objects.isNull(order)) { + order = 1; + } + +// if (Strings.isBlank(autoCollectStrategy)) { +// autoCollectStrategy = "{}"; +// } + +// String user = "hdfs"; + + Date now = new Date(); + + DwLayer layer = new DwLayer(); + layer.setName(name); + layer.setEnName(enName); + layer.setDescription(description); + layer.setPrincipalName(principalName); + layer.setOwner(owner); + layer.setDbs(databases); +// layer.setAutoCollectStrategy(autoCollectStrategy); + // 设置为 FALSE ，因为是自定义分层 + layer.setPreset(Boolean.FALSE); + layer.setIsAvailable(Boolean.TRUE); +// layer.setCreateUser(user); + layer.setSort(order); + layer.setCreateTime(now); + layer.setUpdateTime(now); + layer.setStatus(Boolean.TRUE); + layer.setLockVersion(1L); + + int insert = this.dwLayerMapper.insert(layer); + PreconditionUtil.checkState(1 == insert, DwException.stateReject("create dw layer failed")); + + // 建立关联 + try { + LinkisDataAssetsRemoteClient dataAssetsRemoteClient = LinkisRemoteClientHolder.getDataAssetsRemoteClient(); + CreateModelTypeAction action = new CreateModelTypeAction.Builder().setType(ClassificationConstant.LAYER).setName(enName).setUser(username).build(); + CreateModelTypeResult result = dataAssetsRemoteClient.createModelType(action); + + if (result.getStatus() != 0) { + throw new DwException(result.getStatus(), result.getMessage()); + } + } catch (Exception e) { + if (e instanceof ErrorException) { + ErrorException ee = (ErrorException) e; + throw new DwException(DwExceptionCode.CREATE_MODEL_TYPE_ERROR.getCode(), e.getMessage(), ee.getIp(), ee.getPort(), ee.getServiceKind()); + } else { + throw new DwException(DwExceptionCode.CREATE_MODEL_TYPE_ERROR.getCode(), e.getMessage()); + } + } + + + return Message.ok().data("id", layer.getId()); + } + + @Override + public Message getLayerById(HttpServletRequest request, Long id) throws DwException { + PreconditionUtil.checkArgument(!Objects.isNull(id), DwException.argumentReject("id should not be null")); + + DwLayer layer = this.dwLayerMapper.selectById(id); + PreconditionUtil.checkState(!Objects.isNull(layer), DwException.stateReject("layer not found")); + PreconditionUtil.checkState(layer.getStatus(), DwException.stateReject("layer has been removed")); + +// DwLayerDTO dto = this.dwLayerModelMapper.toDTO(layer); + String username = SecurityFilter.getLoginUsername(request); + boolean inUse = isLayerInUse(layer.getId(), username); + + DwLayerDTO dto = new DwLayerDTO(); + BeanUtils.copyProperties(layer, dto); + dto.setReferenced(inUse); + + return Message.ok().data("item", dto); + } + + + @Override + public Message getAllLayers(HttpServletRequest request, Boolean isAvailable, String db) throws DwException { + QueryWrapper queryWrapper = new QueryWrapper<>(); + queryWrapper.eq("status", Boolean.TRUE); + if (!Objects.isNull(isAvailable)) { + queryWrapper.eq("is_available", isAvailable); + } + List records = this.dwLayerMapper.selectList(queryWrapper); + String username = SecurityFilter.getLoginUsername(request); + + List dtos = new ArrayList<>(); + boolean contain; + for (DwLayer record : records) { + contain = true; + if (Strings.isNotBlank(db)) { + if ("all".equalsIgnoreCase(db)) { + // 查询的时候传入的参数 db = all + contain = true; + } else { + // 预置分层所有库都可以用 + if (record.getPreset()) { + contain = true; + } else { + contain = false; + String dbs = record.getDbs(); + if (Strings.isNotBlank(dbs)) { + String[] splitDbs = dbs.split(","); + for (String splitDb : splitDbs) { + // 如果分层可用库选择了所有，则直接添加到查询记录中 + if ("all".equalsIgnoreCase(splitDb)) { + contain = true; + break; + } + // 否则就遍历判断当前查询库是否在当前分层中存在 + if (Objects.equals(splitDb, db)) { + contain = true; + break; + } + } + } + } + } + } + + if (!contain) { + continue; + } + + DwLayerListItemDTO dto = new DwLayerListItemDTO(); + BeanUtils.copyProperties(record, dto); + // get reference count + int layerReferenceCount = getLayerReferenceCount(record.getId(), username); + dto.setReferenceCount(layerReferenceCount); + dtos.add(dto); + } + + return Message.ok().data("list", dtos); + } + + + @Override + public Message getAllPresetLayers(HttpServletRequest request) throws DwException { + QueryWrapper queryWrapper = new QueryWrapper<>(); + queryWrapper.eq("status", Boolean.TRUE); + queryWrapper.eq("preset", Boolean.TRUE); + List records = this.dwLayerMapper.selectList(queryWrapper); +// List dtos = DwLayerModelMapper.INSTANCE.toList(dwLayers); + List dtos = new ArrayList<>(); + + String username = SecurityFilter.getLoginUsername(request); + + for (DwLayer record : records) { + DwLayerListItemDTO dto = new DwLayerListItemDTO(); + BeanUtils.copyProperties(record, dto); + // get reference count + int layerReferenceCount = getLayerReferenceCount(record.getId(), username); + dto.setReferenceCount(layerReferenceCount); + dtos.add(dto); + } + + return Message.ok().data("list", dtos); + } + + @Override + public Message queryPagedCustomLayers(HttpServletRequest request, DwLayerQueryCommand command) throws DwException { + Integer page = command.getPage(); + Integer size = command.getSize(); + String name = command.getName(); + Boolean enabled = command.getEnabled(); + if (Objects.isNull(page)) + page = 1; + + if (Objects.isNull(size)) + size = 10; + + QueryWrapper queryWrapper = new QueryWrapper<>(); + queryWrapper.eq("status", Boolean.TRUE); + queryWrapper.eq("preset", Boolean.FALSE); + if (!Objects.isNull(enabled)) { + queryWrapper.eq("is_available", enabled); + } + if (Strings.isNotBlank(name)) { + queryWrapper.like("name", name).or().like("en_name", name); + } + + Page queryPage = new Page<>(page, size); + + IPage _page = this.dwLayerMapper.selectPage(queryPage, queryWrapper); + + List records = _page.getRecords(); + +// List list = DwLayerModelMapper.INSTANCE.toList(records); + List list = new ArrayList<>(); + String username = SecurityFilter.getLoginUsername(request); + for (DwLayer record : records) { + DwLayerListItemDTO dto = new DwLayerListItemDTO(); + BeanUtils.copyProperties(record, dto); + // get reference count + int layerReferenceCount = getLayerReferenceCount(record.getId(), username); + dto.setReferenceCount(layerReferenceCount); + + list.add(dto); + + } + + + PageInfo __page = new PageInfo<>(list, _page.getCurrent(), _page.getSize(), _page.getTotal()); + + return Message.ok().data("page", __page); + } + + // TODO 后面需要检查主题域下有没有关联的 Hive 表，才决定是否删除 + @Transactional(rollbackFor = Exception.class) + @Override + public Message deleteById(HttpServletRequest request, Long id) throws DwException { + PreconditionUtil.checkArgument(!Objects.isNull(id), DwException.argumentReject("id should not be null")); + DwLayer layer = this.dwLayerMapper.selectById(id); + PreconditionUtil.checkState(!Objects.isNull(layer), DwException.stateReject("layer not found")); + PreconditionUtil.checkState(Objects.equals(Boolean.FALSE, layer.getPreset()), DwException.stateReject("preset layer could not be removed")); + + String username = SecurityFilter.getLoginUsername(request); + // check in use + // statistical_period & modifier & data_model client api check + boolean inUse = isLayerInUse(layer.getId(), username); + PreconditionUtil.checkState(!inUse, DwException.stateReject("layer is in use")); + +// if (Objects.equals(Boolean.FALSE, layer.getStatus())) { +// return Message.ok(); +// } +// layer.setStatus(Boolean.FALSE); +// int i = this.dwLayerMapper.updateById(layer); + int i = this.dwLayerMapper.deleteById(layer); + PreconditionUtil.checkState(1 == i, DwException.stateReject("remove action failed")); + + // 删除关联 + try { + LinkisDataAssetsRemoteClient dataAssetsRemoteClient = LinkisRemoteClientHolder.getDataAssetsRemoteClient(); + DeleteModelTypeAction action = new DeleteModelTypeAction.Builder().setType(ClassificationConstant.LAYER).setName(layer.getEnName()).setUser(username).build(); + DeleteModelTypeResult result = dataAssetsRemoteClient.deleteModelType(action); + + if (result.getStatus() != 0) { + throw new DwException(result.getStatus(), result.getMessage()); + } + } catch (Exception e) { + if (e instanceof ErrorException) { + ErrorException ee = (ErrorException) e; + throw new DwException(DwExceptionCode.DELETE_MODEL_TYPE_ERROR.getCode(), e.getMessage(), ee.getIp(), ee.getPort(), ee.getServiceKind()); + } else { + throw new DwException(DwExceptionCode.DELETE_MODEL_TYPE_ERROR.getCode(), e.getMessage()); + } + } + + return Message.ok(); + } + + /** + * 更新操作涉及到 预置分层和自定义分层 + * + * 预置分层是不能更新名称的，其它字段的更新和自定义分层一致。 + * 所以针对预置分层，是不需要传递 name 参数的，所以要稍微区分逻辑判断来处理 + * + * 预留分层的 name name_alias charge_user 无法改变 + */ + @Transactional(rollbackFor = Exception.class) + @Override + public Message update(HttpServletRequest request, DwLayerUpdateCommand command) throws DwException { + String username = SecurityFilter.getLoginUsername(request); + // 基本参数校验 + Long id = command.getId(); + PreconditionUtil.checkArgument(!Objects.isNull(id), DwException.argumentReject("id should not be null")); + + // 实体校验 + DwLayer layer = this.dwLayerMapper.selectById(id); + PreconditionUtil.checkState(!Objects.isNull(layer), DwException.stateReject("layer not found")); + String orgEnName = layer.getEnName(); + + // 分层类型后的name参数单独校验 + String name = command.getName(); + String enName = command.getEnName(); + String owner = command.getOwner(); + Integer order = command.getOrder(); + String description = command.getDescription(); + String principalName = command.getPrincipalName(); +// authority = PreconditionUtil.checkStringArgumentNotBlankTrim(authority, DwException.argumentReject("authority should not empty")); + String databases = command.getDatabases(); +// String autoCollectStrategy = command.getAutoCollectStrategy(); + if (Strings.isBlank(databases)) { + databases = "ALL"; + } + + if (Objects.isNull(order)) { + order = 1; + } + +// if (Strings.isBlank(autoCollectStrategy)) { +// autoCollectStrategy = "{}"; +// } + + // 预置分层也能修改 +// if (!layer.getPreset()) { + name = PreconditionUtil.checkStringArgumentNotBlankTrim(name, DwException.argumentReject("name should not empty")); + PreconditionUtil.checkArgument(RegexUtil.checkCnName(name), DwException.argumentReject("name must be digitg, chinese and underline")); + enName = PreconditionUtil.checkStringArgumentNotBlankTrim(enName, DwException.argumentReject("name alias should not empty")); + PreconditionUtil.checkArgument(RegexUtil.checkEnName(enName), DwException.argumentReject("name must be digit, alpha and underline")); + + if (!Objects.equals(name, layer.getName()) || !Objects.equals(enName, layer.getEnName())) { + // check name enName if not equal, we should check if in use + // check in use + // statistical_period & modifier & data_model client api check + + boolean inUse = isLayerInUse(layer.getId(), username); + PreconditionUtil.checkState(!inUse, DwException.stateReject("layer is in use")); + } + + owner = PreconditionUtil.checkStringArgumentNotBlankTrim(owner, DwException.argumentReject("owner should not empty")); + // 并且自定义分层在更新名称的时候，名称不能重复 + QueryWrapper nameUniqueCheckQuery = new QueryWrapper<>(); + nameUniqueCheckQuery.eq("name", name); + nameUniqueCheckQuery.ne("id", id); + nameUniqueCheckQuery.eq("status", Boolean.TRUE); + DwLayer exist = this.dwLayerMapper.selectOne(nameUniqueCheckQuery); + PreconditionUtil.checkState(Objects.isNull(exist), DwException.stateReject("layer name aleardy exists")); + layer.setName(name); + + // 并且自定义分层在更新名称的时候，name alias不能重复 + QueryWrapper nameAliasUniqueCheckQuery = new QueryWrapper<>(); + nameAliasUniqueCheckQuery.eq("en_name", name); + nameAliasUniqueCheckQuery.ne("id", id); + nameAliasUniqueCheckQuery.eq("status", Boolean.TRUE); + DwLayer nameAliasExist = this.dwLayerMapper.selectOne(nameAliasUniqueCheckQuery); + PreconditionUtil.checkState(Objects.isNull(nameAliasExist), DwException.stateReject("layer en name aleardy exists")); + layer.setEnName(enName); + + layer.setOwner(owner); +// } + +// String user = SecurityFilter.getLoginUsername(request); + + Long oldVersion = layer.getLockVersion(); + + Date now = new Date(); + layer.setDbs(databases); +// layer.set(autoCollectStrategy); + layer.setDescription(description); + layer.setPrincipalName(principalName); + layer.setSort(order); +// layer.setModifyUser(user); + layer.setUpdateTime(now); + layer.setLockVersion(oldVersion + 1); + + UpdateWrapper updateWrapper = new UpdateWrapper<>(); + updateWrapper.eq("id", layer.getId()); + updateWrapper.eq("lock_version", oldVersion); + + int i = this.dwLayerMapper.update(layer, updateWrapper); + + PreconditionUtil.checkState(1 == i, DwException.stateReject("update layer failed")); + + // 更新关联 + if (!Objects.equals(orgEnName, layer.getEnName())) { + try { + LinkisDataAssetsRemoteClient dataAssetsRemoteClient = LinkisRemoteClientHolder.getDataAssetsRemoteClient(); + UpdateModelTypeAction action = new UpdateModelTypeAction.Builder().setType(ClassificationConstant.LAYER).setName(layer.getEnName()).setOrgName(orgEnName).setUser(username).build(); + UpdateModelTypeResult result = dataAssetsRemoteClient.updateModelType(action); + + if (result.getStatus() != 0) { + throw new DwException(result.getStatus(), result.getMessage()); + } + } catch (Exception e) { + if (e instanceof ErrorException) { + ErrorException ee = (ErrorException) e; + throw new DwException(DwExceptionCode.UPDATE_MODEL_TYPE_ERROR.getCode(), e.getMessage(), ee.getIp(), ee.getPort(), ee.getServiceKind()); + } else { + throw new DwException(DwExceptionCode.UPDATE_MODEL_TYPE_ERROR.getCode(), e.getMessage()); + } + } + } + + return Message.ok(); + } + + @Transactional(rollbackFor = Exception.class) + @Override + public Message enable(HttpServletRequest request, Long id) throws DwException { + changeEnable(request, id, Boolean.TRUE); + return Message.ok(); + } + + @Transactional(rollbackFor = Exception.class) + @Override + public Message disable(HttpServletRequest request, Long id) throws DwException { + changeEnable(request, id, Boolean.FALSE); + return Message.ok(); + } + + private void changeEnable(HttpServletRequest request, Long id, Boolean enabled) throws DwException { + PreconditionUtil.checkArgument(!Objects.isNull(id), DwException.argumentReject("id should not be null")); + + DwLayer record = this.dwLayerMapper.selectById(id); + PreconditionUtil.checkState(!Objects.isNull(record), DwException.stateReject("layer not found")); + PreconditionUtil.checkState(record.getStatus(), DwException.stateReject("layer has been removed")); + if (Objects.equals(enabled, record.getIsAvailable())) { + return; + } + +// String user = "hdfs"; + Long oldVersion = record.getLockVersion(); +// record.setModifyUser(user); + record.setUpdateTime(new Date()); + record.setIsAvailable(enabled); + record.setLockVersion(oldVersion + 1); + UpdateWrapper updateWrapper = new UpdateWrapper<>(); + updateWrapper.eq("id", record.getId()); + updateWrapper.eq("lock_version", oldVersion); + int update = this.dwLayerMapper.update(record, updateWrapper); + PreconditionUtil.checkState(1 == update, DwException.stateReject(enabled ? "enable" : "disable" + " failed")); + } +} diff --git a/dss-apps/dss-datawarehouse-design/data-warehouse-service/src/main/java/com/webank/wedatasphere/warehouse/service/impl/DwModifierServiceImpl.java b/dss-apps/dss-datawarehouse-design/data-warehouse-service/src/main/java/com/webank/wedatasphere/warehouse/service/impl/DwModifierServiceImpl.java new file mode 100644 index 0000000000..7374442aac --- /dev/null +++ b/dss-apps/dss-datawarehouse-design/data-warehouse-service/src/main/java/com/webank/wedatasphere/warehouse/service/impl/DwModifierServiceImpl.java @@ -0,0 +1,455 @@ +package com.webank.wedatasphere.warehouse.service.impl; + +import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; +import com.baomidou.mybatisplus.core.conditions.update.UpdateWrapper; +import com.baomidou.mybatisplus.core.metadata.IPage; +import com.baomidou.mybatisplus.extension.plugins.pagination.Page; +import org.apache.linkis.server.Message; +import org.apache.linkis.server.security.SecurityFilter; +import com.webank.wedatasphere.warehouse.cqe.DwModifierCreateCommand; +import com.webank.wedatasphere.warehouse.cqe.DwModifierQueryCommand; +import com.webank.wedatasphere.warehouse.cqe.DwModifierUpdateCommand; +import com.webank.wedatasphere.warehouse.dao.domain.*; +import com.webank.wedatasphere.warehouse.dao.domain.DwLayer; +import com.webank.wedatasphere.warehouse.dao.domain.DwThemeDomain; +import com.webank.wedatasphere.warehouse.dao.mapper.*; +import com.webank.wedatasphere.warehouse.dto.DwModifierDTO; +import com.webank.wedatasphere.warehouse.dto.DwModifierListDTO; +import com.webank.wedatasphere.warehouse.dto.DwModifierListItemDTO; +import com.webank.wedatasphere.warehouse.dto.PageInfo; +import com.webank.wedatasphere.warehouse.exception.DwException; +import com.webank.wedatasphere.warehouse.service.DwDomainReferenceAdapter; +import com.webank.wedatasphere.warehouse.service.DwModifierService; +import com.webank.wedatasphere.warehouse.utils.PreconditionUtil; +import com.webank.wedatasphere.warehouse.utils.RegexUtil; +import org.apache.logging.log4j.util.Strings; +import org.springframework.beans.BeanUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +import javax.servlet.http.HttpServletRequest; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; +import java.util.Objects; + +@Service +public class DwModifierServiceImpl implements DwModifierService, DwDomainReferenceAdapter { + + private final DwModifierMapper dwModifierMapper; + private final DwModifierListMapper dwModifierListMapper; + private final DwStatisticalPeriodMapper dwStatisticalPeriodMapper; + private final DwLayerMapper dwLayerMapper; + private final DwThemeDomainMapper dwThemeDomainMapper; + + @Autowired + public DwModifierServiceImpl( + DwModifierMapper dwModifierMapper, + DwStatisticalPeriodMapper dwStatisticalPeriodMapper, + DwModifierListMapper dwModifierListMapper, + DwLayerMapper dwLayerMapper, + DwThemeDomainMapper dwThemeDomainMapper + ) { + this.dwModifierMapper = dwModifierMapper; + this.dwStatisticalPeriodMapper = dwStatisticalPeriodMapper; + this.dwModifierListMapper = dwModifierListMapper; + this.dwLayerMapper = dwLayerMapper; + this.dwThemeDomainMapper = dwThemeDomainMapper; + } + + @Override + public DwLayerMapper getDwLayerMapper() { + return dwLayerMapper; + } + + @Override + public DwThemeDomainMapper getDwThemeDomainMapper() { + return dwThemeDomainMapper; + } + + @Override + public DwStatisticalPeriodMapper getDwStatisticalPeriodMapper() { + return this.dwStatisticalPeriodMapper; + } + + @Override + public DwModifierMapper getDwModifierMapper() { + return dwModifierMapper; + } + + @Override + public Message queryAllModifiers(HttpServletRequest request, DwModifierQueryCommand command) throws DwException { + String typeName = command.getName(); + Boolean isAvailable = command.getEnabled(); + String theme = command.getTheme(); + String layer = command.getLayer(); + + QueryWrapper queryWrapper = new QueryWrapper<>(); + queryWrapper.eq("status", Boolean.TRUE); + if (!Objects.isNull(isAvailable)) { + queryWrapper.eq("is_available", isAvailable); + } + if (Strings.isNotBlank(theme)) { + queryWrapper.eq("theme_area_en", theme); + } + if (Strings.isNotBlank(layer)) { + queryWrapper.eq("layer_area_en", layer); + } + + if (Strings.isNotBlank(typeName)) { + queryWrapper.and(qw -> { + queryWrapper.like("modifier_type", typeName).or().like("modifier_type_en", typeName); + }); + } + List records = this.dwModifierMapper.selectList(queryWrapper); + + List list = new ArrayList<>(); + DwModifierListItemDTO dto; + for (DwModifier modifier : records) { + dto = new DwModifierListItemDTO(); + BeanUtils.copyProperties(modifier, dto); + list.add(dto); + } + return Message.ok().data("list", list); + } + + @Override + public Message queryPage(HttpServletRequest request, DwModifierQueryCommand command) throws DwException { + Integer page = command.getPage(); + Integer size = command.getSize(); + String typeName = command.getName(); + if (Objects.isNull(page)) + page = 1; + + if (Objects.isNull(size)) + size = 10; + + QueryWrapper queryWrapper = new QueryWrapper<>(); + queryWrapper.eq("status", Boolean.TRUE); + if (Strings.isNotBlank(typeName)) { + queryWrapper.like("modifier_type", typeName); + } + + if (!Objects.isNull(command.getEnabled())) { + queryWrapper.eq("is_available", command.getEnabled()); + } + + Page queryPage = new Page<>(page, size); + + IPage _page = this.dwModifierMapper.selectPage(queryPage, queryWrapper); + + List records = _page.getRecords(); + + List list = new ArrayList<>(); + DwModifierListItemDTO dto; + String username = SecurityFilter.getLoginUsername(request); + for (DwModifier modifier : records) { + dto = new DwModifierListItemDTO(); + BeanUtils.copyProperties(modifier, dto); + int modifierReferenceCount = getModifierReferenceCount(modifier.getId(), username); + dto.setReferenceCount(modifierReferenceCount); + list.add(dto); + } + +// List list = DwDecorationModelMapper.INSTANCE.toList(records); +// List list = new ArrayList<>(); + + PageInfo __page = new PageInfo<>(list, _page.getCurrent(), _page.getSize(), _page.getTotal()); + + return Message.ok().data("page", __page); + } + + @Transactional(rollbackFor = Exception.class) + @Override + public Message create(HttpServletRequest request, DwModifierCreateCommand command) throws DwException { + Long themeDomainId = command.getThemeDomainId(); + Long layerId = command.getLayerId(); + String typeName = command.getTypeName(); + String typeEnName = command.getTypeEnName(); + +// String typeNameAlias = command.getTypeNameAlias(); +// String decorationList = command.getDecorationList(); + String description = command.getDescription(); + + PreconditionUtil.checkArgument(!Objects.isNull(themeDomainId), DwException.argumentReject("theme domain id should not empty")); + PreconditionUtil.checkArgument(!Objects.isNull(layerId), DwException.argumentReject("layer id should not empty")); + typeName = PreconditionUtil.checkStringArgumentNotBlankTrim(typeName, DwException.argumentReject("type name should not empty")); + PreconditionUtil.checkArgument(RegexUtil.checkCnName(typeName), DwException.argumentReject("type name must be digitg, chinese and underline")); + typeEnName = PreconditionUtil.checkStringArgumentNotBlankTrim(typeEnName, DwException.argumentReject("type en name should not empty")); + PreconditionUtil.checkArgument(RegexUtil.checkEnName(typeEnName), DwException.argumentReject("type en name must be digitg, alpha and underline")); +// typeNameAlias = PreconditionUtil.checkStringArgumentNotBlankTrim(typeNameAlias, DwException.argumentReject("typeNameAlias should not empty")); + + // 验证 主题域 分层是否存在 + QueryWrapper layerQueryWrapper = new QueryWrapper<>(); + layerQueryWrapper.eq("id", layerId); + layerQueryWrapper.eq("status", Boolean.TRUE); + DwLayer dwLayer = this.dwLayerMapper.selectOne(layerQueryWrapper); + PreconditionUtil.checkState(!Objects.isNull(dwLayer), DwException.stateReject("layer not found")); + PreconditionUtil.checkState(dwLayer.getIsAvailable(), DwException.stateReject("layer disabled")); + + QueryWrapper themeDomainQueryWrapper = new QueryWrapper<>(); + themeDomainQueryWrapper.eq("id", themeDomainId); + themeDomainQueryWrapper.eq("status", Boolean.TRUE); + DwThemeDomain dwThemeDomain = this.dwThemeDomainMapper.selectOne(themeDomainQueryWrapper); + PreconditionUtil.checkState(!Objects.isNull(dwThemeDomain), DwException.stateReject("theme domain not found")); + + // typeName 唯一性检测 + QueryWrapper nameUniqueCheckQuery = new QueryWrapper<>(); + nameUniqueCheckQuery.eq("modifier_type", typeName); + nameUniqueCheckQuery.eq("status", Boolean.TRUE); + DwModifier exist = this.dwModifierMapper.selectOne(nameUniqueCheckQuery); + PreconditionUtil.checkState(Objects.isNull(exist), DwException.stateReject("modifier type name aleardy exists")); + + // typeNameEn 唯一性检测 + QueryWrapper nameAliasUniqueCheckQuery = new QueryWrapper<>(); + nameAliasUniqueCheckQuery.eq("modifier_type_en", typeEnName); + nameAliasUniqueCheckQuery.eq("status", Boolean.TRUE); + DwModifier exist2 = this.dwModifierMapper.selectOne(nameAliasUniqueCheckQuery); + PreconditionUtil.checkState(Objects.isNull(exist2), DwException.stateReject("modifier type en name aleardy exists")); + + Date now = new Date(); + + DwModifier record = new DwModifier(); + record.setThemeDomainId(dwThemeDomain.getId()); + record.setThemeArea(dwThemeDomain.getName()); + record.setThemeAreaEn(dwThemeDomain.getEnName()); + record.setLayerId(dwLayer.getId()); + record.setLayerArea(dwLayer.getName()); + record.setLayerAreaEn(dwLayer.getEnName()); + record.setModifierType(typeName); + record.setModifierTypeEn(typeEnName); + record.setDescription(description); + record.setIsAvailable(Boolean.TRUE); + record.setCreateTime(now); + record.setUpdateTime(now); + record.setStatus(Boolean.TRUE); + record.setLockVersion(1L); + + int insert = this.dwModifierMapper.insert(record); + PreconditionUtil.checkState(1 == insert, DwException.stateReject("create dw modifier failed")); + + // 新增修饰词列表信息 + List list = command.getList(); + if (!Objects.isNull(list) && !list.isEmpty()) { + for (DwModifierCreateCommand.DwModifierCreateListItem item : list) { + String name = item.getName(); + String identifier = item.getIdentifier(); + String formula = item.getFormula(); + if (Strings.isBlank(name) || Strings.isBlank(identifier) || Strings.isBlank(formula)) { + continue; + } + + DwModifierList m = new DwModifierList(); + m.setModifierId(record.getId()); + m.setName(name); + m.setIdentifier(identifier); + m.setFormula(formula); + m.setCreateTime(now); + m.setUpdateTime(now); + this.dwModifierListMapper.insert(m); + } + } + + return Message.ok().data("id", record.getId()); + } + + @Override + public Message getById(HttpServletRequest request, Long id) throws DwException { + PreconditionUtil.checkArgument(!Objects.isNull(id), DwException.argumentReject("id should not be null")); + + DwModifier record = this.dwModifierMapper.selectById(id); + PreconditionUtil.checkState(!Objects.isNull(record), DwException.stateReject("modifier not found")); + PreconditionUtil.checkState(record.getStatus(), DwException.stateReject("modifier has been removed")); + + DwModifierDTO dto = new DwModifierDTO(); +// DwModifierDTO dto = this.dwModifierModelMapper.toDTO(record); + BeanUtils.copyProperties(record, dto); + String username = SecurityFilter.getLoginUsername(request); + boolean inUse = isModifierInUse(record.getId(), username); + dto.setReferenced(inUse); + + QueryWrapper dwModifierListQueryWrapper = new QueryWrapper<>(); + dwModifierListQueryWrapper.eq("modifier_id", record.getId()); + List dwModifierLists = this.dwModifierListMapper.selectList(dwModifierListQueryWrapper); + + List dwModifierListDTOs = new ArrayList<>(); + for (DwModifierList item : dwModifierLists) { + DwModifierListDTO dwModifierListDTO = new DwModifierListDTO();/*this.dwModifierListModelMapper.toListItem(item);*/ + BeanUtils.copyProperties(item, dwModifierListDTO); + dwModifierListDTOs.add(dwModifierListDTO); + } + + dto.setList(dwModifierListDTOs); + + return Message.ok().data("item", dto); + } + + @Transactional(rollbackFor = Exception.class) + @Override + public Message deleteById(HttpServletRequest request, Long id) throws DwException { + PreconditionUtil.checkArgument(!Objects.isNull(id), DwException.argumentReject("id should not be null")); + DwModifier record = this.dwModifierMapper.selectById(id); + PreconditionUtil.checkState(!Objects.isNull(record), DwException.stateReject("modifier not found")); + + String username = SecurityFilter.getLoginUsername(request); + boolean inUse = isModifierInUse(record.getId(), username); + PreconditionUtil.checkState(!inUse, DwException.stateReject("modifier is in use, id = {}, name = {}", record.getId(), record.getModifierType())); + +// if (Objects.equals(Boolean.FALSE, record.getStatus())) { +// return Message.ok(); +// } +// record.setStatus(Boolean.FALSE); +// int i = this.dwModifierMapper.updateById(record); + int i = this.dwModifierMapper.deleteById(record); + PreconditionUtil.checkState(1 == i, DwException.stateReject("remove action failed")); + return Message.ok(); + } + + @Transactional(rollbackFor = Exception.class) + @Override + public Message update(HttpServletRequest request, DwModifierUpdateCommand command) throws DwException { + Long id = command.getId(); + Long themeDomainId = command.getThemeDomainId(); + Long layerId = command.getLayerId(); + String typeName = command.getTypeName(); + String typeEnName = command.getTypeEnName(); + String description = command.getDescription(); + + PreconditionUtil.checkArgument(!Objects.isNull(id), DwException.argumentReject("id should not empty")); + PreconditionUtil.checkArgument(!Objects.isNull(themeDomainId), DwException.argumentReject("theme domain id should not empty")); + PreconditionUtil.checkArgument(!Objects.isNull(layerId), DwException.argumentReject("layer id should not empty")); +// typeName = PreconditionUtil.checkStringArgumentNotBlankTrim(typeName, DwException.argumentReject("modifier type name should not empty")); + typeName = PreconditionUtil.checkStringArgumentNotBlankTrim(typeName, DwException.argumentReject("type name should not empty")); + PreconditionUtil.checkArgument(RegexUtil.checkCnName(typeName), DwException.argumentReject("type name must be digitg, chinese and underline")); + + typeEnName = PreconditionUtil.checkStringArgumentNotBlankTrim(typeEnName, DwException.argumentReject("type en name should not empty")); + PreconditionUtil.checkArgument(RegexUtil.checkEnName(typeEnName), DwException.argumentReject("type en name must be digitg, alpha and underline")); + // 验证 主题域 分层是否存在 + QueryWrapper layerQueryWrapper = new QueryWrapper<>(); + layerQueryWrapper.eq("id", layerId); + layerQueryWrapper.eq("status", Boolean.TRUE); + DwLayer dwLayer = this.dwLayerMapper.selectOne(layerQueryWrapper); + PreconditionUtil.checkState(!Objects.isNull(dwLayer), DwException.stateReject("layer not found")); + PreconditionUtil.checkState(dwLayer.getIsAvailable(), DwException.stateReject("layer disabled")); + + QueryWrapper themeDomainQueryWrapper = new QueryWrapper<>(); + themeDomainQueryWrapper.eq("id", themeDomainId); + themeDomainQueryWrapper.eq("status", Boolean.TRUE); + DwThemeDomain dwThemeDomain = this.dwThemeDomainMapper.selectOne(themeDomainQueryWrapper); + PreconditionUtil.checkState(!Objects.isNull(dwThemeDomain), DwException.stateReject("theme domain not found")); + + // 实体校验 + DwModifier record = this.dwModifierMapper.selectById(id); + PreconditionUtil.checkState(!Objects.isNull(record), DwException.stateReject("modifier not found")); +// String orgName = record.getModifierTypeEn(); + + // typeName 唯一性检测 + QueryWrapper nameUniqueCheckQuery = new QueryWrapper<>(); + nameUniqueCheckQuery.eq("modifier_type", typeName); + nameUniqueCheckQuery.ne("id", id); + nameUniqueCheckQuery.eq("status", Boolean.TRUE); + DwModifier exist = this.dwModifierMapper.selectOne(nameUniqueCheckQuery); + PreconditionUtil.checkState(Objects.isNull(exist), DwException.stateReject("modifier type name aleardy exists")); + // typeNameAlias 唯一性检测 + QueryWrapper nameAliasUniqueCheckQuery = new QueryWrapper<>(); + nameAliasUniqueCheckQuery.eq("modifier_type_en", typeEnName); + nameAliasUniqueCheckQuery.ne("id", id); + nameAliasUniqueCheckQuery.eq("status", Boolean.TRUE); + DwModifier exist2 = this.dwModifierMapper.selectOne(nameAliasUniqueCheckQuery); + PreconditionUtil.checkState(Objects.isNull(exist2), DwException.stateReject("modifier type en name aleardy exists")); + + Long oldVersion = record.getLockVersion(); + + Date now = new Date(); + +// record.setSubjectDomainId(dwThemeDomain.getId()); +// record.setLayerId(dwLayer.getId()); + record.setModifierType(typeName); + record.setThemeDomainId(dwThemeDomain.getId()); + record.setThemeArea(dwThemeDomain.getName()); + record.setLayerId(dwLayer.getId()); + record.setLayerArea(dwLayer.getName()); + record.setDescription(description); +// record.setModifyUser(user); + record.setUpdateTime(now); + record.setLockVersion(oldVersion + 1); + + UpdateWrapper updateWrapper = new UpdateWrapper<>(); + updateWrapper.eq("id", record.getId()); + updateWrapper.eq("lock_version", oldVersion); + + int i = this.dwModifierMapper.update(record, updateWrapper); + PreconditionUtil.checkState(1 == i, DwException.stateReject("update modifier failed")); + + // 更新修饰词列表信息 + List list = command.getList(); + if (!Objects.isNull(list) && !list.isEmpty()) { + // 如果不为空，先清空某个 modifier 下的关联修饰词 + QueryWrapper dwModifierListQueryWrapper = new QueryWrapper<>(); + dwModifierListQueryWrapper.eq("modifier_id", record.getId()); + this.dwModifierListMapper.delete(dwModifierListQueryWrapper); + + // 删除后再新增 + for (DwModifierUpdateCommand.DwModifierUpdateListItem item : list) { + String name = item.getName(); + String identifier = item.getIdentifier(); + String formula = item.getFormula(); + if (Strings.isBlank(name) || Strings.isBlank(identifier) || Strings.isBlank(formula)) { + continue; + } + + DwModifierList m = new DwModifierList(); + m.setModifierId(record.getId()); + m.setName(name); + m.setIdentifier(identifier); + m.setFormula(formula); + m.setCreateTime(now); + m.setUpdateTime(now); + this.dwModifierListMapper.insert(m); + } + } else { + QueryWrapper dwModifierListQueryWrapper = new QueryWrapper<>(); + dwModifierListQueryWrapper.eq("modifier_id", record.getId()); + this.dwModifierListMapper.delete(dwModifierListQueryWrapper); + } + + + return Message.ok(); + } + + @Transactional(rollbackFor = Exception.class) + @Override + public Message enable(HttpServletRequest request, Long id) throws DwException { + changeEnable(request, id, Boolean.TRUE); + return Message.ok(); + } + + @Transactional(rollbackFor = Exception.class) + @Override + public Message disable(HttpServletRequest request, Long id) throws DwException { + changeEnable(request, id, Boolean.FALSE); + return Message.ok(); + } + + private void changeEnable(HttpServletRequest request, Long id, Boolean enabled) throws DwException { + PreconditionUtil.checkArgument(!Objects.isNull(id), DwException.argumentReject("id should not be null")); + + DwModifier record = this.dwModifierMapper.selectById(id); + PreconditionUtil.checkState(!Objects.isNull(record), DwException.stateReject("modifier not found")); + PreconditionUtil.checkState(record.getStatus(), DwException.stateReject("modifier has been removed")); + if (Objects.equals(enabled, record.getIsAvailable())) { + return; + } + + Long oldVersion = record.getLockVersion(); + record.setUpdateTime(new Date()); + record.setIsAvailable(enabled); + record.setLockVersion(oldVersion + 1); + UpdateWrapper updateWrapper = new UpdateWrapper<>(); + updateWrapper.eq("id", record.getId()); + updateWrapper.eq("lock_version", oldVersion); + int update = this.dwModifierMapper.update(record, updateWrapper); + PreconditionUtil.checkState(1 == update, DwException.stateReject(enabled ? "enable" : "disable" + " failed")); + } +} diff --git a/dss-apps/dss-datawarehouse-design/data-warehouse-service/src/main/java/com/webank/wedatasphere/warehouse/service/impl/DwStatisticalPeriodServiceImpl.java b/dss-apps/dss-datawarehouse-design/data-warehouse-service/src/main/java/com/webank/wedatasphere/warehouse/service/impl/DwStatisticalPeriodServiceImpl.java new file mode 100644 index 0000000000..7a36ee0899 --- /dev/null +++ b/dss-apps/dss-datawarehouse-design/data-warehouse-service/src/main/java/com/webank/wedatasphere/warehouse/service/impl/DwStatisticalPeriodServiceImpl.java @@ -0,0 +1,446 @@ +package com.webank.wedatasphere.warehouse.service.impl; + +import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; +import com.baomidou.mybatisplus.core.conditions.update.UpdateWrapper; +import com.baomidou.mybatisplus.extension.plugins.pagination.Page; +import org.apache.linkis.server.Message; +import org.apache.linkis.server.security.SecurityFilter; +import com.webank.wedatasphere.warehouse.cqe.DwStatisticalPeriodCreateCommand; +import com.webank.wedatasphere.warehouse.cqe.DwStatisticalPeriodQueryCommand; +import com.webank.wedatasphere.warehouse.cqe.DwStatisticalPeriodUpdateCommand; +import com.webank.wedatasphere.warehouse.dao.domain.DwLayer; +import com.webank.wedatasphere.warehouse.dao.domain.DwStatisticalPeriod; +import com.webank.wedatasphere.warehouse.dao.domain.DwThemeDomain; +import com.webank.wedatasphere.warehouse.dao.mapper.DwLayerMapper; +import com.webank.wedatasphere.warehouse.dao.mapper.DwModifierMapper; +import com.webank.wedatasphere.warehouse.dao.mapper.DwStatisticalPeriodMapper; +import com.webank.wedatasphere.warehouse.dao.mapper.DwThemeDomainMapper; +import com.webank.wedatasphere.warehouse.dao.vo.DwStatisticalPeriodVo; +import com.webank.wedatasphere.warehouse.dto.PageInfo; +import com.webank.wedatasphere.warehouse.exception.DwException; +import com.webank.wedatasphere.warehouse.service.DwDomainReferenceAdapter; +import com.webank.wedatasphere.warehouse.service.DwStatisticalPeriodService; +import com.webank.wedatasphere.warehouse.utils.PreconditionUtil; +import com.webank.wedatasphere.warehouse.utils.RegexUtil; +import org.apache.logging.log4j.util.Strings; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +import javax.servlet.http.HttpServletRequest; +import java.util.*; + +@Service +public class DwStatisticalPeriodServiceImpl implements DwStatisticalPeriodService, DwDomainReferenceAdapter { + + private final DwLayerMapper dwLayerMapper; + private final DwThemeDomainMapper dwThemeDomainMapper; + private final DwStatisticalPeriodMapper dwStatisticalPeriodMapper; + private final DwModifierMapper dwModifierMapper; + + @Autowired + public DwStatisticalPeriodServiceImpl( + final DwStatisticalPeriodMapper dwStatisticalPeriodMapper, + final DwLayerMapper dwLayerMapper, + final DwThemeDomainMapper dwThemeDomainMapper, + final DwModifierMapper dwModifierMapper + + ) { + this.dwStatisticalPeriodMapper = dwStatisticalPeriodMapper; + this.dwLayerMapper = dwLayerMapper; + this.dwThemeDomainMapper = dwThemeDomainMapper; + this.dwModifierMapper = dwModifierMapper; + } + + @Override + public DwLayerMapper getDwLayerMapper() { + return dwLayerMapper; + } + + @Override + public DwThemeDomainMapper getDwThemeDomainMapper() { + return dwThemeDomainMapper; + } + + @Override + public DwModifierMapper getDwModifierMapper() { + return dwModifierMapper; + } + + @Override + public DwStatisticalPeriodMapper getDwStatisticalPeriodMapper() { + return dwStatisticalPeriodMapper; + } + + @Override + public Message queryAll(HttpServletRequest request, DwStatisticalPeriodQueryCommand command) throws DwException { + String name = command.getName(); + Boolean enabled = command.getEnabled(); + String layer = command.getLayer(); + String theme = command.getTheme(); + + QueryWrapper queryWrapper = new QueryWrapper<>(); + queryWrapper.eq("status", Boolean.TRUE); + if (!Objects.isNull(enabled)) { + queryWrapper.eq("is_available", enabled); + } + + if (Strings.isNotBlank(layer)) { + Optional layerOptional = this.dwLayerMapper.findByName(layer); + layerOptional.ifPresent(dwLayer -> queryWrapper.eq("layer_id", dwLayer.getId())); + } + + if (Strings.isNotBlank(theme)) { + Optional themeOptional = this.dwThemeDomainMapper.findByName(theme); + themeOptional.ifPresent(dwThemeDomain -> queryWrapper.eq("theme_domain_id", dwThemeDomain.getId())); + } + + if (Strings.isNotBlank(name)) { + queryWrapper.and(qw -> { + qw.like("name", name).or().like("en_name", name); + }); + } + + List recs = this.dwStatisticalPeriodMapper.selectList(queryWrapper); + + List records = new ArrayList<>(); + for (DwStatisticalPeriod rec : recs) { + DwStatisticalPeriodVo vo = new DwStatisticalPeriodVo(); + vo.setId(rec.getId()); + vo.setName(rec.getName()); + vo.setEnName(rec.getEnName()); + vo.setDescription(rec.getDescription()); + vo.setOwner(rec.getOwner()); + vo.setStatus(rec.getStatus()); + vo.setCreateTime(rec.getCreateTime()); + vo.setUpdateTime(rec.getUpdateTime()); + vo.setPrincipalName(rec.getPrincipalName()); + vo.setStartTimeFormula(rec.getStartTimeFormula()); + vo.setEndTimeFormula(rec.getEndTimeFormula()); + vo.setLayerId(rec.getLayerId()); + vo.setThemeDomainId(rec.getThemeDomainId()); + vo.setIsAvailable(rec.getIsAvailable()); + DwLayer dwLayer = this.dwLayerMapper.selectById(rec.getLayerId()); + Optional.ofNullable(dwLayer).ifPresent(layerBean -> vo.setLayerArea(layerBean.getName())); + + DwThemeDomain dwThemeDomain = dwThemeDomainMapper.selectById(rec.getThemeDomainId()); + Optional.ofNullable(dwThemeDomain).ifPresent(themeBean -> vo.setThemeArea(themeBean.getName())); + + records.add(vo); + } + +// List records = this.dwStatisticalPeriodMapper.selectItems(queryWrapper); + return Message.ok().data("list", records); + } + + @Override + public Message queryPage(HttpServletRequest request, DwStatisticalPeriodQueryCommand command) throws DwException { + Integer page = command.getPage(); + Integer size = command.getSize(); + String name = command.getName(); + if (Objects.isNull(page)) + page = 1; + + if (Objects.isNull(size)) + size = 10; + + QueryWrapper queryWrapper = new QueryWrapper<>(); + queryWrapper.eq("status", Boolean.TRUE); + if (!Objects.isNull(command.getEnabled())) { + queryWrapper.eq("is_available", command.getEnabled()); + } + + if (Strings.isNotBlank(name)) { + queryWrapper.and(qw -> { + qw.like("name", name).or().like("en_name", name); + }); + } + + Page queryPage = new Page<>(page, size); + +// IPage _page = this.dwStatisticalPeriodMapper.selectPageItems(queryPage, queryWrapper); +// IPage _page = this.dwStatisticalPeriodMapper.selectPageItems(queryPage, params); + Page _page = this.dwStatisticalPeriodMapper.selectPage(queryPage, queryWrapper); + List recs = _page.getRecords(); + List records = new ArrayList<>(); + String username = SecurityFilter.getLoginUsername(request); + for (DwStatisticalPeriod rec : recs) { + DwStatisticalPeriodVo vo = new DwStatisticalPeriodVo(); + vo.setId(rec.getId()); + vo.setName(rec.getName()); + vo.setEnName(rec.getEnName()); + vo.setDescription(rec.getDescription()); + vo.setOwner(rec.getOwner()); + vo.setStatus(rec.getStatus()); + vo.setCreateTime(rec.getCreateTime()); + vo.setUpdateTime(rec.getUpdateTime()); + vo.setPrincipalName(rec.getPrincipalName()); + vo.setStartTimeFormula(rec.getStartTimeFormula()); + vo.setEndTimeFormula(rec.getEndTimeFormula()); + vo.setLayerId(rec.getLayerId()); + vo.setThemeDomainId(rec.getThemeDomainId()); + vo.setIsAvailable(rec.getIsAvailable()); + // 单独查询 + DwLayer dwLayer = this.dwLayerMapper.selectById(rec.getLayerId()); + Optional.ofNullable(dwLayer).ifPresent(layer -> { + vo.setLayerArea(layer.getName()); + vo.setLayerAreaEn(layer.getEnName()); + }); + DwThemeDomain dwThemeDomain = dwThemeDomainMapper.selectById(rec.getThemeDomainId()); + Optional.ofNullable(dwThemeDomain).ifPresent(theme -> { + vo.setThemeArea(theme.getName()); + vo.setThemeAreaEn(theme.getEnName()); + }); + int statisticalPeriodReferenceCount = getStatisticalPeriodReferenceCount(rec.getId(), username); + vo.setReferenceCount(statisticalPeriodReferenceCount); + records.add(vo); + } + + PageInfo __page = new PageInfo<>(records, _page.getCurrent(), _page.getSize(), _page.getTotal()); + + return Message.ok().data("page", __page); + } + + @Transactional(rollbackFor = Exception.class) + @Override + public Message create(HttpServletRequest request, DwStatisticalPeriodCreateCommand command) throws DwException { + String name = command.getName(); + String enName = command.getEnName(); + String principalName = command.getPrincipalName(); + String owner = command.getOwner(); + String description = command.getDescription(); + Long themeDomainId = command.getThemeDomainId(); + Long layerId = command.getLayerId(); + String statStartFormula = command.getStatStartFormula(); + String statEndFormula = command.getStatEndFormula(); + + name = PreconditionUtil.checkStringArgumentNotBlankTrim(name, DwException.argumentReject("name should not empty")); + PreconditionUtil.checkArgument(RegexUtil.checkCnName(name), DwException.argumentReject("name must be digitg, chinese and underline")); + enName = PreconditionUtil.checkStringArgumentNotBlankTrim(enName, DwException.argumentReject("name alias should not empty")); + PreconditionUtil.checkArgument(RegexUtil.checkEnName(enName), DwException.argumentReject("name must be digit, alpha and underline")); + owner = PreconditionUtil.checkStringArgumentNotBlankTrim(owner, DwException.argumentReject("owner should not empty")); + statStartFormula = PreconditionUtil.checkStringArgumentNotBlankTrim(statStartFormula, DwException.argumentReject("stat start formula should not empty")); + statEndFormula = PreconditionUtil.checkStringArgumentNotBlankTrim(statEndFormula, DwException.argumentReject("stat end formula should not empty")); + PreconditionUtil.checkState(!Objects.isNull(layerId), DwException.argumentReject("layer id not empty")); + PreconditionUtil.checkState(!Objects.isNull(themeDomainId), DwException.argumentReject("theme domain id not empty")); + + // 验证 主题域 分层是否存在 + QueryWrapper layerQueryWrapper = new QueryWrapper<>(); + layerQueryWrapper.eq("id", layerId); + layerQueryWrapper.eq("status", Boolean.TRUE); + DwLayer dwLayer = this.dwLayerMapper.selectOne(layerQueryWrapper); + PreconditionUtil.checkState(!Objects.isNull(dwLayer), DwException.stateReject("layer not found")); + PreconditionUtil.checkState(dwLayer.getIsAvailable(), DwException.stateReject("layer disabled")); + + QueryWrapper themeDomainQueryWrapper = new QueryWrapper<>(); + themeDomainQueryWrapper.eq("id", themeDomainId); + themeDomainQueryWrapper.eq("status", Boolean.TRUE); + DwThemeDomain dwThemeDomain = this.dwThemeDomainMapper.selectOne(themeDomainQueryWrapper); + PreconditionUtil.checkState(!Objects.isNull(dwThemeDomain), DwException.stateReject("theme domain not found")); + + Date now = new Date(); + + DwStatisticalPeriod record = new DwStatisticalPeriod(); + record.setName(name); + record.setEnName(enName); + record.setThemeDomainId(themeDomainId); + record.setLayerId(layerId); + record.setDescription(description); + record.setPrincipalName(principalName); + record.setStartTimeFormula(statStartFormula); + record.setEndTimeFormula(statEndFormula); + record.setOwner(owner); + record.setIsAvailable(Boolean.TRUE); + record.setCreateTime(now); + record.setUpdateTime(now); + record.setStatus(Boolean.TRUE); + record.setLockVersion(1L); + + int insert = this.dwStatisticalPeriodMapper.insert(record); + PreconditionUtil.checkState(1 == insert, DwException.stateReject("create statistical period failed")); + + return Message.ok().data("id", record.getId()); + } + + @Override + public Message getById(HttpServletRequest request, Long id) throws DwException { + PreconditionUtil.checkArgument(!Objects.isNull(id), DwException.argumentReject("id should not be null")); + + DwStatisticalPeriod period = this.dwStatisticalPeriodMapper.selectById(id); + PreconditionUtil.checkState(!Objects.isNull(period), DwException.stateReject("statistical period not found")); + PreconditionUtil.checkState(period.getStatus(), DwException.stateReject("statistical period has been removed")); + + DwStatisticalPeriodVo record = new DwStatisticalPeriodVo(); + record.setId(period.getId()); + record.setName(period.getName()); + record.setEnName(period.getEnName()); + record.setDescription(period.getDescription()); + record.setOwner(period.getOwner()); + record.setStatus(period.getStatus()); + record.setCreateTime(period.getCreateTime()); + record.setUpdateTime(period.getUpdateTime()); + record.setPrincipalName(period.getPrincipalName()); + record.setStartTimeFormula(period.getStartTimeFormula()); + record.setEndTimeFormula(period.getEndTimeFormula()); + record.setLayerId(period.getLayerId()); + record.setThemeDomainId(period.getThemeDomainId()); + record.setIsAvailable(period.getIsAvailable()); + // 单独查询 + Optional.ofNullable(period.getLayerId()).ifPresent(lid -> { + DwLayer dwLayer = this.dwLayerMapper.selectById(lid); + Optional.ofNullable(dwLayer).ifPresent(layer -> { + record.setLayerArea(layer.getName()); + }); + }); + + Optional.ofNullable(period.getThemeDomainId()).ifPresent(tid -> { + DwThemeDomain dwThemeDomain = dwThemeDomainMapper.selectById(tid); + Optional.ofNullable(dwThemeDomain).ifPresent(theme -> { + record.setThemeArea(theme.getName()); + }); + }); + // 应用查询 + String username = SecurityFilter.getLoginUsername(request); + boolean inUse = isStatisticalPeriodInUse(record.getId(), username); + record.setReferenced(inUse); + return Message.ok().data("item", record); + } + + @Transactional(rollbackFor = Exception.class) + @Override + public Message deleteById(HttpServletRequest request, Long id) throws DwException { + PreconditionUtil.checkArgument(!Objects.isNull(id), DwException.argumentReject("id should not be null")); + DwStatisticalPeriod record = this.dwStatisticalPeriodMapper.selectById(id); + PreconditionUtil.checkState(!Objects.isNull(record), DwException.stateReject("statistical period not found")); + + String username = SecurityFilter.getLoginUsername(request); + boolean inUse = isStatisticalPeriodInUse(record.getId(), username); + PreconditionUtil.checkState(!inUse, DwException.stateReject("statistical period is in use, id = {}, name = {}", record.getId(), record.getName())); + +// if (Objects.equals(Boolean.FALSE, record.getStatus())) { +// return Message.ok(); +// } +// Long oldLockVersion = record.getLockVersion(); +// QueryWrapper updateWrapper = new QueryWrapper<>(); +// updateWrapper.eq("lock_version", oldLockVersion); +// updateWrapper.eq("id", record.getId()); +// +// DwStatisticalPeriod updateBean = new DwStatisticalPeriod(); +// updateBean.setLockVersion(oldLockVersion + 1); +// updateBean.setStatus(Boolean.FALSE); +// +// int i = this.dwStatisticalPeriodMapper.update(updateBean, updateWrapper); + int i = this.dwStatisticalPeriodMapper.deleteById(record); + PreconditionUtil.checkState(1 == i, DwException.stateReject("remove action failed")); + return Message.ok(); + } + + @Transactional(rollbackFor = Exception.class) + @Override + public Message update(HttpServletRequest request, DwStatisticalPeriodUpdateCommand command) throws DwException { + Long id = command.getId(); + String name = command.getName(); + String enName = command.getEnName(); + String principalName = command.getPrincipalName(); + String owner = command.getOwner(); + String description = command.getDescription(); + Long themeDomainId = command.getThemeDomainId(); + Long layerId = command.getLayerId(); + String statStartFormula = command.getStatStartFormula(); + String statEndFormula = command.getStatEndFormula(); + + PreconditionUtil.checkState(!Objects.isNull(id), DwException.argumentReject("id not empty")); + name = PreconditionUtil.checkStringArgumentNotBlankTrim(name, DwException.argumentReject("name should not empty")); + PreconditionUtil.checkArgument(RegexUtil.checkCnName(name), DwException.argumentReject("name must be digitg, chinese and underline")); + enName = PreconditionUtil.checkStringArgumentNotBlankTrim(enName, DwException.argumentReject("name alias should not empty")); + PreconditionUtil.checkArgument(RegexUtil.checkEnName(enName), DwException.argumentReject("name must be digit, alpha and underline")); + owner = PreconditionUtil.checkStringArgumentNotBlankTrim(owner, DwException.argumentReject("owner should not empty")); + statStartFormula = PreconditionUtil.checkStringArgumentNotBlankTrim(statStartFormula, DwException.argumentReject("stat start formula should not empty")); + statEndFormula = PreconditionUtil.checkStringArgumentNotBlankTrim(statEndFormula, DwException.argumentReject("stat end formula should not empty")); + PreconditionUtil.checkState(!Objects.isNull(layerId), DwException.argumentReject("layer id not empty")); + PreconditionUtil.checkState(!Objects.isNull(themeDomainId), DwException.argumentReject("theme domain id not empty")); + + QueryWrapper statisticalPeriodQueryWrapper = new QueryWrapper<>(); + statisticalPeriodQueryWrapper.eq("id", id); + statisticalPeriodQueryWrapper.eq("status", Boolean.TRUE); + DwStatisticalPeriod record = this.dwStatisticalPeriodMapper.selectOne(statisticalPeriodQueryWrapper); + PreconditionUtil.checkState(!Objects.isNull(record), DwException.stateReject("period not found")); + PreconditionUtil.checkState(record.getIsAvailable(), DwException.stateReject("period disabled")); + + // 验证 主题域 分层是否存在 + QueryWrapper layerQueryWrapper = new QueryWrapper<>(); + layerQueryWrapper.eq("id", layerId); + layerQueryWrapper.eq("status", Boolean.TRUE); + DwLayer dwLayer = this.dwLayerMapper.selectOne(layerQueryWrapper); + PreconditionUtil.checkState(!Objects.isNull(dwLayer), DwException.stateReject("layer not found")); + PreconditionUtil.checkState(dwLayer.getIsAvailable(), DwException.stateReject("layer disabled")); + + QueryWrapper themeDomainQueryWrapper = new QueryWrapper<>(); + themeDomainQueryWrapper.eq("id", themeDomainId); + themeDomainQueryWrapper.eq("status", Boolean.TRUE); + DwThemeDomain dwThemeDomain = this.dwThemeDomainMapper.selectOne(themeDomainQueryWrapper); + PreconditionUtil.checkState(!Objects.isNull(dwThemeDomain), DwException.stateReject("theme domain not found")); + + Date now = new Date(); + Long oldLockVersion = record.getLockVersion(); + + QueryWrapper updateWrapper = new QueryWrapper<>(); + updateWrapper.eq("lock_version", oldLockVersion); + updateWrapper.eq("id", record.getId()); + + record.setName(name); + record.setEnName(enName); + record.setThemeDomainId(themeDomainId); + record.setLayerId(layerId); + record.setDescription(description); + record.setPrincipalName(principalName); + record.setStartTimeFormula(statStartFormula); + record.setEndTimeFormula(statEndFormula); + record.setOwner(owner); + record.setUpdateTime(now); + record.setLockVersion(oldLockVersion + 1); + + int update = this.dwStatisticalPeriodMapper.update(record, updateWrapper); + PreconditionUtil.checkState(1 == update, DwException.stateReject("update statistical period failed")); + + return Message.ok(); + } + + @Transactional(rollbackFor = Exception.class) + @Override + public Message enable(HttpServletRequest request, Long id) throws DwException { + changeEnable(request, id, Boolean.TRUE); + return Message.ok(); + } + + @Transactional(rollbackFor = Exception.class) + @Override + public Message disable(HttpServletRequest request, Long id) throws DwException { + changeEnable(request, id, Boolean.FALSE); + return Message.ok(); + } + + private void changeEnable(HttpServletRequest request, Long id, Boolean enabled) throws DwException { + PreconditionUtil.checkArgument(!Objects.isNull(id), DwException.argumentReject("id should not be null")); + + DwStatisticalPeriod record = this.dwStatisticalPeriodMapper.selectById(id); + PreconditionUtil.checkState(!Objects.isNull(record), DwException.stateReject("statistical period not found")); + PreconditionUtil.checkState(record.getStatus(), DwException.stateReject("statistical period has been removed")); + if (Objects.equals(enabled, record.getIsAvailable())) { + return; + } + + Long oldVersion = record.getLockVersion(); + UpdateWrapper updateWrapper = new UpdateWrapper<>(); + updateWrapper.eq("id", record.getId()); + updateWrapper.eq("lock_version", oldVersion); + + DwStatisticalPeriod updateBean = new DwStatisticalPeriod(); + updateBean.setIsAvailable(enabled); + updateBean.setUpdateTime(new Date()); + updateBean.setLockVersion(oldVersion + 1); + + int update = this.dwStatisticalPeriodMapper.update(updateBean, updateWrapper); + PreconditionUtil.checkState(1 == update, DwException.stateReject(enabled ? "enable" : "disable" + " failed")); + } +} diff --git a/dss-apps/dss-datawarehouse-design/data-warehouse-service/src/main/java/com/webank/wedatasphere/warehouse/service/impl/DwThemeDomainServiceImpl.java b/dss-apps/dss-datawarehouse-design/data-warehouse-service/src/main/java/com/webank/wedatasphere/warehouse/service/impl/DwThemeDomainServiceImpl.java new file mode 100644 index 0000000000..e613e416ee --- /dev/null +++ b/dss-apps/dss-datawarehouse-design/data-warehouse-service/src/main/java/com/webank/wedatasphere/warehouse/service/impl/DwThemeDomainServiceImpl.java @@ -0,0 +1,431 @@ +package com.webank.wedatasphere.warehouse.service.impl; + +import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; +import com.baomidou.mybatisplus.core.conditions.update.UpdateWrapper; +import com.baomidou.mybatisplus.core.metadata.IPage; +import com.baomidou.mybatisplus.extension.plugins.pagination.Page; +import com.webank.wedatasphere.dss.data.governance.entity.ClassificationConstant; +import com.webank.wedatasphere.dss.data.governance.impl.LinkisDataAssetsRemoteClient; +import com.webank.wedatasphere.dss.data.governance.request.CreateModelTypeAction; +import com.webank.wedatasphere.dss.data.governance.request.DeleteModelTypeAction; +import com.webank.wedatasphere.dss.data.governance.request.UpdateModelTypeAction; +import com.webank.wedatasphere.dss.data.governance.response.CreateModelTypeResult; +import com.webank.wedatasphere.dss.data.governance.response.DeleteModelTypeResult; +import com.webank.wedatasphere.dss.data.governance.response.UpdateModelTypeResult; +import org.apache.linkis.common.exception.ErrorException; +import org.apache.linkis.server.Message; +import org.apache.linkis.server.security.SecurityFilter; +import com.webank.wedatasphere.warehouse.LinkisRemoteClientHolder; +import com.webank.wedatasphere.warehouse.cqe.DwThemeDomainCreateCommand; +import com.webank.wedatasphere.warehouse.cqe.DwThemeDomainQueryCommand; +import com.webank.wedatasphere.warehouse.cqe.DwThemeDomainUpdateCommand; +import com.webank.wedatasphere.warehouse.dao.domain.DwThemeDomain; +import com.webank.wedatasphere.warehouse.dao.mapper.DwLayerMapper; +import com.webank.wedatasphere.warehouse.dao.mapper.DwModifierMapper; +import com.webank.wedatasphere.warehouse.dao.mapper.DwStatisticalPeriodMapper; +import com.webank.wedatasphere.warehouse.dao.mapper.DwThemeDomainMapper; +import com.webank.wedatasphere.warehouse.dto.DwThemeDomainDTO; +import com.webank.wedatasphere.warehouse.dto.DwThemeDomainListItemDTO; +import com.webank.wedatasphere.warehouse.dto.PageInfo; +import com.webank.wedatasphere.warehouse.exception.DwException; +import com.webank.wedatasphere.warehouse.exception.DwExceptionCode; +import com.webank.wedatasphere.warehouse.service.DwDomainReferenceAdapter; +import com.webank.wedatasphere.warehouse.service.DwThemeDomainService; +import com.webank.wedatasphere.warehouse.utils.PreconditionUtil; +import com.webank.wedatasphere.warehouse.utils.RegexUtil; +import org.apache.logging.log4j.util.Strings; +import org.springframework.beans.BeanUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +import javax.servlet.http.HttpServletRequest; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; +import java.util.Objects; + +@Service +public class DwThemeDomainServiceImpl implements DwThemeDomainService, DwDomainReferenceAdapter { + + private final DwLayerMapper dwLayerMapper; + private final DwThemeDomainMapper dwThemeDomainMapper; + private final DwModifierMapper dwModifierMapper; + private final DwStatisticalPeriodMapper dwStatisticalPeriodMapper; + + @Autowired + public DwThemeDomainServiceImpl(final DwLayerMapper dwLayerMapper, final DwThemeDomainMapper dwThemeDomainMapper, final DwModifierMapper dwModifierMapper, final DwStatisticalPeriodMapper dwStatisticalPeriodMapper) { + this.dwLayerMapper = dwLayerMapper; + this.dwThemeDomainMapper = dwThemeDomainMapper; + this.dwModifierMapper = dwModifierMapper; + this.dwStatisticalPeriodMapper = dwStatisticalPeriodMapper; + } + + @Override + public DwLayerMapper getDwLayerMapper() { + return dwLayerMapper; + } + + @Override + public DwThemeDomainMapper getDwThemeDomainMapper() { + return dwThemeDomainMapper; + } + + @Override + public DwModifierMapper getDwModifierMapper() { + return dwModifierMapper; + } + + @Override + public DwStatisticalPeriodMapper getDwStatisticalPeriodMapper() { + return dwStatisticalPeriodMapper; + } + + @Override + public Message queryAllThemeDomains(HttpServletRequest request, DwThemeDomainQueryCommand command) throws DwException { + String name = command.getName(); + Boolean enabled = command.getEnabled(); + + QueryWrapper queryWrapper = new QueryWrapper<>(); + queryWrapper.eq("status", Boolean.TRUE); + if (!Objects.isNull(enabled)) { + queryWrapper.eq("is_available", enabled); + } + if (Strings.isNotBlank(name)) { + queryWrapper.and(qw -> { + qw.like("name", name).or().like("en_name", name); + }); + } + + List records = this.dwThemeDomainMapper.selectList(queryWrapper); + + List list = new ArrayList<>(); + for (DwThemeDomain domain : records) { + DwThemeDomainListItemDTO dwThemeDomainListItemDTO = new DwThemeDomainListItemDTO(); + BeanUtils.copyProperties(domain, dwThemeDomainListItemDTO); + list.add(dwThemeDomainListItemDTO); + } + + return Message.ok().data("list", list); + } + + @Transactional(rollbackFor = Exception.class) + @Override + public Message queryPage(HttpServletRequest request, DwThemeDomainQueryCommand command) throws DwException { + Integer page = command.getPage(); + Integer size = command.getSize(); + String name = command.getName(); + Boolean enabled = command.getEnabled(); + if (Objects.isNull(page)) + page = 1; + + if (Objects.isNull(size)) + size = 10; + + QueryWrapper queryWrapper = new QueryWrapper<>(); + queryWrapper.eq("status", Boolean.TRUE); + if (Objects.nonNull(enabled)) { + queryWrapper.eq("is_available", enabled); + } + if (Strings.isNotBlank(name)) { + queryWrapper.and(qw -> { + qw.like("name", name).or().like("en_name", name); + }); + } + Page queryPage = new Page<>(page, size); + + IPage _page = this.dwThemeDomainMapper.selectPage(queryPage, queryWrapper); + + List records = _page.getRecords(); + List list = new ArrayList<>(); + String username = SecurityFilter.getLoginUsername(request); + for (DwThemeDomain domain : records) { + DwThemeDomainListItemDTO dwThemeDomainListItemDTO = new DwThemeDomainListItemDTO();/*= this.dwThemeDomainModelMapper.toListItem(domain)*/; + BeanUtils.copyProperties(domain, dwThemeDomainListItemDTO); + + int themeDomainReferenceCount = getThemeDomainReferenceCount(domain.getId(), username); + dwThemeDomainListItemDTO.setReferenceCount(themeDomainReferenceCount); + + list.add(dwThemeDomainListItemDTO); + } + + PageInfo __page = new PageInfo<>(list, _page.getCurrent(), _page.getSize(), _page.getTotal()); + + return Message.ok().data("page", __page); + } + + @Transactional(rollbackFor = Exception.class) + @Override + public Message create(HttpServletRequest request, DwThemeDomainCreateCommand command) throws DwException { + String username = SecurityFilter.getLoginUsername(request); +// String username = "hdfs"; + String name = command.getName(); + String enName = command.getEnName(); + String principalName = command.getPrincipalName(); + String owner = command.getOwner(); + Integer sort = command.getSort(); + String description = command.getDescription(); +// String authority = command.getAuthority(); + +// name = PreconditionUtil.checkStringArgumentNotBlankTrim(name, DwException.argumentReject("name should not empty")); +// enName = PreconditionUtil.checkStringArgumentNotBlankTrim(enName, DwException.argumentReject("en name should not empty")); + name = PreconditionUtil.checkStringArgumentNotBlankTrim(name, DwException.argumentReject("name should not empty")); + PreconditionUtil.checkArgument(RegexUtil.checkCnName(name), DwException.argumentReject("name must be digitg, chinese and underline")); + enName = PreconditionUtil.checkStringArgumentNotBlankTrim(enName, DwException.argumentReject("name alias should not empty")); + PreconditionUtil.checkArgument(RegexUtil.checkEnName(enName), DwException.argumentReject("name must be digit, alpha and underline")); + owner = PreconditionUtil.checkStringArgumentNotBlankTrim(owner, DwException.argumentReject("owner should not empty")); +// authority = PreconditionUtil.checkStringArgumentNotBlankTrim(authority, DwException.argumentReject("authority should not empty")); + + if (Objects.isNull(sort)) { + sort = 1; + } + +// if (Strings.isBlank(availableRoles)) { +// availableRoles = "ALL"; +// } + +// String user = "hdfs"; + + Date now = new Date(); + + DwThemeDomain record = new DwThemeDomain(); + record.setName(name); + record.setEnName(enName); + record.setDescription(description); + record.setPrincipalName(principalName); +// record.setAvailableRoles(availableRoles); + record.setOwner(owner); + record.setIsAvailable(Boolean.TRUE); +// record.setCreateUser(user); + record.setSort(sort); + record.setCreateTime(now); + record.setUpdateTime(now); + record.setStatus(Boolean.TRUE); + record.setLockVersion(1L); + + int insert = this.dwThemeDomainMapper.insert(record); + PreconditionUtil.checkState(1 == insert, DwException.stateReject("create dw theme domain failed")); + + // 建立关联 + try { + LinkisDataAssetsRemoteClient dataAssetsRemoteClient = LinkisRemoteClientHolder.getDataAssetsRemoteClient(); + CreateModelTypeAction action = new CreateModelTypeAction.Builder().setType(ClassificationConstant.THEME).setName(enName).setUser(username).build(); + CreateModelTypeResult result = dataAssetsRemoteClient.createModelType(action); + if (result.getStatus() != 0) { + throw new DwException(result.getStatus(), result.getMessage()); + } + } catch (Exception e) { + if (e instanceof ErrorException) { + ErrorException ee = (ErrorException) e; + throw new DwException(DwExceptionCode.CREATE_MODEL_TYPE_ERROR.getCode(), e.getMessage(), ee.getIp(), ee.getPort(), ee.getServiceKind()); + } else { + throw new DwException(DwExceptionCode.CREATE_MODEL_TYPE_ERROR.getCode(), e.getMessage()); + } + } + + return Message.ok().data("id", record.getId()); + } + + @Override + public Message getById(HttpServletRequest request, Long id) throws DwException { + PreconditionUtil.checkArgument(!Objects.isNull(id), DwException.argumentReject("id should not be null")); + + DwThemeDomain record = this.dwThemeDomainMapper.selectById(id); + PreconditionUtil.checkState(!Objects.isNull(record), DwException.stateReject("theme domain not found")); + PreconditionUtil.checkState(record.getStatus(), DwException.stateReject("theme domain has been removed")); + +// DwThemeDomainDTO dto = this.dwThemeDomainModelMapper.toDTO(record); + String username = SecurityFilter.getLoginUsername(request); + boolean inUse = isThemeDomainInUse(record.getId(), username); + DwThemeDomainDTO dto = new DwThemeDomainDTO(); + BeanUtils.copyProperties(record, dto); + dto.setReferenced(inUse); + + return Message.ok().data("item", dto); + } + + @Transactional(rollbackFor = Exception.class) + @Override + public Message deleteById(HttpServletRequest request, Long id) throws DwException { + PreconditionUtil.checkArgument(!Objects.isNull(id), DwException.argumentReject("id should not be null")); + DwThemeDomain record = this.dwThemeDomainMapper.selectById(id); + PreconditionUtil.checkState(!Objects.isNull(record), DwException.stateReject("theme domain not found")); + + String username = SecurityFilter.getLoginUsername(request); + + // check in use + // statistical_period & modifier & data_model client api check + boolean inUse = isThemeDomainInUse(record.getId(), username); + PreconditionUtil.checkState(!inUse, DwException.stateReject("theme domain is in use")); + +// if (Objects.equals(Boolean.FALSE, record.getStatus())) { +// return Message.ok(); +// } +// record.setStatus(Boolean.FALSE); +// int i = this.dwThemeDomainMapper.updateById(record); + // physical delete, because of name unique + int i = this.dwThemeDomainMapper.deleteById(record); + PreconditionUtil.checkState(1 == i, DwException.stateReject("remove action failed")); + + // 删除关联 + try { + LinkisDataAssetsRemoteClient dataAssetsRemoteClient = LinkisRemoteClientHolder.getDataAssetsRemoteClient(); + DeleteModelTypeAction action = new DeleteModelTypeAction.Builder().setType(ClassificationConstant.THEME).setName(record.getEnName()).setUser(username).build(); + DeleteModelTypeResult result = dataAssetsRemoteClient.deleteModelType(action); + + if (result.getStatus() != 0) { + throw new DwException(result.getStatus(), result.getMessage()); + } + } catch (Exception e) { + if (e instanceof ErrorException) { + ErrorException ee = (ErrorException) e; + throw new DwException(DwExceptionCode.DELETE_MODEL_TYPE_ERROR.getCode(), e.getMessage(), ee.getIp(), ee.getPort(), ee.getServiceKind()); + } else { + throw new DwException(DwExceptionCode.DELETE_MODEL_TYPE_ERROR.getCode(), e.getMessage()); + } + } + + return Message.ok(); + } + + @Transactional(rollbackFor = Exception.class) + @Override + public Message update(HttpServletRequest request, DwThemeDomainUpdateCommand command) throws DwException { + String username = SecurityFilter.getLoginUsername(request); + // 基本参数校验 + Long id = command.getId(); + String name = command.getName(); + String enName = command.getEnName(); + String owner = command.getOwner(); + Integer sort = command.getSort(); + String principalName = command.getPrincipalName(); +// String authority = command.getAuthority(); + String description = command.getDescription(); + PreconditionUtil.checkArgument(!Objects.isNull(id), DwException.argumentReject("id should not be null")); +// name = PreconditionUtil.checkStringArgumentNotBlankTrim(name, DwException.argumentReject("name should not empty")); +// enName = PreconditionUtil.checkStringArgumentNotBlankTrim(enName, DwException.argumentReject("en name should not empty")); + name = PreconditionUtil.checkStringArgumentNotBlankTrim(name, DwException.argumentReject("name should not empty")); + PreconditionUtil.checkArgument(RegexUtil.checkCnName(name), DwException.argumentReject("name must be digitg, chinese and underline")); + enName = PreconditionUtil.checkStringArgumentNotBlankTrim(enName, DwException.argumentReject("name alias should not empty")); + PreconditionUtil.checkArgument(RegexUtil.checkEnName(enName), DwException.argumentReject("name must be digit, alpha and underline")); + owner = PreconditionUtil.checkStringArgumentNotBlankTrim(owner, DwException.argumentReject("owner should not empty")); +// authority = PreconditionUtil.checkStringArgumentNotBlankTrim(authority, DwException.argumentReject("authority should not empty")); + +// if (Strings.isBlank(principalName)) { +// principalName = "ALL"; +// } + + if (Objects.isNull(sort)) { + sort = 1; + } + + // 实体校验 + DwThemeDomain record = this.dwThemeDomainMapper.selectById(id); + PreconditionUtil.checkState(!Objects.isNull(record), DwException.stateReject("theme domain not found")); + PreconditionUtil.checkState(record.getIsAvailable(), DwException.stateReject("theme domain is unAvailable")); + + String orgName = record.getEnName(); + + // if name enName not equal to database attribute, we should check domain if in use + if (!Objects.equals(name, record.getName()) || !Objects.equals(enName, record.getEnName())) { + // check in use + // statistical_period & modifier & data_model client api check + + boolean inUse = isThemeDomainInUse(record.getId(), username); + PreconditionUtil.checkState(!inUse, DwException.stateReject("theme domain is in use")); + } + + // name 唯一性检测 + QueryWrapper nameUniqueCheckQuery = new QueryWrapper<>(); + nameUniqueCheckQuery.eq("name", name); + nameUniqueCheckQuery.ne("id", id); + nameUniqueCheckQuery.eq("status", Boolean.TRUE); + DwThemeDomain exist = this.dwThemeDomainMapper.selectOne(nameUniqueCheckQuery); + PreconditionUtil.checkState(Objects.isNull(exist), DwException.stateReject("theme domain name aleardy exists")); + + // name alias 唯一性检测 + QueryWrapper nameAliasUniqueCheckQuery = new QueryWrapper<>(); + nameAliasUniqueCheckQuery.eq("en_name", enName); + nameAliasUniqueCheckQuery.ne("id", id); + nameAliasUniqueCheckQuery.eq("status", Boolean.TRUE); + DwThemeDomain nameAliasExist = this.dwThemeDomainMapper.selectOne(nameUniqueCheckQuery); + PreconditionUtil.checkState(Objects.isNull(nameAliasExist), DwException.stateReject("theme domain en name aleardy exists")); + + Long oldVersion = record.getLockVersion(); + + Date now = new Date(); + record.setName(name); + record.setEnName(enName); + record.setDescription(description); + record.setPrincipalName(principalName); + record.setOwner(owner); + record.setSort(sort); + record.setUpdateTime(now); + record.setLockVersion(oldVersion + 1); + + UpdateWrapper updateWrapper = new UpdateWrapper<>(); + updateWrapper.eq("id", record.getId()); + updateWrapper.eq("lock_version", oldVersion); + + int i = this.dwThemeDomainMapper.update(record, updateWrapper); + + PreconditionUtil.checkState(1 == i, DwException.stateReject("update theme domain failed")); + + // 如果英文名称有改动，则更新关联 + if (!Objects.equals(orgName, record.getEnName())) { + try { + LinkisDataAssetsRemoteClient dataAssetsRemoteClient = LinkisRemoteClientHolder.getDataAssetsRemoteClient(); + UpdateModelTypeAction action = new UpdateModelTypeAction.Builder().setType(ClassificationConstant.THEME).setName(record.getEnName()).setOrgName(orgName).setUser(username).build(); + UpdateModelTypeResult result = dataAssetsRemoteClient.updateModelType(action); + + if (result.getStatus() != 0) { + throw new DwException(result.getStatus(), result.getMessage()); + } + } catch (Exception e) { + if (e instanceof ErrorException) { + ErrorException ee = (ErrorException) e; + throw new DwException(DwExceptionCode.UPDATE_MODEL_TYPE_ERROR.getCode(), e.getMessage(), ee.getIp(), ee.getPort(), ee.getServiceKind()); + } else { + throw new DwException(DwExceptionCode.UPDATE_MODEL_TYPE_ERROR.getCode(), e.getMessage()); + } + } + } + return Message.ok(); + } + + @Transactional(rollbackFor = Exception.class) + @Override + public Message enable(HttpServletRequest request, Long id) throws DwException { + changeEnable(request, id, Boolean.TRUE); + return Message.ok(); + } + + @Transactional(rollbackFor = Exception.class) + @Override + public Message disable(HttpServletRequest request, Long id) throws DwException { + changeEnable(request, id, Boolean.FALSE); + return Message.ok(); + } + + private void changeEnable(HttpServletRequest request, Long id, Boolean enabled) throws DwException { + PreconditionUtil.checkArgument(!Objects.isNull(id), DwException.argumentReject("id should not be null")); + + DwThemeDomain record = this.dwThemeDomainMapper.selectById(id); + PreconditionUtil.checkState(!Objects.isNull(record), DwException.stateReject("theme domain not found")); + PreconditionUtil.checkState(record.getStatus(), DwException.stateReject("theme domain has been removed")); + if (Objects.equals(enabled, record.getIsAvailable())) { + return; + } + + Long oldVersion = record.getLockVersion(); + record.setUpdateTime(new Date()); + record.setIsAvailable(enabled); + record.setLockVersion(oldVersion + 1); + UpdateWrapper updateWrapper = new UpdateWrapper<>(); + updateWrapper.eq("id", record.getId()); + updateWrapper.eq("lock_version", oldVersion); + int update = this.dwThemeDomainMapper.update(record, updateWrapper); + PreconditionUtil.checkState(1 == update, DwException.stateReject(enabled ? "enable" : "disable" + " failed")); + } + +} diff --git a/dss-apps/dss-datawarehouse-design/data-warehouse-service/src/main/java/com/webank/wedatasphere/warehouse/utils/PreconditionUtil.java b/dss-apps/dss-datawarehouse-design/data-warehouse-service/src/main/java/com/webank/wedatasphere/warehouse/utils/PreconditionUtil.java new file mode 100644 index 0000000000..6b96d9e5c1 --- /dev/null +++ b/dss-apps/dss-datawarehouse-design/data-warehouse-service/src/main/java/com/webank/wedatasphere/warehouse/utils/PreconditionUtil.java @@ -0,0 +1,84 @@ +package com.webank.wedatasphere.warehouse.utils; + +import com.google.common.base.Strings; +import com.webank.wedatasphere.warehouse.exception.DwException; + +import javax.annotation.Nullable; + +public class PreconditionUtil { + + private PreconditionUtil() {} + + public static void checkState(boolean condition, DwException e) throws DwException { + if (!condition) { + throw e; + } + } + + public static void checkState(boolean condition, @Nullable String errorMessageTemplate, @Nullable Object... errorMessageArgs) throws DwException { + if (!condition) { + throw new DwException(DwException.BUSINESS_ERROR, format(errorMessageTemplate, errorMessageArgs)); + } + } + + public static void checkArgument(boolean condition, DwException e) throws DwException { + if (!condition) { + throw e; + } + } + + public static void checkArgument(boolean condition, @Nullable String errorMessageTemplate, @Nullable Object... errorMessageArgs) throws DwException { + if (!condition) { + throw new DwException(DwException.BUSINESS_ERROR, format(errorMessageTemplate, errorMessageArgs)); + } + } + + public static void checkStringArgumentNotBlank(String argument, DwException e) throws DwException { + if (Strings.isNullOrEmpty(argument)) { + throw e; + } + } + + // 检查字符串参数，成功则返回 trim() 后的字符串，否则抛出异常 + public static String checkStringArgumentNotBlankTrim(String argument, DwException e) throws DwException { + if (Strings.isNullOrEmpty(argument)) { + throw e; + } + return argument.trim(); + } + + private static String format(@Nullable String template, @Nullable Object... args) throws DwException { + int numArgs = args == null ? 0 : args.length; + template = String.valueOf(template); + StringBuilder builder = new StringBuilder(template.length() + 16 * numArgs); + int templateStart = 0; + + int i; + int placeholderStart; + for(i = 0; i < numArgs; templateStart = placeholderStart + 2) { + placeholderStart = template.indexOf("%s", templateStart); + if (placeholderStart == -1) { + break; + } + + builder.append(template.substring(templateStart, placeholderStart)); + builder.append(args[i++]); + } + + builder.append(template.substring(templateStart)); + if (i < numArgs) { + builder.append(" ["); + builder.append(args[i++]); + + while(i < numArgs) { + builder.append(", "); + builder.append(args[i++]); + } + + builder.append(']'); + } + + return builder.toString(); + } + +} diff --git a/dss-apps/dss-datawarehouse-design/data-warehouse-service/src/main/java/com/webank/wedatasphere/warehouse/utils/RegexUtil.java b/dss-apps/dss-datawarehouse-design/data-warehouse-service/src/main/java/com/webank/wedatasphere/warehouse/utils/RegexUtil.java new file mode 100644 index 0000000000..2edaf7eddf --- /dev/null +++ b/dss-apps/dss-datawarehouse-design/data-warehouse-service/src/main/java/com/webank/wedatasphere/warehouse/utils/RegexUtil.java @@ -0,0 +1,30 @@ +package com.webank.wedatasphere.warehouse.utils; + +import java.util.Objects; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public class RegexUtil { + + private static final String REGEX_EN_NAME = "^\\w+$"; + private static final String REGEX_CN_NAME = "^[\\u4E00-\\u9FA50-9_]+$"; + private static final Pattern enNamePattern = Pattern.compile(REGEX_EN_NAME); + private static final Pattern cnNamePattern = Pattern.compile(REGEX_CN_NAME); + + public static boolean checkEnName(String source) { + if (Objects.isNull(source)) { + return false; + } + + return enNamePattern.matcher(source).find(); + } + + public static boolean checkCnName(String source) { + if (Objects.isNull(source)) { + return false; + } + + return cnNamePattern.matcher(source).find(); + } + +} diff --git a/dss-apps/dss-datawarehouse-design/data-warehouse-service/src/main/scala/com/webank/wedatasphere/warehouse/LinkisRemoteClientHolder.scala b/dss-apps/dss-datawarehouse-design/data-warehouse-service/src/main/scala/com/webank/wedatasphere/warehouse/LinkisRemoteClientHolder.scala new file mode 100644 index 0000000000..eae22a948a --- /dev/null +++ b/dss-apps/dss-datawarehouse-design/data-warehouse-service/src/main/scala/com/webank/wedatasphere/warehouse/LinkisRemoteClientHolder.scala @@ -0,0 +1,72 @@ +package com.webank.wedatasphere.warehouse + +import com.webank.wedatasphere.dss.data.governance.impl.LinkisDataAssetsRemoteClient +import com.webank.wedatasphere.dss.datamodel.center.client.impl.LinkisDatamodelCenterRemoteClient +import com.webank.wedatasphere.dss.framework.workspace.client.impl.LinkisWorkSpaceRemoteClient +import org.apache.linkis.datasource.client.impl.{LinkisDataSourceRemoteClient, LinkisMetadataSourceRemoteClient} +import org.apache.linkis.httpclient.dws.authentication.TokenAuthenticationStrategy +import org.apache.linkis.httpclient.dws.config.{DWSClientConfig, DWSClientConfigBuilder} +import com.webank.wedatasphere.warehouse.client.DwDataSourceConfiguration + +import java.lang +import java.util.concurrent.TimeUnit + +object LinkisRemoteClientHolder { + //Linkis Datasource Client Config + val serverUrl: String = DwDataSourceConfiguration.SERVER_URL.getValue + val connectionTimeout: lang.Long = DwDataSourceConfiguration.CONNECTION_TIMEOUT.getValue + val discoveryEnabled: lang.Boolean = DwDataSourceConfiguration.DISCOVERY_ENABLED.getValue + val discoveryFrequencyPeriod: lang.Long = DwDataSourceConfiguration.DISCOVERY_FREQUENCY_PERIOD.getValue + val loadbalancerEnabled: lang.Boolean = DwDataSourceConfiguration.LOAD_BALANCER_ENABLED.getValue + val maxConnectionSize: Integer = DwDataSourceConfiguration.MAX_CONNECTION_SIZE.getValue + val retryEnabled: lang.Boolean = DwDataSourceConfiguration.RETRY_ENABLED.getValue + val readTimeout: lang.Long = DwDataSourceConfiguration.READ_TIMEOUT.getValue + val authTokenKey: String = DwDataSourceConfiguration.AUTHTOKEN_KEY.getValue + val authTokenValue: String = DwDataSourceConfiguration.AUTHTOKEN_VALUE.getValue + val dwsVersion: String = DwDataSourceConfiguration.DWS_VERSION.getValue + + val clientConfig: DWSClientConfig = DWSClientConfigBuilder.newBuilder() + .addServerUrl(serverUrl) + .connectionTimeout(connectionTimeout) + .discoveryEnabled(discoveryEnabled) + .discoveryFrequency(discoveryFrequencyPeriod, TimeUnit.MINUTES) + .loadbalancerEnabled(loadbalancerEnabled) + .maxConnectionSize(maxConnectionSize) + .retryEnabled(retryEnabled) + .readTimeout(readTimeout) + .setAuthenticationStrategy(new TokenAuthenticationStrategy()) + .setAuthTokenKey(authTokenKey) + .setAuthTokenValue(authTokenValue) + .setDWSVersion(dwsVersion) + .build() + + val dataSourceClient = new LinkisDataSourceRemoteClient(clientConfig) + + val metadataSourceRemoteClient = new LinkisMetadataSourceRemoteClient(clientConfig) + + var dataModelRemoteClient = new LinkisDatamodelCenterRemoteClient(clientConfig) + + var dataAssetsRemoteClient = new LinkisDataAssetsRemoteClient(clientConfig) + + var workspaceRemoteClient = new LinkisWorkSpaceRemoteClient(clientConfig) + + def getWorkspaceRemoteClient: LinkisWorkSpaceRemoteClient = { + workspaceRemoteClient + } + + def getLinkisDataSourceRemoteClient: LinkisDataSourceRemoteClient = { + dataSourceClient + } + + def getDataModelRemoteClient: LinkisDatamodelCenterRemoteClient = { + dataModelRemoteClient + } + + def getDataAssetsRemoteClient: LinkisDataAssetsRemoteClient = { + dataAssetsRemoteClient + } + + def getMetadataSourceRemoteClient: LinkisMetadataSourceRemoteClient = { + metadataSourceRemoteClient + } +} diff --git a/dss-apps/dss-datawarehouse-design/data-warehouse-service/src/test/java/com/webank/wedatasphere/warehouse/RemoteClientTest.java b/dss-apps/dss-datawarehouse-design/data-warehouse-service/src/test/java/com/webank/wedatasphere/warehouse/RemoteClientTest.java new file mode 100644 index 0000000000..0db5085132 --- /dev/null +++ b/dss-apps/dss-datawarehouse-design/data-warehouse-service/src/test/java/com/webank/wedatasphere/warehouse/RemoteClientTest.java @@ -0,0 +1,34 @@ +package com.webank.wedatasphere.warehouse; + +import org.apache.linkis.datasource.client.impl.LinkisMetadataSourceRemoteClient; +import org.apache.linkis.datasource.client.request.GetMetadataSourceAllDatabasesAction; +import org.apache.linkis.datasource.client.response.GetMetadataSourceAllDatabasesResult; +import org.apache.linkis.httpclient.dws.authentication.StaticAuthenticationStrategy; +import org.apache.linkis.httpclient.dws.config.DWSClientConfig; +import org.apache.linkis.httpclient.dws.config.DWSClientConfigBuilder; + +import java.util.List; +import java.util.concurrent.TimeUnit; + +public class RemoteClientTest { + + public static void main(String[] args) { + LinkisMetadataSourceRemoteClient client = getMetadataSourceRemoteClient(); + GetMetadataSourceAllDatabasesAction action = GetMetadataSourceAllDatabasesAction.builder().setUser("hdfs").build(); + GetMetadataSourceAllDatabasesResult result = client.getAllDBMetaDataSource(action); + List dbs = result.getDbs(); + System.out.println(dbs); + } + + private static LinkisMetadataSourceRemoteClient getMetadataSourceRemoteClient(){ + DWSClientConfig clientConfig = ((DWSClientConfigBuilder) (DWSClientConfigBuilder.newBuilder().addServerUrl("http://gateway:8088") + .connectionTimeout(30000).discoveryEnabled(true) + .discoveryFrequency(1, TimeUnit.MINUTES) + .loadbalancerEnabled(true).maxConnectionSize(5) + .retryEnabled(false).readTimeout(30000) + .setAuthenticationStrategy(new StaticAuthenticationStrategy()).setAuthTokenKey("hdfs") + .setAuthTokenValue("hdfs"))).setDWSVersion("v1").build(); + return new LinkisMetadataSourceRemoteClient(clientConfig); + } + +} diff --git a/dss-apps/dss-datawarehouse-design/data-warehouse-service/src/test/scala/com/webank/wedatasphere/warehouse/DwLayerRemoteClientTest.scala b/dss-apps/dss-datawarehouse-design/data-warehouse-service/src/test/scala/com/webank/wedatasphere/warehouse/DwLayerRemoteClientTest.scala new file mode 100644 index 0000000000..77c6e79307 --- /dev/null +++ b/dss-apps/dss-datawarehouse-design/data-warehouse-service/src/test/scala/com/webank/wedatasphere/warehouse/DwLayerRemoteClientTest.scala @@ -0,0 +1,52 @@ +package com.webank.wedatasphere.warehouse + +import org.apache.linkis.datasource.client.impl.{LinkisMetaDataRemoteClient, LinkisMetadataSourceRemoteClient} +import org.apache.linkis.datasource.client.request.{GetMetadataSourceAllDatabasesAction, MetadataGetDatabasesAction} +import org.apache.linkis.httpclient.dws.authentication.StaticAuthenticationStrategy +import org.apache.linkis.httpclient.dws.config.{DWSClientConfig, DWSClientConfigBuilder} + +import java.util.concurrent.TimeUnit + +object DwLayerRemoteClientTest { + + + def main(args: Array[String]): Unit = { + val clientCfg: DWSClientConfig = DWSClientConfigBuilder.newBuilder() + .addServerUrl("http://gateway:8088") + .connectionTimeout(30000) + .discoveryEnabled(true) + .loadbalancerEnabled(true) + .maxConnectionSize(5) + .retryEnabled(false) + .readTimeout(30000) + .setAuthenticationStrategy(new StaticAuthenticationStrategy()) + .setAuthTokenKey("hdfs") + .setAuthTokenValue("hdfs") + .setDWSVersion("v1") + .build(); + + val client = new LinkisMetadataSourceRemoteClient(clientCfg) + + val action = GetMetadataSourceAllDatabasesAction.builder().setUser("hdfs").build() + val result = client.getAllDBMetaDataSource(action) + println(result.dbs) + +// val client = new LinkisMetaDataRemoteClient(clientCfg) +// val action = new ListDwLayerAction +// val result = client.listLayers(action) +// println(result.getAll) +// +// val action1 = new ListDwThemeDomainAction +// val result1 = client.listThemeDomains(action1) +// println(result1.getAll) +// +// val action2 = new ListDwModifierAction +// val result2 = client.listModifiers(action2) +// println(result2.getAll) +// +// val action3 = new ListDwThemeDomainAction +// val result3 = client.listThemeDomains(action3) +// println(result3.getAll) + } + +} diff --git a/dss-apps/dss-datawarehouse-design/governance-dao/pom.xml b/dss-apps/dss-datawarehouse-design/governance-dao/pom.xml new file mode 100644 index 0000000000..0baa851007 --- /dev/null +++ b/dss-apps/dss-datawarehouse-design/governance-dao/pom.xml @@ -0,0 +1,69 @@ + + + 4.0.0 + + dss-datawarehouse-design + com.webank.wedatasphere.dss + 1.0.1 + + + governance-dao + jar + + + 8 + 8 + + + + + org.apache.linkis + linkis-module + provided + + + com.webank.wedatasphere.dss + data-warehouse-mybatis + ${dss.version} + + + + org.apache.linkis + linkis-mybatis + + + + commons-beanutils + commons-beanutils + 1.9.4 + + + + org.projectlombok + lombok + provided + + + + + + + org.apache.maven.plugins + maven-deploy-plugin + + + net.alchim31.maven + scala-maven-plugin + + + org.apache.maven.plugins + maven-jar-plugin + + + + ${project.artifactId}-${project.version} + + + \ No newline at end of file diff --git a/dss-apps/dss-datawarehouse-design/governance-dao/src/main/java/com/webank/wedatasphere/warehouse/dao/domain/DwLayer.java b/dss-apps/dss-datawarehouse-design/governance-dao/src/main/java/com/webank/wedatasphere/warehouse/dao/domain/DwLayer.java new file mode 100644 index 0000000000..b172094b08 --- /dev/null +++ b/dss-apps/dss-datawarehouse-design/governance-dao/src/main/java/com/webank/wedatasphere/warehouse/dao/domain/DwLayer.java @@ -0,0 +1,72 @@ +package com.webank.wedatasphere.warehouse.dao.domain; + +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableField; +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableName; +import com.webank.wedatasphere.warehouse.dao.interceptor.NameAttachWorkspaceTrans; +import com.webank.wedatasphere.warehouse.dao.domain.common.DssWorkspaceEntity; +import lombok.*; +import lombok.experimental.Accessors; + +import java.util.Date; + +@Data +@Getter +@Setter +@ToString +@Builder +@AllArgsConstructor +@NoArgsConstructor +@EqualsAndHashCode(callSuper = false) +@Accessors(chain = true) +@TableName("dss_datawarehouse_layer") +public class DwLayer extends DssWorkspaceEntity { + @TableId(value = "id", type = IdType.AUTO) + private Long id; + + @NameAttachWorkspaceTrans + private String name; + + @TableField(value = "en_name") + private String enName; + + private String description; + + // 授权的角色 + @TableField(value = "principal_name") + private String principalName; + + // 是否预置主题 + private Boolean preset; + + @TableField(value = "is_available") + private Boolean isAvailable; + + // 为空代表所有库 + @TableField(value = "dbs") + private String dbs; + + // 负责人 + private String owner; + + private Integer sort; + + // 创建人 +// @TableField(value = "create_user") +// private String createUser; + + @TableField(value = "create_time") + private Date createTime; + +// @TableField(value = "modify_user") +// private String modifyUser; + + @TableField(value = "update_time") + private Date updateTime; + + private Boolean status; + + @TableField(value = "lock_version") + private Long lockVersion; +} diff --git a/dss-apps/dss-datawarehouse-design/governance-dao/src/main/java/com/webank/wedatasphere/warehouse/dao/domain/DwLayerGeneralizeRule.java b/dss-apps/dss-datawarehouse-design/governance-dao/src/main/java/com/webank/wedatasphere/warehouse/dao/domain/DwLayerGeneralizeRule.java new file mode 100644 index 0000000000..7be9976c83 --- /dev/null +++ b/dss-apps/dss-datawarehouse-design/governance-dao/src/main/java/com/webank/wedatasphere/warehouse/dao/domain/DwLayerGeneralizeRule.java @@ -0,0 +1,56 @@ +package com.webank.wedatasphere.warehouse.dao.domain; + +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableField; +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableName; +import com.webank.wedatasphere.warehouse.dao.domain.common.DssWorkspaceEntity; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import lombok.ToString; + +import java.util.Date; + +@Getter +@Setter +@ToString +@NoArgsConstructor +@TableName("dss_datawarehouse_layer_generalize_rule") +public class DwLayerGeneralizeRule extends DssWorkspaceEntity { + + @TableId(value = "id", type = IdType.AUTO) + private Long id; + + @TableField(value = "layer_id") + private Long layerId; + + // 自动归纳表达式 + private String regex; + + private String identifier; + + @TableField(value = "en_identifier") + private String enIdentifier; + + @TableField(value = "desc") + private String description; + +// @TableField(value = "create_user") +// private String createUser; + + @TableField(value = "create_time") + private Date createTime; + +// @TableField(value = "modify_user") +// private String modifyUser; + + @TableField(value = "update_time") + private Date updateTime; + + private Boolean status; + + @TableField(value = "lock_version") + private Long lockVersion; + +} diff --git a/dss-apps/dss-datawarehouse-design/governance-dao/src/main/java/com/webank/wedatasphere/warehouse/dao/domain/DwModifier.java b/dss-apps/dss-datawarehouse-design/governance-dao/src/main/java/com/webank/wedatasphere/warehouse/dao/domain/DwModifier.java new file mode 100644 index 0000000000..6b59e389af --- /dev/null +++ b/dss-apps/dss-datawarehouse-design/governance-dao/src/main/java/com/webank/wedatasphere/warehouse/dao/domain/DwModifier.java @@ -0,0 +1,76 @@ +package com.webank.wedatasphere.warehouse.dao.domain; + +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableField; +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableName; +import com.webank.wedatasphere.warehouse.dao.domain.common.DssWorkspaceEntity; +import lombok.*; +import lombok.experimental.Accessors; + +import java.util.Date; + +@Data +@Getter +@Setter +@ToString +@Builder +@AllArgsConstructor +@NoArgsConstructor +@EqualsAndHashCode(callSuper = false) +@Accessors(chain = true) +@TableName("dss_datawarehouse_modifier") +public class DwModifier extends DssWorkspaceEntity { + + @TableId(value = "id", type = IdType.AUTO) + private Long id; + + @TableField(value = "modifier_type") + private String modifierType; + + @TableField(value = "modifier_type_en") + private String modifierTypeEn; + + @TableField(value = "theme_domain_id") + private Long themeDomainId; + + // 空：代表所有，如果是逗号分隔的字符串则代表对应的theme的names + @TableField(value = "theme_area") + private String themeArea; + + @TableField(value = "theme_area_en") + private String themeAreaEn; + + @TableField(value = "layer_id") + private Long layerId; + + // 空：代表所有，如果是逗号分隔的字符串则代表对应的layer的names + @TableField(value = "layer_area") + private String layerArea; + + @TableField(value = "layer_area_en") + private String layerAreaEn; + + private String description; + + @TableField(value = "is_available") + private Boolean isAvailable; + +// @TableField(value = "create_user") +// private String createUser; + + @TableField(value = "create_time") + private Date createTime; + +// @TableField(value = "modify_user") +// private String modifyUser; + + @TableField(value = "update_time") + private Date updateTime; + + private Boolean status; + + @TableField(value = "lock_version") + private Long lockVersion; + +} diff --git a/dss-apps/dss-datawarehouse-design/governance-dao/src/main/java/com/webank/wedatasphere/warehouse/dao/domain/DwModifierList.java b/dss-apps/dss-datawarehouse-design/governance-dao/src/main/java/com/webank/wedatasphere/warehouse/dao/domain/DwModifierList.java new file mode 100644 index 0000000000..3b712ec740 --- /dev/null +++ b/dss-apps/dss-datawarehouse-design/governance-dao/src/main/java/com/webank/wedatasphere/warehouse/dao/domain/DwModifierList.java @@ -0,0 +1,45 @@ +package com.webank.wedatasphere.warehouse.dao.domain; + +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableField; +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableName; +import com.webank.wedatasphere.warehouse.dao.domain.common.DssWorkspaceEntity; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import lombok.ToString; + +import java.util.Date; + +@Getter +@Setter +@ToString +@NoArgsConstructor +@TableName("dss_datawarehouse_modifier_list") +public class DwModifierList extends DssWorkspaceEntity { + @TableId(value = "id", type = IdType.AUTO) + private Long id; + + @TableField(value = "modifier_id") + private Long modifierId; + + private String name; + + private String identifier; + + private String formula; + +// @TableField(value = "create_user") +// private String createUser; + + @TableField(value = "create_time") + private Date createTime; + +// @TableField(value = "modify_user") +// private String modifyUser; + + @TableField(value = "update_time") + private Date updateTime; + +} diff --git a/dss-apps/dss-datawarehouse-design/governance-dao/src/main/java/com/webank/wedatasphere/warehouse/dao/domain/DwStatisticalPeriod.java b/dss-apps/dss-datawarehouse-design/governance-dao/src/main/java/com/webank/wedatasphere/warehouse/dao/domain/DwStatisticalPeriod.java new file mode 100644 index 0000000000..eb80cc082a --- /dev/null +++ b/dss-apps/dss-datawarehouse-design/governance-dao/src/main/java/com/webank/wedatasphere/warehouse/dao/domain/DwStatisticalPeriod.java @@ -0,0 +1,77 @@ +package com.webank.wedatasphere.warehouse.dao.domain; + +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableField; +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableName; +import com.webank.wedatasphere.warehouse.dao.domain.common.DssWorkspaceEntity; +import com.webank.wedatasphere.warehouse.dao.interceptor.NameAttachWorkspaceTrans; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import lombok.ToString; + +import java.util.Date; + +@Getter +@Setter +@ToString +@NoArgsConstructor +@TableName("dss_datawarehouse_statistical_period") +public class DwStatisticalPeriod extends DssWorkspaceEntity { + @TableId(value = "id", type = IdType.AUTO) + private Long id; + + @TableField(value = "theme_domain_id") + private Long themeDomainId; + +// @TableField(value = "theme_area") +// private String themeArea; + + @TableField(value = "layer_id") + private Long layerId; + +// @TableField(value = "layer_area") +// private String layerArea; + + @NameAttachWorkspaceTrans + private String name; + + @TableField(value = "en_name") + private String enName; + + private String description; + + @TableField(value = "start_time_formula") + private String startTimeFormula; + + @TableField(value = "end_time_formula") + private String endTimeFormula; + + // 授权的名字：userName、roleName + @TableField(value = "principal_name") + private String principalName; + + @TableField(value = "is_available") + private Boolean isAvailable; + + // 负责人 + private String owner; + +// @TableField(value = "create_user") +// private String createUser; + + @TableField(value = "create_time") + private Date createTime; + +// @TableField(value = "modify_user") +// private String modifyUser; + + @TableField(value = "update_time") + private Date updateTime; + + private Boolean status; + + @TableField(value = "lock_version") + private Long lockVersion; +} diff --git a/dss-apps/dss-datawarehouse-design/governance-dao/src/main/java/com/webank/wedatasphere/warehouse/dao/domain/DwThemeDomain.java b/dss-apps/dss-datawarehouse-design/governance-dao/src/main/java/com/webank/wedatasphere/warehouse/dao/domain/DwThemeDomain.java new file mode 100644 index 0000000000..53943da675 --- /dev/null +++ b/dss-apps/dss-datawarehouse-design/governance-dao/src/main/java/com/webank/wedatasphere/warehouse/dao/domain/DwThemeDomain.java @@ -0,0 +1,64 @@ +package com.webank.wedatasphere.warehouse.dao.domain; + +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableField; +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableName; +import com.webank.wedatasphere.warehouse.dao.interceptor.NameAttachWorkspaceTrans; +import com.webank.wedatasphere.warehouse.dao.domain.common.DssWorkspaceEntity; +import lombok.*; +import lombok.experimental.Accessors; + +import java.util.Date; + +@Data +@Getter +@Setter +@ToString +@Builder +@AllArgsConstructor +@NoArgsConstructor +@EqualsAndHashCode(callSuper = false) +@Accessors(chain = true) +@TableName("dss_datawarehouse_theme_domain") +public class DwThemeDomain extends DssWorkspaceEntity { + @TableId(value = "id", type = IdType.AUTO) + private Long id; + + @NameAttachWorkspaceTrans + private String name; + + @TableField(value = "en_name") + private String enName; + + private String description; + + @TableField(value = "is_available") + private Boolean isAvailable; + + // 可用角色，授权角色 + @TableField(value = "principal_name") + private String principalName; + + @TableField(value = "owner") + private String owner; + + private Integer sort; + +// @TableField(value = "create_user") +// private String createUser; + + @TableField(value = "create_time") + private Date createTime; + +// @TableField(value = "modify_user") +// private String modifyUser; + + @TableField(value = "update_time") + private Date updateTime; + + private Boolean status; + + @TableField(value = "lock_version") + private Long lockVersion; +} diff --git a/dss-apps/dss-datawarehouse-design/governance-dao/src/main/java/com/webank/wedatasphere/warehouse/dao/domain/common/DssWorkspaceEntity.java b/dss-apps/dss-datawarehouse-design/governance-dao/src/main/java/com/webank/wedatasphere/warehouse/dao/domain/common/DssWorkspaceEntity.java new file mode 100644 index 0000000000..0f3173da8b --- /dev/null +++ b/dss-apps/dss-datawarehouse-design/governance-dao/src/main/java/com/webank/wedatasphere/warehouse/dao/domain/common/DssWorkspaceEntity.java @@ -0,0 +1,7 @@ +package com.webank.wedatasphere.warehouse.dao.domain.common; + +import java.io.Serializable; + +public class DssWorkspaceEntity implements Serializable { + +} diff --git a/dss-apps/dss-datawarehouse-design/governance-dao/src/main/java/com/webank/wedatasphere/warehouse/dao/interceptor/DssWorkspaceNameAdapter.java b/dss-apps/dss-datawarehouse-design/governance-dao/src/main/java/com/webank/wedatasphere/warehouse/dao/interceptor/DssWorkspaceNameAdapter.java new file mode 100644 index 0000000000..fde7435ef5 --- /dev/null +++ b/dss-apps/dss-datawarehouse-design/governance-dao/src/main/java/com/webank/wedatasphere/warehouse/dao/interceptor/DssWorkspaceNameAdapter.java @@ -0,0 +1,14 @@ +package com.webank.wedatasphere.warehouse.dao.interceptor; + +import org.apache.linkis.common.conf.CommonVars; + +public class DssWorkspaceNameAdapter { + + public static final CommonVars BDP_ENTITY_WORKSPACE_NAME_AUTO_TRANSFORM = CommonVars.apply("wds.entity.workspace.name.auto.transform", "false"); + + public String getWorkspaceName() { + // TODO 真正获取当前用户线程的工作空间名称 + return "workspaceName"; + } + +} diff --git a/dss-apps/dss-datawarehouse-design/governance-dao/src/main/java/com/webank/wedatasphere/warehouse/dao/interceptor/DssWorkspaceNameAutoExtractQueryInterceptor.java b/dss-apps/dss-datawarehouse-design/governance-dao/src/main/java/com/webank/wedatasphere/warehouse/dao/interceptor/DssWorkspaceNameAutoExtractQueryInterceptor.java new file mode 100644 index 0000000000..20696b0f22 --- /dev/null +++ b/dss-apps/dss-datawarehouse-design/governance-dao/src/main/java/com/webank/wedatasphere/warehouse/dao/interceptor/DssWorkspaceNameAutoExtractQueryInterceptor.java @@ -0,0 +1,91 @@ +package com.webank.wedatasphere.warehouse.dao.interceptor; + + +import lombok.AllArgsConstructor; +import org.apache.ibatis.executor.resultset.DefaultResultSetHandler; +import org.apache.ibatis.executor.resultset.ResultSetHandler; +import org.apache.ibatis.mapping.MappedStatement; +import org.apache.ibatis.mapping.ResultMap; +import org.apache.ibatis.plugin.*; +import org.apache.ibatis.reflection.MetaObject; +import org.apache.ibatis.reflection.SystemMetaObject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.BeanUtils; + +import javax.sql.DataSource; +import java.beans.PropertyDescriptor; +import java.lang.reflect.Field; +import java.lang.reflect.InvocationTargetException; +import java.sql.Statement; +import java.util.*; + +@AllArgsConstructor +@Intercepts({ + @Signature(type = ResultSetHandler.class, method = "handleResultSets", args = {Statement.class}) +}) +public class DssWorkspaceNameAutoExtractQueryInterceptor /*extends AbstractSqlParserHandler*/ implements Interceptor { + private static final Logger logger = LoggerFactory.getLogger(DssWorkspaceNameAutoExtractQueryInterceptor.class); + private static final DssWorkspaceNameAdapter dssWorkspaceNameAdapter = new DssWorkspaceNameAdapter(); + private final DataSource dataSource; + private Object extract(Object object) throws InvocationTargetException, IllegalAccessException { + Field[] fields = object.getClass().getDeclaredFields(); + for (Field field : fields) { + NameAttachWorkspaceTrans confidential = field.getAnnotation(NameAttachWorkspaceTrans.class); + if (confidential==null){ + continue; + } + PropertyDescriptor ps = BeanUtils.getPropertyDescriptor(object.getClass(), field.getName()); + if (ps == null || ps.getReadMethod() == null || ps.getWriteMethod() == null) { + continue; + } + Object value = ps.getReadMethod().invoke(object); + if (value != null) { + if (value instanceof String) { + String fieldValue = (String) value; + if (fieldValue.contains(".") && fieldValue.contains(dssWorkspaceNameAdapter.getWorkspaceName())) { + String[] split = fieldValue.split("\\."); + if (split.length == 2) { + ps.getWriteMethod().invoke(object, split[1]); +// resultMap.put(colName, split[1]); + } + } + } + } + } + return object; + } + + @Override + public Object intercept(Invocation invocation) throws Throwable { + String flag = DssWorkspaceNameAdapter.BDP_ENTITY_WORKSPACE_NAME_AUTO_TRANSFORM.getValue(); + if ("false".equalsIgnoreCase(flag)) { + return invocation.proceed(); + } + + logger.info("use mybatis interceptor to handle name field [transform workspace.name ??]"); + + DefaultResultSetHandler defaultResultSetHandler = (DefaultResultSetHandler) invocation.getTarget(); + MetaObject metaObject = SystemMetaObject.forObject(defaultResultSetHandler); + MappedStatement mappedStatement = (MappedStatement) metaObject.getValue("mappedStatement"); + List resultMaps = mappedStatement.getResultMaps(); + + if (resultMaps == null || resultMaps.isEmpty()) { + return invocation.proceed(); + } + + Object result = invocation.proceed(); + if (result instanceof Collection) { + Collection objList= (Collection) result; + List resultList=new ArrayList<>(); + for (Object obj : objList) { + resultList.add(extract(obj)); + } + return resultList; + }else { + return extract(result); + } + } + + +} diff --git a/dss-apps/dss-datawarehouse-design/governance-dao/src/main/java/com/webank/wedatasphere/warehouse/dao/interceptor/DssWorkspaceNameAutoTransformUpdateInteceptor.java b/dss-apps/dss-datawarehouse-design/governance-dao/src/main/java/com/webank/wedatasphere/warehouse/dao/interceptor/DssWorkspaceNameAutoTransformUpdateInteceptor.java new file mode 100644 index 0000000000..8d7d277593 --- /dev/null +++ b/dss-apps/dss-datawarehouse-design/governance-dao/src/main/java/com/webank/wedatasphere/warehouse/dao/interceptor/DssWorkspaceNameAutoTransformUpdateInteceptor.java @@ -0,0 +1,81 @@ +package com.webank.wedatasphere.warehouse.dao.interceptor; + +import com.baomidou.mybatisplus.extension.handlers.AbstractSqlParserHandler; +import lombok.AllArgsConstructor; +import org.apache.ibatis.binding.MapperMethod; +import org.apache.ibatis.executor.Executor; +import org.apache.ibatis.mapping.MappedStatement; +import org.apache.ibatis.mapping.SqlCommandType; +import org.apache.ibatis.plugin.Interceptor; +import org.apache.ibatis.plugin.Intercepts; +import org.apache.ibatis.plugin.Invocation; +import org.apache.ibatis.plugin.Signature; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.sql.DataSource; +import java.lang.reflect.Field; +import java.lang.reflect.InvocationTargetException; +import java.util.Objects; + +@AllArgsConstructor +@Intercepts({ + @Signature(type = Executor.class, method = "update", args = {MappedStatement.class, Object.class}) +}) +public class DssWorkspaceNameAutoTransformUpdateInteceptor extends AbstractSqlParserHandler implements Interceptor { + private static final Logger logger = LoggerFactory.getLogger(DssWorkspaceNameAutoTransformUpdateInteceptor.class); + private static final DssWorkspaceNameAdapter dssWorkspaceNameAdapter = new DssWorkspaceNameAdapter(); + private final DataSource dataSource; + @Override + public Object intercept(Invocation invocation) throws Throwable { + String flag = DssWorkspaceNameAdapter.BDP_ENTITY_WORKSPACE_NAME_AUTO_TRANSFORM.getValue(); + if ("false".equalsIgnoreCase(flag)) { + return invocation.proceed(); + } + + logger.info("use mybatis interceptor to handle name field [transform workspace.name ??]"); + + MappedStatement mappedStatement = (MappedStatement)invocation.getArgs()[0]; + SqlCommandType sqlCommandType = mappedStatement.getSqlCommandType(); + + if (sqlCommandType == SqlCommandType.DELETE) { + return invocation.proceed(); + } + + Object parameter = invocation.getArgs()[1]; + + if (parameter instanceof MapperMethod.ParamMap) { + MapperMethod.ParamMap inParams = (MapperMethod.ParamMap) parameter; + Object et = inParams.get("et"); + + Field[] declaredFields = et.getClass().getDeclaredFields(); + for (Field field : declaredFields) { + if (field.getAnnotation(NameAttachWorkspaceTrans.class) != null) { + field.setAccessible(true); + String finalName = null; + Object origVal = field.get(et); + if (!Objects.isNull(origVal) && origVal instanceof String) { + finalName = dssWorkspaceNameAdapter.getWorkspaceName()+"."+origVal; + } + field.set(et, finalName); + field.setAccessible(false); + } + } + } else { + Field[] declaredFields = parameter.getClass().getDeclaredFields(); + for (Field field : declaredFields) { + if (field.getAnnotation(NameAttachWorkspaceTrans.class) != null) { + field.setAccessible(true); + String finalName = ""; + Object origVal = field.get(parameter); + if (origVal instanceof String) { + finalName = dssWorkspaceNameAdapter.getWorkspaceName()+"."+origVal; + } + field.set(parameter, finalName); + field.setAccessible(false); + } + } + } + return invocation.proceed(); + } +} diff --git a/dss-apps/dss-datawarehouse-design/governance-dao/src/main/java/com/webank/wedatasphere/warehouse/dao/interceptor/NameAttachWorkspaceTrans.java b/dss-apps/dss-datawarehouse-design/governance-dao/src/main/java/com/webank/wedatasphere/warehouse/dao/interceptor/NameAttachWorkspaceTrans.java new file mode 100644 index 0000000000..1cc014c96f --- /dev/null +++ b/dss-apps/dss-datawarehouse-design/governance-dao/src/main/java/com/webank/wedatasphere/warehouse/dao/interceptor/NameAttachWorkspaceTrans.java @@ -0,0 +1,11 @@ +package com.webank.wedatasphere.warehouse.dao.interceptor; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +@Retention(RetentionPolicy.RUNTIME) +@Target({ ElementType.FIELD }) +public @interface NameAttachWorkspaceTrans { +} diff --git a/dss-apps/dss-datawarehouse-design/governance-dao/src/main/java/com/webank/wedatasphere/warehouse/dao/mapper/DwLayerMapper.java b/dss-apps/dss-datawarehouse-design/governance-dao/src/main/java/com/webank/wedatasphere/warehouse/dao/mapper/DwLayerMapper.java new file mode 100644 index 0000000000..a768ffbfd0 --- /dev/null +++ b/dss-apps/dss-datawarehouse-design/governance-dao/src/main/java/com/webank/wedatasphere/warehouse/dao/mapper/DwLayerMapper.java @@ -0,0 +1,15 @@ +package com.webank.wedatasphere.warehouse.dao.mapper; + +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import com.webank.wedatasphere.warehouse.dao.domain.DwLayer; +import org.apache.ibatis.annotations.Mapper; +import org.apache.ibatis.annotations.Param; +import org.apache.ibatis.annotations.Select; + +import java.util.Optional; + +@Mapper +public interface DwLayerMapper extends BaseMapper { + @Select("select * from dss_datawarehouse_layer where name = #{layer} or en_name = #{layer}") + Optional findByName(@Param("layer") String layer); +} diff --git a/dss-apps/dss-datawarehouse-design/governance-dao/src/main/java/com/webank/wedatasphere/warehouse/dao/mapper/DwModifierListMapper.java b/dss-apps/dss-datawarehouse-design/governance-dao/src/main/java/com/webank/wedatasphere/warehouse/dao/mapper/DwModifierListMapper.java new file mode 100644 index 0000000000..5000e80c8b --- /dev/null +++ b/dss-apps/dss-datawarehouse-design/governance-dao/src/main/java/com/webank/wedatasphere/warehouse/dao/mapper/DwModifierListMapper.java @@ -0,0 +1,9 @@ +package com.webank.wedatasphere.warehouse.dao.mapper; + +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import com.webank.wedatasphere.warehouse.dao.domain.DwModifierList; +import org.apache.ibatis.annotations.Mapper; + +@Mapper +public interface DwModifierListMapper extends BaseMapper { +} diff --git a/dss-apps/dss-datawarehouse-design/governance-dao/src/main/java/com/webank/wedatasphere/warehouse/dao/mapper/DwModifierMapper.java b/dss-apps/dss-datawarehouse-design/governance-dao/src/main/java/com/webank/wedatasphere/warehouse/dao/mapper/DwModifierMapper.java new file mode 100644 index 0000000000..06375b9815 --- /dev/null +++ b/dss-apps/dss-datawarehouse-design/governance-dao/src/main/java/com/webank/wedatasphere/warehouse/dao/mapper/DwModifierMapper.java @@ -0,0 +1,9 @@ +package com.webank.wedatasphere.warehouse.dao.mapper; + +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import com.webank.wedatasphere.warehouse.dao.domain.DwModifier; +import org.apache.ibatis.annotations.Mapper; + +@Mapper +public interface DwModifierMapper extends BaseMapper { +} diff --git a/dss-apps/dss-datawarehouse-design/governance-dao/src/main/java/com/webank/wedatasphere/warehouse/dao/mapper/DwStatisticalPeriodMapper.java b/dss-apps/dss-datawarehouse-design/governance-dao/src/main/java/com/webank/wedatasphere/warehouse/dao/mapper/DwStatisticalPeriodMapper.java new file mode 100644 index 0000000000..3155a13543 --- /dev/null +++ b/dss-apps/dss-datawarehouse-design/governance-dao/src/main/java/com/webank/wedatasphere/warehouse/dao/mapper/DwStatisticalPeriodMapper.java @@ -0,0 +1,22 @@ +package com.webank.wedatasphere.warehouse.dao.mapper; + +import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import com.baomidou.mybatisplus.core.metadata.IPage; +import com.baomidou.mybatisplus.core.toolkit.Constants; +import com.webank.wedatasphere.warehouse.dao.domain.DwStatisticalPeriod; +import com.webank.wedatasphere.warehouse.dao.vo.DwStatisticalPeriodVo; +import org.apache.ibatis.annotations.Mapper; +import org.apache.ibatis.annotations.Param; + +import java.util.List; +import java.util.Map; + +@Mapper +public interface DwStatisticalPeriodMapper extends BaseMapper { + IPage selectPageItems(IPage queryPage, Map params); + + List selectItems(@Param(Constants.WRAPPER) QueryWrapper queryWrapper); + + DwStatisticalPeriodVo selectItemById(@Param(value = "id") Long id); +} diff --git a/dss-apps/dss-datawarehouse-design/governance-dao/src/main/java/com/webank/wedatasphere/warehouse/dao/mapper/DwThemeDomainMapper.java b/dss-apps/dss-datawarehouse-design/governance-dao/src/main/java/com/webank/wedatasphere/warehouse/dao/mapper/DwThemeDomainMapper.java new file mode 100644 index 0000000000..314f3a2ddd --- /dev/null +++ b/dss-apps/dss-datawarehouse-design/governance-dao/src/main/java/com/webank/wedatasphere/warehouse/dao/mapper/DwThemeDomainMapper.java @@ -0,0 +1,15 @@ +package com.webank.wedatasphere.warehouse.dao.mapper; + +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import com.webank.wedatasphere.warehouse.dao.domain.DwThemeDomain; +import org.apache.ibatis.annotations.Mapper; +import org.apache.ibatis.annotations.Param; +import org.apache.ibatis.annotations.Select; + +import java.util.Optional; + +@Mapper +public interface DwThemeDomainMapper extends BaseMapper { + @Select("select * from dss_datawarehouse_theme_domain where name = #{theme} or en_name = #{theme}") + Optional findByName(@Param("theme") String theme); +} diff --git a/dss-apps/dss-datawarehouse-design/governance-dao/src/main/java/com/webank/wedatasphere/warehouse/dao/mapper/impl/DwStatisticalPeriodMapper.xml b/dss-apps/dss-datawarehouse-design/governance-dao/src/main/java/com/webank/wedatasphere/warehouse/dao/mapper/impl/DwStatisticalPeriodMapper.xml new file mode 100644 index 0000000000..6da9cecc5a --- /dev/null +++ b/dss-apps/dss-datawarehouse-design/governance-dao/src/main/java/com/webank/wedatasphere/warehouse/dao/mapper/impl/DwStatisticalPeriodMapper.xml @@ -0,0 +1,33 @@ + + + + + + + + + + + \ No newline at end of file diff --git a/dss-apps/dss-datawarehouse-design/governance-dao/src/main/java/com/webank/wedatasphere/warehouse/dao/vo/DwStatisticalPeriodVo.java b/dss-apps/dss-datawarehouse-design/governance-dao/src/main/java/com/webank/wedatasphere/warehouse/dao/vo/DwStatisticalPeriodVo.java new file mode 100644 index 0000000000..0b18c34b9f --- /dev/null +++ b/dss-apps/dss-datawarehouse-design/governance-dao/src/main/java/com/webank/wedatasphere/warehouse/dao/vo/DwStatisticalPeriodVo.java @@ -0,0 +1,60 @@ +package com.webank.wedatasphere.warehouse.dao.vo; + +import com.webank.wedatasphere.warehouse.dao.interceptor.NameAttachWorkspaceTrans; +import lombok.Getter; +import lombok.Setter; +import lombok.ToString; + +import java.io.Serializable; +import java.util.Date; + + +// 不要使用继承，否则 SQL 拦截器 ResultSetHandler 无法获取到父类的 private 字段来转换 +@Setter +@Getter +@ToString +public class DwStatisticalPeriodVo implements Serializable { + private Long id; + + private Long themeDomainId; + + private Long layerId; + + @NameAttachWorkspaceTrans + private String name; + + private String enName; + + private String description; + + private String startTimeFormula; + + private String endTimeFormula; + + private String principalName; + + private Boolean isAvailable; + + private String owner; + + private Date createTime; + + private Date updateTime; + + // 自定义字段 + @NameAttachWorkspaceTrans + private String themeArea; + + private String themeAreaEn; + + @NameAttachWorkspaceTrans + private String layerArea; + + private String layerAreaEn; + + transient private Boolean status; + + private boolean referenced; + + private int referenceCount; +} diff --git a/dss-apps/dss-datawarehouse-design/pom.xml b/dss-apps/dss-datawarehouse-design/pom.xml new file mode 100644 index 0000000000..8462dd4749 --- /dev/null +++ b/dss-apps/dss-datawarehouse-design/pom.xml @@ -0,0 +1,154 @@ + + + + dss + com.webank.wedatasphere.dss + 1.0.1 + ../../pom.xml + + 4.0.0 + + dss-datawarehouse-design + pom + + + UTF-8 + 2.16 + 2.22.2 + 2.22.2 + + + + data-warehouse-server + data-warehouse-service + data-warehouse-mybatis + data-warehouse-client + governance-dao + + + + + + + com.webank.wedatasphere.dss + governance-dao + ${project.version} + + + + com.webank.wedatasphere.dss + data-warehouse-service + ${project.version} + + + + + org.scala-lang + scala-library + ${scala.version} + + + org.scala-lang + scala-compiler + ${scala.version} + + + org.scala-lang + scala-reflect + ${scala.version} + + + org.scala-lang + scalap + ${scala.version} + + + commons-lang + commons-lang + ${commons.lang.version} + + + org.apache.linkis + linkis-mybatis + ${linkis.version} + + + org.apache.linkis + linkis-httpclient + ${linkis.version} + + + org.apache.linkis + linkis-gateway-httpclient-support + ${linkis.version} + + + org.apache.linkis + linkis-module + + + org.springframework.boot + spring-boot-starter-tomcat + + + hibernate-validator + org.hibernate.validator + + + ${linkis.version} + + + org.apache.linkis + linkis-common + ${linkis.version} + + + + org.apache.linkis + linkis-protocol + ${linkis.version} + + + + com.google.code.gson + gson + ${gson.version} + + + com.fasterxml.jackson.core + jackson-databind + ${fasterxml.jackson.version} + + + org.codehaus.jackson + jackson-mapper-asl + ${org.codehaus.jackson.version} + + + org.apache.commons + commons-math3 + ${commons.math.version} + + + xstream + com.thoughtworks.xstream + ${xstream.version} + + + + org.apache.linkis + linkis-datasource-client + ${linkis.version} + + + + org.projectlombok + lombok + 1.18.20 + + + + + \ No newline at end of file diff --git a/dss-framework/dss-framework-workspace-client/pom.xml b/dss-framework/dss-framework-workspace-client/pom.xml new file mode 100644 index 0000000000..439093a016 --- /dev/null +++ b/dss-framework/dss-framework-workspace-client/pom.xml @@ -0,0 +1,56 @@ + + + + dss + com.webank.wedatasphere.dss + 1.0.1 + + 4.0.0 + + dss-framework-workspace-client + + + 8 + 8 + + + + + org.apache.linkis + linkis-gateway-httpclient-support + ${linkis.version} + + + org.projectlombok + lombok + 1.18.16 + compile + + + + + + + org.apache.maven.plugins + maven-deploy-plugin + + + + net.alchim31.maven + scala-maven-plugin + + + org.apache.maven.plugins + maven-jar-plugin + + + + + src/main/resources + + + ${project.artifactId}-${project.version} + + \ No newline at end of file diff --git a/dss-framework/dss-framework-workspace-client/src/main/java/com/webank/wedatasphere/dss/framework/workspace/client/entity/DSSWorkspaceRole.java b/dss-framework/dss-framework-workspace-client/src/main/java/com/webank/wedatasphere/dss/framework/workspace/client/entity/DSSWorkspaceRole.java new file mode 100644 index 0000000000..e55424cf09 --- /dev/null +++ b/dss-framework/dss-framework-workspace-client/src/main/java/com/webank/wedatasphere/dss/framework/workspace/client/entity/DSSWorkspaceRole.java @@ -0,0 +1,45 @@ +package com.webank.wedatasphere.dss.framework.workspace.client.entity; + +import java.io.Serializable; + +public class DSSWorkspaceRole implements Serializable { + private static final long serialVersionUID=1L; + + private int roleId; + private String roleName; + private String roleFrontName; + + + public int getRoleId() { + return roleId; + } + + public void setRoleId(int roleId) { + this.roleId = roleId; + } + + public String getRoleName() { + return roleName; + } + + public void setRoleName(String roleName) { + this.roleName = roleName; + } + + public String getRoleFrontName() { + return roleFrontName; + } + + public void setRoleFrontName(String roleFrontName) { + this.roleFrontName = roleFrontName; + } + + @Override + public String toString() { + return "DSSWorkspaceRole{" + + "roleId=" + roleId + + ", roleName='" + roleName + '\'' + + ", roleFrontName='" + roleFrontName + '\'' + + '}'; + } +} diff --git a/dss-framework/dss-framework-workspace-client/src/main/java/com/webank/wedatasphere/dss/framework/workspace/client/entity/DSSWorkspaceUser.java b/dss-framework/dss-framework-workspace-client/src/main/java/com/webank/wedatasphere/dss/framework/workspace/client/entity/DSSWorkspaceUser.java new file mode 100644 index 0000000000..739233d233 --- /dev/null +++ b/dss-framework/dss-framework-workspace-client/src/main/java/com/webank/wedatasphere/dss/framework/workspace/client/entity/DSSWorkspaceUser.java @@ -0,0 +1,78 @@ +package com.webank.wedatasphere.dss.framework.workspace.client.entity; + + + +import java.io.Serializable; +import java.util.Date; +import java.util.List; + + +public class DSSWorkspaceUser implements Serializable { + private int id; + private String name; + private List roles; + private String department; + private String office; + private String creator; + private Date joinTime; + + public DSSWorkspaceUser() { + } + + + public int getId() { + return id; + } + + public void setId(int id) { + this.id = id; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public List getRoles() { + return roles; + } + + public void setRoles(List roles) { + this.roles = roles; + } + + public String getDepartment() { + return department; + } + + public void setDepartment(String department) { + this.department = department; + } + + public String getOffice() { + return office; + } + + public void setOffice(String office) { + this.office = office; + } + + public String getCreator() { + return creator; + } + + public void setCreator(String creator) { + this.creator = creator; + } + + public Date getJoinTime() { + return joinTime; + } + + public void setJoinTime(Date joinTime) { + this.joinTime = joinTime; + } +} diff --git a/dss-framework/dss-framework-workspace-client/src/main/scala/com/webank/wedatasphere/dss/framework/workspace/client/AbstractRemoteClient.scala b/dss-framework/dss-framework-workspace-client/src/main/scala/com/webank/wedatasphere/dss/framework/workspace/client/AbstractRemoteClient.scala new file mode 100644 index 0000000000..9e6e318ee7 --- /dev/null +++ b/dss-framework/dss-framework-workspace-client/src/main/scala/com/webank/wedatasphere/dss/framework/workspace/client/AbstractRemoteClient.scala @@ -0,0 +1,15 @@ +package com.webank.wedatasphere.dss.framework.workspace.client + +import org.apache.linkis.httpclient.dws.DWSHttpClient +import org.apache.linkis.httpclient.request.Action +import org.apache.linkis.httpclient.response.Result + +abstract class AbstractRemoteClient extends RemoteClient{ + protected val dwsHttpClient:DWSHttpClient + + override def execute(action: Action): Result = action match { + case action: Action => dwsHttpClient.execute(action) + } + + override def close(): Unit = dwsHttpClient.close() +} diff --git a/dss-framework/dss-framework-workspace-client/src/main/scala/com/webank/wedatasphere/dss/framework/workspace/client/RemoteClient.scala b/dss-framework/dss-framework-workspace-client/src/main/scala/com/webank/wedatasphere/dss/framework/workspace/client/RemoteClient.scala new file mode 100644 index 0000000000..26a8f863ac --- /dev/null +++ b/dss-framework/dss-framework-workspace-client/src/main/scala/com/webank/wedatasphere/dss/framework/workspace/client/RemoteClient.scala @@ -0,0 +1,11 @@ +package com.webank.wedatasphere.dss.framework.workspace.client + +import org.apache.linkis.httpclient.request.Action +import org.apache.linkis.httpclient.response.Result +import java.io.Closeable + +trait RemoteClient extends Closeable{ + protected def execute(action: Action): Result + + override def close(): Unit +} diff --git a/dss-framework/dss-framework-workspace-client/src/main/scala/com/webank/wedatasphere/dss/framework/workspace/client/WorkSpaceRemoteClient.scala b/dss-framework/dss-framework-workspace-client/src/main/scala/com/webank/wedatasphere/dss/framework/workspace/client/WorkSpaceRemoteClient.scala new file mode 100644 index 0000000000..fc331606ee --- /dev/null +++ b/dss-framework/dss-framework-workspace-client/src/main/scala/com/webank/wedatasphere/dss/framework/workspace/client/WorkSpaceRemoteClient.scala @@ -0,0 +1,9 @@ +package com.webank.wedatasphere.dss.framework.workspace.client + +import com.webank.wedatasphere.dss.framework.workspace.client.request.{GetWorkspaceRolesAction, GetWorkspaceUsersAction} +import com.webank.wedatasphere.dss.framework.workspace.client.response.{GetWorkspaceRolesResult, GetWorkspaceUsersResult} + +trait WorkSpaceRemoteClient extends RemoteClient { + def getWorkspaceUsers(action: GetWorkspaceUsersAction): GetWorkspaceUsersResult + def getWorkspaceRoles(action: GetWorkspaceRolesAction): GetWorkspaceRolesResult +} diff --git a/dss-framework/dss-framework-workspace-client/src/main/scala/com/webank/wedatasphere/dss/framework/workspace/client/exception/WorkSpaceClientBuilderException.scala b/dss-framework/dss-framework-workspace-client/src/main/scala/com/webank/wedatasphere/dss/framework/workspace/client/exception/WorkSpaceClientBuilderException.scala new file mode 100644 index 0000000000..1fa7b9a4bd --- /dev/null +++ b/dss-framework/dss-framework-workspace-client/src/main/scala/com/webank/wedatasphere/dss/framework/workspace/client/exception/WorkSpaceClientBuilderException.scala @@ -0,0 +1,5 @@ +package com.webank.wedatasphere.dss.framework.workspace.client.exception + +import org.apache.linkis.common.exception.ErrorException + +class WorkSpaceClientBuilderException(errorDesc: String) extends ErrorException(60022, errorDesc) diff --git a/dss-framework/dss-framework-workspace-client/src/main/scala/com/webank/wedatasphere/dss/framework/workspace/client/impl/LinkisWorkSpaceRemoteClient.scala b/dss-framework/dss-framework-workspace-client/src/main/scala/com/webank/wedatasphere/dss/framework/workspace/client/impl/LinkisWorkSpaceRemoteClient.scala new file mode 100644 index 0000000000..e517c9d8df --- /dev/null +++ b/dss-framework/dss-framework-workspace-client/src/main/scala/com/webank/wedatasphere/dss/framework/workspace/client/impl/LinkisWorkSpaceRemoteClient.scala @@ -0,0 +1,16 @@ +package com.webank.wedatasphere.dss.framework.workspace.client.impl + +import com.webank.wedatasphere.dss.framework.workspace.client.request.{GetWorkspaceRolesAction, GetWorkspaceUsersAction} +import com.webank.wedatasphere.dss.framework.workspace.client.response.{GetWorkspaceRolesResult, GetWorkspaceUsersResult} +import com.webank.wedatasphere.dss.framework.workspace.client.{AbstractRemoteClient, WorkSpaceRemoteClient} +import org.apache.linkis.httpclient.dws.DWSHttpClient +import org.apache.linkis.httpclient.dws.config.DWSClientConfig + +class LinkisWorkSpaceRemoteClient(clientConfig: DWSClientConfig) extends AbstractRemoteClient with WorkSpaceRemoteClient { + override protected val dwsHttpClient: DWSHttpClient = new DWSHttpClient(clientConfig, "WorkSpace-Client") + + override def getWorkspaceUsers(action: GetWorkspaceUsersAction): GetWorkspaceUsersResult = execute(action).asInstanceOf[GetWorkspaceUsersResult] + + override def getWorkspaceRoles(action: GetWorkspaceRolesAction): GetWorkspaceRolesResult = execute(action).asInstanceOf[GetWorkspaceRolesResult] + +} diff --git a/dss-framework/dss-framework-workspace-client/src/main/scala/com/webank/wedatasphere/dss/framework/workspace/client/request/GetWorkspaceRolesAction.scala b/dss-framework/dss-framework-workspace-client/src/main/scala/com/webank/wedatasphere/dss/framework/workspace/client/request/GetWorkspaceRolesAction.scala new file mode 100644 index 0000000000..feb3c01e42 --- /dev/null +++ b/dss-framework/dss-framework-workspace-client/src/main/scala/com/webank/wedatasphere/dss/framework/workspace/client/request/GetWorkspaceRolesAction.scala @@ -0,0 +1,42 @@ +package com.webank.wedatasphere.dss.framework.workspace.client.request + +import com.webank.wedatasphere.dss.framework.workspace.client.exception.WorkSpaceClientBuilderException +import org.apache.linkis.httpclient.request.GetAction + +class GetWorkspaceRolesAction extends GetAction with WorkSpaceAction{ + + private var user:String = _ + + override def setUser(user: String): Unit = this.user = user + + override def getUser: String = this.user + + override def suffixURLs: Array[String] = Array("dss", "framework","workspace", "getWorkspaceRoles") +} +object GetWorkspaceRolesAction{ + def builder(): Builder = new Builder + + class Builder private[GetWorkspaceRolesAction]() { + private var workspaceId:String = _ + private var user: String = _ + + def setUser(user: String): Builder ={ + this.user = user + this + } + def setWorkspaceId(workspaceId:String): Builder = { + this.workspaceId = workspaceId + this + } + + def build(): GetWorkspaceRolesAction = { + val action = new GetWorkspaceRolesAction + if(workspaceId == null) throw new WorkSpaceClientBuilderException("workspaceId is needed!") + action.setParameter("workspaceId",workspaceId) + action.setUser(user) + action + } + } +} + + diff --git a/dss-framework/dss-framework-workspace-client/src/main/scala/com/webank/wedatasphere/dss/framework/workspace/client/request/GetWorkspaceUsersAction.scala b/dss-framework/dss-framework-workspace-client/src/main/scala/com/webank/wedatasphere/dss/framework/workspace/client/request/GetWorkspaceUsersAction.scala new file mode 100644 index 0000000000..081ae714bd --- /dev/null +++ b/dss-framework/dss-framework-workspace-client/src/main/scala/com/webank/wedatasphere/dss/framework/workspace/client/request/GetWorkspaceUsersAction.scala @@ -0,0 +1,78 @@ +package com.webank.wedatasphere.dss.framework.workspace.client.request + +import com.webank.wedatasphere.dss.framework.workspace.client.exception.WorkSpaceClientBuilderException +import org.apache.linkis.httpclient.request.GetAction + +class GetWorkspaceUsersAction extends GetAction with WorkSpaceAction{ + private var user:String = _ + + override def setUser(user: String): Unit = this.user = user + + override def getUser: String = this.user + + override def suffixURLs: Array[String] = Array("dss", "framework","workspace", "getWorkspaceUsers") +} +object GetWorkspaceUsersAction{ + def builder(): Builder = new Builder + + class Builder private[GetWorkspaceUsersAction]() { + private var workspaceId:String = _ + private var pageNow:Integer=_ + private var pageSize:Integer=_ + private var department:String=_ + private var username:String=_ + private var roleName:String=_ + private var user: String = _ + + def setUser(user: String): Builder ={ + this.user = user + this + } + def setWorkspaceId(workspaceId:String): Builder = { + this.workspaceId = workspaceId + this + } + + def setPageNow(pageNow:Integer): Builder ={ + this.pageNow = pageNow + this + } + + def setPageSize(pageSize:Integer):Builder={ + this.pageSize = pageSize + this + } + + def setDepartment(department:String): Builder ={ + this.department = department + this + } + + def setRoleName(roleName:String):Builder={ + this.roleName = roleName + this + } + + def setUsername(username:String):Builder={ + this.username = username + this + } + + def build(): GetWorkspaceUsersAction = { + val action = new GetWorkspaceUsersAction + if(workspaceId == null) throw new WorkSpaceClientBuilderException("workspaceId is needed!") + if(user == null) throw new WorkSpaceClientBuilderException("user is needed!") + if(pageNow == null) pageNow = 1 + if(pageSize == null) pageSize = 10 + action.setUser(user) + action.setParameter("workspaceId",workspaceId) + action.setParameter("pageNow",pageNow) + action.setParameter("pageSize",pageSize) + if(department != null) action.setParameter("department",department) + if(roleName != null) action.setParameter("roleName",roleName) + if(username != null) action.setParameter("username",username) + + action + } + } +} diff --git a/dss-framework/dss-framework-workspace-client/src/main/scala/com/webank/wedatasphere/dss/framework/workspace/client/request/WorkSpaceAction.scala b/dss-framework/dss-framework-workspace-client/src/main/scala/com/webank/wedatasphere/dss/framework/workspace/client/request/WorkSpaceAction.scala new file mode 100644 index 0000000000..0be7262a59 --- /dev/null +++ b/dss-framework/dss-framework-workspace-client/src/main/scala/com/webank/wedatasphere/dss/framework/workspace/client/request/WorkSpaceAction.scala @@ -0,0 +1,5 @@ +package com.webank.wedatasphere.dss.framework.workspace.client.request + +import org.apache.linkis.httpclient.dws.request.DWSHttpAction + +trait WorkSpaceAction extends DWSHttpAction with org.apache.linkis.httpclient.request.UserAction diff --git a/dss-framework/dss-framework-workspace-client/src/main/scala/com/webank/wedatasphere/dss/framework/workspace/client/response/GetWorkspaceRolesResult.scala b/dss-framework/dss-framework-workspace-client/src/main/scala/com/webank/wedatasphere/dss/framework/workspace/client/response/GetWorkspaceRolesResult.scala new file mode 100644 index 0000000000..687998f611 --- /dev/null +++ b/dss-framework/dss-framework-workspace-client/src/main/scala/com/webank/wedatasphere/dss/framework/workspace/client/response/GetWorkspaceRolesResult.scala @@ -0,0 +1,22 @@ +package com.webank.wedatasphere.dss.framework.workspace.client.response + +import com.webank.wedatasphere.dss.framework.workspace.client.entity.DSSWorkspaceRole +import org.apache.linkis.httpclient.dws.DWSHttpClient +import org.apache.linkis.httpclient.dws.annotation.DWSHttpMessageResult +import org.apache.linkis.httpclient.dws.response.DWSResult + +import java.util +import scala.beans.BeanProperty + +@DWSHttpMessageResult("/api/rest_j/v\\d+/dss/framework/workspace/getWorkspaceRoles") +class GetWorkspaceRolesResult extends DWSResult{ + @BeanProperty var workspaceRoles:util.List[java.util.Map[String, Any]] = _ + + def getWorkspaceRoleList:util.List[DSSWorkspaceRole]={ + import scala.collection.JavaConverters._ + workspaceRoles.asScala.map(x=>{ + val str = DWSHttpClient.jacksonJson.writeValueAsString(x) + DWSHttpClient.jacksonJson.readValue(str, classOf[DSSWorkspaceRole]) + }).asJava + } +} diff --git a/dss-framework/dss-framework-workspace-client/src/main/scala/com/webank/wedatasphere/dss/framework/workspace/client/response/GetWorkspaceUsersResult.scala b/dss-framework/dss-framework-workspace-client/src/main/scala/com/webank/wedatasphere/dss/framework/workspace/client/response/GetWorkspaceUsersResult.scala new file mode 100644 index 0000000000..6c068e8887 --- /dev/null +++ b/dss-framework/dss-framework-workspace-client/src/main/scala/com/webank/wedatasphere/dss/framework/workspace/client/response/GetWorkspaceUsersResult.scala @@ -0,0 +1,32 @@ +package com.webank.wedatasphere.dss.framework.workspace.client.response + +import com.webank.wedatasphere.dss.framework.workspace.client.entity.{DSSWorkspaceRole, DSSWorkspaceUser} +import org.apache.linkis.httpclient.dws.DWSHttpClient +import org.apache.linkis.httpclient.dws.annotation.DWSHttpMessageResult +import org.apache.linkis.httpclient.dws.response.DWSResult + +import java.util +import scala.beans.BeanProperty + +@DWSHttpMessageResult("/api/rest_j/v\\d+/dss/framework/workspace/getWorkspaceUsers") +class GetWorkspaceUsersResult extends DWSResult{ + @BeanProperty var roles:util.List[java.util.Map[String, Any]] = _ + @BeanProperty var workspaceUsers:util.List[java.util.Map[String, Any]] = _ + @BeanProperty var total:Long = _ + + def getWorkspaceRoleList:util.List[DSSWorkspaceRole]={ + import scala.collection.JavaConverters._ + roles.asScala.map(x=>{ + val str = DWSHttpClient.jacksonJson.writeValueAsString(x) + DWSHttpClient.jacksonJson.readValue(str, classOf[DSSWorkspaceRole]) + }).asJava + } + + def getWorkspaceUserList:util.List[DSSWorkspaceUser]={ + import scala.collection.JavaConverters._ + workspaceUsers.asScala.map(x=>{ + val str = DWSHttpClient.jacksonJson.writeValueAsString(x) + DWSHttpClient.jacksonJson.readValue(str, classOf[DSSWorkspaceUser]) + }).asJava + } +} diff --git a/dss-framework/dss-framework-workspace-client/src/test/java/com/webank/wedatasphere/dss/framework/workspace/client/WorkSpaceRemoteClientTest.scala b/dss-framework/dss-framework-workspace-client/src/test/java/com/webank/wedatasphere/dss/framework/workspace/client/WorkSpaceRemoteClientTest.scala new file mode 100644 index 0000000000..76363881f7 --- /dev/null +++ b/dss-framework/dss-framework-workspace-client/src/test/java/com/webank/wedatasphere/dss/framework/workspace/client/WorkSpaceRemoteClientTest.scala @@ -0,0 +1,40 @@ +package com.webank.wedatasphere.dss.framework.workspace.client + +import com.webank.wedatasphere.dss.framework.workspace.client.impl.LinkisWorkSpaceRemoteClient +import com.webank.wedatasphere.dss.framework.workspace.client.request.{GetWorkspaceRolesAction, GetWorkspaceUsersAction} +import org.apache.linkis.httpclient.dws.authentication.StaticAuthenticationStrategy +import org.apache.linkis.httpclient.dws.config.DWSClientConfigBuilder + +import java.util.concurrent.TimeUnit + +object WorkSpaceRemoteClientTest { + def main(args: Array[String]): Unit = { + val clientConfig = DWSClientConfigBuilder.newBuilder() + .addServerUrl("http://localhost:8088") + .connectionTimeout(30000) + .discoveryEnabled(false) + .discoveryFrequency(1,TimeUnit.MINUTES) + .loadbalancerEnabled(true) + .maxConnectionSize(5) + .retryEnabled(false) + .readTimeout(30000) + .setAuthenticationStrategy(new StaticAuthenticationStrategy()) + .setAuthTokenKey("hdfs") + .setAuthTokenValue("hdfs") + .setDWSVersion("v1") + .build() + + val workSpaceRemoteClient = new LinkisWorkSpaceRemoteClient(clientConfig) + + val workspaceRoles = workSpaceRemoteClient.getWorkspaceRoles(GetWorkspaceRolesAction.builder().setUser("hdfs").setWorkspaceId("224").build()) + println(workspaceRoles.getWorkspaceRoleList) + + + val workspaceUsersResult = workSpaceRemoteClient.getWorkspaceUsers(GetWorkspaceUsersAction.builder().setUser("hdfs").setWorkspaceId("224").build()) + println(workspaceUsersResult.total) + println(workspaceUsersResult.getWorkspaceUserList) + println(workspaceUsersResult.getWorkspaceRoleList) + } + + +} diff --git a/dss-framework/pom.xml b/dss-framework/pom.xml index d48d3d929c..e95966dfff 100644 --- a/dss-framework/pom.xml +++ b/dss-framework/pom.xml @@ -35,6 +35,7 @@ framework-plugins/dss-framework-orchestrator-publish dss-framework-orchestrator-server dss-framework-project-server + dss-framework-workspace-client \ No newline at end of file diff --git a/pom.xml b/pom.xml index 2500454363..3068f7538a 100644 --- a/pom.xml +++ b/pom.xml @@ -35,6 +35,9 @@ plugins/azkaban/linkis-jobtype plugins/linkis/dss-gateway-support assembly + dss-apps/dss-dataasset-management + dss-apps/dss-datawarehouse-design + dss-apps/dss-datamodel-center diff --git a/web/.env b/web/.env index 4d2c4690ee..dc010e7160 100644 --- a/web/.env +++ b/web/.env @@ -2,4 +2,4 @@ VUE_APP_HOST= VUE_APP_MN_CONFIG_PREFIX= VUE_APP_MN_CONFIG_SOCKET=/ws/api/entrance/connect NODE_ENV=pro -VUE_APP_VERSION=v1.0.1 +VUE_APP_VERSION=v1.2.0 diff --git a/web/package.json b/web/package.json index 282ae5bd80..c01a8b676f 100755 --- a/web/package.json +++ b/web/package.json @@ -1,6 +1,6 @@ { "name": "dataspherestudio", - "version": "1.0.1", + "version": "1.2.0", "private": true, "scripts": { "serve": "vue-cli-service serve", @@ -29,8 +29,10 @@ ] }, "dependencies": { + "@antv/g6": "^4.5.3", "axios": "0.19.2", "babel-polyfill": "6.26.0", + "clipboard": "^2.0.8", "core-js": "2.6.11", "dexie": "2.0.4", "dt-sql-parser": "1.2.1", diff --git a/web/src/apps/apiServices/module/apiServices/index.vue b/web/src/apps/apiServices/module/apiServices/index.vue index 491ea52324..fa756de441 100755 --- a/web/src/apps/apiServices/module/apiServices/index.vue +++ b/web/src/apps/apiServices/module/apiServices/index.vue @@ -6,22 +6,22 @@ - {{$t("message.apiServices.label.name")}}： - + {{$t("message.apiServices.label.name")}}： + - {{$t("message.apiServices.label.status")}}： - + {{$t("message.apiServices.label.status")}}： + - {{$t("message.apiServices.label.submitter")}}： - + {{$t("message.apiServices.label.submitter")}}： + - +
@@ -60,18 +60,18 @@ v-for="(item, index) in pageDatalist" :key="item.id + `${index}`" > - - - - + + + + diff --git a/web/src/apps/apiServices/module/servicesMangement/component/APIInfo.vue b/web/src/apps/apiServices/module/servicesMangement/component/APIInfo.vue index 3b7554d08b..94ba1d995c 100755 --- a/web/src/apps/apiServices/module/servicesMangement/component/APIInfo.vue +++ b/web/src/apps/apiServices/module/servicesMangement/component/APIInfo.vue @@ -2,10 +2,10 @@
-
- {{ item.label }} - {{ item.value }} -
+
+ {{ item.label }} + {{ item.value }} +
diff --git a/web/src/apps/dataAssetManage/assets/styles/console.scss b/web/src/apps/dataAssetManage/assets/styles/console.scss new file mode 100644 index 0000000000..77b3484586 --- /dev/null +++ b/web/src/apps/dataAssetManage/assets/styles/console.scss @@ -0,0 +1,69 @@ +/*! + * Copyright 2019 WeBank + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +@import '@/common/style/variables.scss'; + +.console-page{ + position: $relative; + width: $percent-all; + height: $percent-all; + display: flex; + flex-direction: column; + background: $background-color-base; + padding: 20px; + overflow: hidden; + + .console-page-content-head { + display: flex; + justify-content: space-between; + + .console-page-content-title { + font-size: $font-size-large; + font-weight: bold; + } + } + + .console-page-content-body { + display: flex; + height: calc(100% - 68px); + .content-body-side-nav { + width: 200px; + + .content-body-card { + padding: 10px 0 10px; + } + } + .content-body-side-right { + flex: 1; + margin-left: 20px; + background: $body-background; + border-radius: $border-radius-small; + padding: 10px; + overflow: auto; + .content-body-side-right-title { + height: 30px; + border-bottom: $border-width-base $border-style-base $border-color-split; + margin-bottom: 10px + } + .content-body-side-right-content { + overflow: auto; + } + + } + } + +} diff --git a/web/src/apps/dataAssetManage/directive/clipboard/clipboard.js b/web/src/apps/dataAssetManage/directive/clipboard/clipboard.js new file mode 100644 index 0000000000..dd8ff67096 --- /dev/null +++ b/web/src/apps/dataAssetManage/directive/clipboard/clipboard.js @@ -0,0 +1,49 @@ +const Clipboard = require('clipboard'); + +if (!Clipboard) { + throw new Error('you should npm install `clipboard` --save at first '); +} + +export default { + bind(el, binding) { + if (binding.arg === 'success') { + el._v_clipboard_success = binding.value; + } else if (binding.arg === 'error') { + el._v_clipboard_error = binding.value; + } else { + const clipboard = new Clipboard(el, { + text() { return binding.value; }, + action() { return binding.arg === 'cut' ? 'cut' : 'copy'; } + }); + clipboard.on('success', (e) => { + const callback = el._v_clipboard_success; + callback && callback(e) // eslint-disable-line + }); + clipboard.on('error', (e) => { + const callback = el._v_clipboard_error; + callback && callback(e) // eslint-disable-line + }); + el._v_clipboard = clipboard; + } + }, + update(el, binding) { + if (binding.arg === 'success') { + el._v_clipboard_success = binding.value; + } else if (binding.arg === 'error') { + el._v_clipboard_error = binding.value; + } else { + el._v_clipboard.text = function () { return binding.value; }; + el._v_clipboard.action = function () { return binding.arg === 'cut' ? 'cut' : 'copy'; }; + } + }, + unbind(el, binding) { + if (binding.arg === 'success') { + delete el._v_clipboard_success; + } else if (binding.arg === 'error') { + delete el._v_clipboard_error; + } else { + el._v_clipboard.destroy(); + delete el._v_clipboard; + } + } +}; diff --git a/web/src/apps/dataAssetManage/directive/clipboard/index.js b/web/src/apps/dataAssetManage/directive/clipboard/index.js new file mode 100644 index 0000000000..3783c2d4a8 --- /dev/null +++ b/web/src/apps/dataAssetManage/directive/clipboard/index.js @@ -0,0 +1,13 @@ +import Clipboard from './clipboard'; + +const install = (Vue) => { + Vue.directive('Clipboard', Clipboard); +}; + +if (window.Vue) { + window.clipboard = Clipboard; + Vue.use(install); // eslint-disable-line +} + +Clipboard.install = install; +export default Clipboard; diff --git a/web/src/apps/dataAssetManage/i18n/en.json b/web/src/apps/dataAssetManage/i18n/en.json new file mode 100644 index 0000000000..1124b52349 --- /dev/null +++ b/web/src/apps/dataAssetManage/i18n/en.json @@ -0,0 +1,14 @@ +{ + "message": { + "dataAssetManage": { + "dataAssetManage": "Data Governance", + "dataAssets": "Data Assets", + "dataOverview": "Data Overview", + "dataAssetsIndex": "Data Assets Index", + "overallMeasurement": "Overall measurement", + "menu": "Menu", + "pleaseEnterATableName": "Please enter a table name / description to search the database table", + "search": "Search" + } + } +} diff --git a/web/src/apps/dataAssetManage/i18n/zh.json b/web/src/apps/dataAssetManage/i18n/zh.json new file mode 100644 index 0000000000..c68f94c5d9 --- /dev/null +++ b/web/src/apps/dataAssetManage/i18n/zh.json @@ -0,0 +1,14 @@ +{ + "message": { + "dataAssetManage": { + "dataAssetManage": "数据治理", + "dataAssets": "数据资产", + "dataOverview": "数据总览", + "dataAssetsIndex": "数据资产目录", + "overallMeasurement": "总体计量", + "menu": "目录", + "pleaseEnterATableName": "请输入表名/描述搜索数据库表", + "search": "搜索" + } + } +} diff --git a/web/src/apps/dataAssetManage/module/common/eventBus/event-bus.js b/web/src/apps/dataAssetManage/module/common/eventBus/event-bus.js new file mode 100644 index 0000000000..c5dbc39b13 --- /dev/null +++ b/web/src/apps/dataAssetManage/module/common/eventBus/event-bus.js @@ -0,0 +1,3 @@ +// event-bus.js +import Vue from 'vue' +export const EventBus = new Vue() diff --git a/web/src/apps/dataAssetManage/module/common/iCard.vue b/web/src/apps/dataAssetManage/module/common/iCard.vue new file mode 100644 index 0000000000..927326e7e0 --- /dev/null +++ b/web/src/apps/dataAssetManage/module/common/iCard.vue @@ -0,0 +1,74 @@ + + + + + diff --git a/web/src/apps/dataAssetManage/module/common/navMenu.vue b/web/src/apps/dataAssetManage/module/common/navMenu.vue new file mode 100644 index 0000000000..8aac2d29f4 --- /dev/null +++ b/web/src/apps/dataAssetManage/module/common/navMenu.vue @@ -0,0 +1,152 @@ + + + diff --git a/web/src/apps/dataAssetManage/module/common/tabCard/index.vue b/web/src/apps/dataAssetManage/module/common/tabCard/index.vue new file mode 100644 index 0000000000..abb2ee0049 --- /dev/null +++ b/web/src/apps/dataAssetManage/module/common/tabCard/index.vue @@ -0,0 +1,94 @@ + + + + + diff --git a/web/src/apps/dataAssetManage/module/common/tabList/index.scss b/web/src/apps/dataAssetManage/module/common/tabList/index.scss new file mode 100644 index 0000000000..be3cf6437b --- /dev/null +++ b/web/src/apps/dataAssetManage/module/common/tabList/index.scss @@ -0,0 +1,110 @@ +@import '@/common/style/variables.scss'; +.workflowTabContainer { + margin-left: 304px; + transition: margin-left .3s; + position: relative; + height: 100%; + box-sizing: border-box; + display: flex; + flex-direction: column; + &.tree-fold { + margin-left: 54px; + } + .tap-bar { + // background: #fff; + @include bg-color($workspace-body-bg-color, $dark-workspace-body-bg-color); + margin-bottom: $padding-25; + .topTabList { + padding: 25px $padding-25 0px; + flex: none; + display: flex; + justify-content: space-between; + align-items: center; + flex-wrap: wrap; + .topLeft { + flex: 1; + min-width: 320px; + .ivu-breadcrumb { + font-size: 21px; + /deep/.ivu-breadcrumb-item-separator { + // color: rgba(0,0,0,0.65); + @include font-color($light-text-color, $dark-text-color); + } + } + } + .buttonChange { + flex: none; + .ivu-btn { + font-size: 14px; + } + .button-group { + margin-left: 15px; + } + } + } + .bottomTapList { + padding: 0px $padding-25; + border-bottom: $border-width-base $border-style-base $border-color-base; + @include border-color($background-color-base, $dark-workspace-body-bg-color); + @include font-color($workspace-title-color, $dark-workspace-title-color); + margin-top: 12px; + flex: none; + display: flex; + align-items: center; + font-size: $font-size-large; + .bottomLeftText { + cursor: pointer; + flex: none; + font-size: $font-size-large; + padding: 0 15px; + margin-bottom: -1px; + line-height: 40px; + position: relative; + &::after { + content: ""; + border-left: 1px solid #DEE4EC; + @include border-color($border-color-base, $dark-border-color-base); + width: 0; + position: absolute; + right: -15px; + top: 12px; + height: 16px; + margin: 0 15px; + } + } + .active { + border-bottom: 2px solid $primary-color; + @include border-color($primary-color, $dark-primary-color); + } + .bottomRightContainer { + flex: 1; + height: 40px; + } + } + } + .defaultSlot { + flex: 1; + } +} + +.tab-item { + display: inline-block; + height: 40px; + line-height: 40px; + // color: $title-color; + @include font-color($workspace-title-color, $dark-workspace-title-color); + cursor: pointer; + min-width: 100px; + max-width: 200px; + overflow: hidden; + margin-right: 2px; + &.active { + height: 40px; + // color: $primary-color; + @include font-color($primary-color, $dark-primary-color); + border-radius: 4px 4px 0 0; + border-bottom: 2px solid $primary-color; + @include border-color($primary-color, $dark-primary-color); + line-height: 38px; + } +} \ No newline at end of file diff --git a/web/src/apps/dataAssetManage/module/common/tabList/index.vue b/web/src/apps/dataAssetManage/module/common/tabList/index.vue new file mode 100644 index 0000000000..f99b395608 --- /dev/null +++ b/web/src/apps/dataAssetManage/module/common/tabList/index.vue @@ -0,0 +1,175 @@ + + + diff --git a/web/src/apps/dataAssetManage/module/common/tabList/tabs.vue b/web/src/apps/dataAssetManage/module/common/tabList/tabs.vue new file mode 100644 index 0000000000..bea86179ae --- /dev/null +++ b/web/src/apps/dataAssetManage/module/common/tabList/tabs.vue @@ -0,0 +1,80 @@ + + + diff --git a/web/src/apps/dataAssetManage/module/common/title.vue b/web/src/apps/dataAssetManage/module/common/title.vue new file mode 100644 index 0000000000..fac3fff349 --- /dev/null +++ b/web/src/apps/dataAssetManage/module/common/title.vue @@ -0,0 +1,41 @@ + + + + + diff --git a/web/src/apps/dataAssetManage/module/common/tree/tree-item.vue b/web/src/apps/dataAssetManage/module/common/tree/tree-item.vue new file mode 100644 index 0000000000..7f5890c210 --- /dev/null +++ b/web/src/apps/dataAssetManage/module/common/tree/tree-item.vue @@ -0,0 +1,168 @@ + + + diff --git a/web/src/apps/dataAssetManage/module/common/tree/tree.vue b/web/src/apps/dataAssetManage/module/common/tree/tree.vue new file mode 100644 index 0000000000..2387e31164 --- /dev/null +++ b/web/src/apps/dataAssetManage/module/common/tree/tree.vue @@ -0,0 +1,121 @@ + + diff --git a/web/src/apps/dataAssetManage/module/common/treeMenu.vue b/web/src/apps/dataAssetManage/module/common/treeMenu.vue new file mode 100644 index 0000000000..5f279249d9 --- /dev/null +++ b/web/src/apps/dataAssetManage/module/common/treeMenu.vue @@ -0,0 +1,201 @@ + + + diff --git a/web/src/apps/dataAssetManage/module/components/tabCard/index.vue b/web/src/apps/dataAssetManage/module/components/tabCard/index.vue new file mode 100644 index 0000000000..27ff7346dd --- /dev/null +++ b/web/src/apps/dataAssetManage/module/components/tabCard/index.vue @@ -0,0 +1,81 @@ + + + + + diff --git a/web/src/apps/dataAssetManage/module/dataAssetManage/assetsIndex.vue b/web/src/apps/dataAssetManage/module/dataAssetManage/assetsIndex.vue new file mode 100644 index 0000000000..479e3d542c --- /dev/null +++ b/web/src/apps/dataAssetManage/module/dataAssetManage/assetsIndex.vue @@ -0,0 +1,230 @@ + + + diff --git a/web/src/apps/dataAssetManage/module/dataAssetManage/index.js b/web/src/apps/dataAssetManage/module/dataAssetManage/index.js new file mode 100644 index 0000000000..1b6a4f62a9 --- /dev/null +++ b/web/src/apps/dataAssetManage/module/dataAssetManage/index.js @@ -0,0 +1,10 @@ +export default { + name: 'dataAssetManage', + events: [], + dispatchs: { + }, + data: { + API_PATH: process.env.VUE_APP_MN_CONFIG_PREFIX || `http://${window.location.host}/api/rest_j/v1/`, + }, + component: () => import('./index.vue') +}; diff --git a/web/src/apps/dataAssetManage/module/dataAssetManage/index.vue b/web/src/apps/dataAssetManage/module/dataAssetManage/index.vue new file mode 100644 index 0000000000..78335d02c0 --- /dev/null +++ b/web/src/apps/dataAssetManage/module/dataAssetManage/index.vue @@ -0,0 +1,140 @@ + + + + diff --git a/web/src/apps/dataAssetManage/module/dataAssetManage/overview.vue b/web/src/apps/dataAssetManage/module/dataAssetManage/overview.vue new file mode 100644 index 0000000000..c7991de855 --- /dev/null +++ b/web/src/apps/dataAssetManage/module/dataAssetManage/overview.vue @@ -0,0 +1,186 @@ + + + diff --git a/web/src/apps/dataAssetManage/router.js b/web/src/apps/dataAssetManage/router.js new file mode 100644 index 0000000000..70ff6decfb --- /dev/null +++ b/web/src/apps/dataAssetManage/router.js @@ -0,0 +1,61 @@ +const routes = [ + { + name: "dataAssetManage", + path: "/dataAssetManage", + redirect: "/dataAssetManage/overview", + component: () => import("./view/layout/index.vue"), + meta: { + title: "数据资产", + publicPage: true, + icon: "ios-paper" + + }, + children: [ + { + name: "overview", + path: "/dataAssetManage/overview", + component: () => import("./module/dataAssetManage/overview.vue"), + meta: { + title: "数据总览", + publicPage: true, + icon: "ios-paper" + } + }, + { + name: "assets", + path: "/dataAssetManage/assets", + redirect: "/dataAssetManage/assets/search", + component: () => import("./module/dataAssetManage/assetsIndex.vue"), + meta: { + title: "数据资产目录", + publicPage: true, + cover: "assetsSearch" + }, + children: [ + { + name: "assetsSearch", + path: "/dataAssetManage/assets/search", + component: () => import("./view/assetsSearch/index.vue"), + meta: { + title: "数据资产目录", + publicPage: true, + icon: "ios-paper" + } + }, + { + name: "assetsInfo", + path: "/dataAssetManage/assets/info/:guid", + component: () => import("./view/assetsInfo/index.vue"), + meta: { + title: "数据资产详情", + publicPage: true, + cover: "assetsInfo" + } + } + ] + } + ] + } +]; + +export default routes; diff --git a/web/src/apps/dataAssetManage/service/api.js b/web/src/apps/dataAssetManage/service/api.js new file mode 100644 index 0000000000..8e776b9638 --- /dev/null +++ b/web/src/apps/dataAssetManage/service/api.js @@ -0,0 +1,123 @@ +import API_PATH from '@/common/config/apiPath.js' +import api from '@/common/service/api' + +/** + * 数据资产概要 + * @returns {Object.result} + * + */ +export const getHiveSummary = () => + api.fetch(`${API_PATH.DATA_GOVERNANCE}hiveSummary`, 'get') + +/** + * 查询hive表--基础&列 + * @param {*} guid + * @returns + */ +export const getHiveTblBasic = guid => + api.fetch(`${API_PATH.DATA_GOVERNANCE}hiveTbl/${guid}/basic`, {}, 'get') + +/** + * 查询hive表-分区信息 + * @param {*} guid + * @returns + */ +export const getHiveTblPartition = guid => + api.fetch(`${API_PATH.DATA_GOVERNANCE}hiveTbl/${guid}/partition`, {}, 'get') + +/** + * 查询hive表--select语句 + * @param {*} guid + * @returns + */ +export const getSelectSql = guid => + api.fetch(`${API_PATH.DATA_GOVERNANCE}hiveTbl/${guid}/select`, {}, 'get') + +/** + * 查询hive表--create语句 + * @param {*} guid + * @returns + */ +export const getSelectDdl = guid => + api.fetch(`${API_PATH.DATA_GOVERNANCE}hiveTbl/${guid}/create`, {}, 'get') + +/** + * 搜索hive表 + * @param {query} + * @returns + */ +export const getHiveTbls = params => + api.fetch( + `${API_PATH.DATA_GOVERNANCE}hiveTbl/search?query=${params.query}&owner=${params.owner}&label=${params.label}&limit=${params.limit}&offset=${params.offset}`, + {}, + 'get' + ) + +/* + * 查询hive表--血缘 + * @param {*} guid + * @returnsL + */ +export const getLineage = guid => + api.fetch(`${API_PATH.DATA_GOVERNANCE}hiveTbl/${guid}/lineage`, {}, 'get') + +/** + * 批量修改注释 + * @params {Map} + * @returns + */ +export const putCommetBulk = params => { + return api.fetch(`${API_PATH.DATA_GOVERNANCE}comment/bulk`, params, 'put') +} + +/** + * 存储量前10表 + * @params {void} + * @returns Array + */ +export const getTopStorage = () => + api.fetch(`${API_PATH.DATA_GOVERNANCE}hiveTbl/topStorage`, {}, 'get') + +/** + * 设置标签--表或列 + * @params {guid} + * @returns String + */ +export const postSetLabel = (guid, params) => + api.fetch(`${API_PATH.DATA_GOVERNANCE}label/${guid}`, params, 'post') + +/** + * 修改注释--表或列 + * @params {guid, Obiect} + * @returns String + */ +export const postSetComment = (guid, comment) => + api.fetch( + `${API_PATH.DATA_GOVERNANCE}comment/${guid}?comment=${comment}`, + {}, + 'put' + ) + +/** + * 负责人查询 + * @params {workspaceId} + * @returns Array + */ +export const getWorkspaceUsers = (workspackId) => + api.fetch( + `${API_PATH.DATA_GOVERNANCE}users/${workspackId}`, + {}, + 'get' + ) + +/** + * 标签查询 + * @params {workspaceId} + * @returns Array + */ +export const getLabels = (search) => + api.fetch( + `${API_PATH.DATA_GOVERNANCE}labels/search?query=${search}`, + {}, + 'get' + ) diff --git a/web/src/apps/dataAssetManage/service/db/node.js b/web/src/apps/dataAssetManage/service/db/node.js new file mode 100644 index 0000000000..8fce1d1d1b --- /dev/null +++ b/web/src/apps/dataAssetManage/service/db/node.js @@ -0,0 +1,103 @@ +/* + * Copyright 2019 WeBank + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +import {isNil} from 'lodash'; +import { Basic } from '@/common/service/db/index.js'; +import { isArray } from 'util'; +/** + * @class Globalcache + * @extends {Basic} + */ +class Node extends Basic { + /** + *Creates an instance of Globalcache. + * @param {*} table + * @param {*} db + * @memberof Globalcache + */ + constructor(table) { + super(table); + } + + /** + * @param {*} nodeId + * @param {*} key + * @return {*} + */ + async getNodeCache({nodeId, key}) { + let caches = await this.get(nodeId); + if (key) { + return caches[0][key]; + } + return caches[0]; + } + + /** + * @param {*} nodeId + * @param {*} value + * @return {*} + */ + async addNodeCache({nodeId, value}) { + let cache = await this.getNodeCache({nodeId}); + if (!isNil(cache)) { + return this.update(nodeId, value) + } + return this.add(Object.assign({nodeId}, value)); + } + + + /** + * @param {*} nodeId + * @param {*} key + * @param {*} value + * @return {*} + */ + async updateNodeCache({nodeId, key, value}) { + let cache = {}; + if (key) { + if (isArray(key)) { + key.forEach((k) => { + cache[k] = value[k]; + }) + } else { + cache[key] = value[key]; + } + } else { + cache = value; + } + return nodeId ? this.update(nodeId, cache) : null; + } + + + /** + * @param {*} args + * @return {*} + */ + async removeNodeCache({nodeId, key}) { + let cache = await this.getNodeCache({nodeId}); + if (isNil(cache)) { + if (key) { + cache[key] = null; + return this.update(nodeId, cache); + } + return this.remove(nodeId); + } + } +} +const node = new Node('node'); + +export default node; diff --git a/web/src/apps/dataAssetManage/service/db/project.js b/web/src/apps/dataAssetManage/service/db/project.js new file mode 100644 index 0000000000..8a7ed72012 --- /dev/null +++ b/web/src/apps/dataAssetManage/service/db/project.js @@ -0,0 +1,118 @@ +/* + * Copyright 2019 WeBank + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +import {isNil} from 'lodash'; +import { Basic } from '@/common/service/db/index.js'; +import { isArray } from 'util'; +/** + * @class Globalcache + * @extends {Basic} + */ +class Project extends Basic { + /** + *Creates an instance of Globalcache. + * @param {*} table + * @param {*} db + * @memberof Globalcache + */ + constructor(table) { + super(table); + } + + /** + * @param {*} projectID + * @param {*} key + * @return {*} + */ + async getProjectCache({projectID, key}) { + projectID = Number(projectID); + let caches = await this.get(projectID); + if (key) { + return caches[0][key]; + } + return caches[0]; + } + + /** + * @param {*} projectID + * @param {*} value + * @return {*} + */ + async addProjectCache({projectID, value}) { + projectID = Number(projectID); + let cache = await this.getProjectCache({projectID}); + if (!isNil(cache)) { + return this.update(projectID, value) + } + return this.add(Object.assign({projectID}, value)); + } + + + /** + * @param {*} projectID + * @param {*} key + * @param {*} value token是用于第二层比对的字段，sKey是实际写入的key，sValue是用于比对的值 + * @param {*} isDeep 是否要对第二层数据进行修改，如果需要的话，就需要在value中携带 + * @return {*} + */ + async updateProjectCache({projectID, key, value, isDeep}) { + projectID = Number(projectID); + let cache = {}; + if (key) { + if (isDeep) { + cache = await this.getProjectCache({projectID}) || {}; + let arr = cache && cache[key] || []; + arr.forEach((item) => { + if (item[value.token] === Number(value.sValue)) { + item[value.sKey] = value[value.sKey]; + } + }) + } else { + if (isArray(key)) { + key.forEach((k) => { + cache[k] = value[k]; + }) + } else { + cache[key] = value[key]; + } + } + } else { + cache = value; + } + return this.update(projectID, cache); + } + + + /** + * @param {*} args + * @return {*} + */ + async removeProjectCache({projectID, key}) { + projectID = Number(projectID); + let cache = await this.getProjectCache({projectID}); + if (isNil(cache)) { + if (key) { + cache[key] = null; + return this.update(projectID, cache); + } + return this.remove(projectID); + } + } +} +const project = new Project('project'); + +export default project; diff --git a/web/src/apps/dataAssetManage/utils/clipboard.js b/web/src/apps/dataAssetManage/utils/clipboard.js new file mode 100644 index 0000000000..7efc01c495 --- /dev/null +++ b/web/src/apps/dataAssetManage/utils/clipboard.js @@ -0,0 +1,43 @@ +import Vue from 'vue'; +import Clipboard from 'clipboard'; + +/** + *@returns + */ +function clipboardSuccess() { + Vue.prototype.$Message.success({ + content: '复制成功', + duration: 1.5 + }); +} + + +/** + *@returns + */ +function clipboardError() { + Vue.prototype.$Message.success({ + message: '复制失败', + duration: 1.5 + }); +} + +/** + * 把字符串中的内容放到剪切板上 + * @param {String, Object} + * @param {void} + */ +export default function handleClipboard(text, event) { + const clipboard = new Clipboard(event.target, { + text: () => text + }); + clipboard.on('success', () => { + clipboardSuccess(); + clipboard.destroy(); + }); + clipboard.on('error', () => { + clipboardError(); + clipboard.destroy(); + }); + clipboard.onClick(event); +} diff --git a/web/src/apps/dataAssetManage/utils/fomatSQL.js b/web/src/apps/dataAssetManage/utils/fomatSQL.js new file mode 100644 index 0000000000..caba9997b5 --- /dev/null +++ b/web/src/apps/dataAssetManage/utils/fomatSQL.js @@ -0,0 +1,23 @@ +/** + * 转换 后端SQL语句 成 html文本 + * @param (String) + * @return (String) + */ +export function fomatSqlForShow(sql) { + if ( typeof sql !== 'string' ) return '' + const arr = sql.split('@$'); + const res = arr.map((i, idx) => `

${idx+1}${i}

`) + return res.join('') +} + +/** + * + * @param {String} sql + * @return {String} + */ +export function fomatSqlForCopy(sql) { + if ( typeof sql !== 'string' ) return '' + const arr = sql.replaceAll(' ', '').split('@$'); + const res = arr.map(i => `${i}\r\n`) + return res.join('') +} diff --git a/web/src/apps/dataAssetManage/utils/formatDate.js b/web/src/apps/dataAssetManage/utils/formatDate.js new file mode 100644 index 0000000000..d3d68b5957 --- /dev/null +++ b/web/src/apps/dataAssetManage/utils/formatDate.js @@ -0,0 +1,11 @@ +import moment from 'moment' + +/** + * 转换时间戳到字符串 + * @param (Number) + * @return (String) + */ +export default function formatDate (value) { + if (!value) return '' + return moment(value).format("YYYY-MM-DD HH:mm") +} diff --git a/web/src/apps/dataAssetManage/utils/storage.js b/web/src/apps/dataAssetManage/utils/storage.js new file mode 100644 index 0000000000..2ab293c3b9 --- /dev/null +++ b/web/src/apps/dataAssetManage/utils/storage.js @@ -0,0 +1,11 @@ +export const storage = { + setItem(key, val) { + return sessionStorage.setItem(key, val) + }, + getItem(key) { + return sessionStorage.getItem(key) + }, + removeItem(key) { + return sessionStorage.removeItem(key) + } +} diff --git a/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/AtlasServer.png b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/AtlasServer.png new file mode 100644 index 0000000000..56c6f5889e Binary files /dev/null and b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/AtlasServer.png differ diff --git a/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/DataSet.png b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/DataSet.png new file mode 100644 index 0000000000..aa899dea20 Binary files /dev/null and b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/DataSet.png differ diff --git a/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/avro.png b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/avro.png new file mode 100644 index 0000000000..a43a2dabfe Binary files /dev/null and b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/avro.png differ diff --git a/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/aws.png b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/aws.png new file mode 100644 index 0000000000..79097bd920 Binary files /dev/null and b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/aws.png differ diff --git a/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/azure.png b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/azure.png new file mode 100644 index 0000000000..b6d6e89e77 Binary files /dev/null and b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/azure.png differ diff --git a/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/db.png b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/db.png new file mode 100644 index 0000000000..489fcd93c8 Binary files /dev/null and b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/db.png differ diff --git a/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/AtlasServer.png b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/AtlasServer.png new file mode 100644 index 0000000000..ab3a124795 Binary files /dev/null and b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/AtlasServer.png differ diff --git a/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/DataSet.png b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/DataSet.png new file mode 100644 index 0000000000..5ed51de759 Binary files /dev/null and b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/DataSet.png differ diff --git a/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/avro.png b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/avro.png new file mode 100644 index 0000000000..c9a6172cf6 Binary files /dev/null and b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/avro.png differ diff --git a/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/aws.png b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/aws.png new file mode 100644 index 0000000000..d9be6f3ea0 Binary files /dev/null and b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/aws.png differ diff --git a/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/azure.png b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/azure.png new file mode 100644 index 0000000000..720f3ac591 Binary files /dev/null and b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/azure.png differ diff --git a/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/db.png b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/db.png new file mode 100644 index 0000000000..982c6867a2 Binary files /dev/null and b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/db.png differ diff --git a/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/falcon.png b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/falcon.png new file mode 100644 index 0000000000..ef8cd242eb Binary files /dev/null and b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/falcon.png differ diff --git a/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/falcon_feed_creation.png b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/falcon_feed_creation.png new file mode 100644 index 0000000000..f740467883 Binary files /dev/null and b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/falcon_feed_creation.png differ diff --git a/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/falcon_feed_replication.png b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/falcon_feed_replication.png new file mode 100644 index 0000000000..af492dabe6 Binary files /dev/null and b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/falcon_feed_replication.png differ diff --git a/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/falcon_process.png b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/falcon_process.png new file mode 100644 index 0000000000..075628c589 Binary files /dev/null and b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/falcon_process.png differ diff --git a/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/file_system.png b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/file_system.png new file mode 100644 index 0000000000..8b4e50416c Binary files /dev/null and b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/file_system.png differ diff --git a/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/flink.png b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/flink.png new file mode 100644 index 0000000000..af78144be1 Binary files /dev/null and b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/flink.png differ diff --git a/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/flink_application.png b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/flink_application.png new file mode 100644 index 0000000000..af78144be1 Binary files /dev/null and b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/flink_application.png differ diff --git a/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/flink_application_processes.png b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/flink_application_processes.png new file mode 100644 index 0000000000..28d4ed9168 Binary files /dev/null and b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/flink_application_processes.png differ diff --git a/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/flink_process.png b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/flink_process.png new file mode 100644 index 0000000000..af78144be1 Binary files /dev/null and b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/flink_process.png differ diff --git a/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/gcp.png b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/gcp.png new file mode 100644 index 0000000000..e7d83fe36b Binary files /dev/null and b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/gcp.png differ diff --git a/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/hadoop.png b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/hadoop.png new file mode 100644 index 0000000000..0334fc7d78 Binary files /dev/null and b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/hadoop.png differ diff --git a/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/hbase.png b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/hbase.png new file mode 100644 index 0000000000..fa6e21427d Binary files /dev/null and b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/hbase.png differ diff --git a/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/hbase_column.png b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/hbase_column.png new file mode 100644 index 0000000000..ef454267bf Binary files /dev/null and b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/hbase_column.png differ diff --git a/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/hbase_column_family.png b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/hbase_column_family.png new file mode 100644 index 0000000000..0182ec3e0f Binary files /dev/null and b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/hbase_column_family.png differ diff --git a/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/hbase_table.png b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/hbase_table.png new file mode 100644 index 0000000000..556a10a14d Binary files /dev/null and b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/hbase_table.png differ diff --git a/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/hdfs_path.png b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/hdfs_path.png new file mode 100644 index 0000000000..8b4e50416c Binary files /dev/null and b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/hdfs_path.png differ diff --git a/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/hive.png b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/hive.png new file mode 100644 index 0000000000..227f5b3a31 Binary files /dev/null and b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/hive.png differ diff --git a/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/hive_column.png b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/hive_column.png new file mode 100644 index 0000000000..35fbe48234 Binary files /dev/null and b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/hive_column.png differ diff --git a/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/hive_column_lineage.png b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/hive_column_lineage.png new file mode 100644 index 0000000000..c08f4b4200 Binary files /dev/null and b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/hive_column_lineage.png differ diff --git a/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/hive_db.png b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/hive_db.png new file mode 100644 index 0000000000..76b5c3f8c5 Binary files /dev/null and b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/hive_db.png differ diff --git a/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/hive_process.png b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/hive_process.png new file mode 100644 index 0000000000..5cc6d78c6d Binary files /dev/null and b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/hive_process.png differ diff --git a/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/hive_process_execution.png b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/hive_process_execution.png new file mode 100644 index 0000000000..ec1be80618 Binary files /dev/null and b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/hive_process_execution.png differ diff --git a/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/hive_storagedesc.png b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/hive_storagedesc.png new file mode 100644 index 0000000000..c3930a43f4 Binary files /dev/null and b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/hive_storagedesc.png differ diff --git a/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/hive_table.png b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/hive_table.png new file mode 100644 index 0000000000..b22e194d83 Binary files /dev/null and b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/hive_table.png differ diff --git a/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/impala.png b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/impala.png new file mode 100644 index 0000000000..c94be973b7 Binary files /dev/null and b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/impala.png differ diff --git a/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/impala_column_lineage.png b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/impala_column_lineage.png new file mode 100644 index 0000000000..7ef403bc4d Binary files /dev/null and b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/impala_column_lineage.png differ diff --git a/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/impala_process.png b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/impala_process.png new file mode 100644 index 0000000000..7ef403bc4d Binary files /dev/null and b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/impala_process.png differ diff --git a/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/jms_topic.png b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/jms_topic.png new file mode 100644 index 0000000000..54a02c8f44 Binary files /dev/null and b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/jms_topic.png differ diff --git a/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/kafka.png b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/kafka.png new file mode 100644 index 0000000000..ef941d3445 Binary files /dev/null and b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/kafka.png differ diff --git a/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/kafka_consumer.png b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/kafka_consumer.png new file mode 100644 index 0000000000..d7e564f0fa Binary files /dev/null and b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/kafka_consumer.png differ diff --git a/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/kafka_consumer_group.png b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/kafka_consumer_group.png new file mode 100644 index 0000000000..1bc9dadfc2 Binary files /dev/null and b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/kafka_consumer_group.png differ diff --git a/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/kafka_consumer_lineage.png b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/kafka_consumer_lineage.png new file mode 100644 index 0000000000..80248e9f07 Binary files /dev/null and b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/kafka_consumer_lineage.png differ diff --git a/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/kafka_producer.png b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/kafka_producer.png new file mode 100644 index 0000000000..372cff48f1 Binary files /dev/null and b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/kafka_producer.png differ diff --git a/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/kafka_producer_lineage.png b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/kafka_producer_lineage.png new file mode 100644 index 0000000000..c312b54807 Binary files /dev/null and b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/kafka_producer_lineage.png differ diff --git a/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/kafka_topic.png b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/kafka_topic.png new file mode 100644 index 0000000000..84cb234161 Binary files /dev/null and b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/kafka_topic.png differ diff --git a/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/ozone.png b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/ozone.png new file mode 100644 index 0000000000..f498c2d149 Binary files /dev/null and b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/ozone.png differ diff --git a/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/ozone_bucket.png b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/ozone_bucket.png new file mode 100644 index 0000000000..5b0d5d159e Binary files /dev/null and b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/ozone_bucket.png differ diff --git a/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/ozone_key.png b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/ozone_key.png new file mode 100644 index 0000000000..4dba49172d Binary files /dev/null and b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/ozone_key.png differ diff --git a/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/ozone_volume.png b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/ozone_volume.png new file mode 100644 index 0000000000..1e4326b22a Binary files /dev/null and b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/ozone_volume.png differ diff --git a/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/process.png b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/process.png new file mode 100644 index 0000000000..88bc1ae758 Binary files /dev/null and b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/process.png differ diff --git a/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/rdbms.png b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/rdbms.png new file mode 100644 index 0000000000..84f20bae4f Binary files /dev/null and b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/rdbms.png differ diff --git a/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/rdbms_column.png b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/rdbms_column.png new file mode 100644 index 0000000000..200a288216 Binary files /dev/null and b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/rdbms_column.png differ diff --git a/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/rdbms_db.png b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/rdbms_db.png new file mode 100644 index 0000000000..6f1d1690f8 Binary files /dev/null and b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/rdbms_db.png differ diff --git a/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/rdbms_table.png b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/rdbms_table.png new file mode 100644 index 0000000000..e49fc68f84 Binary files /dev/null and b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/rdbms_table.png differ diff --git a/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/spark.png b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/spark.png new file mode 100644 index 0000000000..93f8a4d860 Binary files /dev/null and b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/spark.png differ diff --git a/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/spark_process.png b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/spark_process.png new file mode 100644 index 0000000000..bf191217c1 Binary files /dev/null and b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/spark_process.png differ diff --git a/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/spark_table.png b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/spark_table.png new file mode 100644 index 0000000000..1404026ae4 Binary files /dev/null and b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/spark_table.png differ diff --git a/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/sqoop.png b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/sqoop.png new file mode 100644 index 0000000000..359b77edf9 Binary files /dev/null and b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/sqoop.png differ diff --git a/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/sqoop_db.png b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/sqoop_db.png new file mode 100644 index 0000000000..970ab9a6be Binary files /dev/null and b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/sqoop_db.png differ diff --git a/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/sqoop_process.png b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/sqoop_process.png new file mode 100644 index 0000000000..aae4d56092 Binary files /dev/null and b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/sqoop_process.png differ diff --git a/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/storm.png b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/storm.png new file mode 100644 index 0000000000..cab3f7f021 Binary files /dev/null and b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/storm.png differ diff --git a/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/storm_bolt.png b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/storm_bolt.png new file mode 100644 index 0000000000..f35add3ac5 Binary files /dev/null and b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/storm_bolt.png differ diff --git a/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/storm_spout.png b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/storm_spout.png new file mode 100644 index 0000000000..841375170a Binary files /dev/null and b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/storm_spout.png differ diff --git a/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/storm_topology.png b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/storm_topology.png new file mode 100644 index 0000000000..224a2876fb Binary files /dev/null and b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/storm_topology.png differ diff --git a/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/table.png b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/table.png new file mode 100644 index 0000000000..de1e5cdff0 Binary files /dev/null and b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/disabled/table.png differ diff --git a/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/falcon.png b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/falcon.png new file mode 100644 index 0000000000..ab1c68b15a Binary files /dev/null and b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/falcon.png differ diff --git a/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/falcon_feed_creation.png b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/falcon_feed_creation.png new file mode 100644 index 0000000000..714e31930e Binary files /dev/null and b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/falcon_feed_creation.png differ diff --git a/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/falcon_feed_replication.png b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/falcon_feed_replication.png new file mode 100644 index 0000000000..714e31930e Binary files /dev/null and b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/falcon_feed_replication.png differ diff --git a/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/falcon_process.png b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/falcon_process.png new file mode 100644 index 0000000000..714e31930e Binary files /dev/null and b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/falcon_process.png differ diff --git a/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/file_system.png b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/file_system.png new file mode 100644 index 0000000000..14233ea099 Binary files /dev/null and b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/file_system.png differ diff --git a/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/flink.png b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/flink.png new file mode 100644 index 0000000000..07c5c1473b Binary files /dev/null and b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/flink.png differ diff --git a/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/flink_application.png b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/flink_application.png new file mode 100644 index 0000000000..07c5c1473b Binary files /dev/null and b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/flink_application.png differ diff --git a/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/flink_application_processes.png b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/flink_application_processes.png new file mode 100644 index 0000000000..a8af7322ca Binary files /dev/null and b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/flink_application_processes.png differ diff --git a/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/flink_process.png b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/flink_process.png new file mode 100644 index 0000000000..07c5c1473b Binary files /dev/null and b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/flink_process.png differ diff --git a/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/gcp.png b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/gcp.png new file mode 100644 index 0000000000..567362875a Binary files /dev/null and b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/gcp.png differ diff --git a/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/hadoop.png b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/hadoop.png new file mode 100644 index 0000000000..f8394167bd Binary files /dev/null and b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/hadoop.png differ diff --git a/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/hbase.png b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/hbase.png new file mode 100644 index 0000000000..9f788b8d55 Binary files /dev/null and b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/hbase.png differ diff --git a/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/hbase_column.png b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/hbase_column.png new file mode 100644 index 0000000000..b01300c091 Binary files /dev/null and b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/hbase_column.png differ diff --git a/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/hbase_column_family.png b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/hbase_column_family.png new file mode 100644 index 0000000000..b01300c091 Binary files /dev/null and b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/hbase_column_family.png differ diff --git a/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/hbase_table.png b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/hbase_table.png new file mode 100644 index 0000000000..92a0b78fdf Binary files /dev/null and b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/hbase_table.png differ diff --git a/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/hdfs_path.png b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/hdfs_path.png new file mode 100644 index 0000000000..14233ea099 Binary files /dev/null and b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/hdfs_path.png differ diff --git a/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/hive.png b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/hive.png new file mode 100644 index 0000000000..0a2eb2042a Binary files /dev/null and b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/hive.png differ diff --git a/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/hive_column.png b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/hive_column.png new file mode 100644 index 0000000000..4abdfc71aa Binary files /dev/null and b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/hive_column.png differ diff --git a/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/hive_column_lineage.png b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/hive_column_lineage.png new file mode 100644 index 0000000000..f8a9f040da Binary files /dev/null and b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/hive_column_lineage.png differ diff --git a/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/hive_db.png b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/hive_db.png new file mode 100644 index 0000000000..c47836caa5 Binary files /dev/null and b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/hive_db.png differ diff --git a/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/hive_process.png b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/hive_process.png new file mode 100644 index 0000000000..f8a9f040da Binary files /dev/null and b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/hive_process.png differ diff --git a/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/hive_process_execution.png b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/hive_process_execution.png new file mode 100644 index 0000000000..5b2ea450af Binary files /dev/null and b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/hive_process_execution.png differ diff --git a/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/hive_storagedesc.png b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/hive_storagedesc.png new file mode 100644 index 0000000000..1a9692984b Binary files /dev/null and b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/hive_storagedesc.png differ diff --git a/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/hive_table.png b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/hive_table.png new file mode 100644 index 0000000000..144081f939 Binary files /dev/null and b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/hive_table.png differ diff --git a/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/hourglass.svg b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/hourglass.svg new file mode 100644 index 0000000000..aad49788c0 --- /dev/null +++ b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/hourglass.svg @@ -0,0 +1,3 @@ + + + diff --git a/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/impala.png b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/impala.png new file mode 100644 index 0000000000..6711fc60c0 Binary files /dev/null and b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/impala.png differ diff --git a/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/impala_column_lineage.png b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/impala_column_lineage.png new file mode 100644 index 0000000000..d3db7e6159 Binary files /dev/null and b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/impala_column_lineage.png differ diff --git a/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/impala_process.png b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/impala_process.png new file mode 100644 index 0000000000..d3db7e6159 Binary files /dev/null and b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/impala_process.png differ diff --git a/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/jms_topic.png b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/jms_topic.png new file mode 100644 index 0000000000..b215f0263f Binary files /dev/null and b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/jms_topic.png differ diff --git a/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/kafka.png b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/kafka.png new file mode 100644 index 0000000000..fd479c332d Binary files /dev/null and b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/kafka.png differ diff --git a/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/kafka_consumer.png b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/kafka_consumer.png new file mode 100644 index 0000000000..459143c4d1 Binary files /dev/null and b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/kafka_consumer.png differ diff --git a/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/kafka_consumer_group.png b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/kafka_consumer_group.png new file mode 100644 index 0000000000..157dcb422f Binary files /dev/null and b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/kafka_consumer_group.png differ diff --git a/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/kafka_consumer_lineage.png b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/kafka_consumer_lineage.png new file mode 100644 index 0000000000..ebf891983a Binary files /dev/null and b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/kafka_consumer_lineage.png differ diff --git a/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/kafka_producer.png b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/kafka_producer.png new file mode 100644 index 0000000000..0844afe239 Binary files /dev/null and b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/kafka_producer.png differ diff --git a/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/kafka_producer_lineage.png b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/kafka_producer_lineage.png new file mode 100644 index 0000000000..15a960e19c Binary files /dev/null and b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/kafka_producer_lineage.png differ diff --git a/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/kafka_topic.png b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/kafka_topic.png new file mode 100644 index 0000000000..2556f3a68c Binary files /dev/null and b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/kafka_topic.png differ diff --git a/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/ozone.png b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/ozone.png new file mode 100644 index 0000000000..6004dc6b74 Binary files /dev/null and b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/ozone.png differ diff --git a/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/ozone_bucket.png b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/ozone_bucket.png new file mode 100644 index 0000000000..acee870507 Binary files /dev/null and b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/ozone_bucket.png differ diff --git a/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/ozone_key.png b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/ozone_key.png new file mode 100644 index 0000000000..b38745634a Binary files /dev/null and b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/ozone_key.png differ diff --git a/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/ozone_volume.png b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/ozone_volume.png new file mode 100644 index 0000000000..9222507232 Binary files /dev/null and b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/ozone_volume.png differ diff --git a/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/process.png b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/process.png new file mode 100644 index 0000000000..fb1aab2104 Binary files /dev/null and b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/process.png differ diff --git a/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/rdbms.png b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/rdbms.png new file mode 100644 index 0000000000..cc3cb4fd12 Binary files /dev/null and b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/rdbms.png differ diff --git a/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/rdbms_column.png b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/rdbms_column.png new file mode 100644 index 0000000000..9d3c598caa Binary files /dev/null and b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/rdbms_column.png differ diff --git a/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/rdbms_db.png b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/rdbms_db.png new file mode 100644 index 0000000000..ff40a4d290 Binary files /dev/null and b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/rdbms_db.png differ diff --git a/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/rdbms_table.png b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/rdbms_table.png new file mode 100644 index 0000000000..174221fef1 Binary files /dev/null and b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/rdbms_table.png differ diff --git a/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/spark.png b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/spark.png new file mode 100644 index 0000000000..178d5a6f5b Binary files /dev/null and b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/spark.png differ diff --git a/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/spark_process.png b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/spark_process.png new file mode 100644 index 0000000000..030da23995 Binary files /dev/null and b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/spark_process.png differ diff --git a/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/spark_table.png b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/spark_table.png new file mode 100644 index 0000000000..78db10930f Binary files /dev/null and b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/spark_table.png differ diff --git a/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/sqoop.png b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/sqoop.png new file mode 100644 index 0000000000..dec730b445 Binary files /dev/null and b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/sqoop.png differ diff --git a/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/sqoop_db.png b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/sqoop_db.png new file mode 100644 index 0000000000..d219127b7f Binary files /dev/null and b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/sqoop_db.png differ diff --git a/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/sqoop_process.png b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/sqoop_process.png new file mode 100644 index 0000000000..21eebadf17 Binary files /dev/null and b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/sqoop_process.png differ diff --git a/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/storm.png b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/storm.png new file mode 100644 index 0000000000..4e95d5d40a Binary files /dev/null and b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/storm.png differ diff --git a/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/storm_bolt.png b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/storm_bolt.png new file mode 100644 index 0000000000..7ed692ff44 Binary files /dev/null and b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/storm_bolt.png differ diff --git a/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/storm_spout.png b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/storm_spout.png new file mode 100644 index 0000000000..111a11ce68 Binary files /dev/null and b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/storm_spout.png differ diff --git a/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/storm_topology.png b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/storm_topology.png new file mode 100644 index 0000000000..7642fe0db0 Binary files /dev/null and b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/storm_topology.png differ diff --git a/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/table.png b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/table.png new file mode 100644 index 0000000000..f1672fc089 Binary files /dev/null and b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/icon/table.png differ diff --git a/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/index.scss b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/index.scss new file mode 100644 index 0000000000..d5a783420f --- /dev/null +++ b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/index.scss @@ -0,0 +1,47 @@ +.dagreLayout-page{ + width: 100%; + height: 100%; + position: relative; + .flow-canvas { + width: 100%; + height: 100%; + } + .point { + position: absolute; + width: 8px; + height: 8px; + border-radius: 50%; + border: 1px solid #D9D9D9; + cursor: pointer; + } + .point-1 { + top: 50px; + left: 300px; + } + .point-2 { + top: 300px; + left: 500px; + } + .operate-bar{ + position: absolute; + top: 20px; + right: 20px; + z-index: 999; + box-shadow:0 2px 3px 0 rgba(0,112,204,0.06); + border:1px solid #D9D9D9; + border-radius: 4px; + padding: 10px; + background-color: rgba(255, 255, 255, 0.8); + color:#222; + .operate-bar-title{ + height: 30px; + width: 100%; + line-height: 30px; + text-align: center; + background-color: #F66902; + color: #222; + margin-bottom: 10px; + } + + } +} diff --git a/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/index.vue b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/index.vue new file mode 100644 index 0000000000..2647e458f9 --- /dev/null +++ b/web/src/apps/dataAssetManage/view/assetsInfo/components/lineage/index.vue @@ -0,0 +1,201 @@ + + + diff --git a/web/src/apps/dataAssetManage/view/assetsInfo/index.vue b/web/src/apps/dataAssetManage/view/assetsInfo/index.vue new file mode 100644 index 0000000000..c7110fb4a6 --- /dev/null +++ b/web/src/apps/dataAssetManage/view/assetsInfo/index.vue @@ -0,0 +1,296 @@ + + + + + diff --git a/web/src/apps/dataAssetManage/view/assetsSearch/index.vue b/web/src/apps/dataAssetManage/view/assetsSearch/index.vue new file mode 100644 index 0000000000..320b630caa --- /dev/null +++ b/web/src/apps/dataAssetManage/view/assetsSearch/index.vue @@ -0,0 +1,424 @@ + + + diff --git a/web/src/apps/dataAssetManage/view/fieldInfo/index.vue b/web/src/apps/dataAssetManage/view/fieldInfo/index.vue new file mode 100644 index 0000000000..34d656eb2d --- /dev/null +++ b/web/src/apps/dataAssetManage/view/fieldInfo/index.vue @@ -0,0 +1,295 @@ + + + + + diff --git a/web/src/apps/dataAssetManage/view/layout/emptylayout.vue b/web/src/apps/dataAssetManage/view/layout/emptylayout.vue new file mode 100644 index 0000000000..98240aef81 --- /dev/null +++ b/web/src/apps/dataAssetManage/view/layout/emptylayout.vue @@ -0,0 +1,3 @@ + diff --git a/web/src/apps/dataAssetManage/view/layout/index.vue b/web/src/apps/dataAssetManage/view/layout/index.vue new file mode 100644 index 0000000000..470830eaec --- /dev/null +++ b/web/src/apps/dataAssetManage/view/layout/index.vue @@ -0,0 +1,19 @@ + + + + diff --git a/web/src/apps/dataAssetManage/view/layout/sidebar-sub-menu.vue b/web/src/apps/dataAssetManage/view/layout/sidebar-sub-menu.vue new file mode 100644 index 0000000000..dd958f7a34 --- /dev/null +++ b/web/src/apps/dataAssetManage/view/layout/sidebar-sub-menu.vue @@ -0,0 +1,61 @@ + + + diff --git a/web/src/apps/dataAssetManage/view/layout/sidebar.vue b/web/src/apps/dataAssetManage/view/layout/sidebar.vue new file mode 100644 index 0000000000..4d603d386d --- /dev/null +++ b/web/src/apps/dataAssetManage/view/layout/sidebar.vue @@ -0,0 +1,39 @@ + + + diff --git a/web/src/apps/dataAssetManage/view/rangeInfo/index.vue b/web/src/apps/dataAssetManage/view/rangeInfo/index.vue new file mode 100644 index 0000000000..0761cd9f5c --- /dev/null +++ b/web/src/apps/dataAssetManage/view/rangeInfo/index.vue @@ -0,0 +1,44 @@ + + + + + diff --git a/web/src/apps/dataModelCenter/assets/styles/common.scss b/web/src/apps/dataModelCenter/assets/styles/common.scss new file mode 100644 index 0000000000..a1909b210a --- /dev/null +++ b/web/src/apps/dataModelCenter/assets/styles/common.scss @@ -0,0 +1,4 @@ +.page-content { + box-sizing :border-box; + padding: 20px; +} diff --git a/web/src/apps/dataModelCenter/i18n/en.json b/web/src/apps/dataModelCenter/i18n/en.json new file mode 100644 index 0000000000..e837e0dea1 --- /dev/null +++ b/web/src/apps/dataModelCenter/i18n/en.json @@ -0,0 +1,3 @@ +{ + "message": {} +} diff --git a/web/src/apps/dataModelCenter/i18n/zh.json b/web/src/apps/dataModelCenter/i18n/zh.json new file mode 100644 index 0000000000..e837e0dea1 --- /dev/null +++ b/web/src/apps/dataModelCenter/i18n/zh.json @@ -0,0 +1,3 @@ +{ + "message": {} +} diff --git a/web/src/apps/dataModelCenter/module/dimension/editModal.vue b/web/src/apps/dataModelCenter/module/dimension/editModal.vue new file mode 100644 index 0000000000..707a92cabf --- /dev/null +++ b/web/src/apps/dataModelCenter/module/dimension/editModal.vue @@ -0,0 +1,287 @@ + + + + + diff --git a/web/src/apps/dataModelCenter/module/dimension/index.js b/web/src/apps/dataModelCenter/module/dimension/index.js new file mode 100644 index 0000000000..40f38d29cd --- /dev/null +++ b/web/src/apps/dataModelCenter/module/dimension/index.js @@ -0,0 +1,10 @@ +export default { + name: 'dimensionManage', + events: [], + dispatchs: { + }, + data: { + API_PATH: process.env.VUE_APP_MN_CONFIG_PREFIX || `http://${window.location.host}/api/rest_j/v1/`, + }, + component: () => import('./index.vue'), +}; diff --git a/web/src/apps/dataModelCenter/module/dimension/index.vue b/web/src/apps/dataModelCenter/module/dimension/index.vue new file mode 100644 index 0000000000..eca87d713b --- /dev/null +++ b/web/src/apps/dataModelCenter/module/dimension/index.vue @@ -0,0 +1,284 @@ + + + + + diff --git a/web/src/apps/dataModelCenter/module/indicators/editModal.vue b/web/src/apps/dataModelCenter/module/indicators/editModal.vue new file mode 100644 index 0000000000..c4bd74c90b --- /dev/null +++ b/web/src/apps/dataModelCenter/module/indicators/editModal.vue @@ -0,0 +1,986 @@ + + + + + diff --git a/web/src/apps/dataModelCenter/module/indicators/index.js b/web/src/apps/dataModelCenter/module/indicators/index.js new file mode 100644 index 0000000000..f6acb36a57 --- /dev/null +++ b/web/src/apps/dataModelCenter/module/indicators/index.js @@ -0,0 +1,10 @@ +export default { + name: 'indicatorsManage', + events: [], + dispatchs: { + }, + data: { + API_PATH: process.env.VUE_APP_MN_CONFIG_PREFIX || `http://${window.location.host}/api/rest_j/v1/`, + }, + component: () => import('./index.vue'), +}; diff --git a/web/src/apps/dataModelCenter/module/indicators/index.vue b/web/src/apps/dataModelCenter/module/indicators/index.vue new file mode 100644 index 0000000000..13742d0a32 --- /dev/null +++ b/web/src/apps/dataModelCenter/module/indicators/index.vue @@ -0,0 +1,347 @@ + + + + + diff --git a/web/src/apps/dataModelCenter/module/indicators/selectPage.vue b/web/src/apps/dataModelCenter/module/indicators/selectPage.vue new file mode 100644 index 0000000000..6f1e67f5aa --- /dev/null +++ b/web/src/apps/dataModelCenter/module/indicators/selectPage.vue @@ -0,0 +1,108 @@ + + + + + diff --git a/web/src/apps/dataModelCenter/module/indicators/showVersionModal.vue b/web/src/apps/dataModelCenter/module/indicators/showVersionModal.vue new file mode 100644 index 0000000000..0186898069 --- /dev/null +++ b/web/src/apps/dataModelCenter/module/indicators/showVersionModal.vue @@ -0,0 +1,295 @@ + + + + + diff --git a/web/src/apps/dataModelCenter/module/indicators/statement.js b/web/src/apps/dataModelCenter/module/indicators/statement.js new file mode 100644 index 0000000000..5e104a33ec --- /dev/null +++ b/web/src/apps/dataModelCenter/module/indicators/statement.js @@ -0,0 +1,56 @@ +export const sourceInfoMap = { + 0: { + // 度量 + measure: "", + measureEn: "", + // 计算公式 + formula: "", + // 维度 + dimension: "", + dimensionEn: "" + }, + 1: { + // 指标 + indicatorName: "", + indicatorNameEn: "", + // 计算公式 + formula: "", + // 维度 + dimension: "", + dimensionEn: "" + }, + 2: { + // 指标 + indicatorName: "", + indicatorNameEn: "", + // 周期 + cycle: "", + cycleEn: "", + // 修饰词 + modifier: "", + modifierEn: "", + // 维度 + dimension: "", + dimensionEn: "" + }, + 3: { + // 指标 + indicatorName: "", + indicatorNameEn: "", + // 计算公式 + formula: "", + // 度量 + modifier: "", + modifierEn: "", + // 维度 + dimension: "", + dimensionEn: "" + }, + 4: { + // 计算公式 + formula: "", + // 维度 + dimension: "", + dimensionEn: "" + } +}; diff --git a/web/src/apps/dataModelCenter/module/indicators/versionListModal.vue b/web/src/apps/dataModelCenter/module/indicators/versionListModal.vue new file mode 100644 index 0000000000..9d0579f8d8 --- /dev/null +++ b/web/src/apps/dataModelCenter/module/indicators/versionListModal.vue @@ -0,0 +1,120 @@ + + + + + diff --git a/web/src/apps/dataModelCenter/module/labels/editModal.vue b/web/src/apps/dataModelCenter/module/labels/editModal.vue new file mode 100644 index 0000000000..36d06d6b39 --- /dev/null +++ b/web/src/apps/dataModelCenter/module/labels/editModal.vue @@ -0,0 +1,360 @@ + + + + + diff --git a/web/src/apps/dataModelCenter/module/labels/index.js b/web/src/apps/dataModelCenter/module/labels/index.js new file mode 100644 index 0000000000..c37aa9bed5 --- /dev/null +++ b/web/src/apps/dataModelCenter/module/labels/index.js @@ -0,0 +1,10 @@ +export default { + name: 'labelsManage', + events: [], + dispatchs: { + }, + data: { + API_PATH: process.env.VUE_APP_MN_CONFIG_PREFIX || `http://${window.location.host}/api/rest_j/v1/`, + }, + component: () => import('./index.vue'), +}; diff --git a/web/src/apps/dataModelCenter/module/labels/index.vue b/web/src/apps/dataModelCenter/module/labels/index.vue new file mode 100644 index 0000000000..617c9ffb25 --- /dev/null +++ b/web/src/apps/dataModelCenter/module/labels/index.vue @@ -0,0 +1,304 @@ + + + + + + diff --git a/web/src/apps/dataModelCenter/module/measure/editModal.vue b/web/src/apps/dataModelCenter/module/measure/editModal.vue new file mode 100644 index 0000000000..ae2ff95948 --- /dev/null +++ b/web/src/apps/dataModelCenter/module/measure/editModal.vue @@ -0,0 +1,295 @@ + + + + + diff --git a/web/src/apps/dataModelCenter/module/measure/index.js b/web/src/apps/dataModelCenter/module/measure/index.js new file mode 100644 index 0000000000..a371d29718 --- /dev/null +++ b/web/src/apps/dataModelCenter/module/measure/index.js @@ -0,0 +1,10 @@ +export default { + name: 'measureManage', + events: [], + dispatchs: { + }, + data: { + API_PATH: process.env.VUE_APP_MN_CONFIG_PREFIX || `http://${window.location.host}/api/rest_j/v1/`, + }, + component: () => import('./index.vue'), +}; diff --git a/web/src/apps/dataModelCenter/module/measure/index.vue b/web/src/apps/dataModelCenter/module/measure/index.vue new file mode 100644 index 0000000000..3e4f0ec52b --- /dev/null +++ b/web/src/apps/dataModelCenter/module/measure/index.vue @@ -0,0 +1,289 @@ + + + + + diff --git a/web/src/apps/dataModelCenter/module/tableEditor/columnEditor.vue b/web/src/apps/dataModelCenter/module/tableEditor/columnEditor.vue new file mode 100644 index 0000000000..1b81e6c9a5 --- /dev/null +++ b/web/src/apps/dataModelCenter/module/tableEditor/columnEditor.vue @@ -0,0 +1,373 @@ + + + + + diff --git a/web/src/apps/dataModelCenter/module/tableEditor/index.js b/web/src/apps/dataModelCenter/module/tableEditor/index.js new file mode 100644 index 0000000000..bb8ae7dcf6 --- /dev/null +++ b/web/src/apps/dataModelCenter/module/tableEditor/index.js @@ -0,0 +1,10 @@ +export default { + name: 'tableEditor', + events: [], + dispatchs: { + }, + data: { + API_PATH: process.env.VUE_APP_MN_CONFIG_PREFIX || `http://${window.location.host}/api/rest_j/v1/`, + }, + component: () => import('./index.vue'), +}; diff --git a/web/src/apps/dataModelCenter/module/tableEditor/index.vue b/web/src/apps/dataModelCenter/module/tableEditor/index.vue new file mode 100644 index 0000000000..b7c507cd8d --- /dev/null +++ b/web/src/apps/dataModelCenter/module/tableEditor/index.vue @@ -0,0 +1,721 @@ + + + + + diff --git a/web/src/apps/dataModelCenter/module/tableEditor/selectModel.vue b/web/src/apps/dataModelCenter/module/tableEditor/selectModel.vue new file mode 100644 index 0000000000..1c3db96f32 --- /dev/null +++ b/web/src/apps/dataModelCenter/module/tableEditor/selectModel.vue @@ -0,0 +1,185 @@ + + + diff --git a/web/src/apps/dataModelCenter/module/tableInfo/index.js b/web/src/apps/dataModelCenter/module/tableInfo/index.js new file mode 100644 index 0000000000..a48d6d2a04 --- /dev/null +++ b/web/src/apps/dataModelCenter/module/tableInfo/index.js @@ -0,0 +1,10 @@ +export default { + name: 'tableInfo', + events: [], + dispatchs: { + }, + data: { + API_PATH: process.env.VUE_APP_MN_CONFIG_PREFIX || `http://${window.location.host}/api/rest_j/v1/`, + }, + component: () => import('./index.vue'), +}; diff --git a/web/src/apps/dataModelCenter/module/tableInfo/index.vue b/web/src/apps/dataModelCenter/module/tableInfo/index.vue new file mode 100644 index 0000000000..2c310cbbe8 --- /dev/null +++ b/web/src/apps/dataModelCenter/module/tableInfo/index.vue @@ -0,0 +1,872 @@ + + + + + diff --git a/web/src/apps/dataModelCenter/module/tableSearch/index.js b/web/src/apps/dataModelCenter/module/tableSearch/index.js new file mode 100644 index 0000000000..780f406472 --- /dev/null +++ b/web/src/apps/dataModelCenter/module/tableSearch/index.js @@ -0,0 +1,10 @@ +export default { + name: 'tableSearch', + events: [], + dispatchs: { + }, + data: { + API_PATH: process.env.VUE_APP_MN_CONFIG_PREFIX || `http://${window.location.host}/api/rest_j/v1/`, + }, + component: () => import('./index.vue'), +}; diff --git a/web/src/apps/dataModelCenter/module/tableSearch/index.vue b/web/src/apps/dataModelCenter/module/tableSearch/index.vue new file mode 100644 index 0000000000..7f8ebe0806 --- /dev/null +++ b/web/src/apps/dataModelCenter/module/tableSearch/index.vue @@ -0,0 +1,637 @@ + + + + + diff --git a/web/src/apps/dataModelCenter/module/tableSearch/versionListModal.vue b/web/src/apps/dataModelCenter/module/tableSearch/versionListModal.vue new file mode 100644 index 0000000000..87261ab1f8 --- /dev/null +++ b/web/src/apps/dataModelCenter/module/tableSearch/versionListModal.vue @@ -0,0 +1,129 @@ + + + + + diff --git a/web/src/apps/dataModelCenter/module/tableVersionInfo/index.js b/web/src/apps/dataModelCenter/module/tableVersionInfo/index.js new file mode 100644 index 0000000000..58ed54d7c4 --- /dev/null +++ b/web/src/apps/dataModelCenter/module/tableVersionInfo/index.js @@ -0,0 +1,10 @@ +export default { + name: 'tableVersionInfo', + events: [], + dispatchs: { + }, + data: { + API_PATH: process.env.VUE_APP_MN_CONFIG_PREFIX || `http://${window.location.host}/api/rest_j/v1/`, + }, + component: () => import('./index.vue'), +}; diff --git a/web/src/apps/dataModelCenter/module/tableVersionInfo/index.vue b/web/src/apps/dataModelCenter/module/tableVersionInfo/index.vue new file mode 100644 index 0000000000..643c159121 --- /dev/null +++ b/web/src/apps/dataModelCenter/module/tableVersionInfo/index.vue @@ -0,0 +1,420 @@ + + + + + diff --git a/web/src/apps/dataModelCenter/router.js b/web/src/apps/dataModelCenter/router.js new file mode 100644 index 0000000000..728facca4a --- /dev/null +++ b/web/src/apps/dataModelCenter/router.js @@ -0,0 +1,106 @@ +const routes = [ + { + name: "dataModelCenter", + path: "/dataModelCenter", + redirect: "/dataModelCenter/tableManage", + component: () => import("./view/layout/index.vue"), + meta: { + title: "数据模型中心", + publicPage: true + }, + children: [ + { + name: "tableManage", + path: "/dataModelCenter/tableManage", + component: () => import("./view/layout/emptylayout.vue"), + redirect: "/dataModelCenter/tableManage/tableSearch", + meta: { + publicPage: true, + cover: "tableSearch" + }, + children: [ + { + name: "tableSearch", + path: "/dataModelCenter/tableManage/tableSearch", + component: () => import("./view/tableManage/tableSearch/index.vue"), + meta: { + title: "数据表管理", + publicPage: true, + icon: "ios-paper" + } + }, + { + name: "tableInfo", + path: "/dataModelCenter/tableManage/tableInfo", + component: () => import("./view/tableManage/tableInfo/index.vue"), + meta: { + title: "表详情", + publicPage: true + } + }, + { + name: "tableEditor", + path: "/dataModelCenter/tableManage/tableEditor", + component: () => import("./view/tableManage/tableEditor/index.vue"), + meta: { + title: "表编辑", + publicPage: true + } + }, + { + name: "tableVersionInfo", + path: "/dataModelCenter/tableManage/tableVersionInfo", + component: () => + import("./view/tableManage/tableVersionInfo/index.vue"), + meta: { + title: "历史版本信息", + publicPage: true + } + } + ] + }, + { + name: "dimension", + path: "/dataModelCenter/dimension", + component: () => import("./view/dimension/index.vue"), + meta: { + title: "维度", + publicPage: true, + icon: "ios-paper" + } + }, + { + name: "measure", + path: "/dataModelCenter/measure", + component: () => import("./view/measure/index.vue"), + meta: { + title: "度量", + publicPage: true, + icon: "ios-paper" + } + }, + { + name: "indicators", + path: "/dataModelCenter/indicators", + component: () => import("./view/indicators/index.vue"), + meta: { + title: "指标", + publicPage: true, + icon: "ios-paper" + } + }, + { + name: "labels", + path: "/dataModelCenter/labels", + component: () => import("./view/labels/index.vue"), + meta: { + title: "标签", + publicPage: true, + icon: "ios-paper" + } + } + ] + } +]; + +export default routes; diff --git a/web/src/apps/dataModelCenter/service/api/common.js b/web/src/apps/dataModelCenter/service/api/common.js new file mode 100644 index 0000000000..91b7ff0c45 --- /dev/null +++ b/web/src/apps/dataModelCenter/service/api/common.js @@ -0,0 +1,50 @@ +import API_PATH from "@/common/config/apiPath.js"; +import api from "@/common/service/api"; + +/** + * 主题域列表 + * @returns {Promise} + * + */ +export const getThemesList = () => api.fetch(`${API_PATH.DATAMODEL_PATH}themes/list`, {}, "post"); + +/** + * 分层列表 + * @param dbName {String} 可用库名称 + * @returns {Promise} + * + */ +export const getLayersList = (dbName) => api.fetch(`${API_PATH.DATAMODEL_PATH}layers/list`, { + dbName +}, "post"); + +/** + * 修饰词列表 + * @returns {Promise} + * + */ +export const getModifiersList = () => api.fetch(`${API_PATH.DATAMODEL_PATH}modifiers/list`, {}, "post"); + +/** + * 周期列表 + * @returns {Promise} + * + */ +export const getCyclesList = () => api.fetch(`${API_PATH.DATAMODEL_PATH}cycles/list`, {}, "post"); + + +/** + * 获取可用角色列表 + * @param workspaceid {String} 工作区id + * @returns {Promise} + * + */ +export const getRolesList = (workspaceid) => api.fetch(`${API_PATH.DATAMODEL_PATH}roles/${workspaceid}`, {}, "get"); + +/** + * 获取所有用户列表 + * @param workspaceid {String} 工作区id + * @returns {Promise} + * + */ +export const getUsersList = (workspaceid) => api.fetch(`${API_PATH.DATAMODEL_PATH}users/${workspaceid}`, {}, "get"); diff --git a/web/src/apps/dataModelCenter/service/api/dimensions.js b/web/src/apps/dataModelCenter/service/api/dimensions.js new file mode 100644 index 0000000000..cef6ef8990 --- /dev/null +++ b/web/src/apps/dataModelCenter/service/api/dimensions.js @@ -0,0 +1,64 @@ +import API_PATH from "@/common/config/apiPath.js"; +import api from "@/common/service/api"; + +/** + * 维度列表 + * @returns {Object.result} + * + */ +export const getDimensions = ({ + pageNum, + pageSize, + isAvailable, + owner, + name, + warehouseThemeName +}) => + api.fetch( + `${API_PATH.DATAMODEL_PATH}dimensions/list`, + { pageNum, pageSize, isAvailable, owner, name, warehouseThemeName }, + "post" + ); + +/** + * 新增维度 + * @returns {Object.result} + * + */ +export const createDimensions = body => + api.fetch(`${API_PATH.DATAMODEL_PATH}dimensions`, body, "post"); + +/** + * 修改维度 + * @returns {Object.result} + * + */ +export const editDimensions = (id, body) => + api.fetch(`${API_PATH.DATAMODEL_PATH}dimensions/${id}`, body, "put"); + +/** + * 根据id获取维度 + * @returns {Object.result} + * + */ +export const getDimensionsById = id => + api.fetch(`${API_PATH.DATAMODEL_PATH}dimensions/${id}`, {}, "get"); + +/** + * 启用禁用维度 + * @returns {Object.result} + * + */ +export const switchDimensionsStatus = (id, status) => + api.fetch( + `${API_PATH.DATAMODEL_PATH}dimensions/enable/${id}`, + { isAvailable: status }, + "put" + ); + +/** + * 删除维度 + * @returns {Object.result} + */ +export const delDimensions = id => + api.fetch(`${API_PATH.DATAMODEL_PATH}dimensions/${id}`, {}, "delete"); diff --git a/web/src/apps/dataModelCenter/service/api/indicators.js b/web/src/apps/dataModelCenter/service/api/indicators.js new file mode 100644 index 0000000000..bd2ddee777 --- /dev/null +++ b/web/src/apps/dataModelCenter/service/api/indicators.js @@ -0,0 +1,109 @@ +import API_PATH from "@/common/config/apiPath.js"; +import api from "@/common/service/api"; + +/** + * 指标列表 + * @returns {Object.result} + * + */ +export const getIndicators = ({ + pageNum, + pageSize, + isAvailable, + owner, + name, + indicatorType, + warehouseThemeName +}) => + api.fetch( + `${API_PATH.DATAMODEL_PATH}indicators/list`, + { + pageNum, + pageSize, + isAvailable, + owner, + name, + indicatorType, + warehouseThemeName + }, + "post" + ); + +/** + * 新增指标 + * @returns {Object.result} + * + */ +export const createIndicators = body => + api.fetch(`${API_PATH.DATAMODEL_PATH}indicators`, body, "post"); + +/** + * 修改指标 + * @returns {Object.result} + * + */ +export const editIndicators = (id, body) => + api.fetch(`${API_PATH.DATAMODEL_PATH}indicators/${id}`, body, "put"); + +/** + * 根据id获取指标 + * @returns {Object.result} + * + */ +export const getIndicatorsById = id => + api.fetch(`${API_PATH.DATAMODEL_PATH}indicators/${id}`, {}, "get"); + +/** + * 启用禁用指标 + * @returns {Object.result} + * + */ +export const switcIndicatorsStatus = (id, status) => + api.fetch( + `${API_PATH.DATAMODEL_PATH}indicators/enable/${id}`, + { isAvailable: status }, + "put" + ); + +/** + * 获取版本列表 + * @returns {Object.result} + * + */ +export const getIndicatorsVersionList = name => + api.fetch( + `${API_PATH.DATAMODEL_PATH}indicators/versions/list`, + { name }, + "post" + ); + +/** + * 新增版本 + * @returns {Object.result} + * + */ +export const buildIndicatorsVersion = (id, body) => + api.fetch( + `${API_PATH.DATAMODEL_PATH}indicators/versions/${id}`, + body, + "post" + ); + +/** + * 回退版本 + * @returns {Object.result} + * + */ +export const rollbackIndicatorsVersion = (name, version) => + api.fetch( + `${API_PATH.DATAMODEL_PATH}indicators/versions/rollback`, + { name, version }, + "post" + ); + +/** + * 删除指标 + * @returns {Object.result} + */ +export const delIndicators = id => + api.fetch(`${API_PATH.DATAMODEL_PATH}indicators/${id}`, {}, "delete"); diff --git a/web/src/apps/dataModelCenter/service/api/labels.js b/web/src/apps/dataModelCenter/service/api/labels.js new file mode 100644 index 0000000000..cd6a556823 --- /dev/null +++ b/web/src/apps/dataModelCenter/service/api/labels.js @@ -0,0 +1,69 @@ +import API_PATH from "@/common/config/apiPath.js"; +import api from "@/common/service/api"; + +/** + * 标签列表 + * @returns {Object.result} + */ +export const getLabelList = ({ + pageNum, + pageSize, + isAvailable, + name, + owner, + warehouseThemeName +}) => + api.fetch( + `${API_PATH.DATAMODEL_PATH}labels/list`, + { + pageNum, + pageSize, + isAvailable, + name, + owner, + warehouseThemeName + }, + "post" + ); + +/** + * 根据id获取标签 + * @returns {Object.result} + */ +export const getLabelById = id => + api.fetch(`${API_PATH.DATAMODEL_PATH}labels/${id}`, {}, "get"); + +/** + * 更新标签 + * @returns {Object.result} + */ +export const updateLabel = (id, body) => + api.fetch(`${API_PATH.DATAMODEL_PATH}labels/${id}`, body, "put"); + +/** + * 删除标签 + * @returns {Object.result} + */ +export const delLabel = id => + api.fetch(`${API_PATH.DATAMODEL_PATH}labels/${id}`, {}, "delete"); + +/** + * 创建标签 + * @returns {Object.result} + */ +export const createLabel = body => { + console.log(body); + return api.fetch(`${API_PATH.DATAMODEL_PATH}labels`, body, "post"); +}; + +/** + * 启用禁用指标 + * @returns {Object.result} + * + */ +export const switcLabelStatus = (id, status) => + api.fetch( + `${API_PATH.DATAMODEL_PATH}labels/enable/${id}`, + {isAvailable: status}, + "put" + ); diff --git a/web/src/apps/dataModelCenter/service/api/measures.js b/web/src/apps/dataModelCenter/service/api/measures.js new file mode 100644 index 0000000000..9f70884579 --- /dev/null +++ b/web/src/apps/dataModelCenter/service/api/measures.js @@ -0,0 +1,65 @@ +import API_PATH from "@/common/config/apiPath.js"; +import api from "@/common/service/api"; + +/** + * 度量列表 + * @returns {Object.result} + * + */ +export const getMeasures = ({ + pageNum, + pageSize, + isAvailable, + owner, + name, + warehouseThemeName +}) => + api.fetch( + `${API_PATH.DATAMODEL_PATH}measures/list`, + {pageNum, pageSize, isAvailable, owner, name, warehouseThemeName}, + "post" + ); + +/** + * 新增度量 + * @returns {Object.result} + * + */ +export const createMeasures = body => + api.fetch(`${API_PATH.DATAMODEL_PATH}measures`, body, "post"); + +/** + * 修改度量 + * @returns {Object.result} + * + */ +export const editMeasures = (id, body) => + api.fetch(`${API_PATH.DATAMODEL_PATH}measures/${id}`, body, "put"); + +/** + * 根据id获取度量 + * @returns {Object.result} + * + */ +export const getMeasuresById = id => + api.fetch(`${API_PATH.DATAMODEL_PATH}measures/${id}`, {}, "get"); + +/** + * 启用禁用度量 + * @returns {Object.result} + * + */ +export const switchMeasuresStatus = (id, status) => + api.fetch( + `${API_PATH.DATAMODEL_PATH}measures/enable/${id}`, + {isAvailable: status}, + "put" + ); + +/** + * 删除度量 + * @returns {Object.result} + * + */ +export const delMeasures = id => + api.fetch(`${API_PATH.DATAMODEL_PATH}measures/${id}`, {}, "delete"); diff --git a/web/src/apps/dataModelCenter/service/api/tableManage.js b/web/src/apps/dataModelCenter/service/api/tableManage.js new file mode 100644 index 0000000000..e23712059d --- /dev/null +++ b/web/src/apps/dataModelCenter/service/api/tableManage.js @@ -0,0 +1,230 @@ +import API_PATH from "@/common/config/apiPath.js"; +import api from "@/common/service/api"; + +/** + * @description 表搜索 + * @param name {String} 表名 + * @param warehouseLayerName {String} 分层 + * @param warehouseThemeName {String} 主题 + * @param pageSize {Number} 分页大小 + * @param pageNum {Number} 当前页 + * @param modelType {Number} 模型类型 + * @param modelName {String} 模型名字 + * @param tableType {Number} 逻辑表或物理表 + */ +export const searchTable = ({ + name, + warehouseLayerName, + warehouseThemeName, + pageSize, + pageNum, + modelType, + modelName, + tableType +}) => + api.fetch( + `${API_PATH.DATAMODEL_PATH}tables/list`, + { + name, + warehouseLayerName, + warehouseThemeName, + pageSize, + pageNum, + modelType, + modelName, + tableType + }, + "post" + ); + +/** + * @description 获取收藏表 + * @returns {Promise} + * @param name {String} + */ +export const getCollectList = ({name} = {}) => { + return api.fetch( + `${API_PATH.DATAMODEL_PATH}tables/collect/list`, + {name}, + "post" + ); +}; +/** + * @description 添加收藏 + * @param + * @returns {Promise} + */ +export const addCollect = body => { + return api.fetch(`${API_PATH.DATAMODEL_PATH}tables/collect`, body, "post"); +}; +/** + * 取消收藏 + * @returns {Promise} + */ +export const delCancel = tableName => { + return api.fetch( + `${API_PATH.DATAMODEL_PATH}tables/collect/cancel`, + {tableName}, + "post" + ); +}; + +/** + * 数据库列表 + * @returns {Promise} + */ +export const getDataBasesList = (name = "") => { + return api.fetch( + `${API_PATH.DATAMODEL_PATH}tables/databases/list`, + {name}, + "post" + ); +}; + +/** + * 根据id获取表详情 + * @returns {Promise} + */ +export const getTableInfoById = id => { + return api.fetch(`${API_PATH.DATAMODEL_PATH}tables/${id}`, {}, "get"); +}; + +/** + * 根据name和guid获取表详情 + * @returns {Promise} + */ +export const getTableInfoByName = (name = "", guid = "") => { + return api.fetch( + `${API_PATH.DATAMODEL_PATH}tables/name`, + {name, guid}, + "post" + ); +}; + +/** + * 分区统计信息 + * @returns {Promise} + */ +export const getTablesPartitionStats = (name = "", guid = "") => { + return api.fetch( + `${API_PATH.DATAMODEL_PATH}tables/partition/stats`, + {name, guid}, + "post" + ); +}; + +/** + * 生成建表语句 + * @returns {Promise} + */ +export const getTablesCreateSql = (tableId = "", guid = "") => { + return api.fetch( + `${API_PATH.DATAMODEL_PATH}tables/create/sql`, + {tableId, guid}, + "post" + ); +}; + +/** + * 获取预览信息 + * @returns {Promise} + */ +export const getTablesPreview = tableName => { + return api.fetch( + `${API_PATH.DATAMODEL_PATH}tables/data/preview`, + {tableName}, + "post" + ); +}; + +/** + * 字典列表 + * @returns {Promise} + * + */ +export const getDictionariesList = type => + api.fetch( + `${API_PATH.DATAMODEL_PATH}tables/dictionaries/list`, + {type}, + "post" + ); + +/** + * 新增表 + * @returns {Promise} + * + */ +export const addTable = body => + api.fetch(`${API_PATH.DATAMODEL_PATH}tables`, body, "post"); + +/** + * 更新表 + * @returns {Promise} + * + */ +export const updateTable = (id, body) => + api.fetch(`${API_PATH.DATAMODEL_PATH}tables/${id}`, body, "put"); + +/** + * 执行表创建 + * @returns {Promise} + * + */ +export const createTable = tableId => + api.fetch(`${API_PATH.DATAMODEL_PATH}tables/create`, {tableId}, "post"); + +/** + * 版本列表 + * @returns {Promise} + * + */ +export const getVersionListByName = name => + api.fetch(`${API_PATH.DATAMODEL_PATH}tables/versions/list`, {name}, "post"); + +/** + * 版本回退 + * @returns {Promise} + * + */ +export const tableVersionRollback = (name, version) => + api.fetch( + `${API_PATH.DATAMODEL_PATH}tables/versions/rollback`, + {name, version}, + "post" + ); + +/** + * 新增版本 + * @returns {Promise} + * + */ +export const generatorNewVersion = (id, data) => + api.fetch(`${API_PATH.DATAMODEL_PATH}tables/versions/${id}`, data, "post"); + +/** + * 检查表是否有数据 + * @returns {Promise} + * + */ +export const checkTableData = tableName => + api.fetch( + `${API_PATH.DATAMODEL_PATH}tables/check/data`, + {tableName}, + "post" + ); + +/** + * @description 删除表 + * @param {String} id + * @returns {Promise} + */ +export const deleteTableById = id => + api.fetch(`${API_PATH.DATAMODEL_PATH}tables/${id}`, {}, "delete"); + +/** + * 主动绑定 + * @returns {Promise} + * + */ +export const bindTable = id => + api.fetch(`${API_PATH.DATAMODEL_PATH}tables/bind/${id}`, "put"); diff --git a/web/src/apps/dataModelCenter/utils/clipboard.js b/web/src/apps/dataModelCenter/utils/clipboard.js new file mode 100644 index 0000000000..7efc01c495 --- /dev/null +++ b/web/src/apps/dataModelCenter/utils/clipboard.js @@ -0,0 +1,43 @@ +import Vue from 'vue'; +import Clipboard from 'clipboard'; + +/** + *@returns + */ +function clipboardSuccess() { + Vue.prototype.$Message.success({ + content: '复制成功', + duration: 1.5 + }); +} + + +/** + *@returns + */ +function clipboardError() { + Vue.prototype.$Message.success({ + message: '复制失败', + duration: 1.5 + }); +} + +/** + * 把字符串中的内容放到剪切板上 + * @param {String, Object} + * @param {void} + */ +export default function handleClipboard(text, event) { + const clipboard = new Clipboard(event.target, { + text: () => text + }); + clipboard.on('success', () => { + clipboardSuccess(); + clipboard.destroy(); + }); + clipboard.on('error', () => { + clipboardError(); + clipboard.destroy(); + }); + clipboard.onClick(event); +} diff --git a/web/src/apps/dataModelCenter/utils/fomatSQL.js b/web/src/apps/dataModelCenter/utils/fomatSQL.js new file mode 100644 index 0000000000..caba9997b5 --- /dev/null +++ b/web/src/apps/dataModelCenter/utils/fomatSQL.js @@ -0,0 +1,23 @@ +/** + * 转换 后端SQL语句 成 html文本 + * @param (String) + * @return (String) + */ +export function fomatSqlForShow(sql) { + if ( typeof sql !== 'string' ) return '' + const arr = sql.split('@$'); + const res = arr.map((i, idx) => `

${idx+1}${i}

`) + return res.join('') +} + +/** + * + * @param {String} sql + * @return {String} + */ +export function fomatSqlForCopy(sql) { + if ( typeof sql !== 'string' ) return '' + const arr = sql.replaceAll(' ', '').split('@$'); + const res = arr.map(i => `${i}\r\n`) + return res.join('') +} diff --git a/web/src/apps/dataModelCenter/utils/formatDate.js b/web/src/apps/dataModelCenter/utils/formatDate.js new file mode 100644 index 0000000000..239b49beaf --- /dev/null +++ b/web/src/apps/dataModelCenter/utils/formatDate.js @@ -0,0 +1,10 @@ +import moment from "moment"; +/** + * @description 转换时间戳到字符串 + * @param value {Number} 时间戳 + * @return {String} + */ +export default function(value) { + if (!value) return ""; + return moment(value).format("YYYY-MM-DD HH:mm"); +} diff --git a/web/src/apps/dataModelCenter/view/dimension/index.vue b/web/src/apps/dataModelCenter/view/dimension/index.vue new file mode 100644 index 0000000000..32753ad12d --- /dev/null +++ b/web/src/apps/dataModelCenter/view/dimension/index.vue @@ -0,0 +1,11 @@ + + diff --git a/web/src/apps/dataModelCenter/view/indicators/index.vue b/web/src/apps/dataModelCenter/view/indicators/index.vue new file mode 100644 index 0000000000..db5ffc5366 --- /dev/null +++ b/web/src/apps/dataModelCenter/view/indicators/index.vue @@ -0,0 +1,11 @@ + + diff --git a/web/src/apps/dataModelCenter/view/labels/index.vue b/web/src/apps/dataModelCenter/view/labels/index.vue new file mode 100644 index 0000000000..a8c786b2e7 --- /dev/null +++ b/web/src/apps/dataModelCenter/view/labels/index.vue @@ -0,0 +1,11 @@ + + diff --git a/web/src/apps/dataModelCenter/view/layout/emptylayout.vue b/web/src/apps/dataModelCenter/view/layout/emptylayout.vue new file mode 100644 index 0000000000..98240aef81 --- /dev/null +++ b/web/src/apps/dataModelCenter/view/layout/emptylayout.vue @@ -0,0 +1,3 @@ + diff --git a/web/src/apps/dataModelCenter/view/layout/index.vue b/web/src/apps/dataModelCenter/view/layout/index.vue new file mode 100644 index 0000000000..d3eacab1ea --- /dev/null +++ b/web/src/apps/dataModelCenter/view/layout/index.vue @@ -0,0 +1,28 @@ + + + + diff --git a/web/src/apps/dataModelCenter/view/layout/sidebar-sub-menu.vue b/web/src/apps/dataModelCenter/view/layout/sidebar-sub-menu.vue new file mode 100644 index 0000000000..dd958f7a34 --- /dev/null +++ b/web/src/apps/dataModelCenter/view/layout/sidebar-sub-menu.vue @@ -0,0 +1,61 @@ + + + diff --git a/web/src/apps/dataModelCenter/view/layout/sidebar.vue b/web/src/apps/dataModelCenter/view/layout/sidebar.vue new file mode 100644 index 0000000000..4e8d7699b8 --- /dev/null +++ b/web/src/apps/dataModelCenter/view/layout/sidebar.vue @@ -0,0 +1,41 @@ + + + diff --git a/web/src/apps/dataModelCenter/view/measure/index.vue b/web/src/apps/dataModelCenter/view/measure/index.vue new file mode 100644 index 0000000000..5d4695940c --- /dev/null +++ b/web/src/apps/dataModelCenter/view/measure/index.vue @@ -0,0 +1,11 @@ + + diff --git a/web/src/apps/dataModelCenter/view/tableManage/tableEditor/index.vue b/web/src/apps/dataModelCenter/view/tableManage/tableEditor/index.vue new file mode 100644 index 0000000000..b4b8077126 --- /dev/null +++ b/web/src/apps/dataModelCenter/view/tableManage/tableEditor/index.vue @@ -0,0 +1,11 @@ + + diff --git a/web/src/apps/dataModelCenter/view/tableManage/tableInfo/index.vue b/web/src/apps/dataModelCenter/view/tableManage/tableInfo/index.vue new file mode 100644 index 0000000000..5c11c6829a --- /dev/null +++ b/web/src/apps/dataModelCenter/view/tableManage/tableInfo/index.vue @@ -0,0 +1,11 @@ + + diff --git a/web/src/apps/dataModelCenter/view/tableManage/tableSearch/index.vue b/web/src/apps/dataModelCenter/view/tableManage/tableSearch/index.vue new file mode 100644 index 0000000000..bb17e79695 --- /dev/null +++ b/web/src/apps/dataModelCenter/view/tableManage/tableSearch/index.vue @@ -0,0 +1,11 @@ + + diff --git a/web/src/apps/dataModelCenter/view/tableManage/tableVersionInfo/index.vue b/web/src/apps/dataModelCenter/view/tableManage/tableVersionInfo/index.vue new file mode 100644 index 0000000000..ddb56cf26d --- /dev/null +++ b/web/src/apps/dataModelCenter/view/tableManage/tableVersionInfo/index.vue @@ -0,0 +1,11 @@ + + diff --git a/web/src/apps/dataWarehouseDesign/assets/styles/common.scss b/web/src/apps/dataWarehouseDesign/assets/styles/common.scss new file mode 100644 index 0000000000..a1909b210a --- /dev/null +++ b/web/src/apps/dataWarehouseDesign/assets/styles/common.scss @@ -0,0 +1,4 @@ +.page-content { + box-sizing :border-box; + padding: 20px; +} diff --git a/web/src/apps/dataWarehouseDesign/i18n/en.json b/web/src/apps/dataWarehouseDesign/i18n/en.json new file mode 100644 index 0000000000..0967ef424b --- /dev/null +++ b/web/src/apps/dataWarehouseDesign/i18n/en.json @@ -0,0 +1 @@ +{} diff --git a/web/src/apps/dataWarehouseDesign/i18n/zh.json b/web/src/apps/dataWarehouseDesign/i18n/zh.json new file mode 100644 index 0000000000..0967ef424b --- /dev/null +++ b/web/src/apps/dataWarehouseDesign/i18n/zh.json @@ -0,0 +1 @@ +{} diff --git a/web/src/apps/dataWarehouseDesign/module/layered/editModal.vue b/web/src/apps/dataWarehouseDesign/module/layered/editModal.vue new file mode 100644 index 0000000000..3324ce77b8 --- /dev/null +++ b/web/src/apps/dataWarehouseDesign/module/layered/editModal.vue @@ -0,0 +1,306 @@ + + + + + diff --git a/web/src/apps/dataWarehouseDesign/module/layered/index.js b/web/src/apps/dataWarehouseDesign/module/layered/index.js new file mode 100644 index 0000000000..5ee5da619c --- /dev/null +++ b/web/src/apps/dataWarehouseDesign/module/layered/index.js @@ -0,0 +1,10 @@ +export default { + name: 'layeredManage', + events: [], + dispatchs: { + }, + data: { + API_PATH: process.env.VUE_APP_MN_CONFIG_PREFIX || `http://${window.location.host}/api/rest_j/v1/`, + }, + component: () => import('./index.vue'), +}; diff --git a/web/src/apps/dataWarehouseDesign/module/layered/index.vue b/web/src/apps/dataWarehouseDesign/module/layered/index.vue new file mode 100644 index 0000000000..38ef21a7f3 --- /dev/null +++ b/web/src/apps/dataWarehouseDesign/module/layered/index.vue @@ -0,0 +1,365 @@ + + + + + diff --git a/web/src/apps/dataWarehouseDesign/module/modifier/editModal.vue b/web/src/apps/dataWarehouseDesign/module/modifier/editModal.vue new file mode 100644 index 0000000000..8940694e5f --- /dev/null +++ b/web/src/apps/dataWarehouseDesign/module/modifier/editModal.vue @@ -0,0 +1,348 @@ + + + + + diff --git a/web/src/apps/dataWarehouseDesign/module/modifier/index.js b/web/src/apps/dataWarehouseDesign/module/modifier/index.js new file mode 100644 index 0000000000..cce9a7cb4c --- /dev/null +++ b/web/src/apps/dataWarehouseDesign/module/modifier/index.js @@ -0,0 +1,10 @@ +export default { + name: 'modifierManage', + events: [], + dispatchs: { + }, + data: { + API_PATH: process.env.VUE_APP_MN_CONFIG_PREFIX || `http://${window.location.host}/api/rest_j/v1/`, + }, + component: () => import('./index.vue'), +}; diff --git a/web/src/apps/dataWarehouseDesign/module/modifier/index.vue b/web/src/apps/dataWarehouseDesign/module/modifier/index.vue new file mode 100644 index 0000000000..9f3d3207a5 --- /dev/null +++ b/web/src/apps/dataWarehouseDesign/module/modifier/index.vue @@ -0,0 +1,278 @@ + + + + + diff --git a/web/src/apps/dataWarehouseDesign/module/statPeriod/editModal.vue b/web/src/apps/dataWarehouseDesign/module/statPeriod/editModal.vue new file mode 100644 index 0000000000..a7f78ce8f5 --- /dev/null +++ b/web/src/apps/dataWarehouseDesign/module/statPeriod/editModal.vue @@ -0,0 +1,340 @@ + + + + + diff --git a/web/src/apps/dataWarehouseDesign/module/statPeriod/index.js b/web/src/apps/dataWarehouseDesign/module/statPeriod/index.js new file mode 100644 index 0000000000..ceebed5a83 --- /dev/null +++ b/web/src/apps/dataWarehouseDesign/module/statPeriod/index.js @@ -0,0 +1,10 @@ +export default { + name: 'statPeriodManage', + events: [], + dispatchs: { + }, + data: { + API_PATH: process.env.VUE_APP_MN_CONFIG_PREFIX || `http://${window.location.host}/api/rest_j/v1/`, + }, + component: () => import('./index.vue'), +}; diff --git a/web/src/apps/dataWarehouseDesign/module/statPeriod/index.vue b/web/src/apps/dataWarehouseDesign/module/statPeriod/index.vue new file mode 100644 index 0000000000..76cc7458b9 --- /dev/null +++ b/web/src/apps/dataWarehouseDesign/module/statPeriod/index.vue @@ -0,0 +1,302 @@ + + + + + diff --git a/web/src/apps/dataWarehouseDesign/module/themeDomains/editModal.vue b/web/src/apps/dataWarehouseDesign/module/themeDomains/editModal.vue new file mode 100644 index 0000000000..56cd0ff202 --- /dev/null +++ b/web/src/apps/dataWarehouseDesign/module/themeDomains/editModal.vue @@ -0,0 +1,267 @@ + + + + + diff --git a/web/src/apps/dataWarehouseDesign/module/themeDomains/index.js b/web/src/apps/dataWarehouseDesign/module/themeDomains/index.js new file mode 100644 index 0000000000..30b523b364 --- /dev/null +++ b/web/src/apps/dataWarehouseDesign/module/themeDomains/index.js @@ -0,0 +1,10 @@ +export default { + name: 'themeDomainsManage', + events: [], + dispatchs: { + }, + data: { + API_PATH: process.env.VUE_APP_MN_CONFIG_PREFIX || `http://${window.location.host}/api/rest_j/v1/`, + }, + component: () => import('./index.vue'), +}; diff --git a/web/src/apps/dataWarehouseDesign/module/themeDomains/index.vue b/web/src/apps/dataWarehouseDesign/module/themeDomains/index.vue new file mode 100644 index 0000000000..4ad29290ef --- /dev/null +++ b/web/src/apps/dataWarehouseDesign/module/themeDomains/index.vue @@ -0,0 +1,279 @@ + + + + + diff --git a/web/src/apps/dataWarehouseDesign/router.js b/web/src/apps/dataWarehouseDesign/router.js new file mode 100644 index 0000000000..5b67832ca2 --- /dev/null +++ b/web/src/apps/dataWarehouseDesign/router.js @@ -0,0 +1,56 @@ +const routes = [ + { + name: "dataWarehouseDesign", + path: "/dataWarehouseDesign", + redirect: "/dataWarehouseDesign/themeDomains", + component: () => import("./view/layout/index.vue"), + meta: { + title: "数仓规划", + publicPage: true + }, + children: [ + { + name: "themeDomains", + path: "/dataWarehouseDesign/themeDomains", + component: () => import("./view/themeDomains/index.vue"), + meta: { + title: "主题域配置", + publicPage: true, + icon: "ios-paper" + } + }, + { + name: "layered", + path: "/dataWarehouseDesign/layered", + component: () => import("./view/layered/index.vue"), + meta: { + title: "分层配置", + publicPage: true, + icon: "ios-paper" + } + }, + { + name: "modifier", + path: "/dataWarehouseDesign/modifier", + component: () => import("./view/modifier/index.vue"), + meta: { + title: "修饰词管理", + publicPage: true, + icon: "ios-paper" + } + }, + { + name: "statPeriod", + path: "/dataWarehouseDesign/statPeriod", + component: () => import("./view/statPeriod/index.vue"), + meta: { + title: "统计周期管理", + publicPage: true, + icon: "ios-paper" + } + } + ] + } +]; + +export default routes; diff --git a/web/src/apps/dataWarehouseDesign/service/api/common.js b/web/src/apps/dataWarehouseDesign/service/api/common.js new file mode 100644 index 0000000000..6f621a15f2 --- /dev/null +++ b/web/src/apps/dataWarehouseDesign/service/api/common.js @@ -0,0 +1,16 @@ +import api from "@/common/service/api"; +import API_PATH from "@/common/config/apiPath"; + +/** + * 获取可用角色列表 + * @param workspaceid {String} 工作区id + * @return {Promise} + */ +export const getRolesList = (workspaceid) => api.fetch(`${API_PATH.WAREHOUSE_PATH}workspace/${workspaceid}/principal_roles`, {}, "get"); + +/** + * 获取所有用户列表 + * @param workspaceid {String} 工作区id + * @return {Promise} + */ +export const getUsersList = (workspaceid) => api.fetch(`${API_PATH.WAREHOUSE_PATH}workspace/${workspaceid}/principal_users`, {}, "get"); diff --git a/web/src/apps/dataWarehouseDesign/service/api/layer.js b/web/src/apps/dataWarehouseDesign/service/api/layer.js new file mode 100644 index 0000000000..105285c909 --- /dev/null +++ b/web/src/apps/dataWarehouseDesign/service/api/layer.js @@ -0,0 +1,72 @@ +import API_PATH from "@/common/config/apiPath.js"; +import api from "@/common/service/api"; + +/** + * 查询所有分层 + * @param isAvailable {Number} 是否启用 + * @return {Promise} + */ +export const getLayersAll = ({isAvailable = undefined}) => api.fetch(`${API_PATH.WAREHOUSE_PATH}layers/all`, {isAvailable}, "get") + +/** + * 查询所有预置分层 + * @return {Promise} + */ +export const getLayersPreset = () => api.fetch(`${API_PATH.WAREHOUSE_PATH}layers/preset`,{}, "get"); + +/** + * 分页查询自定义分层 + * @param page {Number} 当前页 + * @param size {Number} 分页数量 + * @returns {Promise} + */ +export const getLayersCustom = ({page = 1,size = 10}) => api.fetch(`${API_PATH.WAREHOUSE_PATH}layers/custom`, { page,size }, "get"); + +/** + * 新增自定义分层 + * @param body {Object} 数据体 + * @returns {Promise} + */ +export const createLayersCustom = body => api.fetch(`${API_PATH.WAREHOUSE_PATH}layers/custom`, body, "post"); + +/** + * 根据ID查询某个分层信息 + * @param id {Number} id + * @returns {Promise} + */ +export const getLayersById = id => api.fetch(`${API_PATH.WAREHOUSE_PATH}layers/${id}`, {}, "get"); + +/** + * 编辑分层 + * @param id {Number} id + * @param body {Object} 数据体 + * @returns {Promise} + */ +export const editLayersCustom = (id, body) => api.fetch(`${API_PATH.WAREHOUSE_PATH}layers/${id}`, body, "put"); + +/** + * 删除分层 + * @param id {Number} id + * @returns {Promise} + */ +export const deleteLayers = id => api.fetch(`${API_PATH.WAREHOUSE_PATH}layers/${id}`, {}, "delete"); + +/** + * 禁用分层 + * @param id {Number} id + * @returns {Promise} + */ +export const disableLayers = id => api.fetch(`${API_PATH.WAREHOUSE_PATH}layers/${id}/disable`, {}, "put"); + +/** + * 启用分层 + * @param id {Number} id + * @returns {Promise} + */ +export const enableLayers = id => api.fetch(`${API_PATH.WAREHOUSE_PATH}layers/${id}/enable`, {}, "put"); + +/** + * 获取可用库 + * @returns {Promise} + */ +export const getDbs = () => api.fetch(`${API_PATH.WAREHOUSE_PATH}dbs/hive`, {}, "get"); diff --git a/web/src/apps/dataWarehouseDesign/service/api/modifiers.js b/web/src/apps/dataWarehouseDesign/service/api/modifiers.js new file mode 100644 index 0000000000..d02812694c --- /dev/null +++ b/web/src/apps/dataWarehouseDesign/service/api/modifiers.js @@ -0,0 +1,55 @@ +import api from "@/common/service/api"; +import API_PATH from "@/common/config/apiPath"; + +/** + * 创建修饰词 + * @param body {Object} 数据体 + * @return {Promise} + */ +export const createModifiers = body => api.fetch(`${API_PATH.WAREHOUSE_PATH}modifiers`, body, "post"); + +/** + * 删除修饰词 + * @param id {Number} id + * @return {Promise} + */ +export const deleteModifiers = id => api.fetch(`${API_PATH.WAREHOUSE_PATH}modifiers/${id}`, {}, "delete"); + +/** + * 编辑修饰词 + * @param id {Number} id + * @param body {Object} 数据体 + * @return {Promise} + */ +export const editModifiers = (id, body) => api.fetch(`${API_PATH.WAREHOUSE_PATH}modifiers/${id}`, body, "put"); + +/** + * 分页获取修饰词 + * @param page {Number} 当前页 + * @param size {分页数量} 数量 + * @param name {String} 搜索名称 + * @param enabled {Boolean} 是否启用 + * @return {Promise} + */ +export const getModifiers = ({page, size, name, enabled} = {}) => api.fetch(`${API_PATH.WAREHOUSE_PATH}modifiers`, {page, size, name, enabled}, "get"); + +/** + * 根据id获取 + * @param id {Number} id + * @return {Promise} + */ +export const getModifiersById = id => api.fetch(`${API_PATH.WAREHOUSE_PATH}modifiers/${id}`, {}, "get"); + +/** + * 禁用修饰词 + * @param id {Number} id + * @return {Promise} + */ +export const disableModifiers = id => api.fetch(`${API_PATH.WAREHOUSE_PATH}modifiers/${id}/disable`, {}, "put"); + +/** + * 启用修饰词 + * @param id {Number} id + * @return {Promise} + */ +export const enableModifiers = id => api.fetch(`${API_PATH.WAREHOUSE_PATH}modifiers/${id}/enable`, {}, "put"); diff --git a/web/src/apps/dataWarehouseDesign/service/api/statisticalPeriods.js b/web/src/apps/dataWarehouseDesign/service/api/statisticalPeriods.js new file mode 100644 index 0000000000..636802a670 --- /dev/null +++ b/web/src/apps/dataWarehouseDesign/service/api/statisticalPeriods.js @@ -0,0 +1,55 @@ +import api from "@/common/service/api"; +import API_PATH from "@/common/config/apiPath"; + +/** + * 创建统计周期 + * @param body {Object} 数据体 + * @return {Promise} + */ +export const createStatisticalPeriods = body => api.fetch(`${API_PATH.WAREHOUSE_PATH}statistical_periods`, body, "post"); + +/** + * 删除统计周期 + * @param id {Number} 统计周期id + * @return {Promise} + */ +export const deleteStatisticalPeriods = id => api.fetch(`${API_PATH.WAREHOUSE_PATH}statistical_periods/${id}`, {}, "delete"); + +/** + * 编辑统计周期 + * @param id {Number} 统计周期id + * @param body {Object} 数据体 + * @return {Promise} + */ +export const editStatisticalPeriods = (id, body) => api.fetch(`${API_PATH.WAREHOUSE_PATH}statistical_periods/${id}`, body, "put"); + +/** + * 分页获取统计周期 + * @param page {Number} 当前页 + * @param size {分页数量} 数量 + * @param name {String} 搜索名称 + * @param enabled {Boolean} 是否启用 + * @return {Promise} + */ +export const getStatisticalPeriods = ({page, size, name, enabled} = {}) => api.fetch(`${API_PATH.WAREHOUSE_PATH}statistical_periods`, {page, size, name, enabled}, "get"); + +/** + * 根据id获取统计周期 + * @param id {Number} 周期id + * @return {Promise} + */ +export const getStatisticalPeriodsById = id => api.fetch(`${API_PATH.WAREHOUSE_PATH}statistical_periods/${id}`, {}, "get"); + +/** + * 禁用统计周期 + * @param id {Number} 周期id + * @return {Promise} + */ +export const disableStatisticalPeriods = id => api.fetch(`${API_PATH.WAREHOUSE_PATH}statistical_periods/${id}/disable`, {}, "put"); + +/** + * 启用统计周期 + * @param id {Number} 周期id + * @return {Promise} + */ +export const enableStatisticalPeriods = id => api.fetch(`${API_PATH.WAREHOUSE_PATH}statistical_periods/${id}/enable`, {}, "put"); diff --git a/web/src/apps/dataWarehouseDesign/service/api/theme.js b/web/src/apps/dataWarehouseDesign/service/api/theme.js new file mode 100644 index 0000000000..c0809389be --- /dev/null +++ b/web/src/apps/dataWarehouseDesign/service/api/theme.js @@ -0,0 +1,56 @@ +import API_PATH from "@/common/config/apiPath.js"; +import api from "@/common/service/api"; + +/** + * 查询主主题域 + * @param page {Number} 当前页 + * @param size {分页数量} 数量 + * @param name {String} 搜索名称 + * @param enabled {Boolean} 是否启用 + * @returns {Promise} + */ +export const getThemedomains = ({page, size, name, enabled} = {}) => api.fetch(`${API_PATH.WAREHOUSE_PATH}themedomains`, {page, size, name, enabled}, "get"); + +/** + * 创建主题域 + * @param body {Object} 数据体 + * @returns {Promise} + */ +export const createThemedomains = body => api.fetch(`${API_PATH.WAREHOUSE_PATH}themedomains`, body, "post"); + +/** + * 删除主题域 + * @params {id} + * @returns {Promise} + */ +export const deleteThemedomains = id => api.fetch(`${API_PATH.WAREHOUSE_PATH}themedomains/${id}`, {}, "delete"); + +/** + * 禁用主题域 + * @param id {Number} id + * @returns {Promise} + */ +export const disableThemedomains = id => api.fetch(`${API_PATH.WAREHOUSE_PATH}themedomains/${id}/disable`, {}, "put"); + +/** + * 启用主题域 + * @param id {Number} id + * @returns {Promise} + */ +export const enableThemedomains = id => api.fetch(`${API_PATH.WAREHOUSE_PATH}themedomains/${id}/enable`, {}, "put"); + +/** + * 根据id获取主题 + * @param id {Number} id + * @returns {Promise} + */ +export const getThemedomainsById = id => api.fetch(`${API_PATH.WAREHOUSE_PATH}themedomains/${id}`, {}, "get"); + +/** + * 更新主题 + * @param id {Number} id + * @param body {Object} 数据体 + * @returns {Promise} + */ +export const editThemedomains = (id, body) => api.fetch(`${API_PATH.WAREHOUSE_PATH}themedomains/${id}`, body, "put"); + diff --git a/web/src/apps/dataWarehouseDesign/utils/formatDate.js b/web/src/apps/dataWarehouseDesign/utils/formatDate.js new file mode 100644 index 0000000000..9c64db3f4e --- /dev/null +++ b/web/src/apps/dataWarehouseDesign/utils/formatDate.js @@ -0,0 +1,11 @@ +import moment from "moment"; + +/** + * 转换时间戳到字符串 + * @param (Number) + * @return (String) + */ +export default function formatDate(value) { + if (!value) return ""; + return moment(value).format("YYYY-MM-DD HH:mm"); +} diff --git a/web/src/apps/dataWarehouseDesign/view/layered/index.vue b/web/src/apps/dataWarehouseDesign/view/layered/index.vue new file mode 100644 index 0000000000..858c7f2b09 --- /dev/null +++ b/web/src/apps/dataWarehouseDesign/view/layered/index.vue @@ -0,0 +1,11 @@ + + diff --git a/web/src/apps/dataWarehouseDesign/view/layout/emptylayout.vue b/web/src/apps/dataWarehouseDesign/view/layout/emptylayout.vue new file mode 100644 index 0000000000..98240aef81 --- /dev/null +++ b/web/src/apps/dataWarehouseDesign/view/layout/emptylayout.vue @@ -0,0 +1,3 @@ + diff --git a/web/src/apps/dataWarehouseDesign/view/layout/index.vue b/web/src/apps/dataWarehouseDesign/view/layout/index.vue new file mode 100644 index 0000000000..b6cef5f965 --- /dev/null +++ b/web/src/apps/dataWarehouseDesign/view/layout/index.vue @@ -0,0 +1,24 @@ + + + + diff --git a/web/src/apps/dataWarehouseDesign/view/layout/sidebar-sub-menu.vue b/web/src/apps/dataWarehouseDesign/view/layout/sidebar-sub-menu.vue new file mode 100644 index 0000000000..dd958f7a34 --- /dev/null +++ b/web/src/apps/dataWarehouseDesign/view/layout/sidebar-sub-menu.vue @@ -0,0 +1,61 @@ + + + diff --git a/web/src/apps/dataWarehouseDesign/view/layout/sidebar.vue b/web/src/apps/dataWarehouseDesign/view/layout/sidebar.vue new file mode 100644 index 0000000000..4353f2610a --- /dev/null +++ b/web/src/apps/dataWarehouseDesign/view/layout/sidebar.vue @@ -0,0 +1,41 @@ + + + diff --git a/web/src/apps/dataWarehouseDesign/view/modifier/index.vue b/web/src/apps/dataWarehouseDesign/view/modifier/index.vue new file mode 100644 index 0000000000..bce20146a1 --- /dev/null +++ b/web/src/apps/dataWarehouseDesign/view/modifier/index.vue @@ -0,0 +1,11 @@ + + diff --git a/web/src/apps/dataWarehouseDesign/view/statPeriod/index.vue b/web/src/apps/dataWarehouseDesign/view/statPeriod/index.vue new file mode 100644 index 0000000000..15c9006e03 --- /dev/null +++ b/web/src/apps/dataWarehouseDesign/view/statPeriod/index.vue @@ -0,0 +1,11 @@ + + diff --git a/web/src/apps/dataWarehouseDesign/view/themeDomains/index.vue b/web/src/apps/dataWarehouseDesign/view/themeDomains/index.vue new file mode 100644 index 0000000000..7e884d7f85 --- /dev/null +++ b/web/src/apps/dataWarehouseDesign/view/themeDomains/index.vue @@ -0,0 +1,11 @@ + + diff --git a/web/src/apps/scriptis/components/importToHive/index.vue b/web/src/apps/scriptis/components/importToHive/index.vue index 2d829c173c..d2aace8a47 100755 --- a/web/src/apps/scriptis/components/importToHive/index.vue +++ b/web/src/apps/scriptis/components/importToHive/index.vue @@ -151,60 +151,60 @@ - -
- {{ secondStep.partition }} - = - -
- + +
+ {{ secondStep.partition }} + = + +
+ - - - + + + - - - + + + - - - + + + - - = - - + + = + + - +
序号
diff --git a/web/src/apps/workflows/service/nodeType.js b/web/src/apps/workflows/service/nodeType.js index 2933f9f31f..5b80444a03 100755 --- a/web/src/apps/workflows/service/nodeType.js +++ b/web/src/apps/workflows/service/nodeType.js @@ -15,136 +15,136 @@ * */ -import shell from '../module/process/images/shell.svg'; -import flow from '../module/process/images/newIcon/flow.svg'; -import eventchecker from '../module/process/images/newIcon/eventcheckerf.svg'; -import sparkPython from '../module/process/images/pyspark.svg'; -import sql from '../module/process/images/newIcon/spark.svg'; -import hivesql from '../module/process/images/newIcon/hive.svg'; -import veceive from '../module/process/images/newIcon/eventcheckerw.svg'; -import connect from '../module/process/images/newIcon/connector.svg'; -import datacheck from '../module/process/images/newIcon/datacheck.svg'; -import send from '../module/process/images/sender.svg'; -import display from '../module/process/images/newIcon/display.svg'; -import dashboard from '../module/process/images/newIcon/Dashboard.svg'; -import widget from '../module/process/images/newIcon/widget.svg'; -import sendmail from '../module/process/images/newIcon/email.svg'; -import scala from '../module/process/images/scala.svg'; -import exchange from '../module/process/images/newIcon/exchange.svg'; -import qualitis from '../module/process/images/newIcon/qualitis.svg'; -import python from '../module/process/images/newIcon/python.svg'; -import projectNode from '../module/process/images/newIcon/projectNode.svg' -import mlss from '../module/process/images/newIcon/mlss.svg' +import shell from "../module/process/images/shell.svg"; +import flow from "../module/process/images/newIcon/flow.svg"; +import eventchecker from "../module/process/images/newIcon/eventcheckerf.svg"; +import sparkPython from "../module/process/images/pyspark.svg"; +import sql from "../module/process/images/newIcon/spark.svg"; +import hivesql from "../module/process/images/newIcon/hive.svg"; +import veceive from "../module/process/images/newIcon/eventcheckerw.svg"; +import connect from "../module/process/images/newIcon/connector.svg"; +import datacheck from "../module/process/images/newIcon/datacheck.svg"; +import send from "../module/process/images/sender.svg"; +import display from "../module/process/images/newIcon/display.svg"; +import dashboard from "../module/process/images/newIcon/Dashboard.svg"; +import widget from "../module/process/images/newIcon/widget.svg"; +import sendmail from "../module/process/images/newIcon/email.svg"; +import scala from "../module/process/images/scala.svg"; +import exchange from "../module/process/images/newIcon/exchange.svg"; +import qualitis from "../module/process/images/newIcon/qualitis.svg"; +import python from "../module/process/images/newIcon/python.svg"; +import projectNode from "../module/process/images/newIcon/projectNode.svg"; +import mlss from "../module/process/images/newIcon/mlss.svg"; const NODETYPE = { - SHELL: 'linkis.shell.sh', - HQL: 'linkis.hive.hql', - SPARKSQL: 'linkis.spark.sql', - SPARKPY: 'linkis.spark.py', - SCALA: 'linkis.spark.scala', - PYTHON: 'linkis.python.python', - CONNECTOR: 'linkis.control.empty', - DISPLAY: 'linkis.appconn.visualis.display', - DASHBOARD: 'linkis.appconn.visualis.dashboard', - WIDGET: 'linkis.appconn.visualis.widget', - SENDMAIL: 'linkis.appconn.sendemail', - EVENTCHECKERF: 'linkis.appconn.eventchecker.eventsender', - EVENTCHECKERW: 'linkis.appconn.eventchecker.eventreceiver', - DATACHECKER: 'linkis.appconn.datachecker', - RMBSENDER: 'azkaban.rmbsender', - FLOW: 'workflow.subflow', - EXCHANGE: 'linkis.data.exchange', - QUALITIS: 'linkis.appconn.qualitis', - PROJECTNODE: 'projectNode', - MLSS: 'linkis.appconn.mlss' -} + SHELL: "linkis.shell.sh", + HQL: "linkis.hive.hql", + SPARKSQL: "linkis.spark.sql", + SPARKPY: "linkis.spark.py", + SCALA: "linkis.spark.scala", + PYTHON: "linkis.python.python", + CONNECTOR: "linkis.control.empty", + DISPLAY: "linkis.appconn.visualis.display", + DASHBOARD: "linkis.appconn.visualis.dashboard", + WIDGET: "linkis.appconn.visualis.widget", + SENDMAIL: "linkis.appconn.sendemail", + EVENTCHECKERF: "linkis.appconn.eventchecker.eventsender", + EVENTCHECKERW: "linkis.appconn.eventchecker.eventreceiver", + DATACHECKER: "linkis.appconn.datachecker", + RMBSENDER: "azkaban.rmbsender", + FLOW: "workflow.subflow", + EXCHANGE: "linkis.data.exchange", + QUALITIS: "linkis.appconn.qualitis", + PROJECTNODE: "projectNode", + MLSS: "linkis.appconn.mlss" +}; const ext = { - [NODETYPE.SHELL]: 'shell', - [NODETYPE.HQL]: 'hql', - [NODETYPE.SPARKSQL]: 'sql', - [NODETYPE.SPARKPY]: 'pyspark', - [NODETYPE.SCALA]: 'scala', - [NODETYPE.PYTHON]: 'python' -} + [NODETYPE.SHELL]: "shell", + [NODETYPE.HQL]: "hql", + [NODETYPE.SPARKSQL]: "sql", + [NODETYPE.SPARKPY]: "pyspark", + [NODETYPE.SCALA]: "scala", + [NODETYPE.PYTHON]: "python" +}; const NODEICON = { [NODETYPE.SHELL]: { icon: shell, - class: {'shell': true} + class: { shell: true } }, [NODETYPE.HQL]: { icon: hivesql, - class: {'hivesql': true} + class: { hivesql: true } }, [NODETYPE.SPARKSQL]: { icon: sql, - class: {'sql': true} + class: { sql: true } }, [NODETYPE.SPARKPY]: { icon: sparkPython, - class: {'sparkPython': true} + class: { sparkPython: true } }, [NODETYPE.SCALA]: { icon: scala, - class: {'scala': true} + class: { scala: true } }, [NODETYPE.PYTHON]: { icon: python, - class: {'python': true} + class: { python: true } }, [NODETYPE.CONNECTOR]: { icon: connect, - class: {'connect': true} + class: { connect: true } }, [NODETYPE.DISPLAY]: { icon: display, - class: {'display': true} + class: { display: true } }, [NODETYPE.DASHBOARD]: { icon: dashboard, - class: {'dashboard': true} + class: { dashboard: true } }, [NODETYPE.WIDGET]: { icon: widget, - class: {'widget': true} + class: { widget: true } }, [NODETYPE.SENDMAIL]: { icon: sendmail, - class: {'sendmail': true} + class: { sendmail: true } }, [NODETYPE.EVENTCHECKERF]: { icon: eventchecker, - class: {'eventchecker': true} + class: { eventchecker: true } }, [NODETYPE.EVENTCHECKERW]: { icon: veceive, - class: {'veceive': true} + class: { veceive: true } }, [NODETYPE.DATACHECKER]: { icon: datacheck, - class: {'datacheck': true} + class: { datacheck: true } }, [NODETYPE.RMBSENDER]: { icon: send, - class: {'send': true} + class: { send: true } }, [NODETYPE.FLOW]: { icon: flow, - class: {'flow': true} + class: { flow: true } }, [NODETYPE.EXCHANGE]: { icon: exchange, - class: {'exchange': true} + class: { exchange: true } }, [NODETYPE.QUALITIS]: { icon: qualitis, - class: {'qualitis': true} + class: { qualitis: true } }, [NODETYPE.PROJECTNODE]: { icon: projectNode, - class: {'projectNode': true} + class: { projectNode: true } }, [NODETYPE.MLSS]: { icon: mlss, - class: {'mlss': true} - }, -} -export { NODETYPE, ext, NODEICON}; + class: { mlss: true } + } +}; +export { NODETYPE, ext, NODEICON }; diff --git a/web/src/apps/workspace/module/management/component/formsechbar.vue b/web/src/apps/workspace/module/management/component/formsechbar.vue index cc4958c0ab..f4c911cb0f 100755 --- a/web/src/apps/workspace/module/management/component/formsechbar.vue +++ b/web/src/apps/workspace/module/management/component/formsechbar.vue @@ -4,9 +4,9 @@ style="width:250px" v-model="searchBar.searchText" :placeholder="searchBar.title"> - + - - + diff --git a/web/src/common/config/apiPath.js b/web/src/common/config/apiPath.js index 50f8d6abe1..35fe370305 100755 --- a/web/src/common/config/apiPath.js +++ b/web/src/common/config/apiPath.js @@ -1,7 +1,10 @@ export default { - WORKSPACE_PATH: '/dss/framework/workspace/', - PROJECT_PATH: '/dss/framework/project/', - ORCHESTRATOR_PATH: '/dss/framework/orchestrator/', - WORKFLOW_PATH: '/dss/workflow/', - PUBLISH_PATH: '/dss/framework/release/' -} \ No newline at end of file + WORKSPACE_PATH: "/dss/framework/workspace/", + PROJECT_PATH: "/dss/framework/project/", + ORCHESTRATOR_PATH: "/dss/framework/orchestrator/", + WORKFLOW_PATH: "/dss/workflow/", + PUBLISH_PATH: "/dss/framework/release/", + DATAMODEL_PATH: "/datamodel/", + WAREHOUSE_PATH: "/data-warehouse/", + DATA_GOVERNANCE: "/data-assets/asset/" +}; diff --git a/web/src/common/style/_theme.scss b/web/src/common/style/_theme.scss new file mode 100644 index 0000000000..8dfbd72d07 --- /dev/null +++ b/web/src/common/style/_theme.scss @@ -0,0 +1,88 @@ +/*! + * Copyright 2019 WeBank + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Color +$primary-color: #2E92F7; +$dark-primary-color: #4B8FF3; + +$workspace-background: #F4F7FB; //$workspace-background +$dark-workspace-background: #151A23; + +$background-color-header: #0B203C; +$dark-background-color-header: #151A23; + +// base +$light-base-color: $body-background; // #FFFFFF +$dark-base-color: #1E232D; +$dark-menu-base-color: #2A303C; +$dark-submenu-color: #242A36; +$dark-input-placeholder-color: rgba(255,255,255,0.25); +$dark-border-color: #404A5D; +$dark-disable-color: #3C4352; +$dark-disable-text-color: rgba(255,255,255,0.25); + +// workspace text color +$light-text-color: rgba(0,0,0,0.65); +$dark-text-color: rgba(255,255,255,0.65); //$text-desc-color: rgba(0,0,0,0.65); + +// border color +$border-color-base: $border-color-base; //#dcdee2 +$dark-border-color-base: rgba(255,255,255,0.20); + +// $dark-text-color: #F4F7FB; + +// workspace title color +$workspace-title-color: $text-title-color; //rgba(0, 0, 0, 0.85) +$dark-workspace-title-color: rgba(255,255,255,0.85); + +$workspace-name-color: #044B93; +$dark-workspace-name-color: #FFFFFF; + +$light-text-desc-color: rgba(0, 0, 0, 0.45); +$dark-text-desc-color: rgba(255, 255, 255, 0.45); +// workspace body backround color +$workspace-body-bg-color: $light-base-color; // #FFFFFF +$dark-workspace-body-bg-color: $dark-base-color; + +// appProcess color +$app-process-bg-color: $light-base-color; +$dark-app-process-bg-color: #2A303C; + +$hover-color-base: #ECF4FF; //#F3F3F3 +$dark-hover-color-base: rgba(75,143,243,0.50); + +$active-menu-item: #ECEFF4; +$dark-active-menu-item: #252A35; + +$menu-list-bg-color: $light-base-color; +$dark-menu-list-bg-color: #3A404B; +// workflow +$workflow-body-bg-color: #F8F9FC; +$dark-workflow-body-bg-color: $dark-base-color; + +$header-tool-bg-color: #F7F7F7; +$dark-header-tool-bg-color: $dark-base-color; + +$dark-workflow-bg-color: #2A303C; +$dark-workflow-font-color: rgba(255,255,255,0.65); + +$dark-header-bg-color: #242934; + +$suspanding-panel-bg-color: #061528; + +$disabled-bg-color: #f3f3f3; +$dark-disabled-bg-color: #252A35; diff --git a/web/src/common/style/default.scss b/web/src/common/style/default.scss new file mode 100644 index 0000000000..db86a69a50 --- /dev/null +++ b/web/src/common/style/default.scss @@ -0,0 +1,320 @@ +/*! + * Copyright 2019 WeBank + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Color +$primary-color: #2d8cf0; +$info-color: #2db7f5; +$success-color: #19be6b; +$processing-color: $primary-color; +$warning-color: #ff9900; +$error-color: #ed4014; +$normal-color: #e6ebf1; +$link-color: #2D8cF0; +$link-hover-color: mix(#ffffff, $link-color, 20%); +$link-active-color: mix(#000000, $link-color, 5%); +$selected-color: rgba($primary-color, 90%); +$tooltip-color: #fff; +$subsidiary-color: #808695; +$rate-star-color: #f5a623; +$pink-color: rgba(243, 133, 243, 0.753); +$yellow-color: #f4cf2a; + +// Base +$body-background: #fff; +$font-family: "Helvetica Neue",Helvetica,"PingFang SC","Hiragino Sans GB","Microsoft YaHei","微软雅黑",Arial,sans-serif; +$code-family: Consolas,Menlo,Courier,monospace; +$title-color: #17233d; +$text-color: #515a6e; +$font-size-base: 14px; +$font-size-small: 12px; +$font-size-large: 16px; +$line-height-base: 1.5; +$line-height-computed: floor(($font-size-base * $line-height-base)); +$border-radius-base: 6px; +$border-radius-small: 4px; +$border-radius-large: 8px; +$cursor-disabled: not-allowed; + +// Border color +$border-color-base: #DEE4EC; +// outside +$border-color-split: #e8eaec; +// inside +$border-color-bule: #3c8dbc; +$border-width-base: 1px; +// width of the border for a component +$border-style-base: solid; +// style of a components border + +// Background color +$background-color-white: #fff; +$background-color-base: #f7f7f7; +// base +$background-color-select-hover: #f3f3f3; +// $background-color-header: #0B203C; +$tooltip-bg: rgba(70, 76, 91, .9); +$head-bg: #f9fafc; +$table-thead-bg: #f8f8f9; +$table-thead-blue-bg: $primary-color; +$table-td-stripe-bg: #f8f8f9; +$table-td-hover-bg: #ebf7ff; +$table-td-highlight-bg: #ebf7ff; +$menu-dark-title: #515a6e; +$menu-dark-active-bg: #363e4f; +$menu-dark-subsidiary-color: rgba(255,255,255,.7); +$menu-dark-group-title-color: rgba(255,255,255,.36); +$date-picker-cell-hover-bg: #e1f0fe; + +// Shadow +$shadow-color: rgba(0, 0, 0, .2); +$shadow-base: -1px 0 6px $shadow-color; +$shadow-card: 0 1px 1px 0 rgba(0,0,0,.1); +$shadow-up: 0 -1px 6px $shadow-color; +$shadow-down: 0 1px 6px $shadow-color; +$shadow-left: -1px 0 6px $shadow-color; +$shadow-right: 1px 0 6px $shadow-color; +$shadow-inset: inset 0 0 5px $shadow-color; + +// Button +$btn-font-weight: normal; +$btn-padding-base: 5px 15px 6px; +$btn-padding-large: 6px 15px 6px 15px; +$btn-padding-small: 1px 7px 2px; +$btn-padding-base-icon: 5px 15px 6px; +$btn-padding-large-icon: 6px 15px 6px 15px; +$btn-padding-small-icon: 1px 7px 2px; +$btn-font-size: 12px; +$btn-font-size-large: 14px; +$btn-border-radius: 4px; +$btn-border-radius-small: 3px; +$btn-group-border: shade($primary-color, 5%); + +$btn-disable-color: #c5c8ce; +$btn-disable-bg: $background-color-base; +$btn-disable-border: $border-color-base; + +$btn-default-color: $text-color; +$btn-default-bg: #fff; +$btn-default-border: $border-color-base; + +$btn-primary-color: #fff; +$btn-primary-bg: $primary-color; + +$btn-ghost-color: $text-color; +$btn-ghost-bg: #fff; +$btn-ghost-border: $border-color-base; + +$btn-circle-size: 32px; +$btn-circle-size-large: 36px; +$btn-circle-size-small: 24px; + +// Layout and Grid +$grid-columns: 24; +$grid-gutter-width: 0; +$layout-body-background: #f5f7f9; +$layout-header-background: #515a6e; +$layout-header-height: 64px; +$layout-header-padding: 0 50px; +$layout-footer-padding: 24px 50px; +$layout-footer-background: $layout-body-background; +$layout-sider-background: $layout-header-background; +$layout-trigger-height: 48px; +$layout-trigger-color: #fff; +$layout-zero-trigger-width: 36px; +$layout-zero-trigger-height: 42px; + +// Legend +$legend-color: #999; + +// Input +$input-height-base: 32px; +$input-height-large: 36px; +$input-height-small: 24px; + +$input-padding-horizontal: 7px; +$input-padding-vertical-base: 4px; +$input-padding-vertical-small: 1px; +$input-padding-vertical-large: 6px; + +$input-placeholder-color: $btn-disable-color; +$input-color: $text-color; +$input-border-color: $border-color-base; +$input-bg: #fff; +$input-group-bg: #f8f8f9; + +$input-hover-border-color: $primary-color; +$input-focus-border-color: $primary-color; +$input-disabled-bg: #f3f3f3; + +// Tag +$tag-font-size: 12px; + +// Media queries breakpoints +// Extra small screen / phone +$screen-xs: 480px; +$screen-xs-min: $screen-xs; +$screen-xs-max: ($screen-xs-min - 1); + +// Small screen / tablet +$screen-sm: 768px; +$screen-sm-min: $screen-sm; +$screen-sm-max: ($screen-sm-min - 1); + +// Medium screen / desktop +$screen-md: 992px; +$screen-md-min: $screen-md; +$screen-md-max: ($screen-md-min - 1); + +// Large screen / wide desktop +$screen-lg: 1200px; +$screen-lg-min: $screen-lg; +$screen-lg-max: ($screen-lg-min - 1); + +// Z-index +$zindex-spin: 8; +$zindex-affix: 10; +$zindex-back-top: 10; +$zindex-select: 900; +$zindex-modal: 1000; +$zindex-drawer: 1000; +$zindex-message: 1010; +$zindex-notification: 1010; +$zindex-tooltip: 1060; +$zindex-transfer: 1060; +$zindex-loading-bar: 2000; +$zindex-spin-fullscreen: 2010; + +// Animation +$animation-time: .3s; +$transition-time: .2s; +$ease-in-out: ease-in-out; + +// Slider +$slider-color: tint($primary-color, 20%); +$slider-height: 4px; +$slider-margin: 16px 0; +$slider-button-wrap-size: 18px; +$slider-button-wrap-offset: -4px; +$slider-disabled-color: #ccc; + +// Avatar +$avatar-size-base: 32px; +$avatar-size-lg: 40px; +$avatar-size-sm: 24px; +$avatar-font-size-base: 18px; +$avatar-font-size-lg: 24px; +$avatar-font-size-sm: 14px; +$avatar-bg: #ccc; +$avatar-color: #fff; +$avatar-border-radius: $border-radius-small; + +// Anchor +$anchor-border-width: 2px; + +$dark-bg: #252a2f; +$dark-bg-light: rgb(0, 33, 64); +$dark-bg-dark: rgb(0, 12, 23); +$dark-color: rgba(255, 255, 255, 0.65); +$dark-selected-bg: rgb(24, 144, 255); +$dark-selected-color: #ffffff; +$dark-active-color: #2E92F7; +// global position variables +$fixed: fixed; +$absolute: absolute; +$relative: relative; + +// scrollbar +$scrollbar-color: #cdcdcd; + +// 新加入的 +// global color variables +// button color schema +$btn-origin-color: #0a9fec; +$btn-over-color: #54bcf2; +$btn-click-color: #076fa5; + +// actionbar color schema +$actionbar-origin-color: #898989; +$actionbar-over-color: #b5b5b5; +$actionbar-click-color: #0a9fec; +$actionbar-run-color: #ff0000; + +// text-color +$text-blue: #0a9fec; +$text-origin-color: darken(#898989, 30%); +$text-click-color: #0a9fec; +$text-over-color: #b5b5b5; +$text-red-color: #ff0000; +$text-color: #515a6e; +$input-placeholder-color: #c5c8ce; +$text-title-color: rgba(0, 0, 0, 0.85); +$text-desc-color: rgba(0,0,0,0.65); +$text-tille-light-color: rgba(255,255,255,0.85); + +$grey-bg-color: #eff1f6; +$darken-grey-bg: darken(#eff1f6, 5%); + +// line color schema +$line-color: darken(#dcdee2, 12%); + +$line-border: 1px solid $line-color; +$blue-line-bd: 3px solid #0a9fec; +$black-line-bd: 3px solid #3D3D3D; + +// global dimension variables +$header-height: 36px; +$sidenav-width: 45px; +$sidewin-width: 306px; +$sidewin-min-width: 206px; +$sidenav-item-height: 110px; + +$actionbar-height: 34px; +$editor-height: 200px; +$editor-tab-height: 35px; +$output-tab-height: 35px; +$common-tab-height: 35px; + +// global font size variables +$text-sz-default: 12px; +$font-size: 12px; +$font-size-14: 14px; + +// global align variables +$align-left: left; +$align-right: right; +$align-center: center; + +// global maring padding variables +$margin: 10px; +$padding: 10px; +$padding-25: 25px; + +// global % variables +$percent-all: 100%; +$percent-half: 50%; + +// @mixin button-stress($color) +// color: white +// background-color: $color + +// @mixin button-normal($color) +// color: $color +// background-color: white + +$workspace-background: #F4F7FB; + +$toolbarWidth: 40px; diff --git a/web/src/common/style/variables.scss b/web/src/common/style/variables.scss index 213c56419c..95f1a8bf53 100755 --- a/web/src/common/style/variables.scss +++ b/web/src/common/style/variables.scss @@ -16,304 +16,37 @@ */ // Color -$primary-color : #2d8cf0; -$info-color : #2db7f5; -$success-color : #19be6b; -$processing-color : $primary-color; -$warning-color : #ff9900; -$error-color : #ed4014; -$normal-color : #e6ebf1; -$link-color : #2D8cF0; -$link-hover-color : mix(#ffffff, $link-color, 20%); -$link-active-color : mix(#000000, $link-color, 5%); -$selected-color : rgba($primary-color, 90%); -$tooltip-color : #fff; -$subsidiary-color : #808695; -$rate-star-color : #f5a623; -$pink-color : rgba(243, 133, 243, 0.753); -$yellow-color : #f4cf2a; -// Base -$body-background : #fff; -$font-family : "Helvetica Neue",Helvetica,"PingFang SC","Hiragino Sans GB","Microsoft YaHei","微软雅黑",Arial,sans-serif; -$code-family : Consolas,Menlo,Courier,monospace; -$title-color : #17233d; -$text-color : #515a6e; -$font-size-base : 14px; -$font-size-small : 12px; -$font-size-large : 16px; -$line-height-base : 1.5; -$line-height-computed : floor(($font-size-base * $line-height-base)); -$border-radius-base : 6px; -$border-radius-small : 4px; -$border-radius-large : 8px; -$cursor-disabled : not-allowed; - -// Border color -$border-color-base : #dcdee2; // outside -$border-color-split : #e8eaec; // inside -$border-color-bule : #3c8dbc; -$border-width-base : 1px; // width of the border for a component -$border-style-base : solid; // style of a components border - -// Background color -$background-color-white : #fff; -$background-color-base : #f7f7f7; // base -$background-color-select-hover: #f3f3f3; -$background-color-header: #001C40; -$tooltip-bg : rgba(70, 76, 91, .9); -$head-bg : #f9fafc; -$table-thead-bg : #f8f8f9; -$table-thead-blue-bg : $primary-color; -$table-td-stripe-bg : #f8f8f9; -$table-td-hover-bg : #ebf7ff; -$table-td-highlight-bg : #ebf7ff; -$menu-dark-title : #515a6e; -$menu-dark-active-bg : #363e4f; -$menu-dark-subsidiary-color : rgba(255,255,255,.7); -$menu-dark-group-title-color : rgba(255,255,255,.36); -$date-picker-cell-hover-bg : #e1f0fe; - -// Shadow -$shadow-color : rgba(0, 0, 0, .2); -$shadow-base : -1px 0 6px $shadow-color; -$shadow-card : 0 1px 1px 0 rgba(0,0,0,.1); -$shadow-up : 0 -1px 6px $shadow-color; -$shadow-down : 0 1px 6px $shadow-color; -$shadow-left : -1px 0 6px $shadow-color; -$shadow-right : 1px 0 6px $shadow-color; -$shadow-inset : inset 0 0 5px $shadow-color; - -// Button -$btn-font-weight : normal; -$btn-padding-base : 5px 15px 6px; -$btn-padding-large : 6px 15px 6px 15px; -$btn-padding-small : 1px 7px 2px; -$btn-padding-base-icon : 5px 15px 6px; -$btn-padding-large-icon : 6px 15px 6px 15px; -$btn-padding-small-icon : 1px 7px 2px; -$btn-font-size : 12px; -$btn-font-size-large : 14px; -$btn-border-radius : 4px; -$btn-border-radius-small: 3px; -$btn-group-border : shade($primary-color, 5%); - -$btn-disable-color : #c5c8ce; -$btn-disable-bg : $background-color-base; -$btn-disable-border : $border-color-base; - -$btn-default-color : $text-color; -$btn-default-bg : #fff; -$btn-default-border : $border-color-base; - -$btn-primary-color : #fff; -$btn-primary-bg : $primary-color; - -$btn-ghost-color : $text-color; -$btn-ghost-bg : #fff; -$btn-ghost-border : $border-color-base; - -$btn-circle-size : 32px; -$btn-circle-size-large : 36px; -$btn-circle-size-small : 24px; - -// Layout and Grid -$grid-columns : 24; -$grid-gutter-width : 0; -$layout-body-background : #f5f7f9; -$layout-header-background : #515a6e; -$layout-header-height : 64px; -$layout-header-padding : 0 50px; -$layout-footer-padding : 24px 50px; -$layout-footer-background : $layout-body-background; -$layout-sider-background : $layout-header-background; -$layout-trigger-height : 48px; -$layout-trigger-color : #fff; -$layout-zero-trigger-width : 36px; -$layout-zero-trigger-height : 42px; - -// Legend -$legend-color : #999; - -// Input -$input-height-base : 32px; -$input-height-large : 36px; -$input-height-small : 24px; - -$input-padding-horizontal : 7px; -$input-padding-vertical-base : 4px; -$input-padding-vertical-small: 1px; -$input-padding-vertical-large: 6px; - -$input-placeholder-color : $btn-disable-color; -$input-color : $text-color; -$input-border-color : $border-color-base; -$input-bg : #fff; -$input-group-bg : #f8f8f9; - -$input-hover-border-color : $primary-color; -$input-focus-border-color : $primary-color; -$input-disabled-bg : #f3f3f3; - -// Tag -$tag-font-size : 12px; - -// Media queries breakpoints -// Extra small screen / phone -$screen-xs : 480px; -$screen-xs-min : $screen-xs; -$screen-xs-max : ($screen-xs-min - 1); - -// Small screen / tablet -$screen-sm : 768px; -$screen-sm-min : $screen-sm; -$screen-sm-max : ($screen-sm-min - 1); - -// Medium screen / desktop -$screen-md : 992px; -$screen-md-min : $screen-md; -$screen-md-max : ($screen-md-min - 1); - -// Large screen / wide desktop -$screen-lg : 1200px; -$screen-lg-min : $screen-lg; -$screen-lg-max : ($screen-lg-min - 1); - -// Z-index -$zindex-spin : 8; -$zindex-affix : 10; -$zindex-back-top : 10; -$zindex-select : 900; -$zindex-modal : 1000; -$zindex-drawer : 1000; -$zindex-message : 1010; -$zindex-notification : 1010; -$zindex-tooltip : 1060; -$zindex-transfer : 1060; -$zindex-loading-bar : 2000; -$zindex-spin-fullscreen : 2010; - -// Animation -$animation-time : .3s; -$transition-time : .2s; -$ease-in-out : ease-in-out; - -// Slider -$slider-color : tint($primary-color, 20%); -$slider-height : 4px; -$slider-margin : 16px 0; -$slider-button-wrap-size : 18px; -$slider-button-wrap-offset : -4px; -$slider-disabled-color : #ccc; - -// Avatar -$avatar-size-base: 32px; -$avatar-size-lg: 40px; -$avatar-size-sm: 24px; -$avatar-font-size-base: 18px; -$avatar-font-size-lg: 24px; -$avatar-font-size-sm: 14px; -$avatar-bg: #ccc; -$avatar-color: #fff; -$avatar-border-radius: $border-radius-small; - -// Anchor -$anchor-border-width: 2px; - - -$dark-bg: #252a2f; -$dark-bg-light: rgb(0, 33, 64); -$dark-bg-dark: rgb(0, 12, 23); -$dark-color: rgba(255, 255, 255, 0.8); -$dark-selected-bg: rgb(24, 144, 255); -$dark-selected-color: #ffffff; -$dark-active-color: #2E92F7; -// global position variables -$fixed: fixed; -$absolute: absolute; -$relative: relative; - -// scrollbar -$scrollbar-color: #cdcdcd; - - -// 新加入的 -// global color variables -// button color schema -$btn-origin-color: #0a9fec; -$btn-over-color: #54bcf2; -$btn-click-color: #076fa5; - -// actionbar color schema -$actionbar-origin-color: #898989; -$actionbar-over-color: #b5b5b5; -$actionbar-click-color: #0a9fec; -$actionbar-run-color: #ff0000; - -// text-color -$text-blue: #0a9fec; -$text-origin-color: darken(#898989, 30%); -$text-click-color: #0a9fec; -$text-over-color: #b5b5b5; -$text-red-color: #ff0000; -$text-color: #515a6e; -$input-placeholder-color: #c5c8ce; -$text-title-color: rgba(0, 0, 0, 0.85); -$text-desc-color: rgba(0,0,0,0.65); -$text-tille-light-color: rgba(255,255,255,0.85); - -$grey-bg-color: #eff1f6; -$darken-grey-bg: darken(#eff1f6, 5%); - -// line color schema -$line-color: darken(#dcdee2, 12%); - - -$line-border: 1px solid $line-color; -$blue-line-bd: 3px solid #0a9fec; -$black-line-bd: 3px solid #3D3D3D; - - -// global dimension variables -$header-height: 36px; -$sidenav-width: 45px; -$sidewin-width: 306px; -$sidewin-min-width: 206px; -$sidenav-item-height: 110px; - -$actionbar-height: 34px; -$editor-height: 200px; -$editor-tab-height: 35px; -$output-tab-height: 35px; -$common-tab-height: 35px; - -// global font size variables -$text-sz-default: 12px; -$font-size: 12px; -$font-size-14: 14px; - -// global align variables -$align-left: left; -$align-right: right; -$align-center: center; - -// global maring padding variables -$margin: 10px; -$padding: 10px; -$padding-25: 25px; - -// global % variables -$percent-all: 100%; -$percent-half: 50%; - -// @mixin button-stress($color) -// color: white -// background-color: $color - -// @mixin button-normal($color) -// color: $color -// background-color: white - -$workspace-background: #F4F7FB; - -$toolbarWidth: 40px; +@charset "utf-8"; +@import "./default"; +@import "./_theme"; +/*引入配置 所有演示相关必须走函数引用，方便后续换肤*/ +@mixin bg-color($color, $darkColor) { + background-color: $color; + + [data-theme="dark"] & { + background-color: $darkColor; + } +} +@mixin font-color($color, $darkColor) { + color: $color; + + [data-theme="dark"] & { + color: $darkColor; + } +} +@mixin border-color($color, $darkColor) { + border-color: $color; + + [data-theme="dark"] & { + border-color: $darkColor; + } +} + +@mixin guide-box-shadow($color, $darkColor) { + box-shadow: -2px 0 10px 0 $color; + + [data-theme="dark"] & { + box-shadow: -2px 0 10px 0 $darkColor; + } +} diff --git a/web/src/components/consoleComponent/progress.vue b/web/src/components/consoleComponent/progress.vue index 6667ff886c..c3f2e9a9bf 100755 --- a/web/src/components/consoleComponent/progress.vue +++ b/web/src/components/consoleComponent/progress.vue @@ -15,18 +15,18 @@ - {{ $t('message.common.progress.title') + '：' }} + {{ $t('message.common.progress.title') + '：' }} - + - {{$t('message.common.process')}} - {{ costTime }} + {{$t('message.common.process')}} + {{ costTime }} {{ $t('message.common.progress.title') + '：' }} - {{ $t('message.common.progress.watingList', { num: waitingSize }) }} + {{ $t('message.common.progress.watingList', { num: waitingSize }) }} -
- - - - - +
+ + + + + + {{ child.label }} +
+

{{ $t('message.common.detail') }}

+

{{ p }}

+
+ {{ child.label }} -
-

{{ $t('message.common.detail') }}

-

{{ p }}

-
- - {{ child.label }} - - -
+ :class="getClasses(child)" + v-else>{{ child.label }} + + +
diff --git a/web/src/components/consoleComponent/toolbar.vue b/web/src/components/consoleComponent/toolbar.vue index 8148396f0c..3b30f49832 100755 --- a/web/src/components/consoleComponent/toolbar.vue +++ b/web/src/components/consoleComponent/toolbar.vue @@ -25,26 +25,26 @@ - {{$t('message.common.toolbar.graphAnalysis')}} + {{$t('message.common.toolbar.graphAnalysis')}}- {{$t('message.common.toolbar.excelAnalysis')}} + {{$t('message.common.toolbar.excelAnalysis')}} - + - + @@ -74,12 +74,12 @@ - CSV + CSV- Excel + Excel @@ -91,12 +91,12 @@ - UTF-8 + UTF-8- GBK + GBK @@ -108,12 +108,12 @@ - NULL + NULL- {{$t('message.common.toolbar.emptyString')}} + {{$t('message.common.toolbar.emptyString')}} @@ -128,14 +128,14 @@ - + - + diff --git a/web/src/components/dynamicForm/index.vue b/web/src/components/dynamicForm/index.vue index ccee60e5c4..e8d0e8d8b0 100755 --- a/web/src/components/dynamicForm/index.vue +++ b/web/src/components/dynamicForm/index.vue @@ -8,38 +8,38 @@ {{ index + 1 }}- - - - + + + =- - - + + + - + diff --git "a/web/src/components/svgIcon/svg/1 - \345\211\257\346\234\254.svg" "b/web/src/components/svgIcon/svg/1 - \345\211\257\346\234\254.svg" deleted file mode 100755 index b73d99cf81..0000000000 --- "a/web/src/components/svgIcon/svg/1 - \345\211\257\346\234\254.svg" +++ /dev/null @@ -1 +0,0 @@ - \ No newline at end of file diff --git a/web/src/components/svgIcon/svg/newUi/shortcutIcon/assets-logo.svg b/web/src/components/svgIcon/svg/newUi/shortcutIcon/assets-logo.svg new file mode 100644 index 0000000000..e7a80ebef6 --- /dev/null +++ b/web/src/components/svgIcon/svg/newUi/shortcutIcon/assets-logo.svg @@ -0,0 +1,1707 @@ + + + assets + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/web/src/components/svgIcon/svg/newUi/shortcutIcon/datamodel-logo.svg b/web/src/components/svgIcon/svg/newUi/shortcutIcon/datamodel-logo.svg new file mode 100644 index 0000000000..fd7a7cb082 --- /dev/null +++ b/web/src/components/svgIcon/svg/newUi/shortcutIcon/datamodel-logo.svg @@ -0,0 +1,6368 @@ + + + datamodel + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/web/src/components/svgIcon/svg/newUi/shortcutIcon/datawarehouse-logo.svg b/web/src/components/svgIcon/svg/newUi/shortcutIcon/datawarehouse-logo.svg new file mode 100644 index 0000000000..096c06b208 --- /dev/null +++ b/web/src/components/svgIcon/svg/newUi/shortcutIcon/datawarehouse-logo.svg @@ -0,0 +1,353 @@ + + + datawarehouse + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/web/src/components/svgIcon/svg/newUi/shortcutIcon/streamis-logo.svg b/web/src/components/svgIcon/svg/newUi/shortcutIcon/streamis-logo.svg new file mode 100644 index 0000000000..60e1bfdd00 --- /dev/null +++ b/web/src/components/svgIcon/svg/newUi/shortcutIcon/streamis-logo.svg @@ -0,0 +1,2640 @@ + + + streamSQL开发 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/web/src/config.json b/web/src/config.json index a1c487cbeb..2b7b74c4b7 100755 --- a/web/src/config.json +++ b/web/src/config.json @@ -1,38 +1,65 @@ { - "workspace": { - "routes": "apps/workspace/router", - "module": "apps/workspace/module", - "i18n": { - "en": "apps/workspace/i18n/en.json", - "zh-CN": "apps/workspace/i18n/zh.json" - }, - "header": "" + "dataAssetManage": { + "routes": "apps/dataAssetManage/router", + "module": "apps/dataAssetManage/module", + "i18n": { + "en": "apps/dataAssetManage/i18n/en.json", + "zh-CN": "apps/dataAssetManage/i18n/zh.json" }, - "scriptis": { - "routes": "apps/scriptis/router", - "module": "apps/scriptis/module", - "i18n": { - "en": "apps/scriptis/i18n/common/en.json", - "zh-CN": "apps/scriptis/i18n/common/zh.json" - }, - "header": "apps/scriptis/module/header" + "header": "" + }, + "dataModelCenter": { + "routes": "apps/dataModelCenter/router", + "module": "apps/dataModelCenter/module", + "i18n": { + "en": "apps/dataModelCenter/i18n/en.json", + "zh-CN": "apps/dataModelCenter/i18n/zh.json" }, - "workflows": { - "routes": "apps/workflows/router", - "module": "apps/workflows/module", - "i18n": { - "en": "apps/workflows/i18n/common/en.json", - "zh-CN": "apps/workflows/i18n/common/zh.json" - }, - "header": "apps/workflows/module/header" + "header": "" + }, + "dataWarehouseDesign": { + "routes": "apps/dataWarehouseDesign/router", + "module": "apps/dataWarehouseDesign/module", + "i18n": { + "en": "apps/dataWarehouseDesign/i18n/en.json", + "zh-CN": "apps/dataWarehouseDesign/i18n/zh.json" }, - "apiServices": { - "routes": "apps/apiServices/router", - "module": "apps/apiServices/module", - "i18n": { - "en": "apps/apiServices/i18n/en.json", - "zh-CN": "apps/apiServices/i18n/zh.json" - }, - "header": "apps/apiServices/module/header" - } -} \ No newline at end of file + "header": "" + }, + "workspace": { + "routes": "apps/workspace/router", + "module": "apps/workspace/module", + "i18n": { + "en": "apps/workspace/i18n/en.json", + "zh-CN": "apps/workspace/i18n/zh.json" + }, + "header": "" + }, + "scriptis": { + "routes": "apps/scriptis/router", + "module": "apps/scriptis/module", + "i18n": { + "en": "apps/scriptis/i18n/common/en.json", + "zh-CN": "apps/scriptis/i18n/common/zh.json" + }, + "header": "apps/scriptis/module/header" + }, + "workflows": { + "routes": "apps/workflows/router", + "module": "apps/workflows/module", + "i18n": { + "en": "apps/workflows/i18n/common/en.json", + "zh-CN": "apps/workflows/i18n/common/zh.json" + }, + "header": "apps/workflows/module/header" + }, + "apiServices": { + "routes": "apps/apiServices/router", + "module": "apps/apiServices/module", + "i18n": { + "en": "apps/apiServices/i18n/en.json", + "zh-CN": "apps/apiServices/i18n/zh.json" + }, + "header": "apps/apiServices/module/header" + } +} diff --git a/web/src/router.js b/web/src/router.js index 2e59561bfc..afb8df27be 100755 --- a/web/src/router.js +++ b/web/src/router.js @@ -16,13 +16,13 @@ */ import VueRouter from "vue-router"; -import { routes } from './dynamic-apps' +import { routes } from "./dynamic-apps"; // 解决重复点击路由跳转报错 const originalPush = VueRouter.prototype.push; VueRouter.prototype.push = function push(location) { - return originalPush.call(this, location).catch(err => err) -} + return originalPush.call(this, location).catch(err => err); +}; const router = new VueRouter({ routes }); @@ -31,22 +31,27 @@ router.beforeEach((to, from, next) => { if (to.meta) { // 给路由添加参数，控制显示对应header if (to.meta.header) { - to.query.showHeader = to.meta.header + to.query.showHeader = to.meta.header; } if (to.meta.publicPage) { + // workspaceId 处理 + if (from.query.workspaceId && !Boolean(to.query.workspaceId)) { + to.query.workspaceId = from.query.workspaceId; + next(to); + } // 公共页面不需要权限控制（404，500） next(); - } else if( to.path != '/workspace') { - next('/workspace'); + } else if (to.path != "/workspace") { + next("/workspace"); } else { - next() + next(); } } }); -router.afterEach((to) => { +router.afterEach(to => { if (to.meta) { - document.title = to.meta.title || 'DataSphere Studio'; + document.title = to.meta.title || "DataSphere Studio"; } }); diff --git a/web/vue.config.js b/web/vue.config.js index a399e75f0f..8cb7a8013c 100755 --- a/web/vue.config.js +++ b/web/vue.config.js @@ -160,7 +160,7 @@ module.exports = { devServer: { proxy: { '/api': { - target: 'x.x.x.x', + target: "x.x.x.x", changeOrigin: true, pathRewrite: { '^/api': '/api' @@ -168,4 +168,4 @@ module.exports = { } } } -} \ No newline at end of file +}