diff --git a/.gitignore b/.gitignore index aa5f9b2526..983d6454a0 100644 --- a/.gitignore +++ b/.gitignore @@ -101,6 +101,9 @@ dss-apps/dss-data-governance/dss-data-warehouse-dao/target dss-apps/dss-data-governance/dss-data-warehouse-service/target dss-apps/dss-data-governance/dss-data-warehouse-server/target +#dss-git +dss-git/dss-git-common/target +dss-git/dss-git-server/target # plugins plugins/azkaban/linkis-jobtype/target diff --git a/README-ZH.md b/README-ZH.md index 0d04b86795..9e4da69b51 100644 --- a/README-ZH.md +++ b/README-ZH.md @@ -170,7 +170,7 @@        您也可以扫描下面的二维码，加入我们的 微信群，以获得更快速的响应。 -![交流](images/zh_CN/readme/communication.png) +![交流](images/zh_CN/readme/communication.jpg) ## 谁在使用 DSS diff --git a/README.md b/README.md index 31bb3436b8..e59fbdc683 100644 --- a/README.md +++ b/README.md @@ -168,7 +168,7 @@ English | [中文](README-ZH.md)        You can scan the QR code below to join our WeChat to get more immediate response. -![communication](images/en_US/readme/communication.png) +![communication](images/en_US/readme/communication.jpg) ## Who is using DSS diff --git a/assembly/bin/appconn-refresh.sh b/assembly/bin/appconn-refresh.sh index 1a648adabb..37df8746fb 100644 --- a/assembly/bin/appconn-refresh.sh +++ b/assembly/bin/appconn-refresh.sh @@ -22,7 +22,8 @@ function isSuccess(){ function refresh() { while true; do - response=$(curl -H "Token-Code:" -H "Token-User:hadoop" -X GET http://${GATEWAY_INSTALL_IP}:${GATEWAY_PORT}/api/rest_j/v1/dss/framework/project/appconn/${APPCONN_NAME}/load) + response=$(curl -H "Token-Code:xxx" -H "Token-User:hadoop" -X GET + http://${GATEWAY_INSTALL_IP}:${GATEWAY_PORT}/api/rest_j/v1/dss/framework/project/appconn/${APPCONN_NAME}/load) if [[ $response == *"not appconn manager node"* ]]; then echo 'not appconn manager node, we will try again 5 seconds later' sleep 5 diff --git a/assembly/bin/dssinstall.sh b/assembly/bin/dssinstall.sh index 1fb1e4b3ab..157ebd72a7 100644 --- a/assembly/bin/dssinstall.sh +++ b/assembly/bin/dssinstall.sh @@ -105,7 +105,7 @@ function changeCommonConf(){ sed -i "s#hostname:.*#hostname: $SERVER_IP#g" $CONF_APPLICATION_YML sed -i "s#wds.linkis.server.mybatis.datasource.url.*#wds.linkis.server.mybatis.datasource.url=jdbc:mysql://${MYSQL_HOST}:${MYSQL_PORT}/${MYSQL_DB}?characterEncoding=UTF-8#g" $CONF_DSS_PROPERTIES sed -i "s#wds.linkis.server.mybatis.datasource.username.*#wds.linkis.server.mybatis.datasource.username=$MYSQL_USER#g" $CONF_DSS_PROPERTIES - sed -i "s#wds.linkis.server.mybatis.datasource.password.*#***REMOVED***$MYSQL_PASSWORD#g" $CONF_DSS_PROPERTIES + sed -i "s#wds.linkis.server.mybatis.datasource.password.*#wds.linkis.server.mybatis.datasource.password=$MYSQL_PASSWORD#g" $CONF_DSS_PROPERTIES sed -i "s#wds.linkis.gateway.ip.*#wds.linkis.gateway.ip=$GATEWAY_INSTALL_IP#g" $CONF_DSS_PROPERTIES sed -i "s#wds.linkis.gateway.port.*#wds.linkis.gateway.port=$GATEWAY_PORT#g" $CONF_DSS_PROPERTIES sed -i "s#wds.linkis.gateway.url.*#wds.linkis.gateway.url=http://$GATEWAY_INSTALL_IP:$GATEWAY_PORT/#g" $CONF_DSS_PROPERTIES diff --git a/assembly/bin/install.sh b/assembly/bin/install.sh new file mode 100644 index 0000000000..bf49bea929 --- /dev/null +++ b/assembly/bin/install.sh @@ -0,0 +1,378 @@ +#!/bin/sh +#Actively load user env +if [ -f "~/.bashrc" ];then + echo "Warning! user bashrc file does not exist." +else + source ~/.bashrc +fi + +shellDir=`dirname $0` +workDir=`cd ${shellDir}/..;pwd` + +SERVER_IP="" +SERVER_HOME="" + +local_host="`hostname --fqdn`" +LOCAL_IP=$(hostname -I) +LOCAL_IP=${LOCAL_IP// /} + +#To be compatible with MacOS and Linux +txt="" +if [[ "$OSTYPE" == "darwin"* ]]; then + txt="''" +elif [[ "$OSTYPE" == "linux-gnu" ]]; then + # linux + txt="" +elif [[ "$OSTYPE" == "cygwin" ]]; then + echo "dss not support Windows operating system" + exit 1 +elif [[ "$OSTYPE" == "msys" ]]; then + echo "dss not support Windows operating system" + exit 1 +elif [[ "$OSTYPE" == "win32" ]]; then + echo "dss not support Windows operating system" + exit 1 +elif [[ "$OSTYPE" == "freebsd"* ]]; then + txt="" +else + echo "Operating system unknown, please tell us(submit issue) for better service" + exit 1 +fi + +function isSuccess(){ + if [ $? -ne 0 ]; then + echo "Failed to " + $1 + exit 1 + else + echo "Succeed to " + $1 + fi +} + +function checkJava(){ + java -version + isSuccess "execute java --version" +} + +checkJava + +dos2unix -q ${workDir}/config/* +isSuccess "execute dos2unix -q ${workDir}/config/*" +dos2unix -q ${workDir}/bin/* + +echo "step1:load config" +source ${workDir}/config/config.sh +source ${workDir}/config/db.sh + +DSS_FILE_PATH="$workDir/$DSS_FILE_NAME" +#dos2unix ${DSS_FILE_PATH}/sbin/* +#dos2unix ${DSS_FILE_PATH}/sbin/ext/* +if [ -z $DSS_FILE_NAME ]; then + echo "DSS_FILE_NAME is null " + exit 1 +fi + +function replaceCommonIp() { + if [ -z "$DSS_FRAMEWORK_PROJECT_SERVER_INSTALL_IP" ]; then + DSS_FRAMEWORK_PROJECT_SERVER_INSTALL_IP=$LOCAL_IP + fi + if [ -z "$DSS_FRAMEWORK_PROJECT_SERVER_PORT" ]; then + DSS_FRAMEWORK_PROJECT_SERVER_PORT=9002 + fi + + if [ -z "$DSS_FRAMEWORK_ORCHESTRATOR_SERVER_INSTALL_IP" ]; then + DSS_FRAMEWORK_ORCHESTRATOR_SERVER_INSTALL_IP=$LOCAL_IP + fi + if [ -z "$DSS_FRAMEWORK_ORCHESTRATOR_SERVER_PORT" ]; then + DSS_FRAMEWORK_ORCHESTRATOR_SERVER_PORT=9003 + fi + + if [ -z "$DSS_APISERVICE_SERVER_INSTALL_IP" ]; then + DSS_APISERVICE_SERVER_INSTALL_IP=$LOCAL_IP + fi + if [ -z "$DSS_APISERVICE_SERVER_PORT" ]; then + DSS_APISERVICE_SERVER_PORT=9004 + fi + + if [ -z "$DSS_WORKFLOW_SERVER_INSTALL_IP" ]; then + DSS_WORKFLOW_SERVER_INSTALL_IP=$LOCAL_IP + fi + if [ -z "$DSS_WORKFLOW_SERVER_PORT" ]; then + DSS_WORKFLOW_SERVER_PORT=9005 + fi + + if [ -z "$DSS_FLOW_EXECUTION_SERVER_INSTALL_IP" ]; then + DSS_FLOW_EXECUTION_SERVER_INSTALL_IP=$LOCAL_IP + fi + if [ -z "$DSS_FLOW_EXECUTION_SERVER_PORT" ]; then + DSS_FLOW_EXECUTION_SERVER_PORT=9006 + fi + + if [ -z "$DSS_SCRIPTIS_SERVER_INSTALL_IP" ]; then + DSS_SCRIPTIS_SERVER_INSTALL_IP=$LOCAL_IP + fi + if [ -z "$DSS_SCRIPTIS_SERVER_PORT" ]; then + DSS_SCRIPTIS_SERVER_PORT=9008 + fi + + if [ -z "$DSS_DATA_API_SERVER_INSTALL_IP" ]; then + DSS_DATA_API_SERVER_INSTALL_IP=$LOCAL_IP + fi + if [ -z "$DSS_DATA_API_SERVER_PORT" ]; then + DSS_DATA_API_SERVER_PORT=9208 + fi + + if [ -z "$DSS_DATA_GOVERNANCE_SERVER_INSTALL_IP" ]; then + DSS_DATA_GOVERNANCE_SERVER_INSTALL_IP=$LOCAL_IP + fi + if [ -z "$DSS_DATA_GOVERNANCE_SERVER_PORT" ]; then + DSS_DATA_GOVERNANCE_SERVER_PORT=9209 + fi + + if [ -z "$DSS_GUIDE_SERVER_INSTALL_IP" ]; then + DSS_GUIDE_SERVER_INSTALL_IP=$LOCAL_IP + fi + if [ -z "$DSS_GUIDE_SERVER_PORT" ]; then + DSS_GUIDE_SERVER_PORT=9210 + fi + + if [[ $GATEWAY_INSTALL_IP == "127.0.0.1" ]] || [ -z "$GATEWAY_INSTALL_IP" ]; then + #echo "GATEWAY_INSTALL_IP is equals $GATEWAY_INSTALL_IP ,we will change it to ip address" + GATEWAY_INSTALL_IP=$LOCAL_IP + fi + if [[ $EUREKA_INSTALL_IP == "127.0.0.1" ]] || [ -z "$EUREKA_INSTALL_IP" ]; then + #echo "EUREKA_INSTALL_IP is equals $EUREKA_INSTALL_IP ,we will change it to ip address" + EUREKA_INSTALL_IP=$LOCAL_IP + fi +} +##替换真实的IP +replaceCommonIp + +EUREKA_URL=http://$EUREKA_INSTALL_IP:$EUREKA_PORT/eureka/ + +## excecute sql +source ${workDir}/bin/executeSQL.sh + +function changeCommonConf(){ + sed -i "s#defaultZone:.*#defaultZone: $EUREKA_URL#g" $CONF_APPLICATION_YML + sed -i "s#hostname:.*#hostname: $SERVER_IP#g" $CONF_APPLICATION_YML + sed -i "s#wds.linkis.server.mybatis.datasource.url.*#wds.linkis.server.mybatis.datasource.url=jdbc:mysql://${MYSQL_HOST}:${MYSQL_PORT}/${MYSQL_DB}?characterEncoding=UTF-8#g" $CONF_DSS_PROPERTIES + sed -i "s#wds.linkis.server.mybatis.datasource.username.*#wds.linkis.server.mybatis.datasource.username=$MYSQL_USER#g" $CONF_DSS_PROPERTIES + sed -i "s#wds.linkis.server.mybatis.datasource.password.*#wds.linkis.server.mybatis.datasource.password=$MYSQL_PASSWORD#g" $CONF_DSS_PROPERTIES + sed -i "s#wds.linkis.gateway.ip.*#wds.linkis.gateway.ip=$GATEWAY_INSTALL_IP#g" $CONF_DSS_PROPERTIES + sed -i "s#wds.linkis.gateway.port.*#wds.linkis.gateway.port=$GATEWAY_PORT#g" $CONF_DSS_PROPERTIES + sed -i "s#wds.linkis.gateway.url.*#wds.linkis.gateway.url=http://$GATEWAY_INSTALL_IP:$GATEWAY_PORT/#g" $CONF_DSS_PROPERTIES + sed -i "s#wds.linkis.gateway.wtss.url.*#wds.linkis.gateway.wtss.url=http://$GATEWAY_INSTALL_IP:$GATEWAY_PORT/#g" $CONF_DSS_PROPERTIES +} + +##function start +function changeConf(){ + sed -i "s#spring.server.port=.*#spring.server.port=$SERVER_PORT#g" $CONF_SERVER_PROPERTIES + if [[ $SERVER_NAME == "dss-framework-orchestrator-server" ]] || [[ $SERVER_NAME == "dss-workflow-server" ]]; then + SERVER_FULL_NAME=$SERVER_NAME + SERVER_FULL_NAME=$SERVER_NAME-$ENV_FLAG + sed -i "s#spring.spring.application.name=.*#spring.spring.application.name=$SERVER_FULL_NAME#g" $CONF_SERVER_PROPERTIES + fi + sed -i "s#wds.dss.appconn.scheduler.project.store.dir.*#wds.dss.appconn.scheduler.project.store.dir=$WDS_SCHEDULER_PATH#g" $CONF_SERVER_PROPERTIES + isSuccess "subsitution $CONF_SERVER_PROPERTIES" +} +##function end + + +UPLOAD_PUBLIC_IPS="" +##function start +function uploadServiceFile(){ + if [[ $SERVER_IP == "127.0.0.1" ]]; then + SERVER_IP=$local_host + fi + #echo "$SERVER_NAME-step3:copy install package" + # upload project conf + # cp -rfp $SSH_PORT ${workDir}/config/{$SERVER_NAME}.properties $CONF_PATH + if [[ $UPLOAD_PUBLIC_IPS == *",${ENV_FLAG}-$SERVER_IP,"* ]]; then + return 0 + fi + cp -rfp ${DSS_FILE_PATH}/* $SERVER_HOME + cp -rfp ${workDir}/bin $SERVER_HOME + cp -rfp ${workDir}/config/* $SERVER_HOME/conf + sudo chown -R $deployUser:$deployUser $SERVER_HOME + UPLOAD_PUBLIC_IPS="$UPLOAD_PUBLIC_IPS,${ENV_FLAG}-$SERVER_IP," + changeCommonConf +# echo "UPLOAD_PUBLIC_IPS-->$UPLOAD_PUBLIC_IPS" +} + +##function start +function installPackage(){ + if [[ $SERVER_IP == "127.0.0.1" ]]; then + SERVER_IP=$local_host + fi + if [ -z $SERVER_NAME ]; then + echo "ERROR:SERVER_NAME is null " + exit 1 + fi + uploadServiceFile + # change configuration + changeConf +} + +function dssWebInstall(){ +if ! test -e ${LINKIS_DSS_HOME}/wedatasphere-dss-web*.zip; then + echo "**********Error: please put wedatasphere-dss-web-xxx.zip in ${LINKIS_DSS_HOME}! " + exit 1 +else + echo "Start to unzip dss web package." + unzip -d ${LINKIS_DSS_HOME}/web/ -o ${LINKIS_DSS_HOME}/wedatasphere-dss-web-*.zip > /dev/null 2>&1 + sed -i "s#linkis_url.*#linkis_url=${LINKIS_GATEWAY_URL}#g" ${LINKIS_DSS_HOME}/web/config.sh + isSuccess "Unzip dss web package to ${LINKIS_DSS_HOME}/web" +fi +} + +##Install dss projects +function installDssProject() { + echo "step2:update config" +# if [ "$DSS_INSTALL_HOME" != "" ] +# then +# rm -rf $DSS_INSTALL_HOME +# fi + #echo "" + #echo "-----------------DSS install start--------------------" + SERVER_HOME=$DSS_INSTALL_HOME + if [ "$SERVER_HOME" == "" ] + then + export SERVER_HOME=${workDir}/DSSInstall + fi + if [ -d $SERVER_HOME ] && [ "$SERVER_HOME" != "$workDir" ]; then + rm -r $SERVER_HOME-bak + echo "mv $SERVER_HOME $SERVER_HOME-bak" + mv $SERVER_HOME $SERVER_HOME-bak + fi + echo "create dir SERVER_HOME: $SERVER_HOME" + sudo mkdir -p $SERVER_HOME + isSuccess "Create the dir of $SERVER_HOME" + sudo chown -R $deployUser:$deployUser $SERVER_HOME + isSuccess "chown -R $deployUser:$deployUser $SERVER_HOME" + + #echo "" + SERVER_NAME=dss-framework-project-server + SERVER_IP=$DSS_FRAMEWORK_PROJECT_SERVER_INSTALL_IP + SERVER_PORT=$DSS_FRAMEWORK_PROJECT_SERVER_PORT + + UPLOAD_LIB_FILES=$DSS_FILE_PATH/lib/dss-framework/$SERVER_NAME + LIB_PATH=$SERVER_HOME/lib/dss-framework + LOG_PATH=$SERVER_HOME/logs/dss-framework/$SERVER_NAME + CONF_SERVER_PROPERTIES=$SERVER_HOME/conf/$SERVER_NAME.properties + CONF_DSS_PROPERTIES=$SERVER_HOME/conf/dss.properties + CONF_APPLICATION_YML=$SERVER_HOME/conf/application-dss.yml + ###install project-Server + installPackage + #echo "" + + SERVER_NAME=dss-framework-orchestrator-server + SERVER_IP=$DSS_FRAMEWORK_ORCHESTRATOR_SERVER_INSTALL_IP + SERVER_PORT=$DSS_FRAMEWORK_ORCHESTRATOR_SERVER_PORT + UPLOAD_LIB_FILES=$DSS_FILE_PATH/lib/dss-framework/$SERVER_NAME + LIB_PATH=$SERVER_HOME/lib/dss-framework + LOG_PATH=$SERVER_HOME/logs/dss-framework/$SERVER_NAME + CONF_SERVER_PROPERTIES=$SERVER_HOME/conf/$SERVER_NAME.properties + CONF_DSS_PROPERTIES=$SERVER_HOME/conf/dss.properties + CONF_APPLICATION_YML=$SERVER_HOME/conf/application-dss.yml + ###install orchestrator-Server + installPackage + #echo "" + + SERVER_NAME=dss-apiservice-server + SERVER_IP=$DSS_APISERVICE_SERVER_INSTALL_IP + SERVER_PORT=$DSS_APISERVICE_SERVER_PORT + UPLOAD_LIB_FILES=$DSS_FILE_PATH/lib/dss-apps/$SERVER_NAME + LIB_PATH=$SERVER_HOME/lib/dss-apps + LOG_PATH=$SERVER_HOME/logs/dss-apps/$SERVER_NAME + CONF_SERVER_PROPERTIES=$SERVER_HOME/conf/$SERVER_NAME.properties + CONF_DSS_PROPERTIES=$SERVER_HOME/conf/dss.properties + CONF_APPLICATION_YML=$SERVER_HOME/conf/application-dss.yml + ###install dss-apiservice-server + installPackage + #echo "" + + SERVER_NAME=dss-scriptis-server + SERVER_IP=$DSS_SCRIPTIS_SERVER_INSTALL_IP + SERVER_PORT=$DSS_SCRIPTIS_SERVER_PORT + UPLOAD_LIB_FILES=$DSS_FILE_PATH/lib/dss-apps/$SERVER_NAME + LIB_PATH=$SERVER_HOME/lib/dss-apps + LOG_PATH=$SERVER_HOME/logs/dss-apps/$SERVER_NAME + CONF_SERVER_PROPERTIES=$SERVER_HOME/conf/$SERVER_NAME.properties + CONF_DSS_PROPERTIES=$SERVER_HOME/conf/dss.properties + CONF_APPLICATION_YML=$SERVER_HOME/conf/application-dss.yml + ###install dss-scriptis-server + installPackage + #echo "" + + ##Flow execution Install + PACKAGE_DIR=dss + SERVER_NAME=dss-flow-execution-server + SERVER_IP=$DSS_FLOW_EXECUTION_SERVER_INSTALL_IP + SERVER_PORT=$DSS_FLOW_EXECUTION_SERVER_PORT + UPLOAD_LIB_FILES=$DSS_FILE_PATH/lib/dss-orchestrator/$SERVER_NAME + LIB_PATH=$SERVER_HOME/lib/dss-orchestrator + LOG_PATH=$SERVER_HOME/logs/dss-orchestrator/$SERVER_NAME + CONF_SERVER_PROPERTIES=$SERVER_HOME/conf/$SERVER_NAME.properties + CONF_DSS_PROPERTIES=$SERVER_HOME/conf/dss.properties + CONF_APPLICATION_YML=$SERVER_HOME/conf/application-dss.yml + ###Install flow execution + installPackage + #echo "" + + SERVER_NAME=dss-workflow-server + SERVER_IP=$DSS_WORKFLOW_SERVER_INSTALL_IP + SERVER_PORT=$DSS_WORKFLOW_SERVER_PORT + UPLOAD_LIB_FILES=$DSS_FILE_PATH/lib/dss-orchestrator/$SERVER_NAME + LIB_PATH=$SERVER_HOME/lib/dss-orchestrator + LOG_PATH=$SERVER_HOME/logs/dss-orchestrator/$SERVER_NAME + CONF_SERVER_PROPERTIES=$SERVER_HOME/conf/$SERVER_NAME.properties + CONF_DSS_PROPERTIES=$SERVER_HOME/conf/dss.properties + CONF_APPLICATION_YML=$SERVER_HOME/conf/application-dss.yml + ###install dss-workflow-server + installPackage + + ###install dss-data-api-server + SERVER_NAME=dss-data-api-server + SERVER_IP=$DSS_DATA_API_SERVER_INSTALL_IP + SERVER_PORT=$DSS_DATA_API_SERVER_PORT + UPLOAD_LIB_FILES=$DSS_FILE_PATH/lib/dss-data-api/$SERVER_NAME + LIB_PATH=$SERVER_HOME/lib/dss-data-api + LOG_PATH=$SERVER_HOME/logs/dss-data-api/$SERVER_NAME + CONF_SERVER_PROPERTIES=$SERVER_HOME/conf/$SERVER_NAME.properties + CONF_DSS_PROPERTIES=$SERVER_HOME/conf/dss.properties + CONF_APPLICATION_YML=$SERVER_HOME/conf/application-dss.yml + installPackage + + ###install dss-data-governance-server + SERVER_NAME=dss-data-governance-server + SERVER_IP=$DSS_DATA_GOVERNANCE_SERVER_INSTALL_IP + SERVER_PORT=$DSS_DATA_GOVERNANCE_SERVER_PORT + UPLOAD_LIB_FILES=$DSS_FILE_PATH/lib/dss-data-governance/$SERVER_NAME + LIB_PATH=$SERVER_HOME/lib/dss-data-governance + LOG_PATH=$SERVER_HOME/logs/dss-data-governance/$SERVER_NAME + CONF_SERVER_PROPERTIES=$SERVER_HOME/conf/$SERVER_NAME.properties + CONF_DSS_PROPERTIES=$SERVER_HOME/conf/dss.properties + CONF_APPLICATION_YML=$SERVER_HOME/conf/application-dss.yml + installPackage + + ###install dss-guide-server + SERVER_NAME=dss-guide-server + SERVER_IP=$DSS_GUIDE_SERVER_INSTALL_IP + SERVER_PORT=$DSS_GUIDE_SERVER_PORT + UPLOAD_LIB_FILES=$DSS_FILE_PATH/lib/dss-guide/$SERVER_NAME + LIB_PATH=$SERVER_HOME/lib/dss-guide + LOG_PATH=$SERVER_HOME/logs/dss-guide/$SERVER_NAME + CONF_SERVER_PROPERTIES=$SERVER_HOME/conf/$SERVER_NAME.properties + CONF_DSS_PROPERTIES=$SERVER_HOME/conf/dss.properties + CONF_APPLICATION_YML=$SERVER_HOME/conf/application-dss.yml + installPackage + + #echo "-----------------DSS install end--------------------" + #echo "" + +} +ENV_FLAG="dev" +installDssProject + +echo "Congratulations! You have installed DSS $DSS_VERSION successfully, please use sbin/dss-start-all.sh to start it!" + diff --git a/assembly/config/config.sh b/assembly/config/config.sh index 76633b44b6..9c1c506bad 100644 --- a/assembly/config/config.sh +++ b/assembly/config/config.sh @@ -25,7 +25,7 @@ GATEWAY_INSTALL_IP=127.0.0.1 GATEWAY_PORT=9001 ### Linkis BML Token -BML_AUTH= +BML_AUTH=BML-AUTH ################### The install Configuration of all Micro-Services start ##################### # @@ -50,16 +50,16 @@ DSS_APPS_SERVER_PORT=9044 ############## ############## dss_appconn_instance configuration start ############## ############## ####eventchecker表的地址，一般就是dss数据库 -EVENTCHECKER_JDBC_URL=jdbc:mysql://172.0.0.1:3305/linkis_kinghao?characterEncoding=UTF-8 +EVENTCHECKER_JDBC_URL=jdbc:mysql://172.16.16.16:3305/linkis_kinghao?characterEncoding=UTF-8 EVENTCHECKER_JDBC_USERNAME=linkis EVENTCHECKER_JDBC_PASSWORD=linkis@Wds #### hive地址 -DATACHECKER_JOB_JDBC_URL=jdbc:mysql://172.0.0.1:3306/hivemeta?useUnicode=true +DATACHECKER_JOB_JDBC_URL=jdbc:mysql://172.16.16.10:3306/hivemeta?useUnicode=true DATACHECKER_JOB_JDBC_USERNAME=hivemeta DATACHECKER_JOB_JDBC_PASSWORD=Linkishivemeta@123 #### 元数据库，可配置成和DATACHECKER_JOB的一致 -DATACHECKER_BDP_JDBC_URL=jdbc:mysql://172.0.0.1:3306/hivemeta?useUnicode=true +DATACHECKER_BDP_JDBC_URL=jdbc:mysql://172.16.16.10:3306/hivemeta?useUnicode=true DATACHECKER_BDP_JDBC_USERNAME=hivemeta DATACHECKER_BDP_JDBC_PASSWORD=Linkishivemeta@123 diff --git a/assembly/dss-package/pom.xml b/assembly/dss-package/pom.xml index 842307e240..53f64fa10b 100644 --- a/assembly/dss-package/pom.xml +++ b/assembly/dss-package/pom.xml @@ -21,7 +21,7 @@ dss com.webank.wedatasphere.dss - 1.1.0.20-SNAPSHOT + 1.10.0-SNAPSHOT ../../pom.xml 4.0.0 @@ -78,6 +78,10 @@ dss-sender-service ${dss.version} + + javax.servlet-api + javax.servlet + @@ -103,10 +107,18 @@ linkis-hadoop-common ${linkis.version} + + javax.servlet + javax.servlet-api + netty io.netty + + javax.servlet-api + javax.servlet + diff --git a/assembly/pom.xml b/assembly/pom.xml index 0e97116a4d..b19ac7a680 100644 --- a/assembly/pom.xml +++ b/assembly/pom.xml @@ -22,7 +22,7 @@ dss com.webank.wedatasphere.dss - 1.1.0.20-SNAPSHOT + 1.10.0-SNAPSHOT ../pom.xml pom diff --git a/conf/application-dss.yml b/conf/application-dss.yml new file mode 100644 index 0000000000..b0473838d8 --- /dev/null +++ b/conf/application-dss.yml @@ -0,0 +1,23 @@ + +eureka: + client: + serviceUrl: + defaultZone: http://127.0.0.1:20303/eureka/ + #instance: + #prefer-ip-address: true + #instance-id: ${spring.cloud.client.ip-address}:${server.port} + #metadata-map: + #test: wedatasphere + +management: + endpoints: + web: + exposure: + include: refresh,info +logging: + config: classpath:log4j2.xml + +#mybatis: +# configuration: +# log-impl: org.apache.ibatis.logging.stdout.StdOutImpl + diff --git a/conf/atlas-application.properties b/conf/atlas-application.properties new file mode 100644 index 0000000000..e69de29bb2 diff --git a/conf/dss-apiservice-server.properties b/conf/dss-apiservice-server.properties new file mode 100644 index 0000000000..5162f42bcd --- /dev/null +++ b/conf/dss-apiservice-server.properties @@ -0,0 +1,42 @@ +# +# Copyright 2019 WeBank +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# + +# Spring configurations +spring.server.port=9206 +spring.spring.application.name=dss-apiservice-server + +wds.linkis.server.mybatis.mapperLocations=classpath*:com/webank/wedatasphere/dss/apiservice/core/dao/mapper/*.xml +wds.linkis.server.mybatis.typeAliasesPackage=com.webank.wedatasphere.dss.apiservice.core.bo,com.webank.wedatasphere.dss.apiservice.core.vo +wds.linkis.server.mybatis.BasePackage=com.webank.wedatasphere.dss.apiservice.core.dao + +wds.linkis.server.restful.scan.packages=com.webank.wedatasphere.dss.apiservice.core.restful + +#sit +wds.linkis.server.version=v1 +wds.linkis.server.url= + +#test +wds.linkis.test.mode=false +wds.linkis.test.user= + + +#dsm +wds.linkis.server.dsm.admin.users= + + +#用于执行的datasource配置 +wds.linkis.datasource.hikari.maximumPoolSize=100 +wds.linkis.datasource.hikari.minimumIdle=10 diff --git a/conf/dss-data-api-server.properties b/conf/dss-data-api-server.properties new file mode 100644 index 0000000000..fe386097ed --- /dev/null +++ b/conf/dss-data-api-server.properties @@ -0,0 +1,39 @@ +# +# /* +# * Copyright 2019 WeBank +# * +# * Licensed under the Apache License, Version 2.0 (the "License"); +# * you may not use this file except in compliance with the License. +# * You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ +# + +# Spring configurations +spring.server.port=9208 +spring.spring.application.name=dss-data-api-server + +wds.linkis.log.clear=true + +wds.linkis.server.version=v1 + +##restful +wds.linkis.server.restful.scan.packages=com.webank.wedatasphere.dss.data.api.server.restful + +##mybatis +wds.linkis.server.mybatis.mapperLocations=classpath*:com/webank/wedatasphere/dss/data/api/server/dao/impl/*.xml + +wds.linkis.server.mybatis.typeAliasesPackage=com.webank.wedatasphere.dss.data.api.server.entity + +wds.linkis.server.mybatis.BasePackage=com.webank.wedatasphere.dss.data.api.server.dao + +#wds.linkis.gateway.ip=127.0.0.1 +#wds.linkis.gateway.port=9001 +#wds.linkis.gateway.url=http://127.0.0.1:9001/ diff --git a/conf/dss-data-governance-server.properties b/conf/dss-data-governance-server.properties new file mode 100644 index 0000000000..b3980c6ccc --- /dev/null +++ b/conf/dss-data-governance-server.properties @@ -0,0 +1,53 @@ +# +# /* +# * Copyright 2019 WeBank +# * +# * Licensed under the Apache License, Version 2.0 (the "License"); +# * you may not use this file except in compliance with the License. +# * You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ +# + +# Spring configurations +spring.server.port=9209 +spring.spring.application.name=dss-data-governance-server + +wds.linkis.log.clear=true + +wds.linkis.server.version=v1 + +##restful +wds.linkis.server.restful.scan.packages=com.webank.wedatasphere.dss.data.asset.restful,com.webank.wedatasphere.dss.data.classification.restful + +##mybatis +wds.linkis.server.mybatis.mapperLocations=classpath*:com/webank/wedatasphere/dss/data/asset/dao/impl/*.xml +wds.linkis.server.mybatis.typeAliasesPackage=com.webank.wedatasphere.dss.data.asset.entity +wds.linkis.server.mybatis.BasePackage=com.webank.wedatasphere.dss.data.asset.dao,com.webank.wedatasphere.dss.data.warehouse.dao,com.webank.wedatasphere.dss.data.warehouse.mapper + +#wds.linkis.gateway.ip=127.0.0.1 +#wds.linkis.gateway.port=9001 +#wds.linkis.gateway.url=http://127.0.0.1:9001/ + + +# atlas config +atlas.rest.address=http://xxxxxxx:21000 +atlas.username=xxxxxxxx +atlas.password=yyyyyyyyy +atlas.client.readTimeoutMSecs=60000 +atlas.client.connectTimeoutMSecs=60000 + +atlas.cluster.name=primary + +# hive metadata config +metastore.datasource.driver=com.mysql.jdbc.Driver +metastore.datasource.url=jdbc:mysql://xxxxxx:yyyy/metastore?characterEncoding=UTF-8 +metastore.datasource.username=xxxxxx +metastore.datasource.password=yyyyyy \ No newline at end of file diff --git a/conf/dss-flow-execution-server.properties b/conf/dss-flow-execution-server.properties new file mode 100644 index 0000000000..1d8eb8d285 --- /dev/null +++ b/conf/dss-flow-execution-server.properties @@ -0,0 +1,55 @@ +# +# Copyright 2019 WeBank +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# + +# Spring configurations +spring.server.port=9006 + +spring.spring.application.name=dss-flow-entrance + +##mybatis +wds.linkis.server.mybatis.mapperLocations=classpath*:com/webank/wedatasphere/dss/flow/execution/entrance/dao/impl/*.xml,classpath*:org/apache/linkis/jobhistory/dao/impl/*.xml + +wds.linkis.server.mybatis.typeAliasesPackage= + +wds.linkis.server.mybatis.BasePackage=com.webank.wedatasphere.dss.flow.execution.entrance.dao,org.apache.linkis.jobhistory.dao + + +wds.linkis.server.restful.scan.packages=org.apache.linkis.entrance.restful,com.webank.wedatasphere.dss.flow.execution.entrance.restful + +#wds.linkis.server.component.exclude.classes=org.apache.linkis.DataWorkCloudApplication + +wds.linkis.engine.application.name=flowExecutionEngine +wds.linkis.enginemanager.application.name=flowExecution + +wds.linkis.query.application.name=linkis-ps-publicservice + +wds.linkis.console.config.application.name=linkis-ps-publicservice +wds.linkis.engine.creation.wait.time.max=20m +wds.linkis.server.version=v1 + +wds.linkis.server.socket.mode=true + +wds.linkis.client.flow.adminuser=ws +wds.linkis.client.flow.author.user.token=WS-AUTH + +wds.linkis.server.component.exclude.classes=org.apache.linkis.entranceclient.conf.ClientForEntranceSpringConfiguration,org.apache.linkis.entranceclient.conf.ClientSpringConfiguration + +wds.linkis.server.component.exclude.packages=org.apache.linkis.entrance.restful. +spring.spring.main.allow-bean-definition-overriding=true + +wds.linkis.entrance.config.log.path=file:///appcom/tmp/dss/ +wds.linkis.spark.engine.version=2.4.3 +wds.linkis.hive.engine.version=2.3.3 diff --git a/conf/dss-framework-orchestrator-server.properties b/conf/dss-framework-orchestrator-server.properties index b70b14792b..d6d0a1c4dd 100644 --- a/conf/dss-framework-orchestrator-server.properties +++ b/conf/dss-framework-orchestrator-server.properties @@ -32,7 +32,6 @@ wds.linkis.server.mybatis.typeAliasesPackage=com.webank.wedatasphere.dss.server. wds.linkis.server.mybatis.BasePackage=com.webank.wedatasphere.dss.framework.appconn.dao,com.webank.wedatasphere.dss.orchestrator.core.dao,com.webank.wedatasphere.dss.server.dao,com.webank.wedatasphere.dss.application.dao,com.webank.wedatasphere.dss.workspace.mapper,com.webank.wedatasphere.dss.workspace.common.dao,com.webank.wedatasphere.dss.workspace.common.dao,com.webank.wedatasphere.dss.orchestrator.db.dao,com.webank.wedatasphere.dss.workflow.dao,com.webank.wedatasphere.dss.framework.appconn.dao,com.webank.wedatasphere.dss.flow.execution.entrance.dao -wds.dss.server.scheduling.clear.cs.cron=0 0 3 * * ? wds.dss.publish.max.remain.version=3 diff --git a/conf/dss-framework-project-server.properties b/conf/dss-framework-project-server.properties new file mode 100644 index 0000000000..6b8086de83 --- /dev/null +++ b/conf/dss-framework-project-server.properties @@ -0,0 +1,37 @@ +# +# Copyright 2019 WeBank +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# + +# Spring configurations +spring.server.port=9202 +spring.spring.application.name=dss-framework-project-server + +wds.linkis.log.clear=true + +wds.linkis.server.version=v1 + +##restful +wds.linkis.server.restful.scan.packages=com.webank.wedatasphere.dss.framework.workspace.restful,com.webank.wedatasphere.dss.framework.project.restful,com.webank.wedatasphere.dss.framework.release.restful,com.webank.wedatasphere.dss.framework.appconn.restful,com.webank.wedatasphere.dss.framework.admin.restful + +##mybatis +wds.linkis.server.mybatis.mapperLocations=classpath*:com/webank/wedatasphere/dss/framework/workspace/dao/impl/*.xml,classpath*:com/webank/wedatasphere/dss/application/dao/impl/*.xml,classpath*:com/webank/wedatasphere/dss/framework/project/dao/impl/*Mapper.xml,classpath*:com/webank/wedatasphere/dss/framework/appconn/dao/impl/*.xml,classpath*:com/webank/wedatasphere/dss/framework/release/dao/impl/*.xml,classpath*:com/webank/wedatasphere/dss/framework/admin/xml/impl/*.xml + +wds.linkis.server.mybatis.typeAliasesPackage=com.webank.wedatasphere.dss.application.entity,com.webank.wedatasphere.dss.common.entity,com.webank.wedatasphere.dss.framework.workspace.bean,com.webank.wedatasphere.dss.framework.project.entity,com.webank.wedatasphere.dss.framework.appconn.entity,com.webank.wedatasphere.dss.framework.release.entity,com.webank.wedatasphere.dss.framework.admin.pojo.entity + +wds.linkis.server.mybatis.BasePackage=com.webank.wedatasphere.dss.framework.workspace.dao,com.webank.wedatasphere.dss.application.dao,com.webank.wedatasphere.dss.framework.project.dao,com.webank.wedatasphere.dss.framework.appconn.dao,com.webank.wedatasphere.dss.framework.release.dao,com.webank.wedatasphere.dss.framework.admin.xml + +wds.dss.appconn.checker.development.ignore.list=workflow,sendemail +wds.dss.appconn.checker.project.ignore.list=visualis + diff --git a/conf/dss-guide-server.properties b/conf/dss-guide-server.properties new file mode 100644 index 0000000000..617935749d --- /dev/null +++ b/conf/dss-guide-server.properties @@ -0,0 +1,55 @@ +# +# /* +# * Copyright 2019 WeBank +# * +# * Licensed under the Apache License, Version 2.0 (the "License"); +# * you may not use this file except in compliance with the License. +# * You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ +# + +# Spring configurations +spring.server.port=9210 +spring.spring.application.name=dss-guide-server + +spring.jackson.date-format=yyyy-MM-dd HH:mm:ss +spring.jackson.time-zone=GMT+8 + +wds.linkis.server.version=v1 + +wds.linkis.log.clear=true + + + +##restful +wds.linkis.server.restful.scan.packages=com.webank.wedatasphere.dss.guide.server.restful + +##mybatis +wds.linkis.server.mybatis.mapperLocations=classpath*:com/webank/wedatasphere/dss/guide/server/dao/impl/*.xml +wds.linkis.server.mybatis.typeAliasesPackage=com.webank.wedatasphere.dss.guide.server.entity +wds.linkis.server.mybatis.BasePackage=com.webank.wedatasphere.dss.guide.server.dao + + +## guide_images_path +guide.content.images.path=/opt/dss/dss-guide-server/guide_images/ +guide.chapter.images.path=/opt/dss/dss-guide-server/guide_images/ + +#gitbook +#The machine where the file exists +target.ip.address=127.0.0.1 +#The file path of the machine where the file is stored +host.gitbook.path=/appcom/Install/ApacheInstall/gitbook_books/_book +#The path to copy the file to the current machine +target.gitbook.path=/appcom/Install/ApacheInstall/gitbook_books +#SUMMARY.md ignore absolve +summary.ignore.model=km +#sync model include: gitbook or database +guide.sync.model=gitbook diff --git a/conf/dss-scriptis-server.properties b/conf/dss-scriptis-server.properties new file mode 100644 index 0000000000..5a37729566 --- /dev/null +++ b/conf/dss-scriptis-server.properties @@ -0,0 +1,39 @@ +# +# Copyright 2019 WeBank +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# + +# Spring configurations +spring.server.port=9009 +spring.spring.application.name=dss-scriptis-server + +wds.linkis.server.mybatis.mapperLocations=classpath*:com/webank/wedatasphere/dss/scriptis/dao/mapper/*.xml +wds.linkis.server.mybatis.typeAliasesPackage=com.webank.wedatasphere.dss.scriptis.vo +wds.linkis.server.mybatis.BasePackage=com.webank.wedatasphere.dss.scriptis.dao + +wds.linkis.server.restful.scan.packages=com.webank.wedatasphere.dss.scriptis.restful + +#sit +wds.linkis.server.version=v1 +wds.linkis.server.url= + +#test +wds.linkis.test.mode=false +wds.linkis.test.user= + +wds.dss.scriptis.global.limits.exportResEnable=false +wds.dss.scriptis.global.limits.exportTableEnable=false +wds.dss.scriptis.global.limits.downloadResEnable=false +wds.dss.scriptis.global.limits.resCopyEnable=false +wds.dss.scriptis.global.limits.proxyEnable=false diff --git a/conf/dss-server.properties b/conf/dss-server.properties index af19be3563..6aef7895bb 100644 --- a/conf/dss-server.properties +++ b/conf/dss-server.properties @@ -22,7 +22,7 @@ wds.dss.appconn.checker.development.ignore.list=orchestrator-framework,workflow, wds.dss.appconn.checker.project.ignore.list= ## filter appconn wds.dss.appconn.disabled= -wds.dss.trust.token=***REMOVED*** +wds.dss.trust.token=FotUrJ9JmhQ= ##import file dir wds.dss.server.scheduling.clear.cs.cron=0/5 * * * * ? @@ -44,7 +44,7 @@ wds.linkis.resultSet.store.path=hdfs:///tmp/linkis wds.linkis.server.socket.mode=true wds.linkis.client.flow.adminuser=ws -wds.linkis.client.flow.author.user.token=172.0.0.1 +wds.linkis.client.flow.author.user.token=WS-AUTH wds.linkis.server.component.exclude.classes=org.apache.linkis.entranceclient.conf.ClientForEntranceSpringConfiguration,org.apache.linkis.entranceclient.conf.ClientSpringConfiguration diff --git a/conf/dss-workflow-server.properties b/conf/dss-workflow-server.properties new file mode 100644 index 0000000000..df4bd6dea5 --- /dev/null +++ b/conf/dss-workflow-server.properties @@ -0,0 +1,44 @@ +# +# Copyright 2019 WeBank +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# + +# Spring configurations +spring.server.port=9207 +spring.spring.application.name=dss-workflow-server-dev + +wds.linkis.test.mode=false + +wds.linkis.test.user= + +wds.linkis.log.clear=true + +wds.linkis.server.version=v1 + +##restful +wds.linkis.server.restful.scan.packages=com.webank.wedatasphere.dss.workflow.restful + +##mybatis +wds.linkis.server.mybatis.mapperLocations=classpath*:com/webank/wedatasphere/dss/workflow/dao/impl/*.xml,classpath*:com/webank/wedatasphere/dss/framework/appconn/dao/impl/*.xml + +wds.linkis.server.mybatis.typeAliasesPackage=com.webank.wedatasphere.dss.workflow.entity,com.webank.wedatasphere.dss.framework.appconn.entity + +wds.linkis.server.mybatis.BasePackage=com.webank.wedatasphere.dss.workflow.dao,com.webank.wedatasphere.dss.framework.appconn.dao + +##import file dir +wds.dss.file.upload.dir=/appcom/tmp/uploads + +wds.dss.server.export.env=DEV +wds.dss.server.import.env=DEV + diff --git a/conf/dss.properties b/conf/dss.properties index 674e0bf403..f61562fc8a 100644 --- a/conf/dss.properties +++ b/conf/dss.properties @@ -22,11 +22,11 @@ wds.linkis.gateway.wtss.url=http://127.0.0.1:9001/ wds.linkis.mysql.is.encrypt=false wds.linkis.server.mybatis.datasource.url= wds.linkis.server.mybatis.datasource.username= -***REMOVED*** -wds.linkis.bml.auth.token.value= -linkis.configuration.linkisclient.auth.token.value= -wds.linkis.context.client.auth.value= -wds.linkis.errorcode.auth.token= +wds.linkis.server.mybatis.datasource.password= +wds.linkis.bml.auth.token.value=BML-AUTH +linkis.configuration.linkisclient.auth.token.value=BML-AUTH +wds.linkis.context.client.auth.value=BML-AUTH +wds.linkis.errorcode.auth.token=BML-AUTH wds.dss.check.server.active.period=30 diff --git a/db/dss_ddl.sql b/db/dss_ddl.sql index 1b52e2c607..551b4b59ab 100644 --- a/db/dss_ddl.sql +++ b/db/dss_ddl.sql @@ -51,6 +51,7 @@ CREATE TABLE `dss_orchestrator_info` ( `orchestrator_level` varchar(32) DEFAULT NULL COMMENT '工作流级别', `update_user` varchar(100) DEFAULT NULL COMMENT '更新人', `update_time` datetime DEFAULT CURRENT_TIMESTAMP COMMENT '更新时间', + `status` VARCHAR(64), PRIMARY KEY (`id`) USING BTREE, UNIQUE KEY `unique_idx_uuid` (`uuid`) ) ENGINE=InnoDB AUTO_INCREMENT=326 DEFAULT CHARSET=utf8mb4 ROW_FORMAT=COMPACT; @@ -70,6 +71,7 @@ CREATE TABLE `dss_orchestrator_version_info` ( `content` varchar(255) DEFAULT NULL, `context_id` varchar(200) DEFAULT NULL COMMENT '上下文ID', `valid_flag` INT(1) DEFAULT '1' COMMENT '版本有效标示，0:无效；1：有效', + `commit_id` varchar(64), PRIMARY KEY (`id`) USING BTREE ) ENGINE=InnoDB AUTO_INCREMENT=422 DEFAULT CHARSET=utf8mb4 ROW_FORMAT=COMPACT; @@ -116,6 +118,7 @@ CREATE TABLE `dss_project` ( `dev_process` varchar(200) COLLATE utf8_bin DEFAULT NULL COMMENT '开发流程，多个以英文逗号分隔，取得的值是dss_workspace_dictionary中的dic_key(parent_key=p_develop_process)', `orchestrator_mode` varchar(200) COLLATE utf8_bin DEFAULT NULL COMMENT '编排模式，多个以英文逗号分隔，取得的值是dss_workspace_dictionary中的dic_key(parent_key=p_arrangement_mode或下面一级)', `visible` tinyint(4) DEFAULT '1' COMMENT '0:已删除；1：未删除(默认)', + `associate_git` TINYINT DEFAULT '0' COMMENT '0:未接入git，1:已接入git', PRIMARY KEY (`id`) ) ENGINE=InnoDB AUTO_INCREMENT=313 DEFAULT CHARSET=utf8 COLLATE=utf8_bin ROW_FORMAT=COMPACT; @@ -639,3 +642,33 @@ key `idx_limit_name` (`limit_name`) ) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4 COLLATE=utf8mb4_bin COMMENT ='dss用户限制表'; +DROP TABLE IF EXISTS `dss_workspace_associate_git`; +CREATE TABLE `dss_workspace_associate_git` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `workspace_id` bigint(20) DEFAULT NULL, + `git_user` varchar(64) DEFAULT NULL COMMENT 'git登录用户名', + `git_password` VARCHAR(255) DEFAULT NULL COMMENT 'git登录密码，用于跳转', + `git_token` varchar(255) COMMENT '用户配置的git token', + `git_url` varchar(255), + `create_time` datetime DEFAULT NULL, + `update_time` datetime DEFAULT NULL, + `create_by` varchar(128) DEFAULT NULL, + `update_by` varchar(128) DEFAULT NULL, + `type` varchar(32) DEFAULT NULL, + `git_user_id` varchar(32) DEFAULT NULL, + PRIMARY KEY (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin COMMENT='工作空间绑定的git信息'; + + +DROP TABLE IF EXISTS `dss_orchestrator_submit_job_info`; +CREATE TABLE `dss_orchestrator_submit_job_info` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT '主键', + `orchestrator_id` bigint(20) NOT NULL, + `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', + `update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '修改时间', + `instance_name` varchar(128) DEFAULT NULL COMMENT '提交任务的实例', + `status` varchar(128) DEFAULT NULL COMMENT '提交任务状态', + `error_msg` varchar(2048) DEFAULT NULL COMMENT '提交任务异常信息', + PRIMARY KEY (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin COMMENT='dss_orchestrator_submit_job_info表'; + diff --git a/dss-appconn/appconns/dss-datachecker-appconn/pom.xml b/dss-appconn/appconns/dss-datachecker-appconn/pom.xml index 51cf70eaa5..7b2ecf1590 100644 --- a/dss-appconn/appconns/dss-datachecker-appconn/pom.xml +++ b/dss-appconn/appconns/dss-datachecker-appconn/pom.xml @@ -21,7 +21,7 @@ dss com.webank.wedatasphere.dss - 1.1.0.20-SNAPSHOT + 1.10.0-SNAPSHOT ../../../pom.xml 4.0.0 @@ -54,7 +54,7 @@ com.webank.wedatasphere.dss dss-origin-sso-integration-standard - 1.1.0.20-SNAPSHOT + 1.10.0-SNAPSHOT org.apache.linkis diff --git a/dss-appconn/appconns/dss-datachecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/datachecker/DataChecker.java b/dss-appconn/appconns/dss-datachecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/datachecker/DataChecker.java index 61ff244b5e..e6aa66b835 100644 --- a/dss-appconn/appconns/dss-datachecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/datachecker/DataChecker.java +++ b/dss-appconn/appconns/dss-datachecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/datachecker/DataChecker.java @@ -28,12 +28,27 @@ public class DataChecker { public final static String SOURCE_TYPE = "source.type"; public final static String DATA_OBJECT = "check.object"; + /** + * 检查对象在节点中的序号 + */ + public final static String DATA_OBJECT_NUM = "check.sn.object.num"; public final static String WAIT_TIME = "max.check.hours"; public final static String QUERY_FREQUENCY = "query.frequency"; public final static String TIME_SCAPE = "time.scape"; public final static String MASK_URL = "bdp.mask.url"; public final static String MASK_APP_ID = "bdp.mask.app.id"; public final static String MASK_APP_TOKEN = "bdp.mask.app.token"; + public final static String CONTEXTID_USER = "contextId.user"; + public final static String CONTEXTID_PROJECT_NAME = "contextId.projectName"; + public final static String CONTEXTID_FLOW_NAME = "contextId.flowName"; + public final static String NAME_NAME = "nodeName"; + + public final static String QUALITIS_CHECK = "qualitis.check"; + public final static String QUALITIS_SWITCH = "job.eventchecker.qualitis.switch"; + public final static String QUALITIS_CHECK_DEFAULT = "qualitis.check.default"; + public final static String EXPAND_SECOND_PARTITION = "hourly.secondary.partition"; + + private Properties p; private static final Logger logger = LoggerFactory.getLogger(DataChecker.class);; @@ -48,7 +63,7 @@ public DataChecker(Properties p, DataCheckerExecutionAction action) { maxWaitTime = Long.valueOf(p.getProperty(DataChecker.WAIT_TIME, "1")) * 3600 * 1000; //test over time // maxWaitTime = Long.valueOf(p.getProperty(DataChecker.WAIT_TIME, "1")) * 120 * 1000; - queryFrequency = Integer.valueOf(p.getProperty(DataChecker.QUERY_FREQUENCY, "30000")); + queryFrequency = Integer.valueOf(p.getProperty(DataChecker.QUERY_FREQUENCY, "60000")); } @@ -66,23 +81,23 @@ public void run() { } }catch (Exception ex){ dataCheckerAction.setState(RefExecutionState.Failed); - throw new RuntimeException("get DataChecker result failed", ex); + throw ex; } } - public void begineCheck(RefExecutionAction action){ + public void begineCheck(DataCheckerExecutionAction action){ boolean success=false; try { success= wbDao.validateTableStatusFunction(p, logger,action); }catch (Exception ex){ dataCheckerAction.setState(RefExecutionState.Failed); logger.error("datacheck error",ex); - throw new RuntimeException("get DataChecker result failed", ex); + throw ex; } if(success) { dataCheckerAction.setState(RefExecutionState.Success); - }else { + } else if (dataCheckerAction.getState() != RefExecutionState.Failed) { dataCheckerAction.setState(RefExecutionState.Running); } } diff --git a/dss-appconn/appconns/dss-datachecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/datachecker/common/CheckDataObject.java b/dss-appconn/appconns/dss-datachecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/datachecker/common/CheckDataObject.java index 60223ca56b..66e44eb220 100644 --- a/dss-appconn/appconns/dss-datachecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/datachecker/common/CheckDataObject.java +++ b/dss-appconn/appconns/dss-datachecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/datachecker/common/CheckDataObject.java @@ -75,6 +75,20 @@ public enum Type{ TABLE, } + public String forMat(String secondPartitionExpand){ + final StringBuilder sb = new StringBuilder(); + sb.append(dbName) + .append(".") + .append(tableName); + if(type==Type.PARTITION){ + sb.append("{").append(partitionName); + if(secondPartitionExpand!=null){ + sb.append("/").append(secondPartitionExpand); + } + sb.append("}"); + } + return sb.toString(); + } @Override public String toString() { diff --git a/dss-appconn/appconns/dss-datachecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/datachecker/connector/DataCheckerDao.java b/dss-appconn/appconns/dss-datachecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/datachecker/connector/DataCheckerDao.java index e3bc2ae270..924fc8504b 100644 --- a/dss-appconn/appconns/dss-datachecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/datachecker/connector/DataCheckerDao.java +++ b/dss-appconn/appconns/dss-datachecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/datachecker/connector/DataCheckerDao.java @@ -24,8 +24,7 @@ import com.webank.wedatasphere.dss.appconn.datachecker.common.MaskCheckNotExistException; import com.webank.wedatasphere.dss.appconn.datachecker.utils.HttpUtils; import com.webank.wedatasphere.dss.appconn.datachecker.utils.QualitisUtil; -import com.webank.wedatasphere.dss.standard.app.development.listener.common.RefExecutionAction; -import com.webank.wedatasphere.dss.standard.app.development.listener.common.RefExecutionState; +import com.webank.wedatasphere.dss.common.exception.DSSRuntimeException; import okhttp3.FormBody; import okhttp3.RequestBody; import okhttp3.Response; @@ -54,13 +53,6 @@ public class DataCheckerDao { private static final String SQL_SOURCE_TYPE_JOB_PARTITION = "SELECT * FROM DBS d JOIN TBLS t ON t.DB_ID = d.DB_ID JOIN PARTITIONS p ON p.TBL_ID = t.TBL_ID WHERE d.NAME=? AND t.TBL_NAME=? AND p.PART_NAME=?"; - private static final String SQL_SOURCE_TYPE_BDP = - "SELECT * FROM desktop_bdapimport WHERE bdap_db_name = ? AND bdap_table_name = ? AND target_partition_name = ? AND status = '1';"; - - private static final String SQL_SOURCE_TYPE_BDP_WITH_TIME_CONDITION = - "SELECT * FROM desktop_bdapimport WHERE bdap_db_name = ? AND bdap_table_name = ? AND target_partition_name = ? " + - "AND (UNIX_TIMESTAMP() - UNIX_TIMESTAMP(STR_TO_DATE(modify_time, '%Y-%m-%d %H:%i:%s'))) <= ? AND status = '1';"; - private static final String SQL_DOPS_CHECK_TABLE = "SELECT * FROM dops_clean_task_list WHERE db_name = ? AND tb_name = ? AND part_name is null AND task_state NOT IN (10,13) order by order_id desc limit 1"; private static final String SQL_DOPS_CHECK_PARTITION = @@ -72,7 +64,6 @@ public class DataCheckerDao { private static final String MASK_SOURCE_TYPE = "maskdb"; private static DataSource jobDS; - private static DataSource bdpDS; private static DataSource dopsDS; private static volatile DataCheckerDao instance; @@ -96,13 +87,6 @@ public boolean validateTableStatusFunction(Properties props, Logger log, DataChe return false; } } - if (bdpDS == null) { - bdpDS = DataDruidFactory.getBDPInstance(props, log); - if (bdpDS == null) { - log.warn("Error getting job Druid DataSource instance"); - return false; - } - } boolean systemCheck = Boolean.valueOf(props.getProperty(DataChecker.QUALITIS_SWITCH)); if (systemCheck && dopsDS == null) { dopsDS = DataDruidFactory.getDopsInstance(props, log);//通过alibaba的druid数据库连接池获取JOB数据库连接 @@ -122,7 +106,7 @@ public boolean validateTableStatusFunction(Properties props, Logger log, DataChe } log.info("(DataChecker info) database table partition info : " + dataCheckerInfo); long waitTime = Long.valueOf(props.getProperty(DataChecker.WAIT_TIME, "1")) * 3600 * 1000; - int queryFrequency = Integer.valueOf(props.getProperty(DataChecker.QUERY_FREQUENCY, "30000")); + int queryFrequency = Integer.valueOf(props.getProperty(DataChecker.QUERY_FREQUENCY, "60000")); // String timeScape = props.getProperty(DataChecker.TIME_SCAPE, "NULL"); log.info("(DataChecker info) wait time : " + waitTime); log.info("(DataChecker info) query frequency : " + queryFrequency); @@ -134,13 +118,12 @@ public boolean validateTableStatusFunction(Properties props, Logger log, DataChe }); QualitisUtil qualitisUtil = new QualitisUtil(props); try (Connection jobConn = jobDS.getConnection(); - Connection bdpConn = bdpDS.getConnection(); Connection dopsConn = dopsDS != null ? dopsDS.getConnection() : null) { List allCheckRes = dataObjectList .parallelStream() .map(proObjectMap -> { log.info("Begin to Check dataObject:" + proObjectMap.entrySet().toString()); - boolean checkRes = getDataCheckResult(proObjectMap, jobConn, bdpConn, dopsConn, props, log,action,qualitisUtil); + boolean checkRes = getDataCheckResult(proObjectMap, jobConn, dopsConn, props, log,action,qualitisUtil); if (null != action.getExecutionRequestRefContext()) { if (checkRes) { action.getExecutionRequestRefContext().appendLog("Database table partition info : " + proObjectMap.get(DataChecker.DATA_OBJECT) + " has arrived"); @@ -178,7 +161,6 @@ public boolean validateTableStatusFunction(Properties props, Logger log, DataChe private boolean getDataCheckResult(Map proObjectMap, Connection jobConn, - Connection bdpConn, Connection dopsConn, Properties props, Logger log, @@ -190,12 +172,9 @@ private boolean getDataCheckResult(Map proObjectMap, } String objectNum = proObjectMap.get(DataChecker.DATA_OBJECT_NUM); CheckDataObject dataObject; - try { + dataObject = parseDataObject(dataObjectStr); - } catch (SQLException e) { - log.error("parse dataObject failed", e); - return false; - } + Predicate> hasDataSource = p -> { if (StringUtils.isEmpty(proObjectMap.get(DataChecker.SOURCE_TYPE))) { return false; @@ -231,7 +210,7 @@ private boolean getDataCheckResult(Map proObjectMap, } log.info("start to check maskis"); proObjectMap.put(DataChecker.SOURCE_TYPE, MASK_SOURCE_TYPE); - normalCheck= (getBdpTotalCount(dataObject, bdpConn, log, props) > 0 || "success".equals(fetchMaskCode(dataObject, log, props).get("maskStatus"))); + normalCheck= "success".equals(fetchMaskCode(dataObject, log, props).get("maskStatus")); if (null != action.getExecutionRequestRefContext()){ action.getExecutionRequestRefContext().appendLog(dataObjectStr+" check maskis end,check result:"+normalCheck); } @@ -265,9 +244,38 @@ private void removeBlankSpace(Properties props) { } private List> extractProperties(Properties p) { - return p.keySet().stream() + List> checkList = p.keySet().stream() .map(key -> key2Map(key, p)).filter(x -> x.size() > 0) .collect(Collectors.toList()); + if (p.containsKey(DataChecker.EXPAND_SECOND_PARTITION) && "true".equalsIgnoreCase(p.getProperty(DataChecker.EXPAND_SECOND_PARTITION).trim())) { + List> expandCheckList = new ArrayList<>(); + for (Map proObjectMap : checkList) { + String dataObjectStr = proObjectMap.get(DataChecker.DATA_OBJECT) ; + CheckDataObject dataObject= parseDataObject(dataObjectStr); + if(dataObject.getType() == CheckDataObject.Type.PARTITION + && !dataObject.getPartitionName().contains("/")){ + for (int i = 0; i < 24; i++) { + Map itemMap = new HashMap<>(3); + String formattedValue = String.format("%02d", i); + String stKey = DataChecker.SOURCE_TYPE; + if (proObjectMap.containsKey(stKey)) { + itemMap.put(DataChecker.SOURCE_TYPE, proObjectMap.get(stKey)); + } + String secondPartitionName = "ph=" + formattedValue; + String nCheckObject = dataObject.forMat(secondPartitionName); + itemMap.put(DataChecker.DATA_OBJECT,nCheckObject); + String nObjectNum = proObjectMap.get(DataChecker.DATA_OBJECT_NUM) + formattedValue; + itemMap.put(DataChecker.DATA_OBJECT_NUM, nObjectNum); + expandCheckList.add(itemMap); + } + }else{ + expandCheckList.add(proObjectMap); + } + + } + checkList = expandCheckList; + } + return checkList; } private Map key2Map(Object key, Properties p) { @@ -316,25 +324,6 @@ private PreparedStatement getJobStatement(Connection conn, CheckDataObject dataO } } - /** - * 构造查询maskis的查询 - */ - private PreparedStatement getBdpStatement(Connection conn, CheckDataObject dataObject, String timeScape) throws SQLException { - PreparedStatement pstmt = null; - if (timeScape.equals("NULL")) { - pstmt = conn.prepareCall(SQL_SOURCE_TYPE_BDP); - } else { - pstmt = conn.prepareCall(SQL_SOURCE_TYPE_BDP_WITH_TIME_CONDITION); - pstmt.setInt(4, Integer.valueOf(timeScape) * 3600); - } - if (dataObject.getPartitionName() == null) { - dataObject.setPartitionName(""); - } - pstmt.setString(1, dataObject.getDbName()); - pstmt.setString(2, dataObject.getTableName()); - pstmt.setString(3, dataObject.getPartitionName()); - return pstmt; - } /** * 构造查询dops库的查询 @@ -371,10 +360,11 @@ private PreparedStatement getDopsStatementCheckAllPartition(Connection conn, Che * @param dataObjectStr 字符串形式的对象 * @return 发序列化后的对象 */ - private CheckDataObject parseDataObject(String dataObjectStr)throws SQLException{ + private CheckDataObject parseDataObject(String dataObjectStr) { CheckDataObject dataObject; if(!dataObjectStr.contains(".")){ - throw new SQLException("Error for DataObject format!"+dataObjectStr); + throw new DSSRuntimeException("check object format error(校验对象配置格式错误，请按照dbname" + + ".tablename{ds=partionname}格式配置)！ error content(错误内容）:"+dataObjectStr); } String dbName = dataObjectStr.split("\\.")[0]; String tableName = dataObjectStr.split("\\.")[1]; @@ -414,27 +404,6 @@ private long getJobTotalCount(CheckDataObject dataObject, Connection conn, Logge } } - /** - * 查mask db - */ - private long getBdpTotalCount(CheckDataObject dataObject, Connection conn, Logger log, Properties props) { - String timeScape = props.getOrDefault(DataChecker.TIME_SCAPE, "NULL").toString(); - log.info("-------------------------------------- search bdp data "); - log.info("-------------------------------------- dataObject: " + dataObject.toString()); - try (PreparedStatement pstmt = getBdpStatement(conn, dataObject, timeScape)) { - ResultSet rs = pstmt.executeQuery(); - long ret = 0L; - while (rs.next()) { - ret ++; - } -// long ret=rs.last() ? rs.getRow() : 0; - log.info("-------------------------------------- bdp data result:"+ret); - return ret; - } catch (SQLException e) { - log.error("fetch data from bdp error", e); - return 0; - } - } /** * - 返回0表示未找到任何记录 ； diff --git a/dss-appconn/appconns/dss-datachecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/datachecker/connector/DataDruidFactory.java b/dss-appconn/appconns/dss-datachecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/datachecker/connector/DataDruidFactory.java index 32dba01ebe..be89365dee 100644 --- a/dss-appconn/appconns/dss-datachecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/datachecker/connector/DataDruidFactory.java +++ b/dss-appconn/appconns/dss-datachecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/datachecker/connector/DataDruidFactory.java @@ -25,7 +25,8 @@ public class DataDruidFactory { private static volatile DruidDataSource jobInstance; - private static volatile DruidDataSource bdpInstance; + + private static volatile DruidDataSource dopsInstance; private static volatile DruidDataSource msgInstance; public static DruidDataSource getJobInstance(Properties props, Logger log) { @@ -35,26 +36,27 @@ public static DruidDataSource getJobInstance(Properties props, Logger log) { try { jobInstance = createDataSource(props, log, "Job"); } catch (Exception e) { - throw new RuntimeException("Error creating Druid DataSource", e); + throw new RuntimeException("Error creating job Druid DataSource", e); } } } } return jobInstance; } - public static DruidDataSource getBDPInstance(Properties props, Logger log) { - if (bdpInstance == null ) { + + public static DruidDataSource getDopsInstance(Properties props, Logger log) { + if (dopsInstance == null) { synchronized (DataDruidFactory.class) { - if(bdpInstance == null) { + if (dopsInstance == null) { try { - bdpInstance = createDataSource(props, log, "BDP"); + dopsInstance = createDataSource(props, log, "Dops"); } catch (Exception e) { - throw new RuntimeException("Error creating Druid DataSource", e); + throw new RuntimeException("Error creating DOPS Druid DataSource", e); } } } } - return bdpInstance; + return dopsInstance; } public static DruidDataSource getMsgInstance(Properties props, Logger log) { @@ -110,6 +112,22 @@ private static DruidDataSource createDataSource(Properties props, Logger log, St } catch (Exception e) { log.error("password decore failed" + e); } + }else if ("Dops".equals(type)) { + name = props.getProperty("dops.datachecker.jdo.option.name"); + url = props.getProperty("dops.datachecker.jdo.option.url"); + username = props.getProperty("dops.datachecker.jdo.option.username"); + password=props.getProperty("dops.datachecker.jdo.option.password"); + loginType = props.getProperty("dops.datachecker.jdo.option.login.type"); + log.info("dops url is:" + url + "and name is:" + username); + try { + if ("base64".equals(loginType)) { + password = new String(Base64.getDecoder().decode(props.getProperty("dops.datachecker.jdo.option.password").getBytes()), "UTF-8"); + } else { + password = props.getProperty("dops.datachecker.jdo.option.password"); + } + } catch (Exception e) { + log.error("password decore failed", e); + } } int initialSize = Integer.valueOf(props.getProperty("datachecker.jdo.option.initial.size", "1")); int maxActive = Integer.valueOf(props.getProperty("datachecker.jdo.option.max.active", "100")); diff --git a/dss-appconn/appconns/dss-datachecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/datachecker/utils/Sha256Utils.java b/dss-appconn/appconns/dss-datachecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/datachecker/utils/Sha256Utils.java new file mode 100644 index 0000000000..242fc1a5ea --- /dev/null +++ b/dss-appconn/appconns/dss-datachecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/datachecker/utils/Sha256Utils.java @@ -0,0 +1,30 @@ +package com.webank.wedatasphere.dss.appconn.datachecker.utils; + +import org.apache.commons.lang.StringUtils; + +import java.math.BigInteger; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; + +/** + * @author lebronwang + * @date 2022/08/04 + **/ +public class Sha256Utils { + + public static String getSHA256L32(String str) throws NoSuchAlgorithmException { + try { + // Generate a instance of sha-256 + MessageDigest hash = MessageDigest.getInstance("SHA-256"); + // calculate + hash.update(str.getBytes()); + String result = new BigInteger(1, hash.digest()).toString(16); + + return StringUtils.leftPad(result, 32, '0'); + } catch (NoSuchAlgorithmException e) { + throw new NoSuchAlgorithmException("Error generating sha256 of string", e); + } + } + + +} diff --git a/dss-appconn/appconns/dss-datachecker-appconn/src/main/scala/com/webank/wedatasphere/dss/appconn/datachecker/DataCheckerRefExecutionOperation.scala b/dss-appconn/appconns/dss-datachecker-appconn/src/main/scala/com/webank/wedatasphere/dss/appconn/datachecker/DataCheckerRefExecutionOperation.scala index 93f0b60a7e..8451e58bb9 100644 --- a/dss-appconn/appconns/dss-datachecker-appconn/src/main/scala/com/webank/wedatasphere/dss/appconn/datachecker/DataCheckerRefExecutionOperation.scala +++ b/dss-appconn/appconns/dss-datachecker-appconn/src/main/scala/com/webank/wedatasphere/dss/appconn/datachecker/DataCheckerRefExecutionOperation.scala @@ -66,10 +66,11 @@ class DataCheckerRefExecutionOperation }else{ rows= descValue.split(";") } + rows=rows.map(_.trim) rows.foreach(row => if (row.contains("=")) { val endLocation = row.indexOf("=") - val rowKey = row.trim.substring(0, endLocation) - val rowEnd = row.trim.substring(endLocation + 1) + val rowKey = row.substring(0, endLocation) + val rowEnd = row.substring(endLocation + 1) tmpProperties.put(rowKey, rowEnd) }) } else { @@ -77,19 +78,25 @@ class DataCheckerRefExecutionOperation } } ) + val tmp: util.HashMap[String, Any] = new util.HashMap[String, Any]() + inputParams.foreach(x=> + if(x._1.equalsIgnoreCase(VariableUtils.RUN_DATE) + ||x._1.equalsIgnoreCase(VariableUtils.RUN_TODAY_H) + ||x._1.equalsIgnoreCase("run_today_hour") + ||x._1.equalsIgnoreCase("run_today_h_std") + ||x._1.equalsIgnoreCase("run_today_hour_std") + ){ + tmp.put(x._1, x._2) + } + ) tmpProperties.foreach { record => logger.info("request params key : " + record._1 + ",value : " + record._2) if (null == record._2) { properties.put(record._1, "") - } - else { - if (inputParams.exists(x => x._1.equalsIgnoreCase(VariableUtils.RUN_DATE))) { - val tmp: util.HashMap[String, Any] = new util.HashMap[String, Any]() - tmp.put(VariableUtils.RUN_DATE, inputParams.getOrElse(VariableUtils.RUN_DATE, null)) - properties.put(record._1, VariableUtils.replace(record._2.toString, tmp)) - } else { - properties.put(record._1, VariableUtils.replace(record._2.toString)) - } + } else if(tmp.isEmpty){ + properties.put(record._1, VariableUtils.replace(record._2.toString)) + } else { + properties.put(record._1, VariableUtils.replace(record._2.toString, tmp)) } } Utils.tryCatch({ diff --git a/dss-appconn/appconns/dss-dolphinscheduler-appconn/pom.xml b/dss-appconn/appconns/dss-dolphinscheduler-appconn/pom.xml index 8543d92a97..d7be23fcdb 100644 --- a/dss-appconn/appconns/dss-dolphinscheduler-appconn/pom.xml +++ b/dss-appconn/appconns/dss-dolphinscheduler-appconn/pom.xml @@ -6,7 +6,7 @@ dss com.webank.wedatasphere.dss - 1.1.0.20-SNAPSHOT + 1.10.0-SNAPSHOT ../../../pom.xml 4.0.0 @@ -18,7 +18,6 @@ com.webank.wedatasphere.dss dss-scheduler-appconn ${dss.version} - provided @@ -81,7 +80,7 @@ com.google.guava guava - 28.2-android + 33.1.0-jre diff --git a/dss-appconn/appconns/dss-dolphinscheduler-appconn/src/main/scala/com/webank/wedatasphere/dss/appconn/schedulis/conf/SchedulisConf.scala b/dss-appconn/appconns/dss-dolphinscheduler-appconn/src/main/scala/com/webank/wedatasphere/dss/appconn/schedulis/conf/SchedulisConf.scala new file mode 100644 index 0000000000..42a6b4cfcc --- /dev/null +++ b/dss-appconn/appconns/dss-dolphinscheduler-appconn/src/main/scala/com/webank/wedatasphere/dss/appconn/schedulis/conf/SchedulisConf.scala @@ -0,0 +1,32 @@ + /* + * + * * Copyright 2019 WeBank + * * + * * Licensed under the Apache License, Version 2.0 (the "License"); + * * you may not use this file except in compliance with the License. + * * You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.schedulis.conf + +import java.lang +import java.lang.reflect.Type + +import com.google.gson.{Gson, GsonBuilder, JsonElement, JsonPrimitive, JsonSerializationContext, JsonSerializer} + +object SchedulisConf { + implicit val gson:Gson = new GsonBuilder().setPrettyPrinting().setDateFormat("yyyy-MM-dd'T'HH:mm:ssZ").serializeNulls + .registerTypeAdapter(classOf[java.lang.Double], new JsonSerializer[java.lang.Double] { + override def serialize(t: lang.Double, `type`: Type, jsonSerializationContext: JsonSerializationContext): JsonElement = + if(t == t.longValue()) new JsonPrimitive(t.longValue()) else new JsonPrimitive(t) + }).create +} diff --git a/dss-appconn/appconns/dss-dolphinscheduler-appconn/src/main/scala/com/webank/wedatasphere/dss/appconn/schedulis/http/SchedulisHttpAction.scala b/dss-appconn/appconns/dss-dolphinscheduler-appconn/src/main/scala/com/webank/wedatasphere/dss/appconn/schedulis/http/SchedulisHttpAction.scala new file mode 100644 index 0000000000..8c1543dd63 --- /dev/null +++ b/dss-appconn/appconns/dss-dolphinscheduler-appconn/src/main/scala/com/webank/wedatasphere/dss/appconn/schedulis/http/SchedulisHttpAction.scala @@ -0,0 +1,87 @@ + /* + * + * * Copyright 2019 WeBank + * * + * * Licensed under the Apache License, Version 2.0 (the "License"); + * * you may not use this file except in compliance with the License. + * * You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.schedulis.http + +import java.io.{File, InputStream} +import java.util + +import com.webank.wedatasphere.dss.appconn.schedulis.conf.SchedulisConf +import org.apache.linkis.httpclient.request.{GetAction, HttpAction, POSTAction, UploadAction, UserAction} + +trait SchedulisHttpAction extends UserAction{ + + private var user:String = _ + + override def setUser(user: String): Unit = this.user = user + + override def getUser: String = this.user + +} + +abstract class SchedulisGetAction extends GetAction with SchedulisHttpAction + + +abstract class ScheudlisPostAction extends POSTAction with SchedulisHttpAction{ + + override def getRequestPayload: String = SchedulisConf.gson.toJson(getRequestPayloads) + +} + + + + +case class SchedulisUploadAction(filePaths:Array[String], + _inputStreams:util.Map[String,InputStream],uploadUrl:String) extends ScheudlisPostAction with UploadAction with SchedulisHttpAction{ + + private val streamNames = new util.HashMap[String,String] + + override val files: util.Map[String, String] = { + if (null == filePaths || filePaths.length == 0) new util.HashMap[String,String]() else{ + val map = new java.util.HashMap[String, String] + filePaths foreach { + filePath => val arr = filePath.split(File.separator) + val fileName = arr(arr.length - 1) + map.put("file", filePath) + } + map + } + } + + override def inputStreams: util.Map[String, InputStream] = _inputStreams + + override def inputStreamNames: util.Map[String, String] = streamNames + + private var _user:String = _ + + override def setUser(user: String): Unit = this._user = user + + override def getUser: String = this._user + + override def getRequestPayload: String = "" + + override def getURL: String = uploadUrl +} + +class SchedulisCreateProjectAction(url:String) extends ScheudlisPostAction{ + + override def getURL: String = url + +} + + diff --git a/dss-appconn/appconns/dss-eventchecker-appconn/pom.xml b/dss-appconn/appconns/dss-eventchecker-appconn/pom.xml index 7539d9e681..01d05f70d8 100644 --- a/dss-appconn/appconns/dss-eventchecker-appconn/pom.xml +++ b/dss-appconn/appconns/dss-eventchecker-appconn/pom.xml @@ -21,7 +21,7 @@ dss com.webank.wedatasphere.dss - 1.1.0.20-SNAPSHOT + 1.10.0-SNAPSHOT ../../../pom.xml 4.0.0 @@ -61,7 +61,22 @@ dss-development-process-standard-execution ${dss.version} - + + com.squareup.okhttp3 + okhttp + 4.2.2 + + + org.jetbrains + annotations + + + + + cn.hutool + hutool-all + 5.8.21 + org.apache.commons commons-lang3 diff --git a/dss-appconn/appconns/dss-eventchecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/eventchecker/entity/EventChecker.java b/dss-appconn/appconns/dss-eventchecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/eventchecker/entity/EventChecker.java index 1854973ae2..fcec4ceca7 100644 --- a/dss-appconn/appconns/dss-eventchecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/eventchecker/entity/EventChecker.java +++ b/dss-appconn/appconns/dss-eventchecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/eventchecker/entity/EventChecker.java @@ -47,6 +47,7 @@ public class EventChecker implements Runnable{ public final static String TODAY="only.receive.today"; public final static String USE_RUN_DATE ="msg.receive.use.rundate"; public final static String AFTERSEND="msg.after.send"; + public final static String CHANNEL_TYPE="msg.channel.type"; private Properties p; private String jobId; @@ -67,10 +68,10 @@ public EventChecker(Properties p, EventCheckerExecutionAction action) { String waitTime = p.getProperty(EventChecker.WAIT_TIME, "1"); Double doubleWaitTime = Double.valueOf(waitTime) * 3600 * 1000; maxWaitTime = Long.valueOf(doubleWaitTime.longValue()); - String query_frequency = p.getProperty(EventChecker.QUERY_FREQUENCY, "30000"); + String query_frequency = p.getProperty(EventChecker.QUERY_FREQUENCY, "60000"); queryFrequency = Integer.valueOf(query_frequency); - if(queryFrequency <10000){ - queryFrequency = 10000; + if(queryFrequency <60000){ + queryFrequency = 60000; } } diff --git a/dss-appconn/appconns/dss-eventchecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/eventchecker/entity/HttpMsgReceiveRequest.java b/dss-appconn/appconns/dss-eventchecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/eventchecker/entity/HttpMsgReceiveRequest.java new file mode 100644 index 0000000000..c540cc64bd --- /dev/null +++ b/dss-appconn/appconns/dss-eventchecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/eventchecker/entity/HttpMsgReceiveRequest.java @@ -0,0 +1,70 @@ +package com.webank.wedatasphere.dss.appconn.eventchecker.entity; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; + +import java.util.HashMap; +import java.util.Map; + +/** + * Author: xlinliu + * Date: 2024/8/2 + */ +public class HttpMsgReceiveRequest { + private String receiver; + private String topic; + private String msgName; + private String runDate; + private boolean onlyReceiveToday; + private boolean receiveUseRunDate; + + + public HttpMsgReceiveRequest(String receiver, String topic, String msgName, String runDate, boolean onlyReceiveToday, boolean receiveUseRunDate) { + this.receiver = receiver; + this.topic = topic; + this.msgName = msgName; + this.runDate = runDate; + this.onlyReceiveToday = onlyReceiveToday; + this.receiveUseRunDate = receiveUseRunDate; + } + + public HttpMsgReceiveRequest() { + + } + + public HttpMsgReceiveRequest setReceiver(String receiver) { + this.receiver = receiver; + return this; + } + + public HttpMsgReceiveRequest setTopic(String topic) { + this.topic = topic; + return this; + } + + public HttpMsgReceiveRequest setMsgName(String msgName) { + this.msgName = msgName; + return this; + } + + public HttpMsgReceiveRequest setRunDate(String runDate) { + this.runDate = runDate; + return this; + } + + public HttpMsgReceiveRequest setOnlyReceiveToday(boolean onlyReceiveToday) { + this.onlyReceiveToday = onlyReceiveToday; + return this; + } + + public HttpMsgReceiveRequest setReceiveUseRunDate(boolean receiveUseRunDate) { + this.receiveUseRunDate = receiveUseRunDate; + return this; + } + + + public String toJson() { + Gson gson = new GsonBuilder().create(); + return gson.toJson(this); + } +} diff --git a/dss-appconn/appconns/dss-eventchecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/eventchecker/entity/HttpMsgReceiveResponse.java b/dss-appconn/appconns/dss-eventchecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/eventchecker/entity/HttpMsgReceiveResponse.java new file mode 100644 index 0000000000..6ff0105051 --- /dev/null +++ b/dss-appconn/appconns/dss-eventchecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/eventchecker/entity/HttpMsgReceiveResponse.java @@ -0,0 +1,60 @@ +package com.webank.wedatasphere.dss.appconn.eventchecker.entity; + +import java.util.Map; + +/** + * Author: xlinliu + * Date: 2024/8/6 + */ +public class HttpMsgReceiveResponse { + private int retCode; + private String message; + private long msgId; + private Map msgBody; + /** + * success成功 + * fail失败 + * running运行中 + */ + private String status; + + public int getRetCode() { + return retCode; + } + + public void setRetCode(int retCode) { + this.retCode = retCode; + } + + public String getMessage() { + return message; + } + + public void setMessage(String message) { + this.message = message; + } + + public long getMsgId() { + return msgId; + } + + public void setMsgId(long msgId) { + this.msgId = msgId; + } + + public Map getMsgBody() { + return msgBody; + } + + public void setMsgBody(Map msgBody) { + this.msgBody = msgBody; + } + + public String getStatus() { + return status; + } + + public void setStatus(String status) { + this.status = status; + } +} diff --git a/dss-appconn/appconns/dss-eventchecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/eventchecker/entity/HttpMsgSendRequest.java b/dss-appconn/appconns/dss-eventchecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/eventchecker/entity/HttpMsgSendRequest.java new file mode 100644 index 0000000000..9240d2fdb2 --- /dev/null +++ b/dss-appconn/appconns/dss-eventchecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/eventchecker/entity/HttpMsgSendRequest.java @@ -0,0 +1,69 @@ +package com.webank.wedatasphere.dss.appconn.eventchecker.entity; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; + +import java.util.HashMap; +import java.util.Map; + +/** + * Author: xlinliu + * Date: 2024/8/2 + */ +public class HttpMsgSendRequest { + private String sender; + private String topic; + private String msgName; + private String runDate; + private String msgId; + private Map msgBody; + + public HttpMsgSendRequest(String sender, String topic, String msgName, String runDate, String msgId, Map msgBody) { + this.sender = sender; + this.topic = topic; + this.msgName = msgName; + this.runDate = runDate; + this.msgId = msgId; + this.msgBody = msgBody; + } + + public HttpMsgSendRequest() { + this.msgBody = new HashMap<>(); + } + + public HttpMsgSendRequest setSender(String sender) { + this.sender = sender; + return this; + } + + public HttpMsgSendRequest setTopic(String topic) { + this.topic = topic; + return this; + } + + public HttpMsgSendRequest setMsgName(String msgName) { + this.msgName = msgName; + return this; + } + + public HttpMsgSendRequest setRunDate(String runDate) { + this.runDate = runDate; + return this; + } + + public HttpMsgSendRequest setMsgId(String msgId) { + this.msgId = msgId; + return this; + } + + + public HttpMsgSendRequest addMsgBodyField(String key, Object value) { + this.msgBody.put(key, value); + return this; + } + + public String toJson() { + Gson gson = new GsonBuilder().create(); + return gson.toJson(this); + } +} diff --git a/dss-appconn/appconns/dss-eventchecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/eventchecker/entity/HttpMsgSendResponse.java b/dss-appconn/appconns/dss-eventchecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/eventchecker/entity/HttpMsgSendResponse.java new file mode 100644 index 0000000000..0ab97825a0 --- /dev/null +++ b/dss-appconn/appconns/dss-eventchecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/eventchecker/entity/HttpMsgSendResponse.java @@ -0,0 +1,26 @@ +package com.webank.wedatasphere.dss.appconn.eventchecker.entity; + +/** + * Author: xlinliu + * Date: 2024/8/6 + */ +public class HttpMsgSendResponse { + private int retCode; + private String message; + + public int getRetCode() { + return retCode; + } + + public void setRetCode(int retCode) { + this.retCode = retCode; + } + + public String getMessage() { + return message; + } + + public void setMessage(String message) { + this.message = message; + } +} diff --git a/dss-appconn/appconns/dss-eventchecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/eventchecker/service/AbstractEventCheck.java b/dss-appconn/appconns/dss-eventchecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/eventchecker/service/AbstractEventCheck.java index 23ed8576fc..8d51d4cf1e 100644 --- a/dss-appconn/appconns/dss-eventchecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/eventchecker/service/AbstractEventCheck.java +++ b/dss-appconn/appconns/dss-eventchecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/eventchecker/service/AbstractEventCheck.java @@ -70,7 +70,7 @@ void initECParams(Properties props){ runDate = props.getProperty("run_date"); userTime = props.getProperty(EventChecker.USER_TIME); waitTime = props.getProperty(EventChecker.WAIT_TIME, "1"); - query_frequency = props.getProperty(EventChecker.QUERY_FREQUENCY, "30000"); + query_frequency = props.getProperty(EventChecker.QUERY_FREQUENCY, "60000"); afterSend = props.getProperty(EventChecker.AFTERSEND); } diff --git a/dss-appconn/appconns/dss-eventchecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/eventchecker/service/AbstractEventCheckReceiver.java b/dss-appconn/appconns/dss-eventchecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/eventchecker/service/AbstractEventCheckReceiver.java index 853eb381c1..f20095ea0a 100644 --- a/dss-appconn/appconns/dss-eventchecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/eventchecker/service/AbstractEventCheckReceiver.java +++ b/dss-appconn/appconns/dss-eventchecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/eventchecker/service/AbstractEventCheckReceiver.java @@ -26,10 +26,29 @@ import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; +import java.util.Arrays; import java.util.Date; import java.util.Properties; -public class AbstractEventCheckReceiver extends AbstractEventCheck{ +public abstract class AbstractEventCheckReceiver extends AbstractEventCheck{ + String todayStartTime; + String todayEndTime; + String allStartTime; + String allEndTime; + String nowStartTime; + + public AbstractEventCheckReceiver(Properties props) { + initECParams(props); + initReceiverTimes(); + } + + void initReceiverTimes(){ + todayStartTime = DateFormatUtils.format(new Date(), "yyyy-MM-dd 00:00:00"); + todayEndTime = DateFormatUtils.format(new Date(), "yyyy-MM-dd 23:59:59"); + allStartTime = DateFormatUtils.format(new Date(), "10000-01-01 00:00:00"); + allEndTime = DateFormatUtils.format(new Date(), "9999-12-31 23:59:59"); + nowStartTime = DateFormatUtils.format(new Date(), "yyyy-MM-dd HH:mm:ss"); + } /** * Fill the result into the source */ @@ -39,14 +58,14 @@ String setConsumedMsg(Properties props, Logger log, String[] consumedMsgInfo){ if(consumedMsgInfo!=null && consumedMsgInfo.length == 4){ vNewMsgID = consumedMsgInfo[0]; String vMsgName = consumedMsgInfo[1]; - String vSender = consumedMsgInfo[2]; + String vReceiver = consumedMsgInfo[2]; String vMsg = consumedMsgInfo[3]; if (null == vMsg) { props.put(EventChecker.MSG, "NULL"); } else { props.put(EventChecker.MSG, vMsg); } - log.info("Received message : messageID: " + vNewMsgID + ", messageName: " + vMsgName + ", receiver: " + vSender + log.info("Received message : messageID: " + vNewMsgID + ", messageName: " + vMsgName + ", receiver: " + vReceiver + ", messageBody: " + vMsg); } }catch (Exception e) { @@ -114,8 +133,7 @@ String getOffset(int jobId, Properties props, Logger log){ String lastMsgId = "0"; try { msgConn = getEventCheckerConnection(props,log); - pstmtForGetID = msgConn.prepareStatement(sqlForReadMsgID, ResultSet.TYPE_SCROLL_INSENSITIVE, ResultSet.CONCUR_READ_ONLY); -// pstmtForGetID = msgConn.prepareCall(sqlForReadMsgID); + pstmtForGetID = msgConn.prepareCall(sqlForReadMsgID); pstmtForGetID.setString(1, receiver); pstmtForGetID.setString(2, topic); pstmtForGetID.setString(3, msgName); @@ -134,71 +152,54 @@ String getOffset(int jobId, Properties props, Logger log){ log.info("The last record id was " + lastMsgId); return lastMsgId; } - - /** - * Consistent entrance to consumer message - */ - String[] getMsg(Properties props, Logger log,String ... params){ - boolean useRunDate=Boolean.parseBoolean(params[3]); - String sqlForReadTMsg; - if(useRunDate){ - sqlForReadTMsg ="SELECT * FROM event_queue WHERE topic=? AND msg_name=? AND send_time >=? AND send_time <=? AND msg_id >? AND run_date =?ORDER BY msg_id ASC LIMIT 1"; - } else{ - sqlForReadTMsg="SELECT * FROM event_queue WHERE topic=? AND msg_name=? AND send_time >=? AND send_time <=? AND msg_id >? ORDER BY msg_id ASC LIMIT 1"; + @Override + public boolean reciveMsg(int jobId, Properties props, Logger log) { + boolean result = false; + try{ + String lastMsgId = getOffset(jobId,props,log); + String[] executeType = createExecuteType(jobId,props,log,lastMsgId); + log.info("event receiver executeType[]:{},{},{},{},{}",executeType[0],executeType[1],executeType[2],executeType[3],executeType[4]); + if(executeType!=null && executeType.length ==5){ + String[] consumedMsgInfo = getMsg(jobId, props, log,executeType); + if(consumedMsgInfo!=null && consumedMsgInfo.length == 4){ + result = updateMsgOffset(jobId,props,log,consumedMsgInfo,lastMsgId); + } + }else{ + log.error("executeType error {} " , Arrays.toString(executeType)); + return result; + } + }catch (Exception e){ + log.error("EventChecker failed to receive the message" , e); + throw e; } + return result; + } - PreparedStatement pstmt = null; - Connection msgConn = null; - ResultSet rs = null; - String[] consumedMsgInfo = null; - try { - msgConn = getEventCheckerConnection(props,log); - pstmt = msgConn.prepareStatement(sqlForReadTMsg, ResultSet.TYPE_SCROLL_INSENSITIVE, ResultSet.CONCUR_READ_ONLY); -// pstmt = msgConn.prepareCall(sqlForReadTMsg); - pstmt.setString(1, topic); - pstmt.setString(2, msgName); - pstmt.setString(3, params[0]); - pstmt.setString(4, params[1]); - pstmt.setString(5, params[2]); - if(useRunDate){ - log.info("use run_date, run_date:{}", params[4]); - pstmt.setString(6,params[4]); - } - log.info("param {} StartTime: " + params[0] + ", EndTime: " + params[1] - + ", Topic: " + topic + ", MessageName: " + msgName + ", LastMessageID: " + params[2]); - rs = pstmt.executeQuery(); - while (rs.next()) { - consumedMsgInfo = new String[4]; - String[] msgKey = new String[]{"msg_id", "msg_name", "sender", "msg"}; - for (int i = 0; i < msgKey.length; i++) { - try { - consumedMsgInfo[i] = rs.getString(msgKey[i]); - } catch (SQLException e) { - throw new RuntimeException("Error while reading data from ResultSet", e); - } + private String[] createExecuteType(int jobId, Properties props, Logger log,String lastMsgId){ + boolean receiveTodayFlag = (null != receiveToday && "true".equals(receiveToday.trim().toLowerCase())); + boolean afterSendFlag = (null != afterSend && "true".equals(afterSend.trim().toLowerCase())); + //只有receiveTodayFlag为true时，useRunDateFlag才有意义。 + Boolean useRunDateFlag = receiveTodayFlag && (null == useRunDate || "true".equalsIgnoreCase(useRunDate.trim())); + String[] executeType = null; + try { + if (receiveTodayFlag && !useRunDateFlag) { + if (afterSendFlag) { + executeType = new String[]{nowStartTime, todayEndTime, lastMsgId, useRunDateFlag.toString(), runDate}; + } else { + executeType = new String[]{todayStartTime, todayEndTime, lastMsgId, useRunDateFlag.toString(), runDate}; + } + } else { + if (afterSendFlag) { + executeType = new String[]{nowStartTime, allEndTime, lastMsgId, useRunDateFlag.toString(), runDate}; + } else { + executeType = new String[]{allStartTime, allEndTime, lastMsgId, useRunDateFlag.toString(), runDate}; } } - -// if(rs.last()){ -// consumedMsgInfo = new String[4]; -// String[] msgKey = new String[]{"msg_id","msg_name","sender","msg"}; -// for (int i = 0;i <= 3;i++) { -// consumedMsgInfo[i] = rs.getString(msgKey[i]); -// } -// } - } catch (SQLException e) { - throw new RuntimeException("EventChecker failed to receive message" + e); - } finally { - closeQueryStmt(pstmt, log); - closeConnection(msgConn, log); - closeQueryRef(rs, log); + }catch(Exception e){ + log.error("create executeType failed {}" + e); } - return consumedMsgInfo; - } - - @Override - public boolean reciveMsg(int jobId, Properties props, Logger log) { - return super.reciveMsg(jobId, props, log); + return executeType; } + public abstract String[] getMsg(int jobId,Properties props, Logger log,String ... params) ; } diff --git a/dss-appconn/appconns/dss-eventchecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/eventchecker/service/DefaultEventcheckReceiver.java b/dss-appconn/appconns/dss-eventchecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/eventchecker/service/DefaultEventcheckReceiver.java index 7689a997fc..4dd895ed4c 100644 --- a/dss-appconn/appconns/dss-eventchecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/eventchecker/service/DefaultEventcheckReceiver.java +++ b/dss-appconn/appconns/dss-eventchecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/eventchecker/service/DefaultEventcheckReceiver.java @@ -18,113 +18,73 @@ -import org.apache.commons.lang3.time.DateFormatUtils; -import org.slf4j.Logger; -import java.text.DateFormat; -import java.text.ParseException; -import java.text.SimpleDateFormat; -import java.util.Arrays; -import java.util.Date; +import org.slf4j.Logger; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; import java.util.Properties; public class DefaultEventcheckReceiver extends AbstractEventCheckReceiver { - String todayStartTime; - String todayEndTime; - String allStartTime; - String allEndTime; - String nowStartTime; public DefaultEventcheckReceiver(Properties props) { - initECParams(props); - initReceiverTimes(); - } - - private void initReceiverTimes(){ - todayStartTime = DateFormatUtils.format(new Date(), "yyyy-MM-dd 00:00:00"); - todayEndTime = DateFormatUtils.format(new Date(), "yyyy-MM-dd 23:59:59"); - allStartTime = DateFormatUtils.format(new Date(), "10000-01-01 00:00:00"); - allEndTime = DateFormatUtils.format(new Date(), "9999-12-31 23:59:59"); - nowStartTime = DateFormatUtils.format(new Date(), "yyyy-MM-dd HH:mm:ss"); + super(props); } - + /** + * Consistent entrance to consumer message + */ @Override - public boolean reciveMsg(int jobId, Properties props, Logger log) { - boolean result = false; - try{ - String lastMsgId = getOffset(jobId,props,log); - String[] executeType = createExecuteType(jobId,props,log,lastMsgId); - log.info("event receiver executeType[]:{},{},{},{},{}",executeType[0],executeType[1],executeType[2],executeType[3],executeType[4]); - if(executeType!=null && executeType.length ==5){ - String[] consumedMsgInfo = getMsg(props, log,executeType); - if(consumedMsgInfo!=null && consumedMsgInfo.length == 4){ - result = updateMsgOffset(jobId,props,log,consumedMsgInfo,lastMsgId); - } - }else{ - log.error("executeType error {} " + Arrays.toString(executeType)); - return result; - } - }catch (Exception e){ - log.error("EventChecker failed to receive the message {}" + e); - return result; + public String[] getMsg(int jobId,Properties props, Logger log,String ... params){ + boolean useRunDate=Boolean.parseBoolean(params[3]); + String sqlForReadTMsg; + if(useRunDate){ + sqlForReadTMsg ="SELECT * FROM event_queue WHERE topic=? AND msg_name=? AND send_time >=? AND send_time <=? AND msg_id >? AND run_date =?ORDER BY msg_id ASC LIMIT 1"; + } else{ + sqlForReadTMsg="SELECT * FROM event_queue WHERE topic=? AND msg_name=? AND send_time >=? AND send_time <=? AND msg_id >? ORDER BY msg_id ASC LIMIT 1"; } - return result; - } - private String[] createExecuteType(int jobId, Properties props, Logger log,String lastMsgId){ - boolean receiveTodayFlag = (null != receiveToday && "true".equals(receiveToday.trim().toLowerCase())); - boolean afterSendFlag = (null != afterSend && "true".equals(afterSend.trim().toLowerCase())); - //只有receiveTodayFlag为true时，useRunDateFlag才有意义。 - Boolean useRunDateFlag = receiveTodayFlag && (null == useRunDate || "true".equalsIgnoreCase(useRunDate.trim())); - String[] executeType = null; + PreparedStatement pstmt = null; + Connection msgConn = null; + ResultSet rs = null; + String[] consumedMsgInfo = null; try { - if (receiveTodayFlag && !useRunDateFlag) { - if (afterSendFlag) { - executeType = new String[]{nowStartTime, todayEndTime, lastMsgId, useRunDateFlag.toString(), runDate}; - } else { - executeType = new String[]{todayStartTime, todayEndTime, lastMsgId, useRunDateFlag.toString(), runDate}; - } - } else { - if (afterSendFlag) { - executeType = new String[]{nowStartTime, allEndTime, lastMsgId, useRunDateFlag.toString(), runDate}; - } else { - executeType = new String[]{allStartTime, allEndTime, lastMsgId, useRunDateFlag.toString(), runDate}; - } + msgConn = getEventCheckerConnection(props,log); + pstmt = msgConn.prepareCall(sqlForReadTMsg); + pstmt.setString(1, topic); + pstmt.setString(2, msgName); + pstmt.setString(3, params[0]); + pstmt.setString(4, params[1]); + pstmt.setString(5, params[2]); + if(useRunDate){ + log.info("use run_date, run_date:{}", params[4]); + pstmt.setString(6,params[4]); } - }catch(Exception e){ - log.error("create executeType failed {}" + e); - } - return executeType; - } - - private void waitForTime(Logger log,Long waitTime){ - String waitForTime = wait_for_time; - String formatWaitForTime = DateFormatUtils.format(new Date(),"yyyy-MM-dd " + waitForTime + ":00"); - DateFormat fmt =new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); - Date targetWaitTime = new Date(); - try { - targetWaitTime = fmt.parse(formatWaitForTime); - } catch (ParseException e) { - log.error("parse date failed {}" + e); - } + log.info("param {} StartTime: " + params[0] + ", EndTime: " + params[1] + + ", Topic: " + topic + ", MessageName: " + msgName + ", LastMessageID: " + params[2]); + rs = pstmt.executeQuery(); - log.info("It will success at a specified time: " + targetWaitTime); - long wt = targetWaitTime.getTime() - System.currentTimeMillis(); - if(wt > 0){ - //wt must less than wait.time - if(wt <= waitTime){ - log.info("EventChecker will wait "+ wt + " milliseconds before starting execution"); - try { - Thread.sleep(wt); - } catch (InterruptedException e) { - throw new RuntimeException("EventChecker throws an exception during the waiting time {}"+e); + while (rs.next()) { + consumedMsgInfo = new String[4]; + String[] msgKey = new String[]{"msg_id", "msg_name", "sender", "msg"}; + for (int i = 0; i < msgKey.length; i++) { + try { + consumedMsgInfo[i] = rs.getString(msgKey[i]); + } catch (SQLException e) { + throw new RuntimeException("Error while reading data from ResultSet", e); + } } - }else{ - throw new RuntimeException("The waiting time from Job starttime to wait.for.time"+ wt +"(ms) greater than wait.time , unreasonable setting！"); + consumedMsgInfo[2] = receiver; } - }else{ - log.info("EventChecker has reached the specified time"); + } catch (SQLException e) { + log.error("EventChecker failed to receive message",e); + throw new RuntimeException(e); + } finally { + closeQueryStmt(pstmt, log); + closeConnection(msgConn, log); + closeQueryRef(rs, log); } + return consumedMsgInfo; } } diff --git a/dss-appconn/appconns/dss-eventchecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/eventchecker/service/EventCheckerService.java b/dss-appconn/appconns/dss-eventchecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/eventchecker/service/EventCheckerService.java index 4583b6ea89..cd6bb36818 100644 --- a/dss-appconn/appconns/dss-eventchecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/eventchecker/service/EventCheckerService.java +++ b/dss-appconn/appconns/dss-eventchecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/eventchecker/service/EventCheckerService.java @@ -16,6 +16,7 @@ package com.webank.wedatasphere.dss.appconn.eventchecker.service; +import com.webank.wedatasphere.dss.appconn.eventchecker.entity.EventChecker; import org.slf4j.Logger; import java.util.Properties; @@ -36,7 +37,17 @@ public static EventCheckerService getInstance() { public boolean sendMsg(int jobId, Properties props, Logger log) { if (props != null) { - return new EventCheckSender(props).sendMsg(jobId, props, log); + String channelType = props.containsKey(EventChecker.CHANNEL_TYPE) ? props.getProperty(EventChecker.CHANNEL_TYPE) + : "DSS"; + AbstractEventCheck eventCheck; + if("DSS".equalsIgnoreCase(channelType)) { + eventCheck = new EventCheckSender(props); + log.info("this is dss send task"); + }else { + eventCheck = new HttpEventCheckSender(props); + log.info("this is kgas receive task"); + } + return eventCheck.sendMsg(jobId, props, log); } else { log.error("create EventCheckSender failed {}"); return false; @@ -52,7 +63,17 @@ public boolean sendMsg(int jobId, Properties props, Logger log) { */ public boolean reciveMsg(int jobId, Properties props, Logger log) { if (props != null) { - return new DefaultEventcheckReceiver(props).reciveMsg(jobId, props, log); + String channelType = props.containsKey(EventChecker.CHANNEL_TYPE) ? props.getProperty(EventChecker.CHANNEL_TYPE) + : "DSS"; + AbstractEventCheck eventCheck; + if("DSS".equalsIgnoreCase(channelType)){ + eventCheck= new DefaultEventcheckReceiver(props); + log.info("this is dss receive task"); + }else{ + eventCheck = new HttpEventcheckerReceiver(props); + log.info("this is kgas receive task"); + } + return eventCheck.reciveMsg(jobId, props, log); } else { log.error("create EventCheckSender failed {}"); return false; diff --git a/dss-appconn/appconns/dss-eventchecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/eventchecker/service/HttpEventCheckSender.java b/dss-appconn/appconns/dss-eventchecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/eventchecker/service/HttpEventCheckSender.java new file mode 100644 index 0000000000..f13f6bbe6c --- /dev/null +++ b/dss-appconn/appconns/dss-eventchecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/eventchecker/service/HttpEventCheckSender.java @@ -0,0 +1,97 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.eventchecker.service; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.JsonSyntaxException; +import com.webank.wedatasphere.dss.appconn.eventchecker.entity.HttpMsgSendRequest; +import com.webank.wedatasphere.dss.appconn.eventchecker.entity.HttpMsgSendResponse; +import com.webank.wedatasphere.dss.appconn.eventchecker.utils.EventCheckerHttpUtils; +import okhttp3.Response; +import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; +import java.util.Properties; + +public class HttpEventCheckSender extends AbstractEventCheck { + private static final String HTTP_EVENT_SIGN_KEY = "msg.eventchecker.http.sign.key"; + private static final String HTTP_EVENT_KGAS_SEND_URL = "msg.eventchecker.http.kgas.send.url"; + + public HttpEventCheckSender(Properties props) { + initECParams(props); + } + + @Override + public boolean sendMsg(int jobId, Properties props, Logger log) { + boolean result = false; + String url = props.getProperty(HTTP_EVENT_KGAS_SEND_URL); + String key = props.getProperty(HTTP_EVENT_SIGN_KEY); + String timestamp = String.valueOf(System.currentTimeMillis()); + String sign = EventCheckerHttpUtils.calculateSign(key, timestamp); + Map header = new HashMap<>(); + header.put("sign", sign); + header.put("timestamp", timestamp); + Gson gson = new GsonBuilder().create(); + Map msgBody = new HashMap<>(); + long currentTimeMillis = System.currentTimeMillis(); + int secondsSinceEpoch = (int) (currentTimeMillis / 1000); + String msgId = String.valueOf(secondsSinceEpoch); + if (StringUtils.isNoneBlank(msg)) { + try { + msgBody = gson.fromJson(msg, Map.class); + }catch (JsonSyntaxException jsonSyntaxException){ + throw new RuntimeException("msg.body格式有误，请输入标准的json格式"+ jsonSyntaxException.getMessage()); + } + } + HttpMsgSendRequest message = new HttpMsgSendRequest(sender, topic, msgName, runDate, msgId, msgBody); + String responseBody = null; + String messageJson = gson.toJson(message); + String requestStr = EventCheckerHttpUtils.requestToString(url, "POST", header, null, messageJson); + log.info("try to send http message,request{}",requestStr); + try (Response response = EventCheckerHttpUtils.post(url, header, null, messageJson)) { + HttpMsgSendResponse msgSendResponse; + try { + responseBody = response.body().string(); + msgSendResponse = gson.fromJson(responseBody, + HttpMsgSendResponse.class); + } catch (Exception e){ + throw new RuntimeException("请求KGAS失败，详情：" + responseBody); + } + + int reCode = msgSendResponse.getRetCode(); + if (reCode == 0) { + result = true; + log.info("send successfully.jobId:{}",jobId); + } else { + String errorMsg ="信号发送失败，详情："+ responseBody; + log.error("send failed,response:{}", errorMsg); + throw new RuntimeException(errorMsg); + } + + } catch (IOException e) { + String errorMsg = responseBody != null ? responseBody : ""; + + throw new RuntimeException(errorMsg,e); + } + + return result; + } +} diff --git a/dss-appconn/appconns/dss-eventchecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/eventchecker/service/HttpEventcheckerReceiver.java b/dss-appconn/appconns/dss-eventchecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/eventchecker/service/HttpEventcheckerReceiver.java new file mode 100644 index 0000000000..3fc71645b8 --- /dev/null +++ b/dss-appconn/appconns/dss-eventchecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/eventchecker/service/HttpEventcheckerReceiver.java @@ -0,0 +1,105 @@ +package com.webank.wedatasphere.dss.appconn.eventchecker.service; + + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.webank.wedatasphere.dss.appconn.eventchecker.entity.HttpMsgReceiveRequest; +import com.webank.wedatasphere.dss.appconn.eventchecker.entity.HttpMsgReceiveResponse; +import com.webank.wedatasphere.dss.appconn.eventchecker.utils.EventCheckerHttpUtils; +import okhttp3.Response; +import org.slf4j.Logger; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; +import java.util.Properties; + +/** + * Author: xlinliu + * Date: 2024/8/1 + */ +public class HttpEventcheckerReceiver extends AbstractEventCheckReceiver{ + private static final String HTTP_EVENT_SIGN_KEY = "msg.eventchecker.http.sign.key"; + private static final String HTTP_EVENT_KGAS_RECEIVE_URL = "msg.eventchecker.http.kgas.receive.url"; + + + + public HttpEventcheckerReceiver(Properties props) { + super(props); + } + + @Override + public String[] getMsg(int jobId,Properties props, Logger log, String... params) { + String url=props.getProperty(HTTP_EVENT_KGAS_RECEIVE_URL); + String key=props.getProperty(HTTP_EVENT_SIGN_KEY); + String timestamp=String.valueOf( System.currentTimeMillis()); + String sign = EventCheckerHttpUtils.calculateSign(key,timestamp); + Map header = new HashMap<>(); + header.put("sign", sign); + header.put("timestamp", timestamp); + //params = new String[]{nowStartTime, todayEndTime, lastMsgId, useRunDateFlag.toString(), runDate} + boolean useRunDate=Boolean.parseBoolean(params[3]); + Long lastMsgId = Long.valueOf(params[2]); + String runDate=params[4]; + Gson gson = new GsonBuilder().create(); + boolean receiveTodayFlag = (null != receiveToday && "true".equalsIgnoreCase(receiveToday.trim())); + HttpMsgReceiveRequest message = new HttpMsgReceiveRequest(receiver, topic, msgName, runDate, receiveTodayFlag, useRunDate); + String[] consumedMsgInfo = null; + String responseBody = null; + String messageJson = gson.toJson(message); + try (Response response = EventCheckerHttpUtils.post(url, header, null, messageJson)) { + HttpMsgReceiveResponse msgReceiveResponse; + try { + responseBody = response.body().string(); + msgReceiveResponse = gson.fromJson(responseBody, + HttpMsgReceiveResponse.class); + }catch (Exception e){ + throw new RuntimeException("请求KGAS失败，详情：" + responseBody); + } + int reCode = msgReceiveResponse.getRetCode(); + if (reCode == 0 ) { + log.info("receive request successfully.jobId:{}",jobId); + if("SUCCESS".equalsIgnoreCase(msgReceiveResponse.getStatus())) { + log.info("receive successfully,now try to parse message.jobId:{}",jobId); + String msgBodyJson = gson.toJson(msgReceiveResponse.getMsgBody()); + Long msgId = msgReceiveResponse.getMsgId(); + if (msgId > lastMsgId) { + log.info("receive new message successfully.jobId:{}", jobId); + //{"msg_id", "msg_name", "receiver", "msg"}; + consumedMsgInfo = new String[]{msgId.toString(), msgName, receiver, msgBodyJson}; + } + }else if ("FAILED".equalsIgnoreCase(msgReceiveResponse.getStatus())){ + log.error("receive failed,response:{}", responseBody); + String errorMsg = "信号接收失败。详情："+responseBody; + throw new RuntimeException(errorMsg); + }else{ + log.info("receive failed,will try again later ,retCode:{},status:{},message:{}", + msgReceiveResponse.getRetCode() + ,msgReceiveResponse.getStatus() + ,msgReceiveResponse.getMessage() + ); + consumedMsgInfo = null; + } + } else if (reCode == 9998) { + log.info("message has not send to third system,will try again later ,retCode:{},status:{},message:{}", + msgReceiveResponse.getRetCode() + ,msgReceiveResponse.getStatus() + ,msgReceiveResponse.getMessage() + ); + consumedMsgInfo = null; + } else { + String requestStr = EventCheckerHttpUtils.requestToString(url, "POST", header, null, messageJson); + log.error("receive failed,request:{}", requestStr); + log.error("receive failed,response:{}", responseBody); + String errorMsg = "信号接收失败。详情："+responseBody; + throw new RuntimeException(errorMsg); + } + return consumedMsgInfo; + } catch (IOException e) { + String errorMsg = responseBody != null ? responseBody: ""; + throw new RuntimeException(errorMsg,e); + } + } + + +} diff --git a/dss-appconn/appconns/dss-eventchecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/eventchecker/utils/EventCheckerHttpUtils.java b/dss-appconn/appconns/dss-eventchecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/eventchecker/utils/EventCheckerHttpUtils.java new file mode 100644 index 0000000000..4aaaae70a2 --- /dev/null +++ b/dss-appconn/appconns/dss-eventchecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/eventchecker/utils/EventCheckerHttpUtils.java @@ -0,0 +1,158 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.eventchecker.utils; + + +import cn.hutool.crypto.digest.DigestUtil; +import okhttp3.*; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import java.io.IOException; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +public class EventCheckerHttpUtils { + + private static final Logger logger = LoggerFactory.getLogger(EventCheckerHttpUtils.class); + + /** + * 发送get请求 + * + * @param url 地址 + * @param params 参数 + * @return 请求结果 + */ + public static Response get(String url, Map headerMap, Map params) throws IOException { + return request("GET", url, headerMap, params,null); + } + + /** + * 发送post请求 + * + * @param url 地址 + * @param params 参数 + * @return 请求结果 + */ + public static Response post(String url, Map headerMap, Map params, String jsonBody) throws IOException { + + return request("POST",url, headerMap, params,jsonBody); + } + + /** + * 发送http请求 + * + * @param method 请求方法 + * @param url 地址 + * @param headerMap 可以为null + * @param params 参数map，可以为null + * @param jsonBody json body，可以为null + * @return 请求结果 + */ + public static Response request(String method, String url, Map headerMap, + Map params, String jsonBody) throws IOException { + + if (method == null) { + throw new RuntimeException("请求方法不能为空"); + } + + if (url == null) { + throw new RuntimeException("url不能为空"); + } + + HttpUrl.Builder httpBuilder = HttpUrl.parse(url).newBuilder(); + + if (params != null) { + for (Map.Entry param : params.entrySet()) { + httpBuilder.addQueryParameter(param.getKey(), param.getValue()); + } + } + Headers headers = setHeaderParams(headerMap); + RequestBody body = jsonBody == null ? null : RequestBody.Companion.create(jsonBody, + MediaType.Companion.parse("application/json")); + Request request = new Request.Builder() + .url(httpBuilder.build()) + .method(method, body) + .headers(headers) + .build(); + + + OkHttpClient client = new OkHttpClient.Builder() + .connectTimeout(10, TimeUnit.SECONDS) + .writeTimeout(20, TimeUnit.SECONDS) + .readTimeout(20, TimeUnit.SECONDS) + .build(); + Response response = client.newCall(request).execute(); + logger.info("eventChecker http request successfully,url：{},retCode:{}", url, response.code()); + return response; + } + + + + /** + * 添加headers + * + * @param headerParams + * @return + */ + private static Headers setHeaderParams(Map headerParams) { + Headers.Builder headersbuilder = new Headers.Builder(); + if (headerParams != null) { + headerParams.forEach(headersbuilder::add); + } + return headersbuilder.build(); + + } + + // 鉴权方法 + public static String calculateSign( String key,String timestamp) { + + // 计算签名 + return DigestUtil.sha256Hex(key + timestamp); + } + public static String requestToString(String url, String method, Map headerMap, + Map params, + String jsonBody){ + HttpUrl.Builder httpBuilder = HttpUrl.parse(url).newBuilder(); + + if (params != null) { + for (Map.Entry param : params.entrySet()) { + httpBuilder.addQueryParameter(param.getKey(), param.getValue()); + } + } + Headers headers = setHeaderParams(headerMap); + RequestBody body = jsonBody == null ? null : RequestBody.Companion.create(jsonBody, + MediaType.Companion.parse("application/json")); + Request request = new Request.Builder() + .url(httpBuilder.build()) + .method(method, body) + .headers(headers) + .build(); + + + StringBuilder sb = new StringBuilder(); + sb.append("Request: ").append(request.method()).append(" ").append(request.url()).append("\n"); + sb.append("Headers: ").append(request.headers()).append("\n"); + sb.append("Body: ").append(jsonBody); + return sb.toString(); + } + + public static void main(String[] args) throws IOException { + Response response = get("http://10.107.116.246:8088/api/rest_j/v1/dss/scriptis/proxy/addUserProxy", null, null); + System.out.println(response.body().string()); + } + +} diff --git a/dss-appconn/appconns/dss-schedulis-appconn/pom.xml b/dss-appconn/appconns/dss-schedulis-appconn/pom.xml index ae044adb08..0cd8349a5d 100644 --- a/dss-appconn/appconns/dss-schedulis-appconn/pom.xml +++ b/dss-appconn/appconns/dss-schedulis-appconn/pom.xml @@ -21,7 +21,7 @@ dss com.webank.wedatasphere.dss - 1.1.0.20-SNAPSHOT + 1.10.0-SNAPSHOT ../../../pom.xml 4.0.0 diff --git a/dss-appconn/appconns/dss-schedulis-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/schedulis/constant/AzkabanConstant.java b/dss-appconn/appconns/dss-schedulis-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/schedulis/constant/AzkabanConstant.java index 821c3a6d0a..c08b0ada54 100644 --- a/dss-appconn/appconns/dss-schedulis-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/schedulis/constant/AzkabanConstant.java +++ b/dss-appconn/appconns/dss-schedulis-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/schedulis/constant/AzkabanConstant.java @@ -29,5 +29,8 @@ public class AzkabanConstant { public final static String FLOW_CONTEXT_ID = "wds.linkis.flow.contextID="; public final static String LINKIS_VERSION = "linkis.version"; public final static String JOB_COMMENT = "comment"; + public final static String AUTO_DISABLED = "auto.disabled"; + + public final static String WTSS_PREFIX = "wtss."; } diff --git a/dss-appconn/appconns/dss-schedulis-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/schedulis/conversion/AzkabanWorkflowToRelSynchronizer.java b/dss-appconn/appconns/dss-schedulis-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/schedulis/conversion/AzkabanWorkflowToRelSynchronizer.java index 31cbb70e0e..a4714db747 100644 --- a/dss-appconn/appconns/dss-schedulis-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/schedulis/conversion/AzkabanWorkflowToRelSynchronizer.java +++ b/dss-appconn/appconns/dss-schedulis-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/schedulis/conversion/AzkabanWorkflowToRelSynchronizer.java @@ -56,7 +56,7 @@ public class AzkabanWorkflowToRelSynchronizer implements WorkflowToRelSynchroniz private DSSToRelConversionOperation dssToRelConversionOperation; //匹配wtss返回的错误信息 private static final Pattern ERROR_PATTERN = Pattern.compile("(?<=Error uploading project properties)[\\s\\S]+.job"); - private static final int SCHEDULIS_MAX_SIZE = 250; + private static final int SCHEDULIS_MAX_SIZE = 1024; public void init() { String baseUrl = dssToRelConversionOperation.getConversionService().getAppInstance().getBaseUrl(); @@ -139,7 +139,7 @@ private String dealSchedulisErrorMsg(String errorMsg) { Matcher matcher = ERROR_PATTERN.matcher(errorMsg); if (matcher.find() && matcher.group().length() >= SCHEDULIS_MAX_SIZE) { errorMsg = "wokflow name " + matcher.group().split("/")[1] + " is to long, please abide the rules of schedulis: projectName + workflowName*3 + 12 <= 250 " + - "(工作流名称太长，需要满足规则 项目名长度 + 工作流长度*3 + 12 <= 250)"; } + "(工作流名称太长，需要满足规则 项目名长度 + 工作流长度*3 + 12 <= 1024)"; } return errorMsg; } } diff --git a/dss-appconn/appconns/dss-schedulis-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/schedulis/conversion/WorkflowToAzkbanNodeRelConverter.java b/dss-appconn/appconns/dss-schedulis-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/schedulis/conversion/WorkflowToAzkbanNodeRelConverter.java index 8e24ac5801..6d7d739ced 100644 --- a/dss-appconn/appconns/dss-schedulis-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/schedulis/conversion/WorkflowToAzkbanNodeRelConverter.java +++ b/dss-appconn/appconns/dss-schedulis-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/schedulis/conversion/WorkflowToAzkbanNodeRelConverter.java @@ -20,6 +20,7 @@ import com.webank.wedatasphere.dss.appconn.schedulis.constant.AzkabanConstant; import com.webank.wedatasphere.dss.appconn.schedulis.entity.AzkabanWorkflow; import com.webank.wedatasphere.dss.appconn.schedulis.linkisjob.LinkisJobConverter; +import com.webank.wedatasphere.dss.appconn.schedulis.linkisjob.WTSSJobConverter; import com.webank.wedatasphere.dss.common.entity.Resource; import com.webank.wedatasphere.dss.common.exception.DSSErrorException; import com.webank.wedatasphere.dss.common.utils.ClassUtils; @@ -78,6 +79,11 @@ private void writeNodeToJobLocal(WorkflowNode workflowNode, String storePath) th FileUtils.forceMkdir(jobDirFile); File jobFile = new File(storePath,workflowNode.getName() + AzkabanConstant.AZKABAN_JOB_SUFFIX); jobFile.createNewFile(); + if(workflowNode.getNodeType()!=null && workflowNode.getNodeType().startsWith(AzkabanConstant.WTSS_PREFIX)){ + nodeConverter = ClassUtils.getInstanceOrDefault(NodeConverter.class, new WTSSJobConverter()); + }else{ + nodeConverter = ClassUtils.getInstanceOrDefault(NodeConverter.class, new LinkisJobConverter()); + } String nodeString = nodeConverter.conversion(workflowNode); os = FileUtils.openOutputStream(jobFile,true); os.write(nodeString.getBytes()); diff --git a/dss-appconn/appconns/dss-schedulis-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/schedulis/conversion/WorkflowToAzkbanRelConverter.java b/dss-appconn/appconns/dss-schedulis-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/schedulis/conversion/WorkflowToAzkbanRelConverter.java index a413d9d933..d949ec7e92 100644 --- a/dss-appconn/appconns/dss-schedulis-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/schedulis/conversion/WorkflowToAzkbanRelConverter.java +++ b/dss-appconn/appconns/dss-schedulis-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/schedulis/conversion/WorkflowToAzkbanRelConverter.java @@ -105,9 +105,6 @@ private String getFlowResourceStringPrefix(String projectStorePath, String store private void writeFlowPropertiesToLocal(AzkabanWorkflow flow) throws DSSErrorException { List> flowProperties = flow.getFlowProperties(); - if (flowProperties == null || flowProperties.isEmpty()) { - return; - } FileOutputStream os = null; try { String storePath = flow.getStorePath(); @@ -115,9 +112,11 @@ private void writeFlowPropertiesToLocal(AzkabanWorkflow flow) throws DSSErrorExc flowPrpsFile.createNewFile(); os = FileUtils.openOutputStream(flowPrpsFile, true); StringBuilder stringBuilder = new StringBuilder(); - flowProperties.forEach(p -> p.forEach((k, v) -> { - stringBuilder.append(AzkabanConstant.LINKIS_FLOW_VARIABLE_KEY + k + "=" + v + "\n"); - })); + if(flowProperties != null) { + flowProperties.forEach(p -> p.forEach((k, v) -> { + stringBuilder.append(AzkabanConstant.LINKIS_FLOW_VARIABLE_KEY + k + "=" + v + "\n"); + })); + } // update by peaceWong add contextID to Flow properties String contextID = flow.getContextID(); if (StringUtils.isNotBlank(contextID)) { diff --git a/dss-appconn/appconns/dss-schedulis-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/schedulis/linkisjob/LinkisJob.java b/dss-appconn/appconns/dss-schedulis-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/schedulis/linkisjob/LinkisJob.java index 76e7ef8858..2d269872f1 100644 --- a/dss-appconn/appconns/dss-schedulis-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/schedulis/linkisjob/LinkisJob.java +++ b/dss-appconn/appconns/dss-schedulis-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/schedulis/linkisjob/LinkisJob.java @@ -23,6 +23,8 @@ public class LinkisJob { private String type; private String linkistype; private String proxyUser; + + private String autoDisabled; private String dependencies; private Map conf; private String command; @@ -60,6 +62,14 @@ public void setProxyUser(String proxyUser) { this.proxyUser = proxyUser; } + public String getAutoDisabled() { + return autoDisabled; + } + + public void setAutoDisabled(String autoDisabled) { + this.autoDisabled = autoDisabled; + } + public String getDependencies() { return dependencies; } diff --git a/dss-appconn/appconns/dss-schedulis-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/schedulis/linkisjob/LinkisJobConverter.java b/dss-appconn/appconns/dss-schedulis-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/schedulis/linkisjob/LinkisJobConverter.java index f175b6ca3c..8f569be455 100644 --- a/dss-appconn/appconns/dss-schedulis-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/schedulis/linkisjob/LinkisJobConverter.java +++ b/dss-appconn/appconns/dss-schedulis-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/schedulis/linkisjob/LinkisJobConverter.java @@ -70,6 +70,7 @@ private String convertJobToString(LinkisJob job){ map.put(WorkflowConstant.PROXY_USER,job.getProxyUser()); map.put(AzkabanConstant.JOB_COMMAND,job.getCommand()); map.put(AzkabanConstant.JOB_COMMENT,job.getComment()); + map.put(AzkabanConstant.AUTO_DISABLED,job.getAutoDisabled()); Map labels = new HashMap<>(1); labels.put("route", SchedulerConf.JOB_LABEL.getValue()); map.put(AzkabanConstant.JOB_LABELS, DSSCommonUtils.COMMON_GSON.toJson(labels)); @@ -114,7 +115,22 @@ private void convertConfiguration(WorkflowNode workflowNode, LinkisJob job){ configuration.forEach((k,v)-> { if(null!=v) { v.forEach((k2, v2) -> { - if(null !=v2) {job.getConf().put(confprefix + k + "." + k2, v2.toString());} + if(v2!=null) { + String vStr; + if (v2 instanceof Number) { + Number numValue = (Number) v2; + vStr = numValue.longValue() == numValue.doubleValue() ? + String.valueOf(numValue.longValue()) : + numValue.toString(); + } else { + vStr = v2.toString(); + } + if (AzkabanConstant.AUTO_DISABLED.equals(k2) ) { + job.setAutoDisabled(vStr); + } else { + job.getConf().put(confprefix + k + "." + k2, vStr); + } + } }); } }); diff --git a/dss-appconn/appconns/dss-schedulis-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/schedulis/linkisjob/WTSSJobConverter.java b/dss-appconn/appconns/dss-schedulis-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/schedulis/linkisjob/WTSSJobConverter.java new file mode 100644 index 0000000000..0705957026 --- /dev/null +++ b/dss-appconn/appconns/dss-schedulis-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/schedulis/linkisjob/WTSSJobConverter.java @@ -0,0 +1,149 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.appconn.schedulis.linkisjob; + +import com.webank.wedatasphere.dss.appconn.scheduler.utils.SchedulerConf; +import com.webank.wedatasphere.dss.appconn.schedulis.conf.AzkabanConf; +import com.webank.wedatasphere.dss.appconn.schedulis.constant.AzkabanConstant; +import com.webank.wedatasphere.dss.appconn.schedulis.conversion.NodeConverter; +import com.webank.wedatasphere.dss.common.utils.DSSCommonUtils; +import com.webank.wedatasphere.dss.workflow.core.constant.WorkflowConstant; +import com.webank.wedatasphere.dss.workflow.core.entity.WorkflowNode; +import org.apache.commons.lang.StringUtils; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class WTSSJobConverter implements NodeConverter { + + private LinkisJobTuning[] linkisJobTunings; + + public WTSSJobConverter(){ + this.linkisJobTunings = new LinkisJobTuning[]{new AzkabanSubFlowJobTuning()}; + } + + @Override + public String conversion(WorkflowNode workflowNode){ + return baseConversion(workflowNode); + } + + private String baseConversion(WorkflowNode workflowNode){ + LinkisJob job = new LinkisJob(); + job.setConf(new HashMap<>()); + job.setName(workflowNode.getName()); + job.setComment(workflowNode.getDSSNode().getDesc()); + convertHead(workflowNode,job); + convertDependencies(workflowNode,job); + convertProxyUser(workflowNode,job); + convertConfiguration(workflowNode,job); + convertJobCommand(workflowNode,job); + Arrays.stream(linkisJobTunings).forEach(t ->{ + if(t.ifJobCantuning(workflowNode.getNodeType())) { + t.tuningJob(job); + } + }); + return convertJobToString(job); + } + + private String convertJobToString(LinkisJob job){ + HashMap map = new HashMap<>(8); + map.put(AzkabanConstant.LINKIS_VERSION, AzkabanConf.LINKIS_VERSION.getValue()); + map.put(AzkabanConstant.JOB_TYPE,job.getType()); + map.put(AzkabanConstant.LINKIS_TYPE,job.getLinkistype()); + map.put(AzkabanConstant.ZAKABAN_DEPENDENCIES_KEY,job.getDependencies()); + map.put(WorkflowConstant.PROXY_USER,job.getProxyUser()); + map.put(AzkabanConstant.JOB_COMMAND,job.getCommand()); + map.put(AzkabanConstant.JOB_COMMENT,job.getComment()); + map.put(AzkabanConstant.AUTO_DISABLED,job.getAutoDisabled()); + Map labels = new HashMap<>(1); + labels.put("route", SchedulerConf.JOB_LABEL.getValue()); + map.put(AzkabanConstant.JOB_LABELS, DSSCommonUtils.COMMON_GSON.toJson(labels)); + map.putAll(job.getConf()); + StringBuilder stringBuilder = new StringBuilder(); + map.forEach((k,v)->{ + if(v != null) { + //for value contains "\n" + v = v.replace("\n", ";"); + stringBuilder.append(k).append("=").append(v).append("\n"); + } + }); + return stringBuilder.toString(); + } + + private void convertHead(WorkflowNode workflowNode, LinkisJob job){ + job.setType("eventchecker"); + job.setLinkistype(workflowNode.getNodeType()); + } + + private void convertDependencies(WorkflowNode workflowNode, LinkisJob job){ + List dependencys = workflowNode.getDSSNode().getDependencys(); + if(dependencys != null && !dependencys.isEmpty()) { + StringBuilder dependencies = new StringBuilder(); + dependencys.forEach(d -> dependencies.append(d).append(",")); + job.setDependencies(dependencies.substring(0,dependencies.length()-1)); + } + } + + private void convertProxyUser(WorkflowNode workflowNode, LinkisJob job){ + String userProxy = workflowNode.getDSSNode().getUserProxy(); + if(!StringUtils.isEmpty(userProxy)) { + job.setProxyUser(userProxy); + } + } + + private void convertConfiguration(WorkflowNode workflowNode, LinkisJob job){ + Map params = workflowNode.getDSSNode().getParams(); + if (params != null && !params.isEmpty()) { + Map> configuration = (Map>) params.get("configuration"); + configuration.forEach((k,v)-> { + if(null!=v) { + v.forEach((k2, v2) -> { + if(AzkabanConstant.AUTO_DISABLED.equals(k2) && null !=v2){job.setAutoDisabled(v2.toString());} + else if(null !=v2) { + String vStr; + if (v2 instanceof Number) { + Number numValue = (Number) v2; + vStr = numValue.longValue() == numValue.doubleValue() ? + String.valueOf(numValue.longValue()) : + numValue.toString(); + } else { + vStr = v2.toString(); + } + if("only.receive.today".equals(k2)){ + job.getConf().put("msg.rece.today", vStr); + }else if("max.receive.hours".equals(k2)){ + job.getConf().put("wait.time", vStr); + }else{ + job.getConf().put(k2, vStr); + } + } + }); + } + }); + } + } + + private void convertJobCommand(WorkflowNode workflowNode, LinkisJob job){ + Map jobContent = workflowNode.getDSSNode().getJobContent(); + if(jobContent != null) { + jobContent.remove("jobParams"); + job.setCommand(DSSCommonUtils.COMMON_GSON.toJson(jobContent)); + } + } +} diff --git a/dss-appconn/appconns/dss-schedulis-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/schedulis/operation/SchedulisProjectSearchOperation.java b/dss-appconn/appconns/dss-schedulis-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/schedulis/operation/SchedulisProjectSearchOperation.java index b15e20d308..7514853d6d 100644 --- a/dss-appconn/appconns/dss-schedulis-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/schedulis/operation/SchedulisProjectSearchOperation.java +++ b/dss-appconn/appconns/dss-schedulis-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/schedulis/operation/SchedulisProjectSearchOperation.java @@ -28,6 +28,7 @@ public ProjectResponseRef searchProject(RefProjectContentRequestRef.RefProjectCo params.put("project", requestRef.getProjectName()); params.put("ajax", "fetchprojectflows"); try { + logger.info("request url from Schedulis is: {}.", queryUrl); String responseBody = SchedulisHttpUtils.getHttpGetResult(queryUrl, params, ssoRequestOperation, requestRef.getWorkspace()); logger.info("responseBody from Schedulis is: {}.", responseBody); Map map = DSSCommonUtils.COMMON_GSON.fromJson(responseBody, new TypeToken>(){}.getType()); diff --git a/dss-appconn/appconns/dss-schedulis-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/schedulis/operation/SchedulisProjectUpdateOperation.java b/dss-appconn/appconns/dss-schedulis-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/schedulis/operation/SchedulisProjectUpdateOperation.java index 5b4d9a4417..b52919478a 100644 --- a/dss-appconn/appconns/dss-schedulis-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/schedulis/operation/SchedulisProjectUpdateOperation.java +++ b/dss-appconn/appconns/dss-schedulis-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/schedulis/operation/SchedulisProjectUpdateOperation.java @@ -43,9 +43,9 @@ public void init() { @Override public ResponseRef updateProject(ProjectUpdateRequestRef.ProjectUpdateRequestRefImpl projectRef) { - if (CollectionUtils.isNotEmpty(projectRef.getDSSProjectPrivilege().getReleaseUsers())) { - // 先校验运维用户是否存在于 Schedulis，如果不存在，则不能成功创建工程。 - projectRef.getDSSProjectPrivilege().getReleaseUsers().forEach(releaseUser -> { + if (CollectionUtils.isNotEmpty(projectRef.getAddedDSSProjectPrivilege().getReleaseUsers())) { + // 先校验新增的运维用户是否存在于 Schedulis，如果不存在，则不能成功创建工程。 + projectRef.getAddedDSSProjectPrivilege().getReleaseUsers().forEach(releaseUser -> { if (!AzkabanUserService.containsUser(releaseUser, getBaseUrl(), ssoRequestOperation, projectRef.getWorkspace())) { throw new ExternalOperationFailedException(100323, "当前设置的发布用户: " + releaseUser + ", 在 Schedulis 系统中不存在，请在Schedulis中创建该用户！"); } diff --git a/dss-appconn/appconns/dss-schedulis-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/schedulis/service/AzkabanUserService.java b/dss-appconn/appconns/dss-schedulis-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/schedulis/service/AzkabanUserService.java index 2a4cccbda0..a5ea2ceeb5 100644 --- a/dss-appconn/appconns/dss-schedulis-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/schedulis/service/AzkabanUserService.java +++ b/dss-appconn/appconns/dss-schedulis-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/schedulis/service/AzkabanUserService.java @@ -62,6 +62,7 @@ private static List requestUserId(String releaseUser, String } } catch (Exception e) { LOGGER.error("update all releaseUsers from Schedulis url {} failed.", baseUrl, e); + throw e; } return newEntityList; } diff --git a/dss-appconn/appconns/dss-schedulis-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/schedulis/utils/AzkabanUtils.java b/dss-appconn/appconns/dss-schedulis-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/schedulis/utils/AzkabanUtils.java index 6d0588d48a..9fa22d3def 100644 --- a/dss-appconn/appconns/dss-schedulis-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/schedulis/utils/AzkabanUtils.java +++ b/dss-appconn/appconns/dss-schedulis-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/schedulis/utils/AzkabanUtils.java @@ -20,6 +20,7 @@ import org.apache.commons.lang.StringUtils; import java.util.Map; +import java.util.Objects; public class AzkabanUtils { @@ -27,7 +28,7 @@ public static String handleAzkabanEntity(String entityString) { if(StringUtils.isNotBlank(entityString)){ if(entityString.startsWith("{") && entityString.endsWith("}")){ Map resMap = DSSCommonUtils.COMMON_GSON.fromJson(entityString, Map.class); - if(resMap.containsKey("error")){ + if(resMap.containsKey("error") && !Objects.isNull(resMap.get("error"))){ return (String)resMap.get("error"); } } diff --git a/dss-appconn/appconns/dss-scriptis-appconn/pom.xml b/dss-appconn/appconns/dss-scriptis-appconn/pom.xml index 1fbb75e667..d41686c53e 100644 --- a/dss-appconn/appconns/dss-scriptis-appconn/pom.xml +++ b/dss-appconn/appconns/dss-scriptis-appconn/pom.xml @@ -21,7 +21,7 @@ dss com.webank.wedatasphere.dss - 1.1.0.20-SNAPSHOT + 1.10.0-SNAPSHOT ../../../pom.xml 4.0.0 diff --git a/dss-appconn/appconns/dss-sendemail-appconn/pom.xml b/dss-appconn/appconns/dss-sendemail-appconn/pom.xml index 792ccf982b..29dcd96a8b 100644 --- a/dss-appconn/appconns/dss-sendemail-appconn/pom.xml +++ b/dss-appconn/appconns/dss-sendemail-appconn/pom.xml @@ -21,7 +21,7 @@ dss com.webank.wedatasphere.dss - 1.1.0.20-SNAPSHOT + 1.10.0-SNAPSHOT ../../../pom.xml 4.0.0 diff --git a/dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/pom.xml b/dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/pom.xml index 050c80d844..19a2c7b403 100644 --- a/dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/pom.xml +++ b/dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/pom.xml @@ -21,7 +21,7 @@ dss com.webank.wedatasphere.dss - 1.1.0.20-SNAPSHOT + 1.10.0-SNAPSHOT ../../../../pom.xml 4.0.0 diff --git a/dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/src/main/scala/com/webank/wedatasphere/dss/appconn/sendemail/emailcontent/parser/AbstractEmailContentParser.scala b/dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/src/main/scala/com/webank/wedatasphere/dss/appconn/sendemail/emailcontent/parser/AbstractEmailContentParser.scala index f58d908f28..54d8cf5ec1 100644 --- a/dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/src/main/scala/com/webank/wedatasphere/dss/appconn/sendemail/emailcontent/parser/AbstractEmailContentParser.scala +++ b/dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/src/main/scala/com/webank/wedatasphere/dss/appconn/sendemail/emailcontent/parser/AbstractEmailContentParser.scala @@ -27,7 +27,7 @@ import org.apache.linkis.common.io.resultset.ResultSetReader import org.apache.linkis.common.io.{MetaData, Record} import org.apache.linkis.common.utils.Utils import org.apache.linkis.storage.LineRecord -import org.apache.linkis.storage.resultset.ResultSetReaderFactory +import org.apache.linkis.storage.resultset.ResultSetReader import org.apache.commons.io.IOUtils abstract class AbstractEmailContentParser[T] extends EmailContentParser { @@ -42,8 +42,8 @@ abstract class AbstractEmailContentParser[T] extends EmailContentParser { case _ => } - protected def getResultSetReader(fsPathStore: FsPathStoreEmailContent): ResultSetReader[_, _ ] = { - val reader = ResultSetReaderFactory.getResultSetReader(fsPathStore.getFsPath.getSchemaPath) + protected def getResultSetReader(fsPathStore: FsPathStoreEmailContent): ResultSetReader[_ <: MetaData, _ <: Record] = { + val reader = ResultSetReader.getResultSetReader(fsPathStore.getFsPath.getSchemaPath) reader.getMetaData reader } diff --git a/dss-appconn/appconns/dss-sso-appconn/pom.xml b/dss-appconn/appconns/dss-sso-appconn/pom.xml index 2b4eb6f8a6..4fcc9d371a 100644 --- a/dss-appconn/appconns/dss-sso-appconn/pom.xml +++ b/dss-appconn/appconns/dss-sso-appconn/pom.xml @@ -21,7 +21,7 @@ dss com.webank.wedatasphere.dss - 1.1.0.20-SNAPSHOT + 1.10.0-SNAPSHOT ../../../pom.xml 4.0.0 diff --git a/dss-appconn/appconns/dss-workflow-appconn/pom.xml b/dss-appconn/appconns/dss-workflow-appconn/pom.xml index 8ec03fe62c..a7c476b66a 100644 --- a/dss-appconn/appconns/dss-workflow-appconn/pom.xml +++ b/dss-appconn/appconns/dss-workflow-appconn/pom.xml @@ -21,7 +21,7 @@ dss com.webank.wedatasphere.dss - 1.1.0.20-SNAPSHOT + 1.10.0-SNAPSHOT ../../../pom.xml 4.0.0 diff --git a/dss-appconn/appconns/dss-workflow-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/workflow/opertion/WorkflowRefCopyOperation.java b/dss-appconn/appconns/dss-workflow-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/workflow/opertion/WorkflowRefCopyOperation.java index de702dd629..ebb4a19914 100644 --- a/dss-appconn/appconns/dss-workflow-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/workflow/opertion/WorkflowRefCopyOperation.java +++ b/dss-appconn/appconns/dss-workflow-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/workflow/opertion/WorkflowRefCopyOperation.java @@ -16,6 +16,7 @@ package com.webank.wedatasphere.dss.appconn.workflow.opertion; +import com.webank.wedatasphere.dss.common.label.DSSLabel; import com.webank.wedatasphere.dss.common.utils.RpcAskUtils; import com.webank.wedatasphere.dss.orchestrator.common.ref.OrchestratorRefConstant; import com.webank.wedatasphere.dss.sender.service.DSSSenderServiceFactory; @@ -28,6 +29,7 @@ import org.apache.linkis.rpc.Sender; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.Optional; @@ -54,7 +56,9 @@ public RefJobContentResponseRef copyRef(ThirdlyRequestRef.CopyWitContextRequestR workflowCopyRequestRef.getWorkspace(), appId, contextIdStr, projectName, version, description, workflowCopyRequestRef.getDSSLabels(), targetProjectId, (String) nodeSuffix.orElse(null), (String) newFlowName.orElse(null)); - ResponseCopyWorkflow responseCopyWorkflow = RpcAskUtils.processAskException(sender.ask(requestCopyWorkflow), + List dssLabels = workflowCopyRequestRef.getDSSLabels(); + Sender tempSend = DSSSenderServiceFactory.getOrCreateServiceInstance().getWorkflowSender(dssLabels); + ResponseCopyWorkflow responseCopyWorkflow = RpcAskUtils.processAskException(tempSend.ask(requestCopyWorkflow), ResponseCopyWorkflow.class, RequestCopyWorkflow.class); Map refJobContent = new HashMap<>(2); refJobContent.put(OrchestratorRefConstant.ORCHESTRATION_ID_KEY, responseCopyWorkflow.getDssFlow().getId()); diff --git a/dss-appconn/appconns/dss-workflow-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/workflow/opertion/WorkflowRefExportOperation.java b/dss-appconn/appconns/dss-workflow-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/workflow/opertion/WorkflowRefExportOperation.java index 038db59133..febdcdf8ae 100644 --- a/dss-appconn/appconns/dss-workflow-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/workflow/opertion/WorkflowRefExportOperation.java +++ b/dss-appconn/appconns/dss-workflow-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/workflow/opertion/WorkflowRefExportOperation.java @@ -48,7 +48,8 @@ public ExportResponseRef exportRef(ThirdlyRequestRef.RefJobContentRequestRefImpl projectId, projectName, toJson(requestRef.getWorkspace()), - requestRef.getDSSLabels()); + requestRef.getDSSLabels(), + true); Sender sender = DSSSenderServiceFactory.getOrCreateServiceInstance().getWorkflowSender(requestRef.getDSSLabels()); ResponseExportWorkflow responseExportWorkflow = RpcAskUtils.processAskException(sender.ask(requestExportWorkflow), ResponseExportWorkflow.class, RequestExportWorkflow.class); diff --git a/dss-appconn/appconns/dss-workflow-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/workflow/opertion/WorkflowRefImportOperation.java b/dss-appconn/appconns/dss-workflow-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/workflow/opertion/WorkflowRefImportOperation.java index 1ccab9a8ca..bee804b055 100644 --- a/dss-appconn/appconns/dss-workflow-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/workflow/opertion/WorkflowRefImportOperation.java +++ b/dss-appconn/appconns/dss-workflow-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/workflow/opertion/WorkflowRefImportOperation.java @@ -17,7 +17,6 @@ package com.webank.wedatasphere.dss.appconn.workflow.opertion; import com.webank.wedatasphere.dss.common.protocol.JobStatus; -import com.webank.wedatasphere.dss.common.utils.MapUtils; import com.webank.wedatasphere.dss.common.utils.RpcAskUtils; import com.webank.wedatasphere.dss.orchestrator.common.ref.OrchestratorRefConstant; import com.webank.wedatasphere.dss.sender.service.DSSSenderServiceFactory; @@ -48,6 +47,9 @@ public RefJobContentResponseRef importRef(ThirdlyRequestRef.ImportWitContextRequ requestRef.getNewVersion(), requestRef.getWorkspace(), requestRef.getContextId(), requestRef.getDSSLabels()); + if("true".equals( requestRef.getParameter("isOldPackageStruct"))){ + requestImportWorkflow.setOldPackageStruct(true); + } Sender sender = DSSSenderServiceFactory.getOrCreateServiceInstance().getWorkflowSender(requestRef.getDSSLabels()); ResponseImportWorkflow responseImportWorkflow = RpcAskUtils.processAskException(sender.ask(requestImportWorkflow), diff --git a/dss-appconn/dss-appconn-core/pom.xml b/dss-appconn/dss-appconn-core/pom.xml index 60f42dc4bb..fb23c61acb 100644 --- a/dss-appconn/dss-appconn-core/pom.xml +++ b/dss-appconn/dss-appconn-core/pom.xml @@ -21,7 +21,7 @@ dss com.webank.wedatasphere.dss - 1.1.0.20-SNAPSHOT + 1.10.0-SNAPSHOT ../../pom.xml 4.0.0 diff --git a/dss-appconn/dss-appconn-loader/pom.xml b/dss-appconn/dss-appconn-loader/pom.xml index 2155e4f32c..46064bb03e 100644 --- a/dss-appconn/dss-appconn-loader/pom.xml +++ b/dss-appconn/dss-appconn-loader/pom.xml @@ -21,7 +21,7 @@ dss com.webank.wedatasphere.dss - 1.1.0.20-SNAPSHOT + 1.10.0-SNAPSHOT ../../pom.xml 4.0.0 diff --git a/dss-appconn/dss-appconn-manager/dss-appconn-manager-client/pom.xml b/dss-appconn/dss-appconn-manager/dss-appconn-manager-client/pom.xml index f12b02dd25..4baf611a53 100644 --- a/dss-appconn/dss-appconn-manager/dss-appconn-manager-client/pom.xml +++ b/dss-appconn/dss-appconn-manager/dss-appconn-manager-client/pom.xml @@ -21,7 +21,7 @@ dss com.webank.wedatasphere.dss - 1.1.0.20-SNAPSHOT + 1.10.0-SNAPSHOT ../../../pom.xml 4.0.0 diff --git a/dss-appconn/dss-appconn-manager/dss-appconn-manager-client/src/main/java/com/webank/wedatasphere/dss/appconn/manager/conf/AppConnManagerClientConfiguration.java b/dss-appconn/dss-appconn-manager/dss-appconn-manager-client/src/main/java/com/webank/wedatasphere/dss/appconn/manager/conf/AppConnManagerClientConfiguration.java index e6fbc5e707..7628fee54d 100644 --- a/dss-appconn/dss-appconn-manager/dss-appconn-manager-client/src/main/java/com/webank/wedatasphere/dss/appconn/manager/conf/AppConnManagerClientConfiguration.java +++ b/dss-appconn/dss-appconn-manager/dss-appconn-manager-client/src/main/java/com/webank/wedatasphere/dss/appconn/manager/conf/AppConnManagerClientConfiguration.java @@ -21,7 +21,8 @@ public class AppConnManagerClientConfiguration { - public final static CommonVars DSS_APPCONN_CLIENT_TOKEN = CommonVars.apply("wds.dss.appconn.client.user.token","172.0.0.1"); + public final static CommonVars DSS_APPCONN_CLIENT_TOKEN = + CommonVars.apply("wds.dss.appconn.client.user.token","xxx"); public final static CommonVars LINKIS_ADMIN_USER = CommonVars.apply("wds.dss.appconn.client.user","ws"); public final static CommonVars APPCONN_WAIT_MAX_TIME = CommonVars.apply("wds.dss.appconn.client.load.wait.max.time",new TimeType("3m")); diff --git a/dss-appconn/dss-appconn-manager/dss-appconn-manager-client/src/main/java/com/webank/wedatasphere/dss/appconn/manager/service/AppConnResourceServiceImpl.java b/dss-appconn/dss-appconn-manager/dss-appconn-manager-client/src/main/java/com/webank/wedatasphere/dss/appconn/manager/service/AppConnResourceServiceImpl.java index 2f16d6fdb8..25a1c0ef3b 100644 --- a/dss-appconn/dss-appconn-manager/dss-appconn-manager-client/src/main/java/com/webank/wedatasphere/dss/appconn/manager/service/AppConnResourceServiceImpl.java +++ b/dss-appconn/dss-appconn-manager/dss-appconn-manager-client/src/main/java/com/webank/wedatasphere/dss/appconn/manager/service/AppConnResourceServiceImpl.java @@ -125,6 +125,8 @@ public String getAppConnHome(AppConnInfo appConnInfo) { } catch (DSSErrorException e) { throw new AppConnHomeNotExistsWarnException(20350, "Unzip " + zipFilePath + " failed, AppConn is " + appConnName, e); } + //解压完了，zip包就没用了，删掉。 + deleteFile(zipFilePath, "Delete the zip file " + zipFilePath.getName() + " of AppConn " + appConnName + " failed"); File oldIndexFile = AppConnIndexFileUtils.getIndexFile(appConnPath); // delete old index file. @@ -157,6 +159,7 @@ public String getAppConnHome(AppConnInfo appConnInfo) { private void deleteFile(File file, String errorMsg) { try { + LOGGER.info("delete appconn file:{}", file.getAbsolutePath()); FileUtils.forceDelete(file); } catch (IOException e) { if(StringUtils.isNotEmpty(errorMsg)) { diff --git a/dss-appconn/dss-appconn-manager/dss-appconn-manager-core/pom.xml b/dss-appconn/dss-appconn-manager/dss-appconn-manager-core/pom.xml index a2693bc65a..e02a3e9919 100644 --- a/dss-appconn/dss-appconn-manager/dss-appconn-manager-core/pom.xml +++ b/dss-appconn/dss-appconn-manager/dss-appconn-manager-core/pom.xml @@ -21,7 +21,7 @@ dss com.webank.wedatasphere.dss - 1.1.0.20-SNAPSHOT + 1.10.0-SNAPSHOT ../../../pom.xml 4.0.0 diff --git a/dss-appconn/dss-appconn-manager/dss-appconn-manager-core/src/main/java/com/webank/wedatasphere/dss/appconn/manager/impl/AbstractAppConnManager.java b/dss-appconn/dss-appconn-manager/dss-appconn-manager-core/src/main/java/com/webank/wedatasphere/dss/appconn/manager/impl/AbstractAppConnManager.java index a6d019e7ab..c811e515c0 100644 --- a/dss-appconn/dss-appconn-manager/dss-appconn-manager-core/src/main/java/com/webank/wedatasphere/dss/appconn/manager/impl/AbstractAppConnManager.java +++ b/dss-appconn/dss-appconn-manager/dss-appconn-manager-core/src/main/java/com/webank/wedatasphere/dss/appconn/manager/impl/AbstractAppConnManager.java @@ -43,6 +43,7 @@ import org.slf4j.LoggerFactory; import java.util.*; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; import java.util.function.Consumer; import java.util.stream.Collectors; @@ -52,7 +53,7 @@ public abstract class AbstractAppConnManager implements AppConnManager { private static final Logger LOGGER = LoggerFactory.getLogger(AbstractAppConnManager.class); private final AppConnLoader appConnLoader = AppConnLoaderFactory.getAppConnLoader(); - private final Map appConns = new HashMap<>(); + private final Map appConns = new ConcurrentHashMap<>(); private volatile boolean isLoaded = false; private List appConnList = null; AppConnInfoService appConnInfoService; @@ -60,6 +61,8 @@ public abstract class AbstractAppConnManager implements AppConnManager { private AppConnRefreshThread appConnRefreshThread; private static volatile AppConnManager appConnManager; + + private static volatile boolean appConnManagerInited = false; private static boolean lazyLoad = false; public static void setLazyLoad() { @@ -67,11 +70,11 @@ public static void setLazyLoad() { } public static AppConnManager getAppConnManager() { - if (appConnManager != null) { + if (appConnManagerInited) { return appConnManager; } synchronized (AbstractAppConnManager.class) { - if (appConnManager == null) { + if (!appConnManagerInited) { //appconn-manager-core包无法引入manager-client包，会有maven循环依赖，这里通过反射获取client的实现类 //ismanager=false时，获取client端的AppConnManager实现类，ismanager=true时，获取appconn-framework端的AppConnManager实现类。 if (Objects.equals(AppConnManagerCoreConf.IS_APPCONN_MANAGER.getValue(), AppConnManagerCoreConf.hostname) @@ -87,6 +90,7 @@ public static AppConnManager getAppConnManager() { LOGGER.info("The instance of AppConnManager is {}.", appConnManager.getClass().getName()); appConnManager.init(); } + appConnManagerInited = true; return appConnManager; } } @@ -101,6 +105,7 @@ public void init() { loadAppConns(); isLoaded = true; } + LOGGER.info("AppConnManager init successfully"); } protected abstract AppConnInfoService createAppConnInfoService(); @@ -117,9 +122,6 @@ protected void loadAppConns() { LOGGER.warn("No AppConnInfos returned, ignore it."); return; } - long refreshInterval = AppInstanceConstants.APP_CONN_REFRESH_INTERVAL.getValue().toLong(); - appConnRefreshThread = new AppConnRefreshThread(this, appConnInfos); - Utils.defaultScheduler().scheduleAtFixedRate(appConnRefreshThread, refreshInterval, refreshInterval, TimeUnit.MILLISECONDS); Map appConns = new HashMap<>(); Consumer loadAndAdd = DSSExceptionUtils.handling(appConnInfo -> { AppConn appConn = loadAppConn(appConnInfo); @@ -150,6 +152,9 @@ protected void loadAppConns() { appConnList = Collections.unmodifiableList(new ArrayList<>(appConns.values())); } LOGGER.info("Inited all AppConns, the AppConn list are {}.", this.appConns.keySet()); + long refreshInterval = AppInstanceConstants.APP_CONN_REFRESH_INTERVAL.getValue().toLong(); + appConnRefreshThread = new AppConnRefreshThread(this, appConnInfos); + Utils.defaultScheduler().scheduleAtFixedRate(appConnRefreshThread, refreshInterval, refreshInterval, TimeUnit.MILLISECONDS); } protected AppConn loadAppConn(AppConnInfo appConnInfo) throws Exception { diff --git a/dss-appconn/dss-appconn-manager/pom.xml b/dss-appconn/dss-appconn-manager/pom.xml index e2b61cf83f..501acce760 100644 --- a/dss-appconn/dss-appconn-manager/pom.xml +++ b/dss-appconn/dss-appconn-manager/pom.xml @@ -21,7 +21,7 @@ dss com.webank.wedatasphere.dss - 1.1.0.20-SNAPSHOT + 1.10.0-SNAPSHOT ../../pom.xml 4.0.0 diff --git a/dss-appconn/dss-scheduler-appconn/pom.xml b/dss-appconn/dss-scheduler-appconn/pom.xml index cc14c6c04e..b760026009 100644 --- a/dss-appconn/dss-scheduler-appconn/pom.xml +++ b/dss-appconn/dss-scheduler-appconn/pom.xml @@ -21,7 +21,7 @@ dss com.webank.wedatasphere.dss - 1.1.0.20-SNAPSHOT + 1.10.0-SNAPSHOT ../../pom.xml 4.0.0 diff --git a/dss-appconn/linkis-appconn-engineplugin/pom.xml b/dss-appconn/linkis-appconn-engineplugin/pom.xml index 363d409dee..4f0be8c54f 100644 --- a/dss-appconn/linkis-appconn-engineplugin/pom.xml +++ b/dss-appconn/linkis-appconn-engineplugin/pom.xml @@ -21,7 +21,7 @@ dss com.webank.wedatasphere.dss - 1.1.0.20-SNAPSHOT + 1.10.0-SNAPSHOT ../../pom.xml 4.0.0 diff --git a/dss-appconn/linkis-appconn-engineplugin/src/main/resources/linkis-engineconn.properties b/dss-appconn/linkis-appconn-engineplugin/src/main/resources/linkis-engineconn.properties index 01b9fe8084..5e3cb3eab5 100644 --- a/dss-appconn/linkis-appconn-engineplugin/src/main/resources/linkis-engineconn.properties +++ b/dss-appconn/linkis-appconn-engineplugin/src/main/resources/linkis-engineconn.properties @@ -37,7 +37,7 @@ wds.linkis.server.mybatis.datasource.url=jdbc:mysql://127.0.0.1:3306/${databaseN wds.linkis.server.mybatis.datasource.username= -***REMOVED*** +wds.linkis.server.mybatis.datasource.password= wds.linkis.gateway.ip=127.0.0.1 wds.linkis.gateway.port=9001 diff --git a/dss-appconn/linkis-appconn-engineplugin/src/main/scala/org/apache/linkis/manager/engineplugin/appconn/executor/AbstractExecutionRequestRefContext.scala b/dss-appconn/linkis-appconn-engineplugin/src/main/scala/org/apache/linkis/manager/engineplugin/appconn/executor/AbstractExecutionRequestRefContext.scala index b631b447d5..3d91c60192 100644 --- a/dss-appconn/linkis-appconn-engineplugin/src/main/scala/org/apache/linkis/manager/engineplugin/appconn/executor/AbstractExecutionRequestRefContext.scala +++ b/dss-appconn/linkis-appconn-engineplugin/src/main/scala/org/apache/linkis/manager/engineplugin/appconn/executor/AbstractExecutionRequestRefContext.scala @@ -17,6 +17,7 @@ package org.apache.linkis.manager.engineplugin.appconn.executor import java.util + import com.webank.wedatasphere.dss.standard.app.development.listener.core.ExecutionRequestRefContext import com.webank.wedatasphere.dss.standard.app.development.listener.exception.AppConnExecutionErrorException import org.apache.linkis.common.io.resultset.{ResultSet, ResultSetReader, ResultSetWriter} @@ -27,7 +28,7 @@ import org.apache.linkis.manager.engineplugin.appconn.conf.AppConnEngineConnConf import org.apache.linkis.rpc.Sender import org.apache.linkis.storage.FSFactory import org.apache.linkis.storage.fs.FileSystem -import org.apache.linkis.storage.resultset.{ResultSetFactory, ResultSetReaderFactory} +import org.apache.linkis.storage.resultset.{ResultSetFactory, ResultSetReader} abstract class AbstractExecutionRequestRefContext(engineExecutorContext: EngineExecutionContext, user: String, @@ -69,10 +70,8 @@ abstract class AbstractExecutionRequestRefContext(engineExecutorContext: EngineE resultSetAlias: String): ResultSetWriter[M, R] = engineExecutorContext.createResultSetWriter(resultSet, resultSetAlias).asInstanceOf[ResultSetWriter[M, R]] - override def getResultSetReader[M <: MetaData, R <: Record](fsPath: FsPath): ResultSetReader[M, R] = { - - ResultSetReaderFactory.getResultSetReader(fsPath.getSchemaPath).asInstanceOf[ResultSetReader[M, R]] - } + override def getResultSetReader[M <: MetaData, R <: Record](fsPath: FsPath): ResultSetReader[M, R] = + ResultSetReader.getResultSetReader(fsPath.getSchemaPath).asInstanceOf[ResultSetReader[M, R]] private def createResultSetWriter[M <: MetaData, R <: Record](resultSetType: String, resultSetAlias: String): ResultSetWriter[M, R] = engineExecutorContext.createResultSetWriter(resultSetType, resultSetAlias).asInstanceOf[ResultSetWriter[M, R]] diff --git a/dss-appconn/pom.xml b/dss-appconn/pom.xml index 616ea2f666..bbdfbf351f 100644 --- a/dss-appconn/pom.xml +++ b/dss-appconn/pom.xml @@ -21,7 +21,7 @@ dss com.webank.wedatasphere.dss - 1.1.0.20-SNAPSHOT + 1.10.0-SNAPSHOT ../pom.xml 4.0.0 diff --git a/dss-apps/dss-apiservice-server/pom.xml b/dss-apps/dss-apiservice-server/pom.xml index 0c82fbb7ec..25e955b29b 100644 --- a/dss-apps/dss-apiservice-server/pom.xml +++ b/dss-apps/dss-apiservice-server/pom.xml @@ -21,7 +21,7 @@ dss com.webank.wedatasphere.dss - 1.1.0.20-SNAPSHOT + 1.10.0-SNAPSHOT ../../pom.xml 4.0.0 @@ -140,11 +140,6 @@ - - org.postgresql - postgresql - 42.3.3 - org.apache.linkis linkis-mybatis @@ -348,6 +343,12 @@ provided + + com.sun.jersey + jersey-core + 1.19.4 + + diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/config/ApiServiceConfiguration.java b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/config/ApiServiceConfiguration.java index 969e055e7b..a4d4c9ac21 100644 --- a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/config/ApiServiceConfiguration.java +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/config/ApiServiceConfiguration.java @@ -20,7 +20,7 @@ public class ApiServiceConfiguration { - public final static CommonVars LINKIS_AUTHOR_USER_TOKEN = CommonVars.apply("wds.linkis.client.api.service.author.user.token","172.0.0.1"); + public final static CommonVars LINKIS_AUTHOR_USER_TOKEN = CommonVars.apply("wds.linkis.client.api.service.author.user.token",""); public final static CommonVars LINKIS_ADMIN_USER = CommonVars.apply("wds.linkis.client.api.service.adminuser","ws"); public final static CommonVars LINKIS_CONNECTION_TIMEOUT = CommonVars.apply("wds.linkis.flow.connection.timeout",30000); diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/execute/ExecuteCodeHelper.java b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/execute/ExecuteCodeHelper.java index 19b0304775..8cac07324f 100644 --- a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/execute/ExecuteCodeHelper.java +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/execute/ExecuteCodeHelper.java @@ -195,10 +195,11 @@ public static String getResultList(JobExecuteResult executeResult,UJESClient cl - public static String getResultContent(String user, String path, int maxSize, UJESClient client) { + public static String getResultContent(String user, String path, int maxSize, UJESClient client, boolean enableLimit) { return client.resultSet(ResultSetAction.builder() .setPath(path) .setUser(user) + .setEnableLimit(enableLimit) .setPageSize(maxSize).build()).getResponseBody(); } diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/restful/ApiServiceExecuteRestfulApi.java b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/restful/ApiServiceExecuteRestfulApi.java index f6a66fc6ed..7a085b2efd 100644 --- a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/restful/ApiServiceExecuteRestfulApi.java +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/restful/ApiServiceExecuteRestfulApi.java @@ -173,7 +173,8 @@ public Message openFile(HttpServletRequest req, @RequestParam(required = false, name = "taskId") String taskId, @RequestParam(required = false, name = "page", defaultValue = "1") Integer page, @RequestParam(required = false, name = "pageSize", defaultValue = "5000") Integer pageSize, - @RequestParam(required = false, name = "charset", defaultValue = "utf-8") String charset) { + @RequestParam(required = false, name = "charset", defaultValue = "utf-8") String charset, + @RequestParam(required = false, name = "enableLimit", defaultValue = "false") Boolean enableLimit) { String userName = SecurityFilter.getLoginUsername(req); logger.info("User {} wants to open resultSet file {} in task {}.", userName, path, taskId); if (!isNumber(taskId)) { @@ -191,7 +192,7 @@ public Message openFile(HttpServletRequest req, } else if (userName.equals(apiServiceJob.getSubmitUser())) { UJESClient client = LinkisJobSubmit.getClient(); try { - String fileContent = ExecuteCodeHelper.getResultContent(apiServiceJob.getProxyUser(), path, pageSize, client); + String fileContent = ExecuteCodeHelper.getResultContent(apiServiceJob.getProxyUser(), path, pageSize, client,enableLimit); return DSSCommonUtils.COMMON_GSON.fromJson(fileContent, Message.class); } catch (Exception e) { logger.error("User {} fetch resultSet {} failed.", userName, path, e); diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/service/impl/ApiServiceImpl.java b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/service/impl/ApiServiceImpl.java index f42bd1e142..8fc1584550 100644 --- a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/service/impl/ApiServiceImpl.java +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/service/impl/ApiServiceImpl.java @@ -491,7 +491,7 @@ private Map uploadBml(String userName, String scriptPath, Map variableList=null; if(metadata.entrySet().size() >0) { Variable[] v = VariableParser.getVariables(metadata); - variableList = Arrays.stream(v).filter(var -> !StringUtils.isEmpty(var.getValue())).collect(Collectors.toList()); + variableList = Arrays.stream(v).filter(var -> !StringUtils.isEmpty(var.value())).collect(Collectors.toList()); } if(variableList!=null) { @@ -520,7 +520,7 @@ private Map updateBml(String userName, String resourceId, String try { ScriptFsWriter writer = StorageScriptFsWriter.getScriptFsWriter(new FsPath(scriptPath), Consts.UTF_8.toString(), null); Variable[] v = VariableParser.getVariables(metadata); - List variableList = Arrays.stream(v).filter(var -> !StringUtils.isEmpty(var.getValue())).collect(Collectors.toList()); + List variableList = Arrays.stream(v).filter(var -> !StringUtils.isEmpty(var.value())).collect(Collectors.toList()); writer.addMetaData(new ScriptMetaData(variableList.toArray(new Variable[0]))); writer.addRecord(new ScriptRecord(scriptContent)); InputStream inputStream = writer.getInputStream(); diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/service/impl/ApiServiceQueryServiceImpl.java b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/service/impl/ApiServiceQueryServiceImpl.java index 492b153fe7..436efbd034 100644 --- a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/service/impl/ApiServiceQueryServiceImpl.java +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/service/impl/ApiServiceQueryServiceImpl.java @@ -383,10 +383,9 @@ private Pair> queryBml(String userName, String resou InputStream inputStream = resource.inputStream(); - try (FileSource fileSource = FileSource.create(new FsPath(scriptPath), inputStream)) { + try (FileSource fileSource = FileSource$.MODULE$.create(new FsPath(scriptPath), inputStream)) { //todo 数组取了第一个 - Pair> sourcePair = fileSource.collect()[0]; - collect = new Pair<>(sourcePair.getKey(), new ArrayList<>(sourcePair.getValue())); + collect = fileSource.collect()[0]; bmlCache.put(key, collect); } } diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/token/JwtManager.java b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/token/JwtManager.java index cb0bea3c7b..6f36464c43 100644 --- a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/token/JwtManager.java +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/token/JwtManager.java @@ -15,7 +15,6 @@ */ package com.webank.wedatasphere.dss.apiservice.core.token; - import com.sun.jersey.core.util.Base64; import com.webank.wedatasphere.dss.apiservice.core.config.ApiServiceConfiguration; import com.webank.wedatasphere.dss.apiservice.core.bo.ApiServiceToken; diff --git a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/token/TokenStatusCheckerTask.java b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/token/TokenStatusCheckerTask.java index 95203fc078..6a9896ffb3 100644 --- a/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/token/TokenStatusCheckerTask.java +++ b/dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/token/TokenStatusCheckerTask.java @@ -38,7 +38,7 @@ public class TokenStatusCheckerTask { @Autowired ApiServiceTokenManagerDao atmd; - @Scheduled(cron = "0 0/5 * * * ?") + @Scheduled(cron = "0 0/10 * * * ?") public void doTokenStatusCheckTask() { // 查询启用状态的token List tokenManagerVos = atmd.queryTokenByStatus(1); diff --git a/dss-apps/dss-apiservice-server/src/test/resources/linkis.properties b/dss-apps/dss-apiservice-server/src/test/resources/linkis.properties index e68b6f9632..69cd5c3fe7 100644 --- a/dss-apps/dss-apiservice-server/src/test/resources/linkis.properties +++ b/dss-apps/dss-apiservice-server/src/test/resources/linkis.properties @@ -42,7 +42,7 @@ wds.linkis.test.mode=false wds.linkis.test.user= wds.linkis.server.mybatis.datasource.url=jdbc:h2:mem:test wds.linkis.server.mybatis.datasource.username= -***REMOVED*** +wds.linkis.server.mybatis.datasource.password= #dsm wds.linkis.server.dsm.admin.users= diff --git a/dss-apps/dss-apps-server/pom.xml b/dss-apps/dss-apps-server/pom.xml index 827480e12c..c00f5c6caf 100644 --- a/dss-apps/dss-apps-server/pom.xml +++ b/dss-apps/dss-apps-server/pom.xml @@ -21,7 +21,7 @@ dss com.webank.wedatasphere.dss - 1.1.0.20-SNAPSHOT + 1.10.0-SNAPSHOT ../../pom.xml 4.0.0 diff --git a/dss-apps/dss-data-api/dss-api-sql-template/pom.xml b/dss-apps/dss-data-api/dss-api-sql-template/pom.xml index 24a2d57c0d..e279065247 100644 --- a/dss-apps/dss-data-api/dss-api-sql-template/pom.xml +++ b/dss-apps/dss-data-api/dss-api-sql-template/pom.xml @@ -5,7 +5,7 @@ dss com.webank.wedatasphere.dss - 1.1.0.20-SNAPSHOT + 1.10.0-SNAPSHOT ../../../pom.xml 4.0.0 diff --git a/dss-apps/dss-data-api/dss-data-api-server/pom.xml b/dss-apps/dss-data-api/dss-data-api-server/pom.xml index ab98a16c4c..5113f6f083 100644 --- a/dss-apps/dss-data-api/dss-data-api-server/pom.xml +++ b/dss-apps/dss-data-api/dss-data-api-server/pom.xml @@ -5,7 +5,7 @@ dss com.webank.wedatasphere.dss - 1.1.0.20-SNAPSHOT + 1.10.0-SNAPSHOT ../../../pom.xml 4.0.0 diff --git a/dss-apps/dss-data-api/dss-data-api-server/src/main/resources/dss.properties b/dss-apps/dss-data-api/dss-data-api-server/src/main/resources/dss.properties index 5d0955dcf2..50a90b064e 100644 --- a/dss-apps/dss-data-api/dss-data-api-server/src/main/resources/dss.properties +++ b/dss-apps/dss-data-api/dss-data-api-server/src/main/resources/dss.properties @@ -22,4 +22,4 @@ wds.linkis.gateway.url=http://127.0.0.1:9001/ wds.linkis.server.mybatis.datasource.url=jdbc:mysql://127.0.0.1:3306/dss_test?characterEncoding=UTF-8 wds.linkis.server.mybatis.datasource.username= -***REMOVED*** \ No newline at end of file +wds.linkis.server.mybatis.datasource.password= \ No newline at end of file diff --git a/dss-apps/dss-data-api/pom.xml b/dss-apps/dss-data-api/pom.xml index 1e17302980..c5a6d5d1f2 100644 --- a/dss-apps/dss-data-api/pom.xml +++ b/dss-apps/dss-data-api/pom.xml @@ -6,7 +6,7 @@ dss com.webank.wedatasphere.dss ../../pom.xml - 1.1.0.20-SNAPSHOT + 1.10.0-SNAPSHOT 4.0.0 dss-data-api diff --git a/dss-apps/dss-data-governance/dss-data-asset-server/pom.xml b/dss-apps/dss-data-governance/dss-data-asset-server/pom.xml index f0061c59ba..0a387b5c57 100644 --- a/dss-apps/dss-data-governance/dss-data-asset-server/pom.xml +++ b/dss-apps/dss-data-governance/dss-data-asset-server/pom.xml @@ -5,7 +5,7 @@ dss com.webank.wedatasphere.dss - 1.1.0.20-SNAPSHOT + 1.10.0-SNAPSHOT ../../../pom.xml 4.0.0 diff --git a/dss-apps/dss-data-governance/dss-data-classification-server/pom.xml b/dss-apps/dss-data-governance/dss-data-classification-server/pom.xml index 8e39df352b..7333bfa2f3 100644 --- a/dss-apps/dss-data-governance/dss-data-classification-server/pom.xml +++ b/dss-apps/dss-data-governance/dss-data-classification-server/pom.xml @@ -5,7 +5,7 @@ dss com.webank.wedatasphere.dss - 1.1.0.20-SNAPSHOT + 1.10.0-SNAPSHOT ../../../pom.xml 4.0.0 diff --git a/dss-apps/dss-data-governance/dss-data-governance-common/pom.xml b/dss-apps/dss-data-governance/dss-data-governance-common/pom.xml index 891faa5f47..ec1e7dde14 100644 --- a/dss-apps/dss-data-governance/dss-data-governance-common/pom.xml +++ b/dss-apps/dss-data-governance/dss-data-governance-common/pom.xml @@ -5,7 +5,7 @@ dss com.webank.wedatasphere.dss - 1.1.0.20-SNAPSHOT + 1.10.0-SNAPSHOT ../../../pom.xml 4.0.0 diff --git a/dss-apps/dss-data-governance/dss-data-governance-server/pom.xml b/dss-apps/dss-data-governance/dss-data-governance-server/pom.xml index 25689c2686..39feb2c775 100644 --- a/dss-apps/dss-data-governance/dss-data-governance-server/pom.xml +++ b/dss-apps/dss-data-governance/dss-data-governance-server/pom.xml @@ -5,7 +5,7 @@ dss com.webank.wedatasphere.dss - 1.1.0.20-SNAPSHOT + 1.10.0-SNAPSHOT ../../../pom.xml 4.0.0 diff --git a/dss-apps/dss-data-governance/dss-data-governance-server/src/main/resources/dss.properties b/dss-apps/dss-data-governance/dss-data-governance-server/src/main/resources/dss.properties index 379830e551..41a8e28207 100644 --- a/dss-apps/dss-data-governance/dss-data-governance-server/src/main/resources/dss.properties +++ b/dss-apps/dss-data-governance/dss-data-governance-server/src/main/resources/dss.properties @@ -22,4 +22,4 @@ wds.linkis.gateway.url=http://127.0.0.1:9001/ wds.linkis.server.mybatis.datasource.url=jdbc:mysql://xxxxxxx:3306/dss_test?characterEncoding=UTF-8 wds.linkis.server.mybatis.datasource.username=xxxxx -***REMOVED***xxxxxx +wds.linkis.server.mybatis.datasource.password=xxxxxx diff --git a/dss-apps/dss-data-governance/pom.xml b/dss-apps/dss-data-governance/pom.xml index 668d5c2c21..c278a5c096 100644 --- a/dss-apps/dss-data-governance/pom.xml +++ b/dss-apps/dss-data-governance/pom.xml @@ -5,7 +5,7 @@ dss com.webank.wedatasphere.dss - 1.1.0.20-SNAPSHOT + 1.10.0-SNAPSHOT ../../pom.xml 4.0.0 diff --git a/dss-apps/dss-scriptis-server/pom.xml b/dss-apps/dss-scriptis-server/pom.xml index 74d84e93f9..903ee325d9 100644 --- a/dss-apps/dss-scriptis-server/pom.xml +++ b/dss-apps/dss-scriptis-server/pom.xml @@ -22,7 +22,7 @@ dss com.webank.wedatasphere.dss - 1.1.0.20-SNAPSHOT + 1.10.0-SNAPSHOT ../../pom.xml @@ -142,11 +142,6 @@ - - org.postgresql - postgresql - 42.3.3 - org.apache.linkis linkis-mybatis diff --git a/dss-apps/dss-scriptis-server/src/main/java/com/webank/wedatasphere/dss/scriptis/config/DSSScriptisConfiguration.java b/dss-apps/dss-scriptis-server/src/main/java/com/webank/wedatasphere/dss/scriptis/config/DSSScriptisConfiguration.java index c82662558a..0b0fd99fab 100644 --- a/dss-apps/dss-scriptis-server/src/main/java/com/webank/wedatasphere/dss/scriptis/config/DSSScriptisConfiguration.java +++ b/dss-apps/dss-scriptis-server/src/main/java/com/webank/wedatasphere/dss/scriptis/config/DSSScriptisConfiguration.java @@ -20,7 +20,7 @@ public class DSSScriptisConfiguration { - public final static CommonVars LINKIS_AUTHOR_USER_TOKEN = CommonVars.apply("wds.linkis.client.api.service.author.user.token","172.0.0.1"); + public final static CommonVars LINKIS_AUTHOR_USER_TOKEN = CommonVars.apply("wds.linkis.client.api.service.author.user.token",""); public final static CommonVars LINKIS_ADMIN_USER = CommonVars.apply("wds.linkis.client.api.service.adminuser","ws"); public final static CommonVars LINKIS_CONNECTION_TIMEOUT = CommonVars.apply("wds.linkis.client.connection.timeout",45000L); @@ -29,5 +29,4 @@ public class DSSScriptisConfiguration { public final static String GLOBAL_LIMITS_PREFIX = "wds.dss.scriptis.global.limits."; public final static String GLOBAL_LIMIT_PREFIX = "wds.dss.scriptis.global.limit."; - } diff --git a/dss-apps/dss-scriptis-server/src/main/java/com/webank/wedatasphere/dss/scriptis/restful/ScriptisAuthRestfulApi.java b/dss-apps/dss-scriptis-server/src/main/java/com/webank/wedatasphere/dss/scriptis/restful/ScriptisAuthRestfulApi.java index 22306ad1aa..3cf28f84ae 100644 --- a/dss-apps/dss-scriptis-server/src/main/java/com/webank/wedatasphere/dss/scriptis/restful/ScriptisAuthRestfulApi.java +++ b/dss-apps/dss-scriptis-server/src/main/java/com/webank/wedatasphere/dss/scriptis/restful/ScriptisAuthRestfulApi.java @@ -2,6 +2,7 @@ import com.webank.wedatasphere.dss.common.conf.DSSCommonConf; import com.webank.wedatasphere.dss.common.utils.GlobalLimitsUtils; +import com.webank.wedatasphere.dss.scriptis.config.DSSScriptisConfiguration; import com.webank.wedatasphere.dss.scriptis.service.ScriptisAuthService; import org.apache.linkis.common.conf.BDPConfiguration; import org.apache.linkis.server.Message; @@ -11,6 +12,7 @@ import javax.annotation.PostConstruct; import javax.servlet.http.HttpServletRequest; +import java.util.HashMap; import java.util.Map; import static com.webank.wedatasphere.dss.scriptis.config.DSSScriptisConfiguration.GLOBAL_LIMITS_PREFIX; diff --git a/dss-apps/dss-user-guide/dss-user-guide-server/pom.xml b/dss-apps/dss-user-guide/dss-user-guide-server/pom.xml index e7e6add8d1..cd70ea7927 100644 --- a/dss-apps/dss-user-guide/dss-user-guide-server/pom.xml +++ b/dss-apps/dss-user-guide/dss-user-guide-server/pom.xml @@ -5,7 +5,7 @@ dss-user-guide com.webank.wedatasphere.dss - 1.1.0.20-SNAPSHOT + 1.10.0-SNAPSHOT ../pom.xml 4.0.0 diff --git a/dss-apps/dss-user-guide/dss-user-guide-server/src/main/java/com/webank/wedatasphere/dss/guide/server/util/ShellUtils.java b/dss-apps/dss-user-guide/dss-user-guide-server/src/main/java/com/webank/wedatasphere/dss/guide/server/util/ShellUtils.java new file mode 100644 index 0000000000..1c0f27fc2e --- /dev/null +++ b/dss-apps/dss-user-guide/dss-user-guide-server/src/main/java/com/webank/wedatasphere/dss/guide/server/util/ShellUtils.java @@ -0,0 +1,59 @@ +package com.webank.wedatasphere.dss.guide.server.util; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.BufferedReader; +import java.io.InputStreamReader; + +public class ShellUtils { + + private static final Logger logger = LoggerFactory.getLogger(ShellUtils.class); + + /** + * 执行shell命令 + * + * @param shellString + */ + public static void callShellByExec(String shellString) { + try { + logger.info("shellString:" + shellString); + Process process = Runtime.getRuntime().exec(shellString); + int exitValue = process.waitFor(); + if (0 != exitValue) { + logger.error("call shell failed. error code is :" + exitValue); + } + } catch (Throwable e) { + logger.error("call shell failed. " + e); + } + } + + /** + * 执行shell命令 + * + * @param shellString + */ + public static String callShellQuery(String shellString) { + logger.info("shellString:" + shellString); + BufferedReader reader = null; + String result = ""; + try { + Process process = Runtime.getRuntime().exec(shellString); + int exitValue = process.waitFor(); + if (0 != exitValue) { + logger.error("call shell failed. error code is :" + exitValue); + } + // 返回值 + reader = new BufferedReader(new InputStreamReader(process.getInputStream())); + String line = null; + while ((line = reader.readLine()) != null) { + result += line; + } + } catch (Throwable e) { + logger.error("call shell failed. " + e); + } + logger.info("call shell query result:" + result); + return result.trim(); + } + +} diff --git a/dss-apps/dss-user-guide/dss-user-guide-server/src/main/resources/dss.properties b/dss-apps/dss-user-guide/dss-user-guide-server/src/main/resources/dss.properties index 29a74250d8..f10c96b3af 100644 --- a/dss-apps/dss-user-guide/dss-user-guide-server/src/main/resources/dss.properties +++ b/dss-apps/dss-user-guide/dss-user-guide-server/src/main/resources/dss.properties @@ -21,7 +21,7 @@ wds.linkis.gateway.url=http://xxxxxxx:9001/ wds.linkis.mysql.is.encrypt=false wds.linkis.server.mybatis.datasource.url=jdbc:mysql://xxxxxxx:3306/dss_dev_1?characterEncoding=UTF-8 wds.linkis.server.mybatis.datasource.username=xxxxxxx -***REMOVED***xxxxxxx +wds.linkis.server.mybatis.datasource.password=xxxxxxx wds.linkis.server.version=v1 spring.server.port=9210 diff --git a/dss-apps/dss-user-guide/pom.xml b/dss-apps/dss-user-guide/pom.xml index b506cc4b3f..61d4471cb6 100644 --- a/dss-apps/dss-user-guide/pom.xml +++ b/dss-apps/dss-user-guide/pom.xml @@ -5,7 +5,7 @@ dss com.webank.wedatasphere.dss - 1.1.0.20-SNAPSHOT + 1.10.0-SNAPSHOT ../../pom.xml 4.0.0 diff --git a/dss-apps/pom.xml b/dss-apps/pom.xml index 3731e59ae0..5f64617096 100644 --- a/dss-apps/pom.xml +++ b/dss-apps/pom.xml @@ -21,7 +21,7 @@ dss com.webank.wedatasphere.dss - 1.1.0.20-SNAPSHOT + 1.10.0-SNAPSHOT ../pom.xml 4.0.0 diff --git a/dss-commons/dss-common/pom.xml b/dss-commons/dss-common/pom.xml index 23c2acc60f..d3fac400d8 100644 --- a/dss-commons/dss-common/pom.xml +++ b/dss-commons/dss-common/pom.xml @@ -22,7 +22,7 @@ dss-commons com.webank.wedatasphere.dss - 1.1.0.20-SNAPSHOT + 1.10.0-SNAPSHOT ../pom.xml dss-common @@ -30,6 +30,7 @@ 5.1.49 2.30.1 + 1.21 @@ -109,7 +110,7 @@ com.monitorjbl xlsx-streamer - 1.2.1 + 2.1.0 com.fasterxml.jackson.core @@ -117,7 +118,11 @@ org.apache.poi - ooxml-schemas + poi-ooxml-schemas + + + org.apache.commons + commons-compress @@ -126,16 +131,19 @@ spring-aop org.springframework ${spring.version} + provided spring-context-support org.springframework ${spring.version} + provided spring-webmvc org.springframework ${spring.version} + provided org.apache.linkis @@ -155,6 +163,11 @@ ${mysql.connector.version} + + org.apache.commons + commons-compress + ${commons-compress.version} + diff --git a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/StaffInfo.java b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/StaffInfo.java similarity index 93% rename from dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/StaffInfo.java rename to dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/StaffInfo.java index 719c6554ed..9596e184d7 100644 --- a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/StaffInfo.java +++ b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/StaffInfo.java @@ -14,7 +14,7 @@ * */ -package com.webank.wedatasphere.dss.framework.workspace.bean; +package com.webank.wedatasphere.dss.common; import com.google.gson.annotations.SerializedName; @@ -40,7 +40,8 @@ public class StaffInfo { private String status; @SerializedName(value = "PersonGroup") private String personGroup; - + @SerializedName(value = "bgName") + private String bgName; public StaffInfo(){ } @@ -130,6 +131,14 @@ public void setPersonGroup(String personGroup) { this.personGroup = personGroup; } + public String getBgName() { + return bgName; + } + + public void setBgName(String bgName) { + this.bgName = bgName; + } + @Override public String toString() { return "StaffInfo{" + diff --git a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/service/StaffInfoGetter.java b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/StaffInfoGetter.java similarity index 86% rename from dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/service/StaffInfoGetter.java rename to dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/StaffInfoGetter.java index 094936ed00..0b04048a3b 100644 --- a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/service/StaffInfoGetter.java +++ b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/StaffInfoGetter.java @@ -14,11 +14,10 @@ * */ -package com.webank.wedatasphere.dss.framework.workspace.service; - -import com.webank.wedatasphere.dss.framework.workspace.bean.StaffInfo; +package com.webank.wedatasphere.dss.common; import java.util.List; +import java.util.Set; public interface StaffInfoGetter { @@ -31,4 +30,6 @@ public interface StaffInfoGetter { StaffInfo getStaffInfoByUsername(String username); + Set getAllUsernames(); + } diff --git a/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/auditlog/TargetTypeEnum.java b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/auditlog/TargetTypeEnum.java index 678bfc3e0b..7710aca4c4 100644 --- a/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/auditlog/TargetTypeEnum.java +++ b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/auditlog/TargetTypeEnum.java @@ -69,6 +69,10 @@ public enum TargetTypeEnum { * 工作流引用默认模板 */ DSS_EC_CONFIG_TEMPLATE_WORKFLOW("dss_ec_config_template_workflow"), + + + DSS_WORKSPACE_DEFAULT_TEMPLATE("dss_workspace_default_template"), + ; private String name; diff --git a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/util/WorkspaceServerConstant.java b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/conf/WorkspaceServerConstant.java similarity index 82% rename from dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/util/WorkspaceServerConstant.java rename to dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/conf/WorkspaceServerConstant.java index ab75cda431..21b8689142 100644 --- a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/util/WorkspaceServerConstant.java +++ b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/conf/WorkspaceServerConstant.java @@ -14,11 +14,12 @@ * */ -package com.webank.wedatasphere.dss.framework.workspace.util; +package com.webank.wedatasphere.dss.common.conf; public interface WorkspaceServerConstant { - String DEFAULT_DEPARTMENT = "基础科技部"; + String DEFAULT_DEPARTMENT = "基础科技产品部"; String DEFAULT_OFFICE = "大数据平台室"; String DEFAULT_STAFF_SPLIT = "-"; + String DEFAULT_RESIGNED = "已离职"; } diff --git a/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/node/DSSNodeDefault.java b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/node/DSSNodeDefault.java index db5efa3111..924d2d4b58 100644 --- a/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/node/DSSNodeDefault.java +++ b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/node/DSSNodeDefault.java @@ -33,10 +33,14 @@ public class DSSNodeDefault implements DSSNode { private List resources; private String title; private String desc; - private String createTime; + private Long createTime; private String userProxy; private String modifyUser; private Long modifyTime; + private String appTag; + private String businessTag; + private String ecConfTemplateId; + private String ecConfTemplateName; /** * dependencys 是该Node的依赖节点 */ @@ -168,10 +172,105 @@ public String toString() { return "dwsNode{" + "layout=" + layout + ", id='" + id + '\'' + + ", key='" + key + '\'' + ", jobType='" + jobType + '\'' + ", lastUpdateTime=" + lastUpdateTime + ", params=" + params + ", resources=" + resources + + ", jobType=" + jobType + + ", title=" + title + + ", createTime=" + createTime + + ", modifyTime=" + modifyTime + + ", modifyUser=" + modifyUser + + ", desc='" + desc + '\'' + '}'; } + + public String getKey() { + return key; + } + + public void setKey(String key) { + this.key = key; + } + + public String getJobType() { + return jobType; + } + + public void setJobType(String jobType) { + this.jobType = jobType; + } + + public String getTitle() { + return title; + } + + public void setTitle(String title) { + this.title = title; + } + + public void setDesc(String desc) { + this.desc = desc; + } + + public Long getCreateTime() { + return createTime; + } + + public void setCreateTime(Long createTime) { + this.createTime = createTime; + } + + public void setModifyUser(String modifyUser) { + this.modifyUser = modifyUser; + } + + public void setModifyTime(Long modifyTime) { + this.modifyTime = modifyTime; + } + + public String getLastUpdateTime() { + return lastUpdateTime; + } + + public void setLastUpdateTime(String lastUpdateTime) { + this.lastUpdateTime = lastUpdateTime; + } + + public String getAppTag() { + return appTag; + } + + public void setAppTag(String appTag) { + this.appTag = appTag; + } + + public String getBusinessTag() { + return businessTag; + } + + public void setBusinessTag(String businessTag) { + this.businessTag = businessTag; + } + + public String getEcConfTemplateId() { + return ecConfTemplateId; + } + + public void setEcConfTemplateId(String ecConfTemplateId) { + this.ecConfTemplateId = ecConfTemplateId; + } + + public String getEcConfTemplateName() { + return ecConfTemplateName; + } + + public void setEcConfTemplateName(String ecConfTemplateName) { + this.ecConfTemplateName = ecConfTemplateName; + } + + public void setDependencys(List dependencys) { + this.dependencys = dependencys; + } } diff --git a/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/project/DSSProject.java b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/project/DSSProject.java index 7595a45ec3..476f05f4af 100644 --- a/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/project/DSSProject.java +++ b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/entity/project/DSSProject.java @@ -17,6 +17,7 @@ package com.webank.wedatasphere.dss.common.entity.project; import java.util.Date; +import java.util.List; public class DSSProject implements Project { @@ -40,6 +41,29 @@ public class DSSProject implements Project { private Boolean editable; + /** + * 1-接入git 0-不接入（默认） + */ + private Boolean associateGit; + + + /** + * 发布用户 list + */ + private List releaseUsers; + + /** + * 编辑用户 list + */ + private List editUsers; + + /** + * 查看用户 list + */ + private List accessUsers; + + + private boolean isAdmin; @Override public Long getId() { @@ -169,6 +193,56 @@ public String toString() { ", workspaceId=" + workspaceId + ", workspaceName='" + workspaceName + '\'' + ", editable=" + editable + + ", associateGit=" + associateGit + '}'; } + + public Boolean getAssociateGit() { + return associateGit; + } + + public void setAssociateGit(Boolean associateGit) { + this.associateGit = associateGit; + } + + public Boolean getEditable() { + return editable; + } + + public void setEditable(Boolean editable) { + this.editable = editable; + } + + public List getReleaseUsers() { + return releaseUsers; + } + + public void setReleaseUsers(List releaseUsers) { + this.releaseUsers = releaseUsers; + } + + public List getEditUsers() { + return editUsers; + } + + public void setEditUsers(List editUsers) { + this.editUsers = editUsers; + } + + public List getAccessUsers() { + return accessUsers; + } + + public void setAccessUsers(List accessUsers) { + this.accessUsers = accessUsers; + } + + + public boolean isAdmin() { + return isAdmin; + } + + public void setAdmin(boolean admin) { + isAdmin = admin; + } } diff --git a/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/protocol/project/ProjectInfoListRequest.java b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/protocol/project/ProjectInfoListRequest.java new file mode 100644 index 0000000000..a6e8540e09 --- /dev/null +++ b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/protocol/project/ProjectInfoListRequest.java @@ -0,0 +1,15 @@ +package com.webank.wedatasphere.dss.common.protocol.project; + +import java.util.List; + +public class ProjectInfoListRequest { + private List projectNames; + + public List getProjectNames() { + return projectNames; + } + + public void setProjectNames(List projectNames) { + this.projectNames = projectNames; + } +} diff --git a/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/protocol/project/ProjectInfoListResponse.java b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/protocol/project/ProjectInfoListResponse.java new file mode 100644 index 0000000000..901ace8057 --- /dev/null +++ b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/protocol/project/ProjectInfoListResponse.java @@ -0,0 +1,24 @@ +package com.webank.wedatasphere.dss.common.protocol.project; + +import com.webank.wedatasphere.dss.common.entity.project.DSSProject; + +import java.util.List; + +public class ProjectInfoListResponse { + private List dssProjects; + + public ProjectInfoListResponse(List dssProjects) { + this.dssProjects = dssProjects; + } + + public ProjectInfoListResponse() { + } + + public List getDssProjects() { + return dssProjects; + } + + public void setDssProjects(List dssProjects) { + this.dssProjects = dssProjects; + } +} diff --git a/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/protocol/project/ProjectListQueryRequest.java b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/protocol/project/ProjectListQueryRequest.java new file mode 100644 index 0000000000..a2df139b85 --- /dev/null +++ b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/protocol/project/ProjectListQueryRequest.java @@ -0,0 +1,32 @@ +package com.webank.wedatasphere.dss.common.protocol.project; + +public class ProjectListQueryRequest { + + private Long workspaceId; + + private String username; + + // ！！！ 新增字段 也不要更改这个构造函数 + public ProjectListQueryRequest(Long workspaceId, String username) { + this.workspaceId = workspaceId; + this.username = username; + } + + + + public Long getWorkspaceId() { + return workspaceId; + } + + public void setWorkspaceId(Long workspaceId) { + this.workspaceId = workspaceId; + } + + public String getUsername() { + return username; + } + + public void setUsername(String username) { + this.username = username; + } +} diff --git a/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/protocol/project/ProjectListQueryResponse.java b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/protocol/project/ProjectListQueryResponse.java new file mode 100644 index 0000000000..d726be19a8 --- /dev/null +++ b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/protocol/project/ProjectListQueryResponse.java @@ -0,0 +1,28 @@ +package com.webank.wedatasphere.dss.common.protocol.project; + +import com.webank.wedatasphere.dss.common.entity.project.DSSProject; + +import java.util.ArrayList; +import java.util.List; + +public class ProjectListQueryResponse { + + private List projectList; + + + // ！！！ 新增字段 也不要更改这个构造函数 + public ProjectListQueryResponse(List projectList) { + this.projectList = projectList; + } + + public ProjectListQueryResponse() { + } + + public List getProjectList() { + return projectList; + } + + public void setProjectList(List projectList) { + this.projectList = projectList; + } +} diff --git a/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/utils/DSSCommonUtils.java b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/utils/DSSCommonUtils.java index fd04be0f72..cedd481e19 100644 --- a/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/utils/DSSCommonUtils.java +++ b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/utils/DSSCommonUtils.java @@ -27,6 +27,7 @@ import java.util.Date; import java.util.List; +import java.util.Map; import java.util.stream.Collectors; public class DSSCommonUtils { @@ -65,10 +66,12 @@ public class DSSCommonUtils { public static final String ENV_LABEL_VALUE_DEV = "dev"; + public static final String ENV_LABEL_VALUE_PROD = "prod"; + public static final String DSS_LABELS_KEY = "labels"; public static final String DSS_EXECUTE_BY_PROXY_USER_KEY = "execByProxyUser"; - public static final CommonVars DSS_HOME = CommonVars.apply("DSS_HOME", ""); + public static final CommonVars DSS_HOME = CommonVars.apply("DSS_HOME", "/appcom/Install/dss"); public static long parseToLong(Object val) { if (val instanceof Double) { @@ -92,6 +95,15 @@ public static List getWorkFlowNodes(String workFlowJson) { return dwsNodes; } + public static List> getFlowAttribute(String workFlowJson, String attribute) { + JsonParser parser = new JsonParser(); + JsonObject jsonObject = parser.parse(workFlowJson).getAsJsonObject(); + JsonArray nodeJsonArray = jsonObject.getAsJsonArray(attribute); + List> mapList = DSSCommonUtils.COMMON_GSON.fromJson(nodeJsonArray, new TypeToken>>() { + }.getType()); + return mapList; + } + public static List getWorkFlowNodesJson(String workFlowJson) { JsonParser parser = new JsonParser(); JsonObject jsonObject = parser.parse(workFlowJson).getAsJsonObject(); diff --git a/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/utils/IoUtils.java b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/utils/IoUtils.java index ab0610e618..555972a13e 100644 --- a/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/utils/IoUtils.java +++ b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/utils/IoUtils.java @@ -24,9 +24,7 @@ import java.io.*; import java.text.SimpleDateFormat; -import java.util.Arrays; -import java.util.Date; -import java.util.Properties; +import java.util.*; /** * IO通用类。提供对一次IO操作的临时目录创建、文件创建、IO流创建功能。 @@ -37,6 +35,7 @@ public class IoUtils { private static Logger logger = LoggerFactory.getLogger(IoUtils.class); private static final String DATE_FORMAT = "yyyyMMddHHmmssSSS"; private static final String DEFAULT_IO_FILE_NAME = "IO.properties"; + public static final String FLOW_META_DIRECTORY_NAME = ".metaConf"; /** * 生成一个基础目录，这个目录用于某次任务的临时文件目录 @@ -51,7 +50,62 @@ public static String generateIOPath(String userName,String projectName,String su return addFileSeparator(baseUrl, dataStr, userName, projectName, subDir); } - private static String addFileSeparator(String... str) { + public static List getSubdirectoriesNames(String directoryPath) { + File directory = new File(directoryPath); + File[] subdirectories = directory.listFiles(File::isDirectory); // 使用文件过滤器获取只有目录的File数组 + + List subdirectoriesNames = new ArrayList<>(); + if (subdirectories != null) { + for (File subdirectory : subdirectories) { + subdirectoriesNames.add(subdirectory.getName()); // 将目录名称添加到列表中 + } + } + return subdirectoriesNames; + } + + /** + * 生成一个临时目录 + * @param userName 用户名 + * @return 临时目录 + */ + public static String generateTempIOPath(String userName){ + String baseUrl = DSSCommonConf.DSS_EXPORT_URL.getValue(); + String dataStr = new SimpleDateFormat(DATE_FORMAT).format(new Date()); + return addFileSeparator(baseUrl, dataStr, userName); + } + + /** + * 生产一个项目文件夹 + * @param userName + * @param projectName + * @return + */ + public static String generateProjectIOPath(String userName,String projectName){ + String baseUrl = DSSCommonConf.DSS_EXPORT_URL.getValue(); + String dataStr = new SimpleDateFormat(DATE_FORMAT).format(new Date()); + return addFileSeparator(baseUrl, dataStr, userName, projectName); + } + + /** + * 生成一个工作流元数据目录，这个目录用于某次任务的临时文件目录 + */ + public static String generateFlowMetaIOPath(String projectPath,String flowName) { + String baseUrl = DSSCommonConf.DSS_EXPORT_URL.getValue(); + String dataStr = new SimpleDateFormat(DATE_FORMAT).format(new Date()); + return addFileSeparator(projectPath, FLOW_META_DIRECTORY_NAME,flowName); + } + + + /** + * 生成一个工作流代码目录，这个目录用于某次任务的临时文件目录 + */ + public static String generateFlowCodeIOPath(String projectPath,String flowName) { + String baseUrl = DSSCommonConf.DSS_EXPORT_URL.getValue(); + String dataStr = new SimpleDateFormat(DATE_FORMAT).format(new Date()); + return addFileSeparator(projectPath, flowName); + } + + public static String addFileSeparator(String... str) { return Arrays.stream(str).reduce((a, b) -> a + File.separator + b).orElse(""); } diff --git a/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/utils/ZipHelper.java b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/utils/ZipHelper.java index 55652b6fbd..766d3996b1 100644 --- a/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/utils/ZipHelper.java +++ b/dss-commons/dss-common/src/main/java/com/webank/wedatasphere/dss/common/utils/ZipHelper.java @@ -24,11 +24,13 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.BufferedReader; -import java.io.File; -import java.io.InputStreamReader; +import java.io.*; import java.util.ArrayList; +import java.util.Enumeration; import java.util.List; +import java.util.zip.ZipEntry; +import java.util.zip.ZipFile; +import java.util.zip.ZipInputStream; public class ZipHelper { @@ -126,8 +128,53 @@ public static String zip(String dirPath, boolean deleteOriginDir)throws DSSError * @return 解压后的文件夹名 * @throws DSSErrorException 解压出现异常 */ - - public static String unzip(String dirPath)throws DSSErrorException { + @Deprecated + public static String unzip(String dirPath)throws DSSErrorException{ + return unzip(dirPath, false); + } + /** + * 解压一个zip文件 + * @param zipFilePath zip文件的全路径名 + * @destDirectory 解压到的目的地 + * @return 解压后的文件夹名 + * @throws DSSErrorException 解压出现异常 + */ + public static void unzipFile(String zipFilePath, String destDirectory, boolean deleteOriginZip) throws DSSErrorException { + File destDir = new File(destDirectory); + // 使用 try-with-resources 确保资源被自动关闭 + File zipFile = new File(zipFilePath); + try (ZipInputStream zis = new ZipInputStream(new FileInputStream(zipFilePath))) { + byte[] buffer = new byte[1024]; + ZipEntry zipEntry = zis.getNextEntry(); + while (zipEntry != null) { + File newFile = new File(destDir, zipEntry.getName()); + if (zipEntry.isDirectory()) { + newFile.mkdirs(); + } else { + // 为解压后的文件创建文件输出流 + try (FileOutputStream fos = new FileOutputStream(newFile)) { + int len; + while ((len = zis.read(buffer)) > 0) { + fos.write(buffer, 0, len); + } + } + } + zipEntry = zis.getNextEntry(); + } + zis.closeEntry(); + }catch(final Exception e){ + logger.error(" 解压缩 zip 文件失败, reason: ", e); + DSSErrorException exception = new DSSErrorException(90009, "unzip" + zipFilePath + " to " + destDirectory + "zip file failed"); + exception.initCause(e); + throw exception; + } + // 如果指定删除原zip文件 + if (deleteOriginZip) { + new File(zipFilePath).delete(); + } + } + @Deprecated + public static String unzip(String dirPath,boolean deleteOriginZip)throws DSSErrorException { File file = new File(dirPath); if(!file.exists()){ logger.error("{} 不存在, 不能解压zip文件", dirPath); @@ -166,6 +213,9 @@ public static String unzip(String dirPath)throws DSSErrorException { if (exitCode != 0){ throw new DSSErrorException(90007,errMsg.toString()); } + if(deleteOriginZip){ + file.delete(); + } }catch(final Exception e){ logger.error("{} 解压缩 zip 文件失败, reason: ", e); DSSErrorException exception = new DSSErrorException(90009,dirPath + " to zip file failed"); @@ -179,8 +229,21 @@ public static String unzip(String dirPath)throws DSSErrorException { } return longZipFilePath; } + public static String readImportZipProjectName(String zipFilePath) throws IOException { + try(ZipFile zipFile =new ZipFile(zipFilePath)){ + Enumeration entries =zipFile.entries(); + if(entries.hasMoreElements()){ + String projectName=entries.nextElement().getName(); + while (projectName.endsWith(File.separator)){ + projectName = projectName.substring(0, projectName.length() - 1); + } + return projectName; + } + } + throw new IOException(); + } - private static boolean deleteDir(File dir) { + public static boolean deleteDir(File dir) { if (dir.isDirectory()) { String[] children = dir.list(); if (children != null && children.length > 0){ diff --git a/dss-commons/dss-common/src/main/scala/com/webank/wedatasphere/dss/common/protocol/DSSProtocolObject.scala b/dss-commons/dss-common/src/main/scala/com/webank/wedatasphere/dss/common/protocol/DSSProtocolObject.scala index 41e6b87c8b..bd2106a200 100644 --- a/dss-commons/dss-common/src/main/scala/com/webank/wedatasphere/dss/common/protocol/DSSProtocolObject.scala +++ b/dss-commons/dss-common/src/main/scala/com/webank/wedatasphere/dss/common/protocol/DSSProtocolObject.scala @@ -41,11 +41,35 @@ case class RequestExportWorkflow(userName: String, projectId: Long, projectName: String, workspaceStr: String, - dssLabelList: java.util.List[DSSLabel] + dssLabelList: java.util.List[DSSLabel], + exportExternalNodeAppConnResource: Boolean ) +case class RequestExportWorkflowList(userName: String, + flowIDList: java.util.List[java.lang.Long], + projectId: Long, + projectName: String, + workspaceStr: String, + dssLabelList: java.util.List[DSSLabel], + exportExternalNodeAppConnResource: Boolean + ) + +case class RequestReadWorkflowNode(userName: String, + flowID: Long, + projectId: Long, + projectName: String, + workspaceStr: String, + dssLabelList: java.util.List[DSSLabel], + exportExternalNodeAppConnResource: Boolean, + filePath: String + ) + case class ResponseExportWorkflow(resourceId: String, version: String, flowID: Long) +case class ResponseExportWorkflowList(resourceId: String, version: String, flowID: java.util.List[java.lang.Long]) + +case class ResponseReadWorkflow(fileContent: String) + case class RequestQueryWorkFlow(userName: String, rootFlowId: Long) case class ProxyUserCheckRequest(userName: String, proxyUser: String) diff --git a/dss-commons/dss-contextservice/pom.xml b/dss-commons/dss-contextservice/pom.xml index e72f0c6728..32f399e3bc 100644 --- a/dss-commons/dss-contextservice/pom.xml +++ b/dss-commons/dss-contextservice/pom.xml @@ -21,7 +21,7 @@ dss-commons com.webank.wedatasphere.dss - 1.1.0.20-SNAPSHOT + 1.10.0-SNAPSHOT ../pom.xml dss-contextservice diff --git a/dss-commons/dss-sender-service/pom.xml b/dss-commons/dss-sender-service/pom.xml index ba32deb186..471ab7aca7 100644 --- a/dss-commons/dss-sender-service/pom.xml +++ b/dss-commons/dss-sender-service/pom.xml @@ -21,7 +21,7 @@ dss-commons com.webank.wedatasphere.dss - 1.1.0.20-SNAPSHOT + 1.10.0-SNAPSHOT ../pom.xml 4.0.0 @@ -56,6 +56,12 @@ com.webank.wedatasphere.dss dss-development-process-standard ${dss.version} + + + javax.servlet + javax.servlet-api + + com.webank.wedatasphere.dss diff --git a/dss-commons/dss-sender-service/src/main/java/com/webank/wedatasphere/dss/sender/service/DSSSenderService.java b/dss-commons/dss-sender-service/src/main/java/com/webank/wedatasphere/dss/sender/service/DSSSenderService.java index 212d631ec1..4aec347e11 100644 --- a/dss-commons/dss-sender-service/src/main/java/com/webank/wedatasphere/dss/sender/service/DSSSenderService.java +++ b/dss-commons/dss-sender-service/src/main/java/com/webank/wedatasphere/dss/sender/service/DSSSenderService.java @@ -36,5 +36,7 @@ public interface DSSSenderService { Sender getProjectServerSender(); + Sender getGitSender(); + } diff --git a/dss-commons/dss-sender-service/src/main/java/com/webank/wedatasphere/dss/sender/service/conf/DSSSenderServiceConf.java b/dss-commons/dss-sender-service/src/main/java/com/webank/wedatasphere/dss/sender/service/conf/DSSSenderServiceConf.java index ff33f617c0..37fd3a90ca 100644 --- a/dss-commons/dss-sender-service/src/main/java/com/webank/wedatasphere/dss/sender/service/conf/DSSSenderServiceConf.java +++ b/dss-commons/dss-sender-service/src/main/java/com/webank/wedatasphere/dss/sender/service/conf/DSSSenderServiceConf.java @@ -28,6 +28,9 @@ public class DSSSenderServiceConf { public static final CommonVars PROJECT_SERVER_NAME = CommonVars.apply("wds.dss.project.sever.name", "dss-framework-project-server"); + public static final CommonVars GIT_SERVER_NAME = + CommonVars.apply("wds.dss.git.sever.name", "dss-framework-git-server"); + public static final CommonVars DSS_SERVER_NAME = CommonVars.apply("wds.dss.sever.name.dev", "dss-server-dev"); diff --git a/dss-commons/dss-sender-service/src/main/java/com/webank/wedatasphere/dss/sender/service/impl/DSSSenderServiceImpl.java b/dss-commons/dss-sender-service/src/main/java/com/webank/wedatasphere/dss/sender/service/impl/DSSSenderServiceImpl.java index ff403f86ca..3723705716 100644 --- a/dss-commons/dss-sender-service/src/main/java/com/webank/wedatasphere/dss/sender/service/impl/DSSSenderServiceImpl.java +++ b/dss-commons/dss-sender-service/src/main/java/com/webank/wedatasphere/dss/sender/service/impl/DSSSenderServiceImpl.java @@ -29,6 +29,8 @@ public class DSSSenderServiceImpl implements DSSSenderService { private final Sender workflowSender = Sender.getSender(DSSSenderServiceConf.DSS_WORKFLOW_APPLICATION_NAME_DEV.getValue()); private final Sender projectSender = Sender.getSender(DSSSenderServiceConf.PROJECT_SERVER_NAME.getValue()); + + private final Sender gitSender = Sender.getSender(DSSSenderServiceConf.GIT_SERVER_NAME.getValue()); @Override public Sender getOrcSender() { return orcSender; @@ -64,4 +66,9 @@ public Sender getProjectServerSender() { return projectSender; } + @Override + public Sender getGitSender() { + return gitSender; + } + } diff --git a/dss-commons/dss-sender-service/src/main/java/com/webank/wedatasphere/dss/sender/service/impl/DSSServerSenderServiceImpl.java b/dss-commons/dss-sender-service/src/main/java/com/webank/wedatasphere/dss/sender/service/impl/DSSServerSenderServiceImpl.java index 4ebe11e5fe..58046caf8a 100644 --- a/dss-commons/dss-sender-service/src/main/java/com/webank/wedatasphere/dss/sender/service/impl/DSSServerSenderServiceImpl.java +++ b/dss-commons/dss-sender-service/src/main/java/com/webank/wedatasphere/dss/sender/service/impl/DSSServerSenderServiceImpl.java @@ -44,4 +44,9 @@ public Sender getSchedulerWorkflowSender() { public Sender getProjectServerSender() { return dssServerSender; } + + @Override + public Sender getGitSender() { + return dssServerSender; + } } diff --git a/dss-commons/pom.xml b/dss-commons/pom.xml index f719aa12c4..29854135a6 100644 --- a/dss-commons/pom.xml +++ b/dss-commons/pom.xml @@ -21,7 +21,7 @@ dss com.webank.wedatasphere.dss - 1.1.0.20-SNAPSHOT + 1.10.0-SNAPSHOT ../pom.xml 4.0.0 diff --git a/dss-framework/dss-appconn-framework/pom.xml b/dss-framework/dss-appconn-framework/pom.xml index be1f80ef56..a46e2809ac 100644 --- a/dss-framework/dss-appconn-framework/pom.xml +++ b/dss-framework/dss-appconn-framework/pom.xml @@ -21,7 +21,7 @@ dss com.webank.wedatasphere.dss - 1.1.0.20-SNAPSHOT + 1.10.0-SNAPSHOT ../../pom.xml 4.0.0 diff --git a/dss-framework/dss-appconn-framework/src/main/java/com/webank/wedatasphere/dss/framework/appconn/restful/AppConn2LinkisRefExecutionRestfulApi.java b/dss-framework/dss-appconn-framework/src/main/java/com/webank/wedatasphere/dss/framework/appconn/restful/AppConn2LinkisRefExecutionRestfulApi.java index b505761062..1992ff816e 100644 --- a/dss-framework/dss-appconn-framework/src/main/java/com/webank/wedatasphere/dss/framework/appconn/restful/AppConn2LinkisRefExecutionRestfulApi.java +++ b/dss-framework/dss-appconn-framework/src/main/java/com/webank/wedatasphere/dss/framework/appconn/restful/AppConn2LinkisRefExecutionRestfulApi.java @@ -13,6 +13,7 @@ import com.webank.wedatasphere.dss.standard.app.sso.Workspace; import com.webank.wedatasphere.dss.standard.common.desc.AppInstance; import com.webank.wedatasphere.dss.standard.common.desc.AppInstanceImpl; +import com.webank.wedatasphere.dss.standard.common.exception.NoSuchAppInstanceException; import org.apache.linkis.server.Message; import org.apache.linkis.server.security.SecurityFilter; import org.slf4j.Logger; @@ -41,7 +42,7 @@ public class AppConn2LinkisRefExecutionRestfulApi { private AppConnManagerRestfulApi appConnManagerRestfulApi; @RequestMapping(path = "execute", method = RequestMethod.POST) - public Message execute(HttpServletRequest request, @RequestBody Map json) { + public Message execute(HttpServletRequest request, @RequestBody Map json) throws NoSuchAppInstanceException { String userName = SecurityFilter.getLoginUsername(request); LOGGER.info("user {} try to transform jobContent to Linkis job with requestBody {}.", userName, json); String workspaceStr = (String) json.get("workspaceStr"); diff --git a/dss-framework/dss-framework-admin-service/pom.xml b/dss-framework/dss-framework-admin-service/pom.xml index 901db373bc..b6ea319315 100644 --- a/dss-framework/dss-framework-admin-service/pom.xml +++ b/dss-framework/dss-framework-admin-service/pom.xml @@ -4,7 +4,7 @@ dss com.webank.wedatasphere.dss - 1.1.0.20-SNAPSHOT + 1.10.0-SNAPSHOT ../../pom.xml diff --git a/dss-framework/dss-framework-common/pom.xml b/dss-framework/dss-framework-common/pom.xml index 942384562a..ebfb3d97ae 100644 --- a/dss-framework/dss-framework-common/pom.xml +++ b/dss-framework/dss-framework-common/pom.xml @@ -21,7 +21,7 @@ dss com.webank.wedatasphere.dss - 1.1.0.20-SNAPSHOT + 1.10.0-SNAPSHOT ../../pom.xml 4.0.0 diff --git a/dss-framework/dss-framework-orchestrator-server/pom.xml b/dss-framework/dss-framework-orchestrator-server/pom.xml index 48b2caf90a..be2a763c7b 100644 --- a/dss-framework/dss-framework-orchestrator-server/pom.xml +++ b/dss-framework/dss-framework-orchestrator-server/pom.xml @@ -21,7 +21,7 @@ dss com.webank.wedatasphere.dss - 1.1.0.20-SNAPSHOT + 1.10.0-SNAPSHOT ../../pom.xml 4.0.0 @@ -29,6 +29,36 @@ dss-framework-orchestrator-server + + com.webank.wedatasphere.dss + dss-git-common + ${dss.version} + + + org.seleniumhq.selenium + selenium-java + 3.9.1 + + + com.google.guava + guava + + + httpclient + org.apache.httpcomponents + + + + + com.webank.wedatasphere.dss + dss-git-common + ${dss.version} + + + com.webank.wedatasphere.dss + dss-workflow-server + ${dss.version} + com.webank.wedatasphere.dss dss-framework-common @@ -197,6 +227,35 @@ + + + com.webank.wedatasphere.dss + dss-framework-workspace-server + ${dss.version} + + + linkis-common + org.apache.linkis + + + commons-logging + commons-logging + + + commons-codec + commons-codec + + + commons-lang3 + org.apache.commons + + + json4s-jackson_2.11 + org.json4s + + + + diff --git a/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/conf/OrchestratorConf.java b/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/conf/OrchestratorConf.java index 4ab4f98671..a8b51fa8b1 100644 --- a/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/conf/OrchestratorConf.java +++ b/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/conf/OrchestratorConf.java @@ -23,6 +23,6 @@ public class OrchestratorConf { public static final CommonVars DSS_UPLOAD_PATH = CommonVars.apply("wds.dss.file.upload.dir", "/appcom/tmp/uploads"); public static final CommonVars DSS_PUBLISH_MAX_VERSION = CommonVars.apply("wds.dss.publish.max.remain.version", 30); public static final CommonVars DSS_CS_CLEAR_ENV = CommonVars.apply("wds.dss.server.cs.clear.env", "DEV"); - public static final CommonVars DSS_CS_CLEAR_CRON = CommonVars.apply("wds.dss.server.scheduling.clear.cs.cron", ""); + public static final CommonVars DSS_CS_CLEAR_CRON = CommonVars.apply("wds.dss.server.scheduling.clear.cs.cron", "0 0 0/6 * * ?"); } diff --git a/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/conf/OrchestratorSpringConf.java b/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/conf/OrchestratorSpringConf.java index 40fe3495a7..ca259265f1 100644 --- a/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/conf/OrchestratorSpringConf.java +++ b/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/conf/OrchestratorSpringConf.java @@ -16,6 +16,7 @@ package com.webank.wedatasphere.dss.orchestrator.server.conf; +import com.google.common.util.concurrent.ThreadFactoryBuilder; import com.webank.wedatasphere.dss.common.service.BMLService; import com.webank.wedatasphere.dss.contextservice.service.ContextService; import com.webank.wedatasphere.dss.contextservice.service.impl.ContextServiceImpl; @@ -42,9 +43,9 @@ public BMLService createBmlService() { } @Bean - public ScheduledThreadPoolExecutor scheduledExecutorService() { - ScheduledThreadPoolExecutor executor = new ScheduledThreadPoolExecutor(3); - return executor; + public ScheduledThreadPoolExecutor dssDefaultScheduledExecutor() { + return new ScheduledThreadPoolExecutor(30, + new ThreadFactoryBuilder().setNameFormat("Dss-Default-Spring-Scheduler-Thread-%d").build()); } @Bean diff --git a/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/constant/DSSOrchestratorConstant.java b/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/constant/DSSOrchestratorConstant.java index 9fe2c841fd..9d51ee2aec 100644 --- a/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/constant/DSSOrchestratorConstant.java +++ b/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/constant/DSSOrchestratorConstant.java @@ -21,4 +21,11 @@ public class DSSOrchestratorConstant { //每次提交linkis清理数量控制在10个以下，超了接口就会返回超时 public static final int MAX_CLEAR_SIZE = 10; + public static final String CHROME_DRIVER_PATH = "chromedriver"; + + + // 升序 + public static final String ASCEND="ascend"; + // 降序 + public static final String DESCEND="descend"; } diff --git a/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/constant/OrchestratorStatusEnum.java b/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/constant/OrchestratorStatusEnum.java new file mode 100644 index 0000000000..19b49c469e --- /dev/null +++ b/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/constant/OrchestratorStatusEnum.java @@ -0,0 +1,73 @@ +package com.webank.wedatasphere.dss.orchestrator.server.constant; + +import com.webank.wedatasphere.dss.orchestrator.common.ref.OrchestratorRefConstant; +import com.webank.wedatasphere.dss.orchestrator.server.entity.vo.OrchestratorStatusVo; + +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +public enum OrchestratorStatusEnum { + SAVE(OrchestratorRefConstant.FLOW_STATUS_SAVE, "待提交"), + PUSH(OrchestratorRefConstant.FLOW_STATUS_PUSH, "待发布"), + PUBLISH(OrchestratorRefConstant.FLOW_STATUS_PUBLISH, "已发布"), + RUNNING(OrchestratorRefConstant.FLOW_STATUS_PUSHING, "提交中"), + PUBLISHING(OrchestratorRefConstant.FLOW_STATUS_PUBLISHING, "发布中"), + STATELESS(OrchestratorRefConstant.FLOW_STATUS_STATELESS, "--"), + SUCCESS(OrchestratorRefConstant.FLOW_STATUS_PUSH_SUCCESS,"发布成功"), + UNPUBLISHED(OrchestratorRefConstant.FLOW_STATUS_UNPUBLISHED,"未发布"), + FAILED(OrchestratorRefConstant.FLOW_STATUS_PUSH_FAILED, "发布失败"); + + + private String status; + private String name; + + OrchestratorStatusEnum(String status, String name) { + this.status = status; + this.name = name; + } + + + public static OrchestratorStatusEnum getEnum(String status) { + if (status == null) { + return STATELESS; + } + for (OrchestratorStatusEnum statusEnum : values()) { + if (statusEnum.getStatus().equals(status)) { + return statusEnum; + } + } + return STATELESS; + } + + + public String getStatus() { + return status; + } + + public void setStatus(String status) { + this.status = status; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + + public static List getOrchestratorGitStatus() { + + return Stream.of( + new OrchestratorStatusVo(PUSH.getStatus(), PUSH.getName()), + new OrchestratorStatusVo(SAVE.getStatus(), SAVE.getName()), + new OrchestratorStatusVo(PUBLISHING.getStatus(), PUBLISHING.getName()), + new OrchestratorStatusVo(RUNNING.getStatus(), RUNNING.getName()), + new OrchestratorStatusVo(PUBLISH.getStatus(),PUBLISH.getName()), + new OrchestratorStatusVo(STATELESS.getStatus(), STATELESS.getName()) + ).collect(Collectors.toList()); + } + +} diff --git a/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/entity/request/BatchPublishFlowCheckRequest.java b/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/entity/request/BatchPublishFlowCheckRequest.java new file mode 100644 index 0000000000..46639ec49a --- /dev/null +++ b/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/entity/request/BatchPublishFlowCheckRequest.java @@ -0,0 +1,17 @@ +package com.webank.wedatasphere.dss.orchestrator.server.entity.request; + +import java.util.List; + +public class BatchPublishFlowCheckRequest { + + private List orchestratorIdList; + + + public List getOrchestratorIdList() { + return orchestratorIdList; + } + + public void setOrchestratorIdList(List orchestratorIdList) { + this.orchestratorIdList = orchestratorIdList; + } +} diff --git a/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/entity/request/ModifyOrchestratorMetaRequest.java b/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/entity/request/ModifyOrchestratorMetaRequest.java new file mode 100644 index 0000000000..e84e8c4254 --- /dev/null +++ b/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/entity/request/ModifyOrchestratorMetaRequest.java @@ -0,0 +1,126 @@ +package com.webank.wedatasphere.dss.orchestrator.server.entity.request; + +import javax.validation.constraints.NotNull; +public class ModifyOrchestratorMetaRequest { + + // 编排ID + @NotNull(message = "编排Id不能为空") + private Long orchestratorId; + + // 工作流ID + @NotNull(message = "工作流Id不能为空") + private String uuid; + + // 编排名称 + @NotNull(message = "工作流名称不能为空") + private String orchestratorName; + + // 描述 + @NotNull(message = "工作流描述不能为空") + private String description; + + // 命名空间ID + private Long workspaceId; + + // 命名空间名称 + private String workspaceName; + + // 项目ID + @NotNull(message = "项目Id不能为空") + private Long projectId; + + // 项目名称 + private String projectName; + + // 是否默认引用资源参数模板 + private String isDefaultReference; + + // 代理用户 + private String proxyUser; + + + public ModifyOrchestratorMetaRequest() { + } + + public Long getOrchestratorId() { + return orchestratorId; + } + + public void setOrchestratorId(Long orchestratorId) { + this.orchestratorId = orchestratorId; + } + + public String getUuid() { + return uuid; + } + + public void setUuid(String uuid) { + this.uuid = uuid; + } + + public String getOrchestratorName() { + return orchestratorName; + } + + public void setOrchestratorName(String orchestratorName) { + this.orchestratorName = orchestratorName; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + + + public String getWorkspaceName() { + return workspaceName; + } + + public void setWorkspaceName(String workspaceName) { + this.workspaceName = workspaceName; + } + + + public String getProjectName() { + return projectName; + } + + public void setProjectName(String projectName) { + this.projectName = projectName; + } + + public String getIsDefaultReference() { + return isDefaultReference; + } + + public void setIsDefaultReference(String isDefaultReference) { + this.isDefaultReference = isDefaultReference; + } + + public String getProxyUser() { + return proxyUser; + } + + public void setProxyUser(String proxyUser) { + this.proxyUser = proxyUser; + } + + public Long getWorkspaceId() { + return workspaceId; + } + + public void setWorkspaceId(Long workspaceId) { + this.workspaceId = workspaceId; + } + + public Long getProjectId() { + return projectId; + } + + public void setProjectId(Long projectId) { + this.projectId = projectId; + } +} diff --git a/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/entity/request/OrchestratorBatchSubmitRequest.java b/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/entity/request/OrchestratorBatchSubmitRequest.java new file mode 100644 index 0000000000..5089c715eb --- /dev/null +++ b/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/entity/request/OrchestratorBatchSubmitRequest.java @@ -0,0 +1,49 @@ +package com.webank.wedatasphere.dss.orchestrator.server.entity.request; + + +import com.webank.wedatasphere.dss.common.label.LabelRouteVO; + +import java.util.List; + +public class OrchestratorBatchSubmitRequest { + private List submitRequestList; + private LabelRouteVO labels; + private String comment; + + public OrchestratorBatchSubmitRequest(List submitRequestList) { + this.submitRequestList = submitRequestList; + } + + public OrchestratorBatchSubmitRequest() { + } + + public OrchestratorBatchSubmitRequest(List submitRequestList, LabelRouteVO labels, String comment) { + this.submitRequestList = submitRequestList; + this.labels = labels; + this.comment = comment; + } + + public List getSubmitRequestList() { + return submitRequestList; + } + + public void setSubmitRequestList(List submitRequestList) { + this.submitRequestList = submitRequestList; + } + + public LabelRouteVO getLabels() { + return labels; + } + + public void setLabels(LabelRouteVO labels) { + this.labels = labels; + } + + public String getComment() { + return comment; + } + + public void setComment(String comment) { + this.comment = comment; + } +} diff --git a/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/entity/request/OrchestratorCreateRequest.java b/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/entity/request/OrchestratorCreateRequest.java index d374b668cd..7c63b2c592 100644 --- a/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/entity/request/OrchestratorCreateRequest.java +++ b/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/entity/request/OrchestratorCreateRequest.java @@ -52,6 +52,8 @@ public class OrchestratorCreateRequest extends OrchestratorRequest { private String workspaceName; + private String isDefaultReference; + public OrchestratorCreateRequest() { } @@ -130,6 +132,14 @@ public void setOrchestratorLevel(String orchestratorLevel) { this.orchestratorLevel = orchestratorLevel; } + public String getIsDefaultReference() { + return isDefaultReference; + } + + public void setIsDefaultReference(String isDefaultReference) { + this.isDefaultReference = isDefaultReference; + } + @Override public String toString() { return "OrchestratorCreateRequest{" + diff --git a/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/entity/request/OrchestratorModifyRequest.java b/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/entity/request/OrchestratorModifyRequest.java index fb62a9fac5..b38bc74d40 100644 --- a/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/entity/request/OrchestratorModifyRequest.java +++ b/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/entity/request/OrchestratorModifyRequest.java @@ -47,6 +47,8 @@ public class OrchestratorModifyRequest extends OrchestratorRequest { private List dssLabels; + private String isDefaultReference; + public Long getId() { return id; } @@ -117,4 +119,12 @@ public String getOrchestratorLevel() { public void setOrchestratorLevel(String orchestratorLevel) { this.orchestratorLevel = orchestratorLevel; } + + public String getIsDefaultReference() { + return isDefaultReference; + } + + public void setIsDefaultReference(String isDefaultReference) { + this.isDefaultReference = isDefaultReference; + } } diff --git a/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/entity/request/OrchestratorRequest.java b/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/entity/request/OrchestratorRequest.java index 3c22462cb6..10e70ba6ec 100644 --- a/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/entity/request/OrchestratorRequest.java +++ b/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/entity/request/OrchestratorRequest.java @@ -42,6 +42,14 @@ public class OrchestratorRequest { */ private LabelRouteVO labels; + public OrchestratorRequest() { + } + + public OrchestratorRequest(Long workspaceId, Long projectId) { + this.workspaceId = workspaceId; + this.projectId = projectId; + } + public LabelRouteVO getLabels() { return labels; } diff --git a/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/entity/request/OrchestratorSubmitRequest.java b/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/entity/request/OrchestratorSubmitRequest.java new file mode 100644 index 0000000000..fa71d1500b --- /dev/null +++ b/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/entity/request/OrchestratorSubmitRequest.java @@ -0,0 +1,83 @@ +package com.webank.wedatasphere.dss.orchestrator.server.entity.request; + +import com.webank.wedatasphere.dss.common.label.LabelRouteVO; + +public class OrchestratorSubmitRequest { + private Long flowId; + private LabelRouteVO labels; + private String projectName; + private String comment; + private Long orchestratorId; + private String filePath; + // 是否获取发布前后文件内容 true-发布 false-提交 + private Boolean publish; + + public OrchestratorSubmitRequest(Long flowId, LabelRouteVO labels, String projectName, String comment, Long orchestratorId, String filePath, Boolean publish) { + this.flowId = flowId; + this.labels = labels; + this.projectName = projectName; + this.comment = comment; + this.orchestratorId = orchestratorId; + this.filePath = filePath; + this.publish = publish; + } + + public OrchestratorSubmitRequest() { + } + + public Long getFlowId() { + return flowId; + } + + public void setFlowId(Long flowId) { + this.flowId = flowId; + } + + public LabelRouteVO getLabels() { + return labels; + } + + public void setLabels(LabelRouteVO labels) { + this.labels = labels; + } + + public String getProjectName() { + return projectName; + } + + public void setProjectName(String projectName) { + this.projectName = projectName; + } + + public String getComment() { + return comment; + } + + public void setComment(String comment) { + this.comment = comment; + } + + public Long getOrchestratorId() { + return orchestratorId; + } + + public void setOrchestratorId(Long orchestratorId) { + this.orchestratorId = orchestratorId; + } + + public String getFilePath() { + return filePath; + } + + public void setFilePath(String filePath) { + this.filePath = filePath; + } + + public Boolean getPublish() { + return publish; + } + + public void setPublish(Boolean publish) { + this.publish = publish; + } +} diff --git a/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/entity/vo/OrchestratorBaseInfo.java b/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/entity/vo/OrchestratorBaseInfo.java index 953307bec4..228ec22557 100644 --- a/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/entity/vo/OrchestratorBaseInfo.java +++ b/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/entity/vo/OrchestratorBaseInfo.java @@ -16,6 +16,9 @@ package com.webank.wedatasphere.dss.orchestrator.server.entity.vo; +import com.webank.wedatasphere.dss.orchestrator.common.entity.DSSOrchestratorInfo; + +import java.util.ArrayList; import java.util.Date; import java.util.List; @@ -34,6 +37,10 @@ public class OrchestratorBaseInfo { * 工程id */ private Long projectId; + /** + * 工作流的唯一id，不同环境里，同一个工作流保持一致，用来判断是否是同一个工作流。 + */ + private String uuid; /** * 编排模式id（工作流,调用orchestrator服务返回的orchestratorId） @@ -94,6 +101,8 @@ public class OrchestratorBaseInfo { private String orchestratorLevel; + private String isDefaultReference; + private boolean flowEditLockExist = false; public Boolean getEditable() { @@ -123,6 +132,7 @@ public void setReleasable(Boolean releasable) { */ private Boolean releasable; + private String status; public Long getId() { return id; @@ -148,6 +158,14 @@ public void setProjectId(Long projectId) { this.projectId = projectId; } + public String getUuid() { + return uuid; + } + + public void setUuid(String uuid) { + this.uuid = uuid; + } + public Long getOrchestratorId() { return orchestratorId; } @@ -251,4 +269,76 @@ public String getOrchestratorLevel() { public void setOrchestratorLevel(String orchestratorLevel) { this.orchestratorLevel = orchestratorLevel; } + + public String getIsDefaultReference() { + return isDefaultReference; + } + + public void setIsDefaultReference(String isDefaultReference) { + this.isDefaultReference = isDefaultReference; + } + + public static OrchestratorBaseInfo convertFrom(DSSOrchestratorInfo dssInfo){ + OrchestratorBaseInfo baseInfo = new OrchestratorBaseInfo(); + + // 直接对应的属性 + baseInfo.setOrchestratorVersionId(dssInfo.getId()); + baseInfo.setProjectId(dssInfo.getProjectId()); + baseInfo.setWorkspaceId(dssInfo.getWorkspaceId()); + baseInfo.setOrchestratorName(dssInfo.getName()); + baseInfo.setDescription(dssInfo.getDesc()); + baseInfo.setUses(dssInfo.getUses()); + baseInfo.setCreateUser(dssInfo.getCreator()); + baseInfo.setCreateTime(dssInfo.getCreateTime()); + baseInfo.setUpdateUser(dssInfo.getUpdateUser()); + baseInfo.setUpdateTime(dssInfo.getUpdateTime()); + + // 可能需要根据实际情况调整的属性 + baseInfo.setOrchestratorMode(dssInfo.getOrchestratorMode()); + baseInfo.setOrchestratorLevel(dssInfo.getOrchestratorLevel()); + + // DSSOrchestratorInfo中的orchestratorWay可能需要映射到OrchestratorBaseInfo的orchestratorWays列表中 + List orchestratorWays = new ArrayList<>(); + orchestratorWays.add(dssInfo.getOrchestratorWay()); + baseInfo.setOrchestratorWays(orchestratorWays); + return baseInfo; + } + + public DSSOrchestratorInfo convertToDSSOrchestratorInfo() { + OrchestratorBaseInfo baseInfo=this; + DSSOrchestratorInfo dssInfo = new DSSOrchestratorInfo(); + + // 直接对应的属性 + dssInfo.setId(baseInfo.getId()); + dssInfo.setProjectId(baseInfo.getProjectId()); + dssInfo.setWorkspaceId(baseInfo.getWorkspaceId()); + dssInfo.setName(baseInfo.getOrchestratorName()); + dssInfo.setDesc(baseInfo.getDescription()); + dssInfo.setUses(baseInfo.getUses()); + dssInfo.setCreator(baseInfo.getCreateUser()); + dssInfo.setCreateTime(baseInfo.getCreateTime()); + dssInfo.setUpdateUser(baseInfo.getUpdateUser()); + dssInfo.setUpdateTime(baseInfo.getUpdateTime()); + dssInfo.setUUID(baseInfo.getUuid()); + + // 可能需要根据实际情况调整的属性 + dssInfo.setOrchestratorMode(baseInfo.getOrchestratorMode()); + dssInfo.setOrchestratorLevel(baseInfo.getOrchestratorLevel()); + + // 处理没有直接对应的字段 + if (baseInfo.getOrchestratorWays() != null && !baseInfo.getOrchestratorWays().isEmpty()) { + // 取第一个作为orchestratorWay，这需要根据实际情况来决定是否合适 + dssInfo.setOrchestratorWay(baseInfo.getOrchestratorWays().get(0)); + } + + return dssInfo; + } + + public String getStatus() { + return status; + } + + public void setStatus(String status) { + this.status = status; + } } diff --git a/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/entity/vo/OrchestratorDiffDirVo.java b/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/entity/vo/OrchestratorDiffDirVo.java new file mode 100644 index 0000000000..cec34e281a --- /dev/null +++ b/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/entity/vo/OrchestratorDiffDirVo.java @@ -0,0 +1,43 @@ +package com.webank.wedatasphere.dss.orchestrator.server.entity.vo; + +import com.webank.wedatasphere.dss.git.common.protocol.GitTree; + +import java.util.List; + +public class OrchestratorDiffDirVo { + private List codeTree; + private List metaTree; + private String commitId; + + public OrchestratorDiffDirVo(List codeTree, List metaTree) { + this.codeTree = codeTree; + this.metaTree = metaTree; + } + + public OrchestratorDiffDirVo() { + } + + public List getCodeTree() { + return codeTree; + } + + public void setCodeTree(List codeTree) { + this.codeTree = codeTree; + } + + public List getMetaTree() { + return metaTree; + } + + public void setMetaTree(List metaTree) { + this.metaTree = metaTree; + } + + public String getCommitId() { + return commitId; + } + + public void setCommitId(String commitId) { + this.commitId = commitId; + } +} diff --git a/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/entity/vo/OrchestratorDiffNodeVo.java b/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/entity/vo/OrchestratorDiffNodeVo.java new file mode 100644 index 0000000000..c62a905261 --- /dev/null +++ b/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/entity/vo/OrchestratorDiffNodeVo.java @@ -0,0 +1,110 @@ +package com.webank.wedatasphere.dss.orchestrator.server.entity.vo; + +import java.util.List; + +public class OrchestratorDiffNodeVo { + + private String orchestratorName; + + private Long orchestratorId; + + private String projectName; + + private Long projectId; + + private List notContainsKeywordsNodeList; + + private String enabledFlowKeywordsCheck; + + private String workspaceName; + + private Long workspaceId; + + private Boolean associateGit; + + + public OrchestratorDiffNodeVo() { + } + + + public OrchestratorDiffNodeVo(String orchestratorName, Long orchestratorId, List notContainsKeywordsNodeList) { + this.orchestratorName = orchestratorName; + this.orchestratorId = orchestratorId; + this.notContainsKeywordsNodeList = notContainsKeywordsNodeList; + } + + public String getOrchestratorName() { + return orchestratorName; + } + + public void setOrchestratorName(String orchestratorName) { + + this.orchestratorName = orchestratorName; + } + + + public Long getOrchestratorId() { + return orchestratorId; + } + + public void setOrchestratorId(Long orchestratorId) { + this.orchestratorId = orchestratorId; + } + + public List getNotContainsKeywordsNodeList() { + return notContainsKeywordsNodeList; + } + + public void setNotContainsKeywordsNodeList(List notContainsKeywordsNodeList) { + this.notContainsKeywordsNodeList = notContainsKeywordsNodeList; + } + + public String getProjectName() { + return projectName; + } + + public void setProjectName(String projectName) { + this.projectName = projectName; + } + + public Long getProjectId() { + return projectId; + } + + public void setProjectId(Long projectId) { + this.projectId = projectId; + } + + + public String getEnabledFlowKeywordsCheck() { + return enabledFlowKeywordsCheck; + } + + public void setEnabledFlowKeywordsCheck(String enabledFlowKeywordsCheck) { + this.enabledFlowKeywordsCheck = enabledFlowKeywordsCheck; + } + + public String getWorkspaceName() { + return workspaceName; + } + + public void setWorkspaceName(String workspaceName) { + this.workspaceName = workspaceName; + } + + public Long getWorkspaceId() { + return workspaceId; + } + + public void setWorkspaceId(Long workspaceId) { + this.workspaceId = workspaceId; + } + + public Boolean getAssociateGit() { + return associateGit; + } + + public void setAssociateGit(Boolean associateGit) { + this.associateGit = associateGit; + } +} diff --git a/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/entity/vo/OrchestratorRelationVo.java b/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/entity/vo/OrchestratorRelationVo.java new file mode 100644 index 0000000000..17387b6617 --- /dev/null +++ b/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/entity/vo/OrchestratorRelationVo.java @@ -0,0 +1,30 @@ +package com.webank.wedatasphere.dss.orchestrator.server.entity.vo; + +public class OrchestratorRelationVo { + private Long orchestratorId; + private Long flowId; + + public OrchestratorRelationVo() { + } + + public OrchestratorRelationVo(Long flowId, Long orchestratorId) { + this.flowId = flowId; + this.orchestratorId = orchestratorId; + } + + public Long getOrchestratorId() { + return orchestratorId; + } + + public void setOrchestratorId(Long orchestratorId) { + this.orchestratorId = orchestratorId; + } + + public Long getFlowId() { + return flowId; + } + + public void setFlowId(Long flowId) { + this.flowId = flowId; + } +} diff --git a/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/entity/vo/OrchestratorRollBackGitVo.java b/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/entity/vo/OrchestratorRollBackGitVo.java new file mode 100644 index 0000000000..d78a0826c6 --- /dev/null +++ b/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/entity/vo/OrchestratorRollBackGitVo.java @@ -0,0 +1,43 @@ +package com.webank.wedatasphere.dss.orchestrator.server.entity.vo; + +import com.webank.wedatasphere.dss.orchestrator.common.entity.DSSOrchestratorInfo; +import com.webank.wedatasphere.dss.orchestrator.common.entity.DSSOrchestratorVersion; + +public class OrchestratorRollBackGitVo { + private DSSOrchestratorVersion oldOrcVersion; + private DSSOrchestratorVersion dssOrchestratorVersion; + private DSSOrchestratorInfo dssOrchestratorInfo; + private String version; + + public DSSOrchestratorVersion getOldOrcVersion() { + return oldOrcVersion; + } + + public void setOldOrcVersion(DSSOrchestratorVersion oldOrcVersion) { + this.oldOrcVersion = oldOrcVersion; + } + + public DSSOrchestratorVersion getDssOrchestratorVersion() { + return dssOrchestratorVersion; + } + + public void setDssOrchestratorVersion(DSSOrchestratorVersion dssOrchestratorVersion) { + this.dssOrchestratorVersion = dssOrchestratorVersion; + } + + public DSSOrchestratorInfo getDssOrchestratorInfo() { + return dssOrchestratorInfo; + } + + public void setDssOrchestratorInfo(DSSOrchestratorInfo dssOrchestratorInfo) { + this.dssOrchestratorInfo = dssOrchestratorInfo; + } + + public String getVersion() { + return version; + } + + public void setVersion(String version) { + this.version = version; + } +} diff --git a/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/entity/vo/OrchestratorStatusVo.java b/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/entity/vo/OrchestratorStatusVo.java new file mode 100644 index 0000000000..fe3b95a12e --- /dev/null +++ b/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/entity/vo/OrchestratorStatusVo.java @@ -0,0 +1,33 @@ +package com.webank.wedatasphere.dss.orchestrator.server.entity.vo; + +public class OrchestratorStatusVo { + + + private String status; + private String name; + + + public OrchestratorStatusVo(String status, String name) { + this.status = status; + this.name = name; + } + + public OrchestratorStatusVo() { + } + + public String getStatus() { + return status; + } + + public void setStatus(String status) { + this.status = status; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } +} diff --git a/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/job/OrchestratorCopyEnv.java b/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/job/OrchestratorCopyEnv.java index fedcda0543..0f925c29b9 100644 --- a/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/job/OrchestratorCopyEnv.java +++ b/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/job/OrchestratorCopyEnv.java @@ -9,6 +9,9 @@ import com.webank.wedatasphere.dss.orchestrator.publish.ExportDSSOrchestratorPlugin; import com.webank.wedatasphere.dss.orchestrator.publish.ImportDSSOrchestratorPlugin; import com.webank.wedatasphere.dss.orchestrator.server.service.OrchestratorFrameworkService; +import com.webank.wedatasphere.dss.workflow.dao.FlowMapper; +import com.webank.wedatasphere.dss.workflow.dao.LockMapper; +import com.webank.wedatasphere.dss.workflow.service.DSSFlowService; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Component; @@ -41,6 +44,12 @@ public class OrchestratorCopyEnv { @Autowired AddOrchestratorVersionHook addOrchestratorVersionHook; + @Autowired + private DSSFlowService flowService; + + @Autowired + private LockMapper lockMapper; + public ContextService getContextService() { return contextService; } @@ -116,4 +125,20 @@ public ImportDSSOrchestratorPlugin getImportDSSOrchestratorPlugin() { public void setImportDSSOrchestratorPlugin(ImportDSSOrchestratorPlugin importDSSOrchestratorPlugin) { this.importDSSOrchestratorPlugin = importDSSOrchestratorPlugin; } + + public DSSFlowService getFlowService() { + return flowService; + } + + public void setFlowService(DSSFlowService flowService) { + this.flowService = flowService; + } + + public LockMapper getLockMapper() { + return lockMapper; + } + + public void setLockMapper(LockMapper lockMapper) { + this.lockMapper = lockMapper; + } } diff --git a/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/job/OrchestratorCopyJob.java b/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/job/OrchestratorCopyJob.java index fc73840193..1d6176b194 100644 --- a/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/job/OrchestratorCopyJob.java +++ b/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/job/OrchestratorCopyJob.java @@ -1,22 +1,20 @@ package com.webank.wedatasphere.dss.orchestrator.server.job; import com.google.common.collect.Lists; -import com.webank.wedatasphere.dss.appconn.scheduler.structure.orchestration.OrchestrationCreationOperation; -import com.webank.wedatasphere.dss.appconn.scheduler.structure.orchestration.OrchestrationService; -import com.webank.wedatasphere.dss.appconn.scheduler.structure.orchestration.ref.DSSOrchestrationContentRequestRef; -import com.webank.wedatasphere.dss.appconn.scheduler.structure.orchestration.ref.OrchestrationResponseRef; +import com.webank.wedatasphere.dss.common.entity.project.DSSProject; import com.webank.wedatasphere.dss.common.exception.DSSErrorException; import com.webank.wedatasphere.dss.common.label.DSSLabel; +import com.webank.wedatasphere.dss.common.label.EnvDSSLabel; +import com.webank.wedatasphere.dss.common.label.LabelRouteVO; import com.webank.wedatasphere.dss.common.utils.MapUtils; -import com.webank.wedatasphere.dss.orchestrator.common.entity.*; +import com.webank.wedatasphere.dss.orchestrator.common.entity.DSSOrchestratorCopyInfo; +import com.webank.wedatasphere.dss.orchestrator.common.entity.DSSOrchestratorInfo; +import com.webank.wedatasphere.dss.orchestrator.common.entity.DSSOrchestratorVersion; import com.webank.wedatasphere.dss.orchestrator.common.ref.OrchestratorRefConstant; import com.webank.wedatasphere.dss.orchestrator.core.DSSOrchestrator; import com.webank.wedatasphere.dss.orchestrator.core.utils.OrchestratorUtils; -import com.webank.wedatasphere.dss.orchestrator.db.dao.OrchestratorMapper; import com.webank.wedatasphere.dss.orchestrator.publish.utils.OrchestrationDevelopmentOperationUtils; import com.webank.wedatasphere.dss.orchestrator.server.entity.vo.OrchestratorCopyVo; -import com.webank.wedatasphere.dss.orchestrator.server.service.OrchestratorService; -import com.webank.wedatasphere.dss.orchestrator.server.service.impl.OrchestratorFrameworkServiceImpl; import com.webank.wedatasphere.dss.standard.app.development.operation.RefCopyOperation; import com.webank.wedatasphere.dss.standard.app.development.ref.CopyRequestRef; import com.webank.wedatasphere.dss.standard.app.development.ref.RefJobContentResponseRef; @@ -24,6 +22,7 @@ import com.webank.wedatasphere.dss.standard.app.development.standard.DevelopmentIntegrationStandard; import com.webank.wedatasphere.dss.standard.app.sso.Workspace; +import com.webank.wedatasphere.dss.workflow.common.entity.DSSFlow; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.BeanUtils; @@ -36,13 +35,9 @@ public class OrchestratorCopyJob implements Runnable { private OrchestratorCopyVo orchestratorCopyVo; - protected OrchestratorCopyEnv orchestratorCopyEnv; - - private OrchestratorFrameworkServiceImpl orchestratorFrameworkServiceImpl; + private DSSProject targetProject; - private OrchestratorService orchestratorService; - - private OrchestratorMapper orchestratorMapper; + protected OrchestratorCopyEnv orchestratorCopyEnv; private DSSOrchestratorCopyInfo orchestratorCopyInfo = new DSSOrchestratorCopyInfo(UUID.randomUUID().toString()); @@ -77,10 +72,9 @@ private void copyOrchestrator() { newOrchestrator.setDesc("copy from " + sourceOrchestrator.getName()); newOrchestrator.setUpdateTime(null); newOrchestrator.setUpdateUser(null); - DSSOrchestratorVersion dssOrchestratorVersion = null; try { - dssOrchestratorVersion = doOrchestratorCopy(orchestratorCopyVo.getUsername(), orchestratorCopyVo.getWorkspace(), newOrchestrator, + doOrchestratorCopy(orchestratorCopyVo.getUsername(), orchestratorCopyVo.getWorkspace(), newOrchestrator, orchestratorCopyVo.getTargetProjectName(), Lists.newArrayList(orchestratorCopyVo.getDssLabel()), appId); } catch (Exception e) { //保存错误信息 @@ -103,41 +97,13 @@ private void copyOrchestrator() { orchestratorCopyInfo.setSuccessNode(Lists.newArrayList("All")); orchestratorCopyInfo.setStatus(1); orchestratorCopyEnv.getOrchestratorCopyJobMapper().updateCopyStatus(orchestratorCopyInfo); - - List dssLabels = new ArrayList<>(); - dssLabels.add(orchestratorCopyVo.getDssLabel()); - - //2.如果调度系统要求同步创建工作流，向调度系统发送创建工作流的请求 - OrchestrationResponseRef orchestrationResponseRef = orchestratorFrameworkServiceImpl.tryOrchestrationOperation(dssLabels, true, orchestratorCopyVo.getUsername(), - orchestratorCopyVo.getTargetProjectName(), orchestratorCopyVo.getWorkspace(), newOrchestrator, - OrchestrationService::getOrchestrationCreationOperation, - (structureOperation, structureRequestRef) -> ((OrchestrationCreationOperation) structureOperation) - .createOrchestration((DSSOrchestrationContentRequestRef) structureRequestRef), "create"); - - try { - orchestratorService.copyOrchestrator(orchestratorCopyVo.getUsername(), orchestratorCopyVo.getWorkspace(), orchestratorCopyVo.getTargetProjectName(), - orchestratorCopyVo.getTargetProjectId(), newOrchestrator.getDesc(), newOrchestrator, dssLabels); - } catch (Exception e) { - throw new RuntimeException("error happened when copying orc.", e); - } - - Long orchestratorId = newOrchestrator.getId(); - Long orchestratorVersionId = dssOrchestratorVersion.getId(); - //4.将工程和orchestrator的关系存储到的数据库中 - if (orchestrationResponseRef != null) { - Long refProjectId = (Long) orchestrationResponseRef.toMap().get("refProjectId"); - orchestratorMapper.addOrchestratorRefOrchestration(new DSSOrchestratorRefOrchestration(orchestratorId, refProjectId, orchestrationResponseRef.getRefOrchestrationId())); - } else { - LOGGER.info("copy orchestration {} with orchestratorId is {}, and versionId is {}, and orchestrationResponseRef is null.", newOrchestrator.getName(), orchestratorId, orchestratorVersionId); - - } } - private DSSOrchestratorVersion doOrchestratorCopy(String userName, + private void doOrchestratorCopy(String userName, Workspace workspace, DSSOrchestratorInfo dssOrchestratorInfo, String projectName, - List dssLabels, Long appId) throws DSSErrorException { + List dssLabels, Long appId) throws Exception { String copyInitVersion = OrchestratorUtils.generateNewCopyVersion(orchestratorCopyVo.getWorkflowNodeSuffix()); String contextId = orchestratorCopyEnv.getContextService().createContextID(workspace.getWorkspaceName(), projectName, dssOrchestratorInfo.getName(), copyInitVersion, userName); DSSOrchestratorVersion dssOrchestratorVersion = new DSSOrchestratorVersion(); @@ -166,13 +132,25 @@ private DSSOrchestratorVersion doOrchestratorCopy(String userName, requestRef.setNewVersion(dssOrchestratorVersion.getVersion()).setRefJobContent(refJobContent); return ((RefCopyOperation) developmentOperation).copyRef(requestRef); }, "copy"); - dssOrchestratorVersion.setAppId((Long) responseRef.getRefJobContent().get(OrchestratorRefConstant.ORCHESTRATION_ID_KEY)); + Long flowId = (Long) responseRef.getRefJobContent().get(OrchestratorRefConstant.ORCHESTRATION_ID_KEY); + dssOrchestratorVersion.setAppId(flowId); dssOrchestratorVersion.setContent((String) responseRef.getRefJobContent().get(OrchestratorRefConstant.ORCHESTRATION_CONTENT_KEY)); List paramConfTemplateIds=(List) responseRef.getRefJobContent().get(OrchestratorRefConstant.ORCHESTRATION_FLOWID_PARAMCONF_TEMPLATEID_TUPLES_KEY); orchestratorCopyEnv.getOrchestratorMapper().addOrchestrator(dssOrchestratorInfo); - dssOrchestratorVersion.setOrchestratorId(dssOrchestratorInfo.getId()); + Long orchestratorId = dssOrchestratorInfo.getId(); + dssOrchestratorVersion.setOrchestratorId(orchestratorId); orchestratorCopyEnv.getOrchestratorMapper().addOrchestratorVersion(dssOrchestratorVersion); orchestratorCopyEnv.getAddOrchestratorVersionHook().afterAdd(dssOrchestratorVersion, Collections.singletonMap(OrchestratorRefConstant.ORCHESTRATION_FLOWID_PARAMCONF_TEMPLATEID_TUPLES_KEY,paramConfTemplateIds)); + // 更新flowJson\ + if (dssLabels.get(0).getValue().get(EnvDSSLabel.DSS_ENV_LABEL_KEY).equals("dev")) { + DSSFlow flowByID = orchestratorCopyEnv.getFlowService().getFlow(flowId); + if (flowByID != null) { + orchestratorCopyEnv.getFlowService().saveFlowMetaData(flowId, flowByID.getFlowJson(), orchestratorId); + } + if (targetProject != null && targetProject.getAssociateGit() && flowByID != null) { + orchestratorCopyEnv.getFlowService().updateTOSaveStatus(targetProject.getId(), flowByID.getId(), orchestratorId); + } + } } @@ -200,27 +178,11 @@ public void setOrchestratorCopyInfo(DSSOrchestratorCopyInfo orchestratorCopyInfo this.orchestratorCopyInfo = orchestratorCopyInfo; } - public OrchestratorFrameworkServiceImpl getOrchestratorFrameworkServiceImpl() { - return orchestratorFrameworkServiceImpl; - } - - public void setOrchestratorFrameworkServiceImpl(OrchestratorFrameworkServiceImpl orchestratorFrameworkServiceImpl) { - this.orchestratorFrameworkServiceImpl = orchestratorFrameworkServiceImpl; - } - - public OrchestratorService getOrchestratorService() { - return orchestratorService; - } - - public void setOrchestratorService(OrchestratorService orchestratorService) { - this.orchestratorService = orchestratorService; - } - - public OrchestratorMapper getOrchestratorMapper() { - return orchestratorMapper; + public DSSProject getTargetProject() { + return targetProject; } - public void setOrchestratorMapper(OrchestratorMapper orchestratorMapper) { - this.orchestratorMapper = orchestratorMapper; + public void setTargetProject(DSSProject targetProject) { + this.targetProject = targetProject; } } diff --git a/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/restful/DSSFrameworkOrchestratorRestful.java b/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/restful/DSSFrameworkOrchestratorRestful.java index f982e439ea..84e7dc9fd5 100644 --- a/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/restful/DSSFrameworkOrchestratorRestful.java +++ b/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/restful/DSSFrameworkOrchestratorRestful.java @@ -19,34 +19,70 @@ import com.webank.wedatasphere.dss.appconn.manager.utils.AppConnManagerUtils; import com.webank.wedatasphere.dss.common.auditlog.OperateTypeEnum; import com.webank.wedatasphere.dss.common.auditlog.TargetTypeEnum; +import com.webank.wedatasphere.dss.common.entity.project.DSSProject; import com.webank.wedatasphere.dss.common.exception.DSSErrorException; import com.webank.wedatasphere.dss.common.label.DSSLabel; import com.webank.wedatasphere.dss.common.label.EnvDSSLabel; +import com.webank.wedatasphere.dss.common.label.LabelRouteVO; +import com.webank.wedatasphere.dss.common.protocol.project.ProjectInfoListRequest; +import com.webank.wedatasphere.dss.common.protocol.project.ProjectInfoListResponse; +import com.webank.wedatasphere.dss.common.protocol.project.ProjectInfoRequest; import com.webank.wedatasphere.dss.common.utils.AuditLogUtils; -import com.webank.wedatasphere.dss.orchestrator.common.entity.DSSOrchestratorCopyInfo; -import com.webank.wedatasphere.dss.orchestrator.common.entity.OrchestratorVo; +import com.webank.wedatasphere.dss.common.utils.DSSExceptionUtils; +import com.webank.wedatasphere.dss.common.utils.RpcAskUtils; +import com.webank.wedatasphere.dss.framework.workspace.service.DSSWorkspaceRoleService; +import com.webank.wedatasphere.dss.framework.workspace.service.DSSWorkspaceService; +import com.webank.wedatasphere.dss.git.common.protocol.GitTree; +import com.webank.wedatasphere.dss.git.common.protocol.config.GitServerConfig; +import com.webank.wedatasphere.dss.git.common.protocol.constant.GitConstant; +import com.webank.wedatasphere.dss.git.common.protocol.request.GitAddMemberRequest; +import com.webank.wedatasphere.dss.git.common.protocol.request.GitUserInfoRequest; +import com.webank.wedatasphere.dss.git.common.protocol.response.GitAddMemberResponse; +import com.webank.wedatasphere.dss.git.common.protocol.response.GitFileContentResponse; +import com.webank.wedatasphere.dss.git.common.protocol.response.GitHistoryResponse; +import com.webank.wedatasphere.dss.git.common.protocol.response.GitUserInfoResponse; +import com.webank.wedatasphere.dss.git.common.protocol.util.UrlUtils; +import com.webank.wedatasphere.dss.orchestrator.common.entity.*; +import com.webank.wedatasphere.dss.orchestrator.common.ref.OrchestratorRefConstant; +import com.webank.wedatasphere.dss.orchestrator.db.dao.OrchestratorMapper; +import com.webank.wedatasphere.dss.orchestrator.server.conf.OrchestratorConf; import com.webank.wedatasphere.dss.orchestrator.server.constant.OrchestratorLevelEnum; import com.webank.wedatasphere.dss.orchestrator.server.entity.request.*; -import com.webank.wedatasphere.dss.orchestrator.server.entity.vo.CommonOrchestratorVo; -import com.webank.wedatasphere.dss.orchestrator.server.entity.vo.OrchestratorCopyHistory; -import com.webank.wedatasphere.dss.orchestrator.server.entity.vo.OrchestratorUnlockVo; +import com.webank.wedatasphere.dss.orchestrator.server.entity.vo.*; import com.webank.wedatasphere.dss.orchestrator.server.service.OrchestratorFrameworkService; +import com.webank.wedatasphere.dss.orchestrator.server.service.OrchestratorPluginService; import com.webank.wedatasphere.dss.orchestrator.server.service.OrchestratorService; +import com.webank.wedatasphere.dss.sender.service.DSSSenderServiceFactory; import com.webank.wedatasphere.dss.standard.app.sso.Workspace; import com.webank.wedatasphere.dss.standard.sso.utils.SSOHelper; +import com.webank.wedatasphere.dss.workflow.common.entity.DSSFlow; +import com.webank.wedatasphere.dss.workflow.common.protocol.RequestLockWorkflow; +import com.webank.wedatasphere.dss.workflow.common.protocol.ResponseLockWorkflow; +import com.webank.wedatasphere.dss.workflow.constant.DSSWorkFlowConstant; +import com.webank.wedatasphere.dss.workflow.dao.LockMapper; +import com.webank.wedatasphere.dss.workflow.entity.DSSFlowEditLock; +import com.webank.wedatasphere.dss.workflow.lock.DSSFlowEditLockManager; +import com.webank.wedatasphere.dss.workflow.service.DSSFlowService; import org.apache.commons.math3.util.Pair; +import org.apache.linkis.rpc.Sender; import org.apache.linkis.server.Message; import org.apache.linkis.server.security.SecurityFilter; +import org.checkerframework.checker.units.qual.A; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.util.CollectionUtils; +import org.springframework.util.StringUtils; import org.springframework.web.bind.annotation.*; -import scala.tools.nsc.typechecker.Implicits; import javax.annotation.PostConstruct; +import javax.servlet.http.Cookie; import javax.servlet.http.HttpServletRequest; -import java.util.Arrays; -import java.util.List; +import javax.servlet.http.HttpServletResponse; +import java.util.*; +import java.util.concurrent.ExecutionException; +import java.util.regex.Pattern; +import java.util.stream.Collectors; @RequestMapping(path = "/dss/framework/orchestrator", produces = {"application/json"}) @RestController @@ -59,12 +95,22 @@ public class DSSFrameworkOrchestratorRestful { @Autowired private OrchestratorService orchestratorService; @Autowired + private OrchestratorPluginService orchestratorPluginService; + @Autowired private HttpServletRequest httpServletRequest; + @Autowired + private LockMapper lockMapper; + @Autowired + private DSSFlowService flowService; + @Autowired + private OrchestratorMapper orchestratorMapper; + + @Autowired + private DSSWorkspaceRoleService dssWorkspaceRoleService; + + @Autowired + private DSSWorkspaceService dssWorkspaceService; - @PostConstruct - public void init() { - AppConnManagerUtils.autoLoadAppConnManager(); - } /** * 创建编排模式 @@ -94,8 +140,25 @@ public Message createOrchestrator(@RequestBody OrchestratorCreateRequest createR public Message getAllOrchestrator(@RequestBody OrchestratorRequest orchestratorRequest) { try { String username = SecurityFilter.getLoginUsername(httpServletRequest); + if (orchestratorRequest.getWorkspaceId() == null) { + orchestratorRequest.setWorkspaceId(SSOHelper.getWorkspace(httpServletRequest).getWorkspaceId()); + } + + try { + dssWorkspaceService.getWorkspacesById(orchestratorRequest.getWorkspaceId(), username); + } catch (DSSErrorException e) { + LOGGER.error("User {} get workspace {} failed.", username, orchestratorRequest.getWorkspaceId(), e); + return Message.error(e); + } + + List roles = dssWorkspaceRoleService.getRoleInWorkspace(username, orchestratorRequest.getWorkspaceId().intValue()); + if (roles == null || roles.isEmpty()) { + LOGGER.error("username {}, in workspace {} roles are null or empty", username, orchestratorRequest.getWorkspaceId()); + return Message.error("can not get roles information"); + } + LOGGER.info("user {} begin to geyAllOrchestrator, requestBody:{}", username, orchestratorRequest); - return Message.ok("获取编排模式成功").data("page", orchestratorService.getListByPage(orchestratorRequest, username)); + return Message.ok("获取编排模式成功").data("page", orchestratorService.getOrchestratorInfos(orchestratorRequest, username)); } catch (Exception e) { LOGGER.error("getAllOrchestratorError ", e); return Message.error("获取编排模式失败:" + e.getMessage()); @@ -134,6 +197,12 @@ public Message deleteOrchestrator(@RequestBody OrchestratorDeleteRequest deleteR if (orchestratorFrameworkService.getOrchestratorCopyStatus(deleteRequest.getId())) { return Message.error("当前工作流正在被复制，不允许删除"); } + ProjectInfoRequest projectInfoRequest = new ProjectInfoRequest(); + projectInfoRequest.setProjectId(deleteRequest.getProjectId()); + DSSProject dssProject = (DSSProject) DSSSenderServiceFactory.getOrCreateServiceInstance().getProjectServerSender().ask(projectInfoRequest); + if (dssProject.getWorkspaceId() != workspace.getWorkspaceId()) { + DSSExceptionUtils.dealErrorException(63335, "工作流所在工作空间和cookie中不一致，请刷新页面后，再次发布！", DSSErrorException.class); + } CommonOrchestratorVo orchestratorVo = orchestratorFrameworkService.deleteOrchestrator(username, deleteRequest, workspace); AuditLogUtils.printLog(username, workspace.getWorkspaceId(), workspace.getWorkspaceName(), TargetTypeEnum.ORCHESTRATOR, orchestratorVo.getOrchestratorId(), orchestratorVo.getOrchestratorName(), OperateTypeEnum.DELETE, deleteRequest); @@ -229,11 +298,17 @@ public Message rollbackOrchestrator(HttpServletRequest request, @RequestBody Rol Long projectId = rollbackOrchestratorRequest.getProjectId(); String projectName = rollbackOrchestratorRequest.getProjectName(); Workspace workspace = SSOHelper.getWorkspace(request); - DSSLabel envDSSLabel = new EnvDSSLabel(rollbackOrchestratorRequest.getLabels().getRoute()); + LabelRouteVO labels = rollbackOrchestratorRequest.getLabels(); try { LOGGER.info("user {} begin to rollbackOrchestrator, params:{}", username, rollbackOrchestratorRequest); - String newVersion = orchestratorService.rollbackOrchestrator(username, projectId, projectName, orchestratorId, version, envDSSLabel, workspace); - Message message = Message.ok("回滚版本成功").data("newVersion", newVersion); + DSSOrchestratorVersion latestVersionById = orchestratorMapper.getLatestOrchestratorVersionById(orchestratorId); + OrchestratorRollBackGitVo rollbackOrchestrator = orchestratorService.rollbackOrchestrator(username, projectId, projectName, orchestratorId, version, labels, workspace); + try { + orchestratorService.rollbackOrchestratorGit(rollbackOrchestrator, username, projectId, projectName, orchestratorId, labels, workspace, latestVersionById.getAppId()); + } catch (Exception e) { + return Message.ok("回滚版本成功,git回滚失败，请重新保存并提交工作流").data("newVersion", rollbackOrchestrator.getVersion()); + } + Message message = Message.ok("回滚版本成功").data("newVersion", rollbackOrchestrator.getVersion()); return message; } catch (final Throwable t) { LOGGER.error("Failed to rollback orchestrator for user {} orchestratorId {}, projectId {} version {}", @@ -272,4 +347,438 @@ public Message getVersionByOrchestratorId(@RequestBody QueryOrchestratorVersion List list = orchestratorService.getVersionByOrchestratorId(queryOrchestratorVersion.getOrchestratorId()); return Message.ok().data("list", list); } + + @RequestMapping(value = "diffFlowJob", method = RequestMethod.POST) + public Message diff(@RequestBody OrchestratorSubmitRequest submitFlowRequest) { + Workspace workspace = SSOHelper.getWorkspace(httpServletRequest); + String userName = SecurityFilter.getLoginUsername(httpServletRequest); + + Long orchestratorId = submitFlowRequest.getOrchestratorId(); + DSSOrchestratorVersion latestOrchestratorVersion = orchestratorFrameworkService.getLatestOrchestratorVersion(orchestratorId); + submitFlowRequest.setFlowId(latestOrchestratorVersion.getAppId()); + + String ticketId = Arrays.stream(httpServletRequest.getCookies()).filter(cookie -> DSSWorkFlowConstant.BDP_USER_TICKET_ID.equals(cookie.getName())) + .findFirst().map(Cookie::getValue).get(); + DSSFlowEditLock flowEditLock = lockMapper.getFlowEditLockByID(submitFlowRequest.getFlowId()); + if (flowEditLock != null && !flowEditLock.getOwner().equals(ticketId)) { + return Message.error("当前工作流被用户" + flowEditLock.getUsername() + "已锁定编辑，您编辑的内容不能再被保存。如有疑问，请与" + flowEditLock.getUsername() + "确认"); + } + try { + Long taskId = orchestratorPluginService.diffFlow(submitFlowRequest, userName, workspace); + return Message.ok().data("taskId", taskId); + } catch (Exception e) { + return Message.error("获取对比内容失败，原因为：" + e.getMessage()); + } + + } + + @RequestMapping(value = "diffStatus", method = RequestMethod.GET) + public Message diffStatus(@RequestParam(required = true, name = "taskId") Long taskId) { + Workspace workspace = SSOHelper.getWorkspace(httpServletRequest); + String userName = SecurityFilter.getLoginUsername(httpServletRequest); + + try { + String status = orchestratorPluginService.diffStatus(taskId); + return Message.ok().data("status", status); + } catch (Exception e) { + return Message.ok().data("status", OrchestratorRefConstant.FLOW_STATUS_PUSH_FAILED).data("errMsg", e.getMessage()); + } + } + + @RequestMapping(value = "diffContent", method = RequestMethod.GET) + public Message diffContent(@RequestParam(required = true, name = "taskId") Long taskId) { + Workspace workspace = SSOHelper.getWorkspace(httpServletRequest); + String userName = SecurityFilter.getLoginUsername(httpServletRequest); + + return Message.ok().data("tree", orchestratorPluginService.diffContent(taskId)); + } + + @RequestMapping(value = "diffOrchestratorPublish", method = RequestMethod.POST) + public Message diffOrchestratorPublish(@RequestBody OrchestratorSubmitRequest submitFlowRequest) { + Workspace workspace = SSOHelper.getWorkspace(httpServletRequest); + String userName = SecurityFilter.getLoginUsername(httpServletRequest); + + OrchestratorDiffDirVo gitTree = orchestratorPluginService.diffPublish(submitFlowRequest, userName, workspace); + return Message.ok().data("tree", gitTree); + } + + @RequestMapping(value = "diffFlowContent", method = RequestMethod.POST) + public Message diffFlowContent(@RequestBody OrchestratorSubmitRequest submitFlowRequest) { + Workspace workspace = SSOHelper.getWorkspace(httpServletRequest); + String userName = SecurityFilter.getLoginUsername(httpServletRequest); + + Long orchestratorId = submitFlowRequest.getOrchestratorId(); + DSSOrchestratorVersion latestOrchestratorVersion = orchestratorFrameworkService.getLatestOrchestratorVersion(orchestratorId); + submitFlowRequest.setFlowId(latestOrchestratorVersion.getAppId()); + + String ticketId = Arrays.stream(httpServletRequest.getCookies()).filter(cookie -> DSSWorkFlowConstant.BDP_USER_TICKET_ID.equals(cookie.getName())) + .findFirst().map(Cookie::getValue).get(); + DSSFlowEditLock flowEditLock = lockMapper.getFlowEditLockByID(submitFlowRequest.getFlowId()); + if (flowEditLock != null && !flowEditLock.getOwner().equals(ticketId)) { + return Message.error("当前工作流被用户" + flowEditLock.getUsername() + "已锁定编辑，您编辑的内容不能再被保存。如有疑问，请与" + flowEditLock.getUsername() + "确认"); + } + try { + GitFileContentResponse contentResponse = orchestratorPluginService.diffFlowContent(submitFlowRequest, userName, workspace); + return Message.ok().data("content", contentResponse); + }catch (Exception e) { + return Message.error("获取文件内容失败，原因为：" + e.getMessage()); + } + } + + @RequestMapping(value = "batchSubmitFlow", method = RequestMethod.POST) + public Message batchSubmitFlow(@RequestBody OrchestratorBatchSubmitRequest batchSubmitRequest) { + Workspace workspace = SSOHelper.getWorkspace(httpServletRequest); + String userName = SecurityFilter.getLoginUsername(httpServletRequest); + + List submitRequestList = batchSubmitRequest.getSubmitRequestList(); + + if (CollectionUtils.isEmpty(submitRequestList)) { + return Message.error("至少需要选择一项工作流进行提交"); + } + + Map> map = new HashMap<>(); + Map projectMap = new HashMap<>(); + String ticketId = Arrays.stream(httpServletRequest.getCookies()).filter(cookie -> DSSWorkFlowConstant.BDP_USER_TICKET_ID.equals(cookie.getName())) + .findFirst().map(Cookie::getValue).get(); + for (OrchestratorSubmitRequest submitFlowRequest: submitRequestList) { + try { + checkSubmitWorkflow(ticketId, submitFlowRequest, workspace, userName); + boolean b = map.containsKey(submitFlowRequest.getProjectName()); + if (b) { + List orchestratorRelationVos = map.get(submitFlowRequest.getProjectName()); + orchestratorRelationVos.add(new OrchestratorRelationVo(submitFlowRequest.getFlowId(), submitFlowRequest.getOrchestratorId())); + } else { + List orchestratorRelationVos = new ArrayList<>(); + orchestratorRelationVos.add(new OrchestratorRelationVo(submitFlowRequest.getFlowId(), submitFlowRequest.getOrchestratorId())); + DSSOrchestratorInfo orchestrator = orchestratorMapper.getOrchestrator(submitFlowRequest.getOrchestratorId()); + long projectId = orchestrator.getProjectId(); + projectMap.put(submitFlowRequest.getProjectName(), projectId); + map.put(submitFlowRequest.getProjectName(), orchestratorRelationVos); + } + } catch (Exception e) { + return Message.error("提交工作流失败，请保存工作流重试，原因为："+ e.getMessage()); + } + + } + + String label = batchSubmitRequest.getLabels().getRoute(); + String comment = batchSubmitRequest.getComment(); + try { + orchestratorPluginService.batchSubmitFlow(map, projectMap, userName, workspace, label, comment); + } catch (Exception e) { + return Message.error("提交工作流失败，请保存工作流重试，原因为："+ e.getMessage()); + } + + return Message.ok(); + } + + + @RequestMapping(value = "submitFlow", method = RequestMethod.POST) + public Message submitFlow(@RequestBody OrchestratorSubmitRequest submitFlowRequest) { + Workspace workspace = SSOHelper.getWorkspace(httpServletRequest); + String userName = SecurityFilter.getLoginUsername(httpServletRequest); + + String ticketId = Arrays.stream(httpServletRequest.getCookies()).filter(cookie -> DSSWorkFlowConstant.BDP_USER_TICKET_ID.equals(cookie.getName())) + .findFirst().map(Cookie::getValue).get(); + + try { + checkSubmitWorkflow(ticketId, submitFlowRequest, workspace, userName); + orchestratorPluginService.submitFlow(submitFlowRequest, userName, workspace); + } catch (Exception e) { + return Message.error("提交工作流失败，请保存工作流重试，原因为："+ e.getMessage()); + } + + + return Message.ok(); + } + + private void checkSubmitWorkflow(String ticketId, OrchestratorSubmitRequest submitFlowRequest, Workspace workspace, String userName) throws DSSErrorException{ + Long orchestratorId = submitFlowRequest.getOrchestratorId(); + try { + checkWorkspace(orchestratorId, workspace); + } catch (Exception e) { + LOGGER.error("check failed, the reason is: ", e); + throw new DSSErrorException(80001, "提交失败，原因为：" + e.getMessage()); + } + + List dssLabelList = new ArrayList<>(); + dssLabelList.add(new EnvDSSLabel(submitFlowRequest.getLabels().getRoute())); + + DSSOrchestratorVersion latestOrchestratorVersion = orchestratorFrameworkService.getLatestOrchestratorVersion(orchestratorId); + Long flowId = latestOrchestratorVersion.getAppId(); + DSSFlowEditLock flowEditLock = lockMapper.getFlowEditLockByID(flowId); + submitFlowRequest.setFlowId(flowId); + if (flowEditLock != null && !flowEditLock.getOwner().equals(ticketId)) { + throw new DSSErrorException(80001,"当前工作流被用户" + flowEditLock.getUsername() + "已锁定编辑，您编辑的内容不能再被保存。如有疑问，请与" + flowEditLock.getUsername() + "确认"); + } + lockFlow(flowId, userName, ticketId); + + + } + + private void lockFlow(Long flowID, String username, String ticketId) throws DSSErrorException { + DSSFlow dssFlow = flowService.getFlowByID(flowID); + // 尝试获取工作流编辑锁 + try { + //只有父工作流才有锁，子工作流复用父工作流的锁 + if(dssFlow.getRootFlow()) { + String flowEditLock = DSSFlowEditLockManager.tryAcquireLock(dssFlow, username, ticketId); + dssFlow.setFlowEditLock(flowEditLock); + } + } catch (DSSErrorException e) { + if (DSSWorkFlowConstant.EDIT_LOCK_ERROR_CODE == e.getErrCode()) { + DSSFlowEditLock flowEditLock = lockMapper.getFlowEditLockByID(flowID); + throw new DSSErrorException(60056,"用户已锁定编辑错误码，editLockInfo:" + flowEditLock); + } + throw e; + } + } + + @RequestMapping(path = "gitUrl", method = RequestMethod.GET) + public Message gitUrl(@RequestParam(required = true, name = "projectName") String projectName, + @RequestParam(required = false, name = "workflowName") String workflowName, + @RequestParam(required = false, name = "workflowNodeName") String workflowNodeName, + @RequestParam(required = false, name = "workflowId") Long workflowId, + HttpServletResponse response) { + Workspace workspace = SSOHelper.getWorkspace(httpServletRequest); + String userName = SecurityFilter.getLoginUsername(httpServletRequest); + try { + String gitUrl = orchestratorService.getGitUrl(workflowName, workflowNodeName, projectName, userName, workspace, workflowId); + return Message.ok().data("gitUrl", gitUrl); + } catch (Exception e) { + return Message.error("跳转git失败，原因为:" + e.getMessage()); + } + + + } + + @RequestMapping(value = "submitFlow/status", method = RequestMethod.GET) + public Message submitFlow(@RequestParam Long orchestratorId) { + Workspace workspace = SSOHelper.getWorkspace(httpServletRequest); + String userName = SecurityFilter.getLoginUsername(httpServletRequest); + + try { + OrchestratorVo orchestratorVoById = orchestratorService.getOrchestratorVoById(orchestratorId); + if (orchestratorVoById == null) { + DSSExceptionUtils.dealErrorException(80001, "编排不存在", DSSErrorException.class); + } + long projectId = orchestratorVoById.getDssOrchestratorInfo().getProjectId(); + ProjectInfoRequest projectInfoRequest = new ProjectInfoRequest(); + projectInfoRequest.setProjectId(projectId); + DSSProject dssProject = (DSSProject) DSSSenderServiceFactory.getOrCreateServiceInstance().getProjectServerSender().ask(projectInfoRequest); + if (dssProject.getWorkspaceId() != workspace.getWorkspaceId()) { + DSSExceptionUtils.dealErrorException(63335, "工作流所在工作空间和cookie中不一致，请刷新页面后，再次发布！", DSSErrorException.class); + } + if (dssProject.getAssociateGit() == null || !dssProject.getAssociateGit()) { + return Message.ok().data("status", OrchestratorRefConstant.FLOW_STATUS_NO_GIT); + } + } catch (Exception e) { + LOGGER.error("check failed, the reason is: ", e); + return Message.error("提交失败，原因为：" + e.getMessage()); + } + + OrchestratorSubmitJob orchestratorSubmitJob = orchestratorFrameworkService.getOrchestratorStatus(orchestratorId); + // 未提交 + if (orchestratorSubmitJob == null) { + return Message.error("该编排未开始提交"); + } + String status = orchestratorSubmitJob.getStatus(); + if (OrchestratorRefConstant.FLOW_STATUS_PUSH_FAILED.equals(status)) { + return Message.error("提交失败，原因为：" + orchestratorSubmitJob.getErrorMsg()); + } + return Message.ok().data("status", status); + } + + @RequestMapping(value = "publish/history", method = RequestMethod.GET) + public Message getHistory(@RequestParam Long orchestratorId, @RequestParam String projectName) { + Workspace workspace = SSOHelper.getWorkspace(httpServletRequest); + String userName = SecurityFilter.getLoginUsername(httpServletRequest); + + try { + checkWorkspace(orchestratorId, workspace); + } catch (Exception e) { + LOGGER.error("check failed, the reason is: ", e); + return Message.error("提交失败，原因为：" + e.getMessage()); + } + + GitHistoryResponse history = orchestratorFrameworkService.getHistory(workspace.getWorkspaceId(), orchestratorId, projectName); + + return Message.ok().data("history", history); + } + + private void checkWorkspace(Long orchestratorId, Workspace workspace) throws DSSErrorException{ + OrchestratorVo orchestratorVoById = orchestratorService.getOrchestratorVoById(orchestratorId); + if (orchestratorVoById == null) { + DSSExceptionUtils.dealErrorException(80001, "编排不存在", DSSErrorException.class); + } + long projectId = orchestratorVoById.getDssOrchestratorInfo().getProjectId(); + ProjectInfoRequest projectInfoRequest = new ProjectInfoRequest(); + projectInfoRequest.setProjectId(projectId); + DSSProject dssProject = (DSSProject) DSSSenderServiceFactory.getOrCreateServiceInstance().getProjectServerSender().ask(projectInfoRequest); + if (dssProject.getWorkspaceId() != workspace.getWorkspaceId()) { + DSSExceptionUtils.dealErrorException(63335, "工作流所在工作空间和cookie中不一致，请刷新页面后，再次发布！", DSSErrorException.class); + } + } + + @RequestMapping(value = "allType", method = RequestMethod.GET) + public Message allType(HttpServletRequest req) { + Workspace workspace = SSOHelper.getWorkspace(httpServletRequest); + String userName = SecurityFilter.getLoginUsername(httpServletRequest); + + return Message.ok().data("type", GitConstant.GIT_SERVER_SEARCH_TYPE); + } + + + @RequestMapping(value = "/getAllOrchestratorMeta", method = RequestMethod.POST) + public Message getAllOrchestratorMeta(HttpServletRequest httpServletRequest, @RequestBody OrchestratorMetaRequest orchestratorMetaRequest) { + + if (orchestratorMetaRequest.getPageNow() == null) { + orchestratorMetaRequest.setPageNow(1); + } + + if (orchestratorMetaRequest.getPageSize() == null) { + orchestratorMetaRequest.setPageSize(10); + } + + if (orchestratorMetaRequest.getWorkspaceId() == null) { + Long workspaceId = SSOHelper.getWorkspace(httpServletRequest).getWorkspaceId(); + orchestratorMetaRequest.setWorkspaceId(workspaceId); + } + + String userName = SecurityFilter.getLoginUsername(httpServletRequest); + + try { + dssWorkspaceService.getWorkspacesById(orchestratorMetaRequest.getWorkspaceId(), userName); + } catch (DSSErrorException e) { + LOGGER.error("User {} get workspace {} failed.", userName, orchestratorMetaRequest.getWorkspaceId(), e); + return Message.error(e); + } + + List roles = dssWorkspaceRoleService.getRoleInWorkspace(userName, orchestratorMetaRequest.getWorkspaceId().intValue()); + if (roles == null || roles.isEmpty()) { + LOGGER.error("username {}, in workspace {} roles are null or empty", userName, orchestratorMetaRequest.getWorkspaceId()); + return Message.error("can not get roles information"); + } + + List totals = new ArrayList<>(); + + List orchestratorMetaList = orchestratorFrameworkService.getAllOrchestratorMeta(orchestratorMetaRequest, totals,userName); + + return Message.ok().data("data", orchestratorMetaList).data("total", totals.get(0)); + } + + + @RequestMapping(value = "/modifyOrchestratorMeta", method = RequestMethod.POST) + public Message modifyOrchestratorMeta(HttpServletRequest httpServletRequest, @RequestBody ModifyOrchestratorMetaRequest modifyOrchestratorMetaRequest) { + + String username = SecurityFilter.getLoginUsername(httpServletRequest); + Workspace workspace = SSOHelper.getWorkspace(httpServletRequest); + + // workspaceId参数如果和cookie中不一致,则抛错 + if(modifyOrchestratorMetaRequest.getWorkspaceId() != null + && !String.valueOf(workspace.getWorkspaceId()).equals(String.valueOf(modifyOrchestratorMetaRequest.getWorkspaceId()))){ + return Message.error(String.format("%s 工作流所在工作空间和cookie中不一致，请刷新页面后，再次编辑！",modifyOrchestratorMetaRequest.getOrchestratorName())); + } + + if (orchestratorFrameworkService.getOrchestratorCopyStatus(modifyOrchestratorMetaRequest.getOrchestratorId())) { + return Message.error("当前工作流正在被复制，不允许编辑"); + } + + DSSOrchestratorVersion orchestratorVersion = orchestratorMapper.getLatestOrchestratorVersionByIdAndValidFlag(modifyOrchestratorMetaRequest.getOrchestratorId(),1); + + if(orchestratorVersion == null){ + return Message.error(String.format("%s工作流不存在或已被删除，请重新查询后进行编辑",modifyOrchestratorMetaRequest.getOrchestratorName())); + } + + String proxyUser = modifyOrchestratorMetaRequest.getProxyUser(); + // 添加代理用户验证是否合法 + if(!StringUtils.isEmpty(proxyUser) && !Pattern.compile("^[a-zA-Z0-9_]+$").matcher(proxyUser).find()){ + return Message.error(String.format("%s代理用名称输入不合法",proxyUser)); + } + + DSSFlowEditLock flowEditLock = lockMapper.getFlowEditLockByID(orchestratorVersion.getAppId()); + //若工作流已经被其他用户抢锁，则当前用户不能再保存成功 + String ticketId = Arrays.stream(httpServletRequest.getCookies()).filter(cookie -> DSSWorkFlowConstant.BDP_USER_TICKET_ID.equals(cookie.getName())) + .findFirst().map(Cookie::getValue).get(); + if (flowEditLock != null && !flowEditLock.getOwner().equals(ticketId)) { + return Message.error("当前工作流被用户" + flowEditLock.getUsername() + "已锁定编辑，您编辑的内容不能再被保存。如有疑问，请与" + flowEditLock.getUsername() + "确认"); + } + + modifyOrchestratorMetaRequest.setWorkspaceId(workspace.getWorkspaceId()); + modifyOrchestratorMetaRequest.setWorkspaceName(workspace.getWorkspaceName()); + + try { + orchestratorFrameworkService.modifyOrchestratorMeta(username, modifyOrchestratorMetaRequest, workspace,orchestratorVersion); + } catch (Exception e) { + LOGGER.error(String.format("%s modify OrchestratorMeta fail", modifyOrchestratorMetaRequest.getOrchestratorName())); + LOGGER.error(e.getMessage(),e); + return Message.error(String.format("%s 工作流信息编辑失败,原因为 %s", modifyOrchestratorMetaRequest.getOrchestratorName(), e.getMessage())); + } + + AuditLogUtils.printLog(username, workspace.getWorkspaceId(), workspace.getWorkspaceName(), TargetTypeEnum.ORCHESTRATOR, + modifyOrchestratorMetaRequest.getOrchestratorId(), modifyOrchestratorMetaRequest.getOrchestratorName(), OperateTypeEnum.UPDATE, modifyOrchestratorMetaRequest); + + return Message.ok(String.format("%s工作流信息编辑成功", modifyOrchestratorMetaRequest.getOrchestratorName())); + } + + + /** + * 获取编排状态 + **/ + @RequestMapping(value = "/getOrchestratorGitStatus", method = RequestMethod.GET) + public Message getOrchestratorGitStatus() { + + return Message.ok().data("status", orchestratorService.getOrchestratorGitStatus()); + } + + + /** + * 获取编排名称 + **/ + @RequestMapping(value = "/getAllOrchestratorName", method = RequestMethod.GET) + public Message getAllOrchestratorName(HttpServletRequest httpServletRequest, + @RequestParam(value = "workspaceId", required = false) Long workspaceId, + @RequestParam(value = "projectName",required = false) String projectName) { + + if (workspaceId == null) { + workspaceId = SSOHelper.getWorkspace(httpServletRequest).getWorkspaceId(); + } + + String username = SecurityFilter.getLoginUsername(httpServletRequest); + + try { + dssWorkspaceService.getWorkspacesById(workspaceId, username); + } catch (DSSErrorException e) { + LOGGER.error("User {} get workspace {} failed.", username, workspaceId, e); + return Message.error(e); + } + + List roles = dssWorkspaceRoleService.getRoleInWorkspace(username, workspaceId.intValue()); + if (roles == null || roles.isEmpty()) { + LOGGER.error("username {}, in workspace {} roles are null or empty", username,workspaceId); + return Message.error("can not get roles information"); + } + + LOGGER.info(String.format("getAllOrchestratorName workspaceId is %s", workspaceId)); + return Message.ok().data("data", orchestratorService.getAllOrchestratorName(workspaceId,projectName)); + } + + + @RequestMapping(value = "publishFlowCheck",method = RequestMethod.GET) + public Message publishFlowCheck(@RequestParam("orchestratorId") Long orchestratorId, + @RequestParam("projectId") Long projectId) throws DSSErrorException{ + + Workspace workspace = SSOHelper.getWorkspace(httpServletRequest); + + return Message.ok().data("data",orchestratorPluginService.getNotContainsKeywordsNode(orchestratorId,projectId,workspace)); + } + + @RequestMapping(value = "batchPublishFlowCheck",method = RequestMethod.POST) + public Message batchPublishFlowCheck(@RequestBody BatchPublishFlowCheckRequest batchPublishFlowCheckRequest) throws DSSErrorException{ + + Workspace workspace = SSOHelper.getWorkspace(httpServletRequest); + + return Message.ok().data("data",orchestratorPluginService.batchPublishFlowCheck(batchPublishFlowCheckRequest,workspace)); + } + } diff --git a/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/restful/OrchestratorIERestful.java b/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/restful/OrchestratorIERestful.java index 29f36fd914..6149bf0e68 100644 --- a/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/restful/OrchestratorIERestful.java +++ b/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/restful/OrchestratorIERestful.java @@ -114,7 +114,7 @@ public Message importOrcFile(HttpServletRequest req, RequestImportOrchestrator importRequest = new RequestImportOrchestrator(userName, projectName, projectID, resultMap.getResourceId(), resultMap.getVersion(), null, dssLabelList, workspace); - dssOrchestratorVersion = orchestratorContext.getDSSOrchestratorPlugin(ImportDSSOrchestratorPlugin.class).importOrchestrator(importRequest); + dssOrchestratorVersion = orchestratorContext.getDSSOrchestratorPlugin(ImportDSSOrchestratorPlugin.class).importOrchestratorNew(importRequest); AuditLogUtils.printLog(userName, workspace.getWorkspaceId(), workspace.getWorkspaceName(), TargetTypeEnum.ORCHESTRATOR, projectID, projectName, OperateTypeEnum.CREATE, importRequest); } catch (Exception e) { @@ -153,7 +153,7 @@ public void exportOrcFile(HttpServletRequest req, OrchestratorFrameworkServiceImpl.validateOperation(orchestratorVo.getDssOrchestratorInfo().getProjectId(), userName); logger.info("export orchestrator orchestratorId " + orchestratorId + ",orcVersionId:" + orcVersionId); try { - res = orchestratorContext.getDSSOrchestratorPlugin(ExportDSSOrchestratorPlugin.class).exportOrchestrator(userName, + res = orchestratorContext.getDSSOrchestratorPlugin(ExportDSSOrchestratorPlugin.class).exportOrchestratorNew(userName, orchestratorId, orcVersionId, projectName, dssLabelList, addOrcVersion, workspace).getBmlResource(); AuditLogUtils.printLog(userName, workspace.getWorkspaceId(), workspace.getWorkspaceName(), TargetTypeEnum.ORCHESTRATOR, orcVersionId, orchestratorVo.getDssOrchestratorInfo().getName(), OperateTypeEnum.UPDATE, orchestratorVo); diff --git a/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/service/OrchestratorFrameworkService.java b/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/service/OrchestratorFrameworkService.java index 3d56192a91..9318c6b34e 100644 --- a/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/service/OrchestratorFrameworkService.java +++ b/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/service/OrchestratorFrameworkService.java @@ -17,15 +17,20 @@ package com.webank.wedatasphere.dss.orchestrator.server.service; import com.webank.wedatasphere.dss.common.exception.DSSErrorException; -import com.webank.wedatasphere.dss.orchestrator.common.entity.DSSOrchestratorCopyInfo; +import com.webank.wedatasphere.dss.common.label.LabelRouteVO; +import com.webank.wedatasphere.dss.git.common.protocol.GitTree; +import com.webank.wedatasphere.dss.git.common.protocol.response.GitHistoryResponse; +import com.webank.wedatasphere.dss.orchestrator.common.entity.*; import com.webank.wedatasphere.dss.orchestrator.server.entity.request.*; import com.webank.wedatasphere.dss.orchestrator.server.entity.vo.CommonOrchestratorVo; import com.webank.wedatasphere.dss.orchestrator.server.entity.vo.OrchestratorCopyHistory; import com.webank.wedatasphere.dss.orchestrator.server.entity.vo.OrchestratorUnlockVo; import com.webank.wedatasphere.dss.standard.app.sso.Workspace; + import org.apache.commons.math3.util.Pair; import java.util.List; +import java.util.Map; public interface OrchestratorFrameworkService { @@ -45,4 +50,17 @@ public interface OrchestratorFrameworkService { Boolean getOrchestratorCopyStatus(Long sourceOrchestratorId); DSSOrchestratorCopyInfo getOrchestratorCopyInfoById(String copyInfoId); + + + OrchestratorSubmitJob getOrchestratorStatus(Long orchestratorId); + + GitHistoryResponse getHistory(Long workspaceId, Long orchestratorId, String projectName); + + + void modifyOrchestratorMeta(String username, ModifyOrchestratorMetaRequest modifyOrchestratorMetaRequest, Workspace workspace, DSSOrchestratorVersion orchestratorVersion) throws Exception; + + List getAllOrchestratorMeta(OrchestratorMetaRequest orchestratorMetaRequest, List total,String username); + + DSSOrchestratorVersion getLatestOrchestratorVersion(Long orchestratorId); + } diff --git a/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/service/OrchestratorPluginService.java b/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/service/OrchestratorPluginService.java index e15ae50503..db1d424ff8 100644 --- a/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/service/OrchestratorPluginService.java +++ b/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/service/OrchestratorPluginService.java @@ -16,8 +16,22 @@ package com.webank.wedatasphere.dss.orchestrator.server.service; +import com.webank.wedatasphere.dss.common.entity.BmlResource; +import com.webank.wedatasphere.dss.common.exception.DSSErrorException; +import com.webank.wedatasphere.dss.git.common.protocol.GitTree; +import com.webank.wedatasphere.dss.git.common.protocol.response.GitCommitResponse; +import com.webank.wedatasphere.dss.git.common.protocol.response.GitFileContentResponse; import com.webank.wedatasphere.dss.orchestrator.common.protocol.RequestFrameworkConvertOrchestration; import com.webank.wedatasphere.dss.orchestrator.common.protocol.ResponseConvertOrchestrator; +import com.webank.wedatasphere.dss.orchestrator.server.entity.request.BatchPublishFlowCheckRequest; +import com.webank.wedatasphere.dss.orchestrator.server.entity.request.OrchestratorSubmitRequest; +import com.webank.wedatasphere.dss.orchestrator.server.entity.vo.OrchestratorDiffDirVo; +import com.webank.wedatasphere.dss.orchestrator.server.entity.vo.OrchestratorDiffNodeVo; +import com.webank.wedatasphere.dss.orchestrator.server.entity.vo.OrchestratorRelationVo; +import com.webank.wedatasphere.dss.standard.app.sso.Workspace; + +import java.util.List; +import java.util.Map; public interface OrchestratorPluginService { @@ -29,4 +43,26 @@ public interface OrchestratorPluginService { ResponseConvertOrchestrator getConvertOrchestrationStatus(String id); + Long diffFlow(OrchestratorSubmitRequest flowRequest, String username, Workspace workspace) throws Exception; + + OrchestratorDiffDirVo diffPublish(OrchestratorSubmitRequest flowRequest, String username, Workspace workspace); + + GitFileContentResponse diffFlowContent(OrchestratorSubmitRequest flowRequest, String username, Workspace workspace) throws DSSErrorException; + + void submitFlow(OrchestratorSubmitRequest flowRequest, String username, Workspace workspace) throws DSSErrorException; + + void batchSubmitFlow(Map> map, Map projectMap, String username, Workspace workspace, String label, String comment) throws DSSErrorException; + + GitCommitResponse submitWorkflowSync(OrchestratorSubmitRequest flowRequest, String username, Workspace workspace) throws DSSErrorException; + + BmlResource uploadWorkflowListToGit(List flowIdList, String projectName, String label, String username, Workspace workspace, Long projectId) throws Exception; + + String diffStatus(Long taskId) throws DSSErrorException; + + OrchestratorDiffDirVo diffContent(Long taskId); + + OrchestratorDiffNodeVo getNotContainsKeywordsNode(long orchestratorId, long projectId, Workspace workspace) throws DSSErrorException; + + List batchPublishFlowCheck(BatchPublishFlowCheckRequest request,Workspace workspace) throws DSSErrorException; + } diff --git a/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/service/OrchestratorService.java b/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/service/OrchestratorService.java index 0e19e5860e..82c1745b9b 100644 --- a/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/service/OrchestratorService.java +++ b/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/service/OrchestratorService.java @@ -17,21 +17,27 @@ package com.webank.wedatasphere.dss.orchestrator.server.service; +import com.webank.wedatasphere.dss.common.entity.BmlResource; +import com.webank.wedatasphere.dss.common.entity.project.DSSProject; import com.webank.wedatasphere.dss.common.exception.DSSErrorException; import com.webank.wedatasphere.dss.common.label.DSSLabel; +import com.webank.wedatasphere.dss.common.label.LabelRouteVO; +import com.webank.wedatasphere.dss.common.service.BMLService; import com.webank.wedatasphere.dss.framework.common.exception.DSSFrameworkErrorException; -import com.webank.wedatasphere.dss.orchestrator.common.entity.DSSOrchestratorInfo; -import com.webank.wedatasphere.dss.orchestrator.common.entity.DSSOrchestratorVersion; -import com.webank.wedatasphere.dss.orchestrator.common.entity.OrchestratorVo; +import com.webank.wedatasphere.dss.orchestrator.common.entity.*; import com.webank.wedatasphere.dss.orchestrator.common.protocol.RequestOrchestratorInfos; import com.webank.wedatasphere.dss.orchestrator.common.protocol.ResponseOrchestratorInfos; +import com.webank.wedatasphere.dss.orchestrator.server.entity.request.ModifyOrchestratorMetaRequest; import com.webank.wedatasphere.dss.orchestrator.server.entity.request.OrchestratorModifyRequest; import com.webank.wedatasphere.dss.orchestrator.server.entity.request.OrchestratorRequest; import com.webank.wedatasphere.dss.orchestrator.server.entity.vo.OrchestratorBaseInfo; +import com.webank.wedatasphere.dss.orchestrator.server.entity.vo.OrchestratorRollBackGitVo; +import com.webank.wedatasphere.dss.orchestrator.server.entity.vo.OrchestratorStatusVo; import com.webank.wedatasphere.dss.orchestrator.server.entity.vo.OrchestratorUnlockVo; import com.webank.wedatasphere.dss.standard.app.sso.Workspace; import java.util.List; +import java.util.concurrent.ExecutionException; public interface OrchestratorService { @@ -137,18 +143,36 @@ OrchestratorUnlockVo unlockOrchestrator(String userName, List getOrchestratorVersions(String username, Long projectId, Long orchestratorId); - String rollbackOrchestrator(String username, Long projectId, String projectName, - Long orchestratorId, String version, DSSLabel dssLabel, Workspace workspace) throws Exception; + OrchestratorRollBackGitVo rollbackOrchestrator(String username, Long projectId, String projectName, + Long orchestratorId, String version, LabelRouteVO labels, Workspace workspace) throws Exception; + void rollbackOrchestratorGit(OrchestratorRollBackGitVo rollBackGitVo, String userName, Long projectId, String projectName, + Long orchestratorId, LabelRouteVO labels, Workspace workspace, Long latestAppId) throws Exception; //**** new method void isExistSameNameBeforeCreate(Long workspaceId, Long projectId, String orchestratorName) throws DSSFrameworkErrorException; - Long isExistSameNameBeforeUpdate(OrchestratorModifyRequest orchestratorModifRequest) throws DSSFrameworkErrorException; + Long isExistSameNameBeforeUpdate(OrchestratorModifyRequest orchestratorModifRequest, DSSProject dssProject, String username) throws DSSFrameworkErrorException; - List getListByPage(OrchestratorRequest orchestratorRequest, String username); + List getOrchestratorInfos(OrchestratorRequest orchestratorRequest, String username); ResponseOrchestratorInfos queryOrchestratorInfos(RequestOrchestratorInfos requestOrchestratorInfos); void batchClearContextId(); + String getAuthenToken(String gitUrl, String gitUsername, String gitPassword) throws ExecutionException; + + OrchestratorVo getOrchestratorByAppId(Long appId); + + List getOrchestratorInfoByLabel(OrchestratorRequest orchestratorRequest); + + List getOrchestratorGitStatus(); + + List getAllOrchestratorName(Long workspaceId,String projectName); + + OrchestratorVo updateOrchestratorBML(Long orchestratorId, BmlResource bmlResource); + + Boolean updateOrchestratorTime(Long orchestratorId); + + String getGitUrl(String workflowName, String workflowNodeName, String projectName, String userName, Workspace workspace, Long workflowId) throws Exception; + } diff --git a/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/service/impl/BatchClearCsTask.java b/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/service/impl/BatchClearCsTask.java new file mode 100644 index 0000000000..4c416246f2 --- /dev/null +++ b/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/service/impl/BatchClearCsTask.java @@ -0,0 +1,24 @@ +//package com.webank.wedatasphere.dss.orchestrator.server.service.impl; +// +//import com.webank.wedatasphere.dss.orchestrator.server.service.OrchestratorService; +//import org.springframework.beans.factory.annotation.Autowired; +//import org.springframework.context.annotation.PropertySource; +//import org.springframework.scheduling.annotation.EnableScheduling; +//import org.springframework.scheduling.annotation.Scheduled; +//import org.springframework.stereotype.Component; +// +///** +// * Description +// */ +//@Component +//@EnableScheduling +//@PropertySource("classpath:dss-framework-orchestrator-server.properties") +//public class BatchClearCsTask { +// @Autowired +// OrchestratorService orchestratorService; +// +// @Scheduled(cron = "${wds.dss.server.scheduling.clear.cs.cron}") +// public void batchClearCsTask(){ +// orchestratorService.batchClearContextId(); +// } +//} diff --git a/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/service/impl/OrchestratorFrameworkServiceImpl.java b/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/service/impl/OrchestratorFrameworkServiceImpl.java index a57950020b..85fbf65090 100644 --- a/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/service/impl/OrchestratorFrameworkServiceImpl.java +++ b/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/service/impl/OrchestratorFrameworkServiceImpl.java @@ -16,8 +16,11 @@ package com.webank.wedatasphere.dss.orchestrator.server.service.impl; +import cn.hutool.core.collection.CollUtil; import com.google.common.collect.Lists; +import com.google.common.reflect.TypeToken; import com.google.common.util.concurrent.ThreadFactoryBuilder; +import com.google.gson.*; import com.webank.wedatasphere.dss.appconn.scheduler.SchedulerAppConn; import com.webank.wedatasphere.dss.appconn.scheduler.structure.orchestration.OrchestrationCreationOperation; import com.webank.wedatasphere.dss.appconn.scheduler.structure.orchestration.OrchestrationDeletionOperation; @@ -31,17 +34,26 @@ import com.webank.wedatasphere.dss.common.constant.project.ProjectUserPrivEnum; import com.webank.wedatasphere.dss.common.entity.project.DSSProject; import com.webank.wedatasphere.dss.common.exception.DSSErrorException; +import com.webank.wedatasphere.dss.common.exception.DSSRuntimeException; import com.webank.wedatasphere.dss.common.label.DSSLabel; import com.webank.wedatasphere.dss.common.label.EnvDSSLabel; +import com.webank.wedatasphere.dss.common.label.LabelRouteVO; import com.webank.wedatasphere.dss.common.protocol.project.*; +import com.webank.wedatasphere.dss.common.service.BMLService; import com.webank.wedatasphere.dss.common.utils.DSSCommonUtils; import com.webank.wedatasphere.dss.common.utils.DSSExceptionUtils; import com.webank.wedatasphere.dss.common.utils.RpcAskUtils; +import com.webank.wedatasphere.dss.contextservice.service.ContextService; +import com.webank.wedatasphere.dss.contextservice.service.impl.ContextServiceImpl; import com.webank.wedatasphere.dss.framework.common.exception.DSSFrameworkErrorException; -import com.webank.wedatasphere.dss.orchestrator.common.entity.DSSOrchestratorCopyInfo; -import com.webank.wedatasphere.dss.orchestrator.common.entity.DSSOrchestratorInfo; -import com.webank.wedatasphere.dss.orchestrator.common.entity.DSSOrchestratorRefOrchestration; -import com.webank.wedatasphere.dss.orchestrator.common.entity.OrchestratorVo; +import com.webank.wedatasphere.dss.git.common.protocol.exception.GitErrorException; +import com.webank.wedatasphere.dss.git.common.protocol.request.GitCommitInfoBetweenRequest; +import com.webank.wedatasphere.dss.git.common.protocol.request.GitRemoveRequest; +import com.webank.wedatasphere.dss.git.common.protocol.request.GitRenameRequest; +import com.webank.wedatasphere.dss.git.common.protocol.response.GitCommitResponse; +import com.webank.wedatasphere.dss.git.common.protocol.response.GitHistoryResponse; +import com.webank.wedatasphere.dss.orchestrator.common.entity.*; +import com.webank.wedatasphere.dss.orchestrator.common.ref.OrchestratorRefConstant; import com.webank.wedatasphere.dss.orchestrator.core.DSSOrchestrator; import com.webank.wedatasphere.dss.orchestrator.core.exception.DSSOrchestratorErrorException; import com.webank.wedatasphere.dss.orchestrator.core.type.DSSOrchestratorRelation; @@ -50,6 +62,8 @@ import com.webank.wedatasphere.dss.orchestrator.db.dao.OrchestratorCopyJobMapper; import com.webank.wedatasphere.dss.orchestrator.db.dao.OrchestratorMapper; import com.webank.wedatasphere.dss.orchestrator.loader.OrchestratorManager; +import com.webank.wedatasphere.dss.orchestrator.server.constant.DSSOrchestratorConstant; +import com.webank.wedatasphere.dss.orchestrator.server.constant.OrchestratorStatusEnum; import com.webank.wedatasphere.dss.orchestrator.server.entity.request.*; import com.webank.wedatasphere.dss.orchestrator.server.entity.vo.CommonOrchestratorVo; import com.webank.wedatasphere.dss.orchestrator.server.entity.vo.OrchestratorCopyHistory; @@ -67,6 +81,12 @@ import com.webank.wedatasphere.dss.standard.common.desc.AppInstance; import com.webank.wedatasphere.dss.standard.common.entity.ref.ResponseRef; import com.webank.wedatasphere.dss.standard.common.exception.operation.ExternalOperationWarnException; +import com.webank.wedatasphere.dss.workflow.common.entity.DSSFlow; + +import com.webank.wedatasphere.dss.workflow.dao.*; +import com.webank.wedatasphere.dss.workflow.dto.NodeContentDO; +import com.webank.wedatasphere.dss.workflow.service.DSSFlowService; +import com.webank.wedatasphere.dss.workflow.service.SaveFlowHook; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang.StringUtils; import org.apache.commons.math3.util.Pair; @@ -75,6 +95,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import java.time.LocalDateTime; import java.time.ZoneId; @@ -84,6 +105,7 @@ import java.util.function.BiFunction; import java.util.function.Function; import java.util.stream.Collectors; +import java.util.stream.Stream; public class OrchestratorFrameworkServiceImpl implements OrchestratorFrameworkService { @@ -104,7 +126,25 @@ public class OrchestratorFrameworkServiceImpl implements OrchestratorFrameworkSe private OrchestratorCopyEnv orchestratorCopyEnv; @Autowired private OrchestratorOperateService orchestratorOperateService; + @Autowired + private LockMapper lockMapper; + @Autowired + @Qualifier("workflowBmlService") + private BMLService bmlService; + @Autowired + private FlowMapper flowMapper; + @Autowired + private NodeMetaMapper nodeMetaMapper; + @Autowired + private SaveFlowHook saveFlowHook; + @Autowired + private NodeContentMapper nodeContentMapper; + @Autowired + private NodeContentUIMapper nodeContentUIMapper; + private static ContextService contextService = ContextServiceImpl.getInstance(); + @Autowired + private DSSFlowService flowService; private static final int MAX_DESC_LENGTH = 250; private static final int MAX_NAME_LENGTH = 128; @@ -116,6 +156,10 @@ public class OrchestratorFrameworkServiceImpl implements OrchestratorFrameworkSe private final ExecutorService orchestratorCopyThreadPool = new ThreadPoolExecutor(0, 200, 60L, TimeUnit.SECONDS, new LinkedBlockingQueue<>(1024), orchestratorCopyThreadFactory, new ThreadPoolExecutor.AbortPolicy()); + protected Sender getOrchestratorSender() { + return DSSSenderServiceFactory.getOrCreateServiceInstance().getOrcSender(); + } + /** * 1.拿到的dss orchestrator的appconn * 2.然后创建 @@ -128,16 +172,22 @@ public class OrchestratorFrameworkServiceImpl implements OrchestratorFrameworkSe public CommonOrchestratorVo createOrchestrator(String username, OrchestratorCreateRequest orchestratorCreateRequest, Workspace workspace) throws Exception { //检查desc字段长度 - if(orchestratorCreateRequest.getDescription().length() > MAX_DESC_LENGTH){ + if (orchestratorCreateRequest.getDescription().length() > MAX_DESC_LENGTH) { DSSFrameworkErrorException.dealErrorException(60000, "描述过长，请限制在" + MAX_DESC_LENGTH + "以内"); } - if(orchestratorCreateRequest.getOrchestratorName().length() > MAX_NAME_LENGTH){ + if (orchestratorCreateRequest.getOrchestratorName().length() > MAX_NAME_LENGTH) { DSSFrameworkErrorException.dealErrorException(60000, "编排名称过长，请限制在" + MAX_NAME_LENGTH + "以内"); } //是否存在相同的编排名称 orchestratorService.isExistSameNameBeforeCreate(orchestratorCreateRequest.getWorkspaceId(), orchestratorCreateRequest.getProjectId(), orchestratorCreateRequest.getOrchestratorName()); //判断工程是否存在,并且取出工程名称和空间名称 DSSProject dssProject = validateOperation(orchestratorCreateRequest.getProjectId(), username); + Integer dssProjectWorkspaceId = dssProject.getWorkspaceId(); + long workspaceId = workspace.getWorkspaceId(); + if(dssProjectWorkspaceId == null || !String.valueOf(dssProjectWorkspaceId).equals(String.valueOf(workspaceId))){ + LOGGER.error("createOrchestrator projectId is {},project workspaceId is {}, workspace params is {}",dssProject.getId(),dssProject.getName(), workspaceId); + DSSFrameworkErrorException.dealErrorException(60000, "无法在当前工作空间创建工作流,"+dssProject.getName()+"项目不属于"+workspace.getWorkspaceName()+"工作空间"); + } //1.创建编排实体bean DSSOrchestratorRelation dssOrchestratorRelation = DSSOrchestratorRelationManager.getDSSOrchestratorRelationByMode(orchestratorCreateRequest.getOrchestratorMode()); DSSOrchestratorInfo dssOrchestratorInfo = new DSSOrchestratorInfo(); @@ -152,10 +202,11 @@ public CommonOrchestratorVo createOrchestrator(String username, OrchestratorCrea dssOrchestratorInfo.setSecondaryType(orchestratorCreateRequest.getOrchestratorWays().toString()); dssOrchestratorInfo.setUses(orchestratorCreateRequest.getUses()); //new field - dssOrchestratorInfo.setWorkspaceId(workspace.getWorkspaceId()); + dssOrchestratorInfo.setWorkspaceId(workspaceId); dssOrchestratorInfo.setOrchestratorWay(OrchestratorUtils.getModeStr(orchestratorCreateRequest.getOrchestratorWays())); dssOrchestratorInfo.setOrchestratorMode(orchestratorCreateRequest.getOrchestratorMode()); dssOrchestratorInfo.setOrchestratorLevel(orchestratorCreateRequest.getOrchestratorLevel()); + dssOrchestratorInfo.setIsDefaultReference(orchestratorCreateRequest.getIsDefaultReference()); //1.去orchestratorFramework创建编排模式 LOGGER.info("{} begins to create a orchestrator {}.", username, orchestratorCreateRequest); List dssLabels = Collections.singletonList(new EnvDSSLabel(orchestratorCreateRequest.getLabels().getRoute())); @@ -169,6 +220,10 @@ public CommonOrchestratorVo createOrchestrator(String username, OrchestratorCrea dssOrchestratorInfo.getProjectId(), dssOrchestratorInfo.getDesc(), dssOrchestratorInfo, dssLabels); Long orchestratorId = orchestratorVo.getDssOrchestratorInfo().getId(); Long orchestratorVersionId = orchestratorVo.getDssOrchestratorVersion().getId(); + // 初始化置为已提交 + if (dssProject.getAssociateGit() != null && dssProject.getAssociateGit()) { + lockMapper.updateOrchestratorStatus(orchestratorId, OrchestratorRefConstant.FLOW_STATUS_SAVE); + } LOGGER.info("created orchestration {} with orchestratorId is {}, and versionId is {}.", orchestratorCreateRequest.getOrchestratorName(), orchestratorId, orchestratorVersionId); //4.将工程和orchestrator的关系存储到的数据库中 if (orchestrationResponseRef != null) { @@ -180,7 +235,7 @@ public CommonOrchestratorVo createOrchestrator(String username, OrchestratorCrea return commonOrchestratorVo; } - public V tryOrchestrationOperation(List dssLabels, Boolean askProjectSender, String userName, String projectName, + private V tryOrchestrationOperation(List dssLabels, Boolean askProjectSender, String userName, String projectName, Workspace workspace, DSSOrchestratorInfo dssOrchestrator, Function getOrchestrationOperation, BiFunction responseRefConsumer, String operationName) { @@ -188,8 +243,8 @@ public V tryOrchestration Long refProjectId, refOrchestrationId; if (askProjectSender) { ProjectRefIdResponse projectRefIdResponse = RpcAskUtils.processAskException(DSSSenderServiceFactory.getOrCreateServiceInstance().getProjectServerSender() - .ask(new ProjectRefIdRequest(Optional.ofNullable(orchestrationPair).map(ImmutablePair::getValue).map(AppInstance::getId).orElse(null), dssOrchestrator.getProjectId())), ProjectRefIdResponse.class, ProjectRefIdRequest.class); - refProjectId = projectRefIdResponse.getRefProjectId(); + .ask(new ProjectRefIdRequest(orchestrationPair.getValue().getId(), dssOrchestrator.getProjectId())), ProjectRefIdResponse.class, ProjectRefIdRequest.class); + refProjectId = projectRefIdResponse.getRefProjectId(); refOrchestrationId = null; } else { DSSOrchestratorRefOrchestration refOrchestration = orchestratorMapper.getRefOrchestrationId(dssOrchestrator.getId()); @@ -225,17 +280,21 @@ public V tryOrchestration @Override public CommonOrchestratorVo modifyOrchestrator(String username, OrchestratorModifyRequest orchestratorModifyRequest, Workspace workspace) throws Exception { //检查desc字段长度 - if(orchestratorModifyRequest.getDescription().length() > MAX_DESC_LENGTH){ + if (orchestratorModifyRequest.getDescription().length() > MAX_DESC_LENGTH) { DSSFrameworkErrorException.dealErrorException(60000, "描述字段过长，请限制在" + MAX_DESC_LENGTH + "以内"); } - if(orchestratorModifyRequest.getOrchestratorName().length() > MAX_NAME_LENGTH){ + if (orchestratorModifyRequest.getOrchestratorName().length() > MAX_NAME_LENGTH) { DSSFrameworkErrorException.dealErrorException(60000, "编排名称过长，请限制在" + MAX_NAME_LENGTH + "以内"); } //判断工程是否存在,并且取出工程名称和空间名称 DSSProject dssProject = validateOperation(orchestratorModifyRequest.getProjectId(), username); + if(dssProject.getWorkspaceId() == null || !String.valueOf(dssProject.getWorkspaceId()).equals(String.valueOf(workspace.getWorkspaceId()))){ + LOGGER.error("modifyOrchestrator projectId is {},project workspaceId is {}, workspace params is {}",dssProject.getId(),dssProject.getName(),workspace.getWorkspaceId()); + DSSFrameworkErrorException.dealErrorException(60000, "无法在当前工作空间修改工作流,"+dssProject.getName()+"项目不属于"+workspace.getWorkspaceName()+"工作空间"); + } workspace.setWorkspaceName(dssProject.getWorkspaceName()); //是否存在相同的编排名称 //todo 返回orchestratorInfo而不是id - Long orchestratorId = orchestratorService.isExistSameNameBeforeUpdate(orchestratorModifyRequest); + Long orchestratorId = orchestratorService.isExistSameNameBeforeUpdate(orchestratorModifyRequest, dssProject, username); LOGGER.info("{} begins to update a orchestrator {}.", username, orchestratorModifyRequest.getOrchestratorName()); List dssLabels = Collections.singletonList(new EnvDSSLabel(orchestratorModifyRequest.getLabels().getRoute())); DSSOrchestratorRelation dssOrchestratorRelation = DSSOrchestratorRelationManager.getDSSOrchestratorRelationByMode(orchestratorModifyRequest.getOrchestratorMode()); @@ -255,6 +314,7 @@ public CommonOrchestratorVo modifyOrchestrator(String username, OrchestratorModi dssOrchestratorInfo.setOrchestratorWay(OrchestratorUtils.getModeStr(orchestratorModifyRequest.getOrchestratorWays())); dssOrchestratorInfo.setOrchestratorLevel(orchestratorModifyRequest.getOrchestratorLevel()); dssOrchestratorInfo.setUses(orchestratorModifyRequest.getUses()); + dssOrchestratorInfo.setIsDefaultReference(orchestratorModifyRequest.getIsDefaultReference()); //1.如果调度系统要求同步创建工作流，向调度系统发送更新工作流的请求 tryOrchestrationOperation(dssLabels, false, username, dssProject.getName(), workspace, dssOrchestratorInfo, OrchestrationService::getOrchestrationUpdateOperation, @@ -264,6 +324,8 @@ public CommonOrchestratorVo modifyOrchestrator(String username, OrchestratorModi //3.将工程和orchestrator的关系存储到的数据库中 CommonOrchestratorVo orchestratorVo = new CommonOrchestratorVo(); orchestratorVo.setOrchestratorId(orchestratorId); + // 清空BML + orchestratorMapper.updateOrchestratorBmlVersion(orchestratorId, null, null); return orchestratorVo; } @@ -274,19 +336,50 @@ public CommonOrchestratorVo deleteOrchestrator(String username, OrchestratorDele DSSOrchestratorInfo orchestratorInfo = orchestratorMapper.getOrchestrator(orchestratorDeleteRequest.getId()); LOGGER.info("{} begins to delete a orchestrator {}.", username, orchestratorInfo.getName()); List dssLabels = Collections.singletonList(new EnvDSSLabel(orchestratorDeleteRequest.getLabels().getRoute())); - if(orchestratorDeleteRequest.getDeleteSchedulerWorkflow()) { + if (orchestratorDeleteRequest.getDeleteSchedulerWorkflow()) { tryOrchestrationOperation(dssLabels, false, username, dssProject.getName(), workspace, orchestratorInfo, OrchestrationService::getOrchestrationDeletionOperation, (structureOperation, structureRequestRef) -> ((OrchestrationDeletionOperation) structureOperation) .deleteOrchestration((RefOrchestrationContentRequestRef) structureRequestRef), "delete"); } - - orchestratorService.deleteOrchestrator(username, workspace, dssProject.getName(), orchestratorInfo.getId(), dssLabels); - orchestratorOperateService.deleteTemplateOperate(orchestratorInfo.getId()); + Long orchestratorInfoId = orchestratorInfo.getId(); + DSSOrchestratorVersion versionById = orchestratorMapper.getLatestOrchestratorVersionById(orchestratorInfoId); + if (dssProject.getAssociateGit() != null && dssProject.getAssociateGit()) { + try { + // git删除成功之后再删除库表记录 + Sender sender = DSSSenderServiceFactory.getOrCreateServiceInstance().getGitSender(); + List path = new ArrayList<>(); + path.add(orchestratorInfo.getName()); + GitRemoveRequest removeRequest = new GitRemoveRequest(workspace.getWorkspaceId(), dssProject.getName(), path, username); + GitCommitResponse commitResponse = RpcAskUtils.processAskException(sender.ask(removeRequest), GitCommitResponse.class, GitRemoveRequest.class); + lockMapper.updateOrchestratorStatus(orchestratorDeleteRequest.getId(), OrchestratorRefConstant.FLOW_STATUS_PUSH); + if (versionById != null) { + lockMapper.updateOrchestratorVersionCommitId(commitResponse.getCommitId(), versionById.getAppId()); + } + } catch (Exception e) { + LOGGER.error("git remove failed, the reason is: ", e); + lockMapper.updateOrchestratorStatus(orchestratorDeleteRequest.getId(), OrchestratorRefConstant.FLOW_STATUS_SAVE); + throw new GitErrorException(800001, e.getMessage()); + } + } + orchestratorService.deleteOrchestrator(username, workspace, dssProject.getName(), orchestratorInfoId, dssLabels); + orchestratorOperateService.deleteTemplateOperate(orchestratorInfoId); LOGGER.info("delete orchestrator {} by orchestrator framework succeed.", orchestratorInfo.getName()); CommonOrchestratorVo orchestratorVo = new CommonOrchestratorVo(); orchestratorVo.setOrchestratorName(orchestratorInfo.getName()); - orchestratorVo.setOrchestratorId(orchestratorInfo.getId()); + orchestratorVo.setOrchestratorId(orchestratorInfoId); + + // 同步更新flowJson + List contentDOS = nodeContentMapper.getContentListByOrchestratorIdAndFlowId(orchestratorInfoId, versionById.getAppId()); + if (CollectionUtils.isEmpty(contentDOS)) { + return orchestratorVo; + } + List contentIdList = contentDOS.stream().map(NodeContentDO::getId).collect(Collectors.toList()); + nodeContentMapper.deleteNodeContentByOrchestratorIdAndFlowId(orchestratorInfoId, versionById.getAppId()); + if (CollectionUtils.isNotEmpty(contentIdList)) { + nodeContentUIMapper.deleteNodeContentUIByContentList(contentIdList); + } + nodeMetaMapper.deleteNodeMetaByOrchestratorId(orchestratorInfoId); return orchestratorVo; } @@ -305,8 +398,13 @@ public OrchestratorUnlockVo unlockOrchestrator(String username, Workspace worksp } @Override - public String copyOrchestrator(String username, OrchestratorCopyRequest orchestratorCopyRequest, Workspace workspace) throws Exception{ - if(orchestratorCopyRequest.getTargetOrchestratorName().length() > MAX_NAME_LENGTH){ + public DSSOrchestratorVersion getLatestOrchestratorVersion(Long orchestratorId) { + return orchestratorMapper.getLatestOrchestratorVersionById(orchestratorId); + } + + @Override + public String copyOrchestrator(String username, OrchestratorCopyRequest orchestratorCopyRequest, Workspace workspace) throws Exception { + if (orchestratorCopyRequest.getTargetOrchestratorName().length() > MAX_NAME_LENGTH) { DSSFrameworkErrorException.dealErrorException(60000, "编排名称过长，请限制在" + MAX_NAME_LENGTH + "以内"); } //校验编排名是可用 @@ -325,11 +423,18 @@ public String copyOrchestrator(String username, OrchestratorCopyRequest orchestr } else if (orchestratorCopyRequest.getWorkflowNodeSuffix().length() > 10) { DSSExceptionUtils.dealErrorException(6014, "The node suffix length can not exceed 10. (节点后缀长度不能超过10)", DSSOrchestratorErrorException.class); } + String dssLabel = null; + if (orchestratorCopyRequest.getLabels() != null && orchestratorCopyRequest.getLabels().getRoute() != null) { + dssLabel = orchestratorCopyRequest.getLabels().getRoute(); + } else { + dssLabel = DSSCommonUtils.ENV_LABEL_VALUE_DEV; + } OrchestratorCopyVo orchestratorCopyVo = new OrchestratorCopyVo.Builder(username, sourceProject.getId(), sourceProject.getName(), targetProject.getId(), targetProject.getName(), sourceOrchestratorInfo, orchestratorCopyRequest.getTargetOrchestratorName(), - orchestratorCopyRequest.getWorkflowNodeSuffix(), new EnvDSSLabel(DSSCommonUtils.ENV_LABEL_VALUE_DEV), + orchestratorCopyRequest.getWorkflowNodeSuffix(), new EnvDSSLabel(dssLabel), workspace, Sender.getThisInstance()).setCopyTaskId(null).build(); OrchestratorCopyJob orchestratorCopyJob = new OrchestratorCopyJob(); + orchestratorCopyJob.setTargetProject(targetProject); orchestratorCopyJob.setOrchestratorCopyVo(orchestratorCopyVo); orchestratorCopyJob.setOrchestratorCopyEnv(orchestratorCopyEnv); orchestratorCopyJob.getOrchestratorCopyInfo().setId(UUID.randomUUID().toString()); @@ -349,9 +454,9 @@ public Pair> getOrchestratorCopyHistory(Stri } total = orchestratorCopyInfoList.size(); - List < OrchestratorCopyHistory > orchestratorCopyHistoryList = new ArrayList<>(); + List orchestratorCopyHistoryList = new ArrayList<>(); OrchestratorCopyHistory orchestratorCopyHistory; - for (DSSOrchestratorCopyInfo orchestratorCopyInfo: orchestratorCopyInfoList) { + for (DSSOrchestratorCopyInfo orchestratorCopyInfo : orchestratorCopyInfoList) { orchestratorCopyHistory = new OrchestratorCopyHistory(orchestratorCopyInfo.getId(), orchestratorCopyInfo.getUsername(), workspace.getWorkspaceName(), orchestratorCopyInfo.getSourceOrchestratorName(), orchestratorCopyInfo.getTargetOrchestratorName(), orchestratorCopyInfo.getSourceProjectName(), orchestratorCopyInfo.getTargetProjectName(), orchestratorCopyInfo.getWorkflowNodeSuffix(), @@ -382,12 +487,11 @@ protected ImmutablePair getOrchestrationServi dssOrchestratorInfo.setLinkedAppConnNames(dssOrchestrator.getLinkedAppConn().stream().map(appConn -> appConn.getAppDesc().getAppName()).collect(Collectors.toList())); } SchedulerAppConn appConn = dssOrchestrator.getSchedulerAppConn(); - if (appConn != null) { - AppInstance appInstance = appConn.getAppDesc().getAppInstances().get(0); - return new ImmutablePair<>(appConn.getOrCreateStructureStandard().getOrchestrationService(appInstance), appInstance); - } else { - return new ImmutablePair<>(null, null); + if (appConn == null) { + throw new ExternalOperationWarnException(50322, "DSSOrchestrator " + dssOrchestrator.getName() + " has no SchedulerAppConn."); } + AppInstance appInstance = appConn.getAppDesc().getAppInstances().get(0); + return new ImmutablePair<>(appConn.getOrCreateStructureStandard().getOrchestrationService(appInstance), appInstance); } /** @@ -423,4 +527,476 @@ private static boolean hasProjectEditPriv(Long projectId, String username) { return hasEditPriv || projectUserAuthResponse.getProjectOwner().equals(username); } + @Override + public OrchestratorSubmitJob getOrchestratorStatus(Long orchestratorId) { + return orchestratorMapper.selectSubmitJobStatus(orchestratorId); + } + + @Override + public GitHistoryResponse getHistory(Long workspaceId, Long orchestratorId, String projectName) { + DSSOrchestratorInfo orchestrator = orchestratorMapper.getOrchestrator(orchestratorId); + List versionByOrchestratorId = orchestratorMapper.getVersionByOrchestratorId(orchestratorId); + if (CollectionUtils.isEmpty(versionByOrchestratorId)) { + return new GitHistoryResponse(); + } + String workflowName = orchestrator.getName(); + Sender sender = DSSSenderServiceFactory.getOrCreateServiceInstance().getGitSender(); + GitHistoryResponse historyResponse = null; + // 当前未有版本发布 + GitCommitInfoBetweenRequest commitInfoBetweenRequest = new GitCommitInfoBetweenRequest(); + String newCommitId = versionByOrchestratorId.get(0).getCommitId(); + commitInfoBetweenRequest.setNewCommitId(newCommitId); + String oldCommitId = null; + int i = 0; + for (DSSOrchestratorVersion version : versionByOrchestratorId) { + if (version.getCommitId() != null && i++ != 0) { + oldCommitId = version.getCommitId(); + break; + } + } + commitInfoBetweenRequest.setOldCommitId(oldCommitId); + commitInfoBetweenRequest.setProjectName(projectName); + commitInfoBetweenRequest.setDirName(workflowName); + commitInfoBetweenRequest.setWorkspaceId(workspaceId); + historyResponse = RpcAskUtils.processAskException(sender.ask(commitInfoBetweenRequest), GitHistoryResponse.class, GitCommitInfoBetweenRequest.class); + + return historyResponse; + } + + + @Override + public void modifyOrchestratorMeta(String username, ModifyOrchestratorMetaRequest modifyOrchestratorMetaRequest, Workspace workspace, DSSOrchestratorVersion orchestratorVersion) throws Exception { + + //检查desc字段长度 + if (modifyOrchestratorMetaRequest.getDescription().length() > MAX_DESC_LENGTH) { + DSSFrameworkErrorException.dealErrorException(60000, "描述字段过长，请限制在" + MAX_DESC_LENGTH + "以内"); + } + if (modifyOrchestratorMetaRequest.getOrchestratorName().length() > MAX_NAME_LENGTH) { + DSSFrameworkErrorException.dealErrorException(60000, "编排名称过长，请限制在" + MAX_NAME_LENGTH + "以内"); + } + //判断工程是否存在,并且取出工程名称和空间名称 + DSSProject dssProject = validateOperation(modifyOrchestratorMetaRequest.getProjectId(), username); + + if(dssProject.getWorkspaceId() == null || !String.valueOf(dssProject.getWorkspaceId()).equals(String.valueOf(workspace.getWorkspaceId()))){ + LOGGER.error("modifyOrchestratorMeta projectId is {},project workspaceId is {}, workspace params is {}",dssProject.getId(),dssProject.getName(),workspace.getWorkspaceId()); + DSSFrameworkErrorException.dealErrorException(60000, "无法在当前工作空间修改工作流,"+dssProject.getName()+"项目不属于"+workspace.getWorkspaceName()+"工作空间"); + } + + workspace.setWorkspaceName(dssProject.getWorkspaceName()); + + OrchestratorModifyRequest orchestratorModifyRequest = new OrchestratorModifyRequest(); + orchestratorModifyRequest.setOrchestratorName(modifyOrchestratorMetaRequest.getOrchestratorName()); + orchestratorModifyRequest.setProjectId(modifyOrchestratorMetaRequest.getProjectId()); + orchestratorModifyRequest.setId(modifyOrchestratorMetaRequest.getOrchestratorId()); + orchestratorModifyRequest.setWorkspaceId(workspace.getWorkspaceId()); + + orchestratorService.isExistSameNameBeforeUpdate(orchestratorModifyRequest, dssProject, username); + + OrchestratorMeta orchestratorMetaInfo = getOrchestratorMetaInfo(modifyOrchestratorMetaRequest, dssProject, username); + + DSSOrchestratorRelation dssOrchestratorRelation = DSSOrchestratorRelationManager.getDSSOrchestratorRelationByMode(orchestratorMetaInfo.getOrchestratorMode()); + + DSSOrchestratorInfo dssOrchestratorInfo = getDssOrchestratorInfo(orchestratorMetaInfo, dssOrchestratorRelation); + + List dssLabels = Collections.singletonList(new EnvDSSLabel(DSSCommonUtils.ENV_LABEL_VALUE_DEV)); + //1.如果调度系统要求同步创建工作流，向调度系统发送更新工作流的请求 + tryOrchestrationOperation(dssLabels, false, username, dssProject.getName(), workspace, dssOrchestratorInfo, + OrchestrationService::getOrchestrationUpdateOperation, + (structureOperation, structureRequestRef) -> ((OrchestrationUpdateOperation) structureOperation) + .updateOrchestration((OrchestrationUpdateRequestRef) structureRequestRef), "update"); + + updateBmlResource(orchestratorMetaInfo, username, orchestratorVersion,dssProject); + + orchestratorService.updateOrchestrator(username, workspace, dssOrchestratorInfo, dssLabels); + + } + + private static DSSOrchestratorInfo getDssOrchestratorInfo(OrchestratorMeta orchestratorMetaInfo, DSSOrchestratorRelation dssOrchestratorRelation) { + DSSOrchestratorInfo dssOrchestratorInfo = new DSSOrchestratorInfo(); + dssOrchestratorInfo.setId(orchestratorMetaInfo.getOrchestratorId()); + dssOrchestratorInfo.setName(orchestratorMetaInfo.getOrchestratorName()); + dssOrchestratorInfo.setUpdateUser(orchestratorMetaInfo.getUpdateUser()); + dssOrchestratorInfo.setUpdateTime(orchestratorMetaInfo.getUpdateTime()); + dssOrchestratorInfo.setDesc(orchestratorMetaInfo.getDescription()); + dssOrchestratorInfo.setCreator(orchestratorMetaInfo.getCreator()); + dssOrchestratorInfo.setProjectId(orchestratorMetaInfo.getProjectId()); + dssOrchestratorInfo.setComment(orchestratorMetaInfo.getDescription()); + if(StringUtils.isEmpty(orchestratorMetaInfo.getIsDefaultReference())){ + dssOrchestratorInfo.setIsDefaultReference(null); + }else{ + dssOrchestratorInfo.setIsDefaultReference(orchestratorMetaInfo.getIsDefaultReference()); + } + dssOrchestratorInfo.setType(dssOrchestratorRelation.getDSSOrchestratorName()); + dssOrchestratorInfo.setAppConnName(dssOrchestratorRelation.getBindingAppConnName()); + dssOrchestratorInfo.setSecondaryType(orchestratorMetaInfo.getOrchestratorWay()); + dssOrchestratorInfo.setOrchestratorMode(orchestratorMetaInfo.getOrchestratorMode()); + dssOrchestratorInfo.setOrchestratorWay(orchestratorMetaInfo.getOrchestratorWay()); + dssOrchestratorInfo.setOrchestratorLevel(orchestratorMetaInfo.getOrchestratorLevel()); + dssOrchestratorInfo.setUses(orchestratorMetaInfo.getUses()); + return dssOrchestratorInfo; + } + + + @Override + public List getAllOrchestratorMeta(OrchestratorMetaRequest orchestratorMetaRequest, List total, String username) { + List orchestratorMetaList = orchestratorMapper.getAllOrchestratorMeta(orchestratorMetaRequest); + List orchestratorMetaInfo = new ArrayList<>(); + if (CollectionUtils.isEmpty(orchestratorMetaList)) { + total.add(0L); + return orchestratorMetaInfo; + } + + List orchestratorIdList = orchestratorMetaList.stream().map(OrchestratorMeta::getOrchestratorId).collect(Collectors.toList()); + List releaseVersionInfos = orchestratorMapper.getOrchestratorReleaseVersionInfo(orchestratorIdList); + + Map versionMap = new HashMap<>(); + if (!CollectionUtils.isEmpty(releaseVersionInfos)) { + // 根据orchestratorId 分组 + Map> map = releaseVersionInfos.stream() + .collect(Collectors.groupingBy(OrchestratorReleaseVersionInfo::getOrchestratorId)); + // 取编排的第一条记录 + for(Long orchestratorId: map.keySet()){ + + OrchestratorReleaseVersionInfo orchestratorReleaseVersionInfo = map.get(orchestratorId).stream() + .max(Comparator.comparing(OrchestratorReleaseVersionInfo::getReleaseTaskId)).orElse(null); + + if(orchestratorReleaseVersionInfo == null){ + continue; + } + + versionMap.put(orchestratorId,orchestratorReleaseVersionInfo); + } + + } + // 获取模板名称 + List templateInfos = orchestratorMapper.getOrchestratorDefaultTemplateInfo(orchestratorIdList); + Map templateMap = new HashMap<>(); + if (!CollectionUtils.isEmpty(templateInfos)) { + // 分组 拼接模板名称 + templateMap = templateInfos.stream().collect(Collectors.groupingBy(OrchestratorTemplateInfo::getOrchestratorId, + Collectors.mapping(OrchestratorTemplateInfo::getTemplateName, Collectors.joining(",")))); + } + + List orchestratorSubmitJobList = orchestratorMapper.getSubmitJobStatus(orchestratorIdList); + Map submitJobMap = new HashMap<>(); + if (!CollectionUtils.isEmpty(orchestratorSubmitJobList)) { + // 分组排序 获取编排最新的第一条记录信息 + submitJobMap = orchestratorSubmitJobList.stream() + .collect(Collectors.groupingBy(OrchestratorSubmitJob::getOrchestratorId)).values() + .stream() + .flatMap(v -> Stream.of(v.stream().max(Comparator.comparing(OrchestratorSubmitJob::getId)).get())) + .collect(Collectors.toMap(OrchestratorSubmitJob::getOrchestratorId, orchestratorSubmitJob -> orchestratorSubmitJob)); + + } + + for (OrchestratorMeta orchestratorMeta : orchestratorMetaList) { + + OrchestratorReleaseVersionInfo releaseVersion = versionMap.getOrDefault(orchestratorMeta.getOrchestratorId(), + new OrchestratorReleaseVersionInfo()); + + OrchestratorSubmitJob orchestratorSubmitJob = submitJobMap.get(orchestratorMeta.getOrchestratorId()); + + orchestratorMeta.setVersion(releaseVersion.getVersion()); + orchestratorMeta.setUpdateTime(releaseVersion.getReleaseTime()); + orchestratorMeta.setUpdateUser(releaseVersion.getReleaseUser()); + orchestratorStatus(orchestratorMeta,orchestratorSubmitJob,releaseVersion); + + if (templateMap.containsKey(orchestratorMeta.getOrchestratorId())) { + orchestratorMeta.setTemplateName(templateMap.get(orchestratorMeta.getOrchestratorId())); + } + + // 若状态为NULL, 则置为 无状态 + if (StringUtils.isBlank(orchestratorMeta.getStatus())) { + orchestratorMeta.setStatus(OrchestratorStatusEnum.STATELESS.getStatus()); + } + + } + + if (!StringUtils.isBlank(orchestratorMetaRequest.getStatus())) { + // 状态过滤 + orchestratorMetaList = orchestratorMetaList.stream().filter(orchestratorMeta -> + orchestratorMeta.getStatus().equalsIgnoreCase(orchestratorMetaRequest.getStatus())).collect(Collectors.toList()); + } + + total.add((long) orchestratorMetaList.size()); + + // 排序 + try { + if (DSSOrchestratorConstant.ASCEND.equalsIgnoreCase(orchestratorMetaRequest.getOrderBy())) { + orchestratorMetaList = orchestratorMetaList.stream().sorted(Comparator.comparing(OrchestratorMeta::getUpdateTime, Comparator.nullsLast(Comparator.naturalOrder()))).collect(Collectors.toList()); + } else { + orchestratorMetaList = orchestratorMetaList.stream().sorted(Comparator.comparing(OrchestratorMeta::getUpdateTime, Comparator.nullsFirst(Comparator.naturalOrder())).reversed()).collect(Collectors.toList()); + } + } catch (Exception e) { + LOGGER.error("排序失败， 原因为：", e); + } + + // 分页处理 + int page = orchestratorMetaRequest.getPageNow() >= 1 ? orchestratorMetaRequest.getPageNow() : 1; + int pageSize = orchestratorMetaRequest.getPageSize() >= 1 ? orchestratorMetaRequest.getPageSize() : 10; + int start = (page - 1) * pageSize; + int end = Math.min(page * pageSize , orchestratorMetaList.size()); + Map map = new HashMap<>(); + for (int i = start; i < end; i++) { + OrchestratorMeta orchestratorMeta = orchestratorMetaList.get(i); + orchestratorMeta.setStatusName(OrchestratorStatusEnum.getEnum(orchestratorMeta.getStatus()).getName()); + + if (!map.containsKey(orchestratorMeta.getProjectId())) { + + ProjectUserAuthResponse projectUserAuthResponse = RpcAskUtils.processAskException(DSSSenderServiceFactory.getOrCreateServiceInstance() + .getProjectServerSender().ask(new ProjectUserAuthRequest(orchestratorMeta.getProjectId(), username)), + ProjectUserAuthResponse.class, ProjectUserAuthRequest.class); + boolean isEditable = projectUserAuthResponse.getProjectOwner().equals(username); + if (!isEditable && !CollectionUtils.isEmpty(projectUserAuthResponse.getPrivList())) { + isEditable = projectUserAuthResponse.getPrivList().contains(ProjectUserPrivEnum.PRIV_EDIT.getRank()) + || projectUserAuthResponse.getPrivList().contains(ProjectUserPrivEnum.PRIV_RELEASE.getRank()); + } + + map.put(orchestratorMeta.getProjectId(), isEditable); + } + + orchestratorMeta.setEditable(map.get(orchestratorMeta.getProjectId())); + + orchestratorMetaInfo.add(orchestratorMeta); + } + + return orchestratorMetaInfo; + } + + + + public void updateBmlResource(OrchestratorMeta orchestratorMeta, String username, DSSOrchestratorVersion orchestratorVersion, DSSProject dssProject) throws Exception { + // 查询dss工作流信息 + DSSFlow dssFlow = flowMapper.selectFlowByID(orchestratorVersion.getAppId()); + if (StringUtils.isEmpty(dssFlow.getCreator())) { + dssFlow.setCreator(username); + } + String creator = dssFlow.getCreator(); + // 从Bml中获取json信息 + String flowJsonOld = flowService.getFlowJson(creator, orchestratorMeta.getProjectName(), dssFlow); + JsonParser jsonParser = new JsonParser(); + JsonObject jsonObject = jsonParser.parse(flowJsonOld).getAsJsonObject(); + String proxyUser = orchestratorMeta.getProxyUser(); + + if(!jsonObject.keySet().contains("nodes")){ + LOGGER.error("flowJson not contains nodes, json is {}",flowJsonOld); + throw new DSSRuntimeException(6005, orchestratorMeta.getOrchestratorName()+ " 工作流未进行保存,请保存后在重新编辑！！！"); + } + + JsonObject scheduleParams = jsonObject.getAsJsonObject("scheduleParams"); + scheduleParams.addProperty("proxyuser", proxyUser); + + // 更新user.to.proxy用户和proxyuser用户 信息 + JsonArray props = jsonObject.getAsJsonArray("props"); + if(props == null){ + props = new JsonArray(); + jsonObject.add("props", props); + } + // JsonArray 转list，是否包含 user.to.proxy key + List> propList = DSSCommonUtils.COMMON_GSON.fromJson(props, + new TypeToken>>() { + }.getType()); + int size = propList.stream().filter(map -> map.containsKey("user.to.proxy")).collect(Collectors.toList()).size(); + if (size == 0) { + JsonObject element = new JsonObject(); + element.addProperty("user.to.proxy", proxyUser); + props.add(element); + } else { + for (JsonElement prop : props) { + if (prop.getAsJsonObject().keySet().contains("user.to.proxy")) { + prop.getAsJsonObject().addProperty("user.to.proxy", proxyUser); + } + } + } + + String jsonFlow = jsonObject.toString(); + + flowService.saveFlow(orchestratorVersion.getAppId(),jsonFlow + ,dssFlow.getDescription(),username + ,dssProject.getWorkspaceName(),dssProject.getName(),null); + + } + + + public void getGitOrchestratorSubmitStatus(OrchestratorSubmitJob orchestratorSubmitJob,OrchestratorMeta orchestratorMeta){ + + if (OrchestratorRefConstant.FLOW_STATUS_PUSH_FAILED.equalsIgnoreCase(orchestratorSubmitJob.getStatus())) { + // 提交失败 -> 待提交 + orchestratorMeta.setStatus(OrchestratorRefConstant.FLOW_STATUS_SAVE); + orchestratorMeta.setErrorMsg(orchestratorSubmitJob.getErrorMsg()); + + } else if (OrchestratorRefConstant.FLOW_STATUS_PUSHING.equalsIgnoreCase(orchestratorSubmitJob.getStatus())) { + // 提交中 + orchestratorMeta.setStatus(OrchestratorRefConstant.FLOW_STATUS_PUSHING); + } else { + // 待提交 + orchestratorMeta.setStatus(OrchestratorRefConstant.FLOW_STATUS_SAVE); + } + + } + + + public void getGitOrchestratorReleaseStatus(OrchestratorReleaseVersionInfo releaseVersion,OrchestratorMeta orchestratorMeta){ + + // 查询 dss_release_task，根据编排Id获取 + if (OrchestratorRefConstant.FLOW_STATUS_PUSH_FAILED.equalsIgnoreCase(releaseVersion.getStatus())) { + // 发布失败 -> 待发布 + orchestratorMeta.setStatus(OrchestratorRefConstant.FLOW_STATUS_PUSH); + orchestratorMeta.setErrorMsg(releaseVersion.getErrorMsg()); + orchestratorMeta.setNewStatus(OrchestratorStatusEnum.FAILED.getStatus()); + orchestratorMeta.setNewStatusName(OrchestratorStatusEnum.FAILED.getName()); + + } else if (OrchestratorRefConstant.FLOW_STATUS_PUSHING.equalsIgnoreCase(releaseVersion.getStatus())) { + // 发布中状态 + orchestratorMeta.setStatus(OrchestratorRefConstant.FLOW_STATUS_PUBLISHING); + + }else if (OrchestratorRefConstant.FLOW_STATUS_PUBLISH.equalsIgnoreCase(orchestratorMeta.getStatus()) + && OrchestratorRefConstant.FLOW_STATUS_PUSH_SUCCESS.equalsIgnoreCase(releaseVersion.getStatus())) { + + orchestratorMeta.setStatus(OrchestratorStatusEnum.PUBLISH.getStatus()); + orchestratorMeta.setNewStatus(OrchestratorStatusEnum.SUCCESS.getStatus()); + orchestratorMeta.setNewStatusName(OrchestratorStatusEnum.SUCCESS.getName()); + + } else{ + // 待发布 + orchestratorMeta.setStatus(OrchestratorRefConstant.FLOW_STATUS_PUSH); + } + } + + + public OrchestratorMeta getOrchestratorMetaInfo(ModifyOrchestratorMetaRequest modifyOrchestratorMetaRequest, DSSProject dssProject, String username) throws DSSFrameworkErrorException { + + OrchestratorMeta orchestratorInfo = orchestratorMapper.getOrchestratorMeta(modifyOrchestratorMetaRequest.getOrchestratorId()); + + if (orchestratorInfo == null) { + DSSFrameworkErrorException.dealErrorException(60000, "编排模式ID=" + modifyOrchestratorMetaRequest.getOrchestratorId() + "不存在"); + } + + OrchestratorReleaseVersionInfo releaseVersion = orchestratorMapper.getOrchestratorVersionById(orchestratorInfo.getOrchestratorId()); + OrchestratorSubmitJob orchestratorSubmitJob = orchestratorMapper.selectSubmitJobStatus(orchestratorInfo.getOrchestratorId()); + orchestratorStatus(orchestratorInfo,orchestratorSubmitJob,releaseVersion); + + // 发布中和提交中的工作流不允许进行更新 + if (OrchestratorRefConstant.FLOW_STATUS_PUSHING.equalsIgnoreCase(orchestratorInfo.getStatus()) + || OrchestratorRefConstant.FLOW_STATUS_PUBLISHING.equalsIgnoreCase(orchestratorInfo.getStatus())) { + + DSSFrameworkErrorException.dealErrorException(60000, + String.format("%s工作流正在提交中或发布中，不允许进行编辑,请稍后重试", modifyOrchestratorMetaRequest.getOrchestratorName())); + + } + + //若修改了编排名称，检查是否存在相同的编排名称 + if (!modifyOrchestratorMetaRequest.getOrchestratorName().equals(orchestratorInfo.getOrchestratorName())) { + orchestratorService.isExistSameNameBeforeCreate(modifyOrchestratorMetaRequest.getWorkspaceId(), modifyOrchestratorMetaRequest.getProjectId(), modifyOrchestratorMetaRequest.getOrchestratorName()); + + if (dssProject.getAssociateGit() != null && dssProject.getAssociateGit()) { + Sender sender = DSSSenderServiceFactory.getOrCreateServiceInstance().getGitSender(); + GitRenameRequest renameRequest = new GitRenameRequest(modifyOrchestratorMetaRequest.getWorkspaceId(), dssProject.getName(), + modifyOrchestratorMetaRequest.getOrchestratorName(), modifyOrchestratorMetaRequest.getOrchestratorName(), username); + RpcAskUtils.processAskException(sender.ask(renameRequest), GitCommitResponse.class, GitRenameRequest.class); + } + } + + // 修改前端传入的信息, 默认模板调用saveTemplateRef接口进行保存 + orchestratorInfo.setOrchestratorName(modifyOrchestratorMetaRequest.getOrchestratorName()); + orchestratorInfo.setDescription(modifyOrchestratorMetaRequest.getDescription()); + orchestratorInfo.setIsDefaultReference(modifyOrchestratorMetaRequest.getIsDefaultReference()); + if (!Objects.equals(orchestratorInfo.getProxyUser(), modifyOrchestratorMetaRequest.getProxyUser())) { + orchestratorInfo.setProxyUser(modifyOrchestratorMetaRequest.getProxyUser()); + } + orchestratorInfo.setUpdateUser(username); + orchestratorInfo.setUpdateTime(new Date(System.currentTimeMillis())); + + return orchestratorInfo; + + } + + + + public void orchestratorStatus(OrchestratorMeta orchestratorMeta,OrchestratorSubmitJob orchestratorSubmitJob, OrchestratorReleaseVersionInfo releaseVersion){ + + /* + * 对于接入git的项目下，并且当前状态为save的编排，dss_orchestrator_submit_job_info根据编排Id获取最新的提交记录， + * 对于当前状态为push、publish或者为空的，查询 dss_release_task，根据编排Id获取 + * + * 非git项目的工作流只有：发布中、无状态 + * git项目下的工作流才有这四个状态：待提交 待发布 提交中 发布中 + * + * git项目Failed就展示 待发布 待提交 + * 非git项目Failed就无状态 + * + * **/ + + if (orchestratorMeta.getAssociateGit() != null && orchestratorMeta.getAssociateGit()) { + // git项目下的工作流才有这四个状态：待提交 待发布 提交中 发布中 + if(OrchestratorRefConstant.FLOW_STATUS_SAVE.equalsIgnoreCase(orchestratorMeta.getStatus()) && orchestratorSubmitJob!= null){ + + getGitOrchestratorSubmitStatus(orchestratorSubmitJob,orchestratorMeta); + + }else if (StringUtils.isBlank(orchestratorMeta.getStatus()) + || OrchestratorRefConstant.FLOW_STATUS_PUSH.equalsIgnoreCase(orchestratorMeta.getStatus()) + || OrchestratorRefConstant.FLOW_STATUS_PUBLISH.equalsIgnoreCase(orchestratorMeta.getStatus())) { + + getGitOrchestratorReleaseStatus(releaseVersion,orchestratorMeta); + + }else{ + orchestratorMeta.setStatus(OrchestratorRefConstant.FLOW_STATUS_SAVE); + } + + + // 获取最近一次的发布状态 |发布成功 or 发布失败 or 未发布 + if(StringUtils.isEmpty(orchestratorMeta.getNewStatus())){ + + // 未发布 + if(StringUtils.isEmpty(releaseVersion.getStatus())){ + orchestratorMeta.setNewStatus(OrchestratorStatusEnum.UNPUBLISHED.getStatus()); + orchestratorMeta.setNewStatusName(OrchestratorStatusEnum.UNPUBLISHED.getName()); + orchestratorMeta.setVersion(null); + + }else if (OrchestratorRefConstant.FLOW_STATUS_PUSH_FAILED.equalsIgnoreCase(releaseVersion.getStatus())){ + + orchestratorMeta.setNewStatus(OrchestratorStatusEnum.FAILED.getStatus()); + orchestratorMeta.setNewStatusName(OrchestratorStatusEnum.FAILED.getName()); + orchestratorMeta.setErrorMsg(releaseVersion.getErrorMsg()); + + } else if (OrchestratorRefConstant.FLOW_STATUS_PUSH_SUCCESS.equalsIgnoreCase(releaseVersion.getStatus())) { + + orchestratorMeta.setNewStatus(OrchestratorStatusEnum.SUCCESS.getStatus()); + orchestratorMeta.setNewStatusName(OrchestratorStatusEnum.SUCCESS.getName()); + } + + } + + }else { + // 非git项目的工作流只有：发布中、无状态 + if (OrchestratorRefConstant.FLOW_STATUS_PUSHING.equalsIgnoreCase(releaseVersion.getStatus())) { + // 发布中 + orchestratorMeta.setStatus(OrchestratorRefConstant.FLOW_STATUS_PUBLISHING); + + } else if (OrchestratorRefConstant.FLOW_STATUS_PUSH_FAILED.equalsIgnoreCase(releaseVersion.getStatus())) { + // 发布失败 -> 无状态 + orchestratorMeta.setStatus(OrchestratorRefConstant.FLOW_STATUS_STATELESS); + orchestratorMeta.setErrorMsg(releaseVersion.getErrorMsg()); + orchestratorMeta.setNewStatus(OrchestratorStatusEnum.FAILED.getStatus()); + orchestratorMeta.setNewStatusName(OrchestratorStatusEnum.FAILED.getName()); + + } else if (OrchestratorRefConstant.FLOW_STATUS_PUSH_SUCCESS.equalsIgnoreCase(releaseVersion.getStatus())) { + + orchestratorMeta.setStatus(OrchestratorRefConstant.FLOW_STATUS_STATELESS); + orchestratorMeta.setNewStatus(OrchestratorStatusEnum.SUCCESS.getStatus()); + orchestratorMeta.setNewStatusName(OrchestratorStatusEnum.SUCCESS.getName()); + + } else { + // 无状态 + orchestratorMeta.setStatus(OrchestratorRefConstant.FLOW_STATUS_STATELESS); + // 未进行发布的工作流，为未发布状态 + if (StringUtils.isEmpty(releaseVersion.getStatus())) { + orchestratorMeta.setNewStatus(OrchestratorStatusEnum.UNPUBLISHED.getStatus()); + orchestratorMeta.setNewStatusName(OrchestratorStatusEnum.UNPUBLISHED.getName()); + orchestratorMeta.setVersion(null); + } + } + } + } + } diff --git a/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/service/impl/OrchestratorPluginServiceImpl.java b/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/service/impl/OrchestratorPluginServiceImpl.java index 999b70bdd2..a462ec0455 100644 --- a/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/service/impl/OrchestratorPluginServiceImpl.java +++ b/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/service/impl/OrchestratorPluginServiceImpl.java @@ -16,37 +16,62 @@ package com.webank.wedatasphere.dss.orchestrator.server.service.impl; +import cn.hutool.core.util.StrUtil; +import com.google.common.reflect.TypeToken; +import com.google.gson.Gson; +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.webank.wedatasphere.dss.common.constant.project.ProjectUserPrivEnum; +import com.webank.wedatasphere.dss.common.entity.BmlResource; +import com.webank.wedatasphere.dss.common.entity.Resource; import com.webank.wedatasphere.dss.common.entity.project.DSSProject; import com.webank.wedatasphere.dss.common.exception.DSSErrorException; import com.webank.wedatasphere.dss.common.label.DSSLabel; import com.webank.wedatasphere.dss.common.label.DSSLabelUtil; -import com.webank.wedatasphere.dss.common.protocol.JobStatus; +import com.webank.wedatasphere.dss.common.label.EnvDSSLabel; +import com.webank.wedatasphere.dss.common.label.LabelRouteVO; +import com.webank.wedatasphere.dss.common.protocol.*; import com.webank.wedatasphere.dss.common.protocol.project.ProjectInfoRequest; -import com.webank.wedatasphere.dss.common.utils.DSSCommonUtils; -import com.webank.wedatasphere.dss.common.utils.DSSExceptionUtils; -import com.webank.wedatasphere.dss.common.utils.RpcAskUtils; -import com.webank.wedatasphere.dss.orchestrator.common.entity.DSSOrchestratorInfo; -import com.webank.wedatasphere.dss.orchestrator.common.entity.DSSOrchestratorRefOrchestration; -import com.webank.wedatasphere.dss.orchestrator.common.entity.DSSOrchestratorVersion; -import com.webank.wedatasphere.dss.orchestrator.common.entity.OrchestratorInfo; +import com.webank.wedatasphere.dss.common.protocol.project.ProjectUserAuthRequest; +import com.webank.wedatasphere.dss.common.protocol.project.ProjectUserAuthResponse; +import com.webank.wedatasphere.dss.common.service.BMLService; +import com.webank.wedatasphere.dss.common.utils.*; +import com.webank.wedatasphere.dss.framework.workspace.bean.DSSWorkspace; +import com.webank.wedatasphere.dss.framework.workspace.dao.DSSWorkspaceMapper; +import com.webank.wedatasphere.dss.git.common.protocol.GitTree; +import com.webank.wedatasphere.dss.git.common.protocol.request.*; +import com.webank.wedatasphere.dss.git.common.protocol.response.GitCommitResponse; +import com.webank.wedatasphere.dss.git.common.protocol.response.GitDiffResponse; +import com.webank.wedatasphere.dss.git.common.protocol.response.GitFileContentResponse; +import com.webank.wedatasphere.dss.orchestrator.common.entity.*; import com.webank.wedatasphere.dss.orchestrator.common.protocol.RequestFrameworkConvertOrchestration; import com.webank.wedatasphere.dss.orchestrator.common.protocol.ResponseConvertOrchestrator; import com.webank.wedatasphere.dss.orchestrator.common.protocol.ResponseOperateOrchestrator; +import com.webank.wedatasphere.dss.orchestrator.common.ref.OrchestratorRefConstant; import com.webank.wedatasphere.dss.orchestrator.core.DSSOrchestratorContext; import com.webank.wedatasphere.dss.orchestrator.core.plugin.DSSOrchestratorPlugin; import com.webank.wedatasphere.dss.orchestrator.db.dao.OrchestratorJobMapper; import com.webank.wedatasphere.dss.orchestrator.db.dao.OrchestratorMapper; -import com.webank.wedatasphere.dss.orchestrator.common.entity.OrchestratorPublishJob; import com.webank.wedatasphere.dss.orchestrator.publish.ConversionDSSOrchestratorPlugin; import com.webank.wedatasphere.dss.orchestrator.publish.ExportDSSOrchestratorPlugin; import com.webank.wedatasphere.dss.orchestrator.publish.conf.DSSOrchestratorConf; import com.webank.wedatasphere.dss.orchestrator.publish.job.CommonUpdateConvertJobStatus; import com.webank.wedatasphere.dss.orchestrator.publish.job.ConversionJobEntity; import com.webank.wedatasphere.dss.orchestrator.publish.job.OrchestratorConversionJob; +import com.webank.wedatasphere.dss.orchestrator.server.entity.request.BatchPublishFlowCheckRequest; +import com.webank.wedatasphere.dss.orchestrator.server.entity.request.OrchestratorSubmitRequest; +import com.webank.wedatasphere.dss.orchestrator.server.entity.vo.OrchestratorDiffDirVo; +import com.webank.wedatasphere.dss.orchestrator.server.entity.vo.OrchestratorDiffNodeVo; +import com.webank.wedatasphere.dss.orchestrator.server.entity.vo.OrchestratorRelationVo; import com.webank.wedatasphere.dss.orchestrator.server.service.OrchestratorPluginService; +import com.webank.wedatasphere.dss.orchestrator.server.service.OrchestratorService; import com.webank.wedatasphere.dss.sender.service.DSSSenderServiceFactory; import com.webank.wedatasphere.dss.standard.app.sso.Workspace; +import com.webank.wedatasphere.dss.workflow.dao.FlowMapper; +import com.webank.wedatasphere.dss.workflow.dao.LockMapper; import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang3.ObjectUtils; +import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.exception.ExceptionUtils; import org.apache.linkis.common.utils.ByteTimeUtils; import org.apache.linkis.common.utils.Utils; @@ -55,9 +80,16 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import java.io.File; +import java.lang.reflect.Type; +import java.nio.file.Files; +import java.nio.file.Paths; import java.util.*; import java.util.concurrent.ExecutorService; +import java.util.regex.Pattern; +import java.util.stream.Collectors; public class OrchestratorPluginServiceImpl implements OrchestratorPluginService { @@ -76,20 +108,39 @@ public class OrchestratorPluginServiceImpl implements OrchestratorPluginService @Autowired private CommonUpdateConvertJobStatus commonUpdateConvertJobStatus; + @Autowired + private LockMapper lockMapper; + + @Autowired + private FlowMapper flowMapper; + + @Autowired + private OrchestratorService orchestratorService; + + @Autowired + private OrchestratorPluginService orchestratorPluginService; + + @Autowired + private DSSWorkspaceMapper dssWorkspaceMapper; + + @Autowired + @Qualifier("workflowBmlService") + private BMLService bmlService; + private ExecutorService releaseThreadPool = Utils.newCachedThreadPool(50, "Convert-Orchestration-Thread-", true); @Override public ResponseConvertOrchestrator convertOrchestration(RequestFrameworkConvertOrchestration requestConversionOrchestration) { // 如果是先导入，后发布的分两步模式，则走专门的逻辑 - if(DSSOrchestratorConf.DSS_IMPORT_VALID_IMMEDIATELY.getValue() && + if (DSSOrchestratorConf.DSS_IMPORT_VALID_IMMEDIATELY.getValue() && requestConversionOrchestration.isConvertAllOrcs() && - requestConversionOrchestration.getProjectId()!=null){ + requestConversionOrchestration.getProjectId() != null) { ResponseOperateOrchestrator responseOperateOrchestrator = convertWholeProjectOrcs(requestConversionOrchestration); return new ResponseConvertOrchestrator("-1", responseOperateOrchestrator); } LOGGER.info("conversion request is called, request is {}.", DSSCommonUtils.COMMON_GSON.toJson(requestConversionOrchestration)); Long toPublishOrcId; - if(requestConversionOrchestration.getOrcAppId() != null) { + if (requestConversionOrchestration.getOrcAppId() != null) { OrchestratorInfo orchestratorInfo = orchestratorMapper.getOrcInfoByAppId(requestConversionOrchestration.getOrcAppId()); toPublishOrcId = orchestratorInfo.getOrchestratorId(); } else if (CollectionUtils.isNotEmpty(requestConversionOrchestration.getOrcIds())) { @@ -105,9 +156,9 @@ public ResponseConvertOrchestrator convertOrchestration(RequestFrameworkConvertO Map orchestrationIdMap = new HashMap<>(); List publishedOrcIds; List labels = LabelBuilderFactoryContext.getLabelBuilderFactory().getLabels(requestConversionOrchestration.getLabels()); - if(requestConversionOrchestration.isConvertAllOrcs()) { + if (requestConversionOrchestration.isConvertAllOrcs()) { //获取所有的已经发布过的orchestrator - publishedOrcIds = orchestratorMapper.getAllOrcIdsByProjectId(projectId,1); + publishedOrcIds = orchestratorMapper.getAllOrcIdsByProjectId(projectId, 1); if (!publishedOrcIds.contains(toPublishOrcId)) { publishedOrcIds.add(toPublishOrcId); } @@ -121,7 +172,7 @@ public ResponseConvertOrchestrator convertOrchestration(RequestFrameworkConvertO continue; } DSSOrchestratorRefOrchestration dssOrchestratorRefOrchestration = orchestratorMapper.getRefOrchestrationId(orcId); - if(dssOrchestratorRefOrchestration != null) { + if (dssOrchestratorRefOrchestration != null) { orchestrationIdMap.put(dssOrchestratorVersion.getAppId(), dssOrchestratorRefOrchestration.getRefOrchestrationId()); } else { orchestrationIdMap.put(dssOrchestratorVersion.getAppId(), null); @@ -129,27 +180,27 @@ public ResponseConvertOrchestrator convertOrchestration(RequestFrameworkConvertO } } else { publishedOrcIds = new ArrayList<>(); - if(requestConversionOrchestration.getOrcAppId() != null) { + if (requestConversionOrchestration.getOrcAppId() != null) { publishedOrcIds.add(toPublishOrcId); - DSSOrchestratorVersion dssOrchestratorVersion = orchestratorMapper.getLatestOrchestratorVersionByIdAndValidFlag(toPublishOrcId,1); + DSSOrchestratorVersion dssOrchestratorVersion = orchestratorMapper.getLatestOrchestratorVersionByIdAndValidFlag(toPublishOrcId, 1); if (dssOrchestratorVersion != null) { DSSOrchestratorRefOrchestration dssOrchestratorRefOrchestration = orchestratorMapper.getRefOrchestrationId(toPublishOrcId); - if(dssOrchestratorRefOrchestration != null) { + if (dssOrchestratorRefOrchestration != null) { orchestrationIdMap.put(dssOrchestratorVersion.getAppId(), dssOrchestratorRefOrchestration.getRefOrchestrationId()); } else { orchestrationIdMap.put(dssOrchestratorVersion.getAppId(), null); } } } - if(requestConversionOrchestration.getOrcIds() != null && !requestConversionOrchestration.getOrcIds().isEmpty()) { + if (requestConversionOrchestration.getOrcIds() != null && !requestConversionOrchestration.getOrcIds().isEmpty()) { for (Long orcId : requestConversionOrchestration.getOrcIds()) { - DSSOrchestratorVersion dssOrchestratorVersion = orchestratorMapper.getLatestOrchestratorVersionByIdAndValidFlag(orcId,1); + DSSOrchestratorVersion dssOrchestratorVersion = orchestratorMapper.getLatestOrchestratorVersionByIdAndValidFlag(orcId, 1); if (dssOrchestratorVersion == null) { continue; } publishedOrcIds.add(orcId); DSSOrchestratorRefOrchestration dssOrchestratorRefOrchestration = orchestratorMapper.getRefOrchestrationId(orcId); - if(dssOrchestratorRefOrchestration != null) { + if (dssOrchestratorRefOrchestration != null) { orchestrationIdMap.put(dssOrchestratorVersion.getAppId(), dssOrchestratorRefOrchestration.getRefOrchestrationId()); } else { orchestrationIdMap.put(dssOrchestratorVersion.getAppId(), null); @@ -157,7 +208,7 @@ public ResponseConvertOrchestrator convertOrchestration(RequestFrameworkConvertO } } } - if(orchestrationIdMap.isEmpty()) { + if (orchestrationIdMap.isEmpty()) { LOGGER.info("the project {} has no suitable workflow, the publish by user {} is ignored.", projectId, requestConversionOrchestration.getUserName()); return new ResponseConvertOrchestrator("no-necessary-id", ResponseOperateOrchestrator.failed("No suitable workflow(s) found, publish is ignored.")); @@ -196,31 +247,668 @@ public ResponseConvertOrchestrator convertOrchestration(RequestFrameworkConvertO } job.setOrchestratorPublishJob(orchestratorPublishJob); orchestratorJobMapper.insertPublishJob(orchestratorPublishJob); + LOGGER.info("trace workflow publish,flowId:{},orchestratorId:{}. 开始提交convert任务", + requestConversionOrchestration.getOrcAppId(), publishedOrcIds); //submit it releaseThreadPool.submit(job); - LOGGER.info("publish orchestrator success. publishedOrcIds:{} ",publishedOrcIds); + LOGGER.info("publish orchestrator success. publishedOrcIds:{} ", publishedOrcIds); return new ResponseConvertOrchestrator(job.getId(), entity.getResponse()); } + @Override + public void submitFlow(OrchestratorSubmitRequest flowRequest, String username, Workspace workspace) throws DSSErrorException { + Long orchestratorId = flowRequest.getOrchestratorId(); + String status = lockMapper.selectOrchestratorStatus(orchestratorId); + if (!StringUtils.isEmpty(status) && !status.equals(OrchestratorRefConstant.FLOW_STATUS_SAVE)) { + throw new DSSErrorException(800001, "工作流无改动或改动未提交，请确认改动并保存再进行提交"); + } + OrchestratorSubmitJob submitJob = new OrchestratorSubmitJob(); + submitJob.setOrchestratorId(orchestratorId); + submitJob.setInstanceName(Sender.getThisInstance()); + submitJob.setStatus(OrchestratorRefConstant.FLOW_STATUS_PUSHING); + orchestratorMapper.insertOrchestratorSubmitJob(submitJob); + Long taskId = submitJob.getId(); + releaseThreadPool.submit(() -> { + //1. 异步提交，更新提交状态 + try { + submitWorkflowToBML(taskId, flowRequest, username, workspace); + } catch (Exception e) { + LOGGER.error("push failed, the reason is : ", e); + orchestratorMapper.updateOrchestratorSubmitJobStatus(taskId, OrchestratorRefConstant.FLOW_STATUS_PUSH_FAILED, e.getMessage()); + } + }); + } + + @Override + public void batchSubmitFlow(Map> map, Map projectMap, String username, Workspace workspace, String label, String comment) throws DSSErrorException { + if (map == null) { + throw new DSSErrorException(80001, "批量提交的工作流不能为空"); + } + Map taskMap = new HashMap<>(); + List submitRequests = new ArrayList<>(); + for (Map.Entry> entry : map.entrySet()) { + List orchestratorRelationVos = entry.getValue(); + String projectName = entry.getKey(); + Long projectId = projectMap.get(projectName); + + List orchestratorIdList = new ArrayList<>(); + + ProjectUserAuthResponse projectUserAuthResponse = RpcAskUtils.processAskException(DSSSenderServiceFactory.getOrCreateServiceInstance() + .getProjectServerSender().ask(new ProjectUserAuthRequest(projectId, username)), + ProjectUserAuthResponse.class, ProjectUserAuthRequest.class); + boolean isEditable = false; + if (!CollectionUtils.isEmpty(projectUserAuthResponse.getPrivList())) { + isEditable = projectUserAuthResponse.getPrivList().contains(ProjectUserPrivEnum.PRIV_EDIT.getRank()) + || projectUserAuthResponse.getPrivList().contains(ProjectUserPrivEnum.PRIV_RELEASE.getRank()); + } + isEditable = isEditable || projectUserAuthResponse.getProjectOwner().equals(username); + + if (!isEditable) { + DSSExceptionUtils.dealErrorException(63335, "用户" + username + "没有项目" + projectName + "编辑权限，请检查后重新提交", DSSErrorException.class); + } + + for (OrchestratorRelationVo relationVo : orchestratorRelationVos) { + Long orchestratorId = relationVo.getOrchestratorId(); + orchestratorIdList.add(orchestratorId); + String status = lockMapper.selectOrchestratorStatus(orchestratorId); + if (StringUtils.isEmpty(status)) { + throw new DSSErrorException(800001, projectName + "项目未接入git，批量提交失败"); + } + DSSOrchestratorInfo orchestrator = orchestratorMapper.getOrchestrator(orchestratorId); + if (!StringUtils.isEmpty(status) && !status.equals(OrchestratorRefConstant.FLOW_STATUS_SAVE)) { + throw new DSSErrorException(800001, orchestrator.getName() + "工作流无改动或改动未提交，请确认改动并保存再进行提交"); + } + LabelRouteVO labelRouteVO = new LabelRouteVO(); + labelRouteVO.setRoute(label); + OrchestratorSubmitRequest orchestratorSubmitRequest = new OrchestratorSubmitRequest(relationVo.getFlowId(), labelRouteVO, projectName, comment, orchestratorId, orchestrator.getName(), false); + submitRequests.add(orchestratorSubmitRequest); + } + // 批量提交更新状态 + for (OrchestratorRelationVo relationVo : orchestratorRelationVos) { + Long orchestratorId = relationVo.getOrchestratorId(); + + OrchestratorSubmitJob submitJob = new OrchestratorSubmitJob(); + submitJob.setOrchestratorId(orchestratorId); + submitJob.setInstanceName(Sender.getThisInstance()); + submitJob.setStatus(OrchestratorRefConstant.FLOW_STATUS_PUSHING); + orchestratorMapper.insertOrchestratorSubmitJob(submitJob); + Long taskId = submitJob.getId(); + taskMap.put(orchestratorId, taskId); + } + } + + releaseThreadPool.submit(() -> { + // 串行提交 + for (OrchestratorSubmitRequest submitRequest : submitRequests) { + Long taskId = taskMap.get(submitRequest.getOrchestratorId()); + try { + submitWorkflowToBML(taskId, submitRequest, username, workspace); + } catch (Exception e) { + LOGGER.error("push failed, the reason is : ", e); + orchestratorMapper.updateOrchestratorSubmitJobStatus(taskId, OrchestratorRefConstant.FLOW_STATUS_PUSH_FAILED, e.getMessage()); + } + } + }); + } + + @Override + public GitCommitResponse submitWorkflowSync(OrchestratorSubmitRequest flowRequest, String username, Workspace workspace) throws DSSErrorException { + Long orchestratorId = flowRequest.getOrchestratorId(); + OrchestratorSubmitJob submitJob = new OrchestratorSubmitJob(); + submitJob.setOrchestratorId(orchestratorId); + submitJob.setInstanceName(Sender.getThisInstance()); + submitJob.setStatus(OrchestratorRefConstant.FLOW_STATUS_PUSHING); + orchestratorMapper.insertOrchestratorSubmitJob(submitJob); + Long taskId = submitJob.getId(); + try { + GitCommitResponse gitCommitResponse = submitWorkflowToBML(taskId, flowRequest, username, workspace); + return gitCommitResponse; + } catch (Exception e) { + LOGGER.error("push failed, the reason is : ", e); + orchestratorMapper.updateOrchestratorSubmitJobStatus(taskId, OrchestratorRefConstant.FLOW_STATUS_PUSH_FAILED, e.toString()); + throw new DSSErrorException(80001, "push failed, the reason is : " + e.toString()); + } + } + + + private GitCommitResponse submitWorkflowToBML(Long taskId, OrchestratorSubmitRequest flowRequest, String username, Workspace workspace) { + LOGGER.info("====begin to submit flow {}====", flowRequest.getFlowId()); + Long orchestratorId = flowRequest.getOrchestratorId(); + Long flowId = flowRequest.getFlowId(); + String projectName = flowRequest.getProjectName(); + String label = flowRequest.getLabels().getRoute(); + //2. 获取编排信息 + DSSOrchestratorInfo orchestrator = orchestratorMapper.getOrchestrator(orchestratorId); + //3. 获取上传工作流信息 + LOGGER.info("=====begin to submit flow {} to BML====", flowRequest.getFlowId()); + BmlResource bmlResource = orchestratorMapper.getOrchestratorBmlVersion(orchestratorId); + if (bmlResource == null || bmlResource.getResourceId() == null || bmlResource.getVersion() == null) { + bmlResource = uploadWorkflowToGit(flowId, projectName, label, username, workspace, orchestrator); + orchestratorMapper.updateOrchestratorBmlVersion(orchestratorId, bmlResource.getResourceId(), bmlResource.getVersion()); + } + LOGGER.info("====finish flowTOBML, begin to submit flow {} to Git====", flowRequest.getFlowId()); + //4. 调用git服务上传 + GitCommitResponse commit = push(orchestrator.getName(), bmlResource, username, workspace.getWorkspaceId(), projectName, flowRequest.getComment()); + if (commit == null) { + LOGGER.info("change is empty"); + } + LOGGER.info("====finish flowTOGIT {}====", flowRequest.getFlowId()); + orchestratorMapper.updateOrchestratorSubmitJobStatus(taskId, OrchestratorRefConstant.FLOW_STATUS_PUSH_SUCCESS, ""); + //5. 返回文件列表 + lockMapper.updateOrchestratorStatus(orchestratorId, OrchestratorRefConstant.FLOW_STATUS_PUSH); + // 更新commitId + lockMapper.updateOrchestratorVersionCommitId(commit.getCommitId(), flowId); + + updateOrchestratorNotContainsKeywordsNode(flowRequest, orchestrator, username, workspace,bmlResource); + + return commit; + } + + @Override + public OrchestratorDiffNodeVo getNotContainsKeywordsNode(long orchestratorId, long projectId, Workspace workspace) throws DSSErrorException { + + OrchestratorDiffNodeVo orchestratorDiffNodeVo = new OrchestratorDiffNodeVo(); + DSSWorkspace dssWorkspace = dssWorkspaceMapper.getWorkspace((int) workspace.getWorkspaceId()); + if (disabledKeywordsCheck(dssWorkspace)) { + LOGGER.info("{} , {} 工作空间未启用工作流关键字校验",dssWorkspace.getId(),dssWorkspace.getName()); + return orchestratorDiffNodeVo; + } + + DSSOrchestratorInfo orchestratorInfo = orchestratorMapper.getOrchestratorNotContainsKeywordsNode(orchestratorId); + // 项目未接入git + if (orchestratorInfo.getAssociateGit() == null || !orchestratorInfo.getAssociateGit()) { + LOGGER.info("{} 项目未接入git,project id is {}",orchestratorInfo.getProjectName(),orchestratorInfo.getProjectId()); + return orchestratorDiffNodeVo; + } + + return getOrchestratorDiffNodeVo(orchestratorInfo,dssWorkspace); + + } + + @Override + public List batchPublishFlowCheck(BatchPublishFlowCheckRequest request, Workspace workspace) throws DSSErrorException { + + List orchestratorDiffNodeVoList = new ArrayList<>(); + DSSWorkspace dssWorkspace = dssWorkspaceMapper.getWorkspace((int) workspace.getWorkspaceId()); + if (disabledKeywordsCheck(dssWorkspace) || CollectionUtils.isEmpty(request.getOrchestratorIdList())) { + return orchestratorDiffNodeVoList; + } + + List orchestratorInfoList = orchestratorMapper.getOrchestratorListById(request.getOrchestratorIdList()); + + if(CollectionUtils.isNotEmpty(orchestratorInfoList)){ + DSSOrchestratorInfo dssOrchestratorInfo = orchestratorInfoList.get(0); + if(!String.valueOf(dssOrchestratorInfo.getWorkspaceId()).equals(String.valueOf(workspace.getWorkspaceId()))){ + LOGGER.error("cookie workspaceId is {}, orchestrator workspaceId,id,name is {},{},{} " + ,workspace.getWorkspaceId(),dssOrchestratorInfo.getWorkspaceId(),dssOrchestratorInfo.getId(),dssOrchestratorInfo.getName()); + throw new DSSErrorException(90058,"工作流对应的工作空间与cookie中不一致，请刷新页面后重试"); + } + + for(DSSOrchestratorInfo orchestratorInfo: orchestratorInfoList){ + + orchestratorDiffNodeVoList.add(getOrchestratorDiffNodeVo(orchestratorInfo,dssWorkspace)); + } + + } + + return orchestratorDiffNodeVoList; + } + + + private OrchestratorDiffNodeVo getOrchestratorDiffNodeVo(DSSOrchestratorInfo orchestratorInfo,DSSWorkspace dssWorkspace){ + + + OrchestratorDiffNodeVo orchestratorDiffNodeVo = new OrchestratorDiffNodeVo(); + // 项目已接入git,notContainsKeywordsNode不为空 + if (Boolean.TRUE.equals(orchestratorInfo.getAssociateGit()) && StringUtils.isNotEmpty(orchestratorInfo.getNotContainsKeywordsNode())) { + orchestratorDiffNodeVo.setNotContainsKeywordsNodeList(Arrays.asList(orchestratorInfo.getNotContainsKeywordsNode().split(","))); + } + orchestratorDiffNodeVo.setProjectName(orchestratorInfo.getProjectName()); + orchestratorDiffNodeVo.setOrchestratorId(orchestratorInfo.getId()); + orchestratorDiffNodeVo.setProjectId(orchestratorInfo.getProjectId()); + orchestratorDiffNodeVo.setOrchestratorName(orchestratorInfo.getName()); + orchestratorDiffNodeVo.setEnabledFlowKeywordsCheck(dssWorkspace.getEnabledFlowKeywordsCheck()); + orchestratorDiffNodeVo.setWorkspaceId((long) dssWorkspace.getId()); + orchestratorDiffNodeVo.setWorkspaceName(dssWorkspace.getName()); + orchestratorDiffNodeVo.setAssociateGit(orchestratorInfo.getAssociateGit()); + return orchestratorDiffNodeVo; + + } + + private boolean disabledKeywordsCheck(DSSWorkspace dssWorkspace) { + + // 命名空间 禁用关键字检查 + return dssWorkspace == null || StringUtils.isEmpty(dssWorkspace.getEnabledFlowKeywordsCheck()) + || !dssWorkspace.getEnabledFlowKeywordsCheck().equals("1"); + } + + + private void updateOrchestratorNotContainsKeywordsNode(OrchestratorSubmitRequest flowRequest, DSSOrchestratorInfo orchestrator, + String username, Workspace workspace,BmlResource bmlResource) { + + DSSWorkspace dssWorkspace = dssWorkspaceMapper.getWorkspace((int) workspace.getWorkspaceId()); + + if (disabledKeywordsCheck(dssWorkspace)) { + return; + } + + List resourceFileName = new ArrayList<>(); + + try { + // 空工作流会获取json失败 + String flowJson = bmlService.query(username,bmlResource.getResourceId(),bmlResource.getVersion()).get("string").toString(); + List> resources = DSSCommonUtils.getFlowAttribute(flowJson, "resources"); + + // 取资源文件名称 + if(!CollectionUtils.isEmpty(resources)){ + resourceFileName = resources.stream().map(resource -> String.valueOf(resource.get("fileName"))).collect(Collectors.toList()); + } + }catch (Exception e){ + LOGGER.error("OrchestratorNotContainsKeywordsNode get resource fail",e); + } + + String commitId = getLastPublishCommitId(orchestrator); + + GitDiffResponse gitDiffResponse = diffPublish(orchestrator.getName(), commitId, username, + workspace.getWorkspaceId(), flowRequest.getProjectName()); + + if (gitDiffResponse == null || CollectionUtils.isEmpty(gitDiffResponse.getCodeTree())) { + LOGGER.info("gitDiffResponse change is empty"); + return; + } + + List paths = new ArrayList<>(); + + Map codeTreeMap = gitDiffResponse.getCodeTree().get(0).getChildren(); + + for(String name: codeTreeMap.keySet()){ + + // 判断是否资源文件 + if(resourceFileName.contains(name)){ + continue; + } + + getCodePath(paths,codeTreeMap.get(name)); + } + + // 获取内容并检查 + List nodePathList = getNotContainsKeywordsNodePath(paths, commitId, workspace, flowRequest.getProjectName(), username); + + String notContainsKeywordsNode = StrUtil.join(",",nodePathList); + + orchestratorMapper.updateOrchestratorNotContainsKeywordsNode(flowRequest.getOrchestratorId(), notContainsKeywordsNode); + } + + + private void getCodePath(List paths, GitTree gitTree) { + + if (MapUtils.isEmpty(gitTree.getChildren())) { + return; + } + + for (Map.Entry entry : gitTree.getChildren().entrySet()) { + + // 过滤后缀 + if (StringUtils.endsWithAny(entry.getValue().getAbsolutePath(), ".sql", ".hql", ".py")) { + paths.add(entry.getValue().getAbsolutePath()); + } + + getCodePath(paths, entry.getValue()); + } + + } + + + private List getNotContainsKeywordsNodePath(List codePaths, String commitId, Workspace workspace, String projectName, String username) { + + List nodePathList = new ArrayList<>(); + + if (CollectionUtils.isEmpty(codePaths)) { + return nodePathList; + } + + codePaths = codePaths.stream().distinct().collect(Collectors.toList()); + + GitDiffFileContentRequest diffFileContentRequest = new GitDiffFileContentRequest(); + diffFileContentRequest.setFilePaths(codePaths); + diffFileContentRequest.setPublish(true); + diffFileContentRequest.setCommitId(commitId); + diffFileContentRequest.setUsername(username); + diffFileContentRequest.setWorkspaceId(workspace.getWorkspaceId()); + diffFileContentRequest.setProjectName(projectName); + Sender gitSender = DSSSenderServiceFactory.getOrCreateServiceInstance().getGitSender(); + // 批量获取脚本内容新 + GitDiffFileContentResponse gitDiffFileContentResponse = RpcAskUtils.processAskException( + gitSender.ask(diffFileContentRequest), GitDiffFileContentResponse.class, GitDiffFileContentRequest.class); + + if (gitDiffFileContentResponse == null) { + LOGGER.info("get script content isEmpty"); + return null; + } + + Pattern pattern = Pattern.compile("\\b(create\\s+table|insert)\\b",Pattern.CASE_INSENSITIVE); + + for (GitFileContentResponse gitFileContentResponse : gitDiffFileContentResponse.getGitFileContentResponseList()) { + + String content = gitFileContentResponse.getAfter(); + + // 内容为NULL或者已匹配到关键字 跳过 + if (content == null || pattern.matcher(content).find()) { + continue; + } + + // 去掉/xxx.sql,hql,py 脚本后缀 + String nodePath = gitFileContentResponse.getFilePath().substring(0, gitFileContentResponse.getFilePath().lastIndexOf("/")); + nodePathList.add(nodePath); + } + + return nodePathList; + + } + + + public GitCommitResponse batchSubmitWorkflowToBML(List taskIdList, List relationVos, String username, Workspace workspace, String projectName, String label, Long projectId, String comment) throws Exception { + + List flowIdList = relationVos.stream().map(OrchestratorRelationVo::getFlowId).collect(Collectors.toList()); + BmlResource bmlResource = orchestratorPluginService.uploadWorkflowListToGit(flowIdList, projectName, label, username, workspace, projectId); + List orchestratorIdList = relationVos.stream().map(OrchestratorRelationVo::getOrchestratorId).collect(Collectors.toList()); + List orchestratorName = orchestratorMapper.getOrchestratorName(orchestratorIdList); + //4. 调用git服务上传 + GitCommitResponse commit = batchPush(orchestratorName, bmlResource, username, workspace.getWorkspaceId(), projectName, comment); + if (commit == null) { + LOGGER.info("change is empty"); + } + orchestratorMapper.batchUpdateOrchestratorSubmitJobStatus(taskIdList, OrchestratorRefConstant.FLOW_STATUS_PUSH_SUCCESS, ""); + //5. 返回文件列表 + lockMapper.batchUpdateOrchestratorStatus(orchestratorIdList, OrchestratorRefConstant.FLOW_STATUS_PUSH); + // 更新commitId + lockMapper.batchUpdateOrchestratorVersionCommitId(commit.getCommitId(), flowIdList); + return commit; + } + + private GitCommitResponse push(String path, BmlResource bmlResource, String username, Long workspaceId, String projectName, String comment) { + LOGGER.info("-------=======================begin to add testGit1=======================-------"); + Sender gitSender = DSSSenderServiceFactory.getOrCreateServiceInstance().getGitSender(); + Map file = new HashMap<>(); + file.put(path, bmlResource); + GitCommitRequest request1 = new GitCommitRequest(workspaceId, projectName, file, comment, username); + GitCommitResponse responseWorkflowValidNode = RpcAskUtils.processAskException(gitSender.ask(request1), GitCommitResponse.class, GitCommitRequest.class); + LOGGER.info("-------=======================End to add testGit1=======================-------: {}", responseWorkflowValidNode); + return responseWorkflowValidNode; + } + + private GitCommitResponse batchPush(List path, BmlResource bmlResource, String username, Long workspaceId, String projectName, String comment) { + LOGGER.info("-------=======================begin to add testGit1=======================-------"); + Sender gitSender = DSSSenderServiceFactory.getOrCreateServiceInstance().getGitSender(); + GitBatchCommitRequest request1 = new GitBatchCommitRequest(workspaceId, projectName, comment, username, path, bmlResource); + GitCommitResponse responseWorkflowValidNode = RpcAskUtils.processAskException(gitSender.ask(request1), GitCommitResponse.class, GitCommitRequest.class); + LOGGER.info("-------=======================End to add testGit1=======================-------: {}", responseWorkflowValidNode); + return responseWorkflowValidNode; + } + + private BmlResource uploadWorkflowToGit(Long flowId, String projectName, String label, String username, Workspace workspace, DSSOrchestratorInfo orchestrator) { + // 1. 将序列化好的工作流文件包提交给git服务，并拿到diff文件列表结果, + + Long projectId = orchestrator.getProjectId(); + List dssLabelList = new ArrayList<>(); + dssLabelList.add(new EnvDSSLabel(label)); + RequestExportWorkflow requestExportWorkflow = new RequestExportWorkflow(username, + flowId, + projectId, + projectName, + DSSCommonUtils.COMMON_GSON.toJson(workspace), + dssLabelList, + false); + Sender sender = DSSSenderServiceFactory.getOrCreateServiceInstance().getWorkflowSender(dssLabelList); + ResponseExportWorkflow responseExportWorkflow = RpcAskUtils.processAskException(sender.ask(requestExportWorkflow), + ResponseExportWorkflow.class, RequestExportWorkflow.class); + + BmlResource bmlResource = new BmlResource(responseExportWorkflow.resourceId(), responseExportWorkflow.version()); + + return bmlResource; + } + + @Override + public String diffStatus(Long taskId) throws DSSErrorException { + OrchestratorSubmitJob orchestratorSubmitJob = orchestratorMapper.selectSubmitJobStatusById(taskId); + if (orchestratorSubmitJob != null) { + String status = orchestratorSubmitJob.getStatus(); + if (OrchestratorRefConstant.FLOW_STATUS_PUSH_FAILED.equals(status)) { + throw new DSSErrorException(80001, "获取工作流diff内容失败，原因为：" + orchestratorSubmitJob.getErrorMsg()); + } + return status; + } + return null; + } + + @Override + public OrchestratorDiffDirVo diffContent(Long taskId) { + String result = orchestratorMapper.selectResult(taskId); + if (StringUtils.isNotEmpty(result)) { + Type listType = new TypeToken() { + }.getType(); + Gson gson = new Gson(); + OrchestratorDiffDirVo gitTrees = gson.fromJson(result, listType); + return gitTrees; + } + return null; + } + + @Override + public BmlResource uploadWorkflowListToGit(List flowIdList, String projectName, String label, String username, Workspace workspace, Long projectId) throws Exception { + // 1. 将序列化好的工作流文件包提交给git服务，并拿到diff文件列表结果, + List dssLabelList = new ArrayList<>(); + dssLabelList.add(new EnvDSSLabel(label)); + RequestExportWorkflowList requestExportWorkflow = new RequestExportWorkflowList(username, + flowIdList, + projectId, + projectName, + DSSCommonUtils.COMMON_GSON.toJson(workspace), + dssLabelList, + false); + Sender sender = DSSSenderServiceFactory.getOrCreateServiceInstance().getWorkflowSender(dssLabelList); + ResponseExportWorkflowList responseExportWorkflow = RpcAskUtils.processAskException(sender.ask(requestExportWorkflow), + ResponseExportWorkflowList.class, RequestExportWorkflowList.class); + Map resourceMap = new HashMap<>(2); + BmlResource bmlResource = new BmlResource(responseExportWorkflow.resourceId(), responseExportWorkflow.version()); + + return bmlResource; + } + + private String readWorkflowNode(BmlResource bmlResource, String username, String projectName, String orchestratorName, String filePath) throws DSSErrorException { + // 1. 将序列化好的工作流文件包提交给git服务，并拿到diff文件列表结果, + String projectPath = IoUtils.generateProjectIOPath(username, projectName); + String fullPath = projectPath + File.separator + projectName + File.separator + filePath; + String zipFilePath = projectPath + File.separator + orchestratorName + ".zip"; + String fileContent = null; + try { + bmlService.downloadToLocalPath(username, bmlResource.getResourceId(), bmlResource.getVersion(), zipFilePath); + ZipHelper.unzipFile(zipFilePath, projectPath, true); + + LOGGER.info("export workflow success. orchestratorName:{},fullPath:{} .", filePath, projectPath); + + File file = new File(fullPath); + if (file.exists()) { + fileContent = new String(Files.readAllBytes(Paths.get(fullPath))); + } + } catch (Exception e) { + LOGGER.error("exportFlowInfoNew failed , the reason is:", e); + throw new DSSErrorException(100098, "工作流导出失败，原因为" + e.getMessage()); + } finally { + //删掉整个目录 + if (StringUtils.isNotEmpty(projectPath)) { + File file = new File(projectPath); + if (ZipHelper.deleteDir(file)) { + LOGGER.info("结束删除目录 {} 成功", projectPath); + } else { + LOGGER.info("删除目录 {} 失败", projectPath); + } + } + } + return fileContent; + } + + @Override + public Long diffFlow(OrchestratorSubmitRequest flowRequest, String username, Workspace workspace) throws Exception { + Long orchestratorId = flowRequest.getOrchestratorId(); + String status = lockMapper.selectOrchestratorStatus(orchestratorId); + if (!StringUtils.isEmpty(status) && !status.equals(OrchestratorRefConstant.FLOW_STATUS_SAVE)) { + throw new DSSErrorException(800001, "工作流无改动，请确认改动并保存再进行提交"); + } + DSSOrchestratorInfo orchestrator = orchestratorMapper.getOrchestrator(orchestratorId); + Long flowId = flowRequest.getFlowId(); + String projectName = flowRequest.getProjectName(); + String label = flowRequest.getLabels().getRoute(); + + BmlResource bmlResource = orchestratorMapper.getOrchestratorBmlVersion(orchestratorId); + Long taskId; + if (bmlResource == null || bmlResource.getResourceId() == null || bmlResource.getVersion() == null) { + OrchestratorSubmitJob diffJob = new OrchestratorSubmitJob(); + diffJob.setOrchestratorId(orchestratorId); + diffJob.setInstanceName(Sender.getThisInstance()); + diffJob.setStatus(OrchestratorRefConstant.FLOW_STATUS_PUSHING); + orchestratorMapper.insertOrchestratorSubmitJob(diffJob); + taskId = diffJob.getId(); + releaseThreadPool.submit(() -> { + try { + BmlResource uploadBmlResource = uploadWorkflowToGit(flowId, projectName, label, username, workspace, orchestrator); + orchestratorMapper.updateOrchestratorBmlVersion(orchestratorId, uploadBmlResource.getResourceId(), uploadBmlResource.getVersion()); + GitDiffResponse diff = diff(orchestrator.getName(), uploadBmlResource, username, workspace.getWorkspaceId(), flowRequest.getProjectName()); + String result = ""; + if (diff != null) { + List codeTree = diff.getCodeTree(); + List metaTree = diff.getMetaTree(); + OrchestratorDiffDirVo diffDirVo = new OrchestratorDiffDirVo(codeTree, metaTree); + Gson gson = new Gson(); + result = gson.toJson(diffDirVo); + } else { + LOGGER.info("change is empty"); + } + orchestratorMapper.updateOrchestratorSubmitResult(taskId, OrchestratorRefConstant.FLOW_STATUS_PUSH_SUCCESS, result); + } catch (Exception e) { + LOGGER.error("push failed, the reason is : ", e); + orchestratorMapper.updateOrchestratorBmlVersion(orchestratorId, null, null); + orchestratorMapper.updateOrchestratorSubmitJobStatus(taskId, OrchestratorRefConstant.FLOW_STATUS_PUSH_FAILED, e.toString()); + } + }); + } else { + taskId = orchestratorMapper.getLatestOrchestratorSubmitJobId(orchestratorId); + } + + return taskId; + } + + @Override + public OrchestratorDiffDirVo diffPublish(OrchestratorSubmitRequest flowRequest, String username, Workspace workspace) { + Long orchestratorId = flowRequest.getOrchestratorId(); + DSSOrchestratorInfo orchestrator = orchestratorMapper.getOrchestrator(orchestratorId); + String commitId = getLastPublishCommitId(orchestrator); + + + GitDiffResponse diff = diffPublish(orchestrator.getName(), commitId, username, workspace.getWorkspaceId(), flowRequest.getProjectName()); + if (diff == null) { + LOGGER.info("change is empty"); + return null; + } + List codeTree = diff.getCodeTree(); + List metaTree = diff.getMetaTree(); + + //4. 返回文件列表 + return new OrchestratorDiffDirVo(codeTree, metaTree); + } + + private String getLastPublishCommitId(DSSOrchestratorInfo orchestrator) { + List orchestratorVersions = orchestratorMapper.getVersionByOrchestratorId(orchestrator.getId()); + String commitId = null; + if (CollectionUtils.isNotEmpty(orchestratorVersions)) { + String commitIdNow = orchestratorVersions.get(0).getCommitId(); + if (commitIdNow != null) { + int i = 0; + for (DSSOrchestratorVersion version : orchestratorVersions) { + if (StringUtils.isNotEmpty(version.getCommitId()) && i++ != 0) { + commitId = version.getCommitId(); + break; + } + } + } + } + return commitId; + } + + @Override + public GitFileContentResponse diffFlowContent(OrchestratorSubmitRequest flowRequest, String username, Workspace workspace) throws DSSErrorException { + Long orchestratorId = flowRequest.getOrchestratorId(); + Long flowId = flowRequest.getFlowId(); + String projectName = flowRequest.getProjectName(); + String label = flowRequest.getLabels().getRoute(); + String filePath = flowRequest.getFilePath(); + DSSOrchestratorInfo orchestrator = orchestratorMapper.getOrchestrator(orchestratorId); + BmlResource bmlResource = orchestratorMapper.getOrchestratorBmlVersion(orchestratorId); + if (bmlResource == null || bmlResource.getVersion() == null || bmlResource.getResourceId() == null) { + bmlResource = uploadWorkflowToGit(flowId, projectName, label, username, workspace, orchestrator); + orchestratorMapper.updateOrchestratorBmlVersion(orchestratorId, bmlResource.getResourceId(), bmlResource.getVersion()); + } + String s = readWorkflowNode(bmlResource, username, projectName, orchestrator.getName(), filePath); + + + GitFileContentRequest fileContentRequest = new GitFileContentRequest(); + fileContentRequest.setFilePath(filePath); + fileContentRequest.setProjectName(projectName); + fileContentRequest.setWorkspaceId(workspace.getWorkspaceId()); + fileContentRequest.setUsername(username); + fileContentRequest.setPublish(flowRequest.getPublish()); + + if (flowRequest.getPublish()) { + String commitId = getLastPublishCommitId(orchestrator); + fileContentRequest.setCommitId(commitId); + } + + Sender gitSender = DSSSenderServiceFactory.getOrCreateServiceInstance().getGitSender(); + GitFileContentResponse contentResponse = RpcAskUtils.processAskException(gitSender.ask(fileContentRequest), GitFileContentResponse.class, GitFileContentRequest.class); + if (flowRequest.getPublish()) { + return contentResponse; + } + contentResponse.setAfter(s); + return contentResponse; + } + + private GitDiffResponse diff(String path, BmlResource bmlResource, String username, Long workspaceId, String projectName) { + Sender gitSender = DSSSenderServiceFactory.getOrCreateServiceInstance().getGitSender(); + Map file = new HashMap<>(); + file.put(path, bmlResource); + GitDiffRequest request1 = new GitDiffRequest(workspaceId, projectName, file, username); + LOGGER.info("-------=======================begin to diff {}=======================-------", request1.getProjectName()); + GitDiffResponse responseWorkflowValidNode = RpcAskUtils.processAskException(gitSender.ask(request1), GitDiffResponse.class, GitDiffRequest.class); + LOGGER.info("-------=======================End to diff testGit1=======================-------: {}", responseWorkflowValidNode); + return responseWorkflowValidNode; + } + + private GitDiffResponse diffPublish(String path, String commitId, String username, Long workspaceId, String projectName) { + Sender gitSender = DSSSenderServiceFactory.getOrCreateServiceInstance().getGitSender(); + GitDiffTargetCommitRequest targetCommitRequest = new GitDiffTargetCommitRequest(workspaceId, projectName, commitId, username, path); + GitDiffResponse responseWorkflowValidNode = RpcAskUtils.processAskException(gitSender.ask(targetCommitRequest), GitDiffResponse.class, GitDiffTargetCommitRequest.class); + return responseWorkflowValidNode; + } + /** * 发布整个项目中所有未发布过的工作流。 * 使用在先导入，后发布两步走的场景中 */ - private ResponseOperateOrchestrator convertWholeProjectOrcs(RequestFrameworkConvertOrchestration requestConversionOrchestration){ - Long projectId=requestConversionOrchestration.getProjectId(); + private ResponseOperateOrchestrator convertWholeProjectOrcs(RequestFrameworkConvertOrchestration requestConversionOrchestration) { + Long projectId = requestConversionOrchestration.getProjectId(); List labels = LabelBuilderFactoryContext.getLabelBuilderFactory().getLabels(requestConversionOrchestration.getLabels()); //这里的 key 为 DSS 具体编排（如 DSS 工作流）的 id；这里的 value 为 DSS 编排所对应的第三方调度系统的工作流 ID //请注意：由于对接的 SchedulerAppConn 调度系统有可能没有实现 OrchestrationService， //所以可能存在 DSS 在创建 DSS 编排时，无法同步去 SchedulerAppConn 创建工作流的情况，从而导致这个 Map 的所有 value 都为 null。 Map orchestrationIdMap = new HashMap<>(); - List orcIds=new ArrayList<>(); + List orcIds = new ArrayList<>(); //所有之前发布过的编排 - List publishedOrcIds = orchestratorMapper.getAllOrcIdsByProjectId(projectId,1); + List publishedOrcIds = orchestratorMapper.getAllOrcIdsByProjectId(projectId, 1); orcIds.addAll(publishedOrcIds); //所有之前未发布过的编排 - List notPublished= orchestratorMapper.getAllOrcIdsByProjectId(projectId,0); + List notPublished = orchestratorMapper.getAllOrcIdsByProjectId(projectId, 0); orcIds.addAll(notPublished); for (Long publishedOrcId : orcIds) { DSSOrchestratorVersion dssOrchestratorVersion = orchestratorMapper.getLatestOrchestratorVersionByIdAndValidFlag(publishedOrcId, 1); @@ -228,13 +916,13 @@ private ResponseOperateOrchestrator convertWholeProjectOrcs(RequestFrameworkConv continue; } DSSOrchestratorRefOrchestration dssOrchestratorRefOrchestration = orchestratorMapper.getRefOrchestrationId(publishedOrcId); - if(dssOrchestratorRefOrchestration != null) { + if (dssOrchestratorRefOrchestration != null) { orchestrationIdMap.put(dssOrchestratorVersion.getAppId(), dssOrchestratorRefOrchestration.getRefOrchestrationId()); } else { orchestrationIdMap.put(dssOrchestratorVersion.getAppId(), null); } } - if(orchestrationIdMap.isEmpty()) { + if (orchestrationIdMap.isEmpty()) { LOGGER.info("the project {} has no suitable workflow, the publish by user {} is ignored.", projectId, requestConversionOrchestration.getUserName()); return ResponseOperateOrchestrator.failed("No suitable workflow(s) found, publish is ignored."); @@ -244,23 +932,23 @@ private ResponseOperateOrchestrator convertWholeProjectOrcs(RequestFrameworkConv LOGGER.info("user {} try to submit a conversion job , the orchestrationIdMap is {}, the orcIdList is {}.", requestConversionOrchestration.getUserName(), orchestrationIdMap, publishedOrcIds); - long startTime=System.currentTimeMillis(); + long startTime = System.currentTimeMillis(); //进行发布到schedulis等调度系统 LOGGER.info(" begin to convert project {} for user {} to scheduler, the orchestrationIds is {}.", - projectId,requestConversionOrchestration.getUserName() , orcIds); + projectId, requestConversionOrchestration.getUserName(), orcIds); ConversionDSSOrchestratorPlugin conversionDSSOrchestratorPlugin = null; - for (DSSOrchestratorPlugin plugin:dssOrchestratorContext.getOrchestratorPlugins()) { - if(plugin instanceof ConversionDSSOrchestratorPlugin) { + for (DSSOrchestratorPlugin plugin : dssOrchestratorContext.getOrchestratorPlugins()) { + if (plugin instanceof ConversionDSSOrchestratorPlugin) { conversionDSSOrchestratorPlugin = (ConversionDSSOrchestratorPlugin) plugin; } } - if(conversionDSSOrchestratorPlugin==null){ + if (conversionDSSOrchestratorPlugin == null) { return ResponseOperateOrchestrator.failed("can not find plugin type of ConversionDSSOrchestratorPlugin,operate failed!"); } ProjectInfoRequest projectInfoRequest = new ProjectInfoRequest(); projectInfoRequest.setProjectId(projectId); ResponseOperateOrchestrator response; - try{ + try { DSSProject project = RpcAskUtils.processAskException(DSSSenderServiceFactory.getOrCreateServiceInstance().getProjectServerSender() .ask(projectInfoRequest), DSSProject.class, ProjectInfoRequest.class); response = conversionDSSOrchestratorPlugin.convert( @@ -270,24 +958,24 @@ private ResponseOperateOrchestrator convertWholeProjectOrcs(RequestFrameworkConv orchestrationIdMap, labels, requestConversionOrchestration.getApproveId()); - if(response.isFailed()) { + if (response.isFailed()) { String msg = response.getMessage() == null ? "Unknown reason, please ask admin for help!" : response.getMessage(); throw new DSSErrorException(50000, msg); } - final ResponseOperateOrchestrator finalResponse=response; + final ResponseOperateOrchestrator finalResponse = response; //最后，把所有之前未发布过的编排，都标记为已发布。 ConversionJobEntity entity = new ConversionJobEntity(); entity.setUserName(requestConversionOrchestration.getUserName()); entity.setOrcIdList(publishedOrcIds); entity.setLabels(labels); entity.setProject(project); - notPublished.forEach(toPublishOrcId-> this.updateDBAfterConversion(toPublishOrcId, finalResponse, entity, requestConversionOrchestration) ); - } catch (final Exception t){ - LOGGER.error("convert for project failed"+projectId, t); + notPublished.forEach(toPublishOrcId -> this.updateDBAfterConversion(toPublishOrcId, finalResponse, entity, requestConversionOrchestration)); + } catch (final Exception t) { + LOGGER.error("convert for project failed" + projectId, t); response = ResponseOperateOrchestrator.failed(ExceptionUtils.getRootCauseMessage(t)); } - LOGGER.info("convert project {} for user {} to Orchestrators {}, costs {}.",projectId, - requestConversionOrchestration.getUserName(),orcIds , ByteTimeUtils.msDurationToString(System.currentTimeMillis()-startTime)); + LOGGER.info("convert project {} for user {} to Orchestrators {}, costs {}.", projectId, + requestConversionOrchestration.getUserName(), orcIds, ByteTimeUtils.msDurationToString(System.currentTimeMillis() - startTime)); return response; } @@ -306,7 +994,7 @@ public ResponseConvertOrchestrator getConvertOrchestrationStatus(String id) { return new ResponseConvertOrchestrator(jobId, responseOperateOrchestrator); } - private void updateDBAfterConversion(Long toPublishOrcId,ResponseOperateOrchestrator response, + private void updateDBAfterConversion(Long toPublishOrcId, ResponseOperateOrchestrator response, ConversionJobEntity conversionJobEntity, RequestFrameworkConvertOrchestration requestConversionOrchestration) { if (response.isSucceed()) { @@ -314,7 +1002,7 @@ private void updateDBAfterConversion(Long toPublishOrcId,ResponseOperateOrchestr List orcIdList = conversionJobEntity.getOrcIdList(); LOGGER.info("the orcIdList is:{}", orcIdList); // 开发环境才新增版本号 - if(DSSLabelUtil.isDevEnv(conversionJobEntity.getLabels())){ + if (DSSLabelUtil.isDevEnv(conversionJobEntity.getLabels())) { orcIdList.forEach(DSSExceptionUtils.handling(orcId -> { DSSOrchestratorInfo dssOrchestratorInfo = orchestratorMapper.getOrchestrator(orcId); dssOrchestratorContext.getDSSOrchestratorPlugin(ExportDSSOrchestratorPlugin.class) @@ -326,18 +1014,18 @@ private void updateDBAfterConversion(Long toPublishOrcId,ResponseOperateOrchestr //2. 做一个标记表示已经发布过了 orcIdList.forEach(orchestratorMapper::setPublished); //3.更新当前的提交的发布编排模式版本的common - updateCurrentPublishOrchestratorCommon(toPublishOrcId,requestConversionOrchestration.getComment(),conversionJobEntity.getLabels()); + updateCurrentPublishOrchestratorCommon(toPublishOrcId, requestConversionOrchestration.getComment(), conversionJobEntity.getLabels()); } } - public void updateCurrentPublishOrchestratorCommon(Long orcId,String realComment,List dssLabels) { + public void updateCurrentPublishOrchestratorCommon(Long orcId, String realComment, List dssLabels) { //DEV环境获取获取最新有效的版本即可 int validFlag = 1; - if (!DSSOrchestratorConf.DSS_IMPORT_VALID_IMMEDIATELY.getValue()&& !DSSLabelUtil.isDevEnv(dssLabels)) { + if (!DSSOrchestratorConf.DSS_IMPORT_VALID_IMMEDIATELY.getValue() && !DSSLabelUtil.isDevEnv(dssLabels)) { //PROD获取当前编排模式最新版本的ID validFlag = 0; } - DSSOrchestratorVersion version = orchestratorMapper.getLatestOrchestratorVersionByIdAndValidFlag(orcId,validFlag); + DSSOrchestratorVersion version = orchestratorMapper.getLatestOrchestratorVersionByIdAndValidFlag(orcId, validFlag); DSSOrchestratorVersion updateCommentVersion = new DSSOrchestratorVersion(); updateCommentVersion.setId(version.getId()); updateCommentVersion.setComment(realComment); diff --git a/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/service/impl/OrchestratorServiceImpl.java b/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/service/impl/OrchestratorServiceImpl.java index 11312bdce5..6e94f76b9c 100644 --- a/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/service/impl/OrchestratorServiceImpl.java +++ b/dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/service/impl/OrchestratorServiceImpl.java @@ -19,20 +19,29 @@ import com.google.common.collect.Lists; import com.webank.wedatasphere.dss.appconn.core.AppConn; import com.webank.wedatasphere.dss.common.constant.project.ProjectUserPrivEnum; +import com.webank.wedatasphere.dss.common.entity.BmlResource; +import com.webank.wedatasphere.dss.common.entity.project.DSSProject; import com.webank.wedatasphere.dss.common.exception.DSSErrorException; import com.webank.wedatasphere.dss.common.label.DSSLabel; import com.webank.wedatasphere.dss.common.label.DSSLabelUtil; import com.webank.wedatasphere.dss.common.label.EnvDSSLabel; +import com.webank.wedatasphere.dss.common.label.LabelRouteVO; import com.webank.wedatasphere.dss.common.protocol.project.ProjectUserAuthRequest; import com.webank.wedatasphere.dss.common.protocol.project.ProjectUserAuthResponse; import com.webank.wedatasphere.dss.common.utils.DSSExceptionUtils; +import com.webank.wedatasphere.dss.common.utils.IoUtils; import com.webank.wedatasphere.dss.common.utils.MapUtils; import com.webank.wedatasphere.dss.common.utils.RpcAskUtils; import com.webank.wedatasphere.dss.contextservice.service.ContextService; import com.webank.wedatasphere.dss.framework.common.exception.DSSFrameworkErrorException; -import com.webank.wedatasphere.dss.orchestrator.common.entity.DSSOrchestratorInfo; -import com.webank.wedatasphere.dss.orchestrator.common.entity.DSSOrchestratorVersion; -import com.webank.wedatasphere.dss.orchestrator.common.entity.OrchestratorVo; +import com.webank.wedatasphere.dss.git.common.protocol.config.GitServerConfig; +import com.webank.wedatasphere.dss.git.common.protocol.request.GitAddMemberRequest; +import com.webank.wedatasphere.dss.git.common.protocol.request.GitRenameRequest; +import com.webank.wedatasphere.dss.git.common.protocol.request.GitRevertRequest; +import com.webank.wedatasphere.dss.git.common.protocol.response.GitAddMemberResponse; +import com.webank.wedatasphere.dss.git.common.protocol.response.GitCommitResponse; +import com.webank.wedatasphere.dss.git.common.protocol.util.UrlUtils; +import com.webank.wedatasphere.dss.orchestrator.common.entity.*; import com.webank.wedatasphere.dss.orchestrator.common.protocol.RequestOrchestratorInfos; import com.webank.wedatasphere.dss.orchestrator.common.protocol.RequestProjectUpdateOrcVersion; import com.webank.wedatasphere.dss.orchestrator.common.protocol.ResponseOrchestratorInfos; @@ -46,10 +55,16 @@ import com.webank.wedatasphere.dss.orchestrator.publish.utils.OrchestrationDevelopmentOperationUtils; import com.webank.wedatasphere.dss.orchestrator.server.conf.OrchestratorConf; import com.webank.wedatasphere.dss.orchestrator.server.constant.DSSOrchestratorConstant; +import com.webank.wedatasphere.dss.orchestrator.server.constant.OrchestratorStatusEnum; +import com.webank.wedatasphere.dss.orchestrator.server.entity.request.ModifyOrchestratorMetaRequest; import com.webank.wedatasphere.dss.orchestrator.server.entity.request.OrchestratorModifyRequest; import com.webank.wedatasphere.dss.orchestrator.server.entity.request.OrchestratorRequest; +import com.webank.wedatasphere.dss.orchestrator.server.entity.request.OrchestratorSubmitRequest; import com.webank.wedatasphere.dss.orchestrator.server.entity.vo.OrchestratorBaseInfo; +import com.webank.wedatasphere.dss.orchestrator.server.entity.vo.OrchestratorRollBackGitVo; +import com.webank.wedatasphere.dss.orchestrator.server.entity.vo.OrchestratorStatusVo; import com.webank.wedatasphere.dss.orchestrator.server.entity.vo.OrchestratorUnlockVo; +import com.webank.wedatasphere.dss.orchestrator.server.service.OrchestratorPluginService; import com.webank.wedatasphere.dss.orchestrator.server.service.OrchestratorService; import com.webank.wedatasphere.dss.sender.service.DSSSenderServiceFactory; import com.webank.wedatasphere.dss.standard.app.development.operation.*; @@ -63,20 +78,33 @@ import com.webank.wedatasphere.dss.standard.common.desc.AppInstance; import com.webank.wedatasphere.dss.standard.common.entity.ref.ResponseRef; import com.webank.wedatasphere.dss.standard.common.exception.operation.ExternalOperationWarnException; +import com.webank.wedatasphere.dss.workflow.common.entity.DSSFlow; import com.webank.wedatasphere.dss.workflow.common.protocol.*; +import com.webank.wedatasphere.dss.workflow.dao.FlowMapper; +import com.webank.wedatasphere.dss.workflow.dao.LockMapper; +import com.webank.wedatasphere.dss.workflow.lock.DSSFlowEditLockManager; +import com.webank.wedatasphere.dss.workflow.service.DSSFlowService; import org.apache.commons.collections.CollectionUtils; import org.apache.linkis.cs.client.ContextClient; import org.apache.linkis.cs.client.builder.ContextClientFactory; import org.apache.linkis.rpc.Sender; +import org.openqa.selenium.*; +import org.openqa.selenium.chrome.ChromeDriver; +import org.openqa.selenium.chrome.ChromeDriverService; +import org.openqa.selenium.chrome.ChromeOptions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.BeanUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; +import org.springframework.util.StringUtils; +import java.io.File; import java.time.LocalDateTime; import java.util.*; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; import java.util.function.BiFunction; import java.util.function.Consumer; import java.util.function.Function; @@ -94,6 +122,14 @@ public class OrchestratorServiceImpl implements OrchestratorService { private ContextService contextService; @Autowired AddOrchestratorVersionHook addOrchestratorVersionHook; + @Autowired + private OrchestratorPluginService orchestratorPluginService; + @Autowired + private LockMapper lockMapper; + @Autowired + private FlowMapper flowMapper; + @Autowired + private DSSFlowService flowService; private static final int VALID_FLAG = 1; @@ -297,7 +333,7 @@ public OrchestratorUnlockVo unlockOrchestrator(String userName, DSSOrchestratorVersion dssOrchestratorVersion = orchestratorMapper.getLatestOrchestratorVersionByIdAndValidFlag(orchestratorInfoId, VALID_FLAG); LOGGER.info("user {} try to unlock the project {} 's orchestration(such as DSS workflow) {} of orchestrator {} in version {}.", userName, projectName, dssOrchestratorVersion.getAppId(), dssOrchestratorInfo.getName(), dssOrchestratorVersion.getVersion()); - RequestUnlockWorkflow requestUnlockWorkflow = new RequestUnlockWorkflow(userName, dssOrchestratorVersion.getAppId(), confirmDelete); + RequestUnlockWorkflow requestUnlockWorkflow = new RequestUnlockWorkflow(userName, dssOrchestratorVersion.getAppId(), confirmDelete, workspace); ResponseUnlockWorkflow responseUnlockWorkflow = RpcAskUtils.processAskException(DSSSenderServiceFactory.getOrCreateServiceInstance() .getWorkflowSender(dssLabels).ask(requestUnlockWorkflow), ResponseUnlockWorkflow.class, RequestUnlockWorkflow.class); switch (responseUnlockWorkflow.getUnlockStatus()) { @@ -387,16 +423,16 @@ public List getOrchestratorVersions(String username, Lon @Override @Transactional(rollbackFor = Exception.class) - public String rollbackOrchestrator(String userName, Long projectId, String projectName, - Long orchestratorId, String version, DSSLabel dssLabel, Workspace workspace) throws Exception { + public OrchestratorRollBackGitVo rollbackOrchestrator(String userName, Long projectId, String projectName, + Long orchestratorId, String version, LabelRouteVO labels, Workspace workspace) throws Exception { //1.新建一个版本 //2.然后将version的版本内容进行去workflow进行cp //3.然后把生产的内容进行update到数据库 - DSSOrchestratorVersion oldOrcVersion=orchestratorMapper.getLatestOrchestratorVersionByIdAndValidFlag(orchestratorId, 1); + DSSOrchestratorVersion oldOrcVersion = orchestratorMapper.getLatestOrchestratorVersionByIdAndValidFlag(orchestratorId, 1); String latestVersion = oldOrcVersion.getVersion(); - List labels = new ArrayList<>(); - labels.add(dssLabel); + DSSLabel envDSSLabel = new EnvDSSLabel(labels.getRoute()); DSSOrchestratorInfo dssOrchestratorInfo = orchestratorMapper.getOrchestrator(orchestratorId); + String newVersion = OrchestratorUtils.increaseVersion(latestVersion); DSSOrchestratorVersion dssOrchestratorVersion = new DSSOrchestratorVersion(); String comment = "回滚工作流到版本:" + version; @@ -415,7 +451,7 @@ public String rollbackOrchestrator(String userName, Long projectId, String proje String contextId = contextService.createContextID(workspace.getWorkspaceName(), projectName, dssOrchestratorInfo.getName(), dssOrchestratorVersion.getVersion(), userName); dssOrchestratorVersion.setContextId(contextId); LOGGER.info("Create a new ContextId {} for rollback the orchestration {} to version {}.", contextId, dssOrchestratorInfo.getName(), version); - RefJobContentResponseRef responseRef = tryRefOperation(dssOrchestratorInfo, userName, workspace, Collections.singletonList(dssLabel), null, + RefJobContentResponseRef responseRef = tryRefOperation(dssOrchestratorInfo, userName, workspace, Collections.singletonList(envDSSLabel), null, developmentService -> ((RefCRUDService) developmentService).getRefCopyOperation(), dssContextRequestRef -> dssContextRequestRef.setContextId(contextId), projectRefRequestRef -> projectRefRequestRef.setRefProjectId(dssOrchestratorInfo.getProjectId()).setProjectName(projectName), @@ -429,14 +465,43 @@ public String rollbackOrchestrator(String userName, Long projectId, String proje }, "copy"); dssOrchestratorVersion.setAppId((Long) responseRef.getRefJobContent().get(OrchestratorRefConstant.ORCHESTRATION_ID_KEY)); dssOrchestratorVersion.setContent((String) responseRef.getRefJobContent().get(OrchestratorRefConstant.ORCHESTRATION_CONTENT_KEY)); - List paramConfTemplateIds=(List) responseRef.getRefJobContent().get(OrchestratorRefConstant.ORCHESTRATION_FLOWID_PARAMCONF_TEMPLATEID_TUPLES_KEY); + List paramConfTemplateIds = (List) responseRef.getRefJobContent().get(OrchestratorRefConstant.ORCHESTRATION_FLOWID_PARAMCONF_TEMPLATEID_TUPLES_KEY); dssOrchestratorVersion.setFormatContextId(contextId); //update appConn node contextId addOrchestratorVersionHook.beforeAdd(oldOrcVersion, Collections.emptyMap()); orchestratorMapper.addOrchestratorVersion(dssOrchestratorVersion); - addOrchestratorVersionHook.afterAdd(dssOrchestratorVersion, Collections.singletonMap(OrchestratorRefConstant.ORCHESTRATION_FLOWID_PARAMCONF_TEMPLATEID_TUPLES_KEY,paramConfTemplateIds)); + addOrchestratorVersionHook.afterAdd(dssOrchestratorVersion, Collections.singletonMap(OrchestratorRefConstant.ORCHESTRATION_FLOWID_PARAMCONF_TEMPLATEID_TUPLES_KEY, paramConfTemplateIds)); // synProjectOrchestratorVersionId(dssOrchestratorVersion, labels); - return dssOrchestratorVersion.getVersion(); + + OrchestratorRollBackGitVo rollBackGitVo = new OrchestratorRollBackGitVo(); + rollBackGitVo.setOldOrcVersion(dbOrcVersion); + rollBackGitVo.setDssOrchestratorInfo(dssOrchestratorInfo); + rollBackGitVo.setDssOrchestratorVersion(dssOrchestratorVersion); + rollBackGitVo.setVersion(dssOrchestratorVersion.getVersion()); + + return rollBackGitVo; + } + + @Override + public void rollbackOrchestratorGit(OrchestratorRollBackGitVo rollBackGitVo, String userName, Long projectId, String projectName, + Long orchestratorId, LabelRouteVO labels, Workspace workspace, Long latestAppId) throws Exception { + if (rollBackGitVo == null) { + return; + } + + DSSOrchestratorVersion dssOrchestratorVersion = rollBackGitVo.getDssOrchestratorVersion(); + DSSProject projectInfo = DSSFlowEditLockManager.getProjectInfo(projectId); + DSSFlow dssFlow = flowService.getFlow(dssOrchestratorVersion.getAppId()); + if (projectInfo.getAssociateGit() != null && projectInfo.getAssociateGit()) { + lockMapper.updateOrchestratorStatus(orchestratorId, OrchestratorRefConstant.FLOW_STATUS_SAVE); + // 回滚清空存量BML + orchestratorMapper.updateOrchestratorBmlVersion(orchestratorId, null, null); + } + if (labels.getRoute().equals("dev")) { + flowService.deleteFlowMetaData(orchestratorId); + flowService.saveAllFlowMetaData(dssFlow.getId(), orchestratorId); + } + } @@ -465,14 +530,21 @@ public void isExistSameNameBeforeCreate(Long workspaceId, Long projectId, String //是否存在相同的编排名称,如果不存在相同的编排名称則返回编排id @Override - public Long isExistSameNameBeforeUpdate(OrchestratorModifyRequest orchestratorModifRequest) throws DSSFrameworkErrorException { - DSSOrchestratorInfo orchestratorInfo = orchestratorMapper.getOrchestrator(orchestratorModifRequest.getId()); + public Long isExistSameNameBeforeUpdate(OrchestratorModifyRequest orchestratorModifRequest, DSSProject dssProject, String username) throws DSSFrameworkErrorException { + Long orchestratorId = orchestratorModifRequest.getId(); + DSSOrchestratorInfo orchestratorInfo = orchestratorMapper.getOrchestrator(orchestratorId); if (orchestratorInfo == null) { - DSSFrameworkErrorException.dealErrorException(60000, "编排模式ID=" + orchestratorModifRequest.getId() + "不存在"); + DSSFrameworkErrorException.dealErrorException(60000, "编排模式ID=" + orchestratorId + "不存在"); } //若修改了编排名称，检查是否存在相同的编排名称 if (!orchestratorModifRequest.getOrchestratorName().equals(orchestratorInfo.getName())) { isExistSameNameBeforeCreate(orchestratorModifRequest.getWorkspaceId(), orchestratorModifRequest.getProjectId(), orchestratorModifRequest.getOrchestratorName()); + if (dssProject.getAssociateGit() != null && dssProject.getAssociateGit()) { + Sender sender = DSSSenderServiceFactory.getOrCreateServiceInstance().getGitSender(); + GitRenameRequest renameRequest = new GitRenameRequest(orchestratorInfo.getWorkspaceId(), dssProject.getName(), orchestratorInfo.getName(), orchestratorModifRequest.getOrchestratorName(), username); + RpcAskUtils.processAskException(sender.ask(renameRequest), GitCommitResponse.class, GitRenameRequest.class); + lockMapper.updateOrchestratorStatus(orchestratorId, OrchestratorRefConstant.FLOW_STATUS_SAVE); + } } return orchestratorInfo.getId(); } @@ -485,12 +557,17 @@ public Long isExistSameNameBeforeUpdate(OrchestratorModifyRequest orchestratorMo * @return list of OrchestratorBaseInfo */ @Override - public List getListByPage(OrchestratorRequest orchestratorRequest, String username) { - List list = orchestratorMapper.queryOrchestratorInfos(new HashMap() {{ - put("workspace_id", orchestratorRequest.getWorkspaceId()); - put("project_id", orchestratorRequest.getProjectId()); - put("orchestrator_mode", orchestratorRequest.getOrchestratorMode()); - }}); + public List getOrchestratorInfos(OrchestratorRequest orchestratorRequest, String username) { + LabelRouteVO labels = orchestratorRequest.getLabels(); + List list = new ArrayList<>(); + if (labels == null || "dev".equals(labels.getRoute())) { + list = getOrchestratorInfoByLabel(orchestratorRequest); + } else { + List dssLabelList = Arrays.asList(new EnvDSSLabel(labels.getRoute())); + Sender sender = DSSSenderServiceFactory.getOrCreateServiceInstance().getOrcSender(dssLabelList); + DSSOrchestratorInfoList orchestratorInfoList = RpcAskUtils.processAskException(sender.ask(orchestratorRequest), DSSOrchestratorInfoList.class, OrchestratorRequest.class); + list = orchestratorInfoList.getOrchestratorInfos(); + } List retList = new ArrayList<>(list.size()); if (!CollectionUtils.isEmpty(list)) { //todo Is used in front-end? @@ -514,12 +591,27 @@ public List getListByPage(OrchestratorRequest orchestrator orchestratorBaseInfo.setOrchestratorId(dssOrchestratorInfo.getId()); orchestratorBaseInfo.setEditable(isEditable || isReleasable); orchestratorBaseInfo.setReleasable(isReleasable); + orchestratorBaseInfo.setIsDefaultReference(dssOrchestratorInfo.getIsDefaultReference()); + orchestratorBaseInfo.setStatus(dssOrchestratorInfo.getStatus()); + retList.add(orchestratorBaseInfo); } } return retList; } + @Override + public List getOrchestratorInfoByLabel(OrchestratorRequest orchestratorRequest) { + List dssOrchestratorInfos = orchestratorMapper.queryOrchestratorInfos(new HashMap() {{ + put("workspace_id", orchestratorRequest.getWorkspaceId()); + put("project_id", orchestratorRequest.getProjectId()); + put("orchestrator_mode", orchestratorRequest.getOrchestratorMode()); + }}); + if (dssOrchestratorInfos == null) + return new ArrayList<>(); + return dssOrchestratorInfos; + } + @Override public ResponseOrchestratorInfos queryOrchestratorInfos(RequestOrchestratorInfos requestOrchestratorInfos) { List orchestratorInfos = orchestratorMapper.queryOrchestratorInfos(new HashMap() {{ @@ -538,8 +630,8 @@ public void batchClearContextId() { // 1、先去查询dss_orchestrator_version_info表，筛选出发布过n次及以上的编排，并获取老的发布记录。 //为了不影响正常使用，需要频繁下载工作流bml文件，每次只拿50条工作流进行清理 List historyOrcVersionList = orchestratorMapper.getHistoryOrcVersion(OrchestratorConf.DSS_PUBLISH_MAX_VERSION.getValue()); - while (historyOrcVersionList.size()>0) { - LOGGER.info("Clear historyOrcVersionList size is "+ historyOrcVersionList.size()); + while (historyOrcVersionList.size() > 0) { + LOGGER.info("Clear historyOrcVersionList size is " + historyOrcVersionList.size()); if (historyOrcVersionList == null || historyOrcVersionList.isEmpty()) { LOGGER.info("--------------------{} end clear old contextId------------------------", LocalDateTime.now()); return; @@ -558,7 +650,7 @@ public void batchClearContextId() { contextIdList.addAll(response.getContextIdList()); } } - LOGGER.info("Clear contextIdList size is "+ contextIdList.size()); + LOGGER.info("Clear contextIdList size is " + contextIdList.size()); // 3、调用linkis接口批量删除contextId ContextClient contextClient = ContextClientFactory.getOrCreateContextClient(); // 每次处理1000条数据 @@ -580,7 +672,7 @@ public void batchClearContextId() { orchestratorMapper.batchUpdateOrcInfo(historyOrcVersionList); Thread.sleep(5000); - historyOrcVersionList=orchestratorMapper.getHistoryOrcVersion(OrchestratorConf.DSS_PUBLISH_MAX_VERSION.getValue()); + historyOrcVersionList = orchestratorMapper.getHistoryOrcVersion(OrchestratorConf.DSS_PUBLISH_MAX_VERSION.getValue()); } } catch (Exception e) { @@ -588,4 +680,143 @@ public void batchClearContextId() { } } + @Override + public String getAuthenToken(String gitUrlPre, String gitUsername, String gitPassword) throws ExecutionException { + // 启动chromedriver + WebDriver driver = generateChromeDriver(this.getClass().getClassLoader().getResource(DSSOrchestratorConstant.CHROME_DRIVER_PATH).getPath(), null); + String token = ""; + try { + //设置超时时间 + driver.manage().timeouts().implicitlyWait(Long.parseLong(GitServerConfig.GIT_TIME.getValue()), TimeUnit.SECONDS); + driver.manage().window().maximize(); + driver.manage().window().setSize(new Dimension(1920, 1080)); + driver.get(UrlUtils.normalizeIp(gitUrlPre)); + WebElement elementUserName = driver.findElement(By.id(GitServerConfig.GIT_USER.getValue())); + WebElement elementPassWord = driver.findElement(By.id(GitServerConfig.GIT_PASSWD.getValue())); + WebElement elementBtn = driver.findElement(By.cssSelector(GitServerConfig.GIT_SUBMIT.getValue())); + elementUserName.sendKeys(gitUsername); + elementPassWord.sendKeys(gitPassword); + elementBtn.submit(); + driver.navigate().refresh(); + LOGGER.info("for user getting... " + UrlUtils.normalizeIp(gitUrlPre)); + Set cookies = driver.manage().getCookies(); + LOGGER.info("cookies： {}", cookies.toString()); + for (Cookie cookie : cookies) { + if (cookie.getName().equals("_gitlab_session")) { + token = cookie.getValue(); + break; + } + } + LOGGER.info("driver cookies: {}", token); + } catch (Exception e) { + LOGGER.info("error bescause: ", e); + } finally { + driver.manage().deleteAllCookies(); + driver.quit(); + } + return token; + } + + + private WebDriver generateChromeDriver(String path, Proxy seleniumProxy) throws ExecutionException { + File file = new File(path); + if (!file.canExecute() && !file.setExecutable(true)) { + throw new ExecutionException(new Exception(path + "is not executable!")); + } + + System.setProperty(ChromeDriverService.CHROME_DRIVER_EXE_PROPERTY, path); + + ChromeOptions options = new ChromeOptions(); + options.setProxy(seleniumProxy); + options.addArguments("headless"); + options.addArguments("no-sandbox"); + options.addArguments("disable-gpu"); + options.addArguments("--disable-notifications"); + options.addArguments("--disable-popup-blocking"); + options.addArguments("disable-features=NetworkService"); + options.addArguments("ignore-certificate-errors"); + options.addArguments("silent"); + options.addArguments("--disable-application-cache"); + + options.addArguments("disable-dev-shm-usage"); + options.addArguments("remote-debugging-port=9012"); + + + return new ChromeDriver(options); + } + + @Override + public OrchestratorVo getOrchestratorByAppId(Long appId) { + OrchestratorInfo orcInfoByAppId = orchestratorMapper.getOrcInfoByAppId(appId); + if (orcInfoByAppId == null) { + return null; + } + return getOrchestratorVoById(orcInfoByAppId.getOrchestratorId()); + } + + + public List getOrchestratorGitStatus() { + + return OrchestratorStatusEnum.getOrchestratorGitStatus(); + } + + @Override + public List getAllOrchestratorName(Long workspaceId, String projectName) { + return orchestratorMapper.getAllOrchestratorName(workspaceId, projectName); + } + + @Override + public OrchestratorVo updateOrchestratorBML(Long flowId, BmlResource bmlResource) { + OrchestratorVo orchestratorByAppId = getOrchestratorByAppId(flowId); + orchestratorMapper.updateOrchestratorBmlVersion(orchestratorByAppId.getDssOrchestratorInfo().getId(), bmlResource.getResourceId(), bmlResource.getVersion()); + return orchestratorByAppId; + } + + @Override + public Boolean updateOrchestratorTime(Long orchestratorId) { + DSSOrchestratorVersion orcVersion = orchestratorMapper.getLatestOrchestratorVersionByIdAndValidFlag(orchestratorId, 1); + if (orcVersion != null) { + orchestratorMapper.updateOrchestratorVersionUpdateTime(orcVersion.getId()); + return true; + } + + return false; + } + + @Override + public String getGitUrl(String workflowName, String workflowNodeName, String projectName, String userName, Workspace workspace, Long workflowId) throws Exception{ + Sender sender = DSSSenderServiceFactory.getOrCreateServiceInstance().getGitSender(); + String filePath; + if (workflowId == null) { + filePath = workflowName; + } else { + filePath = getFlowPath(workflowId); + } + if (!StringUtils.isEmpty(workflowNodeName)) { + filePath += "/" + workflowNodeName; + } + GitAddMemberRequest gitAddMemberRequest = new GitAddMemberRequest(workspace.getWorkspaceId(), projectName, userName, filePath); + GitAddMemberResponse addMemberResponse = RpcAskUtils.processAskException(sender.ask(gitAddMemberRequest), GitAddMemberResponse.class, GitAddMemberRequest.class); + if (addMemberResponse == null) { + return null; + } else { + return addMemberResponse.getGitUrl(); + } + } + + private String getFlowPath(Long flowId) { + if (flowId == null) { + return null; + } + DSSFlow dssFlow = flowMapper.selectFlowByID(flowId); + if (dssFlow == null) { + return null; + } + if (dssFlow.getRootFlow()) { + return dssFlow.getName(); + } else { + Long parentFlowID = flowMapper.getParentFlowID(flowId); + return getFlowPath(parentFlowID) + "/" + dssFlow.getName(); + } + } } \ No newline at end of file diff --git a/dss-framework/dss-framework-orchestrator-server/src/main/scala/com/webank/wedatasphere/dss/orchestrator/server/receiver/DSSOrchestratorChooser.scala b/dss-framework/dss-framework-orchestrator-server/src/main/scala/com/webank/wedatasphere/dss/orchestrator/server/receiver/DSSOrchestratorChooser.scala index b26472b078..ed70e4de55 100644 --- a/dss-framework/dss-framework-orchestrator-server/src/main/scala/com/webank/wedatasphere/dss/orchestrator/server/receiver/DSSOrchestratorChooser.scala +++ b/dss-framework/dss-framework-orchestrator-server/src/main/scala/com/webank/wedatasphere/dss/orchestrator/server/receiver/DSSOrchestratorChooser.scala @@ -18,7 +18,9 @@ package com.webank.wedatasphere.dss.orchestrator.server.receiver import com.webank.wedatasphere.dss.orchestrator.common.protocol._ import com.webank.wedatasphere.dss.orchestrator.core.DSSOrchestratorContext -import com.webank.wedatasphere.dss.orchestrator.server.service.{OrchestratorPluginService, OrchestratorService} +import com.webank.wedatasphere.dss.orchestrator.server.entity.request.OrchestratorRequest +import com.webank.wedatasphere.dss.orchestrator.server.service.{OrchestratorFrameworkService, OrchestratorPluginService, OrchestratorService} + import javax.annotation.PostConstruct import org.apache.linkis.rpc.{RPCMessageEvent, Receiver, ReceiverChooser} import org.springframework.beans.factory.annotation.Autowired @@ -52,6 +54,11 @@ class DSSOrchestratorChooser extends ReceiverChooser { case _: RequestOrchestratorVersion => receiver case _: RequestOrchestratorInfos => receiver case _: RequestQueryByIdOrchestrator => receiver + case _: RequestQuertByAppIdOrchestrator => receiver + case _: RequestSubmitOrchestratorSync => receiver + case _: OrchestratorRequest => receiver + case _: RequestUpdateOrchestratorBML => receiver + case _: RequestOrchestratorVersionUpdateTime => receiver case _ => None } } \ No newline at end of file diff --git a/dss-framework/dss-framework-orchestrator-server/src/main/scala/com/webank/wedatasphere/dss/orchestrator/server/receiver/DSSOrchestratorReceiver.scala b/dss-framework/dss-framework-orchestrator-server/src/main/scala/com/webank/wedatasphere/dss/orchestrator/server/receiver/DSSOrchestratorReceiver.scala index 25add1c6ed..18d1e10af7 100644 --- a/dss-framework/dss-framework-orchestrator-server/src/main/scala/com/webank/wedatasphere/dss/orchestrator/server/receiver/DSSOrchestratorReceiver.scala +++ b/dss-framework/dss-framework-orchestrator-server/src/main/scala/com/webank/wedatasphere/dss/orchestrator/server/receiver/DSSOrchestratorReceiver.scala @@ -18,12 +18,14 @@ package com.webank.wedatasphere.dss.orchestrator.server.receiver import com.webank.wedatasphere.dss.common.exception.DSSErrorException import com.webank.wedatasphere.dss.common.protocol.{ResponseExportOrchestrator, ResponseImportOrchestrator} -import com.webank.wedatasphere.dss.orchestrator.common.entity.OrchestratorVo +import com.webank.wedatasphere.dss.orchestrator.common.entity.{DSSOrchestratorInfoList, OrchestratorSubmitJob, OrchestratorVo} import com.webank.wedatasphere.dss.orchestrator.common.protocol._ +import com.webank.wedatasphere.dss.orchestrator.common.ref.OrchestratorRefConstant import com.webank.wedatasphere.dss.orchestrator.core.DSSOrchestratorContext import com.webank.wedatasphere.dss.orchestrator.publish.entity.OrchestratorExportResult import com.webank.wedatasphere.dss.orchestrator.publish.{ExportDSSOrchestratorPlugin, ImportDSSOrchestratorPlugin} -import com.webank.wedatasphere.dss.orchestrator.server.service.{OrchestratorPluginService, OrchestratorService} +import com.webank.wedatasphere.dss.orchestrator.server.entity.request.{OrchestratorRequest, OrchestratorSubmitRequest} +import com.webank.wedatasphere.dss.orchestrator.server.service.{ OrchestratorPluginService, OrchestratorService} import org.apache.linkis.rpc.{Receiver, Sender} import org.slf4j.{Logger, LoggerFactory} @@ -40,7 +42,7 @@ class DSSOrchestratorReceiver(orchestratorService: OrchestratorService, orchestr override def receiveAndReply(message: Any, sender: Sender): Any = message match { case reqExportOrchestrator: RequestExportOrchestrator => - val dssExportOrcResource: OrchestratorExportResult = orchestratorContext.getDSSOrchestratorPlugin(classOf[ExportDSSOrchestratorPlugin]).exportOrchestrator( + val dssExportOrcResource: OrchestratorExportResult = orchestratorContext.getDSSOrchestratorPlugin(classOf[ExportDSSOrchestratorPlugin]).exportOrchestratorNew( reqExportOrchestrator.getUserName, reqExportOrchestrator.getOrchestratorId, reqExportOrchestrator.getOrcVersionId, @@ -53,7 +55,7 @@ class DSSOrchestratorReceiver(orchestratorService: OrchestratorService, orchestr ) case requestImportOrchestrator: RequestImportOrchestrator => - val dssOrchestratorVersion = orchestratorContext.getDSSOrchestratorPlugin(classOf[ImportDSSOrchestratorPlugin]).importOrchestrator(requestImportOrchestrator) + val dssOrchestratorVersion = orchestratorContext.getDSSOrchestratorPlugin(classOf[ImportDSSOrchestratorPlugin]).importOrchestratorNew(requestImportOrchestrator) ResponseImportOrchestrator(dssOrchestratorVersion.getOrchestratorId,dssOrchestratorVersion.getVersion) case addVersionAfterPublish: RequestAddVersionAfterPublish => @@ -90,13 +92,38 @@ class DSSOrchestratorReceiver(orchestratorService: OrchestratorService, orchestr case requestQueryByIdOrchestrator: RequestQueryByIdOrchestrator => { val orcVersionId = requestQueryByIdOrchestrator.getOrcVersionId val orchestratorId = requestQueryByIdOrchestrator.getOrchestratorId - if (orchestratorId != null) { + if (orcVersionId != null) { orchestratorService.getOrchestratorVoByIdAndOrcVersionId(orchestratorId, orcVersionId) } else { orchestratorService.getOrchestratorVoById(orchestratorId) } } + case requestQuertByAppIdOrchestrator: RequestQuertByAppIdOrchestrator => + orchestratorService.getOrchestratorByAppId(requestQuertByAppIdOrchestrator.getAppId) + case requestSubmitOrchestratorSync: RequestSubmitOrchestratorSync => + val request = new OrchestratorSubmitRequest() + request.setOrchestratorId(requestSubmitOrchestratorSync.getOrchestratorId) + request.setProjectName(requestSubmitOrchestratorSync.getProjectName) + request.setFlowId(requestSubmitOrchestratorSync.getFlowId) + request.setLabels(requestSubmitOrchestratorSync.getLabels) + request.setComment(requestSubmitOrchestratorSync.getComment) + val username = requestSubmitOrchestratorSync.getUsername + val workspace = requestSubmitOrchestratorSync.getWorkspace + orchestratorPluginService.submitWorkflowSync(request, username, workspace) + + case orchestratorRequest: OrchestratorRequest => + val list = orchestratorService.getOrchestratorInfoByLabel(orchestratorRequest) + new DSSOrchestratorInfoList(list) + + case orchestratorUpdateBMLRequest: RequestUpdateOrchestratorBML => + val orchestratorVo = orchestratorService.updateOrchestratorBML(orchestratorUpdateBMLRequest.getFlowId,orchestratorUpdateBMLRequest.getBmlResource) + orchestratorVo + + case orchestratorVersionUpdateTime: RequestOrchestratorVersionUpdateTime => + val boolean = orchestratorService.updateOrchestratorTime(orchestratorVersionUpdateTime.getOrchestratorId) + boolean + case _ => throw new DSSErrorException(90000, "Not support message type " + message) } diff --git a/dss-framework/dss-framework-project-server/pom.xml b/dss-framework/dss-framework-project-server/pom.xml index d8bd3ed2dd..47e15fe7b4 100644 --- a/dss-framework/dss-framework-project-server/pom.xml +++ b/dss-framework/dss-framework-project-server/pom.xml @@ -21,7 +21,7 @@ dss com.webank.wedatasphere.dss - 1.1.0.20-SNAPSHOT + 1.10.0-SNAPSHOT ../../pom.xml 4.0.0 @@ -29,6 +29,11 @@ dss-framework-project-server + + com.webank.wedatasphere.dss + dss-git-common + ${dss.version} + com.webank.wedatasphere.dss dss-framework-common @@ -207,6 +212,11 @@ + + com.webank.wedatasphere.dss + dss-framework-orchestrator-server + ${dss.version} + diff --git a/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/contant/DSSProjectConstant.java b/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/contant/DSSProjectConstant.java new file mode 100644 index 0000000000..8ecca9c88f --- /dev/null +++ b/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/contant/DSSProjectConstant.java @@ -0,0 +1,42 @@ +package com.webank.wedatasphere.dss.framework.project.contant; + +import org.apache.commons.lang3.StringUtils; + +public class DSSProjectConstant { + + // 升序 + public static final String ASCEND = "ascend"; + // 降序 + public static final String DESCEND = "descend"; + + + public static String concatOrderBySql(String sortBy, String orderBy) { + + if (StringUtils.isEmpty(sortBy) || StringUtils.isEmpty(orderBy)) { + return null; + } + + String[] sortArray = sortBy.split(","); + String[] orderArray = orderBy.split(","); + if (sortArray.length != orderArray.length) { + return null; + } + StringBuffer buffer = new StringBuffer(); + for (int i = 0; i < sortArray.length; i++) { + buffer.append(sortArray[i]); + if (ASCEND.equalsIgnoreCase(orderArray[i])) { + buffer.append(" asc"); + } else { + buffer.append(" desc"); + } + + if (i < sortArray.length - 1) { + buffer.append(","); + } + } + + return buffer.toString(); + + } + +} diff --git a/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/contant/ProjectServerResponse.java b/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/contant/ProjectServerResponse.java index f47d010117..96459c7206 100644 --- a/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/contant/ProjectServerResponse.java +++ b/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/contant/ProjectServerResponse.java @@ -21,7 +21,7 @@ public enum ProjectServerResponse { PROJECT_NOT_EDIT_NAME(6002,"不能修改工程名称"), PROJECT_NOT_EXIST(6003,"工程不存在"), PROJECT_USER_NOT_IN_WORKSPACE(6004,"只有工作空间用户或管理员才能创建工程"), - PROJECT_IS_NOT_ADMIN(6005,"只有创建人或管理员才能编辑工程") + PROJECT_IS_NOT_ADMIN_OR_RELEASE(6005,"只有创建人、管理员或发布者权限用户才能编辑工程") ; ProjectServerResponse(int code, String msg) { diff --git a/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/dao/DSSProjectMapper.java b/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/dao/DSSProjectMapper.java index c3f9dfc198..f09714ddba 100644 --- a/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/dao/DSSProjectMapper.java +++ b/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/dao/DSSProjectMapper.java @@ -32,8 +32,6 @@ @Mapper public interface DSSProjectMapper extends BaseMapper { - void addProject(DSSProjectPo dssProjectPo); - @Select("select id from dss_project where `name` = #{projectName}") Long getProjectIdByName(@Param("projectName") String projectName); @@ -70,4 +68,19 @@ public interface DSSProjectMapper extends BaseMapper { * */ List getDeletedProjects(ProjectQueryRequest projectRequest); + + + + List queryListByParam(ProjectQueryRequest projectRequest); + + + List getProjectIdByUser(@Param("priv") Integer priv,@Param("userList") List userList); + + List getProjectByName(List list); + + + List queryProjectName(@Param("workspaceId") Long workspaceId,@Param("queryUser") String queryUser); + + + List queryProjectList(@Param("workspaceId") Long workspaceId,@Param("queryUser") String queryUser); } diff --git a/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/dao/impl/DSSProjectMapper.xml b/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/dao/impl/DSSProjectMapper.xml index fb1e37b25a..db21fabaf8 100644 --- a/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/dao/impl/DSSProjectMapper.xml +++ b/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/dao/impl/DSSProjectMapper.xml @@ -23,14 +23,6 @@ , `user_id`, `create_by_str`, `update_by_str` - - INSERT INTO dss_project () - VALUES - (#{id},#{name},#{source}, #{workspaceId}, #{description},#{orgID},#{visibility},#{isTransfer},#{initialOrgID}, - #{username},#{createTime},#{createBy},#{product},#{applicationArea},#{business}, - #{userID},#{createByStr},#{updateByStr}) - @@ -80,6 +82,8 @@ b.product,b.source,b.isArchive,b.create_time createTime,b.update_time updateTime, b.dev_process devProcess,b.orchestrator_mode orchestratorMode, b.visible visible, + b.associate_git associateGit, + b.data_source_list_json dataSourceListJson, GROUP_CONCAT(CONCAT(a.project_id,'-',a.priv,'-',a.username)) pusername FROM dss_project_user a LEFT JOIN dss_project b ON a.project_id = b.id AND a.workspace_id=b.workspace_id @@ -87,6 +91,14 @@ AND b.id = #{id} + + + AND b.label=#{creatorLabel} + + + AND b.label is null + + GROUP BY b.id ) c order by updateTime desc @@ -99,6 +111,7 @@ b.product,b.source,b.isArchive,b.create_time createTime,b.update_time updateTime, b.dev_process devProcess,b.orchestrator_mode orchestratorMode, b.visible visible, + b.data_source_list_json dataSourceListJson, GROUP_CONCAT(CONCAT(a.project_id,'-',a.priv,'-',a.username)) pusername FROM dss_project_user a LEFT JOIN dss_project b ON a.project_id = b.id AND a.workspace_id=b.workspace_id @@ -106,7 +119,158 @@ AND b.id = #{id} + + + AND b.label=#{creatorLabel} + + + AND b.label is null + + GROUP BY b.id ) c where c.pusername like concat('%',#{username},'%') order by updateTime desc + + + + + + + + + + + + + + \ No newline at end of file diff --git a/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/entity/DSSProjectDO.java b/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/entity/DSSProjectDO.java index 0b900239d1..8da327339c 100644 --- a/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/entity/DSSProjectDO.java +++ b/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/entity/DSSProjectDO.java @@ -16,10 +16,7 @@ package com.webank.wedatasphere.dss.framework.project.entity; -import com.baomidou.mybatisplus.annotation.IdType; -import com.baomidou.mybatisplus.annotation.TableField; -import com.baomidou.mybatisplus.annotation.TableId; -import com.baomidou.mybatisplus.annotation.TableName; +import com.baomidou.mybatisplus.annotation.*; import java.io.Serializable; import java.util.Date; @@ -104,6 +101,18 @@ public class DSSProjectDO implements Serializable { private Integer visible; + /** + * 1-接入git 0-不接入（默认） + */ + private Boolean associateGit; + @TableField(updateStrategy = FieldStrategy.IGNORED) + private String dataSourceListJson; + + /** + * 标签 + */ + private String label; + public Long getId() { return id; } @@ -320,6 +329,22 @@ public void setVisible(Integer visible) { this.visible = visible; } + public String getDataSourceListJson() { + return dataSourceListJson; + } + + public void setDataSourceListJson(String dataSourceListJson) { + this.dataSourceListJson = dataSourceListJson; + } + + public String getLabel() { + return label; + } + + public void setLabel(String label) { + this.label = label; + } + @Override public String toString() { return "DSSProjectDO{" + @@ -350,6 +375,15 @@ public String toString() { ", devProcess='" + devProcess + '\'' + ", orchestratorMode='" + orchestratorMode + '\'' + ", visible=" + visible + + ", associateGit=" + associateGit + '}'; } + + public Boolean getAssociateGit() { + return associateGit; + } + + public void setAssociateGit(Boolean associateGit) { + this.associateGit = associateGit; + } } diff --git a/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/entity/OrchestratorBatchImportInfo.java b/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/entity/OrchestratorBatchImportInfo.java new file mode 100644 index 0000000000..1fe2346ed4 --- /dev/null +++ b/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/entity/OrchestratorBatchImportInfo.java @@ -0,0 +1,46 @@ +package com.webank.wedatasphere.dss.framework.project.entity; + +import com.webank.wedatasphere.dss.framework.project.entity.po.OrchestratorImportInfo; +import com.webank.wedatasphere.dss.orchestrator.server.entity.vo.OrchestratorBaseInfo; + +import java.util.List; + +/** + * 批量导入编排的信息记录 + * Author: xlinliu + * Date: 2022/9/14 + */ +public class OrchestratorBatchImportInfo { + /** + * 来源编排信息 + */ + List from; + /** + * 导入后的编排信息 + */ + List to; + + public OrchestratorBatchImportInfo() { + } + + public OrchestratorBatchImportInfo(List from, List to) { + this.from = from; + this.to = to; + } + + public List getFrom() { + return from; + } + + public void setFrom(List from) { + this.from = from; + } + + public List getTo() { + return to; + } + + public void setTo(List to) { + this.to = to; + } +} diff --git a/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/entity/po/OrchestratorImportInfo.java b/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/entity/po/OrchestratorImportInfo.java new file mode 100644 index 0000000000..7483d7b571 --- /dev/null +++ b/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/entity/po/OrchestratorImportInfo.java @@ -0,0 +1,47 @@ +package com.webank.wedatasphere.dss.framework.project.entity.po; + +/** + * Author: xlinliu + * Date: 2024/8/27 + */ +public class OrchestratorImportInfo { + private String orchestratorName; + + private Long orchestratorId; + + private String orchestratorVersion; + + + + public String getOrchestratorName() { + return orchestratorName; + } + + public void setOrchestratorName(String orchestratorName) { + this.orchestratorName = orchestratorName; + } + + public Long getOrchestratorId() { + return orchestratorId; + } + + public void setOrchestratorId(Long orchestratorId) { + this.orchestratorId = orchestratorId; + } + + public String getOrchestratorVersion() { + return orchestratorVersion; + } + + public void setOrchestratorVersion(String orchestratorVersion) { + this.orchestratorVersion = orchestratorVersion; + } + + public static OrchestratorImportInfo newInstance(Long orchestratorId,String orchestratorName, String orchestratorVersion){ + OrchestratorImportInfo orchestratorReleaseInfo = new OrchestratorImportInfo(); + orchestratorReleaseInfo.setOrchestratorId(orchestratorId); + orchestratorReleaseInfo.setOrchestratorName(orchestratorName); + orchestratorReleaseInfo.setOrchestratorVersion(orchestratorVersion); + return orchestratorReleaseInfo; + } +} diff --git a/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/entity/request/ExportAllOrchestratorsReqest.java b/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/entity/request/ExportAllOrchestratorsReqest.java new file mode 100644 index 0000000000..8879f44264 --- /dev/null +++ b/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/entity/request/ExportAllOrchestratorsReqest.java @@ -0,0 +1,53 @@ +package com.webank.wedatasphere.dss.framework.project.entity.request; + +import javax.validation.constraints.NotNull; +import java.util.List; + +/** + * Author: xlinliu + * Date: 2024/4/15 + */ +public class ExportAllOrchestratorsReqest { + + /** + * 项目id + */ + @NotNull + private Long projectId; + + /** + * 环境标签 + */ + @NotNull + private String labels; + + + /** + * 导出备注 + */ + private String comment; + + public Long getProjectId() { + return projectId; + } + + public void setProjectId(Long projectId) { + this.projectId = projectId; + } + + public String getLabels() { + return labels; + } + + public void setLabels(String labels) { + this.labels = labels; + } + + public String getComment() { + return comment; + } + + public void setComment(String comment) { + this.comment = comment; + } +} diff --git a/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/entity/request/ProjectCreateRequest.java b/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/entity/request/ProjectCreateRequest.java index 87f90a5b58..ce727edd90 100644 --- a/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/entity/request/ProjectCreateRequest.java +++ b/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/entity/request/ProjectCreateRequest.java @@ -16,6 +16,8 @@ package com.webank.wedatasphere.dss.framework.project.entity.request; +import com.webank.wedatasphere.dss.standard.app.structure.project.ref.DSSProjectDataSource; + import javax.validation.constraints.NotNull; import javax.xml.bind.annotation.XmlRootElement; import java.util.List; @@ -60,7 +62,26 @@ public class ProjectCreateRequest { * 编码模式 list */ private List orchestratorModeList; + /** + * 项目上的数据源列表 + */ + private List dataSourceList; + + /** + * 1-接入git 0-不接入（默认） + */ + private Boolean associateGit; + + /** + * 接入git的读写用户名及token + */ + private String gitUser; + private String gitToken; + /** + * 标签 + */ + private String creatorLabel; public String getName() { return name; @@ -158,6 +179,14 @@ public void setOrchestratorModeList(List orchestratorModeList) { this.orchestratorModeList = orchestratorModeList; } + public List getDataSourceList() { + return dataSourceList; + } + + public void setDataSourceList(List dataSourceList) { + this.dataSourceList = dataSourceList; + } + @Override public String toString() { return "ProjectCreateRequest{" + @@ -173,6 +202,42 @@ public String toString() { ", workspaceName='" + workspaceName + '\'' + ", devProcessList=" + devProcessList + ", orchestratorModeList=" + orchestratorModeList + + ", associateGit=" + associateGit + + ", gitToken=" + gitToken + + ", gitUser=" + gitUser + + ", creatorLabel=" + creatorLabel + '}'; } + + public Boolean getAssociateGit() { + return associateGit; + } + + public void setAssociateGit(Boolean associateGit) { + this.associateGit = associateGit; + } + + public String getCreatorLabel() { + return creatorLabel; + } + + public void setCreatorLabel(String creatorLabel) { + this.creatorLabel = creatorLabel; + } + + public String getGitUser() { + return gitUser; + } + + public void setGitUser(String gitUser) { + this.gitUser = gitUser; + } + + public String getGitToken() { + return gitToken; + } + + public void setGitToken(String gitToken) { + this.gitToken = gitToken; + } } diff --git a/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/entity/request/ProjectModifyRequest.java b/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/entity/request/ProjectModifyRequest.java index 78b28ab614..79ab1ce55a 100644 --- a/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/entity/request/ProjectModifyRequest.java +++ b/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/entity/request/ProjectModifyRequest.java @@ -16,6 +16,8 @@ package com.webank.wedatasphere.dss.framework.project.entity.request; +import com.webank.wedatasphere.dss.standard.app.structure.project.ref.DSSProjectDataSource; + import javax.validation.constraints.NotNull; import javax.xml.bind.annotation.XmlRootElement; import java.util.List; @@ -77,6 +79,23 @@ public void setWorkspaceName(String workspaceName) { * 编排模式 code list */ private List orchestratorModeList; + private List dataSourceList; + + /** + * 1-接入git 0-不接入（默认） + */ + private Boolean associateGit; + + /** + * 接入git的读写用户名及token + */ + private String gitUser; + private String gitToken; + + /** + * 标签 + */ + private String creatorLabel; public String getName() { return name; @@ -174,6 +193,15 @@ public void setOrchestratorModeList(List orchestratorModeList) { this.orchestratorModeList = orchestratorModeList; } + + public List getDataSourceList() { + return dataSourceList; + } + + public void setDataSourceList(List dataSourceList) { + this.dataSourceList = dataSourceList; + } + @Override public String toString() { return "ProjectModifyRequest{" + @@ -189,6 +217,42 @@ public String toString() { ", workspaceId=" + workspaceId + ", devProcessList=" + devProcessList + ", orchestratorModeList=" + orchestratorModeList + + ", associateGit=" + associateGit + + ", gitUser=" + gitUser + + ", gitToken=" + gitToken + + ", creatorLabel=" + creatorLabel + '}'; } + + public Boolean getAssociateGit() { + return associateGit; + } + + public void setAssociateGit(Boolean associateGit) { + this.associateGit = associateGit; + } + + public String getGitUser() { + return gitUser; + } + + public void setGitUser(String gitUser) { + this.gitUser = gitUser; + } + + public String getGitToken() { + return gitToken; + } + + public void setGitToken(String gitToken) { + this.gitToken = gitToken; + } + + public String getCreatorLabel() { + return creatorLabel; + } + + public void setCreatorLabel(String creatorLabel) { + this.creatorLabel = creatorLabel; + } } diff --git a/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/entity/request/ProjectQueryRequest.java b/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/entity/request/ProjectQueryRequest.java index 29ba634627..c8b1dd571c 100644 --- a/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/entity/request/ProjectQueryRequest.java +++ b/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/entity/request/ProjectQueryRequest.java @@ -19,6 +19,7 @@ import javax.validation.constraints.NotNull; import javax.xml.bind.annotation.XmlRootElement; import java.io.Serializable; +import java.util.List; @XmlRootElement @@ -34,6 +35,108 @@ public class ProjectQueryRequest implements Serializable { private String username; private boolean filterProject; + /** + * 标签 + */ + private String creatorLabel; + + /** + * 项目名称列表 + * **/ + private List projectNames; + + + /** + * 项目创建人 + * **/ + private List createUsers; + + /** + * 发布权限用户 + * **/ + private List releaseUsers; + + /** + * 编辑权限用户 + * **/ + private List editUsers; + + /** + * 查看权限用户 + * **/ + private List accessUsers; + + private Integer pageNow; + + private Integer pageSize; + + private String sortBy; + + private String orderBy; + + private String orderBySql; + + private String queryUser; + + private List projectIdList; + + private String proxyUser; + + public List getProjectIdList() { + return projectIdList; + } + + public void setProjectIdList(List projectIdList) { + this.projectIdList = projectIdList; + } + + public String getQueryUser() { + return queryUser; + } + + public void setQueryUser(String queryUser) { + this.queryUser = queryUser; + } + + public String getOrderBySql() { + return orderBySql; + } + + public void setOrderBySql(String orderBySql) { + this.orderBySql = orderBySql; + } + + public String getSortBy() { + return sortBy; + } + + public void setSortBy(String sortBy) { + this.sortBy = sortBy; + } + + public String getOrderBy() { + return orderBy; + } + + public void setOrderBy(String orderBy) { + this.orderBy = orderBy; + } + + public Integer getPageNow() { + return pageNow; + } + + public void setPageNow(Integer pageNow) { + this.pageNow = pageNow; + } + + public Integer getPageSize() { + return pageSize; + } + + public void setPageSize(Integer pageSize) { + this.pageSize = pageSize; + } public boolean getFilterProject() { return filterProject; @@ -43,6 +146,14 @@ public void setFilterProject(boolean filterProject) { this.filterProject = filterProject; } + public String getCreatorLabel() { + return creatorLabel; + } + + public void setCreatorLabel(String creatorLabel) { + this.creatorLabel = creatorLabel; + } + public Long getId() { return id; } @@ -66,4 +177,56 @@ public String getUsername() { public void setUsername(String username) { this.username = username; } + + public boolean isFilterProject() { + return filterProject; + } + + public List getProjectNames() { + return projectNames; + } + + public void setProjectNames(List projectNames) { + this.projectNames = projectNames; + } + + public List getCreateUsers() { + return createUsers; + } + + public void setCreateUsers(List createUsers) { + this.createUsers = createUsers; + } + + public List getReleaseUsers() { + return releaseUsers; + } + + public void setReleaseUsers(List releaseUsers) { + this.releaseUsers = releaseUsers; + } + + public List getEditUsers() { + return editUsers; + } + + public void setEditUsers(List editUsers) { + this.editUsers = editUsers; + } + + public List getAccessUsers() { + return accessUsers; + } + + public void setAccessUsers(List accessUsers) { + this.accessUsers = accessUsers; + } + + public String getProxyUser() { + return proxyUser; + } + + public void setProxyUser(String proxyUser) { + this.proxyUser = proxyUser; + } } diff --git a/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/entity/request/ProjectTransferRequest.java b/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/entity/request/ProjectTransferRequest.java new file mode 100644 index 0000000000..81423f72b0 --- /dev/null +++ b/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/entity/request/ProjectTransferRequest.java @@ -0,0 +1,56 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.framework.project.entity.request; + +import javax.validation.constraints.NotNull; + +public class ProjectTransferRequest { + + @NotNull(message = "工程转移用户不能为空") + private String transferUserName; + + @NotNull(message = "工程名称不能为空") + private String projectName; + + + public String getTransferUserName() { + return transferUserName; + } + + public void setTransferUserName(String transferUserName) { + this.transferUserName = transferUserName; + } + + + public String getProjectName() { + return projectName; + } + + public void setProjectName(String projectName) { + this.projectName = projectName; + } + + + @Override + public String toString() { + final StringBuffer sb = new StringBuffer("ProjectTransferRequest{"); + sb.append("transferUserName='").append(transferUserName).append('\''); + sb.append(", projectName='").append(projectName).append('\''); + sb.append('}'); + return sb.toString(); + } +} \ No newline at end of file diff --git a/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/entity/response/BatchExportResult.java b/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/entity/response/BatchExportResult.java new file mode 100644 index 0000000000..43d50c490f --- /dev/null +++ b/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/entity/response/BatchExportResult.java @@ -0,0 +1,42 @@ +package com.webank.wedatasphere.dss.framework.project.entity.response; + +import com.webank.wedatasphere.dss.common.entity.BmlResource; + +import java.util.List; + +/** + * Author: xlinliu + * Date: 2022/9/9 + */ +public class BatchExportResult { + /** + * 导出文件上传后的bml资源 + */ + private BmlResource bmlResource; + + /** + * 导出文件的md5校验码 + */ + private String checkSum; + + public BatchExportResult(BmlResource bmlResource, String checkSum) { + this.bmlResource = bmlResource; + this.checkSum = checkSum; + } + + public BmlResource getBmlResource() { + return bmlResource; + } + + public void setBmlResource(BmlResource bmlResource) { + this.bmlResource = bmlResource; + } + + public String getCheckSum() { + return checkSum; + } + + public void setCheckSum(String checkSum) { + this.checkSum = checkSum; + } +} diff --git a/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/entity/response/ExportResult.java b/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/entity/response/ExportResult.java new file mode 100644 index 0000000000..ba00588da3 --- /dev/null +++ b/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/entity/response/ExportResult.java @@ -0,0 +1,55 @@ +/* + * + * * Copyright 2019 WeBank + * * + * * Licensed under the Apache License, Version 2.0 (the "License"); + * * you may not use this file except in compliance with the License. + * * You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.framework.project.entity.response; + +import com.webank.wedatasphere.dss.common.entity.BmlResource; + +/** + * created by cooperyang on 2021/1/25 + * Description: + */ +public class ExportResult { + + private BmlResource bmlResource; + /** + * 导出编排的版本id + */ + private Long orchestratorVersionId; + + public ExportResult(BmlResource bmlResource, Long orchestratorVersionId) { + this.bmlResource = bmlResource; + this.orchestratorVersionId = orchestratorVersionId; + } + + public BmlResource getBmlResource() { + return bmlResource; + } + + public void setBmlResource(BmlResource bmlResource) { + this.bmlResource = bmlResource; + } + + public Long getOrchestratorVersionId() { + return orchestratorVersionId; + } + + public void setOrchestratorVersionId(Long orchestratorVersionId) { + this.orchestratorVersionId = orchestratorVersionId; + } +} diff --git a/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/entity/response/ProjectResponse.java b/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/entity/response/ProjectResponse.java index e049ffde45..8930d5001b 100644 --- a/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/entity/response/ProjectResponse.java +++ b/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/entity/response/ProjectResponse.java @@ -16,6 +16,10 @@ package com.webank.wedatasphere.dss.framework.project.entity.response; +import com.fasterxml.jackson.annotation.JsonFormat; +import com.webank.wedatasphere.dss.standard.app.structure.project.ref.DSSProjectDataSource; +import org.springframework.format.annotation.DateTimeFormat; + import java.io.Serializable; import java.util.Date; import java.util.List; @@ -41,9 +45,14 @@ public class ProjectResponse implements Serializable { private String product; private Boolean isArchive; //工程创建时间 + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss", timezone = "GMT+8") + @DateTimeFormat(pattern = "yyyy-MM-dd HH:mm:ss") private Date createTime; //工程修改时间 + @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss", timezone = "GMT+8") + @DateTimeFormat(pattern = "yyyy-MM-dd HH:mm:ss") private Date updateTime; + private List dataSourceList; public Boolean getEditable() { return editable; @@ -80,6 +89,27 @@ public void setEditable(Boolean editable) { */ private List orchestratorModeList; + private Boolean associateGit; + + private String gitUser; + + private String gitToken; + + private String workspaceName; + + /** + * 开发流程 下拉列表 + */ + private List devProcessPermission; + + public String getWorkspaceName() { + return workspaceName; + } + + public void setWorkspaceName(String workspaceName) { + this.workspaceName = workspaceName; + } + public Date getCreateTime() { return createTime; } @@ -208,6 +238,14 @@ public void setUpdateTime(Date updateTime) { this.updateTime = updateTime; } + public List getDataSourceList() { + return dataSourceList; + } + + public void setDataSourceList(List dataSourceList) { + this.dataSourceList = dataSourceList; + } + @Override public String toString() { return "ResponseProjectVo{" + @@ -228,6 +266,42 @@ public String toString() { ", devProcessList=" + devProcessList + ", orchestratorModeList=" + orchestratorModeList + ", editable=" + editable + + ", associateGit=" + associateGit + + ", gitUser=" + gitUser + + ", gitToken=" + gitToken + + ", devProcessPermission=" + devProcessPermission + '}'; } + + public Boolean getAssociateGit() { + return associateGit; + } + + public void setAssociateGit(Boolean associateGit) { + this.associateGit = associateGit; + } + + public String getGitUser() { + return gitUser; + } + + public void setGitUser(String gitUser) { + this.gitUser = gitUser; + } + + public String getGitToken() { + return gitToken; + } + + public void setGitToken(String gitToken) { + this.gitToken = gitToken; + } + + public List getDevProcessPermission() { + return devProcessPermission; + } + + public void setDevProcessPermission(List devProcessPermission) { + this.devProcessPermission = devProcessPermission; + } } diff --git a/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/entity/vo/QueryProjectVo.java b/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/entity/vo/QueryProjectVo.java index 687b6d3617..929d13d20e 100644 --- a/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/entity/vo/QueryProjectVo.java +++ b/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/entity/vo/QueryProjectVo.java @@ -42,6 +42,49 @@ public class QueryProjectVo implements Serializable { private Integer visible; + private Boolean associateGit; + private String dataSourceListJson; + + + private String accessUsers; + + private String editUsers; + + private String releaseUsers; + + private String workspaceName; + + public String getWorkspaceName() { + return workspaceName; + } + + public void setWorkspaceName(String workspaceName) { + this.workspaceName = workspaceName; + } + + public String getAccessUsers() { + return accessUsers; + } + + public void setAccessUsers(String accessUsers) { + this.accessUsers = accessUsers; + } + + public String getEditUsers() { + return editUsers; + } + + public void setEditUsers(String editUsers) { + this.editUsers = editUsers; + } + + public String getReleaseUsers() { + return releaseUsers; + } + + public void setReleaseUsers(String releaseUsers) { + this.releaseUsers = releaseUsers; + } public Long getId() { return id; @@ -162,4 +205,20 @@ public Integer getVisible() { public void setVisible(Integer visible) { this.visible = visible; } + + public Boolean getAssociateGit() { + return associateGit; + } + + public void setAssociateGit(Boolean associateGit) { + this.associateGit = associateGit; + } + + public String getDataSourceListJson() { + return dataSourceListJson; + } + + public void setDataSourceListJson(String dataSourceListJson) { + this.dataSourceListJson = dataSourceListJson; + } } diff --git a/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/restful/DSSFrameworkProjectRestfulApi.java b/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/restful/DSSFrameworkProjectRestfulApi.java index 598e051000..d37330ceb2 100644 --- a/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/restful/DSSFrameworkProjectRestfulApi.java +++ b/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/restful/DSSFrameworkProjectRestfulApi.java @@ -18,27 +18,36 @@ import com.webank.wedatasphere.dss.common.auditlog.OperateTypeEnum; import com.webank.wedatasphere.dss.common.auditlog.TargetTypeEnum; +import com.webank.wedatasphere.dss.common.exception.DSSErrorException; import com.webank.wedatasphere.dss.common.utils.AuditLogUtils; import com.webank.wedatasphere.dss.common.utils.DSSExceptionUtils; +import com.webank.wedatasphere.dss.common.utils.RpcAskUtils; import com.webank.wedatasphere.dss.framework.project.conf.ProjectConf; +import com.webank.wedatasphere.dss.framework.project.contant.DSSProjectConstant; import com.webank.wedatasphere.dss.framework.project.entity.DSSProjectDO; -import com.webank.wedatasphere.dss.framework.project.entity.request.ProjectCreateRequest; -import com.webank.wedatasphere.dss.framework.project.entity.request.ProjectDeleteRequest; -import com.webank.wedatasphere.dss.framework.project.entity.request.ProjectModifyRequest; -import com.webank.wedatasphere.dss.framework.project.entity.request.ProjectQueryRequest; +import com.webank.wedatasphere.dss.framework.project.entity.request.*; import com.webank.wedatasphere.dss.framework.project.entity.response.ProjectResponse; import com.webank.wedatasphere.dss.framework.project.entity.vo.DSSProjectVo; -import com.webank.wedatasphere.dss.framework.project.exception.DSSProjectErrorException; import com.webank.wedatasphere.dss.framework.project.service.DSSFrameworkProjectService; import com.webank.wedatasphere.dss.framework.project.service.DSSProjectService; import com.webank.wedatasphere.dss.framework.project.service.ProjectHttpRequestHook; import com.webank.wedatasphere.dss.framework.project.utils.ApplicationArea; +import com.webank.wedatasphere.dss.framework.proxy.conf.ProxyUserConfiguration; import com.webank.wedatasphere.dss.framework.proxy.exception.DSSProxyUserErrorException; +import com.webank.wedatasphere.dss.framework.proxy.service.DssProxyUserService; +import com.webank.wedatasphere.dss.framework.workspace.service.DSSWorkspaceRoleService; +import com.webank.wedatasphere.dss.framework.workspace.service.DSSWorkspaceService; +import com.webank.wedatasphere.dss.common.StaffInfoGetter; +import com.webank.wedatasphere.dss.git.common.protocol.GitUserEntity; +import com.webank.wedatasphere.dss.git.common.protocol.request.GitSearchRequest; +import com.webank.wedatasphere.dss.git.common.protocol.response.GitSearchResponse; +import com.webank.wedatasphere.dss.sender.service.DSSSenderServiceFactory; import com.webank.wedatasphere.dss.standard.app.sso.Workspace; import com.webank.wedatasphere.dss.standard.sso.utils.SSOHelper; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.exception.ExceptionUtils; import org.apache.linkis.common.exception.WarnException; +import org.apache.linkis.rpc.Sender; import org.apache.linkis.server.Message; import org.apache.linkis.server.security.SecurityFilter; import org.slf4j.Logger; @@ -49,15 +58,15 @@ import javax.servlet.http.HttpServletRequest; import javax.validation.Valid; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Objects; +import java.util.*; import java.util.function.Consumer; import java.util.function.Function; import java.util.function.Supplier; import java.util.stream.Collectors; +import static com.webank.wedatasphere.dss.standard.sso.utils.SSOHelper.WORKSPACE_ID_COOKIE_KEY; +import static com.webank.wedatasphere.dss.standard.sso.utils.SSOHelper.WORKSPACE_NAME_COOKIE_KEY; + @RequestMapping(path = "/dss/framework/project", produces = {"application/json"}) @RestController public class DSSFrameworkProjectRestfulApi { @@ -72,11 +81,21 @@ public class DSSFrameworkProjectRestfulApi { private DSSProjectService dssProjectService; @Autowired private List projectHttpRequestHooks; + @Autowired + private DssProxyUserService dssProxyUserService; + @Autowired + private StaffInfoGetter staffInfoGetter; + + @Autowired + private DSSWorkspaceRoleService dssWorkspaceRoleService; + + @Autowired + private DSSWorkspaceService dssWorkspaceService; private Message executePreHook(Function function) { String errorMsg = projectHttpRequestHooks.stream().map(function).filter(Objects::nonNull).map(Message::getMessage) .collect(Collectors.joining(", ")); - if(StringUtils.isNotBlank(errorMsg)) { + if (StringUtils.isNotBlank(errorMsg)) { return Message.error(errorMsg); } else { return null; @@ -103,7 +122,7 @@ private Message executeAfterHook(Consumer consumer, Supp * @param projectRequest * @return */ - @RequestMapping(path = "getAllProjects", method = RequestMethod.POST) + @RequestMapping(path = {"getAllProjects","getAllProdProjects"}, method = RequestMethod.POST) public Message getAllProjects(HttpServletRequest request, @RequestBody ProjectQueryRequest projectRequest) { String username = SecurityFilter.getLoginUsername(request); projectRequest.setUsername(username); @@ -111,6 +130,29 @@ public Message getAllProjects(HttpServletRequest request, @RequestBody ProjectQu if(message != null) { return message; } + String proxyUser=username; + if(ProxyUserConfiguration.isProxyUserEnable()) { + try { + proxyUser = dssProxyUserService.getProxyUser(request); + } catch (DSSProxyUserErrorException e) { + //do nothing + } + } + final String finalProxyUsername = proxyUser; + projectRequest.setProxyUser(finalProxyUsername); + + try { + dssWorkspaceService.getWorkspacesById(projectRequest.getWorkspaceId(), username); + } catch (DSSErrorException e) { + LOGGER.error("User {} get workspace {} failed.", username, projectRequest.getWorkspaceId(), e); + return Message.error(e); + } + List roles = dssWorkspaceRoleService.getRoleInWorkspace(username, projectRequest.getWorkspaceId().intValue()); + if (roles == null || roles.isEmpty()) { + LOGGER.error("username {}, in workspace {} roles are null or empty", username, projectRequest.getWorkspaceId()); + return Message.error("can not get roles information"); + } + LOGGER.info("user {} begin to getAllProjects, projectId: {}.", username, projectRequest.getId()); List dssProjectVos = projectService.getListByParam(projectRequest); if(!CollectionUtils.isEmpty(dssProjectVos) && projectRequest.getFilterProject()){ @@ -118,6 +160,13 @@ public Message getAllProjects(HttpServletRequest request, @RequestBody ProjectQu (item.getEditUsers().contains(username) || item.getReleaseUsers().contains(username)) && item.getEditable()).collect(Collectors.toList()); } + //如果是生产中心请求，则靠考虑用户或者代理用户是否有发布权限 + if(request.getPathInfo().endsWith("getAllProdProjects")){ + dssProjectVos = dssProjectVos.stream().filter( + item-> item.getReleaseUsers().contains(username) + ||item.getReleaseUsers().contains(finalProxyUsername) + ).collect(Collectors.toList()); + } return Message.ok("获取工作空间的工程成功").data("projects", dssProjectVos); } @@ -129,14 +178,14 @@ public Message createProject(HttpServletRequest request, @RequestBody ProjectCre String username = SecurityFilter.getLoginUsername(request); Workspace workspace = SSOHelper.getWorkspace(request); LOGGER.info("user {} begin to createProject, workspace {}, project entity: {}.", username, workspace.getWorkspaceName(), projectCreateRequest); - if(projectCreateRequest.getEditUsers() == null) { + if (projectCreateRequest.getEditUsers() == null) { projectCreateRequest.setEditUsers(new ArrayList<>()); } - if(projectCreateRequest.getReleaseUsers() == null) { + if (projectCreateRequest.getReleaseUsers() == null) { projectCreateRequest.setReleaseUsers(new ArrayList<>()); } Message message = executePreHook(projectHttpRequestHook -> projectHttpRequestHook.beforeCreateProject(request, projectCreateRequest)); - if(message != null) { + if (message != null) { return message; } //將创建人默认为发布权限和編輯权限 @@ -149,12 +198,12 @@ public Message createProject(HttpServletRequest request, @RequestBody ProjectCre return DSSExceptionUtils.getMessage(() -> { DSSProjectVo dssProjectVo = dssFrameworkProjectService.createProject(projectCreateRequest, username, workspace); AuditLogUtils.printLog(username, workspace.getWorkspaceId(), workspace.getWorkspaceName(), TargetTypeEnum.PROJECT, - dssProjectVo.getId(),dssProjectVo.getName(), OperateTypeEnum.CREATE,projectCreateRequest); + dssProjectVo.getId(), dssProjectVo.getName(), OperateTypeEnum.CREATE, projectCreateRequest); return dssProjectVo; }, dssProjectVo -> - executeAfterHook(projectHttpRequestHook -> projectHttpRequestHook.afterCreateProject(request, projectCreateRequest, dssProjectVo), - () -> Message.ok("创建工程成功.").data("project", dssProjectVo)), + executeAfterHook(projectHttpRequestHook -> projectHttpRequestHook.afterCreateProject(request, projectCreateRequest, dssProjectVo), + () -> Message.ok("创建工程成功.").data("project", dssProjectVo)), String.format("用户 %s 创建工程 %s 失败. ", username, projectCreateRequest.getName())); } @@ -168,21 +217,45 @@ public Message checkProjectName(HttpServletRequest request, @RequestParam(name = } @RequestMapping(path = "getProjectInfo", method = RequestMethod.GET) - public Message getProjectInfoByName(@RequestParam(name = "projectName") String projectName){ + public Message getProjectInfoByName(@RequestParam(name = "projectName") String projectName) { DSSProjectDO dbProject = dssProjectService.getProjectByName(projectName); Message message; - if(dbProject==null){ + if (dbProject == null) { String msg = String.format("project %s does not exist.", projectName); message = Message.error(msg); - }else{ + } else { DSSProjectVo dssProjectVo = new DSSProjectVo(); dssProjectVo.setDescription(dbProject.getDescription()); dssProjectVo.setId(dbProject.getId()); dssProjectVo.setName(dbProject.getName()); message = Message.ok().data("project", dssProjectVo); } - return message; + return message; + } + + @RequestMapping(path = "getProjectGitInfo", method = RequestMethod.GET) + public Message getProjectInfoByName(HttpServletRequest request, + @RequestParam(name = "projectId") Long projectId) { + DSSProjectDO dbProject = dssProjectService.getProjectById(projectId); + Message message; + if (dbProject == null) { + message = Message.error("project does not exist."); + } else { + String username = SecurityFilter.getLoginUsername(request); + Workspace workspace = SSOHelper.getWorkspace(request); + Map projectGitUserInfo = dssProjectService.getProjectGitUserInfo(username, workspace.getWorkspaceId()); + GitUserEntity gitUserEntity = projectGitUserInfo.get(dbProject.getName()); + String gitUser = null; + String gitToken = null; + if (gitUserEntity != null) { + gitUser = gitUserEntity.getGitUser(); + gitToken = gitUserEntity.getGitToken(); + } + message = Message.ok().data("gitUser", gitUser).data("gitToken", gitToken); + } + return message; } + /** * 编辑工程 * @@ -199,13 +272,13 @@ public Message modifyProject(HttpServletRequest request, @RequestBody ProjectMod Workspace workspace = SSOHelper.getWorkspace(request); LOGGER.info("user {} begin to modifyProject, workspace {}, project entity: {}.", username, workspace.getWorkspaceName(), projectModifyRequest); Message message = executePreHook(projectHttpRequestHook -> projectHttpRequestHook.beforeModifyProject(request, projectModifyRequest)); - if(message != null) { + if (message != null) { return message; } if (projectModifyRequest.getDescription().length() > MAX_DESC_LENGTH) { return Message.error("The project description information is too long, exceeding the maximum length:" + MAX_DESC_LENGTH); } - if(org.apache.commons.lang.StringUtils.isNotEmpty(projectModifyRequest.getBusiness()) && projectModifyRequest.getBusiness().length() > MAX_BUSSINESS_SIZE){ + if (org.apache.commons.lang.StringUtils.isNotEmpty(projectModifyRequest.getBusiness()) && projectModifyRequest.getBusiness().length() > MAX_BUSSINESS_SIZE) { return Message.error("The project bussiness is too long, exceeding the maximum length:" + MAX_BUSSINESS_SIZE); } DSSProjectDO dbProject = dssProjectService.getProjectById(projectModifyRequest.getId()); @@ -214,6 +287,9 @@ public Message modifyProject(HttpServletRequest request, @RequestBody ProjectMod LOGGER.error("project {} is not exists.", projectModifyRequest.getName()); return Message.error(String.format("project %s is not exists.", projectModifyRequest.getName())); } + if (dbProject.getWorkspaceId() != workspace.getWorkspaceId()) { + return Message.error("工作流所在工作空间和cookie中不一致，请刷新页面后，再次修改！"); + } String createUsername = dbProject.getUsername(); //將创建人默认为发布权限和編輯权限 if (!projectModifyRequest.getEditUsers().contains(createUsername)) { @@ -230,11 +306,11 @@ public Message modifyProject(HttpServletRequest request, @RequestBody ProjectMod return DSSExceptionUtils.getMessage(() -> { dssFrameworkProjectService.modifyProject(projectModifyRequest, dbProject, username, workspace); AuditLogUtils.printLog(username, workspace.getWorkspaceId(), workspace.getWorkspaceName(), TargetTypeEnum.PROJECT, - projectModifyRequest.getId(),projectModifyRequest.getName(), OperateTypeEnum.UPDATE,projectModifyRequest); - }, + projectModifyRequest.getId(), projectModifyRequest.getName(), OperateTypeEnum.UPDATE, projectModifyRequest); + }, () -> - executeAfterHook(projectHttpRequestHook -> projectHttpRequestHook.afterModifyProject(request, projectModifyRequest), - () -> Message.ok("修改工程成功.")), + executeAfterHook(projectHttpRequestHook -> projectHttpRequestHook.afterModifyProject(request, projectModifyRequest), + () -> Message.ok("修改工程成功.")), String.format("用户 %s 修改工程 %s 失败. ", username, projectModifyRequest.getName())); } @@ -250,20 +326,21 @@ public Message deleteProject(HttpServletRequest request, @RequestBody ProjectDel String username = SecurityFilter.getLoginUsername(request); Workspace workspace = SSOHelper.getWorkspace(request); Message message = executePreHook(projectHttpRequestHook -> projectHttpRequestHook.beforeDeleteProject(request, projectDeleteRequest)); - if(message != null) { + if (message != null) { return message; } LOGGER.info("user {} begin to deleteProject, workspace {}, project entity: {}.", username, workspace.getWorkspaceName(), projectDeleteRequest); return DSSExceptionUtils.getMessage(() -> { // 检查是否具有删除项目权限 projectService.isDeleteProjectAuth(projectDeleteRequest.getId(), username); - DSSProjectDO dssProjectDO= projectService.getProjectById(projectDeleteRequest.getId()); - projectService.deleteProject(username, projectDeleteRequest, workspace,dssProjectDO); + DSSProjectDO dssProjectDO = projectService.getProjectById(projectDeleteRequest.getId()); + projectService.deleteProject(username, projectDeleteRequest, workspace, dssProjectDO); AuditLogUtils.printLog(username, workspace.getWorkspaceId(), workspace.getWorkspaceName(), TargetTypeEnum.PROJECT, - dssProjectDO.getId(),dssProjectDO.getName(), OperateTypeEnum.DELETE,projectDeleteRequest); }, + dssProjectDO.getId(), dssProjectDO.getName(), OperateTypeEnum.DELETE, projectDeleteRequest); + }, () -> - executeAfterHook(projectHttpRequestHook -> projectHttpRequestHook.afterDeleteProject(request, projectDeleteRequest), - () -> Message.ok("删除工程成功.")), + executeAfterHook(projectHttpRequestHook -> projectHttpRequestHook.afterDeleteProject(request, projectDeleteRequest), + () -> Message.ok("删除工程成功.")), String.format("用户 %s 删除工程失败. ", username)); } @@ -304,12 +381,202 @@ public Message getDeletedProjects(HttpServletRequest request, @Valid @RequestBod String username = SecurityFilter.getLoginUsername(request); projectRequest.setUsername(username); Message message = executePreHook(projectHttpRequestHook -> projectHttpRequestHook.beforeGetDeletedProject(request, projectRequest)); - if(message != null) { + if (message != null) { return message; } List dssProjectVos = projectService.getDeletedProjects(projectRequest); return Message.ok("获取工作空间已删除的工程成功").data("projects", dssProjectVos); } + @RequestMapping(path = "searchGit", method = RequestMethod.POST) + public Message searchGit(HttpServletRequest request, @RequestBody GitSearchRequest searchRequest) { + String username = SecurityFilter.getLoginUsername(request); + Sender sender = DSSSenderServiceFactory.getOrCreateServiceInstance().getGitSender(); + GitSearchResponse gitSearchResponse = RpcAskUtils.processAskException(sender.ask(searchRequest), GitSearchResponse.class, GitSearchRequest.class); + + return Message.ok().data("data", gitSearchResponse); + } + + @RequestMapping(path = "modifyProjectMeta", method = RequestMethod.POST) + public Message modifyProjectMeta(HttpServletRequest request, @RequestBody ProjectModifyRequest projectModifyRequest) { + + if (projectModifyRequest.getId() == null || projectModifyRequest.getId() < 0) { + return Message.error("project id is null, cannot modify it."); + } + + String username = SecurityFilter.getLoginUsername(request); + Workspace workspace = SSOHelper.getWorkspace(request); + + if(projectModifyRequest.getWorkspaceId() == null){ + projectModifyRequest.setWorkspaceId(workspace.getWorkspaceId()); + + }else if(!projectModifyRequest.getWorkspaceId().equals(workspace.getWorkspaceId())){ + return Message.error("命名空间信息和cookie中的命名空间不一致,请确认！！！"); + } + + if (projectModifyRequest.getDescription().length() > MAX_DESC_LENGTH) { + return Message.error("The project description information is too long, exceeding the maximum length:" + MAX_DESC_LENGTH); + } + + DSSProjectDO dbProject = dssProjectService.getProjectById(projectModifyRequest.getId()); + //工程不存在 + if (dbProject == null) { + LOGGER.error("project {} is not exists.", projectModifyRequest.getName()); + return Message.error(String.format("project %s is not exists.", projectModifyRequest.getName())); + } + + String createUsername = dbProject.getUsername(); + //將创建人默认为发布权限和编辑权限 + if (!projectModifyRequest.getEditUsers().contains(createUsername)) { + projectModifyRequest.getEditUsers().add(createUsername); + } + + if (!projectModifyRequest.getReleaseUsers().contains(createUsername)) { + projectModifyRequest.getReleaseUsers().add(createUsername); + } + + return DSSExceptionUtils.getMessage(() -> { + dssFrameworkProjectService.modifyProjectMeta(projectModifyRequest, dbProject, username, workspace); + AuditLogUtils.printLog(username, workspace.getWorkspaceId(), workspace.getWorkspaceName(), TargetTypeEnum.PROJECT, + projectModifyRequest.getId(), projectModifyRequest.getName(), OperateTypeEnum.UPDATE, projectModifyRequest); + }, + () -> executeAfterHook(projectHttpRequestHook -> projectHttpRequestHook.afterModifyProject(request, projectModifyRequest), + () -> Message.ok("修改工程成功.")), + String.format("用户 %s 修改工程 %s 失败. ", username, projectModifyRequest.getName())); + } + + + @RequestMapping(path = "listAllProjectName", method = RequestMethod.POST) + public Message listAllProjectName(HttpServletRequest request, @RequestBody ProjectQueryRequest projectRequest) { + + String username = SecurityFilter.getLoginUsername(request); + projectRequest.setUsername(username); + + if (projectRequest.getWorkspaceId() == null) { + Workspace workspace = SSOHelper.getWorkspace(request); + projectRequest.setWorkspaceId(workspace.getWorkspaceId()); + } + + try { + dssWorkspaceService.getWorkspacesById(projectRequest.getWorkspaceId(), username); + } catch (DSSErrorException e) { + LOGGER.error("User {} get workspace {} failed.", username, projectRequest.getWorkspaceId(), e); + return Message.error(e); + } + List roles = dssWorkspaceRoleService.getRoleInWorkspace(username, projectRequest.getWorkspaceId().intValue()); + if (roles == null || roles.isEmpty()) { + LOGGER.error("username {}, in workspace {} roles are null or empty", username, projectRequest.getWorkspaceId()); + return Message.error("can not get roles information"); + } + + LOGGER.info("user {} begin to listAllProjectName, workspaceId: {}.", username, projectRequest.getWorkspaceId()); + + List projectNames = projectService.queryProjectName(projectRequest); + + return Message.ok().data("data", projectNames); + } + + /** + * 项目转移 + */ + @RequestMapping(path = "transferProject", method = RequestMethod.POST) + public Message transferProject(HttpServletRequest request, @RequestBody ProjectTransferRequest projectTransferRequest) { + if (projectTransferRequest.getProjectName() ==null|| projectTransferRequest.getTransferUserName()==null) { + return Message.error("project id is null, cannot modify it."); + } + LOGGER.info("begin to transferProject, project entity: {}.", projectTransferRequest); + DSSProjectDO dbProject = dssProjectService.getProjectByName(projectTransferRequest.getProjectName()); + //工程不存在 + if (dbProject == null) { + LOGGER.error("project {} is not exists.", projectTransferRequest.getProjectName()); + return Message.error(String.format("project %s is not exists.", projectTransferRequest.getProjectName())); + } + String oldProjectOwner = dbProject.getCreateBy(); + if(projectTransferRequest.getTransferUserName().equals(oldProjectOwner)){ + LOGGER.warn("交接人和被交接人不能是同一个人。"); + return Message.error("交接人和被交接人不能是同一个人。"); + } + String operator =SecurityFilter.getLoginUsername(request); ; + String workspaceName = dssWorkspaceService.getWorkspaceName(dbProject.getWorkspaceId()); + Long workspaceId = dbProject.getWorkspaceId(); + Workspace workspace = new Workspace(); + workspace.setWorkspaceId(workspaceId); + workspace.setWorkspaceName(workspaceName); + SSOHelper.addWorkspaceInfo(request, workspace); + workspace.addCookie(WORKSPACE_NAME_COOKIE_KEY, workspaceName); + workspace.addCookie(WORKSPACE_ID_COOKIE_KEY,workspaceId.toString()); + + //1.把用户加到工作空间里。 + //2.，把operator加到查看、编辑、发布权限里。 + //3.把create_by和username换成transferUserName，并且把transferUserName加到查看、编辑、发布权限里 + //4.同步git + + try { + dssFrameworkProjectService.transferProject(projectTransferRequest, dbProject, oldProjectOwner, workspace, + operator); + AuditLogUtils.printLog(oldProjectOwner, workspace.getWorkspaceId(), workspace.getWorkspaceName(), TargetTypeEnum.PROJECT, + dbProject.getId(), projectTransferRequest.getProjectName(), OperateTypeEnum.UPDATE, projectTransferRequest); + } catch (Exception e) { + String errorMsg = String.format("用户 %s 转移工程 %s 失败. ", operator, + projectTransferRequest.getProjectName()); + LOGGER.error(errorMsg, e); + return Message.error(errorMsg + e.getMessage()); + } + return Message.ok("转移工程成功."); + } + + + @RequestMapping(path = "queryAllProjects", method = RequestMethod.POST) + public Message queryAllProject(HttpServletRequest request, @RequestBody ProjectQueryRequest projectRequest){ + + String username = SecurityFilter.getLoginUsername(request); + projectRequest.setUsername(username); + Message message = executePreHook(projectHttpRequestHook -> projectHttpRequestHook.beforeGetAllProjects(request, projectRequest)); + if (message != null) { + return message; + } + + if (projectRequest.getWorkspaceId() == null) { + Workspace workspace = SSOHelper.getWorkspace(request); + projectRequest.setWorkspaceId(workspace.getWorkspaceId()); + } + + try { + dssWorkspaceService.getWorkspacesById(projectRequest.getWorkspaceId(), username); + } catch (DSSErrorException e) { + LOGGER.error("User {} get workspace {} failed.", username, projectRequest.getWorkspaceId(), e); + return Message.error(e); + } + List roles = dssWorkspaceRoleService.getRoleInWorkspace(username, projectRequest.getWorkspaceId().intValue()); + if (roles == null || roles.isEmpty()) { + LOGGER.error("username {}, in workspace {} roles are null or empty", username, projectRequest.getWorkspaceId()); + return Message.error("can not get roles information"); + } + + LOGGER.info("user {} begin to queryAllProjects, projectId: {}.", username, projectRequest.getId()); + if(!StringUtils.isEmpty(projectRequest.getSortBy()) && !StringUtils.isEmpty(projectRequest.getOrderBy())){ + String orderBySql = DSSProjectConstant.concatOrderBySql(projectRequest.getSortBy(),projectRequest.getOrderBy()); + LOGGER.info(String.format("queryAllProjects sort orderBySql is %s", orderBySql)); + if(orderBySql == null){ + return Message.error("queryAllProjects sortBy or orderBy params input error"); + } + projectRequest.setOrderBySql(orderBySql); + } + + + if(projectRequest.getPageNow() == null || projectRequest.getPageNow() <= 0){ + projectRequest.setPageNow(1); + } + + if(projectRequest.getPageSize() == null || projectRequest.getPageSize() <= 0 ){ + projectRequest.setPageSize(10); + } + + List totals = new ArrayList<>(); + List dssProjectVos = projectService.queryListByParam(projectRequest,totals); + + return Message.ok("获取工作空间的工程成功").data("projects", dssProjectVos).data("total",totals.get(0)); + + } } diff --git a/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/service/DSSFrameworkProjectService.java b/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/service/DSSFrameworkProjectService.java index 46c57a0816..1f8ec5cf72 100644 --- a/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/service/DSSFrameworkProjectService.java +++ b/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/service/DSSFrameworkProjectService.java @@ -19,6 +19,7 @@ import com.webank.wedatasphere.dss.framework.project.entity.DSSProjectDO; import com.webank.wedatasphere.dss.framework.project.entity.request.ProjectCreateRequest; import com.webank.wedatasphere.dss.framework.project.entity.request.ProjectModifyRequest; +import com.webank.wedatasphere.dss.framework.project.entity.request.ProjectTransferRequest; import com.webank.wedatasphere.dss.framework.project.entity.vo.DSSProjectDetailVo; import com.webank.wedatasphere.dss.framework.project.entity.vo.DSSProjectVo; import com.webank.wedatasphere.dss.framework.project.exception.DSSProjectErrorException; @@ -41,4 +42,18 @@ public interface DSSFrameworkProjectService { void modifyProject(ProjectModifyRequest projectModifyRequest, DSSProjectDO dbProject, String username, Workspace workspace) throws Exception; void checkProjectName(String name, Workspace workspace,String username) throws DSSProjectErrorException; + + void modifyProjectMeta(ProjectModifyRequest projectModifyRequest, DSSProjectDO dbProject, String username, Workspace workspace) throws Exception; + + /** + * 项目交接 + * @param projectTransferRequest 交接请求 + * @param dbProject 原始项目 + * @param oldProjectOwner 原始项目owner + * @param workspace 工作空间 + * @param operator 操作人 + * @throws Exception + */ + void transferProject(ProjectTransferRequest projectTransferRequest, DSSProjectDO dbProject, + String oldProjectOwner, Workspace workspace,String operator); } diff --git a/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/service/DSSProjectService.java b/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/service/DSSProjectService.java index a8781eca50..29c610b951 100644 --- a/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/service/DSSProjectService.java +++ b/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/service/DSSProjectService.java @@ -17,16 +17,17 @@ package com.webank.wedatasphere.dss.framework.project.service; import com.baomidou.mybatisplus.extension.service.IService; +import com.webank.wedatasphere.dss.common.entity.BmlResource; +import com.webank.wedatasphere.dss.common.entity.project.DSSProject; import com.webank.wedatasphere.dss.common.label.DSSLabel; +import com.webank.wedatasphere.dss.common.label.EnvDSSLabel; +import com.webank.wedatasphere.dss.common.protocol.project.ProjectListQueryRequest; import com.webank.wedatasphere.dss.framework.project.entity.DSSProjectDO; -import com.webank.wedatasphere.dss.framework.project.entity.request.ProjectCreateRequest; -import com.webank.wedatasphere.dss.framework.project.entity.request.ProjectDeleteRequest; -import com.webank.wedatasphere.dss.framework.project.entity.request.ProjectModifyRequest; -import com.webank.wedatasphere.dss.framework.project.entity.request.ProjectQueryRequest; +import com.webank.wedatasphere.dss.framework.project.entity.request.*; import com.webank.wedatasphere.dss.framework.project.entity.response.ProjectResponse; import com.webank.wedatasphere.dss.framework.project.entity.vo.ProjectInfoVo; import com.webank.wedatasphere.dss.framework.project.exception.DSSProjectErrorException; -import com.webank.wedatasphere.dss.orchestrator.common.protocol.RequestProjectImportOrchestrator; +import com.webank.wedatasphere.dss.git.common.protocol.GitUserEntity; import com.webank.wedatasphere.dss.standard.app.sso.Workspace; import com.webank.wedatasphere.dss.standard.common.desc.AppInstance; @@ -39,7 +40,7 @@ public interface DSSProjectService extends IService { DSSProjectDO createProject(String username, ProjectCreateRequest projectCreateRequest); - void modifyProject(String username, ProjectModifyRequest modifyRequest) throws DSSProjectErrorException; + DSSProjectDO modifyProject(String username, ProjectModifyRequest modifyRequest) throws DSSProjectErrorException; /** * 旧工程导入到新环境的，修改新环境工程相关字段 @@ -57,6 +58,7 @@ public interface DSSProjectService extends IService { List getListByParam(ProjectQueryRequest projectRequest); + Map getProjectGitUserInfo(String username, Long workspaceId); ProjectInfoVo getProjectInfoById(Long id); @@ -80,5 +82,37 @@ public interface DSSProjectService extends IService { */ List getDeletedProjects(ProjectQueryRequest projectRequest); + /** + * 批量导出工程内的所有编排 + */ + BmlResource exportProject(ExportAllOrchestratorsReqest exportAllOrchestratorsReqest, String username, String proxyUser, Workspace workspace) throws Exception; + + /** + * 批量导入编排到指定工程 + * @param projectInfo 要导入的目标工程 + * @param importResource 导入的编排资源 + * @param username 导入人 + * @param checkCode 校验码 + * @param packageInfo 导入包路径或者文件名 + * @param envLabel 环境标签 + * @param workspace 导入工作空间 + + */ + void importProject(ProjectInfoVo projectInfo, BmlResource importResource, String username, + String checkCode, String packageInfo, EnvDSSLabel envLabel, Workspace workspace) throws Exception; + + BmlResource exportOnlyProjectMeta(ExportAllOrchestratorsReqest exportAllOrchestratorsReqest, + String username, String proxyUser, Workspace workspace) throws Exception; + + DSSProjectDO modifyProjectMeta(String username, ProjectModifyRequest projectModifyRequest) throws DSSProjectErrorException; + + + List queryListByParam(ProjectQueryRequest projectRequest,List totals); + + List getDSSProjectByName(List name); + + List queryProjectName(ProjectQueryRequest projectRequest); + + List queryProject(ProjectListQueryRequest projectListQueryRequest); } diff --git a/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/service/DSSProjectUserService.java b/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/service/DSSProjectUserService.java index b012ff17e0..beef42a04d 100644 --- a/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/service/DSSProjectUserService.java +++ b/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/service/DSSProjectUserService.java @@ -62,12 +62,10 @@ public interface DSSProjectUserService { * the db project * @param projectModifyRequest * the project modify request - * @param loginuser - * the loginuser * @throws Exception * the exception */ - void modifyProjectUser(DSSProjectDO dbProject, ProjectModifyRequest projectModifyRequest, String loginuser,Workspace workspace)throws Exception; + void modifyProjectUser(DSSProjectDO dbProject, ProjectModifyRequest projectModifyRequest); List getListByParam(Long workspaceId, String username); diff --git a/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/service/ExportService.java b/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/service/ExportService.java new file mode 100644 index 0000000000..4aaab4e2a7 --- /dev/null +++ b/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/service/ExportService.java @@ -0,0 +1,63 @@ +/* + * + * * Copyright 2019 WeBank + * * + * * Licensed under the Apache License, Version 2.0 (the "License"); + * * you may not use this file except in compliance with the License. + * * You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.framework.project.service; + +import com.webank.wedatasphere.dss.common.label.DSSLabel; +import com.webank.wedatasphere.dss.framework.project.entity.response.BatchExportResult; +import com.webank.wedatasphere.dss.framework.project.entity.response.ExportResult; +import com.webank.wedatasphere.dss.orchestrator.common.entity.OrchestratorInfo; +import com.webank.wedatasphere.dss.orchestrator.server.entity.vo.OrchestratorBaseInfo; +import com.webank.wedatasphere.dss.standard.app.sso.Workspace; +import org.apache.linkis.common.exception.ErrorException; + +import java.util.List; + +/** + * created by cooperyang on 2020/12/9 + * Description: 专门使用导出的 + */ +public interface ExportService { + + /** + * 导出一个编排到bml + * @param releaseUser 导出人 + * @param projectId 项目id + * @param projectName 项目名 + * @param dssLabel 标签 + * @param workspace 工作孔家 + * @return 编排的导出结果 + * @throws ErrorException + */ + ExportResult export(String releaseUser, Long projectId, OrchestratorInfo orchestrator, + String projectName, DSSLabel dssLabel, Workspace workspace) throws ErrorException; + + /** + * 批量导出编排,打包到一起生成一个bml文件 + * @param orchestrators 需要导出的编排们 + * @return 编排导出后的目录。目录下是一个个编排 + */ + String batchExport(String userName, Long projectId, List orchestrators, + String projectName, DSSLabel dssLabel, Workspace workspace) throws ErrorException; + + + + Long addVersionAfterPublish(String releaseUser, Long projectId, Long orchestratorId, Long orchestratorVersionId, + String projectName, Workspace workspace, DSSLabel dssLabel,String comment) throws ErrorException; + +} diff --git a/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/service/ImportService.java b/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/service/ImportService.java new file mode 100644 index 0000000000..f5f3381681 --- /dev/null +++ b/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/service/ImportService.java @@ -0,0 +1,57 @@ +/* + * + * * Copyright 2019 WeBank + * * + * * Licensed under the Apache License, Version 2.0 (the "License"); + * * you may not use this file except in compliance with the License. + * * You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.framework.project.service; + +import com.webank.wedatasphere.dss.common.label.DSSLabel; +import com.webank.wedatasphere.dss.framework.project.entity.OrchestratorBatchImportInfo; +import com.webank.wedatasphere.dss.common.entity.BmlResource; +import com.webank.wedatasphere.dss.framework.project.entity.po.OrchestratorImportInfo; +import com.webank.wedatasphere.dss.orchestrator.server.entity.vo.OrchestratorBaseInfo; +import com.webank.wedatasphere.dss.standard.app.sso.Workspace; +import org.apache.linkis.common.exception.ErrorException; + +import java.util.List; + +/** + * created by cooperyang on 2020/12/9 + * Description: 导入的操作都在此处进行 + * 导入需要携带 + * 1.工程需要的zip包，应该是是需要bml的resourceid和version就可以 + */ +public interface ImportService { + + + /** + * 通过发布的导入,只需要将resourceId和version给到nextLabel的OrcFramework + */ + OrchestratorImportInfo importOrc(String orchestratorName, String releaseUser, Long projectId, String projectName, + BmlResource bmlResource, DSSLabel dssLabel, String workspaceName, Workspace workspace) throws ErrorException; + + /** + * 批量导入编排 + * @param userName 导入人用户名 + * @param projectPath 导入的资源目录文件夹路径 + * @param checkCode 校验码 + * @param dssLabel 标签 + * @param workspace 导入的工作空间 + * @return 导入的编排信息 + */ + OrchestratorBatchImportInfo batchImportOrc(String userName, Long projectId, String projectName, String projectPath, + String checkCode, DSSLabel dssLabel, Workspace workspace)throws ErrorException; +} diff --git a/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/service/ProjectHttpRequestHook.java b/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/service/ProjectHttpRequestHook.java index 2b1572971d..26693507eb 100644 --- a/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/service/ProjectHttpRequestHook.java +++ b/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/service/ProjectHttpRequestHook.java @@ -1,9 +1,6 @@ package com.webank.wedatasphere.dss.framework.project.service; -import com.webank.wedatasphere.dss.framework.project.entity.request.ProjectCreateRequest; -import com.webank.wedatasphere.dss.framework.project.entity.request.ProjectDeleteRequest; -import com.webank.wedatasphere.dss.framework.project.entity.request.ProjectModifyRequest; -import com.webank.wedatasphere.dss.framework.project.entity.request.ProjectQueryRequest; +import com.webank.wedatasphere.dss.framework.project.entity.request.*; import com.webank.wedatasphere.dss.framework.project.entity.vo.DSSProjectVo; import org.apache.linkis.server.Message; @@ -81,4 +78,5 @@ default void afterDeleteProject(HttpServletRequest request, ProjectDeleteRequest * @return 前置操作失败，请返回 Message，否则返回 null */ Message beforeGetDeletedProject(HttpServletRequest request, ProjectQueryRequest projectRequest); + } diff --git a/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/service/impl/DSSFrameworkProjectServiceImpl.java b/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/service/impl/DSSFrameworkProjectServiceImpl.java index d9b0ab3b1c..557ea5deec 100644 --- a/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/service/impl/DSSFrameworkProjectServiceImpl.java +++ b/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/service/impl/DSSFrameworkProjectServiceImpl.java @@ -16,37 +16,57 @@ package com.webank.wedatasphere.dss.framework.project.service.impl; +import com.baomidou.mybatisplus.core.conditions.update.UpdateWrapper; import com.webank.wedatasphere.dss.appconn.core.AppConn; import com.webank.wedatasphere.dss.appconn.core.ext.OnlyStructureAppConn; +import com.webank.wedatasphere.dss.common.auditlog.OperateTypeEnum; +import com.webank.wedatasphere.dss.common.auditlog.TargetTypeEnum; +import com.webank.wedatasphere.dss.common.entity.BmlResource; import com.webank.wedatasphere.dss.common.entity.project.DSSProject; -import com.webank.wedatasphere.dss.common.utils.DSSExceptionUtils; +import com.webank.wedatasphere.dss.common.utils.*; +import com.webank.wedatasphere.dss.framework.admin.service.DssAdminUserService; import com.webank.wedatasphere.dss.framework.project.conf.ProjectConf; import com.webank.wedatasphere.dss.framework.project.contant.ProjectServerResponse; import com.webank.wedatasphere.dss.framework.project.entity.DSSProjectDO; import com.webank.wedatasphere.dss.framework.project.entity.DSSProjectUser; +import com.webank.wedatasphere.dss.framework.project.entity.request.ExportAllOrchestratorsReqest; import com.webank.wedatasphere.dss.framework.project.entity.request.ProjectCreateRequest; import com.webank.wedatasphere.dss.framework.project.entity.request.ProjectModifyRequest; +import com.webank.wedatasphere.dss.framework.project.entity.request.ProjectTransferRequest; import com.webank.wedatasphere.dss.framework.project.entity.vo.DSSProjectDetailVo; import com.webank.wedatasphere.dss.framework.project.entity.vo.DSSProjectVo; import com.webank.wedatasphere.dss.framework.project.exception.DSSProjectErrorException; import com.webank.wedatasphere.dss.framework.project.service.DSSFrameworkProjectService; import com.webank.wedatasphere.dss.framework.project.service.DSSProjectService; import com.webank.wedatasphere.dss.framework.project.service.DSSProjectUserService; +import com.webank.wedatasphere.dss.framework.project.utils.ProjectStringUtils; +import com.webank.wedatasphere.dss.framework.workspace.bean.vo.StaffInfoVO; +import com.webank.wedatasphere.dss.framework.workspace.service.DSSWorkspaceUserService; +import com.webank.wedatasphere.dss.git.common.protocol.exception.GitErrorException; +import com.webank.wedatasphere.dss.git.common.protocol.request.*; +import com.webank.wedatasphere.dss.git.common.protocol.response.*; +import com.webank.wedatasphere.dss.orchestrator.common.entity.DSSOrchestratorInfo; +import com.webank.wedatasphere.dss.orchestrator.common.ref.OrchestratorRefConstant; +import com.webank.wedatasphere.dss.orchestrator.db.dao.OrchestratorMapper; +import com.webank.wedatasphere.dss.orchestrator.server.entity.request.OrchestratorRequest; +import com.webank.wedatasphere.dss.orchestrator.server.entity.vo.OrchestratorBaseInfo; +import com.webank.wedatasphere.dss.orchestrator.server.service.OrchestratorService; +import com.webank.wedatasphere.dss.sender.service.DSSSenderServiceFactory; import com.webank.wedatasphere.dss.standard.app.sso.Workspace; import com.webank.wedatasphere.dss.standard.app.structure.project.*; -import com.webank.wedatasphere.dss.standard.app.structure.project.ref.DSSProjectContentRequestRef; -import com.webank.wedatasphere.dss.standard.app.structure.project.ref.DSSProjectPrivilege; -import com.webank.wedatasphere.dss.standard.app.structure.project.ref.ProjectUpdateRequestRef; -import com.webank.wedatasphere.dss.standard.app.structure.project.ref.RefProjectContentRequestRef; +import com.webank.wedatasphere.dss.standard.app.structure.project.ref.*; import com.webank.wedatasphere.dss.standard.app.structure.utils.StructureOperationUtils; import com.webank.wedatasphere.dss.standard.common.desc.AppInstance; import com.webank.wedatasphere.dss.standard.common.exception.operation.ExternalOperationFailedException; +import com.webank.wedatasphere.dss.framework.project.dao.DSSProjectMapper; +import com.webank.wedatasphere.dss.workflow.dao.LockMapper; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.exception.ExceptionUtils; import org.apache.linkis.common.conf.CommonVars; import org.apache.linkis.common.exception.WarnException; import org.apache.linkis.protocol.util.ImmutablePair; +import org.apache.linkis.rpc.Sender; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.BeanUtils; @@ -69,6 +89,18 @@ public class DSSFrameworkProjectServiceImpl implements DSSFrameworkProjectServic private DSSProjectService dssProjectService; @Autowired private DSSProjectUserService projectUserService; + @Autowired + private DSSProjectMapper projectMapper; + @Autowired + private DssAdminUserService dssUserService; + @Autowired + private DSSWorkspaceUserService dssWorkspaceUserService; + @Autowired + private OrchestratorService orchestratorService; + @Autowired + private LockMapper lockMapper; + @Autowired + private OrchestratorMapper orchestratorMapper; private static final boolean STRICT_PROJECT_CREATE_MODE = CommonVars.apply("wds.dss.project.strict.mode", false).getValue(); @@ -96,15 +128,32 @@ public DSSProjectVo createProject(ProjectCreateRequest projectCreateRequest, Str } else if(StringUtils.isNotEmpty(projectCreateRequest.getBusiness()) && projectCreateRequest.getBusiness().length() > MAX_PROJECT_BUSSINESS_SIZE){ DSSExceptionUtils.dealErrorException(60021,"project bussiness is too long. the length must be less than " + MAX_PROJECT_BUSSINESS_SIZE, DSSProjectErrorException.class); } + //校验项目名合法性 + checkProjectName(projectCreateRequest.getName(), workspace, username); + // 对于接入git的项目校验项目名称 + if(projectCreateRequest.getAssociateGit() != null && projectCreateRequest.getAssociateGit()) { + // 校验gitUser gitToken合法性以及projectName是否重复 + boolean repeat = checkGitName(projectCreateRequest.getName(), workspace, username, projectCreateRequest.getGitUser(), projectCreateRequest.getGitToken()); + if (repeat) { + throw new DSSProjectErrorException(71000, "git中存在同名项目，请更换名字或删去同名项目再重试"); + } + } - this.checkProjectName(projectCreateRequest.getName(), workspace, username); - - Map projectMap = createAppConnProject(projectCreateRequest, workspace, username); //3.保存dss_project DSSProjectDO project = dssProjectService.createProject(username, projectCreateRequest); + // 同步项目信息至git + if(projectCreateRequest.getAssociateGit() != null && projectCreateRequest.getAssociateGit()) { + ExportAllOrchestratorsReqest exportAllOrchestratorsReqest = new ExportAllOrchestratorsReqest(); + exportAllOrchestratorsReqest.setProjectId(project.getId()); + exportAllOrchestratorsReqest.setComment("create Project:" + projectCreateRequest.getName()); + exportAllOrchestratorsReqest.setLabels(DSSCommonUtils.ENV_LABEL_VALUE_DEV); + BmlResource bmlResource = dssProjectService.exportProject(exportAllOrchestratorsReqest, username, "", workspace); + createGitProject(workspace.getWorkspaceId(), project.getName(), bmlResource, username, projectCreateRequest.getGitUser(), projectCreateRequest.getGitToken()); + } //4.保存dss_project_user 工程与用户关系 projectUserService.saveProjectUser(project.getId(), username, projectCreateRequest, workspace); //5.保存dss工程与其他工程的对应关系,应该都是以id来作为标识 + Map projectMap = createAppConnProject(projectCreateRequest, workspace, username); if (projectMap.size() > 0) { dssProjectService.saveProjectRelation(project, projectMap); } @@ -115,6 +164,15 @@ public DSSProjectVo createProject(ProjectCreateRequest projectCreateRequest, Str return dssProjectVo; } + private void createGitProject(Long workspaceId, String projectName, BmlResource bmlResource, String username, String gitUser, String gitToken) { + Sender gitSender = DSSSenderServiceFactory.getOrCreateServiceInstance().getGitSender(); + GitCreateProjectRequest request1 = new GitCreateProjectRequest(workspaceId, projectName, bmlResource, username, gitUser, gitToken); + LOGGER.info("-------=======================begin to create project: {}=======================-------", projectName); + Object ask = gitSender.ask(request1); + GitCreateProjectResponse responseWorkflowValidNode = RpcAskUtils.processAskException(ask, GitCreateProjectResponse.class, GitCreateProjectResponse.class); + LOGGER.info("-------=======================End to create project: {}=======================-------: {}", projectName, responseWorkflowValidNode); + } + @Override public void checkProjectName(String name, Workspace workspace, String username) throws DSSProjectErrorException { @@ -141,39 +199,83 @@ public void checkProjectName(String name, Workspace workspace, String username) @Override public void modifyProject(ProjectModifyRequest projectModifyRequest, DSSProjectDO dbProject, String username, Workspace workspace) throws Exception { - //如果不是工程的创建人，则校验是否管理员 - if (!username.equalsIgnoreCase(dbProject.getCreateBy())) { - boolean isAdmin = projectUserService.isAdminByUsername(projectModifyRequest.getWorkspaceId(), username); - //非管理员 - if (!isAdmin) { - DSSExceptionUtils.dealErrorException(ProjectServerResponse.PROJECT_IS_NOT_ADMIN.getCode(), ProjectServerResponse.PROJECT_IS_NOT_ADMIN.getMsg(), DSSProjectErrorException.class); - } + Boolean dbProjectAssociateGit = dbProject.getAssociateGit(); + // 校验参数正确性 + checkProjectRight(projectModifyRequest, dbProject, username, workspace); + // 对于首次接入git的项目需要校验项目名称 + boolean repeat = false; + Boolean requestAssociateGit = projectModifyRequest.getAssociateGit(); + if ((dbProjectAssociateGit == null || !dbProjectAssociateGit) && requestAssociateGit != null && requestAssociateGit) { + // 校验gitUser gitToken合法性以及projectName是否重复 + repeat = checkGitName(projectModifyRequest.getName(), workspace, username, projectModifyRequest.getGitUser(), projectModifyRequest.getGitToken()); } - //不允许修改工程名称 - if (!dbProject.getName().equalsIgnoreCase(projectModifyRequest.getName())) { - DSSExceptionUtils.dealErrorException(ProjectServerResponse.PROJECT_NOT_EDIT_NAME.getCode(), ProjectServerResponse.PROJECT_NOT_EDIT_NAME.getMsg(), DSSProjectErrorException.class); + if ((requestAssociateGit == null || !requestAssociateGit) && (dbProjectAssociateGit != null && dbProjectAssociateGit)) { + // 不接入 清空编排bml缓存 + OrchestratorRequest orchestratorRequest = new OrchestratorRequest(workspace.getWorkspaceId(), dbProject.getId()); + List orchestratorInfos = orchestratorService.getOrchestratorInfos(orchestratorRequest, username); + for (OrchestratorBaseInfo baseInfo : orchestratorInfos) { + orchestratorMapper.updateOrchestratorBmlVersion(baseInfo.getId(), null, null); + } + // 清空工作流状态 + List orchestratorInfoByLabel = orchestratorService.getOrchestratorInfoByLabel(orchestratorRequest); + if (CollectionUtils.isNotEmpty(orchestratorInfoByLabel)) { + List orchestratorIdList = orchestratorInfoByLabel.stream().map(DSSOrchestratorInfo::getId).collect(Collectors.toList()); + lockMapper.batchUpdateOrchestratorStatus(orchestratorIdList, null); + } } + + // 1.统一修改各个接入的第三方的系统的工程状态信息 //调用第三方的工程修改接口 dbProject.setUsername(username); dbProject.setApplicationArea(Integer.valueOf(projectModifyRequest.getApplicationArea())); dbProject.setDescription(projectModifyRequest.getDescription()); dbProject.setBusiness(projectModifyRequest.getBusiness()); dbProject.setProduct(projectModifyRequest.getProduct()); - modifyThirdProject(projectModifyRequest, dbProject, workspace); + modifyThirdProject(projectModifyRequest, dbProject, workspace, dbProject.getCreateBy()); - //1.统一修改各个接入的第三方的系统的工程状态信息 //2.修改dss_project_user 工程与用户关系 - projectUserService.modifyProjectUser(dbProject, projectModifyRequest, username, workspace); + projectUserService.modifyProjectUser(dbProject, projectModifyRequest); //3.修改dss_project DSS基本工程信息 - dssProjectService.modifyProject(username, projectModifyRequest); + DSSProjectDO dssProjectDO = dssProjectService.modifyProject(username, projectModifyRequest); + + // 同步项目配置元数据到git + try { + syncGitProject(projectModifyRequest, dbProject, username, workspace, repeat); + }catch (Exception e) { + DSSProjectDO project = new DSSProjectDO(); + project.setAssociateGit(dbProjectAssociateGit); + UpdateWrapper updateWrapper = new UpdateWrapper<>(); + updateWrapper.eq("id", projectModifyRequest.getId()); + updateWrapper.eq("workspace_id", projectModifyRequest.getWorkspaceId()); + projectMapper.update(project, updateWrapper); + throw new DSSProjectErrorException(71000, "修改项目接入Git失败，原因为：" + e.getMessage()); + } } + private void updateProject(Long workspaceId, String projectName,BmlResource bmlResource, String username, String gitUser, String gitToken) { + Sender gitSender = DSSSenderServiceFactory.getOrCreateServiceInstance().getGitSender(); + Map file = new HashMap<>(); + // 测试数据 key表示项目名、value为项目BmlResource资源 + file.put(".projectmeta", bmlResource); + GitCommitRequest request1 = new GitCommitRequest(workspaceId, projectName, file, "update Project", username, gitUser, gitToken); + LOGGER.info("-------=======================begin to update project: {}=======================-------", projectName); + GitCommitResponse responseWorkflowValidNode = RpcAskUtils.processAskException(gitSender.ask(request1), GitCommitResponse.class, GitCommitRequest.class); + LOGGER.info("-------=======================End to update project: {}=======================-------: {}", projectName, responseWorkflowValidNode); + } + /** + * 统一修改各个接入的第三方的系统的工程状态信息，修改 dss_project 调用。 + */ + private void modifyThirdProject(ProjectModifyRequest projectModifyRequest, + DSSProjectDO dbProject, Workspace workspace, String username){ + modifyThirdProject(projectModifyRequest, dbProject, workspace, username, null); + } /** * 统一修改各个接入的第三方的系统的工程状态信息，修改 dss_project 调用。 */ private void modifyThirdProject(ProjectModifyRequest projectModifyRequest, - DSSProjectDO dbProject, Workspace workspace){ + DSSProjectDO dbProject, Workspace workspace, String username, + String newProjectOwner){ DSSProject dssProject = new DSSProject(); BeanUtils.copyProperties(dbProject, dssProject); DSSProjectPrivilege privilege = DSSProjectPrivilege.newBuilder().setAccessUsers(projectModifyRequest.getAccessUsers()) @@ -200,6 +302,7 @@ private void modifyThirdProject(ProjectModifyRequest projectModifyRequest, addedPrivilege = getPrivilege.apply(true); removedPrivilege = getPrivilege.apply(false); } + List dataSourceList = projectModifyRequest.getDataSourceList(); Map appInstanceToRefProjectId = new HashMap<>(10); tryProjectOperation((appConn, appInstance) -> { Long refProjectId = dssProjectService.getAppConnProjectId(appInstance.getId(), dbProject.getId()); @@ -212,11 +315,11 @@ private void modifyThirdProject(ProjectModifyRequest projectModifyRequest, return true; } }, workspace, projectService -> projectService.getProjectUpdateOperation(), - dssProjectContentRequestRef -> dssProjectContentRequestRef.setDSSProject(dssProject).setDSSProjectPrivilege(privilege).setUserName(dbProject.getUpdateBy()).setWorkspace(workspace), + dssProjectContentRequestRef -> dssProjectContentRequestRef.setDSSProject(dssProject).setDSSProjectPrivilege(privilege).setDSSProjectDataSources(dataSourceList).setUserName(username).setWorkspace(workspace), (appInstance, refProjectContentRequestRef) -> refProjectContentRequestRef.setRefProjectId(appInstanceToRefProjectId.get(appInstance)), (structureOperation, structureRequestRef) -> { ProjectUpdateRequestRef projectUpdateRequestRef = (ProjectUpdateRequestRef) structureRequestRef; - projectUpdateRequestRef.setAddedDSSProjectPrivilege(addedPrivilege).setRemovedDSSProjectPrivilege(removedPrivilege); + projectUpdateRequestRef.setAddedDSSProjectPrivilege(addedPrivilege).setRemovedDSSProjectPrivilege(removedPrivilege).setNewOwner(newProjectOwner); return ((ProjectUpdateOperation) structureOperation).updateProject(projectUpdateRequestRef); }, null, "update refProject " + projectModifyRequest.getName()); } @@ -252,10 +355,11 @@ private Map createAppConnProject(ProjectCreateRequest dssProj DSSProjectPrivilege privilege = DSSProjectPrivilege.newBuilder().setAccessUsers(dssProjectCreateRequest.getAccessUsers()) .setEditUsers(dssProjectCreateRequest.getEditUsers()) .setReleaseUsers(dssProjectCreateRequest.getReleaseUsers()).build(); + List dataSourceList = dssProjectCreateRequest.getDataSourceList(); try { tryProjectOperation(null, workspace, ProjectService::getProjectCreationOperation, dssProjectContentRequestRef -> dssProjectContentRequestRef.setDSSProject(dssProject) - .setDSSProjectPrivilege(privilege).setUserName(username), null, + .setDSSProjectPrivilege(privilege).setDSSProjectDataSources(dataSourceList).setUserName(username), null, (structureOperation, structureRequestRef) -> ((ProjectCreationOperation) structureOperation).createProject((DSSProjectContentRequestRef) structureRequestRef), (pair, projectResponseRef) -> { projectMap.put(pair.right, projectResponseRef.getRefProjectId()); @@ -298,4 +402,204 @@ private Map createAppConnProject(ProjectCreateRequest dssProj return projectMap; } + private boolean checkGitName(String name, Workspace workspace, String username, String gitUser, String gitToken) throws DSSProjectErrorException { + // 校验是否为虚拟用户，此处禁止使用DSS实名用户接入Git + List all = dssWorkspaceUserService.listAllDSSUsers(); + Set allDSSUsers = all.stream().map(StaffInfoVO::getUsername).collect(Collectors.toSet()); + if (allDSSUsers.contains(gitUser)) { + throw new DSSProjectErrorException(71000, "Git读写账号用户名必须为虚拟用户，不能是DSS实名用户！"); + } + // 校验Git名称 + Sender gitSender = DSSSenderServiceFactory.getOrCreateServiceInstance().getGitSender(); + GitCheckProjectRequest request1 = new GitCheckProjectRequest(workspace.getWorkspaceId(), name, username, gitUser, gitToken); + LOGGER.info("-------=======================begin to check project: {}=======================-------", name); + Object ask = gitSender.ask(request1); + GitCheckProjectResponse responseWorkflowValidNode = RpcAskUtils.processAskException(ask, GitCheckProjectResponse.class, GitCheckProjectRequest.class); + LOGGER.info("-------=======================End to check project: {}=======================-------: {}", name, responseWorkflowValidNode); + if (responseWorkflowValidNode == null) { + throw new DSSProjectErrorException(71000, "向Git发起检查工程名是否重复失败，请稍后重试 "); + } else if (responseWorkflowValidNode.getRepeat()) { + LOGGER.info("项目{}重新接入git", name); + return true; + } + return false; + } + + @Override + public void modifyProjectMeta(ProjectModifyRequest projectModifyRequest, DSSProjectDO dbProject, String username, Workspace workspace) throws Exception { + // 校验项目参数正确性 + checkProjectRight(projectModifyRequest, dbProject, username, workspace); + // 对于首次接入git的项目需要校验项目名称 + boolean repeat = false; + if ((dbProject.getAssociateGit() == null || !dbProject.getAssociateGit()) && projectModifyRequest.getAssociateGit()!= null && projectModifyRequest.getAssociateGit()) { + // 校验gitUser gitToken合法性以及projectName是否重复 + repeat = checkGitName(projectModifyRequest.getName(), workspace, username, projectModifyRequest.getGitUser(), projectModifyRequest.getGitToken()); + } + + // 1.统一修改各个接入的第三方的系统的工程状态信息 + //调用第三方的工程修改接口 + dbProject.setDescription(projectModifyRequest.getDescription()); + dbProject.setUsername(username); + modifyThirdProject(projectModifyRequest, dbProject, workspace, dbProject.getCreateBy()); + + //2.修改dss_project_user 工程与用户关系 + projectUserService.modifyProjectUser(dbProject, projectModifyRequest); + + //3.修改dss_project DSS基本工程信息 + dssProjectService.modifyProjectMeta(username, projectModifyRequest); + + // 同步项目配置元数据到git + syncGitProject(projectModifyRequest, dbProject, username, workspace, repeat); + + } + + + public void checkProjectRight(ProjectModifyRequest projectModifyRequest, DSSProjectDO dbProject, String username, Workspace workspace) throws Exception { + + //如果不是工程的创建人，则校验是否管理员或发布者权限 + if (!username.equalsIgnoreCase(dbProject.getCreateBy())) { + //获取发布者权限用户 + List projectUsers = projectUserService.getProjectPriv(projectModifyRequest.getId()).stream() + .filter(projectUser -> projectUser.getPriv() == 3).map(DSSProjectUser::getUsername).collect(Collectors.toList()); + boolean isAdmin = projectUserService.isAdminByUsername(projectModifyRequest.getWorkspaceId(), username); + //非管理员非发布者权限 + if (!isAdmin && !projectUsers.contains(username)) { + DSSExceptionUtils.dealErrorException(ProjectServerResponse.PROJECT_IS_NOT_ADMIN_OR_RELEASE.getCode(), ProjectServerResponse.PROJECT_IS_NOT_ADMIN_OR_RELEASE.getMsg(), DSSProjectErrorException.class); + } + } + + //不允许修改工程名称 + if (!dbProject.getName().equalsIgnoreCase(projectModifyRequest.getName())) { + DSSExceptionUtils.dealErrorException(ProjectServerResponse.PROJECT_NOT_EDIT_NAME.getCode(), ProjectServerResponse.PROJECT_NOT_EDIT_NAME.getMsg(), DSSProjectErrorException.class); + } + + } + + public void syncGitProject(ProjectModifyRequest projectModifyRequest, DSSProjectDO dbProject, String username, Workspace workspace, boolean repeat) throws Exception{ + + // 同步项目配置元数据到git + if (projectModifyRequest.getAssociateGit() != null && projectModifyRequest.getAssociateGit()) { + ExportAllOrchestratorsReqest exportAllOrchestratorsReqest = new ExportAllOrchestratorsReqest(); + exportAllOrchestratorsReqest.setProjectId(dbProject.getId()); + exportAllOrchestratorsReqest.setComment("modify Project"); + exportAllOrchestratorsReqest.setLabels(DSSCommonUtils.ENV_LABEL_VALUE_DEV); + //如果不是工程的创建人，则校验是否管理员或发布者权限 + String uploadUserName = username; + if (!username.equalsIgnoreCase(dbProject.getCreateBy())) { + //获取发布者权限用户 + List projectUsers = projectUserService.getProjectPriv(projectModifyRequest.getId()).stream() + .filter(projectUser -> projectUser.getPriv() == 3).map(DSSProjectUser::getUsername).collect(Collectors.toList()); + boolean isAdmin = projectUserService.isAdminByUsername(projectModifyRequest.getWorkspaceId(), username); + // 管理员非发布者权限 + if (!projectUsers.contains(username) && isAdmin) { + uploadUserName = dbProject.getCreateBy(); + } + } + BmlResource bmlResource = dssProjectService.exportOnlyProjectMeta(exportAllOrchestratorsReqest, uploadUserName, "", workspace); + if (!repeat && (dbProject.getAssociateGit() == null || !dbProject.getAssociateGit()) && projectModifyRequest.getAssociateGit() != null && projectModifyRequest.getAssociateGit()) { + createGitProject(workspace.getWorkspaceId(), dbProject.getName(), bmlResource, username, projectModifyRequest.getGitUser(), projectModifyRequest.getGitToken()); + OrchestratorRequest orchestratorRequest = new OrchestratorRequest(workspace.getWorkspaceId(), projectModifyRequest.getId()); + List orchestratorInfoByLabel = orchestratorService.getOrchestratorInfoByLabel(orchestratorRequest); + if (CollectionUtils.isNotEmpty(orchestratorInfoByLabel)) { + List orchestratorIdList = orchestratorInfoByLabel.stream().map(DSSOrchestratorInfo::getId).collect(Collectors.toList()); + lockMapper.batchUpdateOrchestratorStatus(orchestratorIdList, OrchestratorRefConstant.FLOW_STATUS_SAVE); + } + + } else { + updateProject(workspace.getWorkspaceId(), dbProject.getName(), bmlResource, username, projectModifyRequest.getGitUser(), projectModifyRequest.getGitToken()); + } + // 如果重复，则表示之前接入过git，重新接入则需要更新所有编排状态 + if (repeat) { + OrchestratorRequest orchestratorRequest = new OrchestratorRequest(workspace.getWorkspaceId(), projectModifyRequest.getId()); + List orchestratorInfoByLabel = orchestratorService.getOrchestratorInfoByLabel(orchestratorRequest); + if (CollectionUtils.isNotEmpty(orchestratorInfoByLabel)) { + List orchestratorIdList = orchestratorInfoByLabel.stream().map(DSSOrchestratorInfo::getId).collect(Collectors.toList()); + lockMapper.batchUpdateOrchestratorStatus(orchestratorIdList, OrchestratorRefConstant.FLOW_STATUS_SAVE); + } + } + } + } + + @Override + @Transactional + public void transferProject(ProjectTransferRequest projectTransferRequest, DSSProjectDO dbProject, + String oldProjectOwner, + Workspace workspace,String operator){ + ProjectModifyRequest projectModifyRequest = new ProjectModifyRequest(); + + BeanUtils.copyProperties(dbProject, projectModifyRequest); + if (dbProject.getApplicationArea() != null) { + projectModifyRequest.setApplicationArea(dbProject.getApplicationArea().toString()); + } + if (StringUtils.isNotBlank(dbProject.getDevProcess())) { + projectModifyRequest.setDevProcessList(ProjectStringUtils.convertList(dbProject.getDevProcess())); + } + projectModifyRequest.setOrchestratorModeList(ProjectStringUtils.convertList(dbProject.getOrchestratorMode())); + if (StringUtils.isNotEmpty(dbProject.getLabel())) { + projectModifyRequest.setCreatorLabel(dbProject.getLabel()); + } + //项目交接，清空数据源 +// dbProject.setDataSourceListJson(null); + initProjectModifyRequestPermissionInfo(projectModifyRequest,projectTransferRequest, oldProjectOwner); + + String newProjectOwner = projectTransferRequest.getTransferUserName(); + Long count = dssWorkspaceUserService.getCountByUsername(newProjectOwner, (int)workspace.getWorkspaceId()); + if (count == null || count == 0) { + dssUserService.insertIfNotExist(newProjectOwner, workspace); + List roles = Collections.singletonList(4); + dssWorkspaceUserService.addWorkspaceUser(roles, workspace.getWorkspaceId(), newProjectOwner, "system", + null); + } + // 1.统一修改各个接入的第三方的系统的工程状态信息 + //调用第三方的工程修改接口 + modifyThirdProject(projectModifyRequest, dbProject, workspace, oldProjectOwner,newProjectOwner); + + //2.修改dss_project_user 工程与用户关系 + projectUserService.modifyProjectUser(dbProject, projectModifyRequest); + + dbProject.setCreateBy(newProjectOwner); + dbProject.setUsername(newProjectOwner); + + //3.修改dss_project owner信息 + updateProject4Transfer(dbProject, operator); + //4.修改git权限 + try { + syncGitProject(projectModifyRequest, dbProject, newProjectOwner, workspace, true); + }catch (Exception e){ + LOGGER.error("transfer git failed",e); + } + } + + private void updateProject4Transfer(DSSProjectDO dbProject,String operator){ + dbProject.setUpdateByStr(operator); + dbProject.setUpdateTime(new Date()); + UpdateWrapper updateWrapper = new UpdateWrapper<>(); + updateWrapper.eq("id", dbProject.getId()); + updateWrapper.eq("workspace_id", dbProject.getWorkspaceId()); + projectMapper.update(dbProject, updateWrapper); + } + + private void initProjectModifyRequestPermissionInfo(ProjectModifyRequest projectModifyRequest, ProjectTransferRequest projectTransferRequest, String oldProjectOwner){ + List projectPriv = projectUserService.getProjectPriv(projectModifyRequest.getId()); + List releaseUsers = projectPriv.stream().filter(projectUser -> projectUser.getPriv() == 3).map(DSSProjectUser::getUsername).collect(Collectors.toList()); + if(!releaseUsers.contains(projectTransferRequest.getTransferUserName())){ + releaseUsers.add(projectTransferRequest.getTransferUserName()); + } + releaseUsers.removeIf(oldProjectOwner::equals); + projectModifyRequest.setReleaseUsers(releaseUsers); + + List editUsers = projectPriv.stream().filter(projectUser -> projectUser.getPriv() == 2).map(DSSProjectUser::getUsername).collect(Collectors.toList()); + if(!editUsers.contains(projectTransferRequest.getTransferUserName())){ + editUsers.add(projectTransferRequest.getTransferUserName()); + } + editUsers.removeIf(oldProjectOwner::equals); + projectModifyRequest.setEditUsers(editUsers); + List accessUsers = projectPriv.stream().filter(projectUser -> projectUser.getPriv() == 1).map(DSSProjectUser::getUsername).collect(Collectors.toList()); + if(!accessUsers.contains(projectTransferRequest.getTransferUserName())){ + accessUsers.add(projectTransferRequest.getTransferUserName()); + } + accessUsers.removeIf(oldProjectOwner::equals); + projectModifyRequest.setAccessUsers(accessUsers); + } + } diff --git a/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/service/impl/DSSProjectServiceImpl.java b/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/service/impl/DSSProjectServiceImpl.java index 804712fb83..de1e8f387a 100644 --- a/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/service/impl/DSSProjectServiceImpl.java +++ b/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/service/impl/DSSProjectServiceImpl.java @@ -19,41 +19,78 @@ import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; import com.baomidou.mybatisplus.core.conditions.update.UpdateWrapper; import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl; +import com.github.pagehelper.PageHelper; +import com.github.pagehelper.PageInfo; +import com.google.common.util.concurrent.ThreadFactoryBuilder; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.reflect.TypeToken; import com.webank.wedatasphere.dss.appconn.core.AppConn; import com.webank.wedatasphere.dss.appconn.manager.AppConnManager; +import com.webank.wedatasphere.dss.common.constant.project.ProjectUserPrivEnum; +import com.webank.wedatasphere.dss.common.entity.BmlResource; +import com.webank.wedatasphere.dss.common.entity.project.DSSProject; import com.webank.wedatasphere.dss.common.exception.DSSErrorException; +import com.webank.wedatasphere.dss.common.exception.DSSRuntimeException; import com.webank.wedatasphere.dss.common.label.DSSLabel; +import com.webank.wedatasphere.dss.common.label.EnvDSSLabel; +import com.webank.wedatasphere.dss.common.label.LabelRouteVO; +import com.webank.wedatasphere.dss.common.protocol.project.ProjectInfoListRequest; +import com.webank.wedatasphere.dss.common.protocol.project.ProjectListQueryRequest; +import com.webank.wedatasphere.dss.common.service.BMLService; import com.webank.wedatasphere.dss.common.utils.DSSExceptionUtils; +import com.webank.wedatasphere.dss.common.utils.IoUtils; +import com.webank.wedatasphere.dss.common.utils.RpcAskUtils; +import com.webank.wedatasphere.dss.common.utils.ZipHelper; import com.webank.wedatasphere.dss.framework.project.conf.ProjectConf; import com.webank.wedatasphere.dss.framework.project.contant.ProjectServerResponse; -import com.webank.wedatasphere.dss.common.constant.project.ProjectUserPrivEnum; import com.webank.wedatasphere.dss.framework.project.dao.DSSProjectMapper; import com.webank.wedatasphere.dss.framework.project.dao.DSSProjectUserMapper; import com.webank.wedatasphere.dss.framework.project.entity.DSSProjectDO; import com.webank.wedatasphere.dss.framework.project.entity.po.ProjectRelationPo; -import com.webank.wedatasphere.dss.framework.project.entity.request.ProjectCreateRequest; -import com.webank.wedatasphere.dss.framework.project.entity.request.ProjectDeleteRequest; -import com.webank.wedatasphere.dss.framework.project.entity.request.ProjectModifyRequest; -import com.webank.wedatasphere.dss.framework.project.entity.request.ProjectQueryRequest; +import com.webank.wedatasphere.dss.framework.project.entity.request.*; import com.webank.wedatasphere.dss.framework.project.entity.response.ProjectResponse; import com.webank.wedatasphere.dss.framework.project.entity.vo.ProjectInfoVo; import com.webank.wedatasphere.dss.framework.project.entity.vo.QueryProjectVo; import com.webank.wedatasphere.dss.framework.project.exception.DSSProjectErrorException; import com.webank.wedatasphere.dss.framework.project.service.DSSProjectService; -import com.webank.wedatasphere.dss.framework.project.service.DSSProjectUserService; +import com.webank.wedatasphere.dss.framework.project.service.ExportService; +import com.webank.wedatasphere.dss.framework.project.service.ImportService; import com.webank.wedatasphere.dss.framework.project.utils.ProjectStringUtils; +import com.webank.wedatasphere.dss.git.common.protocol.GitUserEntity; +import com.webank.wedatasphere.dss.git.common.protocol.request.GitArchiveProjectRequest; +import com.webank.wedatasphere.dss.git.common.protocol.request.GitUserByWorkspaceIdRequest; +import com.webank.wedatasphere.dss.git.common.protocol.response.GitArchivePorjectResponse; +import com.webank.wedatasphere.dss.git.common.protocol.response.GitUserByWorkspaceResponse; +import com.webank.wedatasphere.dss.orchestrator.server.entity.request.OrchestratorRequest; +import com.webank.wedatasphere.dss.orchestrator.server.entity.vo.OrchestratorBaseInfo; +import com.webank.wedatasphere.dss.orchestrator.server.service.OrchestratorService; +import com.webank.wedatasphere.dss.sender.service.DSSSenderServiceFactory; import com.webank.wedatasphere.dss.standard.app.sso.Workspace; import com.webank.wedatasphere.dss.standard.app.structure.project.ProjectDeletionOperation; import com.webank.wedatasphere.dss.standard.app.structure.project.ProjectService; +import com.webank.wedatasphere.dss.standard.app.structure.project.ref.DSSProjectDataSource; import com.webank.wedatasphere.dss.standard.app.structure.project.ref.RefProjectContentRequestRef; import com.webank.wedatasphere.dss.standard.common.desc.AppInstance; import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.io.FileUtils; import org.apache.commons.lang.StringUtils; +import org.apache.linkis.rpc.Sender; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.beans.BeanUtils; import org.springframework.beans.factory.annotation.Autowired; - +import org.springframework.beans.factory.annotation.Qualifier; + +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; import java.util.*; +import java.util.concurrent.*; import java.util.stream.Collectors; import static com.webank.wedatasphere.dss.framework.project.utils.ProjectOperationUtils.tryProjectOperation; @@ -63,13 +100,30 @@ public class DSSProjectServiceImpl extends ServiceImpl(1024), projectOperateThreadFactory, new ThreadPoolExecutor.AbortPolicy()); @Override public DSSProjectDO createProject(String username, ProjectCreateRequest projectCreateRequest) { @@ -84,16 +138,24 @@ public DSSProjectDO createProject(String username, ProjectCreateRequest projectC project.setUpdateTime(new Date()); project.setDescription(projectCreateRequest.getDescription()); project.setApplicationArea(projectCreateRequest.getApplicationArea()); + project.setAssociateGit(projectCreateRequest.getAssociateGit()); //开发流程，编排模式组拼接 前后进行英文逗号接口 project.setDevProcess(ProjectStringUtils.getModeStr(projectCreateRequest.getDevProcessList())); project.setOrchestratorMode(ProjectStringUtils.getModeStr(projectCreateRequest.getOrchestratorModeList())); + List dataSourceList = projectCreateRequest.getDataSourceList(); + if (dataSourceList != null && !dataSourceList.isEmpty()) { + project.setDataSourceListJson(new Gson().toJson(dataSourceList)); + } + if (projectCreateRequest.getCreatorLabel() != null) { + project.setLabel(projectCreateRequest.getCreatorLabel()); + } projectMapper.insert(project); return project; } //修改dss_project工程字段 @Override - public void modifyProject(String username, ProjectModifyRequest projectModifyRequest) throws DSSProjectErrorException { + public DSSProjectDO modifyProject(String username, ProjectModifyRequest projectModifyRequest) throws DSSProjectErrorException { //校验当前登录用户是否含有修改权限 // projectUserService.isEditProjectAuth(projectModifyRequest.getId(), username); DSSProjectDO project = new DSSProjectDO(); @@ -108,11 +170,22 @@ public void modifyProject(String username, ProjectModifyRequest projectModifyReq project.setOrchestratorMode(ProjectStringUtils.getModeStr(projectModifyRequest.getOrchestratorModeList())); project.setBusiness(projectModifyRequest.getBusiness()); project.setProduct(projectModifyRequest.getProduct()); + project.setAssociateGit(projectModifyRequest.getAssociateGit()); + List dataSourceList = projectModifyRequest.getDataSourceList(); + if (dataSourceList != null && !dataSourceList.isEmpty()) { + project.setDataSourceListJson(new Gson().toJson(dataSourceList)); + } + + if(StringUtils.isNotEmpty(projectModifyRequest.getCreatorLabel())){ + project.setLabel(projectModifyRequest.getCreatorLabel()); + } UpdateWrapper updateWrapper = new UpdateWrapper<>(); updateWrapper.eq("id", projectModifyRequest.getId()); updateWrapper.eq("workspace_id", projectModifyRequest.getWorkspaceId()); projectMapper.update(project, updateWrapper); + + return project; } /** @@ -153,12 +226,9 @@ public List getListByParam(ProjectQueryRequest projectRequest) List list; //判断工作空间是否设置了管理员能否查看该工作空间下所有项目的权限 Integer workspaceAdminPermission = projectUserMapper.getWorkspaceAdminPermission(projectRequest.getWorkspaceId()); - if (isWorkspaceAdmin(projectRequest.getWorkspaceId(), projectRequest.getUsername())) { - if (workspaceAdminPermission == 1){ - list = projectMapper.getListForAdmin(projectRequest); - } else { - list = projectMapper.getListByParam(projectRequest); - } + boolean isAdmin = isWorkspaceAdmin(projectRequest.getWorkspaceId(), projectRequest.getUsername()); + if (isAdmin && workspaceAdminPermission == 1) { + list = projectMapper.getListForAdmin(projectRequest); } else { list = projectMapper.getListByParam(projectRequest); } @@ -166,6 +236,7 @@ public List getListByParam(ProjectQueryRequest projectRequest) return new ArrayList<>(); } + Map projectGitUserInfo = getProjectGitUserInfo(projectRequest.getUsername(), projectRequest.getWorkspaceId()); List projectResponseList = new ArrayList<>(); ProjectResponse projectResponse; for (QueryProjectVo projectVo : list) { @@ -184,9 +255,22 @@ public List getListByParam(ProjectQueryRequest projectRequest) projectResponse.setArchive(projectVo.getArchive()); projectResponse.setCreateTime(projectVo.getCreateTime()); projectResponse.setUpdateTime(projectVo.getUpdateTime()); + projectResponse.setAssociateGit(projectVo.getAssociateGit()); + if (projectVo.getAssociateGit()) { + GitUserEntity gitUserEntity = projectGitUserInfo.get(projectVo.getName()); + if (gitUserEntity != null) { + projectResponse.setGitUser(gitUserEntity.getGitUser()); + projectResponse.setGitToken(gitUserEntity.getGitToken()); + } + } projectResponse.setDevProcessList(ProjectStringUtils.convertList(projectVo.getDevProcess())); projectResponse.setOrchestratorModeList(ProjectStringUtils.convertList(projectVo.getOrchestratorMode())); - projectResponseList.add(projectResponse); + if (projectVo.getDataSourceListJson() != null) { + List dataSourceList = new Gson().fromJson(projectVo.getDataSourceListJson(), + new TypeToken>() { + }.getType()); + projectResponse.setDataSourceList(dataSourceList); + } String pusername = projectVo.getPusername(); String editPriv = projectVo.getId() + KEY_SPLIT + ProjectUserPrivEnum.PRIV_EDIT.getRank() @@ -198,11 +282,11 @@ public List getListByParam(ProjectQueryRequest projectRequest) Map> userPricMap = new HashMap<>(); String[] tempstrArr = pusername.split(MODE_SPLIT); - // 拆分有projectId +"-" + priv + "-" + username的拼接而成的字段， - // 从而得到：查看权限用户、编辑权限用户、发布权限用户 + // 拆分有projectId +"-" + priv + "-" + username的拼接而成的字段， + // 从而得到：查看权限用户、编辑权限用户、发布权限用a for (String s : tempstrArr) { String[] strArr = s.split(KEY_SPLIT); - if(strArr.length >= 3) { + if (strArr.length >= 3) { String key = strArr[0] + KEY_SPLIT + strArr[1]; userPricMap.computeIfAbsent(key, k -> new ArrayList<>()); userPricMap.get(key).add(strArr[2]); @@ -218,18 +302,18 @@ public List getListByParam(ProjectQueryRequest projectRequest) LOGGER.info("user:{} get project access users info, workspaceId:{}, projectId:{}, projectName:{}, accessUsers:{}, editUsers:{}, releaseUsers:{}", projectRequest.getUsername(), projectRequest.getWorkspaceId(), projectVo.getId(), projectVo.getName(), accessUsers, editUsers, releaseUsers); - // 用户是否具有编辑权限 编辑权限和创建者都有 + // 用户是否具有编辑权限 发布权限和创建者和空间管理员都有 if (!StringUtils.isEmpty(pusername) && - (pusername.contains(editPriv) || - projectVo.getCreateBy().equals(projectRequest.getUsername()) || - isWorkspaceAdmin(projectRequest.getWorkspaceId(), projectRequest.getUsername())) || projectResponse.getReleaseUsers().contains(projectRequest.getUsername())) { + (projectVo.getCreateBy().equals(projectRequest.getUsername()) || isAdmin) + || projectResponse.getReleaseUsers().contains(projectRequest.getUsername())) { projectResponse.setEditable(true); - } else if (isWorkspaceAdmin(projectRequest.getWorkspaceId(), projectRequest.getUsername()) || - projectVo.getCreateBy().equals(projectRequest.getUsername())) { + } else if (isAdmin || projectVo.getCreateBy().equals(projectRequest.getUsername())) { projectResponse.setEditable(true); } else { projectResponse.setEditable(false); } + + projectResponseList.add(projectResponse); } return projectResponseList; } @@ -267,38 +351,67 @@ public Long getAppConnProjectId(Long appInstanceId, Long dssProjectId) { } @Override - public void deleteProject(String username, ProjectDeleteRequest projectDeleteRequest, Workspace workspace, DSSProjectDO dssProjectDO) throws Exception { + public void deleteProject(String username, ProjectDeleteRequest projectDeleteRequest, Workspace workspace, DSSProjectDO dssProjectDO) throws Exception { if (dssProjectDO == null) { throw new DSSErrorException(600001, "工程不存在!"); } LOGGER.warn("user {} begins to delete project {} in workspace {}.", username, dssProjectDO.getName(), workspace.getWorkspaceName()); - if(!dssProjectDO.getUsername().equalsIgnoreCase(username)){ - throw new DSSErrorException(600002, "刪除工程失敗，沒有删除权限!" ); + if (!dssProjectDO.getUsername().equalsIgnoreCase(username)) { + throw new DSSErrorException(600002, "刪除工程失敗，沒有删除权限!"); } - if(projectDeleteRequest.isIfDelOtherSys()) { + if (projectDeleteRequest.isIfDelOtherSys()) { LOGGER.warn("User {} requires to delete all projects with name {} in third-party AppConns.", username, dssProjectDO.getName()); Map appInstanceToRefProjectId = new HashMap<>(10); tryProjectOperation((appConn, appInstance) -> { - Long refProjectId = getAppConnProjectId(appInstance.getId(), projectDeleteRequest.getId()); - if(refProjectId == null) { - LOGGER.warn("delete project {} for third-party AppConn {} is ignored, appInstance is {}. Caused by: the refProjectId is null.", - dssProjectDO.getName(), appConn.getAppDesc().getAppName(), appInstance.getBaseUrl()); - return false; - } else { - appInstanceToRefProjectId.put(appInstance, refProjectId); - return true; - } - }, workspace, ProjectService::getProjectDeletionOperation, - null, - (appInstance, refProjectContentRequestRef) -> refProjectContentRequestRef.setProjectName(dssProjectDO.getName()) - .setRefProjectId(appInstanceToRefProjectId.get(appInstance)).setUserName(username), - (structureOperation, structureRequestRef) -> ((ProjectDeletionOperation) structureOperation).deleteProject((RefProjectContentRequestRef) structureRequestRef), - null, "delete refProject " + dssProjectDO.getName()); + Long refProjectId = getAppConnProjectId(appInstance.getId(), projectDeleteRequest.getId()); + if (refProjectId == null) { + LOGGER.warn("delete project {} for third-party AppConn {} is ignored, appInstance is {}. Caused by: the refProjectId is null.", + dssProjectDO.getName(), appConn.getAppDesc().getAppName(), appInstance.getBaseUrl()); + return false; + } else { + appInstanceToRefProjectId.put(appInstance, refProjectId); + return true; + } + }, workspace, ProjectService::getProjectDeletionOperation, + null, + (appInstance, refProjectContentRequestRef) -> refProjectContentRequestRef.setProjectName(dssProjectDO.getName()) + .setRefProjectId(appInstanceToRefProjectId.get(appInstance)).setUserName(username), + (structureOperation, structureRequestRef) -> ((ProjectDeletionOperation) structureOperation).deleteProject((RefProjectContentRequestRef) structureRequestRef), + null, "delete refProject " + dssProjectDO.getName()); } + // 对于DSS项目进行归档 + archiveGitProject(username, projectDeleteRequest, workspace, dssProjectDO); projectMapper.deleteProject(projectDeleteRequest.getId()); LOGGER.warn("User {} deleted project {}.", username, dssProjectDO.getName()); } + private void archiveGitProject(String username, ProjectDeleteRequest projectDeleteRequest, Workspace workspace, DSSProjectDO dssProjectDO) { + if (dssProjectDO.getAssociateGit() != null && dssProjectDO.getAssociateGit()) { + Sender gitSender = DSSSenderServiceFactory.getOrCreateServiceInstance().getGitSender(); + Map file = new HashMap<>(); + // 测试数据 key表示项目名、value为项目BmlResource资源 + GitArchiveProjectRequest request1 = new GitArchiveProjectRequest(workspace.getWorkspaceId(), dssProjectDO.getName(), username); + LOGGER.info("-------=======================begin to archive project: {}=======================-------", dssProjectDO.getName()); + Object ask = gitSender.ask(request1); + GitArchivePorjectResponse responseWorkflowValidNode = RpcAskUtils.processAskException(ask, GitArchivePorjectResponse.class, GitArchiveProjectRequest.class); + LOGGER.info("-------=======================End to archive project: {}=======================-------: {}", dssProjectDO.getName(), responseWorkflowValidNode); + } + } + + @Override + public Map getProjectGitUserInfo(String username, Long workspaceId) { + Sender gitSender = DSSSenderServiceFactory.getOrCreateServiceInstance().getGitSender(); + Map file = new HashMap<>(); + // 测试数据 key表示项目名、value为项目BmlResource资源 + GitUserByWorkspaceIdRequest request1 = new GitUserByWorkspaceIdRequest(); + request1.setWorkspaceId(workspaceId); + request1.setUsername(username); + Object ask = gitSender.ask(request1); + GitUserByWorkspaceResponse workspaceResponse = RpcAskUtils.processAskException(ask, GitUserByWorkspaceResponse.class, GitUserByWorkspaceIdRequest.class); + + return workspaceResponse.getMap(); + } + @Override public List getProjectAbilities(String username) { LOGGER.info("{} begins to get project ability", username); @@ -345,6 +458,12 @@ public List getDeletedProjects(ProjectQueryRequest projectReque projectResponse.setUpdateTime(projectVo.getUpdateTime()); projectResponse.setDevProcessList(ProjectStringUtils.convertList(projectVo.getDevProcess())); projectResponse.setOrchestratorModeList(ProjectStringUtils.convertList(projectVo.getOrchestratorMode())); + if (projectVo.getDataSourceListJson() != null) { + List dataSourceList = new Gson().fromJson(projectVo.getDataSourceListJson(), + new TypeToken>() { + }.getType()); + projectResponse.setDataSourceList(dataSourceList); + } projectResponseList.add(projectResponse); /** * 拆分有projectId +"-" + priv + "-" + username的拼接而成的字段， @@ -373,4 +492,359 @@ public List getDeletedProjects(ProjectQueryRequest projectReque return projectResponseList; } + @Override + public BmlResource exportProject(ExportAllOrchestratorsReqest exportAllOrchestratorsReqest, + String username, String proxyUser, Workspace workspace) throws Exception { + Long projectId = exportAllOrchestratorsReqest.getProjectId(); + EnvDSSLabel envLabel = new EnvDSSLabel(exportAllOrchestratorsReqest.getLabels()); + OrchestratorRequest orchestratorRequest = new OrchestratorRequest(workspace.getWorkspaceId(), exportAllOrchestratorsReqest.getProjectId()); + LabelRouteVO labelRouteVO = new LabelRouteVO(); + labelRouteVO.setRoute(exportAllOrchestratorsReqest.getLabels()); + orchestratorRequest.setLabels(labelRouteVO); + List orchestrators = orchestratorService.getOrchestratorInfos(orchestratorRequest, username); + DSSProjectDO projectDO = projectService.getProjectById(projectId); + String projectName = projectDO.getName(); + String projectPath = exportService.batchExport(username, projectId, orchestrators, projectName, envLabel, workspace); + exportProjectMeta(projectDO, projectPath); + String zipFile = ZipHelper.zip(projectPath, true); + LOGGER.info("export project file locate at {}", zipFile); + //先上传 + InputStream inputStream = bmlService.readLocalResourceFile(username, zipFile); + BmlResource bmlResource = bmlService.upload(username, inputStream, projectName + ".OrcsExport", projectName); + + LOGGER.info("export zip file upload to bmlResourceId:{} bmlResourceVersion:{}", + bmlResource.getResourceId(), bmlResource.getVersion()); + FileUtils.deleteQuietly(new File(zipFile)); + return bmlResource; + + } + + @Override + public void importProject(ProjectInfoVo projectInfo, BmlResource importResource, String username, + String checkCode, String packageInfo, EnvDSSLabel envLabel, Workspace workspace) throws Exception { + String projectName = projectInfo.getProjectName(); + //下载到本地处理 + String importSaveBasePath = IoUtils.generateTempIOPath(username); + String importFile = importSaveBasePath + File.separator + projectName + ".zip"; + LOGGER.info("import zip file locate at {}", importFile); + //下载压缩包 + bmlService.downloadToLocalPath(username, importResource.getResourceId(), importResource.getVersion(), importFile); + try { + String originProjectName = ZipHelper.readImportZipProjectName(importFile); + if (!projectName.equals(originProjectName)) { + String msg = String.format("target project name must be same with origin project name.origin project name:%s,target project name:%s(导入的目标工程名必须与导出时源工程名保持一致。源工程名：%s，目标工程名：%s)" + , originProjectName, projectName, originProjectName, projectName); + throw new DSSRuntimeException(msg); + } + } catch (IOException e) { + throw new DSSRuntimeException("upload file format error(导入包格式错误)"); + } + //解压 + ZipHelper.unzipFile(importFile, importSaveBasePath, true); + String projectPath = IoUtils.addFileSeparator(importSaveBasePath, projectName); + importService.batchImportOrc(username, projectInfo.getId(), + projectInfo.getProjectName(), projectPath, checkCode, envLabel, workspace); + + + } + + /** + * 导出项目元数据 + * + * @param projectDO 项目元数据 + * @param projectPath 项目目录 + * @throws IOException + */ + private void exportProjectMeta(DSSProjectDO projectDO, String projectPath) throws Exception { + String path = projectPath + File.separator + PROJECT_META_FILE_NAME; + File projectMetaFile = new File(path); + if (!projectMetaFile.exists()) { + try { + Path filePath = Paths.get(path); + // 确保父级路径存在 + Files.createDirectories(filePath.getParent()); + // 创建文件 + Files.createFile(filePath); + LOGGER.info("create success"); + } catch (Exception e) { + LOGGER.error("create File {} failed, the reason is : {}", path, e); + throw new Exception("create File failed"); + } + } + Gson gson = new GsonBuilder().setPrettyPrinting().create(); + // 文件不存在，直接创建并写入orchestratorInfo信息 + try (FileWriter writer = new FileWriter(projectMetaFile)) { + gson.toJson(projectDO, writer); + } + + } + + @Override + public BmlResource exportOnlyProjectMeta(ExportAllOrchestratorsReqest exportAllOrchestratorsReqest, + String username, String proxyUser, Workspace workspace) throws Exception { + Long projectId = exportAllOrchestratorsReqest.getProjectId(); + EnvDSSLabel envLabel = new EnvDSSLabel(exportAllOrchestratorsReqest.getLabels()); + OrchestratorRequest orchestratorRequest = new OrchestratorRequest(workspace.getWorkspaceId(), exportAllOrchestratorsReqest.getProjectId()); + LabelRouteVO labelRouteVO = new LabelRouteVO(); + labelRouteVO.setRoute(exportAllOrchestratorsReqest.getLabels()); + orchestratorRequest.setLabels(labelRouteVO); + List orchestrators = orchestratorService.getOrchestratorInfos(orchestratorRequest, username); + DSSProjectDO projectDO = projectService.getProjectById(projectId); + String projectName = projectDO.getName(); + String exportSaveBasePath = IoUtils.generateTempIOPath(username); + String projectPath = IoUtils.addFileSeparator(exportSaveBasePath, projectName); + exportProjectMeta(projectDO, projectPath); + String zipFile = ZipHelper.zip(projectPath, true); + LOGGER.info("export project file locate at {}", zipFile); + //先上传 + InputStream inputStream = bmlService.readLocalResourceFile(username, zipFile); + BmlResource bmlResource = bmlService.upload(username, inputStream, projectName + ".OrcsExport", projectName); + + LOGGER.info("export zip file upload to bmlResourceId:{} bmlResourceVersion:{}", + bmlResource.getResourceId(), bmlResource.getVersion()); + FileUtils.deleteQuietly(new File(zipFile)); + return bmlResource; + } + + + @Override + public DSSProjectDO modifyProjectMeta(String username, ProjectModifyRequest projectModifyRequest) throws DSSProjectErrorException { + + DSSProjectDO project = new DSSProjectDO(); + //修改的字段 + project.setDescription(projectModifyRequest.getDescription()); + project.setUpdateTime(new Date()); + project.setUpdateByStr(username); + + UpdateWrapper updateWrapper = new UpdateWrapper<>(); + updateWrapper.eq("id", projectModifyRequest.getId()); + updateWrapper.eq("workspace_id", projectModifyRequest.getWorkspaceId()); + projectMapper.update(project, updateWrapper); + + return project; + } + + @Override + public List queryListByParam(ProjectQueryRequest projectRequest, List totals) { + + //判断工作空间是否设置了管理员能否查看该工作空间下所有项目的权限 + Integer workspaceAdminPermission = projectUserMapper.getWorkspaceAdminPermission(projectRequest.getWorkspaceId()); + boolean isAdmin = isWorkspaceAdmin(projectRequest.getWorkspaceId(), projectRequest.getUsername()); + if (!isAdmin || workspaceAdminPermission != 1) { + projectRequest.setQueryUser(projectRequest.getUsername()); + } + + List projectIdList = getProjectId(projectRequest); + + if((!CollectionUtils.isEmpty(projectRequest.getAccessUsers()) || !CollectionUtils.isEmpty(projectRequest.getEditUsers()) + || !CollectionUtils.isEmpty(projectRequest.getReleaseUsers()) ) && CollectionUtils.isEmpty(projectIdList)){ + + totals.add(0L); + return new ArrayList<>(); + } + // 去重 + projectIdList = projectIdList.stream().distinct().collect(Collectors.toList()); + + //根据dss_project、dss_project_user查询出所在空间登录用户相关的工程 + if (StringUtils.isEmpty(projectRequest.getOrderBySql())) { + projectRequest.setOrderBySql("createTime desc"); + } + LOGGER.info("queryListByParam order by sql is {} ", projectRequest.getOrderBySql()); + projectRequest.setProjectIdList(projectIdList); + PageHelper.startPage(projectRequest.getPageNow(), projectRequest.getPageSize(), projectRequest.getOrderBySql()); + List list = projectMapper.queryListByParam(projectRequest); + if (CollectionUtils.isEmpty(list)) { + totals.add(0L); + return new ArrayList<>(); + } + + PageInfo pageInfo = new PageInfo(list); + totals.add(pageInfo.getTotal()); + List projectResponseList = new ArrayList<>(); + + for (QueryProjectVo projectVo : list) { + + ProjectResponse projectResponse = new ProjectResponse(); + projectResponse.setApplicationArea(projectVo.getApplicationArea()); + projectResponse.setId(projectVo.getId()); + projectResponse.setBusiness(projectVo.getBusiness()); + projectResponse.setCreateBy(projectVo.getCreateBy()); + projectResponse.setDescription(projectVo.getDescription()); + projectResponse.setName(projectVo.getName()); + projectResponse.setProduct(projectVo.getProduct()); + projectResponse.setSource(projectVo.getSource()); + projectResponse.setArchive(projectVo.getArchive()); + projectResponse.setCreateTime(projectVo.getCreateTime()); + projectResponse.setUpdateTime(projectVo.getUpdateTime()); + projectResponse.setDevProcessList(ProjectStringUtils.convertList(projectVo.getDevProcess())); + projectResponse.setOrchestratorModeList(ProjectStringUtils.convertList(projectVo.getOrchestratorMode())); + projectResponse.setWorkspaceName(projectVo.getWorkspaceName()); + + if (projectVo.getDataSourceListJson() != null) { + List dataSourceList = new Gson().fromJson(projectVo.getDataSourceListJson(), + new TypeToken>() { + }.getType()); + projectResponse.setDataSourceList(dataSourceList); + } + + List accessUsers = StringUtils.isBlank(projectVo.getAccessUsers()) ? new ArrayList<>() + : Arrays.asList(projectVo.getAccessUsers().split(MODE_SPLIT)); + List editUsers = StringUtils.isBlank(projectVo.getEditUsers()) ? new ArrayList<>() + : Arrays.asList(projectVo.getEditUsers().split(MODE_SPLIT)); + List releaseUsers = StringUtils.isBlank(projectVo.getReleaseUsers()) ? new ArrayList<>() + : Arrays.asList(projectVo.getReleaseUsers().split(MODE_SPLIT)); + + projectResponse.setAccessUsers(accessUsers.stream().distinct().collect(Collectors.toList())); + projectResponse.setEditUsers(editUsers.stream().distinct().collect(Collectors.toList())); + projectResponse.setReleaseUsers(releaseUsers.stream().distinct().collect(Collectors.toList())); + + // 用户是否具有编辑权限 发布者和创建者、管理员都有 + if (projectVo.getCreateBy().equals(projectRequest.getUsername()) + || isAdmin + || projectResponse.getReleaseUsers().contains(projectRequest.getUsername()) + ) { + projectResponse.setEditable(true); + } else { + projectResponse.setEditable(false); + } + + projectResponseList.add(projectResponse); + } + + return projectResponseList; + + } + + + public List getProjectId(ProjectQueryRequest projectRequest){ + List projectId = new ArrayList<>(); + + Map map = new HashMap(); + map.put(ProjectUserPrivEnum.PRIV_ACCESS.getRank(), projectRequest.getAccessUsers()); + map.put(ProjectUserPrivEnum.PRIV_EDIT.getRank(), projectRequest.getEditUsers()); + map.put(ProjectUserPrivEnum.PRIV_RELEASE.getRank(), projectRequest.getReleaseUsers()); + + for(Integer rank: map.keySet()){ + + if(CollectionUtils.isEmpty(map.get(rank))){ + continue; + } + + List projectIds = projectMapper.getProjectIdByUser(rank, map.get(rank)); + + if(CollectionUtils.isEmpty(projectIds)){ + return new ArrayList<>(); + } + + if(CollectionUtils.isEmpty(projectId)){ + projectId.addAll(projectIds); + }else{ + // 取交集 + projectId.retainAll(projectIds); + // 取完交集后为空，则直接返回 + if(CollectionUtils.isEmpty(projectId)){ + return new ArrayList<>(); + } + } + + + + } + + return projectId; + } + + @Override + public List getDSSProjectByName(List name) { + QueryWrapper projectQueryWrapper = new QueryWrapper<>(); + projectQueryWrapper.in("name", name); + List projectList = projectMapper.selectList(projectQueryWrapper); + List projects = new ArrayList<>(); + for (DSSProjectDO projectDO : projectList) { + DSSProject dssProject = new DSSProject(); + BeanUtils.copyProperties(projectDO, dssProject); + projects.add(dssProject); + } + return projects; + } + + @Override + public List queryProjectName(ProjectQueryRequest projectRequest){ + //判断工作空间是否设置了管理员能否查看该工作空间下所有项目的权限 + Integer workspaceAdminPermission = projectUserMapper.getWorkspaceAdminPermission(projectRequest.getWorkspaceId()); + + String queryUser = null; + if (!isWorkspaceAdmin(projectRequest.getWorkspaceId(), projectRequest.getUsername()) || workspaceAdminPermission != 1) { + queryUser = projectRequest.getUsername(); + } + + return projectMapper.queryProjectName(projectRequest.getWorkspaceId(),queryUser); + + } + + public List queryProject(ProjectListQueryRequest projectListQueryRequest){ + + List projectList = new ArrayList<>(); + + String queryUser = projectListQueryRequest.getUsername(); + Long workspaceId = projectListQueryRequest.getWorkspaceId(); + //判断工作空间是否设置了管理员能否查看该工作空间下所有项目的权限 + Integer workspaceAdminPermission = projectUserMapper.getWorkspaceAdminPermission(workspaceId); + boolean isAdmin = isWorkspaceAdmin(workspaceId,projectListQueryRequest.getUsername()); + if(isAdmin && workspaceAdminPermission == 1){ + queryUser = null; + } + + List queryProjectVoList = projectMapper.queryProjectList(projectListQueryRequest.getWorkspaceId(),queryUser); + + if(CollectionUtils.isEmpty(queryProjectVoList)){ + return projectList; + } + + + for (QueryProjectVo projectVo : queryProjectVoList) { + + DSSProject dssProject = new DSSProject(); + dssProject.setApplicationArea(projectVo.getApplicationArea()); + dssProject.setId(projectVo.getId()); + dssProject.setBusiness(projectVo.getBusiness()); + dssProject.setCreateBy(projectVo.getCreateBy()); + dssProject.setDescription(projectVo.getDescription()); + dssProject.setName(projectVo.getName()); + dssProject.setProduct(projectVo.getProduct()); + dssProject.setCreateTime(projectVo.getCreateTime()); + dssProject.setUpdateTime(projectVo.getUpdateTime()); + dssProject.setWorkspaceName(projectVo.getWorkspaceName()); + dssProject.setAssociateGit(projectVo.getAssociateGit()); + + List accessUsers = StringUtils.isBlank(projectVo.getAccessUsers()) ? new ArrayList<>() + : Arrays.asList(projectVo.getAccessUsers().split(MODE_SPLIT)); + List editUsers = StringUtils.isBlank(projectVo.getEditUsers()) ? new ArrayList<>() + : Arrays.asList(projectVo.getEditUsers().split(MODE_SPLIT)); + List releaseUsers = StringUtils.isBlank(projectVo.getReleaseUsers()) ? new ArrayList<>() + : Arrays.asList(projectVo.getReleaseUsers().split(MODE_SPLIT)); + + dssProject.setAccessUsers(accessUsers.stream().distinct().collect(Collectors.toList())); + dssProject.setEditUsers(editUsers.stream().distinct().collect(Collectors.toList())); + dssProject.setReleaseUsers(releaseUsers.stream().distinct().collect(Collectors.toList())); + + // 用户是否具有编辑权限 发布者和创建者、管理员都有 + if (projectVo.getCreateBy().equals(projectListQueryRequest.getUsername()) + || isAdmin + || dssProject.getReleaseUsers().contains(projectListQueryRequest.getUsername()) + ) { + dssProject.setEditable(true); + } else { + dssProject.setEditable(false); + } + + dssProject.setAdmin(isAdmin); + + projectList.add(dssProject); + } + + + return projectList; + } + } diff --git a/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/service/impl/DSSProjectUserServiceImpl.java b/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/service/impl/DSSProjectUserServiceImpl.java index 548c375bf5..9320e84796 100644 --- a/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/service/impl/DSSProjectUserServiceImpl.java +++ b/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/service/impl/DSSProjectUserServiceImpl.java @@ -99,7 +99,7 @@ public void saveProjectUser(Long projectID, String username, ProjectCreateReques * @param modifyRequest */ @Override - public void modifyProjectUser(DSSProjectDO dbProject, ProjectModifyRequest modifyRequest, String loginuser, Workspace workspace) throws Exception { + public void modifyProjectUser(DSSProjectDO dbProject, ProjectModifyRequest modifyRequest) { projectUserMapper.deleteAllPriv(dbProject.getId()); //將创建人默认为发布权限 List releaseUsers = modifyRequest.getReleaseUsers(); diff --git a/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/service/impl/ExportServiceImpl.java b/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/service/impl/ExportServiceImpl.java new file mode 100644 index 0000000000..e19876285f --- /dev/null +++ b/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/service/impl/ExportServiceImpl.java @@ -0,0 +1,143 @@ +/* + * + * * Copyright 2019 WeBank + * * + * * Licensed under the Apache License, Version 2.0 (the "License"); + * * you may not use this file except in compliance with the License. + * * You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.framework.project.service.impl; + +import com.google.gson.Gson; +import com.webank.wedatasphere.dss.appconn.core.AppConn; +import com.webank.wedatasphere.dss.appconn.core.ext.OnlyDevelopmentAppConn; +import com.webank.wedatasphere.dss.appconn.manager.AppConnManager; +import com.webank.wedatasphere.dss.common.exception.DSSRuntimeException; +import com.webank.wedatasphere.dss.common.label.DSSLabel; +import com.webank.wedatasphere.dss.common.utils.*; +import com.webank.wedatasphere.dss.framework.common.exception.DSSFrameworkErrorException; +import com.webank.wedatasphere.dss.common.entity.BmlResource; +import com.webank.wedatasphere.dss.framework.project.entity.response.BatchExportResult; +import com.webank.wedatasphere.dss.framework.project.entity.response.ExportResult; +import com.webank.wedatasphere.dss.framework.project.service.ExportService; +import com.webank.wedatasphere.dss.orchestrator.common.entity.OrchestratorInfo; +import com.webank.wedatasphere.dss.orchestrator.common.protocol.RequestAddVersionAfterPublish; +import com.webank.wedatasphere.dss.orchestrator.common.ref.OrchestratorRefConstant; +import com.webank.wedatasphere.dss.common.service.BMLService; +import com.webank.wedatasphere.dss.orchestrator.server.entity.vo.OrchestratorBaseInfo; +import com.webank.wedatasphere.dss.sender.service.DSSSenderServiceFactory; +import com.webank.wedatasphere.dss.standard.app.development.operation.RefExportOperation; +import com.webank.wedatasphere.dss.standard.app.development.ref.ExportResponseRef; +import com.webank.wedatasphere.dss.standard.app.development.ref.ImportRequestRef; +import com.webank.wedatasphere.dss.standard.app.development.ref.impl.ThirdlyRequestRef; +import com.webank.wedatasphere.dss.standard.app.development.service.RefExportService; +import com.webank.wedatasphere.dss.standard.app.development.standard.DevelopmentIntegrationStandard; +import com.webank.wedatasphere.dss.standard.app.sso.Workspace; +import com.webank.wedatasphere.dss.standard.common.desc.AppInstance; +import org.apache.commons.io.FileUtils; +import org.apache.linkis.common.exception.ErrorException; + +import java.io.*; +import java.nio.charset.StandardCharsets; +import java.util.*; + +import org.apache.linkis.rpc.Sender; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.stereotype.Service; +import org.apache.commons.codec.digest.DigestUtils; + +/** + * created by cooperyang on 2020/12/9 + * Description: + */ +@Service +public class ExportServiceImpl implements ExportService { + private final String metaFileName = "meta.json"; + + private static final Logger LOGGER = LoggerFactory.getLogger(ExportServiceImpl.class); + + private static final String ORCHESTRATOR_FRAMEWORK_NAME = "orchestrator-framework"; + @Autowired + @Qualifier("projectBmlService") + private BMLService bmlService; + + + @Override + @SuppressWarnings("ConstantConditions") + public ExportResult export(String releaseUser, Long projectId, OrchestratorInfo orchestrator, + String projectName, DSSLabel dssLabel, Workspace workspace) throws ErrorException { + LOGGER.info("begin to export projectId {} orchestratorId {} orchestratorVersionId {} for user {}", projectId, + orchestrator.getOrchestratorId(), orchestrator.getOrchestratorVersionId(), releaseUser); + //获取到orchestratorFramework的appconn + AppConn appConn = AppConnManager.getAppConnManager().getAppConn(ORCHESTRATOR_FRAMEWORK_NAME); + //2.通过Label拿到对应的Service + DevelopmentIntegrationStandard developmentIntegrationStandard = ((OnlyDevelopmentAppConn) appConn).getOrCreateDevelopmentStandard(); + if (developmentIntegrationStandard == null) { + LOGGER.error("orchestrator framework development is null"); + DSSExceptionUtils.dealErrorException(60021, "orchestrator framework development is null", DSSFrameworkErrorException.class); + } + List dssLabelList = new ArrayList<>(); + dssLabelList.add(dssLabel); + AppInstance appInstance = appConn.getAppDesc().getAppInstancesByLabels(dssLabelList).get(0); + RefExportService refExportService = developmentIntegrationStandard.getRefExportService(appInstance); + RefExportOperation refExportOperation = refExportService.getRefExportOperation(); + ThirdlyRequestRef.RefJobContentRequestRefImpl requestRef = new ThirdlyRequestRef.RefJobContentRequestRefImpl(); + Long orchestratorId = orchestrator.getOrchestratorId(); + Long orchestratorVersionId = orchestrator.getOrchestratorVersionId(); + requestRef.setRefJobContent(MapUtils.newCommonMap(OrchestratorRefConstant.ORCHESTRATOR_ID_KEY, orchestratorId, + OrchestratorRefConstant.ORCHESTRATOR_VERSION_ID_KEY, orchestratorVersionId)); + requestRef.setDSSLabels(Collections.singletonList(dssLabel)); + requestRef.setRefProjectId(projectId); + requestRef.setUserName(releaseUser); + requestRef.setWorkspace(workspace); + requestRef.setProjectName(projectName); + ExportResponseRef responseRef = refExportOperation.exportRef(requestRef); + BmlResource bmlResource = new BmlResource((String) responseRef.getResourceMap().get(ImportRequestRef.RESOURCE_ID_KEY), + (String) responseRef.getResourceMap().get(ImportRequestRef.RESOURCE_VERSION_KEY)); + return new ExportResult(bmlResource, (Long) responseRef.getResourceMap().get(OrchestratorRefConstant.ORCHESTRATOR_VERSION_ID_KEY)); + + } + + @Override + public String batchExport(String userName, Long projectId, List orchestrators, + String projectName, DSSLabel dssLabel, Workspace workspace) throws ErrorException { + String exportSaveBasePath = IoUtils.generateTempIOPath(userName); + for (OrchestratorBaseInfo orchestrator : orchestrators) { + OrchestratorInfo orchestratorInfo = new OrchestratorInfo(orchestrator.getOrchestratorId(), orchestrator.getOrchestratorVersionId()); + BmlResource bmlOneOrc= export(userName, projectId, orchestratorInfo, + projectName, dssLabel, workspace) + .getBmlResource(); + String orcPath=exportSaveBasePath+orchestrator.getOrchestratorName()+".zip"; + bmlService.downloadToLocalPath(userName,bmlOneOrc.getResourceId(),bmlOneOrc.getVersion(),orcPath); + LOGGER.info("export orchestrator file locate at {}",orcPath); + ZipHelper.unzipFile(orcPath, exportSaveBasePath,true); + } + return IoUtils.addFileSeparator(exportSaveBasePath, projectName); + } + + @Override + public Long addVersionAfterPublish(String releaseUser, Long projectId, Long orchestratorId, + Long orchestratorVersionId, String projectName, Workspace workspace, + DSSLabel dssLabel, String comment) { + List dssLabelList = new ArrayList<>(); + dssLabelList.add(dssLabel); + RequestAddVersionAfterPublish requestAddVersionAfterPublish = new RequestAddVersionAfterPublish(releaseUser, workspace, + orchestratorId, orchestratorVersionId, projectName, dssLabelList, comment); + Sender sender = DSSSenderServiceFactory.getOrCreateServiceInstance().getOrcSender(dssLabelList); + return RpcAskUtils.processAskException(sender.ask(requestAddVersionAfterPublish), Long.class, RequestAddVersionAfterPublish.class); + } + +} diff --git a/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/service/impl/ImportServiceImpl.java b/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/service/impl/ImportServiceImpl.java new file mode 100644 index 0000000000..cb96039b1f --- /dev/null +++ b/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/service/impl/ImportServiceImpl.java @@ -0,0 +1,179 @@ +/* + * + * * Copyright 2019 WeBank + * * + * * Licensed under the Apache License, Version 2.0 (the "License"); + * * you may not use this file except in compliance with the License. + * * You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.framework.project.service.impl; + + +import com.google.gson.Gson; +import com.google.gson.reflect.TypeToken; +import com.webank.wedatasphere.dss.appconn.core.AppConn; +import com.webank.wedatasphere.dss.appconn.core.ext.OnlyDevelopmentAppConn; +import com.webank.wedatasphere.dss.appconn.manager.AppConnManager; +import com.webank.wedatasphere.dss.common.exception.DSSRuntimeException; +import com.webank.wedatasphere.dss.common.label.DSSLabel; +import com.webank.wedatasphere.dss.common.utils.DSSExceptionUtils; +import com.webank.wedatasphere.dss.common.utils.IoUtils; +import com.webank.wedatasphere.dss.common.utils.MapUtils; +import com.webank.wedatasphere.dss.common.utils.ZipHelper; +import com.webank.wedatasphere.dss.framework.common.exception.DSSFrameworkErrorException; +import com.webank.wedatasphere.dss.framework.project.entity.OrchestratorBatchImportInfo; +import com.webank.wedatasphere.dss.common.entity.BmlResource; +import com.webank.wedatasphere.dss.framework.project.entity.po.OrchestratorImportInfo; +import com.webank.wedatasphere.dss.framework.project.service.ImportService; +import com.webank.wedatasphere.dss.framework.project.utils.ExportAndImportSupportUtils; +import com.webank.wedatasphere.dss.orchestrator.common.entity.DSSOrchestratorInfo; +import com.webank.wedatasphere.dss.orchestrator.common.ref.OrchestratorRefConstant; +import com.webank.wedatasphere.dss.common.service.BMLService; +import com.webank.wedatasphere.dss.orchestrator.publish.io.input.MetaInputService; +import com.webank.wedatasphere.dss.orchestrator.server.entity.vo.OrchestratorBaseInfo; +import com.webank.wedatasphere.dss.standard.app.development.operation.RefImportOperation; +import com.webank.wedatasphere.dss.standard.app.development.ref.ImportRequestRef; +import com.webank.wedatasphere.dss.standard.app.development.ref.RefJobContentResponseRef; +import com.webank.wedatasphere.dss.standard.app.development.ref.impl.ThirdlyRequestRef; +import com.webank.wedatasphere.dss.standard.app.development.service.RefImportService; +import com.webank.wedatasphere.dss.standard.app.development.standard.DevelopmentIntegrationStandard; +import com.webank.wedatasphere.dss.standard.app.sso.Workspace; +import com.webank.wedatasphere.dss.standard.common.desc.AppInstance; +import org.apache.commons.codec.digest.DigestUtils; +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.FilenameUtils; +import org.apache.linkis.common.exception.ErrorException; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.lang.reflect.Type; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.*; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import java.util.zip.ZipEntry; +import java.util.zip.ZipFile; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.stereotype.Service; + +import static com.webank.wedatasphere.dss.common.utils.IoUtils.FLOW_META_DIRECTORY_NAME; + +/** + * created by cooperyang on 2020/12/9 + * Description: + */ +@Service +public class ImportServiceImpl implements ImportService { + + private static final Logger LOGGER = LoggerFactory.getLogger(ImportServiceImpl.class); + + private static final String ORC_FRAMEWORK_NAME = "orchestrator-framework"; + + @Autowired + @Qualifier("projectBmlService") + private BMLService bmlService; + @Autowired + private MetaInputService metaInputService; + + @Override + @SuppressWarnings("ConstantConditions") + public OrchestratorImportInfo importOrc(String orchestratorName, String releaseUser, Long projectId, String projectName, + BmlResource bmlResource, DSSLabel dssLabel, String workspaceName, Workspace workspace) throws ErrorException { + //导入之后我们应该拿到的是导入之后的orchestrator的内容,这样我们在做同步到调度中心的时候才是同步的生产中心的内容 + LOGGER.info("Begin to import orc for project {} and orc resource is {}", projectName, bmlResource); + OrchestratorImportInfo orchestratorReleaseInfo =null; + AppConn appConn = AppConnManager.getAppConnManager().getAppConn(ORC_FRAMEWORK_NAME); + DevelopmentIntegrationStandard standard = ((OnlyDevelopmentAppConn)appConn).getOrCreateDevelopmentStandard(); + if (standard == null) { + LOGGER.error("developStandard is null, can not do import operation"); + DSSExceptionUtils.dealErrorException(60045, "developStandard is null, can not do import operation", DSSFrameworkErrorException.class); + } + List dssLabelList = Collections.singletonList(dssLabel); + AppInstance appInstance = appConn.getAppDesc().getAppInstancesByLabels(dssLabelList).get(0); + RefImportService refImportService = standard.getRefImportService(appInstance); + RefImportOperation refImportOperation = refImportService.getRefImportOperation(); + try { + ThirdlyRequestRef.ImportRequestRefImpl importRequestRef = new ThirdlyRequestRef.ImportRequestRefImpl(); + + importRequestRef.setResourceMap(MapUtils.newCommonMap( + ImportRequestRef.RESOURCE_ID_KEY, bmlResource.getResourceId(), ImportRequestRef.RESOURCE_VERSION_KEY, bmlResource.getVersion())); + importRequestRef.setDSSLabels(Collections.singletonList(dssLabel)); + importRequestRef.setRefProjectId(projectId); + importRequestRef.setUserName(releaseUser); + importRequestRef.setProjectName(projectName); + importRequestRef.setName(orchestratorName); + importRequestRef.setWorkspace(workspace); + RefJobContentResponseRef responseRef = refImportOperation.importRef(importRequestRef); + Long refOrcId = (Long) responseRef.getRefJobContent().get(OrchestratorRefConstant.ORCHESTRATOR_ID_KEY); + String orchestratorVersion=(String) responseRef.getRefJobContent().get(OrchestratorRefConstant.ORCHESTRATOR_VERSION_KEY); + LOGGER.info("Succeed to get responseRef for projectInfo {}, refOrcId is {}.", projectName, refOrcId); + //从返回的的内容中搞到导入之后的orc的信息 + orchestratorReleaseInfo= OrchestratorImportInfo.newInstance(refOrcId,orchestratorName, + orchestratorVersion); + } catch (Exception e) { + DSSExceptionUtils.dealErrorException(60035, "Failed to import Ref for project: " + e.getMessage(), e, DSSFrameworkErrorException.class); + } + return orchestratorReleaseInfo; + } + + @Override + public OrchestratorBatchImportInfo batchImportOrc(String userName, Long projectId, String projectName, String projectPath, String checkCode, + DSSLabel dssLabel, Workspace workspace) + throws ErrorException{ + String importSaveBasePath = Paths.get(projectPath).getParent().toString(); + LOGGER.info("import unziped file locate at {}",projectPath); + //解析元数据 + String metaPath = IoUtils.addFileSeparator(projectPath, FLOW_META_DIRECTORY_NAME); + List importOrcMetas; + try (Stream paths = Files.walk(Paths.get(metaPath), 1)) { + importOrcMetas = paths.filter(Files::isDirectory) // 确保它是一个目录 + .filter(path -> !path.equals(Paths.get(metaPath))) // 排除根目录本身 + .map(path -> path.toAbsolutePath().toString()) + .map(metaInputService::importOrchestratorNew) + .map(OrchestratorBaseInfo::convertFrom) + .collect(Collectors.toList()); + } catch (IOException e) { + throw new DSSRuntimeException("read import package failed,please check package struct(导入包格式错误，请检查导入包格式）"); + } + String separateTargetPath = IoUtils.generateTempIOPath(userName); + //分离各个工作流 + Map flowProjectPaths = ExportAndImportSupportUtils.separateFlows(importSaveBasePath, projectName, separateTargetPath); + + + + List importResultInfo = new ArrayList<>(); + for (Path path : flowProjectPaths.values()) { + String zipFilePath = ZipHelper.zip(path.toAbsolutePath().toString()); + File orcZipFile=new File(zipFilePath); + InputStream inputStream = bmlService.readLocalResourceFile(userName, orcZipFile.getAbsolutePath()); + BmlResource uploadResult = bmlService.upload(userName, inputStream, + orcZipFile.getName() , projectName); + String orcName = FilenameUtils.getBaseName(orcZipFile.getName()); + OrchestratorImportInfo importInfo = importOrc(orcName, userName, projectId, projectName, uploadResult, dssLabel, workspace.getWorkspaceName(), workspace); + importResultInfo.add(importInfo); + } + //清理文件 + FileUtils.deleteQuietly(new File(importSaveBasePath)); + FileUtils.deleteQuietly(new File(separateTargetPath)); + + return new OrchestratorBatchImportInfo(importOrcMetas,importResultInfo); + } + +} diff --git a/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/service/impl/ProxyUserProjectHttpRequestHook.java b/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/service/impl/ProxyUserProjectHttpRequestHook.java index 8122693901..9edefe20f7 100644 --- a/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/service/impl/ProxyUserProjectHttpRequestHook.java +++ b/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/service/impl/ProxyUserProjectHttpRequestHook.java @@ -1,11 +1,14 @@ package com.webank.wedatasphere.dss.framework.project.service.impl; +import com.webank.wedatasphere.dss.common.constant.project.ProjectUserPrivEnum; +import com.webank.wedatasphere.dss.framework.project.entity.DSSProjectUser; import com.webank.wedatasphere.dss.framework.project.entity.request.ProjectCreateRequest; import com.webank.wedatasphere.dss.framework.project.entity.request.ProjectDeleteRequest; import com.webank.wedatasphere.dss.framework.project.entity.request.ProjectModifyRequest; import com.webank.wedatasphere.dss.framework.project.entity.request.ProjectQueryRequest; import com.webank.wedatasphere.dss.framework.project.entity.response.ProjectResponse; import com.webank.wedatasphere.dss.framework.project.service.DSSProjectService; +import com.webank.wedatasphere.dss.framework.project.service.DSSProjectUserService; import com.webank.wedatasphere.dss.framework.project.service.ProjectHttpRequestHook; import com.webank.wedatasphere.dss.framework.proxy.conf.ProxyUserConfiguration; import com.webank.wedatasphere.dss.framework.proxy.exception.DSSProxyUserErrorException; @@ -21,8 +24,10 @@ import org.springframework.stereotype.Component; import javax.servlet.http.HttpServletRequest; +import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.Objects; import java.util.function.Consumer; import java.util.function.Function; import java.util.stream.Collectors; @@ -36,14 +41,17 @@ */ @Component public class ProxyUserProjectHttpRequestHook implements ProjectHttpRequestHook { - private static final Logger LOGGER= LoggerFactory.getLogger(ProxyUserProjectHttpRequestHook.class); + private static final Logger LOGGER = LoggerFactory.getLogger(ProxyUserProjectHttpRequestHook.class); @Autowired private DSSProjectService dssProjectService; @Autowired private DssProxyUserService dssProxyUserService; + @Autowired + private DSSProjectUserService dssProjectUserService; + private Message doProxyUserFunction(HttpServletRequest request, Function function) { - if(!ProxyUserConfiguration.isProxyUserEnable()) { + if (!ProxyUserConfiguration.isProxyUserEnable()) { return null; } String proxyUser; @@ -51,8 +59,8 @@ private Message doProxyUserFunction(HttpServletRequest request, FunctionString.format("%s=%s",cookie.getName(),cookie.getValue())).collect(Collectors.joining(","))); - LOGGER.error("getProxyUser failed.",e); + .map(cookie -> String.format("%s=%s", cookie.getName(), cookie.getValue())).collect(Collectors.joining(","))); + LOGGER.error("getProxyUser failed.", e); return Message.error(e.getMessage()); } return function.apply(proxyUser); @@ -73,14 +81,13 @@ public Message beforeGetAllProjects(HttpServletRequest request, ProjectQueryRequ @Override public Message beforeCreateProject(HttpServletRequest request, ProjectCreateRequest projectCreateRequest) { return doProxyUserFunction(request, proxyUser -> { - if(CollectionUtils.isNotEmpty(projectCreateRequest.getAccessUsers()) || - CollectionUtils.isNotEmpty(projectCreateRequest.getEditUsers()) || CollectionUtils.isNotEmpty(projectCreateRequest.getReleaseUsers())) { - return Message.error("This environment is not allowed to set accessUsers, editUsers or ReleaseUsers(本环境不允许设置发布权限、编辑权限和查看权限，请删除相关权限后再重试)."); + if (CollectionUtils.isNotEmpty(projectCreateRequest.getEditUsers()) || CollectionUtils.isNotEmpty(projectCreateRequest.getReleaseUsers())) { + return Message.error("This environment is not allowed to set editUsers or ReleaseUsers(本环境不允许设置发布权限、编辑权限，请删除相关权限后再重试)."); } - String userName= SecurityFilter.getLoginUsername(request); - if(userName.equals(proxyUser) - &&!StringUtils.startsWithIgnoreCase(proxyUser,"WTSS_") - &&!StringUtils.startsWithIgnoreCase(proxyUser,"hduser")){ + String userName = SecurityFilter.getLoginUsername(request); + if (userName.equals(proxyUser) + && !StringUtils.startsWithIgnoreCase(proxyUser, "WTSS_") + && !StringUtils.startsWithIgnoreCase(proxyUser, "hduser")) { return Message.error("only ops proxy user can create project(只允许代理用户创建工程)."); } projectCreateRequest.getEditUsers().add(proxyUser); @@ -98,14 +105,26 @@ public Message beforeModifyProject(HttpServletRequest request, ProjectModifyRequ // 这里直接使用代理用户来查询 projectQueryRequest.setUsername(proxyUser); projectQueryRequest.setWorkspaceId(workspace.getWorkspaceId()); - List projectResponseList = dssProjectService.getListByParam(projectQueryRequest); - if(CollectionUtils.isEmpty(projectResponseList)) { + + List dssProjectUsers = dssProjectUserService.getProjectPriv(projectModifyRequest.getId()); + + if (CollectionUtils.isEmpty(dssProjectUsers)) { return Message.error("You have no permission to modify this project."); - } else if(!CollectionUtils.isEqualCollection(projectModifyRequest.getEditUsers(), projectResponseList.get(0).getEditUsers()) || - !CollectionUtils.isEqualCollection(projectModifyRequest.getReleaseUsers(), projectResponseList.get(0).getReleaseUsers()) || - CollectionUtils.isNotEmpty(projectModifyRequest.getAccessUsers())) { - return Message.error("This environment is not allowed to set accessUsers, editUsers or ReleaseUsers(本环境不允许设置发布权限、编辑权限和查看权限，请删除相关权限后再重试)."); } + + Integer editPriv = ProjectUserPrivEnum.PRIV_EDIT.getRank(); + List editUsers = projectModifyRequest.getEditUsers(); + + Integer releasePriv = ProjectUserPrivEnum.PRIV_RELEASE.getRank(); + List releaseUsers = projectModifyRequest.getReleaseUsers(); + + if (!isSameUser(editPriv, dssProjectUsers, editUsers, projectModifyRequest) + || !isSameUser(releasePriv, dssProjectUsers, releaseUsers, projectModifyRequest)) { + + return Message.error("This environment is not allowed to set accessUsers, " + + "editUsers or ReleaseUsers(本环境不允许设置编辑、发布权限，请删除相关权限后再重试)."); + } + return null; }); } @@ -119,4 +138,39 @@ public Message beforeDeleteProject(HttpServletRequest request, ProjectDeleteRequ public Message beforeGetDeletedProject(HttpServletRequest request, ProjectQueryRequest projectRequest) { return doProxyUserConsumer(request, projectRequest::setUsername); } + + + public Boolean isSameUser(Integer priv, List dssProjectUsers, List inputUser, ProjectModifyRequest projectModifyRequest) { + + // 根据priv过滤出需要的用户信息 + List projectUsers = dssProjectUsers.stream() + .filter(projectUser -> Objects.equals(priv, projectUser.getPriv())) + .map(DSSProjectUser::getUsername).collect(Collectors.toList()); + + // 过滤代理用户信息，取实名用户 + List realUsers = projectUsers.stream().filter( + user -> !StringUtils.startsWithIgnoreCase(user, "WTSS_") + && !StringUtils.startsWithIgnoreCase(user, "hduser") + ).collect(Collectors.toList()); + + if (!CollectionUtils.isEqualCollection(inputUser, projectUsers) + && !CollectionUtils.isEqualCollection(inputUser, realUsers) + ) { + return false; + + } + + // 判断输入的用户是否和实名用户一致 + if (CollectionUtils.isEqualCollection(inputUser, realUsers)) { + + if (Objects.equals(ProjectUserPrivEnum.PRIV_EDIT.getRank(), priv)) { + projectModifyRequest.setEditUsers(projectUsers); + } else if (Objects.equals(ProjectUserPrivEnum.PRIV_RELEASE.getRank(), priv)) { + projectModifyRequest.setReleaseUsers(projectUsers); + } + } + + return true; + + } } diff --git a/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/utils/ExportAndImportSupportUtils.java b/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/utils/ExportAndImportSupportUtils.java new file mode 100644 index 0000000000..a953bc29a3 --- /dev/null +++ b/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/utils/ExportAndImportSupportUtils.java @@ -0,0 +1,131 @@ +package com.webank.wedatasphere.dss.framework.project.utils; +import com.webank.wedatasphere.dss.common.exception.DSSErrorException; +import com.webank.wedatasphere.dss.common.exception.DSSRuntimeException; +import com.webank.wedatasphere.dss.common.utils.IoUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.IOException; +import java.nio.file.*; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * 支持导入导出的工具类 + * Author: xlinliu + * Date: 2024/4/19 + */ +public class ExportAndImportSupportUtils { + private static final Logger LOGGER = LoggerFactory.getLogger(ExportAndImportSupportUtils.class); + private static final String EXTERNAL_RESOURCES = ".external-resources"; + private static final String META_CONF = ".metaConf"; + + /** + * 分离一个工作流导出包，得到各个工作流的导出包 + * @param sourceDir 原始导出包的的路劲，路径下存在至少一个项目名，项目下有两个以上的工作流 + * @param projectName 项目名 + * @param targetDir 指定分离后的工作流导出包位置路径。导出后，这个路径下会出现以各工作流名命名的文件夹，这些文件夹下都只有一个工作流导出包。 + * @return 分离后的导出包路径，key是工作流名，value是一个项目文件夹。此时项目文件夹里只有一个工作流。 + */ + public static Map separateFlows(String sourceDir, String projectName, String targetDir) { + + // 检测工作流目录 + List workflowNames = detectWorkflowNames(new File(sourceDir, projectName)); + Map workFlowProjectPaths = new HashMap<>(workflowNames.size()); + for (String workflow : workflowNames) { + try { + // 创建新的工作流项目目录 + String newWorkflowDir = IoUtils.addFileSeparator(targetDir,workflow,projectName) ; + new File(newWorkflowDir).mkdirs(); + + // 复制项目中的所有文件到新的工作流项目目录 + copyProjectFiles(new File(sourceDir, projectName), newWorkflowDir); + + // 复制 EXTERNAL_RESOURCES + copyDirectory(new File(sourceDir, projectName + File.separator + EXTERNAL_RESOURCES).getPath(), newWorkflowDir + File.separator + EXTERNAL_RESOURCES); + + // 复制 META_CONF，仅包括指定工作流的子目录 + copyWorkflowMetaConf(new File(sourceDir, projectName + File.separator + META_CONF).getPath(), newWorkflowDir + File.separator + META_CONF, workflow); + + // 复制工作流的代码目录 + String workflowDir = new File(sourceDir, projectName + File.separator + workflow).getPath(); + copyDirectory(workflowDir, newWorkflowDir + File.separator + workflow); + workFlowProjectPaths.put(workflow, Paths.get(newWorkflowDir)); + } catch (IOException e) { + LOGGER.error("Error while copying workflow directory " , e); + throw new DSSRuntimeException(70001, "导入多个工作流时分离工作流失败，原因为:" + e.getMessage(),e); + } + + + } + + return workFlowProjectPaths; + } + + // 复制整个目录 + private static void copyDirectory(String sourceDirectory, String destinationDirectory) throws IOException { + File file = new File(sourceDirectory); + if (!file.exists()) { + return ; + } + Files.walk(Paths.get(sourceDirectory)) + .forEach(source -> { + Path destination = Paths.get(destinationDirectory, source.toString().substring(sourceDirectory.length())); + try { + if (Files.isDirectory(source)) { + Files.createDirectories(destination); + } else { + Files.copy(source, destination, StandardCopyOption.REPLACE_EXISTING); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + }); + } + + // 检测工作流目录 + private static List detectWorkflowNames(File projectDir) { + List workflowNames = new ArrayList<>(); + File[] projectSubFolders = projectDir.listFiles(); + if (projectSubFolders != null) { + for (File file : projectSubFolders) { + if (file.isDirectory() && !file.getName().equals(EXTERNAL_RESOURCES) && !file.getName().equals(META_CONF)) { + workflowNames.add(file.getName()); + } + } + } + return workflowNames; + } + + // 复制项目目录中的所有文件 + private static void copyProjectFiles(File sourceDir, String destinationDir) throws IOException { + File[] files = sourceDir.listFiles(); + if (files != null) { + for (File file : files) { + if (file.isFile()) { // 确保只复制文件 + Path destination = Paths.get(destinationDir, file.getName()); + Files.copy(file.toPath(), destination, StandardCopyOption.REPLACE_EXISTING); + } + } + } + } + + // 专门为 META_CONF 文件夹的复制，只包括特定工作流的子目录 + private static void copyWorkflowMetaConf(String sourceDir, String destinationDir, String workflow) throws IOException { + File sourceMetaConfDir = new File(sourceDir); + File[] filesAndDirs = sourceMetaConfDir.listFiles(); + if (filesAndDirs != null) { + for (File item : filesAndDirs) { + if (item.isDirectory() && item.getName().equals(workflow)) { + copyDirectory(item.getPath(), Paths.get(destinationDir, item.getName()).toString()); + } else if (item.isFile()) { + Files.copy(item.toPath(), Paths.get(destinationDir, item.getName()), StandardCopyOption.REPLACE_EXISTING); + } + } + } + } + +} diff --git a/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/utils/ProjectOperationUtils.java b/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/utils/ProjectOperationUtils.java index 5ef95a427c..73e3f70483 100644 --- a/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/utils/ProjectOperationUtils.java +++ b/dss-framework/dss-framework-project-server/src/main/java/com/webank/wedatasphere/dss/framework/project/utils/ProjectOperationUtils.java @@ -15,6 +15,8 @@ import com.webank.wedatasphere.dss.standard.common.desc.AppInstance; import com.webank.wedatasphere.dss.standard.common.entity.ref.ResponseRef; import org.apache.commons.lang3.tuple.ImmutablePair; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.util.function.*; @@ -24,6 +26,7 @@ * @since 0.5.0 */ public class ProjectOperationUtils { + private static final Logger LOGGER = LoggerFactory.getLogger(ProjectOperationUtils.class); public static void tryProjectOperation(BiPredicate isTryOperation, Workspace workspace, Function getProjectOperation, @@ -45,6 +48,7 @@ public static void tryPr StructureIntegrationStandard structureStandard = ((OnlyStructureAppConn) appConn).getOrCreateStructureStandard(); appConn.getAppDesc().getAppInstances().forEach(appInstance -> { if(isTryOperation == null || isTryOperation.test(appConn, appInstance)) { + long startTime = System.currentTimeMillis(); V responseRef = StructureOperationUtils.tryProjectOperation(() -> structureStandard.getProjectService(appInstance), getProjectOperation, dssProjectContentRequestRefConsumer, @@ -56,6 +60,9 @@ public static void tryPr if(dealResponseRefConsumer != null) { dealResponseRefConsumer.accept(new ImmutablePair<>(appConn, appInstance), responseRef); } + long endTime = System.currentTimeMillis(); + String url =""+ appInstance.getBaseUrl() + appInstance.getHomepageUri(); + LOGGER.info("try project operation cost info.cost:{}ms url:{}", (endTime - startTime), url); } }); }); diff --git a/dss-framework/dss-framework-project-server/src/main/resources/application-dss.yml b/dss-framework/dss-framework-project-server/src/main/resources/application-dss.yml new file mode 100644 index 0000000000..b0473838d8 --- /dev/null +++ b/dss-framework/dss-framework-project-server/src/main/resources/application-dss.yml @@ -0,0 +1,23 @@ + +eureka: + client: + serviceUrl: + defaultZone: http://127.0.0.1:20303/eureka/ + #instance: + #prefer-ip-address: true + #instance-id: ${spring.cloud.client.ip-address}:${server.port} + #metadata-map: + #test: wedatasphere + +management: + endpoints: + web: + exposure: + include: refresh,info +logging: + config: classpath:log4j2.xml + +#mybatis: +# configuration: +# log-impl: org.apache.ibatis.logging.stdout.StdOutImpl + diff --git a/dss-framework/dss-framework-project-server/src/main/resources/dss.properties b/dss-framework/dss-framework-project-server/src/main/resources/dss.properties index fcefc00537..753837afb1 100644 --- a/dss-framework/dss-framework-project-server/src/main/resources/dss.properties +++ b/dss-framework/dss-framework-project-server/src/main/resources/dss.properties @@ -23,7 +23,7 @@ wds.linkis.gateway.url=http://127.0.0.1:9001/ wds.linkis.server.mybatis.datasource.url=jdbc:mysql://127.0.0.1:3306/dss_dev_1?characterEncoding=UTF-8 wds.linkis.server.mybatis.datasource.username= -***REMOVED*** +wds.linkis.server.mybatis.datasource.password= spring.server.port=9202 spring.spring.application.name=dss-framework-project-server diff --git a/dss-framework/dss-framework-project-server/src/main/scala/com/webank/wedatasphere/dss/framework/project/server/rpc/ProjectReceiver.scala b/dss-framework/dss-framework-project-server/src/main/scala/com/webank/wedatasphere/dss/framework/project/server/rpc/ProjectReceiver.scala index 660bd04445..bc84de344a 100644 --- a/dss-framework/dss-framework-project-server/src/main/scala/com/webank/wedatasphere/dss/framework/project/server/rpc/ProjectReceiver.scala +++ b/dss-framework/dss-framework-project-server/src/main/scala/com/webank/wedatasphere/dss/framework/project/server/rpc/ProjectReceiver.scala @@ -76,6 +76,15 @@ class ProjectReceiver(projectService: DSSProjectService, val projectDo: DSSProjectDO = projectService.getProjectById(projectId) val privList = projectUserService.getProjectUserPriv(projectId, userName).map(_.getPriv) new ProjectUserAuthResponse(projectId, userName, privList, projectDo.getCreateBy) + + case projectInfoListRequest: ProjectInfoListRequest => + val projects = projectService.getDSSProjectByName(projectInfoListRequest.getProjectNames) + new ProjectInfoListResponse(projects) + + case projectListQueryRequest: ProjectListQueryRequest => + val projectList = projectService.queryProject(projectListQueryRequest) + new ProjectListQueryResponse(projectList) + } } diff --git a/dss-framework/dss-framework-project-server/src/main/scala/com/webank/wedatasphere/dss/framework/project/server/rpc/ProjectReceiverChooser.scala b/dss-framework/dss-framework-project-server/src/main/scala/com/webank/wedatasphere/dss/framework/project/server/rpc/ProjectReceiverChooser.scala index 6555d78c0a..9455d86b5a 100644 --- a/dss-framework/dss-framework-project-server/src/main/scala/com/webank/wedatasphere/dss/framework/project/server/rpc/ProjectReceiverChooser.scala +++ b/dss-framework/dss-framework-project-server/src/main/scala/com/webank/wedatasphere/dss/framework/project/server/rpc/ProjectReceiverChooser.scala @@ -53,6 +53,7 @@ class ProjectReceiverChooser extends ReceiverChooser { case _: RequestProjectImportOrchestrator => receiver case _: RequestProjectUpdateOrcVersion => receiver case _: ProxyUserCheckRequest => receiver + case _: ProjectInfoRequest => receiver case _ => None } } diff --git a/dss-framework/dss-framework-project-server/src/main/scala/com/webank/wedatasphere/dss/framework/project/server/service/BMLService.scala b/dss-framework/dss-framework-project-server/src/main/scala/com/webank/wedatasphere/dss/framework/project/server/service/BMLService.scala new file mode 100644 index 0000000000..c3f339a8c7 --- /dev/null +++ b/dss-framework/dss-framework-project-server/src/main/scala/com/webank/wedatasphere/dss/framework/project/server/service/BMLService.scala @@ -0,0 +1,263 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.framework.project.server.service + +import java.io.{ByteArrayInputStream, InputStream} +import java.util +import java.util.UUID + +import com.webank.wedatasphere.dss.common.exception.DSSErrorException +import com.webank.wedatasphere.dss.common.utils.IoUtils +import org.apache.linkis.bml.client.{BmlClient, BmlClientFactory} +import org.apache.linkis.bml.protocol.{BmlDownloadResponse, BmlUpdateResponse, BmlUploadResponse} +import org.apache.linkis.common.utils.{JavaLog, Logging, Utils} +import org.apache.commons.io.IOUtils +import org.springframework.stereotype.Component + +import scala.collection.JavaConversions._ + + +@Component("projectServerBMLService") +class BMLService extends Logging{ + @volatile + private var bmlClient: BmlClient = _ + + /** + * 单例，获取bml client + * @param userName + * @return + */ + private def getBmlClient(userName: String): BmlClient = { + if (bmlClient == null) synchronized { + if (bmlClient == null) { + bmlClient = BmlClientFactory.createBmlClient(userName) + } + } + bmlClient + } + + /** + * 上传一段文本 + * @param content 要上传的文本内容 + * @param fileName 指定生成的文件名 + * @param projectName 指定项目名 + * @return 返回BmlResource + */ + def upload(userName: String, content: String, fileName: String, projectName:String): util.Map[String, Object] = { + val inputStream = new ByteArrayInputStream(content.getBytes("utf-8")) + val client: BmlClient = getBmlClient(userName) + val resource: BmlUploadResponse = client.uploadShareResource(userName, projectName, fileName, inputStream) + if (!resource.isSuccess) throw new DSSErrorException(911113, "上传失败") + val map = new util.HashMap[String, Object] + map += "resourceId" -> resource.resourceId + map += "version" -> resource.version + } + + /** + * 上传二进制流 + * + * @param inputStream 要上传的二进制流 + * @param fileName 指定生成的文件名 + * @param projectName 指定项目名 + * @return 上传后的结果BmlResource + */ + def upload(userName: String, inputStream: InputStream, fileName: String, projectName:String): util.Map[String, Object] = { + val client: BmlClient = getBmlClient(userName) + val resource: BmlUploadResponse = client.uploadShareResource(userName, projectName, fileName, inputStream) + if (!resource.isSuccess) throw new DSSErrorException(911113, "上传失败") + val map = new util.HashMap[String, Object] + map += "resourceId" -> resource.resourceId + map += "version" -> resource.version + } + + /** + * 将特定资源更新为指定二进制流 + * @param resourceId 要更新的资源 + * @param inputStream 二进制流 + * @return 更新后的结果BmlResource + */ + def update(userName: String, resourceId: String, inputStream: InputStream): util.Map[String, Object] = { + val client: BmlClient = getBmlClient(userName) + val resource: BmlUpdateResponse = client.updateShareResource(userName, resourceId, "", inputStream) + if (!resource.isSuccess) throw new DSSErrorException(911114, "更新失败") + val map = new util.HashMap[String, Object] + map += "resourceId" -> resource.resourceId + map += "version" -> resource.version + } + + /** + * 将特定资源更新为指定文本 + * + * @param resourceId 要更新的资源 + * @param inputStream 文本内容 + * @return 更新后的结果BmlResource + */ + def update(userName: String, resourceId: String, content: String): util.Map[String, Object] = { + val inputStream = new ByteArrayInputStream(content.getBytes("utf-8")) + val client: BmlClient = getBmlClient(userName) + val resource: BmlUpdateResponse = client.updateShareResource(userName, resourceId, UUID.randomUUID().toString+".json", inputStream) + if (!resource.isSuccess) throw new DSSErrorException(911114, "更新失败") + val map = new util.HashMap[String, Object] + map += "resourceId" -> resource.resourceId + map += "version" -> resource.version + } + + /** + * 读取一个resource并转为文本返回。 + */ + def query(userName: String, resourceId: String, version: String): util.Map[String, Object] = { + val client: BmlClient = getBmlClient(userName) + var resource: BmlDownloadResponse = null + if (version == null) { + resource = client.downloadShareResource(userName, resourceId) + } else { + resource = client.downloadShareResource(userName, resourceId, version) + } + if (!resource.isSuccess) throw new DSSErrorException(911115, "下载失败") + val map = new util.HashMap[String, Object] + map += "path" -> resource.fullFilePath + map += "string" -> inputstremToString(resource.inputStream) + } + + /** + * 下载一个资源 + * @return 资源的路径和输入流 + */ + def download(userName: String, resourceId: String, version: String): util.Map[String, Object] = { + val client: BmlClient = getBmlClient(userName) + var resource: BmlDownloadResponse = null + if (version == null) { + resource = client.downloadShareResource(userName, resourceId) + } else { + resource = client.downloadShareResource(userName, resourceId, version) + } + if (!resource.isSuccess) throw new DSSErrorException(911115, "下载失败") + val map = new util.HashMap[String, Object] + map += "path" -> resource.fullFilePath + map += "is" -> resource.inputStream + } + + /** + * 把资源下载到指定的本地文件，返 + * @param path 本地文件全路径名 + */ + def downloadToLocalPath(userName: String, resourceId: String, version: String, path: String): String = { + val result = download(userName,resourceId,version) + val is = result.get("is").asInstanceOf[InputStream] + val os = IoUtils.generateExportOutputStream(path) + Utils.tryFinally(IOUtils.copy(is,os)){ + IOUtils.closeQuietly(os) + IOUtils.closeQuietly(is) + } + path + } + + def downloadAndGetText(userName: String, resourceId: String, version: String, path: String): String = { + downloadToLocalPath(userName,resourceId,version,path) + //因为下载到指定目录后返回的resource的Stream为null,只能从文件重新读取。 + val is = IoUtils.generateInputInputStream(path) + Utils.tryFinally(inputstremToString(is))(IOUtils.closeQuietly(is)) + } + + /** + * 读取本地文件，返回二进制输入流 + * + * @param userName 用户名 + * @param readPath 本地文件全路径名 + * @return 二进制输入流 + */ + def readLocalResourceFile(userName: String,readPath: String): InputStream ={ + IoUtils.generateInputInputStream(readPath) + } + + /** + * 读取本地指定文件名的一段文本 + * @param readPath 本地文件全路径名 + * @return 文件里的文本内容 + */ + def readLocalTextFile(userName: String,readPath: String): String ={ + var inputStream:InputStream = null + var text:String = null + Utils.tryFinally{ + inputStream = IoUtils.generateInputInputStream(readPath) + text = inputstremToString(inputStream) + }{ + IOUtils.closeQuietly(inputStream) + } + text + } + + private def inputstremToString(inputStream: InputStream): String = { + scala.io.Source.fromInputStream(inputStream).mkString + } + + /** + * 获取Bml client 实例 + * + * @param userName 用户名 + * @return bml client + */ + + /** + * 创建BML工程 + * @param username 用户名 + * @param projectName 工程名 + * @param editUsers 编辑者 + * @param accessUsers 访问者 + */ + def createBmlProject(username:String, projectName:String, editUsers:util.List[String], + accessUsers:util.List[String] ): Unit ={ + val client = getBmlClient(username) + val response = client.createBmlProject(username, projectName, accessUsers, editUsers) + if (response.isSuccess){ + logger.info(s"for user $username create bml project $projectName success") + }else{ + logger.error(s"for user $username create bml project $projectName failed") + } + } + + def attachResourceAndProject(username:String, projectName:String, resourceId:String):Unit = { + val client = getBmlClient(username) + val response = client.attachResourceAndProject(projectName, resourceId) + if (response.isSuccess){ + logger.info(s"attach $username and $projectName success") + }else{ + logger.error(s"attach $username and $projectName failed") + } + } + + /** + * 修改BML工程中的编辑者、访问者等权限 + * @param projectName 工程名 + * @param username 操作用户名 + * @param editUsers 新的编辑者名单 + * @param accessUsers 新的访问者名单 + */ + def updateProjectPriv(projectName:String, username:String, editUsers:util.List[String], + accessUsers:util.List[String]): Unit ={ + val client = getBmlClient(username) + val response = client.updateProjectPriv(username, projectName, editUsers, accessUsers) + if (response.isSuccess){ + logger.info(s"attach $username and $projectName success") + }else{ + logger.error(s"attach $username and $projectName failed") + } + } + + + +} diff --git a/dss-framework/dss-framework-proxy-user-service/pom.xml b/dss-framework/dss-framework-proxy-user-service/pom.xml index 7faa50d571..eca9ba4c44 100644 --- a/dss-framework/dss-framework-proxy-user-service/pom.xml +++ b/dss-framework/dss-framework-proxy-user-service/pom.xml @@ -5,7 +5,7 @@ dss-framework com.webank.wedatasphere.dss - 1.1.0.20-SNAPSHOT + 1.10.0-SNAPSHOT 4.0.0 diff --git a/dss-framework/dss-framework-workspace-server/pom.xml b/dss-framework/dss-framework-workspace-server/pom.xml index 2ff2f09049..858c1cd404 100644 --- a/dss-framework/dss-framework-workspace-server/pom.xml +++ b/dss-framework/dss-framework-workspace-server/pom.xml @@ -21,7 +21,7 @@ dss com.webank.wedatasphere.dss - 1.1.0.20-SNAPSHOT + 1.10.0-SNAPSHOT ../../pom.xml 4.0.0 @@ -45,6 +45,13 @@ provided + + com.webank.wedatasphere.dss + dss-git-common + ${dss.version} + provided + + org.apache.commons commons-lang3 @@ -110,6 +117,13 @@ 2.1.210 test + + + cn.hutool + hutool-all + 5.8.21 + + diff --git a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/DSSWorkspace.java b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/DSSWorkspace.java index bf9829176e..de39a5efa6 100644 --- a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/DSSWorkspace.java +++ b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/DSSWorkspace.java @@ -33,6 +33,9 @@ public class DSSWorkspace { private String lastUpdateUser; private String workspaceType; private int adminPermission; + private String enabledFlowKeywordsCheck; + + private String isDefaultReference; public int getAdminPermission() { return adminPermission; @@ -139,4 +142,19 @@ public void setLastUpdateUser(String lastUpdateUser) { } + public String getEnabledFlowKeywordsCheck() { + return enabledFlowKeywordsCheck; + } + + public void setEnabledFlowKeywordsCheck(String enabledFlowKeywordsCheck) { + this.enabledFlowKeywordsCheck = enabledFlowKeywordsCheck; + } + + public String getIsDefaultReference() { + return isDefaultReference; + } + + public void setIsDefaultReference(String isDefaultReference) { + this.isDefaultReference = isDefaultReference; + } } diff --git a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/itsm/ItsmRequest.java b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/itsm/ItsmRequest.java new file mode 100644 index 0000000000..c5fb23916c --- /dev/null +++ b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/itsm/ItsmRequest.java @@ -0,0 +1,116 @@ +package com.webank.wedatasphere.dss.framework.workspace.bean.itsm; + +import com.google.gson.Gson; +import com.google.gson.JsonArray; +import com.google.gson.JsonObject; +import com.google.gson.reflect.TypeToken; +import org.springframework.util.StringUtils; + +import java.lang.reflect.Type; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +public class ItsmRequest { + + private static final String dataKey = "dataList"; + + private String createDate; + private String createUser; + private String data; + private String externalId; + private String flowId; + private String operateUser; + private String requestTitle; + private String style; + private String taskId; + + + // getters和setters + public String getCreateDate() { + return createDate; + } + + public void setCreateDate(String createDate) { + this.createDate = createDate; + } + + public String getCreateUser() { + return createUser; + } + + public void setCreateUser(String createUser) { + this.createUser = createUser; + } + + public String getData() { + return data; + } + + public void setData(String data) { + this.data = data; + } + + public String getExternalId() { + return externalId; + } + + public void setExternalId(String externalId) { + this.externalId = externalId; + } + + public String getFlowId() { + return flowId; + } + + public void setFlowId(String flowId) { + this.flowId = flowId; + } + + public String getOperateUser() { + return operateUser; + } + + public void setOperateUser(String operateUser) { + this.operateUser = operateUser; + } + + public String getRequestTitle() { + return requestTitle; + } + + public void setRequestTitle(String requestTitle) { + this.requestTitle = requestTitle; + } + + public String getStyle() { + return style; + } + + public void setStyle(String style) { + this.style = style; + } + + public String getTaskId() { + return taskId; + } + + public void setTaskId(String taskId) { + this.taskId = taskId; + } + + public List> getDataList(){ + String dataStr = getData(); + if(!StringUtils.isEmpty(dataStr)) + { + Gson gson = new Gson(); + JsonObject jsonObject = gson.fromJson(dataStr, JsonObject.class); + JsonArray dataList = jsonObject.getAsJsonArray("dataList"); + + Type listType = new TypeToken>>(){}.getType(); + return gson.fromJson(dataList.toString(), listType); + } + return Collections.emptyList(); + } + +} diff --git a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/itsm/ItsmResponse.java b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/itsm/ItsmResponse.java new file mode 100644 index 0000000000..d36336857a --- /dev/null +++ b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/itsm/ItsmResponse.java @@ -0,0 +1,58 @@ +package com.webank.wedatasphere.dss.framework.workspace.bean.itsm; + +public class ItsmResponse { + + private String data; + private int retCode; + private String retDetail; + + public String getData() { + return data; + } + + public void setData(String data) { + this.data = data; + } + + public ItsmResponse data(String data) { + this.data = data; + return this; + } + + public int getRetCode() { + return retCode; + } + + public void setRetCode(int retCode) { + this.retCode = retCode; + } + + public ItsmResponse retCode(int retCode) { + this.retCode = retCode; + return this; + } + + public String getRetDetail() { + return retDetail; + } + + public void setRetDetail(String retDetail) { + this.retDetail = retDetail; + } + + public ItsmResponse retDetail(String retDetail) { + this.retDetail = retDetail; + return this; + } + + public static ItsmResponse ok(){ + ItsmResponse itsmResponse = new ItsmResponse(); + return itsmResponse.retCode(0); + } + public static ItsmResponse error(){ + ItsmResponse itsmResponse = new ItsmResponse(); + return itsmResponse.retCode(-1); + } + + +} diff --git a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/vo/DSSWorkspaceUsersDepartmentVo.java b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/vo/DSSWorkspaceUsersDepartmentVo.java new file mode 100644 index 0000000000..5f83c0b343 --- /dev/null +++ b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/vo/DSSWorkspaceUsersDepartmentVo.java @@ -0,0 +1,19 @@ +package com.webank.wedatasphere.dss.framework.workspace.bean.vo; + +import lombok.Data; + +import java.io.Serializable; +import java.util.List; +import java.util.Map; +@Data +public class DSSWorkspaceUsersDepartmentVo implements Serializable { + + private static final long serialVersionUID=1L; + private List editUsers; + + private List accessUsers; + + private List releaseUsers; + + +} diff --git a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/vo/DSSWorkspaceUsersVo.java b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/vo/DSSWorkspaceUsersVo.java index 82a562dc0f..e6412f2fd6 100644 --- a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/vo/DSSWorkspaceUsersVo.java +++ b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/vo/DSSWorkspaceUsersVo.java @@ -4,33 +4,45 @@ public class DSSWorkspaceUsersVo { + private List editUsers; private List accessUsers; private List releaseUsers; - public List getAccessUsers() { - return accessUsers; + private List createUsers; + + public void setAccessUsers(List accessUsers) { + this.accessUsers = accessUsers; + } + + public void setReleaseUsers(List releaseUsers) { + this.releaseUsers = releaseUsers; + } + + public void setEditUsers(List editUsers) { + this.editUsers = editUsers; } public List getEditUsers() { return editUsers; } + public List getAccessUsers() { + return accessUsers; + } + public List getReleaseUsers() { return releaseUsers; } - public void setAccessUsers(List accessUsers) { - this.accessUsers = accessUsers; - } - public void setReleaseUsers(List releaseUsers) { - this.releaseUsers = releaseUsers; + public List getCreateUsers() { + return createUsers; } - public void setEditUsers(List editUsers) { - this.editUsers = editUsers; + public void setCreateUsers(List createUsers) { + this.createUsers = createUsers; } } diff --git a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/vo/DepartmentUserTreeVo.java b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/vo/DepartmentUserTreeVo.java new file mode 100644 index 0000000000..0009cbd3fb --- /dev/null +++ b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/vo/DepartmentUserTreeVo.java @@ -0,0 +1,22 @@ +package com.webank.wedatasphere.dss.framework.workspace.bean.vo; + +import lombok.Data; + +import java.io.Serializable; +import java.util.List; + +@Data +public class DepartmentUserTreeVo implements Serializable { + private static final long serialVersionUID=1L; + private Integer id; + private String name; + private String type; + private List child; + + public DepartmentUserTreeVo(Integer id, String name, String type, List child){ + this.id = id; + this.name = name; + this.type = type; + this.child = child; + } +} diff --git a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/vo/DepartmentUserVo.java b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/vo/DepartmentUserVo.java new file mode 100644 index 0000000000..2de4a915a1 --- /dev/null +++ b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/bean/vo/DepartmentUserVo.java @@ -0,0 +1,35 @@ +package com.webank.wedatasphere.dss.framework.workspace.bean.vo; + +import java.io.Serializable; +import java.util.List; + +public class DepartmentUserVo implements Serializable { + private static final long serialVersionUID=1L; + private String name; + private String department; + private String office; + + public String getOffice() { + return office; + } + + public void setOffice(String office) { + this.office = office; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getDepartment() { + return department; + } + + public void setDepartment(String department) { + this.department = department; + } +} diff --git a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/constant/ApplicationConf.java b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/constant/ApplicationConf.java index df18f0a84c..7f673238ca 100644 --- a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/constant/ApplicationConf.java +++ b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/constant/ApplicationConf.java @@ -27,4 +27,8 @@ public class ApplicationConf { public static final CommonVars HOMEPAGE_URL = CommonVars.apply("wds.linkis.special.homepage.module.url", "/newHome?workspaceId="); + + public static final CommonVars ITSM_SECRETKEY = CommonVars.apply("wds.dss.itsm.secretkey","350965f1d6dfc38757cba3c34478163176aafcb2ed5ff2478d94a43b40d3ae42"); + + public static final CommonVars HOMEPAGE_DEFAULT_WORKSPACE = CommonVars.apply("wds.linkis.special.homepage.default.workspace","BRM_WORKSPACE,CF_WORKSPACE"); } diff --git a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/dao/DSSComponentRoleMapper.java b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/dao/DSSComponentRoleMapper.java index a52fb355b7..31b34d3491 100644 --- a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/dao/DSSComponentRoleMapper.java +++ b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/dao/DSSComponentRoleMapper.java @@ -30,4 +30,8 @@ public interface DSSComponentRoleMapper extends BaseMapper { public void insertBatch(@Param("list") List list); + List queryDSSComponentRole(@Param("workspaceId") int workspaceId,@Param("updateBy") String updateBy); + + void updateDSSComponentRoleById(@Param("list") List idList,@Param("updateBy") String updateBy); + } diff --git a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/dao/DSSMenuRoleMapper.java b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/dao/DSSMenuRoleMapper.java index d7f215aaea..cdcd8b34ef 100644 --- a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/dao/DSSMenuRoleMapper.java +++ b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/dao/DSSMenuRoleMapper.java @@ -28,4 +28,8 @@ public interface DSSMenuRoleMapper extends BaseMapper { public void insertBatch(@Param("list") List menuRoles); + List queryWorkspaceMenuRole(@Param("workspaceId") int workspaceId,@Param("updateBy") String updateBy); + + void updateWorkspaceMenuRoleById(@Param("list") List idList,@Param("updateBy") String updateBy); + } diff --git a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/dao/DSSWorkspaceMapper.java b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/dao/DSSWorkspaceMapper.java index 3396ceccf2..6f79db8a72 100644 --- a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/dao/DSSWorkspaceMapper.java +++ b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/dao/DSSWorkspaceMapper.java @@ -36,6 +36,8 @@ public interface DSSWorkspaceMapper { @Result(property = "createTime", column = "create_time"), @Result(property = "lastUpdateTime", column = "last_update_time"), @Result(property = "lastUpdateUser", column = "last_update_user"), + @Result(property = "enabledFlowKeywordsCheck",column = "enabled_flow_keywords_check"), + @Result(property = "isDefaultReference",column = "is_default_reference") }) DSSWorkspace getWorkspace(@Param("workspaceId") int workspaceId); @@ -79,4 +81,9 @@ void updateDepartmentsForWorkspace(@Param("workspaceId") Long workspaceId, @Para @Select("select * from dss_workspace_associate_departments") List getWorkspaceAssociateDepartments(); + + + void updateWorkSpace(DSSWorkspace dssWorkspace); + + void updateWorkspaceInfo(DSSWorkspace dssWorkspace); } diff --git a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/dao/DSSWorkspaceUserMapper.java b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/dao/DSSWorkspaceUserMapper.java index da7c66c2b0..4d42bb6f0b 100644 --- a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/dao/DSSWorkspaceUserMapper.java +++ b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/dao/DSSWorkspaceUserMapper.java @@ -122,4 +122,9 @@ void insertUserRoleInWorkspace(@Param("workspaceId") long workspaceId, @Param("r @MapKey("userName") Map getWorkspaceRolePrivByUsername(@Param("list") List list); + @Select("select distinct username from dss_workspace_user_role where workspace_id = #{workspaceId} and role_id = #{roleId} order by id desc") + List getWorkspaceUserByRoleId(@Param("workspaceId") Long workspaceId,@Param("roleId") Integer roleId); + + @Delete("delete from dss_project_user where username = #{username} ") + void deleteProjectUserByUserName(String userName); } diff --git a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/dao/NoticeMapper.java b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/dao/NoticeMapper.java index 268f3c77cc..fc7fddb662 100644 --- a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/dao/NoticeMapper.java +++ b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/dao/NoticeMapper.java @@ -1,9 +1,9 @@ -package com.webank.wedatasphere.dss.framework.workspace.dao; - -import com.baomidou.mybatisplus.core.mapper.BaseMapper; -import com.webank.wedatasphere.dss.framework.workspace.bean.NoticeContent; -import org.apache.ibatis.annotations.Mapper; - -@Mapper -public interface NoticeMapper extends BaseMapper { -} +//package com.webank.wedatasphere.dss.framework.workspace.dao; +// +//import com.baomidou.mybatisplus.core.mapper.BaseMapper; +//import com.webank.wedatasphere.dss.framework.workspace.bean.NoticeContent; +//import org.apache.ibatis.annotations.Mapper; +// +//@Mapper +//public interface NoticeMapper extends BaseMapper { +//} diff --git a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/dao/impl/DSSComponentRoleMapper.xml b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/dao/impl/DSSComponentRoleMapper.xml index 2eda66f6fc..519b213b03 100644 --- a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/dao/impl/DSSComponentRoleMapper.xml +++ b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/dao/impl/DSSComponentRoleMapper.xml @@ -28,4 +28,23 @@ + + + + + update dss_workspace_appconn_role set update_time = now(),updateby=#{updateBy} where id in + + #{id} + + + + \ No newline at end of file diff --git a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/dao/impl/DSSMenuRoleMapper.xml b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/dao/impl/DSSMenuRoleMapper.xml index dfa15e0ed8..de4e0b19e3 100644 --- a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/dao/impl/DSSMenuRoleMapper.xml +++ b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/dao/impl/DSSMenuRoleMapper.xml @@ -18,6 +18,7 @@ + insert into dss_workspace_menu_role (workspace_id,menu_id,role_id,priv, update_time,updateby) values @@ -27,5 +28,22 @@ + + + + update dss_workspace_menu_role set update_time = now(),updateby=#{updateBy} where id in + + #{id} + + \ No newline at end of file diff --git a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/dao/impl/DSSWorkspaceMapper.xml b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/dao/impl/DSSWorkspaceMapper.xml index 29df8eb462..69680d8c8c 100644 --- a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/dao/impl/DSSWorkspaceMapper.xml +++ b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/dao/impl/DSSWorkspaceMapper.xml @@ -54,7 +54,7 @@ + + update dss_workspace set create_by = #{createBy},last_update_time=#{lastUpdateTime}, + last_update_user=#{lastUpdateUser},description=#{description} + where id = #{id} + + + + update dss_workspace set enabled_flow_keywords_check = #{enabledFlowKeywordsCheck}, + is_default_reference = #{isDefaultReference} + where id = #{id} + + \ No newline at end of file diff --git a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/dao/impl/WorkspaceMapper.xml b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/dao/impl/WorkspaceMapper.xml index 5d86da48ea..d5d2c9562a 100644 --- a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/dao/impl/WorkspaceMapper.xml +++ b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/dao/impl/WorkspaceMapper.xml @@ -61,7 +61,9 @@ last_update_time, last_update_user, workspace_type, - admin_permission + admin_permission, + enabled_flow_keywords_check as enabledFlowKeywordsCheck, + is_default_reference as isDefaultReference FROM dss_workspace WHERE id = #{workspaceId} diff --git a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/restful/DSSWorkspaceRestful.java b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/restful/DSSWorkspaceRestful.java index 103fbfb726..92d9dd055a 100644 --- a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/restful/DSSWorkspaceRestful.java +++ b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/restful/DSSWorkspaceRestful.java @@ -16,35 +16,50 @@ package com.webank.wedatasphere.dss.framework.workspace.restful; +import cn.hutool.core.util.StrUtil; +import com.webank.wedatasphere.dss.common.StaffInfoGetter; import com.webank.wedatasphere.dss.common.auditlog.OperateTypeEnum; import com.webank.wedatasphere.dss.common.auditlog.TargetTypeEnum; + import com.webank.wedatasphere.dss.common.exception.DSSErrorException; import com.webank.wedatasphere.dss.common.utils.AuditLogUtils; import com.webank.wedatasphere.dss.common.utils.DSSCommonUtils; +import com.webank.wedatasphere.dss.common.utils.RpcAskUtils; import com.webank.wedatasphere.dss.framework.admin.service.DssAdminUserService; import com.webank.wedatasphere.dss.framework.workspace.bean.DSSWorkspace; -import com.webank.wedatasphere.dss.framework.workspace.bean.NoticeContent; import com.webank.wedatasphere.dss.framework.workspace.bean.dto.response.WorkspaceFavoriteVo; import com.webank.wedatasphere.dss.framework.workspace.bean.dto.response.WorkspaceMenuVo; +import com.webank.wedatasphere.dss.framework.workspace.bean.itsm.ItsmRequest; +import com.webank.wedatasphere.dss.framework.workspace.bean.itsm.ItsmResponse; import com.webank.wedatasphere.dss.framework.workspace.bean.request.CreateWorkspaceRequest; import com.webank.wedatasphere.dss.framework.workspace.bean.vo.DSSWorkspaceHomePageVO; import com.webank.wedatasphere.dss.framework.workspace.bean.vo.DSSWorkspaceOverviewVO; import com.webank.wedatasphere.dss.framework.workspace.bean.vo.DSSWorkspaceVO; +import com.webank.wedatasphere.dss.framework.workspace.service.DSSWorkspaceRoleCheckService; import com.webank.wedatasphere.dss.framework.workspace.service.DSSWorkspaceRoleService; import com.webank.wedatasphere.dss.framework.workspace.service.DSSWorkspaceService; -import com.webank.wedatasphere.dss.framework.workspace.service.NoticeService; import com.webank.wedatasphere.dss.framework.workspace.util.WorkspaceDBHelper; import com.webank.wedatasphere.dss.framework.workspace.util.WorkspaceUtils; +import com.webank.wedatasphere.dss.git.common.protocol.GitUserEntity; +import com.webank.wedatasphere.dss.git.common.protocol.request.GitConnectRequest; +import com.webank.wedatasphere.dss.git.common.protocol.request.GitUserInfoRequest; +import com.webank.wedatasphere.dss.git.common.protocol.request.GitUserUpdateRequest; +import com.webank.wedatasphere.dss.git.common.protocol.response.GitConnectResponse; +import com.webank.wedatasphere.dss.git.common.protocol.response.GitUserInfoResponse; +import com.webank.wedatasphere.dss.git.common.protocol.response.GitUserUpdateResponse; +import com.webank.wedatasphere.dss.sender.service.DSSSenderServiceFactory; import com.webank.wedatasphere.dss.standard.app.sso.Workspace; import com.webank.wedatasphere.dss.standard.common.exception.AppStandardWarnException; import com.webank.wedatasphere.dss.standard.sso.utils.SSOHelper; import org.apache.linkis.common.exception.ErrorException; +import org.apache.linkis.rpc.Sender; import org.apache.linkis.server.Message; import org.apache.linkis.server.security.SecurityFilter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.util.CollectionUtils; +import org.springframework.util.StringUtils; import org.springframework.web.bind.annotation.*; import javax.servlet.http.HttpServletRequest; @@ -53,6 +68,7 @@ import java.util.stream.Collectors; import static com.webank.wedatasphere.dss.framework.workspace.util.DSSWorkspaceConstant.WORKSPACE_ID_STR; +import static com.webank.wedatasphere.dss.standard.sso.utils.SSOHelper.USERNAME_NAME_COOKIE_KEY; @RequestMapping(path = "/dss/framework/workspace", produces = {"application/json"}) @RestController @@ -73,9 +89,9 @@ public class DSSWorkspaceRestful { private HttpServletRequest httpServletRequest; @Autowired private HttpServletResponse httpServletResponse; - @Autowired - private NoticeService noticeService; + private StaffInfoGetter staffInfoGetter; + @RequestMapping(path = "createWorkspace", method = RequestMethod.POST) public Message createWorkspace(@RequestBody CreateWorkspaceRequest createWorkspaceRequest) throws ErrorException { @@ -94,6 +110,122 @@ public Message createWorkspace(@RequestBody CreateWorkspaceRequest createWorkspa return Message.ok().data("workspaceId", workspaceId).data("workspaceName", workSpaceName); } + + /** + * 提供给ITSM 单使用， 新建或者修改工作空间信息接口 + */ + @RequestMapping(path = "updateWorkspace", method = RequestMethod.POST) + public ItsmResponse updateWorkspace(@RequestBody ItsmRequest itsmRequest, HttpServletRequest req, HttpServletResponse resp) { + LOGGER.info("itrtsm try to update workspace, itsm id:{}.", itsmRequest.getExternalId()); + // 获取请求头中的timestamp和sign字段 + String timestamp = req.getHeader("timeStamp"); + String sign = req.getHeader("sign"); + // 验证鉴权 + if (!WorkspaceUtils.validateAuth(timestamp, sign)) { + // 鉴权失败，返回错误信息 + LOGGER.error("Authentication failed."); + // 设置HTTP状态码为403 + resp.setStatus(HttpServletResponse.SC_FORBIDDEN); + return ItsmResponse.error().retDetail("Authentication failed."); + } + List> dataList = itsmRequest.getDataList(); + + // 鉴权成功 + if (dataList.isEmpty()) { + return ItsmResponse.error().retDetail("data is empty"); + } + + String createUser = itsmRequest.getCreateUser(); + + for (Map table : dataList) { + String workspaceName = table.get("workspaceName"); + + if(StrUtil.isEmpty(workspaceName)){ + LOGGER.error(workspaceName + " workspace is empty"); + return ItsmResponse.error().retDetail("工作空间信息不能为空！"); + } + + workspaceName = workspaceName.trim(); + // 判断命名空间是否存在 + boolean exists = dssWorkspaceService.existWorkspaceName(workspaceName); + String option = table.get("option"); + String desc = table.get("desc"); + // 工作空间owner为NULl，则获取提单人信息 + String workspaceOwner = table.getOrDefault("workspaceOwner", createUser).trim(); + String oldOwner = table.get("oldOwner"); + String newOwner = table.get("newOwner"); + + if ("add".equalsIgnoreCase(option)) { + // 新建 + if (exists) { + LOGGER.error(workspaceName + " workspace info is exists!"); + return ItsmResponse.error().retDetail(workspaceName + "同名工作空间已存在,不能再次创建！"); + } + + try { + // 判断用户是否可以设置为管理员 + if(!dssWorkspaceService.checkUserIfSettingAdmin(workspaceOwner)){ + return ItsmResponse.error().retDetail("无权限进行该操作"); + } + + int workspaceId = dssWorkspaceService.createWorkspace(workspaceName, "", workspaceOwner, desc, + null, "DSS", "project"); + + AuditLogUtils.printLog(workspaceOwner, workspaceId, workspaceName, TargetTypeEnum.WORKSPACE, workspaceId, workspaceName, + OperateTypeEnum.CREATE, itsmRequest); + + } catch (Exception e) { + LOGGER.info("{} workspace add fail , (工作空间 申请失败)", workspaceName); + LOGGER.error(e.getMessage()); + return ItsmResponse.error().retDetail(workspaceName + "工作空间新建失败!\n" + e.getMessage()); + } + + } else if ("modify".equalsIgnoreCase(option)) { + // 修改 + if (!exists) { + LOGGER.error(workspaceName + " workspace info not exists, not modify!"); + return ItsmResponse.error().retDetail(workspaceName + "工作空间名称不存在,不能进行修改！"); + } + + if(StrUtil.isEmpty(oldOwner) || StrUtil.isEmpty(newOwner)){ + return ItsmResponse.error().retDetail(workspaceName + "工作空间的新Owner和原Owner不能为空！"); + } + + oldOwner = oldOwner.trim(); + newOwner = newOwner.trim(); + + if(oldOwner.equalsIgnoreCase(newOwner)){ + return ItsmResponse.error().retDetail(workspaceName + "工作空间的新Owner和原Owner不能是同一个用户！"); + } + + try { + + // 判断用户是否可以设置为管理员 + if(!dssWorkspaceService.checkUserIfSettingAdmin(newOwner)){ + return ItsmResponse.error().retDetail("无权限进行该操作"); + } + + int workspaceId = dssWorkspaceService.transferWorkspace(workspaceName,oldOwner,newOwner,desc); + + AuditLogUtils.printLog(newOwner, workspaceId, workspaceName, TargetTypeEnum.WORKSPACE, workspaceId, workspaceName, + OperateTypeEnum.UPDATE, itsmRequest); + + } catch (Exception e) { + LOGGER.error("{} workspace modify fail , (工作空间 申请失败)", workspaceName); + LOGGER.error(e.getMessage(), e); + return ItsmResponse.error().retDetail(workspaceName + "工作空间信息修改失败!\n"+ e.getMessage()); + } + + } else { + return ItsmResponse.error().retDetail(workspaceName + "无法进行该操作,请正确选择操作选项!"); + } + + } + LOGGER.info("success to update workspace, itsm id:{}.", itsmRequest.getExternalId()); + return ItsmResponse.ok().retDetail("Success to update workspace"); + + } + /** * 返回所有office,格式为：a-b-c * @@ -106,6 +238,18 @@ public Message listAllDepartments(@RequestParam(value = WORKSPACE_ID_STR, requir return Message.ok().data("departmentWithOffices", departments); } + /** + * 返回所有department + * + * @param workspaceId + * @return + */ + @RequestMapping(path = "getAllDepartments", method = RequestMethod.GET) + public Message getAllDepartments(@RequestParam(value = WORKSPACE_ID_STR, required = false) String workspaceId) { + List departments = dssWorkspaceService.getAllDepartments(); + return Message.ok().data("departments", departments); + } + /** * 绑定工作空间和部门-科室关系 * @@ -201,15 +345,8 @@ public Message getAllWorkspaces() { */ @RequestMapping(path = "getWorkSpaceStr", method = RequestMethod.GET) public Message getWorkSpaceStr(@RequestParam(name = "workspaceName") String workspaceName) { - String username = SecurityFilter.getLoginUsername(httpServletRequest); - DSSWorkspace workspaceEntity; - try { - workspaceEntity = dssWorkspaceService.getWorkspacesByName(workspaceName, username); - } catch (DSSErrorException e) { - LOGGER.error("User {} get workspace {} failed.", username, workspaceName, e); - return Message.error(e); - } - Workspace workspace = SSOHelper.setAndGetWorkspace(httpServletRequest, httpServletResponse, workspaceEntity.getId(), workspaceName); + int workspaceId = dssWorkspaceService.getWorkspaceId(workspaceName); + Workspace workspace = SSOHelper.setAndGetWorkspace(httpServletRequest, httpServletResponse, workspaceId, workspaceName); return Message.ok("succeed.").data("workspaceStr", DSSCommonUtils.COMMON_GSON.toJson(workspace)); } @@ -223,7 +360,16 @@ public Message getWorkspacesById(@PathVariable("id") Long workspaceId) { LOGGER.error("User {} get workspace {} failed.", username, workspaceId, e); return Message.error(e); } + SSOHelper.setAndGetWorkspace(httpServletRequest, httpServletResponse, workspace.getId(), workspace.getName()); + + Set allUsernames = staffInfoGetter.getAllUsernames(); + if(allUsernames.contains(username)) { + SSOHelper.addUsernameCookie(httpServletRequest, httpServletResponse, username); + }else{ + SSOHelper.deleteCookieByName(httpServletRequest, httpServletResponse, USERNAME_NAME_COOKIE_KEY); + } + List roles = dssWorkspaceRoleService.getRoleInWorkspace(username, workspaceId.intValue()); if (roles == null || roles.isEmpty()) { LOGGER.error("username {}, in workspace {} roles are null or empty", username, workspaceId); @@ -339,11 +485,38 @@ public Message deleteFavorite(@PathVariable("workspaceId") Long workspaceId, @Pa return Message.ok().data("favoriteId", favoriteId); } - @GetMapping("getNotice") - public Message getNotice(){ - List noticeContent= noticeService.getNoticeContent(); - return Message.ok("公告获取成功").data("notices", noticeContent); - } + @RequestMapping(value = "/updateWorkspaceInfo",method = RequestMethod.POST) + public Message updateWorkspaceInfo(@RequestBody Map json) throws DSSErrorException{ + + Workspace workspace = SSOHelper.getWorkspace(httpServletRequest); + String username = SecurityFilter.getLoginUsername(httpServletRequest); + + String workspaceId= json.getOrDefault("workspaceId",String.valueOf(workspace.getWorkspaceId())); + + if(!workspaceId.equals(String.valueOf(workspace.getWorkspaceId()))){ + throw new DSSErrorException(90053,"当前工作空间与cookie中的不一致，重新刷新页面后在操作"); + } + + String enabledFlowKeywordsCheck = json.getOrDefault("enabledFlowKeywordsCheck","0"); + String isDefaultReference = json.getOrDefault("isDefaultReference","0"); + + DSSWorkspace dssWorkspace = new DSSWorkspace(); + dssWorkspace.setId(Integer.parseInt(workspaceId)); + dssWorkspace.setEnabledFlowKeywordsCheck(enabledFlowKeywordsCheck); + dssWorkspace.setIsDefaultReference(isDefaultReference); + + if (!dssWorkspaceService.checkAdminByWorkspace(username, dssWorkspace.getId())) { + return Message.error(String.format("%s 用户不是当前工作空间管理员,无权限进行该操作!",username)); + } + + dssWorkspaceService.updateWorkspaceInfo(dssWorkspace); + + AuditLogUtils.printLog(username, workspaceId, workspace.getWorkspaceName(), TargetTypeEnum.WORKSPACE, workspaceId, workspace.getWorkspaceName(), + OperateTypeEnum.UPDATE, json); + + return Message.ok().data("workspaceId", workspaceId); + + } } diff --git a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/restful/DSSWorkspaceUserRestful.java b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/restful/DSSWorkspaceUserRestful.java index 52d75a14dc..5af07a4820 100644 --- a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/restful/DSSWorkspaceUserRestful.java +++ b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/restful/DSSWorkspaceUserRestful.java @@ -20,16 +20,15 @@ import com.webank.wedatasphere.dss.common.auditlog.TargetTypeEnum; import com.webank.wedatasphere.dss.common.utils.AuditLogUtils; import com.webank.wedatasphere.dss.framework.admin.service.DssAdminUserService; +import com.webank.wedatasphere.dss.common.StaffInfo; import com.webank.wedatasphere.dss.framework.workspace.bean.request.DeleteWorkspaceUserRequest; import com.webank.wedatasphere.dss.framework.workspace.bean.request.RevokeUserRole; import com.webank.wedatasphere.dss.framework.workspace.bean.request.UpdateWorkspaceUserRequest; -import com.webank.wedatasphere.dss.framework.workspace.bean.vo.DSSWorkspaceRoleVO; -import com.webank.wedatasphere.dss.framework.workspace.bean.vo.DSSWorkspaceUserVO; -import com.webank.wedatasphere.dss.framework.workspace.bean.vo.DSSWorkspaceUsersVo; -import com.webank.wedatasphere.dss.framework.workspace.bean.vo.StaffInfoVO; +import com.webank.wedatasphere.dss.framework.workspace.bean.vo.*; import com.webank.wedatasphere.dss.framework.workspace.service.DSSWorkspaceRoleCheckService; import com.webank.wedatasphere.dss.framework.workspace.service.DSSWorkspaceService; import com.webank.wedatasphere.dss.framework.workspace.service.DSSWorkspaceUserService; +import com.webank.wedatasphere.dss.common.StaffInfoGetter; import com.webank.wedatasphere.dss.framework.workspace.util.WorkspaceDBHelper; import com.webank.wedatasphere.dss.standard.app.sso.Workspace; import com.webank.wedatasphere.dss.standard.common.exception.AppStandardWarnException; @@ -45,9 +44,7 @@ import org.springframework.web.bind.annotation.*; import javax.servlet.http.HttpServletRequest; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; +import java.util.*; import java.util.stream.Collectors; import static com.webank.wedatasphere.dss.framework.common.conf.TokenConf.HPMS_USER_TOKEN; @@ -71,16 +68,18 @@ public class DSSWorkspaceUserRestful { private HttpServletRequest httpServletRequest; @Autowired private DSSWorkspaceRoleCheckService roleCheckService; + @Autowired + private StaffInfoGetter staffInfoGetter; @RequestMapping(path = "getWorkspaceUsers", method = RequestMethod.GET) - public Message getWorkspaceUsers( @RequestParam(WORKSPACE_ID_STR) String workspaceId, + public Message getWorkspaceUsers(@RequestParam(WORKSPACE_ID_STR) String workspaceId, @RequestParam(required = false, name = "pageNow") Integer pageNow, @RequestParam(required = false, name = "pageSize") Integer pageSize, @RequestParam(required = false, name = "department") String department, @RequestParam(required = false, name = "userName") String username, @RequestParam(required = false, name = "roleName") String roleName) { //todo 获取工作空间中所有的用户以及他们的角色信息 - if(pageNow==null&&pageSize==null){ - pageSize=Integer.MAX_VALUE; - pageNow=1; + if (pageNow == null && pageSize == null) { + pageSize = Integer.MAX_VALUE; + pageNow = 1; } if (pageNow == null) { pageNow = 1; @@ -97,16 +96,33 @@ public Message getWorkspaceUsers( @RequestParam(WORKSPACE_ID_STR) String workspa } @RequestMapping(path = "getAllWorkspaceUsers", method = RequestMethod.GET) - public Message getAllWorkspaceUsers() { + public Message getAllWorkspaceUsers(@RequestParam(value = WORKSPACE_ID_STR, required = false) Integer workspaceId) { DSSWorkspaceUsersVo dssWorkspaceUsersVo = new DSSWorkspaceUsersVo(); + if(workspaceId == null){ + // workspaceId改为从cookie取 + workspaceId = (int) SSOHelper.getWorkspace(httpServletRequest).getWorkspaceId(); + } + List users = dssWorkspaceUserService.getAllWorkspaceUsers(workspaceId); + dssWorkspaceUsersVo.setAccessUsers(users); +// dssWorkspaceUsersVo.setEditUsers(dssWorkspaceUserService.getWorkspaceEditUsers(workspaceId)); +// dssWorkspaceUsersVo.setReleaseUsers(dssWorkspaceUserService.getWorkspaceReleaseUsers(workspaceId)); + dssWorkspaceUsersVo.setEditUsers(users); + dssWorkspaceUsersVo.setReleaseUsers(users); + dssWorkspaceUsersVo.setCreateUsers(users); + return Message.ok().data("users", dssWorkspaceUsersVo); + } + + @RequestMapping(path = "getAllWorkspaceUsersWithDepartment", method = RequestMethod.GET) + public Message getAllWorkspaceUsersWithDepartment() { + DSSWorkspaceUsersDepartmentVo dSSWorkspaceUsersDepartmentVo = new DSSWorkspaceUsersDepartmentVo(); // workspaceId改为从cookie取 int workspaceId = (int) SSOHelper.getWorkspace(httpServletRequest).getWorkspaceId(); - dssWorkspaceUsersVo.setAccessUsers(dssWorkspaceUserService.getAllWorkspaceUsers(workspaceId)); + dSSWorkspaceUsersDepartmentVo.setAccessUsers(dssWorkspaceUserService.getAllWorkspaceUsersDepartment(workspaceId)); // dssWorkspaceUsersVo.setEditUsers(dssWorkspaceUserService.getWorkspaceEditUsers(workspaceId)); // dssWorkspaceUsersVo.setReleaseUsers(dssWorkspaceUserService.getWorkspaceReleaseUsers(workspaceId)); - dssWorkspaceUsersVo.setEditUsers(dssWorkspaceUserService.getAllWorkspaceUsers(workspaceId)); - dssWorkspaceUsersVo.setReleaseUsers(dssWorkspaceUserService.getAllWorkspaceUsers(workspaceId)); - return Message.ok().data("users", dssWorkspaceUsersVo); + dSSWorkspaceUsersDepartmentVo.setEditUsers(dssWorkspaceUserService.getAllWorkspaceUsersDepartment(workspaceId)); + dSSWorkspaceUsersDepartmentVo.setReleaseUsers(dssWorkspaceUserService.getAllWorkspaceUsersDepartment(workspaceId)); + return Message.ok().data("users", dSSWorkspaceUsersDepartmentVo); } @@ -153,40 +169,40 @@ public Message addWorkspaceUser(@RequestBody UpdateWorkspaceUserRequest updateWo } dssUserService.insertIfNotExist(userName, workspace); dssWorkspaceUserService.addWorkspaceUser(roles, workspace.getWorkspaceId(), userName, creator, userId); - AuditLogUtils.printLog(userName,workspaceId, workspace.getWorkspaceName(), TargetTypeEnum.WORKSPACE,workspaceId, - workspace.getWorkspaceName(), OperateTypeEnum.ADD_USERS,updateWorkspaceUserRequest); + AuditLogUtils.printLog(userName, workspaceId, workspace.getWorkspaceName(), TargetTypeEnum.WORKSPACE, workspaceId, + workspace.getWorkspaceName(), OperateTypeEnum.ADD_USERS, updateWorkspaceUserRequest); return Message.ok(); } @RequestMapping(path = "updateWorkspaceUser", method = RequestMethod.POST) - public Message updateWorkspaceUser( @RequestBody UpdateWorkspaceUserRequest updateWorkspaceUserRequest) { + public Message updateWorkspaceUser(@RequestBody UpdateWorkspaceUserRequest updateWorkspaceUserRequest) { String creator = SecurityFilter.getLoginUsername(httpServletRequest); List roles = updateWorkspaceUserRequest.getRoles(); int workspaceId = updateWorkspaceUserRequest.getWorkspaceId(); - String workspaceName= dssWorkspaceService.getWorkspaceName((long)workspaceId); + String workspaceName = dssWorkspaceService.getWorkspaceName((long) workspaceId); String userName = updateWorkspaceUserRequest.getUserName(); if (!roleCheckService.checkRolesOperation(workspaceId, creator, userName, roles)) { return Message.error("无权限进行该操作"); } dssWorkspaceUserService.updateWorkspaceUser(roles, workspaceId, userName, creator); - AuditLogUtils.printLog(userName,workspaceId, workspaceName, TargetTypeEnum.WORKSPACE,workspaceId, - workspaceName, OperateTypeEnum.UPDATE_USERS,updateWorkspaceUserRequest); + AuditLogUtils.printLog(userName, workspaceId, workspaceName, TargetTypeEnum.WORKSPACE, workspaceId, + workspaceName, OperateTypeEnum.UPDATE_USERS, updateWorkspaceUserRequest); return Message.ok(); } @RequestMapping(path = "deleteWorkspaceUser", method = RequestMethod.POST) - public Message deleteWorkspaceUser( @RequestBody DeleteWorkspaceUserRequest deleteWorkspaceUserRequest) { + public Message deleteWorkspaceUser(@RequestBody DeleteWorkspaceUserRequest deleteWorkspaceUserRequest) { //todo 删除工作空间中的用户 String userName = deleteWorkspaceUserRequest.getUserName(); int workspaceId = deleteWorkspaceUserRequest.getWorkspaceId(); - String workspaceName= dssWorkspaceService.getWorkspaceName((long)workspaceId); + String workspaceName = dssWorkspaceService.getWorkspaceName((long) workspaceId); String creator = SecurityFilter.getLoginUsername(httpServletRequest); if (!roleCheckService.checkRolesOperation(workspaceId, creator, userName, new ArrayList<>())) { return Message.error("无权限进行该操作"); } dssWorkspaceUserService.deleteWorkspaceUser(userName, workspaceId); - AuditLogUtils.printLog(userName,workspaceId, workspaceName, TargetTypeEnum.WORKSPACE,workspaceId, - workspaceName, OperateTypeEnum.UPDATE_USERS,deleteWorkspaceUserRequest); + AuditLogUtils.printLog(userName, workspaceId, workspaceName, TargetTypeEnum.WORKSPACE, workspaceId, + workspaceName, OperateTypeEnum.UPDATE_USERS, deleteWorkspaceUserRequest); return Message.ok(); } @@ -196,8 +212,31 @@ public Message listAllUsers() { return Message.ok().data("users", dssUsers); } + /** + * 判断username是否离职 + * @param usernames + * @return 离职返回true，未离职false + */ + @RequestMapping(path = "isDismissed", method = RequestMethod.POST) + public Message isDismissed(@RequestBody Map> body) { + List usernames = body.get("usernames"); + List> userStatus = new ArrayList<>(usernames.size()); + for (String username : usernames) { + boolean isDismissed; + if(username.startsWith("hduser")||username.startsWith("WTSS_")){ + isDismissed = false; + }else{ + StaffInfo staffInfo= staffInfoGetter.getStaffInfoByUsername(username); + isDismissed = staffInfo != null && "2".equals(staffInfo.getStatus()); + } + Map tuple=Collections.singletonMap(username, isDismissed); + userStatus.add(tuple); + } + return Message.ok().data("isDismissed", userStatus); + } + @RequestMapping(path = "getWorkspaceIdByUserName", method = RequestMethod.GET) - public Message getWorkspaceIdByUserName( @RequestParam(required = false, name = "userName") String userName) { + public Message getWorkspaceIdByUserName(@RequestParam(required = false, name = "userName") String userName) { String loginUserName = SecurityFilter.getLoginUsername(httpServletRequest); String queryUserName = userName; if (StringUtils.isEmpty(userName)) { @@ -212,10 +251,10 @@ public Message getWorkspaceIdByUserName( @RequestParam(required = false, name = public Message getWorkspaceUserRole(@RequestParam(name = "userName") String username) { String token = ModuleUserUtils.getToken(httpServletRequest); if (StringUtils.isNotBlank(token)) { - if(!token.equals(HPMS_USER_TOKEN)){ + if (!token.equals(HPMS_USER_TOKEN)) { return Message.error("Token:" + token + " has no permission to get user info."); } - }else { + } else { return Message.error("User:" + username + " has no permission to get user info."); } List userRoles = dssWorkspaceUserService.getUserRoleByUserName(username); @@ -223,18 +262,18 @@ public Message getWorkspaceUserRole(@RequestParam(name = "userName") String user } @RequestMapping(path = "/clearUser", method = RequestMethod.GET) - public Message clearUser(@RequestParam("userName") String userName) { + public Message clearUser(@RequestParam("userName") String userName,@RequestParam(name="handover_ename",required = false) String handover_ename) { String token = ModuleUserUtils.getToken(httpServletRequest); if (StringUtils.isNotBlank(token)) { - if(!token.equals(HPMS_USER_TOKEN)){ + if (!token.equals(HPMS_USER_TOKEN)) { return Message.error("Token:" + token + " has no permission to clear user."); } - }else { + } else { return Message.error("User:" + userName + " has no permission to clear user."); } dssWorkspaceUserService.clearUserByUserName(userName); - AuditLogUtils.printLog(userName,null, null, TargetTypeEnum.WORKSPACE_ROLE,null, - "clearUser", OperateTypeEnum.DELETE,null); + AuditLogUtils.printLog(userName, null, null, TargetTypeEnum.WORKSPACE_ROLE, null, + "clearUser", OperateTypeEnum.DELETE, null); return Message.ok("清理成功"); } @@ -246,16 +285,52 @@ public Message revokeUserRole(@Validated @RequestBody RevokeUserRole revokeUserR Integer[] roleIds = revokeUserRole.getRoleIds(); String token = ModuleUserUtils.getToken(httpServletRequest); if (StringUtils.isNotBlank(token)) { - if(!token.equals(HPMS_USER_TOKEN)){ + if (!token.equals(HPMS_USER_TOKEN)) { return Message.error("Token:" + token + " has no permission to revoke userRole."); } - }else { + } else { return Message.error("User:" + userName + " has no permission to revoke userRole."); } dssWorkspaceUserService.revokeUserRoles(userName, workspaceIds, roleIds); - AuditLogUtils.printLog(userName,null, null, TargetTypeEnum.WORKSPACE_ROLE,null, - "revokeUserRole", OperateTypeEnum.DELETE,revokeUserRole); + AuditLogUtils.printLog(userName, null, null, TargetTypeEnum.WORKSPACE_ROLE, null, + "revokeUserRole", OperateTypeEnum.DELETE, revokeUserRole); return Message.ok("回收成功"); } + + + /** + * 根据角色获取工作空间用户信息 + */ + @RequestMapping(path = "/getWorkspaceUserByRole", method = RequestMethod.GET) + public Message getWorkspaceUserByRole(@RequestParam(value = WORKSPACE_ID_STR, required = false) Long workspaceId, + @RequestParam(value = "roleId", required = false, defaultValue = "1") Integer roleId) { + + if (workspaceId == null) { + workspaceId = SSOHelper.getWorkspace(httpServletRequest).getWorkspaceId(); + } + LOGGER.info(String.format("getWorkspaceUserByRole workspaceId is %s, roleId is %s", workspaceId, roleId)); + List users = dssWorkspaceUserService.getWorkspaceUserByRoleId(workspaceId, roleId); + String workspaceName = dssWorkspaceService.getWorkspaceName(workspaceId); + return Message.ok().data("users", users).data("workspaceName", workspaceName); + + } + + @RequestMapping(path = "/getWorkspaceProxyUsers", method = RequestMethod.GET) + public Message getWorkspaceProxyUsers(@RequestParam(value = WORKSPACE_ID_STR, required = false) Long workspaceId) { + + + if (workspaceId == null) { + workspaceId = SSOHelper.getWorkspace(httpServletRequest).getWorkspaceId(); + } + + LOGGER.info("getWorkspaceProxyUsers workspaceId is {}", workspaceId); + + List users = dssWorkspaceUserService.getAllWorkspaceUsers(workspaceId); + + return Message.ok().data("users", users); + + } + + } diff --git a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/service/DSSWorkspaceRoleCheckService.java b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/service/DSSWorkspaceRoleCheckService.java index 9ac67cc63f..4083ccc5e2 100644 --- a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/service/DSSWorkspaceRoleCheckService.java +++ b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/service/DSSWorkspaceRoleCheckService.java @@ -14,4 +14,7 @@ public interface DSSWorkspaceRoleCheckService { */ boolean checkRolesOperation(int workspaceId, String loginUser, String username, List roles); + + boolean checkUserRolesOperation(String username,List roles); + } diff --git a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/service/DSSWorkspaceService.java b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/service/DSSWorkspaceService.java index 4d57123d01..bf8331ae68 100644 --- a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/service/DSSWorkspaceService.java +++ b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/service/DSSWorkspaceService.java @@ -64,6 +64,8 @@ List getWorkspaceUsers(String workspaceId, String department String getWorkspaceName(Long workspaceId); + int getWorkspaceId(String workspaceName); + /** * 工作空间管理员权限判断(is_admin=1的用户才拥有工作空间操作权限) * @param userName @@ -81,6 +83,8 @@ List getWorkspaceUsers(String workspaceId, String department List getAllDepartmentWithOffices(); + List getAllDepartments(); + void associateDepartments(Long workspaceId, String departments, String roles,String user) throws DSSErrorException; DSSWorkspaceAssociateDepartments getAssociateDepartmentsInfo(Long workspaceId); @@ -111,4 +115,10 @@ List getWorkspaceAppConns(Workspace workspace, Long workspaceId boolean checkAdminByWorkspace(String username, int workspaceId); PageInfo getAllUserPrivs(Integer currentPage, Integer pageSize); + + int transferWorkspace(String workspaceName,String oldOwner,String newOwner,String desc) throws DSSErrorException; + + boolean checkUserIfSettingAdmin(String username); + + void updateWorkspaceInfo(DSSWorkspace dssWorkspace); } diff --git a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/service/DSSWorkspaceUserService.java b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/service/DSSWorkspaceUserService.java index c3b3e95fe1..6bb81c0a30 100644 --- a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/service/DSSWorkspaceUserService.java +++ b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/service/DSSWorkspaceUserService.java @@ -19,6 +19,7 @@ import com.webank.wedatasphere.dss.common.entity.PageInfo; import com.webank.wedatasphere.dss.framework.workspace.bean.vo.DSSWorkspaceRoleVO; +import com.webank.wedatasphere.dss.framework.workspace.bean.vo.DepartmentUserTreeVo; import com.webank.wedatasphere.dss.framework.workspace.bean.vo.StaffInfoVO; import java.util.List; @@ -50,7 +51,10 @@ public interface DSSWorkspaceUserService { List listAllDSSUsers(); List getAllWorkspaceUsers(long workspaceId); - PageInfo getAllWorkspaceUsersPage(long workspaceId, Integer pageNow, Integer pageSize); + + List getAllWorkspaceUsersDepartment(long workspaceId) ; + + PageInfo getAllWorkspaceUsersPage(long workspaceId, Integer pageNow, Integer pageSize,boolean paged); List getUserWorkspaceIds(String userName); @@ -66,4 +70,8 @@ public interface DSSWorkspaceUserService { void clearUserByUserName(String userName); void revokeUserRoles(String userName, Integer[] workspaceIds, Integer[] roleIds); + + List getWorkspaceUserByRoleId(Long workspaceId,Integer roleId); + + void updateWorkspaceRole(List roles, long workspaceId,String newOwner); } diff --git a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/service/NoticeService.java b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/service/NoticeService.java index 24e03763f5..e66d07a12c 100644 --- a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/service/NoticeService.java +++ b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/service/NoticeService.java @@ -1,11 +1,11 @@ -package com.webank.wedatasphere.dss.framework.workspace.service; - -import com.webank.wedatasphere.dss.framework.workspace.bean.NoticeContent; - -import java.util.List; - -public interface NoticeService { - - List getNoticeContent(); - -} +//package com.webank.wedatasphere.dss.framework.workspace.service; +// +//import com.webank.wedatasphere.dss.framework.workspace.bean.NoticeContent; +// +//import java.util.List; +// +//public interface NoticeService { +// +// List getNoticeContent(); +// +//} diff --git a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/service/impl/DSSWorkspaceRoleCheckServiceImpl.java b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/service/impl/DSSWorkspaceRoleCheckServiceImpl.java index 415718ad6d..a0211d1481 100644 --- a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/service/impl/DSSWorkspaceRoleCheckServiceImpl.java +++ b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/service/impl/DSSWorkspaceRoleCheckServiceImpl.java @@ -25,4 +25,11 @@ public boolean checkRolesOperation(int workspaceId, String loginUser, String use (dssWorkspaceService.isAdminUser((long) workspaceId, loginUser) && ((!dssWorkspaceService.isAdminUser((long) workspaceId, username) && !roles.contains(1)) || (StringUtils.equals(loginUser, username) && roles.contains(1)))); } + + @Override + public boolean checkUserRolesOperation(String username, List roles) { + return true; + } + + } diff --git a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/service/impl/DSSWorkspaceServiceImpl.java b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/service/impl/DSSWorkspaceServiceImpl.java index 8da745db45..a92db04432 100644 --- a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/service/impl/DSSWorkspaceServiceImpl.java +++ b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/service/impl/DSSWorkspaceServiceImpl.java @@ -16,6 +16,7 @@ package com.webank.wedatasphere.dss.framework.workspace.service.impl; +import cn.hutool.core.util.ObjectUtil; import com.github.pagehelper.PageHelper; import com.github.pagehelper.PageInfo; import com.github.pagehelper.page.PageMethod; @@ -23,7 +24,9 @@ import com.webank.wedatasphere.dss.appconn.core.ext.OnlySSOAppConn; import com.webank.wedatasphere.dss.appconn.manager.AppConnManager; import com.webank.wedatasphere.dss.appconn.manager.utils.AppInstanceConstants; +import com.webank.wedatasphere.dss.common.StaffInfoGetter; import com.webank.wedatasphere.dss.common.exception.DSSErrorException; +import com.webank.wedatasphere.dss.common.exception.DSSRuntimeException; import com.webank.wedatasphere.dss.common.label.EnvDSSLabel; import com.webank.wedatasphere.dss.framework.admin.conf.AdminConf; import com.webank.wedatasphere.dss.framework.admin.service.DssAdminUserService; @@ -37,23 +40,20 @@ import com.webank.wedatasphere.dss.framework.workspace.constant.ApplicationConf; import com.webank.wedatasphere.dss.framework.workspace.dao.*; import com.webank.wedatasphere.dss.framework.workspace.exception.DSSWorkspaceDuplicateNameException; -import com.webank.wedatasphere.dss.framework.workspace.service.DSSWorkspaceRoleService; -import com.webank.wedatasphere.dss.framework.workspace.service.DSSWorkspaceService; -import com.webank.wedatasphere.dss.framework.workspace.service.DSSWorkspaceUserService; -import com.webank.wedatasphere.dss.framework.workspace.service.StaffInfoGetter; +import com.webank.wedatasphere.dss.framework.workspace.service.*; import com.webank.wedatasphere.dss.framework.workspace.util.CommonRoleEnum; import com.webank.wedatasphere.dss.framework.workspace.util.DSSWorkspaceConstant; import com.webank.wedatasphere.dss.framework.workspace.util.WorkspaceDBHelper; -import com.webank.wedatasphere.dss.framework.workspace.util.WorkspaceServerConstant; +import com.webank.wedatasphere.dss.common.conf.WorkspaceServerConstant; import com.webank.wedatasphere.dss.standard.app.sso.Workspace; import com.webank.wedatasphere.dss.standard.app.sso.builder.SSOUrlBuilderOperation; import com.webank.wedatasphere.dss.standard.common.desc.AppInstance; import com.webank.wedatasphere.dss.standard.sso.utils.SSOHelper; +import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.collections4.MapUtils; import org.apache.commons.lang.ArrayUtils; import org.apache.commons.lang3.StringUtils; import org.apache.linkis.common.exception.ErrorException; -import org.apache.linkis.protocol.util.ImmutablePair; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -97,6 +97,9 @@ public class DSSWorkspaceServiceImpl implements DSSWorkspaceService { @Autowired private DSSWorkspaceRoleService dssWorkspaceRoleService; + @Autowired + private DSSWorkspaceRoleCheckService roleCheckService; + //创建工作空间 @Override @Transactional(rollbackFor = Exception.class) @@ -122,7 +125,7 @@ public int createWorkspace(String workspaceName, String tags, String userName, } Long userId = dssWorkspaceUserMapper.getUserID(userName); dssWorkspaceUserMapper.insertUserRoleInWorkspace(dssWorkspace.getId(), - workspaceDBHelper.getRoleIdByName(CommonRoleEnum.ADMIN.getName()),new Date(), userName, userName, userId, userName); + workspaceDBHelper.getRoleIdByName(CommonRoleEnum.ADMIN.getName()), new Date(), userName, userName, userId, userName); dssMenuRoleMapper.insertBatch(workspaceDBHelper.generateDefaultWorkspaceMenuRole(dssWorkspace.getId(), userName)); dssWorkspaceHomepageMapper.insertBatch(workspaceDBHelper.generateDefaultWorkspaceHomepage(dssWorkspace.getId(), userName)); dssComponentRoleMapper.insertBatch(workspaceDBHelper.generateDefaultWorkspaceComponentPrivs(dssWorkspace.getId(), userName)); @@ -138,11 +141,70 @@ private void setAllRolesToWorkspaceCreator(int workspaceId, String userName) { dssWorkspaceUserService.updateWorkspaceUser(roleIds, workspaceId, userName, userName); } + @Transactional(rollbackFor = Throwable.class) + @Override + public int transferWorkspace(String workspaceName, String oldOwner, String newOwner, String desc) throws DSSErrorException { + + List dssWorkspaces = workspaceMapper.findByWorkspaceName(workspaceName); + + if (dssWorkspaces == null || dssWorkspaces.isEmpty()) { + throw new DSSFrameworkWarnException(30021, workspaceName + "工作空间不存在!"); + } else if (dssWorkspaces.size() > 1) { + throw new DSSFrameworkWarnException(30021, "Too many workspaces named " + workspaceName); + } + + DSSWorkspace dssWorkspace = dssWorkspaces.get(0); + + LOGGER.info("workspace name is {}, oldOwner is {}, newOwner is {}", workspaceName, oldOwner, newOwner); + dssWorkspace.setCreateBy(newOwner); + dssWorkspace.setLastUpdateTime(new Date()); + dssWorkspace.setLastUpdateUser(newOwner); + dssWorkspace.setDescription(desc); + LOGGER.info("updateWorkSpace name is {}", dssWorkspace.getName()); + // 更改创建者 和更新时间 + dssWorkspaceMapper.updateWorkSpace(dssWorkspace); + + List dssWorkspaceMenuRoleList = dssMenuRoleMapper.queryWorkspaceMenuRole(dssWorkspace.getId(), oldOwner); + + if (CollectionUtils.isNotEmpty(dssWorkspaceMenuRoleList)) { + LOGGER.info("updateWorkspaceMenuRoleById workspace name is {}, menuRoleIdList size is {}", workspaceName, dssWorkspaceMenuRoleList.size()); + List menuRoleIdList = dssWorkspaceMenuRoleList.stream().map(DSSWorkspaceMenuRole::getId).collect(Collectors.toList()); + dssMenuRoleMapper.updateWorkspaceMenuRoleById(menuRoleIdList, newOwner); + + } + + List dssWorkspaceComponentList = dssComponentRoleMapper.queryDSSComponentRole(dssWorkspace.getId(), oldOwner); + + if (CollectionUtils.isNotEmpty(dssWorkspaceComponentList)) { + LOGGER.info("updateDSSComponentRoleById workspace name is {}, componentList size is {}", workspaceName, dssWorkspaceComponentList.size()); + List componetPrivIdList = dssWorkspaceComponentList.stream().map(DSSWorkspaceComponentPriv::getId).collect(Collectors.toList()); + dssComponentRoleMapper.updateDSSComponentRoleById(componetPrivIdList, newOwner); + } + + List roleIds = dssWorkspaceService.getWorkspaceRoles(dssWorkspace.getId()) + .stream() + .map(DSSWorkspaceRoleVO::getRoleId) + .collect(Collectors.toList()); + dssWorkspaceUserService.updateWorkspaceRole(roleIds, dssWorkspace.getId(), newOwner); + + LOGGER.info("success transfer workspace {} ", workspaceName); + return dssWorkspace.getId(); + } + + + @Override + public boolean checkUserIfSettingAdmin(String username){ + + int roleId= workspaceDBHelper.getRoleIdByName(CommonRoleEnum.ADMIN.getName()); + // 判断用户是否可以设置为管理员(代理和非合作伙伴不能设置为管理员) + return roleCheckService.checkUserRolesOperation(username, Collections.singletonList(roleId)); + } //获取所有的工作空间 @Override public List getWorkspaces(String userName) { List workspaces = dssWorkspaceMapper.getWorkspaces(userName); + workspaces = sortDssWorkspacesIndex(workspaces); //用于展示demo的工作空间是不应该返回的,除非用户是管理员 if (dssWorkspaceUserMapper.isAdmin(userName) == 1) { return workspaces; @@ -162,24 +224,23 @@ public List getWorkspaces(String userName) { @Override public DSSWorkspaceHomePageVO getWorkspaceHomePage(String userName, String moduleName) throws DSSErrorException { List dssWorkspaces = dssWorkspaceMapper.getWorkspaces(userName); - List workspaceIds = dssWorkspaces.stream().filter(l -> !DEFAULT_DEMO_WORKSPACE_NAME.getValue().equals(l.getName())) - .map(DSSWorkspace::getId).collect(Collectors.toList()); + List workspaceIds = sortDssWorkspacesIndex(dssWorkspaces).stream().map(DSSWorkspace::getId).collect(Collectors.toList()); DSSWorkspaceHomePageVO dssWorkspaceHomePageVO = new DSSWorkspaceHomePageVO(); if (workspaceIds.size() == 0) { Long userId = dssWorkspaceUserMapper.getUserID(userName); int workspaceId = dssWorkspaceMapper.getWorkspaceIdByName(DSSWorkspaceConstant.DEFAULT_WORKSPACE_NAME.getValue()); Integer analyserRole = workspaceDBHelper.getRoleIdByName(CommonRoleEnum.ANALYSER.getName()); - dssWorkspaceUserMapper.insertUserRoleInWorkspace(workspaceId,analyserRole ,new Date(), + dssWorkspaceUserMapper.insertUserRoleInWorkspace(workspaceId, analyserRole, new Date(), userName, "system", userId, "system"); Integer workspace0xId = dssWorkspaceMapper.getWorkspaceIdByName(DSSWorkspaceConstant.DEFAULT_0XWORKSPACE_NAME.getValue()); if (workspace0xId != null) { - dssWorkspaceUserMapper.insertUserRoleInWorkspace(workspace0xId, analyserRole ,new Date(), + dssWorkspaceUserMapper.insertUserRoleInWorkspace(workspace0xId, analyserRole, new Date(), userName, "system", userId, "system"); } //todo 初始化做的各项事情改为listener模式 //为新用户自动加入部门关联的工作空间 - joinWorkspaceForNewUser(userName, userId); + joinWorkspaceForNewUser(userName, userId, workspaceId, workspace0xId, analyserRole); //若路径没有workspaceId会出现页面没有首页、管理台 String homepageUrl = "/home" + "?workspaceId=" + workspaceId; @@ -197,7 +258,7 @@ public DSSWorkspaceHomePageVO getWorkspaceHomePage(String userName, String modul if ("/workspace".equals(homepageUrl)) { //兼容旧的默认首页配置 homepageUrl = "/workspaceHome"; - }else if("/workspaceHome/scheduleCenter".equals(homepageUrl)){ + } else if ("/workspaceHome/scheduleCenter".equals(homepageUrl)) { //兼容旧的生产中心首页配置 homepageUrl = "/scheduleCenter"; } @@ -214,6 +275,13 @@ public DSSWorkspaceHomePageVO getWorkspaceHomePage(String userName, String modul dssWorkspaceHomePageVO.setRoleName(workspaceDBHelper.getRoleNameById(minRoleId)); } else { String homepageUrl = "/workspaceHome?workspaceId=" + workspaceIds.get(0); + String[] defaultWorkspaceName = ApplicationConf.HOMEPAGE_DEFAULT_WORKSPACE.getValue().split(","); + DSSWorkspace defaultWorkspace = dssWorkspaces.stream() + .filter(workspcae -> Arrays.stream(defaultWorkspaceName).anyMatch(n->n.equalsIgnoreCase(workspcae.getName()))).findFirst().orElse(null); + if(ObjectUtil.isNotEmpty(defaultWorkspace)){ + homepageUrl = "/workspaceHome?workspaceId=" + defaultWorkspace.getId(); + } + if (ApplicationConf.HOMEPAGE_MODULE_NAME.getValue().equalsIgnoreCase(moduleName)) { homepageUrl = ApplicationConf.HOMEPAGE_URL.getValue() + workspaceIds.get(0); } @@ -223,6 +291,24 @@ public DSSWorkspaceHomePageVO getWorkspaceHomePage(String userName, String modul return dssWorkspaceHomePageVO; } + private List sortDssWorkspacesIndex(List dssWorkspaces) { + List dssWorkspaceReq = dssWorkspaces.stream().filter(l -> !DEFAULT_DEMO_WORKSPACE_NAME.getValue().equals(l.getName())) + .collect(Collectors.toList()); + if(dssWorkspaces.size()>1){ + dssWorkspaceReq = dssWorkspaces.stream().filter(l -> !DSSWorkspaceConstant.DEFAULT_WORKSPACE_NAME.getValue().equals(l.getName()) + &&!DSSWorkspaceConstant.DEFAULT_0XWORKSPACE_NAME.getValue().equals(l.getName())) + .collect(Collectors.toList()); + Map workspaceMap = dssWorkspaces.stream().collect(Collectors.toMap(DSSWorkspace::getName, w->w)); + if(workspaceMap.keySet().contains(DSSWorkspaceConstant.DEFAULT_WORKSPACE_NAME.getValue())){ + dssWorkspaceReq.add(workspaceMap.get(DSSWorkspaceConstant.DEFAULT_WORKSPACE_NAME.getValue())); + } + if(workspaceMap.keySet().contains(DSSWorkspaceConstant.DEFAULT_0XWORKSPACE_NAME.getValue())){ + dssWorkspaceReq.add(workspaceMap.get(DSSWorkspaceConstant.DEFAULT_0XWORKSPACE_NAME.getValue())); + } + } + return dssWorkspaceReq; + } + @Override public List getWorkspaceUsers(String workspaceId, String department, String username, String roleName, int pageNow, int pageSize, List total) { @@ -255,8 +341,12 @@ private DSSWorkspaceUserVO changeToUserVO(DSSWorkspaceUser dssWorkspaceUser, Lis String orgFullName = staffInfoGetter.getFullOrgNameByUsername(userName); if (StringUtils.isNotEmpty(orgFullName)) { try { - String departmentName = orgFullName.split(WorkspaceServerConstant.DEFAULT_STAFF_SPLIT)[0]; - String officeName = orgFullName.split(WorkspaceServerConstant.DEFAULT_STAFF_SPLIT)[1]; + String[] fullNameArray = orgFullName.split(WorkspaceServerConstant.DEFAULT_STAFF_SPLIT); + String departmentName = fullNameArray[0]; + String officeName = ""; + if (fullNameArray.length > 1) { + officeName = fullNameArray[1]; + } vo.setDepartment(departmentName); vo.setOffice(officeName); } catch (Exception e) { @@ -419,6 +509,14 @@ public DSSWorkspaceHomepageSettingVO getWorkspaceHomepageSettings(int workspaceI public String getWorkspaceName(Long workspaceId) { return dssWorkspaceMapper.getWorkspaceNameById(workspaceId); } + @Override + public int getWorkspaceId(String workspaceName) { + Integer workspaceId= dssWorkspaceMapper.getWorkspaceIdByName(workspaceName); + if (workspaceId == null || workspaceId < 0) { + throw new DSSRuntimeException("workspace is not exist.workspaceName:" + workspaceName); + } + return workspaceId; + } @Override public boolean checkAdmin(String userName) { @@ -437,6 +535,13 @@ public List getAllDepartmentWithOffices() { return allDepartments; } + @Override + public List getAllDepartments() { + List allDepartments = staffInfoGetter.getAllDepartments().stream() + .map(department -> department.split(WorkspaceServerConstant.DEFAULT_STAFF_SPLIT)[0]).distinct().collect(Collectors.toList()); + return allDepartments; + } + @Override public void associateDepartments(Long workspaceId, String departments, String roles, String user) throws DSSErrorException { List userRoles = dssWorkspaceUserMapper.getRoleInWorkspace(workspaceId.intValue(), user); @@ -666,9 +771,11 @@ public boolean isAdminUser(Long workspaceId, String username) { @Override public PageInfo getAllUserPrivs(Integer currentPage, Integer pageSize) { - PageMethod.startPage(currentPage,pageSize); + PageMethod.startPage(currentPage, pageSize); List users = dssWorkspaceUserMapper.getAllUsers(); Map userRolePrivMap = dssWorkspaceUserMapper.getWorkspaceRolePrivByUsername(users); + Set allUsernames = staffInfoGetter.getAllUsernames(); + users = users.stream().filter(user->allUsernames.contains(user.getUserName())).collect(Collectors.toList()); //处理用户没有角色权限时只返回用户 for (DSSUserRoleComponentPriv user : users) { if (MapUtils.isEmpty(userRolePrivMap) || Objects.isNull(userRolePrivMap.get(user.getUserName()))) { @@ -679,7 +786,8 @@ public PageInfo getAllUserPrivs(Integer currentPage, I return new PageInfo<>(users); } - private void joinWorkspaceForNewUser(String userName, Long userId) { + + private void joinWorkspaceForNewUser(String userName, Long userId, int workspaceId, Integer workspace0xId, Integer analyserRole) { String userOrgName = staffInfoGetter.getFullOrgNameByUsername(userName); String orgName = userOrgName.split("-")[0]; List workspaceAssociateDepartments = dssWorkspaceMapper.getWorkspaceAssociateDepartments(); @@ -698,10 +806,22 @@ private void joinWorkspaceForNewUser(String userName, Long userId) { } needToAdd.forEach(map -> { map.forEach((key, value) -> { - Arrays.stream(value.split(",")).forEach(roleId -> { - dssWorkspaceUserMapper.insertUserRoleInWorkspace(key.intValue(), Integer.parseInt(roleId), new Date(), userName, "system", userId, "system"); - }); + if (key.intValue() == workspaceId || (workspace0xId != null && key.intValue() == workspace0xId)) { + Arrays.stream(value.split(",")).filter(roleId -> analyserRole == null || Integer.parseInt(roleId) != analyserRole).forEach(roleId -> { + dssWorkspaceUserMapper.insertUserRoleInWorkspace(key.intValue(), Integer.parseInt(roleId), new Date(), userName, "system", userId, "system"); + }); + } else { + Arrays.stream(value.split(",")).forEach(roleId -> { + dssWorkspaceUserMapper.insertUserRoleInWorkspace(key.intValue(), Integer.parseInt(roleId), new Date(), userName, "system", userId, "system"); + }); + } }); }); } + + + @Override + public void updateWorkspaceInfo(DSSWorkspace dssWorkspace){ + dssWorkspaceMapper.updateWorkspaceInfo(dssWorkspace); + } } diff --git a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/service/impl/DSSWorkspaceUserServiceImpl.java b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/service/impl/DSSWorkspaceUserServiceImpl.java index 4bde06b77e..56973790ee 100644 --- a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/service/impl/DSSWorkspaceUserServiceImpl.java +++ b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/service/impl/DSSWorkspaceUserServiceImpl.java @@ -20,15 +20,17 @@ import com.webank.wedatasphere.dss.common.entity.PageInfo; import com.webank.wedatasphere.dss.framework.admin.service.DssAdminUserService; import com.webank.wedatasphere.dss.framework.workspace.bean.DSSWorkspaceUser; -import com.webank.wedatasphere.dss.framework.workspace.bean.StaffInfo; +import com.webank.wedatasphere.dss.common.StaffInfo; import com.webank.wedatasphere.dss.framework.workspace.bean.vo.DSSWorkspaceRoleVO; +import com.webank.wedatasphere.dss.framework.workspace.bean.vo.DepartmentUserTreeVo; +import com.webank.wedatasphere.dss.framework.workspace.bean.vo.DepartmentUserVo; import com.webank.wedatasphere.dss.framework.workspace.bean.vo.StaffInfoVO; import com.webank.wedatasphere.dss.framework.workspace.dao.DSSWorkspaceUserMapper; import com.webank.wedatasphere.dss.framework.workspace.service.DSSWorkspaceAddUserHook; import com.webank.wedatasphere.dss.framework.workspace.service.DSSWorkspaceUserService; -import com.webank.wedatasphere.dss.framework.workspace.service.StaffInfoGetter; +import com.webank.wedatasphere.dss.common.StaffInfoGetter; import com.webank.wedatasphere.dss.framework.workspace.util.WorkspaceDBHelper; -import com.webank.wedatasphere.dss.framework.workspace.util.WorkspaceServerConstant; +import com.webank.wedatasphere.dss.common.conf.WorkspaceServerConstant; import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -38,6 +40,8 @@ import java.util.Date; import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Collectors; @@ -59,6 +63,12 @@ public class DSSWorkspaceUserServiceImpl implements DSSWorkspaceUserService { @Autowired private DSSWorkspaceAddUserHook dssWorkspaceAddUserHook; + private static final String NODE_TYPE_DEPARTMENT = "department"; + + private static final String NODE_TYPE_OFFICE = "office"; + + private static final String NODE_TYPE_USER = "user"; + @Override @Transactional(rollbackFor = Exception.class) public void addWorkspaceUser(List roleIds, long workspaceId, String userName, String creator, String userId) { @@ -67,7 +77,7 @@ public void addWorkspaceUser(List roleIds, long workspaceId, String use for (Integer roleId : roleIds) { dssWorkspaceUserMapper.insertUserRoleInWorkspace((int) workspaceId, roleId, new Date(), userName, creator, userId == null ? null : Long.parseLong(userId), creator); } - dssWorkspaceAddUserHook.afterAdd(userName,workspaceId); + dssWorkspaceAddUserHook.afterAdd(userName, workspaceId); } @Override @@ -76,11 +86,22 @@ public void updateWorkspaceUser(List roles, long workspaceId, String us //获取用户创建时间 DSSWorkspaceUser workspaceUsers = dssWorkspaceUserMapper.getWorkspaceUsers(String.valueOf(workspaceId), userName, null).stream().findFirst().get(); dssWorkspaceUserMapper.removeAllRolesForUser(userName, workspaceId); - roles.forEach(role ->{ + roles.forEach(role -> { dssWorkspaceUserMapper.insertUserRoleInWorkspace(workspaceId, role, workspaceUsers.getJoinTime(), userName, workspaceUsers.getCreator(), 0L, creator); }); } + @Override + @Transactional(rollbackFor = Throwable.class) + public void updateWorkspaceRole(List roles, long workspaceId,String newOwner){ + //获取用户创建时间 + dssWorkspaceUserMapper.removeAllRolesForUser(newOwner, workspaceId); + roles.forEach(role -> { + dssWorkspaceUserMapper.insertUserRoleInWorkspace(workspaceId, role, new Date() , newOwner, newOwner, 0L, newOwner); + }); + } + + @Override @Transactional(rollbackFor = Exception.class) public void deleteWorkspaceUser(String userName, int workspaceId) { @@ -94,16 +115,20 @@ public List listAllDSSUsers() { return staffInfos.stream().map(this::staffToDSSUser).collect(Collectors.toList()); } - private StaffInfoVO staffToDSSUser(StaffInfo staffInfo){ + private StaffInfoVO staffToDSSUser(StaffInfo staffInfo) { StaffInfoVO staffInfoVO = new StaffInfoVO(); String orgFullName = staffInfo.getOrgFullName(); - if (StringUtils.isNotEmpty(orgFullName)){ - try{ - String departmentName = orgFullName.split(WorkspaceServerConstant.DEFAULT_STAFF_SPLIT)[0]; - String officeName = orgFullName.split(WorkspaceServerConstant.DEFAULT_STAFF_SPLIT)[1]; + if (StringUtils.isNotEmpty(orgFullName)) { + try { + String[] fullNameArray = orgFullName.split(WorkspaceServerConstant.DEFAULT_STAFF_SPLIT); + String departmentName = fullNameArray[0]; + String officeName = ""; + if(fullNameArray.length>1){ + officeName = fullNameArray[1]; + } staffInfoVO.setDepartment(departmentName); staffInfoVO.setOffice(officeName); - }catch(Exception e){ + } catch (Exception e) { //LOGGER.warn("fail to get department and office {} ", e.getMessage()); staffInfoVO.setDepartment(WorkspaceServerConstant.DEFAULT_DEPARTMENT); staffInfoVO.setOffice(WorkspaceServerConstant.DEFAULT_OFFICE); @@ -120,9 +145,61 @@ public List getAllWorkspaceUsers(long workspaceId) { } @Override - public PageInfo getAllWorkspaceUsersPage(long workspaceId, Integer pageNow, Integer pageSize) { - PageHelper.startPage(pageNow,pageSize); - List dos= dssWorkspaceUserMapper.getAllWorkspaceUsers(workspaceId); + public List getAllWorkspaceUsersDepartment(long workspaceId) { + List userList = dssWorkspaceUserMapper.getAllWorkspaceUsers(workspaceId).stream().map( + this::changeToUserVO) + .collect(Collectors.toList()); + return structureVo2Tree(userList); + } + + private List structureVo2Tree(List userList) { + AtomicReference id = new AtomicReference<>(0); + Map>> userMap = userList.stream().collect(Collectors.groupingBy(DepartmentUserVo::getDepartment, + Collectors.groupingBy(DepartmentUserVo::getOffice, Collectors.mapping(user -> + new DepartmentUserTreeVo(id.getAndSet(id.get() + 1), user.getName(), NODE_TYPE_USER, null), Collectors.toList())))); + return userMap.keySet().stream().map(key -> + new DepartmentUserTreeVo(id.getAndSet(id.get() + 1), key, NODE_TYPE_DEPARTMENT, buildTree4Office(userMap.get(key), id)) + ).collect(Collectors.toList()); + } + + private List buildTree4Office(Map> officeMap, AtomicReference id) { + return officeMap.keySet().stream().map(key -> + new DepartmentUserTreeVo(id.getAndSet(id.get() + 1), key, NODE_TYPE_OFFICE, officeMap.get(key)) + ).collect(Collectors.toList()); + } + + private DepartmentUserVo changeToUserVO(String userName) { + DepartmentUserVo vo = new DepartmentUserVo(); + vo.setName(userName); + String orgFullName = staffInfoGetter.getFullOrgNameByUsername(userName); + if (org.apache.commons.lang3.StringUtils.isNotEmpty(orgFullName)) { + try { + String[] fullNameArray = orgFullName.split(WorkspaceServerConstant.DEFAULT_STAFF_SPLIT); + String departmentName = fullNameArray[0]; + String officeName = WorkspaceServerConstant.DEFAULT_STAFF_SPLIT; + if(fullNameArray.length>1){ + officeName = fullNameArray[1]; + } + vo.setDepartment(departmentName); + vo.setOffice(officeName); + } catch (Exception e) { + //LOGGER.warn("fail to get department and office {} ", e.getMessage()); + vo.setDepartment(WorkspaceServerConstant.DEFAULT_DEPARTMENT); + vo.setOffice(WorkspaceServerConstant.DEFAULT_OFFICE); + } + }else{ + vo.setDepartment(WorkspaceServerConstant.DEFAULT_RESIGNED); + vo.setOffice(WorkspaceServerConstant.DEFAULT_RESIGNED); + } + return vo; + } + + @Override + public PageInfo getAllWorkspaceUsersPage(long workspaceId, Integer pageNow, Integer pageSize,boolean paged) { + if(paged) { + PageHelper.startPage(pageNow, pageSize); + } + List dos = dssWorkspaceUserMapper.getAllWorkspaceUsers(workspaceId); com.github.pagehelper.PageInfo doPage = new com.github.pagehelper.PageInfo<>(dos); return new PageInfo<>(doPage.getList(), doPage.getTotal()); } @@ -144,8 +221,8 @@ public List getWorkspaceReleaseUsers(int workspaceId) { } @Override - public Long getCountByUsername(String username,int workspaceId){ - return dssWorkspaceUserMapper.getCountByUsername(username,workspaceId); + public Long getCountByUsername(String username, int workspaceId) { + return dssWorkspaceUserMapper.getCountByUsername(username, workspaceId); } @Override @@ -166,6 +243,7 @@ public void clearUserByUserName(String userName) { dssWorkspaceUserMapper.deleteUserByUserName(userName); dssWorkspaceUserMapper.deleteUserRolesByUserName(userName); dssWorkspaceUserMapper.deleteProxyUserByUserName(userName); + dssWorkspaceUserMapper.deleteProjectUserByUserName(userName); } @Override @@ -173,4 +251,9 @@ public void clearUserByUserName(String userName) { public void revokeUserRoles(String userName, Integer[] workspaceIds, Integer[] roleIds) { dssWorkspaceUserMapper.deleteUserRoles(userName, workspaceIds, roleIds); } + + @Override + public List getWorkspaceUserByRoleId(Long workspaceId, Integer roleId) { + return dssWorkspaceUserMapper.getWorkspaceUserByRoleId(workspaceId, roleId); + } } diff --git a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/service/impl/DefaultStaffInfoGetter.java b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/service/impl/DefaultStaffInfoGetter.java index 8b5232a401..d4eda9a91e 100644 --- a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/service/impl/DefaultStaffInfoGetter.java +++ b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/service/impl/DefaultStaffInfoGetter.java @@ -16,15 +16,15 @@ package com.webank.wedatasphere.dss.framework.workspace.service.impl; +import cn.hutool.core.collection.CollUtil; import com.google.common.collect.Lists; -import com.webank.wedatasphere.dss.framework.workspace.bean.StaffInfo; -import com.webank.wedatasphere.dss.framework.workspace.service.StaffInfoGetter; -import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; -import org.springframework.stereotype.Component; +import com.webank.wedatasphere.dss.common.StaffInfo; +import com.webank.wedatasphere.dss.common.StaffInfoGetter; import java.util.Arrays; +import java.util.HashSet; import java.util.List; - +import java.util.Set; public class DefaultStaffInfoGetter implements StaffInfoGetter { @@ -49,4 +49,9 @@ public List getAllDepartments() { public StaffInfo getStaffInfoByUsername(String username) { return new StaffInfo(); } + + @Override + public Set getAllUsernames() { + return CollUtil.newHashSet(); + } } diff --git a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/service/impl/NoticeServiceImpl.java b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/service/impl/NoticeServiceImpl.java index 9262a453dc..62bd8ab31c 100644 --- a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/service/impl/NoticeServiceImpl.java +++ b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/service/impl/NoticeServiceImpl.java @@ -1,27 +1,27 @@ -package com.webank.wedatasphere.dss.framework.workspace.service.impl; - -import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; -import com.webank.wedatasphere.dss.framework.workspace.bean.NoticeContent; -import com.webank.wedatasphere.dss.framework.workspace.dao.NoticeMapper; -import com.webank.wedatasphere.dss.framework.workspace.service.NoticeService; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; - -import java.util.Date; -import java.util.List; - -@Service -public class NoticeServiceImpl implements NoticeService { - - @Autowired - private NoticeMapper noticeMapper; - - @Override - public List getNoticeContent() { - QueryWrapper queryWrapper = new QueryWrapper<>(); - Date now = new Date(); - queryWrapper.gt("end_time", now); - queryWrapper.orderByDesc("id"); - return noticeMapper.selectList(queryWrapper); - } -} +//package com.webank.wedatasphere.dss.framework.workspace.service.impl; +// +//import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; +//import com.webank.wedatasphere.dss.framework.workspace.bean.NoticeContent; +//import com.webank.wedatasphere.dss.framework.workspace.dao.NoticeMapper; +//import com.webank.wedatasphere.dss.framework.workspace.service.NoticeService; +//import org.springframework.beans.factory.annotation.Autowired; +//import org.springframework.stereotype.Service; +// +//import java.util.Date; +//import java.util.List; +// +//@Service +//public class NoticeServiceImpl implements NoticeService { +// +// @Autowired +// private NoticeMapper noticeMapper; +// +// @Override +// public List getNoticeContent() { +// QueryWrapper queryWrapper = new QueryWrapper<>(); +// Date now = new Date(); +// queryWrapper.gt("end_time", now); +// queryWrapper.orderByDesc("id"); +// return noticeMapper.selectList(queryWrapper); +// } +//} diff --git a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/util/WorkspaceSpringConfig.java b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/util/WorkspaceSpringConfig.java index 4924e032b3..cb455ebce8 100644 --- a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/util/WorkspaceSpringConfig.java +++ b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/util/WorkspaceSpringConfig.java @@ -17,7 +17,7 @@ package com.webank.wedatasphere.dss.framework.workspace.util; import com.webank.wedatasphere.dss.framework.workspace.service.DSSWorkspaceService; -import com.webank.wedatasphere.dss.framework.workspace.service.StaffInfoGetter; +import com.webank.wedatasphere.dss.common.StaffInfoGetter; import com.webank.wedatasphere.dss.framework.workspace.service.impl.DSSWorkspaceServiceImpl; import com.webank.wedatasphere.dss.framework.workspace.service.impl.DefaultStaffInfoGetter; import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; diff --git a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/util/WorkspaceUtils.java b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/util/WorkspaceUtils.java index 7b8184c7ba..3f5a870a05 100644 --- a/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/util/WorkspaceUtils.java +++ b/dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/util/WorkspaceUtils.java @@ -16,9 +16,12 @@ package com.webank.wedatasphere.dss.framework.workspace.util; +import cn.hutool.crypto.digest.DigestUtil; import com.webank.wedatasphere.dss.common.exception.DSSErrorException; +import com.webank.wedatasphere.dss.framework.workspace.constant.ApplicationConf; import com.webank.wedatasphere.dss.standard.app.sso.Workspace; import com.webank.wedatasphere.dss.standard.sso.utils.SSOHelper; +import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -57,4 +60,14 @@ public static void validateWorkspace(long workspaceId, HttpServletRequest httpSe } } + + // 鉴权方法 + public static boolean validateAuth(String timestamp, String sign) { + // 计算签名 + String calculatedSign = DigestUtil.sha256Hex(ApplicationConf.ITSM_SECRETKEY.getValue() + timestamp); + + // 检查签名是否匹配，并且检查timestamp是否在5分钟内 + return StringUtils.equals(calculatedSign, sign) && System.currentTimeMillis() - Long.parseLong(timestamp) <= 5 * 60 * 1000; + } + } diff --git a/dss-framework/dss-framework-workspace-server/src/test/java/com/webank/wedatasphere/dss/framework/workspace/restful/DSSWorkspaceUserRestfulTest.java b/dss-framework/dss-framework-workspace-server/src/test/java/com/webank/wedatasphere/dss/framework/workspace/restful/DSSWorkspaceUserRestfulTest.java index df33a3d77f..2c965821c7 100644 --- a/dss-framework/dss-framework-workspace-server/src/test/java/com/webank/wedatasphere/dss/framework/workspace/restful/DSSWorkspaceUserRestfulTest.java +++ b/dss-framework/dss-framework-workspace-server/src/test/java/com/webank/wedatasphere/dss/framework/workspace/restful/DSSWorkspaceUserRestfulTest.java @@ -1,66 +1,66 @@ -package com.webank.wedatasphere.dss.framework.workspace.restful; - -import com.webank.wedatasphere.dss.framework.admin.service.DssAdminUserService; -import lombok.SneakyThrows; -import org.junit.jupiter.api.*; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.boot.test.mock.mockito.MockBean; -import org.springframework.test.web.servlet.MockMvc; -import org.springframework.test.web.servlet.request.MockMvcRequestBuilders; -import org.springframework.test.web.servlet.setup.MockMvcBuilders; -import org.springframework.web.context.WebApplicationContext; - -import javax.ws.rs.core.MediaType; - -@TestMethodOrder(MethodOrderer.OrderAnnotation. class ) -@SpringBootTest(classes=com.webank.wedatasphere.dss.Application.class) -//@AutoConfigureMockMvc -class DSSWorkspaceUserRestfulTest { - -// @Autowired - private MockMvc mockMvc; - - @MockBean - private DssAdminUserService dssUserService; - - @BeforeEach - void before(WebApplicationContext context) { - mockMvc = MockMvcBuilders.webAppContextSetup(context) - .addFilter((request, response, chain) -> { - response.setCharacterEncoding("UTF-8"); - chain.doFilter(request, response); - }, "/*") - .build(); - } - @Order ( 2 ) - @SneakyThrows - @org.junit.jupiter.api.Test - void revokeUserRole() { - String content = mockMvc.perform(MockMvcRequestBuilders.post("/dss/framework/workspace/revokeUserRole") - .content("{\n" + - " \"userName\": \"v_xiangbiaowu\",\n" + - " \"workspaceIds\": [224],\n" + - " \"roleIds\": [1,2,3]\n" + - "}") - .header("Token-Code", "HPMS-KhFGSQkdaaCPBYfE") - .contentType(MediaType.APPLICATION_JSON) - ).andReturn().getResponse().getContentAsString(); - } - @Order ( 1 ) - @SneakyThrows - @Test - void getWorkspaceUserRole() { - String content = mockMvc.perform(MockMvcRequestBuilders.get("/dss/framework/workspace/getUserRole") - .param("userName", "hadoop") - .header("Token-Code", "HPMS-KhFGSQkdaaCPBYfE") - .contentType("application/json") - ).andReturn().getResponse().getContentAsString(); - } - - @Order ( 3 ) - @Test - void getWorkspaceUserRole2() { - getWorkspaceUserRole(); - } - -} \ No newline at end of file +//package com.webank.wedatasphere.dss.framework.workspace.restful; +// +//import com.webank.wedatasphere.dss.framework.admin.service.DssAdminUserService; +//import lombok.SneakyThrows; +//import org.junit.jupiter.api.*; +//import org.springframework.boot.test.context.SpringBootTest; +//import org.springframework.boot.test.mock.mockito.MockBean; +//import org.springframework.test.web.servlet.MockMvc; +//import org.springframework.test.web.servlet.request.MockMvcRequestBuilders; +//import org.springframework.test.web.servlet.setup.MockMvcBuilders; +//import org.springframework.web.context.WebApplicationContext; +// +//import javax.ws.rs.core.MediaType; +// +//@TestMethodOrder(MethodOrderer.OrderAnnotation. class ) +//@SpringBootTest(classes=com.webank.wedatasphere.dss.Application.class) +////@AutoConfigureMockMvc +//class DSSWorkspaceUserRestfulTest { +// +//// @Autowired +// private MockMvc mockMvc; +// +// @MockBean +// private DssAdminUserService dssUserService; +// +// @BeforeEach +// void before(WebApplicationContext context) { +// mockMvc = MockMvcBuilders.webAppContextSetup(context) +// .addFilter((request, response, chain) -> { +// response.setCharacterEncoding("UTF-8"); +// chain.doFilter(request, response); +// }, "/*") +// .build(); +// } +// @Order ( 2 ) +// @SneakyThrows +// @org.junit.jupiter.api.Test +// void revokeUserRole() { +// String content = mockMvc.perform(MockMvcRequestBuilders.post("/dss/framework/workspace/revokeUserRole") +// .content("{\n" + +// " \"userName\": \"v_xiangbiaowu\",\n" + +// " \"workspaceIds\": [224],\n" + +// " \"roleIds\": [1,2,3]\n" + +// "}") +// .header("Token-Code", "HPMS-KhFGSQkdaaCPBYfE") +// .contentType(MediaType.APPLICATION_JSON) +// ).andReturn().getResponse().getContentAsString(); +// } +// @Order ( 1 ) +// @SneakyThrows +// @Test +// void getWorkspaceUserRole() { +// String content = mockMvc.perform(MockMvcRequestBuilders.get("/dss/framework/workspace/getUserRole") +// .param("userName", "hadoop") +// .header("Token-Code", "HPMS-KhFGSQkdaaCPBYfE") +// .contentType("application/json") +// ).andReturn().getResponse().getContentAsString(); +// } +// +// @Order ( 3 ) +// @Test +// void getWorkspaceUserRole2() { +// getWorkspaceUserRole(); +// } +// +//} \ No newline at end of file diff --git a/dss-framework/framework-plugins/dss-framework-migrate-server/pom.xml b/dss-framework/framework-plugins/dss-framework-migrate-server/pom.xml index 4e68515859..4f1677bd85 100644 --- a/dss-framework/framework-plugins/dss-framework-migrate-server/pom.xml +++ b/dss-framework/framework-plugins/dss-framework-migrate-server/pom.xml @@ -5,7 +5,7 @@ dss com.webank.wedatasphere.dss - 1.1.0.20-SNAPSHOT + 1.10.0-SNAPSHOT ../../../pom.xml 4.0.0 diff --git a/dss-framework/framework-plugins/dss-framework-migrate-server/src/main/java/com/webank/wedatasphere/dss/migrate/conf/MigrateConf.java b/dss-framework/framework-plugins/dss-framework-migrate-server/src/main/java/com/webank/wedatasphere/dss/migrate/conf/MigrateConf.java new file mode 100644 index 0000000000..2a46b8f419 --- /dev/null +++ b/dss-framework/framework-plugins/dss-framework-migrate-server/src/main/java/com/webank/wedatasphere/dss/migrate/conf/MigrateConf.java @@ -0,0 +1,12 @@ +package com.webank.wedatasphere.dss.migrate.conf; + +import org.apache.linkis.common.conf.CommonVars; + +public class MigrateConf { + + public static final String ORC_SERVER_NAME = + CommonVars.apply("wds.dss.framework.migrate.orc.name", "dss-framework-orchestrator-server-dev").getValue(); + + public static final String WORKFLOW_SERVER_NAME = + CommonVars.apply("wds.dss.workflow.server.name", "dss-workflow-server-dev").getValue(); +} diff --git a/dss-framework/framework-plugins/dss-framework-migrate-server/src/main/java/com/webank/wedatasphere/dss/migrate/restful/DSSMigrateRestful.java b/dss-framework/framework-plugins/dss-framework-migrate-server/src/main/java/com/webank/wedatasphere/dss/migrate/restful/DSSMigrateRestful.java index 4cdfd1c636..0dac927b1f 100644 --- a/dss-framework/framework-plugins/dss-framework-migrate-server/src/main/java/com/webank/wedatasphere/dss/migrate/restful/DSSMigrateRestful.java +++ b/dss-framework/framework-plugins/dss-framework-migrate-server/src/main/java/com/webank/wedatasphere/dss/migrate/restful/DSSMigrateRestful.java @@ -398,61 +398,14 @@ public void exportOrcSqlFile(HttpServletRequest req, // 解压下载文件 ZipHelper.unzip(zipFilePath); // 解压后工程文件夹路径 - String projectBasePath = pathRoot + "default_orc"; - // 解压工作流zip文件 - String orcFlowZipFile = projectBasePath + File.separator + "orc_flow.zip"; - ZipHelper.unzip(orcFlowZipFile); - // 解压后工作流文件夹路径 - String flowBasePath = projectBasePath + File.separator + projectName; - // 读取工作流信息 - List dssFlows = metaService.readFlow(flowBasePath); - // 获取工作流节点解析对象 - JsonToFlowParser jsonToFlowParser = WorkflowFactory.INSTANCE.getJsonToFlowParser(); - List uploadResourceNewFileNameList = new ArrayList<>(); - for (DSSFlow dssFlow : dssFlows) { - String workflowPath = flowBasePath + File.separator + dssFlow.getName() + File.separator; - String flowJsonFile = workflowPath + dssFlow.getName() + ".json"; - String jsonContent = FileHelper.readFile(flowJsonFile); - - if (StringUtils.isNotBlank(jsonContent)) { - //将json读取为string，存入workflow - dssFlow.setFlowJson(jsonContent); - // 获取工作流节点信息 - Workflow workflow = jsonToFlowParser.parse(dssFlow); - // 替换用户上传的资源文件名称 - List uploadResources = workflow.getFlowResources(); - if (uploadResources != null && uploadResources.size() > 0) { - for (Resource uploadResource : uploadResources) { - String oldUploadResourceName = workflowPath + "resource" + File.separator + uploadResource.getResourceId() + ".re"; - String newUploadResourceName = workflowPath + "resource" + File.separator + uploadResource.getFileName(); - FileUtils.getFile(oldUploadResourceName).renameTo(new File(newUploadResourceName)); - uploadResourceNewFileNameList.add(newUploadResourceName); - } - } - // 替换脚本sql文件名称为节点名称 - workflow.getWorkflowNodes().forEach(workflowNode -> { - List resources = workflowNode.getDSSNode().getResources(); - if (resources != null && resources.size() > 0) { - for (Resource resource : resources) { - String oldSqlFilePath = workflowPath + "resource" + File.separator + resource.getResourceId() + "_" + resource.getVersion() + ".re"; - String newSqlFilePath = workflowPath + "resource" + File.separator + workflowNode.getDSSNode().getName() + "_" + resource.getVersion() + ".re"; - FileUtils.getFile(oldSqlFilePath).renameTo(new File(newSqlFilePath)); - } - } - }); - } + String flowBasePath = pathRoot + File.separator + projectName; + // 删除 .metaconf文件夹 + String flowMetaPath = flowBasePath + File.separator + ".metaConf"; + File metaFile = new File(flowMetaPath); + if (metaFile.exists()) { + FileUtils.deleteDirectory(metaFile); } - //删除掉无用文件，包括zip包,.json,.txt,.properties - List fileNameList = new ArrayList<>(); - FileHelper.getAllFileNames(pathRoot, fileNameList); - fileNameList.forEach(fileName -> { - if (fileName.endsWith(".zip") || fileName.endsWith(".json") || fileName.endsWith(".txt") || fileName.endsWith(".properties")) { - // 上传的资源文件不能删除 - if (uploadResourceNewFileNameList.stream().noneMatch(fileName::equals)) { - new File(fileName).delete(); - } - } - }); + //将文件打成zip包并输出 String orcZipPath = ZipHelper.zip(flowBasePath, true); try (InputStream inputStream = new FileInputStream(orcZipPath)) { diff --git a/dss-framework/framework-plugins/dss-framework-orchestrator-publish/pom.xml b/dss-framework/framework-plugins/dss-framework-orchestrator-publish/pom.xml index d7b57582ed..7ba373d77b 100644 --- a/dss-framework/framework-plugins/dss-framework-orchestrator-publish/pom.xml +++ b/dss-framework/framework-plugins/dss-framework-orchestrator-publish/pom.xml @@ -21,7 +21,7 @@ dss com.webank.wedatasphere.dss - 1.1.0.20-SNAPSHOT + 1.10.0-SNAPSHOT ../../../pom.xml 4.0.0 @@ -29,6 +29,12 @@ dss-framework-orchestrator-publish + + com.webank.wedatasphere.dss + dss-workflow-server + ${dss.version} + compile + com.webank.wedatasphere.dss dss-orchestrator-core diff --git a/dss-framework/framework-plugins/dss-framework-orchestrator-publish/src/main/java/com/webank/wedatasphere/dss/orchestrator/publish/ExportDSSOrchestratorPlugin.java b/dss-framework/framework-plugins/dss-framework-orchestrator-publish/src/main/java/com/webank/wedatasphere/dss/orchestrator/publish/ExportDSSOrchestratorPlugin.java index d2130b45f2..ebe3bb9679 100644 --- a/dss-framework/framework-plugins/dss-framework-orchestrator-publish/src/main/java/com/webank/wedatasphere/dss/orchestrator/publish/ExportDSSOrchestratorPlugin.java +++ b/dss-framework/framework-plugins/dss-framework-orchestrator-publish/src/main/java/com/webank/wedatasphere/dss/orchestrator/publish/ExportDSSOrchestratorPlugin.java @@ -39,6 +39,19 @@ OrchestratorExportResult exportOrchestrator(String userName, String projectName, List dssLabels, boolean addOrcVersion, Workspace workspace) throws DSSErrorException; + + /** + * 导出Orchestrator基本信息和工作流，并放到BML中。 + * 注意：导入导出接口只适合于不同环境下，先从A环境导出工作流，再导入到B环境的情况。 + * 不适合用于在同一环境下的复制。同一环境的复制操作需使用copyOperation。 + * @param addOrcVersion 导出之后，是否要升级一个版本 + */ + OrchestratorExportResult exportOrchestratorNew(String userName, + Long orchestratorId, + Long orcVersionId, + String projectName, + List dssLabels, + boolean addOrcVersion, Workspace workspace) throws DSSErrorException; Long orchestratorVersionIncrease(Long orcId, String userName, String comment, diff --git a/dss-framework/framework-plugins/dss-framework-orchestrator-publish/src/main/java/com/webank/wedatasphere/dss/orchestrator/publish/ImportDSSOrchestratorPlugin.java b/dss-framework/framework-plugins/dss-framework-orchestrator-publish/src/main/java/com/webank/wedatasphere/dss/orchestrator/publish/ImportDSSOrchestratorPlugin.java index bb09a55cf1..455fbc7dd5 100644 --- a/dss-framework/framework-plugins/dss-framework-orchestrator-publish/src/main/java/com/webank/wedatasphere/dss/orchestrator/publish/ImportDSSOrchestratorPlugin.java +++ b/dss-framework/framework-plugins/dss-framework-orchestrator-publish/src/main/java/com/webank/wedatasphere/dss/orchestrator/publish/ImportDSSOrchestratorPlugin.java @@ -28,6 +28,17 @@ public interface ImportDSSOrchestratorPlugin extends DSSOrchestratorPlugin { + /** + * 导入Orchestrator + * 注意：导入导出接口只适合于不同环境下，先从A环境导出工作流，再导入到B环境的情况。 + * 不适合用于在同一环境下的复制。同一环境的复制操作需使用copyOperation。 + * @param requestImportOrchestrator + * @return + * @throws DSSErrorException + * @throws IOException + * @throws ExternalOperationFailedException + */ + DSSOrchestratorVersion importOrchestratorNew(RequestImportOrchestrator requestImportOrchestrator) throws Exception; /** * 导入Orchestrator diff --git a/dss-framework/framework-plugins/dss-framework-orchestrator-publish/src/main/java/com/webank/wedatasphere/dss/orchestrator/publish/impl/ExportDSSOrchestratorPluginImpl.java b/dss-framework/framework-plugins/dss-framework-orchestrator-publish/src/main/java/com/webank/wedatasphere/dss/orchestrator/publish/impl/ExportDSSOrchestratorPluginImpl.java index de33c0cdcb..ba3eeebad7 100644 --- a/dss-framework/framework-plugins/dss-framework-orchestrator-publish/src/main/java/com/webank/wedatasphere/dss/orchestrator/publish/impl/ExportDSSOrchestratorPluginImpl.java +++ b/dss-framework/framework-plugins/dss-framework-orchestrator-publish/src/main/java/com/webank/wedatasphere/dss/orchestrator/publish/impl/ExportDSSOrchestratorPluginImpl.java @@ -64,6 +64,7 @@ import java.util.function.BiFunction; import java.util.function.Consumer; +import static com.webank.wedatasphere.dss.common.utils.ZipHelper.unzip; import static com.webank.wedatasphere.dss.common.utils.ZipHelper.zip; @@ -87,6 +88,88 @@ public class ExportDSSOrchestratorPluginImpl extends AbstractDSSOrchestratorPlug @Autowired AddOrchestratorVersionHook addOrchestratorVersionHook; + @Override + public OrchestratorExportResult exportOrchestratorNew(String userName, Long orchestratorId, Long orcVersionId, String projectName, + List dssLabels, boolean addOrcVersion, Workspace workspace) throws DSSErrorException { + //1、导出info信息 + if (orcVersionId == null || orcVersionId < 0){ + LOGGER.info("orchestratorVersionId is {}.", orcVersionId); + //最简单的就是通过orcId来找到最新的versionId + orcVersionId = orchestratorMapper.findLatestOrcVersionId(orchestratorId); + } + DSSOrchestratorVersion dssOrchestratorVersion = orchestratorMapper.getOrchestratorVersion(orcVersionId); + DSSOrchestratorInfo dssOrchestratorInfo = orchestratorMapper.getOrchestrator(dssOrchestratorVersion.getOrchestratorId()); + String orcName = dssOrchestratorInfo.getName(); + if (dssOrchestratorInfo != null) { + + LOGGER.info("{} 开始导出Orchestrator: {} 版本ID为: {}.", userName, dssOrchestratorInfo.getName(), orcVersionId); + + //2、导出第三方应用信息，如工作流、Visualis、Qualitis + DSSOrchestrator dssOrchestrator = orchestratorManager.getOrCreateOrchestrator(userName, workspace.getWorkspaceName(), dssOrchestratorInfo.getType(), + dssLabels); + //定义操作结果处理器 + BiFunction responseRefConsumer = (developmentOperation, developmentRequestRef) -> { + RefJobContentRequestRef requestRef = (RefJobContentRequestRef) developmentRequestRef; + requestRef.setRefJobContent(MapUtils.newCommonMap(OrchestratorRefConstant.ORCHESTRATION_ID_KEY, dssOrchestratorVersion.getAppId())); + return ((RefExportOperation) developmentOperation).exportRef(requestRef); + }; + //定义项目相关的处理器，处着编排的RequestRef的项目相关信息 + Consumer projectRefRequestRefConsumer = projectRefRequestRef -> projectRefRequestRef.setProjectName(projectName).setRefProjectId(dssOrchestratorVersion.getProjectId()); + + ExportResponseRef responseRef = OrchestrationDevelopmentOperationUtils.tryOrchestrationOperation( + dssOrchestratorInfo, + dssOrchestrator, + userName, + workspace, + dssLabels, + //指明DevelopmentService是RefExportService + DevelopmentIntegrationStandard::getRefExportService, + //指明operation是ExportOperation + developmentService -> ((RefExportService) developmentService).getRefExportOperation(), + null, + projectRefRequestRefConsumer, + responseRefConsumer, + "export"); + // /appcom/tmp/dss/yyyyMMddHHmmssSSS/arionliu + String tmpPath = IoUtils.generateTempIOPath(userName); + // /appcom/tmp/dss/yyyyMMddHHmmssSSS/arionliu/projectxxx.zip + String flowZipPath = IoUtils.addFileSeparator(tmpPath, projectName + ".zip"); + + String resourceId = (String) responseRef.getResourceMap().get(ImportRequestRef.RESOURCE_ID_KEY); + String version = (String) responseRef.getResourceMap().get(ImportRequestRef.RESOURCE_VERSION_KEY); + bmlService.downloadToLocalPath(userName, resourceId, version, flowZipPath); + // /appcom/tmp/dss/yyyyMMddHHmmssSSS/arionliu/projectxxx + String projectPath = unzip(flowZipPath,true); + // /appcom/tmp/dss/yyyyMMddHHmmssSSS/arionliu/projectxxx/.flowmeta/flow_all_type_node/ + String flowMetaPath = IoUtils.generateFlowMetaIOPath(projectPath, orcName); + + try { + metaExportService.exportNew(dssOrchestratorInfo, flowMetaPath); + } catch (IOException e) { + LOGGER.error("Failed to export metaInfo in orchestrator server for orc({}) in version {}.", orchestratorId, orcVersionId, e); + DSSExceptionUtils.dealErrorException(60099, "Failed to export metaInfo in orchestrator server.", e, DSSOrchestratorErrorException.class); + } + + //打包导出工程 + String exportPath = zip(projectPath); + + //3、打包新的zip包上传BML + InputStream inputStream = bmlService.readLocalResourceFile(userName, exportPath); + BmlResource uploadResult = bmlService.upload(userName, inputStream, + dssOrchestratorInfo.getName() + ".OrcExport", projectName); + + //4、判断导出后是否改变Orc的版本 + if (addOrcVersion) { + orcVersionId = orchestratorVersionIncrease(dssOrchestratorInfo.getId(), + userName, dssOrchestratorInfo.getComment(), + workspace, dssOrchestratorInfo, projectName, dssLabels); + } + return new OrchestratorExportResult(uploadResult,String.valueOf(orcVersionId)); + //4、返回BML存储信息 + } else { + throw new DSSErrorException(90038, "该Orchestrator的版本号不存在，请检查版本号是否正确."); + } + } @Override public OrchestratorExportResult exportOrchestrator(String userName, Long orchestratorId, Long orcVersionId, String projectName, List dssLabels, boolean addOrcVersion, Workspace workspace) throws DSSErrorException { @@ -99,6 +182,7 @@ public OrchestratorExportResult exportOrchestrator(String userName, Long orchest DSSOrchestratorVersion dssOrchestratorVersion = orchestratorMapper.getOrchestratorVersion(orcVersionId); DSSOrchestratorInfo dssOrchestratorInfo = orchestratorMapper.getOrchestrator(dssOrchestratorVersion.getOrchestratorId()); if (dssOrchestratorInfo != null) { + // /appcom/tmp/dss/yyyyMMddHHmmssSSS/arionliu/default_orc/ String orcExportSaveBasePath = IoUtils.generateIOPath(userName, DEFAULT_ORC_NAME, ""); try { Files.createDirectories(Paths.get(orcExportSaveBasePath).toAbsolutePath().normalize()); diff --git a/dss-framework/framework-plugins/dss-framework-orchestrator-publish/src/main/java/com/webank/wedatasphere/dss/orchestrator/publish/impl/ImportDSSOrchestratorPluginImpl.java b/dss-framework/framework-plugins/dss-framework-orchestrator-publish/src/main/java/com/webank/wedatasphere/dss/orchestrator/publish/impl/ImportDSSOrchestratorPluginImpl.java index 8531c8312f..2323a15d60 100644 --- a/dss-framework/framework-plugins/dss-framework-orchestrator-publish/src/main/java/com/webank/wedatasphere/dss/orchestrator/publish/impl/ImportDSSOrchestratorPluginImpl.java +++ b/dss-framework/framework-plugins/dss-framework-orchestrator-publish/src/main/java/com/webank/wedatasphere/dss/orchestrator/publish/impl/ImportDSSOrchestratorPluginImpl.java @@ -17,9 +17,12 @@ package com.webank.wedatasphere.dss.orchestrator.publish.impl; import com.webank.wedatasphere.dss.common.entity.BmlResource; +import com.webank.wedatasphere.dss.common.entity.project.DSSProject; import com.webank.wedatasphere.dss.common.exception.DSSErrorException; +import com.webank.wedatasphere.dss.common.exception.DSSRuntimeException; import com.webank.wedatasphere.dss.common.label.DSSLabel; import com.webank.wedatasphere.dss.common.label.DSSLabelUtil; +import com.webank.wedatasphere.dss.common.label.EnvDSSLabel; import com.webank.wedatasphere.dss.common.utils.*; import com.webank.wedatasphere.dss.contextservice.service.ContextService; import com.webank.wedatasphere.dss.orchestrator.common.entity.DSSOrchestratorInfo; @@ -44,6 +47,9 @@ import com.webank.wedatasphere.dss.standard.app.development.service.RefImportService; import com.webank.wedatasphere.dss.standard.app.development.standard.DevelopmentIntegrationStandard; import com.webank.wedatasphere.dss.standard.app.sso.Workspace; +import com.webank.wedatasphere.dss.workflow.common.entity.DSSFlow; +import com.webank.wedatasphere.dss.workflow.lock.DSSFlowEditLockManager; +import com.webank.wedatasphere.dss.workflow.service.DSSFlowService; import org.apache.commons.lang.StringUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; @@ -55,6 +61,7 @@ import java.util.List; import java.util.UUID; +import static com.webank.wedatasphere.dss.common.utils.IoUtils.FLOW_META_DIRECTORY_NAME; import static com.webank.wedatasphere.dss.orchestrator.publish.impl.ExportDSSOrchestratorPluginImpl.DEFAULT_ORC_NAME; @@ -76,10 +83,165 @@ public class ImportDSSOrchestratorPluginImpl extends AbstractDSSOrchestratorPlug private ContextService contextService; @Autowired private OrchestratorManager orchestratorManager; + @Autowired + private DSSFlowService dssFlowService; @Autowired private AddOrchestratorVersionHook addOrchestratorVersionHook; + @Override + public DSSOrchestratorVersion importOrchestratorNew(RequestImportOrchestrator requestImportOrchestrator) throws Exception { + String userName = requestImportOrchestrator.getUserName(); + String projectName = requestImportOrchestrator.getProjectName(); + Long projectId = requestImportOrchestrator.getProjectId(); + String resourceId = requestImportOrchestrator.getResourceId(); + String version = requestImportOrchestrator.getBmlVersion(); + List dssLabels = requestImportOrchestrator.getDssLabels(); + Workspace workspace = requestImportOrchestrator.getWorkspace(); + + //1、下载BML的Orchestrator的导入包 + // /appcom/tmp/dss/yyyyMMddHHmmssSSS/arionliu + String tempPath = IoUtils.generateTempIOPath(userName); + // /appcom/tmp/dss/yyyyMMddHHmmssSSS/arionliu/projectxxx.zip + String inputZipPath = IoUtils.addFileSeparator(tempPath, projectName + ".zip"); + bmlService.downloadToLocalPath(userName, resourceId, version, inputZipPath); + // /appcom/tmp/dss/yyyyMMddHHmmssSSS/arionliu/projectxxx + String projectPath = ZipHelper.unzip(inputZipPath,true); + String flowName = IoUtils.getSubdirectoriesNames(projectPath).stream().filter(name -> !name.startsWith(".")) + .findFirst().orElseThrow(() -> new DSSRuntimeException("import package has no flow（未导入任何工作流，请检查导入包格式）")); + String flowMetaPath = IoUtils.addFileSeparator(projectPath, FLOW_META_DIRECTORY_NAME, flowName); + + //2、导入Info信息(导入冲突处理) + DSSOrchestratorInfo importDssOrchestratorInfo = metaInputService.importOrchestratorNew(flowMetaPath); + importDssOrchestratorInfo.setProjectId(projectId); + importDssOrchestratorInfo.setWorkspaceId(workspace.getWorkspaceId()); + //复制工程，直接使用新的UUID和复制后的工程ID + if (requestImportOrchestrator.getCopyProjectId() != null + && StringUtils.isNotBlank(requestImportOrchestrator.getCopyProjectName())) { + projectId = requestImportOrchestrator.getCopyProjectId(); + importDssOrchestratorInfo.setProjectId(projectId); + importDssOrchestratorInfo.setUUID(UUID.randomUUID().toString()); + } + //根据工程ID和编排名称查询 + String uuid = orchestratorMapper.getOrcNameByParam(importDssOrchestratorInfo.getProjectId(), importDssOrchestratorInfo.getName()); + //通过UUID查找是否已经导入过 + DSSOrchestratorInfo existFlag = orchestratorMapper.getOrchestratorByUUID(importDssOrchestratorInfo.getUUID()); + //add 和update都需要更新成当前环境id信息，放到新的版本记录中 + //todo 跨环境必须保证工程ID一样，或者需要更新导入包中的所有工程ID为当前最新ID,不一致的话关系到上下文、第三方工程的映射问题 + if (null != existFlag) { + //判断是否存在相同名称的编排 + if (StringUtils.isNotBlank(uuid) && !uuid.equals(importDssOrchestratorInfo.getUUID())) { + DSSExceptionUtils + .dealErrorException(61002, "The same orchestration name already exists", DSSErrorException.class); + } + importDssOrchestratorInfo.setId(existFlag.getId()); + } else { + //判断是否存在相同名称的编排 + if (StringUtils.isNotBlank(uuid)) { + DSSExceptionUtils + .dealErrorException(61002, "The same orchestration name already exists", DSSErrorException.class); + } + //使用生产环境的id + importDssOrchestratorInfo.setId(null); + importDssOrchestratorInfo.setCreator(userName); + importDssOrchestratorInfo.setCreateTime(new Date()); + //兼容 + if (importDssOrchestratorInfo.getWorkspaceId() == null) { + importDssOrchestratorInfo.setWorkspaceId(workspace.getWorkspaceId()); + } + if (StringUtils.isEmpty(importDssOrchestratorInfo.getOrchestratorMode())) { + importDssOrchestratorInfo.setOrchestratorMode("pom_work_flow"); + } + if (StringUtils.isEmpty(importDssOrchestratorInfo.getOrchestratorWay())) { + importDssOrchestratorInfo.setOrchestratorWay(",pom_work_flow_DAG,"); + } + } + + //4、导入版本Version信息 + DSSOrchestratorVersion dssOrchestratorVersion = new DSSOrchestratorVersion(); + dssOrchestratorVersion.setAppId(null); + dssOrchestratorVersion.setComment("orchestrator import"); + dssOrchestratorVersion.setOrchestratorId(importDssOrchestratorInfo.getId()); + dssOrchestratorVersion.setContent(""); + dssOrchestratorVersion.setProjectId(projectId); + dssOrchestratorVersion.setSource("Orchestrator create"); + dssOrchestratorVersion.setUpdater(userName); + //生产导入：默认是为无效，除非开启直接有效（在生成环境独立部署的时候，导入和发布是分开的）；开发环境为有效 + int valid = DSSOrchestratorConf.DSS_IMPORT_VALID_IMMEDIATELY.getValue() || DSSLabelUtil.isDevEnv(dssLabels) ? 1 : 0; + dssOrchestratorVersion.setValidFlag(valid); + + DSSOrchestratorVersion oldVersion = orchestratorMapper.getLatestOrchestratorVersionByIdAndValidFlag(importDssOrchestratorInfo.getId(), 1); + Long oldAppId = null; + if (oldVersion!=null) { + dssOrchestratorVersion.setVersion(OrchestratorUtils.increaseVersion(oldVersion.getVersion())); + oldAppId = oldVersion.getAppId(); + } else { + dssOrchestratorVersion.setVersion(OrchestratorUtils.generateNewVersion()); + } + + //5、生成上下文ContextId + String contextId = contextService.createContextID(workspace.getWorkspaceName(), projectName, importDssOrchestratorInfo.getName(), dssOrchestratorVersion.getVersion(), userName); + dssOrchestratorVersion.setFormatContextId(contextId); + LOGGER.info("Create a new ContextId for import: {} ", contextId); + + //6、导入第三方应用信息，如工作流、Visualis、Qualities + DSSOrchestrator dssOrchestrator = orchestratorManager.getOrCreateOrchestrator(userName, + workspace.getWorkspaceName(), importDssOrchestratorInfo.getType(), dssLabels); + Long finalProjectId = projectId; + RefJobContentResponseRef responseRef = OrchestrationDevelopmentOperationUtils.tryOrchestrationOperation(importDssOrchestratorInfo, + dssOrchestrator, userName, workspace, dssLabels, + DevelopmentIntegrationStandard::getRefImportService, + developmentService -> ((RefImportService) developmentService).getRefImportOperation(), + dssContextRequestRef -> dssContextRequestRef.setContextId(contextId), + projectRefRequestRef -> projectRefRequestRef.setRefProjectId(finalProjectId).setProjectName(projectName), + (developmentOperation, developmentRequestRef) -> { + ImportRequestRef requestRef = (ImportRequestRef) developmentRequestRef; + requestRef.setResourceMap(MapUtils.newCommonMap(ImportRequestRef.RESOURCE_ID_KEY, resourceId, ImportRequestRef.RESOURCE_VERSION_KEY, version)); + requestRef.setNewVersion(dssOrchestratorVersion.getVersion()); + return ((RefImportOperation) developmentOperation).importRef(requestRef); + }, "import"); + long orchestrationId = (Long) responseRef.getRefJobContent().get(OrchestratorRefConstant.ORCHESTRATION_ID_KEY); + String orchestrationContent = (String) responseRef.getRefJobContent().get(OrchestratorRefConstant.ORCHESTRATION_CONTENT_KEY); + List paramConfTemplateIds = (List) responseRef.getRefJobContent().get(OrchestratorRefConstant.ORCHESTRATION_FLOWID_PARAMCONF_TEMPLATEID_TUPLES_KEY); + if(null != existFlag){ + if(oldVersion!=null) { + //如果生产中心的所有orc版本的valid_flag都是0（之前的所有发布都在convert期间失败了），那么oldVersion是为空的。 + addOrchestratorVersionHook.beforeAdd(oldVersion, Collections.emptyMap()); + } + //如果Orchestrator已经导入过，目前只更新版本信息，并更新基础信息name,其它信息不修改。 + orchestratorMapper.updateOrchestrator(importDssOrchestratorInfo); + }else{ + orchestratorMapper.addOrchestrator(importDssOrchestratorInfo); + } + + //更新返回內容 + dssOrchestratorVersion.setUpdateTime(new Date()); + dssOrchestratorVersion.setAppId(orchestrationId); + dssOrchestratorVersion.setContent(orchestrationContent); + dssOrchestratorVersion.setOrchestratorId(importDssOrchestratorInfo.getId()); + + orchestratorMapper.addOrchestratorVersion(dssOrchestratorVersion); + addOrchestratorVersionHook.afterAdd(dssOrchestratorVersion, Collections.singletonMap(OrchestratorRefConstant.ORCHESTRATION_FLOWID_PARAMCONF_TEMPLATEID_TUPLES_KEY,paramConfTemplateIds)); + LOGGER.info("import orchestrator success,orcId:{},appId:{}",importDssOrchestratorInfo.getId(),orchestrationId); + + // 更新flowJson + if (dssLabels.get(0).getValue().get(EnvDSSLabel.DSS_ENV_LABEL_KEY).equals("dev")) { + Long appId = dssOrchestratorVersion.getAppId(); + Long versionOrchestratorId = dssOrchestratorVersion.getOrchestratorId(); + DSSFlow flowByID = dssFlowService.getFlow(appId); + if (flowByID != null) { + dssFlowService.deleteFlowMetaData(versionOrchestratorId); + dssFlowService.saveAllFlowMetaData(appId, versionOrchestratorId); + } + DSSProject projectInfo = DSSFlowEditLockManager.getProjectInfo(projectId); + if (projectInfo.getAssociateGit()) { + dssFlowService.updateTOSaveStatus(projectInfo.getId(), appId, versionOrchestratorId); + } + } + + return dssOrchestratorVersion; + } + @Override public DSSOrchestratorVersion importOrchestrator(RequestImportOrchestrator requestImportOrchestrator) throws Exception { String userName = requestImportOrchestrator.getUserName(); @@ -99,6 +261,7 @@ public DSSOrchestratorVersion importOrchestrator(RequestImportOrchestrator reque List dssOrchestratorInfos = metaInputService.importOrchestrator(inputPath); DSSOrchestratorInfo importDssOrchestratorInfo = dssOrchestratorInfos.get(0); importDssOrchestratorInfo.setProjectId(projectId); + importDssOrchestratorInfo.setWorkspaceId(workspace.getWorkspaceId()); //复制工程，直接使用新的UUID和复制后的工程ID if (requestImportOrchestrator.getCopyProjectId() != null && StringUtils.isNotBlank(requestImportOrchestrator.getCopyProjectName())) { @@ -186,6 +349,7 @@ public DSSOrchestratorVersion importOrchestrator(RequestImportOrchestrator reque ImportRequestRef requestRef = (ImportRequestRef) developmentRequestRef; requestRef.setResourceMap(MapUtils.newCommonMap(ImportRequestRef.RESOURCE_ID_KEY, orcResourceId, ImportRequestRef.RESOURCE_VERSION_KEY, orcBmlVersion)); requestRef.setNewVersion(dssOrchestratorVersion.getVersion()); + requestRef.setParameter("isOldPackageStruct","true"); return ((RefImportOperation) developmentOperation).importRef(requestRef); }, "import"); long orchestrationId = (Long) responseRef.getRefJobContent().get(OrchestratorRefConstant.ORCHESTRATION_ID_KEY); diff --git a/dss-framework/framework-plugins/dss-framework-orchestrator-publish/src/main/java/com/webank/wedatasphere/dss/orchestrator/publish/io/export/MetaExportService.java b/dss-framework/framework-plugins/dss-framework-orchestrator-publish/src/main/java/com/webank/wedatasphere/dss/orchestrator/publish/io/export/MetaExportService.java index 01afcd2b8a..a1af87a924 100644 --- a/dss-framework/framework-plugins/dss-framework-orchestrator-publish/src/main/java/com/webank/wedatasphere/dss/orchestrator/publish/io/export/MetaExportService.java +++ b/dss-framework/framework-plugins/dss-framework-orchestrator-publish/src/main/java/com/webank/wedatasphere/dss/orchestrator/publish/io/export/MetaExportService.java @@ -35,6 +35,7 @@ public interface MetaExportService { * @throws IOException */ void export(DSSOrchestratorInfo dssOrchestratorInfo, String savePath) throws IOException; + void exportNew(DSSOrchestratorInfo dssOrchestratorInfo, String flowMetaPath) throws IOException; void exportFlowBaseInfo(List allDSSFlows, List allFlowRelations, String savePath) throws IOException; diff --git a/dss-framework/framework-plugins/dss-framework-orchestrator-publish/src/main/java/com/webank/wedatasphere/dss/orchestrator/publish/io/export/impl/MetaExportServiceImpl.java b/dss-framework/framework-plugins/dss-framework-orchestrator-publish/src/main/java/com/webank/wedatasphere/dss/orchestrator/publish/io/export/impl/MetaExportServiceImpl.java index fc6d20153d..2b17c14b28 100644 --- a/dss-framework/framework-plugins/dss-framework-orchestrator-publish/src/main/java/com/webank/wedatasphere/dss/orchestrator/publish/io/export/impl/MetaExportServiceImpl.java +++ b/dss-framework/framework-plugins/dss-framework-orchestrator-publish/src/main/java/com/webank/wedatasphere/dss/orchestrator/publish/io/export/impl/MetaExportServiceImpl.java @@ -17,6 +17,10 @@ package com.webank.wedatasphere.dss.orchestrator.publish.io.export.impl; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; import com.webank.wedatasphere.dss.common.utils.IoUtils; import com.webank.wedatasphere.dss.orchestrator.common.entity.DSSOrchestratorInfo; import com.webank.wedatasphere.dss.orchestrator.publish.io.export.MetaExportService; @@ -33,12 +37,42 @@ @Service("orcMetaExportService") public class MetaExportServiceImpl implements MetaExportService { + public static final String ORCHESTRATOR_META_KEY = "dss_orchestrator"; + public static final String FLOW_META_FILE_NAME = ".flowmeta"; private Logger logger = LoggerFactory.getLogger(this.getClass()); - - private final String fileName = "meta.txt"; + @Override + public void exportNew(DSSOrchestratorInfo dssOrchestratorInfo, String flowMetaPath) throws IOException { + File flowMetaFile = new File(flowMetaPath + File.separator + FLOW_META_FILE_NAME); + Gson gson = new GsonBuilder().setPrettyPrinting().create(); + + if (flowMetaFile.exists()) { + // 初始化JsonObject为null + JsonObject jsonObject = null; + // 先读取现有内容 + try (FileReader reader = new FileReader(flowMetaFile)) { + jsonObject = new JsonParser().parse(reader).getAsJsonObject(); + } // try-with-resources会自动关闭reader + // 更新JsonObject + if (jsonObject != null) { + jsonObject.add(ORCHESTRATOR_META_KEY, gson.toJsonTree(dssOrchestratorInfo)); + } + + // 写回修改后的内容 + try (FileWriter writer = new FileWriter(flowMetaFile)) { + gson.toJson(jsonObject, writer); + } // try-with-resources会自动关闭writer + + System.out.println("JSON文件已更新并保存。"); + } else { + // 文件不存在，直接创建并写入orchestratorInfo信息 + try (FileWriter writer = new FileWriter(flowMetaFile)) { + gson.toJson(dssOrchestratorInfo, writer); + } // try-with-resources会自动关闭writer + } + } @Override public void export(DSSOrchestratorInfo dssOrchestratorInfo, String savePath) throws IOException { diff --git a/dss-framework/framework-plugins/dss-framework-orchestrator-publish/src/main/java/com/webank/wedatasphere/dss/orchestrator/publish/io/input/MetaInputService.java b/dss-framework/framework-plugins/dss-framework-orchestrator-publish/src/main/java/com/webank/wedatasphere/dss/orchestrator/publish/io/input/MetaInputService.java index a672c6c4f4..89a356d781 100644 --- a/dss-framework/framework-plugins/dss-framework-orchestrator-publish/src/main/java/com/webank/wedatasphere/dss/orchestrator/publish/io/input/MetaInputService.java +++ b/dss-framework/framework-plugins/dss-framework-orchestrator-publish/src/main/java/com/webank/wedatasphere/dss/orchestrator/publish/io/input/MetaInputService.java @@ -33,6 +33,14 @@ public interface MetaInputService { List importOrchestrator(String basePath) throws IOException; + /** + * 读取编排元数据 + * @param flowMetaPath 工作流的元数据目录，比如/temp/projectName/.metaconf/demoflow/ + * @return + * @throws IOException + */ + DSSOrchestratorInfo importOrchestratorNew(String flowMetaPath) ; + List inputFlow(String basePath) throws IOException; List inputFlowRelation(String basePath) throws IOException; diff --git a/dss-framework/framework-plugins/dss-framework-orchestrator-publish/src/main/java/com/webank/wedatasphere/dss/orchestrator/publish/io/input/impl/MetaInputServiceImpl.java b/dss-framework/framework-plugins/dss-framework-orchestrator-publish/src/main/java/com/webank/wedatasphere/dss/orchestrator/publish/io/input/impl/MetaInputServiceImpl.java index 2691959d89..07c8e2d747 100644 --- a/dss-framework/framework-plugins/dss-framework-orchestrator-publish/src/main/java/com/webank/wedatasphere/dss/orchestrator/publish/io/input/impl/MetaInputServiceImpl.java +++ b/dss-framework/framework-plugins/dss-framework-orchestrator-publish/src/main/java/com/webank/wedatasphere/dss/orchestrator/publish/io/input/impl/MetaInputServiceImpl.java @@ -17,6 +17,10 @@ package com.webank.wedatasphere.dss.orchestrator.publish.io.input.impl; +import com.google.gson.Gson; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import com.webank.wedatasphere.dss.common.exception.DSSRuntimeException; import com.webank.wedatasphere.dss.common.utils.IoUtils; import com.webank.wedatasphere.dss.orchestrator.common.entity.DSSOrchestratorInfo; import com.webank.wedatasphere.dss.orchestrator.publish.io.input.MetaInputService; @@ -26,12 +30,11 @@ import com.webank.wedatasphere.dss.workflow.common.entity.DSSFlowRelation; import org.springframework.stereotype.Service; -import java.io.File; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.io.InputStream; +import java.io.*; import java.util.List; +import static com.webank.wedatasphere.dss.orchestrator.publish.io.export.impl.MetaExportServiceImpl.*; + @Service("orcMetaInputService") public class MetaInputServiceImpl implements MetaInputService { @@ -40,6 +43,18 @@ public class MetaInputServiceImpl implements MetaInputService { private final String fileName = "meta.txt"; + @Override + public DSSOrchestratorInfo importOrchestratorNew( String flowMetaPath) { + File flowMetaFile = new File(flowMetaPath + File.separator + FLOW_META_FILE_NAME); + JsonParser jsonParser = new JsonParser(); + Gson gson = new Gson(); + try (FileReader reader = new FileReader(flowMetaFile)) { + JsonObject jsonObject = jsonParser.parse(reader).getAsJsonObject(); + return gson.fromJson(jsonObject.get(ORCHESTRATOR_META_KEY), DSSOrchestratorInfo.class); + }catch (IOException e){ + throw new DSSRuntimeException(100000, "read flowMeta file failed,path" + flowMetaPath, e); + } + } @Override public List importOrchestrator(String basePath) throws IOException { try (InputStream inputStream = generateInputstream(basePath)) { diff --git a/dss-framework/framework-plugins/dss-framework-orchestrator-publish/src/main/java/com/webank/wedatasphere/dss/orchestrator/publish/job/OrchestratorConversionJob.java b/dss-framework/framework-plugins/dss-framework-orchestrator-publish/src/main/java/com/webank/wedatasphere/dss/orchestrator/publish/job/OrchestratorConversionJob.java index a7ebf544f9..0bc24971df 100644 --- a/dss-framework/framework-plugins/dss-framework-orchestrator-publish/src/main/java/com/webank/wedatasphere/dss/orchestrator/publish/job/OrchestratorConversionJob.java +++ b/dss-framework/framework-plugins/dss-framework-orchestrator-publish/src/main/java/com/webank/wedatasphere/dss/orchestrator/publish/job/OrchestratorConversionJob.java @@ -92,6 +92,8 @@ public void setCommonUpdateConvertJobStatus(CommonUpdateConvertJobStatus commonU public void run() { //1.从编排中心导出一次工作流,进行一次版本升级 //2.进行发布到schedulis等调度系统 + LOGGER.info("trace workflow publish,flowId:{},orchestratorId:{}.开始convert工作流 ", + conversionJobEntity.getOrchestrationIdMap().keySet(),conversionJobEntity.getOrcIdList() ); LOGGER.info("Job {} begin to convert project {} for user {} to scheduler, the orchestrationIds is {}.", id, conversionJobEntity.getProject().getId(), conversionJobEntity.getUserName(), conversionJobEntity.getOrchestrationIdMap().keySet()); conversionJobEntity.setResponse(ResponseOperateOrchestrator.running()); @@ -127,7 +129,12 @@ public void run() { consumer.accept(response); this.commonUpdateConvertJobStatus.toFailedStatus(this.orchestratorPublishJob, response.getMessage()); } + String costTime = + ByteTimeUtils.msDurationToString(conversionJobEntity.getUpdateTime().getTime() - conversionJobEntity.getCreateTime().getTime()); + LOGGER.info("trace workflow publish,flowId:{},orchestratorId:{}.convert工作流完成，耗时{} ", + conversionJobEntity.getOrchestrationIdMap().keySet(),conversionJobEntity.getOrcIdList(),costTime ); LOGGER.info("Job {} convert project {} for user {} to Orchestrator {}, costs {}.", id, conversionJobEntity.getProject().getId(), - conversionJobEntity.getUserName(), conversionJobEntity.getResponse().getJobStatus(), ByteTimeUtils.msDurationToString(conversionJobEntity.getUpdateTime().getTime() - conversionJobEntity.getCreateTime().getTime())); + conversionJobEntity.getUserName(), conversionJobEntity.getOrcIdList(),costTime + ); } } diff --git a/dss-framework/pom.xml b/dss-framework/pom.xml index 5327fb147b..30ab7146ce 100644 --- a/dss-framework/pom.xml +++ b/dss-framework/pom.xml @@ -21,7 +21,7 @@ dss com.webank.wedatasphere.dss - 1.1.0.20-SNAPSHOT + 1.10.0-SNAPSHOT ../pom.xml 4.0.0 diff --git a/dss-git/dss-git-common/pom.xml b/dss-git/dss-git-common/pom.xml new file mode 100644 index 0000000000..779eb1b78c --- /dev/null +++ b/dss-git/dss-git-common/pom.xml @@ -0,0 +1,22 @@ + + + + dss-git + com.webank.wedatasphere.dss + 1.10.0-SNAPSHOT + + 4.0.0 + + dss-git-common + + + + com.webank.wedatasphere.dss + dss-common + ${dss.version} + + + + \ No newline at end of file diff --git a/dss-git/dss-git-common/src/main/java/com.webank.wedatasphere.dss.git.common/protocol/GitSearchLine.java b/dss-git/dss-git-common/src/main/java/com.webank.wedatasphere.dss.git.common/protocol/GitSearchLine.java new file mode 100644 index 0000000000..659070ef52 --- /dev/null +++ b/dss-git/dss-git-common/src/main/java/com.webank.wedatasphere.dss.git.common/protocol/GitSearchLine.java @@ -0,0 +1,30 @@ +package com.webank.wedatasphere.dss.git.common.protocol; + +public class GitSearchLine { + private Integer number; + private String line; + + public GitSearchLine(Integer number, String line) { + this.number = number; + this.line = line; + } + + public GitSearchLine() { + } + + public Integer getNumber() { + return number; + } + + public void setNumber(Integer number) { + this.number = number; + } + + public String getLine() { + return line; + } + + public void setLine(String line) { + this.line = line; + } +} diff --git a/dss-git/dss-git-common/src/main/java/com.webank.wedatasphere.dss.git.common/protocol/GitSearchResult.java b/dss-git/dss-git-common/src/main/java/com.webank.wedatasphere.dss.git.common/protocol/GitSearchResult.java new file mode 100644 index 0000000000..a1e7258be9 --- /dev/null +++ b/dss-git/dss-git-common/src/main/java/com.webank.wedatasphere.dss.git.common/protocol/GitSearchResult.java @@ -0,0 +1,32 @@ +package com.webank.wedatasphere.dss.git.common.protocol; + +import java.util.List; + +public class GitSearchResult { + private String path; + private List keyLines; + + public GitSearchResult() { + } + + public GitSearchResult(String path, List keyLines) { + this.path = path; + this.keyLines = keyLines; + } + + public String getPath() { + return path; + } + + public void setPath(String path) { + this.path = path; + } + + public List getKeyLines() { + return keyLines; + } + + public void setKeyLines(List keyLines) { + this.keyLines = keyLines; + } +} diff --git a/dss-git/dss-git-common/src/main/java/com.webank.wedatasphere.dss.git.common/protocol/GitTree.java b/dss-git/dss-git-common/src/main/java/com.webank.wedatasphere.dss.git.common/protocol/GitTree.java new file mode 100644 index 0000000000..abdd9e204e --- /dev/null +++ b/dss-git/dss-git-common/src/main/java/com.webank.wedatasphere.dss.git.common/protocol/GitTree.java @@ -0,0 +1,71 @@ +package com.webank.wedatasphere.dss.git.common.protocol; + +import java.util.HashMap; +import java.util.Map; + +public class GitTree { + private String name; + private Boolean meta; + // 仅子节点拥有该属性 文件相对路径 + private String absolutePath; + private Map children = new HashMap<>(); + + public GitTree(String name) { + this.name = name; + } + + public GitTree(String name, Boolean meta) { + this.name = name; + this.meta = meta; + } + + // 添加子节点 + public void addChild(String path) { + String[] parts = path.split("/", 2); + String currentPart = parts[0]; + String restPart = parts.length > 1 ? parts[1] : null; + + GitTree gitTree = new GitTree(currentPart); + + children.putIfAbsent(currentPart, gitTree); + GitTree child = children.get(currentPart); + child.setAbsolutePath(this.getAbsolutePath()); + + if (restPart != null) { + child.addChild(restPart); + } + } + + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public Map getChildren() { + return children; + } + + public void setChildren(Map children) { + this.children = children; + } + + public Boolean getMeta() { + return meta; + } + + public void setMeta(Boolean meta) { + this.meta = meta; + } + + public String getAbsolutePath() { + return absolutePath; + } + + public void setAbsolutePath(String absolutePath) { + this.absolutePath = absolutePath; + } +} diff --git a/dss-git/dss-git-common/src/main/java/com.webank.wedatasphere.dss.git.common/protocol/GitUserEntity.java b/dss-git/dss-git-common/src/main/java/com.webank.wedatasphere.dss.git.common/protocol/GitUserEntity.java new file mode 100644 index 0000000000..a86241da75 --- /dev/null +++ b/dss-git/dss-git-common/src/main/java/com.webank.wedatasphere.dss.git.common/protocol/GitUserEntity.java @@ -0,0 +1,33 @@ +package com.webank.wedatasphere.dss.git.common.protocol; + + +import java.util.Date; + +public class GitUserEntity { + private String gitUser; + private String gitToken; + + public GitUserEntity() { + } + + public GitUserEntity(String gitUser, String gitToken) { + this.gitUser = gitUser; + this.gitToken = gitToken; + } + + public String getGitUser() { + return gitUser; + } + + public void setGitUser(String gitUser) { + this.gitUser = gitUser; + } + + public String getGitToken() { + return gitToken; + } + + public void setGitToken(String gitToken) { + this.gitToken = gitToken; + } +} diff --git a/dss-git/dss-git-common/src/main/java/com.webank.wedatasphere.dss.git.common/protocol/config/GitServerConfig.java b/dss-git/dss-git-common/src/main/java/com.webank.wedatasphere.dss.git.common/protocol/config/GitServerConfig.java new file mode 100644 index 0000000000..df98467dc3 --- /dev/null +++ b/dss-git/dss-git-common/src/main/java/com.webank.wedatasphere.dss.git.common/protocol/config/GitServerConfig.java @@ -0,0 +1,26 @@ +package com.webank.wedatasphere.dss.git.common.protocol.config; + +import org.apache.linkis.common.conf.CommonVars; + +public class GitServerConfig { + public static final CommonVars GIT_SERVER_PATH = CommonVars.apply("wds.dss.git.server.path", "/data/GitInstall"); + + public static final CommonVars GIT_RESTFUL_API_CREATE_PROJECTS = CommonVars.apply("wds.dss.git.server.rest.create.project", "api/v4/projects/"); + + public static final CommonVars GIT_URL_PRE = CommonVars.apply("wds.dss.git.url", "***REMOVED***"); + + public static final CommonVars GIT_THREAD_NUM = CommonVars.apply("wds.dss.git.thread.num", 23); + + public static final CommonVars GIT_SEARCH_RESULT_LIMIT = CommonVars.apply("wds.dss.git.search.limit", 30); + + public static final CommonVars GIT_USER = CommonVars.apply("wds.dss.server.git.user.uiid", "user_login"); + public static final CommonVars GIT_PASSWD = CommonVars.apply("wds.dss.server.git.password.uiid", "user_password"); + public static final CommonVars GIT_SUBMIT = CommonVars.apply("wds.dss.server.git.submit.uiid", "input[class='btn btn-success qa-sign-in-button']"); + public static final CommonVars GIT_TIME = CommonVars.apply("wds.dss.server.git.time", "1"); + + public static final CommonVars GIT_SEARCH_EXCLUDE_DIRECTORY = CommonVars.apply("wds.dss.server.git.search.exclude.dir", ".metaConf"); + + public static final CommonVars GIT_SEARCH_EXCLUDE_FILE = CommonVars.apply("wds.dss.server.git.search.exclude.file", ".properties,.projectmeta"); + + public static final CommonVars LINKIS_MYSQL_PRI_KEY = CommonVars.apply("wds.linkis.mysql.pri.key", "abc"); +} diff --git a/dss-git/dss-git-common/src/main/java/com.webank.wedatasphere.dss.git.common/protocol/constant/GitConstant.java b/dss-git/dss-git-common/src/main/java/com.webank.wedatasphere.dss.git.common/protocol/constant/GitConstant.java new file mode 100644 index 0000000000..d6914caf92 --- /dev/null +++ b/dss-git/dss-git-common/src/main/java/com.webank.wedatasphere.dss.git.common/protocol/constant/GitConstant.java @@ -0,0 +1,12 @@ +package com.webank.wedatasphere.dss.git.common.protocol.constant; + + +import java.util.Arrays; +import java.util.List; + +public class GitConstant { + public static String GIT_ACCESS_WRITE_TYPE = "write"; + public static String GIT_ACCESS_READ_TYPE = "read"; + public static String GIT_SERVER_META_PATH = ".metaConf"; + public static List GIT_SERVER_SEARCH_TYPE = Arrays.asList(".sql",".hql",".jdbc", ".py", ".python", ".scala", ".sh"); +} diff --git a/dss-git/dss-git-common/src/main/java/com.webank.wedatasphere.dss.git.common/protocol/exception/GitErrorException.java b/dss-git/dss-git-common/src/main/java/com.webank.wedatasphere.dss.git.common/protocol/exception/GitErrorException.java new file mode 100644 index 0000000000..9d1fa4a8f4 --- /dev/null +++ b/dss-git/dss-git-common/src/main/java/com.webank.wedatasphere.dss.git.common/protocol/exception/GitErrorException.java @@ -0,0 +1,15 @@ +package com.webank.wedatasphere.dss.git.common.protocol.exception; + +import com.webank.wedatasphere.dss.common.exception.DSSErrorException; + +public class GitErrorException extends DSSErrorException { + + public GitErrorException(int errCode, String desc) { + super(errCode, desc); + } + + public GitErrorException(int errCode, String desc, Throwable cause) { + super(errCode, desc); + initCause(cause); + } +} diff --git a/dss-git/dss-git-common/src/main/java/com.webank.wedatasphere.dss.git.common/protocol/request/GitAddMemberRequest.java b/dss-git/dss-git-common/src/main/java/com.webank.wedatasphere.dss.git.common/protocol/request/GitAddMemberRequest.java new file mode 100644 index 0000000000..ce63c7eba8 --- /dev/null +++ b/dss-git/dss-git-common/src/main/java/com.webank.wedatasphere.dss.git.common/protocol/request/GitAddMemberRequest.java @@ -0,0 +1,41 @@ +package com.webank.wedatasphere.dss.git.common.protocol.request; + +public class GitAddMemberRequest extends GitBaseRequest{ + private String username; + private String flowNodeName; + + public GitAddMemberRequest(String username, String flowNodeName) { + this.username = username; + this.flowNodeName = flowNodeName; + } + + public GitAddMemberRequest(Long workspaceId, String projectName, String username, String flowNodeName) { + super(workspaceId, projectName); + this.username = username; + this.flowNodeName = flowNodeName; + } + + public GitAddMemberRequest() { + } + + public GitAddMemberRequest(Long workspaceId, String projectName) { + super(workspaceId, projectName); + } + + + public String getUsername() { + return username; + } + + public void setUsername(String username) { + this.username = username; + } + + public String getFlowNodeName() { + return flowNodeName; + } + + public void setFlowNodeName(String flowNodeName) { + this.flowNodeName = flowNodeName; + } +} diff --git a/dss-git/dss-git-common/src/main/java/com.webank.wedatasphere.dss.git.common/protocol/request/GitArchiveProjectRequest.java b/dss-git/dss-git-common/src/main/java/com.webank.wedatasphere.dss.git.common/protocol/request/GitArchiveProjectRequest.java new file mode 100644 index 0000000000..07397e777a --- /dev/null +++ b/dss-git/dss-git-common/src/main/java/com.webank.wedatasphere.dss.git.common/protocol/request/GitArchiveProjectRequest.java @@ -0,0 +1,30 @@ +package com.webank.wedatasphere.dss.git.common.protocol.request; + + +public class GitArchiveProjectRequest extends GitBaseRequest{ + /** + * DSS用户名 + */ + private String username; + + public GitArchiveProjectRequest() { + } + + public GitArchiveProjectRequest(String username) { + this.username = username; + } + + public GitArchiveProjectRequest(Long workspaceId, String projectName, String username) { + super(workspaceId, projectName); + this.username = username; + } + + + public String getUsername() { + return username; + } + + public void setUsername(String username) { + this.username = username; + } +} diff --git a/dss-git/dss-git-common/src/main/java/com.webank.wedatasphere.dss.git.common/protocol/request/GitBaseRequest.java b/dss-git/dss-git-common/src/main/java/com.webank.wedatasphere.dss.git.common/protocol/request/GitBaseRequest.java new file mode 100644 index 0000000000..28e9e849c2 --- /dev/null +++ b/dss-git/dss-git-common/src/main/java/com.webank.wedatasphere.dss.git.common/protocol/request/GitBaseRequest.java @@ -0,0 +1,30 @@ +package com.webank.wedatasphere.dss.git.common.protocol.request; + +public class GitBaseRequest{ + private Long workspaceId; + private String projectName; + + public GitBaseRequest() { + } + + public GitBaseRequest(Long workspaceId, String projectName) { + this.workspaceId = workspaceId; + this.projectName = projectName; + } + + public Long getWorkspaceId() { + return workspaceId; + } + + public void setWorkspaceId(Long workspaceId) { + this.workspaceId = workspaceId; + } + + public String getProjectName() { + return projectName; + } + + public void setProjectName(String projectName) { + this.projectName = projectName; + } +} diff --git a/dss-git/dss-git-common/src/main/java/com.webank.wedatasphere.dss.git.common/protocol/request/GitBatchCommitRequest.java b/dss-git/dss-git-common/src/main/java/com.webank.wedatasphere.dss.git.common/protocol/request/GitBatchCommitRequest.java new file mode 100644 index 0000000000..d30b493e6c --- /dev/null +++ b/dss-git/dss-git-common/src/main/java/com.webank.wedatasphere.dss.git.common/protocol/request/GitBatchCommitRequest.java @@ -0,0 +1,66 @@ +package com.webank.wedatasphere.dss.git.common.protocol.request; + +import com.webank.wedatasphere.dss.common.entity.BmlResource; + +import java.util.List; + +public class GitBatchCommitRequest extends GitBaseRequest{ + private String comment; + private String username; + private List filePath; + private BmlResource bmlResource; + + public GitBatchCommitRequest(String comment, String username, List filePath, BmlResource bmlResource) { + this.comment = comment; + this.username = username; + this.filePath = filePath; + this.bmlResource = bmlResource; + } + + public GitBatchCommitRequest(Long workspaceId, String projectName, String comment, String username, List filePath, BmlResource bmlResource) { + super(workspaceId, projectName); + this.comment = comment; + this.username = username; + this.filePath = filePath; + this.bmlResource = bmlResource; + } + + public GitBatchCommitRequest() { + } + + public GitBatchCommitRequest(Long workspaceId, String projectName) { + super(workspaceId, projectName); + } + + public String getComment() { + return comment; + } + + public void setComment(String comment) { + this.comment = comment; + } + + public String getUsername() { + return username; + } + + public void setUsername(String username) { + this.username = username; + } + + public List getFilePath() { + return filePath; + } + + public void setFilePath(List filePath) { + this.filePath = filePath; + } + + public BmlResource getBmlResource() { + return bmlResource; + } + + public void setBmlResource(BmlResource bmlResource) { + this.bmlResource = bmlResource; + } +} diff --git a/dss-git/dss-git-common/src/main/java/com.webank.wedatasphere.dss.git.common/protocol/request/GitCheckProjectRequest.java b/dss-git/dss-git-common/src/main/java/com.webank.wedatasphere.dss.git.common/protocol/request/GitCheckProjectRequest.java new file mode 100644 index 0000000000..7d592f999a --- /dev/null +++ b/dss-git/dss-git-common/src/main/java/com.webank.wedatasphere.dss.git.common/protocol/request/GitCheckProjectRequest.java @@ -0,0 +1,63 @@ +package com.webank.wedatasphere.dss.git.common.protocol.request; + + +public class GitCheckProjectRequest extends GitBaseRequest { + /** + * DSS用户名 + */ + private String username; + private String gitUser; + private String gitToken; + + + + public GitCheckProjectRequest() { + } + + public GitCheckProjectRequest(String username) { + this.username = username; + } + + public GitCheckProjectRequest(Long workspaceId, String projectName, String username) { + super(workspaceId, projectName); + this.username = username; + } + + public GitCheckProjectRequest(String username, String gitUser, String gitToken) { + this.username = username; + this.gitUser = gitUser; + this.gitToken = gitToken; + } + + public GitCheckProjectRequest(Long workspaceId, String projectName, String username, String gitUser, String gitToken) { + super(workspaceId, projectName); + this.username = username; + this.gitUser = gitUser; + this.gitToken = gitToken; + } + + + public String getUsername() { + return username; + } + + public void setUsername(String username) { + this.username = username; + } + + public String getGitUser() { + return gitUser; + } + + public void setGitUser(String gitUser) { + this.gitUser = gitUser; + } + + public String getGitToken() { + return gitToken; + } + + public void setGitToken(String gitToken) { + this.gitToken = gitToken; + } +} diff --git a/dss-git/dss-git-common/src/main/java/com.webank.wedatasphere.dss.git.common/protocol/request/GitCommitInfoBetweenRequest.java b/dss-git/dss-git-common/src/main/java/com.webank.wedatasphere.dss.git.common/protocol/request/GitCommitInfoBetweenRequest.java new file mode 100644 index 0000000000..0860f8abd1 --- /dev/null +++ b/dss-git/dss-git-common/src/main/java/com.webank.wedatasphere.dss.git.common/protocol/request/GitCommitInfoBetweenRequest.java @@ -0,0 +1,47 @@ +package com.webank.wedatasphere.dss.git.common.protocol.request; + +public class GitCommitInfoBetweenRequest extends GitBaseRequest{ + private String oldCommitId; + private String newCommitId; + private String dirName; + + public GitCommitInfoBetweenRequest(String oldCommitId, String newCommitId, String dirName) { + this.oldCommitId = oldCommitId; + this.newCommitId = newCommitId; + this.dirName = dirName; + } + + public GitCommitInfoBetweenRequest(Long workspaceId, String projectName, String oldCommitId, String newCommitId, String dirName) { + super(workspaceId, projectName); + this.oldCommitId = oldCommitId; + this.newCommitId = newCommitId; + this.dirName = dirName; + } + + public GitCommitInfoBetweenRequest() { + } + + public String getOldCommitId() { + return oldCommitId; + } + + public void setOldCommitId(String oldCommitId) { + this.oldCommitId = oldCommitId; + } + + public String getNewCommitId() { + return newCommitId; + } + + public void setNewCommitId(String newCommitId) { + this.newCommitId = newCommitId; + } + + public String getDirName() { + return dirName; + } + + public void setDirName(String dirName) { + this.dirName = dirName; + } +} diff --git a/dss-git/dss-git-common/src/main/java/com.webank.wedatasphere.dss.git.common/protocol/request/GitCommitRequest.java b/dss-git/dss-git-common/src/main/java/com.webank.wedatasphere.dss.git.common/protocol/request/GitCommitRequest.java new file mode 100644 index 0000000000..e9e79ccb41 --- /dev/null +++ b/dss-git/dss-git-common/src/main/java/com.webank.wedatasphere.dss.git.common/protocol/request/GitCommitRequest.java @@ -0,0 +1,99 @@ +package com.webank.wedatasphere.dss.git.common.protocol.request; + +import com.webank.wedatasphere.dss.common.entity.BmlResource; + +import java.util.List; +import java.util.Map; + + +public class GitCommitRequest extends GitBaseRequest{ + /** + * 待修改的文件（key：文件相对路径，value：文件对应BML） + */ + private Map bmlResourceMap; + /** + * 提交更新时的comment + */ + private String comment; + /** + * 下载BMLReource使用的用户名 + */ + private String username; + private String gitUser; + private String gitToken; + + + public GitCommitRequest() { + } + + public GitCommitRequest(Map bmlResourceMap, String comment, String username, String gitUser, String gitToken) { + this.bmlResourceMap = bmlResourceMap; + this.comment = comment; + this.username = username; + this.gitUser = gitUser; + this.gitToken = gitToken; + } + + public GitCommitRequest(Long workspaceId, String projectName, Map bmlResourceMap, String comment, String username, String gitUser, String gitToken) { + super(workspaceId, projectName); + this.bmlResourceMap = bmlResourceMap; + this.comment = comment; + this.username = username; + this.gitUser = gitUser; + this.gitToken = gitToken; + } + + public GitCommitRequest(Long workspaceId, String projectName, Map bmlResourceMap, String comment, String username) { + super(workspaceId, projectName); + this.bmlResourceMap = bmlResourceMap; + this.comment = comment; + this.username = username; + } + + public GitCommitRequest(Map bmlResourceMap, String comment, String username) { + this.bmlResourceMap = bmlResourceMap; + this.comment = comment; + this.username = username; + } + + + public Map getBmlResourceMap() { + return bmlResourceMap; + } + + public void setBmlResourceMap(Map bmlResourceMap) { + this.bmlResourceMap = bmlResourceMap; + } + + public String getComment() { + return comment; + } + + public void setComment(String comment) { + this.comment = comment; + } + + public String getUsername() { + return username; + } + + public void setUsername(String username) { + this.username = username; + } + + public String getGitUser() { + return gitUser; + } + + public void setGitUser(String gitUser) { + this.gitUser = gitUser; + } + + public String getGitToken() { + return gitToken; + } + + public void setGitToken(String gitToken) { + this.gitToken = gitToken; + } +} diff --git a/dss-git/dss-git-common/src/main/java/com.webank.wedatasphere.dss.git.common/protocol/request/GitConnectRequest.java b/dss-git/dss-git-common/src/main/java/com.webank.wedatasphere.dss.git.common/protocol/request/GitConnectRequest.java new file mode 100644 index 0000000000..ff3e9e296f --- /dev/null +++ b/dss-git/dss-git-common/src/main/java/com.webank.wedatasphere.dss.git.common/protocol/request/GitConnectRequest.java @@ -0,0 +1,30 @@ +package com.webank.wedatasphere.dss.git.common.protocol.request; + +public class GitConnectRequest { + private String username; + private String token; + + public GitConnectRequest(String username, String token) { + this.username = username; + this.token = token; + } + + public GitConnectRequest() { + } + + public String getUsername() { + return username; + } + + public void setUsername(String username) { + this.username = username; + } + + public String getToken() { + return token; + } + + public void setToken(String token) { + this.token = token; + } +} diff --git a/dss-git/dss-git-common/src/main/java/com.webank.wedatasphere.dss.git.common/protocol/request/GitCreateProjectRequest.java b/dss-git/dss-git-common/src/main/java/com.webank.wedatasphere.dss.git.common/protocol/request/GitCreateProjectRequest.java new file mode 100644 index 0000000000..7d180f4255 --- /dev/null +++ b/dss-git/dss-git-common/src/main/java/com.webank.wedatasphere.dss.git.common/protocol/request/GitCreateProjectRequest.java @@ -0,0 +1,68 @@ +package com.webank.wedatasphere.dss.git.common.protocol.request; + + +import com.webank.wedatasphere.dss.common.entity.BmlResource; + +public class GitCreateProjectRequest extends GitBaseRequest{ + /** + * DSS项目对应的BML + */ + private BmlResource bmlResource; + /** + * 下载BMLReource使用的用户名 + */ + private String username; + private String gitUser; + private String gitToken; + + public GitCreateProjectRequest() { + } + + + public GitCreateProjectRequest(BmlResource bmlResource, String username, String gitUser, String gitToken) { + this.bmlResource = bmlResource; + this.username = username; + this.gitUser = gitUser; + this.gitToken = gitToken; + } + + public GitCreateProjectRequest(Long workspaceId, String projectName, BmlResource bmlResource, String username, String gitUser, String gitToken) { + super(workspaceId, projectName); + this.bmlResource = bmlResource; + this.username = username; + this.gitUser = gitUser; + this.gitToken = gitToken; + } + + public BmlResource getBmlResource() { + return bmlResource; + } + + public void setBmlResource(BmlResource bmlResource) { + this.bmlResource = bmlResource; + } + + public String getUsername() { + return username; + } + + public void setUsername(String username) { + this.username = username; + } + + public String getGitUser() { + return gitUser; + } + + public void setGitUser(String gitUser) { + this.gitUser = gitUser; + } + + public String getGitToken() { + return gitToken; + } + + public void setGitToken(String gitToken) { + this.gitToken = gitToken; + } +} diff --git a/dss-git/dss-git-common/src/main/java/com.webank.wedatasphere.dss.git.common/protocol/request/GitCurrentCommitRequest.java b/dss-git/dss-git-common/src/main/java/com.webank.wedatasphere.dss.git.common/protocol/request/GitCurrentCommitRequest.java new file mode 100644 index 0000000000..be53533c4d --- /dev/null +++ b/dss-git/dss-git-common/src/main/java/com.webank.wedatasphere.dss.git.common/protocol/request/GitCurrentCommitRequest.java @@ -0,0 +1,37 @@ +package com.webank.wedatasphere.dss.git.common.protocol.request; + +public class GitCurrentCommitRequest extends GitBaseRequest{ + private String username; + private String filepath; + + + public GitCurrentCommitRequest() { + } + + public GitCurrentCommitRequest(String username, String filepath) { + this.username = username; + this.filepath = filepath; + } + + public GitCurrentCommitRequest(Long workspaceId, String projectName, String username, String filepath) { + super(workspaceId, projectName); + this.username = username; + this.filepath = filepath; + } + + public String getUsername() { + return username; + } + + public void setUsername(String username) { + this.username = username; + } + + public String getFilepath() { + return filepath; + } + + public void setFilepath(String filepath) { + this.filepath = filepath; + } +} diff --git a/dss-git/dss-git-common/src/main/java/com.webank.wedatasphere.dss.git.common/protocol/request/GitDeleteRequest.java b/dss-git/dss-git-common/src/main/java/com.webank.wedatasphere.dss.git.common/protocol/request/GitDeleteRequest.java new file mode 100644 index 0000000000..1b42cb895f --- /dev/null +++ b/dss-git/dss-git-common/src/main/java/com.webank.wedatasphere.dss.git.common/protocol/request/GitDeleteRequest.java @@ -0,0 +1,42 @@ +package com.webank.wedatasphere.dss.git.common.protocol.request; + + +import java.util.List; + +public class GitDeleteRequest extends GitBaseRequest{ + /** + * 提交更新时的comment + */ + private String comment; + /** + * 需删除的文件列表 + */ + private List deleteFileList; + + public GitDeleteRequest() { + } + + public GitDeleteRequest(Long workspaceId, String projectName) { + super(workspaceId, projectName); + } + + public GitDeleteRequest(String comment, List deleteFileList) { + this.comment = comment; + this.deleteFileList = deleteFileList; + } + + public GitDeleteRequest(Long workspaceId, String projectName, String comment, List deleteFileList) { + super(workspaceId, projectName); + this.comment = comment; + this.deleteFileList = deleteFileList; + } + + public List getDeleteFileList() { + return deleteFileList; + } + + public void setDeleteFileList(List deleteFileList) { + this.deleteFileList = deleteFileList; + } + +} diff --git a/dss-git/dss-git-common/src/main/java/com.webank.wedatasphere.dss.git.common/protocol/request/GitDiffFileContentRequest.java b/dss-git/dss-git-common/src/main/java/com.webank.wedatasphere.dss.git.common/protocol/request/GitDiffFileContentRequest.java new file mode 100644 index 0000000000..402dc81cac --- /dev/null +++ b/dss-git/dss-git-common/src/main/java/com.webank.wedatasphere.dss.git.common/protocol/request/GitDiffFileContentRequest.java @@ -0,0 +1,50 @@ +package com.webank.wedatasphere.dss.git.common.protocol.request; + +import java.util.List; + +public class GitDiffFileContentRequest extends GitBaseRequest { + + + private String commitId; + /** + * 需获取内容的文件相对路径 + */ + private List filePaths; + + private String username; + + private Boolean publish; + + + public String getCommitId() { + return commitId; + } + + public void setCommitId(String commitId) { + this.commitId = commitId; + } + + public List getFilePaths() { + return filePaths; + } + + public void setFilePaths(List filePaths) { + this.filePaths = filePaths; + } + + public String getUsername() { + return username; + } + + public void setUsername(String username) { + this.username = username; + } + + public Boolean getPublish() { + return publish; + } + + public void setPublish(Boolean publish) { + this.publish = publish; + } +} diff --git a/dss-git/dss-git-common/src/main/java/com.webank.wedatasphere.dss.git.common/protocol/request/GitDiffFileContentResponse.java b/dss-git/dss-git-common/src/main/java/com.webank.wedatasphere.dss.git.common/protocol/request/GitDiffFileContentResponse.java new file mode 100644 index 0000000000..654ee733ab --- /dev/null +++ b/dss-git/dss-git-common/src/main/java/com.webank.wedatasphere.dss.git.common/protocol/request/GitDiffFileContentResponse.java @@ -0,0 +1,25 @@ +package com.webank.wedatasphere.dss.git.common.protocol.request; + +import com.webank.wedatasphere.dss.git.common.protocol.response.GitFileContentResponse; + +import java.util.ArrayList; +import java.util.List; + +public class GitDiffFileContentResponse { + + + private List gitFileContentResponseList; + + + public GitDiffFileContentResponse(List gitFileContentResponseList) { + this.gitFileContentResponseList = gitFileContentResponseList; + } + + public List getGitFileContentResponseList() { + return gitFileContentResponseList; + } + + public void setGitFileContentResponseList(List gitFileContentResponseList) { + this.gitFileContentResponseList = gitFileContentResponseList; + } +} diff --git a/dss-git/dss-git-common/src/main/java/com.webank.wedatasphere.dss.git.common/protocol/request/GitDiffRequest.java b/dss-git/dss-git-common/src/main/java/com.webank.wedatasphere.dss.git.common/protocol/request/GitDiffRequest.java new file mode 100644 index 0000000000..bb85039f11 --- /dev/null +++ b/dss-git/dss-git-common/src/main/java/com.webank.wedatasphere.dss.git.common/protocol/request/GitDiffRequest.java @@ -0,0 +1,48 @@ +package com.webank.wedatasphere.dss.git.common.protocol.request; + +import com.webank.wedatasphere.dss.common.entity.BmlResource; + +import java.util.Map; + + +public class GitDiffRequest extends GitBaseRequest{ + /** + * 待修改的文件（key：文件相对路径，value：文件对应BML） + */ + private Map bmlResourceMap; + /** + * 下载BMLReource使用的用户名 + */ + private String username; + + public GitDiffRequest() { + } + + public GitDiffRequest(Map bmlResourceMap, String username) { + this.bmlResourceMap = bmlResourceMap; + this.username = username; + } + + public GitDiffRequest(Long workspaceId, String projectName, Map bmlResourceMap, String username) { + super(workspaceId, projectName); + this.bmlResourceMap = bmlResourceMap; + this.username = username; + } + + + public Map getBmlResourceMap() { + return bmlResourceMap; + } + + public void setBmlResourceMap(Map bmlResourceMap) { + this.bmlResourceMap = bmlResourceMap; + } + + public String getUsername() { + return username; + } + + public void setUsername(String username) { + this.username = username; + } +} diff --git a/dss-git/dss-git-common/src/main/java/com.webank.wedatasphere.dss.git.common/protocol/request/GitDiffTargetCommitRequest.java b/dss-git/dss-git-common/src/main/java/com.webank.wedatasphere.dss.git.common/protocol/request/GitDiffTargetCommitRequest.java new file mode 100644 index 0000000000..a194345efb --- /dev/null +++ b/dss-git/dss-git-common/src/main/java/com.webank.wedatasphere.dss.git.common/protocol/request/GitDiffTargetCommitRequest.java @@ -0,0 +1,51 @@ +package com.webank.wedatasphere.dss.git.common.protocol.request; + +public class GitDiffTargetCommitRequest extends GitBaseRequest{ + private String commitId; + private String username; + private String filePath; + + public GitDiffTargetCommitRequest(String commitId, String username, String filePath) { + this.commitId = commitId; + this.username = username; + this.filePath = filePath; + } + + public GitDiffTargetCommitRequest(Long workspaceId, String projectName, String commitId, String username, String filePath) { + super(workspaceId, projectName); + this.commitId = commitId; + this.username = username; + this.filePath = filePath; + } + + public GitDiffTargetCommitRequest() { + } + + public GitDiffTargetCommitRequest(Long workspaceId, String projectName) { + super(workspaceId, projectName); + } + + public String getCommitId() { + return commitId; + } + + public void setCommitId(String commitId) { + this.commitId = commitId; + } + + public String getUsername() { + return username; + } + + public void setUsername(String username) { + this.username = username; + } + + public String getFilePath() { + return filePath; + } + + public void setFilePath(String filePath) { + this.filePath = filePath; + } +} diff --git a/dss-git/dss-git-common/src/main/java/com.webank.wedatasphere.dss.git.common/protocol/request/GitFileContentRequest.java b/dss-git/dss-git-common/src/main/java/com.webank.wedatasphere.dss.git.common/protocol/request/GitFileContentRequest.java new file mode 100644 index 0000000000..26e0e72368 --- /dev/null +++ b/dss-git/dss-git-common/src/main/java/com.webank.wedatasphere.dss.git.common/protocol/request/GitFileContentRequest.java @@ -0,0 +1,73 @@ +package com.webank.wedatasphere.dss.git.common.protocol.request; + + +import com.webank.wedatasphere.dss.common.entity.BmlResource; + +import java.util.Map; + +public class GitFileContentRequest extends GitBaseRequest{ + private String commitId; + /** + * 需获取内容的文件相对路径 + */ + private String filePath; + + private String username; + + private Boolean publish; + + public GitFileContentRequest() { + } + + public GitFileContentRequest(Long workspaceId, String projectName) { + super(workspaceId, projectName); + } + + public GitFileContentRequest(String commitId, String filePath, String username, Boolean publish) { + this.commitId = commitId; + this.filePath = filePath; + this.username = username; + this.publish = publish; + } + + public GitFileContentRequest(Long workspaceId, String projectName, String commitId, String filePath, String username, Boolean publish) { + super(workspaceId, projectName); + this.commitId = commitId; + this.filePath = filePath; + this.username = username; + this.publish = publish; + } + + + public String getCommitId() { + return commitId; + } + + public void setCommitId(String commitId) { + this.commitId = commitId; + } + + public String getFilePath() { + return filePath; + } + + public void setFilePath(String filePath) { + this.filePath = filePath; + } + + public Boolean getPublish() { + return publish; + } + + public void setPublish(Boolean publish) { + this.publish = publish; + } + + public String getUsername() { + return username; + } + + public void setUsername(String username) { + this.username = username; + } +} diff --git a/dss-git/dss-git-common/src/main/java/com.webank.wedatasphere.dss.git.common/protocol/request/GitHistoryRequest.java b/dss-git/dss-git-common/src/main/java/com.webank.wedatasphere.dss.git.common/protocol/request/GitHistoryRequest.java new file mode 100644 index 0000000000..39d4d8d642 --- /dev/null +++ b/dss-git/dss-git-common/src/main/java/com.webank.wedatasphere.dss.git.common/protocol/request/GitHistoryRequest.java @@ -0,0 +1,29 @@ +package com.webank.wedatasphere.dss.git.common.protocol.request; + +public class GitHistoryRequest extends GitBaseRequest{ + private String filePath; + + public GitHistoryRequest() { + } + + public GitHistoryRequest(Long workspaceId, String projectName) { + super(workspaceId, projectName); + } + + public GitHistoryRequest(String filePath) { + this.filePath = filePath; + } + + public GitHistoryRequest(Long workspaceId, String projectName, String filePath) { + super(workspaceId, projectName); + this.filePath = filePath; + } + + public String getFilePath() { + return filePath; + } + + public void setFilePath(String filePath) { + this.filePath = filePath; + } +} diff --git a/dss-git/dss-git-common/src/main/java/com.webank.wedatasphere.dss.git.common/protocol/request/GitRemoveRequest.java b/dss-git/dss-git-common/src/main/java/com.webank.wedatasphere.dss.git.common/protocol/request/GitRemoveRequest.java new file mode 100644 index 0000000000..c0ab56dd48 --- /dev/null +++ b/dss-git/dss-git-common/src/main/java/com.webank.wedatasphere.dss.git.common/protocol/request/GitRemoveRequest.java @@ -0,0 +1,44 @@ +package com.webank.wedatasphere.dss.git.common.protocol.request; + +import java.util.List; + +public class GitRemoveRequest extends GitBaseRequest{ + private List path; + + public String getUsername() { + return username; + } + + public void setUsername(String username) { + this.username = username; + } + + private String username; + + + public GitRemoveRequest(List path, String username) { + this.path = path; + this.username = username; + } + + public GitRemoveRequest(Long workspaceId, String projectName, List path, String username) { + super(workspaceId, projectName); + this.path = path; + this.username = username; + } + + public GitRemoveRequest() { + } + + public GitRemoveRequest(Long workspaceId, String projectName) { + super(workspaceId, projectName); + } + + public List getPath() { + return path; + } + + public void setPath(List path) { + this.path = path; + } +} diff --git a/dss-git/dss-git-common/src/main/java/com.webank.wedatasphere.dss.git.common/protocol/request/GitRenameRequest.java b/dss-git/dss-git-common/src/main/java/com.webank.wedatasphere.dss.git.common/protocol/request/GitRenameRequest.java new file mode 100644 index 0000000000..93a099d431 --- /dev/null +++ b/dss-git/dss-git-common/src/main/java/com.webank.wedatasphere.dss.git.common/protocol/request/GitRenameRequest.java @@ -0,0 +1,49 @@ +package com.webank.wedatasphere.dss.git.common.protocol.request; + +public class GitRenameRequest extends GitBaseRequest{ + private String oldName; + + private String name; + + private String username; + + public GitRenameRequest(String oldName, String name, String username) { + this.oldName = oldName; + this.name = name; + this.username = username; + } + + public GitRenameRequest(Long workspaceId, String projectName, String oldName, String name, String username) { + super(workspaceId, projectName); + this.oldName = oldName; + this.name = name; + this.username = username; + } + + public GitRenameRequest() { + } + + public String getOldName() { + return oldName; + } + + public void setOldName(String oldName) { + this.oldName = oldName; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getUsername() { + return username; + } + + public void setUsername(String username) { + this.username = username; + } +} diff --git a/dss-git/dss-git-common/src/main/java/com.webank.wedatasphere.dss.git.common/protocol/request/GitRevertRequest.java b/dss-git/dss-git-common/src/main/java/com.webank.wedatasphere.dss.git.common/protocol/request/GitRevertRequest.java new file mode 100644 index 0000000000..2b42b27f7d --- /dev/null +++ b/dss-git/dss-git-common/src/main/java/com.webank.wedatasphere.dss.git.common/protocol/request/GitRevertRequest.java @@ -0,0 +1,44 @@ +package com.webank.wedatasphere.dss.git.common.protocol.request; + + + +public class GitRevertRequest extends GitBaseRequest{ + private String commitId; + private String path; + + public String getUsername() { + return username; + } + + public void setUsername(String username) { + this.username = username; + } + + private String username; + + public GitRevertRequest(Long workspaceId, String projectName, String commitId, String path, String username) { + super(workspaceId, projectName); + this.commitId = commitId; + this.path = path; + this.username = username; + } + + public GitRevertRequest() { + } + + public String getCommitId() { + return commitId; + } + + public void setCommitId(String commitId) { + this.commitId = commitId; + } + + public String getPath() { + return path; + } + + public void setPath(String path) { + this.path = path; + } +} diff --git a/dss-git/dss-git-common/src/main/java/com.webank.wedatasphere.dss.git.common/protocol/request/GitSearchRequest.java b/dss-git/dss-git-common/src/main/java/com.webank.wedatasphere.dss.git.common/protocol/request/GitSearchRequest.java new file mode 100644 index 0000000000..818cfbd49b --- /dev/null +++ b/dss-git/dss-git-common/src/main/java/com.webank.wedatasphere.dss.git.common/protocol/request/GitSearchRequest.java @@ -0,0 +1,77 @@ +package com.webank.wedatasphere.dss.git.common.protocol.request; + +import java.util.List; + +public class GitSearchRequest extends GitBaseRequest{ + /** + * 搜索条件：工作流（相对路径） + */ + private List workflowNameList; + /** + * 搜索条件：关键字 + */ + private String searchContent; + /** + * 搜索条件：文件名 + */ + private String nodeName; + /** + * 搜索条件：节点类型（后缀） + */ + private List typeList; + private Integer pageSize; + private Integer pageNow; + + public GitSearchRequest() { + } + + + public List getWorkflowNameList() { + return workflowNameList; + } + + public void setWorkflowNameList(List workflowNameList) { + this.workflowNameList = workflowNameList; + } + + public String getSearchContent() { + return searchContent; + } + + public void setSearchContent(String searchContent) { + this.searchContent = searchContent; + } + + public String getNodeName() { + return nodeName; + } + + public void setNodeName(String nodeName) { + this.nodeName = nodeName; + } + + public List getTypeList() { + return typeList; + } + + public void setTypeList(List typeList) { + this.typeList = typeList; + } + + + public Integer getPageSize() { + return pageSize; + } + + public void setPageSize(Integer pageSize) { + this.pageSize = pageSize; + } + + public Integer getPageNow() { + return pageNow; + } + + public void setPageNow(Integer pageNow) { + this.pageNow = pageNow; + } +} diff --git a/dss-git/dss-git-common/src/main/java/com.webank.wedatasphere.dss.git.common/protocol/request/GitUserByWorkspaceIdRequest.java b/dss-git/dss-git-common/src/main/java/com.webank.wedatasphere.dss.git.common/protocol/request/GitUserByWorkspaceIdRequest.java new file mode 100644 index 0000000000..7f4a5a5d66 --- /dev/null +++ b/dss-git/dss-git-common/src/main/java/com.webank.wedatasphere.dss.git.common/protocol/request/GitUserByWorkspaceIdRequest.java @@ -0,0 +1,31 @@ +package com.webank.wedatasphere.dss.git.common.protocol.request; + + +public class GitUserByWorkspaceIdRequest extends GitBaseRequest{ + private String username; + + public GitUserByWorkspaceIdRequest(String username) { + this.username = username; + } + + public GitUserByWorkspaceIdRequest(Long workspaceId, String projectName, String username) { + super(workspaceId, projectName); + this.username = username; + } + + public GitUserByWorkspaceIdRequest() { + } + + public GitUserByWorkspaceIdRequest(Long workspaceId, String projectName) { + super(workspaceId, projectName); + } + + + public String getUsername() { + return username; + } + + public void setUsername(String username) { + this.username = username; + } +} diff --git a/dss-git/dss-git-common/src/main/java/com.webank.wedatasphere.dss.git.common/protocol/request/GitUserInfoByRequest.java b/dss-git/dss-git-common/src/main/java/com.webank.wedatasphere.dss.git.common/protocol/request/GitUserInfoByRequest.java new file mode 100644 index 0000000000..7f9c94b73a --- /dev/null +++ b/dss-git/dss-git-common/src/main/java/com.webank.wedatasphere.dss.git.common/protocol/request/GitUserInfoByRequest.java @@ -0,0 +1,31 @@ +package com.webank.wedatasphere.dss.git.common.protocol.request; + +public class GitUserInfoByRequest { + private String type; + private Long workspaceId; + private String gitUserName; + + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } + + public Long getWorkspaceId() { + return workspaceId; + } + + public void setWorkspaceId(Long workspaceId) { + this.workspaceId = workspaceId; + } + + public String getGitUserName() { + return gitUserName; + } + + public void setGitUserName(String gitUserName) { + this.gitUserName = gitUserName; + } +} diff --git a/dss-git/dss-git-common/src/main/java/com.webank.wedatasphere.dss.git.common/protocol/request/GitUserInfoRequest.java b/dss-git/dss-git-common/src/main/java/com.webank.wedatasphere.dss.git.common/protocol/request/GitUserInfoRequest.java new file mode 100644 index 0000000000..aa170f2dba --- /dev/null +++ b/dss-git/dss-git-common/src/main/java/com.webank.wedatasphere.dss.git.common/protocol/request/GitUserInfoRequest.java @@ -0,0 +1,35 @@ +package com.webank.wedatasphere.dss.git.common.protocol.request; + + +public class GitUserInfoRequest { + private Long workspaceId; + private String type; + /** + * 密码 token 是否解密 true-解密 + */ + private Boolean decrypt; + + public Long getWorkspaceId() { + return workspaceId; + } + + public void setWorkspaceId(Long workspaceId) { + this.workspaceId = workspaceId; + } + + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } + + public Boolean getDecrypt() { + return decrypt; + } + + public void setDecrypt(Boolean decrypt) { + this.decrypt = decrypt; + } +} diff --git a/dss-git/dss-git-common/src/main/java/com.webank.wedatasphere.dss.git.common/protocol/request/GitUserUpdateRequest.java b/dss-git/dss-git-common/src/main/java/com.webank.wedatasphere.dss.git.common/protocol/request/GitUserUpdateRequest.java new file mode 100644 index 0000000000..d3d577b9d2 --- /dev/null +++ b/dss-git/dss-git-common/src/main/java/com.webank.wedatasphere.dss.git.common/protocol/request/GitUserUpdateRequest.java @@ -0,0 +1,25 @@ +package com.webank.wedatasphere.dss.git.common.protocol.request; + +import com.webank.wedatasphere.dss.git.common.protocol.GitUserEntity; + + +public class GitUserUpdateRequest { + private GitUserEntity gitUser; + private String username; + + public GitUserEntity getGitUser() { + return gitUser; + } + + public void setGitUser(GitUserEntity gitUser) { + this.gitUser = gitUser; + } + + public String getUsername() { + return username; + } + + public void setUsername(String username) { + this.username = username; + } +} diff --git a/dss-git/dss-git-common/src/main/java/com.webank.wedatasphere.dss.git.common/protocol/response/GitAddMemberResponse.java b/dss-git/dss-git-common/src/main/java/com.webank.wedatasphere.dss.git.common/protocol/response/GitAddMemberResponse.java new file mode 100644 index 0000000000..d41cb77118 --- /dev/null +++ b/dss-git/dss-git-common/src/main/java/com.webank.wedatasphere.dss.git.common/protocol/response/GitAddMemberResponse.java @@ -0,0 +1,20 @@ +package com.webank.wedatasphere.dss.git.common.protocol.response; + +public class GitAddMemberResponse { + private String gitUrl; + + public GitAddMemberResponse(String gitUrl) { + this.gitUrl = gitUrl; + } + + public GitAddMemberResponse() { + } + + public String getGitUrl() { + return gitUrl; + } + + public void setGitUrl(String gitUrl) { + this.gitUrl = gitUrl; + } +} diff --git a/dss-git/dss-git-common/src/main/java/com.webank.wedatasphere.dss.git.common/protocol/response/GitArchivePorjectResponse.java b/dss-git/dss-git-common/src/main/java/com.webank.wedatasphere.dss.git.common/protocol/response/GitArchivePorjectResponse.java new file mode 100644 index 0000000000..88884222a6 --- /dev/null +++ b/dss-git/dss-git-common/src/main/java/com.webank.wedatasphere.dss.git.common/protocol/response/GitArchivePorjectResponse.java @@ -0,0 +1,21 @@ +package com.webank.wedatasphere.dss.git.common.protocol.response; + + +public class GitArchivePorjectResponse{ + private String projectName; + + public GitArchivePorjectResponse(String projectName) { + this.projectName = projectName; + } + + public GitArchivePorjectResponse() { + } + + public String getProjectName() { + return projectName; + } + + public void setProjectName(String projectName) { + this.projectName = projectName; + } +} diff --git a/dss-git/dss-git-common/src/main/java/com.webank.wedatasphere.dss.git.common/protocol/response/GitBaseResponse.java b/dss-git/dss-git-common/src/main/java/com.webank.wedatasphere.dss.git.common/protocol/response/GitBaseResponse.java new file mode 100644 index 0000000000..9eeaa6169b --- /dev/null +++ b/dss-git/dss-git-common/src/main/java/com.webank.wedatasphere.dss.git.common/protocol/response/GitBaseResponse.java @@ -0,0 +1,42 @@ +package com.webank.wedatasphere.dss.git.common.protocol.response; + + +public class GitBaseResponse { + private String status; + private int index; + private String message; + + public GitBaseResponse(String status, int index, String message) { + this.status = status; + this.index = index; + this.message = message; + } + + public GitBaseResponse() { + } + + + public String getStatus() { + return status; + } + + public void setStatus(String status) { + this.status = status; + } + + public int getIndex() { + return index; + } + + public void setIndex(int index) { + this.index = index; + } + + public String getMessage() { + return message; + } + + public void setMessage(String message) { + this.message = message; + } +} diff --git a/dss-git/dss-git-common/src/main/java/com.webank.wedatasphere.dss.git.common/protocol/response/GitCheckProjectResponse.java b/dss-git/dss-git-common/src/main/java/com.webank.wedatasphere.dss.git.common/protocol/response/GitCheckProjectResponse.java new file mode 100644 index 0000000000..ffba6c4628 --- /dev/null +++ b/dss-git/dss-git-common/src/main/java/com.webank.wedatasphere.dss.git.common/protocol/response/GitCheckProjectResponse.java @@ -0,0 +1,34 @@ +package com.webank.wedatasphere.dss.git.common.protocol.response; + + +public class GitCheckProjectResponse{ + private String projectName; + /** + * true为重复 + */ + private Boolean repeat; + + public GitCheckProjectResponse(String projectName, Boolean repeat) { + this.projectName = projectName; + this.repeat = repeat; + } + + public GitCheckProjectResponse() { + } + + public String getProjectName() { + return projectName; + } + + public void setProjectName(String projectName) { + this.projectName = projectName; + } + + public Boolean getRepeat() { + return repeat; + } + + public void setRepeat(Boolean repeat) { + this.repeat = repeat; + } +} diff --git a/dss-git/dss-git-common/src/main/java/com.webank.wedatasphere.dss.git.common/protocol/response/GitCommitResponse.java b/dss-git/dss-git-common/src/main/java/com.webank.wedatasphere.dss.git.common/protocol/response/GitCommitResponse.java new file mode 100644 index 0000000000..e7f258c24d --- /dev/null +++ b/dss-git/dss-git-common/src/main/java/com.webank.wedatasphere.dss.git.common/protocol/response/GitCommitResponse.java @@ -0,0 +1,90 @@ +package com.webank.wedatasphere.dss.git.common.protocol.response; + + +import com.google.common.base.Objects; + +public class GitCommitResponse{ + private String commitId; + private String commitTime; + private String commitUser; + private String comment; + private String commentFull; + + public GitCommitResponse(String commitId, String commitTime, String commitUser, String comment) { + this.commitId = commitId; + this.commitTime = commitTime; + this.commitUser = commitUser; + this.comment = comment; + } + + public GitCommitResponse() { + } + + public String getCommitId() { + return commitId; + } + + public void setCommitId(String commitId) { + this.commitId = commitId; + } + + public String getCommitTime() { + return commitTime; + } + + public void setCommitTime(String commitTime) { + this.commitTime = commitTime; + } + + public String getCommitUser() { + return commitUser; + } + + public void setCommitUser(String commitUser) { + this.commitUser = commitUser; + } + + public String getComment() { + return comment; + } + + public void setComment(String comment) { + this.comment = comment; + } + + public String getCommentFull() { + return commentFull; + } + + public void setCommentFull(String commentFull) { + this.commentFull = commentFull; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + GitCommitResponse that = (GitCommitResponse) o; + return Objects.equal(commitId, that.commitId); + } + + @Override + public int hashCode() { + return Objects.hashCode(commitId); + } + + @Override + public String toString() { + return "GitCommitResponse{" + + "commitId=" + commitId + + ", commitTime='" + commitTime + '\'' + + ", commitUser='" + commitUser + '\'' + + ", comment='" + comment + '\'' + + ", commentFull='" + commentFull + '\'' + + '}'; + } +} diff --git a/dss-git/dss-git-common/src/main/java/com.webank.wedatasphere.dss.git.common/protocol/response/GitConnectResponse.java b/dss-git/dss-git-common/src/main/java/com.webank.wedatasphere.dss.git.common/protocol/response/GitConnectResponse.java new file mode 100644 index 0000000000..52dbcd4cc9 --- /dev/null +++ b/dss-git/dss-git-common/src/main/java/com.webank.wedatasphere.dss.git.common/protocol/response/GitConnectResponse.java @@ -0,0 +1,23 @@ +package com.webank.wedatasphere.dss.git.common.protocol.response; + +public class GitConnectResponse { + /** + * token-用户名是否匹配 true-匹配 false-不匹配 + */ + private Boolean connect; + + public GitConnectResponse(Boolean connect) { + this.connect = connect; + } + + public GitConnectResponse() { + } + + public Boolean getConnect() { + return connect; + } + + public void setConnect(Boolean connect) { + this.connect = connect; + } +} diff --git a/dss-git/dss-git-common/src/main/java/com.webank.wedatasphere.dss.git.common/protocol/response/GitCreateProjectResponse.java b/dss-git/dss-git-common/src/main/java/com.webank.wedatasphere.dss.git.common/protocol/response/GitCreateProjectResponse.java new file mode 100644 index 0000000000..e51403b0ea --- /dev/null +++ b/dss-git/dss-git-common/src/main/java/com.webank.wedatasphere.dss.git.common/protocol/response/GitCreateProjectResponse.java @@ -0,0 +1,32 @@ +package com.webank.wedatasphere.dss.git.common.protocol.response; + + +public class GitCreateProjectResponse{ + private String commitId; + private String projectName; + + public GitCreateProjectResponse(String commitId, String projectName) { + this.commitId = commitId; + this.projectName = projectName; + } + + public GitCreateProjectResponse() { + } + + + public String getCommitId() { + return commitId; + } + + public void setCommitId(String commitId) { + this.commitId = commitId; + } + + public String getProjectName() { + return projectName; + } + + public void setProjectName(String projectName) { + this.projectName = projectName; + } +} diff --git a/dss-git/dss-git-common/src/main/java/com.webank.wedatasphere.dss.git.common/protocol/response/GitDeleteResponse.java b/dss-git/dss-git-common/src/main/java/com.webank.wedatasphere.dss.git.common/protocol/response/GitDeleteResponse.java new file mode 100644 index 0000000000..3682bb6360 --- /dev/null +++ b/dss-git/dss-git-common/src/main/java/com.webank.wedatasphere.dss.git.common/protocol/response/GitDeleteResponse.java @@ -0,0 +1,33 @@ +package com.webank.wedatasphere.dss.git.common.protocol.response; + +import java.util.List; + +public class GitDeleteResponse { + private String projectName; + + private List path; + + public GitDeleteResponse(String projectName, List path) { + this.projectName = projectName; + this.path = path; + } + + public GitDeleteResponse() { + } + + public String getProjectName() { + return projectName; + } + + public void setProjectName(String projectName) { + this.projectName = projectName; + } + + public List getPath() { + return path; + } + + public void setPath(List path) { + this.path = path; + } +} diff --git a/dss-git/dss-git-common/src/main/java/com.webank.wedatasphere.dss.git.common/protocol/response/GitDiffResponse.java b/dss-git/dss-git-common/src/main/java/com.webank.wedatasphere.dss.git.common/protocol/response/GitDiffResponse.java new file mode 100644 index 0000000000..9301d6f2d6 --- /dev/null +++ b/dss-git/dss-git-common/src/main/java/com.webank.wedatasphere.dss.git.common/protocol/response/GitDiffResponse.java @@ -0,0 +1,46 @@ +package com.webank.wedatasphere.dss.git.common.protocol.response; + + +import com.webank.wedatasphere.dss.git.common.protocol.GitTree; + +import java.util.List; + +public class GitDiffResponse { + private List codeTree; + private List metaTree; + private String commitId; + + public GitDiffResponse(List codeTree, List metaTree, String commitId) { + this.codeTree = codeTree; + this.metaTree = metaTree; + this.commitId = commitId; + } + + public GitDiffResponse() { + } + + + public List getCodeTree() { + return codeTree; + } + + public void setCodeTree(List codeTree) { + this.codeTree = codeTree; + } + + public List getMetaTree() { + return metaTree; + } + + public void setMetaTree(List metaTree) { + this.metaTree = metaTree; + } + + public String getCommitId() { + return commitId; + } + + public void setCommitId(String commitId) { + this.commitId = commitId; + } +} diff --git a/dss-git/dss-git-common/src/main/java/com.webank.wedatasphere.dss.git.common/protocol/response/GitFileContentResponse.java b/dss-git/dss-git-common/src/main/java/com.webank.wedatasphere.dss.git.common/protocol/response/GitFileContentResponse.java new file mode 100644 index 0000000000..b20281f339 --- /dev/null +++ b/dss-git/dss-git-common/src/main/java/com.webank.wedatasphere.dss.git.common/protocol/response/GitFileContentResponse.java @@ -0,0 +1,97 @@ +package com.webank.wedatasphere.dss.git.common.protocol.response; + +public class GitFileContentResponse{ + // 提交前 发布前 文件内容 + private String before; + // 提交后 发布后 文件内容 + private String after; + // 反显注释内容 --仅发布时diff需要 + private String beforeAnnotate; + // 反显CommitId --仅发布时diff需要 + private String beforeCommitId; + // 反显注释内容 --仅发布时diff需要 + private String AfterAnnotate; + // 反显CommitId --仅发布时diff需要 + private String AfterCommitId; + + // 文件路径 + private String filePath; + + public GitFileContentResponse() { + } + + public GitFileContentResponse(String before, String after, String beforeAnnotate, String beforeCommitId, String afterAnnotate, String afterCommitId) { + this.before = before; + this.after = after; + this.beforeAnnotate = beforeAnnotate; + this.beforeCommitId = beforeCommitId; + AfterAnnotate = afterAnnotate; + AfterCommitId = afterCommitId; + } + + public GitFileContentResponse(String before, String after, String beforeAnnotate, String beforeCommitId, String afterAnnotate, String afterCommitId, String filePath) { + this.before = before; + this.after = after; + this.beforeAnnotate = beforeAnnotate; + this.beforeCommitId = beforeCommitId; + AfterAnnotate = afterAnnotate; + AfterCommitId = afterCommitId; + this.filePath = filePath; + } + + public String getBefore() { + return before; + } + + public void setBefore(String before) { + this.before = before; + } + + public String getAfter() { + return after; + } + + public void setAfter(String after) { + this.after = after; + } + + public String getBeforeAnnotate() { + return beforeAnnotate; + } + + public void setBeforeAnnotate(String beforeAnnotate) { + this.beforeAnnotate = beforeAnnotate; + } + + public String getBeforeCommitId() { + return beforeCommitId; + } + + public void setBeforeCommitId(String beforeCommitId) { + this.beforeCommitId = beforeCommitId; + } + + public String getAfterAnnotate() { + return AfterAnnotate; + } + + public void setAfterAnnotate(String afterAnnotate) { + AfterAnnotate = afterAnnotate; + } + + public String getAfterCommitId() { + return AfterCommitId; + } + + public void setAfterCommitId(String afterCommitId) { + AfterCommitId = afterCommitId; + } + + public String getFilePath() { + return filePath; + } + + public void setFilePath(String filePath) { + this.filePath = filePath; + } +} diff --git a/dss-git/dss-git-common/src/main/java/com.webank.wedatasphere.dss.git.common/protocol/response/GitHistoryResponse.java b/dss-git/dss-git-common/src/main/java/com.webank.wedatasphere.dss.git.common/protocol/response/GitHistoryResponse.java new file mode 100644 index 0000000000..1254ce1be4 --- /dev/null +++ b/dss-git/dss-git-common/src/main/java/com.webank.wedatasphere.dss.git.common/protocol/response/GitHistoryResponse.java @@ -0,0 +1,22 @@ +package com.webank.wedatasphere.dss.git.common.protocol.response; + +import java.util.List; + +public class GitHistoryResponse { + private List responses; + + public GitHistoryResponse(List responses) { + this.responses = responses; + } + + public GitHistoryResponse() { + } + + public List getResponses() { + return responses; + } + + public void setResponses(List responses) { + this.responses = responses; + } +} diff --git a/dss-git/dss-git-common/src/main/java/com.webank.wedatasphere.dss.git.common/protocol/response/GitSearchResponse.java b/dss-git/dss-git-common/src/main/java/com.webank.wedatasphere.dss.git.common/protocol/response/GitSearchResponse.java new file mode 100644 index 0000000000..846a014797 --- /dev/null +++ b/dss-git/dss-git-common/src/main/java/com.webank.wedatasphere.dss.git.common/protocol/response/GitSearchResponse.java @@ -0,0 +1,36 @@ +package com.webank.wedatasphere.dss.git.common.protocol.response; + + +import com.webank.wedatasphere.dss.git.common.protocol.GitSearchResult; + +import java.util.List; +import java.util.Map; + +public class GitSearchResponse{ + private List result; + private Integer total; + + public GitSearchResponse(List result, Integer total) { + this.result = result; + this.total = total; + } + + public GitSearchResponse() { + } + + public List getResult() { + return result; + } + + public void setResult(List result) { + this.result = result; + } + + public Integer getTotal() { + return total; + } + + public void setTotal(Integer total) { + this.total = total; + } +} diff --git a/dss-git/dss-git-common/src/main/java/com.webank.wedatasphere.dss.git.common/protocol/response/GitUserByWorkspaceResponse.java b/dss-git/dss-git-common/src/main/java/com.webank.wedatasphere.dss.git.common/protocol/response/GitUserByWorkspaceResponse.java new file mode 100644 index 0000000000..a52ddc2df4 --- /dev/null +++ b/dss-git/dss-git-common/src/main/java/com.webank.wedatasphere.dss.git.common/protocol/response/GitUserByWorkspaceResponse.java @@ -0,0 +1,24 @@ +package com.webank.wedatasphere.dss.git.common.protocol.response; + +import com.webank.wedatasphere.dss.git.common.protocol.GitUserEntity; + +import java.util.Map; + +public class GitUserByWorkspaceResponse { + private Map map; + + public GitUserByWorkspaceResponse(Map map) { + this.map = map; + } + + public GitUserByWorkspaceResponse() { + } + + public Map getMap() { + return map; + } + + public void setMap(Map map) { + this.map = map; + } +} diff --git a/dss-git/dss-git-common/src/main/java/com.webank.wedatasphere.dss.git.common/protocol/response/GitUserInfoListResponse.java b/dss-git/dss-git-common/src/main/java/com.webank.wedatasphere.dss.git.common/protocol/response/GitUserInfoListResponse.java new file mode 100644 index 0000000000..c7152173b9 --- /dev/null +++ b/dss-git/dss-git-common/src/main/java/com.webank.wedatasphere.dss.git.common/protocol/response/GitUserInfoListResponse.java @@ -0,0 +1,17 @@ +package com.webank.wedatasphere.dss.git.common.protocol.response; + +import com.webank.wedatasphere.dss.git.common.protocol.GitUserEntity; + +import java.util.List; + +public class GitUserInfoListResponse { + private List gitUserEntities; + + public List getGitUserEntities() { + return gitUserEntities; + } + + public void setGitUserEntities(List gitUserEntities) { + this.gitUserEntities = gitUserEntities; + } +} diff --git a/dss-git/dss-git-common/src/main/java/com.webank.wedatasphere.dss.git.common/protocol/response/GitUserInfoResponse.java b/dss-git/dss-git-common/src/main/java/com.webank.wedatasphere.dss.git.common/protocol/response/GitUserInfoResponse.java new file mode 100644 index 0000000000..b31a09ab49 --- /dev/null +++ b/dss-git/dss-git-common/src/main/java/com.webank.wedatasphere.dss.git.common/protocol/response/GitUserInfoResponse.java @@ -0,0 +1,16 @@ +package com.webank.wedatasphere.dss.git.common.protocol.response; + +import com.webank.wedatasphere.dss.git.common.protocol.GitUserEntity; + + +public class GitUserInfoResponse { + private GitUserEntity gitUser; + + public GitUserEntity getGitUser() { + return gitUser; + } + + public void setGitUser(GitUserEntity gitUser) { + this.gitUser = gitUser; + } +} diff --git a/dss-git/dss-git-common/src/main/java/com.webank.wedatasphere.dss.git.common/protocol/response/GitUserUpdateResponse.java b/dss-git/dss-git-common/src/main/java/com.webank.wedatasphere.dss.git.common/protocol/response/GitUserUpdateResponse.java new file mode 100644 index 0000000000..248c61fd7f --- /dev/null +++ b/dss-git/dss-git-common/src/main/java/com.webank.wedatasphere.dss.git.common/protocol/response/GitUserUpdateResponse.java @@ -0,0 +1,39 @@ +package com.webank.wedatasphere.dss.git.common.protocol.response; + + +public class GitUserUpdateResponse { + private int status; + private String msg; + + private Long workspaceId; + + public GitUserUpdateResponse(int status, String msg, Long workspaceId) { + this.status = status; + this.msg = msg; + this.workspaceId = workspaceId; + } + + public int getStatus() { + return status; + } + + public void setStatus(int status) { + this.status = status; + } + + public String getMsg() { + return msg; + } + + public void setMsg(String msg) { + this.msg = msg; + } + + public Long getWorkspaceId() { + return workspaceId; + } + + public void setWorkspaceId(Long workspaceId) { + this.workspaceId = workspaceId; + } +} diff --git a/dss-git/dss-git-common/src/main/java/com.webank.wedatasphere.dss.git.common/protocol/util/UrlUtils.java b/dss-git/dss-git-common/src/main/java/com.webank.wedatasphere.dss.git.common/protocol/util/UrlUtils.java new file mode 100644 index 0000000000..fc785cfd49 --- /dev/null +++ b/dss-git/dss-git-common/src/main/java/com.webank.wedatasphere.dss.git.common/protocol/util/UrlUtils.java @@ -0,0 +1,18 @@ +package com.webank.wedatasphere.dss.git.common.protocol.util; + +import java.io.File; + + +public class UrlUtils { + + public static String normalizeIp(String ip) { + ip = ip.trim(); + if(!ip.contains("http")) { + ip = "http://" + ip; + } + while(ip.endsWith("/")) { + ip = ip.substring(0, ip.length()-1); + } + return ip; + } +} diff --git a/dss-git/dss-git-server/pom.xml b/dss-git/dss-git-server/pom.xml new file mode 100644 index 0000000000..93a59d667b --- /dev/null +++ b/dss-git/dss-git-server/pom.xml @@ -0,0 +1,132 @@ + + + + dss-git + com.webank.wedatasphere.dss + 1.10.0-SNAPSHOT + ../pom.xml + + 4.0.0 + + dss-git-server + + + + org.apache.linkis + linkis-mybatis + ${linkis.version} + provided + + + com.webank.wedatasphere.dss + dss-common + ${dss.version} + + + org.apache.httpcomponents + httpclient + ${httpclient.version} + + + com.webank.wedatasphere.dss + dss-git-common + ${dss.version} + + + org.eclipse.jgit + org.eclipse.jgit + 5.13.0.202109080827-r + + + org.apache.linkis + linkis-rpc + ${linkis.version} + provided + + + spring-cloud-starter-netflix-eureka-client + org.springframework.cloud + + + org.springframework.boot + spring-boot-starter-log4j2 + + + + + com.vaadin.external.google + android-json + 0.0.20131108.vaadin1 + compile + + + + + + + + org.apache.maven.plugins + maven-deploy-plugin + + + + net.alchim31.maven + scala-maven-plugin + + + org.apache.maven.plugins + maven-jar-plugin + + + org.apache.maven.plugins + maven-compiler-plugin + + 8 + 8 + + + + org.apache.maven.plugins + maven-assembly-plugin + 2.3 + false + + + make-assembly + package + + single + + + + src/main/assembly/distribution.xml + + + + + + false + out + false + false + + src/main/assembly/distribution.xml + + + + + + + src/main/java + + **/*.xml + **/*.properties + **/*.yml + + + + + + \ No newline at end of file diff --git a/dss-git/dss-git-server/src/main/assembly/distribution.xml b/dss-git/dss-git-server/src/main/assembly/distribution.xml new file mode 100644 index 0000000000..9bcd9d08ed --- /dev/null +++ b/dss-git/dss-git-server/src/main/assembly/distribution.xml @@ -0,0 +1,46 @@ + + + + dss-framework-git-server + + dir + + true + dss-framework-git-server + + + + + + lib + true + true + false + true + true + + + + + + + diff --git a/dss-git/dss-git-server/src/main/java/com/webank/wedatasphere/dss/git/constant/DSSGitConstant.java b/dss-git/dss-git-server/src/main/java/com/webank/wedatasphere/dss/git/constant/DSSGitConstant.java new file mode 100644 index 0000000000..3f9c74fdae --- /dev/null +++ b/dss-git/dss-git-server/src/main/java/com/webank/wedatasphere/dss/git/constant/DSSGitConstant.java @@ -0,0 +1,21 @@ +package com.webank.wedatasphere.dss.git.constant; + +import com.webank.wedatasphere.dss.git.common.protocol.config.GitServerConfig; +import com.webank.wedatasphere.dss.git.utils.FileUtils; + +import java.io.File; + +public class DSSGitConstant { + public static String GIT_PATH_PRE = File.separator + FileUtils.normalizePath(GitServerConfig.GIT_SERVER_PATH.getValue()) + File.separator; + public static String GIT_PATH_SUFFIX = File.separator + ".git"; + + public static String GIT_USERNAME_FLAG = " by: "; + + // 修改未暂存 + public static String GIT_DIFF_STATUS_MODIFIED = "modified"; + // 新增文件未暂存 + public static String GIT_DIFF_STATUS_UNTRACKED = "untracked"; + // 删除未暂存 + public static String GIT_DIFF_STATUS_Missing = "missing"; + +} diff --git a/dss-git/dss-git-server/src/main/java/com/webank/wedatasphere/dss/git/dao/DSSWorkspaceGitMapper.java b/dss-git/dss-git-server/src/main/java/com/webank/wedatasphere/dss/git/dao/DSSWorkspaceGitMapper.java new file mode 100644 index 0000000000..0b767645ee --- /dev/null +++ b/dss-git/dss-git-server/src/main/java/com/webank/wedatasphere/dss/git/dao/DSSWorkspaceGitMapper.java @@ -0,0 +1,36 @@ +package com.webank.wedatasphere.dss.git.dao; + +import com.webank.wedatasphere.dss.git.common.protocol.GitUserEntity; + +import com.webank.wedatasphere.dss.git.dto.GitProjectGitInfo; +import org.apache.ibatis.annotations.Mapper; +import org.apache.ibatis.annotations.Param; + +import java.util.List; + +@Mapper +public interface DSSWorkspaceGitMapper { + void insert(GitUserEntity gitUserDO); + + void update(GitUserEntity gitUserDO); + + GitUserEntity selectByWorkspaceId(@Param("workspaceId") Long workspaceId, @Param("type") String type); + + GitUserEntity selectByUser(@Param("gitUser") String gitUser); + + List selectGitUser(@Param("workspaceId") Long workspaceId, @Param("type") String type, @Param("gitUser") String gitUser); + + List getAllWorkspaceId(); + + List getProjectInfoByWorkspaceId(@Param ("workspaceId") Long workspaceId); + + Long getWorkspaceIdByUserName(@Param("gitUser") String gitUser); + + GitProjectGitInfo getProjectInfoByProjectName(@Param ("projectName") String projectName); + + void insertProjectInfo(GitProjectGitInfo projectGitInfo); + + void updateProjectToken(@Param("projectName") String projectName, @Param("gitToken") String gitToken); + + void updateProjectId(@Param("projectName") String projectName, @Param("gitProjectId") String gitProjectId); +} diff --git a/dss-git/dss-git-server/src/main/java/com/webank/wedatasphere/dss/git/dao/impl/DSSWorkspaceGitMapper.xml b/dss-git/dss-git-server/src/main/java/com/webank/wedatasphere/dss/git/dao/impl/DSSWorkspaceGitMapper.xml new file mode 100644 index 0000000000..b59dee86de --- /dev/null +++ b/dss-git/dss-git-server/src/main/java/com/webank/wedatasphere/dss/git/dao/impl/DSSWorkspaceGitMapper.xml @@ -0,0 +1,116 @@ + + + + + + + + `id`, `workspace_id`, `git_project_id`, `project_name`, `git_user`, `git_token`, `git_url`, `create_time`, `update_time` + + + + insert into dss_workspace_associate_git (workspace_id, git_user, git_password, git_token, git_url, create_time, update_time, create_by, update_by, type, git_user_id) + VALUES (#{workspaceId}, #{gitUser}, #{gitPassword}, #{gitToken}, #{gitUrl}, now(), now(), #{createBy}, #{updateBy}, #{type}, #{gitUserId}) + + + + insert into dss_project_associate_git (workspace_id, git_project_id, project_name, git_user, git_token, git_url, create_time, update_time) + values (#{workspaceId}, #{gitProjectId}, #{projectName}, #{gitUser}, #{gitToken}, #{gitUrl}, now(), now()) + + + + update dss_project_associate_git + + git_token = #{gitToken}, update_time = now() + + where project_name = #{projectName} + + + + update dss_project_associate_git + + git_project_id = #{gitProjectId}, update_time = now() + + where project_name = #{projectName} + + + + + + + + + + + update dss_workspace_associate_git + + + git_user = #{gitUser}, + + + git_password = #{gitPassword}, + + + git_token = #{gitToken}, + + + git_user_id = #{gitUserId}, + + update_by = #{updateBy}, + + where workspace_id = #{workspaceId} and type = #{type} + + + + + + + + + + \ No newline at end of file diff --git a/dss-git/dss-git-server/src/main/java/com/webank/wedatasphere/dss/git/dto/GitProjectGitInfo.java b/dss-git/dss-git-server/src/main/java/com/webank/wedatasphere/dss/git/dto/GitProjectGitInfo.java new file mode 100644 index 0000000000..798223ecf0 --- /dev/null +++ b/dss-git/dss-git-server/src/main/java/com/webank/wedatasphere/dss/git/dto/GitProjectGitInfo.java @@ -0,0 +1,128 @@ +package com.webank.wedatasphere.dss.git.dto; + +import java.util.Date; + +public class GitProjectGitInfo { + private Long id; + private Long workspaceId; + private String gitProjectId; + private String projectName; + private String gitUser; + private String gitToken; + private String gitUrl; + private Date createTime; + private Date updateTime; + private String gitTokenEncrypt; + + public GitProjectGitInfo() { + } + + public GitProjectGitInfo(Long id, Long workspaceId, String gitProjectId, String projectName, String gitUser, String gitToken, String gitUrl, Date createTime, Date updateTime) { + this.id = id; + this.workspaceId = workspaceId; + this.gitProjectId = gitProjectId; + this.projectName = projectName; + this.gitUser = gitUser; + this.gitToken = gitToken; + this.gitUrl = gitUrl; + this.createTime = createTime; + this.updateTime = updateTime; + } + + public GitProjectGitInfo(Long workspaceId, String gitProjectId, String projectName, String gitUser, String gitToken, String gitUrl) { + this.workspaceId = workspaceId; + this.gitProjectId = gitProjectId; + this.projectName = projectName; + this.gitUser = gitUser; + this.gitToken = gitToken; + this.gitUrl = gitUrl; + } + + public GitProjectGitInfo(Long workspaceId, String projectName, String gitUser, String gitToken, String gitUrl) { + this.workspaceId = workspaceId; + this.projectName = projectName; + this.gitUser = gitUser; + this.gitToken = gitToken; + this.gitUrl = gitUrl; + } + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public Long getWorkspaceId() { + return workspaceId; + } + + public void setWorkspaceId(Long workspaceId) { + this.workspaceId = workspaceId; + } + + public String getGitProjectId() { + return gitProjectId; + } + + public void setGitProjectId(String gitProjectId) { + this.gitProjectId = gitProjectId; + } + + public String getProjectName() { + return projectName; + } + + public void setProjectName(String projectName) { + this.projectName = projectName; + } + + public String getGitUser() { + return gitUser; + } + + public void setGitUser(String gitUser) { + this.gitUser = gitUser; + } + + public String getGitToken() { + return gitToken; + } + + public void setGitToken(String gitToken) { + this.gitToken = gitToken; + } + + public Date getCreateTime() { + return createTime; + } + + public void setCreateTime(Date createTime) { + this.createTime = createTime; + } + + public Date getUpdateTime() { + return updateTime; + } + + public void setUpdateTime(Date updateTime) { + this.updateTime = updateTime; + } + + public String getGitUrl() { + return gitUrl; + } + + public void setGitUrl(String gitUrl) { + this.gitUrl = gitUrl; + } + + public String getGitTokenEncrypt() { + return gitTokenEncrypt; + } + + public void setGitTokenEncrypt(String gitTokenEncrypt) { + this.gitTokenEncrypt = gitTokenEncrypt; + } +} diff --git a/dss-git/dss-git-server/src/main/java/com/webank/wedatasphere/dss/git/manage/GitProjectManager.java b/dss-git/dss-git-server/src/main/java/com/webank/wedatasphere/dss/git/manage/GitProjectManager.java new file mode 100644 index 0000000000..e88ba34b8d --- /dev/null +++ b/dss-git/dss-git-server/src/main/java/com/webank/wedatasphere/dss/git/manage/GitProjectManager.java @@ -0,0 +1,226 @@ +package com.webank.wedatasphere.dss.git.manage; + + +import com.webank.wedatasphere.dss.common.exception.DSSErrorException; +import com.webank.wedatasphere.dss.git.common.protocol.GitUserEntity; +import com.webank.wedatasphere.dss.git.common.protocol.config.GitServerConfig; +import com.webank.wedatasphere.dss.git.common.protocol.request.*; +import com.webank.wedatasphere.dss.git.common.protocol.response.*; +import com.webank.wedatasphere.dss.git.common.protocol.util.UrlUtils; +import com.webank.wedatasphere.dss.git.dao.DSSWorkspaceGitMapper; +import com.webank.wedatasphere.dss.git.dto.GitProjectGitInfo; +import com.webank.wedatasphere.dss.git.utils.DSSGitUtils; +import com.webank.wedatasphere.dss.git.utils.FileUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.http.HttpResponse; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.utils.URIBuilder; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClients; +import org.apache.http.util.EntityUtils; +import org.apache.linkis.DataWorkCloudApplication; +import org.apache.linkis.common.utils.Utils; +import org.json.JSONObject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.crypto.Cipher; +import javax.crypto.spec.SecretKeySpec; +import java.io.IOException; +import java.security.Key; +import java.util.*; +import java.util.concurrent.TimeUnit; + +public class GitProjectManager { + + private static final Logger LOGGER = LoggerFactory.getLogger(GitProjectManager.class); + + private volatile static boolean isInit; + + private static DSSWorkspaceGitMapper workspaceGitMapper; + + + protected GitProjectManager() { + } + + static { + LOGGER.info("已归档项目移除定时线程开启..."); + init(); + Utils.defaultScheduler().scheduleAtFixedRate(() -> { + try { + List allWorkspaceId = workspaceGitMapper.getAllWorkspaceId(); + for (Long workspaceId : allWorkspaceId) { + List projectInfoByWorkspaceId = getProjectInfo(workspaceId); + Map> map = getMap(projectInfoByWorkspaceId); + for (Map.Entry> entry : map.entrySet()) { + String gitUser = entry.getKey(); + Map value = entry.getValue(); + for (Map.Entry entry1 : value.entrySet()) { + String gitUrl = entry1.getKey(); + String gitToken = generateKeys(entry1.getValue(), Cipher.DECRYPT_MODE); + List allGitProjectName = DSSGitUtils.getAllProjectName(gitToken, gitUrl); + List localProjectName = FileUtils.getLocalProjectName(workspaceId, gitUser); + LOGGER.info("localProjectName is : {}", localProjectName); + localProjectName.removeAll(allGitProjectName); + if (!localProjectName.isEmpty()) { + LOGGER.info("Project to delete : {}", localProjectName.toString()); + for (String projectName : localProjectName) { + // 删除本地项目 + DSSGitUtils.archiveLocal(projectName, gitUser, workspaceId); + } + } else { + LOGGER.info("Nothing need to delete"); + } + } + } + } + } catch (Exception e) { + LOGGER.error("定时清理归档项目错误", e); + } + }, 0,1, TimeUnit.DAYS); + } + + private static Map> getMap(List projectInfoByWorkspaceId) { + Map> map = new HashMap<>(); + for (GitProjectGitInfo projectGitInfo : projectInfoByWorkspaceId) { + String gitUserName = projectGitInfo.getGitUser(); + String gitToken = projectGitInfo.getGitToken(); + String gitUrl = projectGitInfo.getGitUrl(); + Map tempMap = new HashMap<>(); + if (map.containsKey(gitUserName)) { + tempMap = map.get(gitUserName); + } + tempMap.put(gitUrl, gitToken); + map.put(gitUserName, tempMap); + } + return map; + } + + public static void init() { + if (!isInit) { + synchronized (GitProjectManager.class) { + if (!isInit) { + workspaceGitMapper = DataWorkCloudApplication.getApplicationContext().getBean(DSSWorkspaceGitMapper.class); + isInit = true; + } + } + } + } + + + public static GitUserUpdateResponse associateGit(GitUserUpdateRequest gitUserCreateRequest) throws DSSErrorException, IOException { + return new GitUserUpdateResponse(0, "", 0L); + } + + public static Long getWorkspaceIdByUser(String gitUser) { + return workspaceGitMapper.getWorkspaceIdByUserName(gitUser); + } + + + public static void updateProjectInfo(GitProjectGitInfo projectGitInfo, Boolean isExist) throws DSSErrorException { + String projectName = projectGitInfo.getProjectName(); + // 加密处理密码 + String gitToken = projectGitInfo.getGitToken(); + String encryptToken = generateKeys(gitToken, Cipher.ENCRYPT_MODE); + projectGitInfo.setGitToken(encryptToken); + + if (isExist) { + workspaceGitMapper.updateProjectToken(projectName, encryptToken); + } else { + workspaceGitMapper.insertProjectInfo(projectGitInfo); + } + } + + public static GitProjectGitInfo getProjectInfoByProjectName(String projectName) throws DSSErrorException { + GitProjectGitInfo projectInfoByProjectName = workspaceGitMapper.getProjectInfoByProjectName(projectName); + if (projectInfoByProjectName != null) { + String gitToken = projectInfoByProjectName.getGitToken(); + String token = generateKeys(gitToken, Cipher.DECRYPT_MODE); + projectInfoByProjectName.setGitToken(token); + projectInfoByProjectName.setGitTokenEncrypt(gitToken); + } + return projectInfoByProjectName; + } + + public static List getProjectInfo(Long workspaceId) { + return workspaceGitMapper.getProjectInfoByWorkspaceId(workspaceId); + } + + public static String generateKeys(String password, int mode) throws DSSErrorException{ + // 定义一个字符串作为密钥源 + String keyString = GitServerConfig.LINKIS_MYSQL_PRI_KEY.getValue(); + if (keyString.length() < 16) { + throw new DSSErrorException(800001, "密钥长度必须大于16位"); + } + if (StringUtils.isEmpty(password)) { + throw new DSSErrorException(800001, "密码或token为空"); + } + try { + // 确保密钥长度合适，AES 密钥长度为 128 位 (16 字节) + byte[] keyBytes = keyString.substring(0, 16).getBytes(); + Key key = new SecretKeySpec(keyBytes, "AES"); + // 加密Cipher对象 + Cipher encryptCipher = Cipher.getInstance("AES"); + encryptCipher.init(mode, key); + if (mode == Cipher.ENCRYPT_MODE) { + byte[] encrypted = encryptCipher.doFinal(password.getBytes()); + return Base64.getEncoder().encodeToString(encrypted); + } else { + byte[] decode = Base64.getDecoder().decode(password); + byte[] encrypted = encryptCipher.doFinal(decode); + return new String(encrypted); + } + } catch (Exception e) { + throw new DSSErrorException(800001, "加密失败,原因为" + e); + } + } + + public static Boolean gitTokenTest(String token, String expectedUsername)throws DSSErrorException { + // GitLab API 用户信息接口 + String apiUrl = UrlUtils.normalizeIp(GitServerConfig.GIT_URL_PRE.getValue()) + "/api/v4/user"; + + try (CloseableHttpClient client = HttpClients.createDefault()) { + URIBuilder builder = new URIBuilder(apiUrl); + HttpGet request = new HttpGet(builder.build()); + request.setHeader("PRIVATE-TOKEN", token); + + HttpResponse response = client.execute(request); + String jsonResponse = EntityUtils.toString(response.getEntity()); + + + if (response.getStatusLine().getStatusCode() == 200) { + JSONObject userData = new JSONObject(jsonResponse); + + String actualUsername = userData.getString("username"); + if (actualUsername.equals(expectedUsername)) { + LOGGER.info("Token is valid and matches the username: " + actualUsername); + return true; + }else { + LOGGER.info("当前token与用户名" + actualUsername + "匹配，与当前用户名" + expectedUsername + "不匹配"); + throw new DSSErrorException(800001, "当前用户名 token 不匹配，请检查"); + } + } else if (response.getStatusLine().getStatusCode() == 401){ + throw new DSSErrorException(800001, "请检查token是否正确"); + } + } catch (DSSErrorException e) { + LOGGER.info("Error verifying token: " + e.getMessage()); + throw new DSSErrorException(800001, "校验失败" + e.getMessage()); + }catch (Exception e) { + throw new DSSErrorException(800001, "校验token失败，请确认当前环境git是否可以正常访问" + e.getMessage()); + } + return false; + } + + public static GitUserInfoListResponse getGitUserByType(GitUserInfoByRequest infoByRequest) { + List gitUserEntities = workspaceGitMapper.selectGitUser(infoByRequest.getWorkspaceId(), infoByRequest.getType(), infoByRequest.getGitUserName()); + GitUserInfoListResponse gitUserInfoListResponse = new GitUserInfoListResponse(); + gitUserInfoListResponse.setGitUserEntities(gitUserEntities); + return gitUserInfoListResponse; + } + + public static void updateGitProjectId(String projectName, String gitProjectId) { + workspaceGitMapper.updateProjectId(projectName, gitProjectId); + } + + +} diff --git a/dss-git/dss-git-server/src/main/java/com/webank/wedatasphere/dss/git/service/DSSGitProjectManagerService.java b/dss-git/dss-git-server/src/main/java/com/webank/wedatasphere/dss/git/service/DSSGitProjectManagerService.java new file mode 100644 index 0000000000..e58f10734b --- /dev/null +++ b/dss-git/dss-git-server/src/main/java/com/webank/wedatasphere/dss/git/service/DSSGitProjectManagerService.java @@ -0,0 +1,22 @@ +package com.webank.wedatasphere.dss.git.service; + +import com.webank.wedatasphere.dss.common.exception.DSSErrorException; +import com.webank.wedatasphere.dss.git.common.protocol.exception.GitErrorException; +import com.webank.wedatasphere.dss.git.common.protocol.request.*; +import com.webank.wedatasphere.dss.git.common.protocol.response.*; + +import java.io.IOException; +import java.util.concurrent.ExecutionException; + +public interface DSSGitProjectManagerService { + GitCreateProjectResponse create(GitCreateProjectRequest request) throws DSSErrorException; + + GitArchivePorjectResponse archive(GitArchiveProjectRequest request) throws GitErrorException, DSSErrorException; + + GitCheckProjectResponse checkProject(GitCheckProjectRequest request) throws DSSErrorException; + + GitUserByWorkspaceResponse getProjectGitUserInfo(GitUserByWorkspaceIdRequest request); + + GitAddMemberResponse addMember(GitAddMemberRequest request) throws Exception; + +} diff --git a/dss-git/dss-git-server/src/main/java/com/webank/wedatasphere/dss/git/service/DSSGitWorkflowManagerService.java b/dss-git/dss-git-server/src/main/java/com/webank/wedatasphere/dss/git/service/DSSGitWorkflowManagerService.java new file mode 100644 index 0000000000..e5555dfc73 --- /dev/null +++ b/dss-git/dss-git-server/src/main/java/com/webank/wedatasphere/dss/git/service/DSSGitWorkflowManagerService.java @@ -0,0 +1,44 @@ +package com.webank.wedatasphere.dss.git.service; + + +import com.webank.wedatasphere.dss.common.exception.DSSErrorException; +import com.webank.wedatasphere.dss.git.common.protocol.request.*; +import com.webank.wedatasphere.dss.git.common.protocol.response.*; +import org.eclipse.jgit.lib.Repository; + +import java.io.File; +import java.io.IOException; +import java.util.concurrent.ExecutionException; + +public interface DSSGitWorkflowManagerService { + GitDiffResponse diff(GitDiffRequest request) throws DSSErrorException; + + GitCommitResponse commit(GitCommitRequest request) throws DSSErrorException; + + GitSearchResponse search(GitSearchRequest request) throws DSSErrorException; + + GitDeleteResponse delete(GitDeleteRequest request) throws DSSErrorException; + + GitFileContentResponse getFileContent(GitFileContentRequest request) throws DSSErrorException; + + GitHistoryResponse getHistory(GitHistoryRequest request) throws DSSErrorException; + + GitCommitResponse getCurrentCommit(GitCurrentCommitRequest request) throws DSSErrorException; + + GitCommitResponse gitCheckOut(GitRevertRequest request) throws DSSErrorException; + + GitCommitResponse removeFile(GitRemoveRequest request) throws DSSErrorException; + + GitCommitResponse rename(GitRenameRequest request) throws DSSErrorException; + + GitHistoryResponse getHistory(GitCommitInfoBetweenRequest request) throws DSSErrorException; + + GitDiffResponse diffGit(GitDiffTargetCommitRequest request) throws DSSErrorException; + + GitCommitResponse batchCommit(GitBatchCommitRequest request) throws DSSErrorException; + + Repository getRepository(File repoDir, String projectName, Long workspaceId, String gitUser, String gitToken, String gitUrl) throws DSSErrorException; + + GitDiffFileContentResponse getDiffFileContent(GitDiffFileContentRequest request) throws DSSErrorException; + +} diff --git a/dss-git/dss-git-server/src/main/java/com/webank/wedatasphere/dss/git/service/impl/DSSGitProjectManagerServiceImpl.java b/dss-git/dss-git-server/src/main/java/com/webank/wedatasphere/dss/git/service/impl/DSSGitProjectManagerServiceImpl.java new file mode 100644 index 0000000000..48eba73ed0 --- /dev/null +++ b/dss-git/dss-git-server/src/main/java/com/webank/wedatasphere/dss/git/service/impl/DSSGitProjectManagerServiceImpl.java @@ -0,0 +1,246 @@ +package com.webank.wedatasphere.dss.git.service.impl; + +import com.webank.wedatasphere.dss.common.exception.DSSErrorException; +import com.webank.wedatasphere.dss.common.service.BMLService; +import com.webank.wedatasphere.dss.git.common.protocol.GitUserEntity; +import com.webank.wedatasphere.dss.git.common.protocol.config.GitServerConfig; +import com.webank.wedatasphere.dss.git.common.protocol.exception.GitErrorException; +import com.webank.wedatasphere.dss.git.common.protocol.request.*; +import com.webank.wedatasphere.dss.git.common.protocol.response.*; +import com.webank.wedatasphere.dss.git.common.protocol.util.UrlUtils; +import com.webank.wedatasphere.dss.git.constant.DSSGitConstant; +import com.webank.wedatasphere.dss.git.dto.GitProjectGitInfo; +import com.webank.wedatasphere.dss.git.manage.GitProjectManager; +import com.webank.wedatasphere.dss.git.service.DSSGitProjectManagerService; +import com.webank.wedatasphere.dss.git.service.DSSGitWorkflowManagerService; +import com.webank.wedatasphere.dss.git.utils.DSSGitUtils; +import com.webank.wedatasphere.dss.git.utils.FileUtils; +import org.apache.commons.lang.StringUtils; +import org.eclipse.jgit.api.errors.JGitInternalException; +import org.eclipse.jgit.lib.Repository; +import org.eclipse.jgit.storage.file.FileRepositoryBuilder; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.stereotype.Service; + +import javax.crypto.Cipher; +import java.io.File; +import java.util.*; + +@Service +public class DSSGitProjectManagerServiceImpl implements DSSGitProjectManagerService { + private final Logger logger = LoggerFactory.getLogger(this.getClass()); + + @Autowired + @Qualifier("projectBmlService") + private BMLService bmlService; + + @Autowired + private DSSGitWorkflowManagerService workflowManagerService; + + @Override + public GitCreateProjectResponse create(GitCreateProjectRequest request) throws DSSErrorException{ + Long workspaceId = request.getWorkspaceId(); + String projectName = request.getProjectName(); + String requestGitToken = request.getGitToken(); + + GitProjectGitInfo projectInfoByProjectName = GitProjectManager.getProjectInfoByProjectName(projectName); + if (projectInfoByProjectName == null) { + logger.error("the projectName : {} don't associate with git", projectName); + return null; + } + String gitUser = projectInfoByProjectName.getGitUser(); + String gitToken = projectInfoByProjectName.getGitToken(); + String gitUrl = projectInfoByProjectName.getGitUrl(); + + if (request.getGitUser() != null && !gitUser.equals(request.getGitUser())) { + throw new DSSErrorException(80001, "Git用户名不允许更换"); + } + + if (requestGitToken != null && !gitToken.equals(requestGitToken)) { + Boolean tokenTest = GitProjectManager.gitTokenTest(requestGitToken, gitUser); + if (tokenTest) { + GitProjectGitInfo projectGitInfo = new GitProjectGitInfo(); + projectGitInfo.setProjectName(projectName); + projectGitInfo.setGitToken(requestGitToken); + // 仅更新token + GitProjectManager.updateProjectInfo(projectGitInfo, true); + } + } + + Repository repository = null; + try { + // Http请求Git，创建project + DSSGitUtils.init(projectName, gitUser, gitToken, gitUrl); + // 解压BML文件到本地 + FileUtils.downloadBMLResource(bmlService, projectName, request.getBmlResource(), request.getUsername(), workspaceId, gitUser); + FileUtils.removeProject(projectName, workspaceId, gitUser); + FileUtils.unzipBMLResource(projectName, workspaceId, gitUser); + // 本地创建Git项目 + DSSGitUtils.create(projectName, workspaceId, gitUser); + // 获取git项目 + String gitPath = DSSGitUtils.generateGitPath(projectName, workspaceId, gitUser); + File repoDir = new File(gitPath); + repository = new FileRepositoryBuilder().setGitDir(repoDir).build(); + // 关联远端Git + DSSGitUtils.remote(repository, projectName, gitUser, gitUrl); + // 提交 + String comment = "init project: " + projectName + DSSGitConstant.GIT_USERNAME_FLAG + request.getUsername(); + // 首次创建提交项目整体 + List paths = Collections.singletonList("."); + DSSGitUtils.push(repository, projectName, gitUser, gitToken, comment, paths); + + // 获取项目ProjectId + String projectIdByName = DSSGitUtils.getProjectIdByName(projectName, gitUser, gitToken, gitUrl); +// DSSGitUtils.addProjectMember(gitUser, readGitUser.getGitUserId(), projectIdByName, 20); + // 存储projectId + GitProjectManager.updateGitProjectId(projectName, projectIdByName); + return new GitCreateProjectResponse(); + } catch (Exception e) { + logger.error("create project failed, the reason is: ", e); + throw new DSSErrorException(80001, "create project failed, the reason is: " + e); + } finally { + if (repository != null) { + repository.close(); + } + } + } + + @Override + public GitArchivePorjectResponse archive(GitArchiveProjectRequest request) throws GitErrorException, DSSErrorException { + String projectName = request.getProjectName(); + + GitProjectGitInfo projectInfoByProjectName = GitProjectManager.getProjectInfoByProjectName(projectName); + if (projectInfoByProjectName == null) { + logger.error("the projectName : {} don't associate with git", projectName); + return null; + } + String gitUser = projectInfoByProjectName.getGitUser(); + String gitToken = projectInfoByProjectName.getGitToken(); + String gitUrl = projectInfoByProjectName.getGitUrl(); + // 远程归档 + DSSGitUtils.archive(projectName, gitUser, gitToken, gitUrl); + // 删除本地项目 + DSSGitUtils.archiveLocal(projectName, gitUser, request.getWorkspaceId()); + return new GitArchivePorjectResponse(); + } + + @Override + public GitCheckProjectResponse checkProject(GitCheckProjectRequest request) throws DSSErrorException { + String gitUser = request.getGitUser(); + String gitToken = request.getGitToken(); + String projectName = request.getProjectName(); + Long workspaceId = request.getWorkspaceId(); + Long workspaceIdByUser = GitProjectManager.getWorkspaceIdByUser(gitUser); + if (workspaceIdByUser != null && !workspaceIdByUser.equals(workspaceId)) { + throw new DSSErrorException(80001, "该Git用户已在" + workspaceIdByUser + "配置，请更换Git用户重试"); + } + // 数据库是否存在标志 + Boolean isExist = false; + GitProjectGitInfo projectGitInfo = GitProjectManager.getProjectInfoByProjectName(projectName); + if (projectGitInfo != null ) { + if (!projectGitInfo.getGitUser().equals(gitUser)) { + throw new DSSErrorException(80001, "Git用户名不允许更换"); + } + isExist = true; + } + // 检测token合法性 数据库中已存在的配置无需再次校验 + if (isExist && projectGitInfo.getGitTokenEncrypt().equals(gitToken)) { + // 加密token需解密再验证 + gitToken = GitProjectManager.generateKeys(gitToken, Cipher.DECRYPT_MODE); + } + Boolean tokenTest = GitProjectManager.gitTokenTest(gitToken, gitUser); + String gitUrl = UrlUtils.normalizeIp(GitServerConfig.GIT_URL_PRE.getValue()); + if (tokenTest) { + String projectPath = gitUser + "/" + projectName; + // 检测项目名称是否重复 数据库中已存在的配置无需再次校验 + if (isExist) { + // 存在说明项目已创建，本次为更新 + GitProjectGitInfo gitProjectGitInfo = new GitProjectGitInfo(workspaceId, projectName, gitUser, gitToken, gitUrl); + GitProjectManager.updateProjectInfo(gitProjectGitInfo, true); + } else { + // 否则需要检测Git是否已有同名项目，若没有才可以新增 + boolean projectExists = DSSGitUtils.checkIfProjectExists(gitToken, projectPath); + if (!projectExists) { + GitProjectGitInfo gitProjectGitInfo = new GitProjectGitInfo(workspaceId, projectName, gitUser, gitToken, gitUrl); + GitProjectManager.updateProjectInfo(gitProjectGitInfo, false); + } else { + throw new GitErrorException(80101, "git账号: "+ gitUser+ "下已存在同名项目"+ projectName +"，请更换git账号或项目名称"); + } + } + } else { + throw new GitErrorException(80101, "git init failed, the reason is: projectName " + projectName +" already exists"); + } + + return new GitCheckProjectResponse(projectName, isExist); + } + + @Override + public GitUserByWorkspaceResponse getProjectGitUserInfo(GitUserByWorkspaceIdRequest request) { + String username = request.getUsername(); + Long workspaceId = request.getWorkspaceId(); + + List projectInfoByWorkspaceId = GitProjectManager.getProjectInfo(workspaceId); + + Map map = new HashMap<>(); + for (GitProjectGitInfo gitInfo : projectInfoByWorkspaceId) { + GitUserEntity gitUserEntity = new GitUserEntity(gitInfo.getGitUser(), gitInfo.getGitToken()); + map.put(gitInfo.getProjectName(), gitUserEntity); + } + + return new GitUserByWorkspaceResponse(map); + } + + @Override + public GitAddMemberResponse addMember(GitAddMemberRequest request) throws Exception { + String projectName = request.getProjectName(); + String username = request.getUsername(); + String flowNodeName = request.getFlowNodeName(); + GitProjectGitInfo projectInfoByProjectName = GitProjectManager.getProjectInfoByProjectName(projectName); + if (projectInfoByProjectName == null) { + logger.error("the projectName : {} don't associate with git", projectName); + return null; + } + String gitProjectId = projectInfoByProjectName.getGitProjectId(); + String gitUser = projectInfoByProjectName.getGitUser(); + String gitToken = projectInfoByProjectName.getGitToken(); + String gitUrl = projectInfoByProjectName.getGitUrl(); + Long workspaceId = request.getWorkspaceId(); + + GitHistoryResponse response = new GitHistoryResponse(); + // 拼接.git路径 + String gitPath = DSSGitUtils.generateGitPath(projectName, request.getWorkspaceId(), gitUser); + // 获取git仓库 + File repoDir = new File(gitPath); + if (StringUtils.isNotEmpty(flowNodeName)) { + try (Repository repository = workflowManagerService.getRepository(repoDir, projectName, workspaceId, gitUser, gitToken, gitUrl)) { + // 本地保持最新状态 + DSSGitUtils.pull(repository, projectName, gitUser, gitToken); + String filePath = DSSGitUtils.generateGitPrePath(projectName, workspaceId, gitUser) + "/" + flowNodeName; + File file = new File(filePath); + if (!file.exists()) { + logger.error("{}不存在", filePath); + throw new DSSErrorException(80001, "当前工作流或工作流节点未提交到git，请提交后再跳转"); + } + } catch (JGitInternalException e) { + logger.error("get git failed, the reason is", e); + throw new DSSErrorException(80001, "当前项目下已有工作流在进行git操作，请稍后重试"); + } catch (Exception e) { + logger.error("git init failed, the reason is ", e); + throw new DSSErrorException(80001, "diff failed, the reason is" + e.getMessage()); + } + } + String userIdByUsername = DSSGitUtils.getUserIdByUsername(gitUrl, gitToken, username); + DSSGitUtils.addProjectMember(gitUrl, gitToken, userIdByUsername, gitProjectId, 20); + String jumpUrl = gitUrl + "/" + gitUser + "/" + projectName; + + if (StringUtils.isNotEmpty(flowNodeName)) { + jumpUrl += "/tree/master/" + flowNodeName; + } + return new GitAddMemberResponse(jumpUrl); + } + + +} diff --git a/dss-git/dss-git-server/src/main/java/com/webank/wedatasphere/dss/git/service/impl/DSSGitWorkflowManagerServiceImpl.java b/dss-git/dss-git-server/src/main/java/com/webank/wedatasphere/dss/git/service/impl/DSSGitWorkflowManagerServiceImpl.java new file mode 100644 index 0000000000..c241a916a6 --- /dev/null +++ b/dss-git/dss-git-server/src/main/java/com/webank/wedatasphere/dss/git/service/impl/DSSGitWorkflowManagerServiceImpl.java @@ -0,0 +1,998 @@ +package com.webank.wedatasphere.dss.git.service.impl; + +import com.webank.wedatasphere.dss.common.entity.BmlResource; +import com.webank.wedatasphere.dss.common.exception.DSSErrorException; +import com.webank.wedatasphere.dss.common.service.BMLService; +import com.webank.wedatasphere.dss.git.common.protocol.GitSearchLine; +import com.webank.wedatasphere.dss.git.common.protocol.GitSearchResult; +import com.webank.wedatasphere.dss.git.common.protocol.GitTree; +import com.webank.wedatasphere.dss.git.common.protocol.GitUserEntity; +import com.webank.wedatasphere.dss.git.common.protocol.constant.GitConstant; +import com.webank.wedatasphere.dss.git.common.protocol.exception.GitErrorException; +import com.webank.wedatasphere.dss.git.common.protocol.request.*; +import com.webank.wedatasphere.dss.git.common.protocol.response.*; +import com.webank.wedatasphere.dss.git.common.protocol.config.GitServerConfig; +import com.webank.wedatasphere.dss.git.constant.DSSGitConstant; +import com.webank.wedatasphere.dss.git.dto.GitProjectGitInfo; +import com.webank.wedatasphere.dss.git.manage.GitProjectManager; +import com.webank.wedatasphere.dss.git.service.DSSGitWorkflowManagerService; +import com.webank.wedatasphere.dss.git.utils.DSSGitUtils; +import com.webank.wedatasphere.dss.git.utils.FileUtils; +import org.apache.commons.lang.StringUtils; +import org.eclipse.jgit.api.errors.JGitInternalException; +import org.eclipse.jgit.errors.LockFailedException; +import org.eclipse.jgit.lib.Repository; +import org.eclipse.jgit.revwalk.RevCommit; +import org.eclipse.jgit.storage.file.FileRepositoryBuilder; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.stereotype.Service; +import org.springframework.util.CollectionUtils; + +import javax.crypto.Cipher; +import java.io.BufferedReader; +import java.io.File; +import java.io.IOException; +import java.io.InputStreamReader; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.*; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +@Service +public class DSSGitWorkflowManagerServiceImpl implements DSSGitWorkflowManagerService { + private final Logger logger = LoggerFactory.getLogger(this.getClass()); + + @Autowired + @Qualifier("workflowBmlService") + private BMLService bmlService; + @Override + public GitDiffResponse diff(GitDiffRequest request) throws DSSErrorException { + Long workspaceId = request.getWorkspaceId(); + String projectName = request.getProjectName(); + + GitProjectGitInfo projectInfoByProjectName = GitProjectManager.getProjectInfoByProjectName(projectName); + if (projectInfoByProjectName == null) { + logger.error("the projectName : {} don't associate with git", projectName); + return null; + } + String gitUser = projectInfoByProjectName.getGitUser(); + String gitToken = projectInfoByProjectName.getGitToken(); + String gitUrl = projectInfoByProjectName.getGitUrl(); + GitDiffResponse diff = null; + // 拼接.git路径 + String gitPath = DSSGitUtils.generateGitPath(projectName, workspaceId, gitUser); + // 获取git仓库 + File repoDir = new File(gitPath); + try (Repository repository = getRepository(repoDir, projectName, workspaceId, gitUser, gitToken, gitUrl)){ + // 解压BML文件到本地 + Map bmlResourceMap = request.getBmlResourceMap(); + List fileList = new ArrayList<>(bmlResourceMap.keySet()); + // 本地保持最新状态 + DSSGitUtils.pull(repository, projectName, gitUser, gitToken); + for (Map.Entry entry : bmlResourceMap.entrySet()) { + fileList.add(entry.getKey()); + // 解压BML文件到本地 + FileUtils.downloadBMLResource(bmlService, entry.getKey(), entry.getValue(), request.getUsername(), workspaceId, gitUser); + FileUtils.removeFlowNode(entry.getKey(), projectName, workspaceId, gitUser); + String metaConfPath = GitConstant.GIT_SERVER_META_PATH + File.separator + entry.getKey(); + fileList.add(metaConfPath); + FileUtils.removeFlowNode(metaConfPath, projectName, workspaceId, gitUser); + FileUtils.unzipBMLResource(entry.getKey(), workspaceId, gitUser); + + } + diff = DSSGitUtils.diff(repository, projectName, fileList, gitUser, workspaceId); + // 重置本地 + DSSGitUtils.reset(repository, projectName); + }catch (JGitInternalException e) { + logger.error("get git failed, the reason is", e); + throw new DSSErrorException(80001, "当前项目下已有工作流在进行git操作，请稍后重试"); + } catch (Exception e) { + logger.error("diff failed, the reason is ",e); + throw new DSSErrorException(80001, "diff failed, the reason is" + e.getMessage()); + } + return diff; + + } + + @Override + public GitDiffResponse diffGit(GitDiffTargetCommitRequest request) throws DSSErrorException { + Long workspaceId = request.getWorkspaceId(); + String projectName = request.getProjectName(); + String filePath = request.getFilePath(); + + GitProjectGitInfo projectInfoByProjectName = GitProjectManager.getProjectInfoByProjectName(projectName); + if (projectInfoByProjectName == null) { + logger.error("the projectName : {} don't associate with git", projectName); + return null; + } + String gitUser = projectInfoByProjectName.getGitUser(); + String gitToken = projectInfoByProjectName.getGitToken(); + String gitUrl = projectInfoByProjectName.getGitUrl(); + GitDiffResponse diff = null; + // 拼接.git路径 + String gitPath = DSSGitUtils.generateGitPath(projectName, workspaceId, gitUser); + String gitPrePath = DSSGitUtils.generateGitPrePath(projectName, workspaceId, gitUser); + // 获取git仓库 + File repoDir = new File(gitPath); + try (Repository repository = getRepository(repoDir, projectName, workspaceId, gitUser, gitToken, gitUrl)){ + // 本地保持最新状态 + DSSGitUtils.pull(repository, projectName, gitUser, gitToken); + + if (StringUtils.isEmpty(request.getCommitId())) { + List codeTree = new ArrayList<>(); + List metaTree = new ArrayList<>(); + String path = gitPrePath + File.separator + filePath; + String metaPath = gitPrePath+ File.separator + GitConstant.GIT_SERVER_META_PATH + File.separator + filePath; + GitTree fileTree = getFileTree(path, false); + GitTree metaFileTree = getFileTree(metaPath, true); + + for (Map.Entry entry : fileTree.getChildren().entrySet()) { + codeTree.add(entry.getValue()); + DSSGitUtils.printTree("", entry.getValue()); + } + + for (Map.Entry entry : metaFileTree.getChildren().entrySet()) { + metaTree.add(entry.getValue()); + DSSGitUtils.printTree("", entry.getValue()); + } + + diff = new GitDiffResponse(codeTree, metaTree, null); + } else { + diff = DSSGitUtils.diffGit(repository, projectName, request.getCommitId(), filePath); + } + + } catch (JGitInternalException e) { + logger.error("get git failed, the reason is", e); + throw new DSSErrorException(80001, "当前项目下已有工作流在进行git操作，请稍后重试"); + } catch (Exception e) { + logger.error("diffGit failed, the reason is ",e); + throw new DSSErrorException(80001, "diffGit failed, the reason is" + e.getMessage()); + } + return diff; + + } + + private GitTree getFileTree(String path, Boolean meta) throws GitErrorException { + Path currentDir = Paths.get(path); + GitTree root = new GitTree(""); + + File tempfile =new File(path); + if (!tempfile.exists()) { + return root; + } + + String substring = path.substring(path.lastIndexOf("/") + 1); + + String result; + String metaPath = GitConstant.GIT_SERVER_META_PATH + File.separator + substring; + result = meta? metaPath : substring; + + + try (Stream paths = Files.walk(currentDir)) { + paths + .filter(Files::isRegularFile)// 只过滤出文件 + .forEach(file -> { + // 获取相对路径 + Path relativePath = currentDir.relativize(file); + String fullPath = result + File.separator + relativePath.toString(); + if (!meta) { + List typeList = new ArrayList<>(); + typeList.addAll(GitConstant.GIT_SERVER_SEARCH_TYPE); + typeList.add(".properties"); + for (String type : typeList) { + if (fullPath.endsWith(type)) { + root.setAbsolutePath(fullPath); + root.addChild(fullPath); + break; + } + } + } else { + root.setAbsolutePath(fullPath); + root.addChild(fullPath); + } + }); + } catch (IOException e) { + logger.error("get failed, the reason is ", e); + throw new GitErrorException(800001, "get failed, the reason is " + e.getMessage()); + } + + return root; + } + + @Override + public GitCommitResponse commit(GitCommitRequest request) throws DSSErrorException { + Long workspaceId = request.getWorkspaceId(); + String projectName = request.getProjectName(); + String requestGitToken = request.getGitToken(); + + GitProjectGitInfo projectInfoByProjectName = GitProjectManager.getProjectInfoByProjectName(projectName); + if (projectInfoByProjectName == null) { + logger.error("the projectName : {} don't associate with git", projectName); + throw new DSSErrorException(80001, "DSS项目:" + projectName +" 未关联git"); + } + String gitUser = projectInfoByProjectName.getGitUser(); + String gitToken = projectInfoByProjectName.getGitToken(); + String gitTokenEncrypt = projectInfoByProjectName.getGitTokenEncrypt(); + String gitUrl = projectInfoByProjectName.getGitUrl(); + + if (request.getGitUser() != null && !gitUser.equals(request.getGitUser())) { + throw new DSSErrorException(80001, "Git用户名不允许更换"); + } + + if (requestGitToken != null && !gitTokenEncrypt.equals(requestGitToken)) { + Boolean tokenTest = GitProjectManager.gitTokenTest(requestGitToken, gitUser); + if (tokenTest) { + GitProjectGitInfo projectGitInfo = new GitProjectGitInfo(); + projectGitInfo.setProjectName(projectName); + projectGitInfo.setGitToken(requestGitToken); + // 仅更新token + GitProjectManager.updateProjectInfo(projectGitInfo, true); + gitToken = requestGitToken; + } + } + + GitCommitResponse commitResponse = null; + // 拼接.git路径 + String gitPath = DSSGitUtils.generateGitPath(projectName, workspaceId, gitUser); + // 获取git仓库 + File repoDir = new File(gitPath); + + try (Repository repository = getRepository(repoDir, projectName, workspaceId, gitUser, gitToken, gitUrl)){ + // 解压BML文件到本地 + Map bmlResourceMap = request.getBmlResourceMap(); + // 本地保持最新状态 + DSSGitUtils.pull(repository, projectName, gitUser, gitToken); + List paths = new ArrayList<>(); + for (Map.Entry entry : bmlResourceMap.entrySet()) { + paths.add(entry.getKey()); + // 解压BML文件到本地 + FileUtils.downloadBMLResource(bmlService, entry.getKey(), entry.getValue(), request.getUsername(), workspaceId, gitUser); + FileUtils.removeFlowNode(entry.getKey(), projectName, workspaceId, gitUser); + String metaConfPath = GitConstant.GIT_SERVER_META_PATH + File.separator + entry.getKey(); + paths.add(metaConfPath); + FileUtils.removeFlowNode(metaConfPath, projectName, workspaceId, gitUser); + FileUtils.unzipBMLResource(entry.getKey(), workspaceId, gitUser); + } + // 提交 + String comment = request.getComment() + DSSGitConstant.GIT_USERNAME_FLAG + request.getUsername(); + // 提交前再次pull， 降低多节点同时提交不同工作流任务导致冲突频率 + DSSGitUtils.pull(repository, projectName, gitUser, gitToken); + DSSGitUtils.push(repository, projectName, gitUser, gitToken, comment, paths); + + commitResponse = DSSGitUtils.getCurrentCommit(repository); + } catch (JGitInternalException e) { + logger.error("get git failed, the reason is", e); + throw new DSSErrorException(80001, "当前项目下已有工作流在进行git操作，请稍后重试"); + }catch (Exception e) { + logger.error("commit failed, the reason is ",e); + throw new DSSErrorException(8001, "commit workflow failed, the reason is: " + e.getMessage()); + } + return commitResponse; + } + + @Override + public GitCommitResponse batchCommit(GitBatchCommitRequest request) throws DSSErrorException { + Long workspaceId = request.getWorkspaceId(); + String projectName = request.getProjectName(); + + GitProjectGitInfo projectInfoByProjectName = GitProjectManager.getProjectInfoByProjectName(projectName); + if (projectInfoByProjectName == null) { + logger.error("the projectName : {} don't associate with git", projectName); + return null; + } + String gitUser = projectInfoByProjectName.getGitUser(); + String gitToken = projectInfoByProjectName.getGitToken(); + String gitUrl = projectInfoByProjectName.getGitUrl(); + GitCommitResponse commitResponse = null; + // 拼接.git路径 + String gitPath = DSSGitUtils.generateGitPath(projectName, workspaceId, gitUser); + // 获取git仓库 + File repoDir = new File(gitPath); + + try (Repository repository = getRepository(repoDir, projectName, workspaceId, gitUser, gitToken, gitUrl)){ + // 解压BML文件到本地 + BmlResource bmlResource = request.getBmlResource(); + List paths = new ArrayList<>(); + paths.addAll(request.getFilePath()); + String fileName = projectName + bmlResource.getResourceId(); + // 本地保持最新状态 + DSSGitUtils.pull(repository, projectName, gitUser, gitToken); + // 解压BML文件到本地 + FileUtils.downloadBMLResource(bmlService, fileName, bmlResource, request.getUsername(), workspaceId, gitUser); + for (String path : request.getFilePath()) { + FileUtils.removeFlowNode(path, projectName, workspaceId, gitUser); + String metaConfPath = GitConstant.GIT_SERVER_META_PATH + File.separator + path; + paths.add(metaConfPath); + FileUtils.removeFlowNode(metaConfPath, projectName, workspaceId, gitUser); + } + FileUtils.unzipBMLResource(fileName, workspaceId, gitUser); + + + + // 提交 + String comment = request.getComment() + DSSGitConstant.GIT_USERNAME_FLAG + request.getUsername(); + // 提交前再次pull， 降低多节点同时提交不同工作流任务导致冲突频率 + DSSGitUtils.pull(repository, projectName, gitUser, gitToken); + DSSGitUtils.push(repository, projectName, gitUser, gitToken, comment, paths); + + commitResponse = DSSGitUtils.getCurrentCommit(repository); + } catch (JGitInternalException e) { + logger.error("get git failed, the reason is", e); + throw new DSSErrorException(80001, "当前项目下已有工作流在进行git操作，请稍后重试"); + }catch (Exception e) { + logger.error("commit failed, the reason is ",e); + throw new DSSErrorException(8001, "commit workflow failed, the reason is: " + e); + } + return commitResponse; + } + + @Override + public GitSearchResponse search(GitSearchRequest request) throws DSSErrorException { + Long workspaceId = request.getWorkspaceId(); + String projectName = request.getProjectName(); + + GitProjectGitInfo projectInfoByProjectName = GitProjectManager.getProjectInfoByProjectName(projectName); + if (projectInfoByProjectName == null) { + logger.error("the projectName : {} don't associate with git", projectName); + return null; + } + String gitUser = projectInfoByProjectName.getGitUser(); + String gitToken = projectInfoByProjectName.getGitToken(); + String gitUrl = projectInfoByProjectName.getGitUrl(); + // 拼接.git路径 + String gitPath = DSSGitUtils.generateGitPath(projectName, workspaceId, gitUser); + // 获取git仓库 + File repoDir = new File(gitPath); + try (Repository repository = getRepository(repoDir, projectName, workspaceId, gitUser, gitToken, gitUrl)){ + // 本地保持最新状态 + DSSGitUtils.pull(repository, projectName, gitUser, gitToken); + } catch (JGitInternalException e) { + logger.error("get git failed, the reason is", e); + throw new DSSErrorException(80001, "当前项目下已有工作流在进行git操作，请稍后重试"); + } catch (Exception e) { + logger.error("pull failed, the reason is ",e); + throw new DSSErrorException(80001, "更新本地git项目失败，原因为" + e.getMessage()); + } + if (CollectionUtils.isEmpty(request.getTypeList())) { + request.setTypeList(GitConstant.GIT_SERVER_SEARCH_TYPE); + } + String gitDir = DSSGitUtils.generateGitPath(projectName, request.getWorkspaceId(), gitUser); + String gitPathPre = DSSGitConstant.GIT_PATH_PRE + request.getWorkspaceId() + File.separator + gitUser + File.separator; + String workTree = gitPathPre + projectName; + List gitCommands = new ArrayList<>(Arrays.asList( + "git", "--git-dir=" + gitDir, "--work-tree=" + workTree, "grep", "-F", "-l", request.getSearchContent() + )); + List workflowNode = request.getWorkflowNameList(); + String fileName = request.getNodeName(); + List typeList = request.getTypeList(); + List path = new ArrayList<>(); + if (!CollectionUtils.isEmpty(workflowNode)) { + if (!StringUtils.isEmpty(fileName)) { + workflowNode = workflowNode.stream().map(s -> s + "*" + fileName + "*").collect(Collectors.toList()); + if (!CollectionUtils.isEmpty(typeList)) { + for (String workflow : workflowNode) { + for (String type : typeList) { + path.add(workflow + "/*" + type); + } + } + } + }else if (!CollectionUtils.isEmpty(typeList)) { + for (String workflow : workflowNode) { + for (String type : typeList) { + path.add(workflow + "/**/*" + type); + } + } + } + + if (CollectionUtils.isEmpty(path)) { + path = workflowNode; + } + }else if (!StringUtils.isEmpty(fileName)) { + if (!CollectionUtils.isEmpty(typeList)) { + for (String type : typeList) { + path.add("*" + fileName + "*/*" + type); + } + }else { + path.add("*" + fileName + "*"); + } + }else if (!CollectionUtils.isEmpty(typeList)) { + for (String type : typeList) { + path.add("*" + type); + } + } + + if (!CollectionUtils.isEmpty(path)) { + gitCommands.add("--"); + gitCommands.addAll(path); + } + logger.info(gitCommands.toString()); + List fileList = process(gitCommands); + List result = new ArrayList<>(); + + if (CollectionUtils.isEmpty(fileList)) { + return new GitSearchResponse(result, 0); + } + + String excludeDirectory = GitServerConfig.GIT_SEARCH_EXCLUDE_DIRECTORY.getValue(); + String excludeFile = GitServerConfig.GIT_SEARCH_EXCLUDE_FILE.getValue(); + + List excludeDirList = StringUtils.isEmpty(excludeDirectory)? new ArrayList<>() : Arrays.asList(excludeDirectory.split(",")); + List excludeFileList = StringUtils.isEmpty(excludeFile)? new ArrayList<>() : Arrays.asList(excludeFile.split(",")); + + Set excludeResult = new HashSet<>(); + for (String file : fileList) { + //排除指定文件夹下的内容 (.metaConf) + for (String excludeDir : excludeDirList) { + if (file.startsWith(excludeDir)) { + excludeResult.add(file); + break; + } + } + // 排除指定文件下的内容(.properties) + for (String exclude : excludeFileList) { + if (file.endsWith(exclude)) { + excludeResult.add(file); + } + } + } + + if (!CollectionUtils.isEmpty(excludeResult)) { + fileList.removeAll(excludeResult); + if (CollectionUtils.isEmpty(fileList)) { + return new GitSearchResponse(result, 0); + } + } + + int start = (request.getPageNow()-1) * request.getPageSize(); + int end = Math.min((start + request.getPageSize()), fileList.size()); + if (request.getPageNow() < 0 || start >= fileList.size()) { + logger.error("当前请求页" + request.getPageNow() + "超出搜索指定范围"); + return new GitSearchResponse(result, 0); + } + // subList 截断的List及子集 不允许List变更 + List subList = new ArrayList<>(fileList.subList(start, end)); + List filePathList = new ArrayList<>(); + for (String file : subList) { + filePathList.add(workTree + File.separator + file); + } + + + + List gitBaseCommand = new ArrayList<>(Arrays.asList( + "git", "--git-dir=" + gitDir, "--work-tree=" + workTree, "grep", "-F", "-n", request.getSearchContent() + )); + + + for (String file : filePathList) { + List gitSearchCommand = new ArrayList<>(gitBaseCommand); + gitSearchCommand.add(file); + logger.info(gitSearchCommand.toString()); + + List searchResult = process(gitSearchCommand); + + List keyLines = new ArrayList<>(); + for (String resultLine : searchResult) { + // 找到第一个冒号的位置 + int colonIndex = resultLine.indexOf(':'); + + // 如果存在冒号，去除它之前的所有内容 /testGit/test/.sql:1:test -> 1:test + if (colonIndex != -1 && colonIndex + 1 < resultLine.length()) { + resultLine = resultLine.substring(colonIndex + 1); + // 找到第二个冒号位置 + int colonIndexSec = resultLine.indexOf(':'); + if (colonIndexSec != -1 && colonIndexSec + 1 < resultLine.length()) { + Integer num = Integer.valueOf(resultLine.substring(0, colonIndexSec)); + String line = resultLine.substring(colonIndexSec + 1); + if (!StringUtils.isEmpty(line)) { + // 只添加符合要求的结果 + GitSearchLine searchLine = new GitSearchLine(num, line); + keyLines.add(searchLine); + } + } + } + } + // 处理文件路径 /data/GitInstall/testGit/test/.sql -> testGit/test/.sql + if (file.startsWith(workTree + File.separator)) { + file = file.substring(workTree.length() + File.separator.length()); + } + + result.add(new GitSearchResult(file, keyLines)); + } + + + return new GitSearchResponse(result, fileList.size()); + } + + private List process(List commands) { + List result = new ArrayList<>(); + ProcessBuilder processBuilder = new ProcessBuilder(commands); + processBuilder.redirectErrorStream(true); // Merge error stream with the standard output stream + Process process = null; + try { + process = processBuilder.start(); + BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream())); + String line; + while ((line = reader.readLine()) != null) { + logger.info(line); + result.add(line); + } + boolean b = process.waitFor(5, TimeUnit.SECONDS); + if (b) { + int exitCode = process.waitFor(); + logger.info("Exit code: " + exitCode); + + }else { + logger.info("search timeout"); + process.destroy(); + } + } catch (IOException | InterruptedException e) { + logger.error("grep failed ,the reason is :", e); + } finally { + if (process != null) { + process.destroy(); + } + } + return result; + } + + @Override + public GitDeleteResponse delete(GitDeleteRequest request) throws DSSErrorException { + Long workspaceId = request.getWorkspaceId(); + String projectName = request.getProjectName(); + + GitProjectGitInfo projectInfoByProjectName = GitProjectManager.getProjectInfoByProjectName(projectName); + if (projectInfoByProjectName == null) { + logger.error("the projectName : {} don't associate with git", projectName); + return null; + } + String gitUser = projectInfoByProjectName.getGitUser(); + String gitToken = projectInfoByProjectName.getGitToken(); + String gitUrl = projectInfoByProjectName.getGitUrl(); + GitDeleteResponse deleteResponse = null; + // 拼接.git路径 + String gitPath = DSSGitUtils.generateGitPath(projectName, request.getWorkspaceId(), gitUser); + // 获取git仓库 + File repoDir = new File(gitPath); + try (Repository repository = getRepository(repoDir, projectName, workspaceId, gitUser, gitToken, gitUrl)){ + // 本地保持最新状态 + DSSGitUtils.pull(repository, projectName, gitUser, gitToken); + List deleteFileList = request.getDeleteFileList(); + for (String path : deleteFileList) { + File file = new File(path); + if (file.exists()) { + if (file.isDirectory()) { + FileUtils.removeDirectory(path); + } else { + FileUtils.removeFiles(path); + } + } + } + // 提交前再次pull， 降低多节点同时提交不同工作流任务导致冲突频率 + DSSGitUtils.pull(repository, projectName, gitUser, gitToken); + // 提交 + DSSGitUtils.push(repository, projectName, gitUser, gitToken, "delete " + request.getDeleteFileList(), request.getDeleteFileList()); + } catch (JGitInternalException e) { + logger.error("get git failed, the reason is", e); + throw new DSSErrorException(80001, "当前项目下已有工作流在进行git操作，请稍后重试"); + }catch (Exception e) { + logger.error("delete failed, the reason is ",e); + throw new DSSErrorException(80001, "delete workflow failed, the reason is: " + e); + } + return null; + } + + @Override + public GitFileContentResponse getFileContent(GitFileContentRequest request) throws DSSErrorException { + Long workspaceId = request.getWorkspaceId(); + String projectName = request.getProjectName(); + String filePath = request.getFilePath(); + + GitProjectGitInfo projectInfoByProjectName = GitProjectManager.getProjectInfoByProjectName(projectName); + if (projectInfoByProjectName == null) { + logger.error("the projectName : {} don't associate with git", projectName); + return null; + } + String gitUser = projectInfoByProjectName.getGitUser(); + String gitToken = projectInfoByProjectName.getGitToken(); + String gitUrl = projectInfoByProjectName.getGitUrl(); + GitFileContentResponse contentResponse = new GitFileContentResponse(); + // 拼接.git路径 + String gitPath = DSSGitUtils.generateGitPath(projectName, workspaceId, gitUser); + // 获取git仓库 + File repoDir = new File(gitPath); + try (Repository repository = getRepository(repoDir, projectName, workspaceId, gitUser, gitToken, gitUrl)){ + // 本地保持最新状态 + DSSGitUtils.pull(repository, projectName, gitUser, gitToken); + String before = null; + String after = null; + + if (request.getPublish()) { + if (StringUtils.isNotEmpty(request.getCommitId())) { + // 获取 before commit以及content + DSSGitUtils.getTargetCommitFileContent(repository, request.getCommitId(), filePath, contentResponse); + } + after = DSSGitUtils.getFileContent(filePath, projectName, gitUser, workspaceId); + GitCommitResponse currentCommit = DSSGitUtils.getCurrentCommit(repository); + if (currentCommit.getCommitId() != null) { + contentResponse.setAfterCommitId(currentCommit.getCommitId()); + contentResponse.setAfterAnnotate(currentCommit.getCommentFull()); + } + contentResponse.setAfter(after); + } else { + // 获取当前提交前的文件内容 + before = DSSGitUtils.getFileContent(filePath, projectName, gitUser, workspaceId); + RevCommit latestCommitInfo = DSSGitUtils.getLatestCommitInfo(repository, filePath, projectName, workspaceId, gitUser); + if (latestCommitInfo != null) { + contentResponse.setBeforeAnnotate(latestCommitInfo.getShortMessage()); + contentResponse.setBeforeCommitId(latestCommitInfo.getId().getName()); + } + contentResponse.setBefore(before); + } + + return contentResponse; + } catch (JGitInternalException e) { + logger.error("get git failed, the reason is", e); + throw new DSSErrorException(80001, "当前项目下已有工作流在进行git操作，请稍后重试"); + }catch (Exception e) { + throw new DSSErrorException(80001, "getFileContent failed, the reason is: " + e); + } + } + + + @Override + public GitDiffFileContentResponse getDiffFileContent(GitDiffFileContentRequest request) throws DSSErrorException{ + + Long workspaceId = request.getWorkspaceId(); + String projectName = request.getProjectName(); + List filePaths = request.getFilePaths(); + + GitProjectGitInfo projectInfoByProjectName = GitProjectManager.getProjectInfoByProjectName(projectName); + if (projectInfoByProjectName == null) { + logger.error("getDiffFileContent the projectName : {} don't associate with git", projectName); + return null; + } + + String gitUser = projectInfoByProjectName.getGitUser(); + String gitToken = projectInfoByProjectName.getGitToken(); + String gitUrl = projectInfoByProjectName.getGitUrl(); + + // 拼接.git路径 + String gitPath = DSSGitUtils.generateGitPath(projectName, workspaceId, gitUser); + // 获取git仓库 + File repoDir = new File(gitPath); + + + try { + Repository repository = getRepository(repoDir, projectName, workspaceId, gitUser, gitToken, gitUrl); + // 本地保持最新状态 + DSSGitUtils.pull(repository, projectName, gitUser, gitToken); + // 获取的是提交之后的文件内容信息 + List gitFileContentResponseList = new ArrayList<>(); + for(String filePath: filePaths){ + + GitFileContentResponse contentResponse = new GitFileContentResponse(); + String after = DSSGitUtils.getFileContent(filePath, projectName, gitUser, workspaceId); + GitCommitResponse currentCommit = DSSGitUtils.getCurrentCommit(repository); + if (currentCommit.getCommitId() != null) { + contentResponse.setAfterCommitId(currentCommit.getCommitId()); + contentResponse.setAfterAnnotate(currentCommit.getCommentFull()); + } + contentResponse.setAfter(after); + contentResponse.setFilePath(filePath); + + gitFileContentResponseList.add(contentResponse); + } + + + return new GitDiffFileContentResponse(gitFileContentResponseList); + } catch (JGitInternalException e) { + logger.error("get git failed, the reason is", e); + throw new DSSErrorException(80001, "当前项目下已有工作流在进行git操作，请稍后重试"); + }catch (Exception e) { + throw new DSSErrorException(80001, "getFileContent failed, the reason is: " + e); + } + + } + + @Override + public GitHistoryResponse getHistory(GitHistoryRequest request) throws DSSErrorException { + Long workspaceId = request.getWorkspaceId(); + String projectName = request.getProjectName(); + + GitProjectGitInfo projectInfoByProjectName = GitProjectManager.getProjectInfoByProjectName(projectName); + if (projectInfoByProjectName == null) { + logger.error("the projectName : {} don't associate with git", projectName); + return null; + } + String gitUser = projectInfoByProjectName.getGitUser(); + String gitToken = projectInfoByProjectName.getGitToken(); + String gitUrl = projectInfoByProjectName.getGitUrl(); + GitHistoryResponse response = new GitHistoryResponse(); + // 拼接.git路径 + String gitPath = DSSGitUtils.generateGitPath(projectName, request.getWorkspaceId(), gitUser); + // 获取git仓库 + File repoDir = new File(gitPath); + try (Repository repository = getRepository(repoDir, projectName, workspaceId, gitUser, gitToken, gitUrl)){ + + List fileList = Collections.singletonList(request.getFilePath()); + // 本地保持最新状态 + DSSGitUtils.pull(repository, projectName, gitUser, gitToken); + + List latestCommit = DSSGitUtils.getLatestCommit(repository, request.getFilePath(), null); + if (CollectionUtils.isEmpty(latestCommit)) { + logger.error("get Commit failed, the reason is null"); + }else { + response.setResponses(latestCommit); + } + + } catch (JGitInternalException e) { + logger.error("get git failed, the reason is", e); + throw new DSSErrorException(80001, "当前项目下已有工作流在进行git操作，请稍后重试"); + }catch (Exception e) { + throw new DSSErrorException(80001, "getHistory failed, the reason is: " + e); + } + return response; + } + + @Override + public Repository getRepository(File repoDir, String projectName, Long workspaceId, String gitUser, String gitToken, String gitUrl) throws DSSErrorException { + Repository repository = null; + try { + // 当前机器不存在就新建 + if (repoDir.exists()) { + repository = new FileRepositoryBuilder().setGitDir(repoDir).build(); + } else { + // 本地创建Git项目 + DSSGitUtils.create(projectName, workspaceId, gitUser); + // 获取git项目 + repository = new FileRepositoryBuilder().setGitDir(repoDir).build(); + DSSGitUtils.remote(repository, projectName, gitUser, gitUrl); + DSSGitUtils.pull(repository, projectName, gitUser, gitToken); + } + } catch (JGitInternalException e) { + logger.error("get git failed, the reason is", e); + throw new DSSErrorException(80001, "当前项目下已有工作流在进行git操作，请稍后重试"); + }catch (Exception e) { + logger.info("get repository failed, the reason is: ", e); + throw new DSSErrorException(80001, "get repository failed, the reason is: " + e); + } + return repository; + } + + @Override + public GitCommitResponse getCurrentCommit(GitCurrentCommitRequest request) throws DSSErrorException { + Long workspaceId = request.getWorkspaceId(); + String projectName = request.getProjectName(); + + GitProjectGitInfo projectInfoByProjectName = GitProjectManager.getProjectInfoByProjectName(projectName); + if (projectInfoByProjectName == null) { + logger.error("the projectName : {} don't associate with git", projectName); + return null; + } + String gitUser = projectInfoByProjectName.getGitUser(); + String gitToken = projectInfoByProjectName.getGitToken(); + String gitUrl = projectInfoByProjectName.getGitUrl(); + GitCommitResponse commitResponse = null; + // 拼接.git路径 + String gitPath = DSSGitUtils.generateGitPath(projectName, request.getWorkspaceId(), gitUser); + // 获取git仓库 + File repoDir = new File(gitPath); + try (Repository repository = getRepository(repoDir, projectName, workspaceId, gitUser, gitToken, gitUrl)){ + // 本地保持最新状态 + DSSGitUtils.pull(repository, projectName, gitUser, gitToken); + + List latestCommit = DSSGitUtils.getLatestCommit(repository, request.getFilepath(), 1); + if (CollectionUtils.isEmpty(latestCommit)) { + logger.error("get latestCommit failed, the reason is null"); + } else { + commitResponse = latestCommit.get(0); + return commitResponse; + } + + } catch (JGitInternalException e) { + logger.error("get git failed, the reason is", e); + throw new DSSErrorException(80001, "当前项目下已有工作流在进行git操作，请稍后重试"); + }catch (Exception e) { + logger.error("getCurrentCommit, the reason is ",e); + throw new DSSErrorException(80001, "getCurrentCommit failed, the reason is: " + e); + } + return null; + } + + @Override + public GitCommitResponse gitCheckOut(GitRevertRequest request) throws DSSErrorException { + Long workspaceId = request.getWorkspaceId(); + String projectName = request.getProjectName(); + + GitProjectGitInfo projectInfoByProjectName = GitProjectManager.getProjectInfoByProjectName(projectName); + if (projectInfoByProjectName == null) { + logger.error("the projectName : {} don't associate with git", projectName); + return null; + } + String gitUser = projectInfoByProjectName.getGitUser(); + String gitToken = projectInfoByProjectName.getGitToken(); + String gitUrl = projectInfoByProjectName.getGitUrl(); + GitCommitResponse commitResponse = null; + // 拼接.git路径 + String gitPath = DSSGitUtils.generateGitPath(projectName, request.getWorkspaceId(), gitUser); + // 获取git仓库 + File repoDir = new File(gitPath); + try (Repository repository = getRepository(repoDir, projectName, workspaceId, gitUser, gitToken, gitUrl)){ + // 本地保持最新状态 + DSSGitUtils.pull(repository, projectName, gitUser, gitToken); + // 回滚 + DSSGitUtils.checkoutTargetCommit(repository, request, gitUser); + // push + List paths = Collections.singletonList(request.getPath()); + DSSGitUtils.push(repository, projectName, gitUser, gitToken,"revert "+ DSSGitConstant.GIT_USERNAME_FLAG + request.getUsername(), paths); + + List latestCommit = DSSGitUtils.getLatestCommit(repository, request.getPath(), 1); + if (CollectionUtils.isEmpty(latestCommit)) { + logger.error("get latestCommit failed, the reason is null"); + } else { + return latestCommit.get(0); + } + + } catch (JGitInternalException e) { + logger.error("get git failed, the reason is", e); + throw new DSSErrorException(80001, "当前项目下已有工作流在进行git操作，请稍后重试"); + }catch (Exception e) { + logger.error("checkOut failed, the reason is ",e); + throw new DSSErrorException(80001, "checkOut failed, the reason is: " + e); + } + return null; + } + + @Override + public GitCommitResponse removeFile(GitRemoveRequest request) throws DSSErrorException { + Long workspaceId = request.getWorkspaceId(); + String projectName = request.getProjectName(); + + GitProjectGitInfo projectInfoByProjectName = GitProjectManager.getProjectInfoByProjectName(projectName); + if (projectInfoByProjectName == null) { + logger.error("the projectName : {} don't associate with git", projectName); + return null; + } + String gitUser = projectInfoByProjectName.getGitUser(); + String gitToken = projectInfoByProjectName.getGitToken(); + String gitUrl = projectInfoByProjectName.getGitUrl(); + GitCommitResponse commitResponse = null; + // 拼接.git路径 + String gitPath = DSSGitUtils.generateGitPath(projectName, workspaceId, gitUser); + // 获取git仓库 + File repoDir = new File(gitPath); + try (Repository repository = getRepository(repoDir, projectName, workspaceId, gitUser, gitToken, gitUrl)){ + // 本地保持最新状态 + DSSGitUtils.pull(repository, projectName, gitUser, gitToken); + List paths = new ArrayList<>(); + // 同步删除对应节点 + for (String path : request.getPath()) { + paths.add(path); + FileUtils.removeFlowNode(path, projectName, workspaceId, gitUser); + String metaConfPath = GitConstant.GIT_SERVER_META_PATH + File.separator + path; + paths.add(metaConfPath); + FileUtils.removeFlowNode(metaConfPath, projectName, workspaceId, gitUser); + } + // 提交 + String comment = "delete workflowNode " + request.getPath().toString() + DSSGitConstant.GIT_USERNAME_FLAG + request.getUsername(); + // 提交前再次pull， 降低多节点同时提交不同工作流任务导致冲突频率 + DSSGitUtils.pull(repository, projectName, gitUser, gitToken); + DSSGitUtils.push(repository, projectName, gitUser, gitToken, comment, paths); + + commitResponse = DSSGitUtils.getCurrentCommit(repository); + + } catch (JGitInternalException e) { + logger.error("get git failed, the reason is", e); + throw new DSSErrorException(80001, "当前项目下已有工作流在进行git操作，请稍后重试"); + }catch (Exception e) { + logger.error("removeFile failed, the reason is ",e); + throw new DSSErrorException(80001, "removeFile failed, the reason is: " + e); + } + return commitResponse; + } + + @Override + public GitCommitResponse rename(GitRenameRequest request) throws DSSErrorException { + Long workspaceId = request.getWorkspaceId(); + String projectName = request.getProjectName(); + + GitProjectGitInfo projectInfoByProjectName = GitProjectManager.getProjectInfoByProjectName(projectName); + if (projectInfoByProjectName == null) { + logger.error("the projectName : {} don't associate with git", projectName); + return null; + } + String gitUser = projectInfoByProjectName.getGitUser(); + String gitToken = projectInfoByProjectName.getGitToken(); + String gitUrl = projectInfoByProjectName.getGitUrl(); + + GitCommitResponse commitResponse = null; + // 拼接.git路径 + String gitPath = DSSGitUtils.generateGitPath(projectName, request.getWorkspaceId(), gitUser); + // 获取git仓库 + File repoDir = new File(gitPath); + try (Repository repository = getRepository(repoDir, projectName, workspaceId, gitUser, gitToken, gitUrl)){ + // 同步删除对应节点 eg: /data/GitInstall/224/testGit/flowGitOld -> /data/GitInstall/224/testGit/flowGitNew + String projectPath = DSSGitUtils.generateGitPrePath(projectName, workspaceId, gitUser) + File.separator; + String olfFilePath = projectPath + FileUtils.normalizePath(request.getOldName()); + String filePath = projectPath + FileUtils.normalizePath(request.getName()); + // 同步删除对应节点 eg: /data/GitInstall/224/testGit/.metaConf/flowGitOld -> /data/GitInstall/224/testGit/.metaConf/flowGitNew + String metaPath = FileUtils.normalizePath(GitConstant.GIT_SERVER_META_PATH) + File.separator + FileUtils.normalizePath(request.getOldName()); + String oldFileMetaPath = projectPath + metaPath; + String oldMetaPath = FileUtils.normalizePath(GitConstant.GIT_SERVER_META_PATH) + File.separator + FileUtils.normalizePath(request.getName()); + String fileMetaPath = projectPath + oldMetaPath; + // 本地保持最新状态 + DSSGitUtils.pull(repository, projectName, gitUser, gitToken); + FileUtils.renameFile(olfFilePath, filePath); + FileUtils.renameFile(oldFileMetaPath, fileMetaPath); + // 提交 + String comment = "rename workflowNode " + request.getName() + DSSGitConstant.GIT_USERNAME_FLAG + request.getUsername(); + // 提交前再次pull， 降低多节点同时提交不同工作流任务导致冲突频率 + DSSGitUtils.pull(repository, projectName, gitUser, gitToken); + List paths = new ArrayList<>(); + paths.add(FileUtils.normalizePath(request.getOldName())); + paths.add(FileUtils.normalizePath(request.getName())); + + paths.add(metaPath); + paths.add(oldMetaPath); + DSSGitUtils.push(repository, projectName, gitUser, gitToken, comment, paths); + + commitResponse = DSSGitUtils.getCurrentCommit(repository); + + } catch (JGitInternalException e) { + logger.error("get git failed, the reason is", e); + throw new DSSErrorException(80001, "当前项目下已有工作流在进行git操作，请稍后重试"); + }catch (Exception e) { + logger.error("rename failed, the reason is ",e); + throw new DSSErrorException(80001, "rename failed, the reason is: " + e); + } + return commitResponse; + } + + @Override + public GitHistoryResponse getHistory(GitCommitInfoBetweenRequest request) throws DSSErrorException { + Long workspaceId = request.getWorkspaceId(); + String projectName = request.getProjectName(); + + GitProjectGitInfo projectInfoByProjectName = GitProjectManager.getProjectInfoByProjectName(projectName); + String gitUser = projectInfoByProjectName.getGitUser(); + String gitToken = projectInfoByProjectName.getGitToken(); + String gitUrl = projectInfoByProjectName.getGitUrl(); + if (projectInfoByProjectName == null) { + logger.error("the projectName : {} don't associate with git", projectName); + return null; + } + GitHistoryResponse response = new GitHistoryResponse(); + // 拼接.git路径 + String gitPath = DSSGitUtils.generateGitPath(projectName, request.getWorkspaceId(), gitUser); + // 获取git仓库 + File repoDir = new File(gitPath); + try (Repository repository = getRepository(repoDir, projectName, workspaceId, gitUser, gitToken, gitUrl)){ + // 本地保持最新状态 + DSSGitUtils.pull(repository, projectName, gitUser, gitToken); + if (StringUtils.isEmpty(request.getOldCommitId())) { + // 去掉上线 + List latestCommit = DSSGitUtils.getLatestCommit(repository, request.getDirName(), null); + if (CollectionUtils.isEmpty(latestCommit)) { + logger.error("get Commit failed, the reason is null"); + }else { + response.setResponses(latestCommit); + } + } else { + response = DSSGitUtils.listCommitsBetween(repository, request.getOldCommitId(), request.getNewCommitId(), request.getDirName()); + } + } catch (JGitInternalException e) { + logger.error("get git failed, the reason is", e); + throw new DSSErrorException(80001, "当前项目下已有工作流在进行git操作，请稍后重试"); + }catch (Exception e) { + logger.error("getHistory failed, the reason is ",e); + throw new DSSErrorException(80001, "getHistory failed, the reason is: " + e); + } + return response; + } +} diff --git a/dss-git/dss-git-server/src/main/java/com/webank/wedatasphere/dss/git/thread/GitServerThreadFactory.java b/dss-git/dss-git-server/src/main/java/com/webank/wedatasphere/dss/git/thread/GitServerThreadFactory.java new file mode 100644 index 0000000000..90a0a8fd0b --- /dev/null +++ b/dss-git/dss-git-server/src/main/java/com/webank/wedatasphere/dss/git/thread/GitServerThreadFactory.java @@ -0,0 +1,12 @@ +package com.webank.wedatasphere.dss.git.thread; + +import com.webank.wedatasphere.dss.git.common.protocol.config.GitServerConfig; + + +public class GitServerThreadFactory { + private static GitServerThreadPool gitServerThreadPool = new GitServerThreadPool(GitServerConfig.GIT_THREAD_NUM.getValue()); + + public static GitServerThreadPool getGitServerThreadPool() { + return gitServerThreadPool; + } +} diff --git a/dss-git/dss-git-server/src/main/java/com/webank/wedatasphere/dss/git/thread/GitServerThreadPool.java b/dss-git/dss-git-server/src/main/java/com/webank/wedatasphere/dss/git/thread/GitServerThreadPool.java new file mode 100644 index 0000000000..9a1c5a4a6d --- /dev/null +++ b/dss-git/dss-git-server/src/main/java/com/webank/wedatasphere/dss/git/thread/GitServerThreadPool.java @@ -0,0 +1,31 @@ +package com.webank.wedatasphere.dss.git.thread; + +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.*; + +public class GitServerThreadPool { + private final Map taskExecutors = new HashMap<>(); + private final int POOL_SIZE; + + public GitServerThreadPool(int num) { + this.POOL_SIZE = num; + // 初始化每个任务的ExecutorService + for (int i = 0; i < POOL_SIZE; i++) { + taskExecutors.put("TaskExecutor_" + i, Executors.newSingleThreadExecutor()); + } + } + + public Future submitTask(String taskName, Callable task) { + // 根据任务名称选择对应的ExecutorService，这里简化为根据hashCode选取 + int index = Math.abs(taskName.hashCode()) % POOL_SIZE; + ExecutorService executorService = taskExecutors.get("TaskExecutor_" + index); + return executorService.submit(task); + } + + public void shutdown() { + // 关闭所有ExecutorService + taskExecutors.values().forEach(ExecutorService::shutdown); + } + +} diff --git a/dss-git/dss-git-server/src/main/java/com/webank/wedatasphere/dss/git/utils/DSSGitUtils.java b/dss-git/dss-git-server/src/main/java/com/webank/wedatasphere/dss/git/utils/DSSGitUtils.java new file mode 100644 index 0000000000..33d0ede6b4 --- /dev/null +++ b/dss-git/dss-git-server/src/main/java/com/webank/wedatasphere/dss/git/utils/DSSGitUtils.java @@ -0,0 +1,1034 @@ +package com.webank.wedatasphere.dss.git.utils; + +import com.google.gson.JsonArray; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import com.google.gson.JsonElement; +import com.webank.wedatasphere.dss.common.exception.DSSErrorException; +import com.webank.wedatasphere.dss.git.common.protocol.constant.GitConstant; +import com.webank.wedatasphere.dss.git.common.protocol.response.GitFileContentResponse; +import org.apache.http.client.methods.HttpDelete; +import com.webank.wedatasphere.dss.git.common.protocol.GitTree; +import com.webank.wedatasphere.dss.git.common.protocol.GitUserEntity; +import com.webank.wedatasphere.dss.git.common.protocol.exception.GitErrorException; +import com.webank.wedatasphere.dss.git.common.protocol.request.GitRevertRequest; +import com.webank.wedatasphere.dss.git.common.protocol.response.GitDiffResponse; +import com.webank.wedatasphere.dss.git.common.protocol.response.GitCommitResponse; +import com.webank.wedatasphere.dss.git.common.protocol.response.GitHistoryResponse; +import com.webank.wedatasphere.dss.git.common.protocol.config.GitServerConfig; +import com.webank.wedatasphere.dss.git.common.protocol.util.UrlUtils; +import com.webank.wedatasphere.dss.git.constant.DSSGitConstant; +import org.apache.commons.lang.StringUtils; +import org.apache.http.HttpEntity; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.entity.StringEntity; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClients; +import org.apache.http.util.EntityUtils; +import org.eclipse.jgit.api.*; +import org.eclipse.jgit.api.errors.CheckoutConflictException; +import org.eclipse.jgit.api.errors.GitAPIException; +import org.eclipse.jgit.diff.DiffEntry; +import org.eclipse.jgit.errors.AmbiguousObjectException; +import org.eclipse.jgit.errors.IncorrectObjectTypeException; +import org.eclipse.jgit.errors.MissingObjectException; +import org.eclipse.jgit.lib.*; +import org.eclipse.jgit.revwalk.RevCommit; +import org.eclipse.jgit.revwalk.RevTree; +import org.eclipse.jgit.revwalk.RevWalk; +import org.eclipse.jgit.storage.file.FileRepositoryBuilder; +import org.eclipse.jgit.transport.URIish; +import org.eclipse.jgit.transport.UsernamePasswordCredentialsProvider; +import org.eclipse.jgit.treewalk.CanonicalTreeParser; +import org.eclipse.jgit.treewalk.TreeWalk; +import org.eclipse.jgit.treewalk.filter.OrTreeFilter; +import org.eclipse.jgit.treewalk.filter.PathFilter; +import org.eclipse.jgit.treewalk.filter.PathFilterGroup; +import org.eclipse.jgit.treewalk.filter.TreeFilter; +import org.json.JSONArray; +import org.json.JSONObject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.util.CollectionUtils; + +import java.io.*; +import java.net.HttpURLConnection; +import java.net.URISyntaxException; +import java.net.URL; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.text.SimpleDateFormat; +import java.util.*; +import java.util.stream.Collectors; + +public class DSSGitUtils { + private static final Logger logger = LoggerFactory.getLogger(DSSGitUtils.class); + + public static void init(String projectName, String gitUser, String gitToken, String gitUrl) throws Exception, GitErrorException{ + String projectPath = gitUser + "/" + projectName; + + String url = gitUrl + "/" + GitServerConfig.GIT_RESTFUL_API_CREATE_PROJECTS.getValue(); + try (CloseableHttpClient httpClient = HttpClients.createDefault()) { + HttpPost post = new HttpPost(url); + post.addHeader("PRIVATE-TOKEN", gitToken); + post.addHeader("Content-Type", "application/json"); + String jsonInputString = String.format("{\"name\": \"%s\", \"description\": \"%s\"}", projectName, projectName); + post.setEntity(new StringEntity(jsonInputString)); + + try (CloseableHttpResponse response = httpClient.execute(post)) { + int statusCode = response.getStatusLine().getStatusCode(); + if (statusCode == 201) { + logger.info("init success"); + } else { + throw new GitErrorException(80001, "创建Git项目失败，请检查工作空间token是否过期"); + } + } + } + } + + public static void remote(Repository repository, String projectName, String gitUser, String gitUrl)throws GitErrorException { + // 拼接git remote Url + String remoteUrl = gitUrl + "/" + gitUser + File.separator + projectName + ".git"; + try { + Git git = new Git(repository); + + // 添加远程仓库引用 + git.remoteAdd() + .setName("origin") + .setUri(new URIish(remoteUrl)) + .call(); + + logger.info("remote success"); + } catch (URISyntaxException e) { + throw new GitErrorException(80102, "connect Uri " + remoteUrl +" failed, the reason is: ", e); + } catch (GitAPIException e) { + throw new GitErrorException(80102, "remote git failed, the reason is: ", e); + } + + } + + + public static void create(String projectName, Long workspaceId, String gitUser) throws GitErrorException{ + logger.info("start success"); + File repoDir = new File(generateGitPrePath(projectName, workspaceId, gitUser)); // 指定仓库的目录 + File respo = new File(generateGitPath(projectName, workspaceId, gitUser)); + if (!respo.exists()) { + try { + // 初始化仓库 + Git git = Git.init() + .setDirectory(repoDir) + .call(); + + logger.info("Initialized empty Git repository in " + git.getRepository().getDirectory()); + + git.close(); // 不再需要时关闭Git对象 + } catch (GitAPIException e) { + throw new GitErrorException(80103, "git Create failed, the reason is: ", e); + } + } else { + logger.info(respo + " already exists"); + } + } + + public static void pull(Repository repository, String projectName, String gitUser, String gitToken)throws GitErrorException { + try { + Git git = new Git(repository); + + int i = 1; + while (true) { + i += 1; + // 拉取远程仓库更新至本地 + PullCommand pullCmd = git.pull().setCredentialsProvider(new UsernamePasswordCredentialsProvider(gitUser, gitToken)); + PullResult result = pullCmd.call(); + + // 成功直接返回，失败清空本地修改重试最多3次 + if (result.isSuccessful()) { + logger.info("Pull successful!"); + break; + } else if (i <= 3){ + logger.info("Pull failed : " + result.toString()); + // 冲突时以远程仓库为准 + if (result.getMergeResult().getConflicts() != null) { + logger.info("Conflicts occurred. Resolving with remote as priority..."); + // 丢失本地修改，处理冲突 + reset(repository, projectName); + } + }else { + throw new GitErrorException(80104, "git pull failed"); + } + } + } catch (Exception e) { + // 丢失本地修改，处理冲突 + reset(repository, projectName); + throw new GitErrorException(80001, "拉取git最新代码失败，原因为:" + e.getMessage()); + } + } + + public static void pullTargetFile(Repository repository, String projectName, GitUserEntity gitUser, List paths) throws Exception { + try { + if (CollectionUtils.isEmpty(paths)) { + return ; + } + + // Opening the repository + Git git = new Git(repository); + // 拉取最新提交记录，但不进行合并 + git.fetch().call(); + // 设置需要更新的文件或文件夹路径 + try { + git.checkout() + .setStartPoint("origin/master") + .addPaths(paths) + .call(); + } catch (CheckoutConflictException e) { + logger.info("Conflicts occurred, resetting local changes to match remote..."); + git.reset() + .setMode(ResetCommand.ResetType.HARD) + .setRef("refs/remotes/origin/master") + .call(); + logger.info("reset success"); + } + + } catch (GitAPIException e) { + e.printStackTrace(); + } + } + + public static GitDiffResponse diff(Repository repository, String projectName, List fileList, String gitUser, Long workspaceId)throws GitErrorException{ + + Set status = status(projectName, fileList, gitUser, workspaceId); + List codeTree = new ArrayList<>(); + List metaTree = new ArrayList<>(); + GitCommitResponse currentCommit = getCurrentCommit(repository); + String commitId = currentCommit.getCommitId(); + if (status.isEmpty()) { + return new GitDiffResponse(codeTree, metaTree, commitId); + } + GitTree root = new GitTree("", false); + GitTree rootMeta = new GitTree("", true); + for (String statu : status) { + if (statu.startsWith(GitConstant.GIT_SERVER_META_PATH)) { + rootMeta.setAbsolutePath(statu); + rootMeta.addChild(statu); + } else { + filterRoot(statu, root); + } + } + Map children = root.getChildren(); + for (Map.Entry entry: children.entrySet()) { + codeTree.add(entry.getValue()); + printTree("", entry.getValue()); + } + + for (Map.Entry entry : rootMeta.getChildren().entrySet()) { + metaTree.add(entry.getValue()); + DSSGitUtils.printTree("", entry.getValue()); + } + + return new GitDiffResponse(codeTree, metaTree, commitId); + } + + public static GitDiffResponse diffGit(Repository repository, String projectName, String commitId, String filePath) { + List codeTree = new ArrayList<>(); + List metaTree = new ArrayList<>(); + try (Git git = new Git(repository)) { + ObjectId commitObjectId = ObjectId.fromString(commitId); + try (RevWalk revWalk = new RevWalk(repository)) { + RevCommit commit = revWalk.parseCommit(commitObjectId); + + // Get the tree of the specified commit + ObjectId commitTree = commit.getTree().getId(); + + // Get the tree of the current working directory + ObjectId workTree = repository.resolve("HEAD^{tree}"); + + // Prepare the tree iterators for the diff + CanonicalTreeParser commitTreeParser = new CanonicalTreeParser(); + CanonicalTreeParser workTreeParser = new CanonicalTreeParser(); + + try (ObjectReader reader = repository.newObjectReader()) { + commitTreeParser.reset(reader, commitTree); + workTreeParser.reset(reader, workTree); + } + + List paths = new ArrayList<>(); + paths.add(filePath); + paths.add(GitConstant.GIT_SERVER_META_PATH + File.separator + filePath); + + String[] array = paths.toArray(new String[paths.size()]); + TreeFilter pathFilter = createPathFilter(array); + + // Perform the diff + List diffs = git.diff() + .setOldTree(commitTreeParser) + .setNewTree(workTreeParser) + .setPathFilter(pathFilter) + .call(); + + GitTree root = new GitTree("", false); + GitTree rootMeta = new GitTree("", true); + // Filter the diffs by the specified file path + for (DiffEntry entry : diffs) { + // 获取变更类型和文件路径 + DiffEntry.ChangeType changeType = entry.getChangeType(); + String oldPath = entry.getOldPath(); + String newPath = entry.getNewPath(); + + if (changeType.equals(DiffEntry.ChangeType.RENAME)) { + addChild(root, rootMeta, newPath, changeType, oldPath); + } else { + if (oldPath != null) { + addChild(root, rootMeta, oldPath, changeType, null); + } + if (newPath != null) { + addChild(root, rootMeta, newPath, changeType, null); + } + } + } + Map children = root.getChildren(); + for (Map.Entry entry: children.entrySet()) { + codeTree.add(entry.getValue()); + printTree("", entry.getValue()); + } + for (Map.Entry entry: rootMeta.getChildren().entrySet()) { + metaTree.add(entry.getValue()); + printTree("", entry.getValue()); + } + + } catch (GitAPIException | IOException e) { + throw new RuntimeException(e); + } + } + return new GitDiffResponse(codeTree, metaTree, null); + } + + private static TreeFilter createPathFilter(String[] paths) { + TreeFilter[] filters = new TreeFilter[paths.length]; + for (int i = 0; i < paths.length; i++) { + filters[i] = PathFilter.create(paths[i]); + } + return OrTreeFilter.create(filters); + } + + private static void addChild(GitTree root, GitTree rootMeta, String path, DiffEntry.ChangeType status, String oldFilePath) { + if (path == null) { + return; + } + String pathName = path; + if (pathName.contains("/dev/null")) { + return ; + } + // 处理重命名文件 + if (status.equals(DiffEntry.ChangeType.RENAME)) { + String oldFileName = oldFilePath.substring(oldFilePath.lastIndexOf("/") + 1); + pathName = path + "--(" + oldFileName + ")--"; + } + if (path.startsWith(GitConstant.GIT_SERVER_META_PATH)) { + rootMeta.setAbsolutePath(path); + rootMeta.addChild(pathName); + } else { + filterRoot(path, root); + } + } + + private static void filterRoot(String path, GitTree root) { + List typeList = new ArrayList<>(); + typeList.addAll(GitConstant.GIT_SERVER_SEARCH_TYPE); + typeList.add(".properties"); + for (String type : typeList) { + if (path.endsWith(type)) { + root.setAbsolutePath(path); + root.addChild(path); + break; + } + } + } + + // 打印树结构 + public static void printTree(String prefix, GitTree tree) { + logger.info(prefix + tree.getName()); + for (GitTree child : tree.getChildren().values()) { + printTree(prefix + " ", child); + } + } + + + private static List getDiffPath(List paths,Git git){ + List diffPaths = new ArrayList<>(); + try { + + String devNull = "/dev/null"; + String[] array = paths.toArray(new String[paths.size()]); + TreeFilter pathFilter = createPathFilter(array); + List diff = git.diff() + .setShowNameAndStatusOnly(true) + .setPathFilter(pathFilter) + .call(); + + // 未获取到差异文件信息或者第一次提交(oldPath 全为/dev/null) + if(CollectionUtils.isEmpty(diff) + || diff.stream().allMatch(diffEntry -> diffEntry.getOldPath().equalsIgnoreCase(devNull))){ + + diffPaths = new ArrayList<>(paths); + + }else{ + for (DiffEntry diffEntry: diff){ + logger.info("diff path is {} ,{}",diffEntry.getOldPath(),diffEntry.getNewPath()); + String diffPath = diffEntry.getNewPath().equalsIgnoreCase(devNull) ? diffEntry.getOldPath():diffEntry.getNewPath(); + diffPaths.add(diffPath); + } + } + + }catch (Exception e){ + logger.error("getDiffPath diff error msg is ", e); + diffPaths = new ArrayList<>(paths); + } + + return diffPaths; + } + + + public static void push(Repository repository, String projectName, String gitUser, String gitToken, String comment, List paths) throws GitErrorException{ + + try { + Git git = new Git(repository); + + logger.info("{} project workflow path is {}",projectName, StringUtils.join(paths,",")); + List diffPaths = getDiffPath(paths,git); + + // 添加新增、更改到暂存区 + for (String path : diffPaths) { + logger.info("diff path is {}",path); + // 添加所有更改 + git.add().addFilepattern(path).call(); + // 添加删除到暂存区 + git.add().setUpdate(true).addFilepattern(path).call(); + } + + // 创建新的提交 + git.commit() + .setMessage(comment) + .call(); + + logger.info("Changes committed to local repository."); + + // 推送到远程仓库 + git.push() + .setCredentialsProvider(new UsernamePasswordCredentialsProvider(gitUser, gitToken)) + .call(); + + logger.info("Changes pushed to remote repository."); + } catch (GitAPIException e) { + reset(repository, projectName); + throw new GitErrorException(80105, "提交失败，请重试或检查token是否过期", e); + } + } + + + public static void reset(Repository repository, String projectName)throws GitErrorException { + try { + Git git = new Git(repository); + + git.reset().setMode(ResetCommand.ResetType.HARD).call(); + + git.clean().setForce(true).setCleanDirectories(true).call(); + + logger.info("git reset success : " + projectName); + } catch (GitAPIException e) { + throw new GitErrorException(80106, "git reset failed, the reason is: ", e); + } + } + + public static void checkoutTargetCommit(Repository repository, GitRevertRequest request, String gitUser) throws GitAPIException, IOException, GitErrorException { + File repoDir = new File(generateGitPath(request.getProjectName(), request.getWorkspaceId(), gitUser)); + String commitId = request.getCommitId(); // 替换为目标commit的完整哈希值 + + try { + Git git = new Git(repository); + // 检出（回滚）指定commit的文件版本 + git.checkout() + .setStartPoint(commitId) + .addPath(request.getPath()) + .call(); + + logger.info("git check out success"); + logger.info("File " + repoDir.getAbsolutePath() + " has been rolled back to the version at commit: " + commitId); + + } catch (GitAPIException e) { + reset(repository, request.getProjectName()); + throw new GitErrorException(80107, "git check out failed, the reason is: ", e); + } + } + + public static boolean checkIfProjectExists(String gitToken, String projectPath) throws GitErrorException { + String url = UrlUtils.normalizeIp(GitServerConfig.GIT_URL_PRE.getValue()) + "/api/v4/projects/" + projectPath.replace("/", "%2F"); + try (CloseableHttpClient httpClient = HttpClients.createDefault()) { + HttpGet request = new HttpGet(url); + request.addHeader("PRIVATE-TOKEN", gitToken); + + try (CloseableHttpResponse response = httpClient.execute(request)) { + int statusCode = response.getStatusLine().getStatusCode(); + if (statusCode == 200) { + return true; + } else if (statusCode == 404) { + return false; + } else { + String responseBody = EntityUtils.toString(response.getEntity()); + logger.info("Unexpected response status: " + statusCode); + logger.info("Response body: " + responseBody); + return false; + } + } + } catch (Exception e) { + throw new GitErrorException(80108, "检查项目名称失败，请检查工作空间token是否过期", e); + } + } + + public static String getUserIdByUsername(String gitUrl, String gitToken, String username) throws GitErrorException, IOException { + String url = gitUrl + "/api/v4/users?username=" + username; + BufferedReader in = null; + try (CloseableHttpClient httpClient = HttpClients.createDefault()) { + HttpGet request = new HttpGet(url); + request.addHeader("PRIVATE-TOKEN", gitToken); + + try (CloseableHttpResponse response = httpClient.execute(request)) { + int statusCode = response.getStatusLine().getStatusCode(); + if (statusCode == 200) { + in = new BufferedReader(new InputStreamReader(response.getEntity().getContent())); + String inputLine; + StringBuilder content = new StringBuilder(); + + while ((inputLine = in.readLine()) != null) { + content.append(inputLine); + } + + String responseBody = content.toString(); + logger.info("Response Body: " + responseBody); + + JsonArray jsonArray = JsonParser.parseString(responseBody).getAsJsonArray(); + + if (jsonArray.size() > 0) { + JsonObject userObject = jsonArray.get(0).getAsJsonObject(); + return userObject.get("id").toString(); + } else { + throw new GitErrorException(80109, "获取userId失败，请检查该用户是否为git用户并激活"); + } + } else { + throw new GitErrorException(80109, "获取userId失败，请检查编辑用户token是否过期或git服务是否正常"); + } + } + } catch (Exception e) { + throw new GitErrorException(80109, "获取该git用户Id失败，原因为", e); + } finally { + if (in != null) { + in.close(); + } + } + } + + public static String getProjectIdByName(String projectName, String gitUser, String gitToken, String gitUrl) throws GitErrorException{ + String urlString = gitUrl + "/api/v4/projects?search=" + projectName; + + try (CloseableHttpClient httpClient = HttpClients.createDefault()) { + HttpGet request = new HttpGet(urlString); + request.setHeader("PRIVATE-TOKEN", gitToken); + + CloseableHttpResponse response = httpClient.execute(request); + System.out.println("Response Status Line: " + response.getStatusLine()); + + int responseCode = response.getStatusLine().getStatusCode(); + if (responseCode == HttpURLConnection.HTTP_OK) { + BufferedReader in = new BufferedReader(new InputStreamReader(response.getEntity().getContent())); + String inputLine; + StringBuilder content = new StringBuilder(); + + while ((inputLine = in.readLine()) != null) { + content.append(inputLine); + } + + in.close(); + + String responseBody = content.toString(); + logger.info("Response Body: " + responseBody); + + JsonArray jsonArray = JsonParser.parseString(responseBody).getAsJsonArray(); + + if (jsonArray.size() > 0) { + for (JsonElement element : jsonArray) { + JsonObject projectObject = element.getAsJsonObject(); + if (projectObject.get("name").getAsString().equals(projectName)) { + return projectObject.get("id").toString(); + } + } + } else { + throw new GitErrorException(80110, "项目创建失败，请稍后重试"); + } + } else { + throw new GitErrorException(80110, "请检查编辑用户token是否过期或git服务是否正常"); + } + } catch (Exception e) { + throw new GitErrorException(80110, "获取该项目git Id 失败，原因为", e); + } + return null; + } + + public static boolean addProjectMember(String gitUrl, String gitToken, String userId, String projectId, int accessLevel) throws GitErrorException, IOException { + String url = gitUrl + "/api/v4/projects/" + projectId + "/members"; + try (CloseableHttpClient httpClient = HttpClients.createDefault()) { + HttpPost request = new HttpPost(url); + request.addHeader("PRIVATE-TOKEN", gitToken); + request.addHeader("Content-Type", "application/json"); + + String json = String.format("{\"user_id\": \"%s\", \"access_level\": \"%d\"}", userId, accessLevel); + request.setEntity(new StringEntity(json)); + + try (CloseableHttpResponse response = httpClient.execute(request)) { + int statusCode = response.getStatusLine().getStatusCode(); + String responseBody = EntityUtils.toString(response.getEntity()); + if (statusCode == 201 || (statusCode == 409 && responseBody.contains("Member already exists"))) { + return true; + } else { + throw new GitErrorException(80111, "添加用户失败，请检查只读用户是否存在或编辑用户token是否过期"); + } + } + } catch (Exception e) { + throw new GitErrorException(80111, "添加用户失败，请检查编辑用户token是否过期或git服务是否正常"); + } + } + +// public static boolean removeProjectMember(GitUserEntity gitUser, String userId, String projectId) throws GitErrorException { +// String urlString = UrlUtils.normalizeIp(gitUser.getGitUrl()) + "/api/v4/projects/" + projectId + "/members/" + userId; +// try (CloseableHttpClient httpClient = HttpClients.createDefault()) { +// HttpDelete request = new HttpDelete(urlString); +// request.addHeader("PRIVATE-TOKEN", gitUser.getGitToken()); +// +// try (CloseableHttpResponse response = httpClient.execute(request)) { +// int responseCode = response.getStatusLine().getStatusCode(); +// if (responseCode == 204) { +// return true; +// } else { +// throw new GitErrorException(80112, "请检查工作空间Git只读用户是否存在"); +// } +// } +// } catch (IOException e) { +// throw new GitErrorException(80112, "更新用户权限失败", e); +// } +// } + + public static List getAllProjectName(String gitToken, String gitUrl) throws DSSErrorException { + int page = 1; + List allProjectNames = new ArrayList<>(); + + List projectNames = new ArrayList<>(); + do { + // 修改为GitLab实例的URL + String gitLabUrl = gitUrl + "/api/v4/projects?per_page=100&page=" + page; + // 创建HttpClient实例 + try (CloseableHttpClient httpClient = HttpClients.createDefault()) { + // 创建HttpGet请求 + HttpGet request = new HttpGet(gitLabUrl); + // 添加认证头部 + request.addHeader("PRIVATE-TOKEN", gitToken); + // 执行请求 + try (CloseableHttpResponse response = httpClient.execute(request)) { + // 获取响应实体 + HttpEntity entity = response.getEntity(); + // 将响应实体转换为字符串 + String result = EntityUtils.toString(entity); + // 解析项目名称 + projectNames = parseProjectNames(result); + // 打印项目名称 + logger.info("projectNames is: {}", projectNames); + // 添加到总项目列表中 + allProjectNames.addAll(projectNames); + } + } catch (IOException e) { + throw new GitErrorException(80113, "检查项目名称失败，请检查工作空间token是否过期", e); + } catch (Exception e) { + throw new GitErrorException(80113, "检查项目名称时解析JSON失败，请确认git当前是否可访问 ", e); + } + page++; + } while (!projectNames.isEmpty()); + + return allProjectNames; + } + + public static List parseProjectNames(String json) throws org.json.JSONException { + JSONArray projects = new JSONArray(json); + List projectNames = new ArrayList<>(); + + for (int i = 0; i < projects.length(); i++) { + JSONObject project = projects.getJSONObject(i); + projectNames.add(project.getString("name")); + } + + return projectNames; + } + + public static Set status(String projectName, List fileList, String gitUser, Long workspaceId)throws GitErrorException { + File repoDir = new File(generateGitPath(projectName, workspaceId, gitUser)); // 修改为你的仓库路径 + + try (Repository repository = new FileRepositoryBuilder().setGitDir(repoDir).build()) { + Git git = new Git(repository); + StatusCommand statusCommand = git.status(); + // 仅关注当前diff改动涉及的文件夹 + for (String file : fileList) { + statusCommand.addPath(file); + } + + Status status = statusCommand.call(); + + logger.info("Modified files:"); + logger.info("Modified files: {} , \nUntracked files: {}, \nAdded to index: {}, \nChanged files: {}, \nRemoved files: {}, \nMissing files: {}, \nConflicting files: {} ", + status.getModified().toString(), + status.getUntracked().toString(), + status.getAdded().toString(), + status.getChanged().toString(), + status.getRemoved().toString(), + status.getMissing().toString(), + status.getConflicting().toString() + ); + // 仅关注 修改、删除、新增并且未暂存的文件 + Set fileSet = new HashSet<>(); + if (!CollectionUtils.isEmpty(status.getModified())) { + fileSet.addAll(status.getModified()); + } + if (!CollectionUtils.isEmpty(status.getMissing())) { + fileSet.addAll(status.getMissing()); + } + if (!CollectionUtils.isEmpty(status.getUntracked())) { + fileSet.addAll(status.getUntracked()); + } + + return fileSet; + } catch (IOException | GitAPIException e) { + throw new GitErrorException(80114, "git status failed, the reason is : ", e); + } + } + + public static void archive(String projectName, String gitUser, String gitToken, String gitUrl) throws GitErrorException { + try { + String projectUrlEncoded = java.net.URLEncoder.encode(gitUser + "/" + projectName, "UTF-8"); + URL url = new URL(gitUrl + "/api/v4/projects/" + projectUrlEncoded + "/archive"); + HttpURLConnection conn = (HttpURLConnection) url.openConnection(); + conn.setRequestMethod("POST"); + conn.setRequestProperty("PRIVATE-TOKEN", gitToken); + + int responseCode = conn.getResponseCode(); + logger.info("Response Code: " + responseCode); + + BufferedReader in = new BufferedReader(new InputStreamReader(conn.getInputStream())); + String inputLine; + StringBuilder response = new StringBuilder(); + + while ((inputLine = in.readLine()) != null) { + response.append(inputLine); + } + in.close(); + + // 打印结果 + logger.info(response.toString()); + + } catch (Exception e) { + throw new GitErrorException(80115, "归档失败，请检查当前token是否过期 ", e); + } + } + + public static void archiveLocal(String projectName, String gitUser, Long workspaceId) throws GitErrorException{ + File repoDir = new File(generateGitPath(projectName, workspaceId, gitUser)); + if (!repoDir.exists()) { + logger.info("file {} not exists", repoDir.getAbsolutePath()); + return ; + } + try (Repository repository = new FileRepositoryBuilder().setGitDir(repoDir).build()) { + Git git = new Git(repository); + // 删除名为"origin"的远程仓库配置 + git.remoteRemove().setRemoteName("origin").call(); + // 删除本地文件 + FileUtils.removeDirectory(generateGitPrePath(projectName, workspaceId, gitUser)); + logger.info("Remote 'origin' removed successfully."); + } catch (GitAPIException e) { + throw new GitErrorException(80116, "git archive failed, the reason is : ", e); + } catch (IOException e) { + throw new GitErrorException(80116, "archive failed, the reason is : ", e); + } + } + + public static String getFileContent(String path, String projectName, String gitUser, Long workspaceId) throws IOException { + String filePath = generateGitPrePath(projectName, workspaceId, gitUser) + File.separator + path; + File file = new File(filePath); + if (file.exists()) { + return new String(Files.readAllBytes(Paths.get(filePath))); + } + return null; + } + + public static void getTargetCommitFileContent(Repository repository, String commitId, String filePath, GitFileContentResponse contentResponse) throws GitErrorException { + String content = ""; + try { + // 获取最新的commitId + ObjectId lastCommitId = repository.resolve(commitId); + // 获取提交记录 + try (RevWalk revWalk = new RevWalk(repository)) { + RevCommit commit = revWalk.parseCommit(lastCommitId); + if (commit != null) { + contentResponse.setBeforeAnnotate(commit.getShortMessage()); + contentResponse.setBeforeCommitId(commit.getId().getName()); + } + RevTree tree = commit.getTree(); + logger.info("Having tree: {}", tree); + // 遍历获取最近提交记录 + try (TreeWalk treeWalk = new TreeWalk(repository)) { + treeWalk.addTree(tree); + treeWalk.setRecursive(true); + treeWalk.setFilter(PathFilter.create(filePath)); + if (!treeWalk.next()) { + logger.warn("Did not find expected file '{}'，忽略", filePath); + contentResponse.setBefore(null); + } + + ObjectId objectId = treeWalk.getObjectId(0); + try { + ObjectLoader loader = repository.open(objectId); + byte[] bytes = loader.getBytes(); + content = new String(bytes); + logger.info("File content: " + content); + } catch (Exception e) { + logger.error("getFileContent Failed, the reason is: ", e); + } + } + revWalk.dispose(); + } + } catch (IOException e) { + throw new GitErrorException(80117, "getFileContent failed, the reason is : ", e); + } + contentResponse.setBefore(content); + } + + public static RevCommit getLatestCommitInfo(Repository repository, String filePath, String projectName, Long workspaceId, String gitUser) throws GitErrorException { + try { + Git git = new Git(repository); + Iterable commits = git.log().addPath(filePath).call(); + String path = DSSGitUtils.generateGitPrePath(projectName, workspaceId, gitUser) + filePath; + File file = new File(path); + if (file.exists()) { + for (RevCommit commit : commits) { + return commit; + } + } else { + RevCommit previousCommit = null; + for (RevCommit commit : commits) { + if (commit != null) { + try (TreeWalk treeWalk = TreeWalk.forPath(git.getRepository(), path, commit.getTree())) { + if (treeWalk == null) { + // The file does not exist in this commit, so this is the deletion commit + return commit; + } + } + } + previousCommit = commit; + } + } + return null; + } catch (Exception e) { + throw new GitErrorException(80118, "getLatestCommitInfo failed, the reason is : ", e); + } + } + + public static RevCommit getTargetCommitInfo(Repository repository, String commitId) throws GitErrorException { + try { + Git git = new Git(repository); + ObjectId commitIdInfo = ObjectId.fromString("commitId"); + RevWalk walk = new RevWalk(repository); + RevCommit commit = walk.parseCommit(commitIdInfo); + return commit; + } catch (Exception e) { + throw new GitErrorException(80118, "getTargetCommitInfo failed, the reason is : ", e); + } + } + + + public static void getCommitId(Repository repository, String projectName, int num, Long workspaceId)throws GitErrorException { + // 获取当前CommitId， + File repoDir = new File(File.separator + FileUtils.normalizePath(GitServerConfig.GIT_SERVER_PATH.getValue()) + File.separator + workspaceId + File.separator + projectName + File.separator +".git"); + + try { + Git git = new Git(repository); + Iterable commits = git.log().setMaxCount(num).call(); + SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + + for (RevCommit commit : commits) { + PersonIdent authorIdent = commit.getAuthorIdent(); + String commitHash = commit.getName(); // Commit hash + String commitTime = sdf.format(authorIdent.getWhen()); // Commit time + String commitMessage = commit.getShortMessage(); // Commit message + String commitAuthor = authorIdent.getName(); // Commit author + + logger.info("Commit Hash: " + commitHash); + logger.info("Commit Time: " + commitTime); + logger.info("Commit Message: " + commitMessage); + logger.info("Commit Author: " + commitAuthor); + } + } catch (Exception e) { + throw new GitErrorException(80118, "git log failed, the reason is : ", e); + } + } + + public static GitCommitResponse getCurrentCommit(Repository repository) throws GitErrorException{ + GitCommitResponse commitResponse = new GitCommitResponse(); + try { + // 获取HEAD引用 + Ref head = repository.exactRef("HEAD"); + SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + // 使用RevWalk解析当前的提交 + try (RevWalk walk = new RevWalk(repository)) { + RevCommit commit = walk.parseCommit(head.getObjectId()); + walk.dispose(); + commitResponse.setCommitId(commit.getId().getName()); + commitResponse.setCommitTime(sdf.format(commit.getAuthorIdent().getWhen())); + String shortMessage = commit.getShortMessage(); + commitResponse.setCommentFull(shortMessage); + getUserName(shortMessage, commitResponse, commit); + // 返回commitId字符串 + return commitResponse; + } + } catch (IOException e) { + throw new GitErrorException(80119, "get current commit failed, the reason is : ", e); + } + } + + public static void getUserName(String shortMessage, GitCommitResponse commitResponse, RevCommit commit) { + if (StringUtils.isEmpty(shortMessage)) { + return ; + } + + int lastIndexOf = shortMessage.lastIndexOf(DSSGitConstant.GIT_USERNAME_FLAG); + + if (lastIndexOf != -1) { + String username = shortMessage.substring(lastIndexOf + DSSGitConstant.GIT_USERNAME_FLAG.length()); + String comment = shortMessage.substring(0, lastIndexOf); + commitResponse.setCommitUser(username); + commitResponse.setComment(comment); + } else { + commitResponse.setCommitUser(commit.getCommitterIdent().getName()); + commitResponse.setComment(shortMessage); + } + } + + public static List getLatestCommit(Repository repository, String filePath, Integer num) throws GitErrorException{ + Set commitResponseList = new HashSet<>(); + SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + String metaPath = GitConstant.GIT_SERVER_META_PATH + "/" + filePath; + + try (Git git = new Git(repository)) { + Iterable commits = null; + Iterable metaConfCommits = null; + if (num == null) { + commits = git.log().addPath(filePath).call(); + metaConfCommits = git.log().addPath(metaPath).call(); + } else { + commits = git.log().addPath(filePath).setMaxCount(num).call(); + metaConfCommits = git.log().addPath(metaPath).setMaxCount(num).call(); + } + for (RevCommit commit : commits) { + GitCommitResponse commitResponse = new GitCommitResponse(); + commitResponse.setCommitId(commit.getId().getName()); + commitResponse.setCommitTime(sdf.format(commit.getAuthorIdent().getWhen())); + String shortMessage = commit.getShortMessage(); + getUserName(shortMessage, commitResponse, commit); + logger.info("提交ID: " + commit.getId().getName()); + commitResponseList.add(commitResponse); + } + + for (RevCommit commit : metaConfCommits) { + GitCommitResponse commitResponse = new GitCommitResponse(); + commitResponse.setCommitId(commit.getId().getName()); + commitResponse.setCommitTime(sdf.format(commit.getAuthorIdent().getWhen())); + String shortMessage = commit.getShortMessage(); + getUserName(shortMessage, commitResponse, commit); + logger.info("提交ID: " + commit.getId().getName()); + commitResponseList.add(commitResponse); + } + + return new ArrayList<>(commitResponseList); + } catch (GitAPIException e) { + throw new GitErrorException(80120, "get latestCommitId failed, the reason is : ", e); + } + } + + public static GitHistoryResponse listCommitsBetween(Repository repository, String oldCommitId, String newCommitId, String path) throws Exception { + List gitCommitResponseList = new ArrayList<>(); + Set commitIdSet = new HashSet<>(); + + + try (RevWalk walk = new RevWalk(repository)) { + Git git = new Git(repository); + ObjectId commitIdNow = null; + if (newCommitId == null) { + ObjectId head = repository.resolve("HEAD"); + commitIdNow = walk.parseCommit(head); + } else { + commitIdNow = repository.resolve(newCommitId); + } + // 代码改动 + gitLogHistory(git, repository, oldCommitId, commitIdNow, path, gitCommitResponseList, commitIdSet); + // 元数据改动 + gitLogHistory(git, repository, oldCommitId, commitIdNow, GitConstant.GIT_SERVER_META_PATH + File.separator + path, gitCommitResponseList, commitIdSet); + + gitCommitResponseList = gitCommitResponseList.stream().sorted(Comparator.comparing(GitCommitResponse::getCommitTime).reversed()).collect(Collectors.toList()); + } catch (Exception e) { + throw new GitErrorException(80121, "get log between " + oldCommitId + " and " + newCommitId + "failed, the reason is : ", e); + } + GitHistoryResponse historyResponse = new GitHistoryResponse(); + historyResponse.setResponses(gitCommitResponseList); + return historyResponse; + } + + private static void gitLogHistory(Git git, Repository repository, String oldCommitId, ObjectId commitIdNow, String path, List gitCommitResponseList, Set commitIdSet) throws IOException, GitAPIException { + SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + Iterable commits = git.log() + .addRange(repository.resolve(oldCommitId), commitIdNow) + .addPath(path) + .call(); + + for (RevCommit commit : commits) { + PersonIdent authorIdent = commit.getAuthorIdent(); // 获取提交人信息 + GitCommitResponse commitResponse = new GitCommitResponse(); + String commitId = commit.getId().getName(); + if (!commitIdSet.contains(commitId)) { + commitIdSet.add(commitId); + commitResponse.setCommitId(commitId); + commitResponse.setCommitTime(sdf.format(commit.getAuthorIdent().getWhen())); + String shortMessage = commit.getShortMessage(); + getUserName(shortMessage, commitResponse, commit); + gitCommitResponseList.add(commitResponse); + logger.info("Commit Hash: " + commit.getName()); // 提交的Hash值 + logger.info("Commit Time: " + authorIdent.getWhen()); // 提交时间 + logger.info("Commit Message: " + commit.getFullMessage()); // 提交信息 + logger.info("Author: " + authorIdent.getName() + " <" + authorIdent.getEmailAddress() + ">"); // 提交人 + } + } + } + + + + public static String generateGitPath(String projectName, Long workspaceId, String gitUser) { + // eg ： /data/GitInstall/224/testGit/.git + return generateGitPrePath(projectName, workspaceId, gitUser) + File.separator + DSSGitConstant.GIT_PATH_SUFFIX; + } + + public static String generateGitPrePath(String projectName, Long workspaceId, String gitUser) { + // eg ： /data/GitInstall/224/testGit + return DSSGitConstant.GIT_PATH_PRE + workspaceId + File.separator + gitUser + File.separator + projectName ; + } + + + + + + + + +} diff --git a/dss-git/dss-git-server/src/main/java/com/webank/wedatasphere/dss/git/utils/FileUtils.java b/dss-git/dss-git-server/src/main/java/com/webank/wedatasphere/dss/git/utils/FileUtils.java new file mode 100644 index 0000000000..e915ab6ec1 --- /dev/null +++ b/dss-git/dss-git-server/src/main/java/com/webank/wedatasphere/dss/git/utils/FileUtils.java @@ -0,0 +1,225 @@ +package com.webank.wedatasphere.dss.git.utils; + + +import com.webank.wedatasphere.dss.common.entity.BmlResource; +import com.webank.wedatasphere.dss.common.exception.DSSErrorException; +import com.webank.wedatasphere.dss.common.exception.DSSRuntimeException; +import com.webank.wedatasphere.dss.common.service.BMLService; +import com.webank.wedatasphere.dss.common.utils.ZipHelper; +import com.webank.wedatasphere.dss.git.common.protocol.config.GitServerConfig; +import com.webank.wedatasphere.dss.git.common.protocol.constant.GitConstant; +import com.webank.wedatasphere.dss.git.constant.DSSGitConstant; +import org.apache.commons.lang.StringUtils; +import org.apache.poi.util.IOUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.*; +import java.nio.file.*; +import java.nio.file.attribute.BasicFileAttributes; +import java.util.*; +import java.util.zip.ZipEntry; +import java.util.zip.ZipFile; + +public class FileUtils { + private static final Logger logger = LoggerFactory.getLogger(DSSGitUtils.class); + + public static void removeFiles (String filePath) { + try { + File file = new File(filePath); + if (file.exists()) { + file.delete(); + } + } catch (Exception e) { + logger.error("remove Files Failed, the reason is: ", e); + } + } + + public static void removeDirectory (String removeDirectoryPath) { + try { + File file = new File(removeDirectoryPath); + if (!file.exists()) { + logger.info("file {} not exist", removeDirectoryPath); + return; + } + Path dirToBeDeleted = Paths.get(removeDirectoryPath); + // 使用Files.walk收集所有路径，然后按照逆序排序，确保文件/子文件夹在其父文件夹之前被删除 + Files.walk(dirToBeDeleted) + .sorted(Comparator.reverseOrder()) + .forEach(path -> { + try { + Files.deleteIfExists(path); // 删除每个路径，如果存在的话 + logger.info("{} 删除成功", path); + } catch (IOException e) { + logger.info("无法删除路径: "+ path, e); + } + }); + logger.info("{} Directory and all its content deleted successfully.", removeDirectoryPath); + } catch (IOException e) { + logger.error("remove Directory Failed, the reason is: ", e); + } + } + + + public static void removeFlowNode(String path, String projectName, Long workspaceId, String gitUser) { + // 删除node节点 + String flowNodePath = DSSGitUtils.generateGitPrePath(projectName, workspaceId, gitUser) + File.separator + path; + // 1.删除对应文件代码 + removeDirectory(flowNodePath); + } + + public static void removeProject(String path, Long workspaceId, String gitUser) { + // 删除node节点 + String projectPath = DSSGitConstant.GIT_PATH_PRE + workspaceId + File.separator + gitUser + File.separator + path ; + // 1.删除项目 + removeDirectory(projectPath); + } + + public static void downloadBMLResource(BMLService bmlService, String path, BmlResource bmlResource, String username, Long workspaceId, String gitUser) { + //下载到本地处理 + String importFile = DSSGitConstant.GIT_PATH_PRE + workspaceId + File.separator + gitUser + File.separator + path + ".zip"; + logger.info("import zip file locate at {}",importFile); + + try{ + // 确保父目录存在 + Files.createDirectories(Paths.get(importFile).getParent()); + //下载压缩包 + bmlService.downloadToLocalPath(username, bmlResource.getResourceId(), bmlResource.getVersion(), importFile); + }catch (Exception e){ + logger.error("download failed, the reason is :", e); + throw new DSSRuntimeException("upload file format error(导入包格式错误)"); + } + } + + public static void unzipBMLResource(String path, Long workspaceId, String gitUser) throws DSSErrorException { + //下载到本地处理 + String dirPath = DSSGitConstant.GIT_PATH_PRE + workspaceId + File.separator + gitUser ; + String importFile= dirPath + File.separator + path + ".zip"; + //解压 + ZipHelper.unzipFile(importFile, dirPath, true); + logger.info("import unzip file locate at {}",importFile); + } + + private static String readImportZipProjectName(String zipFilePath) throws IOException { + try(ZipFile zipFile =new ZipFile(zipFilePath)){ + Enumeration entries =zipFile.entries(); + if(entries.hasMoreElements()){ + String projectName=entries.nextElement().getName(); + while (projectName.endsWith(File.separator)){ + projectName = projectName.substring(0, projectName.length() - 1); + } + return projectName; + } + } + throw new IOException(); + } + + public static String normalizePath(String path) { + path = path.trim(); + while(path.startsWith(File.separator)) { + path = path.substring(File.separator.length()); + } + while(path.endsWith(File.separator)) { + path = path.substring(0, path.length()-File.separator.length()); + } + + return path; + } + + public static String unzip(String dirPath,boolean deleteOriginZip)throws DSSErrorException { + File file = new File(dirPath); + if(!file.exists()){ + logger.error("{} 不存在, 不能解压zip文件", dirPath); + throw new DSSErrorException(90001,dirPath + " does not exist, can not unzip"); + } + //先用简单的方法，调用新进程进行压缩 + String[] strArr = dirPath.split(File.separator); + String shortPath = new File(dirPath).getName(); + String workPath = dirPath.substring(0, dirPath.length() - shortPath.length() - 1); + List list = new ArrayList<>(); + list.add("unzip"); + list.add("-o"); + String longZipFilePath = dirPath.replace(".zip",""); + list.add(shortPath); + ProcessBuilder processBuilder = new ProcessBuilder(list); + processBuilder.redirectErrorStream(true); + processBuilder.directory(new File(workPath)); + BufferedReader infoReader = null; + BufferedReader errorReader = null; + try{ + Process process = processBuilder.start(); + infoReader = new BufferedReader(new InputStreamReader(process.getInputStream())); + errorReader = new BufferedReader(new InputStreamReader(process.getErrorStream())); + String infoLine = null; + while((infoLine = infoReader.readLine()) != null){ + logger.info("process output: {} ", infoLine); + } + String errorLine = null; + StringBuilder errMsg = new StringBuilder(); + while((errorLine = errorReader.readLine()) != null){ + if (StringUtils.isNotEmpty(errorLine)){ + errMsg.append(errorLine).append("\n"); + } + logger.error("process error: {} ", errorLine); + } + int exitCode = process.waitFor(); + if (exitCode != 0){ + throw new DSSErrorException(90007,errMsg.toString()); + } + if(deleteOriginZip){ + logger.info("开始删除目录 {}", file); + if (file.delete()){ + logger.info("结束删除目录 {} 成功", file); + }else{ + logger.info("删除目录 {} 失败", file); + } + } + }catch(final Exception e){ + logger.error( file + " 解压缩 zip 文件失败, reason: ", e); + DSSErrorException exception = new DSSErrorException(90009,dirPath + " to zip file failed"); + exception.initCause(e); + throw exception; + } finally { + + logger.info("生成解压目录{}", longZipFilePath); + IOUtils.closeQuietly(infoReader); + IOUtils.closeQuietly(errorReader); + } + return longZipFilePath; + } + + public static void renameFile(String oldFileName, String fileName) { + File oldFile = new File(oldFileName); + File file = new File(fileName); + // 检查原文件夹是否存在 + if (oldFile.exists()) { + // 尝试重命名文件夹 + if (oldFile.renameTo(file)) { + logger.info("Folder renamed successfully."); + } else { + logger.error("Failed to rename folder."); + } + } else { + logger.error("Folder does not exist."); + } + } + + public static List getLocalProjectName(Long workspaceId, String gitUser) throws IOException { + String path = DSSGitConstant.GIT_PATH_PRE + workspaceId + File.separator + gitUser + File.separator; + Path dir = Paths.get(path); + List localProjectList = new ArrayList<>(); + try (DirectoryStream stream = Files.newDirectoryStream(dir)) { + for (Path entry : stream) { + // 获取基本文件属性 + BasicFileAttributes attrs = Files.readAttributes(entry, BasicFileAttributes.class); + if (attrs.isDirectory()) { + localProjectList.add(entry.getFileName().toString()); + } + } + } catch (IOException | DirectoryIteratorException e) { + logger.error("get Local Project Failed", e); + throw e; + } + return localProjectList; + } +} diff --git a/dss-git/dss-git-server/src/main/scala/com.webank.wedatasphere.dss.git/receiver/DSSGitChooser.scala b/dss-git/dss-git-server/src/main/scala/com.webank.wedatasphere.dss.git/receiver/DSSGitChooser.scala new file mode 100644 index 0000000000..f8681cdfcc --- /dev/null +++ b/dss-git/dss-git-server/src/main/scala/com.webank.wedatasphere.dss.git/receiver/DSSGitChooser.scala @@ -0,0 +1,68 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.git.receiver + +import com.webank.wedatasphere.dss.git.common.protocol.request.{GitAddMemberRequest, GitArchiveProjectRequest, GitBaseRequest, GitBatchCommitRequest, GitCheckProjectRequest, GitCommitInfoBetweenRequest, GitCommitRequest, GitConnectRequest, GitCreateProjectRequest, GitCurrentCommitRequest, GitDeleteRequest, GitDiffFileContentRequest, GitDiffRequest, GitDiffTargetCommitRequest, GitFileContentRequest, GitHistoryRequest, GitRemoveRequest, GitRenameRequest, GitRevertRequest, GitSearchRequest, GitUserByWorkspaceIdRequest, GitUserInfoByRequest, GitUserInfoRequest, GitUserUpdateRequest} +import com.webank.wedatasphere.dss.git.service.{DSSGitProjectManagerService, DSSGitWorkflowManagerService} + +import javax.annotation.PostConstruct +import org.apache.linkis.rpc.{RPCMessageEvent, Receiver, ReceiverChooser} +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.stereotype.Component + + +@Component +class DSSGitChooser extends ReceiverChooser { + + @Autowired + var gitProjectManagerService: DSSGitProjectManagerService = _ + + @Autowired + var gitWorkflowManagerService: DSSGitWorkflowManagerService = _ + + var receiver: Option[DSSGitReceiver] = _ + + @PostConstruct + def init(): Unit = receiver = Some(new DSSGitReceiver(gitProjectManagerService, gitWorkflowManagerService)) + + override def chooseReceiver(event: RPCMessageEvent): Option[Receiver] = event.message match { + case _: GitCreateProjectRequest => receiver + case _: GitArchiveProjectRequest => receiver + case _: GitCheckProjectRequest => receiver + case _: GitCommitRequest => receiver + case _: GitDiffRequest => receiver + case _: GitSearchRequest => receiver + case _: GitFileContentRequest => receiver + case _: GitDeleteRequest => receiver + case _: GitHistoryRequest => receiver + case _: GitCommitInfoBetweenRequest => receiver + case _: GitUserUpdateRequest => receiver + case _: GitUserInfoRequest => receiver + case _: GitCurrentCommitRequest => receiver + case _: GitRevertRequest => receiver + case _: GitRemoveRequest => receiver + case _: GitRenameRequest => receiver + case _: GitConnectRequest => receiver + case _: GitUserInfoByRequest => receiver + case _: GitDiffTargetCommitRequest => receiver + case _: GitBatchCommitRequest => receiver + case _: GitUserByWorkspaceIdRequest => receiver + case _: GitAddMemberRequest => receiver + case _: GitDiffFileContentRequest => receiver + case _ => None + } +} \ No newline at end of file diff --git a/dss-git/dss-git-server/src/main/scala/com.webank.wedatasphere.dss.git/receiver/DSSGitReceiver.scala b/dss-git/dss-git-server/src/main/scala/com.webank.wedatasphere.dss.git/receiver/DSSGitReceiver.scala new file mode 100644 index 0000000000..3ed8c81143 --- /dev/null +++ b/dss-git/dss-git-server/src/main/scala/com.webank.wedatasphere.dss.git/receiver/DSSGitReceiver.scala @@ -0,0 +1,77 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.git.receiver + +import com.webank.wedatasphere.dss.git.common.protocol.request.{GitAddMemberRequest, GitArchiveProjectRequest, GitBatchCommitRequest, GitCheckProjectRequest, GitCommitInfoBetweenRequest, GitCommitRequest, GitConnectRequest, GitCreateProjectRequest, GitCurrentCommitRequest, GitDeleteRequest, GitDiffRequest, GitDiffTargetCommitRequest, GitFileContentRequest, GitHistoryRequest, GitRemoveRequest, GitRenameRequest, GitRevertRequest, GitSearchRequest, GitUserByWorkspaceIdRequest, GitUserInfoByRequest, GitUserInfoRequest, GitUserUpdateRequest,GitDiffFileContentRequest} +import com.webank.wedatasphere.dss.git.manage.GitProjectManager +import com.webank.wedatasphere.dss.git.service.{DSSGitProjectManagerService, DSSGitWorkflowManagerService} +import org.apache.linkis.rpc.{Receiver, Sender} + +import scala.concurrent.duration.Duration + +class DSSGitReceiver(gitProjectManagerService: DSSGitProjectManagerService, gitWorkflowManagerService: DSSGitWorkflowManagerService) extends Receiver { + + override def receive(message: Any, sender: Sender): Unit = {} + + override def receiveAndReply(message: Any, sender: Sender): Any = message match { + case gitCreateProjectRequest: GitCreateProjectRequest => + gitProjectManagerService.create(gitCreateProjectRequest) + case gitArchiveProjectRequest: GitArchiveProjectRequest => + gitProjectManagerService.archive(gitArchiveProjectRequest) + case gitCheckProjectRequest: GitCheckProjectRequest => + gitProjectManagerService.checkProject(gitCheckProjectRequest) + case gitDiffRequest: GitDiffRequest => + gitWorkflowManagerService.diff(gitDiffRequest) + case gitCommitRequest: GitCommitRequest => + gitWorkflowManagerService.commit(gitCommitRequest) + case gitSearchRequest: GitSearchRequest => + gitWorkflowManagerService.search(gitSearchRequest) + case gitDeleteRequest: GitDeleteRequest => + gitWorkflowManagerService.delete(gitDeleteRequest) + case gitFileContentRequest: GitFileContentRequest => + gitWorkflowManagerService.getFileContent(gitFileContentRequest) + case gitHistoryRequest: GitHistoryRequest => + gitWorkflowManagerService.getHistory(gitHistoryRequest) + case gitCommitInfoBetweenRequest: GitCommitInfoBetweenRequest => + gitWorkflowManagerService.getHistory(gitCommitInfoBetweenRequest) + case gitUserUpdateRequest: GitUserUpdateRequest => + GitProjectManager.associateGit(gitUserUpdateRequest) + case gitCurrentCommitRequest: GitCurrentCommitRequest => + gitWorkflowManagerService.getCurrentCommit(gitCurrentCommitRequest) + case gitRevertRequest: GitRevertRequest => + gitWorkflowManagerService.gitCheckOut(gitRevertRequest) + case gitRemoveRequest: GitRemoveRequest => + gitWorkflowManagerService.removeFile(gitRemoveRequest) + case gitRenameRequest: GitRenameRequest => + gitWorkflowManagerService.rename(gitRenameRequest) + case gitUserInfoByTypeRequest: GitUserInfoByRequest => + GitProjectManager.getGitUserByType(gitUserInfoByTypeRequest) + case gitDiffTargetCommitRequest: GitDiffTargetCommitRequest => + gitWorkflowManagerService.diffGit(gitDiffTargetCommitRequest) + case gitBatchCommitRequest: GitBatchCommitRequest => + gitWorkflowManagerService.batchCommit(gitBatchCommitRequest) + case gitUserByWorkspaceIdRequest: GitUserByWorkspaceIdRequest => + gitProjectManagerService.getProjectGitUserInfo(gitUserByWorkspaceIdRequest) + case gitAddMemberRequest: GitAddMemberRequest => + gitProjectManagerService.addMember(gitAddMemberRequest) + case gitDiffFileContentRequest: GitDiffFileContentRequest => + gitWorkflowManagerService.getDiffFileContent(gitDiffFileContentRequest) + case _ => None + } + + override def receiveAndReply(message: Any, duration: Duration, sender: Sender): Any = {} +} diff --git a/dss-git/pom.xml b/dss-git/pom.xml new file mode 100644 index 0000000000..47fce9f4c3 --- /dev/null +++ b/dss-git/pom.xml @@ -0,0 +1,20 @@ + + + + dss + com.webank.wedatasphere.dss + 1.10.0-SNAPSHOT + ../pom.xml + + 4.0.0 + + dss-git + pom + + dss-git-common + dss-git-server + + + \ No newline at end of file diff --git a/dss-orchestrator/dss-orchestrator-common/pom.xml b/dss-orchestrator/dss-orchestrator-common/pom.xml index 10b97e09dd..91c5d621c5 100644 --- a/dss-orchestrator/dss-orchestrator-common/pom.xml +++ b/dss-orchestrator/dss-orchestrator-common/pom.xml @@ -21,7 +21,7 @@ dss com.webank.wedatasphere.dss - 1.1.0.20-SNAPSHOT + 1.10.0-SNAPSHOT ../../pom.xml 4.0.0 diff --git a/dss-orchestrator/dss-orchestrator-common/src/main/java/com/webank/wedatasphere/dss/orchestrator/common/entity/DSSOrchestratorInfo.java b/dss-orchestrator/dss-orchestrator-common/src/main/java/com/webank/wedatasphere/dss/orchestrator/common/entity/DSSOrchestratorInfo.java index 2035ae9124..6d0cb49b2a 100644 --- a/dss-orchestrator/dss-orchestrator-common/src/main/java/com/webank/wedatasphere/dss/orchestrator/common/entity/DSSOrchestratorInfo.java +++ b/dss-orchestrator/dss-orchestrator-common/src/main/java/com/webank/wedatasphere/dss/orchestrator/common/entity/DSSOrchestratorInfo.java @@ -86,6 +86,25 @@ public class DSSOrchestratorInfo implements DSSOrchestration { */ private Date updateTime; + /** + * 是否默认引用资源参数模板 + */ + private String isDefaultReference; + + /** + * 工作流状态：save-已保存 push-已提交 publish-已发布 + */ + private String status; + + /** 不包含关键字的节点**/ + private String notContainsKeywordsNode; + + private String projectName; + + + /**所属项目是否接入git**/ + private Boolean associateGit; + public DSSOrchestratorInfo() { } @@ -277,6 +296,14 @@ public void setUpdateTime(Date updateTime) { this.updateTime = updateTime; } + public String getIsDefaultReference() { + return isDefaultReference; + } + + public void setIsDefaultReference(String isDefaultReference) { + this.isDefaultReference = isDefaultReference; + } + @Override public String toString() { return "DSSOrchestratorInfo{" + @@ -293,6 +320,44 @@ public String toString() { ", secondaryType='" + secondaryType + '\'' + ", linkedAppConnNames=" + linkedAppConnNames + ", comment='" + comment + '\'' + + ", status='" + status + '\'' + '}'; } + + public String getStatus() { + return status; + } + + public void setStatus(String status) { + this.status = status; + } + + public String getNotContainsKeywordsNode() { + return notContainsKeywordsNode; + } + + public void setNotContainsKeywordsNode(String notContainsKeywordsNode) { + this.notContainsKeywordsNode = notContainsKeywordsNode; + } + + public void setProjectId(Long projectId) { + this.projectId = projectId; + } + + + public String getProjectName() { + return projectName; + } + + public void setProjectName(String projectName) { + this.projectName = projectName; + } + + public Boolean getAssociateGit() { + return associateGit; + } + + public void setAssociateGit(Boolean associateGit) { + this.associateGit = associateGit; + } } diff --git a/dss-orchestrator/dss-orchestrator-common/src/main/java/com/webank/wedatasphere/dss/orchestrator/common/entity/DSSOrchestratorInfoList.java b/dss-orchestrator/dss-orchestrator-common/src/main/java/com/webank/wedatasphere/dss/orchestrator/common/entity/DSSOrchestratorInfoList.java new file mode 100644 index 0000000000..2d3275ddd5 --- /dev/null +++ b/dss-orchestrator/dss-orchestrator-common/src/main/java/com/webank/wedatasphere/dss/orchestrator/common/entity/DSSOrchestratorInfoList.java @@ -0,0 +1,23 @@ +package com.webank.wedatasphere.dss.orchestrator.common.entity; + + +import java.util.List; + +public class DSSOrchestratorInfoList { + private List orchestratorInfos; + + public DSSOrchestratorInfoList(List orchestratorInfos) { + this.orchestratorInfos = orchestratorInfos; + } + + public DSSOrchestratorInfoList() { + } + + public List getOrchestratorInfos() { + return orchestratorInfos; + } + + public void setOrchestratorInfos(List orchestratorInfos) { + this.orchestratorInfos = orchestratorInfos; + } +} diff --git a/dss-orchestrator/dss-orchestrator-common/src/main/java/com/webank/wedatasphere/dss/orchestrator/common/entity/DSSOrchestratorVersion.java b/dss-orchestrator/dss-orchestrator-common/src/main/java/com/webank/wedatasphere/dss/orchestrator/common/entity/DSSOrchestratorVersion.java index 1175731d95..3c7b4c1124 100644 --- a/dss-orchestrator/dss-orchestrator-common/src/main/java/com/webank/wedatasphere/dss/orchestrator/common/entity/DSSOrchestratorVersion.java +++ b/dss-orchestrator/dss-orchestrator-common/src/main/java/com/webank/wedatasphere/dss/orchestrator/common/entity/DSSOrchestratorVersion.java @@ -72,6 +72,10 @@ public class DSSOrchestratorVersion { * 有效标示 0:无效；1：有效，默认是有效 */ private Integer validFlag; + /** + * 项目发布当前版本的commitId + */ + private String commitId; public Long getId() { @@ -205,6 +209,15 @@ public String toString() { ", content='" + content + '\'' + ", contextId='" + contextId + '\'' + ", validFlag='" + validFlag + '\'' + + ", commitId='" + commitId + '\'' + '}'; } + + public String getCommitId() { + return commitId; + } + + public void setCommitId(String commitId) { + this.commitId = commitId; + } } diff --git a/dss-orchestrator/dss-orchestrator-common/src/main/java/com/webank/wedatasphere/dss/orchestrator/common/entity/OrchestratorMeta.java b/dss-orchestrator/dss-orchestrator-common/src/main/java/com/webank/wedatasphere/dss/orchestrator/common/entity/OrchestratorMeta.java new file mode 100644 index 0000000000..2351967888 --- /dev/null +++ b/dss-orchestrator/dss-orchestrator-common/src/main/java/com/webank/wedatasphere/dss/orchestrator/common/entity/OrchestratorMeta.java @@ -0,0 +1,342 @@ +package com.webank.wedatasphere.dss.orchestrator.common.entity; + +import com.fasterxml.jackson.annotation.JsonFormat; + +import java.util.Date; + +public class OrchestratorMeta { + + private Long orchestratorId; + private String proxyUser; + private String metaResource; + private String globalVar; + private String creator; + //项目名称 + private String projectName; + + // 项目是否关联git + private Boolean associateGit; + + // 发布的错误消息 + private String errorMsg; + + // 项目ID + private Long projectId; + + // 工作空间ID + private Long workspaceId; + + private String workspaceName; + + // 工作流名称 + private String orchestratorName; + + // 项目描述 + private String description; + + // 工作流状态 + private String status; + + private String statusName; + + // 当前版本 + private String version; + + // 当前版本更新时间 + @JsonFormat(pattern="yyyy-MM-dd HH:mm:ss",timezone="GMT+8") + private Date updateTime; + + // 工作流ID + private String uuid; + + /** + * 用途标签，作为描述信息 + */ + private String uses; + /** + * 可以支持的实现的appconn节点 + */ + private String appConnName; + + /** + * 实现的二级类型，比如workflow_DAG + */ + private String secondaryType; + + + private String orchestratorMode; + + private String orchestratorWay; + /** + * 编排的重要程度，调度会考察这个重要程度 + */ + private String orchestratorLevel; + /** + * 更新人 + */ + private String updateUser; + + /** + * 是否默认引用资源参数模板 + */ + private String isDefaultReference; + + + // 模板名称 + private String templateName; + + private Boolean editable; + + + // 最新状态 + private String newStatus; + + // 最新状态名称 + private String newStatusName; + + public Boolean getEditable() { + return editable; + } + + public void setEditable(Boolean editable) { + this.editable = editable; + } + + public String getTemplateName() { + return templateName; + } + + public void setTemplateName(String templateName) { + this.templateName = templateName; + } + + public OrchestratorMeta() { + } + + + public String getErrorMsg() { + return errorMsg; + } + + public void setErrorMsg(String errorMsg) { + this.errorMsg = errorMsg; + } + + public Boolean getAssociateGit() { + return associateGit; + } + + public void setAssociateGit(Boolean associateGit) { + this.associateGit = associateGit; + } + + public Long getOrchestratorId() { + return orchestratorId; + } + + public void setOrchestratorId(Long orchestratorId) { + this.orchestratorId = orchestratorId; + } + + public String getProxyUser() { + return proxyUser; + } + + public void setProxyUser(String proxyUser) { + this.proxyUser = proxyUser; + } + + public String getMetaResource() { + return metaResource; + } + + public void setMetaResource(String metaResource) { + this.metaResource = metaResource; + } + + public String getGlobalVar() { + return globalVar; + } + + public void setGlobalVar(String globalVar) { + this.globalVar = globalVar; + } + + public Long getProjectId() { + return projectId; + } + + public void setProjectId(Long projectId) { + this.projectId = projectId; + } + + public Long getWorkspaceId() { + return workspaceId; + } + + public void setWorkspaceId(Long workspaceId) { + this.workspaceId = workspaceId; + } + + public String getWorkspaceName() { + return workspaceName; + } + + public void setWorkspaceName(String workspaceName) { + this.workspaceName = workspaceName; + } + + public String getProjectName() { + return projectName; + } + + public void setProjectName(String projectName) { + this.projectName = projectName; + } + + public String getIsDefaultReference() { + return isDefaultReference; + } + + public void setIsDefaultReference(String isDefaultReference) { + this.isDefaultReference = isDefaultReference; + } + + public String getOrchestratorName() { + return orchestratorName; + } + + public void setOrchestratorName(String orchestratorName) { + this.orchestratorName = orchestratorName; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + + public String getStatus() { + return status; + } + + public void setStatus(String status) { + this.status = status; + } + + public String getStatusName() { + return statusName; + } + + public void setStatusName(String statusName) { + this.statusName = statusName; + } + + public String getVersion() { + return version; + } + + public void setVersion(String version) { + this.version = version; + } + + public Date getUpdateTime() { + return updateTime; + } + + public void setUpdateTime(Date updateTime) { + this.updateTime = updateTime; + } + + public String getUuid() { + return uuid; + } + + public void setUuid(String uuid) { + this.uuid = uuid; + } + + + public String getUses() { + return uses; + } + + public void setUses(String uses) { + this.uses = uses; + } + + public String getAppConnName() { + return appConnName; + } + + public void setAppConnName(String appConnName) { + this.appConnName = appConnName; + } + + public String getSecondaryType() { + return secondaryType; + } + + public void setSecondaryType(String secondaryType) { + this.secondaryType = secondaryType; + } + + public String getOrchestratorMode() { + return orchestratorMode; + } + + public void setOrchestratorMode(String orchestratorMode) { + this.orchestratorMode = orchestratorMode; + } + + public String getOrchestratorWay() { + return orchestratorWay; + } + + public void setOrchestratorWay(String orchestratorWay) { + this.orchestratorWay = orchestratorWay; + } + + public String getOrchestratorLevel() { + return orchestratorLevel; + } + + public void setOrchestratorLevel(String orchestratorLevel) { + this.orchestratorLevel = orchestratorLevel; + } + + public String getUpdateUser() { + return updateUser; + } + + public void setUpdateUser(String updateUser) { + this.updateUser = updateUser; + } + + public String getCreator() { + return creator; + } + + public void setCreator(String creator) { + this.creator = creator; + } + + + public String getNewStatus() { + return newStatus; + } + + public void setNewStatus(String newStatus) { + this.newStatus = newStatus; + } + + public String getNewStatusName() { + return newStatusName; + } + + public void setNewStatusName(String newStatusName) { + this.newStatusName = newStatusName; + } +} diff --git a/dss-orchestrator/dss-orchestrator-common/src/main/java/com/webank/wedatasphere/dss/orchestrator/common/entity/OrchestratorMetaRequest.java b/dss-orchestrator/dss-orchestrator-common/src/main/java/com/webank/wedatasphere/dss/orchestrator/common/entity/OrchestratorMetaRequest.java new file mode 100644 index 0000000000..4b22d9dc93 --- /dev/null +++ b/dss-orchestrator/dss-orchestrator-common/src/main/java/com/webank/wedatasphere/dss/orchestrator/common/entity/OrchestratorMetaRequest.java @@ -0,0 +1,80 @@ +package com.webank.wedatasphere.dss.orchestrator.common.entity; + +import java.util.List; + +public class OrchestratorMetaRequest { + + + private Integer pageNow; + private Integer pageSize; + + private List projectList; + + private String orchestratorName; + + private String status; + + private Long workspaceId; + + private String orderBy; + + + public OrchestratorMetaRequest() { + } + + public Long getWorkspaceId() { + return workspaceId; + } + + public void setWorkspaceId(Long workspaceId) { + this.workspaceId = workspaceId; + } + + public Integer getPageNow() { + return pageNow; + } + + public void setPageNow(Integer pageNow) { + this.pageNow = pageNow; + } + + public Integer getPageSize() { + return pageSize; + } + + public void setPageSize(Integer pageSize) { + this.pageSize = pageSize; + } + + public List getProjectList() { + return projectList; + } + + public void setProjectList(List projectList) { + this.projectList = projectList; + } + + public String getOrchestratorName() { + return orchestratorName; + } + + public void setOrchestratorName(String orchestratorName) { + this.orchestratorName = orchestratorName; + } + + public String getStatus() { + return status; + } + + public void setStatus(String status) { + this.status = status; + } + + public String getOrderBy() { + return orderBy; + } + + public void setOrderBy(String orderBy) { + this.orderBy = orderBy; + } +} diff --git a/dss-orchestrator/dss-orchestrator-common/src/main/java/com/webank/wedatasphere/dss/orchestrator/common/entity/OrchestratorReleaseVersionInfo.java b/dss-orchestrator/dss-orchestrator-common/src/main/java/com/webank/wedatasphere/dss/orchestrator/common/entity/OrchestratorReleaseVersionInfo.java new file mode 100644 index 0000000000..8f290a8dd5 --- /dev/null +++ b/dss-orchestrator/dss-orchestrator-common/src/main/java/com/webank/wedatasphere/dss/orchestrator/common/entity/OrchestratorReleaseVersionInfo.java @@ -0,0 +1,130 @@ +package com.webank.wedatasphere.dss.orchestrator.common.entity; + +import java.util.Date; + +public class OrchestratorReleaseVersionInfo { + + + private Long id; + // 编排id + private Long orchestratorId; + + // bml版本号 + private String version; + + // 版本更新时间 + private Date updateTime; + + // 发布状态 + private String status; + + // 发布状态fail时的错误信息 + private String errorMsg; + + // 更新人 + private String updater; + + private Date releaseTime; + + private Long releaseTaskId; + + + private String releaseUser; + + public OrchestratorReleaseVersionInfo(Long id, Long orchestratorId, String version, Date updateTime, String status, String errorMsg, String updater, Date releaseTime, Long releaseTaskId) { + this.id = id; + this.orchestratorId = orchestratorId; + this.version = version; + this.updateTime = updateTime; + this.status = status; + this.errorMsg = errorMsg; + this.updater = updater; + this.releaseTime = releaseTime; + this.releaseTaskId = releaseTaskId; + } + + public Long getReleaseTaskId() { + return releaseTaskId; + } + + public void setReleaseTaskId(Long releaseTaskId) { + this.releaseTaskId = releaseTaskId; + } + + public Date getReleaseTime() { + return releaseTime; + } + + public void setReleaseTime(Date releaseTime) { + this.releaseTime = releaseTime; + } + + public OrchestratorReleaseVersionInfo() { + } + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + + public String getUpdater() { + return updater; + } + + public void setUpdater(String updater) { + this.updater = updater; + } + + public Long getOrchestratorId() { + return orchestratorId; + } + + public void setOrchestratorId(Long orchestratorId) { + this.orchestratorId = orchestratorId; + } + + public String getVersion() { + return version; + } + + public void setVersion(String version) { + this.version = version; + } + + public Date getUpdateTime() { + return updateTime; + } + + public void setUpdateTime(Date updateTime) { + this.updateTime = updateTime; + } + + public String getStatus() { + return status; + } + + public void setStatus(String status) { + this.status = status; + } + + public String getErrorMsg() { + return errorMsg; + } + + public void setErrorMsg(String errorMsg) { + this.errorMsg = errorMsg; + } + + + public String getReleaseUser() { + return releaseUser; + } + + public void setReleaseUser(String releaseUser) { + this.releaseUser = releaseUser; + } +} diff --git a/dss-orchestrator/dss-orchestrator-common/src/main/java/com/webank/wedatasphere/dss/orchestrator/common/entity/OrchestratorSubmitJob.java b/dss-orchestrator/dss-orchestrator-common/src/main/java/com/webank/wedatasphere/dss/orchestrator/common/entity/OrchestratorSubmitJob.java new file mode 100644 index 0000000000..d55afd228d --- /dev/null +++ b/dss-orchestrator/dss-orchestrator-common/src/main/java/com/webank/wedatasphere/dss/orchestrator/common/entity/OrchestratorSubmitJob.java @@ -0,0 +1,78 @@ +package com.webank.wedatasphere.dss.orchestrator.common.entity; + +import java.util.Date; + +public class OrchestratorSubmitJob { + private Long id; + private Long orchestratorId; + private Date createTime; + private Date updateTime; + private String instanceName; + private String status; + private String errorMsg; + private String submit_result; + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public Long getOrchestratorId() { + return orchestratorId; + } + + public void setOrchestratorId(Long orchestratorId) { + this.orchestratorId = orchestratorId; + } + + public Date getCreateTime() { + return createTime; + } + + public void setCreateTime(Date createTime) { + this.createTime = createTime; + } + + public Date getUpdateTime() { + return updateTime; + } + + public void setUpdateTime(Date updateTime) { + this.updateTime = updateTime; + } + + public String getInstanceName() { + return instanceName; + } + + public void setInstanceName(String instanceName) { + this.instanceName = instanceName; + } + + public String getStatus() { + return status; + } + + public void setStatus(String status) { + this.status = status; + } + + public String getErrorMsg() { + return errorMsg; + } + + public void setErrorMsg(String errorMsg) { + this.errorMsg = errorMsg; + } + + public String getSubmit_result() { + return submit_result; + } + + public void setSubmit_result(String submit_result) { + this.submit_result = submit_result; + } +} diff --git a/dss-orchestrator/dss-orchestrator-common/src/main/java/com/webank/wedatasphere/dss/orchestrator/common/entity/OrchestratorTemplateInfo.java b/dss-orchestrator/dss-orchestrator-common/src/main/java/com/webank/wedatasphere/dss/orchestrator/common/entity/OrchestratorTemplateInfo.java new file mode 100644 index 0000000000..3490cab6f3 --- /dev/null +++ b/dss-orchestrator/dss-orchestrator-common/src/main/java/com/webank/wedatasphere/dss/orchestrator/common/entity/OrchestratorTemplateInfo.java @@ -0,0 +1,33 @@ +package com.webank.wedatasphere.dss.orchestrator.common.entity; + +public class OrchestratorTemplateInfo { + + private Long orchestratorId; + + private String templateName; + + + public OrchestratorTemplateInfo() { + } + + public OrchestratorTemplateInfo(Long orchestratorId, String templateName) { + this.orchestratorId = orchestratorId; + this.templateName = templateName; + } + + public Long getOrchestratorId() { + return orchestratorId; + } + + public void setOrchestratorId(Long orchestratorId) { + this.orchestratorId = orchestratorId; + } + + public String getTemplateName() { + return templateName; + } + + public void setTemplateName(String templateName) { + this.templateName = templateName; + } +} diff --git a/dss-orchestrator/dss-orchestrator-common/src/main/java/com/webank/wedatasphere/dss/orchestrator/common/protocol/RequestOrchestratorVersionUpdateTime.java b/dss-orchestrator/dss-orchestrator-common/src/main/java/com/webank/wedatasphere/dss/orchestrator/common/protocol/RequestOrchestratorVersionUpdateTime.java new file mode 100644 index 0000000000..469f1aed95 --- /dev/null +++ b/dss-orchestrator/dss-orchestrator-common/src/main/java/com/webank/wedatasphere/dss/orchestrator/common/protocol/RequestOrchestratorVersionUpdateTime.java @@ -0,0 +1,20 @@ +package com.webank.wedatasphere.dss.orchestrator.common.protocol; + +public class RequestOrchestratorVersionUpdateTime { + private Long orchestratorId; + + public RequestOrchestratorVersionUpdateTime(Long orchestratorId) { + this.orchestratorId = orchestratorId; + } + + public RequestOrchestratorVersionUpdateTime() { + } + + public Long getOrchestratorId() { + return orchestratorId; + } + + public void setOrchestratorId(Long orchestratorId) { + this.orchestratorId = orchestratorId; + } +} diff --git a/dss-orchestrator/dss-orchestrator-common/src/main/java/com/webank/wedatasphere/dss/orchestrator/common/protocol/RequestQuertByAppIdOrchestrator.java b/dss-orchestrator/dss-orchestrator-common/src/main/java/com/webank/wedatasphere/dss/orchestrator/common/protocol/RequestQuertByAppIdOrchestrator.java new file mode 100644 index 0000000000..24c197d6b3 --- /dev/null +++ b/dss-orchestrator/dss-orchestrator-common/src/main/java/com/webank/wedatasphere/dss/orchestrator/common/protocol/RequestQuertByAppIdOrchestrator.java @@ -0,0 +1,20 @@ +package com.webank.wedatasphere.dss.orchestrator.common.protocol; + +public class RequestQuertByAppIdOrchestrator { + private Long appId; + + public RequestQuertByAppIdOrchestrator(Long appId) { + this.appId = appId; + } + + public RequestQuertByAppIdOrchestrator() { + } + + public Long getAppId() { + return appId; + } + + public void setAppId(Long appId) { + this.appId = appId; + } +} diff --git a/dss-orchestrator/dss-orchestrator-common/src/main/java/com/webank/wedatasphere/dss/orchestrator/common/protocol/RequestSubmitOrchestratorSync.java b/dss-orchestrator/dss-orchestrator-common/src/main/java/com/webank/wedatasphere/dss/orchestrator/common/protocol/RequestSubmitOrchestratorSync.java new file mode 100644 index 0000000000..89eb86b6c4 --- /dev/null +++ b/dss-orchestrator/dss-orchestrator-common/src/main/java/com/webank/wedatasphere/dss/orchestrator/common/protocol/RequestSubmitOrchestratorSync.java @@ -0,0 +1,71 @@ +package com.webank.wedatasphere.dss.orchestrator.common.protocol; + +import com.webank.wedatasphere.dss.common.label.LabelRouteVO; +import com.webank.wedatasphere.dss.standard.app.sso.Workspace; + + +public class RequestSubmitOrchestratorSync { + private Long flowId; + private LabelRouteVO labels; + private String projectName; + private String comment; + private Long orchestratorId; + private String username; + private Workspace workspace; + + public String getUsername() { + return username; + } + + public void setUsername(String username) { + this.username = username; + } + + public Workspace getWorkspace() { + return workspace; + } + + public void setWorkspace(Workspace workspace) { + this.workspace = workspace; + } + + public Long getFlowId() { + return flowId; + } + + public void setFlowId(Long flowId) { + this.flowId = flowId; + } + + public LabelRouteVO getLabels() { + return labels; + } + + public void setLabels(LabelRouteVO labels) { + this.labels = labels; + } + + public String getProjectName() { + return projectName; + } + + public void setProjectName(String projectName) { + this.projectName = projectName; + } + + public String getComment() { + return comment; + } + + public void setComment(String comment) { + this.comment = comment; + } + + public Long getOrchestratorId() { + return orchestratorId; + } + + public void setOrchestratorId(Long orchestratorId) { + this.orchestratorId = orchestratorId; + } +} diff --git a/dss-orchestrator/dss-orchestrator-common/src/main/java/com/webank/wedatasphere/dss/orchestrator/common/protocol/RequestUpdateOrchestratorBML.java b/dss-orchestrator/dss-orchestrator-common/src/main/java/com/webank/wedatasphere/dss/orchestrator/common/protocol/RequestUpdateOrchestratorBML.java new file mode 100644 index 0000000000..58a4e77c06 --- /dev/null +++ b/dss-orchestrator/dss-orchestrator-common/src/main/java/com/webank/wedatasphere/dss/orchestrator/common/protocol/RequestUpdateOrchestratorBML.java @@ -0,0 +1,34 @@ +package com.webank.wedatasphere.dss.orchestrator.common.protocol; + + +import com.webank.wedatasphere.dss.common.entity.BmlResource; + +public class RequestUpdateOrchestratorBML { + private Long flowId; + private BmlResource bmlResource; + + public RequestUpdateOrchestratorBML() { + } + + public RequestUpdateOrchestratorBML(Long flowId, BmlResource bmlResource) { + this.flowId = flowId; + this.bmlResource = bmlResource; + } + + + public Long getFlowId() { + return flowId; + } + + public void setFlowId(Long flowId) { + this.flowId = flowId; + } + + public BmlResource getBmlResource() { + return bmlResource; + } + + public void setBmlResource(BmlResource bmlResource) { + this.bmlResource = bmlResource; + } +} diff --git a/dss-orchestrator/dss-orchestrator-common/src/main/java/com/webank/wedatasphere/dss/orchestrator/common/ref/OrchestratorRefConstant.java b/dss-orchestrator/dss-orchestrator-common/src/main/java/com/webank/wedatasphere/dss/orchestrator/common/ref/OrchestratorRefConstant.java index 6504ef7c6b..6caab05e48 100644 --- a/dss-orchestrator/dss-orchestrator-common/src/main/java/com/webank/wedatasphere/dss/orchestrator/common/ref/OrchestratorRefConstant.java +++ b/dss-orchestrator/dss-orchestrator-common/src/main/java/com/webank/wedatasphere/dss/orchestrator/common/ref/OrchestratorRefConstant.java @@ -7,6 +7,10 @@ */ public interface OrchestratorRefConstant { + String ORCHESTRATOR_ID_LIST_KEY = "orchestratorIdList"; + String ORCHESTRATOR_VERSION_ID_KEY = "orchestratorVersionId"; + String ORCHESTRATOR_ADD_VERSION_FLAG_KEY = "addOrcVersionFlag"; + String DSS_ORCHESTRATOR_INFO_KEY = "dssOrchestratorInfo"; /*************************************************************/ @@ -34,5 +38,27 @@ public interface OrchestratorRefConstant { //workflow node suffix input by the user during workflow replication. String ORCHESTRATION_NODE_SUFFIX = "nodeSuffix"; + /** + * 工作流状态：save-已发布 push-已提交 publish-已保存 noGit-未接入 + */ + public static final String FLOW_STATUS_NO_GIT = "noGit"; + + public static final String FLOW_STATUS_SAVE = "save"; + + public static final String FLOW_STATUS_PUSHING = "running"; + + public static final String FLOW_STATUS_PUSH_FAILED = "failed"; + + public static final String FLOW_STATUS_PUSH_SUCCESS = "success"; + + public static final String FLOW_STATUS_PUSH = "push"; + + public static final String FLOW_STATUS_PUBLISH = "publish"; + + public static final String FLOW_STATUS_PUBLISHING = "publishing"; + + public static final String FLOW_STATUS_STATELESS = "stateless"; + + public static final String FLOW_STATUS_UNPUBLISHED = "unpublished"; } diff --git a/dss-orchestrator/dss-orchestrator-conversion-standard/pom.xml b/dss-orchestrator/dss-orchestrator-conversion-standard/pom.xml index 3335bfbee1..f66d3d6cd2 100644 --- a/dss-orchestrator/dss-orchestrator-conversion-standard/pom.xml +++ b/dss-orchestrator/dss-orchestrator-conversion-standard/pom.xml @@ -21,7 +21,7 @@ dss-orchestrator com.webank.wedatasphere.dss - 1.1.0.20-SNAPSHOT + 1.10.0-SNAPSHOT ../pom.xml 4.0.0 diff --git a/dss-orchestrator/dss-orchestrator-core/pom.xml b/dss-orchestrator/dss-orchestrator-core/pom.xml index 7d2ffcb84c..0999625d17 100644 --- a/dss-orchestrator/dss-orchestrator-core/pom.xml +++ b/dss-orchestrator/dss-orchestrator-core/pom.xml @@ -21,7 +21,7 @@ dss com.webank.wedatasphere.dss - 1.1.0.20-SNAPSHOT + 1.10.0-SNAPSHOT ../../pom.xml 4.0.0 diff --git a/dss-orchestrator/dss-orchestrator-core/src/main/scala/com/webank/wedatasphere/dss/orchestrator/core/service/BMLService.scala b/dss-orchestrator/dss-orchestrator-core/src/main/scala/com/webank/wedatasphere/dss/orchestrator/core/service/BMLService.scala new file mode 100644 index 0000000000..33936d6664 --- /dev/null +++ b/dss-orchestrator/dss-orchestrator-core/src/main/scala/com/webank/wedatasphere/dss/orchestrator/core/service/BMLService.scala @@ -0,0 +1,182 @@ +///* +// * Copyright 2019 WeBank +// * Licensed under the Apache License, Version 2.0 (the "License"); +// * you may not use this file except in compliance with the License. +// * You may obtain a copy of the License at +// * +// * http://www.apache.org/licenses/LICENSE-2.0 +// * +// * Unless required by applicable law or agreed to in writing, software +// * distributed under the License is distributed on an "AS IS" BASIS, +// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// * See the License for the specific language governing permissions and +// * limitations under the License. +// * +// */ +// +//package com.webank.wedatasphere.dss.orchestrator.core.service +// +//import com.webank.wedatasphere.dss.common.entity.BmlResource +// +//import java.io.{ByteArrayInputStream, InputStream} +//import java.util +//import java.util.UUID +//import com.webank.wedatasphere.dss.common.exception.DSSErrorException +//import com.webank.wedatasphere.dss.common.utils.IoUtils +//import org.apache.linkis.bml.client.{BmlClient, BmlClientFactory} +//import org.apache.linkis.bml.protocol.{BmlDownloadResponse, BmlUpdateResponse, BmlUploadResponse} +//import org.apache.linkis.common.utils.{JavaLog, Utils} +//import org.apache.commons.io.IOUtils +//import org.springframework.stereotype.Component +// +//import scala.collection.JavaConversions._ +// +// +//@Component +//class BMLService extends JavaLog{ +// +// def upload(userName: String, content: String, fileName: String, projectName:String): util.Map[String, Object] = { +// val inputStream = new ByteArrayInputStream(content.getBytes("utf-8")) +// val client: BmlClient = createBMLClient(userName) +// val resource: BmlUploadResponse = client.uploadShareResource(userName, projectName, fileName, inputStream) +// if (!resource.isSuccess) throw new DSSErrorException(911113, "上传失败") +// val map = new util.HashMap[String, Object] +// map += "resourceId" -> resource.resourceId +// map += "version" -> resource.version +// } +// +// def upload(userName: String, inputStream: InputStream, fileName: String, projectName:String): BmlResource = { +// val client: BmlClient = createBMLClient(userName) +// val resource: BmlUploadResponse = client.uploadShareResource(userName, projectName, fileName, inputStream) +// if (!resource.isSuccess) throw new DSSErrorException(911113, "上传失败") +// new BmlResource( resource.resourceId,resource.version); +// } +// +// def update(userName: String, resourceId: String, inputStream: InputStream): util.Map[String, Object] = { +// val client: BmlClient = createBMLClient(userName) +// val resource: BmlUpdateResponse = client.updateShareResource(userName, resourceId, "", inputStream) +// if (!resource.isSuccess) throw new DSSErrorException(911114, "更新失败") +// val map = new util.HashMap[String, Object] +// map += "resourceId" -> resource.resourceId +// map += "version" -> resource.version +// } +// +// def update(userName: String, resourceId: String, content: String): util.Map[String, Object] = { +// val inputStream = new ByteArrayInputStream(content.getBytes("utf-8")) +// val client: BmlClient = createBMLClient(userName) +// val resource: BmlUpdateResponse = client.updateShareResource(userName, resourceId, UUID.randomUUID().toString+".json", inputStream) +// if (!resource.isSuccess) throw new DSSErrorException(911114, "更新失败") +// val map = new util.HashMap[String, Object] +// map += "resourceId" -> resource.resourceId +// map += "version" -> resource.version +// } +// +// def query(userName: String, resourceId: String, version: String): util.Map[String, Object] = { +// val client: BmlClient = createBMLClient(userName) +// var resource: BmlDownloadResponse = null +// if (version == null) { +// resource = client.downloadShareResource(userName, resourceId) +// } else { +// resource = client.downloadShareResource(userName, resourceId, version) +// } +// if (!resource.isSuccess) throw new DSSErrorException(911115, "下载失败") +// val map = new util.HashMap[String, Object] +// map += "path" -> resource.fullFilePath +// map += "string" -> inputstremToString(resource.inputStream) +// } +// +// def download(userName: String, resourceId: String, version: String): util.Map[String, Object] = { +// val client: BmlClient = createBMLClient(userName) +// var resource: BmlDownloadResponse = null +// if (version == null) { +// resource = client.downloadShareResource(userName, resourceId) +// } else { +// resource = client.downloadShareResource(userName, resourceId, version) +// } +// if (!resource.isSuccess) throw new DSSErrorException(911115, "下载失败") +// val map = new util.HashMap[String, Object] +// map += "path" -> resource.fullFilePath +// map += "is" -> resource.inputStream +// } +// +// def downloadToLocalPath(userName: String, resourceId: String, version: String, path: String): String = { +// val result = download(userName,resourceId,version) +// val is = result.get("is").asInstanceOf[InputStream] +// val os = IoUtils.generateExportOutputStream(path) +// Utils.tryFinally(IOUtils.copy(is,os)){ +// IOUtils.closeQuietly(os) +// IOUtils.closeQuietly(is) +// } +// path +// } +// +// def downloadAndGetText(userName: String, resourceId: String, version: String, path: String): String = { +// downloadToLocalPath(userName,resourceId,version,path) +// //因为下载到指定目录后返回的resource的Stream为null,只能从文件重新读取。 +// val is = IoUtils.generateInputInputStream(path) +// Utils.tryFinally(inputstremToString(is))(IOUtils.closeQuietly(is)) +// } +// +// def readLocalResourceFile(userName: String,readPath: String): InputStream ={ +// IoUtils.generateInputInputStream(readPath) +// } +// +// def readLocalTextFile(userName: String,readPath: String): String ={ +// var inputStream:InputStream = null +// var text:String = null +// Utils.tryFinally{ +// inputStream = IoUtils.generateInputInputStream(readPath) +// text = inputstremToString(inputStream) +// }{ +// IOUtils.closeQuietly(inputStream) +// } +// text +// } +// +// private def inputstremToString(inputStream: InputStream): String = { +// scala.io.Source.fromInputStream(inputStream).mkString +// } +// +// private def createBMLClient(userName: String): BmlClient = { +// if (userName == null) +// BmlClientFactory.createBmlClient() +// else +// BmlClientFactory.createBmlClient(userName) +// } +// +// def createBmlProject(username:String, projectName:String, editUsers:util.List[String], +// accessUsers:util.List[String] ): Unit ={ +// val client = createBMLClient(username) +// val response = client.createBmlProject(username, projectName, accessUsers, editUsers) +// if (response.isSuccess){ +// logger.info(s"for user $username create bml project $projectName success") +// }else{ +// logger.error(s"for user $username create bml project $projectName failed") +// } +// } +// +// def attachResourceAndProject(username:String, projectName:String, resourceId:String):Unit = { +// val client = createBMLClient(username) +// val response = client.attachResourceAndProject(projectName, resourceId) +// if (response.isSuccess){ +// logger.info(s"attach $username and $projectName success") +// }else{ +// logger.error(s"attach $username and $projectName failed") +// } +// } +// +// +// def updateProjectPriv(projectName:String, username:String, editUsers:util.List[String], +// accessUsers:util.List[String]): Unit ={ +// val client = createBMLClient(username) +// val response = client.updateProjectPriv(username, projectName, editUsers, accessUsers) +// if (response.isSuccess){ +// logger.info(s"attach $username and $projectName success") +// }else{ +// logger.error(s"attach $username and $projectName failed") +// } +// } +// +// +// +//} diff --git a/dss-orchestrator/dss-orchestrator-db/pom.xml b/dss-orchestrator/dss-orchestrator-db/pom.xml index fd75675ec3..153503c098 100644 --- a/dss-orchestrator/dss-orchestrator-db/pom.xml +++ b/dss-orchestrator/dss-orchestrator-db/pom.xml @@ -21,7 +21,7 @@ dss com.webank.wedatasphere.dss - 1.1.0.20-SNAPSHOT + 1.10.0-SNAPSHOT ../../pom.xml 4.0.0 diff --git a/dss-orchestrator/dss-orchestrator-db/src/main/java/com/webank/wedatasphere/dss/orchestrator/db/dao/OrchestratorMapper.java b/dss-orchestrator/dss-orchestrator-db/src/main/java/com/webank/wedatasphere/dss/orchestrator/db/dao/OrchestratorMapper.java index b95f7df7c1..3b852849a0 100644 --- a/dss-orchestrator/dss-orchestrator-db/src/main/java/com/webank/wedatasphere/dss/orchestrator/db/dao/OrchestratorMapper.java +++ b/dss-orchestrator/dss-orchestrator-db/src/main/java/com/webank/wedatasphere/dss/orchestrator/db/dao/OrchestratorMapper.java @@ -16,6 +16,7 @@ package com.webank.wedatasphere.dss.orchestrator.db.dao; +import com.webank.wedatasphere.dss.common.entity.BmlResource; import com.webank.wedatasphere.dss.orchestrator.common.entity.*; import org.apache.ibatis.annotations.*; @@ -33,6 +34,10 @@ public interface OrchestratorMapper { void updateOrchestrator(DSSOrchestratorInfo dssOrchestratorInfo); + void updateOrchestratorBmlVersion(@Param("id") Long id, @Param("resourceId") String resourceId, @Param("bmlVersion") String bmlVersion); + + BmlResource getOrchestratorBmlVersion(@Param("id") Long id); + void deleteOrchestrator(Long id); /** @@ -74,6 +79,8 @@ DSSOrchestratorVersion getOrcVersionByIdAndOrcVersionId(@Param("orchestratorId") void updateOrchestratorVersion(DSSOrchestratorVersion dssOrchestratorVersion); + void updateOrchestratorVersionUpdateTime(@Param("id") Long id); + void deleteOrchestratorVersion(Long versionId); /** @@ -126,4 +133,48 @@ DSSOrchestratorVersion getOrcVersionByIdAndOrcVersionId(@Param("orchestratorId") List getHistoryOrcVersion(@Param("remainVersion") int remainVersion); void batchUpdateOrcInfo(@Param("list") List historyOrcVersion); + + void updateOrchestratorSubmitJobStatus(@Param("id") Long id, @Param("status") String status, @Param("errMsg") String errMsg); + + Long getLatestOrchestratorSubmitJobId(@Param("orchestratorId") Long orchestratorId); + + void updateOrchestratorSubmitResult(@Param("id") Long id, @Param("status") String status, @Param("result") String result); + + void batchUpdateOrchestratorSubmitJobStatus(@Param("list") List list, @Param("status") String status, @Param("errMsg") String errMsg); + + void insertOrchestratorSubmitJob(OrchestratorSubmitJob orchestratorSubmitJob); + + OrchestratorSubmitJob selectSubmitJobStatus(@Param("orchestratorId") Long orchestratorId); + + OrchestratorSubmitJob selectSubmitJobStatusById(@Param("id") Long id); + + String selectResult(@Param("id") Long id); + + List getAllOrchestratorMeta(OrchestratorMetaRequest orchestratorMetaRequest); + + OrchestratorMeta getOrchestratorMeta(@Param("orchestratorId") Long orchestratorId); + + List getOrchestratorName(List list); + + List getOrchestratorReleaseVersionInfo(@Param("orchestratorIdList") List orchestratorIdList); + + List getOrchestratorDefaultTemplateInfo(@Param("orchestratorIdList")List orchestratorIdList); + + + List getAllOrchestratorName(@Param("workspaceId") Long workspaceId,@Param("projectName") String projectName); + + + OrchestratorReleaseVersionInfo getOrchestratorVersionById(@Param("orchestratorId") Long orchestratorId); + + + List getSubmitJobStatus(@Param("orchestratorIdList") List orchestratorIdList); + + + void updateOrchestratorNotContainsKeywordsNode(@Param("orchestratorId") Long orchestratorId, @Param("notContainsKeywordsNode") String notContainsKeywordsNode); + + DSSOrchestratorInfo getOrchestratorNotContainsKeywordsNode(@Param("orchestratorId") Long orchestratorId); + + List getOrchestratorListById(@Param("orchestratorIdList") List orchestratorIdList); + + } diff --git a/dss-orchestrator/dss-orchestrator-db/src/main/java/com/webank/wedatasphere/dss/orchestrator/db/dao/impl/orchestratorMapper.xml b/dss-orchestrator/dss-orchestrator-db/src/main/java/com/webank/wedatasphere/dss/orchestrator/db/dao/impl/orchestratorMapper.xml index 77e8669fd6..7459c7b7f8 100644 --- a/dss-orchestrator/dss-orchestrator-db/src/main/java/com/webank/wedatasphere/dss/orchestrator/db/dao/impl/orchestratorMapper.xml +++ b/dss-orchestrator/dss-orchestrator-db/src/main/java/com/webank/wedatasphere/dss/orchestrator/db/dao/impl/orchestratorMapper.xml @@ -22,11 +22,11 @@ id,`name`,`type`,`desc`,`creator`,`create_time`,`project_id`,`uses`,`appconn_name`,`uuid`,`secondary_type` , - `workspace_id`,`orchestrator_mode`,`orchestrator_way`,`orchestrator_level`,`update_user`,`update_time` + `workspace_id`,`orchestrator_mode`,`orchestrator_way`,`orchestrator_level`,`update_user`,`update_time`,`is_default_reference` - id,`orchestrator_id`,`app_id`,`source`,`version`,`comment`,`update_time`,`updater`,`project_id`,`content`,`context_id`,`valid_flag` + id,`orchestrator_id`,`app_id`,`source`,`version`,`comment`,`update_time`,`updater`,`project_id`,`content`,`context_id`,`valid_flag`, `commit_id` @@ -35,12 +35,16 @@ `orchestrator_level`=#{orchestratorLevel}, `update_user`=#{updateUser}, `update_time`=#{updateTime}, + `is_default_reference`=#{isDefaultReference}, WHERE id = #{id} + + UPDATE dss_orchestrator_info + + `bml_version` = #{bmlVersion}, `resource_id` = #{resourceId} + + where `id` = #{id} + + + + DELETE @@ -80,7 +98,7 @@ parameterType="com.webank.wedatasphere.dss.orchestrator.common.entity.DSSOrchestratorVersion"> INSERT INTO dss_orchestrator_version_info () VALUES - (#{id},#{orchestratorId},#{appId},#{source},#{version},#{comment},#{updateTime},#{updater},#{projectId},#{content},#{contextId},#{validFlag}) + (#{id},#{orchestratorId},#{appId},#{source},#{version},#{comment},NOW(),#{updater},#{projectId},#{content},#{contextId},#{validFlag}, #{commitId}) @@ -100,10 +118,17 @@ WHERE id = #{id} + + UPDATE dss_orchestrator_version_info + set `update_time` = Now() + WHERE id = #{id} + + SELECT - * + FROM `dss_orchestrator_version_info` WHERE @@ -128,7 +153,7 @@ - SELECT * FROM dss_orchestrator_version_info + SELECT + , commit_id + FROM dss_orchestrator_version_info WHERE orchestrator_id = #{orchestratorId} and valid_flag = 1 and context_id != '' ORDER BY id DESC @@ -181,10 +208,13 @@ + @@ -200,7 +230,9 @@ @@ -213,7 +245,9 @@ - select * + select + from dss_orchestrator_ref_orchestration_relation where orchestrator_id = #{orchestratorId} @@ -274,4 +309,239 @@ #{item.id} + + + insert into dss_orchestrator_submit_job_info (`orchestrator_id`, `create_time`, `update_time`, `instance_name`, `status`) values + (#{orchestratorId}, Now(), Now(), #{instanceName}, #{status}) + + + + update dss_orchestrator_submit_job_info + + `error_msg` = #{errMsg} ,`update_time` = Now() , `status` = #{status} + + where `id` = #{id} + + + + + + update dss_orchestrator_submit_job_info + + `submit_result` = #{result},`update_time` = Now() , `status` = #{status} + + where `id` = #{id} + + + + update dss_orchestrator_submit_job_info + + `error_msg` = #{errMsg} ,`update_time` = Now() , `status` = #{status} + + where `id` IN + + #{item} + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + update dss_orchestrator_info set not_contains_keywords_node = #{notContainsKeywordsNode} + where id = #{orchestratorId} + + + + + + + + \ No newline at end of file diff --git a/dss-orchestrator/dss-orchestrator-db/src/main/java/com/webank/wedatasphere/dss/orchestrator/db/entity/OrchestratorPublishJob.java b/dss-orchestrator/dss-orchestrator-db/src/main/java/com/webank/wedatasphere/dss/orchestrator/db/entity/OrchestratorPublishJob.java new file mode 100644 index 0000000000..854eebbfa2 --- /dev/null +++ b/dss-orchestrator/dss-orchestrator-db/src/main/java/com/webank/wedatasphere/dss/orchestrator/db/entity/OrchestratorPublishJob.java @@ -0,0 +1,56 @@ +package com.webank.wedatasphere.dss.orchestrator.db.entity; + +import java.util.Date; + +public class OrchestratorPublishJob { + + private Long id; + private String jobId; + private String conversionJobJson; + private Date createdTime; + private Date updatedTime; + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public String getJobId() { + return jobId; + } + + public OrchestratorPublishJob setJobId(String jobId) { + this.jobId = jobId; + return this; + } + + public String getConversionJobJson() { + return conversionJobJson; + } + + public OrchestratorPublishJob setConversionJobJson(String conversionJobJson) { + this.conversionJobJson = conversionJobJson; + return this; + } + + public Date getCreatedTime() { + return createdTime; + } + + public OrchestratorPublishJob setCreatedTime(Date createdTime) { + this.createdTime = createdTime; + return this; + } + + public Date getUpdatedTime() { + return updatedTime; + } + + public OrchestratorPublishJob setUpdatedTime(Date updatedTime) { + this.updatedTime = updatedTime; + return this; + } +} diff --git a/dss-orchestrator/dss-orchestrator-loader/pom.xml b/dss-orchestrator/dss-orchestrator-loader/pom.xml index 1f4a05191c..432c3ac771 100644 --- a/dss-orchestrator/dss-orchestrator-loader/pom.xml +++ b/dss-orchestrator/dss-orchestrator-loader/pom.xml @@ -21,7 +21,7 @@ dss com.webank.wedatasphere.dss - 1.1.0.20-SNAPSHOT + 1.10.0-SNAPSHOT ../../pom.xml 4.0.0 diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/pom.xml b/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/pom.xml index 1ac625b498..9e94e580ed 100644 --- a/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/pom.xml +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/pom.xml @@ -21,7 +21,7 @@ dss com.webank.wedatasphere.dss - 1.1.0.20-SNAPSHOT + 1.10.0-SNAPSHOT ../../../../pom.xml 4.0.0 diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/java/com/webank/wedatasphere/dss/flow/execution/entrance/restful/FlowEntranceRestfulApi.java b/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/java/com/webank/wedatasphere/dss/flow/execution/entrance/restful/FlowEntranceRestfulApi.java index 301fa9a91d..f324581818 100644 --- a/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/java/com/webank/wedatasphere/dss/flow/execution/entrance/restful/FlowEntranceRestfulApi.java +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/java/com/webank/wedatasphere/dss/flow/execution/entrance/restful/FlowEntranceRestfulApi.java @@ -160,16 +160,23 @@ public Message status(HttpServletRequest req, @PathVariable("id") String id, @Re logger.error("Got {} task for taskID : {}", size, realTaskID); message = Message.error("Got " + size + " task for taskId : " + realTaskID); } else { - String status = taskList.get(0).getStatus(); - message.data("status", status).data("execID", id); + WorkflowQueryTask task = taskList.get(0); + String status = task.getStatus(); + String logPath = task.getLogPath(); + message.data("status", status).data("execID", id).data("logPath",logPath); } message.setMethod("/api/entrance/" + id + "/status"); return message; } if (job.isDefined()) { + long realTaskID = Long.parseLong(taskID); + WorkflowQueryTask queryTask = new WorkflowQueryTask(); + queryTask.setTaskID(realTaskID); + List taskList = taskMapper.selectTask(queryTask); + String logPath = taskList.isEmpty() ? null : taskList.get(taskList.size() - 1).getLogPath(); message = Message.ok(); message.setMethod("/api/entrance/" + id + "/status"); - message.data("status", job.get().getState().toString()).data("execID", id); + message.data("status", job.get().getState().toString()).data("execID", id).data("logPath",logPath); } else { message = Message.error("ID The corresponding job is empty and cannot obtain the corresponding task status.(ID 对应的job为空，不能获取相应的任务状态)"); } diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/conf/FlowExecutionEntranceConfiguration.scala b/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/conf/FlowExecutionEntranceConfiguration.scala index 3df35204d9..82a65438b4 100644 --- a/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/conf/FlowExecutionEntranceConfiguration.scala +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/conf/FlowExecutionEntranceConfiguration.scala @@ -71,4 +71,12 @@ object FlowExecutionEntranceConfiguration { val FLOW_RESOURCES = "flow_resources" + val START_UP = "startup" + + val YARN_QUEUE = "wds.linkis.rm.yarnqueue" + + val CONFIGURATION = "configuration" + + val USER_TO_PROXY = "user.to.proxy" + } diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/execution/DefaultFlowExecution.scala b/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/execution/DefaultFlowExecution.scala index be502890b5..159dcb9f1f 100644 --- a/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/execution/DefaultFlowExecution.scala +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/execution/DefaultFlowExecution.scala @@ -16,13 +16,15 @@ package com.webank.wedatasphere.dss.flow.execution.entrance.execution -import java.util.concurrent.{Executors, LinkedBlockingQueue, TimeUnit} +import com.fasterxml.jackson.databind.ObjectMapper +import java.util.concurrent.{Executors, LinkedBlockingQueue, TimeUnit} import com.webank.wedatasphere.dss.flow.execution.entrance.conf.FlowExecutionEntranceConfiguration import com.webank.wedatasphere.dss.flow.execution.entrance.job.FlowEntranceJob import com.webank.wedatasphere.dss.flow.execution.entrance.node.{NodeExecutionState, NodeRunner} import com.webank.wedatasphere.dss.flow.execution.entrance.utils.FlowExecutionUtils import org.apache.linkis.common.utils.Logging +import com.google.common.util.concurrent.ThreadFactoryBuilder import org.springframework.stereotype.Service import scala.collection.JavaConversions._ @@ -35,7 +37,8 @@ class DefaultFlowExecution extends FlowExecution with Logging { private val nodeRunnerQueue: LinkedBlockingQueue[NodeRunner] = new LinkedBlockingQueue[NodeRunner]() - private val scheduledThreadPool = Executors.newScheduledThreadPool(FlowExecutionEntranceConfiguration.FLOW_EXECUTION_SCHEDULER_POOL_SIZE.getValue) + private val scheduledThreadPool = Executors.newScheduledThreadPool(FlowExecutionEntranceConfiguration.FLOW_EXECUTION_SCHEDULER_POOL_SIZE.getValue, + new ThreadFactoryBuilder().setNameFormat("Dss-Node-Execute-Status-Poll-Thread-%d").build) private var pollerCount = 0 @@ -71,6 +74,9 @@ class DefaultFlowExecution extends FlowExecution with Logging { info(s"${flowEntranceJob.getId} Submit nodes(${runningNodes.size}) to running") runningNodes.foreach { node => node.getNode.getDSSNode.getParams.get(FlowExecutionEntranceConfiguration.PROPS_MAP).asInstanceOf[java.util.Map[String, Any]].putAll(flowEntranceJob.getParams) + if (node.getNode.getDSSNode.getParams.get(FlowExecutionEntranceConfiguration.CONFIGURATION) != null && node.getNode.getDSSNode.getParams.get(FlowExecutionEntranceConfiguration.CONFIGURATION).asInstanceOf[java.util.Map[String, Any]].get(FlowExecutionEntranceConfiguration.START_UP) != null) { + node.getNode.getDSSNode.getParams.get(FlowExecutionEntranceConfiguration.CONFIGURATION).asInstanceOf[java.util.Map[String, Any]].get(FlowExecutionEntranceConfiguration.START_UP).asInstanceOf[java.util.Map[String, Any]].put(FlowExecutionEntranceConfiguration.YARN_QUEUE, null) + } node.run() nodeRunnerQueue.put(node) if (pollerCount < FlowExecutionEntranceConfiguration.NODE_STATUS_POLLER_THREAD_SIZE.getValue) { diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/job/FlowEntranceJob.scala b/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/job/FlowEntranceJob.scala index f97475fb2f..611abd3957 100644 --- a/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/job/FlowEntranceJob.scala +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/job/FlowEntranceJob.scala @@ -171,7 +171,7 @@ class FlowEntranceJob(persistManager:PersistenceManager) extends EntranceExecuti } - - + override def clear(): Unit = { + } } diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/job/parser/FlowJobNodeParser.scala b/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/job/parser/FlowJobNodeParser.scala index 5fe99b9f4d..b2695cf70c 100644 --- a/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/job/parser/FlowJobNodeParser.scala +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/job/parser/FlowJobNodeParser.scala @@ -98,11 +98,14 @@ class FlowJobNodeParser extends FlowEntranceJobParser with Logging{ run_date = new CustomDateType(getYesterday(false), false) }) } - if (!flowVar.containsKey("run_today_h")) { - val run_today = new CustomDateType(getToday(std = false, run_date + 1), false) - val run_today_h = new CustomHourType(getCurHour(std = false, run_today.toString), false) - flowVar.put("run_today_h", run_today_h.toString()) - } + val run_today = new CustomDateType(getToday(std = false, run_date + 1), false) + val cur_hour=getCurHour(std = false, run_today.toString) + val run_today_h = new CustomHourType(cur_hour, false) + flowVar.put("run_today_h", run_today_h.toString()) + flowVar.put("run_today_hour", run_today_h.toString()) + val run_today_h_std = new CustomHourType(cur_hour, true) + flowVar.put("run_today_h_std", run_today_h_std.toString()) + flowVar.put("run_today_hour_std", run_today_h_std.toString()) params.put(PROPS_MAP, propsMap) params.put(FLOW_VAR_MAP, flowVar) val flowNameAndResources = new util.HashMap[String, util.List[BMLResource]]() diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/utils/FlowExecutionUtils.scala b/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/utils/FlowExecutionUtils.scala index 00a3d65a79..f7f482d34d 100644 --- a/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/utils/FlowExecutionUtils.scala +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/utils/FlowExecutionUtils.scala @@ -40,6 +40,23 @@ import scala.collection.JavaConverters.asScalaBufferConverter object FlowExecutionUtils { def isSkippedNode(node: WorkflowNode, paramsMap: java.util.Map[String, Any]): Boolean = { + val skip = Option(node.getDSSNode) + .flatMap(node => Option(node.getParams)) + .flatMap(params => Option(params.get("configuration"))) + .flatMap { + case config: java.util.Map[_, _] => Option(config.get("special")) + case _ => None + } + .flatMap { + case special: java.util.Map[_, _] => Option(special.get("auto.disabled")) + case _ => None + } + .map(_.toString) + .exists("true".equalsIgnoreCase) + + if (skip) { + return true + } val executeStrategy = paramsMap.get("executeStrategy") if (executeStrategy != null) { return StrategyFactory.getNodeSkipStrategy(executeStrategy.toString).isSkippedNode(node, paramsMap, isReversedChoose = false) diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-linkis-node-execution/pom.xml b/dss-orchestrator/orchestrators/dss-workflow/dss-linkis-node-execution/pom.xml index d88e327d67..c17e7bc66d 100644 --- a/dss-orchestrator/orchestrators/dss-workflow/dss-linkis-node-execution/pom.xml +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-linkis-node-execution/pom.xml @@ -23,7 +23,7 @@ dss com.webank.wedatasphere.dss - 1.1.0.20-SNAPSHOT + 1.10.0-SNAPSHOT ../../../../pom.xml diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/parser/AppConn2LinkisJobParser.java b/dss-orchestrator/orchestrators/dss-workflow/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/parser/AppConn2LinkisJobParser.java index cd50eb1222..fd264d616e 100644 --- a/dss-orchestrator/orchestrators/dss-workflow/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/parser/AppConn2LinkisJobParser.java +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/parser/AppConn2LinkisJobParser.java @@ -73,6 +73,8 @@ protected Map getExecutionParams(BMLResource bmlResource, Linkis throw new LinkisJobExecutionErrorException(50063, "Failed to get workspace str, responseBody is: " + result.getResponseBody()); } + }catch(LinkisJobExecutionErrorException e){ + throw new RuntimeException(e); } finally { if(client != null) { client.close(); diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/service/impl/BuildJobActionImpl.java b/dss-orchestrator/orchestrators/dss-workflow/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/service/impl/BuildJobActionImpl.java index e4eb53887f..699ebf9ec6 100644 --- a/dss-orchestrator/orchestrators/dss-workflow/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/service/impl/BuildJobActionImpl.java +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/service/impl/BuildJobActionImpl.java @@ -25,6 +25,7 @@ import com.webank.wedatasphere.dss.linkis.node.execution.utils.LinkisJobExecutionUtils; import org.apache.commons.lang3.SerializationUtils; import org.apache.commons.lang3.StringUtils; +import org.apache.linkis.common.conf.CommonVars; import org.apache.linkis.manager.label.constant.LabelKeyConstant; import org.apache.linkis.manager.label.entity.engine.EngineTypeLabel; import org.apache.linkis.manager.label.utils.EngineTypeLabelCreator; @@ -48,6 +49,9 @@ public class BuildJobActionImpl implements BuildJobAction { private Logger logger = LoggerFactory.getLogger(BuildJobActionImpl.class); private static BuildJobAction buildJobAction = new BuildJobActionImpl(); + private static final String NEBULA = "nebula"; + private static final CommonVars NEBULA_ENGINE_VERSION = + CommonVars.apply("wds.linkis.nebula.engine.version", "3.0.0"); private BuildJobActionImpl() { @@ -132,6 +136,10 @@ public JobSubmitAction getSubmitAction(Job job) throws LinkisJobExecutionErrorEx String code = parseExecutionCodeFor1X(job); EngineTypeLabel engineTypeLabel = EngineTypeLabelCreator.createEngineTypeLabel(parseAppConnEngineType(job.getEngineType(), job)); + //TODO 升级linkis1.7.0版本之后，这段特殊的硬编码逻辑要去掉 + if(NEBULA.equalsIgnoreCase( engineTypeLabel.getEngineType())){ + engineTypeLabel.setVersion(NEBULA_ENGINE_VERSION.getValue()); + } labels.put(LabelKeyConstant.ENGINE_TYPE_KEY, engineTypeLabel.getStringValue()); labels.put(LabelKeyConstant.USER_CREATOR_TYPE_KEY, job.getUser() + "-" + LINKIS_JOB_CREATOR_1_X.getValue(job.getJobProps())); @@ -139,7 +147,7 @@ public JobSubmitAction getSubmitAction(Job job) throws LinkisJobExecutionErrorEx //是否复用引擎，不复用就为空 - if(!isReuseEngine(job.getParams())){ + if(!isAppconnJob(job) && !isReuseEngine(job.getParams())){ labels.put("executeOnce", ""); } Map paramMapCopy = (HashMap) SerializationUtils.clone(new HashMap(job.getParams())); @@ -189,6 +197,13 @@ public boolean isReuseEngine(Map params) { return true; } + /** + * 是否为appconnjob + */ + private boolean isAppconnJob(Job job){ + return APPCONN.equals(job.getEngineType()); + } + /** * spark自定义参数配置输入，例如spark.sql.shuffle.partitions=10。多个参数使用分号分隔。 * @@ -210,6 +225,8 @@ private void replaceSparkConfParams(Map paramMapCopy) throws Lin startupMap.remove("spark.executor.instances"); startupMap.remove("wds.linkis.engineconn.java.driver.memory"); startupMap.remove("spark.conf"); + startupMap.remove("mapreduce.job.running.map.limit"); + startupMap.remove("mapreduce.job.running.reduce.limit"); logger.info("after remove startup map:{}",startupMap.keySet()); } Map configurationMap = TaskUtils.getMap(paramMapCopy, TaskConstant.PARAMS_CONFIGURATION); diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-common/pom.xml b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-common/pom.xml index e749913122..bbd2952b9c 100644 --- a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-common/pom.xml +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-common/pom.xml @@ -21,7 +21,7 @@ dss com.webank.wedatasphere.dss - 1.1.0.20-SNAPSHOT + 1.10.0-SNAPSHOT ../../../../pom.xml 4.0.0 diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-common/src/main/java/com/webank/wedatasphere/dss/workflow/common/entity/DSSFlow.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-common/src/main/java/com/webank/wedatasphere/dss/workflow/common/entity/DSSFlow.java index d2a3701cd6..5f1e619967 100644 --- a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-common/src/main/java/com/webank/wedatasphere/dss/workflow/common/entity/DSSFlow.java +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-common/src/main/java/com/webank/wedatasphere/dss/workflow/common/entity/DSSFlow.java @@ -53,6 +53,10 @@ public class DSSFlow implements Flow { */ private List flowIdParamConfTemplateIdTuples; + private Long orchestratorId; + + private String defaultProxyUser; + public List getFlowIdParamConfTemplateIdTuples() { return flowIdParamConfTemplateIdTuples; } @@ -275,4 +279,20 @@ public String toString() { ", flowJson='" + flowJson + '\'' + '}'; } + + public Long getOrchestratorId() { + return orchestratorId; + } + + public void setOrchestratorId(Long orchestratorId) { + this.orchestratorId = orchestratorId; + } + + public String getDefaultProxyUser() { + return defaultProxyUser; + } + + public void setDefaultProxyUser(String defaultProxyUser) { + this.defaultProxyUser = defaultProxyUser; + } } diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-common/src/main/java/com/webank/wedatasphere/dss/workflow/common/parser/NodeParser.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-common/src/main/java/com/webank/wedatasphere/dss/workflow/common/parser/NodeParser.java index 53721e0406..3745b98395 100644 --- a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-common/src/main/java/com/webank/wedatasphere/dss/workflow/common/parser/NodeParser.java +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-common/src/main/java/com/webank/wedatasphere/dss/workflow/common/parser/NodeParser.java @@ -30,4 +30,6 @@ public interface NodeParser { String updateSubFlowID(String nodeJson, long subflowId) throws IOException; String getNodeValue(String key, String nodeJson) throws IOException; List getNodeResource(String nodeJson); + + Map getNodeJobContent(String nodeJson) throws IOException; } diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-common/src/main/java/com/webank/wedatasphere/dss/workflow/common/protocol/RequestImportWorkflow.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-common/src/main/java/com/webank/wedatasphere/dss/workflow/common/protocol/RequestImportWorkflow.java index 5c55288752..9a6acfa63b 100644 --- a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-common/src/main/java/com/webank/wedatasphere/dss/workflow/common/protocol/RequestImportWorkflow.java +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-common/src/main/java/com/webank/wedatasphere/dss/workflow/common/protocol/RequestImportWorkflow.java @@ -31,6 +31,10 @@ public class RequestImportWorkflow { private Workspace workspace; private String contextId; private List dssLabels; + /** + * 是否迁移dws的请求,默认不是 + */ + private boolean oldPackageStruct = false; public String getContextId() { return contextId; @@ -106,6 +110,14 @@ public void setDssLabels(List dssLabels) { this.dssLabels = dssLabels; } + public boolean getOldPackageStruct() { + return oldPackageStruct; + } + + public void setOldPackageStruct(boolean oldPackageStruct) { + this.oldPackageStruct = oldPackageStruct; + } + public RequestImportWorkflow(String userName, String resourceId, String bmlVersion, diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-common/src/main/java/com/webank/wedatasphere/dss/workflow/common/protocol/RequestLockWorkflow.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-common/src/main/java/com/webank/wedatasphere/dss/workflow/common/protocol/RequestLockWorkflow.java new file mode 100644 index 0000000000..e581e2b42d --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-common/src/main/java/com/webank/wedatasphere/dss/workflow/common/protocol/RequestLockWorkflow.java @@ -0,0 +1,43 @@ +package com.webank.wedatasphere.dss.workflow.common.protocol; + + +import javax.servlet.http.Cookie; + +public class RequestLockWorkflow { + public RequestLockWorkflow() { + } + + private String username; + private Cookie[] cookies; + private Long flowId; + + public RequestLockWorkflow(String username, Cookie[] cookies, Long flowId) { + this.username = username; + this.cookies = cookies; + this.flowId = flowId; + } + + public String getUsername() { + return username; + } + + public void setUsername(String username) { + this.username = username; + } + + public Cookie[] getCookies() { + return cookies; + } + + public void setCookies(Cookie[] cookies) { + this.cookies = cookies; + } + + public Long getFlowId() { + return flowId; + } + + public void setFlowId(Long flowId) { + this.flowId = flowId; + } +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-common/src/main/java/com/webank/wedatasphere/dss/workflow/common/protocol/RequestUnlockWorkflow.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-common/src/main/java/com/webank/wedatasphere/dss/workflow/common/protocol/RequestUnlockWorkflow.java index 492eb467bc..bae910f395 100644 --- a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-common/src/main/java/com/webank/wedatasphere/dss/workflow/common/protocol/RequestUnlockWorkflow.java +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-common/src/main/java/com/webank/wedatasphere/dss/workflow/common/protocol/RequestUnlockWorkflow.java @@ -1,16 +1,27 @@ package com.webank.wedatasphere.dss.workflow.common.protocol; +import com.webank.wedatasphere.dss.standard.app.sso.Workspace; + public class RequestUnlockWorkflow { private String username; private Long flowId; private Boolean confirmDelete; + private Workspace workspace; + public RequestUnlockWorkflow(String username, Long flowId, boolean confirmDelete) { this.username = username; this.flowId = flowId; this.confirmDelete = confirmDelete; } + public RequestUnlockWorkflow(String username, Long flowId, Boolean confirmDelete, Workspace workspace) { + this.username = username; + this.flowId = flowId; + this.confirmDelete = confirmDelete; + this.workspace = workspace; + } + public String getUsername() { return username; } @@ -34,4 +45,12 @@ public Boolean getConfirmDelete() { public void setConfirmDelete(Boolean confirmDelete) { this.confirmDelete = confirmDelete; } + + public Workspace getWorkspace() { + return workspace; + } + + public void setWorkspace(Workspace workspace) { + this.workspace = workspace; + } } diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-common/src/main/java/com/webank/wedatasphere/dss/workflow/common/protocol/ResponseLockWorkflow.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-common/src/main/java/com/webank/wedatasphere/dss/workflow/common/protocol/ResponseLockWorkflow.java new file mode 100644 index 0000000000..6b951aa174 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-common/src/main/java/com/webank/wedatasphere/dss/workflow/common/protocol/ResponseLockWorkflow.java @@ -0,0 +1,34 @@ +package com.webank.wedatasphere.dss.workflow.common.protocol; + + +public class ResponseLockWorkflow { + public static final int LOCK_SUCCESS = 0; + public static final int LOCK_FAILED = 1; + + private int unlockStatus; + private String lockOwner; + + public ResponseLockWorkflow(int unlockStatus, String lockOwner) { + this.unlockStatus = unlockStatus; + this.lockOwner = lockOwner; + } + + public ResponseLockWorkflow() { + } + + public int getUnlockStatus() { + return unlockStatus; + } + + public void setUnlockStatus(int unlockStatus) { + this.unlockStatus = unlockStatus; + } + + public String getLockOwner() { + return lockOwner; + } + + public void setLockOwner(String lockOwner) { + this.lockOwner = lockOwner; + } +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-conversion-standard/pom.xml b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-conversion-standard/pom.xml index fb2f99ce6c..41499e4e03 100644 --- a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-conversion-standard/pom.xml +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-conversion-standard/pom.xml @@ -21,7 +21,7 @@ dss-workflow com.webank.wedatasphere.dss - 1.1.0.20-SNAPSHOT + 1.10.0-SNAPSHOT ../pom.xml 4.0.0 diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-sdk/pom.xml b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-sdk/pom.xml index 7bf055d48b..ab9a9361b5 100644 --- a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-sdk/pom.xml +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-sdk/pom.xml @@ -21,7 +21,7 @@ dss-workflow com.webank.wedatasphere.dss - 1.1.0.20-SNAPSHOT + 1.10.0-SNAPSHOT ../pom.xml 4.0.0 diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-sdk/src/main/java/com/webank/wedatasphere/dss/workflow/core/json2flow/parser/WorkflowPropsParser.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-sdk/src/main/java/com/webank/wedatasphere/dss/workflow/core/json2flow/parser/WorkflowPropsParser.java index 5c47eeec99..5f2e6c6f4d 100644 --- a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-sdk/src/main/java/com/webank/wedatasphere/dss/workflow/core/json2flow/parser/WorkflowPropsParser.java +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-sdk/src/main/java/com/webank/wedatasphere/dss/workflow/core/json2flow/parser/WorkflowPropsParser.java @@ -23,6 +23,8 @@ import com.webank.wedatasphere.dss.common.utils.DSSCommonUtils; import com.webank.wedatasphere.dss.workflow.core.entity.Workflow; import com.webank.wedatasphere.dss.workflow.core.entity.WorkflowImpl; + +import java.util.ArrayList; import java.util.List; import java.util.Map; diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/pom.xml b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/pom.xml index fd970e1d41..1dfd79a086 100644 --- a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/pom.xml +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/pom.xml @@ -21,13 +21,25 @@ dss com.webank.wedatasphere.dss - 1.1.0.20-SNAPSHOT + 1.10.0-SNAPSHOT ../../../../pom.xml 4.0.0 dss-workflow-server + + com.webank.wedatasphere.dss + dss-git-common + ${dss.version} + provided + + + com.webank.wedatasphere.dss + dss-orchestrator-common + ${dss.version} + provided + com.webank.wedatasphere.dss dss-common @@ -148,6 +160,12 @@ provided + + cn.hutool + hutool-all + 5.8.21 + + diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/DefaultWorkFlowManager.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/DefaultWorkFlowManager.java index 29b6490695..b2480d0cf0 100644 --- a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/DefaultWorkFlowManager.java +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/DefaultWorkFlowManager.java @@ -43,10 +43,9 @@ import com.webank.wedatasphere.dss.workflow.common.entity.DSSFlow; import com.webank.wedatasphere.dss.workflow.common.entity.DSSFlowRelation; import com.webank.wedatasphere.dss.workflow.common.parser.WorkFlowParser; -import com.webank.wedatasphere.dss.workflow.common.protocol.RequestSubFlowContextIds; -import com.webank.wedatasphere.dss.workflow.common.protocol.ResponseSubFlowContextIds; -import com.webank.wedatasphere.dss.workflow.common.protocol.ResponseUnlockWorkflow; +import com.webank.wedatasphere.dss.workflow.common.protocol.*; import com.webank.wedatasphere.dss.workflow.constant.DSSWorkFlowConstant; +import com.webank.wedatasphere.dss.workflow.dao.FlowMapper; import com.webank.wedatasphere.dss.workflow.dao.LockMapper; import com.webank.wedatasphere.dss.workflow.entity.DSSFlowEditLock; import com.webank.wedatasphere.dss.workflow.entity.DSSFlowImportParam; @@ -57,6 +56,7 @@ import com.webank.wedatasphere.dss.common.service.BMLService; import com.webank.wedatasphere.dss.workflow.service.DSSFlowService; import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.io.IOUtils; import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.exception.ExceptionUtils; import org.apache.linkis.common.exception.ErrorException; @@ -68,15 +68,19 @@ import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Component; +import javax.servlet.http.Cookie; +import java.io.File; import java.io.IOException; import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; import java.util.*; -import java.util.regex.Matcher; -import java.util.regex.Pattern; import java.util.stream.Collectors; import java.util.zip.ZipEntry; import java.util.zip.ZipFile; +import static com.webank.wedatasphere.dss.common.utils.IoUtils.FLOW_META_DIRECTORY_NAME; import static com.webank.wedatasphere.dss.workflow.constant.DSSWorkFlowConstant.DEFAULT_SCHEDULER_APP_CONN; @Component @@ -102,6 +106,8 @@ public class DefaultWorkFlowManager implements WorkFlowManager { private WorkFlowParser workFlowParser; @Autowired private LockMapper lockMapper; + @Autowired + private FlowMapper flowMapper; @Override public DSSFlow createWorkflow(String userName, @@ -190,17 +196,81 @@ public void deleteWorkflow(String userName, Long flowId) throws DSSErrorExceptio } @Override - public ResponseUnlockWorkflow unlockWorkflow(String userName, Long flowId, Boolean confirmDelete) throws DSSErrorException { + public ResponseUnlockWorkflow unlockWorkflow(String userName, Long flowId, Boolean confirmDelete, Workspace workspace) throws DSSErrorException { DSSFlowEditLock editLock = lockMapper.getFlowEditLockByID(flowId); if (editLock == null) { return new ResponseUnlockWorkflow(ResponseUnlockWorkflow.NONEED_UNLOCK, null); } else if (!Boolean.TRUE.equals(confirmDelete)) { return new ResponseUnlockWorkflow(ResponseUnlockWorkflow.NEED_SECOND_CONFIRM, editLock.getUsername()); } - DSSFlowEditLockManager.deleteLock(editLock.getLockContent()); + DSSFlowEditLockManager.deleteLock(editLock.getLockContent(), workspace); return new ResponseUnlockWorkflow(ResponseUnlockWorkflow.UNLOCK_SUCCESS, null); } + @Override + public BmlResource exportWorkflowNew(String userName, Long flowId, Long dssProjectId, + String projectName, Workspace workspace, + List dssLabels,boolean exportExternalNodeAppConnResource) throws Exception { + DSSFlow dssFlow = flowService.getFlowByID(flowId); + String projectPath = workFlowExportService.exportFlowInfoNew(dssProjectId, projectName, flowId, userName, workspace, dssLabels,exportExternalNodeAppConnResource); + String exportPath = ZipHelper.zip(projectPath); + InputStream inputStream = bmlService.readLocalResourceFile(userName, exportPath); + BmlResource bmlResource = bmlService.upload(userName, inputStream, dssFlow.getName() + ".export", projectName); + logger.info("export workflow success. flowId:{},bmlResource:{} .",flowId,bmlResource); + return bmlResource; + } + @Override + public BmlResource exportWorkflowListNew(String userName, List flowIdList, Long dssProjectId, + String projectName, Workspace workspace, + List dssLabels,boolean exportExternalNodeAppConnResource) throws Exception { + List dssFlowList = flowMapper.selectFlowListByID(flowIdList); + String projectPath = workFlowExportService.exportFlowListNew(dssProjectId, projectName, flowIdList, userName, workspace, dssLabels,exportExternalNodeAppConnResource); + String exportPath = ZipHelper.zip(projectPath); + File file = new File(exportPath); + if (file.exists() && file.isFile()) { + String limitValue = DSSWorkFlowConstant.DEFAULT_ZIP_FILE_LIMIT.getValue(); + // GB + Long limits = Long.parseLong(limitValue) * 1024 *1024 * 1024; + if (file.length() > limits) { + throw new DSSErrorException(100098, "工作流导出失败，原因为本次导出总大小超过" + limitValue + "GB"); + } + } + InputStream inputStream = bmlService.readLocalResourceFile(userName, exportPath); + BmlResource bmlResource = bmlService.upload(userName, inputStream, projectName + ".export", projectName); + logger.info("export workflow success. flowId:{},bmlResource:{} .",flowIdList,bmlResource); + return bmlResource; + } + @Override + public String readWorkflowNew(String userName, Long flowId, Long dssProjectId, + String projectName, Workspace workspace, + List dssLabels,boolean exportExternalNodeAppConnResource, String filePath) throws Exception { + DSSFlow dssFlow = flowService.getFlowByID(flowId); + String projectPath = null; + String fileContent = null; + try { + projectPath = workFlowExportService.exportFlowInfoNew(dssProjectId, projectName, flowId, userName, workspace, dssLabels,exportExternalNodeAppConnResource); + String fullPath = projectPath + File.separator + filePath; + logger.info("export workflow success. flowId:{},fullPath:{} .",flowId,fullPath); + File file = new File(fullPath); + if (file.exists()) { + fileContent = new String(Files.readAllBytes(Paths.get(fullPath))); + } + } catch (Exception e) { + logger.error("exportFlowInfoNew failed , the reason is:", e); + throw new DSSErrorException(100098, "工作流导出失败，原因为" + e.getMessage()); + } finally { + //删掉整个目录 + if (StringUtils.isNotEmpty(projectPath)) { + File file = new File(projectPath); + if (ZipHelper.deleteDir(file)){ + logger.info("结束删除目录 {} 成功", projectPath); + }else{ + logger.info("删除目录 {} 失败", projectPath); + } + } + } + return fileContent; + } @Override public BmlResource exportWorkflow(String userName, Long flowId, Long dssProjectId, String projectName, Workspace workspace, @@ -212,6 +282,59 @@ public BmlResource exportWorkflow(String userName, Long flowId, Long dssProjectI logger.info("export workflow success. flowId:{},bmlResource:{} .",flowId,bmlResource); return bmlResource; } + @Override + public List importWorkflowNew(String userName, + String resourceId, + String bmlVersion, + DSSFlowImportParam dssFlowImportParam, + List dssLabels) throws DSSErrorException, IOException { + + //todo download workflow bml file contains flowInfo and flowRelationInfo + String projectName = dssFlowImportParam.getProjectName(); + // /appcom/tmp/dss/yyyyMMddHHmmssSSS/arionliu + String tempPath = IoUtils.generateTempIOPath(userName); + // /appcom/tmp/dss/yyyyMMddHHmmssSSS/arionliu/projectxxx.zip + String inputZipPath = IoUtils.addFileSeparator(tempPath, projectName + ".zip"); + bmlService.downloadToLocalPath(userName, resourceId, bmlVersion, inputZipPath); + try{ + String originProjectName=readImportZipProjectName(inputZipPath); + if(!projectName.equals(originProjectName)){ + String msg=String.format("target project name must be same with origin project name.origin project name:%s,target project name:%s(导入的目标工程名必须与导出时源工程名保持一致。源工程名：%s，目标工程名：%s)" + ,originProjectName,projectName,originProjectName,projectName); + throw new DSSRuntimeException(msg); + } + }catch (IOException e){ + throw new DSSRuntimeException("upload file format error(导入包格式错误)"); + } + String projectPath = ZipHelper.unzip(inputZipPath,true); + String flowName = IoUtils.getSubdirectoriesNames(projectPath).stream().filter(name -> !name.startsWith(".")) + .findFirst().orElseThrow(() -> new DSSRuntimeException("import package has no flow（未导入任何工作流，请检查导入包格式）")); + String flowMetaPath=IoUtils.addFileSeparator(projectPath, FLOW_META_DIRECTORY_NAME, flowName); + ImmutablePair,List> meta= metaInputService.inputFlowNew(flowMetaPath); + //导入工作流数据库信息 + List dssFlows = meta.getKey(); + //导入工作流关系信息 + List dwsFlowRelations = meta.getValue(); + + List dwsFlowRelationList = workFlowInputService.persistenceFlow(dssFlowImportParam.getProjectID(), + dssFlowImportParam.getUserName(), + dssFlows, + dwsFlowRelations); + //这里其实只会有1个元素 + List rootFlows = dwsFlowRelationList.stream().filter(DSSFlow::getRootFlow).collect(Collectors.toList()); + for (DSSFlow rootFlow : rootFlows) { + String flowCodePath0=IoUtils.addFileSeparator(projectPath, flowName); + String flowMetaPath0=IoUtils.addFileSeparator(projectPath, FLOW_META_DIRECTORY_NAME, flowName); + workFlowInputService.inputWorkFlowNew(dssFlowImportParam.getUserName(), + rootFlow, + projectName, + flowCodePath0, + flowMetaPath0,null, dssFlowImportParam.getWorkspace(), dssFlowImportParam.getOrcVersion(), + dssFlowImportParam.getContextId(), dssLabels); + } + logger.info("import workflow success.orcVersion:{},context Id:{}", dssFlowImportParam.getOrcVersion(), dssFlowImportParam.getContextId()); + return rootFlows; + } @Override public List importWorkflow(String userName, @@ -358,11 +481,11 @@ private ResponseOperateOrchestrator convert(RequestConvertOrchestrations request } private String readImportZipProjectName(String zipFilePath) throws IOException { - try(ZipFile zipFile =new ZipFile(zipFilePath)){ - Enumeration entries =zipFile.entries(); - if(entries.hasMoreElements()){ - String name=entries.nextElement().getName(); - if(name.endsWith("\\")||name.endsWith("/")){ + try (ZipFile zipFile = new ZipFile(zipFilePath)) { + Enumeration entries = zipFile.entries(); + if (entries.hasMoreElements()) { + String name = entries.nextElement().getName(); + if (name.endsWith("\\") || name.endsWith("/")) { name = name.substring(0, name.length() - 1); } return name; diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/WorkFlowManager.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/WorkFlowManager.java index 281e3405ab..c998059abb 100644 --- a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/WorkFlowManager.java +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/WorkFlowManager.java @@ -25,15 +25,12 @@ import com.webank.wedatasphere.dss.orchestrator.common.protocol.ResponseOperateOrchestrator; import com.webank.wedatasphere.dss.standard.app.sso.Workspace; import com.webank.wedatasphere.dss.workflow.common.entity.DSSFlow; -import com.webank.wedatasphere.dss.workflow.common.protocol.RequestSubFlowContextIds; -import com.webank.wedatasphere.dss.workflow.common.protocol.ResponseSubFlowContextIds; -import com.webank.wedatasphere.dss.workflow.common.protocol.ResponseUnlockWorkflow; +import com.webank.wedatasphere.dss.workflow.common.protocol.*; import com.webank.wedatasphere.dss.workflow.entity.DSSFlowImportParam; import org.apache.linkis.common.exception.ErrorException; import java.io.IOException; import java.util.List; -import java.util.Map; public interface WorkFlowManager { @@ -95,8 +92,47 @@ void updateWorkflow(String userName, void deleteWorkflow(String userName, Long flowID) throws DSSErrorException; - ResponseUnlockWorkflow unlockWorkflow(String userName, Long flowId, Boolean confirmDelete) throws DSSErrorException; + ResponseUnlockWorkflow unlockWorkflow(String userName, Long flowId, Boolean confirmDelete, Workspace workspace) throws DSSErrorException; + /** + * 导出工作流 + * @param userName + * @param flowID + * @param dssProjectId + * @param projectName + * @param workspace + * @param dssLabels + * @param exportExternalNodeAppConnResource 是否导出第三方节点的物料 + * @return 导出的工作流，以Bml资源的形式返回 + * @throws Exception + */ + BmlResource exportWorkflowNew(String userName, + Long flowID, + Long dssProjectId, + String projectName, + Workspace workspace, + List dssLabels, + boolean exportExternalNodeAppConnResource) throws Exception; + + BmlResource exportWorkflowListNew(String userName, List flowIdList, Long dssProjectId, + String projectName, Workspace workspace, + List dssLabels,boolean exportExternalNodeAppConnResource) throws Exception; + /** + * 读取当前最新BML存储的指定文件内容，若文件不存在，则返回空字符串 + * @param userName + * @param flowId + * @param dssProjectId + * @param projectName + * @param workspace + * @param dssLabels + * @param exportExternalNodeAppConnResource + * @param filePath：文件路径 + * @return + * @throws Exception + */ + String readWorkflowNew(String userName, Long flowId, Long dssProjectId, + String projectName, Workspace workspace, + List dssLabels,boolean exportExternalNodeAppConnResource,String filePath) throws Exception; /** * 导出工作流 * @param userName @@ -115,6 +151,11 @@ BmlResource exportWorkflow(String userName, Workspace workspace, List dssLabels) throws Exception; + List importWorkflowNew(String userName, + String resourceId, + String bmlVersion, + DSSFlowImportParam dssFlowImportParam, + List dssLabels) throws Exception; List importWorkflow(String userName, String resourceId, String bmlVersion, @@ -139,4 +180,5 @@ List importWorkflow(String userName, */ ResponseSubFlowContextIds getSubFlowContextIdsByFlowIds(RequestSubFlowContextIds requestSubFlowContextIds) throws ErrorException; + } diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/constant/DSSWorkFlowConstant.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/constant/DSSWorkFlowConstant.java index d85658d7e6..8e5308f794 100644 --- a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/constant/DSSWorkFlowConstant.java +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/constant/DSSWorkFlowConstant.java @@ -60,4 +60,9 @@ public class DSSWorkFlowConstant { * 仅仅用于兼容老的、已经创建的工作量，用于自动路由到一个默认的调度系统。 */ public static final CommonVars DEFAULT_SCHEDULER_APP_CONN = CommonVars.apply("wds.dss.workflow.scheduler.default", "schedulis"); + /** + * 压缩文件大小限制，默认5GB + */ + public static final CommonVars DEFAULT_ZIP_FILE_LIMIT = CommonVars.apply("wds.dss.workflow.export.default.zip.limit", "5"); + } diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/constant/SignalNodeConstant.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/constant/SignalNodeConstant.java new file mode 100644 index 0000000000..00f281c433 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/constant/SignalNodeConstant.java @@ -0,0 +1,14 @@ +package com.webank.wedatasphere.dss.workflow.constant; + +public class SignalNodeConstant { + + public static final String dataCheckerNode = "datachecker"; + + public static final String eventSenderNode = "eventsender"; + + public static final String eventReceiverNode = "eventreceiver"; + + + + +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/constant/WorkflowNodeGroupEnum.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/constant/WorkflowNodeGroupEnum.java new file mode 100644 index 0000000000..1d4fe9ccd1 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/constant/WorkflowNodeGroupEnum.java @@ -0,0 +1,33 @@ +package com.webank.wedatasphere.dss.workflow.constant; + +public enum WorkflowNodeGroupEnum { + DataDevelopment("数据开发","Data development"), + DataVisualization("数据可视化","Data visualization"), + SignalNode("信号节点","Signal node"); + + private String name; + private String nameEn; + + + WorkflowNodeGroupEnum(String name, String nameEn) { + this.name = name; + this.nameEn = nameEn; + } + + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getNameEn() { + return nameEn; + } + + public void setNameEn(String nameEn) { + this.nameEn = nameEn; + } +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/dao/FlowMapper.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/dao/FlowMapper.java index c10f8468b7..22101d78f0 100644 --- a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/dao/FlowMapper.java +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/dao/FlowMapper.java @@ -30,6 +30,8 @@ public interface FlowMapper { DSSFlow selectFlowByID(Long id); + List selectFlowListByID(List list); + void insertFlow(DSSFlow dssFlow) throws DuplicateKeyException; void insertFlowRelation(@Param("flowID") Long flowID, @Param("parentFlowID") Long parentFlowID); @@ -59,4 +61,6 @@ public interface FlowMapper { @Select("select creator from dss_flow where id = #{flowId}") String getCreatorById(@Param("flowId") Long flowId); List getSubflowName(Long parentFlowID); + + List selectFlowListByOrchestratorId(@Param("orchestratorIdList") List orchestratorIdList); } diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/dao/LockMapper.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/dao/LockMapper.java index c2f8ab5e39..d577202dc8 100644 --- a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/dao/LockMapper.java +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/dao/LockMapper.java @@ -19,6 +19,8 @@ import org.apache.ibatis.annotations.Param; import org.springframework.dao.DuplicateKeyException; +import java.util.List; + public interface LockMapper { @@ -48,4 +50,22 @@ Boolean flowNotExistEditLock(@Param("flowID") Long flowID, void clearExpire(@Param("expireTime") String expireTime, @Param("flowId") long flowId); void deleteALL(); + + + void deleteExpectAfterSave(@Param("list") List list); + + + void insertFlowStatus(@Param("flowID") Long flowID, @Param("status") String status); + + void updateOrchestratorStatus(@Param("id") Long id, @Param("status") String status); + + void batchUpdateOrchestratorStatus(@Param("list") List list, @Param("status") String status); + + List selectOrchestratorByStatus( @Param("status") String status); + + String selectOrchestratorStatus(@Param("id") Long id); + + void updateOrchestratorVersionCommitId(@Param("commitId")String commitId, @Param("flowID")Long flowID); + + void batchUpdateOrchestratorVersionCommitId(@Param("commitId")String commitId, @Param("list")List list); } diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/dao/NodeContentMapper.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/dao/NodeContentMapper.java new file mode 100644 index 0000000000..9a25d8ca5e --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/dao/NodeContentMapper.java @@ -0,0 +1,66 @@ +package com.webank.wedatasphere.dss.workflow.dao; + +import com.webank.wedatasphere.dss.workflow.dto.NodeContentDO; + +import com.webank.wedatasphere.dss.workflow.entity.DSSFlowNodeInfo; +import com.webank.wedatasphere.dss.workflow.entity.DSSFlowNodeInfoOfFlow; +import com.webank.wedatasphere.dss.workflow.entity.DSSFlowNodeTemplate; +import org.apache.ibatis.annotations.Param; +import org.apache.ibatis.annotations.Select; +import org.mapstruct.Mapper; + +import java.util.List; + +@Mapper +public interface NodeContentMapper { + void insert(NodeContentDO nodeContentDO); + + void batchInsert(List list); + + void update(NodeContentDO nodeContentDO); + + void updateByKey(NodeContentDO nodeContentDO); + + NodeContentDO getNodeContentByKey(@Param("nodeKey") String nodeKey); + + NodeContentDO getNodeContentByContentId(@Param("id") Long id); + + List getNodeContentByKeyList(@Param("list") List list, @Param("orchestratorId") Long orchestratorId, @Param("flowId") Long flowId); + + List getNodeContentListByOrchestratorId(@Param("orchestratorId") Long orchestratorId); + + List getContentListByOrchestratorIdAndFlowId(@Param("orchestratorId") Long orchestratorId, @Param("flowId") Long flowId); + + List getContentListByFlowId(List list); + + void deleteNodeContentByKey(@Param("nodeKey") String nodeKey, @Param("orchestratorId") Long orchestratorId, @Param("flowId") Long flowId); + + void batchDelete(@Param("list") List list, @Param("orchestratorId") Long orchestratorId, @Param("flowId") Long flowId); + + void deleteNodeContentByOrchestratorIdAndFlowId(@Param("orchestratorId") Long orchestratorId, @Param("flowId") Long flowId); + + void deleteNodeContentByOrchestratorId(@Param("orchestratorId") Long orchestratorId); + + void updateFlowId(@Param("flowId") Long flowId, @Param("oldFlowId") Long oldFlowId); + + void deleteNodeContentByFlowId(List list); + + List queryFlowNodeInfo(@Param("projectIdList") List projectIdList,@Param("nodeTypeList") List nodeTypeList); + + List queryFlowNodeTemplate(@Param("orchestratorIdList") List orchestratorIdList); + + NodeContentDO getNodeContentById(@Param("contentId") Long contentId,@Param("nodeId") String nodeId); + @Select( + "SELECT t2.flow_id AS appId,t2.orchestrator_id AS orchestratorId " + + "FROM dss_orchestrator_info t1 " + + "INNER JOIN dss_workflow_node_content t2 ON t1.id = t2.orchestrator_id " + + "INNER JOIN dss_workflow_node_content_to_ui t3 ON t2.id = t3.content_id " + + "WHERE t1.project_id = #{projectId} " + + "AND t1.name = #{orchestratorName} " + + "AND t3.node_ui_key = 'title' " + + "AND t3.node_ui_value = #{nodeName} LIMIT 1" + ) + DSSFlowNodeInfoOfFlow getNodeInfoOfFLow(@Param("projectId") Long projectId, + @Param("orchestratorName")String orchestratorName, + @Param("nodeName")String nodeName); +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/dao/NodeContentUIMapper.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/dao/NodeContentUIMapper.java new file mode 100644 index 0000000000..df6c1bf1ae --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/dao/NodeContentUIMapper.java @@ -0,0 +1,31 @@ +package com.webank.wedatasphere.dss.workflow.dao; + + +import com.webank.wedatasphere.dss.workflow.dto.NodeContentUIDO; + +import org.apache.ibatis.annotations.Param; +import org.mapstruct.Mapper; + +import java.util.List; +import java.util.Set; + +@Mapper +public interface NodeContentUIMapper { + void insertNodeContentUI(NodeContentUIDO contentUIDO); + + void batchInsertNodeContentUI(List list); + + void updateNodeContentUI(NodeContentUIDO contentUIDO); + + NodeContentUIDO getNodeContentUI(@Param("contentId") Long contentId); + + void batchUpdateNodeContentUI(NodeContentUIDO contentUIDO); + + void deleteNodeContentUIByContentList(@Param("list") List list); + + List queryNodeContentUIList(@Param("contentIdList") List contentIdList); + + List getNodeContentUIByContentId(@Param("contentId") Long contentId); + + List getNodeContentUIByNodeUIKey(@Param("contentIdList") List contentIdList,@Param("nodeUIKey")String nodeKey); +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/dao/NodeInfoMapper.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/dao/NodeInfoMapper.java index 3a950b7a4b..55507ef2d7 100644 --- a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/dao/NodeInfoMapper.java +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/dao/NodeInfoMapper.java @@ -17,9 +17,15 @@ package com.webank.wedatasphere.dss.workflow.dao; +import com.webank.wedatasphere.dss.workflow.dto.NodeTypeDO; import com.webank.wedatasphere.dss.workflow.entity.NodeGroup; import com.webank.wedatasphere.dss.workflow.entity.NodeInfo; +import com.webank.wedatasphere.dss.workflow.entity.NodeUIInfo; +import org.apache.ibatis.annotations.MapKey; +import org.apache.ibatis.annotations.Param; + import java.util.List; +import java.util.Map; public interface NodeInfoMapper { @@ -28,5 +34,14 @@ public interface NodeInfoMapper { NodeInfo getWorkflowNodeByType(String nodeType); + List getWorkflowNodeNumberType(); + + List queryNodeUIInfoList(@Param("nodeTypeList") List nodeTypeList); + + List getNodeUIInfoByNodeType(@Param("nodeType") String nodeType); + + List getNodeTypeByGroupName(@Param("groupName") String groupName,@Param("nodeTypeName") String nodeTypeName); + + List getWorkflowNodeList(); } diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/dao/NodeMetaMapper.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/dao/NodeMetaMapper.java new file mode 100644 index 0000000000..10e8e345c2 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/dao/NodeMetaMapper.java @@ -0,0 +1,19 @@ +package com.webank.wedatasphere.dss.workflow.dao; + +import com.webank.wedatasphere.dss.workflow.dto.NodeMetaDO; + +import org.apache.ibatis.annotations.Param; +import org.mapstruct.Mapper; + + + +@Mapper +public interface NodeMetaMapper { + void insertNodeMeta(NodeMetaDO nodeMetaDO); + + void updateNodeMeta(NodeMetaDO nodeMetaDO); + + NodeMetaDO getNodeMetaByOrchestratorId(@Param("orchestratorId") Long orchestratorId); + + void deleteNodeMetaByOrchestratorId(@Param ("orchestratorId") Long orchestratorId); +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/dao/impl/NodeContentMapper.xml b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/dao/impl/NodeContentMapper.xml new file mode 100644 index 0000000000..750557ceb4 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/dao/impl/NodeContentMapper.xml @@ -0,0 +1,187 @@ + + + + + + + + + + + + + + + + + + + + + + `id`, `node_key`, `node_id`, `job_type`, `orchestrator_id`, `flow_id`, `create_time`, `modify_time`, `modify_user` + + + + INSERT INTO dss_workflow_node_content(`node_key`, `node_id`, `job_type`, `orchestrator_id`, `flow_id`, `create_time`, `modify_time`, `modify_user`) + VAlUES(#{nodeKey}, #{nodeId}, #{jobType}, #{orchestratorId}, #{flowId}, #{createTime}, #{modifyTime}, #{modifyUser}) + + + + INSERT INTO dss_workflow_node_content(node_key, node_id, job_type, orchestrator_id, flow_id, create_time, modify_time, modify_user) + VALUES + + (#{item.nodeKey}, #{item.nodeId}, #{item.jobType}, #{item.orchestratorId}, #{item.flowId}, #{item.createTime}, #{item.modifyTime}, #{item.modifyUser}) + + + + + UPDATE dss_workflow_node_content + SET `node_key` = #{nodeKey}, `node_id` = #{nodeId}, `job_type` = #{jobType}, `orchestrator_id` = #{orchestratorId}, `flow_id` = #{flowId}, `modify_time` = #{modifyTime}, `modify_user` = #{modifyUser} + WHERE `id` = #{id} + + + + UPDATE dss_workflow_node_content + SET `node_id` = #{nodeId}, `job_type` = #{jobType}, `modify_time` = #{modifyTime}, `modify_user` = #{modifyUser} + WHERE `node_key` = #{nodeKey} and `orchestrator_id` = #{orchestratorId} and `flow_id` = #{flowId} + + + + + + + + + + + + + + + + + DELETE FROM dss_workflow_node_content WHERE `node_key` = #{nodeKey} and `orchestrator_id` = #{orchestratorId} + + + + DELETE FROM dss_workflow_node_content WHERE `orchestrator_id` = #{orchestratorId} and `node_key` IN + + #{item.nodeKey} + + + + + DELETE FROM dss_workflow_node_content WHERE orchestrator_id = #{orchestratorId} and flow_id = #{flowId} + + + + DELETE FROM dss_workflow_node_content WHERE orchestrator_id = #{orchestratorId} + + + + DELETE FROM dss_workflow_node_content WHERE flow_id IN + + #{item} + + + + + UPDATE dss_workflow_node_content set flow_id = #{flowId} where flow_id = #{oldFlowId} + + + + + + + + + + + + \ No newline at end of file diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/dao/impl/NodeContentUIMapper.xml b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/dao/impl/NodeContentUIMapper.xml new file mode 100644 index 0000000000..667bf86032 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/dao/impl/NodeContentUIMapper.xml @@ -0,0 +1,91 @@ + + + + + + + + + + + + + + + `content_id`, `node_ui_key`, `node_ui_value` + + + + INSERT INTO dss_workflow_node_content_to_ui(`content_id`, `node_ui_key`, `node_ui_value`, `node_type`, `node_content_type`) VAlUES (#{contentId}, #{nodeUIKey}, #{nodeUIValue}, #{item.nodeType}, #{item.nodeContentType}) + + + + INSERT INTO dss_workflow_node_content_to_ui(`content_id`, `node_ui_key`, `node_ui_value`, `node_type`, `node_content_type`) VALUES + + (#{item.contentId}, #{item.nodeUIKey}, #{item.nodeUIValue}, #{item.nodeType}, #{item.nodeContentType}) + + + + + UPDATE dss_workflow_node_content_to_ui + SET `node_ui_value` = #{nodeUIValue} WHERE `node_ui_key` = #{nodeUIKey} and `content_id` = #{contentId} + + + + UPDAtE dss_workflow_node_content_to_ui + + SET `node_ui_value` = #{item.nodeUIValue} WHERE `node_ui_key` = #{item.nodeUIKey} and `content_id` = #{item.contentId} + + + + + + + DELETE FROM dss_workflow_node_content_to_ui WHERE `content_id` IN + + #{item} + + + + + + + + + + + + \ No newline at end of file diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/dao/impl/NodeMetaMapper.xml b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/dao/impl/NodeMetaMapper.xml new file mode 100644 index 0000000000..696e942315 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/dao/impl/NodeMetaMapper.xml @@ -0,0 +1,57 @@ + + + + + + + + + + + + + + + + + `id`, `orchestrator_id`, `proxy_user`, `meta_resource`, `global_var` + + + + INSERT INTO dss_workflow_node_meta(`id`, `orchestrator_id`, `proxy_user`, `meta_resource`, `global_var`) + VALUES (#{id}, #{orchestratorId}, #{proxyUser}, #{metaResource}, #{globalVar}) + + + + UPDATE dss_workflow_node_meta + SET `orchestrator_id` = #{orchestratorId}, + `proxy_user` = #{proxyUser}, + `meta_resource` = #{metaResource}, + `global_var` = #{globalVar} + WHERE `id` = #{id} + + + + + + Delete from dss_workflow_node_meta where orchestrator_id = #{orchestratorId} + + + \ No newline at end of file diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/dao/impl/flowMapper.xml b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/dao/impl/flowMapper.xml index ed259f1564..5491aa8e7c 100644 --- a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/dao/impl/flowMapper.xml +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/dao/impl/flowMapper.xml @@ -28,6 +28,13 @@ SELECT * FROM `dss_workflow` WHERE id = #{id} + + INSERT INTO dss_workflow () VALUES @@ -143,4 +150,19 @@ + + + + + \ No newline at end of file diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/dao/impl/lockMapper.xml b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/dao/impl/lockMapper.xml index 41e45c40c5..0e6a07b23c 100644 --- a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/dao/impl/lockMapper.xml +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/dao/impl/lockMapper.xml @@ -73,4 +73,64 @@ delete from dss_workflow_edit_lock + + + delete from dss_workflow_edit_lock where flow_id not in + + #{index} + + + + + INSERT INTO dss_workflow_status (`flow_id`, `status`, `create_time`, `update_time`) + VALUES + (#{flowID}, #{status}, NOW(), NOW()) + + + + update dss_orchestrator_info + + `status` = #{status} + + where `id` = #{id} + + + + update dss_orchestrator_info + + `status` = #{status} + + where `id` IN + + #{item} + + + + + + + + + update dss_orchestrator_version_info + + `commit_id` = #{commitId} + + where `app_id` = #{flowID} + + + + update dss_orchestrator_version_info + + `commit_id` = #{commitId} + + where `app_id` IN + + #{item} + + + \ No newline at end of file diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/dao/impl/nodeInfoMapper.xml b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/dao/impl/nodeInfoMapper.xml index 805f903c28..ff5852e2c7 100644 --- a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/dao/impl/nodeInfoMapper.xml +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/dao/impl/nodeInfoMapper.xml @@ -135,8 +135,70 @@ AND nutv.validate_id = nuv.id + + + + + + + + + + + + + \ No newline at end of file diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/dto/NodeContentDO.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/dto/NodeContentDO.java new file mode 100644 index 0000000000..029f06b738 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/dto/NodeContentDO.java @@ -0,0 +1,148 @@ +package com.webank.wedatasphere.dss.workflow.dto; + + +import com.google.common.base.Objects; + +import java.util.Date; + +public class NodeContentDO { + private Long id; + private String nodeKey; + private String nodeId; + private String jobType; + private Long orchestratorId; + private Long flowId; + private Date createTime; + private Date modifyTime; + private String modifyUser; + + public NodeContentDO() { + } + + public NodeContentDO(Long id, String nodeKey, String nodeId, String jobType, Long orchestratorId, Date createTime, Date modifyTime, String modifyUser) { + this.id = id; + this.nodeKey = nodeKey; + this.nodeId = nodeId; + this.jobType = jobType; + this.orchestratorId = orchestratorId; + this.createTime = createTime; + this.modifyTime = modifyTime; + this.modifyUser = modifyUser; + } + + public NodeContentDO(String nodeKey, String nodeId, String jobType, Long orchestratorId, Long flowId, Date createTime, Date modifyTime, String modifyUser) { + this.nodeKey = nodeKey; + this.nodeId = nodeId; + this.jobType = jobType; + this.orchestratorId = orchestratorId; + this.flowId = flowId; + this.createTime = createTime; + this.modifyTime = modifyTime; + this.modifyUser = modifyUser; + } + + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + NodeContentDO that = (NodeContentDO) o; + return Objects.equal(nodeKey, that.nodeKey); + } + + @Override + public int hashCode() { + return Objects.hashCode(nodeKey); + } + + @Override + public String toString() { + return "NodeContentDO{" + + "id=" + id + + ", nodeKey='" + nodeKey + '\'' + + ", nodeId='" + nodeId + '\'' + + ", jobType='" + jobType + '\'' + + ", orchestratorId='" + orchestratorId + '\'' + + ", flowId='" + flowId + '\'' + + ", createTime='" + createTime + '\'' + + ", modifyTime='" + modifyTime + '\'' + + ", modifyUser='" + modifyUser + '\'' + + '}'; + } + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public String getNodeKey() { + return nodeKey; + } + + public void setNodeKey(String nodeKey) { + this.nodeKey = nodeKey; + } + + public String getNodeId() { + return nodeId; + } + + public void setNodeId(String nodeId) { + this.nodeId = nodeId; + } + + public String getJobType() { + return jobType; + } + + public void setJobType(String jobType) { + this.jobType = jobType; + } + + public Long getOrchestratorId() { + return orchestratorId; + } + + public void setOrchestratorId(Long orchestratorId) { + this.orchestratorId = orchestratorId; + } + + public Date getCreateTime() { + return createTime; + } + + public void setCreateTime(Date createTime) { + this.createTime = createTime; + } + + public Date getModifyTime() { + return modifyTime; + } + + public void setModifyTime(Date modifyTime) { + this.modifyTime = modifyTime; + } + + public String getModifyUser() { + return modifyUser; + } + + public void setModifyUser(String modifyUser) { + this.modifyUser = modifyUser; + } + + public Long getFlowId() { + return flowId; + } + + public void setFlowId(Long flowId) { + this.flowId = flowId; + } +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/dto/NodeContentUIDO.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/dto/NodeContentUIDO.java new file mode 100644 index 0000000000..de705fd187 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/dto/NodeContentUIDO.java @@ -0,0 +1,98 @@ +package com.webank.wedatasphere.dss.workflow.dto; + + +import java.util.Objects; + +public class NodeContentUIDO { + private Long contentId; + private String nodeUIKey; + private String nodeUIValue; + private String nodeType; + private String nodeContentType; + + public NodeContentUIDO() { + } + + public NodeContentUIDO(Long contentId, String nodeUIKey, String nodeUIValue) { + this.contentId = contentId; + this.nodeUIKey = nodeUIKey; + this.nodeUIValue = nodeUIValue; + } + + public NodeContentUIDO(Long contentId, String nodeUIKey, String nodeUIValue, String nodeType, String nodeContentType) { + this.contentId = contentId; + this.nodeUIKey = nodeUIKey; + this.nodeUIValue = nodeUIValue; + this.nodeType = nodeType; + this.nodeContentType = nodeContentType; + } + + public Long getContentId() { + return contentId; + } + + public void setContentId(Long contentId) { + this.contentId = contentId; + } + + public String getNodeUIKey() { + return nodeUIKey; + } + + public void setNodeUIKey(String nodeUIKey) { + this.nodeUIKey = nodeUIKey; + } + + public String getNodeUIValue() { + return nodeUIValue; + } + + public void setNodeUIValue(String nodeUIValue) { + this.nodeUIValue = nodeUIValue; + } + + public String getNodeType() { + return nodeType; + } + + public void setNodeType(String nodeType) { + this.nodeType = nodeType; + } + + public String getNodeContentType() { + return nodeContentType; + } + + public void setNodeContentType(String nodeContentType) { + this.nodeContentType = nodeContentType; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + NodeContentUIDO that = (NodeContentUIDO) o; + return nodeUIKey.equals(that.nodeUIKey) && + contentId.equals(that.contentId); + } + + @Override + public int hashCode() { + return Objects.hash(nodeUIKey, contentId); + } + + @Override + public String toString() { + return "NodeContentUIDO{" + + "contentId='" + contentId + '\'' + + ", nodeUIKey='" + nodeUIKey + '\'' + + ", nodeUIValue='" + nodeUIValue + '\'' + + ", nodeType='" + nodeType + '\'' + + ", nodeContentType='" + nodeContentType + '\'' + + '}'; + } +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/dto/NodeMetaDO.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/dto/NodeMetaDO.java new file mode 100644 index 0000000000..fd147157f7 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/dto/NodeMetaDO.java @@ -0,0 +1,69 @@ +package com.webank.wedatasphere.dss.workflow.dto; + + +public class NodeMetaDO { + private Long id; + private Long orchestratorId; + private String proxyUser; + private String metaResource; + private String globalVar; + + public NodeMetaDO() { + } + + public NodeMetaDO(Long id, Long orchestratorId, String proxyUser, String metaResource, String globalVar) { + this.id = id; + this.orchestratorId = orchestratorId; + this.proxyUser = proxyUser; + this.metaResource = metaResource; + this.globalVar = globalVar; + } + + public NodeMetaDO(Long orchestratorId, String proxyUser, String metaResource, String globalVar) { + this.orchestratorId = orchestratorId; + this.proxyUser = proxyUser; + this.metaResource = metaResource; + this.globalVar = globalVar; + } + + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public Long getOrchestratorId() { + return orchestratorId; + } + + public void setOrchestratorId(Long orchestratorId) { + this.orchestratorId = orchestratorId; + } + + public String getProxyUser() { + return proxyUser; + } + + public void setProxyUser(String proxyUser) { + this.proxyUser = proxyUser; + } + + public String getMetaResource() { + return metaResource; + } + + public void setMetaResource(String metaResource) { + this.metaResource = metaResource; + } + + public String getGlobalVar() { + return globalVar; + } + + public void setGlobalVar(String globalVar) { + this.globalVar = globalVar; + } +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/dto/NodeTypeDO.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/dto/NodeTypeDO.java new file mode 100644 index 0000000000..03f96bb33e --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/dto/NodeTypeDO.java @@ -0,0 +1,24 @@ +package com.webank.wedatasphere.dss.workflow.dto; + +import java.io.Serializable; + +public class NodeTypeDO implements Serializable { + private String nodeType; + private String key; + + public String getNodeType() { + return nodeType; + } + + public void setNodeType(String nodeType) { + this.nodeType = nodeType; + } + + public String getKey() { + return key; + } + + public void setKey(String key) { + this.key = key; + } +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/DSSFlowName.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/DSSFlowName.java new file mode 100644 index 0000000000..7194c0d960 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/DSSFlowName.java @@ -0,0 +1,39 @@ +package com.webank.wedatasphere.dss.workflow.entity; + +import java.util.ArrayList; +import java.util.List; + +public class DSSFlowName { + + + private List nodeNameList = new ArrayList<>(); + + private List orchestratorNameList = new ArrayList<>(); + + private List templateNameList = new ArrayList<>(); + + + public List getNodeNameList() { + return nodeNameList; + } + + public void setNodeNameList(List nodeNameList) { + this.nodeNameList = nodeNameList; + } + + public List getOrchestratorNameList() { + return orchestratorNameList; + } + + public void setOrchestratorNameList(List orchestratorNameList) { + this.orchestratorNameList = orchestratorNameList; + } + + public List getTemplateNameList() { + return templateNameList; + } + + public void setTemplateNameList(List templateNameList) { + this.templateNameList = templateNameList; + } +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/DSSFlowNodeInfo.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/DSSFlowNodeInfo.java new file mode 100644 index 0000000000..67ebeb84f1 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/DSSFlowNodeInfo.java @@ -0,0 +1,122 @@ +package com.webank.wedatasphere.dss.workflow.entity; + +import java.util.Date; + +public class DSSFlowNodeInfo { + + private String nodeId; + private Long orchestratorId; + + private String nodeKey; + + private String orchestratorName; + + private Long contentId; + + private String jobType; + + private String nodeTypeName; + + private Long projectId; + + + private Date modifyTime; + + private Date createTime; + + private Long flowId; + + + public String getNodeId() { + return nodeId; + } + + public void setNodeId(String nodeId) { + this.nodeId = nodeId; + } + + public Long getOrchestratorId() { + return orchestratorId; + } + + public void setOrchestratorId(Long orchestratorId) { + this.orchestratorId = orchestratorId; + } + + public String getOrchestratorName() { + return orchestratorName; + } + + public void setOrchestratorName(String orchestratorName) { + this.orchestratorName = orchestratorName; + } + + + public String getJobType() { + return jobType; + } + + public void setJobType(String jobType) { + this.jobType = jobType; + } + + + public String getNodeTypeName() { + return nodeTypeName; + } + + public void setNodeTypeName(String nodeTypeName) { + this.nodeTypeName = nodeTypeName; + } + + public Long getContentId() { + return contentId; + } + + public void setContentId(Long contentId) { + this.contentId = contentId; + } + + public Long getProjectId() { + return projectId; + } + + public void setProjectId(Long projectId) { + this.projectId = projectId; + } + + public Date getModifyTime() { + return modifyTime; + } + + public void setModifyTime(Date modifyTime) { + this.modifyTime = modifyTime; + } + + + public Date getCreateTime() { + return createTime; + } + + public void setCreateTime(Date createTime) { + this.createTime = createTime; + } + + + public String getNodeKey() { + return nodeKey; + } + + public void setNodeKey(String nodeKey) { + this.nodeKey = nodeKey; + } + + + public Long getFlowId() { + return flowId; + } + + public void setFlowId(Long flowId) { + this.flowId = flowId; + } +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/DSSFlowNodeInfoOfFlow.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/DSSFlowNodeInfoOfFlow.java new file mode 100644 index 0000000000..b61850a7d9 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/DSSFlowNodeInfoOfFlow.java @@ -0,0 +1,29 @@ +package com.webank.wedatasphere.dss.workflow.entity; + +public class DSSFlowNodeInfoOfFlow { + + /** + *(子)工作流ID + */ + private Long appId; + /** + * 编排ID + */ + private Long orchestratorId; + + public Long getAppId() { + return appId; + } + + public void setAppId(Long appId) { + this.appId = appId; + } + + public Long getOrchestratorId() { + return orchestratorId; + } + + public void setOrchestratorId(Long orchestratorId) { + this.orchestratorId = orchestratorId; + } +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/DSSFlowNodeTemplate.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/DSSFlowNodeTemplate.java new file mode 100644 index 0000000000..c505c17f94 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/DSSFlowNodeTemplate.java @@ -0,0 +1,25 @@ +package com.webank.wedatasphere.dss.workflow.entity; + +public class DSSFlowNodeTemplate { + + private String templateName; + + private String templateId; + + + public String getTemplateName() { + return templateName; + } + + public void setTemplateName(String templateName) { + this.templateName = templateName; + } + + public String getTemplateId() { + return templateId; + } + + public void setTemplateId(String templateId) { + this.templateId = templateId; + } +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/DataCheckerNodeInfo.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/DataCheckerNodeInfo.java new file mode 100644 index 0000000000..15e906f167 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/DataCheckerNodeInfo.java @@ -0,0 +1,75 @@ +package com.webank.wedatasphere.dss.workflow.entity; + +import java.util.Date; +import java.util.Map; + +public class DataCheckerNodeInfo extends NodeBaseInfo { + + + + // 节点描述 + private String nodeDesc; + + // source.type + private String sourceType; + + // check.object + private String checkObject; + + // max.check.hours + private String maxCheckHours; + + //job.desc + private String jobDesc; + + // qualities 校验 + private Boolean qualitisCheck; + + public String getNodeDesc() { + return nodeDesc; + } + + public void setNodeDesc(String nodeDesc) { + this.nodeDesc = nodeDesc; + } + + public String getSourceType() { + return sourceType; + } + + public void setSourceType(String sourceType) { + this.sourceType = sourceType; + } + + public String getCheckObject() { + return checkObject; + } + + public void setCheckObject(String checkObject) { + this.checkObject = checkObject; + } + + public String getMaxCheckHours() { + return maxCheckHours; + } + + public void setMaxCheckHours(String maxCheckHours) { + this.maxCheckHours = maxCheckHours; + } + + public String getJobDesc() { + return jobDesc; + } + + public void setJobDesc(String jobDesc) { + this.jobDesc = jobDesc; + } + + public Boolean getQualitisCheck() { + return qualitisCheck; + } + + public void setQualitisCheck(Boolean qualitisCheck) { + this.qualitisCheck = qualitisCheck; + } +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/DataDevelopNodeInfo.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/DataDevelopNodeInfo.java new file mode 100644 index 0000000000..5b5dbbc463 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/DataDevelopNodeInfo.java @@ -0,0 +1,74 @@ +package com.webank.wedatasphere.dss.workflow.entity; + +import java.util.Date; +import java.util.Map; + +public class DataDevelopNodeInfo extends NodeBaseInfo{ + + // 是否引用参数模板 + private Boolean isRefTemplate; + + // 模板名称 + private String templateName; + + // 模板ID + + private String templateId; + + // 资源名称 + private String resource; + + // 是否复用引擎 + private Boolean reuseEngine; + + private String script; + + public Boolean getRefTemplate() { + return isRefTemplate; + } + + public void setRefTemplate(Boolean refTemplate) { + isRefTemplate = refTemplate; + } + + public String getTemplateName() { + return templateName; + } + + public void setTemplateName(String templateName) { + this.templateName = templateName; + } + + public String getResource() { + return resource; + } + + public void setResource(String resource) { + this.resource = resource; + } + + public Boolean getReuseEngine() { + return reuseEngine; + } + + public void setReuseEngine(Boolean reuseEngine) { + this.reuseEngine = reuseEngine; + } + + public String getTemplateId() { + return templateId; + } + + public void setTemplateId(String templateId) { + this.templateId = templateId; + } + + + public String getScript() { + return script; + } + + public void setScript(String script) { + this.script = script; + } +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/DataViewNodeInfo.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/DataViewNodeInfo.java new file mode 100644 index 0000000000..c79734c2f6 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/DataViewNodeInfo.java @@ -0,0 +1,30 @@ +package com.webank.wedatasphere.dss.workflow.entity; + +import java.util.Date; +import java.util.Map; + +public class DataViewNodeInfo extends NodeBaseInfo { + + + // 视图ID + private String viewId; + + // 节点描述 + private String nodeDesc; + + public String getViewId() { + return viewId; + } + + public void setViewId(String viewId) { + this.viewId = viewId; + } + + public String getNodeDesc() { + return nodeDesc; + } + + public void setNodeDesc(String nodeDesc) { + this.nodeDesc = nodeDesc; + } +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/EventReceiverNodeInfo.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/EventReceiverNodeInfo.java new file mode 100644 index 0000000000..7ff8b3a691 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/EventReceiverNodeInfo.java @@ -0,0 +1,109 @@ +package com.webank.wedatasphere.dss.workflow.entity; + +import java.util.Date; +import java.util.Map; + +public class EventReceiverNodeInfo extends NodeBaseInfo { + + + // 节点描述 + private String nodeDesc; + + private String msgType; + + private String msgReceiver; + + private String msgTopic; + + private String msgName; + + private String queryFrequency; + + private String maxReceiveHours; + + private String msgSaveKey; + + private Boolean onlyReceiveToday; + + private Boolean msgReceiveUseRunDate; + + public String getNodeDesc() { + return nodeDesc; + } + + public void setNodeDesc(String nodeDesc) { + this.nodeDesc = nodeDesc; + } + + public String getMsgType() { + return msgType; + } + + public void setMsgType(String msgType) { + this.msgType = msgType; + } + + public String getMsgReceiver() { + return msgReceiver; + } + + public void setMsgReceiver(String msgReceiver) { + this.msgReceiver = msgReceiver; + } + + public String getMsgTopic() { + return msgTopic; + } + + public void setMsgTopic(String msgTopic) { + this.msgTopic = msgTopic; + } + + public String getMsgName() { + return msgName; + } + + public void setMsgName(String msgName) { + this.msgName = msgName; + } + + public String getQueryFrequency() { + return queryFrequency; + } + + public void setQueryFrequency(String queryFrequency) { + this.queryFrequency = queryFrequency; + } + + public String getMaxReceiveHours() { + return maxReceiveHours; + } + + public void setMaxReceiveHours(String maxReceiveHours) { + this.maxReceiveHours = maxReceiveHours; + } + + public String getMsgSaveKey() { + return msgSaveKey; + } + + public void setMsgSaveKey(String msgSaveKey) { + this.msgSaveKey = msgSaveKey; + } + + public Boolean getOnlyReceiveToday() { + return onlyReceiveToday; + } + + public void setOnlyReceiveToday(Boolean onlyReceiveToday) { + this.onlyReceiveToday = onlyReceiveToday; + } + + public Boolean getMsgReceiveUseRunDate() { + return msgReceiveUseRunDate; + } + + public void setMsgReceiveUseRunDate(Boolean msgReceiveUseRunDate) { + this.msgReceiveUseRunDate = msgReceiveUseRunDate; + } +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/EventSenderNodeInfo.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/EventSenderNodeInfo.java new file mode 100644 index 0000000000..7b8b29833a --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/EventSenderNodeInfo.java @@ -0,0 +1,68 @@ +package com.webank.wedatasphere.dss.workflow.entity; + +import java.util.Date; +import java.util.Map; + +public class EventSenderNodeInfo extends NodeBaseInfo { + + // 节点描述 + private String nodeDesc; + + private String msgType; + + private String msgSender; + + private String msgBody; + + private String msgTopic; + + private String msgName; + + public String getNodeDesc() { + return nodeDesc; + } + + public void setNodeDesc(String nodeDesc) { + this.nodeDesc = nodeDesc; + } + + public String getMsgType() { + return msgType; + } + + public void setMsgType(String msgType) { + this.msgType = msgType; + } + + public String getMsgSender() { + return msgSender; + } + + public void setMsgSender(String msgSender) { + this.msgSender = msgSender; + } + + public String getMsgBody() { + return msgBody; + } + + public void setMsgBody(String msgBody) { + this.msgBody = msgBody; + } + + public String getMsgTopic() { + return msgTopic; + } + + public void setMsgTopic(String msgTopic) { + this.msgTopic = msgTopic; + } + + public String getMsgName() { + return msgName; + } + + public void setMsgName(String msgName) { + this.msgName = msgName; + } +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/NodeBaseInfo.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/NodeBaseInfo.java new file mode 100644 index 0000000000..9e33e6dd08 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/NodeBaseInfo.java @@ -0,0 +1,186 @@ +package com.webank.wedatasphere.dss.workflow.entity; + +import java.util.Date; +import java.util.Map; + +public class NodeBaseInfo { + + // 项目名称 + private String projectName; + + // 工作流名称 + private String orchestratorName; + + // 节点名称 + private String nodeName; + + // 节点类型 + private String nodeType; + + // 节点小类 + private String nodeTypeName; + + //节点ID + private String nodeId; + + // 工作流ID + private Long orchestratorId; + + // 节点配置ID + private Long contentId; + + // 项目ID + private Long projectId; + + // 更新时间 + private Date updateTime; + + // 创建时间 + private Date createTime; + + // 节点key + private String nodeKey; + + // 节点配置信息 + private Map nodeContent; + + // 是否接入git + private Boolean associateGit; + + + // 是否有编辑权限 + private boolean editable; + + // DSSFlow id + + private Long flowId; + + public String getProjectName() { + return projectName; + } + + public void setProjectName(String projectName) { + this.projectName = projectName; + } + + public String getOrchestratorName() { + return orchestratorName; + } + + public void setOrchestratorName(String orchestratorName) { + this.orchestratorName = orchestratorName; + } + + public String getNodeName() { + return nodeName; + } + + public void setNodeName(String nodeName) { + this.nodeName = nodeName; + } + + public String getNodeType() { + return nodeType; + } + + public void setNodeType(String nodeType) { + this.nodeType = nodeType; + } + + public String getNodeTypeName() { + return nodeTypeName; + } + + public void setNodeTypeName(String nodeTypeName) { + this.nodeTypeName = nodeTypeName; + } + + public String getNodeId() { + return nodeId; + } + + public void setNodeId(String nodeId) { + this.nodeId = nodeId; + } + + public Long getOrchestratorId() { + return orchestratorId; + } + + public void setOrchestratorId(Long orchestratorId) { + this.orchestratorId = orchestratorId; + } + + public Long getContentId() { + return contentId; + } + + public void setContentId(Long contentId) { + this.contentId = contentId; + } + + public Long getProjectId() { + return projectId; + } + + public void setProjectId(Long projectId) { + this.projectId = projectId; + } + + public Date getUpdateTime() { + return updateTime; + } + + public void setUpdateTime(Date updateTime) { + this.updateTime = updateTime; + } + + public Date getCreateTime() { + return createTime; + } + + public void setCreateTime(Date createTime) { + this.createTime = createTime; + } + + public String getNodeKey() { + return nodeKey; + } + + public void setNodeKey(String nodeKey) { + this.nodeKey = nodeKey; + } + + public Map getNodeContent() { + return nodeContent; + } + + public void setNodeContent(Map nodeContent) { + this.nodeContent = nodeContent; + } + + public Boolean getAssociateGit() { + return associateGit; + } + + public void setAssociateGit(Boolean associateGit) { + this.associateGit = associateGit; + } + + + public boolean isEditable() { + return editable; + } + + public void setEditable(boolean editable) { + this.editable = editable; + } + + public Long getFlowId() { + return flowId; + } + + public void setFlowId(Long flowId) { + this.flowId = flowId; + } +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/NodeUIInfo.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/NodeUIInfo.java new file mode 100644 index 0000000000..d35397d76e --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/NodeUIInfo.java @@ -0,0 +1,58 @@ +package com.webank.wedatasphere.dss.workflow.entity; + +import java.io.Serializable; + +public class NodeUIInfo implements Serializable { + + private String nodeType; + + private String name; + + private String key; + + private String position; + + private String defaultValue; + + + public String getNodeType() { + return nodeType; + } + + public void setNodeType(String nodeType) { + this.nodeType = nodeType; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getKey() { + return key; + } + + public void setKey(String key) { + this.key = key; + } + + public String getPosition() { + return position; + } + + public void setPosition(String position) { + this.position = position; + } + + + public String getDefaultValue() { + return defaultValue; + } + + public void setDefaultValue(String defaultValue) { + this.defaultValue = defaultValue; + } +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/request/BatchEditFlowRequest.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/request/BatchEditFlowRequest.java new file mode 100644 index 0000000000..9c4c5ba703 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/request/BatchEditFlowRequest.java @@ -0,0 +1,16 @@ +package com.webank.wedatasphere.dss.workflow.entity.request; + +import java.util.List; + +public class BatchEditFlowRequest { + private List editNodeList; + + + public List getEditNodeList() { + return editNodeList; + } + + public void setEditNodeList(List editNodeList) { + this.editNodeList = editNodeList; + } +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/request/BatchPublishWorkflowRequest.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/request/BatchPublishWorkflowRequest.java new file mode 100644 index 0000000000..4c59626b80 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/request/BatchPublishWorkflowRequest.java @@ -0,0 +1,62 @@ +package com.webank.wedatasphere.dss.workflow.entity.request; + +import com.webank.wedatasphere.dss.common.label.LabelRouteVO; + +import java.util.List; + +public class BatchPublishWorkflowRequest { + private List orchestratorList; + private String comment; + private LabelRouteVO labels; + + private String dssLabel; + + public BatchPublishWorkflowRequest(String comment, LabelRouteVO labels, String dssLabel) { + this.comment = comment; + this.labels = labels; + this.dssLabel = dssLabel; + } + + public BatchPublishWorkflowRequest() { + } + + public BatchPublishWorkflowRequest(List orchestratorList, String comment, LabelRouteVO labels, String dssLabel) { + this.orchestratorList = orchestratorList; + this.comment = comment; + this.labels = labels; + this.dssLabel = dssLabel; + } + + + public String getComment() { + return comment; + } + + public void setComment(String comment) { + this.comment = comment; + } + + public LabelRouteVO getLabels() { + return labels; + } + + public void setLabels(LabelRouteVO labels) { + this.labels = labels; + } + + public String getDssLabel() { + return dssLabel; + } + + public void setDssLabel(String dssLabel) { + this.dssLabel = dssLabel; + } + + public List getOrchestratorList() { + return orchestratorList; + } + + public void setOrchestratorList(List orchestratorList) { + this.orchestratorList = orchestratorList; + } +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/request/DataCheckerNodeRequest.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/request/DataCheckerNodeRequest.java new file mode 100644 index 0000000000..39e89c016a --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/request/DataCheckerNodeRequest.java @@ -0,0 +1,102 @@ +package com.webank.wedatasphere.dss.workflow.entity.request; + +import java.util.List; + +public class DataCheckerNodeRequest { + + private Integer pageNow; + + private Integer pageSize; + + // 工作流名称 + private String orchestratorName; + + // 项目名称 + private List projectNameList; + + // 节点名称 + private List nodeNameList; + + private String sourceType; + + + private String checkObject; + + private String jobDesc; + + + private Boolean qualitisCheck; + + + public Integer getPageNow() { + return pageNow; + } + + public void setPageNow(Integer pageNow) { + this.pageNow = pageNow; + } + + public Integer getPageSize() { + return pageSize; + } + + public void setPageSize(Integer pageSize) { + this.pageSize = pageSize; + } + + public String getOrchestratorName() { + return orchestratorName; + } + + public void setOrchestratorName(String orchestratorName) { + this.orchestratorName = orchestratorName; + } + + public List getProjectNameList() { + return projectNameList; + } + + public void setProjectNameList(List projectNameList) { + this.projectNameList = projectNameList; + } + + public List getNodeNameList() { + return nodeNameList; + } + + public void setNodeNameList(List nodeNameList) { + this.nodeNameList = nodeNameList; + } + + public String getSourceType() { + return sourceType; + } + + public void setSourceType(String sourceType) { + this.sourceType = sourceType; + } + + public String getCheckObject() { + return checkObject; + } + + public void setCheckObject(String checkObject) { + this.checkObject = checkObject; + } + + public String getJobDesc() { + return jobDesc; + } + + public void setJobDesc(String jobDesc) { + this.jobDesc = jobDesc; + } + + public Boolean getQualitisCheck() { + return qualitisCheck; + } + + public void setQualitisCheck(Boolean qualitisCheck) { + this.qualitisCheck = qualitisCheck; + } +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/request/DataDevelopNodeRequest.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/request/DataDevelopNodeRequest.java new file mode 100644 index 0000000000..22a434ac6d --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/request/DataDevelopNodeRequest.java @@ -0,0 +1,129 @@ +package com.webank.wedatasphere.dss.workflow.entity.request; + +import java.util.List; + +public class DataDevelopNodeRequest { + + + + private Long workspaceId; + + private String username; + + // 是否引用模板 + private Boolean isRefTemplate; + + // 是否复用引擎 + private Boolean reuseEngine; + + private Integer pageNow; + + private Integer pageSize; + + // 节点类型名称 + private List nodeTypeNameList; + + // 节点名称 + private List nodeNameList; + + // 模板名称 + private List templateNameList; + + + // 工作流名称 + private String orchestratorName; + + // 项目名称 + private List projectNameList; + + + public Long getWorkspaceId() { + return workspaceId; + } + + public void setWorkspaceId(Long workspaceId) { + this.workspaceId = workspaceId; + } + + public String getUsername() { + return username; + } + + public void setUsername(String username) { + this.username = username; + } + + public Integer getPageNow() { + return pageNow; + } + + public void setPageNow(Integer pageNow) { + this.pageNow = pageNow; + } + + public Integer getPageSize() { + return pageSize; + } + + public void setPageSize(Integer pageSize) { + this.pageSize = pageSize; + } + + + public Boolean getRefTemplate() { + return isRefTemplate; + } + + public void setRefTemplate(Boolean refTemplate) { + isRefTemplate = refTemplate; + } + + public Boolean getReuseEngine() { + return reuseEngine; + } + + public void setReuseEngine(Boolean reuseEngine) { + this.reuseEngine = reuseEngine; + } + + public List getNodeTypeNameList() { + return nodeTypeNameList; + } + + public void setNodeTypeNameList(List nodeTypeNameList) { + this.nodeTypeNameList = nodeTypeNameList; + } + + public List getNodeNameList() { + return nodeNameList; + } + + public void setNodeNameList(List nodeNameList) { + this.nodeNameList = nodeNameList; + } + + public List getTemplateNameList() { + return templateNameList; + } + + public void setTemplateNameList(List templateNameList) { + this.templateNameList = templateNameList; + } + + + public String getOrchestratorName() { + return orchestratorName; + } + + public void setOrchestratorName(String orchestratorName) { + this.orchestratorName = orchestratorName; + } + + public List getProjectNameList() { + return projectNameList; + } + + public void setProjectNameList(List projectNameList) { + this.projectNameList = projectNameList; + } +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/request/DataViewNodeRequest.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/request/DataViewNodeRequest.java new file mode 100644 index 0000000000..3b8311ff81 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/request/DataViewNodeRequest.java @@ -0,0 +1,78 @@ +package com.webank.wedatasphere.dss.workflow.entity.request; + +import java.util.List; + +public class DataViewNodeRequest { + + + private Integer pageNow; + + private Integer pageSize; + + private List nodeTypeNameList; + + private List nodeNameList; + + private String orchestratorName; + + private List projectNameList; + + private String viewId; + + + public Integer getPageNow() { + return pageNow; + } + + public void setPageNow(Integer pageNow) { + this.pageNow = pageNow; + } + + public Integer getPageSize() { + return pageSize; + } + + public void setPageSize(Integer pageSize) { + this.pageSize = pageSize; + } + + public List getNodeTypeNameList() { + return nodeTypeNameList; + } + + public void setNodeTypeNameList(List nodeTypeNameList) { + this.nodeTypeNameList = nodeTypeNameList; + } + + public List getNodeNameList() { + return nodeNameList; + } + + public void setNodeNameList(List nodeNameList) { + this.nodeNameList = nodeNameList; + } + + public String getOrchestratorName() { + return orchestratorName; + } + + public void setOrchestratorName(String orchestratorName) { + this.orchestratorName = orchestratorName; + } + + public List getProjectNameList() { + return projectNameList; + } + + public void setProjectNameList(List projectNameList) { + this.projectNameList = projectNameList; + } + + public String getViewId() { + return viewId; + } + + public void setViewId(String viewId) { + this.viewId = viewId; + } +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/request/EditFlowRequest.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/request/EditFlowRequest.java new file mode 100644 index 0000000000..1b045898b9 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/request/EditFlowRequest.java @@ -0,0 +1,98 @@ +package com.webank.wedatasphere.dss.workflow.entity.request; + +import com.webank.wedatasphere.dss.workflow.common.entity.DSSFlow; + +import java.util.Map; + +public class EditFlowRequest { + private Long id; + private Long orchestratorId; + private String nodeKey; + private String title; + private String desc; + private String businessTag; + private String appTag; + private String params; + private String ecConfTemplateId; + private String ecConfTemplateName; + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public Long getOrchestratorId() { + return orchestratorId; + } + + public void setOrchestratorId(Long orchestratorId) { + this.orchestratorId = orchestratorId; + } + + public String getNodeKey() { + return nodeKey; + } + + public void setNodeKey(String nodeKey) { + this.nodeKey = nodeKey; + } + + public String getParams() { + return params; + } + + public void setParams(String params) { + this.params = params; + } + + public String getTitle() { + return title; + } + + public void setTitle(String title) { + this.title = title; + } + + public String getDesc() { + return desc; + } + + public void setDesc(String desc) { + this.desc = desc; + } + + public String getBusinessTag() { + return businessTag; + } + + public void setBusinessTag(String businessTag) { + this.businessTag = businessTag; + } + + public String getAppTag() { + return appTag; + } + + public void setAppTag(String appTag) { + this.appTag = appTag; + } + + public String getEcConfTemplateId() { + return ecConfTemplateId; + } + + public void setEcConfTemplateId(String ecConfTemplateId) { + this.ecConfTemplateId = ecConfTemplateId; + } + + public String getEcConfTemplateName() { + return ecConfTemplateName; + } + + public void setEcConfTemplateName(String ecConfTemplateName) { + this.ecConfTemplateName = ecConfTemplateName; + } +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/request/EventReceiverNodeRequest.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/request/EventReceiverNodeRequest.java new file mode 100644 index 0000000000..1d5dfe0359 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/request/EventReceiverNodeRequest.java @@ -0,0 +1,131 @@ +package com.webank.wedatasphere.dss.workflow.entity.request; + +import java.util.List; + +public class EventReceiverNodeRequest { + + private Integer pageNow; + + private Integer pageSize; + + // 工作流名称 + private String orchestratorName; + + // 项目名称 + private List projectNameList; + + // 节点名称 + private List nodeNameList; + + + private String msgReceiver; + + private String msgTopic; + + private String msgName; + + private String maxReceiveHours; + + private String msgSaveKey; + + private Boolean onlyReceiveToday; + + private Boolean msgReceiveUseRunDate; + + + public Integer getPageNow() { + return pageNow; + } + + public void setPageNow(Integer pageNow) { + this.pageNow = pageNow; + } + + public Integer getPageSize() { + return pageSize; + } + + public void setPageSize(Integer pageSize) { + this.pageSize = pageSize; + } + + public String getOrchestratorName() { + return orchestratorName; + } + + public void setOrchestratorName(String orchestratorName) { + this.orchestratorName = orchestratorName; + } + + public List getProjectNameList() { + return projectNameList; + } + + public void setProjectNameList(List projectNameList) { + this.projectNameList = projectNameList; + } + + public List getNodeNameList() { + return nodeNameList; + } + + public void setNodeNameList(List nodeNameList) { + this.nodeNameList = nodeNameList; + } + + public String getMsgReceiver() { + return msgReceiver; + } + + public void setMsgReceiver(String msgReceiver) { + this.msgReceiver = msgReceiver; + } + + public String getMsgTopic() { + return msgTopic; + } + + public void setMsgTopic(String msgTopic) { + this.msgTopic = msgTopic; + } + + public String getMsgName() { + return msgName; + } + + public void setMsgName(String msgName) { + this.msgName = msgName; + } + + public String getMaxReceiveHours() { + return maxReceiveHours; + } + + public void setMaxReceiveHours(String maxReceiveHours) { + this.maxReceiveHours = maxReceiveHours; + } + + public String getMsgSaveKey() { + return msgSaveKey; + } + + public void setMsgSaveKey(String msgSaveKey) { + this.msgSaveKey = msgSaveKey; + } + + public Boolean getOnlyReceiveToday() { + return onlyReceiveToday; + } + + public void setOnlyReceiveToday(Boolean onlyReceiveToday) { + this.onlyReceiveToday = onlyReceiveToday; + } + + public Boolean getMsgReceiveUseRunDate() { + return msgReceiveUseRunDate; + } + + public void setMsgReceiveUseRunDate(Boolean msgReceiveUseRunDate) { + this.msgReceiveUseRunDate = msgReceiveUseRunDate; + } +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/request/EventSenderNodeRequest.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/request/EventSenderNodeRequest.java new file mode 100644 index 0000000000..b97c63c5e9 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/request/EventSenderNodeRequest.java @@ -0,0 +1,102 @@ +package com.webank.wedatasphere.dss.workflow.entity.request; + + +import java.util.ArrayList; +import java.util.List; + +public class EventSenderNodeRequest { + + private Integer pageNow; + + private Integer pageSize; + + // 工作流名称 + private String orchestratorName; + + // 项目名称 + private List projectNameList; + + // 节点名称 + private List nodeNameList; + + + private String msgSender; + + private String msgBody; + + private String msgTopic; + + private String msgName; + + public Integer getPageNow() { + return pageNow; + } + + public void setPageNow(Integer pageNow) { + this.pageNow = pageNow; + } + + public Integer getPageSize() { + return pageSize; + } + + public void setPageSize(Integer pageSize) { + this.pageSize = pageSize; + } + + public String getOrchestratorName() { + return orchestratorName; + } + + public void setOrchestratorName(String orchestratorName) { + this.orchestratorName = orchestratorName; + } + + public List getProjectNameList() { + return projectNameList; + } + + public void setProjectNameList(List projectNameList) { + this.projectNameList = projectNameList; + } + + public List getNodeNameList() { + return nodeNameList; + } + + public void setNodeNameList(List nodeNameList) { + this.nodeNameList = nodeNameList; + } + + public String getMsgSender() { + return msgSender; + } + + public void setMsgSender(String msgSender) { + this.msgSender = msgSender; + } + + public String getMsgBody() { + return msgBody; + } + + public void setMsgBody(String msgBody) { + this.msgBody = msgBody; + } + + public String getMsgTopic() { + return msgTopic; + } + + public void setMsgTopic(String msgTopic) { + this.msgTopic = msgTopic; + } + + public String getMsgName() { + return msgName; + } + + public void setMsgName(String msgName) { + this.msgName = msgName; + } +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/request/QueryNodeInfoByPathRequest.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/request/QueryNodeInfoByPathRequest.java new file mode 100644 index 0000000000..29049fa294 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/request/QueryNodeInfoByPathRequest.java @@ -0,0 +1,28 @@ +package com.webank.wedatasphere.dss.workflow.entity.request; + +public class QueryNodeInfoByPathRequest { + /** + * 节点所在项目id + */ + private Long projectId; + + + // 节点在git中的路径 + private String path; + + public Long getProjectId() { + return projectId; + } + + public void setProjectId(Long projectId) { + this.projectId = projectId; + } + + public String getPath() { + return path; + } + + public void setPath(String path) { + this.path = path; + } +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/response/DataCheckerNodeResponse.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/response/DataCheckerNodeResponse.java new file mode 100644 index 0000000000..0259982706 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/response/DataCheckerNodeResponse.java @@ -0,0 +1,32 @@ +package com.webank.wedatasphere.dss.workflow.entity.response; + + +import com.webank.wedatasphere.dss.workflow.entity.DataCheckerNodeInfo; + +import java.util.ArrayList; +import java.util.List; + +public class DataCheckerNodeResponse { + + private Long total = 0L; + + private List dataCheckerNodeInfoList = new ArrayList<>(); + + + public Long getTotal() { + return total; + } + + public void setTotal(Long total) { + this.total = total; + } + + + public List getDataCheckerNodeInfoList() { + return dataCheckerNodeInfoList; + } + + public void setDataCheckerNodeInfoList(List dataCheckerNodeInfoList) { + this.dataCheckerNodeInfoList = dataCheckerNodeInfoList; + } +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/response/DataDevelopNodeResponse.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/response/DataDevelopNodeResponse.java new file mode 100644 index 0000000000..198ef6d229 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/response/DataDevelopNodeResponse.java @@ -0,0 +1,30 @@ +package com.webank.wedatasphere.dss.workflow.entity.response; + +import com.webank.wedatasphere.dss.workflow.entity.DataDevelopNodeInfo; + +import java.util.ArrayList; +import java.util.List; + +public class DataDevelopNodeResponse { + + private Long total = 0L; + + private List dataDevelopNodeInfoList = new ArrayList<>(); + + + public Long getTotal() { + return total; + } + + public void setTotal(Long total) { + this.total = total; + } + + public List getDataDevelopNodeInfoList() { + return dataDevelopNodeInfoList; + } + + public void setDataDevelopNodeInfoList(List dataDevelopNodeInfoList) { + this.dataDevelopNodeInfoList = dataDevelopNodeInfoList; + } +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/response/DataViewNodeResponse.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/response/DataViewNodeResponse.java new file mode 100644 index 0000000000..564a3497f9 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/response/DataViewNodeResponse.java @@ -0,0 +1,32 @@ +package com.webank.wedatasphere.dss.workflow.entity.response; + +import com.webank.wedatasphere.dss.workflow.entity.DataDevelopNodeInfo; +import com.webank.wedatasphere.dss.workflow.entity.DataViewNodeInfo; + +import java.util.ArrayList; +import java.util.List; + +public class DataViewNodeResponse { + + + private Long total = 0L; + + private List dataDevelopNodeInfoList = new ArrayList<>(); + + + public Long getTotal() { + return total; + } + + public void setTotal(Long total) { + this.total = total; + } + + public List getDataDevelopNodeInfoList() { + return dataDevelopNodeInfoList; + } + + public void setDataDevelopNodeInfoList(List dataDevelopNodeInfoList) { + this.dataDevelopNodeInfoList = dataDevelopNodeInfoList; + } +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/response/EventReceiveNodeResponse.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/response/EventReceiveNodeResponse.java new file mode 100644 index 0000000000..7d5d2cb42f --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/response/EventReceiveNodeResponse.java @@ -0,0 +1,32 @@ +package com.webank.wedatasphere.dss.workflow.entity.response; + + +import com.webank.wedatasphere.dss.workflow.entity.EventReceiverNodeInfo; + +import java.util.ArrayList; +import java.util.List; + +public class EventReceiveNodeResponse { + + + private Long total = 0L; + + private List eventReceiverNodeInfoList = new ArrayList<>(); + + + public Long getTotal() { + return total; + } + + public void setTotal(Long total) { + this.total = total; + } + + public List getEventReceiverNodeInfoList() { + return eventReceiverNodeInfoList; + } + + public void setEventReceiverNodeInfoList(List eventReceiverNodeInfoList) { + this.eventReceiverNodeInfoList = eventReceiverNodeInfoList; + } +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/response/EventSenderNodeResponse.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/response/EventSenderNodeResponse.java new file mode 100644 index 0000000000..e3f469a18e --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/response/EventSenderNodeResponse.java @@ -0,0 +1,31 @@ +package com.webank.wedatasphere.dss.workflow.entity.response; + +import com.webank.wedatasphere.dss.workflow.entity.EventSenderNodeInfo; + +import java.util.ArrayList; +import java.util.List; + +public class EventSenderNodeResponse { + + + private Long total = 0L; + + private List eventSenderNodeInfoList = new ArrayList<>(); + + + public Long getTotal() { + return total; + } + + public void setTotal(Long total) { + this.total = total; + } + + public List getEventSenderNodeInfoList() { + return eventSenderNodeInfoList; + } + + public void setEventSenderNodeInfoList(List eventSenderNodeInfoList) { + this.eventSenderNodeInfoList = eventSenderNodeInfoList; + } +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/response/QueryNodeInfoByPathResponse.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/response/QueryNodeInfoByPathResponse.java new file mode 100644 index 0000000000..23aa15597e --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/entity/response/QueryNodeInfoByPathResponse.java @@ -0,0 +1,61 @@ +package com.webank.wedatasphere.dss.workflow.entity.response; + + +public class QueryNodeInfoByPathResponse { + /** + * 节点所在项目id + */ + private Long projectId; + + + /** + *(子)工作流ID + */ + private Long appId; + /** + * 编排ID + */ + private Long orchestratorId; + private String orchestratorName; + private String nodeName; + + public Long getProjectId() { + return projectId; + } + + public void setProjectId(Long projectId) { + this.projectId = projectId; + } + + public Long getAppId() { + return appId; + } + + public void setAppId(Long appId) { + this.appId = appId; + } + + public Long getOrchestratorId() { + return orchestratorId; + } + + public void setOrchestratorId(Long orchestratorId) { + this.orchestratorId = orchestratorId; + } + + public String getOrchestratorName() { + return orchestratorName; + } + + public void setOrchestratorName(String orchestratorName) { + this.orchestratorName = orchestratorName; + } + + public String getNodeName() { + return nodeName; + } + + public void setNodeName(String nodeName) { + this.nodeName = nodeName; + } +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/export/MetaExportService.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/export/MetaExportService.java index 87216241d9..2181f14bb2 100644 --- a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/export/MetaExportService.java +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/export/MetaExportService.java @@ -16,6 +16,7 @@ package com.webank.wedatasphere.dss.workflow.io.export; +import com.webank.wedatasphere.dss.orchestrator.common.entity.OrchestratorVo; import com.webank.wedatasphere.dss.workflow.common.entity.DSSFlow; import com.webank.wedatasphere.dss.workflow.common.entity.DSSFlowRelation; @@ -26,5 +27,6 @@ public interface MetaExportService { void exportFlowBaseInfo(List allDSSFlows, List allFlowRelations, String savePath) throws IOException; + void exportFlowBaseInfoNew(OrchestratorVo orchestratorVo, List allDSSFlows, List allFlowRelations, String savePath) throws IOException; } diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/export/NodeExportService.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/export/NodeExportService.java index 8aa063b23e..f472e96df0 100644 --- a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/export/NodeExportService.java +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/export/NodeExportService.java @@ -25,4 +25,6 @@ public interface NodeExportService { void downloadNodeResourceToLocal(String userName, DSSNode dwsNode, String savePath); void downloadAppConnResourceToLocal(String userName, Long projectId, String projectName, DSSNode dwsNode, String savePath, Workspace workspace, List dssLabels) throws Exception; + void downloadNodeResourceToLocalNew(String userName, DSSNode dwsNode, String flowCodePath); + void downloadAppConnResourceToLocalNew(String userName, Long projectId, String projectName, DSSNode dwsNode, String flowCodePath, Workspace workspace, List dssLabels) throws Exception; } diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/export/WorkFlowExportService.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/export/WorkFlowExportService.java index 55eef00641..6d28191128 100644 --- a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/export/WorkFlowExportService.java +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/export/WorkFlowExportService.java @@ -22,6 +22,24 @@ import java.util.List; public interface WorkFlowExportService { + /** + * 将一个工作流导出成压缩包，存放到本地磁盘，并返回压缩包的路径 + * @param dssProjectId 所属工作空间id + * @param projectName 所属项目名称 + * @param rootFlowId 工作流根节点id + * @param userName 用户名 + * @param workspace 所属工作空间 + * @param dssLabels label列表 + * @param exportExternalNodeAppConnResource 是否导出第三方节点的物料 + * @return 导出的工作流压缩包的文件地址 + * @throws Exception + */ + String exportFlowInfoNew(Long dssProjectId, String projectName, long rootFlowId, String userName, Workspace workspace, + List dssLabels,boolean exportExternalNodeAppConnResource) throws Exception; + + + String exportFlowListNew(Long dssProjectId, String projectName, List rootFlowIdList, String userName, + Workspace workspace, List dssLabels,boolean exportExternalNodeAppConnResource) throws Exception; /** * 将一个工作流导出成压缩包，存放到本地磁盘，并返回压缩包的路径 * @param dssProjectId 所属工作空间id diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/export/impl/MetaExportServiceImpl.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/export/impl/MetaExportServiceImpl.java index 263c2d05d8..019ea3ec0b 100644 --- a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/export/impl/MetaExportServiceImpl.java +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/export/impl/MetaExportServiceImpl.java @@ -16,26 +16,51 @@ package com.webank.wedatasphere.dss.workflow.io.export.impl; +import com.google.gson.GsonBuilder; +import com.webank.wedatasphere.dss.common.utils.DSSCommonUtils; import com.webank.wedatasphere.dss.common.utils.IoUtils; +import com.webank.wedatasphere.dss.orchestrator.common.entity.OrchestratorVo; +import com.webank.wedatasphere.dss.sender.service.DSSSenderServiceFactory; import com.webank.wedatasphere.dss.workflow.common.entity.DSSFlow; import com.webank.wedatasphere.dss.workflow.common.entity.DSSFlowRelation; import com.webank.wedatasphere.dss.workflow.io.export.MetaExportService; import com.webank.wedatasphere.dss.workflow.io.export.MetaWriter; +import org.apache.linkis.rpc.Sender; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Service; import java.io.*; +import java.util.HashMap; import java.util.List; +import java.util.Map; @Service public class MetaExportServiceImpl implements MetaExportService { private Logger logger = LoggerFactory.getLogger(this.getClass()); - + public static final String FLOW_META_FILE_NAME = ".flowmeta"; + public static final String FLOW_META_KEY = "dss_flow"; + public static final String FLOW_RELATION_META_KEY = "dss_flow_relation"; + public static final String ORCHESTRATOR_META_KEY = "dss_orchestrator"; + public static final String ORCHESTRATOR_VERSION_META_KEY = "dss_orchestrator_version"; private final String fileName = "meta.txt"; + @Override + public void exportFlowBaseInfoNew(OrchestratorVo orchestratorVo, List allDSSFlows, List allFlowRelations, String savePath) throws IOException { + try ( + OutputStream outputStream = IoUtils.generateExportOutputStream(savePath + File.separator + FLOW_META_FILE_NAME) + ) { + Map flowMetaMap = new HashMap<>(2); + flowMetaMap.put(FLOW_META_KEY, allDSSFlows); + flowMetaMap.put(FLOW_RELATION_META_KEY, allFlowRelations); + flowMetaMap.put(ORCHESTRATOR_META_KEY, orchestratorVo.getDssOrchestratorInfo()); + flowMetaMap.put(ORCHESTRATOR_VERSION_META_KEY, orchestratorVo.getDssOrchestratorVersion()); + String flowMetaStr = new GsonBuilder().setPrettyPrinting().create().toJson(flowMetaMap); + org.apache.commons.io.IOUtils.write(flowMetaStr,outputStream,"UTF-8"); + } + } @Override public void exportFlowBaseInfo(List allDSSFlows, List allFlowRelations, String savePath) throws IOException { diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/export/impl/NodeExportServiceImpl.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/export/impl/NodeExportServiceImpl.java index 0e4aca71e5..fcd4bb1f3c 100644 --- a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/export/impl/NodeExportServiceImpl.java +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/export/impl/NodeExportServiceImpl.java @@ -40,11 +40,13 @@ import java.io.InputStream; import java.util.List; import java.util.Map; +import java.util.Optional; @Service public class NodeExportServiceImpl implements NodeExportService { + public static final String APPCONN_FILE_NAME = ".appconnre"; private final Logger LOGGER = LoggerFactory.getLogger(this.getClass()); @Autowired @@ -55,6 +57,69 @@ public class NodeExportServiceImpl implements NodeExportService { @Autowired private WorkflowNodeService workflowNodeService; + @Override + public void downloadNodeResourceToLocalNew(String userName, DSSNode dwsNode, String flowCodePath) { + List resources = dwsNode.getResources(); + String scriptName = Optional.ofNullable(dwsNode.getJobContent()).map(jobContent -> jobContent.get("script")) + .map(Object::toString).orElse(null); + if (resources != null) { + boolean hasScriptSaved=false; + for (Resource x : resources) { + // TODO: 2020/6/9 防止前台传的 resources：{{}],后期要去掉 + if (x.getResourceId() != null && x.getFileName() != null && x.getVersion() != null) { + String nodePath = flowCodePath + File.separator + dwsNode.getName(); + // 如果是scriptis节点，且资源名和jobContent中的script相同，则下载后的文件名为script名的后缀，如.sql。 否则说明是用户自定义文件，直接用filename + String resourceName = x.getFileName(); + int index = resourceName.lastIndexOf('.'); + String extensionName= resourceName.substring(index); + String shortName = resourceName.substring(0, index); + if(resourceName.equals(scriptName) && !hasScriptSaved){ + resourceName = Optional.ofNullable(dwsNode.getName()).orElse("") + extensionName; + hasScriptSaved = true; + }else if(shortName.equals(dwsNode.getName())){ + //其他与节点名同名的资源，直接跳过不要导出。相当于被节点自身的代码覆盖掉了 + return; + } + String nodeResourcePath = nodePath + File.separator + resourceName; + bmlService.downloadToLocalPath(userName, x.getResourceId(), x.getVersion(), nodeResourcePath); + } else { + LOGGER.warn("Illegal resource information"); + LOGGER.warn("username:{},nodeId:{},nodeName:{},fileName:{},version:{},resourceId:{}", userName, dwsNode.getId(), dwsNode.getName(), x.getFileName(), x.getVersion(), x.getResourceId()); + } + }; + } + } + + @Override + public void downloadAppConnResourceToLocalNew(String userName, Long projectId, String projectName, DSSNode dwsNode, + String flowCodePath, Workspace workspace, List dssLabels) throws Exception { + CommonAppConnNode node = new CommonAppConnNode(); + node.setJobContent(dwsNode.getJobContent()); + node.setProjectId(projectId); + node.setWorkspace(workspace); + node.setDssLabels(dssLabels); + node.setNodeType(dwsNode.getNodeType()); + node.setName(dwsNode.getName()); + node.setId(dwsNode.getId()); + ExportResponseRef responseRef = workflowNodeService.exportNode(userName, node); + Map resourceMap = responseRef.getResourceMap(); + String nodePath = flowCodePath + File.separator + dwsNode.getName(); + String nodeResourcePath = nodePath + File.separator + APPCONN_FILE_NAME; + if(responseRef.isLinkisBMLResources()) { + String resourceId = resourceMap.get(ImportRequestRef.RESOURCE_ID_KEY).toString(); + String version = resourceMap.get(ImportRequestRef.RESOURCE_VERSION_KEY).toString(); + bmlService.downloadToLocalPath(userName, resourceId, version, nodeResourcePath); + } else { + InputStream inputStream = (InputStream) resourceMap.get(ImportRequestRef.INPUT_STREAM_KEY); + Closeable closeable = (Closeable) resourceMap.get(ImportRequestRef.CLOSEABLE_KEY); + try { + FileUtils.copyInputStreamToFile(inputStream, new File(nodeResourcePath)); + } finally { + IOUtils.closeQuietly(inputStream); + IOUtils.closeQuietly(closeable); + } + } + } @Override public void downloadNodeResourceToLocal(String userName, DSSNode dwsNode, String savePath) { List resources = dwsNode.getResources(); diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/export/impl/WorkFlowExportServiceImpl.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/export/impl/WorkFlowExportServiceImpl.java index 7417a682fc..50c8a30978 100644 --- a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/export/impl/WorkFlowExportServiceImpl.java +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/export/impl/WorkFlowExportServiceImpl.java @@ -17,16 +17,27 @@ package com.webank.wedatasphere.dss.workflow.io.export.impl; +import com.google.gson.*; import com.webank.wedatasphere.dss.common.entity.IOType; import com.webank.wedatasphere.dss.common.entity.Resource; import com.webank.wedatasphere.dss.common.entity.node.DSSEdge; import com.webank.wedatasphere.dss.common.entity.node.DSSNode; import com.webank.wedatasphere.dss.common.entity.node.Node; +import com.webank.wedatasphere.dss.common.entity.project.DSSProject; import com.webank.wedatasphere.dss.common.exception.DSSErrorException; import com.webank.wedatasphere.dss.common.exception.DSSRuntimeException; import com.webank.wedatasphere.dss.common.label.DSSLabel; +import com.webank.wedatasphere.dss.common.protocol.project.ProjectInfoRequest; import com.webank.wedatasphere.dss.common.utils.IoUtils; +import com.webank.wedatasphere.dss.common.utils.RpcAskUtils; import com.webank.wedatasphere.dss.common.utils.ZipHelper; +import com.webank.wedatasphere.dss.orchestrator.common.entity.DSSOrchestratorInfo; +import com.webank.wedatasphere.dss.orchestrator.common.entity.OrchestratorVo; +import com.webank.wedatasphere.dss.orchestrator.common.protocol.RequestOrchestratorInfos; +import com.webank.wedatasphere.dss.orchestrator.common.protocol.RequestQuertByAppIdOrchestrator; +import com.webank.wedatasphere.dss.orchestrator.common.protocol.RequestQueryByIdOrchestrator; +import com.webank.wedatasphere.dss.orchestrator.common.protocol.ResponseOrchestratorInfos; +import com.webank.wedatasphere.dss.sender.service.DSSSenderServiceFactory; import com.webank.wedatasphere.dss.standard.app.sso.Workspace; import com.webank.wedatasphere.dss.workflow.common.entity.DSSFlow; import com.webank.wedatasphere.dss.workflow.common.entity.DSSFlowRelation; @@ -42,6 +53,8 @@ import com.webank.wedatasphere.dss.workflow.service.DSSFlowService; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang.StringUtils; +import org.apache.linkis.common.utils.Utils; +import org.apache.linkis.rpc.Sender; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -49,10 +62,10 @@ import org.springframework.stereotype.Service; import java.io.File; +import java.io.IOException; +import java.io.OutputStream; import java.util.*; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.FutureTask; -import java.util.concurrent.TimeUnit; +import java.util.concurrent.*; import java.util.stream.Collectors; import static com.webank.wedatasphere.dss.workflow.constant.DSSWorkFlowConstant.NODE_EXPORT_IMPORT_TIMEOUT_MINUTES; @@ -64,6 +77,9 @@ public class WorkFlowExportServiceImpl implements WorkFlowExportService { private Logger logger = LoggerFactory.getLogger(this.getClass()); + public static final String FLOW_FILE_NAME = ".flow"; + public static final String NODE_PARAMS_FILE_NAME = ".properties"; + @Autowired @Qualifier("workflowBmlService") private BMLService bmlService; @@ -87,6 +103,122 @@ public class WorkFlowExportServiceImpl implements WorkFlowExportService { private DSSFlowService flowService; + @Override + public String exportFlowInfoNew(Long dssProjectId, String projectName, long rootFlowId, String userName, + Workspace workspace, List dssLabels, boolean exportExternalNodeAppConnResource) throws Exception { + //获取rootFlow,和所有子Flow + DSSFlow rootFlow = flowMapper.selectFlowByID(rootFlowId); + List dssFlowList = new ArrayList<>(); + //生成rootflow及所有子flow + dssFlowList.add(rootFlow); + getAllDssFlowsByRootflowId(rootFlow, dssFlowList); + //生成rootflow及所有子flow的Relations + List flowIds = dssFlowList.stream().map(DSSFlow::getId).collect(Collectors.toList()); + List flowRelations = flowIds.isEmpty() ? new ArrayList<>() : flowMapper.listFlowRelation(flowIds); + // /appcom/tmp/dss/yyyyMMddHHmmssSSS/projectxxx + String projectPath = IoUtils.generateProjectIOPath(userName, projectName); + // /appcom/tmp/dss/yyyyMMddHHmmssSSS/projectxxx/.flowmeta/flow_all_type_node/ + List dssFlows = new ArrayList<>(); + for (DSSFlow dssFlow : dssFlowList) { + //不出意外，永远只会有一个Root工作流，所以这里的循环if，只会进入一次 + if (dssFlow.getRootFlow()) { + // 生成rootflow orchestrator信息 + Sender orcSender = DSSSenderServiceFactory.getOrCreateServiceInstance().getOrcSender(dssLabels); + OrchestratorVo orchestratorVo = RpcAskUtils.processAskException(orcSender.ask(new RequestQuertByAppIdOrchestrator(dssFlow.getId())), + OrchestratorVo.class, RequestQueryByIdOrchestrator.class); + DSSOrchestratorInfo dssOrchestratorInfo = orchestratorVo.getDssOrchestratorInfo(); + String flowMetaPath = IoUtils.generateFlowMetaIOPath(projectPath, dssOrchestratorInfo.getName()); + metaExportService.exportFlowBaseInfoNew(orchestratorVo, dssFlowList, flowRelations, flowMetaPath); + logger.info(userName + "-开始导出Flow：" + dssOrchestratorInfo.getName()); + String flowMetaFilePath = IoUtils.addFileSeparator(flowMetaPath, FLOW_FILE_NAME); + //导出工作流json文件 + String flowJson = bmlService.readTextFromBML(userName, dssFlow.getResourceId(), dssFlow.getBmlVersion()); + if (!dssFlow.getHasSaved()) { + logger.info("工作流{}从未保存过，忽略", dssOrchestratorInfo.getName()); + } else if (StringUtils.isNotBlank(flowJson)) { + // /appcom/tmp/dss/yyyyMMddHHmmssSSS/projectxxx/flow_all_type_node/ + String flowCodePath = IoUtils.generateFlowCodeIOPath(projectPath, dssOrchestratorInfo.getName()); + exportFlowResourcesNewForMultiThread(userName, dssProjectId, projectName, flowCodePath, flowJson, dssOrchestratorInfo.getName(), workspace, dssLabels, exportExternalNodeAppConnResource); + exportAllSubFlowsNew(userName, dssFlow, dssProjectId, projectName, flowCodePath, flowMetaPath, workspace, dssLabels, exportExternalNodeAppConnResource); + String flowJsonWithoutParams = extractAndExportParams(flowJson, flowCodePath); + try ( + OutputStream outputStream = IoUtils.generateExportOutputStream(flowMetaFilePath) + ) { + org.apache.commons.io.IOUtils.write(flowJsonWithoutParams, outputStream, "UTF-8"); + } + dssFlows.add(dssFlow); + } else { + String warnMsg = String.format(DSSWorkFlowConstant.PUBLISH_FLOW_REPORT_FORMATE, dssFlow.getName(), dssFlow.getBmlVersion()); + logger.info(warnMsg); + throw new DSSErrorException(90033, warnMsg); + } + } + } + if (dssFlows.isEmpty()) { + throw new DSSErrorException(90037, "该工程没有可以导出的工作流,请检查工作流是否都为空"); + } + + return projectPath; + } + + @Override + public String exportFlowListNew(Long dssProjectId, String projectName, List rootFlowIdList, String userName, + Workspace workspace, List dssLabels, boolean exportExternalNodeAppConnResource) throws Exception { + //获取rootFlow,和所有子Flow + List rootFlowList = flowMapper.selectFlowListByID(rootFlowIdList); + //生成rootflow及所有子flow + List dssFlowList = new ArrayList<>(); + dssFlowList.addAll(rootFlowList); + for (DSSFlow rootFlow : rootFlowList) { + getAllDssFlowsByRootflowId(rootFlow, dssFlowList); + } + //生成rootflow及所有子flow的Relations + List flowIds = dssFlowList.stream().map(DSSFlow::getId).collect(Collectors.toList()); + List flowRelations = flowIds.isEmpty() ? new ArrayList<>() : flowMapper.listFlowRelation(flowIds); + // /appcom/tmp/dss/yyyyMMddHHmmssSSS/projectxxx + String projectPath = IoUtils.generateProjectIOPath(userName, projectName); + // /appcom/tmp/dss/yyyyMMddHHmmssSSS/projectxxx/.flowmeta/flow_all_type_node/ + List dssFlows = new ArrayList<>(); + for (DSSFlow dssFlow : dssFlowList) { + if (dssFlow.getRootFlow()) { + // 生成rootflow orchestrator信息 + Sender orcSender = DSSSenderServiceFactory.getOrCreateServiceInstance().getOrcSender(dssLabels); + OrchestratorVo orchestratorVo = RpcAskUtils.processAskException(orcSender.ask(new RequestQuertByAppIdOrchestrator(dssFlow.getId())), + OrchestratorVo.class, RequestQueryByIdOrchestrator.class); + DSSOrchestratorInfo dssOrchestratorInfo = orchestratorVo.getDssOrchestratorInfo(); + String flowMetaPath = IoUtils.generateFlowMetaIOPath(projectPath, dssOrchestratorInfo.getName()); + metaExportService.exportFlowBaseInfoNew(orchestratorVo, dssFlowList, flowRelations, flowMetaPath); + logger.info(userName + "-开始导出Flow：" + dssOrchestratorInfo.getName()); + String flowMetaFilePath = IoUtils.addFileSeparator(flowMetaPath, FLOW_FILE_NAME); + //导出工作流json文件 + String flowJson = bmlService.readTextFromBML(userName, dssFlow.getResourceId(), dssFlow.getBmlVersion()); + if (!dssFlow.getHasSaved()) { + logger.info("工作流{}从未保存过，忽略", dssOrchestratorInfo.getName()); + } else if (StringUtils.isNotBlank(flowJson)) { + // /appcom/tmp/dss/yyyyMMddHHmmssSSS/projectxxx/flow_all_type_node/ + String flowCodePath = IoUtils.generateFlowCodeIOPath(projectPath, dssOrchestratorInfo.getName()); + exportFlowResourcesNew(userName, dssProjectId, projectName, flowCodePath, flowJson, dssOrchestratorInfo.getName(), workspace, dssLabels, exportExternalNodeAppConnResource); + exportAllSubFlowsNew(userName, dssFlow, dssProjectId, projectName, flowCodePath, flowMetaPath, workspace, dssLabels, exportExternalNodeAppConnResource); + String flowJsonWithoutParams = extractAndExportParams(flowJson, flowCodePath); + try ( + OutputStream outputStream = IoUtils.generateExportOutputStream(flowMetaFilePath) + ) { + org.apache.commons.io.IOUtils.write(flowJsonWithoutParams, outputStream, "UTF-8"); + } + dssFlows.add(dssFlow); + } else { + String warnMsg = String.format(DSSWorkFlowConstant.PUBLISH_FLOW_REPORT_FORMATE, dssFlow.getName(), dssFlow.getBmlVersion()); + logger.info(warnMsg); + throw new DSSErrorException(90033, warnMsg); + } + } + } + if (dssFlows.isEmpty()) { + throw new DSSErrorException(90037, "该工程没有可以导出的工作流,请检查工作流是否都为空"); + } + return projectPath; + } + @Override public String exportFlowInfo(Long dssProjectId, String projectName, long rootFlowId, String userName, Workspace workspace, List dssLabels) throws Exception { //获取rootFlow,和所有子Flow @@ -132,6 +264,39 @@ public String exportFlowInfo(Long dssProjectId, String projectName, long rootFlo return ZipHelper.zip(flowExportSaveBasePath); } + private void exportAllSubFlowsNew(String userName, DSSFlow dssFlowParent, Long projectId, String projectName, + String parentFlowCodePath, String parentFlowMetaPath, Workspace workspace, + List dssLabels, boolean exportExternalNodeAppConnResource) throws Exception { + List subFlows = dssFlowParent.getChildren(); + if (subFlows != null) { + for (DSSFlow subFlow : subFlows) { + String subFlowCodePath = parentFlowCodePath + File.separator + subFlow.getName(); + String subFlowMetaPath = parentFlowMetaPath + File.separator + subFlow.getName(); + + String subFlowMetaSavePath = subFlowMetaPath + File.separator + FLOW_FILE_NAME; + //导出子flow的json文件 + String flowJson = bmlService.readTextFromBML(userName, subFlow.getResourceId(), subFlow.getBmlVersion()); + + if (!subFlow.getHasSaved()) { + logger.info("工作流{}从未保存过，忽略", subFlow.getName()); + } else if (StringUtils.isNotBlank(flowJson)) { + exportFlowResourcesNewForMultiThread(userName, projectId, projectName, subFlowCodePath, flowJson, subFlow.getName(), workspace, dssLabels, exportExternalNodeAppConnResource); + exportAllSubFlowsNew(userName, subFlow, projectId, projectName, subFlowCodePath, subFlowMetaPath, workspace, dssLabels, exportExternalNodeAppConnResource); + String subFlowWithoutParams = extractAndExportParams(flowJson, subFlowCodePath); + try ( + OutputStream outputStream = IoUtils.generateExportOutputStream(subFlowMetaSavePath) + ) { + org.apache.commons.io.IOUtils.write(subFlowWithoutParams, outputStream, "UTF-8"); + } + + } else { + String warnMsg = String.format(DSSWorkFlowConstant.PUBLISH_FLOW_REPORT_FORMATE, subFlow.getName(), subFlow.getBmlVersion()); + logger.info(warnMsg); + throw new DSSErrorException(90014, warnMsg); + } + } + } + } private void exportAllSubFlows(String userName, DSSFlow dssFlowParent, Long projectId, String projectName, String projectExportBasePath, Workspace workspace, List dssLabels) throws Exception { @@ -160,6 +325,163 @@ private String genWorkFlowExportDir(String projectExportPath, String flowName) { return projectExportPath + File.separator + flowName; } + /** + * 导出工作流中的各种资源，放到flowCodePath中。 + * 工作流中的资源包括工作流资源和节点资源 + * + * @param flowCodePath 保存工作流代码的目录 + * @param flowJson 工作流元信息 + * @param flowName 工作流明 + * @param dssLabels label列表 + * @param exportExternalNodeAppConnResource 是否导出第三方节点的appconn资源 + * @throws Exception + */ + private void exportFlowResourcesNew(String userName, Long projectId, String projectName, + String flowCodePath, String flowJson, String flowName, + Workspace workspace, List dssLabels, boolean exportExternalNodeAppConnResource) throws Exception { + if (StringUtils.isNotEmpty(flowCodePath)) { + //导出工作流资源文件 + List resources = workFlowParser.getWorkFlowResources(flowJson); + if (resources != null) { + resources.forEach(resource -> { + downloadFlowResourceFromBmlNew(userName, resource, flowCodePath); + }); + } + + //导出工作流节点资源文件,工作流节点appconn文件 + List nodes = workFlowParser.getWorkFlowNodes(flowJson); + if (nodes != null) { + for (DSSNode node : nodes) { + nodeExportService.downloadNodeResourceToLocalNew(userName, node, flowCodePath); + NodeInfo nodeInfo = nodeInfoMapper.getWorkflowNodeByType(node.getNodeType()); + if (nodeInfo == null) { + String msg = String.format("%s note type not exist,please check appconn install successfully", node.getNodeType()); + logger.error(msg); + throw new DSSRuntimeException(msg); + } + if (exportExternalNodeAppConnResource && Boolean.TRUE.equals(nodeInfo.getSupportJump()) && nodeInfo.getJumpType() == 1) { + logger.info("node.getJobContent() is :{}", node.getJobContent()); + nodeExportService.downloadAppConnResourceToLocalNew(userName, projectId, projectName, node, flowCodePath, workspace, dssLabels); + } + } + } + + } else { + throw new DSSErrorException(90067, "工作流导出生成路径为空"); + } + } + + + private void exportFlowResourcesNewForMultiThread(String userName, Long projectId, String projectName, + String flowCodePath, String flowJson, String flowName, + Workspace workspace, List dssLabels, boolean exportExternalNodeAppConnResource) throws Exception { + + + if (StringUtils.isEmpty(flowCodePath)) { + throw new DSSErrorException(90067, "工作流导出生成路径为空"); + } + logger.info("export flow resources start,flow name is {}", flowName); + // ExecutorService executor = Utils.newFixedThreadPool(5, "export-flow-resource-Thread-", true); + List resources = workFlowParser.getWorkFlowResources(flowJson); + List futureList = new LinkedList<>(); + if (resources != null) { + //导出工作流资源文件 + for (Resource resource : resources) { + Future future = nodeExportThreadPool.submit(new Runnable() { + @Override + public void run() { + downloadFlowResourceFromBmlNew(userName, resource, flowCodePath); + } + }); + + futureList.add(future); + } + } + //导出工作流节点资源文件,工作流节点appconn文件 + List nodes = workFlowParser.getWorkFlowNodes(flowJson); + if (nodes != null) { + List nodeInfoList = nodeInfoMapper.getWorkflowNodeList(); + Map nodeTypeMap = nodeInfoList.stream().collect(Collectors.toMap(NodeInfo::getNodeType, nodeInfo -> nodeInfo)); + + for (DSSNode node : nodes) { + Future future = nodeExportThreadPool.submit(new Runnable() { + @Override + public void run() { + try { + + nodeExportService.downloadNodeResourceToLocalNew(userName, node, flowCodePath); + NodeInfo nodeInfo = nodeTypeMap.get(node.getNodeType()); + if (nodeInfo == null) { + String msg = String.format("%s note type not exist,please check appconn install successfully", node.getNodeType()); + logger.error(msg); + throw new DSSRuntimeException(msg); + } + if (exportExternalNodeAppConnResource && Boolean.TRUE.equals(nodeInfo.getSupportJump()) && nodeInfo.getJumpType() == 1) { + logger.info("node.getJobContent() is :{}", node.getJobContent()); + nodeExportService.downloadAppConnResourceToLocalNew(userName, projectId, projectName, node, flowCodePath, workspace, dssLabels); + } + + } catch (Exception e) { + logger.error("{} node download error msg is {}", node.getName(), e); + throw new RuntimeException(node.getName() + " node export flow resource error msg is " + e); + } + } + }); + + futureList.add(future); + } + } + + try { + // 等待线程完成 + for (Future future : futureList) { + future.get(); + } + } catch (Exception e) { + logger.error("{} flow export fail msg is ｛｝ ", flowName, e); + throw new RuntimeException(e); + } + + logger.info("export flow resources end,flow name is {}", flowName); + } + + + /** + * 从flowjson中分离各个节点的params参数，以及flow本身的参数，把参数写到磁盘，并返回分离后的flow + * + * @param flowJson 要分离参数的flow + * @param flowCodePath 工作流代码导出目录 + * @return 分离后的flow + */ + private String extractAndExportParams(String flowJson, String flowCodePath) throws IOException { + JsonParser parser = new JsonParser(); + JsonObject jsonObject = parser.parse(flowJson).getAsJsonObject(); + JsonArray nodeJsonArray = jsonObject.getAsJsonArray("nodes"); + Gson gson = new GsonBuilder().setPrettyPrinting().create(); + if (nodeJsonArray == null) { + return gson.toJson(jsonObject); + } + for (JsonElement element : nodeJsonArray) { + JsonObject node = element.getAsJsonObject(); + JsonElement params = node.remove("params"); + String nodeName = Optional.ofNullable(node.get("title")).map(JsonElement::getAsString).orElse(null); + String propertiesName = nodeName + NODE_PARAMS_FILE_NAME; + String nodeParamsPath = IoUtils.addFileSeparator(flowCodePath, nodeName, propertiesName); + String paramsJson = gson.toJson(params); + try ( + + OutputStream outputStream = IoUtils.generateExportOutputStream(nodeParamsPath) + ) { + + org.apache.commons.io.IOUtils.write(paramsJson, outputStream, "UTF-8"); + + } + + } + return gson.toJson(jsonObject); + + } + @Override public void exportFlowResources(String userName, Long projectId, String projectName, String projectSavePath, String flowJson, String flowName, @@ -182,7 +504,7 @@ public void exportFlowResources(String userName, Long projectId, String projectN for (DSSNode node : nodes) { nodeExportService.downloadNodeResourceToLocal(userName, node, workFlowResourceSavePath); NodeInfo nodeInfo = nodeInfoMapper.getWorkflowNodeByType(node.getNodeType()); - if(nodeInfo==null){ + if (nodeInfo == null) { String msg = String.format("%s note type not exist,please check appconn install successfully", node.getNodeType()); logger.error(msg); throw new DSSRuntimeException(msg); @@ -253,7 +575,7 @@ public void exportFlowResources_for_multi_thread(String userName, Long projectId try { nodeExportService.downloadNodeResourceToLocal(userName, node, workFlowResourceSavePath); NodeInfo nodeInfo = nodeInfoMapper.getWorkflowNodeByType(node.getNodeType()); - if(nodeInfo==null){ + if (nodeInfo == null) { String msg = String.format("%s note type not exist,please check appconn install successfully", node.getNodeType()); logger.error(msg); throw new DSSRuntimeException(msg); @@ -301,6 +623,11 @@ public String downloadFlowJsonFromBml(String userName, String resourceId, String return bmlService.downloadAndGetText(userName, resourceId, version, savePath); } + private String downloadFlowResourceFromBmlNew(String userName, Resource resource, String flowCodePath) { + String flowResourcePath = IoUtils.addFileSeparator(flowCodePath, resource.getFileName()); + return bmlService.downloadToLocalPath(userName, resource.getResourceId(), resource.getVersion(), flowResourcePath); + } + private String downloadFlowResourceFromBml(String userName, Resource resource, String savePath) { String flowResourcePath = savePath + File.separator + resource.getResourceId() + ".re"; return bmlService.downloadToLocalPath(userName, resource.getResourceId(), resource.getVersion(), flowResourcePath); diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/input/MetaInputService.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/input/MetaInputService.java index 506de99e43..114052642d 100644 --- a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/input/MetaInputService.java +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/input/MetaInputService.java @@ -19,6 +19,7 @@ import com.webank.wedatasphere.dss.workflow.common.entity.DSSFlow; import com.webank.wedatasphere.dss.workflow.common.entity.DSSFlowRelation; +import org.apache.linkis.protocol.util.ImmutablePair; import java.io.IOException; import java.util.List; @@ -26,6 +27,7 @@ public interface MetaInputService { List inputFlow(String basePath) throws IOException; + ImmutablePair,List> inputFlowNew(String flowMetaPath) throws IOException; List inputFlowRelation(String basePath) throws IOException; diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/input/NodeInputService.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/input/NodeInputService.java index e01c1f092d..ba6c4f9d36 100644 --- a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/input/NodeInputService.java +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/input/NodeInputService.java @@ -29,9 +29,14 @@ public interface NodeInputService { String uploadResourceToBml(String userName, String nodeJson, String inputResourcePath, String projectName) throws IOException; + String uploadResourceToBmlNew(String userName, String nodeJson, String nodePath, String nodeName,String projectName) throws IOException; + String uploadAppConnResource(String userName, String projectName, DSSFlow dssFlow, String nodeJson, String flowContextId, String appConnResourcePath, Workspace workspace, String orcVersion, List dssLabels) throws DSSErrorException,IOException; + String uploadAppConnResourceNew(String userName, String projectName, DSSFlow dssFlow, + String nodeJson, String flowContextId, String nodePath, + Workspace workspace, String orcVersion, List dssLabels) throws IOException; String updateNodeSubflowID(String nodeJson, long subflowID) throws IOException; diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/input/WorkFlowInputService.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/input/WorkFlowInputService.java index b954d8d285..0b7ef14702 100644 --- a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/input/WorkFlowInputService.java +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/input/WorkFlowInputService.java @@ -28,6 +28,10 @@ public interface WorkFlowInputService { + void inputWorkFlowNew(String userName, DSSFlow dssFlow, String projectName, + String flowCodePath, String flowMetaPath, Long parentFlowId, Workspace workspace, + String orcVersion, String contextId, List dssLabels) throws DSSErrorException,IOException; + /** * * @param userName diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/input/impl/MetaInputServiceImpl.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/input/impl/MetaInputServiceImpl.java index 0d181ebdcd..f0971bed80 100644 --- a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/input/impl/MetaInputServiceImpl.java +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/input/impl/MetaInputServiceImpl.java @@ -16,19 +16,24 @@ package com.webank.wedatasphere.dss.workflow.io.input.impl; +import com.google.gson.Gson; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import com.google.gson.reflect.TypeToken; import com.webank.wedatasphere.dss.common.utils.IoUtils; import com.webank.wedatasphere.dss.workflow.common.entity.DSSFlow; import com.webank.wedatasphere.dss.workflow.common.entity.DSSFlowRelation; import com.webank.wedatasphere.dss.workflow.io.input.MetaInputService; import com.webank.wedatasphere.dss.workflow.io.input.MetaReader; +import org.apache.linkis.protocol.util.ImmutablePair; import org.springframework.stereotype.Service; -import java.io.File; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.io.InputStream; +import java.io.*; +import java.lang.reflect.Type; import java.util.List; +import static com.webank.wedatasphere.dss.workflow.io.export.impl.MetaExportServiceImpl.*; + @Service public class MetaInputServiceImpl implements MetaInputService { @@ -37,6 +42,20 @@ public class MetaInputServiceImpl implements MetaInputService { private final String fileName = "meta.txt"; + @Override + public ImmutablePair,List> inputFlowNew(String flowMetaPath) throws IOException { + File flowMetaFile = new File(flowMetaPath + File.separator + FLOW_META_FILE_NAME); + JsonParser jsonParser = new JsonParser(); + Gson gson = new Gson(); + try (FileReader reader = new FileReader(flowMetaFile)) { + Type flowListType = new TypeToken>() {}.getType(); + Type flowRelationListType = new TypeToken>() {}.getType(); + JsonObject jsonObject = jsonParser.parse(reader).getAsJsonObject(); + List flows = gson.fromJson(jsonObject.get(FLOW_META_KEY), flowListType); + List flowRelations = gson.fromJson(jsonObject.get(FLOW_RELATION_META_KEY), flowRelationListType); + return new ImmutablePair<>(flows, flowRelations); + } + } @Override public List inputFlow(String basePath) throws IOException { diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/input/impl/NodeInputServiceImpl.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/input/impl/NodeInputServiceImpl.java index df2869718b..da185f234e 100644 --- a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/input/impl/NodeInputServiceImpl.java +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/input/impl/NodeInputServiceImpl.java @@ -21,7 +21,9 @@ import com.webank.wedatasphere.dss.common.entity.BmlResource; import com.webank.wedatasphere.dss.common.entity.Resource; import com.webank.wedatasphere.dss.common.exception.DSSErrorException; +import com.webank.wedatasphere.dss.common.exception.DSSRuntimeException; import com.webank.wedatasphere.dss.common.label.DSSLabel; +import com.webank.wedatasphere.dss.common.utils.IoUtils; import com.webank.wedatasphere.dss.common.utils.MapUtils; import com.webank.wedatasphere.dss.contextservice.service.ContextService; import com.webank.wedatasphere.dss.contextservice.service.impl.ContextServiceImpl; @@ -47,6 +49,8 @@ import java.util.*; import java.util.function.Supplier; +import static com.webank.wedatasphere.dss.workflow.io.export.impl.NodeExportServiceImpl.APPCONN_FILE_NAME; + @Service public class NodeInputServiceImpl implements NodeInputService { @Autowired @@ -63,7 +67,35 @@ public class NodeInputServiceImpl implements NodeInputService { private final Logger logger = LoggerFactory.getLogger(this.getClass()); - + @Override + public String uploadResourceToBmlNew(String userName, String nodeJson, String nodePath,String nodeName, String projectName) + throws IOException { + List resources = nodeParser.getNodeResource(nodeJson); + Map jobContent = nodeParser.getNodeJobContent(nodeJson); + String scriptName = Optional.ofNullable(jobContent).map(e->e.get("script")).map(Object::toString).orElse(null); + if (resources != null && resources.size() > 0) { + resources.forEach(resource -> { + if (resource.getVersion() != null && resource.getFileName() != null && resource.getResourceId() != null) { + //需要区分代码节点和非代码节点。非代码节点直接根据filename上传即可 + String fileName=resource.getFileName(); + if(fileName.equals(scriptName)){ + String extensionName = fileName.substring(fileName.lastIndexOf('.')); + fileName=Optional.ofNullable(nodeName).orElse("") + extensionName; + } + String filePath = IoUtils.addFileSeparator(nodePath, fileName); + InputStream resourceInputStream = bmlService.readLocalResourceFile(userName, filePath); + BmlResource bmlReturnMap = bmlService.upload(userName, + resourceInputStream, UUID.randomUUID().toString() + ".json", projectName); + resource.setResourceId(bmlReturnMap.getResourceId()); + resource.setVersion(bmlReturnMap.getVersion()); + } else { + logger.warn("Illegal resource information"); + logger.warn("username:{},fileName:{},version:{},resourceId:{}", userName, resource.getFileName(), resource.getVersion(), resource.getResourceId()); + } + }); + } + return nodeParser.updateNodeResource(nodeJson, resources); + } @Override public String uploadResourceToBml(String userName, String nodeJson, String inputResourcePath, String projectName) throws IOException { List resources = nodeParser.getNodeResource(nodeJson); @@ -88,7 +120,65 @@ private InputStream readResource(String userName, Resource resource, String flow String readPath = flowResourcePath + resource.getResourceId() + "_" + resource.getVersion() + ".re"; return bmlService.readLocalResourceFile(userName, readPath); } + @Override + public String uploadAppConnResourceNew(String userName, String projectName, DSSFlow dssFlow, + String nodeJson, String flowContextId, String nodePath, + Workspace workspace, String orcVersion, List dssLabels) throws IOException { + Map nodeJsonMap = BDPJettyServerHelper.jacksonJson().readValue(nodeJson, Map.class); + String nodeType = nodeJsonMap.get("jobType").toString(); + String nodeId = nodeJsonMap.get("id").toString(); + String nodeResourcePath = IoUtils.addFileSeparator(nodePath,APPCONN_FILE_NAME) ; + Map nodeContent = (LinkedHashMap) nodeJsonMap.get("jobContent"); + CommonAppConnNode appConnNode = new CommonAppConnNode(); + appConnNode.setId(nodeId); + appConnNode.setName((String) nodeJsonMap.get("title")); + appConnNode.setDssLabels(dssLabels); + appConnNode.setNodeType(nodeType); + appConnNode.setJobContent(nodeContent); + appConnNode.setFlowId(dssFlow.getId()); + appConnNode.setProjectId(dssFlow.getProjectId()); + appConnNode.setWorkspace(workspace); + appConnNode.setContextId(flowContextId); + + Map nodeExportContent = null; + logger.info("nodeResourcePath:{}", nodeResourcePath); + File file = new File(nodeResourcePath); + if (file.exists()) { + InputStream resourceInputStream = bmlService.readLocalResourceFile(userName, nodeResourcePath); + Supplier> bmlResourceMap = () -> { + BmlResource resource = bmlService.upload(userName, resourceInputStream, UUID.randomUUID().toString() + ".json", + projectName); + return ImmutableMap.of( + "resourceId", resource.getResourceId(), + "version", resource.getVersion() + ); + }; + Supplier> streamResourceMap = () -> MapUtils.newCommonMap(ImportRequestRef.INPUT_STREAM_KEY, resourceInputStream); + try { + nodeExportContent = nodeService.importNode(userName, appConnNode, bmlResourceMap, streamResourceMap, orcVersion); + } catch (ExternalOperationFailedException e) { + logger.error("failed to import node.", e); + throw new DSSRuntimeException(e.getErrCode(), e.getMessage()); + } catch (Exception e) { + logger.error("failed to import node.", e); + throw new DSSRuntimeException(90011, e.getMessage()); + } + if (nodeExportContent != null) { + if (nodeExportContent.get("project_id") != null) { + Long newProjectId = Long.parseLong(nodeExportContent.get("project_id").toString()); + logger.warn(String.format("new appConn node add into dss,dssProjectId: %s,newProjectId: %s", appConnNode.getProjectId(), newProjectId)); + nodeExportContent.remove("project_id"); + } + nodeJsonMap.replace("jobContent", nodeExportContent); + appConnNode.setJobContent(nodeExportContent); + return BDPJettyServerHelper.jacksonJson().writeValueAsString(nodeJsonMap); + } + } else { + logger.warn("appConn node resource file does not exists. nodeId: {}" + nodeId); + } + return nodeJson ; + } @Override public String uploadAppConnResource(String userName, String projectName, DSSFlow dssFlow, String nodeJson, String flowContextId, String appConnResourcePath, @@ -133,10 +223,10 @@ public String uploadAppConnResource(String userName, String projectName, DSSFlow nodeExportContent = nodeService.importNode(userName, appConnNode, bmlResourceMap, streamResourceMap, orcVersion); } catch (ExternalOperationFailedException e) { logger.error("failed to import node.", e); - throw new DSSErrorException(e.getErrCode(), e.getMessage()); + throw new DSSRuntimeException(e.getErrCode(), e.getMessage()); } catch (Exception e) { logger.error("failed to import node.", e); - throw new DSSErrorException(90011, e.getMessage()); + throw new DSSRuntimeException(90011, e.getMessage()); } if (nodeExportContent != null) { if (nodeExportContent.get("project_id") != null) { diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/input/impl/WorkFlowInputServiceImpl.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/input/impl/WorkFlowInputServiceImpl.java index e2e85305e1..cac5dd8ae5 100644 --- a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/input/impl/WorkFlowInputServiceImpl.java +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/io/input/impl/WorkFlowInputServiceImpl.java @@ -22,7 +22,9 @@ import com.webank.wedatasphere.dss.common.entity.BmlResource; import com.webank.wedatasphere.dss.common.entity.Resource; import com.webank.wedatasphere.dss.common.exception.DSSErrorException; +import com.webank.wedatasphere.dss.common.exception.DSSRuntimeException; import com.webank.wedatasphere.dss.common.label.DSSLabel; +import com.webank.wedatasphere.dss.common.utils.IoUtils; import com.webank.wedatasphere.dss.common.utils.MapUtils; import com.webank.wedatasphere.dss.contextservice.service.ContextService; import com.webank.wedatasphere.dss.contextservice.service.impl.ContextServiceImpl; @@ -36,6 +38,7 @@ import com.webank.wedatasphere.dss.workflow.io.input.NodeInputService; import com.webank.wedatasphere.dss.workflow.io.input.WorkFlowInputService; import com.webank.wedatasphere.dss.common.service.BMLService; +import com.webank.wedatasphere.dss.workflow.scheduler.DssJobThreadPool; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang.StringUtils; import org.apache.linkis.cs.common.utils.CSCommonUtils; @@ -54,9 +57,16 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.util.*; +import java.util.concurrent.CompletionService; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorCompletionService; +import java.util.concurrent.Future; import java.util.function.Function; import java.util.stream.Collectors; +import static com.webank.wedatasphere.dss.workflow.io.export.impl.WorkFlowExportServiceImpl.FLOW_FILE_NAME; +import static com.webank.wedatasphere.dss.workflow.io.export.impl.WorkFlowExportServiceImpl.NODE_PARAMS_FILE_NAME; + @Service public class WorkFlowInputServiceImpl implements WorkFlowInputService { @@ -79,6 +89,56 @@ public class WorkFlowInputServiceImpl implements WorkFlowInputService { private FlowMapper flowMapper; private static ContextService contextService = ContextServiceImpl.getInstance(); + @Override + public void inputWorkFlowNew(String userName, + DSSFlow dssFlow, + String projectName, + String flowCodePath, + String flowMetaPath, + Long parentFlowId, + Workspace workspace, + String orcVersion, + String contextId, + List dssLabels) throws DSSErrorException, IOException { + String flowMetaFilePath = IoUtils.addFileSeparator(flowMetaPath, FLOW_FILE_NAME); + String flowJson = bmlService.readLocalTextFile(userName, flowMetaFilePath); + + // TODO: 2020/7/31 优化update方法里面的saveContent + flowJson = updateFlowContextIdAndVersion(userName, + workspace.getWorkspaceName(), + projectName, + flowJson, + dssFlow, + parentFlowId, + contextId, + orcVersion); + flowJson = inputWorkFlowNodesNewForMultiThread(userName, projectName, flowJson, dssFlow, + flowCodePath, workspace, orcVersion, dssLabels); + //如果包含subflow,需要一同导入subflow内容，并更新parentflow的json内容 + List subFlows = dssFlow.getChildren(); + List templateIds = new ArrayList<>(); + if (subFlows != null) { + for (DSSFlow subFlow : subFlows) { + String subCodePath = IoUtils.addFileSeparator(flowCodePath, subFlow.getName()); + String subMetaPath = IoUtils.addFileSeparator(flowMetaPath, subFlow.getName()); + inputWorkFlowNew(userName, subFlow, projectName, subCodePath,subMetaPath, dssFlow.getId(), + workspace, orcVersion, contextId, dssLabels); + templateIds.addAll(subFlow.getFlowIdParamConfTemplateIdTuples()); + } + } + + flowJson = uploadFlowResourceToBmlNew(userName, flowJson, flowCodePath, projectName); + + DSSFlow updateDssFlow = uploadFlowJsonToBml(userName, projectName, dssFlow, flowJson); + List tempIds = workFlowParser.getParamConfTemplate(flowJson); + List templateIdsInRoot = tempIds.stream() + .map(e->new String[]{updateDssFlow.getId().toString(),e}) + .collect(Collectors.toList()); + templateIds.addAll(templateIdsInRoot); + updateDssFlow.setFlowIdParamConfTemplateIdTuples(templateIds); + contextService.checkAndSaveContext(flowJson, String.valueOf(parentFlowId)); + flowMapper.updateFlowInputInfo(updateDssFlow) ; + } @Override public void inputWorkFlow(String userName, DSSFlow dssFlow, @@ -93,7 +153,7 @@ public void inputWorkFlow(String userName, String flowInputPath = inputProjectPath + File.separator + dssFlow.getName(); String flowJsonPath = flowInputPath + File.separator + dssFlow.getName() + ".json"; String flowJson = bmlService.readLocalTextFile(userName, flowJsonPath); - //生成新的节点key。 + //生成新的节点key。为了解决复制工程后，新工程出现原有工程工作流的历史执行记录 Set nodeKeys=findFlowNodeKeys(flowJson); Map oldNewNodeKeyMap = nodeKeys.stream().collect(Collectors.toMap(Function.identity(), v -> UUID.randomUUID().toString(),(e1,e2)->e1)); String updateFlowJson=flowJson; @@ -195,6 +255,169 @@ private String updateFlowContextIdAndVersion(String userName, } return workFlowParser.updateFlowJsonWithMap(flowJson, MapUtils.newCommonMap(CSCommonUtils.CONTEXT_ID_STR, contextId, DSSJobContentConstant.ORC_VERSION_KEY, orcVersion)); } + /** + * 导入工作流节点 + * @param userName + * @param projectName + * @param flowJson + * @param dssFlow + * @param workspace + * @param orcVersion + * @param dssLabels + * @return + * @throws DSSErrorException + * @throws IOException + */ + private String inputWorkFlowNodesNew(String userName, String projectName, String flowJson, + DSSFlow dssFlow, String flowCodePath, Workspace workspace, + String orcVersion, List dssLabels) throws DSSErrorException, IOException { + List nodeJsonList = workFlowParser.getWorkFlowNodesJson(flowJson); + if (nodeJsonList == null) { + throw new DSSErrorException(90073, "工作流内没有工作流节点，导入失败 " + dssFlow.getName()); + } + String updateContextId = workFlowParser.getValueWithKey(flowJson, CSCommonUtils.CONTEXT_ID_STR); + if (nodeJsonList.size() == 0) { + return flowJson; + } + List subflows = (List) dssFlow.getChildren(); + List> nodeJsonListRes = new ArrayList<>(); + if (nodeJsonList.size() > 0) { + for (String nodeJson : nodeJsonList) { + String nodeName = nodeParser.getNodeValue("title", nodeJson); + String nodePath = IoUtils.addFileSeparator(flowCodePath, nodeName); + //上传节点资源到bml，实现节点资源导入，返回新的节点json + String updateNodeJson = nodeInputService.uploadResourceToBmlNew(userName, nodeJson, nodePath, + nodeName, projectName); + //导入第三方节点 + updateNodeJson = nodeInputService.uploadAppConnResourceNew(userName, projectName, + dssFlow, updateNodeJson, updateContextId, nodePath, + workspace, orcVersion, dssLabels); + Map nodeJsonMap = BDPJettyServerHelper.jacksonJson().readValue(updateNodeJson, Map.class); + String nodeParamsJson = readNodeParam(userName, nodePath,nodeName); + + //更新subflowID + String nodeType = nodeJsonMap.get("jobType").toString(); + if ("workflow.subflow".equals(nodeType) && CollectionUtils.isNotEmpty(subflows)) { + String subFlowName = nodeJsonMap.get("title").toString(); + logger.info("subflows:{}", subflows); + List dssFlowList = subflows.stream().filter(subflow -> + subflow.getName().equals(subFlowName) + ).collect(Collectors.toList()); + if (dssFlowList.size() == 1) { + updateNodeJson = nodeInputService.updateNodeSubflowID(updateNodeJson, dssFlowList.get(0).getId()); + nodeJsonMap = BDPJettyServerHelper.jacksonJson().readValue(updateNodeJson, Map.class); + + } else if (dssFlowList.size() > 1) { + logger.error("工程内存在重复的子工作流节点名称，导入失败" + subFlowName); + throw new DSSErrorException(90077, "工程内存在重复的子工作流节点名称，导入失败" + subFlowName); + } else { + logger.error("工程内未能找到子工作流节点，导入失败" + subFlowName); + throw new DSSErrorException(90078, "工程内未能找到子工作流节点，导入失败" + subFlowName); + } + } + if (nodeParamsJson != null && !"null".equalsIgnoreCase(nodeParamsJson)) { + nodeJsonMap.put("params", BDPJettyServerHelper.jacksonJson().readValue(nodeParamsJson, Map.class)); + } + nodeJsonListRes.add(nodeJsonMap); + + } + } + + return workFlowParser.updateFlowJsonWithKey(flowJson, "nodes", nodeJsonListRes); + } + /** + * 导入工作流节点 + * @param userName + * @param projectName + * @param flowJson + * @param dssFlow + * @param workspace + * @param orcVersion + * @param dssLabels + * @return + * @throws DSSErrorException + * @throws IOException + */ + private String inputWorkFlowNodesNewForMultiThread(String userName, String projectName, String flowJson, + DSSFlow dssFlow, String flowCodePath, Workspace workspace, + String orcVersion, List dssLabels) throws DSSErrorException, IOException { + List nodeJsonList = workFlowParser.getWorkFlowNodesJson(flowJson); + if (nodeJsonList == null) { + throw new DSSErrorException(90073, "工作流内没有工作流节点，导入失败 " + dssFlow.getName()); + } + String updateContextId = workFlowParser.getValueWithKey(flowJson, CSCommonUtils.CONTEXT_ID_STR); + if (nodeJsonList.size() == 0) { + return flowJson; + } + List subflows = (List) dssFlow.getChildren(); + List> nodeJsonListRes = new ArrayList<>(); + if (nodeJsonList.size() > 0) { + List>> futures = new ArrayList<>(nodeJsonList.size()); + for (String nodeJson : nodeJsonList) { + Future> future =DssJobThreadPool.nodeImportThreadPool.submit( + ()-> { + try{ + String nodeName = nodeParser.getNodeValue("title", nodeJson); + String nodePath = IoUtils.addFileSeparator(flowCodePath, nodeName); + //上传节点资源到bml，实现节点资源导入，返回新的节点json + String updateNodeJson = nodeInputService.uploadResourceToBmlNew(userName, nodeJson, nodePath, + nodeName, projectName); + //导入第三方节点 + updateNodeJson = nodeInputService.uploadAppConnResourceNew(userName, projectName, + dssFlow, updateNodeJson, updateContextId, nodePath, + workspace, orcVersion, dssLabels); + Map nodeJsonMap = BDPJettyServerHelper.jacksonJson().readValue(updateNodeJson, Map.class); + String nodeParamsJson = readNodeParam(userName, nodePath,nodeName); + + //更新subflowID + String nodeType = nodeJsonMap.get("jobType").toString(); + if ("workflow.subflow".equals(nodeType) && CollectionUtils.isNotEmpty(subflows)) { + String subFlowName = nodeJsonMap.get("title").toString(); + logger.info("subflows:{}", subflows); + List dssFlowList = subflows.stream().filter(subflow -> + subflow.getName().equals(subFlowName) + ).collect(Collectors.toList()); + if (dssFlowList.size() == 1) { + updateNodeJson = nodeInputService.updateNodeSubflowID(updateNodeJson, dssFlowList.get(0).getId()); + nodeJsonMap = BDPJettyServerHelper.jacksonJson().readValue(updateNodeJson, Map.class); + + } else if (dssFlowList.size() > 1) { + logger.error("工程内存在重复的子工作流节点名称，导入失败" + subFlowName); + throw new DSSRuntimeException(90077, "工程内存在重复的子工作流节点名称，导入失败" + subFlowName); + } else { + logger.error("工程内未能找到子工作流节点，导入失败" + subFlowName); + throw new DSSRuntimeException(90078, "工程内未能找到子工作流节点，导入失败" + subFlowName); + } + } + if (nodeParamsJson != null && !"null".equalsIgnoreCase(nodeParamsJson)) { + nodeJsonMap.put("params", BDPJettyServerHelper.jacksonJson().readValue(nodeParamsJson, Map.class)); + } + return nodeJsonMap; + + } catch (IOException e) { + throw new RuntimeException(e); + } + }); + futures.add(future); + } + for (Future> future : futures) { + try{ + Map nodeJsonMap=future.get(); + nodeJsonListRes.add(nodeJsonMap); + } catch (InterruptedException|ExecutionException e) { + logger.error("import-flow-node thread fail msg is ｛｝ ", e); + throw new RuntimeException(e); + } + } + } + return workFlowParser.updateFlowJsonWithKey(flowJson, "nodes", nodeJsonListRes); + } + + private String readNodeParam(String userName,String nodePath,String nodeName){ + String propertiesName = nodeName + NODE_PARAMS_FILE_NAME; + String paramsFilePath = IoUtils.addFileSeparator(nodePath, propertiesName); + return bmlService.readLocalTextFile(userName, paramsFilePath); + } private String inputWorkFlowNodes(String userName, String projectName, String flowJson, DSSFlow dssFlow, String flowPath, Workspace workspace, @@ -254,7 +477,23 @@ private String inputWorkFlowNodes(String userName, String projectName, String fl return workFlowParser.updateFlowJsonWithKey(flowJson, "nodes", nodeJsonListRes); } + private String uploadFlowResourceToBmlNew(String userName, String flowJson, String flowCodePath, String projectName) throws IOException { + List resourceList = workFlowParser.getWorkFlowResources(flowJson); + //上传文件获取resourceId和version save应该是已经有 + if (resourceList != null) { + resourceList.forEach(resource -> { + InputStream resourceInputStream = readFlowResourceNew(userName, resource, flowCodePath); + BmlResource bmlReturnMap = bmlService.upload(userName, resourceInputStream, UUID.randomUUID().toString() + ".json", projectName); + resource.setResourceId(bmlReturnMap.getResourceId()); + resource.setVersion(bmlReturnMap.getVersion()); + }); + if (resourceList.size() == 0) { + return flowJson; + } + } + return workFlowParser.updateFlowJsonWithKey(flowJson, "resources", resourceList); + } private String uploadFlowResourceToBml(String userName, String flowJson, String flowResourcePath, String projectName) throws IOException { List resourceList = workFlowParser.getWorkFlowResources(flowJson); @@ -272,7 +511,10 @@ private String uploadFlowResourceToBml(String userName, String flowJson, String } return workFlowParser.updateFlowJsonWithKey(flowJson, "resources", resourceList); } - + private InputStream readFlowResourceNew(String userName, Resource resource, String flowCodePath) { + String readPath = IoUtils.addFileSeparator(flowCodePath,resource.getFileName()); + return bmlService.readLocalResourceFile(userName, readPath); + } private InputStream readFlowResource(String userName, Resource resource, String flowResourcePath) { // TODO: 2020/3/20 和导出统一,资源都放resouce 如有问题,后再修改 String readPath = flowResourcePath + File.separator + "resource" + File.separator + resource.getResourceId() + ".re"; diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/lock/DSSFlowEditLockManager.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/lock/DSSFlowEditLockManager.java index 2bb3298aa3..efdea30528 100644 --- a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/lock/DSSFlowEditLockManager.java +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/lock/DSSFlowEditLockManager.java @@ -16,23 +16,39 @@ package com.webank.wedatasphere.dss.workflow.lock; +import com.webank.wedatasphere.dss.common.entity.BmlResource; +import com.webank.wedatasphere.dss.common.entity.project.DSSProject; import com.webank.wedatasphere.dss.common.exception.DSSErrorException; +import com.webank.wedatasphere.dss.common.label.LabelRouteVO; +import com.webank.wedatasphere.dss.common.protocol.project.ProjectInfoRequest; +import com.webank.wedatasphere.dss.common.utils.RpcAskUtils; +import com.webank.wedatasphere.dss.git.common.protocol.request.GitCommitRequest; +import com.webank.wedatasphere.dss.git.common.protocol.response.GitCommitResponse; +import com.webank.wedatasphere.dss.orchestrator.common.entity.DSSOrchestratorInfo; +import com.webank.wedatasphere.dss.orchestrator.common.entity.DSSOrchestratorVersion; +import com.webank.wedatasphere.dss.orchestrator.common.entity.OrchestratorVo; +import com.webank.wedatasphere.dss.orchestrator.common.protocol.RequestQuertByAppIdOrchestrator; +import com.webank.wedatasphere.dss.orchestrator.common.protocol.RequestQueryByIdOrchestrator; +import com.webank.wedatasphere.dss.orchestrator.common.protocol.RequestSubmitOrchestratorSync; +import com.webank.wedatasphere.dss.orchestrator.common.ref.OrchestratorRefConstant; +import com.webank.wedatasphere.dss.sender.service.DSSSenderServiceFactory; +import com.webank.wedatasphere.dss.standard.app.sso.Workspace; import com.webank.wedatasphere.dss.workflow.common.entity.DSSFlow; import com.webank.wedatasphere.dss.workflow.constant.DSSWorkFlowConstant; +import com.webank.wedatasphere.dss.workflow.dao.FlowMapper; import com.webank.wedatasphere.dss.workflow.dao.LockMapper; import com.webank.wedatasphere.dss.workflow.entity.DSSFlowEditLock; import org.apache.linkis.DataWorkCloudApplication; import org.apache.linkis.common.utils.Utils; import org.apache.commons.lang.StringUtils; -import org.apache.poi.util.StringUtil; +import org.apache.linkis.rpc.Sender; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.dao.DuplicateKeyException; +import org.springframework.util.CollectionUtils; import java.text.SimpleDateFormat; -import java.util.Date; -import java.util.Optional; -import java.util.UUID; +import java.util.*; import java.util.concurrent.DelayQueue; import java.util.concurrent.Delayed; import java.util.concurrent.TimeUnit; @@ -49,6 +65,8 @@ public class DSSFlowEditLockManager { private static LockMapper lockMapper; + private static FlowMapper flowMapper; + private static final DelayQueue unLockEvents = new DelayQueue<>(); private static final ThreadLocal sdf = ThreadLocal.withInitial(() -> new SimpleDateFormat("yyyy-MM-dd HH:mm:ss")); @@ -60,8 +78,14 @@ protected DSSFlowEditLockManager() { LOGGER.info("unLockEvents移除定时线程开启..."); LOGGER.info("编辑锁超时时间为：{} ms", DSSWorkFlowConstant.DSS_FLOW_EDIT_LOCK_TIMEOUT.getValue()); init(); - //程序重启时，删除所有编辑锁 - lockMapper.deleteALL(); + // 对于工作流状态为已保存的，不自动解锁，该工作流的锁永不过期 + List saveList = lockMapper.selectOrchestratorByStatus(OrchestratorRefConstant.FLOW_STATUS_SAVE); + if (!CollectionUtils.isEmpty(saveList)) { + lockMapper.deleteExpectAfterSave(saveList); + }else { + //程序重启时，删除所有编辑锁 + lockMapper.deleteALL(); + } Utils.defaultScheduler().scheduleAtFixedRate(() -> { try { UnLockEvent pop = unLockEvents.poll(); @@ -168,12 +192,13 @@ private static String generateLock(Long flowID, String username, String owner) t } } - public static void deleteLock(String flowEditLock) throws DSSErrorException { + public static void deleteLock(String flowEditLock, Workspace workspace) throws DSSErrorException { try { if (StringUtils.isNotBlank(flowEditLock)) { DSSFlowEditLock dssFlowEditLock = lockMapper.getFlowEditLockByLockContent(flowEditLock); if (dssFlowEditLock != null) { - lockMapper.clearExpire(sdf.get().format(new Date(System.currentTimeMillis() - DSSWorkFlowConstant.DSS_FLOW_EDIT_LOCK_TIMEOUT.getValue())), dssFlowEditLock.getFlowID()); + Long flowID = dssFlowEditLock.getFlowID(); + lockMapper.clearExpire(sdf.get().format(new Date(System.currentTimeMillis() - DSSWorkFlowConstant.DSS_FLOW_EDIT_LOCK_TIMEOUT.getValue())), flowID); } } } catch (Exception e) { @@ -182,6 +207,46 @@ public static void deleteLock(String flowEditLock) throws DSSErrorException { } } + public static DSSProject getProjectInfo(Long projectId) throws DSSErrorException { + ProjectInfoRequest projectInfoRequest = new ProjectInfoRequest(); + projectInfoRequest.setProjectId(projectId); + DSSProject dssProject = RpcAskUtils.processAskException(DSSSenderServiceFactory.getOrCreateServiceInstance().getProjectServerSender() + .ask(projectInfoRequest), DSSProject.class, ProjectInfoRequest.class); + if (dssProject == null) { + throw new DSSErrorException(90001, "工程不存在"); + } + return dssProject; + } + + public static OrchestratorVo getOrchestratorInfo(Long flowID) throws DSSErrorException{ + Sender orcSender = DSSSenderServiceFactory.getOrCreateServiceInstance().getOrcSender(); + OrchestratorVo orchestratorVo = RpcAskUtils.processAskException(orcSender.ask(new RequestQuertByAppIdOrchestrator(flowID)), + OrchestratorVo.class, RequestQueryByIdOrchestrator.class); + if (orchestratorVo == null) { + throw new DSSErrorException(800001, "编排不存在"); + } + return orchestratorVo; + } + + public static void submitOrchestrator(Long orchestratorId, Long flowId, Workspace workspace, String projectName) { + LOGGER.info("-------=======================begin to add testGit1=======================-------"); + Sender orcSender = DSSSenderServiceFactory.getOrCreateServiceInstance().getOrcSender(); + // 同步提交编排至BML以及push到git + RequestSubmitOrchestratorSync submitOrchestratorSync = new RequestSubmitOrchestratorSync(); + submitOrchestratorSync.setOrchestratorId(orchestratorId); + submitOrchestratorSync.setFlowId(flowId); + submitOrchestratorSync.setComment("force unlock"); + submitOrchestratorSync.setProjectName(projectName); + LabelRouteVO labelRouteVO = new LabelRouteVO(); + labelRouteVO.setRoute("dev"); + submitOrchestratorSync.setLabels(labelRouteVO); + submitOrchestratorSync.setUsername("system"); + submitOrchestratorSync.setWorkspace(workspace); + GitCommitResponse responseWorkflowValidNode = RpcAskUtils.processAskException(orcSender.ask(submitOrchestratorSync), GitCommitResponse.class, RequestSubmitOrchestratorSync.class); + LOGGER.info("-------=======================End to add submit=======================-------: {}", responseWorkflowValidNode); + } + + public static String updateLock(String lock) throws DSSErrorException { if (StringUtils.isBlank(lock)) { throw new DSSErrorException(60066, "update workflow failed because you do not have flowEditLock!"); @@ -205,7 +270,17 @@ public static String updateLock(String lock) throws DSSErrorException { } } - public static boolean isLockExpire(DSSFlowEditLock flowEditLock) { + public static boolean isLockExpire(DSSFlowEditLock flowEditLock) throws DSSErrorException{ + try { + OrchestratorVo orchestratorVo = getOrchestratorInfo(flowEditLock.getFlowID()); + DSSOrchestratorInfo orchestratorInfo = orchestratorVo.getDssOrchestratorInfo(); + String status = lockMapper.selectOrchestratorStatus(orchestratorInfo.getId()); + if (!StringUtils.isEmpty(status) && OrchestratorRefConstant.FLOW_STATUS_SAVE.equals(status)) { + return false; + } + }catch (Exception e) { + return true; + } return System.currentTimeMillis() - flowEditLock.getUpdateTime().getTime() >= DSSWorkFlowConstant.DSS_FLOW_EDIT_LOCK_TIMEOUT.getValue(); } @@ -214,6 +289,7 @@ public static void init() { synchronized (DSSFlowEditLockManager.class) { if (!isInit) { lockMapper = DataWorkCloudApplication.getApplicationContext().getBean(LockMapper.class); + flowMapper = DataWorkCloudApplication.getApplicationContext().getBean(FlowMapper.class); isInit = true; } } diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/parser/DefaultNodeParser.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/parser/DefaultNodeParser.java index bcaf0dce7b..391c347236 100644 --- a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/parser/DefaultNodeParser.java +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/parser/DefaultNodeParser.java @@ -28,6 +28,7 @@ import java.io.IOException; import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -71,6 +72,14 @@ public String getNodeValue(String key, String nodeJson) throws IOException { } + + @Override + public Map getNodeJobContent(String nodeJson) throws IOException { + Map nodeJsonMap = BDPJettyServerHelper.jacksonJson().readValue(nodeJson, Map.class); + return (LinkedHashMap) nodeJsonMap.get("jobContent"); + } + + @Override public List getNodeResource(String nodeJson) { Gson gson = new Gson(); diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/restful/FlowRestfulApi.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/restful/FlowRestfulApi.java index 156b181240..86ed0caca4 100644 --- a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/restful/FlowRestfulApi.java +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/restful/FlowRestfulApi.java @@ -25,10 +25,13 @@ import com.webank.wedatasphere.dss.common.exception.DSSErrorException; import com.webank.wedatasphere.dss.common.label.DSSLabel; import com.webank.wedatasphere.dss.common.label.EnvDSSLabel; +import com.webank.wedatasphere.dss.common.label.LabelRouteVO; import com.webank.wedatasphere.dss.common.utils.AuditLogUtils; +import com.webank.wedatasphere.dss.common.utils.DSSCommonUtils; import com.webank.wedatasphere.dss.common.utils.DSSExceptionUtils; import com.webank.wedatasphere.dss.contextservice.service.ContextService; import com.webank.wedatasphere.dss.contextservice.service.impl.ContextServiceImpl; +import com.webank.wedatasphere.dss.git.common.protocol.GitTree; import com.webank.wedatasphere.dss.orchestrator.common.protocol.ResponseConvertOrchestrator; import com.webank.wedatasphere.dss.standard.app.sso.Workspace; import com.webank.wedatasphere.dss.standard.sso.utils.SSOHelper; @@ -84,11 +87,9 @@ public class FlowRestfulApi { private LockMapper lockMapper; @Autowired private HttpServletRequest httpServletRequest; + @Autowired + private DSSFlowService dssFlowService; - @PostConstruct - public void init() { - AppConnManagerUtils.autoLoadAppConnManager(); - } /** * 添加subflow节点 @@ -106,6 +107,9 @@ public Message addFlow( @RequestBody AddFlowRequest addFlowRequest) throws Error Workspace workspace = SSOHelper.getWorkspace(httpServletRequest); // TODO: 2019/5/23 flowName工程名下唯一校验 String name = addFlowRequest.getName(); + if (name != null && name.length() > 127) { + return Message.error("名称超限，请保持在1到128个字符之间。超限名称：" + name); + } Long parentFlowID = addFlowRequest.getParentFlowID(); // 判断parentFlowID中是否已存在名称为name的subflow if (flowService.checkExistSameSubflow(parentFlowID, name)){ @@ -150,6 +154,7 @@ public Message publishWorkflow(@RequestBody PublishWorkflowRequest publishWorkfl String comment = publishWorkflowRequest.getComment(); Workspace workspace = SSOHelper.getWorkspace(httpServletRequest); String publishUser = SecurityFilter.getLoginUsername(httpServletRequest); + return DSSExceptionUtils.getMessage(() -> publishService.submitPublish(publishUser, workflowId, labels, workspace, comment), taskId -> { if (DSSWorkFlowConstant.PUBLISHING_ERROR_CODE.equals(taskId)) { @@ -167,6 +172,22 @@ public Message publishWorkflow(@RequestBody PublishWorkflowRequest publishWorkfl String.format("用户 %s 发布工作流 %s 失败.", publishUser, workflowId)); } + @RequestMapping(value = "batchPublishWorkflow", method = RequestMethod.POST) + public Message batchPublishWorkflow(@RequestBody BatchPublishWorkflowRequest publishWorkflowRequest) throws Exception { + Workspace workspace = SSOHelper.getWorkspace(httpServletRequest); + String publishUser = SecurityFilter.getLoginUsername(httpServletRequest); + Map labels = new HashMap<>(); + labels.put(EnvDSSLabel.DSS_ENV_LABEL_KEY, publishWorkflowRequest.getLabels().getRoute()); + + try { + publishService.batchPublish(publishWorkflowRequest, workspace, publishUser, labels); + } catch (Exception e) { + return Message.error("批量发布失败，原因为：" + e.getMessage()); + } + return Message.ok("批量发布提交成功"); + + } + /** * 获取发布任务状态 * @@ -247,7 +268,8 @@ public Message updateFlowBaseInfo(@RequestBody UpdateFlowBaseInfoRequest updateF @RequestMapping(value = "get", method = RequestMethod.GET) public Message get(@RequestParam(required = false, name = "flowId") Long flowID, - @RequestParam(required = false, name = "isNotHaveLock") Boolean isNotHaveLock) throws DSSErrorException { + @RequestParam(required = false, name = "isNotHaveLock") Boolean isNotHaveLock, + @RequestParam(required = false, name = "labels") String labels) throws DSSErrorException { String username = SecurityFilter.getLoginUsername(httpServletRequest); LOGGER.info("User {} start to open workflow {}", username, flowID); DSSFlow dssFlow; @@ -256,7 +278,7 @@ public Message get(@RequestParam(required = false, name = "flowId") Long flowID, } catch (NullPointerException e) { return Message.error("The workflow is not exists, please check to delete. (打开了不存在的工作流，请确保是否已删除.)"); } - if (isNotHaveLock != null && isNotHaveLock) { + if ((isNotHaveLock != null && isNotHaveLock) || (StringUtils.isNotEmpty(labels) && DSSCommonUtils.ENV_LABEL_VALUE_PROD.equals(labels))) { return Message.ok().data("flow", dssFlow); } Cookie[] cookies = httpServletRequest.getCookies(); @@ -269,6 +291,7 @@ public Message get(@RequestParam(required = false, name = "flowId") Long flowID, String flowEditLock = DSSFlowEditLockManager.tryAcquireLock(dssFlow, username, ticketId); dssFlow.setFlowEditLock(flowEditLock); } + flowService.getRootFlowProxy(dssFlow); } catch (DSSErrorException e) { if (DSSWorkFlowConstant.EDIT_LOCK_ERROR_CODE == e.getErrCode()) { DSSFlowEditLock flowEditLock = lockMapper.getFlowEditLockByID(flowID); @@ -313,18 +336,9 @@ public Message saveFlow( @RequestBody SaveFlowRequest saveFlowRequest) throws IO Long flowID = saveFlowRequest.getId(); String jsonFlow = saveFlowRequest.getJson(); Workspace workspace = SSOHelper.getWorkspace(httpServletRequest); - DSSFlow dssFlow = flowService.getFlowByID(flowID); String workspaceName = saveFlowRequest.getWorkspaceName(); String projectName = saveFlowRequest.getProjectName(); - // 判断工作流中是否存在命名相同的节点 - if (flowService.checkIsExistSameFlow(jsonFlow)) { - return Message.error("It exists same flow.(存在相同的节点)"); - } - // 判断工作流中是否有子工作流未被保存 - List unSaveNodes = flowService.checkIsSave(flowID, jsonFlow); - if (CollectionUtils.isNotEmpty(unSaveNodes)) { - return Message.error("工作流中存在子工作流未被保存，请先保存子工作流：" + unSaveNodes); - } + LabelRouteVO labels = saveFlowRequest.getLabels(); String userName = SecurityFilter.getLoginUsername(httpServletRequest); String version; @@ -335,9 +349,14 @@ public Message saveFlow( @RequestBody SaveFlowRequest saveFlowRequest) throws IO if (flowEditLock != null && !flowEditLock.getOwner().equals(ticketId)) { return Message.error("当前工作流被用户" + flowEditLock.getUsername() + "已锁定编辑，您编辑的内容不能再被保存。如有疑问，请与" + flowEditLock.getUsername() + "确认"); } - version = flowService.saveFlow(flowID, jsonFlow, null, userName, workspaceName, projectName); - AuditLogUtils.printLog( username,workspace.getWorkspaceId(), workspaceName,TargetTypeEnum.WORKFLOW, - flowID,dssFlow.getName(),OperateTypeEnum.UPDATE,saveFlowRequest); + try { + version = flowService.saveFlow(flowID, jsonFlow, null, userName, workspaceName, projectName, labels); + }catch (Exception e) { + return Message.error("保存失败，原因为：" + e.getMessage()); + } + DSSFlow dssFlow = flowService.getFlowByID(flowID); + AuditLogUtils.printLog(username, workspace.getWorkspaceId(), workspaceName, TargetTypeEnum.WORKFLOW, + flowID, dssFlow.getName(), OperateTypeEnum.UPDATE, saveFlowRequest); return Message.ok().data("flowVersion", version); } @@ -366,8 +385,25 @@ public Message getExtraToolBars( @RequestBody GetExtraToolBarsRequest getExtraTo @RequestMapping(value = "/deleteFlowEditLock/{flowEditLock}", method = RequestMethod.POST) public Message deleteFlowEditLock(HttpServletRequest req, @PathVariable("flowEditLock") String flowEditLock) throws DSSErrorException { - DSSFlowEditLockManager.deleteLock(flowEditLock); + String userName = SecurityFilter.getLoginUsername(httpServletRequest); + Workspace workspace = SSOHelper.getWorkspace(httpServletRequest); + DSSFlowEditLockManager.deleteLock(flowEditLock, workspace); return Message.ok(); } + @RequestMapping(value = "/batchEditFlowNode", method = RequestMethod.POST) + public Message batchEditFlow(HttpServletRequest req, @RequestBody BatchEditFlowRequest batchEditFlowRequest) { + String userName = SecurityFilter.getLoginUsername(httpServletRequest); + Workspace workspace = SSOHelper.getWorkspace(httpServletRequest); + //若工作流已经被其他用户抢锁，则当前用户不能再保存成功 + String ticketId = Arrays.stream(httpServletRequest.getCookies()).filter(cookie -> DSSWorkFlowConstant.BDP_USER_TICKET_ID.equals(cookie.getName())) + .findFirst().map(Cookie::getValue).get(); + try { + dssFlowService.batchEditFlow(batchEditFlowRequest, ticketId, workspace, userName); + } catch (Exception e) { + return Message.error("批量编辑失败，原因为：" + e.getMessage()); + } + return Message.ok("批量编辑成功"); + } + } diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/restful/NodeRestfulApi.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/restful/NodeRestfulApi.java index 5ed20bbb48..bea16815ee 100644 --- a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/restful/NodeRestfulApi.java +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/restful/NodeRestfulApi.java @@ -30,10 +30,8 @@ import com.webank.wedatasphere.dss.workflow.common.parser.WorkFlowParser; import com.webank.wedatasphere.dss.workflow.cs.DSSCSHelper; import com.webank.wedatasphere.dss.workflow.entity.*; -import com.webank.wedatasphere.dss.workflow.entity.request.AppConnNodeUrlRequest; -import com.webank.wedatasphere.dss.workflow.entity.request.BatchDeleteAppConnNodeRequest; -import com.webank.wedatasphere.dss.workflow.entity.request.CreateExternalNodeRequest; -import com.webank.wedatasphere.dss.workflow.entity.request.UpdateExternalNodeRequest; +import com.webank.wedatasphere.dss.workflow.entity.request.*; +import com.webank.wedatasphere.dss.workflow.entity.response.*; import com.webank.wedatasphere.dss.workflow.entity.vo.NodeGroupVO; import com.webank.wedatasphere.dss.workflow.entity.vo.NodeInfoVO; import com.webank.wedatasphere.dss.workflow.entity.vo.NodeUiVO; @@ -42,6 +40,7 @@ import com.webank.wedatasphere.dss.workflow.service.WorkflowNodeService; import org.apache.commons.io.FileUtils; import org.apache.commons.lang.ArrayUtils; +import org.apache.commons.lang.StringUtils; import org.apache.linkis.server.Message; import org.apache.linkis.server.security.SecurityFilter; import org.slf4j.Logger; @@ -69,7 +68,7 @@ public class NodeRestfulApi { @Autowired private WorkFlowParser workFlowParser; - @RequestMapping(value = "/listNodeType",method = RequestMethod.GET) + @RequestMapping(value = "/listNodeType", method = RequestMethod.GET) public Message listNodeType(HttpServletRequest req) { Function supplier = internationalization(req, NodeGroup::getNameEn, NodeGroup::getName); List groupVos = new ArrayList<>(); @@ -128,7 +127,7 @@ private NodeInfoVO transfer(NodeInfo nodeInfo, HttpServletRequest req) throws IO Set keySet = new HashSet<>(nodeInfo.getNodeUis().size()); for (NodeUi nodeUi : nodeInfo.getNodeUis()) { //避免重复的ui key，因为第三方组件可能会重复配置。 - if(keySet.contains(nodeUi.getKey())){ + if (keySet.contains(nodeUi.getKey())) { continue; } NodeUiVO nodeUiVO = new NodeUiVO(); @@ -145,7 +144,7 @@ private NodeInfoVO transfer(NodeInfo nodeInfo, HttpServletRequest req) throws IO } - @RequestMapping(path ="nodeIcon/{nodeType}", method = RequestMethod.GET) + @RequestMapping(path = "nodeIcon/{nodeType}", method = RequestMethod.GET) public void getIcon(HttpServletResponse response, @PathVariable("nodeType") String nodeType) throws IOException { byte[] icon = workflowNodeService.getNodeIcon(nodeType); response.setContentType("image/svg+xml"); @@ -154,7 +153,7 @@ public void getIcon(HttpServletResponse response, @PathVariable("nodeType") Stri response.getOutputStream().write(icon); } - @RequestMapping(value = "/createAppConnNode",method = RequestMethod.POST) + @RequestMapping(value = "/createAppConnNode", method = RequestMethod.POST) public Message createExternalNode(HttpServletRequest req, @RequestBody CreateExternalNodeRequest createExternalNodeRequest) throws DSSErrorException, IllegalAccessException, ExternalOperationFailedException, InstantiationException { String userName = SecurityFilter.getLoginUsername(req); Workspace workspace = SSOHelper.getWorkspace(req); @@ -184,9 +183,9 @@ public Message createExternalNode(HttpServletRequest req, @RequestBody CreateExt node.setParams(params); node.setJobContent(params); //补充json信息,方便appConn去解析获取响应的值 - if(params.containsKey(DSSJobContentConstant.UP_STREAM_KEY)) { + if (params.containsKey(DSSJobContentConstant.UP_STREAM_KEY)) { List dssNodes = null; - if(params.get(DSSJobContentConstant.UP_STREAM_KEY).equals("empty")) { + if (params.get(DSSJobContentConstant.UP_STREAM_KEY).equals("empty")) { // dssNodes = workFlowParser.getWorkFlowNodes(flowContent); params.remove(DSSJobContentConstant.UP_STREAM_KEY); logger.info("Create a node that is not bound to an upstream node."); @@ -194,7 +193,7 @@ public Message createExternalNode(HttpServletRequest req, @RequestBody CreateExt String[] upStreams = ((String) params.get(DSSJobContentConstant.UP_STREAM_KEY)).split(","); dssNodes = workFlowParser.getWorkFlowNodes(flowContent).stream() .filter(dssNode -> ArrayUtils.contains(upStreams, dssNode.getId())).collect(Collectors.toList()); - if(dssNodes.isEmpty() && upStreams.length > 0) { + if (dssNodes.isEmpty() && upStreams.length > 0) { return Message.error("Create node failed! Caused by: the banding up-stream nodes are not exists(绑定的上游节点不存在)."); } params.put(DSSJobContentConstant.UP_STREAM_KEY, dssNodes); @@ -205,7 +204,7 @@ public Message createExternalNode(HttpServletRequest req, @RequestBody CreateExt return Message.ok().data("result", jobContent); } - @RequestMapping(value = "/updateAppConnNode",method = RequestMethod.POST) + @RequestMapping(value = "/updateAppConnNode", method = RequestMethod.POST) public Message updateExternalNode(HttpServletRequest req, @RequestBody UpdateExternalNodeRequest updateExternalNodeRequest) throws IllegalAccessException, ExternalOperationFailedException, InstantiationException { String userName = SecurityFilter.getLoginUsername(req); Workspace workspace = SSOHelper.getWorkspace(req); @@ -228,7 +227,7 @@ public Message updateExternalNode(HttpServletRequest req, @RequestBody UpdateExt return Message.ok().data("result", node.getJobContent()); } - @RequestMapping(value = "/deleteAppConnNode",method = RequestMethod.POST) + @RequestMapping(value = "/deleteAppConnNode", method = RequestMethod.POST) public Message deleteExternalNode(HttpServletRequest req, @RequestBody UpdateExternalNodeRequest updateExternalNodeRequest) throws IllegalAccessException, ExternalOperationFailedException, InstantiationException { String userName = SecurityFilter.getLoginUsername(req); Workspace workspace = SSOHelper.getWorkspace(req); @@ -251,12 +250,12 @@ public Message deleteExternalNode(HttpServletRequest req, @RequestBody UpdateExt return Message.ok().data("result", node.getJobContent()); } - @RequestMapping(value = "/batchDeleteAppConnNode",method = RequestMethod.POST) + @RequestMapping(value = "/batchDeleteAppConnNode", method = RequestMethod.POST) public Message batchDeleteAppConnNode(HttpServletRequest req, @RequestBody BatchDeleteAppConnNodeRequest batchDeleteAppConnNodeRequest) throws IllegalAccessException, ExternalOperationFailedException, InstantiationException { String userName = SecurityFilter.getLoginUsername(req); Workspace workspace = SSOHelper.getWorkspace(req); - List> jsonList = batchDeleteAppConnNodeRequest.getNodes(); - jsonList.forEach(json ->{ + List> jsonList = batchDeleteAppConnNodeRequest.getNodes(); + jsonList.forEach(json -> { Long projectId = Long.parseLong(json.get("projectID").toString()); String nodeType = json.get("nodeType").toString(); Long flowId = (Long) json.get("flowID"); @@ -275,10 +274,10 @@ public Message batchDeleteAppConnNode(HttpServletRequest req, @RequestBody Batch workflowNodeService.deleteNode(userName, node); }); - return Message.ok("success").data("result","success"); + return Message.ok("success").data("result", "success"); } - @RequestMapping(value = "/getAppConnNodeUrl",method = RequestMethod.POST) + @RequestMapping(value = "/getAppConnNodeUrl", method = RequestMethod.POST) public Message getAppConnNodeUrl(HttpServletRequest req, @RequestBody AppConnNodeUrlRequest appConnNodeUrlRequest) { String userName = SecurityFilter.getLoginUsername(req); Workspace workspace = SSOHelper.getWorkspace(req); @@ -299,5 +298,226 @@ public Message getAppConnNodeUrl(HttpServletRequest req, @RequestBody AppConnNod String jumpUrl = workflowNodeService.getNodeJumpUrl(params, node, userName); return Message.ok().data("jumpUrl", jumpUrl); } + + + /** + * 查询数据节点信息 + **/ + @RequestMapping(value = "/queryDataDevelopNode", method = RequestMethod.POST) + public Message queryDataDevelopNode(HttpServletRequest req, @RequestBody DataDevelopNodeRequest dataDevelopNodeRequest) { + + String username = SecurityFilter.getLoginUsername(req); + Workspace workspace = SSOHelper.getWorkspace(req); + + if(dataDevelopNodeRequest.getPageNow() == null){ + dataDevelopNodeRequest.setPageNow(1); + } + + if(dataDevelopNodeRequest.getPageSize() == null){ + dataDevelopNodeRequest.setPageSize(10); + } + + DataDevelopNodeResponse dataDevelopNodeResponse = dssFlowService.queryDataDevelopNodeList(username, workspace, dataDevelopNodeRequest); + + return Message.ok().data("data", dataDevelopNodeResponse.getDataDevelopNodeInfoList()).data("total", dataDevelopNodeResponse.getTotal()); + + } + + + /** + * 查询数据节点参数详情 + **/ + @RequestMapping(value = "/getDataDevelopNodeContent", method = RequestMethod.GET) + public Message getDataDevelopNodeContent(HttpServletRequest req, + @RequestParam("nodeId") String nodeId, + @RequestParam("contentId") Long contentId) throws DSSErrorException { + + if (StringUtils.isBlank(nodeId) || contentId == null) { + return Message.error("输入的参数为空，请检查参数信息"); + } + + Map data = dssFlowService.getDataDevelopNodeContent(nodeId, contentId); + return Message.ok().data("data", data); + + } + + /** + * 查询数据可视化节点 + **/ + @RequestMapping(value = "/queryDataViewNode", method = RequestMethod.POST) + public Message queryDataViewNode(HttpServletRequest req, @RequestBody DataViewNodeRequest dataViewNodeRequest) { + + String username = SecurityFilter.getLoginUsername(req); + Workspace workspace = SSOHelper.getWorkspace(req); + + if(dataViewNodeRequest.getPageNow() == null){ + dataViewNodeRequest.setPageNow(1); + } + + if(dataViewNodeRequest.getPageSize() == null){ + dataViewNodeRequest.setPageSize(10); + } + + DataViewNodeResponse dataViewNodeResponse = dssFlowService.queryDataViewNode(username, workspace, dataViewNodeRequest); + + return Message.ok().data("data", dataViewNodeResponse.getDataDevelopNodeInfoList()).data("total", dataViewNodeResponse.getTotal()); + + } + + + /** + * 节点类型 + **/ + @RequestMapping(value = "/queryNodeTypeNameList", method = RequestMethod.GET) + public Message queryNodeTypeName(@RequestParam("groupNameEn") String groupNameEn) { + + List nodeInfoList = dssFlowService.getNodeInfoByGroupName(groupNameEn); + + List nodeTypeNameList = nodeInfoList.stream().map(NodeInfo::getName).collect(Collectors.toList()); + + return Message.ok().data("data", nodeTypeNameList); + + } + + @RequestMapping(value = "/queryFlowNameList",method = RequestMethod.GET) + public Message queryNodeNameList(HttpServletRequest req, + @RequestParam("groupNameEn") String groupNameEn, + @RequestParam(value = "nodeTypeName",required = false) String nodeTypeName) { + + String username = SecurityFilter.getLoginUsername(req); + Workspace workspace = SSOHelper.getWorkspace(req); + + DSSFlowName dssFlowName = dssFlowService.queryFlowNameList(username, workspace, groupNameEn,nodeTypeName); + + return Message.ok().data("nodeNameList", dssFlowName.getNodeNameList()) + .data("orchestratorNameList", dssFlowName.getOrchestratorNameList()) + .data("templateNameList", dssFlowName.getTemplateNameList()); + + } + + + /* + * 视图ID + * **/ + @RequestMapping(value = "/queryViewId",method = RequestMethod.GET) + public Message queryViewId(HttpServletRequest req) { + String username = SecurityFilter.getLoginUsername(req); + Workspace workspace = SSOHelper.getWorkspace(req); + List viewIdList = dssFlowService.queryViewId(workspace, username); + return Message.ok().data("data", viewIdList); + } + + + + @RequestMapping(value = "/queryDataCheckerNode",method = RequestMethod.POST) + public Message queryDataCheckerNode(HttpServletRequest req,@RequestBody DataCheckerNodeRequest request){ + + String username = SecurityFilter.getLoginUsername(req); + Workspace workspace = SSOHelper.getWorkspace(req); + + if(request.getPageNow() == null){ + request.setPageNow(1); + } + + if(request.getPageSize() == null){ + request.setPageSize(10); + } + + DataCheckerNodeResponse dataCheckerNodeResponse = dssFlowService.queryDataCheckerNode(username, workspace, request); + + return Message.ok().data("data", dataCheckerNodeResponse.getDataCheckerNodeInfoList()).data("total", dataCheckerNodeResponse.getTotal()); + + } + + + @RequestMapping(value = "/queryEventSenderNode",method = RequestMethod.POST) + public Message queryEventSenderNode(HttpServletRequest req,@RequestBody EventSenderNodeRequest request){ + + String username = SecurityFilter.getLoginUsername(req); + Workspace workspace = SSOHelper.getWorkspace(req); + + if(request.getPageNow() == null){ + request.setPageNow(1); + } + + if(request.getPageSize() == null){ + request.setPageSize(10); + } + + EventSenderNodeResponse eventSenderNodeResponse = dssFlowService.queryEventSenderNode(username, workspace, request); + + return Message.ok().data("data", eventSenderNodeResponse.getEventSenderNodeInfoList()).data("total", eventSenderNodeResponse.getTotal()); + + } + + @RequestMapping(value = "/queryNodeInfoByPath",method = RequestMethod.POST) + public Message queryEventSenderNode(HttpServletRequest req,@RequestBody QueryNodeInfoByPathRequest request){ + // 1.根据path解析出节点名，编排名。test1/subFlow_9264/subFlow_8959/sql_5205/sql_5205.sql + Long projectId = request.getProjectId(); + String path = request.getPath(); + if (path == null || !path.contains("/")) { + return Message.error("请求参数不合法，必须包含节点path。path:"+path); + } + String[] strArray=path.split("/"); + int nodeNameIndex=strArray.length-2; + int orchestratorNameIndex=0; + String nodeName = strArray[nodeNameIndex]; + String orchestratorName = strArray[orchestratorNameIndex]; + //2.通过项目id、编排名，找打编排id。 select id from dss_orchestrator_info doi where project_id =? and name=? + //3.通过编排id，找到所有的节点。select id from dss_workflow_node_content where orchestrator_id in 第二步 + //4.根据节点名，过滤节点。拿到节点在的flowid。 + QueryNodeInfoByPathResponse queryNodeInfoByPathResponse = dssFlowService.queryNodeInfo(projectId, + orchestratorName,nodeName); + if (queryNodeInfoByPathResponse == null) { + logger.error("找不到节点所在的工作流信息。projectId:{},path:{}", request.getProjectId(), request.getPath()); + return Message.error("找不到节点所在的工作流信息。请检查工作流最新版本是否成功提交。"); + } + return Message.ok().data("data", queryNodeInfoByPathResponse); + + } + + + + @RequestMapping(value = "/queryEventReceiveNode",method = RequestMethod.POST) + public Message queryEventReceiveNode(HttpServletRequest req,@RequestBody EventReceiverNodeRequest request){ + + String username = SecurityFilter.getLoginUsername(req); + Workspace workspace = SSOHelper.getWorkspace(req); + + if(request.getPageNow() == null){ + request.setPageNow(1); + } + + if(request.getPageSize() == null){ + request.setPageSize(10); + } + + EventReceiveNodeResponse eventReceiveNodeResponse = dssFlowService.queryEventReceiveNode(username, workspace, request); + + return Message.ok().data("data", eventReceiveNodeResponse.getEventReceiverNodeInfoList()).data("total", eventReceiveNodeResponse.getTotal()); + + } + + /* + * 信号节点 sourceType + * **/ + @RequestMapping(value = "/querySourceType",method = RequestMethod.GET) + public Message querySourceType(HttpServletRequest req) { + String username = SecurityFilter.getLoginUsername(req); + Workspace workspace = SSOHelper.getWorkspace(req); + return Message.ok().data("data", dssFlowService.querySourceType(workspace, username)); + } + + + /* + * 信号节点 JobDesc + * **/ + @RequestMapping(value = "/queryJobDesc",method = RequestMethod.GET) + public Message queryJobDesc(HttpServletRequest req) { + String username = SecurityFilter.getLoginUsername(req); + Workspace workspace = SSOHelper.getWorkspace(req); + return Message.ok().data("jobDesc", dssFlowService.queryJobDesc(workspace, username)); + } + } diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/scheduler/DssJobThreadPool.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/scheduler/DssJobThreadPool.java index e715566612..4fbf932141 100644 --- a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/scheduler/DssJobThreadPool.java +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/scheduler/DssJobThreadPool.java @@ -24,10 +24,11 @@ public class DssJobThreadPool { - private static ExecutorService executorService = Utils.newFixedThreadPool(1000, "project-publish", false); - private static ExecutorService executorServiceDeamon = Utils.newFixedThreadPool(1000, "project-publish-deamon", true); - public static ExecutorService nodeExportThreadPool = Utils.newFixedThreadPool(NODE_EXPORT_IMPORT_THREAD_NUM.getValue(), "workflowNode-ExportImport-Thread-", false); - + private static ExecutorService executorService = Utils.newFixedThreadPool(100, "project-publish", false); + private static ExecutorService executorServiceDeamon = Utils.newFixedThreadPool(100, "project-publish-deamon", true); + public static ExecutorService nodeExportThreadPool = Utils.newFixedThreadPool(NODE_EXPORT_IMPORT_THREAD_NUM.getValue(), "workflowNode-Export-Thread-", true); + public static ExecutorService nodeImportThreadPool = Utils.newFixedThreadPool(NODE_EXPORT_IMPORT_THREAD_NUM.getValue(), "workflowNode-Import-Thread-", true); + public static ExecutorService nodeUploadThreadPool = Utils.newFixedThreadPool(NODE_EXPORT_IMPORT_THREAD_NUM.getValue(), "workflowNode-Upload-Thread-", true); public static ExecutorService get() { return executorService; } diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/service/DSSFlowService.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/service/DSSFlowService.java index a001f038ac..ec248bc3a9 100644 --- a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/service/DSSFlowService.java +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/service/DSSFlowService.java @@ -18,13 +18,19 @@ import com.webank.wedatasphere.dss.common.exception.DSSErrorException; import com.webank.wedatasphere.dss.common.label.DSSLabel; +import com.webank.wedatasphere.dss.common.label.LabelRouteVO; import com.webank.wedatasphere.dss.standard.app.sso.Workspace; import com.webank.wedatasphere.dss.workflow.common.entity.DSSFlow; +import com.webank.wedatasphere.dss.workflow.entity.DSSFlowName; +import com.webank.wedatasphere.dss.workflow.entity.NodeInfo; +import com.webank.wedatasphere.dss.workflow.entity.request.*; +import com.webank.wedatasphere.dss.workflow.entity.response.*; import com.webank.wedatasphere.dss.workflow.entity.vo.ExtraToolBarsVO; import org.apache.linkis.common.exception.ErrorException; import java.io.IOException; import java.util.List; +import java.util.Map; public interface DSSFlowService { DSSFlow getFlowByID(Long id); @@ -53,8 +59,19 @@ String saveFlow(Long flowId, String comment, String userName, String workspaceName, - String projectName - ) throws IOException; + String projectName, + LabelRouteVO labels + ) throws Exception; + + void batchEditFlow(BatchEditFlowRequest batchEditFlowRequest, String ticketId, Workspace workspace, String userName) throws Exception; + + void updateTOSaveStatus(Long projectId, Long flowID, Long orchestratorId) throws Exception; + + void deleteFlowMetaData(Long orchestratorId); + + void saveAllFlowMetaData(Long flowId, Long orchestratorId); + + void saveFlowMetaData(Long flowID, String jsonFlow, Long orchestratorId); DSSFlow copyRootFlow(Long rootFlowId, String userName, Workspace workspace, String projectName, String version, String contextIdStr, @@ -72,4 +89,36 @@ DSSFlow copyRootFlow(Long rootFlowId, String userName, Workspace workspace, boolean checkIsExistSameFlow(String jsonFlow); List checkIsSave(Long parentFlowID, String jsonFlow); + + String getFlowJson(String userName, String projectName, DSSFlow dssFlow); + + DataDevelopNodeResponse queryDataDevelopNodeList(String username, Workspace workspace, DataDevelopNodeRequest request); + + Map getDataDevelopNodeContent(String nodeId, Long contentId) throws DSSErrorException; + + DataViewNodeResponse queryDataViewNode(String username, Workspace workspace, DataViewNodeRequest request); + + List getNodeInfoByGroupName(String groupNameEn); + + DSSFlowName queryFlowNameList(String username, Workspace workspace, String groupNameEn,String nodeTypeName); + + List queryViewId(Workspace workspace,String username); + + DataCheckerNodeResponse queryDataCheckerNode(String username, Workspace workspace, DataCheckerNodeRequest request); + + EventSenderNodeResponse queryEventSenderNode(String username, Workspace workspace, EventSenderNodeRequest request); + QueryNodeInfoByPathResponse queryNodeInfo(Long projectId, + String orchestratorName,String nodeName); + + EventReceiveNodeResponse queryEventReceiveNode(String username, Workspace workspace, EventReceiverNodeRequest request); + + List queryJobDesc(Workspace workspace, String username); + + List querySourceType(Workspace workspace, String username); + + void getAllOldFlowId(Long flowId, List flowIdList); + + void deleteNodeContent(List flowIdList); + + void getRootFlowProxy(DSSFlow dssFlow); } diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/service/PublishService.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/service/PublishService.java index a458c70987..48ac60b112 100644 --- a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/service/PublishService.java +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/service/PublishService.java @@ -16,17 +16,22 @@ package com.webank.wedatasphere.dss.workflow.service; +import com.webank.wedatasphere.dss.common.exception.DSSErrorException; import com.webank.wedatasphere.dss.orchestrator.common.protocol.ResponseConvertOrchestrator; import com.webank.wedatasphere.dss.standard.app.sso.Workspace; +import com.webank.wedatasphere.dss.workflow.common.entity.DSSFlow; +import com.webank.wedatasphere.dss.workflow.entity.request.BatchPublishWorkflowRequest; import java.util.Map; public interface PublishService { - String submitPublish(String publishUser, Long workflowId, + String submitPublish(String publishUser, Long workFlowId, Map dssLabel, Workspace workspace, String comment) throws Exception; + void batchPublish(BatchPublishWorkflowRequest publishWorkflowRequest, Workspace workspace, String convertUser, Map dssLabel) throws Exception; + ResponseConvertOrchestrator getStatus(String username, String taskId) throws Exception; } diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/service/impl/DSSFlowServiceImpl.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/service/impl/DSSFlowServiceImpl.java index 07c78ae627..b85d1d3cd9 100644 --- a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/service/impl/DSSFlowServiceImpl.java +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/service/impl/DSSFlowServiceImpl.java @@ -16,55 +16,73 @@ package com.webank.wedatasphere.dss.workflow.service.impl; - +import cn.hutool.core.collection.CollUtil; import com.google.common.reflect.TypeToken; -import com.google.gson.Gson; -import com.google.gson.JsonObject; -import com.google.gson.JsonParser; +import com.google.gson.*; import com.webank.wedatasphere.dss.common.entity.BmlResource; import com.webank.wedatasphere.dss.common.entity.Resource; import com.webank.wedatasphere.dss.common.entity.node.DSSEdge; import com.webank.wedatasphere.dss.common.entity.node.DSSNode; import com.webank.wedatasphere.dss.common.entity.node.DSSNodeDefault; import com.webank.wedatasphere.dss.common.entity.node.Node; +import com.webank.wedatasphere.dss.common.entity.project.DSSProject; import com.webank.wedatasphere.dss.common.exception.DSSErrorException; import com.webank.wedatasphere.dss.common.exception.DSSRuntimeException; import com.webank.wedatasphere.dss.common.label.DSSLabel; -import com.webank.wedatasphere.dss.common.utils.DSSCommonUtils; -import com.webank.wedatasphere.dss.common.utils.IoUtils; -import com.webank.wedatasphere.dss.common.utils.MapUtils; +import com.webank.wedatasphere.dss.common.label.LabelRouteVO; +import com.webank.wedatasphere.dss.common.protocol.project.*; +import com.webank.wedatasphere.dss.common.utils.*; import com.webank.wedatasphere.dss.contextservice.service.ContextService; import com.webank.wedatasphere.dss.contextservice.service.impl.ContextServiceImpl; +import com.webank.wedatasphere.dss.orchestrator.common.entity.DSSOrchestratorVersion; +import com.webank.wedatasphere.dss.orchestrator.common.entity.OrchestratorVo; +import com.webank.wedatasphere.dss.orchestrator.common.protocol.*; +import com.webank.wedatasphere.dss.orchestrator.common.ref.OrchestratorRefConstant; +import com.webank.wedatasphere.dss.sender.service.DSSSenderServiceFactory; import com.webank.wedatasphere.dss.standard.app.development.utils.DSSJobContentConstant; import com.webank.wedatasphere.dss.standard.app.sso.Workspace; +import com.webank.wedatasphere.dss.workflow.WorkFlowManager; import com.webank.wedatasphere.dss.workflow.common.entity.DSSFlow; import com.webank.wedatasphere.dss.workflow.common.entity.DSSFlowRelation; import com.webank.wedatasphere.dss.workflow.common.parser.NodeParser; import com.webank.wedatasphere.dss.workflow.common.parser.WorkFlowParser; -import com.webank.wedatasphere.dss.workflow.constant.DSSWorkFlowConstant; +import com.webank.wedatasphere.dss.workflow.constant.SignalNodeConstant; +import com.webank.wedatasphere.dss.workflow.constant.WorkflowNodeGroupEnum; import com.webank.wedatasphere.dss.workflow.core.WorkflowFactory; import com.webank.wedatasphere.dss.workflow.core.entity.Workflow; import com.webank.wedatasphere.dss.workflow.core.entity.WorkflowWithContextImpl; import com.webank.wedatasphere.dss.workflow.core.json2flow.JsonToFlowParser; -import com.webank.wedatasphere.dss.workflow.dao.FlowMapper; -import com.webank.wedatasphere.dss.workflow.dao.NodeInfoMapper; -import com.webank.wedatasphere.dss.workflow.entity.CommonAppConnNode; -import com.webank.wedatasphere.dss.workflow.entity.NodeInfo; +import com.webank.wedatasphere.dss.workflow.dao.*; +import com.webank.wedatasphere.dss.workflow.dto.NodeContentDO; +import com.webank.wedatasphere.dss.workflow.dto.NodeContentUIDO; +import com.webank.wedatasphere.dss.workflow.dto.NodeMetaDO; +import com.webank.wedatasphere.dss.workflow.dto.NodeTypeDO; +import com.webank.wedatasphere.dss.workflow.entity.*; +import com.webank.wedatasphere.dss.workflow.entity.request.*; +import com.webank.wedatasphere.dss.workflow.entity.response.*; +import com.webank.wedatasphere.dss.workflow.entity.request.BatchEditFlowRequest; +import com.webank.wedatasphere.dss.workflow.entity.request.DataDevelopNodeRequest; +import com.webank.wedatasphere.dss.workflow.entity.request.DataViewNodeRequest; +import com.webank.wedatasphere.dss.workflow.entity.request.EditFlowRequest; +import com.webank.wedatasphere.dss.workflow.entity.response.DataDevelopNodeResponse; +import com.webank.wedatasphere.dss.workflow.entity.response.DataViewNodeResponse; import com.webank.wedatasphere.dss.workflow.entity.vo.ExtraToolBarsVO; import com.webank.wedatasphere.dss.workflow.io.export.NodeExportService; import com.webank.wedatasphere.dss.workflow.io.input.NodeInputService; +import com.webank.wedatasphere.dss.workflow.lock.DSSFlowEditLockManager; import com.webank.wedatasphere.dss.workflow.lock.Lock; import com.webank.wedatasphere.dss.common.service.BMLService; +import com.webank.wedatasphere.dss.workflow.scheduler.DssJobThreadPool; import com.webank.wedatasphere.dss.workflow.service.DSSFlowService; import com.webank.wedatasphere.dss.workflow.service.SaveFlowHook; import com.webank.wedatasphere.dss.workflow.service.WorkflowNodeService; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.exception.ExceptionUtils; -import org.apache.linkis.common.conf.CommonVars; import org.apache.linkis.common.exception.ErrorException; import org.apache.linkis.cs.client.utils.SerializeHelper; import org.apache.linkis.cs.common.utils.CSCommonUtils; +import org.apache.linkis.rpc.Sender; import org.apache.linkis.server.BDPJettyServerHelper; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -78,7 +96,10 @@ import java.io.File; import java.io.IOException; import java.io.InputStream; +import java.time.format.DateTimeFormatter; import java.util.*; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; import java.util.stream.Collectors; import static com.webank.wedatasphere.dss.workflow.constant.DSSWorkFlowConstant.*; @@ -89,6 +110,8 @@ public class DSSFlowServiceImpl implements DSSFlowService { @Autowired private FlowMapper flowMapper; @Autowired + private LockMapper lockMapper; + @Autowired private NodeInfoMapper nodeInfoMapper; @Autowired private NodeInputService nodeInputService; @@ -108,9 +131,26 @@ public class DSSFlowServiceImpl implements DSSFlowService { @Autowired private SaveFlowHook saveFlowHook; + @Autowired + private WorkFlowManager workFlowManager; + @Autowired + private NodeContentMapper nodeContentMapper; + @Autowired + private NodeContentUIMapper nodeContentUIMapper; + @Autowired + private NodeMetaMapper nodeMetaMapper; + + private static final DateTimeFormatter DTF = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"); + private static ContextService contextService = ContextServiceImpl.getInstance(); - private static final String TOKEN = CommonVars.apply("wds.dss.workspace.token", "").getValue(); + private static final String nodeUITitleKey = "title"; + + private static final String nodeUIViewIdKey = "viewId"; + + protected Sender getOrchestratorSender() { + return DSSSenderServiceFactory.getOrCreateServiceInstance().getOrcSender(); + } @Override public DSSFlow getFlowByID(Long id) { @@ -186,7 +226,7 @@ public DSSFlow addFlow(DSSFlow dssFlow, @Transactional(rollbackFor = DSSErrorException.class) @Override public DSSFlow addSubFlow(DSSFlow dssFlow, Long parentFlowID, String contextIDStr, String orcVersion, String schedulerAppConn) throws DSSErrorException { - if (checkExistSameSubflow(parentFlowID, dssFlow.getName())){ + if (checkExistSameSubflow(parentFlowID, dssFlow.getName())) { throw new DSSErrorException(90003, "子工作流名不能重复"); } DSSFlow parentFlow = flowMapper.selectFlowByID(parentFlowID); @@ -207,6 +247,39 @@ public DSSFlow getFlow(Long flowID) throws NullPointerException { return dssFlow; } + @Override + public void getRootFlowProxy(DSSFlow dssFlow){ + + try { + + if(Boolean.FALSE.equals(dssFlow.getRootFlow())){ + Long rootFlowId = getRootFlowId(dssFlow.getId()); + if(rootFlowId == null){ + return; + } + DSSFlow rootFlow = getFlowByID(rootFlowId); + Map json = bmlService.query(rootFlow.getName(), rootFlow.getResourceId(), rootFlow.getBmlVersion()); + + List> props = DSSCommonUtils.getFlowAttribute(json.get("string").toString(), "props"); + String proxyUser = ""; + if (CollectionUtils.isNotEmpty(props)) { + for (Map prop : props) { + if (prop.containsKey("user.to.proxy") && prop.get("user.to.proxy") != null) { + proxyUser = prop.get("user.to.proxy").toString(); + break; + } + } + } + + dssFlow.setDefaultProxyUser(proxyUser); + } + + }catch (Exception e){ + logger.error("getRootFlowProxy dssFlow id is {}, error msg is {}", dssFlow.getId(), e.getMessage()); + } + } + + @Override public List getSubFlowContextIdsByFlowIds(List flowIdList) throws ErrorException { ArrayList contextIdList = new ArrayList<>(); @@ -263,7 +336,32 @@ public String saveFlow(Long flowID, String comment, String userName, String workspaceName, - String projectName) throws IOException{ + String projectName, + LabelRouteVO labels) throws Exception { + + // 判断工作流中是否存在命名相同的节点 + if (checkIsExistSameFlow(jsonFlow)) { + throw new DSSErrorException(80001, "It exists same flow.(存在相同的节点)"); + } + + // 判断工作流中是否有子工作流未被保存 + List unSaveNodes = checkIsSave(flowID, jsonFlow); + + if (CollectionUtils.isNotEmpty(unSaveNodes)) { + throw new DSSErrorException(80001, "工作流中存在子工作流未被保存，请先保存子工作流：" + unSaveNodes); + } + + //判断该工作流对应编排是否已发布，若已发布则不允许修改 + Long rootFlowId = getRootFlowId(flowID); + OrchestratorVo orchestratorVo = null; + try { + orchestratorVo = RpcAskUtils.processAskException(getOrchestratorSender().ask(new RequestQuertByAppIdOrchestrator(rootFlowId)), + OrchestratorVo.class, RequestQuertByAppIdOrchestrator.class); + } catch (Exception e) { + logger.error("保存工作流时获取编排失败，原因为：", e); + } + + DSSOrchestratorVersion dssOrchestratorVersion = orchestratorVo != null ? orchestratorVo.getDssOrchestratorVersion() : null; DSSFlow dssFlow = flowMapper.selectFlowByID(flowID); String creator = dssFlow.getCreator(); @@ -271,20 +369,35 @@ public String saveFlow(Long flowID, userName = creator; } String flowJsonOld = getFlowJson(userName, projectName, dssFlow); - if (isEqualTwoJson(flowJsonOld, jsonFlow)) { + + // 解析并保存元数据 + Long orchestratorId = dssOrchestratorVersion != null ? dssOrchestratorVersion.getOrchestratorId() : null; + try { + // 当前保存的工作流为最新版本工作流时，再执行保存元数据，否则会有重复数据 + if (orchestratorId != null && dssOrchestratorVersion.getAppId().equals(rootFlowId)) { + saveFlowMetaData(flowID, jsonFlow, orchestratorId); + // 更新版本更新时间 + RpcAskUtils.processAskException(getOrchestratorSender().ask(new RequestOrchestratorVersionUpdateTime(orchestratorId)), + Boolean.class, RequestOrchestratorVersionUpdateTime.class); + } + } catch (Exception e) { + logger.error(e.getMessage(), e); + } + + if (StringUtils.isNotEmpty(flowJsonOld) && isEqualTwoJson(flowJsonOld, jsonFlow)) { logger.info("saveFlow is not change"); return dssFlow.getBmlVersion(); } else { logger.info("saveFlow is change"); } - checkSubflowDependencies(userName, flowID, jsonFlow); + checkSubflowDependencies(userName, flowID, jsonFlow); String resourceId = dssFlow.getResourceId(); Long parentFlowID = flowMapper.getParentFlowID(flowID); // 这里不要检查ContextID具体版本等，只要存在就不创建 2020-0423 jsonFlow = contextService.checkAndCreateContextID(jsonFlow, dssFlow.getBmlVersion(), workspaceName, projectName, dssFlow.getName(), userName, false); - saveFlowHook.beforeSave(jsonFlow,dssFlow,parentFlowID); + saveFlowHook.beforeSave(jsonFlow, dssFlow, parentFlowID); Map bmlReturnMap = bmlService.update(userName, resourceId, jsonFlow); dssFlow.setId(flowID); @@ -300,13 +413,326 @@ public String saveFlow(Long flowID, contextService.checkAndSaveContext(jsonFlow, String.valueOf(parentFlowID)); } catch (DSSErrorException e) { logger.error("Failed to saveContext: ", e); - throw new DSSRuntimeException(e.getErrCode(),"保存ContextId失败，您可以尝试重新发布工作流！原因：" + ExceptionUtils.getRootCauseMessage(e),e); + throw new DSSRuntimeException(e.getErrCode(), "保存ContextId失败，您可以尝试重新发布工作流！原因：" + ExceptionUtils.getRootCauseMessage(e), e); } - saveFlowHook.afterSave(jsonFlow,dssFlow,parentFlowID); + saveFlowHook.afterSave(jsonFlow, dssFlow, parentFlowID); String version = bmlReturnMap.get("version").toString(); + // 对子工作流,需更新父工作流状态，以便提交 + Long updateFlowId = parentFlowID == null ? dssFlow.getId() : parentFlowID; + updateTOSaveStatus(dssFlow.getProjectId(), updateFlowId, orchestratorId); + return version; } + @Override + public void updateTOSaveStatus(Long projectId, Long flowID, Long orchestratorId) throws Exception { + try { + DSSProject projectInfo = DSSFlowEditLockManager.getProjectInfo(projectId); + //仅对接入Git的项目 更新状态为 保存 + if (projectInfo.getAssociateGit() != null && projectInfo.getAssociateGit()) { + Long rootFlowId = getRootFlowId(flowID); + if (rootFlowId != null) { + // 工作流变更时，清空BML + BmlResource bmlResource = new BmlResource(); + bmlResource.setResourceId(null); + bmlResource.setVersion(null); + RpcAskUtils.processAskException(getOrchestratorSender().ask(new RequestUpdateOrchestratorBML(rootFlowId, bmlResource)), + OrchestratorVo.class, RequestUpdateOrchestratorBML.class); + lockMapper.updateOrchestratorStatus(orchestratorId, OrchestratorRefConstant.FLOW_STATUS_SAVE); + } + } + } catch (DSSErrorException e) { + logger.error("getProjectInfo failed by:", e); + throw new DSSRuntimeException(e.getErrCode(), "更新工作流状态失败，您可以尝试重新保存工作流！原因：" + ExceptionUtils.getRootCauseMessage(e), e); + } + } + + private Long getRootFlowId(Long flowId) { + if (flowId == null) { + return null; + } + DSSFlow dssFlow = flowMapper.selectFlowByID(flowId); + if (dssFlow == null) { + return null; + } + if (dssFlow.getRootFlow()) { + return dssFlow.getId(); + } else { + Long parentFlowID = flowMapper.getParentFlowID(flowId); + return getRootFlowId(parentFlowID); + } + } + + @Override + public void deleteFlowMetaData(Long orchestratorId) { + List nodeContentListByOrchestratorId = nodeContentMapper.getNodeContentListByOrchestratorId(orchestratorId); + nodeContentMapper.deleteNodeContentByOrchestratorId(orchestratorId); + if (CollectionUtils.isNotEmpty(nodeContentListByOrchestratorId)) { + List collect = nodeContentListByOrchestratorId.stream().map(NodeContentDO::getId).collect(Collectors.toList()); + nodeContentUIMapper.deleteNodeContentUIByContentList(collect); + } + } + + @Override + public void saveAllFlowMetaData(Long flowId, Long orchestratorId) { + DSSFlow flowWithJsonAndSubFlowsByID = getFlowWithJsonAndSubFlowsByID(flowId); + saveAllMetaData(flowWithJsonAndSubFlowsByID, orchestratorId); + } + + private void saveAllMetaData(DSSFlow dssFlow, Long orchestratorId) { + if (dssFlow == null) return; + List subFlows = dssFlow.getChildren(); + saveFlowMetaData(dssFlow.getId(), dssFlow.getFlowJson(), orchestratorId); + if (subFlows != null) { + // 递归遍历子节点保存 + for (DSSFlow subFlow : subFlows) { + saveAllMetaData(subFlow, orchestratorId); + } + } + } + + @Override + public void saveFlowMetaData(Long flowID, String jsonFlow, Long orchestratorId) { + // 解析 jsonflow + // 解析 proxyUser + try { + List> props = DSSCommonUtils.getFlowAttribute(jsonFlow, "props"); + String proxyUser = null; + StringBuilder globalVar = new StringBuilder(); + if (CollectionUtils.isNotEmpty(props)) { + for (Map prop : props) { + if (prop.containsKey("user.to.proxy") && prop.get("user.to.proxy") != null) { + proxyUser = prop.get("user.to.proxy").toString(); + } else { + for (Map.Entry map : prop.entrySet()) { + if (map.getValue() != null) { + globalVar.append(map.getKey() + "=" + map.getValue()); + globalVar.append(";"); + } + } + } + } + } + // 解析 resources + List> resources = DSSCommonUtils.getFlowAttribute(jsonFlow, "resources"); + StringBuilder resourceToString = new StringBuilder(); + if (CollectionUtils.isNotEmpty(resources)) { + for (Map resource : resources) { + if (resource.containsKey("fileName")) { + resourceToString.append(resource.get("fileName")); + resourceToString.append(";"); + } + } + } + + List workFlowNodes = DSSCommonUtils.getWorkFlowNodes(jsonFlow); + + if (CollectionUtils.isEmpty(workFlowNodes)) { + return; + } + + NodeMetaDO nodeMetaByOrchestratorId = nodeMetaMapper.getNodeMetaByOrchestratorId(orchestratorId); + + if (nodeMetaByOrchestratorId == null) { + nodeMetaMapper.insertNodeMeta(new NodeMetaDO(orchestratorId, proxyUser, resourceToString.toString(), globalVar.toString())); + } else { + nodeMetaMapper.updateNodeMeta(new NodeMetaDO(nodeMetaByOrchestratorId.getId(), orchestratorId, proxyUser, resourceToString.toString(), globalVar.toString())); + } + + List contentDOS = nodeContentMapper.getContentListByOrchestratorIdAndFlowId(orchestratorId, flowID); + + logger.info("workFlowNodes:{}", workFlowNodes); + Set nodeContentUIDOS = new HashSet<>(); + List nodeContentDOS = new ArrayList<>(); + List keyList = new ArrayList<>(); + Map map = new HashMap<>(); + for (DSSNodeDefault nodeDefault : workFlowNodes) { + String key = nodeDefault.getKey(); + if (StringUtils.isNotEmpty(key)) { + keyList.add(key); + map.put(key, nodeDefault); + } + String id = nodeDefault.getId(); + String jobType = nodeDefault.getJobType(); + Long createTime = nodeDefault.getCreateTime(); + Long modifyTime = nodeDefault.getModifyTime(); + Date createDate = null; + Date modifyDate = null; + if (createTime != null && modifyTime != null) { + createDate = new Date(createTime); + } + if (modifyTime != null) { + modifyDate = new Date(modifyTime); + } + String modifyUser = nodeDefault.getModifyUser(); + NodeContentDO contentDO = new NodeContentDO(key, id, jobType, orchestratorId, flowID, createDate, modifyDate, modifyUser); + nodeContentDOS.add(contentDO); + } + + if (CollectionUtils.isEmpty(nodeContentDOS)) { + + return; + } + + // 获取相同的部分（交集） + Set intersection = new HashSet<>(contentDOS); + intersection.retainAll(nodeContentDOS); + + // 获取删除的部分（差集） + Set difference1 = new HashSet<>(contentDOS); + difference1.removeAll(nodeContentDOS); + + // 获取新增的部分（差集） + Set difference2 = new HashSet<>(nodeContentDOS); + difference2.removeAll(contentDOS); + + if (CollectionUtils.isNotEmpty(difference2)) { + nodeContentMapper.batchInsert(new ArrayList<>(difference2)); + } + + if (CollectionUtils.isNotEmpty(intersection)) { + for (NodeContentDO nodeContentDO : intersection) { + + nodeContentMapper.updateByKey(nodeContentDO); + } + } + + if (CollectionUtils.isNotEmpty(difference1)) { + nodeContentMapper.batchDelete(new ArrayList<>(difference1), orchestratorId, flowID); + } + + List nodeContents = nodeContentMapper.getNodeContentByKeyList(keyList, orchestratorId, flowID); + List workflowNodeNumberType = nodeInfoMapper.getWorkflowNodeNumberType(); + Map nodeNumberMap = new HashMap<>(); + for (NodeTypeDO nodeTypeDO : workflowNodeNumberType) { + nodeNumberMap.put(nodeTypeDO.getNodeType(), nodeTypeDO.getKey()); + } + for (NodeContentDO nodeContentDO : nodeContents) { + String nodeKey = nodeContentDO.getNodeKey(); + Long contentByKeyId = nodeContentDO.getId(); + DSSNodeDefault nodeDefault = map.get(nodeKey); + String jobType = nodeDefault.getJobType(); + Map jobContent = nodeDefault.getJobContent(); + if (jobContent != null) { + for (Map.Entry entry : jobContent.entrySet()) { + String value = entry.getValue() != null ? entry.getValue().toString() : ""; + nodeContentUIDOS.add(new NodeContentUIDO(contentByKeyId, entry.getKey(), value, jobType, "String")); + } + } + + String title = nodeDefault.getTitle(); + if (StringUtils.isNotEmpty(title)) { + nodeContentUIDOS.add(new NodeContentUIDO(contentByKeyId, "title", title, jobType, "String")); + } + String desc = nodeDefault.getDesc(); + if (StringUtils.isNotEmpty(desc)) { + nodeContentUIDOS.add(new NodeContentUIDO(contentByKeyId, "desc", desc, jobType, "String")); + } + String businessTag = nodeDefault.getBusinessTag(); + if (StringUtils.isNotEmpty(businessTag)) { + nodeContentUIDOS.add(new NodeContentUIDO(contentByKeyId, "businessTag", businessTag, jobType, "String")); + } + String appTag = nodeDefault.getAppTag(); + if (StringUtils.isNotEmpty(appTag)) { + nodeContentUIDOS.add(new NodeContentUIDO(contentByKeyId, "appTag", appTag, jobType, "String")); + } + String ecConfTemplateName = nodeDefault.getEcConfTemplateName(); + if (StringUtils.isNotEmpty(ecConfTemplateName)) { + nodeContentUIDOS.add(new NodeContentUIDO(contentByKeyId, "ecConfTemplateName", ecConfTemplateName, jobType, "String")); + } + String ecConfTemplateId = nodeDefault.getEcConfTemplateId(); + if (StringUtils.isNotEmpty(ecConfTemplateId)) { + nodeContentUIDOS.add(new NodeContentUIDO(contentByKeyId, "ecConfTemplateId", ecConfTemplateId, jobType, "String")); + } + List nodeDefaultResources = nodeDefault.getResources(); + if (CollectionUtils.isNotEmpty(nodeDefaultResources)) { + StringBuilder nodeResources = new StringBuilder(); + for (Resource resource : nodeDefaultResources) { + nodeResources.append(resource.getFileName()); + nodeResources.append(";"); + } + nodeContentUIDOS.add(new NodeContentUIDO(contentByKeyId, "resources", new String(nodeResources), jobType, "String")); + } + + + Map params = nodeDefault.getParams(); + if (params != null) { + Map> mapParams = (Map>) params.get("configuration"); + if (mapParams != null) { + for (Map.Entry> entry : mapParams.entrySet()) { + // special runtime startup + String paramKey = entry.getKey(); + Map paramValue = entry.getValue(); + for (Map.Entry paramEntry : paramValue.entrySet()) { + String paramName = paramEntry.getKey(); + if (paramEntry.getValue() != null) { + String paramVal; + Object value = paramEntry.getValue(); + if (value instanceof Number) { + Number numValue = (Number) value; + paramVal = numValue.longValue() == numValue.doubleValue() ? + String.valueOf(numValue.longValue()) : + numValue.toString(); + } else { + paramVal = value.toString(); + } + logger.info("{}:{}", paramName, paramVal); + String nodeContentType = null; + if (nodeNumberMap.containsKey(jobType) && nodeNumberMap.get(jobType).equals(paramName)) { + nodeContentType = "NumInterval"; + } else if (paramName.endsWith("memory")) { + nodeContentType = "Memory"; + } else { + nodeContentType = "String"; + } + nodeContentUIDOS.add(new NodeContentUIDO(contentByKeyId, paramName, paramVal, jobType, nodeContentType)); + } + + } + } + logger.info(mapParams.toString()); + } + } + } + List contentIdListByOrchestratorId = contentDOS.stream().map(NodeContentDO::getId).collect(Collectors.toList()); + // 先删后增 + if (CollectionUtils.isNotEmpty(contentIdListByOrchestratorId)) { + nodeContentUIMapper.deleteNodeContentUIByContentList(contentIdListByOrchestratorId); + } + if (CollectionUtils.isNotEmpty(nodeContentUIDOS)) { + nodeContentUIMapper.batchInsertNodeContentUI(new ArrayList<>(nodeContentUIDOS)); + } + } catch (Exception e) { + logger.error("saveFlowMeta error, the reason is: ", e); + } + } + + + private void deleteFlowMetaDataByFlowId(Long flowID) { + Long rootFlowId = getRootFlowId(flowID); + OrchestratorVo orchestratorVo = null; + try { + orchestratorVo = RpcAskUtils.processAskException(getOrchestratorSender().ask(new RequestQuertByAppIdOrchestrator(rootFlowId)), + OrchestratorVo.class, RequestQuertByAppIdOrchestrator.class); + } catch (Exception e) { + logger.error("保存工作流时获取编排失败，原因为：", e); + } + DSSOrchestratorVersion dssOrchestratorVersion = orchestratorVo != null ? orchestratorVo.getDssOrchestratorVersion() : null; + // 解析并保存元数据 + Long orchestratorId = dssOrchestratorVersion != null ? dssOrchestratorVersion.getOrchestratorId() : null; + if (orchestratorId != null && dssOrchestratorVersion.getAppId().equals(rootFlowId)) { + List contentDOList = nodeContentMapper.getContentListByOrchestratorIdAndFlowId(orchestratorId, Long.valueOf(flowID)); + if (!CollectionUtils.isEmpty(contentDOList)) { + List contentIdList = contentDOList.stream().map(NodeContentDO::getId).collect(Collectors.toList()); + nodeContentMapper.deleteNodeContentByOrchestratorIdAndFlowId(orchestratorId, Long.valueOf(flowID)); + if (CollectionUtils.isNotEmpty(contentIdList)) { + nodeContentUIMapper.deleteNodeContentUIByContentList(contentIdList); + } + nodeMetaMapper.deleteNodeMetaByOrchestratorId(orchestratorId); + } + } + } + /** * 当数据库的父子工作流依赖关系和json中不一致时，该方法会删除数据库中的脏数据 * @@ -369,6 +795,7 @@ public boolean isEqualTwoJson(String oldJsonNode, String newJsonNode) { return tempOldJson.equals(tempNewJson); } + @Override public String getFlowJson(String userName, String projectName, DSSFlow dssFlow) { String flowExportSaveBasePath = IoUtils.generateIOPath(userName, projectName, ""); String savePath = flowExportSaveBasePath + File.separator + dssFlow.getName() + File.separator + dssFlow.getName() + ".json"; @@ -384,7 +811,7 @@ public Long getParentFlowID(Long flowID) { @Override public List getExtraToolBars(long workspaceId, long projectId) { List retList = new ArrayList<>(); - retList.add(new ExtraToolBarsVO("前往调度中心", DSSWorkFlowConstant.GOTO_SCHEDULER_CENTER_URL.getValue() + "?workspaceId=" + workspaceId, "icon:null")); + retList.add(new ExtraToolBarsVO("前往调度中心", GOTO_SCHEDULER_CENTER_URL.getValue() + "?workspaceId=" + workspaceId, "icon:null")); return retList; } @@ -400,6 +827,7 @@ public void deleteFlow(Long flowId) { // TODO: 2019/6/5 json中资源的删除 // TODO: 2019/6/5 事务的保证 } + deleteFlowMetaDataByFlowId(flowId); deleteDWSDB(flowId); } @@ -433,10 +861,15 @@ public DSSFlow copyRootFlow(Long rootFlowId, String userName, Workspace workspac String description, List dssLabels, String nodeSuffix, String newFlowName, Long newProjectId) throws DSSErrorException, IOException { DSSFlow dssFlow = flowMapper.selectFlowByID(rootFlowId); + Sender orcSender = DSSSenderServiceFactory.getOrCreateServiceInstance().getOrcSender(dssLabels); + OrchestratorVo orchestratorVo = RpcAskUtils.processAskException(orcSender.ask(new RequestQuertByAppIdOrchestrator(dssFlow.getId())), + OrchestratorVo.class, RequestQueryByIdOrchestrator.class); + Long orchestratorId = orchestratorVo.getDssOrchestratorInfo().getId(); + deleteFlowMetaData(orchestratorId); DSSFlow rootFlowWithSubFlows = copyFlowAndSetSubFlowInDB(dssFlow, userName, description, nodeSuffix, newFlowName, newProjectId); updateFlowJson(userName, projectName, rootFlowWithSubFlows, version, null, - contextIdStr, workspace, dssLabels, nodeSuffix); - DSSFlow copyFlow= flowMapper.selectFlowByID(rootFlowWithSubFlows.getId()); + contextIdStr, workspace, dssLabels, nodeSuffix, orchestratorId); + DSSFlow copyFlow = flowMapper.selectFlowByID(rootFlowWithSubFlows.getId()); copyFlow.setFlowIdParamConfTemplateIdTuples(rootFlowWithSubFlows.getFlowIdParamConfTemplateIdTuples()); return copyFlow; } @@ -519,9 +952,38 @@ private DSSFlow copyFlowAndSetSubFlowInDB(DSSFlow dssFlow, String userName, Stri return cyFlow; } + @Override + public void getAllOldFlowId(Long flowId, List flowIdList) { + DSSFlow dssFlow = getFlow(flowId); + List subFlowIDs = flowMapper.selectSubFlowIDByParentFlowID(flowId); + flowIdList.add(flowId); + for (Long subFlowID : subFlowIDs) { + DSSFlow subDSSFlow = flowMapper.selectFlowByID(subFlowID); + if (dssFlow.getChildren() == null) { + dssFlow.setChildren(new ArrayList()); + } + getAllOldFlowId(subFlowID, flowIdList); + } + } + + @Override + public void deleteNodeContent(List flowIdList) { + if (CollectionUtils.isEmpty(flowIdList)) { + return; + } + List contentDOS = nodeContentMapper.getContentListByFlowId(flowIdList); + List contentIdListByOrchestratorId = contentDOS.stream().map(NodeContentDO::getId).collect(Collectors.toList()); + // 删除当前节点的属性信息 + if (CollectionUtils.isNotEmpty(contentIdListByOrchestratorId)) { + nodeContentUIMapper.deleteNodeContentUIByContentList(contentIdListByOrchestratorId); + } + nodeContentMapper.deleteNodeContentByFlowId(flowIdList); + } + private void updateFlowJson(String userName, String projectName, DSSFlow rootFlow, String version, Long parentFlowId, String contextIdStr, - Workspace workspace, List dssLabels, String nodeSuffix) throws DSSErrorException, IOException { + Workspace workspace, List dssLabels, String nodeSuffix, + Long orchestratorId) throws DSSErrorException, IOException { String flowJson = bmlService.readTextFromBML(userName, rootFlow.getResourceId(), rootFlow.getBmlVersion()); //如果包含subflow,需要一同导入subflow内容，并更新parrentflow的json内容 // TODO: 2020/7/31 优化update方法里面的saveContent @@ -537,14 +999,16 @@ private void updateFlowJson(String userName, String projectName, DSSFlow rootFlo //重新上传工作流资源 updateFlowJson = uploadFlowResourceToBml(userName, updateFlowJson, projectName, rootFlow); //上传节点的资源或调用appconn的copyRef - updateFlowJson = updateWorkFlowNodeJson(userName, projectName, updateFlowJson, rootFlow, + updateFlowJson = updateWorkFlowNodeJsonForMultiThread(userName, projectName, updateFlowJson, rootFlow, version, workspace, dssLabels); + // 更新对应节点的FlowJson + saveFlowMetaData(rootFlow.getId(), updateFlowJson, orchestratorId); List subFlows = rootFlow.getChildren(); List templateIds = new ArrayList<>(); if (subFlows != null) { for (DSSFlow subflow : subFlows) { updateFlowJson(userName, projectName, subflow, version, rootFlow.getId(), - contextIdStr, workspace, dssLabels, nodeSuffix); + contextIdStr, workspace, dssLabels, nodeSuffix, orchestratorId); templateIds.addAll(subflow.getFlowIdParamConfTemplateIdTuples()); } } @@ -552,7 +1016,7 @@ private void updateFlowJson(String userName, String projectName, DSSFlow rootFlo DSSFlow updateDssFlow = uploadFlowJsonToBml(userName, projectName, rootFlow, updateFlowJson); List tempIds = workFlowParser.getParamConfTemplate(updateFlowJson); List templateIdsInRoot = tempIds.stream() - .map(e->new String[]{updateDssFlow.getId().toString(),e}) + .map(e -> new String[]{updateDssFlow.getId().toString(), e}) .collect(Collectors.toList()); templateIds.addAll(templateIdsInRoot); rootFlow.setFlowIdParamConfTemplateIdTuples(templateIds); @@ -694,7 +1158,7 @@ private String updateWorkFlowNodeJson(String userName, String projectName, throw new DSSErrorException(90078, "工程内未能找到子工作流节点，导入失败" + subFlowName); } // } else if (nodeJsonMap.get("jobContent") != null && !((Map) nodeJsonMap.get("jobContent")).containsKey("script")) { - }else if(nodeInfo==null){ + } else if (nodeInfo == null) { String msg = String.format("%s note type not exist,please check appconn install successfully", nodeType); logger.error(msg); throw new DSSRuntimeException(msg); @@ -724,6 +1188,114 @@ private String updateWorkFlowNodeJson(String userName, String projectName, return workFlowParser.updateFlowJsonWithKey(flowJson, "nodes", nodeJsonListRes); } + + + + private String updateWorkFlowNodeJsonForMultiThread(String userName, String projectName, + String flowJson, DSSFlow dssFlow, + String version, Workspace workspace, List dssLabels) throws DSSErrorException, IOException{ + + if (StringUtils.isEmpty(version)) { + logger.warn("version id is null when updateWorkFlowNodeJson"); + version = String.valueOf((new Random().nextLong())); + } + + List nodeJsonList = workFlowParser.getWorkFlowNodesJson(flowJson); + if (nodeJsonList == null) { + throw new DSSErrorException(90073, "工作流内没有工作流节点，导入失败." + dssFlow.getName()); + } + String updateContextId = workFlowParser.getValueWithKey(flowJson, CSCommonUtils.CONTEXT_ID_STR); + if (nodeJsonList.size() == 0) { + return flowJson; + } + List subflows = (List) dssFlow.getChildren(); + List> nodeJsonListRes = new ArrayList<>(); + + List>> futures = new ArrayList<>(nodeJsonList.size()); + + String orcVersion = version; + + for (String nodeJson : nodeJsonList) { + + Future> future = DssJobThreadPool.nodeUploadThreadPool.submit( ()-> { + + try { + //重新上传一份jar文件到bml + String updateNodeJson = inputNodeFiles(userName, projectName, nodeJson); + + Map nodeJsonMap = BDPJettyServerHelper.jacksonJson().readValue(updateNodeJson, Map.class); + //更新subflowID + String nodeType = nodeJsonMap.get(JOBTYPE_KEY).toString(); + NodeInfo nodeInfo = nodeInfoMapper.getWorkflowNodeByType(nodeType); + if ("workflow.subflow".equals(nodeType)) { + String subFlowName = nodeJsonMap.get(TITLE_KEY).toString(); + List dssFlowList = subflows.stream().filter(subflow -> + subflow.getName().equals(subFlowName) + ).collect(Collectors.toList()); + if (dssFlowList.size() == 1) { + updateNodeJson = nodeInputService.updateNodeSubflowID(updateNodeJson, dssFlowList.get(0).getId()); + nodeJsonMap = BDPJettyServerHelper.jacksonJson().readValue(updateNodeJson, Map.class); + + } else if (dssFlowList.size() > 1) { + logger.error("工程内存在重复的子工作流节点名称，导入失败" + subFlowName); + throw new DSSErrorException(90077, "工程内存在重复的子工作流节点名称，导入失败" + subFlowName); + } else { + logger.error("工程内存在重复的子工作流节点名称，导入失败" + subFlowName); + throw new DSSErrorException(90078, "工程内未能找到子工作流节点，导入失败" + subFlowName); + } + } else if (nodeInfo == null) { + String msg = String.format("%s note type not exist,please check appconn install successfully", nodeType); + logger.error(msg); + throw new DSSRuntimeException(msg); + } else if (Boolean.TRUE.equals(nodeInfo.getSupportJump()) && nodeInfo.getJumpType() == 1) { + logger.info("nodeJsonMap.jobContent is:{}", nodeJsonMap.get("jobContent")); + CommonAppConnNode newNode = new CommonAppConnNode(); + CommonAppConnNode oldNode = new CommonAppConnNode(); + oldNode.setJobContent((Map) nodeJsonMap.get("jobContent")); + oldNode.setContextId(updateContextId); + oldNode.setNodeType(nodeType); + oldNode.setName((String) nodeJsonMap.get(TITLE_KEY)); + oldNode.setFlowId(dssFlow.getId()); + oldNode.setWorkspace(workspace); + oldNode.setDssLabels(dssLabels); + oldNode.setFlowName(dssFlow.getName()); + oldNode.setProjectId(dssFlow.getProjectId()); + oldNode.setProjectName(projectName); + newNode.setName(oldNode.getName()); + Map jobContent = workflowNodeService.copyNode(userName, newNode, oldNode, orcVersion); + nodeJsonMap.put("jobContent", jobContent); + + } + + return nodeJsonMap; + + }catch (IOException e){ + throw new RuntimeException(e); + } + }); + + futures.add(future); + + } + + + for (Future> future : futures) { + try{ + Map nodeJsonMap=future.get(); + nodeJsonListRes.add(nodeJsonMap); + } catch (InterruptedException | ExecutionException e) { + logger.error("upload-flow-node thread fail msg is ｛｝ ", e); + throw new RuntimeException(e); + } + } + + return workFlowParser.updateFlowJsonWithKey(flowJson, "nodes", nodeJsonListRes); + + + + } + + //由于每一个节点可能含有jar文件，这个功能不能直接复制使用，因为删掉新版本节点会直接删掉旧版本的node中的jar文件 //所以重新上传一份jar文件到bml private String inputNodeFiles(String userName, String projectName, String nodeJson) throws IOException { @@ -748,4 +1320,1320 @@ private void persistenceFlowRelation(Long flowID, Long parentFlowID) { } } -} + + @Override + public DataDevelopNodeResponse queryDataDevelopNodeList(String username, Workspace workspace, DataDevelopNodeRequest request) { + + DataDevelopNodeResponse dataDevelopNodeResponse = new DataDevelopNodeResponse(); + + // 获取项目 + List dssProjectList = getDSSProject(workspace, username); + + if (CollectionUtils.isEmpty(dssProjectList)) { + logger.error("queryDataDevelopNodeList find project is empty, workspaceId is {}, username is {}", workspace.getWorkspaceId(), username); + return dataDevelopNodeResponse; + } + + Map dssProjectMap = projectListToMap(dssProjectList); + + List projectIdList = new ArrayList<>(dssProjectMap.keySet()); + + List nodeInfoList = getNodeInfoByGroupName(WorkflowNodeGroupEnum.DataDevelopment.getNameEn()); + + if (CollectionUtils.isEmpty(nodeInfoList)) { + logger.error("queryDataDevelopNodeList not find node type info"); + return dataDevelopNodeResponse; + } + + List nodeTypeList = nodeInfoList.stream().map(NodeInfo::getNodeType).collect(Collectors.toList()); + // 查询节点信息 + List flowNodeInfoList = nodeContentMapper.queryFlowNodeInfo(projectIdList, nodeTypeList); + +// Map> nodeInfoGroup = nodeUIInfoGroupByNodeType(nodeTypeList); + + if (CollectionUtils.isEmpty(flowNodeInfoList)) { + logger.error("queryDataDevelopNodeList find node info is empty, example project id is {}", projectIdList.get(0)); + return dataDevelopNodeResponse; + } + + Map dssFlowNodeInfoMap = flowNodeToMap(flowNodeInfoList); + List contentIdList = new ArrayList<>(dssFlowNodeInfoMap.keySet()); + + // 查询节点保存的配置信息 + Map> nodeContentUIGroup = getNodeContentUIGroup(contentIdList); + if (nodeContentUIGroup.isEmpty()) { + logger.error("queryDataDevelopNodeList not find nodeUI info , example contextId is {}", contentIdList.get(0)); + return dataDevelopNodeResponse; + } + + + + // 查询模板信息 + // List orchestratorIdList = flowNodeInfoList.stream().map(DSSFlowNodeInfo::getOrchestratorId).collect(Collectors.toList()); + // Map templateMap = getTemplateMap(orchestratorIdList); + + List dataDevelopNodeInfoList = new ArrayList<>(); + + for (Long contentId : nodeContentUIGroup.keySet()) { + + try { + DataDevelopNodeInfo dataDevelopNodeInfo = new DataDevelopNodeInfo(); + DSSFlowNodeInfo dssFlowNodeInfo = dssFlowNodeInfoMap.get(contentId); + List nodeUIList = nodeContentUIGroup.get(contentId); + DSSProject dssProject = dssProjectMap.get(dssFlowNodeInfo.getProjectId()); + +// Map nodeDefaultValue = getNodeDefaultValue(nodeInfoGroup, dssFlowNodeInfo.getJobType()); + + List nodeUIKeys = nodeUIList.stream().map(NodeContentUIDO::getNodeUIKey).collect(Collectors.toList()); + List nodeUIValues = nodeUIList.stream().map(NodeContentUIDO::getNodeUIValue).collect(Collectors.toList()); + + Map nodeMap = CollUtil.zip(nodeUIKeys, nodeUIValues); + NodeBaseInfo nodeBaseInfo = getNodeBaseInfo(dssFlowNodeInfo, dssProject, nodeMap,username); + BeanUtils.copyProperties(nodeBaseInfo, dataDevelopNodeInfo); + + dataDevelopNodeInfo.setResource(nodeMap.get("resources")); + + if (nodeMap.containsKey("ec.conf.templateId") && StringUtils.isNotEmpty(nodeMap.get("ec.conf.templateId"))) { + dataDevelopNodeInfo.setRefTemplate(Boolean.TRUE); + String templateId = nodeMap.get("ec.conf.templateId"); + dataDevelopNodeInfo.setTemplateId(templateId); + if (nodeMap.containsKey("ecConfTemplateName")) { + dataDevelopNodeInfo.setTemplateName(nodeMap.get("ecConfTemplateName")); + } else if (nodeMap.containsKey("ec.conf.templateName")) { + dataDevelopNodeInfo.setTemplateName(nodeMap.get("ec.conf.templateName")); + } else { + logger.error("queryDataDevelopNodeList not find template,contentId is {},template id is {}", contentId, templateId); + } + + } else { + dataDevelopNodeInfo.setRefTemplate(Boolean.FALSE); + } + + if (nodeMap.containsKey("ReuseEngine") && StringUtils.isNotEmpty(nodeMap.get("ReuseEngine"))) { + dataDevelopNodeInfo.setReuseEngine(Boolean.valueOf(nodeMap.get("ReuseEngine"))); + } + + if(nodeMap.containsKey("script") && StringUtils.isNotEmpty(nodeMap.get("script"))){ + dataDevelopNodeInfo.setScript(nodeMap.get("script")); + } + + dataDevelopNodeInfoList.add(dataDevelopNodeInfo); + } catch (Exception exception) { + logger.error("queryDataDevelopNodeList error content id is {}", contentId); + logger.error(exception.getMessage()); + } + } + + // 条件筛选、排序 + dataDevelopNodeInfoList = dataDevelopNodeResultFilter(request, dataDevelopNodeInfoList); + + dataDevelopNodeResponse.setTotal((long) dataDevelopNodeInfoList.size()); + // 分页处理 + int page = request.getPageNow() >= 1 ? request.getPageNow() : 1; + int pageSize = request.getPageSize() >= 1 ? request.getPageSize() : 10; + int start = (page - 1) * pageSize; + int end = Math.min(page * pageSize, dataDevelopNodeInfoList.size()); + + for (int i = start; i < end; i++) { + DataDevelopNodeInfo dataDevelopNodeInfo = dataDevelopNodeInfoList.get(i); + dataDevelopNodeResponse.getDataDevelopNodeInfoList().add(dataDevelopNodeInfo); + } + + return dataDevelopNodeResponse; + + } + + + @Override + public Map getDataDevelopNodeContent(String nodeId, Long contentId) throws DSSErrorException { + NodeContentDO nodeContentDO = nodeContentMapper.getNodeContentById(contentId, nodeId); + Map content = new HashMap<>(); + if (nodeContentDO == null) { + logger.error("not find node info , id is {}, nodeId is {}", contentId, nodeId); + throw new DSSRuntimeException(90004, "未找到" + nodeId + "节点信息，请查看节点是否被删除"); + } + + List nodeContentUIDOList = nodeContentUIMapper.getNodeContentUIByContentId(contentId); + + if (CollectionUtils.isEmpty(nodeContentUIDOList)) { + logger.error("not find node params info , content id is {}", contentId); + throw new DSSRuntimeException(90004, "未找到" + nodeId + "节点信息，请查看节点是否被删除"); + } + + List nodeUIInfoList = nodeInfoMapper.getNodeUIInfoByNodeType(nodeContentDO.getJobType()); + + if (CollectionUtils.isEmpty(nodeUIInfoList)) { + logger.error("node type not params content, node type is {}, content id is {}", nodeContentDO.getJobType(), contentId); + return content; + } + + + List nodeKey = nodeContentUIDOList.stream().map(NodeContentUIDO::getNodeUIKey).collect(Collectors.toList()); + List nodeValue = nodeContentUIDOList.stream().map(NodeContentUIDO::getNodeUIValue).collect(Collectors.toList()); + + Map nodeMap = CollUtil.zip(nodeKey, nodeValue); + + for (NodeUIInfo nodeUIInfo : nodeUIInfoList) { + + content.put(nodeUIInfo.getKey(), nodeMap.getOrDefault(nodeUIInfo.getKey(), "")); + + } + + return content; + + } + + + @Override + public DataViewNodeResponse queryDataViewNode(String username, Workspace workspace, DataViewNodeRequest request) { + + DataViewNodeResponse dataViewNodeResponse = new DataViewNodeResponse(); + + List dssProjectList = getDSSProject(workspace, username); + + if (CollectionUtils.isEmpty(dssProjectList)) { + logger.error("queryDataViewNode workspace is {}, username is {}", workspace, username); + return dataViewNodeResponse; + } + + Map dssProjectMap = projectListToMap(dssProjectList); + + List projectIdList = new ArrayList<>(dssProjectMap.keySet()); + // 查询节点信息 + List nodeInfoList = getNodeInfoByGroupName(WorkflowNodeGroupEnum.DataVisualization.getNameEn()); + if (CollectionUtils.isEmpty(nodeInfoList)) { + logger.error("queryDataViewNode not find node type info"); + return dataViewNodeResponse; + } + List nodeTypeList = nodeInfoList.stream().map(NodeInfo::getNodeType).collect(Collectors.toList()); + List flowNodeInfoList = nodeContentMapper.queryFlowNodeInfo(projectIdList, nodeTypeList); + + if (CollectionUtils.isEmpty(flowNodeInfoList)) { + logger.error("queryDataViewNode not find node info , example projectId is {}", projectIdList.get(0)); + return dataViewNodeResponse; + } + + Map dssFlowNodeInfoMap = flowNodeToMap(flowNodeInfoList); + List contentIdList = new ArrayList<>(dssFlowNodeInfoMap.keySet()); + + // 查询节点保存的配置信息 + Map> nodeContentUIGroup = getNodeContentUIGroup(contentIdList); + if (nodeContentUIGroup.isEmpty()) { + logger.error("queryDataViewNode not find nodeUI info , example contextId is {}", contentIdList.get(0)); + return dataViewNodeResponse; + } + + List dataViewNodeInfoList = new ArrayList<>(); + + for (Long contentId : nodeContentUIGroup.keySet()) { + + DataViewNodeInfo dataViewNodeInfo = new DataViewNodeInfo(); + DSSFlowNodeInfo dssFlowNodeInfo = dssFlowNodeInfoMap.get(contentId); + List nodeUIList = nodeContentUIGroup.get(contentId); + DSSProject dssProject = dssProjectMap.get(dssFlowNodeInfo.getProjectId()); + + List nodeUIKeys = nodeUIList.stream().map(NodeContentUIDO::getNodeUIKey).collect(Collectors.toList()); + List nodeUIValues = nodeUIList.stream().map(NodeContentUIDO::getNodeUIValue).collect(Collectors.toList()); + Map nodeMap = CollUtil.zip(nodeUIKeys, nodeUIValues); + + NodeBaseInfo nodeBaseInfo = getNodeBaseInfo(dssFlowNodeInfo, dssProject, nodeMap,username); + BeanUtils.copyProperties(nodeBaseInfo, dataViewNodeInfo); + + dataViewNodeInfo.setNodeDesc(nodeMap.get("desc")); + if(nodeMap.containsKey("viewId")){ + dataViewNodeInfo.setViewId(nodeMap.get("viewId")); + } else if (nodeMap.containsKey("datasourceId")) { + dataViewNodeInfo.setViewId(nodeMap.get("datasourceId")); + } + + dataViewNodeInfoList.add(dataViewNodeInfo); + } + + dataViewNodeInfoList = dataViewNodeResultFilter(request, dataViewNodeInfoList); + + dataViewNodeResponse.setTotal((long) dataViewNodeInfoList.size()); + // 分页处理 + int page = request.getPageNow() >= 1 ? request.getPageNow() : 1; + int pageSize = request.getPageSize() >= 1 ? request.getPageSize() : 10; + int start = (page - 1) * pageSize; + int end = Math.min(page * pageSize, dataViewNodeInfoList.size()); + + for (int i = start; i < end; i++) { + DataViewNodeInfo dataViewNodeInfo = dataViewNodeInfoList.get(i); + dataViewNodeResponse.getDataDevelopNodeInfoList().add(dataViewNodeInfo); + } + + return dataViewNodeResponse; + + } + + + public List getDSSProject(Workspace workspace, String username) { + + ProjectListQueryResponse response = RpcAskUtils.processAskException(DSSSenderServiceFactory.getOrCreateServiceInstance() + .getProjectServerSender().ask(new ProjectListQueryRequest(workspace.getWorkspaceId(), username)), + ProjectListQueryResponse.class, ProjectListQueryRequest.class); + + + if (CollectionUtils.isEmpty(response.getProjectList())) { + response.setProjectList(new ArrayList<>()); + } + + return response.getProjectList().stream().filter(project -> { + // 过滤具有编辑工作流权限的项目 + return (CollectionUtils.isNotEmpty(project.getEditUsers()) && project.getEditUsers().contains(username)) + || (CollectionUtils.isNotEmpty(project.getReleaseUsers()) && project.getReleaseUsers().contains(username)) + || (StringUtils.isNotEmpty(project.getCreateBy()) && project.getCreateBy().equals(username)) + || (CollectionUtils.isNotEmpty(project.getAccessUsers()) && project.getAccessUsers().contains(username)) + || project.isAdmin(); + }).collect(Collectors.toList()); + + } + + + public Map projectListToMap(List dssProjectList) { + + Map dssProjectMap = new HashMap<>(); + + for (DSSProject project : dssProjectList) { + + dssProjectMap.put(project.getId(), project); + } + + return dssProjectMap; + } + + + public Map getTemplateMap(List orchestratorIdList) { + + // 查询模板信息 + List dssFlowNodeTemplateList = nodeContentMapper.queryFlowNodeTemplate(orchestratorIdList); + + Map templateMap = new HashMap<>(); + + if (!CollectionUtils.isEmpty(dssFlowNodeTemplateList)) { + List templateIdList = dssFlowNodeTemplateList.stream().map(DSSFlowNodeTemplate::getTemplateId).collect(Collectors.toList()); + List templateNameList = dssFlowNodeTemplateList.stream().map(DSSFlowNodeTemplate::getTemplateName).collect(Collectors.toList()); + templateMap = CollUtil.zip(templateIdList, templateNameList); + } + + return templateMap; + } + + + public Map flowNodeToMap(List flowNodeInfoList) { + Map dssFlowNodeInfoMap = new HashMap<>(); + + for (DSSFlowNodeInfo dssFlowNodeInfo : flowNodeInfoList) { + dssFlowNodeInfoMap.put(dssFlowNodeInfo.getContentId(), dssFlowNodeInfo); + } + + return dssFlowNodeInfoMap; + } + + + public Map> getNodeContentUIGroup(List contentIdList) { + Map> nodeContentUIGroup = new HashMap<>(); + List nodeContentUIDOList = nodeContentUIMapper.queryNodeContentUIList(contentIdList); + + if (CollectionUtils.isEmpty(nodeContentUIDOList)) { + return nodeContentUIGroup; + } + + nodeContentUIGroup = nodeContentUIDOList.stream() + .collect(Collectors.groupingBy(NodeContentUIDO::getContentId)); + return nodeContentUIGroup; + } + + + public List dataDevelopNodeResultFilter(DataDevelopNodeRequest request, List dataDevelopNodeInfoList) { + + dataDevelopNodeInfoList = dataDevelopNodeInfoList.stream().filter(dataDevelopNodeInfo -> { + boolean flag = true; + + if (request.getRefTemplate() != null) { + // 是否引用模板 + flag = request.getRefTemplate().equals(dataDevelopNodeInfo.getRefTemplate()); + } + + if (request.getReuseEngine() != null && flag) { + // 是否引用引擎 + flag = request.getReuseEngine().equals(dataDevelopNodeInfo.getReuseEngine()); + } + + if (!CollectionUtils.isEmpty(request.getNodeTypeNameList()) && flag) { + // 节点小类 + flag = request.getNodeTypeNameList().contains(dataDevelopNodeInfo.getNodeTypeName()); + } + + if (!CollectionUtils.isEmpty(request.getNodeNameList()) && flag) { + // 节点名称 + flag = request.getNodeNameList().contains(dataDevelopNodeInfo.getNodeName()); + } + + if (!CollectionUtils.isEmpty(request.getTemplateNameList()) && flag) { + // 模板名称 + flag = request.getTemplateNameList().contains(dataDevelopNodeInfo.getTemplateName()); + } + + + // 工作流名称 + if (!StringUtils.isBlank(request.getOrchestratorName()) && flag) { + flag = request.getOrchestratorName().equals(dataDevelopNodeInfo.getOrchestratorName()); + } + + // 项目名称 + if (!CollectionUtils.isEmpty(request.getProjectNameList()) && flag) { + flag = request.getProjectNameList().contains(dataDevelopNodeInfo.getProjectName()); + } + + return flag; + }).collect(Collectors.toList()); + + // 创建时间倒序排列 + dataDevelopNodeInfoList = dataDevelopNodeInfoList.stream().sorted(Comparator.comparing(DataDevelopNodeInfo::getCreateTime, + Comparator.nullsFirst(Comparator.naturalOrder())).reversed()).collect(Collectors.toList()); + + return dataDevelopNodeInfoList; + + } + + + public List dataViewNodeResultFilter(DataViewNodeRequest request, List dataViewNodeInfoList) { + + dataViewNodeInfoList = dataViewNodeInfoList.stream().filter(dataViewNodeInfo -> { + boolean flag = true; + // 工作流名称 + if (!StringUtils.isBlank(request.getOrchestratorName())) { + flag = request.getOrchestratorName().equals(dataViewNodeInfo.getOrchestratorName()); + } + + // 项目名称 + if (!CollectionUtils.isEmpty(request.getProjectNameList()) && flag) { + flag = request.getProjectNameList().contains(dataViewNodeInfo.getProjectName()); + } + + // 节点类型 + if (!CollectionUtils.isEmpty(request.getNodeTypeNameList()) && flag) { + flag = request.getNodeTypeNameList().contains(dataViewNodeInfo.getNodeTypeName()); + } + + // 节点名称 + if (!CollectionUtils.isEmpty(request.getNodeNameList()) && flag) { + flag = request.getNodeNameList().contains(dataViewNodeInfo.getNodeName()); + } + + if (!StringUtils.isEmpty(request.getViewId()) && flag) { + flag = StringUtils.isNotEmpty(dataViewNodeInfo.getViewId()) + && dataViewNodeInfo.getViewId().contains(request.getViewId()); + } + + return flag; + + }).collect(Collectors.toList()); + + // 创建时间倒序排列 + dataViewNodeInfoList = dataViewNodeInfoList.stream().sorted(Comparator.comparing(DataViewNodeInfo::getCreateTime, + Comparator.nullsFirst(Comparator.naturalOrder())).reversed()).collect(Collectors.toList()); + + return dataViewNodeInfoList; + + + } + + + @Override + public List getNodeInfoByGroupName(String groupNameEn) { + + return nodeInfoMapper.getNodeTypeByGroupName(groupNameEn, null); + } + + + @Override + public DSSFlowName queryFlowNameList(String username, Workspace workspace, String groupNameEn, String nodeTypeName) { + + DSSFlowName dssFlowName = new DSSFlowName(); + + // 获取项目 + List dssProjectList = getDSSProject(workspace, username); + + if (CollectionUtils.isEmpty(dssProjectList)) { + logger.error("queryFlowNameList find project is empty, workspaceId is {}, username is {}", workspace.getWorkspaceId(), username); + return dssFlowName; + } + + List projectIdList = dssProjectList.stream().map(DSSProject::getId).collect(Collectors.toList()); + + List nodeInfoList = nodeInfoMapper.getNodeTypeByGroupName(groupNameEn, nodeTypeName); + + if (CollectionUtils.isEmpty(nodeInfoList)) { + logger.error("queryFlowNameList groupNameEn is {},nodeTypeName is {}", groupNameEn, nodeTypeName); + return dssFlowName; + } + + List nodeTypeList = nodeInfoList.stream().map(NodeInfo::getNodeType).collect(Collectors.toList()); + // 查询节点信息 + List flowNodeInfoList = nodeContentMapper.queryFlowNodeInfo(projectIdList, nodeTypeList); + + if (CollectionUtils.isEmpty(flowNodeInfoList)) { + logger.error("queryFlowNameList find node info is empty, example project id is {}", projectIdList.get(0)); + return dssFlowName; + } + + if (WorkflowNodeGroupEnum.DataDevelopment.getNameEn().equals(groupNameEn)) { + + List orchestratorIdList = flowNodeInfoList.stream().map(DSSFlowNodeInfo::getOrchestratorId).distinct().collect(Collectors.toList()); + List dssFlowNodeTemplateList = nodeContentMapper.queryFlowNodeTemplate(orchestratorIdList); + List templateNameList = dssFlowNodeTemplateList.stream().map(DSSFlowNodeTemplate::getTemplateName).distinct().collect(Collectors.toList()); + dssFlowName.setTemplateNameList(templateNameList); + } + + + List orchestratorNameList = flowNodeInfoList.stream().map(DSSFlowNodeInfo::getOrchestratorName).distinct().collect(Collectors.toList()); + dssFlowName.setOrchestratorNameList(orchestratorNameList); + + List contentIdList = flowNodeInfoList.stream().map(DSSFlowNodeInfo::getContentId).collect(Collectors.toList()); + List nodeContentUIDOList = nodeContentUIMapper.getNodeContentUIByNodeUIKey(contentIdList, nodeUITitleKey); + List nodeNameList = nodeContentUIDOList.stream().map(NodeContentUIDO::getNodeUIValue).distinct().collect(Collectors.toList()); + dssFlowName.setNodeNameList(nodeNameList); + + return dssFlowName; + + } + + + @Override + public List queryViewId(Workspace workspace, String username) { + + List viewIdList = queryNodeUiValueByKey(workspace, username, nodeUIViewIdKey, WorkflowNodeGroupEnum.DataVisualization.getNameEn()); + + List dataSourceIdList = queryNodeUiValueByKey(workspace, username, "datasourceId", WorkflowNodeGroupEnum.DataVisualization.getNameEn()); + + viewIdList.addAll(dataSourceIdList); + + return viewIdList.stream().distinct().collect(Collectors.toList()); + } + + @Override + public void batchEditFlow(BatchEditFlowRequest batchEditFlowRequest, String ticketId, Workspace workspace, String userName) throws Exception { + if (batchEditFlowRequest == null) { + return; + } + List editFlowRequestList = batchEditFlowRequest.getEditNodeList(); + Map> editFlowRequestMap = new HashMap<>(); + Set flowIdList = new HashSet<>(); + + for (EditFlowRequest editFlowRequest : editFlowRequestList) { + Long orchestratorId = editFlowRequest.getOrchestratorId(); + List editFlowRequests = editFlowRequestMap.containsKey(orchestratorId) ? + editFlowRequestMap.get(orchestratorId) : new ArrayList<>(); + editFlowRequests.add(editFlowRequest); + editFlowRequestMap.put(orchestratorId, editFlowRequests); + } + + Set orchestratorIdList = editFlowRequestList.stream().map(EditFlowRequest::getOrchestratorId).collect(Collectors.toSet()); + // 批量获取编辑锁 + Sender sender = DSSSenderServiceFactory.getOrCreateServiceInstance().getOrcSender(); + RequestQueryOrchestrator requestQueryOrchestrator = new RequestQueryOrchestrator(new ArrayList<>(orchestratorIdList)); + ResponseQueryOrchestrator responseQueryOrchestrator = RpcAskUtils.processAskException(sender.ask(requestQueryOrchestrator), ResponseQueryOrchestrator.class, RequestQueryOrchestrator.class); + + if (responseQueryOrchestrator == null) { + throw new DSSErrorException(80001, "该工作流节点没有对应编排"); + } + List orchestratorVoes = responseQueryOrchestrator.getOrchestratorVoes(); + + Long modifyTime = System.currentTimeMillis(); + Map> editFlowRequestTOFlowIDMap = new HashMap<>(); + + for (OrchestratorVo orchestratorVo : orchestratorVoes) { + DSSOrchestratorVersion dssOrchestratorVersion = orchestratorVo.getDssOrchestratorVersion(); + if (dssOrchestratorVersion != null) { + Long orchestratorId = dssOrchestratorVersion.getOrchestratorId(); + Long flowId = dssOrchestratorVersion.getAppId(); + DSSFlow dssFlow = getFlow(flowId); + DSSFlowEditLock flowEditLock = lockMapper.getFlowEditLockByID(flowId); + if (flowEditLock != null && !flowEditLock.getOwner().equals(ticketId)) { + throw new DSSErrorException(80001, "当前工作流" + dssFlow.getName() +"被用户" + flowEditLock.getUsername() + "已锁定编辑，您编辑的内容不能再被保存。如有疑问，请与" + flowEditLock.getUsername() + "确认"); + } + + lockFlow(dssFlow, userName, ticketId); + List editFlowRequests = editFlowRequestMap.get(orchestratorId); + + for (EditFlowRequest editFlowRequest : editFlowRequests) { + NodeContentDO nodeContentByContentId = nodeContentMapper.getNodeContentByContentId(editFlowRequest.getId()); + Long targetFlowId = nodeContentByContentId.getFlowId(); + List editFlowRequestsList = editFlowRequestTOFlowIDMap.containsKey(targetFlowId) ? + editFlowRequestTOFlowIDMap.get(targetFlowId) : new ArrayList<>(); + editFlowRequestsList.add(editFlowRequest); + editFlowRequestTOFlowIDMap.put(targetFlowId, editFlowRequestsList); + } + } + } + + for (Map.Entry> entry : editFlowRequestTOFlowIDMap.entrySet()) { + Long targetFlowId = entry.getKey(); + DSSFlow targetFlow = getFlow(targetFlowId); + DSSProject project = getProjectByProjectId(targetFlow.getProjectId()); + StringBuilder modifyJson = new StringBuilder(targetFlow.getFlowJson()); + List value = entry.getValue(); + if (value != null) { + for (EditFlowRequest editFlow : value) { + String json = modifyJson(String.valueOf(modifyJson), editFlow, modifyTime, userName); + modifyJson.setLength(0); + modifyJson.append(json); + } + // 批量修改属性 + String resultJson = modifyFlowJsonTime(String.valueOf(modifyJson), modifyTime, userName); + saveFlow(targetFlowId, String.valueOf(resultJson) + , targetFlow.getDescription(), userName + , workspace.getWorkspaceName(), project.getName(), null); + } + } + + // 完成后批量解锁 + for (OrchestratorVo orchestratorVo : orchestratorVoes) { + DSSOrchestratorVersion dssOrchestratorVersion = orchestratorVo.getDssOrchestratorVersion(); + if (dssOrchestratorVersion != null) { + Long flowId = dssOrchestratorVersion.getAppId(); + workFlowManager.unlockWorkflow(userName, flowId, true, workspace); + } + } + + } + + private DSSProject getProjectByProjectId(Long projectId) throws DSSErrorException { + ProjectInfoRequest projectInfoRequest = new ProjectInfoRequest(); + projectInfoRequest.setProjectId(projectId); + DSSProject dssProject = RpcAskUtils.processAskException(DSSSenderServiceFactory.getOrCreateServiceInstance().getProjectServerSender() + .ask(projectInfoRequest), DSSProject.class, ProjectInfoRequest.class); + if (dssProject == null) { + DSSExceptionUtils.dealErrorException(6003, "工程不存在", DSSErrorException.class); + } + return dssProject; + } + + private String modifyFlowJsonTime(String flowJson, Long modifyTime, String userName) { + JsonObject jsonObject = JsonParser.parseString(flowJson).getAsJsonObject(); + + jsonObject.addProperty("updateTime", modifyTime); + jsonObject.addProperty("updateUser", userName); + + return jsonObject.toString(); + } + + private String modifyJson(String flowJson, EditFlowRequest editFlowRequest, Long modifyTime, String username) { + String nodeKey = editFlowRequest.getNodeKey(); + String params = editFlowRequest.getParams(); + String title = editFlowRequest.getTitle(); + String desc = editFlowRequest.getDesc(); + String appTag = editFlowRequest.getAppTag(); + String businessTag = editFlowRequest.getBusinessTag(); + String ecConfTemplateId = editFlowRequest.getEcConfTemplateId(); + String ecConfTemplateName = editFlowRequest.getEcConfTemplateName(); + JsonObject jsonObject = JsonParser.parseString(flowJson).getAsJsonObject(); + + JsonArray listArray = jsonObject.getAsJsonArray("nodes"); + + for (JsonElement element : listArray) { + JsonObject obj = element.getAsJsonObject(); + if (obj.get("key").getAsString().equals(nodeKey)) { + JsonElement jsonElement = JsonParser.parseString(params); + if (StringUtils.isNotEmpty(params)) { + obj.add("params", jsonElement); + } + obj.addProperty("modifyTime", modifyTime); + obj.addProperty("modifyUser", username); + flowJsonAddPointProperty(obj, "title", title); + flowJsonAddPointProperty(obj, "desc", desc); + flowJsonAddProperty(obj, "appTag", appTag); + flowJsonAddProperty(obj, "businessTag", businessTag); + flowJsonAddProperty(obj, "ecConfTemplateId", ecConfTemplateId); + flowJsonAddProperty(obj, "ecConfTemplateName", ecConfTemplateName); + break; + } + } + String modifyJson = jsonObject.toString(); + return modifyJson; + } + + private void flowJsonAddPointProperty(JsonObject obj, String contentName, String content) { + if (content != null) { + obj.addProperty(contentName, content); + } + } + + private void flowJsonAddProperty(JsonObject obj, String contentName, String content) { + if (StringUtils.isNotEmpty(content)) { + obj.addProperty(contentName, content); + } else { + obj.remove(contentName); + } + } + + private void lockFlow(DSSFlow dssFlow, String username, String ticketId) throws DSSErrorException { + + // 尝试获取工作流编辑锁 + try { + //只有父工作流才有锁，子工作流复用父工作流的锁 + if (dssFlow.getRootFlow()) { + String flowEditLock = DSSFlowEditLockManager.tryAcquireLock(dssFlow, username, ticketId); + dssFlow.setFlowEditLock(flowEditLock); + } + } catch (DSSErrorException e) { + if (EDIT_LOCK_ERROR_CODE == e.getErrCode()) { + DSSFlowEditLock flowEditLock = lockMapper.getFlowEditLockByID(dssFlow.getId()); + throw new DSSErrorException(60056, "用户已锁定编辑错误码，editLockInfo:" + flowEditLock); + } + throw e; + } + } + + + public DataCheckerNodeResponse queryDataCheckerNode(String username, Workspace workspace, DataCheckerNodeRequest request) { + + DataCheckerNodeResponse dataCheckerNodeResponse = new DataCheckerNodeResponse(); + + List nodeInfoList = nodeInfoMapper.getNodeTypeByGroupName(WorkflowNodeGroupEnum.SignalNode.getNameEn() + , SignalNodeConstant.dataCheckerNode); + + if (CollectionUtils.isEmpty(nodeInfoList)) { + logger.error("queryDataCheckerNode not find dataChecker node info"); + return dataCheckerNodeResponse; + } + + List dssProjectList = getDSSProject(workspace, username); + + if (CollectionUtils.isEmpty(dssProjectList)) { + logger.error("queryDataCheckerNode workspace is {}, username is {}", workspace, username); + return dataCheckerNodeResponse; + } + + Map dssProjectMap = projectListToMap(dssProjectList); + + List projectIdList = new ArrayList<>(dssProjectMap.keySet()); + List nodeTypeList = nodeInfoList.stream().map(NodeInfo::getNodeType).collect(Collectors.toList()); + List flowNodeInfoList = nodeContentMapper.queryFlowNodeInfo(projectIdList, nodeTypeList); + +// Map> nodeInfoGroup = nodeUIInfoGroupByNodeType(nodeTypeList); + + if (CollectionUtils.isEmpty(flowNodeInfoList)) { + logger.error("queryDataCheckerNode query not find node info , example projectId is {}", projectIdList.get(0)); + return dataCheckerNodeResponse; + } + + Map dssFlowNodeInfoMap = flowNodeToMap(flowNodeInfoList); + List contentIdList = new ArrayList<>(dssFlowNodeInfoMap.keySet()); + + + // 查询节点保存的配置信息 + Map> nodeContentUIGroup = getNodeContentUIGroup(contentIdList); + if (nodeContentUIGroup.isEmpty()) { + logger.error("queryDataCheckerNode query not find nodeUI info , example contextId is {}", contentIdList.get(0)); + return dataCheckerNodeResponse; + } + + + List dataCheckerNodeInfoList = new ArrayList<>(); + + for (Long contentId : nodeContentUIGroup.keySet()) { + + DataCheckerNodeInfo dataCheckerNodeInfo = new DataCheckerNodeInfo(); + + DSSFlowNodeInfo dssFlowNodeInfo = dssFlowNodeInfoMap.get(contentId); + List nodeUIList = nodeContentUIGroup.get(contentId); + DSSProject dssProject = dssProjectMap.get(dssFlowNodeInfo.getProjectId()); + +// Map nodeDefaultValue = getNodeDefaultValue(nodeInfoGroup, dssFlowNodeInfo.getJobType()); + + List nodeUIKeys = nodeUIList.stream().map(NodeContentUIDO::getNodeUIKey).collect(Collectors.toList()); + List nodeUIValues = nodeUIList.stream().map(NodeContentUIDO::getNodeUIValue).collect(Collectors.toList()); + Map nodeMap = CollUtil.zip(nodeUIKeys, nodeUIValues); + + NodeBaseInfo nodeBaseInfo = getNodeBaseInfo(dssFlowNodeInfo, dssProject, nodeMap,username); + BeanUtils.copyProperties(nodeBaseInfo, dataCheckerNodeInfo); + + dataCheckerNodeInfo.setMaxCheckHours(nodeMap.get("max.check.hours")); + dataCheckerNodeInfo.setJobDesc(nodeMap.get("job.desc")); + dataCheckerNodeInfo.setCheckObject(nodeMap.get("check.object")); + dataCheckerNodeInfo.setNodeDesc(nodeMap.get("desc")); + dataCheckerNodeInfo.setSourceType(nodeMap.get("source.type")); + + if (nodeMap.containsKey("qualitis.check") && StringUtils.isNotEmpty(nodeMap.get("qualitis.check"))) { + dataCheckerNodeInfo.setQualitisCheck(Boolean.valueOf(nodeMap.get("qualitis.check"))); + } + + dataCheckerNodeInfoList.add(dataCheckerNodeInfo); + + } + + dataCheckerNodeInfoList = dataCheckerNodeResultFilter(dataCheckerNodeInfoList, request); + + + dataCheckerNodeResponse.setTotal((long) dataCheckerNodeInfoList.size()); + // 分页处理 + int page = request.getPageNow() >= 1 ? request.getPageNow() : 1; + int pageSize = request.getPageSize() >= 1 ? request.getPageSize() : 10; + int start = (page - 1) * pageSize; + int end = Math.min(page * pageSize, dataCheckerNodeInfoList.size()); + + for (int i = start; i < end; i++) { + DataCheckerNodeInfo dataCheckerNodeInfo = dataCheckerNodeInfoList.get(i); + dataCheckerNodeResponse.getDataCheckerNodeInfoList().add(dataCheckerNodeInfo); + } + + return dataCheckerNodeResponse; + } + + + public List dataCheckerNodeResultFilter(List dataCheckerNodeInfoList, DataCheckerNodeRequest request) { + + + dataCheckerNodeInfoList = dataCheckerNodeInfoList.stream().filter(dataCheckerNodeInfo -> { + + boolean flag = true; + + if (!CollectionUtils.isEmpty(request.getProjectNameList())) { + flag = request.getProjectNameList().contains(dataCheckerNodeInfo.getProjectName()); + } + + if (!StringUtils.isEmpty(request.getOrchestratorName()) && flag) { + flag = request.getOrchestratorName().equals(dataCheckerNodeInfo.getOrchestratorName()); + } + + if (!CollectionUtils.isEmpty(request.getNodeNameList()) && flag) { + flag = request.getNodeNameList().contains(dataCheckerNodeInfo.getNodeName()); + } + + if (!StringUtils.isEmpty(request.getSourceType()) && flag) { + flag = request.getSourceType().equals(dataCheckerNodeInfo.getSourceType()); + } + + if (!StringUtils.isEmpty(request.getCheckObject()) && flag) { + flag = StringUtils.isNotEmpty(dataCheckerNodeInfo.getCheckObject()) + && dataCheckerNodeInfo.getCheckObject().contains(request.getCheckObject()); + } + + if (!StringUtils.isEmpty(request.getJobDesc()) && flag) { + + flag = StringUtils.isNotEmpty(dataCheckerNodeInfo.getJobDesc()) + && dataCheckerNodeInfo.getJobDesc().contains(request.getJobDesc()); + } + + if (request.getQualitisCheck() != null && flag) { + flag = request.getQualitisCheck().equals(dataCheckerNodeInfo.getQualitisCheck()); + } + + return flag; + + }).collect(Collectors.toList()); + + + dataCheckerNodeInfoList = dataCheckerNodeInfoList.stream().sorted(Comparator.comparing(DataCheckerNodeInfo::getCreateTime, + Comparator.nullsFirst(Comparator.naturalOrder())).reversed()).collect(Collectors.toList()); + + + return dataCheckerNodeInfoList; + + } + + + public EventSenderNodeResponse queryEventSenderNode(String username, Workspace workspace, EventSenderNodeRequest request) { + + EventSenderNodeResponse eventSenderNodeResponse = new EventSenderNodeResponse(); + + List nodeInfoList = nodeInfoMapper.getNodeTypeByGroupName(WorkflowNodeGroupEnum.SignalNode.getNameEn() + , SignalNodeConstant.eventSenderNode); + + if (CollectionUtils.isEmpty(nodeInfoList)) { + logger.error("queryEventSenderNode not find dataChecker node info"); + return eventSenderNodeResponse; + } + + List dssProjectList = getDSSProject(workspace, username); + + if (CollectionUtils.isEmpty(dssProjectList)) { + logger.error("queryEventSenderNode workspace is {}, username is {}", workspace, username); + return eventSenderNodeResponse; + } + + Map dssProjectMap = projectListToMap(dssProjectList); + + List projectIdList = new ArrayList<>(dssProjectMap.keySet()); + List nodeTypeList = nodeInfoList.stream().map(NodeInfo::getNodeType).collect(Collectors.toList()); + List flowNodeInfoList = nodeContentMapper.queryFlowNodeInfo(projectIdList, nodeTypeList); + + if (CollectionUtils.isEmpty(flowNodeInfoList)) { + logger.error("queryEventSenderNode query not find node info , example projectId is {}", projectIdList.get(0)); + return eventSenderNodeResponse; + } + + Map dssFlowNodeInfoMap = flowNodeToMap(flowNodeInfoList); + List contentIdList = new ArrayList<>(dssFlowNodeInfoMap.keySet()); + + // 查询节点保存的配置信息 + Map> nodeContentUIGroup = getNodeContentUIGroup(contentIdList); + if (nodeContentUIGroup.isEmpty()) { + logger.error("queryEventSenderNode query not find nodeUI info , example contextId is {}", contentIdList.get(0)); + return eventSenderNodeResponse; + } + + + List eventSenderNodeInfoList = new ArrayList<>(); + + for (Long contentId : nodeContentUIGroup.keySet()) { + + EventSenderNodeInfo eventSenderNodeInfo = new EventSenderNodeInfo(); + + DSSFlowNodeInfo dssFlowNodeInfo = dssFlowNodeInfoMap.get(contentId); + List nodeUIList = nodeContentUIGroup.get(contentId); + DSSProject dssProject = dssProjectMap.get(dssFlowNodeInfo.getProjectId()); + + List nodeUIKeys = nodeUIList.stream().map(NodeContentUIDO::getNodeUIKey).collect(Collectors.toList()); + List nodeUIValues = nodeUIList.stream().map(NodeContentUIDO::getNodeUIValue).collect(Collectors.toList()); + Map nodeMap = CollUtil.zip(nodeUIKeys, nodeUIValues); + + NodeBaseInfo nodeBaseInfo = getNodeBaseInfo(dssFlowNodeInfo, dssProject, nodeMap,username); + BeanUtils.copyProperties(nodeBaseInfo, eventSenderNodeInfo); + eventSenderNodeInfo.setNodeDesc(nodeMap.get("desc")); + eventSenderNodeInfo.setMsgType(nodeMap.get("msg.type")); + eventSenderNodeInfo.setMsgSender(nodeMap.get("msg.sender")); + eventSenderNodeInfo.setMsgBody(nodeMap.get("msg.body")); + eventSenderNodeInfo.setMsgTopic(nodeMap.get("msg.topic")); + eventSenderNodeInfo.setMsgName(nodeMap.get("msg.name")); + + eventSenderNodeInfoList.add(eventSenderNodeInfo); + + } + + eventSenderNodeInfoList = eventSenderNodeResultFilter(eventSenderNodeInfoList, request); + + eventSenderNodeResponse.setTotal((long) eventSenderNodeInfoList.size()); + // 分页处理 + int page = request.getPageNow() >= 1 ? request.getPageNow() : 1; + int pageSize = request.getPageSize() >= 1 ? request.getPageSize() : 10; + int start = (page - 1) * pageSize; + int end = Math.min(page * pageSize, eventSenderNodeInfoList.size()); + + for (int i = start; i < end; i++) { + EventSenderNodeInfo eventSenderNodeInfo = eventSenderNodeInfoList.get(i); + eventSenderNodeResponse.getEventSenderNodeInfoList().add(eventSenderNodeInfo); + } + + return eventSenderNodeResponse; + } + + @Override + public QueryNodeInfoByPathResponse queryNodeInfo(Long projectId, String orchestratorName, String nodeName) { + QueryNodeInfoByPathResponse result = new QueryNodeInfoByPathResponse(); + DSSFlowNodeInfoOfFlow info= nodeContentMapper.getNodeInfoOfFLow(projectId, orchestratorName, nodeName); + if(info==null){ + return null; + } + result.setAppId(info.getAppId()); + result.setNodeName(nodeName); + result.setProjectId(projectId); + result.setOrchestratorId(info.getOrchestratorId()); + result.setOrchestratorName(orchestratorName); + return result; + } + + public List eventSenderNodeResultFilter(List eventSenderNodeInfoList, EventSenderNodeRequest request) { + + eventSenderNodeInfoList = eventSenderNodeInfoList.stream().filter(eventSenderNodeInfo -> { + + boolean flag = true; + + if (!CollectionUtils.isEmpty(request.getProjectNameList())) { + flag = request.getProjectNameList().contains(eventSenderNodeInfo.getProjectName()); + } + + if (!StringUtils.isEmpty(request.getOrchestratorName()) && flag) { + flag = request.getOrchestratorName().equals(eventSenderNodeInfo.getOrchestratorName()); + } + + if (!CollectionUtils.isEmpty(request.getNodeNameList()) && flag) { + flag = request.getNodeNameList().contains(eventSenderNodeInfo.getNodeName()); + } + + if (!StringUtils.isEmpty(request.getMsgSender()) && flag) { + flag = StringUtils.isNotEmpty(eventSenderNodeInfo.getMsgSender()) + && eventSenderNodeInfo.getMsgSender().contains(request.getMsgSender()); + } + + if (!StringUtils.isEmpty(request.getMsgBody()) && flag) { + + flag = StringUtils.isNotEmpty(eventSenderNodeInfo.getMsgBody()) + && eventSenderNodeInfo.getMsgBody().contains(request.getMsgBody()); + } + + if (!StringUtils.isEmpty(request.getMsgTopic()) && flag) { + flag = StringUtils.isNotEmpty(eventSenderNodeInfo.getMsgTopic()) + && eventSenderNodeInfo.getMsgTopic().contains(request.getMsgTopic()); + } + + if (!StringUtils.isEmpty(request.getMsgName()) && flag) { + flag = StringUtils.isNotEmpty(eventSenderNodeInfo.getMsgName()) + && eventSenderNodeInfo.getMsgName().contains(request.getMsgName()); + } + + return flag; + + }).collect(Collectors.toList()); + + eventSenderNodeInfoList = eventSenderNodeInfoList.stream().sorted(Comparator.comparing(EventSenderNodeInfo::getCreateTime, + Comparator.nullsFirst(Comparator.naturalOrder())).reversed()).collect(Collectors.toList()); + + return eventSenderNodeInfoList; + + } + + + public EventReceiveNodeResponse queryEventReceiveNode(String username, Workspace workspace, EventReceiverNodeRequest request) { + + EventReceiveNodeResponse eventReceiveNodeResponse = new EventReceiveNodeResponse(); + + List nodeInfoList = nodeInfoMapper.getNodeTypeByGroupName(WorkflowNodeGroupEnum.SignalNode.getNameEn() + , SignalNodeConstant.eventReceiverNode); + + if (CollectionUtils.isEmpty(nodeInfoList)) { + logger.error("queryEventReceiveNode not find dataChecker node info"); + return eventReceiveNodeResponse; + } + + List dssProjectList = getDSSProject(workspace, username); + + if (CollectionUtils.isEmpty(dssProjectList)) { + logger.error("queryEventReceiveNode workspace is {}, username is {}", workspace, username); + return eventReceiveNodeResponse; + } + + Map dssProjectMap = projectListToMap(dssProjectList); + + List projectIdList = new ArrayList<>(dssProjectMap.keySet()); + List nodeTypeList = nodeInfoList.stream().map(NodeInfo::getNodeType).collect(Collectors.toList()); + List flowNodeInfoList = nodeContentMapper.queryFlowNodeInfo(projectIdList, nodeTypeList); + +// Map> nodeInfoGroup = nodeUIInfoGroupByNodeType(nodeTypeList); + + if (CollectionUtils.isEmpty(flowNodeInfoList)) { + logger.error("queryEventReceiveNode query not find node info , example projectId is {}", projectIdList.get(0)); + return eventReceiveNodeResponse; + } + + Map dssFlowNodeInfoMap = flowNodeToMap(flowNodeInfoList); + List contentIdList = new ArrayList<>(dssFlowNodeInfoMap.keySet()); + + // 查询节点保存的配置信息 + Map> nodeContentUIGroup = getNodeContentUIGroup(contentIdList); + if (nodeContentUIGroup.isEmpty()) { + logger.error("queryEventSenderNode query not find nodeUI info , example contextId is {}", contentIdList.get(0)); + return eventReceiveNodeResponse; + } + + + List eventReceiverNodeInfoList = new ArrayList<>(); + + for (Long contentId : nodeContentUIGroup.keySet()) { + + EventReceiverNodeInfo eventReceiverNodeInfo = new EventReceiverNodeInfo(); + + DSSFlowNodeInfo dssFlowNodeInfo = dssFlowNodeInfoMap.get(contentId); + List nodeUIList = nodeContentUIGroup.get(contentId); + DSSProject dssProject = dssProjectMap.get(dssFlowNodeInfo.getProjectId()); +// Map nodeDefaultValue = getNodeDefaultValue(nodeInfoGroup, dssFlowNodeInfo.getJobType()); + + List nodeUIKeys = nodeUIList.stream().map(NodeContentUIDO::getNodeUIKey).collect(Collectors.toList()); + List nodeUIValues = nodeUIList.stream().map(NodeContentUIDO::getNodeUIValue).collect(Collectors.toList()); + Map nodeMap = CollUtil.zip(nodeUIKeys, nodeUIValues); + + NodeBaseInfo nodeBaseInfo = getNodeBaseInfo(dssFlowNodeInfo, dssProject, nodeMap,username); + BeanUtils.copyProperties(nodeBaseInfo, eventReceiverNodeInfo); + eventReceiverNodeInfo.setNodeDesc(nodeMap.get("desc")); + eventReceiverNodeInfo.setMsgType(nodeMap.get("msg.type")); + eventReceiverNodeInfo.setMsgReceiver(nodeMap.get("msg.receiver")); + eventReceiverNodeInfo.setMsgTopic(nodeMap.get("msg.topic")); + eventReceiverNodeInfo.setMsgName(nodeMap.get("msg.name")); + eventReceiverNodeInfo.setQueryFrequency(nodeMap.get("query.frequency")); + eventReceiverNodeInfo.setMaxReceiveHours(nodeMap.get("max.receive.hours")); + eventReceiverNodeInfo.setMsgSaveKey(nodeMap.get("msg.savekey")); + + if (nodeMap.containsKey("only.receive.today") && StringUtils.isNotEmpty(nodeMap.get("only.receive.today"))) { + eventReceiverNodeInfo.setOnlyReceiveToday(Boolean.valueOf(nodeMap.get("only.receive.today"))); + } + + if (nodeMap.containsKey("msg.receive.use.rundate") && StringUtils.isNotEmpty(nodeMap.get("msg.receive.use.rundate"))) { + eventReceiverNodeInfo.setMsgReceiveUseRunDate(Boolean.valueOf(nodeMap.get("msg.receive.use.rundate"))); + } + + eventReceiverNodeInfoList.add(eventReceiverNodeInfo); + } + + eventReceiverNodeInfoList = eventReceiveNodeResultFilter(eventReceiverNodeInfoList, request); + + eventReceiveNodeResponse.setTotal((long) eventReceiverNodeInfoList.size()); + // 分页处理 + int page = request.getPageNow() >= 1 ? request.getPageNow() : 1; + int pageSize = request.getPageSize() >= 1 ? request.getPageSize() : 10; + int start = (page - 1) * pageSize; + int end = Math.min(page * pageSize, eventReceiverNodeInfoList.size()); + + for (int i = start; i < end; i++) { + EventReceiverNodeInfo eventReceiverNodeInfo = eventReceiverNodeInfoList.get(i); + eventReceiveNodeResponse.getEventReceiverNodeInfoList().add(eventReceiverNodeInfo); + } + + return eventReceiveNodeResponse; + + } + + + public List eventReceiveNodeResultFilter(List eventReceiverNodeInfoList, + EventReceiverNodeRequest request) { + + eventReceiverNodeInfoList = eventReceiverNodeInfoList.stream().filter(eventReceiverNodeInfo -> { + + boolean flag = true; + + if (!CollectionUtils.isEmpty(request.getProjectNameList())) { + flag = request.getProjectNameList().contains(eventReceiverNodeInfo.getProjectName()); + } + + if (!StringUtils.isEmpty(request.getOrchestratorName()) && flag) { + flag = request.getOrchestratorName().equals(eventReceiverNodeInfo.getOrchestratorName()); + } + + if (!CollectionUtils.isEmpty(request.getNodeNameList()) && flag) { + flag = request.getNodeNameList().contains(eventReceiverNodeInfo.getNodeName()); + } + + if (!StringUtils.isEmpty(request.getMsgReceiver()) && flag) { + + flag = StringUtils.isNotEmpty(eventReceiverNodeInfo.getMsgReceiver()) + && eventReceiverNodeInfo.getMsgReceiver().contains(request.getMsgReceiver()); + } + + if (!StringUtils.isEmpty(request.getMsgTopic()) && flag) { + flag = StringUtils.isNotEmpty(eventReceiverNodeInfo.getMsgTopic()) + && eventReceiverNodeInfo.getMsgTopic().contains(request.getMsgTopic()); + } + + if (!StringUtils.isEmpty(request.getMsgName()) && flag) { + + flag = StringUtils.isNotEmpty(eventReceiverNodeInfo.getMsgName()) + && eventReceiverNodeInfo.getMsgName().contains(request.getMsgName()); + } + + if (!StringUtils.isEmpty(request.getMaxReceiveHours()) && flag) { +// flag = request.getMaxReceiveHours().equals(eventReceiverNodeInfo.getMaxReceiveHours()); + flag = StringUtils.isNotEmpty(eventReceiverNodeInfo.getMaxReceiveHours()) + && eventReceiverNodeInfo.getMaxReceiveHours().contains(request.getMaxReceiveHours()); + } + + if (!StringUtils.isEmpty(request.getMsgSaveKey()) && flag) { + flag = StringUtils.isNotEmpty(eventReceiverNodeInfo.getMsgSaveKey()) + && eventReceiverNodeInfo.getMsgSaveKey().contains(request.getMsgSaveKey()); + } + + if (request.getOnlyReceiveToday() != null && flag) { + flag = request.getOnlyReceiveToday().equals(eventReceiverNodeInfo.getOnlyReceiveToday()); + } + + if (request.getMsgReceiveUseRunDate() != null && flag) { + flag = request.getMsgReceiveUseRunDate().equals(eventReceiverNodeInfo.getMsgReceiveUseRunDate()); + } + + return flag; + + }).collect(Collectors.toList()); + + eventReceiverNodeInfoList = eventReceiverNodeInfoList.stream().sorted(Comparator.comparing(EventReceiverNodeInfo::getCreateTime, + Comparator.nullsFirst(Comparator.naturalOrder())).reversed()).collect(Collectors.toList()); + + return eventReceiverNodeInfoList; + + } + + + public List queryNodeUiValueByKey(Workspace workspace, String username, String nodeUiKey, String groupNameEn) { + + logger.info("nodeUiKey is {}", nodeUiKey); + List nodeUIValue = new ArrayList<>(); + try { + + // 获取项目 + List dssProjectList = getDSSProject(workspace, username); + + if (CollectionUtils.isEmpty(dssProjectList)) { + logger.error("queryNodeUiValueByKey find project is empty, workspaceId is {}, username is {}, nodeUikey is {}", workspace.getWorkspaceId(), username, nodeUiKey); + return nodeUIValue; + } + + List projectIdList = dssProjectList.stream().map(DSSProject::getId).collect(Collectors.toList()); + + List nodeInfoList = getNodeInfoByGroupName(groupNameEn); + if (CollectionUtils.isEmpty(nodeInfoList)) { + logger.error("queryNodeUiValueByKey groupNameEn is {}", groupNameEn); + return nodeUIValue; + } + List nodeTypeList = nodeInfoList.stream().map(NodeInfo::getNodeType).collect(Collectors.toList()); + // 查询节点信息 + List flowNodeInfoList = nodeContentMapper.queryFlowNodeInfo(projectIdList, nodeTypeList); + + if (CollectionUtils.isEmpty(flowNodeInfoList)) { + logger.error("queryNodeUiValueByKey find node info is empty, example project id is {}, nodeUikey is {}", projectIdList.get(0), nodeUiKey); + return nodeUIValue; + } + + List contentIdList = flowNodeInfoList.stream().map(DSSFlowNodeInfo::getContentId).collect(Collectors.toList()); + List nodeContentUIDOList = nodeContentUIMapper.getNodeContentUIByNodeUIKey(contentIdList, nodeUiKey); + nodeUIValue = nodeContentUIDOList.stream().map(NodeContentUIDO::getNodeUIValue).distinct().collect(Collectors.toList()); + + } catch (Exception e) { + logger.error("queryNodeUiValueByKey error, workspaceId is {}, username is {}, nodeUikey is {}", workspace.getWorkspaceId(), username, nodeUiKey); + logger.error(e.getMessage()); + } + + // 过滤出不为空的信息 + return nodeUIValue.stream().filter(StringUtils::isNotEmpty).collect(Collectors.toList()); + } + + + @Override + public List querySourceType(Workspace workspace, String username) { + String nodeUIkey = "source.type"; + return queryNodeUiValueByKey(workspace, username, nodeUIkey, WorkflowNodeGroupEnum.SignalNode.getNameEn()); + } + + + @Override + public List queryJobDesc(Workspace workspace, String username) { + String nodeUIkey = "job.desc"; + + List jobDescValue = queryNodeUiValueByKey(workspace, username, nodeUIkey, WorkflowNodeGroupEnum.SignalNode.getNameEn()); + + List descValue = new ArrayList<>(); + for (String value : jobDescValue) { + if (StringUtils.isEmpty(value)) { + continue; + } + + descValue.addAll(splitJobDesc(value)); + } + + return descValue.stream().filter(value -> { + return !StringUtils.isEmpty(value.trim()); + }).distinct().collect(Collectors.toList()); + + } + + + public Map> nodeUIInfoGroupByNodeType(List nodeTypeList) { + + List nodeUIInfoList = nodeInfoMapper.queryNodeUIInfoList(nodeTypeList); + return nodeUIInfoList.stream().collect(Collectors.groupingBy(NodeUIInfo::getNodeType)); + + } + + + public Map getNodeDefaultValue(Map> nodeInfoGroup, String nodeType) { + + List nodeUIInfoList = nodeInfoGroup.get(nodeType); + Map nodeDefaultValue = new HashMap<>(); + if (CollectionUtils.isNotEmpty(nodeUIInfoList)) { + List key = nodeUIInfoList.stream().map(NodeUIInfo::getKey).collect(Collectors.toList()); + List value = nodeUIInfoList.stream().map(NodeUIInfo::getDefaultValue).collect(Collectors.toList()); + nodeDefaultValue = CollUtil.zip(key, value); + } + + return nodeDefaultValue; + + } + + + + + public NodeBaseInfo getNodeBaseInfo(DSSFlowNodeInfo dssFlowNodeInfo, DSSProject dssProject, Map nodeMap,String username) { + + NodeBaseInfo nodeBaseInfo = new NodeBaseInfo(); + + nodeBaseInfo.setNodeName(nodeMap.get("title")); + nodeBaseInfo.setNodeTypeName(dssFlowNodeInfo.getNodeTypeName()); + nodeBaseInfo.setNodeType(dssFlowNodeInfo.getJobType()); + nodeBaseInfo.setNodeId(dssFlowNodeInfo.getNodeId()); + nodeBaseInfo.setOrchestratorId(dssFlowNodeInfo.getOrchestratorId()); + nodeBaseInfo.setNodeKey(dssFlowNodeInfo.getNodeKey()); + nodeBaseInfo.setOrchestratorName(dssFlowNodeInfo.getOrchestratorName()); + nodeBaseInfo.setProjectName(dssProject.getName()); + nodeBaseInfo.setProjectId(dssProject.getId()); + nodeBaseInfo.setContentId(dssFlowNodeInfo.getContentId()); + nodeBaseInfo.setUpdateTime(dssFlowNodeInfo.getModifyTime()); + nodeBaseInfo.setCreateTime(dssFlowNodeInfo.getCreateTime()); + nodeBaseInfo.setNodeContent(nodeMap); + nodeBaseInfo.setAssociateGit(dssProject.getAssociateGit()); + boolean editable = (CollectionUtils.isNotEmpty(dssProject.getEditUsers()) && dssProject.getEditUsers().contains(username)) + || (CollectionUtils.isNotEmpty(dssProject.getReleaseUsers()) && dssProject.getReleaseUsers().contains(username)) + || (StringUtils.isNotEmpty(dssProject.getCreateBy()) && dssProject.getCreateBy().equals(username)); + nodeBaseInfo.setEditable(editable); + nodeBaseInfo.setFlowId(dssFlowNodeInfo.getFlowId()); + + return nodeBaseInfo; + + } + + + public List splitJobDesc(String jobDesc) { + + if (jobDesc.contains(";")) { + return Arrays.asList(jobDesc.trim().split(";")); + } else if (jobDesc.contains("\n")) { + return Arrays.asList(jobDesc.trim().split("\n")); + } else { + return Arrays.asList(jobDesc.trim().split("")); + } + + } + + + public Map getFlowMap(List flowNodeInfoList){ + + Map flowMap = new HashMap<>(); + + List orchestratorIdList = flowNodeInfoList.stream().map(DSSFlowNodeInfo::getOrchestratorId) + .collect(Collectors.toList()); + + if(CollectionUtils.isEmpty(orchestratorIdList)){ + logger.error("getFlowMap orchestratorIdList is empty"); + return flowMap; + } + + List dssFlowList = flowMapper.selectFlowListByOrchestratorId(orchestratorIdList); + + if(CollectionUtils.isEmpty(dssFlowList)){ + logger.error("example orchestratorId is {}",orchestratorIdList.get(0)); + logger.error("getFlowMap dssFlow list is empty"); + return flowMap; + } + + Map> dssFlowMap = dssFlowList.stream().collect(Collectors.groupingBy(DSSFlow::getOrchestratorId)); + + + for(Long orchestratorId : dssFlowMap.keySet()){ + + DSSFlow dssFlow = dssFlowMap.get(orchestratorId).stream().max(Comparator.comparing(DSSFlow::getId)).orElse(null); + + if(dssFlow == null){ + continue; + } + + flowMap.put(orchestratorId,dssFlow); + } + + return flowMap; + + } + + +} + + + diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/service/impl/PublishServiceImpl.java b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/service/impl/PublishServiceImpl.java index 92bdc280b5..d065b976a0 100644 --- a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/service/impl/PublishServiceImpl.java +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/java/com/webank/wedatasphere/dss/workflow/service/impl/PublishServiceImpl.java @@ -18,31 +18,49 @@ import com.webank.wedatasphere.dss.appconn.manager.AppConnManager; import com.webank.wedatasphere.dss.appconn.scheduler.SchedulerAppConn; +import com.webank.wedatasphere.dss.common.auditlog.OperateTypeEnum; +import com.webank.wedatasphere.dss.common.auditlog.TargetTypeEnum; +import com.webank.wedatasphere.dss.common.constant.project.ProjectUserPrivEnum; import com.webank.wedatasphere.dss.common.entity.project.DSSProject; import com.webank.wedatasphere.dss.common.exception.DSSErrorException; +import com.webank.wedatasphere.dss.common.label.DSSLabel; +import com.webank.wedatasphere.dss.common.label.EnvDSSLabel; import com.webank.wedatasphere.dss.common.protocol.project.ProjectInfoRequest; +import com.webank.wedatasphere.dss.common.protocol.project.ProjectUserAuthRequest; +import com.webank.wedatasphere.dss.common.protocol.project.ProjectUserAuthResponse; +import com.webank.wedatasphere.dss.common.utils.AuditLogUtils; import com.webank.wedatasphere.dss.common.utils.DSSExceptionUtils; import com.webank.wedatasphere.dss.common.utils.RpcAskUtils; -import com.webank.wedatasphere.dss.orchestrator.common.protocol.RequestFrameworkConvertOrchestration; -import com.webank.wedatasphere.dss.orchestrator.common.protocol.RequestFrameworkConvertOrchestrationStatus; -import com.webank.wedatasphere.dss.orchestrator.common.protocol.ResponseConvertOrchestrator; +import com.webank.wedatasphere.dss.git.common.protocol.request.GitCurrentCommitRequest; +import com.webank.wedatasphere.dss.git.common.protocol.response.GitCommitResponse; +import com.webank.wedatasphere.dss.orchestrator.common.entity.DSSOrchestratorVersion; +import com.webank.wedatasphere.dss.orchestrator.common.entity.OrchestratorVo; +import com.webank.wedatasphere.dss.orchestrator.common.protocol.*; +import com.webank.wedatasphere.dss.orchestrator.common.ref.OrchestratorRefConstant; import com.webank.wedatasphere.dss.sender.service.DSSSenderServiceFactory; import com.webank.wedatasphere.dss.standard.app.sso.Workspace; import com.webank.wedatasphere.dss.standard.common.desc.AppInstance; import com.webank.wedatasphere.dss.workflow.common.entity.DSSFlow; import com.webank.wedatasphere.dss.workflow.common.parser.WorkFlowParser; import com.webank.wedatasphere.dss.workflow.constant.DSSWorkFlowConstant; +import com.webank.wedatasphere.dss.workflow.dao.LockMapper; +import com.webank.wedatasphere.dss.workflow.entity.DSSFlowEditLock; +import com.webank.wedatasphere.dss.workflow.entity.request.BatchPublishWorkflowRequest; +import com.webank.wedatasphere.dss.workflow.lock.DSSFlowEditLockManager; import com.webank.wedatasphere.dss.workflow.service.DSSFlowService; import com.webank.wedatasphere.dss.workflow.service.PublishService; +import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang.StringUtils; import org.apache.linkis.rpc.Sender; +import org.apache.linkis.server.Message; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; -import java.util.Map; +import java.util.*; +import java.util.stream.Collectors; -import static com.webank.wedatasphere.dss.workflow.constant.DSSWorkFlowConstant.DEFAULT_SCHEDULER_APP_CONN; +import static com.webank.wedatasphere.dss.workflow.constant.DSSWorkFlowConstant.*; public class PublishServiceImpl implements PublishService { @@ -50,6 +68,10 @@ public class PublishServiceImpl implements PublishService { private DSSFlowService dssFlowService; @Autowired private WorkFlowParser workFlowParser; + @Autowired + private LockMapper lockMapper; + @Autowired + private DSSFlowService flowService; public void setDssFlowService(DSSFlowService dssFlowService) { this.dssFlowService = dssFlowService; @@ -73,17 +95,9 @@ public String submitPublish(String convertUser, Long workflowId, //2.进行提交 DSSFlow dssFlow = null; try { - dssFlow = dssFlowService.getFlow(workflowId); - if (dssFlow == null) { - DSSExceptionUtils.dealErrorException(63325, "workflow " + workflowId + " is not exists.", DSSErrorException.class); - return null; - } - ProjectInfoRequest projectInfoRequest = new ProjectInfoRequest(); - projectInfoRequest.setProjectId(dssFlow.getProjectId()); - DSSProject dssProject = (DSSProject) DSSSenderServiceFactory.getOrCreateServiceInstance().getProjectServerSender().ask(projectInfoRequest); - if (dssProject.getWorkspaceId() != workspace.getWorkspaceId()) { - DSSExceptionUtils.dealErrorException(63335, "工作流所在工作空间和cookie中不一致，请刷新页面后，再次发布！", DSSErrorException.class); - } + LOGGER.info("trace workflow publish,flowId:{}. start submit publish",workflowId); + dssFlow = checkFlowBeforeSubmit(workflowId, convertUser, workspace); + LOGGER.info("trace workflow publish,flowId:{}. 发布前校验完成",workflowId); String schedulerAppConnName = workFlowParser.getValueWithKey(dssFlow.getFlowJson(), DSSWorkFlowConstant.SCHEDULER_APP_CONN_NAME); if (StringUtils.isBlank(schedulerAppConnName)) { // 向下兼容老版本 @@ -93,10 +107,11 @@ public String submitPublish(String convertUser, Long workflowId, // 只是为了获取是否需要发布所有Orc，这里直接拿第一个AppInstance即可。 AppInstance appInstance = schedulerAppConn.getAppDesc().getAppInstances().get(0); ResponseConvertOrchestrator response = requestConvertOrchestration(comment, workflowId, convertUser, - workspace, schedulerAppConn, dssLabel, appInstance); + workspace, schedulerAppConn, dssLabel, appInstance, null); if (response.getResponse().isFailed()) { throw new DSSErrorException(50311, response.getResponse().getMessage()); } + return response.getId(); } catch (DSSErrorException e) { throw e; @@ -108,13 +123,126 @@ public String submitPublish(String convertUser, Long workflowId, return null; } + private DSSFlow checkFlowBeforeSubmit(Long workflowId, String convertUser, Workspace workspace) throws Exception{ + DSSFlow dssFlow = dssFlowService.getFlow(workflowId); + if (dssFlow == null) { + DSSExceptionUtils.dealErrorException(63325, "workflow " + workflowId + " is not exists.", DSSErrorException.class); + return null; + } + LOGGER.info("User {} begins to convert workflow {}.", convertUser, dssFlow.getName()); + Long projectId = dssFlow.getProjectId(); + ProjectInfoRequest projectInfoRequest = new ProjectInfoRequest(); + projectInfoRequest.setProjectId(projectId); + // 校验发布权限 + ProjectUserAuthResponse projectUserAuthResponse = RpcAskUtils.processAskException(DSSSenderServiceFactory.getOrCreateServiceInstance() + .getProjectServerSender().ask(new ProjectUserAuthRequest(projectId, convertUser)), + ProjectUserAuthResponse.class, ProjectUserAuthRequest.class); + boolean isReleasable = false; + if (!CollectionUtils.isEmpty(projectUserAuthResponse.getPrivList())) { + isReleasable = projectUserAuthResponse.getPrivList().contains(ProjectUserPrivEnum.PRIV_RELEASE.getRank()); + } + isReleasable = isReleasable || projectUserAuthResponse.getProjectOwner().equals(convertUser); + + + + DSSProject dssProject = (DSSProject) DSSSenderServiceFactory.getOrCreateServiceInstance().getProjectServerSender().ask(projectInfoRequest); + if (!isReleasable) { + throw new DSSErrorException(800001, "用户" + convertUser + "没有项目" + dssProject.getName() + "发布权限，请检查后重新发布"); + } + + if (dssProject.getWorkspaceId() != workspace.getWorkspaceId()) { + DSSExceptionUtils.dealErrorException(63335, "工作流所在工作空间和cookie中不一致，请刷新页面后，再次发布！", DSSErrorException.class); + } + //仅对接入Git的项目更新状态为 发布-publish + GitCommitResponse gitCommitResponse = null; + OrchestratorVo orchestratorVo = null; + Long orchestratorId = null; + if (dssProject.getAssociateGit() != null && dssProject.getAssociateGit()) { + orchestratorVo = RpcAskUtils.processAskException(getOrchestratorSender().ask(new RequestQuertByAppIdOrchestrator(workflowId)), + OrchestratorVo.class, RequestQueryByIdOrchestrator.class); + if (orchestratorVo == null) { + throw new DSSErrorException(800001, "编排不存在"); + } + orchestratorId = orchestratorVo.getDssOrchestratorInfo().getId(); + String status = lockMapper.selectOrchestratorStatus(orchestratorId); + if (OrchestratorRefConstant.FLOW_STATUS_SAVE.equals(status)) { + throw new DSSErrorException(800001, "发布前请先提交工作流"); + } + + try { + // 获取当前文件Commit + Sender sender = DSSSenderServiceFactory.getOrCreateServiceInstance().getGitSender(); + GitCurrentCommitRequest currentCommitRequest = new GitCurrentCommitRequest(workspace.getWorkspaceId(), dssProject.getName(), convertUser, dssFlow.getName()); + gitCommitResponse = RpcAskUtils.processAskException(sender.ask(currentCommitRequest), GitCommitResponse.class, GitCurrentCommitRequest.class); + // 更新commitId + lockMapper.updateOrchestratorVersionCommitId(gitCommitResponse.getCommitId(), orchestratorId); + // 更新工作流状态 + lockMapper.updateOrchestratorStatus(orchestratorId, OrchestratorRefConstant.FLOW_STATUS_PUBLISH); + } catch (Exception e) { + throw new DSSErrorException(800001, "获取工作流CommitId失败，请检查工作流是否为空或git服务是否异常"); + } + } + return dssFlow; + } + + @Override + public void batchPublish(BatchPublishWorkflowRequest publishWorkflowRequest, Workspace workspace, String convertUser, Map dssLabel) throws Exception{ + List orcIds = publishWorkflowRequest.getOrchestratorList(); + String labelStr = publishWorkflowRequest.getLabels().getRoute(); + Map labels = new HashMap<>(); + labels.put(EnvDSSLabel.DSS_ENV_LABEL_KEY, labelStr); + String comment = publishWorkflowRequest.getComment(); + + List dssLabelList = Arrays.asList(new EnvDSSLabel(labelStr)); + Sender sender = DSSSenderServiceFactory.getOrCreateServiceInstance().getOrcSender(dssLabelList); + RequestQueryOrchestrator queryRequest = new RequestQueryOrchestrator(orcIds); + List workflowIdList = new ArrayList<>(); + + + try { + ResponseQueryOrchestrator queryResponse = RpcAskUtils.processAskException(sender.ask(queryRequest), ResponseQueryOrchestrator.class, RequestQueryOrchestrator.class); + if (queryResponse == null) { + LOGGER.error("query response is null, it is a fatal error"); + throw new DSSErrorException(80001, "查询编排失败，请确认编排是否存在") ; + } + Set orchestratorVersions = queryResponse.getOrchestratorVoes().stream().map(OrchestratorVo::getDssOrchestratorVersion).collect(Collectors.toSet()); + workflowIdList = orchestratorVersions.stream().map(DSSOrchestratorVersion::getAppId).collect(Collectors.toList()); + } catch (Exception e) { + DSSExceptionUtils.dealErrorException(60015, "query orchestrator ref failed", + DSSErrorException.class); + } + + List dssFlowList = new ArrayList<>(); + for (Long flowId : workflowIdList) { + DSSFlow dssFlow = checkFlowBeforeSubmit(flowId, convertUser, workspace); + dssFlowList.add(dssFlow); + } + + String schedulerAppConnName = workFlowParser.getValueWithKey(dssFlowList.get(0).getFlowJson(), DSSWorkFlowConstant.SCHEDULER_APP_CONN_NAME); + if (StringUtils.isBlank(schedulerAppConnName)) { + // 向下兼容老版本 + schedulerAppConnName = DEFAULT_SCHEDULER_APP_CONN.getValue(); + } + SchedulerAppConn schedulerAppConn = (SchedulerAppConn) AppConnManager.getAppConnManager().getAppConn(schedulerAppConnName); + // 只是为了获取是否需要发布所有Orc，这里直接拿第一个AppInstance即可。 + AppInstance appInstance = schedulerAppConn.getAppDesc().getAppInstances().get(0); + ResponseConvertOrchestrator response = requestConvertOrchestration(comment, null, convertUser, + workspace, schedulerAppConn, dssLabel, appInstance, workflowIdList); + + if (response.getResponse().isFailed()) { + throw new DSSErrorException(50311, response.getResponse().getMessage()); + } + + return ; + } + /** * 可覆写该方法自定义实现request对象 * * @return */ protected ResponseConvertOrchestrator requestConvertOrchestration(String comment, Long workflowId, String convertUser, Workspace workspace, - SchedulerAppConn schedulerAppConn, Map dssLabel, AppInstance appInstance) { + SchedulerAppConn schedulerAppConn, Map dssLabel, AppInstance appInstance, List flowIdList) { RequestFrameworkConvertOrchestration requestFrameworkConvertOrchestration = new RequestFrameworkConvertOrchestration(); requestFrameworkConvertOrchestration.setComment(comment); requestFrameworkConvertOrchestration.setOrcAppId(workflowId); diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/scala/com/webank/wedatasphere/dss/workflow/receiver/DSSWorkflowChooser.scala b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/scala/com/webank/wedatasphere/dss/workflow/receiver/DSSWorkflowChooser.scala index 7501faf90c..dded51ff4e 100644 --- a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/scala/com/webank/wedatasphere/dss/workflow/receiver/DSSWorkflowChooser.scala +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/scala/com/webank/wedatasphere/dss/workflow/receiver/DSSWorkflowChooser.scala @@ -16,10 +16,10 @@ package com.webank.wedatasphere.dss.workflow.receiver -import com.webank.wedatasphere.dss.common.protocol.{RequestDeleteWorkflow, RequestExportWorkflow, RequestQueryWorkFlow, RequestUpdateWorkflow} +import com.webank.wedatasphere.dss.common.protocol.{RequestDeleteWorkflow, RequestExportWorkflow, RequestExportWorkflowList, RequestQueryWorkFlow, RequestReadWorkflowNode, RequestUpdateWorkflow} import com.webank.wedatasphere.dss.orchestrator.common.protocol._ import com.webank.wedatasphere.dss.workflow.WorkFlowManager -import com.webank.wedatasphere.dss.workflow.common.protocol.{RequestCopyWorkflow, RequestCreateWorkflow, RequestDeleteBmlSource, RequestImportWorkflow, RequestSubFlowContextIds, RequestUnlockWorkflow} +import com.webank.wedatasphere.dss.workflow.common.protocol.{RequestCopyWorkflow, RequestCreateWorkflow, RequestDeleteBmlSource, RequestImportWorkflow, RequestLockWorkflow, RequestSubFlowContextIds, RequestUnlockWorkflow} import org.apache.linkis.rpc.{RPCMessageEvent, Receiver, ReceiverChooser} import javax.annotation.PostConstruct @@ -50,6 +50,8 @@ class DSSWorkflowChooser extends ReceiverChooser { case _: RequestConvertOrchestrations => receiver case _: RequestSubFlowContextIds => receiver case _: RequestDeleteBmlSource => receiver + case _: RequestReadWorkflowNode => receiver + case _: RequestExportWorkflowList => receiver case _ => None } diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/scala/com/webank/wedatasphere/dss/workflow/receiver/DSSWorkflowReceiver.scala b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/scala/com/webank/wedatasphere/dss/workflow/receiver/DSSWorkflowReceiver.scala index 954166b0da..1ca3a7f46f 100644 --- a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/scala/com/webank/wedatasphere/dss/workflow/receiver/DSSWorkflowReceiver.scala +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/scala/com/webank/wedatasphere/dss/workflow/receiver/DSSWorkflowReceiver.scala @@ -56,19 +56,44 @@ class DSSWorkflowReceiver(workflowManager: WorkFlowManager) extends Receiver { new ResponseDeleteWorkflow(JobStatus.Success) case reqUnlockWorkflow: RequestUnlockWorkflow => - workflowManager.unlockWorkflow(reqUnlockWorkflow.getUsername, reqUnlockWorkflow.getFlowId, reqUnlockWorkflow.getConfirmDelete) + workflowManager.unlockWorkflow(reqUnlockWorkflow.getUsername, reqUnlockWorkflow.getFlowId, reqUnlockWorkflow.getConfirmDelete, reqUnlockWorkflow.getWorkspace) case reqExportFlow: RequestExportWorkflow => - val dssExportFlowResource: BmlResource = workflowManager.exportWorkflow( + val dssExportFlowResource: BmlResource = workflowManager.exportWorkflowNew( reqExportFlow.userName, reqExportFlow.flowID, reqExportFlow.projectId, reqExportFlow.projectName, DSSCommonUtils.COMMON_GSON.fromJson(reqExportFlow.workspaceStr, classOf[Workspace]), - reqExportFlow.dssLabelList) + reqExportFlow.dssLabelList, + reqExportFlow.exportExternalNodeAppConnResource) ResponseExportWorkflow(dssExportFlowResource.getResourceId, dssExportFlowResource.getVersion, reqExportFlow.flowID) + case requestExportWorkflowList: RequestExportWorkflowList => + val dssExportFlowResource : BmlResource = workflowManager.exportWorkflowListNew(requestExportWorkflowList.userName, + requestExportWorkflowList.flowIDList, + requestExportWorkflowList.projectId, + requestExportWorkflowList.projectName, + DSSCommonUtils.COMMON_GSON.fromJson(requestExportWorkflowList.workspaceStr, classOf[Workspace]), + requestExportWorkflowList.dssLabelList, + requestExportWorkflowList.exportExternalNodeAppConnResource) + ResponseExportWorkflowList(dssExportFlowResource.getResourceId, dssExportFlowResource.getVersion, + requestExportWorkflowList.flowIDList) + + case reqReadFlow: RequestReadWorkflowNode => + val workflowNode: String = workflowManager.readWorkflowNew( + reqReadFlow.userName, + reqReadFlow.flowID, + reqReadFlow.projectId, + reqReadFlow.projectName, + DSSCommonUtils.COMMON_GSON.fromJson(reqReadFlow.workspaceStr, classOf[Workspace]), + reqReadFlow.dssLabelList, + reqReadFlow.exportExternalNodeAppConnResource, + reqReadFlow.filePath + ) + ResponseReadWorkflow(workflowNode) + case requestImportWorkflow: RequestImportWorkflow => val dssFlowImportParam: DSSFlowImportParam = new DSSFlowImportParam() dssFlowImportParam.setProjectID(requestImportWorkflow.getProjectId) @@ -77,7 +102,12 @@ class DSSWorkflowReceiver(workflowManager: WorkFlowManager) extends Receiver { dssFlowImportParam.setOrcVersion(requestImportWorkflow.getOrcVersion) dssFlowImportParam.setWorkspace(requestImportWorkflow.getWorkspace) dssFlowImportParam.setContextId(requestImportWorkflow.getContextId) - val dssFlows = workflowManager.importWorkflow(requestImportWorkflow.getUserName, + val dssFlows = if(requestImportWorkflow.getOldPackageStruct) + workflowManager.importWorkflow(requestImportWorkflow.getUserName, + requestImportWorkflow.getResourceId, + requestImportWorkflow.getBmlVersion, + dssFlowImportParam, requestImportWorkflow.getDssLabels) + else workflowManager.importWorkflowNew(requestImportWorkflow.getUserName, requestImportWorkflow.getResourceId, requestImportWorkflow.getBmlVersion, dssFlowImportParam, requestImportWorkflow.getDssLabels) diff --git a/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/scala/com/webank/wedatasphere/dss/workflow/service/BMLService.scala b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/scala/com/webank/wedatasphere/dss/workflow/service/BMLService.scala new file mode 100644 index 0000000000..74f5dd78f1 --- /dev/null +++ b/dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/src/main/scala/com/webank/wedatasphere/dss/workflow/service/BMLService.scala @@ -0,0 +1,214 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.workflow.service + +import com.webank.wedatasphere.dss.common.entity.BmlResource + +import java.io.{ByteArrayInputStream, InputStream} +import java.util +import java.util.UUID +import com.webank.wedatasphere.dss.common.exception.DSSErrorException +import com.webank.wedatasphere.dss.common.utils.IoUtils +import org.apache.linkis.bml.client.{BmlClient, BmlClientFactory} +import org.apache.linkis.bml.protocol.{BmlDownloadResponse, BmlUpdateResponse, BmlUploadResponse} +import org.apache.linkis.common.utils.{JavaLog, Utils} +import org.apache.commons.io.IOUtils +import org.springframework.stereotype.Component + +import scala.collection.JavaConversions._ + + +@Component +class BMLService extends JavaLog { + + @volatile var bmlClient: BmlClient = _ + + def getBmlClient(userName: String): BmlClient = { + if (bmlClient == null) synchronized { + if (bmlClient == null) { + bmlClient = BmlClientFactory.createBmlClient(userName) + } + } + bmlClient + } + + def upload(userName: String, content: String, fileName: String, projectName: String): util.Map[String, Object] = { + val inputStream = new ByteArrayInputStream(content.getBytes("utf-8")) + val client: BmlClient = getBmlClient(userName) + val resource: BmlUploadResponse = client.uploadShareResource(userName, projectName, fileName, inputStream) + if (!resource.isSuccess) throw new DSSErrorException(911113, "上传失败") + val map = new util.HashMap[String, Object] + map += "resourceId" -> resource.resourceId + map += "version" -> resource.version + } + + def upload(userName: String, inputStream: InputStream, fileName: String, projectName: String): BmlResource = { + val client: BmlClient = getBmlClient(userName) + val resource: BmlUploadResponse = client.uploadShareResource(userName, projectName, fileName, inputStream) + if (!resource.isSuccess) throw new DSSErrorException(911113, "上传失败") + new BmlResource( resource.resourceId,resource.version) + } + + def update(userName: String, resourceId: String, inputStream: InputStream): util.Map[String, Object] = { + val client: BmlClient = getBmlClient(userName) + val resource: BmlUpdateResponse = client.updateShareResource(userName, resourceId, "", inputStream) + if (!resource.isSuccess) throw new DSSErrorException(911114, "更新失败") + val map = new util.HashMap[String, Object] + map += "resourceId" -> resource.resourceId + map += "version" -> resource.version + } + + def update(userName: String, resourceId: String, content: String): util.Map[String, Object] = { + val inputStream = new ByteArrayInputStream(content.getBytes("utf-8")) + val client: BmlClient = getBmlClient(userName) + val resource: BmlUpdateResponse = client.updateShareResource(userName, resourceId, UUID.randomUUID().toString + ".json", inputStream) + if (!resource.isSuccess) throw new DSSErrorException(911114, "更新失败") + val map = new util.HashMap[String, Object] + map += "resourceId" -> resource.resourceId + map += "version" -> resource.version + } + + def query(userName: String, resourceId: String, version: String): util.Map[String, Object] = { + val client: BmlClient = getBmlClient(userName) + var resource: BmlDownloadResponse = null + if (version == null) { + resource = client.downloadShareResource(userName, resourceId) + } else { + resource = client.downloadShareResource(userName, resourceId, version) + } + if (!resource.isSuccess) throw new DSSErrorException(911115, "下载失败") + val map = new util.HashMap[String, Object] + map += "path" -> resource.fullFilePath + map += "string" -> inputstremToString(resource.inputStream) + } + + def download(userName: String, resourceId: String, version: String): util.Map[String, Object] = { + val client: BmlClient = getBmlClient(userName) + var resource: BmlDownloadResponse = null + if (version == null) { + resource = client.downloadShareResource(userName, resourceId) + } else { + resource = client.downloadShareResource(userName, resourceId, version) + } + if (!resource.isSuccess) throw new DSSErrorException(911115, "下载失败") + val map = new util.HashMap[String, Object] + map += "path" -> resource.fullFilePath + map += "is" -> resource.inputStream + } + + def downloadToLocalPath(userName: String, resourceId: String, version: String, path: String): String = { + val result = download(userName, resourceId, version) + val is = result.get("is").asInstanceOf[InputStream] + val os = IoUtils.generateExportOutputStream(path) + Utils.tryFinally(IOUtils.copy(is, os)) { + IOUtils.closeQuietly(os) + IOUtils.closeQuietly(is) + } + path + } + + def downloadAndGetText(userName: String, resourceId: String, version: String, path: String): String = { + downloadToLocalPath(userName, resourceId, version, path) + //因为下载到指定目录后返回的resource的Stream为null,只能从文件重新读取。 + val is = IoUtils.generateInputInputStream(path) + Utils.tryFinally(inputstremToString(is))(IOUtils.closeQuietly(is)) + } + + def readLocalResourceFile(userName: String, readPath: String): InputStream = { + IoUtils.generateInputInputStream(readPath) + } + + def readLocalTextFile(userName: String, readPath: String): String = { + var inputStream: InputStream = null + var text: String = null + Utils.tryFinally { + inputStream = IoUtils.generateInputInputStream(readPath) + text = inputstremToString(inputStream) + } { + IOUtils.closeQuietly(inputStream) + } + text + } + + + def readTextFromBML(userName: String, resourceId: String, version: String): String = { + + val result = download(userName, resourceId, version) + val is = result.get("is").asInstanceOf[InputStream] + + var inputStream: InputStream = null + var text: String = null + Utils.tryFinally { + inputStream = is + text = inputstremToString(inputStream) + } { + IOUtils.closeQuietly(inputStream) + } + text + } + + + + private def inputstremToString(inputStream: InputStream): String = { + scala.io.Source.fromInputStream(inputStream).mkString + } + + + def createBmlProject(username: String, projectName: String, editUsers: util.List[String], + accessUsers: util.List[String]): Unit = { + val client = getBmlClient(username) + val response = client.createBmlProject(username, projectName, accessUsers, editUsers) + if (response.isSuccess) { + logger.info(s"for user $username create bml project $projectName success") + } else { + logger.error(s"for user $username create bml project $projectName failed") + } + } + + def attachResourceAndProject(username: String, projectName: String, resourceId: String): Unit = { + val client = getBmlClient(username) + val response = client.attachResourceAndProject(projectName, resourceId) + if (response.isSuccess) { + logger.info(s"attach $username and $projectName success") + } else { + logger.error(s"attach $username and $projectName failed") + } + } + + + def updateProjectPriv(projectName: String, username: String, editUsers: util.List[String], + accessUsers: util.List[String]): Unit = { + val client = getBmlClient(username) + val response = client.updateProjectPriv(username, projectName, editUsers, accessUsers) + if (response.isSuccess) { + logger.info(s"attach $username and $projectName success") + } else { + logger.error(s"attach $username and $projectName failed") + } + } + + def deleteBmlResource(username: String, resourceId: String): Unit = { + val client = getBmlClient(username) + val response = client.deleteResource(username, resourceId) + if (response.isSuccess) { + logger.info(s"delete $username bml resource file success, resourceId: $resourceId") + } else { + logger.error(s"delete $username bml resource file failed, resourceId: $resourceId") + } + } + +} diff --git a/dss-orchestrator/orchestrators/dss-workflow/pom.xml b/dss-orchestrator/orchestrators/dss-workflow/pom.xml index f15cbb3710..6301f4c154 100644 --- a/dss-orchestrator/orchestrators/dss-workflow/pom.xml +++ b/dss-orchestrator/orchestrators/dss-workflow/pom.xml @@ -21,7 +21,7 @@ dss com.webank.wedatasphere.dss - 1.1.0.20-SNAPSHOT + 1.10.0-SNAPSHOT ../../../pom.xml 4.0.0 diff --git a/dss-orchestrator/pom.xml b/dss-orchestrator/pom.xml index dc9f4c6fad..2437fcf978 100644 --- a/dss-orchestrator/pom.xml +++ b/dss-orchestrator/pom.xml @@ -21,7 +21,7 @@ dss com.webank.wedatasphere.dss - 1.1.0.20-SNAPSHOT + 1.10.0-SNAPSHOT ../pom.xml 4.0.0 diff --git a/dss-server/pom.xml b/dss-server/pom.xml index ed7e68cf77..4161526b95 100644 --- a/dss-server/pom.xml +++ b/dss-server/pom.xml @@ -21,7 +21,7 @@ dss com.webank.wedatasphere.dss - 1.1.0.20-SNAPSHOT + 1.10.0-SNAPSHOT ../pom.xml 4.0.0 @@ -49,6 +49,11 @@ + + com.webank.wedatasphere.dss + dss-git-server + ${dss.version} + com.webank.wedatasphere.dss dss-framework-project-server diff --git a/dss-standard/development-standard/development-process-standard-execution/pom.xml b/dss-standard/development-standard/development-process-standard-execution/pom.xml index 40bb6c0800..698b821896 100644 --- a/dss-standard/development-standard/development-process-standard-execution/pom.xml +++ b/dss-standard/development-standard/development-process-standard-execution/pom.xml @@ -21,7 +21,7 @@ dss com.webank.wedatasphere.dss - 1.1.0.20-SNAPSHOT + 1.10.0-SNAPSHOT ../../../pom.xml 4.0.0 diff --git a/dss-standard/development-standard/development-process-standard/pom.xml b/dss-standard/development-standard/development-process-standard/pom.xml index f9f761d5be..f06df8a531 100644 --- a/dss-standard/development-standard/development-process-standard/pom.xml +++ b/dss-standard/development-standard/development-process-standard/pom.xml @@ -21,7 +21,7 @@ dss com.webank.wedatasphere.dss - 1.1.0.20-SNAPSHOT + 1.10.0-SNAPSHOT ../../../pom.xml 4.0.0 diff --git a/dss-standard/dss-standard-common/pom.xml b/dss-standard/dss-standard-common/pom.xml index ee9a581afd..22c89e7981 100644 --- a/dss-standard/dss-standard-common/pom.xml +++ b/dss-standard/dss-standard-common/pom.xml @@ -21,7 +21,7 @@ dss com.webank.wedatasphere.dss - 1.1.0.20-SNAPSHOT + 1.10.0-SNAPSHOT ../../pom.xml 4.0.0 diff --git a/dss-standard/pom.xml b/dss-standard/pom.xml index a136fb2e92..82a472bb06 100644 --- a/dss-standard/pom.xml +++ b/dss-standard/pom.xml @@ -21,7 +21,7 @@ dss com.webank.wedatasphere.dss - 1.1.0.20-SNAPSHOT + 1.10.0-SNAPSHOT ../pom.xml 4.0.0 diff --git a/dss-standard/sso-standard/origin-sso-integration-standard/pom.xml b/dss-standard/sso-standard/origin-sso-integration-standard/pom.xml index 592b59d6ce..584c378e4f 100644 --- a/dss-standard/sso-standard/origin-sso-integration-standard/pom.xml +++ b/dss-standard/sso-standard/origin-sso-integration-standard/pom.xml @@ -21,7 +21,7 @@ dss com.webank.wedatasphere.dss - 1.1.0.20-SNAPSHOT + 1.10.0-SNAPSHOT ../../../pom.xml 4.0.0 diff --git a/dss-standard/sso-standard/spring-origin-sso-integration-plugin/pom.xml b/dss-standard/sso-standard/spring-origin-sso-integration-plugin/pom.xml index 5fdab09428..6b371fc29f 100644 --- a/dss-standard/sso-standard/spring-origin-sso-integration-plugin/pom.xml +++ b/dss-standard/sso-standard/spring-origin-sso-integration-plugin/pom.xml @@ -21,7 +21,7 @@ dss com.webank.wedatasphere.dss - 1.1.0.20-SNAPSHOT + 1.10.0-SNAPSHOT ../../../pom.xml 4.0.0 diff --git a/dss-standard/sso-standard/sso-integration-standard/pom.xml b/dss-standard/sso-standard/sso-integration-standard/pom.xml index 2ac6ce59e1..b5a61dff7e 100644 --- a/dss-standard/sso-standard/sso-integration-standard/pom.xml +++ b/dss-standard/sso-standard/sso-integration-standard/pom.xml @@ -21,7 +21,7 @@ dss com.webank.wedatasphere.dss - 1.1.0.20-SNAPSHOT + 1.10.0-SNAPSHOT ../../../pom.xml 4.0.0 diff --git a/dss-standard/sso-standard/sso-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/sso/utils/SSOHelper.java b/dss-standard/sso-standard/sso-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/sso/utils/SSOHelper.java index a26f615233..a61fa52f0a 100644 --- a/dss-standard/sso-standard/sso-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/sso/utils/SSOHelper.java +++ b/dss-standard/sso-standard/sso-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/sso/utils/SSOHelper.java @@ -37,8 +37,9 @@ public class SSOHelper { - private static final String WORKSPACE_ID_COOKIE_KEY = "workspaceId"; - private static final String WORKSPACE_NAME_COOKIE_KEY = "workspaceName"; + public static final String WORKSPACE_ID_COOKIE_KEY = "workspaceId"; + public static final String WORKSPACE_NAME_COOKIE_KEY = "workspaceName"; + public static final String USERNAME_NAME_COOKIE_KEY = "dss_user_name"; private static SSOBuilderService ssoBuilderService; private static final Logger LOGGER = LoggerFactory.getLogger(SSOHelper.class); @@ -191,4 +192,29 @@ public static String getCookieDomain(String host) { } } + public static void addUsernameCookie(HttpServletRequest request, HttpServletResponse response, String username) { + boolean isWorkspaceExists = Arrays.stream(request.getCookies()) + .anyMatch(cookie -> USERNAME_NAME_COOKIE_KEY.equals(cookie.getName()) && username.equals(cookie.getValue())); + if (isWorkspaceExists) { + LOGGER.warn("username {} already exists in DSS cookies, ignore to set it again.", username); + return ; + } + Cookie usernameCookie = new Cookie(USERNAME_NAME_COOKIE_KEY, username); + usernameCookie.setPath("/"); + response.addCookie(usernameCookie); + } + + public static void deleteCookieByName(HttpServletRequest request, HttpServletResponse response, String cookieKey) { + boolean isCookieKeyExists = Arrays.stream(request.getCookies()) + .anyMatch(cookie -> cookieKey.equals(cookie.getName())); + if (!isCookieKeyExists) { + LOGGER.warn("cookie {} is not exists in DSS cookies.", cookieKey); + return ; + } + Cookie cookie = new Cookie(cookieKey, null); + cookie.setMaxAge(0); // 设置为0，即立即删除 + cookie.setPath("/"); + response.addCookie(cookie); + } + } diff --git a/dss-standard/structure-standard/dss-project-plugin/pom.xml b/dss-standard/structure-standard/dss-project-plugin/pom.xml index 745e6b0132..397672fcbe 100644 --- a/dss-standard/structure-standard/dss-project-plugin/pom.xml +++ b/dss-standard/structure-standard/dss-project-plugin/pom.xml @@ -21,7 +21,7 @@ dss com.webank.wedatasphere.dss - 1.1.0.20-SNAPSHOT + 1.10.0-SNAPSHOT ../../../pom.xml 4.0.0 diff --git a/dss-standard/structure-standard/dss-role-plugin/pom.xml b/dss-standard/structure-standard/dss-role-plugin/pom.xml index e5a5626945..cd10cb07d2 100644 --- a/dss-standard/structure-standard/dss-role-plugin/pom.xml +++ b/dss-standard/structure-standard/dss-role-plugin/pom.xml @@ -21,7 +21,7 @@ dss com.webank.wedatasphere.dss - 1.1.0.20-SNAPSHOT + 1.10.0-SNAPSHOT ../../../pom.xml 4.0.0 diff --git a/dss-standard/structure-standard/dss-structure-integration-standard/pom.xml b/dss-standard/structure-standard/dss-structure-integration-standard/pom.xml index 00ce1d53a0..a01d37eac2 100644 --- a/dss-standard/structure-standard/dss-structure-integration-standard/pom.xml +++ b/dss-standard/structure-standard/dss-structure-integration-standard/pom.xml @@ -21,7 +21,7 @@ dss com.webank.wedatasphere.dss - 1.1.0.20-SNAPSHOT + 1.10.0-SNAPSHOT ../../../pom.xml 4.0.0 diff --git a/dss-standard/structure-standard/dss-structure-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/project/ref/DSSProjectContentRequestRef.java b/dss-standard/structure-standard/dss-structure-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/project/ref/DSSProjectContentRequestRef.java index 64e68a2bc0..1a9cefb37d 100644 --- a/dss-standard/structure-standard/dss-structure-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/project/ref/DSSProjectContentRequestRef.java +++ b/dss-standard/structure-standard/dss-structure-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/project/ref/DSSProjectContentRequestRef.java @@ -1,9 +1,14 @@ package com.webank.wedatasphere.dss.standard.app.structure.project.ref; +import com.google.gson.Gson; +import com.google.gson.reflect.TypeToken; import com.webank.wedatasphere.dss.common.entity.project.DSSProject; import com.webank.wedatasphere.dss.standard.app.structure.StructureRequestRef; import com.webank.wedatasphere.dss.standard.app.structure.StructureRequestRefImpl; +import java.util.Collections; +import java.util.List; + /** * @author enjoyyin * @date 2022-03-13 @@ -34,6 +39,27 @@ default R setDSSProjectPrivilege(DSSProjectPrivilege dssProjectPrivilege) { return (R) this; } + /** + * DSS 工程的全量最新权数据源列表，包含了 DSS 工程所有的数据源 + * @return DSSProjectPrivilege + */ + default List getDSSProjectDataSources() { + String json = (String) this.getParameter("dssProjectDataSources"); + if(json != null) { + return new Gson().fromJson(json, new TypeToken>(){}.getType()); + } + return Collections.emptyList(); + } + + + default R setDSSProjectDataSources(List dssProjectDataSources) { + //为了让第三方组件可以不用升级dss依赖包，转成json string再存储 + if(dssProjectDataSources != null) { + setParameter("dssProjectDataSources", new Gson().toJson(dssProjectDataSources)); + } + return (R) this; + } + class DSSProjectContentRequestRefImpl extends StructureRequestRefImpl implements DSSProjectContentRequestRef { diff --git a/dss-standard/structure-standard/dss-structure-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/project/ref/DSSProjectDataSource.java b/dss-standard/structure-standard/dss-structure-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/project/ref/DSSProjectDataSource.java new file mode 100644 index 0000000000..2d8556ace3 --- /dev/null +++ b/dss-standard/structure-standard/dss-structure-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/project/ref/DSSProjectDataSource.java @@ -0,0 +1,64 @@ +package com.webank.wedatasphere.dss.standard.app.structure.project.ref; + +import java.util.Date; + +/** + * Author: xlinliu + * Date: 2024/6/6 + */ +public class DSSProjectDataSource { + private String dataSourceName; + private String dataSourceDesc; + private Date createTime; + private Date modifyTime; + private String createUser; + private String dataSourceType; + + public String getDataSourceName() { + return dataSourceName; + } + + public void setDataSourceName(String dataSourceName) { + this.dataSourceName = dataSourceName; + } + + public String getDataSourceDesc() { + return dataSourceDesc; + } + + public void setDataSourceDesc(String dataSourceDesc) { + this.dataSourceDesc = dataSourceDesc; + } + + public Date getCreateTime() { + return createTime; + } + + public void setCreateTime(Date createTime) { + this.createTime = createTime; + } + + public Date getModifyTime() { + return modifyTime; + } + + public void setModifyTime(Date modifyTime) { + this.modifyTime = modifyTime; + } + + public String getCreateUser() { + return createUser; + } + + public void setCreateUser(String createUser) { + this.createUser = createUser; + } + + public String getDataSourceType() { + return dataSourceType; + } + + public void setDataSourceType(String dataSourceType) { + this.dataSourceType = dataSourceType; + } +} diff --git a/dss-standard/structure-standard/dss-structure-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/project/ref/ProjectUpdateRequestRef.java b/dss-standard/structure-standard/dss-structure-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/project/ref/ProjectUpdateRequestRef.java index 618b93edfc..87cdbb81be 100644 --- a/dss-standard/structure-standard/dss-structure-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/project/ref/ProjectUpdateRequestRef.java +++ b/dss-standard/structure-standard/dss-structure-integration-standard/src/main/java/com/webank/wedatasphere/dss/standard/app/structure/project/ref/ProjectUpdateRequestRef.java @@ -1,7 +1,12 @@ package com.webank.wedatasphere.dss.standard.app.structure.project.ref; +import com.google.gson.Gson; +import com.google.gson.reflect.TypeToken; import com.webank.wedatasphere.dss.standard.app.structure.StructureRequestRefImpl; +import java.util.Collections; +import java.util.List; + /** * @author enjoyyin * @date 2022-03-13 @@ -34,6 +39,20 @@ default R setAddedDSSProjectPrivilege(DSSProjectPrivilege addedDSSProjectPrivile return (R) this; } + + /** + * 新的项目owner，用于项目交接 + * @return + */ + default String getNewOwner() { + return (String) this.getParameter("newOwner"); + } + + default R setNewOwner(String newOwner) { + setParameter("newOwner", newOwner); + return (R) this; + } + /** * 只包含本次移除的 DSS 工程相关权限用户 * @return @@ -46,7 +65,26 @@ default R setRemovedDSSProjectPrivilege(DSSProjectPrivilege removedDSSProjectPri setParameter("removedDSSProjectPrivilege", removedDSSProjectPrivilege); return (R) this; } + /** + * DSS 工程的全量最新权数据源列表，包含了 DSS 工程所有的数据源 + * 第三方组件需要根据这个列表来判断哪些数据源是新增的，哪些数据源是删除的，时刻保持同步 + * @return DSSProjectPrivilege + */ + default List getDSSProjectDataSources() { + String json = (String) this.getParameter("dssProjectDataSources"); + if(json != null) { + return new Gson().fromJson(json, new TypeToken>(){}.getType()); + } + return Collections.emptyList(); + } + + default R setDSSProjectDataSources(List dssProjectDataSources) { + //为了让第三方组件可以不用升级dss依赖包，转成json string再存储 + String json = new Gson().toJson(dssProjectDataSources); + setParameter("dssProjectDataSources", json); + return (R) this; + } class ProjectUpdateRequestRefImpl extends StructureRequestRefImpl implements ProjectUpdateRequestRef {} diff --git a/dss-standard/structure-standard/spring-origin-dss-project-plugin/pom.xml b/dss-standard/structure-standard/spring-origin-dss-project-plugin/pom.xml index 645724e18f..fd89ccaea9 100644 --- a/dss-standard/structure-standard/spring-origin-dss-project-plugin/pom.xml +++ b/dss-standard/structure-standard/spring-origin-dss-project-plugin/pom.xml @@ -21,7 +21,7 @@ dss com.webank.wedatasphere.dss - 1.1.0.20-SNAPSHOT + 1.10.0-SNAPSHOT ../../../pom.xml 4.0.0 diff --git a/images/en_US/readme/communication.jpg b/images/en_US/readme/communication.jpg new file mode 100644 index 0000000000..55a0dde5d8 Binary files /dev/null and b/images/en_US/readme/communication.jpg differ diff --git a/images/zh_CN/readme/communication.jpg b/images/zh_CN/readme/communication.jpg new file mode 100644 index 0000000000..55a0dde5d8 Binary files /dev/null and b/images/zh_CN/readme/communication.jpg differ diff --git a/next-web/.commitlintrc b/next-web/.commitlintrc new file mode 100644 index 0000000000..cda374c83f --- /dev/null +++ b/next-web/.commitlintrc @@ -0,0 +1,49 @@ +{ + "extends": [ + "@commitlint/config-conventional" + ], + "rules": { + "type-enum": [ + 2, + "always", + [ + "build", + "chore", + "ci", + "docs", + "feat", + "fix", + "perf", + "refactor", + "revert", + "style", + "test" + ] + ], + "type-case": [ + 0 + ], + "type-empty": [ + 0 + ], + "scope-empty": [ + 0 + ], + "scope-case": [ + 0 + ], + "subject-full-stop": [ + 0, + "never" + ], + "subject-case": [ + 0, + "never" + ], + "header-max-length": [ + 0, + "always", + 72 + ] + } +} diff --git a/next-web/.env b/next-web/.env new file mode 100644 index 0000000000..903fa8e228 --- /dev/null +++ b/next-web/.env @@ -0,0 +1 @@ +VITE_REQUEST_PREFIX=/api/rest_j/v1/ \ No newline at end of file diff --git a/next-web/.env.development b/next-web/.env.development new file mode 100644 index 0000000000..903fa8e228 --- /dev/null +++ b/next-web/.env.development @@ -0,0 +1 @@ +VITE_REQUEST_PREFIX=/api/rest_j/v1/ \ No newline at end of file diff --git a/next-web/.eslintignore b/next-web/.eslintignore new file mode 100644 index 0000000000..76add878f8 --- /dev/null +++ b/next-web/.eslintignore @@ -0,0 +1,2 @@ +node_modules +dist \ No newline at end of file diff --git a/next-web/.eslintrc.json b/next-web/.eslintrc.json new file mode 100644 index 0000000000..68b1f7f950 --- /dev/null +++ b/next-web/.eslintrc.json @@ -0,0 +1,23 @@ +{ + "parser": "vue-eslint-parser", + "parserOptions": { + "ecmaVersion": "latest", + "sourceType": "module", + "jsx": true, + "parser": "@typescript-eslint/parser" + }, + + "plugins": ["@typescript-eslint", "prettier"], + + "extends": [ + "plugin:@typescript-eslint/recommended", + "plugin:vue/vue3-recommended", + "plugin:prettier/recommended" + ], + + "rules": { + "@typescript-eslint/ban-ts-comment": "warn", + "vue/multi-word-component-names": "off", + "no-unused-vars": "off" + } +} diff --git a/next-web/.gitignore b/next-web/.gitignore new file mode 100644 index 0000000000..8e1d0f2da7 --- /dev/null +++ b/next-web/.gitignore @@ -0,0 +1,23 @@ +# Logs +logs +*.log +npm-debug.log* +yarn-debug.log* +yarn-error.log* +pnpm-debug.log* +lerna-debug.log* +package-lock.json +node_modules +dist +dist-ssr +*.local + +# Editor directories and files +.vscode/* +.idea +.DS_Store +*.suo +*.ntvs* +*.njsproj +*.sln +*.sw? diff --git a/next-web/.jsconfig.json b/next-web/.jsconfig.json new file mode 100644 index 0000000000..eb1b272dc9 --- /dev/null +++ b/next-web/.jsconfig.json @@ -0,0 +1,8 @@ +{ + "compilerOptions": { + "baseUrl": ".", + "paths": { + "@dataspherestudio": ["./packages/"] + } + } +} diff --git a/next-web/.prettierrc b/next-web/.prettierrc new file mode 100644 index 0000000000..78075da4ef --- /dev/null +++ b/next-web/.prettierrc @@ -0,0 +1,12 @@ +{ + "trailingComma": "es5", + "tabWidth": 2, + "semi": true, + "singleQuote": true, + "overrides": [ + { + "files": [".prettierrc",".commitlintrc"], + "options": { "parser": "json" } + } + ] +} diff --git a/next-web/README.md b/next-web/README.md new file mode 100644 index 0000000000..bb01c4b1bb --- /dev/null +++ b/next-web/README.md @@ -0,0 +1,43 @@ +# DSS Template + +vue3、vite、fes-design，是否使用 TS + +新版 DSS 初始化项目，原 DSS 项目已有功能逐步迁移 + +## 本地开发及接口调试： + +``` +lerna bootstrap +wnpm run prepare +wnpm run dev +``` + +### 方式一 + +本地开发请求后端接口通过配置 vite proxy 来进行，cookie 可以从 dss 登录接口后 set-cookie 里取 + +### 方式二 + +浏览器插件 SwitchyOmega 配置规则，非接口的资源转发至 http://localhost:5173，开发时访问目标域名即可 +网址正则示例： +^http://sit.dss.bdap.weoa.com(?!/api)._ +^http://10.107.97.166:8088(?!/api)._ + +## 代码提交规范 + +请执行 wnpm run prepare 配置好提交前 eslint 检测及 commit 信息规范校验的 git hook + +## 项目结构 + +``` +├─dist # 构建后静态资源 +├─node_modules +├─exts # 扩展插件 +└─packages # 各应用模块 + ├─dss + ├─shared + └─workspace # 工作空间管理 + +``` + +### 其它 diff --git a/next-web/config/dss.js b/next-web/config/dss.js new file mode 100644 index 0000000000..d3f59a53ac --- /dev/null +++ b/next-web/config/dss.js @@ -0,0 +1,30 @@ +export default { + apps: { + workspace: { + routes: 'workspace/router', + module: 'workspace/module', + i18n: { + en: 'workspace/i18n/en.json', + 'zh-CN': 'workspace/i18n/zh.json', + }, + }, + }, + exts: {}, + conf: { + app_name: 'DataSphere Studio', + // app_logo: 'src/assets/images/dssLogo.png', + user_guide: '', + // hide_view_tb_detail: true, + // hide_view_db_detail: true, + watermark: { + show: false, + template: '${username} ${time}', + timeupdate: 60000, + }, + // lsp_service: { + // sql: "${protocol}//${host}/server", + // py: "${protocol}//${host}/python", + // }, + }, + version: '1.1.12', +}; diff --git a/next-web/config/workspace.js b/next-web/config/workspace.js new file mode 100644 index 0000000000..ff8b4c5632 --- /dev/null +++ b/next-web/config/workspace.js @@ -0,0 +1 @@ +export default {}; diff --git a/next-web/index.html b/next-web/index.html new file mode 100644 index 0000000000..e8974504c7 --- /dev/null +++ b/next-web/index.html @@ -0,0 +1,13 @@ + + + + + + + + + +
+ + + diff --git a/next-web/package.json b/next-web/package.json new file mode 100644 index 0000000000..01eea09712 --- /dev/null +++ b/next-web/package.json @@ -0,0 +1,78 @@ +{ + "name": "dataspherestudio", + "private": true, + "version": "0.0.0", + "type": "module", + "scripts": { + "dev": "vite", + "build": "vite build", + "preview": "vite preview", + "prepare": "cd .. && husky install next-web/.husky", + "lint": "eslint . --ext .vue,.js,.ts,.jsx,.tsx", + "lint:fix": "eslint . --ext .vue,.js,.ts,.jsx,.tsx --fix", + "lint-staged": "lint-staged", + "format": "prettier --write ./**/*.{vue,ts,tsx,js,jsx,css,less,scss,json,md}", + "commit": "git-cz", + "typecheck": "vue-tsc --noEmit" + }, + "dependencies": { + "@fesjs/fes-design": "^0.7.28", + "@fesjs/traction-widget": "1.9.1", + "@vueuse/core": "^10.2.0", + "dev-ip": "^1.0.1", + "lodash-es": "^4.17.21", + "vue": "^3.2.47", + "vue-i18n": "^9.3.0-beta.19", + "vue-router": "^4.2.2" + }, + "devDependencies": { + "@commitlint/config-conventional": "^15.0.0", + "@types/node": "^20.3.1", + "@typescript-eslint/eslint-plugin": "^5.59.11", + "@typescript-eslint/parser": "^5.59.11", + "@vitejs/plugin-vue": "^4.1.0", + "@vue/tsconfig": "^0.4.0", + "commitlint": "^15.0.0", + "cz-conventional-changelog": "^3.3.0", + "eslint": "^8.41.0", + "eslint-config-prettier": "^8.8.0", + "eslint-plugin-prettier": "^4.2.1", + "eslint-plugin-vue": "^9.14.1", + "husky": "^7.0.4", + "less": "^4.1.2", + "less-loader": "^10.2.0", + "lint-staged": "^12.1.2", + "prettier": "^2.8.8", + "typescript": "^5.1.3", + "vite": "^4.3.9", + "vite-plugin-eslint": "^1.8.1", + "vue-eslint-parser": "^9.3.0", + "vue-tsc": "^1.6.5" + }, + "config": { + "commitizen": { + "path": "./next-web/node_modules/cz-conventional-changelog" + } + }, + "lint-staged": { + "*.{js,jsx,vue,ts,tsx}": [ + "eslint --fix", + "prettier --write" + ], + "*.{scss,less,css,html,md,json}": [ + "prettier --write" + ] + }, + "husky": { + "hooks": { + "pre-commit": "npm run lint-staged", + "commit-msg": "commitlint -E $HUSKY_GIT_PARAMS" + } + }, + "workspaces": [ + "packages/cyeditor", + "packages/dss", + "packages/shared", + "packages/workspace" + ] +} diff --git a/next-web/packages/dss/package.json b/next-web/packages/dss/package.json new file mode 100644 index 0000000000..a1be21712e --- /dev/null +++ b/next-web/packages/dss/package.json @@ -0,0 +1,7 @@ +{ + "name": "@dataspherestudio/dss", + "version": "0.0.1", + "dependencies": {}, + "devDependencies": {}, + "scripts": {} +} diff --git a/next-web/packages/dss/router.js b/next-web/packages/dss/router.js new file mode 100644 index 0000000000..e69de29bb2 diff --git a/next-web/packages/shared/index.ts b/next-web/packages/shared/index.ts new file mode 100644 index 0000000000..23e56d7ec4 --- /dev/null +++ b/next-web/packages/shared/index.ts @@ -0,0 +1,3 @@ +import request from './requst'; + +export { request }; diff --git a/next-web/packages/shared/package.json b/next-web/packages/shared/package.json new file mode 100644 index 0000000000..49c6bd6fbb --- /dev/null +++ b/next-web/packages/shared/package.json @@ -0,0 +1,11 @@ +{ + "name": "@dataspherestudio/shared", + "version": "0.0.1", + "dependencies": { + "axios": "1.6.4" + }, + "main": "./index.js", + "module": "./index.js", + "devDependencies": {}, + "scripts": {} +} diff --git a/next-web/packages/shared/requst.ts b/next-web/packages/shared/requst.ts new file mode 100644 index 0000000000..949ad3eaad --- /dev/null +++ b/next-web/packages/shared/requst.ts @@ -0,0 +1,238 @@ +import type { AxiosInstance, AxiosRequestConfig, AxiosResponse } from 'axios'; +import axios from 'axios'; + +const pending: { u: string; c: AbortController }[] = []; +const removePending = (config: AxiosRequestConfig) => { + for (let p = 0; p < pending.length; p++) { + const key = getReqestKey(config); + // 如果存在则执行取消操作 + if (pending[p].u === key) { + pending[p].c.abort(); //执行取消操作 + pending.splice(p, 1); // 移除记录 + } + } +}; + +function getReqestKey(config: AxiosRequestConfig) { + const params = JSON.stringify(config.params); + return config.url + '&' + config.method + '&' + params; +} + +function cutReq(config: AxiosRequestConfig) { + for (let p = 0; p < pending.length; p++) { + if (pending[p].u === getReqestKey(config)) { + return true; + } + } +} + +const instance = axios.create({ + baseURL: + import.meta.env.VITE_REQUEST_PREFIX || + `${location.protocol}//${window.location.host}`, + timeout: 600000, + withCredentials: true, + headers: { + 'Content-Type': 'application/json;charset=UTF-8', + }, +}); + +// request拦截器 +instance.interceptors.request.use( + (config) => { + const flag = cutReq(config); + if (flag === true) { + // 取消第二次相同请求 + removePending(config); + return config; + } + const abortController = new AbortController(); + const key = getReqestKey(config); + pending.push({ u: key, c: abortController }); + config.signal = abortController.signal; + return config; + }, + (err) => { + return Promise.reject(err); + } +); +// response拦截器 +instance.interceptors.response.use( + (response) => { + removePending(response.config); + return response; + }, + (error) => { + if (axios.isCancel(error)) { + return {}; + } + return Promise.reject(error); + } +); + +// 参数处理 +const param = function (url: string, data?: object, option?: FetchOptions) { + if (typeof option === 'string') { + option = { + method: option, + }; + } + if (option && option.method) { + if ( + option.method == 'get' || + option.method == 'delete' || + option.method == 'head' || + option.method == 'options' + ) { + option.params = data; + } + if ( + option.method == 'post' || + option.method == 'put' || + option.method == 'patch' + ) { + option.data = data; + } + } else { + option = { + method: 'get', + params: data, + }; + } + + option.url = url; + return instance.request(option); +}; + +// 请求返回数据处理 +const success = function (response: AxiosResponse) { + let data; + if (response) { + if (response.status === 401 && api.error[response.status]) { + api.error[response.status](response); + } + if (typeof response.data === 'string' && response.data) { + try { + data = JSON.parse(response.data); + } catch (err) { + throw new Error('接口返回数据格式错误'); + } + } else if (response.data) { + data = response.data; + } + } else { + throw new Error('接口返回数据格式错误'); + } + if (data && data.status !== 0) { + if (api.error[data.status]) { + api.error[data.status](response); + throw new Error(''); + } else { + throw new Error(data.message || ''); + } + } + return data || {}; +}; + +// 错误处理 +const fail = function (error: { response: any; code?: any }) { + const response = error.response; + if (error?.code == 'ERR_CANCELED') return; + if (response && api.error[response.status]) { + api.error[response.status](response); + } else { + if (response && response.data) { + // + } + } + throw error; +}; + +/** + * 请求方法 + * @param {*} url 接口路径 + * @param {*} data 请求参数 + * @param {*} option 配置 + * @returns promise + * usages： + * fetch('https://dss-open.wedatasphere.com/api/rest_j/v1/user/publicKey') + * fetch('https://dss-open.wedatasphere.com/api/rest_j/v1/user/publicKey', {}, 'get') + * fetch('/user/XX', {}, 'post') + * fetch('/user/XX', {}, {axios config options}) + */ +function fetch(url: string, data?: object, option?: FetchOptions) { + return param(url, data, option) + .then(success, fail) + .then(function (response) { + return response; + }) + .catch(function (error) { + api.showErrorMsg(error); + throw error; + }); +} + +type FetchOptions = string | AxiosRequestConfig; + +/** + * 定义接口返回的固定格式 + */ +export interface ResponseResult { + status: number; + method: string; + message: string; + data: T; +} + +export type ApiRequest = { + instance: AxiosInstance; + setError: (option: any) => void; + fetch: ( + url: string, + data?: any, + option?: FetchOptions + ) => Promise>; + error: Record void>; + showErrorMsg: (error: Error) => void; +}; + +const api: ApiRequest = { + instance: instance, + error: { + '-1': function (res) { + if (res.data && res.data.enableSSO && res.data.SSOURL) { + return window.location.replace(res.data.SSOURL); + } + if (window.location.href.indexOf('/#/login') < 0) { + if (self != top) { + window.parent.location.href = '/#/login'; + } else { + window.location.href = '/#/login'; + } + } + }, + '401': function () { + if (window.location.href.indexOf('/#/login') < 0) { + if (self != top) { + window.parent.location.href = '/#/login'; + } else { + window.location.href = '/#/login'; + } + } + }, + }, + setError: function ({ error, showErrorMsg }) { + if (error && typeof error === 'object') { + Object.assign(api.error, error); + } + if (showErrorMsg) { + api.showErrorMsg = showErrorMsg; + } + }, + showErrorMsg: function (err) { + // empty + }, + fetch, +}; + +export default api; diff --git a/next-web/packages/workspace/i18n/en.json b/next-web/packages/workspace/i18n/en.json new file mode 100644 index 0000000000..e4abbac3b4 --- /dev/null +++ b/next-web/packages/workspace/i18n/en.json @@ -0,0 +1,65 @@ +{ + "common": { + "search": "Search", + "reset": "Reset", + "moreAction": "More Action", + "more": "More", + "createUser": "Create User", + "createTime": "Create Time", + "updateUser": "Update User", + "updateTime": "Update Time", + "action": "Action", + "edit": "Edit", + "delete": "Delete", + "pleaseInput": "Please Input", + "pleaseSelect": "Please Select", + "pleaseFillIn": "Please FillIn", + "userName": "UserName", + "save": "Save" + }, + "workspace": { + "tmlName": "Template Name", + "engineType": "Engine Type", + "relateUser": "Relate User", + "createTml": "Create Template", + "tmlApplyRule": "Template Apply Rule", + "visibility": "Visibility", + "visibleUserCount": "Visible User Count", + "visibleUser": "Visible User", + "allUser": "All User", + "specifyUser": "Specify User", + "resParams": "Resource Params", + "tmlDes": "Template Description", + "workFlowRefList": "WorkFlow Reference List", + "appRefList": "APP Reference List", + "overwriteUser": "Overwrite User", + "appName": "App Name", + "overwriteScope": "Overwrite Scope", + "appRefTable": "App Ref Table", + "addTemplate": "Add Template", + "editTemplate": "Edit Template", + "projectName": "Project Name", + "workFlowName": "WorkFlow Name" + }, + "announcement": { + "addAnnouncement": "Add announcement", + "expired": "Expired", + "inEffect": " In effect", + "notActivated": "Not activated", + "delete": "Delete", + "announcementMessageManage": "Announcement message management", + "announcementId": "Announcement ID", + "announcementContent": "Announcement content", + "announcementStartTime": "Announcement start time", + "announcementEndTime": "Announcement end time", + "announcementStatus": "Announcement status", + "creator": "Create user", + "createTime": "Creation time", + "operation": "Operation", + "pleaseEnter": "Please enter", + "pleaseSelect": "Please select", + "startTime": "Start time", + "endTime": "End time", + "addAnnouncementMessage": "The announcement was added successfully!" + } +} diff --git a/next-web/packages/workspace/i18n/zh.json b/next-web/packages/workspace/i18n/zh.json new file mode 100644 index 0000000000..7da431a48d --- /dev/null +++ b/next-web/packages/workspace/i18n/zh.json @@ -0,0 +1,66 @@ +{ + "common": { + "search": "查询", + "reset": "重置", + "moreAction": "更多操作", + "more": "更多", + "createUser": "创建人", + "createTime": "创建时间", + "updateUser": "更新人", + "updateTime": "更新时间", + "action": "操作", + "edit": "编辑", + "delete": "删除", + "pleaseInput": "请输入", + "pleaseSelect": "请选择", + "pleaseFillIn": "请填写", + "userName": "用户名", + "save": "保存", + "notEmpty": "不能为空" + }, + "workSpace": { + "tmlName": "模板名称", + "engineType": "引擎类型", + "relateUser": "关联用户", + "createTml": "创建模板", + "tmlApplyRule": "模板应用规则", + "visibility": "可见范围", + "visibleUserCount": "可见用户数", + "visibleUser": "可见用户", + "allUser": "全部用户", + "specifyUser": "指定用户", + "resParams": "资源参数", + "tmlDes": "模板描述", + "workFlowRefList": "工作流引用列表", + "appRefList": "应用引用列表", + "overwriteUser": "覆盖用户", + "appName": "应用名称", + "overwriteScope": "覆盖范围", + "appRefTable": "应用引用列表", + "addTemplate": "新建模板", + "editTemplate": "编辑模板", + "projectName": "项目名称", + "workFlowName": "工作流名称" + }, + "announcement": { + "addAnnouncement": "新增公告", + "expired": "已过期", + "inEffect": "生效中", + "notActivated": "未生效", + "delete": "删除", + "announcementMessageManage": "公告消息管理", + "announcementId": "公告ID", + "announcementContent": "公告内容", + "announcementStartTime": "公告起始时间", + "announcementEndTime": "公告结束时间", + "announcementStatus": "公告状态", + "creator": "创建人", + "createTime": "创建时间", + "operation": "操作", + "pleaseEnter": "请输入", + "pleaseSelect": "请选择", + "startTime": "起始时间", + "endTime": "结束时间", + "addAnnouncementMessage": "新增公告成功！" + } +} diff --git a/next-web/packages/workspace/module/announcement/api.ts b/next-web/packages/workspace/module/announcement/api.ts new file mode 100644 index 0000000000..034cad0c29 --- /dev/null +++ b/next-web/packages/workspace/module/announcement/api.ts @@ -0,0 +1,10 @@ +export default { + getNoticeList: 'dss/framework/workspace/notice/list', // 获取公告接口 + deleteNotice: 'dss/framework/workspace/notice/delete', // 删除公告 + createNotice: 'dss/framework/workspace/notice/create', //新增公告 + scripts: { + getNoticeList: 'dss/scriptis/notice/list', // 获取公告接口 + deleteNotice: 'dss/scriptis/notice/delete', // 删除公告 + createNotice: 'dss/scriptis/notice/create', //新增公告 + }, +}; diff --git a/next-web/packages/workspace/module/announcement/editAnnouncement.vue b/next-web/packages/workspace/module/announcement/editAnnouncement.vue new file mode 100644 index 0000000000..30504ca867 --- /dev/null +++ b/next-web/packages/workspace/module/announcement/editAnnouncement.vue @@ -0,0 +1,129 @@ + + diff --git a/next-web/packages/workspace/module/announcement/index.vue b/next-web/packages/workspace/module/announcement/index.vue new file mode 100644 index 0000000000..59ac85847f --- /dev/null +++ b/next-web/packages/workspace/module/announcement/index.vue @@ -0,0 +1,318 @@ + + + diff --git a/next-web/packages/workspace/module/components/NavHeader.vue b/next-web/packages/workspace/module/components/NavHeader.vue new file mode 100644 index 0000000000..30f10cf8be --- /dev/null +++ b/next-web/packages/workspace/module/components/NavHeader.vue @@ -0,0 +1,89 @@ + + + diff --git a/next-web/packages/workspace/module/hooks/usePagination.ts b/next-web/packages/workspace/module/hooks/usePagination.ts new file mode 100644 index 0000000000..abfd46705a --- /dev/null +++ b/next-web/packages/workspace/module/hooks/usePagination.ts @@ -0,0 +1,98 @@ +import { ref, computed, type Ref, type ComputedRef, onMounted } from 'vue'; +import { useDateFormat } from '@vueuse/core'; + +export type PaginationAndParams = { + pageNow?: number; + pageSize?: number; + totalCount?: number; + totalPage?: number; + [index: string]: any; +}; + +export const usePagination = ( + param?: Ref, + cb?: (param: PaginationAndParams) => Promise +) => { + // 分页信息初始化 + const init = () => ({ + pageNow: 1, + pageSize: 10, + totalCount: 0, + totalPage: 0, + }); + const pagination = ref(init()); + const isLoading = ref(false); // 加载中 + + // 查询参数 + const parameter: ComputedRef = computed(() => ({ + ...param?.value, + pageNow: pagination.value.pageNow, + pageSize: pagination.value.pageSize, + })); + + // 设置分页 + const setPagination = (params: { [index: string]: unknown } = {}): void => { + Object.keys(params).forEach((key: string) => { + pagination.value[key] = params[key]; + }); + }; + + // 回调函数 + const callback = async () => { + try { + if (cb) { + isLoading.value = true; + const pagedata = await cb(parameter.value); + isLoading.value = false; + setPagination(pagedata); + } + } catch (error) { + isLoading.value = false; + console.log(error); + } + }; + + // 查询表单 + const handleInit = async () => { + pagination.value.pageNow = 1; + await callback(); + }; + + // 查询表单 + const handleCurrent = async () => { + await callback(); + }; + + // 分页事件 + const handleCurrentChange = async (currentPage: number, pageSize: number) => { + pagination.value.pageSize = pageSize; + pagination.value.pageNow = currentPage; + await callback(); + }; + + onMounted(() => { + pagination.value.pageSize = param?.value?.pageSize || 10; + pagination.value.pageNow = param?.value?.pageNow || 1; + }); + + // 表格内容filter + const fillText = (row: { [index: string]: unknown }) => + ['null', 'undefined', ''].includes(String(row.cellValue)) + ? '- -' + : row.cellValue; + + // 表格内容filter + const fillTimeText = (row: { [index: string]: unknown }) => + ['null', 'undefined', ''].includes(String(row.cellValue)) + ? '- -' + : useDateFormat(row.cellValue as string, 'YYYY-MM-DD HH:mm:ss').value; + return { + isLoading, + pagination, + handleInit, + handleCurrent, + handleCurrentChange, + fillText, + fillTimeText, + }; +}; diff --git a/next-web/packages/workspace/module/hooks/useTableConfig.ts b/next-web/packages/workspace/module/hooks/useTableConfig.ts new file mode 100644 index 0000000000..fcd1421b40 --- /dev/null +++ b/next-web/packages/workspace/module/hooks/useTableConfig.ts @@ -0,0 +1,21 @@ +import { ref, computed, type Ref } from 'vue'; +type ObjectType = Record; +export const useTableConfig = (origins: Ref) => { + // 用于接收当前需要展示的表头的列 + const headers = ref([]); + const originHeaders = computed(() => + (origins.value as ObjectType[]).filter( + (v) => !['action', 'selection'].includes(v.type as string) + ) + ); + const showTableHeaderConfig = ref(false); + // 判断表头是否展示 + const checkTColShow = (col: string) => + headers.value.map((item: ObjectType) => item.prop).includes(col); + return { + headers, + originHeaders, + showTableHeaderConfig, + checkTColShow, + }; +}; diff --git a/next-web/packages/workspace/module/resourceTmlManage/api.ts b/next-web/packages/workspace/module/resourceTmlManage/api.ts new file mode 100644 index 0000000000..f0667f19eb --- /dev/null +++ b/next-web/packages/workspace/module/resourceTmlManage/api.ts @@ -0,0 +1,267 @@ +import { request } from '@dataspherestudio/shared'; + +// 工作流引用列表获取工作流名称下拉框 +export function fetchworkFlowNames(templateId: string) { + return request + .fetch( + 'dss/framework/orchestrator/getTemplateflowNames?templateId=' + templateId + ) + .then((res: any) => { + if (!res) { + return {}; + } + return { + orchestratorNames: res.data.orchestratorNames, + }; + }) + .catch((error: any) => { + // + }); +} +// 工作流引用列表获取项目名称下拉框 +export function fetchworkFlowProjectNames(templateId: string) { + return request + .fetch( + 'dss/framework/orchestrator/getTemplateProjectNames?templateId=' + + templateId + ) + .then((res: any) => { + if (!res) { + return {}; + } + return { + projectNames: res.data.projectNames, + }; + }) + .catch((error: any) => { + // + }); +} + +// 指定模板资源参数 +export function fetchResParams(paramString: string): Promise<{ conf: any }> { + const url = + 'dss/framework/workspace/engineconf/getConfTemplateParamDetail?' + + paramString; + return new Promise((resolve, reject) => { + request + .fetch(url) + .then((res: any) => { + if (!res) { + resolve({ conf: {} }); + return; + } + resolve({ + conf: res.data.conf, + }); + }) + .catch((error: any) => { + reject(error); + }); + }); +} +// 模板列表数据 +export function fetchTemplateTableData(paramsString = {}) { + const url = + 'dss/framework/workspace/engineconf/getConfTemplateList?' + paramsString; + return request + .fetch(url) + .then((res: any) => { + if (!res) { + return { data: {} }; + } + return res; + }) + .catch((error: any) => { + // + }); +} + +// 引擎类型下拉框数据 +export function fetchEngineTypes() { + return request + .fetch('dss/framework/workspace/engineconf/getEngineTypeList') + .then((res: any) => { + if (!res) { + return { engineTypes: {} }; + } + return { + engineTypes: res.data.engineTypes, + }; + }) + .catch((error: any) => { + // + }); +} + +// 获取所有部门科室 +export function fetchDepartments() { + const url = 'dss/framework/workspace/getAllDepartments'; + return request + .fetch(url) + .then((res: any) => { + if (!res) { + return { departments: [] }; + } + return { + departments: res.data.departments, + }; + }) + .catch((error: any) => { + // + }); +} + +export function fetchRelateUsersDepts(workspaceId: string) { + const url = + 'dss/framework/workspace/getAllWorkspaceUsersWithDepartment?workspaceId=' + + workspaceId; + return request + .fetch(url) + .then((res: any) => { + if (!res) { + return { users: {} }; + } + return { + users: res.data.users, + }; + }) + .catch((error: any) => { + // + }); +} + +// 指定工作空间下所有用户下拉框数据 +export function fetchRelateUsers(workspaceId: string) { + const url = + 'dss/framework/workspace/getAllWorkspaceUsers?workspaceId=' + workspaceId; + return request + .fetch(url) + .then((res: any) => { + if (!res) { + return { users: {} }; + } + return { + users: res.data.users, + }; + }) + .catch((error: any) => { + // + }); +} + +// 指定模板下可见用户列表数据 表格和下拉框均用该接口 +export function fetchTemplateUser(paramsString: string) { + const url = + 'dss/framework/workspace/engineconf/getConfTemplateUserList?' + + paramsString; + return request.fetch(url); +} +// 查询工作流引用列表数据 +export function fetchWorkFlows(paramsString: string): Promise<{ data: any }> { + const url = + 'dss/framework/orchestrator/getTemplateWorkflowPageInfo?' + paramsString; + return new Promise((resolve, reject) => { + request + .fetch(url) + .then((res: any) => { + if (!res) { + resolve({ data: {} }); + return; + } + resolve({ + data: res.data, + }); + }) + .catch((error: any) => { + reject(error); + }); + }); +} +// 查询应用引用列表数据 +export function fetchApps(paramsString: string): Promise<{ data: any }> { + const url = + 'dss/framework/workspace/engineconf/getConfTemplateApplyInfo?' + + paramsString; + return new Promise((resolve, reject) => { + request + .fetch(url) + .then((res: any) => { + if (!res) { + resolve({ data: {} }); + return; + } + resolve({ + data: res.data, + }); + }) + .catch((error: any) => { + reject(error); + }); + }); +} + +// 删除模板 +export function deleteTemplate(templateId: string) { + const url = + 'dss/framework/workspace/engineconf/deleteConfTemplate/' + templateId; + return new Promise((resolve, reject) => { + request + .fetch(url, {}, { method: 'post' }) + .then((res: any) => { + if (!res) { + resolve({ data: {} }); + return; + } + resolve({ + data: res.data, + }); + }) + .catch((error: any) => { + reject(error); + }); + }); +} + +// 模板名称唯一性校验 +export function fetchUniqueName(name: string) { + const url = + 'dss/framework/workspace/engineconf/checkConfTemplateName?name=' + name; + return new Promise((resolve, reject) => { + request + .fetch(url) + .then((res: any) => { + if (!res) { + resolve({ data: {} }); + return; + } + resolve({ + data: res.data, + }); + }) + .catch((error: any) => { + reject(error); + }); + }); +} + +// 创建或编辑模板 +export function fetchSaveTemplate(params: object) { + const url = 'dss/framework/workspace/engineconf/saveConfTemplate'; + return new Promise((resolve, reject) => { + request + .fetch(url, JSON.stringify(params), { method: 'put' }) + .then((res: any) => { + if (!res) { + resolve({ data: {} }); + return; + } + resolve({ + data: res.data, + }); + }) + .catch((error: any) => { + reject(error); + }); + }); +} diff --git a/next-web/packages/workspace/module/resourceTmlManage/applicationRule/api.ts b/next-web/packages/workspace/module/resourceTmlManage/applicationRule/api.ts new file mode 100644 index 0000000000..6bfd11724e --- /dev/null +++ b/next-web/packages/workspace/module/resourceTmlManage/applicationRule/api.ts @@ -0,0 +1,18 @@ +export default { + // 公共接口 + getApplicationList: 'dss/framework/workspace/engineconf/getApplicationList', // 加载关联应用 + getEngineTypeList: 'dss/framework/workspace/engineconf/getEngineTypeList', // 加载引擎类型 + getEngineNameList: 'dss/framework/workspace/engineconf/getEngineNameList', // 加载引擎名 + getConfTemplateList: 'dss/framework/workspace/engineconf/getConfTemplateList', // 加载模板名称 + getConfTemplateUserList: + 'dss/framework/workspace/engineconf/getConfTemplateApplyRuleUserList', // 加载覆盖用户 + // 获取应用规则接口 + getConfTemplateApplyRuleList: + 'dss/framework/workspace/engineconf/getConfTemplateApplyRuleList', // 查询模板应用规则 + saveConfTemplateApplyRule: + 'dss/framework/workspace/engineconf/saveConfTemplateApplyRule', // 新增模板应用规则 + deleteConfTemplateApplyRule: + 'dss/framework/workspace/engineconf/deleteConfTemplateApplyRule', // 删除模板应用规则 + getConfTemplateApplyHistory: + 'dss/framework/workspace/engineconf/getConfTemplateApplyHistory', // 查询模板应用执行记录 +}; diff --git a/next-web/packages/workspace/module/resourceTmlManage/applicationRule/editRule.vue b/next-web/packages/workspace/module/resourceTmlManage/applicationRule/editRule.vue new file mode 100644 index 0000000000..0909e52a38 --- /dev/null +++ b/next-web/packages/workspace/module/resourceTmlManage/applicationRule/editRule.vue @@ -0,0 +1,382 @@ + + + diff --git a/next-web/packages/workspace/module/resourceTmlManage/applicationRule/executionRecord.vue b/next-web/packages/workspace/module/resourceTmlManage/applicationRule/executionRecord.vue new file mode 100644 index 0000000000..baa75e3256 --- /dev/null +++ b/next-web/packages/workspace/module/resourceTmlManage/applicationRule/executionRecord.vue @@ -0,0 +1,230 @@ + + + diff --git a/next-web/packages/workspace/module/resourceTmlManage/applicationRule/hooks/useDataList.ts b/next-web/packages/workspace/module/resourceTmlManage/applicationRule/hooks/useDataList.ts new file mode 100644 index 0000000000..fda07fa535 --- /dev/null +++ b/next-web/packages/workspace/module/resourceTmlManage/applicationRule/hooks/useDataList.ts @@ -0,0 +1,130 @@ +import { ref, type ComputedRef } from 'vue'; +import api from '../api'; +import { useDateFormat } from '@vueuse/core'; +import { request } from '@dataspherestudio/shared'; +import { FMessage } from '@fesjs/fes-design'; +const maxPageSize = 10000; + +type ObjectType = Record; + +export const useDataList = (workspaceId?: ComputedRef) => { + const bindApplications = ref([]); // 关联应用 + const engineTypes = ref([]); // 引擎类型 + const engineNames = ref([]); // 引擎名 + const templateNames = ref([]); // 模板名称 + const ruleTypes = ref([ + { value: '1', label: '工作空间新用户规则' }, + { value: '0', label: '临时规则' }, + ]); // 规则类型 + const overlayAreas = ref([ + { value: '0', label: '全部用户' }, + { value: '1', label: '指定用户' }, + { value: '2', label: '工作空间新用户' }, + ]); // 覆盖范围 + + // 加载关联应用 + async function loadBindApplications() { + const res = await request.fetch(api.getApplicationList, {}); + bindApplications.value = (res.data?.applications || []).map( + (item: string) => ({ + valueField: item, + labelField: item, + }) + ); + } + + // 加载引擎类型 + async function loadEngineTypes(application?: string) { + const res = await request.fetch(api.getEngineTypeList, { application }); + engineTypes.value = (res.data?.engineTypes || []).map((item: string) => ({ + valueField: item, + labelField: item, + })); + } + + // 加载引擎名 + async function loadEngineNames(application?: string) { + if (!application) { + FMessage.warn('请先选择关联应用'); + return; + } + const res = await request.fetch(api.getEngineNameList, { application }); + engineNames.value = (res.data?.engineTypes || []).map((item: string) => ({ + valueField: item, + labelField: item, + })); + } + + // 加载模板名称 + async function loadTemplateNames(engineName?: string) { + if (!engineName) { + FMessage.warn('请先选择引擎类型'); + return; + } + const param = { + workspaceId: workspaceId?.value, + engineName, + pageNow: 1, + pageSize: maxPageSize, + }; + const res = await request.fetch(api.getConfTemplateList, param); + templateNames.value = (res.data?.templateList || []).map( + (item: ObjectType) => ({ + valueField: item.templateId, + labelField: item.name, + }) + ); + } + + // 函数映射关系 + const loadMap = { + bind_application: loadBindApplications, + engine_type: loadEngineTypes, + engine_name: loadEngineNames, + template_name: loadTemplateNames, + }; + + function handleSelect(type: keyof typeof loadMap, data?: string): void { + const handler = loadMap[type]; + handler(data); + } + + // 过滤出有值的对象 + function jsonFilter(obj: ObjectType): ObjectType { + if (obj.toString().toLowerCase() !== '[object object]') { + return {}; + } + const result: ObjectType = {}; + Object.keys(obj).forEach((key: string) => { + if (obj[key]) { + result[key as keyof typeof obj] = obj[key]; + } + }); + return result; + } + + // 表格内容filter + const fillText = (row: ObjectType) => + ['null', 'undefined', ''].includes(String(row.cellValue)) + ? '- -' + : row.cellValue; + + // 表格内容filter + const fillTimeText = (row: ObjectType) => + ['null', 'undefined', ''].includes(String(row.cellValue)) + ? '- -' + : useDateFormat(row.cellValue as string, 'YYYY-MM-DD HH:mm:ss').value; + + return { + bindApplications, // 关联应用 + engineTypes, // 引擎类型 + engineNames, // 引擎名 + templateNames, // 模板名称 + ruleTypes, // 规则类型 + overlayAreas, // 覆盖范围 + handleSelect, + jsonFilter, + fillText, + fillTimeText, + }; +}; diff --git a/next-web/packages/workspace/module/resourceTmlManage/applicationRule/index.vue b/next-web/packages/workspace/module/resourceTmlManage/applicationRule/index.vue new file mode 100644 index 0000000000..3f36fc3d55 --- /dev/null +++ b/next-web/packages/workspace/module/resourceTmlManage/applicationRule/index.vue @@ -0,0 +1,496 @@ + + diff --git a/next-web/packages/workspace/module/resourceTmlManage/applicationRule/viewUsers.vue b/next-web/packages/workspace/module/resourceTmlManage/applicationRule/viewUsers.vue new file mode 100644 index 0000000000..46362c56fd --- /dev/null +++ b/next-web/packages/workspace/module/resourceTmlManage/applicationRule/viewUsers.vue @@ -0,0 +1,140 @@ + + + diff --git a/next-web/packages/workspace/module/resourceTmlManage/component/appDrawer.vue b/next-web/packages/workspace/module/resourceTmlManage/component/appDrawer.vue new file mode 100644 index 0000000000..a796406388 --- /dev/null +++ b/next-web/packages/workspace/module/resourceTmlManage/component/appDrawer.vue @@ -0,0 +1,157 @@ + + diff --git a/next-web/packages/workspace/module/resourceTmlManage/component/permissionUserDrawer.vue b/next-web/packages/workspace/module/resourceTmlManage/component/permissionUserDrawer.vue new file mode 100644 index 0000000000..80725620b4 --- /dev/null +++ b/next-web/packages/workspace/module/resourceTmlManage/component/permissionUserDrawer.vue @@ -0,0 +1,137 @@ + + diff --git a/next-web/packages/workspace/module/resourceTmlManage/component/templateOperateDrawer.vue b/next-web/packages/workspace/module/resourceTmlManage/component/templateOperateDrawer.vue new file mode 100644 index 0000000000..9ad9d0f220 --- /dev/null +++ b/next-web/packages/workspace/module/resourceTmlManage/component/templateOperateDrawer.vue @@ -0,0 +1,582 @@ + + + diff --git a/next-web/packages/workspace/module/resourceTmlManage/component/workFlowDrawer.vue b/next-web/packages/workspace/module/resourceTmlManage/component/workFlowDrawer.vue new file mode 100644 index 0000000000..0e162212b3 --- /dev/null +++ b/next-web/packages/workspace/module/resourceTmlManage/component/workFlowDrawer.vue @@ -0,0 +1,186 @@ + + diff --git a/next-web/packages/workspace/module/resourceTmlManage/hooks/useDataList.ts b/next-web/packages/workspace/module/resourceTmlManage/hooks/useDataList.ts new file mode 100644 index 0000000000..f1dc14c094 --- /dev/null +++ b/next-web/packages/workspace/module/resourceTmlManage/hooks/useDataList.ts @@ -0,0 +1,158 @@ +import { ref } from 'vue'; +import { + fetchTemplateUser, + fetchRelateUsers, + fetchRelateUsersDepts, + fetchEngineTypes, + fetchworkFlowProjectNames, + fetchworkFlowNames, + fetchDepartments, +} from '../api'; +import { utils } from '../hooks/utils'; +import { useDateFormat } from '@vueuse/core'; + +export const useDataList = () => { + const templateUserList = ref([]); + const allWorkSpaceUserList = ref([]); + const allWorkSpaceUserDeptsList = ref([]); + const allDeptList = ref([]); + const engineList = ref([]); + const projectNameList = ref([]); + const workflowNameList = ref([]); + const permissionTypeList = ref([ + { + value: 0, + label: '全部用户', + }, + { + value: 1, + label: '指定用户', + }, + ]); + const { paramsObjectToString } = utils(); + + const loadworkFlowNames = async (templateId: string) => { + try { + const data = await fetchworkFlowNames(templateId); + workflowNameList.value = data?.orchestratorNames.map((v: string) => ({ + value: v, + label: v, + })); + } catch (error) { + console.log(error); + } + }; + + const loadworkFlowProjectNames = async (templateId: string) => { + try { + const data = await fetchworkFlowProjectNames(templateId); + projectNameList.value = data?.projectNames.map((v: string) => ({ + value: v, + label: v, + })); + } catch (error) { + console.log(error); + } + }; + + const loadEngineList = async () => { + try { + const data = await fetchEngineTypes(); + engineList.value = data?.engineTypes.map((v: string) => ({ + value: v, + label: v, + })); + } catch (error) { + console.log(error); + } + }; + + const loadAllWorkSpaceUserList = async (workspaceId: string) => { + try { + const data = await fetchRelateUsers(workspaceId); + allWorkSpaceUserList.value = data?.users.accessUsers.map((v: string) => ({ + value: v, + label: v, + })); + } catch (error) { + console.log(error); + } + }; + + const loadAllWorkSpaceUserDeptsList = async (workspaceId: string) => { + try { + const data = await fetchRelateUsersDepts(workspaceId); + if (data && data.users && data.users.accessUsers) { + allWorkSpaceUserDeptsList.value = data.users.accessUsers.map( + (v: any) => v + ); + } else { + allWorkSpaceUserDeptsList.value = []; + } + } catch (error) { + console.log(error); + } + }; + + const loadAllDeptList = async () => { + try { + const data = await fetchDepartments(); + allDeptList.value = data?.departments.map((v: string) => ({ + value: v, + label: v, + })); + } catch (error) { + console.log(error); + } + }; + + const loadTemplateUserList = async (templateId: string, name: string) => { + const params = { + pageNow: 1, + pageSize: 50, + templateId: templateId, + username: name, + }; + const strParams = paramsObjectToString(params); + try { + const res = await fetchTemplateUser(strParams); + templateUserList.value = res.data.users.map((v: { name: string }) => ({ + value: v.name, + label: v.name, + })); + } catch (error) { + console.log(error); + } + }; + // 表格内容filter + const fillText = (row: ObjectType) => + ['null', 'undefined', ''].includes(String(row.cellValue)) + ? '- -' + : row.cellValue; + + // 表格内容filter + const fillTimeText = (row: ObjectType) => + ['null', 'undefined', ''].includes(String(row.cellValue)) + ? '- -' + : useDateFormat(row.cellValue as string, 'YYYY-MM-DD HH:mm:ss').value; + + return { + templateUserList, + allWorkSpaceUserList, + engineList, + permissionTypeList, + projectNameList, + workflowNameList, + allDeptList, + allWorkSpaceUserDeptsList, + loadAllWorkSpaceUserDeptsList, + loadAllDeptList, + loadTemplateUserList, + loadAllWorkSpaceUserList, + loadEngineList, + loadworkFlowProjectNames, + loadworkFlowNames, + fillText, + fillTimeText, + }; +}; diff --git a/next-web/packages/workspace/module/resourceTmlManage/hooks/utils.ts b/next-web/packages/workspace/module/resourceTmlManage/hooks/utils.ts new file mode 100644 index 0000000000..1bb22a4682 --- /dev/null +++ b/next-web/packages/workspace/module/resourceTmlManage/hooks/utils.ts @@ -0,0 +1,27 @@ +export const utils = () => { + // 值为数组、对象前的情况未处理 + const paramsObjectToString = (params: object) => { + const strArray: string[] = []; + for (const [key, value] of Object.entries(params)) { + // console.log(`Key: ${key}, Value: ${value}`); + if (value) { + strArray.push(key + '=' + value); + } + } + const strParams = strArray.join('&'); + return strParams; + }; + + // 时间戳转为yy-MM-dd hh:mm:ss + const convertTimeStampToYMDHMS = (time: number | string) => { + let timeStr = new Date(Number(time)).toLocaleString('zh', { + hour12: false, + }); + timeStr = timeStr.replace(/\//g, '-'); + return timeStr; + }; + return { + paramsObjectToString, + convertTimeStampToYMDHMS, + }; +}; diff --git a/next-web/packages/workspace/module/resourceTmlManage/index.vue b/next-web/packages/workspace/module/resourceTmlManage/index.vue new file mode 100644 index 0000000000..e9b13a1362 --- /dev/null +++ b/next-web/packages/workspace/module/resourceTmlManage/index.vue @@ -0,0 +1,467 @@ + + + diff --git a/next-web/packages/workspace/module/resourceTmlManage/layout.vue b/next-web/packages/workspace/module/resourceTmlManage/layout.vue new file mode 100644 index 0000000000..94e71a5350 --- /dev/null +++ b/next-web/packages/workspace/module/resourceTmlManage/layout.vue @@ -0,0 +1,4 @@ + + diff --git a/next-web/packages/workspace/module/workFlow/advanceModal.vue b/next-web/packages/workspace/module/workFlow/advanceModal.vue new file mode 100644 index 0000000000..5620ffca07 --- /dev/null +++ b/next-web/packages/workspace/module/workFlow/advanceModal.vue @@ -0,0 +1,148 @@ + + + + + diff --git a/next-web/packages/workspace/module/workFlow/api.ts b/next-web/packages/workspace/module/workFlow/api.ts new file mode 100644 index 0000000000..28b365e461 --- /dev/null +++ b/next-web/packages/workspace/module/workFlow/api.ts @@ -0,0 +1,6 @@ +export default { + getFlowList: 'dss/workflow/get', // 获取工作流 + getNodeTypes: 'dss/workflow/listNodeType', // 获取节点类型 + getAllWorkspaceUsers: 'dss/framework/workspace/getAllWorkspaceUsers', // 获取用户 + getConfTemplateList: 'dss/framework/workspace/engineconf/getConfTemplateList', // 工作空间参数模板 +}; diff --git a/next-web/packages/workspace/module/workFlow/hooks/useDataList.ts b/next-web/packages/workspace/module/workFlow/hooks/useDataList.ts new file mode 100644 index 0000000000..2e2b6d18c5 --- /dev/null +++ b/next-web/packages/workspace/module/workFlow/hooks/useDataList.ts @@ -0,0 +1,65 @@ +import { ref } from 'vue'; +import { request } from '@dataspherestudio/shared'; +import api from '../api'; +type ObjectType = Record; + +export const useDataList = () => { + const nodeTypes = ref([]); + const templateList = ref([]); + const userList = ref([]); + + // 获取节点类型 + async function getNodeTypes(param: ObjectType) { + await request + .fetch(api.getNodeTypes, { labels: param.labels }) + .then((res) => { + nodeTypes.value = (res.data.nodeTypes || []) + .map((item: ObjectType) => item.children) + .flat() + .map((item: ObjectType) => ({ label: item.title, value: item.type })); + }); + } + + async function getConfTemplateList(param: ObjectType, pageNow = 1) { + await request + .fetch(api.getConfTemplateList, { + workspaceId: param.workspaceId, + pageNow, + pageSize: 10000, + }) + .then((res) => { + const list = (res.data.templateList || []).map((item: ObjectType) => ({ + engineType: item.engineType, + label: item.name, + value: item.templateId, + })); + templateList.value = [...templateList.value, ...list]; + if (pageNow * 10000 < res.data.total) { + getConfTemplateList(param, pageNow + 1); + } + }); + } + + // 获取用户 + function getUserList(param: ObjectType) { + request + .fetch(api.getAllWorkspaceUsers, { + workspaceId: param.workspaceId, + }) + .then((res) => { + userList.value = res.data.users.accessUsers.map((item: ObjectType) => ({ + value: item, + label: item, + })); + }); + } + + return { + nodeTypes, + templateList, + userList, + getNodeTypes, + getConfTemplateList, + getUserList, + }; +}; diff --git a/next-web/packages/workspace/module/workFlow/hooks/useDataUtils.ts b/next-web/packages/workspace/module/workFlow/hooks/useDataUtils.ts new file mode 100644 index 0000000000..d498c726ba --- /dev/null +++ b/next-web/packages/workspace/module/workFlow/hooks/useDataUtils.ts @@ -0,0 +1,83 @@ +import { ref } from 'vue'; +type ObjectType = Record; + +export const useDataUtils = () => { + // 按照条件过滤 + function filterate(item: ObjectType, param: ObjectType) { + const validList = []; + if (param['nodeName']) { + const valid1 = item.title === param['nodeName']; + validList.push(valid1); + } + if (param['nodeType'] && (param['nodeType'] as string[]).length > 0) { + const valid2 = (param['nodeType'] as string[]).includes( + item.jobType as string + ); + validList.push(valid2); + } + if (param['templateId']) { + const valid3 = item.ecConfTemplateId === param['templateId']; + validList.push(valid3); + } + if (param['modifyUser']) { + const valid4 = item.modifyUser === param['modifyUser']; + validList.push(valid4); + } + if (param['updateTimes'] && (param['updateTimes'] as number[]).length > 0) { + const [startTime, endTime] = param['updateTimes'] as number[]; + const valid5 = + (item.modifyTime as number) >= startTime && + (item.modifyTime as number) <= endTime; + validList.push(valid5); + } + return !validList.some((valid) => valid === false); + } + + const timer = ref(); + + function handleNodeClick( + node: ObjectType | null, + action: string, + flowId: string | number + ) { + if (timer.value) { + window.clearTimeout(timer.value); + } + timer.value = window.setTimeout(() => { + window.parent.postMessage( + JSON.stringify({ + action, + flowId, + type: 'dss-nextweb', + node, + }), + '*' + ); + }, 300); + } + + function handleNodeDblClick( + node: ObjectType, + action: string, + flowId: string | number + ) { + if (timer.value) { + clearTimeout(timer.value); + } + window.parent.postMessage( + JSON.stringify({ + action, + flowId, + type: 'dss-nextweb', + node, + }), + '*' + ); + } + + return { + filterate, + handleNodeClick, + handleNodeDblClick, + }; +}; diff --git a/next-web/packages/workspace/module/workFlow/hooks/useTemplateList.ts b/next-web/packages/workspace/module/workFlow/hooks/useTemplateList.ts new file mode 100644 index 0000000000..91dd6ee391 --- /dev/null +++ b/next-web/packages/workspace/module/workFlow/hooks/useTemplateList.ts @@ -0,0 +1,36 @@ +import { ref } from 'vue'; +type ObjectType = Record; +export const useTemplateList = (templates: { [key: string]: any }) => { + const templateData = ref([]); + + function createData(data: ObjectType[]): ObjectType[] | undefined { + if (!data) return; + return data.map((v) => ({ + label: v.label, + value: v.value, + scopeBus: v.scopeBus, + children: createData(v.children as ObjectType[]), + prefix: null, + suffix: null, + })); + } + + const loadTemplates = async (node: ObjectType) => { + if (!node) { + return Promise.resolve([]); + } + const list = templates.value + .filter((item: ObjectType) => item.engineType === node.value) + .map((item: ObjectType) => { + item.isLeaf = true; + return item; + }); + return Promise.resolve(list); + }; + + return { + templateData, + createData, + loadTemplates, + }; +}; diff --git a/next-web/packages/workspace/module/workFlow/index.vue b/next-web/packages/workspace/module/workFlow/index.vue new file mode 100644 index 0000000000..29a91fd1d1 --- /dev/null +++ b/next-web/packages/workspace/module/workFlow/index.vue @@ -0,0 +1,638 @@ + + + diff --git a/next-web/packages/workspace/module/workFlow/nodeDrawer.vue b/next-web/packages/workspace/module/workFlow/nodeDrawer.vue new file mode 100644 index 0000000000..9e015e2348 --- /dev/null +++ b/next-web/packages/workspace/module/workFlow/nodeDrawer.vue @@ -0,0 +1,283 @@ + + + diff --git a/next-web/packages/workspace/package.json b/next-web/packages/workspace/package.json new file mode 100644 index 0000000000..b150f78799 --- /dev/null +++ b/next-web/packages/workspace/package.json @@ -0,0 +1,9 @@ +{ + "name": "@dataspherestudio/workspace", + "version": "0.0.1", + "dependencies": { + "@dataspherestudio/shared": "^0.0.1" + }, + "devDependencies": {}, + "scripts": {} +} diff --git a/next-web/packages/workspace/router.js b/next-web/packages/workspace/router.js new file mode 100644 index 0000000000..983e537a17 --- /dev/null +++ b/next-web/packages/workspace/router.js @@ -0,0 +1,64 @@ +const routes = [ + { + path: '/workspace', + name: 'workspace', + meta: { + title: 'Workspace', + publicPage: true, + }, + component: () => import('./view/layout.vue'), + redirect: '/workspace/resourceTmlManage', + children: [ + { + path: 'resourceTmlManage', + name: 'resourceTmlManage', + meta: { + title: 'Resource Template', + publicPage: true, + }, + component: () => import('./module/resourceTmlManage/layout.vue'), + children: [ + { + path: '', + name: 'resourceTmlManage', + meta: { + title: 'Resource Template', + publicPage: true, + }, + component: () => import('./module/resourceTmlManage/index.vue'), + }, + { + path: 'applicationRule', + name: 'applicationRule', + meta: { + title: 'Application Rule', + publicPage: true, + }, + component: () => + import('./module/resourceTmlManage/applicationRule/index.vue'), + }, + ], + }, + { + path: 'announcement', + name: 'announcement', + meta: { + title: 'Announcement', + publicPage: true, + }, + component: () => import('./module/announcement/index.vue'), + }, + { + path: 'workflow', + name: 'workflow', + meta: { + title: 'Workflow', + publicPage: true, + }, + component: () => import('./module/workFlow/index.vue'), + }, + ], + }, +]; + +export default routes; diff --git a/next-web/packages/workspace/view/layout.vue b/next-web/packages/workspace/view/layout.vue new file mode 100644 index 0000000000..32ca531e93 --- /dev/null +++ b/next-web/packages/workspace/view/layout.vue @@ -0,0 +1,6 @@ + diff --git a/next-web/public/vite.svg b/next-web/public/vite.svg new file mode 100644 index 0000000000..e7b8dfb1b2 --- /dev/null +++ b/next-web/public/vite.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/next-web/src/App.vue b/next-web/src/App.vue new file mode 100644 index 0000000000..52ed022a95 --- /dev/null +++ b/next-web/src/App.vue @@ -0,0 +1,20 @@ + + + + + diff --git a/next-web/src/assets/vue.svg b/next-web/src/assets/vue.svg new file mode 100644 index 0000000000..770e9d333e --- /dev/null +++ b/next-web/src/assets/vue.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/next-web/src/base.less b/next-web/src/base.less new file mode 100644 index 0000000000..d8c5a1e9f6 --- /dev/null +++ b/next-web/src/base.less @@ -0,0 +1,75 @@ +* { + box-sizing: border-box; + font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', 'PingFang SC', + 'Hiragino Sans GB', 'Microsoft YaHei', 'Helvetica Neue', Helvetica, Arial, + sans-serif, 'Apple Color Emoji', 'Segoe UI Emoji', 'Segoe UI Symbol', + 'JinbiaoSong', 'JinbiaoSongExt'; +} +html { + height: 100%; +} +body { + margin: 0; + padding: 0; + height: 100%; + color: #0f1222; + font-size: 14px; +} + +h1, +h2, +h3, +h4, +h5, +h6, +p { + margin: 0; + padding: 0; +} + +a { + text-decoration: none; + color: #5384ff; +} + +ul { + padding: 0; + margin: 0; + list-style: none; +} + +#app { + height: 100%; +} + +.table-container { + .table-operation-item { + padding: 0; + width: 100%; + min-width: 100%; + border: 0; + font-family: PingFangSC-Regular; + font-size: 14px; + color: #5384ff; + letter-spacing: 0; + line-height: 22px; + font-weight: 400; + &:hover { + color: #5384ff; + } + } +} + +.a-link { + cursor: pointer; +} +.fes-ellipsis { + .a-link { + color: #5384ff; + } +} +.fes-popper { + .a-link { + color: #fff; + } +} diff --git a/next-web/src/i18n.ts b/next-web/src/i18n.ts new file mode 100644 index 0000000000..d88cde1502 --- /dev/null +++ b/next-web/src/i18n.ts @@ -0,0 +1,26 @@ +import { createI18n } from 'vue-i18n'; +import Apps from 'virtual-app-module'; +import { merge } from 'lodash'; + +const messages = { + en: {}, + 'zh-CN': {}, +}; + +Apps.appsI18n.forEach((appI18n: any) => { + Promise.all([appI18n['zh-CN'], appI18n['en']]).then((item) => { + const msgconf = { + 'zh-CN': item[0].default, + en: item[1].default, + }; + merge(messages, msgconf); + }); +}); + +const i18n = createI18n({ + legacy: false, + locale: 'zh-CN', + messages, +}); + +export default i18n; diff --git a/next-web/src/main.ts b/next-web/src/main.ts new file mode 100644 index 0000000000..cd48633a38 --- /dev/null +++ b/next-web/src/main.ts @@ -0,0 +1,65 @@ +import { createApp } from 'vue'; +import router from './router'; +import i18n from './i18n'; +import App from './App.vue'; +import { + FSpace, + FMessage, + FInput, + FButton, + FTable, + FPagination, + FSelect, + FTableColumn, + FDropdown, + FTooltip, + FCard, + FEllipsis, + FModal, + FDrawer, + FForm, + FDatePicker, + FSelectCascader, +} from '@fesjs/fes-design'; +import { + BTablePage, + BSearch, + BTableHeaderConfig, +} from '@fesjs/traction-widget'; +import { request } from '@dataspherestudio/shared'; +import './base.less'; +// 设置请求错误提示 +request.setError({ + showErrorMsg: function (error: any) { + FMessage.error(error.message || '这是一条消息'); + }, +}); + +router().then((router) => { + const app = createApp(App); + app.use(BTablePage); + app.use(BSearch); + app.use(BTableHeaderConfig); + app.use(FDrawer); + app.use(FSpace); + // @ts-ignore + app.use(FMessage); + app.use(FInput); + app.use(FButton); + app.use(FTable); + app.use(FPagination); + app.use(FSelect); + app.use(FTableColumn); + app.use(FDropdown); + app.use(FTooltip); + app.use(FCard); + app.use(FEllipsis); + app.use(FModal); + app.use(FForm); + app.use(FSpace); + app.use(FDatePicker); + app.use(FSelectCascader); + app.use(i18n); + app.use(router); + app.mount('#app'); +}); diff --git a/next-web/src/router.ts b/next-web/src/router.ts new file mode 100644 index 0000000000..2748fc1671 --- /dev/null +++ b/next-web/src/router.ts @@ -0,0 +1,35 @@ +import type { RouteRecordRaw } from 'vue-router'; +import { createRouter, createWebHashHistory } from 'vue-router'; + +import Apps from 'virtual-app-module'; + +console.log(Apps.appsRoutes); +function initRoutes() { + return Promise.all(Object.values(Apps.appsRoutes)).then((route) => { + let routes: RouteRecordRaw[] = []; + //@ts-ignore + route.forEach((it) => (routes = routes.concat(it.default))); + return routes; + }); +} + +export default async function init() { + const routes: RouteRecordRaw[] = await initRoutes(); + const router = createRouter({ + history: createWebHashHistory(), + routes: routes, + }); + + // 全局前置守卫 + router.beforeEach((to, from, next) => { + next(); + }); + + router.afterEach((to) => { + const _title = to.meta.title as string; + if (_title) { + window.document.title = _title; + } + }); + return router; +} diff --git a/next-web/src/vite-env.d.ts b/next-web/src/vite-env.d.ts new file mode 100644 index 0000000000..11f02fe2a0 --- /dev/null +++ b/next-web/src/vite-env.d.ts @@ -0,0 +1 @@ +/// diff --git a/next-web/tsconfig.json b/next-web/tsconfig.json new file mode 100644 index 0000000000..e649c6cd98 --- /dev/null +++ b/next-web/tsconfig.json @@ -0,0 +1,46 @@ +{ + "extends": "@vue/tsconfig/tsconfig.dom.json", + "compilerOptions": { + "target": "ES2020", + + "typeRoots": ["./node_modules/@types", "./types"], + "paths": { + "@/*": ["./*"], + "@dataspherestudio/*": ["./packages/*"] + }, + + /* Bundler mode */ + "moduleResolution": "bundler", + "allowImportingTsExtensions": true, + "resolveJsonModule": true, + "isolatedModules": true, + "noEmit": true, + "jsx": "preserve", + + /* Linting */ + "strict": true, + "noUnusedLocals": true, + "noUnusedParameters": false, + "noFallthroughCasesInSwitch": true + }, + "include": [ + "src/**/*.ts", + "src/**/*.d.ts", + "src/**/*.tsx", + "src/**/*.vue", + "packages/**/*.ts", + "packages/**/*.d.ts", + "packages/**/*.tsx", + "packages/**/*.vue", + "exts/**/*.ts", + "exts/**/*.d.ts", + "exts/**/*.tsx", + "exts/**/*.vue", + "types/**/*.d.ts" + ], + "references": [ + { + "path": "./tsconfig.node.json" + } + ] +} diff --git a/next-web/tsconfig.node.json b/next-web/tsconfig.node.json new file mode 100644 index 0000000000..49b0ac29cc --- /dev/null +++ b/next-web/tsconfig.node.json @@ -0,0 +1,12 @@ +{ + "compilerOptions": { + "target": "es2020", + "composite": true, + "skipLibCheck": true, + "module": "ESNext", + "moduleResolution": "bundler", + "allowSyntheticDefaultImports": true + }, + + "include": ["vite.config.ts", "vite-plugin/*.ts", "types/**/*.d.ts"] +} diff --git a/next-web/types/index.d.ts b/next-web/types/index.d.ts new file mode 100644 index 0000000000..91e74db7f0 --- /dev/null +++ b/next-web/types/index.d.ts @@ -0,0 +1,23 @@ +declare type AppModule = 'workspace'; + +declare type ConfigOfApps = { + routes: string; + module: string; + i18n: { + en: string; + 'zh-CN': string; + }; +}; + +declare type AppConf = { + apps: { + [key in AppModule]: ConfigOfApps; + }; + exts: object; + conf: object; + version: string; +}; + +declare interface Window { + __MICRO_APP_ENVIRONMENT__: any; +} diff --git a/next-web/types/modules.d.ts b/next-web/types/modules.d.ts new file mode 100644 index 0000000000..6ccf7e2595 --- /dev/null +++ b/next-web/types/modules.d.ts @@ -0,0 +1,16 @@ +declare module 'virtual-app-module' { + import type { RouteRecordRaw } from 'vue-router'; + type AppModuleType = 'dss' | 'workspace' | 'scriptis'; + + let exts: object; + let appsRoutes: { + [key in AppModuleType]: RouteRecordRaw[]; + }; + let appsI18n: []; + let conf: { + [key: string]: any; + }; + let gitInfo: { + [key: string]: any; + }; +} diff --git a/next-web/types/shims-vue.d.ts b/next-web/types/shims-vue.d.ts new file mode 100644 index 0000000000..798e8fcfac --- /dev/null +++ b/next-web/types/shims-vue.d.ts @@ -0,0 +1,5 @@ +declare module '*.vue' { + import { defineComponent } from 'vue'; + const component: ReturnType; + export default component; +} diff --git a/next-web/vite-plugin/vite-dynamic-module.ts b/next-web/vite-plugin/vite-dynamic-module.ts new file mode 100644 index 0000000000..cd7314a0e6 --- /dev/null +++ b/next-web/vite-plugin/vite-dynamic-module.ts @@ -0,0 +1,108 @@ +import child_process from 'child_process'; + +// 默认打包DSS +let configFile = `dss.js`; +if (process.env.npm_config_configfile) { + configFile = `.${process.env.npm_config_configfile}`; +} + +const config = await import(`./config/${configFile}`); +const { apps = {}, exts = {}, conf = {}, version } = config.default as AppConf; + +if (version) { + process.env.VUE_APP_VERSION = version; +} + +const appsRoutes: string[] = []; +const appsI18n: string[] = []; +const confs: string[] = []; + +Object.entries(apps).forEach((item) => { + // 处理路由 + const conf = item[1] as ConfigOfApps; + if (conf.routes) { + appsRoutes.push(`${item[0]}: import('@/packages/${conf.routes}')`); + } + // 处理国际化 + if (conf.i18n) { + appsI18n.push(`{ + 'zh-CN': import('@/packages/${conf.i18n['zh-CN']}'), + 'en': import('@/packages/${conf.i18n['en']}') + }`); + } +}); + +// 扩展模块 +const extsMoule: string[] = []; +Object.keys(exts).forEach((item, index) => { + extsMoule[index] = ` + '${item}': { + 'module': import('@/exts/${exts[item].module}'), + 'options': ${JSON.stringify(exts[item].options)} + } + `; + // 扩展模块国际化文件 + if (exts[item].i18n) { + appsI18n.push(`{ + 'zh-CN': import('@/exts/${exts[item].i18n['zh-CN']}'), + 'en': import('@/exts/${exts[item].i18n['en']}') + }`); + } +}); + +// config +Object.keys(conf).forEach((item) => { + if (['app_logo'].includes(item)) { + confs.push(`${item}: import('@/${conf[item]}')`); + } else if (typeof conf[item] == 'string') { + confs.push(`${item}: '${conf[item]}'`); + } else if (typeof conf[item] == 'object') { + confs.push(`${item}: ${JSON.stringify(conf[item])}`); + } else { + confs.push(`${item}: ${conf[item]}`); + } +}); + +const gitInfo: { branchName: string; gitUrl: string; commitInfo: string } = { + branchName: '', + gitUrl: '', + commitInfo: '', +}; +try { + // 当前构建分支信息 + gitInfo.branchName = child_process + .execSync('git branch --show-current') + .toString(); + gitInfo.gitUrl = child_process.execSync('git remote -v').toString(); + const commitInfo = child_process.execSync('git branch -vv').toString(); + gitInfo.commitInfo = commitInfo + .split(/[\r\n]/) + .filter((item) => item.indexOf('*') > -1) + .join(''); +} catch (e) { + /* empty */ +} + +export default function dynamicPlugin() { + const virtualModuleId = 'virtual-app-module'; + const resolvedVirtualModuleId = '\0' + virtualModuleId; + return { + name: 'virtual-module-plugin', + resolveId(id) { + if (id === virtualModuleId) { + return resolvedVirtualModuleId; + } + }, + load(id) { + if (id === resolvedVirtualModuleId) { + return `export default { + exts: {${extsMoule.join(',')}}, + appsRoutes: {${appsRoutes.join(',')}}, + appsI18n: [${appsI18n.join(',')}], + conf: {${confs.join(',')}}, + gitInfo: ${JSON.stringify(gitInfo)} + };`; + } + }, + }; +} diff --git a/next-web/vite-plugin/vite-host.ts b/next-web/vite-plugin/vite-host.ts new file mode 100644 index 0000000000..15f775f52d --- /dev/null +++ b/next-web/vite-plugin/vite-host.ts @@ -0,0 +1,25 @@ +import devip from 'dev-ip'; +import { Plugin } from 'vite'; + +export default function vitePluginHost(): Plugin { + return { + name: 'vite-plugin-host', + apply: 'serve', + config: () => ({ + server: { + host: '0.0.0.0', + hmr: { + host: devip()[0], + port: 5173, + }, + }, + }), + configureServer(server) { + server.middlewares.use((req, _, next) => { + // @ts-ignore + req.url = req._parsedUrl.path; + next(); + }); + }, + }; +} diff --git a/next-web/vite.config.ts b/next-web/vite.config.ts new file mode 100644 index 0000000000..883fd3fe1d --- /dev/null +++ b/next-web/vite.config.ts @@ -0,0 +1,54 @@ +import { resolve } from 'path'; +import vue from '@vitejs/plugin-vue'; +import { ConfigEnv, defineConfig, loadEnv, UserConfig } from 'vite'; +import host from './vite-plugin/vite-host'; +import eslintPlugin from 'vite-plugin-eslint'; +import virtualModule from './vite-plugin/vite-dynamic-module'; + +// https://vitejs.dev/config/ +export default defineConfig(({ command, mode }: ConfigEnv): UserConfig => { + // 根据当前工作目录中的 `mode` 加载 .env 文件 + // 设置第三个参数为 '' 来加载所有环境变量，而不管是否有 `VITE_` 前缀。 + const env = loadEnv(mode, process.cwd(), ''); + console.log(env, command, mode); + return { + define: {}, + plugins: [host(), vue(), virtualModule(), eslintPlugin()], + resolve: { + alias: { + '@': resolve(__dirname, './'), + '@dataspherestudio': resolve(__dirname, './packages'), + }, + }, + css: { + preprocessorOptions: { + less: { + javascriptEnabled: true, + }, + }, + }, + server: { + https: false, + proxy: { + '/api': { + target: 'http://10.107.97.166:8088', + // target: 'http://sit.dss.bdap.weoa.com', + // target: 'http://uat.dss.bdap.weoa.com/', + secure: false, + changeOrigin: true, + followRedirects: true, + headers: { + cookie: '', + // cookie: + // 'linkis_user_session_ticket_id_v1=FmzyJmP2DGUzEo5Hy/D0OqyinoWZ4+L52vOV55MqIl4=; workspaceId=104; workspaceName=bdapWorkspace_move', + TokenCode: 'admin-kmsnd', + TokenUser: 'stacyyan', + TokenAlive: 'true', + }, + }, + }, + }, + base: './', + build: { target: 'chrome66' }, + }; +}); diff --git a/plugins/azkaban/linkis-jobtype/bin/config.sh b/plugins/azkaban/linkis-jobtype/bin/config.sh index 734b464d79..f1b71ec2d0 100644 --- a/plugins/azkaban/linkis-jobtype/bin/config.sh +++ b/plugins/azkaban/linkis-jobtype/bin/config.sh @@ -1,8 +1,8 @@ ##Linkis gateway url LINKIS_GATEWAY_URL=http://127.0.0.1:9001 -##Linkis gateway token default 172.0.0.1 -LINKIS_GATEWAY_TOKEN=172.0.0.1 +##Linkis gateway token default WS-AUTH +LINKIS_GATEWAY_TOKEN=xxx ##Azkaban executor host diff --git a/plugins/azkaban/linkis-jobtype/pom.xml b/plugins/azkaban/linkis-jobtype/pom.xml index e2ace7bb30..531993f253 100644 --- a/plugins/azkaban/linkis-jobtype/pom.xml +++ b/plugins/azkaban/linkis-jobtype/pom.xml @@ -22,7 +22,7 @@ dss com.webank.wedatasphere.dss - 1.1.0.20-SNAPSHOT + 1.10.0-SNAPSHOT ../../../pom.xml linkis-jobtype diff --git a/plugins/azkaban/linkis-jobtype/src/main/java/com/webank/wedatasphere/dss/plugins/azkaban/linkis/jobtype/AzkabanDssJobType.java b/plugins/azkaban/linkis-jobtype/src/main/java/com/webank/wedatasphere/dss/plugins/azkaban/linkis/jobtype/AzkabanDssJobType.java index 0007b771d0..a69e8de88c 100644 --- a/plugins/azkaban/linkis-jobtype/src/main/java/com/webank/wedatasphere/dss/plugins/azkaban/linkis/jobtype/AzkabanDssJobType.java +++ b/plugins/azkaban/linkis-jobtype/src/main/java/com/webank/wedatasphere/dss/plugins/azkaban/linkis/jobtype/AzkabanDssJobType.java @@ -96,6 +96,7 @@ public void run() throws Exception { String runTodayH = getRunTodayh(false); if (StringUtils.isNotBlank(runTodayH)) { this.jobPropsMap.put("run_today_h", runTodayH); + this.jobPropsMap.put("run_today_hour", runTodayH); } this.job = JobBuilder.getAzkanbanBuilder().setJobProps(this.jobPropsMap).build(); this.job.setLogObj(new AzkabanJobLog(this)); diff --git a/plugins/azkaban/linkis-jobtype/src/main/java/com/webank/wedatasphere/dss/plugins/azkaban/linkis/jobtype/job/AzkanbanBuilder.java b/plugins/azkaban/linkis-jobtype/src/main/java/com/webank/wedatasphere/dss/plugins/azkaban/linkis/jobtype/job/AzkanbanBuilder.java index 3bc7fe1c26..04ee6036d4 100644 --- a/plugins/azkaban/linkis-jobtype/src/main/java/com/webank/wedatasphere/dss/plugins/azkaban/linkis/jobtype/job/AzkanbanBuilder.java +++ b/plugins/azkaban/linkis-jobtype/src/main/java/com/webank/wedatasphere/dss/plugins/azkaban/linkis/jobtype/job/AzkanbanBuilder.java @@ -38,7 +38,8 @@ public class AzkanbanBuilder extends Builder { private static final Logger LOGGER = LoggerFactory.getLogger(AzkanbanBuilder.class); private static final String RUN_DATE_KEY = "run_date"; - private static final String RUN_DATE_HOUR_KEY = "run_today_h"; + private static final String RUN_DATE_H_KEY = "run_today_h"; + private static final String RUN_DATE_HOUR_KEY = "run_today_hour"; private Map jobProps; public AzkanbanBuilder setJobProps(Map jobProps) { @@ -103,6 +104,7 @@ protected void fillLinkisJobInfo(LinkisJob linkisJob) { // 只有工作流参数中没有设置,我们才会去进行替换 // 改为不管工作流是否设置，在 Schedulis 这边都需要统一使用 Schedulis 设置的 run_date和un_date_h,防止出现批量调度的误导作用 setNewRunDateVariable(variables, RUN_DATE_KEY); + setNewRunDateVariable(variables, RUN_DATE_H_KEY); setNewRunDateVariable(variables, RUN_DATE_HOUR_KEY); linkisJob.setVariables(variables); linkisJob.setSource(getSource()); diff --git a/plugins/azkaban/linkis-jobtype/src/main/resources/plugin.properties b/plugins/azkaban/linkis-jobtype/src/main/resources/plugin.properties index 67b5ed0659..6ddd23b4c4 100644 --- a/plugins/azkaban/linkis-jobtype/src/main/resources/plugin.properties +++ b/plugins/azkaban/linkis-jobtype/src/main/resources/plugin.properties @@ -16,6 +16,6 @@ wds.linkis.gateway.url.v1=http://127.0.0.1:9001 wds.linkis.gateway.url.v0=http://127.0.0.1:9001 -wds.linkis.client.flow.author.user.token= +wds.linkis.client.flow.author.user.token=xxx wds.linkis.flow.job.creator=scheduler wds.linkis.flow.job.creator.v1=schedulis diff --git a/plugins/dolphinscheduler/dolphinscheduler-prod-metrics/pom.xml b/plugins/dolphinscheduler/dolphinscheduler-prod-metrics/pom.xml index ca0cbacb82..4cc7490ae6 100644 --- a/plugins/dolphinscheduler/dolphinscheduler-prod-metrics/pom.xml +++ b/plugins/dolphinscheduler/dolphinscheduler-prod-metrics/pom.xml @@ -22,7 +22,7 @@ dss-plugin-dolphinscheduler com.webank.wedatasphere.dss - 1.1.0.20-SNAPSHOT + 1.10.0-SNAPSHOT ../pom.xml 4.0.0 diff --git a/plugins/dolphinscheduler/dss-dolphinscheduler-client/pom.xml b/plugins/dolphinscheduler/dss-dolphinscheduler-client/pom.xml index 0a01b3df13..4521ab9f34 100644 --- a/plugins/dolphinscheduler/dss-dolphinscheduler-client/pom.xml +++ b/plugins/dolphinscheduler/dss-dolphinscheduler-client/pom.xml @@ -22,7 +22,7 @@ dss-plugin-dolphinscheduler com.webank.wedatasphere.dss - 1.1.0.20-SNAPSHOT + 1.10.0-SNAPSHOT ../pom.xml 4.0.0 diff --git a/plugins/dolphinscheduler/dss-dolphinscheduler-token/pom.xml b/plugins/dolphinscheduler/dss-dolphinscheduler-token/pom.xml index e2d8af4fd0..e3855390ad 100644 --- a/plugins/dolphinscheduler/dss-dolphinscheduler-token/pom.xml +++ b/plugins/dolphinscheduler/dss-dolphinscheduler-token/pom.xml @@ -22,7 +22,7 @@ dss-plugin-dolphinscheduler com.webank.wedatasphere.dss - 1.1.0.20-SNAPSHOT + 1.10.0-SNAPSHOT ../pom.xml diff --git a/plugins/dolphinscheduler/pom.xml b/plugins/dolphinscheduler/pom.xml index c4fc2db6c2..719467a388 100644 --- a/plugins/dolphinscheduler/pom.xml +++ b/plugins/dolphinscheduler/pom.xml @@ -21,7 +21,7 @@ dss com.webank.wedatasphere.dss - 1.1.0.20-SNAPSHOT + 1.10.0-SNAPSHOT ../../pom.xml 4.0.0 diff --git a/plugins/linkis/dss-gateway-support/pom.xml b/plugins/linkis/dss-gateway-support/pom.xml index 4d24332da9..80be3cf343 100644 --- a/plugins/linkis/dss-gateway-support/pom.xml +++ b/plugins/linkis/dss-gateway-support/pom.xml @@ -21,7 +21,7 @@ dss com.webank.wedatasphere.dss - 1.1.0.20-SNAPSHOT + 1.10.0-SNAPSHOT ../../../pom.xml 4.0.0 diff --git a/plugins/linkis/dss-gateway-support/src/main/scala/org/apache/linkis/gateway/parser/DSSGatewayConfiguration.scala b/plugins/linkis/dss-gateway-support/src/main/scala/org/apache/linkis/gateway/parser/DSSGatewayConfiguration.scala new file mode 100644 index 0000000000..c62734f7b4 --- /dev/null +++ b/plugins/linkis/dss-gateway-support/src/main/scala/org/apache/linkis/gateway/parser/DSSGatewayConfiguration.scala @@ -0,0 +1,32 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.linkis.gateway.parser + +import org.apache.linkis.common.conf.CommonVars + + +object DSSGatewayConfiguration { + val DSS_SPRING_NAME = CommonVars("wds.linkis.dss.name", "dss-server") + + val DSS_URL_LABEL_PREFIX = CommonVars("wds.dss.gateway.url.prefix.name", "labels") + + val DSS_URL_ROUTE_LABEL_PREFIX = CommonVars("wds.dss.gateway.url.prefix.name", "labelsRoute") + + val DSS_URL_APPCONNS = CommonVars("wds.dss.gateway.url.appconns", "visualis") + + +} diff --git a/plugins/linkis/dss-gateway-support/src/main/scala/org/apache/linkis/gateway/parser/DSSGatewayParser.scala b/plugins/linkis/dss-gateway-support/src/main/scala/org/apache/linkis/gateway/parser/DSSGatewayParser.scala new file mode 100644 index 0000000000..9b71d64ecd --- /dev/null +++ b/plugins/linkis/dss-gateway-support/src/main/scala/org/apache/linkis/gateway/parser/DSSGatewayParser.scala @@ -0,0 +1,209 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.linkis.gateway.parser + +import org.apache.linkis.common.ServiceInstance +import org.apache.linkis.gateway.exception.TooManyServiceException +import org.apache.linkis.gateway.http.GatewayContext +import org.apache.linkis.gateway.springcloud.SpringCloudGatewayConfiguration.{API_URL_PREFIX, normalPath} +import org.apache.linkis.manager.label.builder.factory.LabelBuilderFactoryContext +import org.apache.linkis.manager.label.entity.Label +import org.apache.linkis.manager.label.entity.route.RouteLabel +import org.apache.linkis.protocol.constants.TaskConstant +import org.apache.linkis.rpc.sender.SpringCloudFeignConfigurationCache +import org.apache.linkis.server.BDPJettyServerHelper +import org.springframework.stereotype.Component + +import java.util +import scala.collection.JavaConversions._ + + +@Component +class DSSGatewayParser extends AbstractGatewayParser { + + val appConns = DSSGatewayConfiguration.DSS_URL_APPCONNS.getValue.split(",") + + override def shouldContainRequestBody(gatewayContext: GatewayContext): Boolean = { + var contentType = gatewayContext.getRequest.getHeaders.get("Content-Type") + if(null == contentType) { + contentType = gatewayContext.getRequest.getHeaders.get("content-type") + } + + if(contentType != null && contentType.nonEmpty + && contentType(0).contains("form-data")){ + logger.info("DSS gateway get request type is form-data") + return false + } + + gatewayContext.getRequest.getRequestURI match { + case DSSGatewayParser.DSS_URL_DEFAULT_REGEX(_, _) => true + case DSSGatewayParser.DSS_URL_REGEX(_, _,_) => true + case DSSGatewayParser.APPCONN_URL_DEFAULT_REGEX(_,appconn, _) if appConns.contains(appconn) => true + case _ => false + } + } + + override def parse(gatewayContext: GatewayContext): Unit = gatewayContext.getRequest.getRequestURI match { + + case DSSGatewayParser.DSS_URL_REGEX(version, firstName, secondName) => + if (sendResponseWhenNotMatchVersion(gatewayContext, version)) return + var tmpServerName = "dss-" + firstName + "-" + secondName + "-server" + tmpServerName = getServiceNameFromLabel(gatewayContext, tmpServerName) + val serviceName: Option[String] = findCommonService("dss/" + firstName + "/" + secondName, tmpServerName) + if (serviceName.isDefined) { + gatewayContext.getGatewayRoute.setServiceInstance(ServiceInstance(serviceName.get, null)) + } else { + logger.info("Now set default serviceInstance name " + DSSGatewayConfiguration.DSS_SPRING_NAME.getValue + "," + gatewayContext.getRequest.getRequestURI) + gatewayContext.getGatewayRoute.setServiceInstance(ServiceInstance(DSSGatewayConfiguration.DSS_SPRING_NAME.getValue, null)) + } + case DSSGatewayParser.DSS_URL_DEFAULT_REGEX(version, firstName) => + if (sendResponseWhenNotMatchVersion(gatewayContext, version)) return + var tmpServerName = "dss-" + firstName + "-server" + tmpServerName = getServiceNameFromLabel(gatewayContext, tmpServerName) + val serviceName: Option[String] = findCommonService("dss/" + firstName, tmpServerName) + if (serviceName.isDefined) { + gatewayContext.getGatewayRoute.setServiceInstance(ServiceInstance(serviceName.get, null)) + } else { + logger.info("Now set default serviceInstance name " + DSSGatewayConfiguration.DSS_SPRING_NAME.getValue + "," + gatewayContext.getRequest.getRequestURI) + gatewayContext.getGatewayRoute.setServiceInstance(ServiceInstance(DSSGatewayConfiguration.DSS_SPRING_NAME.getValue, null)) + } + case DSSGatewayParser.APPCONN_URL_DEFAULT_REGEX(version, serverName, _) if appConns.contains(serverName) => + if (sendResponseWhenNotMatchVersion(gatewayContext, version)) return + var tmpServerName = serverName + tmpServerName = getServiceNameFromLabel(gatewayContext, tmpServerName) + val serviceName: Option[String] = findCommonService(tmpServerName, tmpServerName) + if (serviceName.isDefined) { + gatewayContext.getGatewayRoute.setServiceInstance(ServiceInstance(serviceName.get, null)) + } else { + logger.info("Now set default serviceInstance name " + DSSGatewayConfiguration.DSS_SPRING_NAME.getValue + "," + gatewayContext.getRequest.getRequestURI) + gatewayContext.getGatewayRoute.setServiceInstance(ServiceInstance(DSSGatewayConfiguration.DSS_SPRING_NAME.getValue, null)) + } + case _ => + } + + private def getServiceNameFromLabel(gatewayContext: GatewayContext, tmpServiceName: String): String = { + var requestUrlLabels = gatewayContext.getRequest.getQueryParams.getOrDefault(DSSGatewayConfiguration.DSS_URL_LABEL_PREFIX.getValue, null) + if(requestUrlLabels == null) requestUrlLabels = gatewayContext.getRequest.getQueryParams.getOrDefault(DSSGatewayConfiguration.DSS_URL_ROUTE_LABEL_PREFIX.getValue,null) + logger.info("Get ServiceName From Label and method is "+ gatewayContext.getRequest.getMethod.toString+",and urlLabels is "+requestUrlLabels) + val requestMethod = gatewayContext.getRequest.getMethod.toLowerCase + if (requestUrlLabels == null && (requestMethod.equals("post") || requestMethod.equals("put") || requestMethod.equals("delete"))) { + val requestBody = Option(gatewayContext.getRequest.getRequestBody) + val routeLabelList = new util.ArrayList[RouteLabel]() + + requestBody match { + //todo form-data resolve + case Some(body) => + val labelBuilderFactory = LabelBuilderFactoryContext.getLabelBuilderFactory + val json = BDPJettyServerHelper.gson.fromJson(body, classOf[java.util.Map[String, Object]]) + val labels: util.List[Label[_]] = json.get(TaskConstant.LABELS) match { + case map: util.Map[String, Object] => labelBuilderFactory.getLabels(map) + case map: util.Map[String, Any] => labelBuilderFactory.getLabels(map.asInstanceOf) + case _ => new util.ArrayList[Label[_]]() + } + labels.filter(label => label.isInstanceOf[RouteLabel]).foreach(label => { + routeLabelList.add(label.asInstanceOf[RouteLabel]) + }) + + case _ => null + } + val labelNameList = routeLabelList.map(routeLabel => routeLabel.getStringValue).toList + if(labelNameList != null && labelNameList.size>0) { + genServiceNameByDSSLabel(labelNameList, tmpServiceName) + }else if(null != requestUrlLabels){ + genServiceNameByDSSLabel(requestUrlLabels.toList, tmpServiceName) + } else tmpServiceName + + } else { + if(requestUrlLabels != null) { + genServiceNameByDSSLabel(requestUrlLabels.toList, tmpServiceName) + }else tmpServiceName + } + } + + private def genServiceNameByDSSLabel(labelList: List[String], tmpServiceName: String): String = { + var resultName = tmpServiceName + if (null != labelList && labelList.size > 0) { + val labelNameList = labelList(0).replace(" ", "").split(",").toList + if (labelNameList.size > 0) { + if (labelNameList.find(name => name.equalsIgnoreCase("dev")).isDefined) { + resultName = tmpServiceName + "-dev" + } else if (labelNameList.find(name => name.equalsIgnoreCase("prod")).isDefined) { + resultName = tmpServiceName + "-prod" + } else if (labelNameList.find(name => name.equalsIgnoreCase("test")).isDefined){ + resultName = tmpServiceName + "-test" + }else{ + resultName = tmpServiceName + } + } + } + resultName + } + + + private def findCommonService(parsedServiceId: String, tmpServerName: String) = findService(parsedServiceId, tmpServerName, services => { + val errorMsg = new TooManyServiceException(s"Cannot find a correct serviceId for parsedServiceId $parsedServiceId, service list is: " + services) + warn("", errorMsg) + throw errorMsg + }) + + protected def findService(parsedServiceId: String, tmpServerName: String, tooManyDeal: List[String] => Option[String]): Option[String] = { + val findIt: (String => Boolean) => Option[String] = op => { + val services = SpringCloudFeignConfigurationCache.getDiscoveryClient + .getServices.filter(op).toList + if (services.length == 1) Some(services.head) + else if (services.length > 1) tooManyDeal(services) + else None + } + //通过匹配到最多的url中的path进行路由,如/dss/framework/workspace/ 会匹配到 dss-framework-workspace-server 而不是 dss-server + // 如果产生了相等的情况，则按照短的service名字为准 比如/dss/getProject, + // 我们可能会匹配到dss-server以及 dss-framework-workspace-server,则选择短名称的dss-server + val findMostCorrect: (String => (String, Int)) => Option[String] = { op => { + val serviceMap = SpringCloudFeignConfigurationCache.getDiscoveryClient + .getServices.map(op).toMap + var count = 0 + var retService: Option[String] = None + serviceMap.foreach { + case (k, v) => if (v > count) { + count = v + retService = Some(k) + } else if (retService.isDefined && v == count && k.length < retService.get.length) { + retService = Some(k) + } + } + retService + } + } + val lowerServiceId = parsedServiceId.toLowerCase + val serverName = tmpServerName.toLowerCase + //findIt(_.toLowerCase == lowerServiceId).orElse(findIt(_.toLowerCase.contains(lowerServiceId))) + findIt(_.toLowerCase == serverName).orElse(findMostCorrect(service => { + (service, lowerServiceId.split("/").count(word => service.contains(word))) + })) + } + + +} + +object DSSGatewayParser { + val DSS_HEADER = normalPath(API_URL_PREFIX) + "rest_[a-zA-Z][a-zA-Z_0-9]*/(v\\d+)/dss/" + val DSS_URL_REGEX = (DSS_HEADER + "([^/]+)/" + "([^/]+)/.+").r + val DSS_URL_DEFAULT_REGEX = (DSS_HEADER + "([^/]+).+").r + + val APPCONN_HEADER = normalPath(API_URL_PREFIX) + "rest_[a-zA-Z][a-zA-Z_0-9]*/(v\\d+)/([^/]+)/" + val APPCONN_URL_DEFAULT_REGEX = (APPCONN_HEADER + "([^/]+).+").r + +} diff --git a/plugins/linkis/dss-gateway-support/src/main/scala/org/apache/linkis/gateway/parser/DSSRouteLabelParser.scala b/plugins/linkis/dss-gateway-support/src/main/scala/org/apache/linkis/gateway/parser/DSSRouteLabelParser.scala new file mode 100644 index 0000000000..a56dbe5944 --- /dev/null +++ b/plugins/linkis/dss-gateway-support/src/main/scala/org/apache/linkis/gateway/parser/DSSRouteLabelParser.scala @@ -0,0 +1,32 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.linkis.gateway.parser + +import org.apache.linkis.gateway.http.GatewayContext +import org.apache.linkis.gateway.ujes.route.label.RouteLabelParser +import org.apache.linkis.manager.label.entity.route.RouteLabel +import org.springframework.stereotype.Component + +import java.util + + +@Component +class DSSRouteLabelParser extends RouteLabelParser{ + override def parse(gatewayContext: GatewayContext): util.List[RouteLabel] = { + new util.ArrayList[RouteLabel]() + } +} diff --git a/pom.xml b/pom.xml index b1372a31f3..1a98b1fb9e 100644 --- a/pom.xml +++ b/pom.xml @@ -23,7 +23,7 @@ pom com.webank.wedatasphere.dss dss - 1.1.0.20-SNAPSHOT + 1.10.0-SNAPSHOT dss-commons @@ -37,25 +37,26 @@ plugins/linkis/dss-gateway-support assembly plugins/dolphinscheduler + dss-git - 1.1.0.20-SNAPSHOT - 1.1.19-wds-SNAPSHOT + 1.10.0-SNAPSHOT + 1.6.1-wds 2.11.12 1.8 3.3.3 - 2.8.5 + 2.8.9 2.14.1 3.1.1 - 4.5.4 - 4.5.4 + 4.5.13 + 4.5.13 1.9.4 UTF-8 - 5.2.22.RELEASE + 5.3.34 2.1.2 - 2.3.12.RELEASE - 2.2.6.RELEASE + 2.7.18 + 2021.0.8 3.1.1 3.8.1 2.6 diff --git a/sbin/ext/dss-apiservice-server b/sbin/ext/dss-apiservice-server new file mode 100644 index 0000000000..58ad10ff0c --- /dev/null +++ b/sbin/ext/dss-apiservice-server @@ -0,0 +1,86 @@ +#!/bin/bash +# +# description: ecm start cmd +# + +# get log directory +cd `dirname $0` +cd .. +INSTALL_HOME=`pwd` + +# set DSS_HOME +if [ "$DSS_HOME" = "" ]; then + export DSS_HOME=$INSTALL_HOME +fi + +# set DSS_CONF_DIR +if [ "$DSS_CONF_DIR" = "" ]; then + export DSS_CONF_DIR=$DSS_HOME/conf +fi + +SERVER_SUFFIX="dss-apiservice-server" +## set log +if [ "$DSS_LOG_DIR" = "" ]; then + export DSS_LOG_DIR="$DSS_HOME/logs" +fi +export SERVER_LOG_PATH=$DSS_LOG_DIR +if [ ! -w "$SERVER_LOG_PATH" ] ; then + mkdir -p "$SERVER_LOG_PATH" +fi + +if test -z "$SERVER_HEAP_SIZE" +then + export SERVER_HEAP_SIZE="512M" +fi + +DEBUG_PORT= +if [ "$DEBUG_PORT" ]; +then + export DEBUG_CMD="-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=$DEBUG_PORT" +fi + +if test -z "$SERVER_JAVA_OPTS" +then + export SERVER_JAVA_OPTS="-DserviceName=$SERVER_SUFFIX -Xmx$SERVER_HEAP_SIZE -XX:+UseG1GC -Xloggc:$SERVER_LOG_PATH/$SERVER_SUFFIX.gc $DEBUG_CMD" +fi + +export SERVER_CLASS=com.webank.wedatasphere.dss.apiservice.DSSApiServiceServerApplication + +## conf dir +export SERVER_CONF_PATH=$DSS_CONF_DIR:$DSS_CONF_DIR/$SERVER_SUFFIX + +## commons lib +export DSS_COMMONS_LIB="$DSS_HOME/lib/dss-commons" + + +if [ ! -r "$DSS_COMMONS_LIB" ] ; then + echo "dss commons lib not exists $DSS_COMMONS_LIB" + exit 1 +fi + +## server lib +export SERVER_LIB=$DSS_HOME/lib/dss-apps/$SERVER_SUFFIX + + +if [ ! -r "$SERVER_LIB" ] ; then + echo "server lib not exists $SERVER_LIB" + exit 1 +fi + +export SERVER_PID=$DSS_HOME/pid/${SERVER_SUFFIX}.pid +## set class path +export SERVER_CLASS_PATH=$SERVER_CONF_PATH:$DSS_COMMONS_LIB/*:$SERVER_LIB/* + +nohup java $SERVER_JAVA_OPTS -cp $SERVER_CLASS_PATH $SERVER_CLASS 2>&1 > $SERVER_LOG_PATH/$SERVER_SUFFIX.out & +pid=$! + +sleep 2 +if [[ -z "${pid}" ]]; then + echo "server $SERVER_SUFFIX start failed!" + exit 1 +else + echo "server $SERVER_SUFFIX start succeeded!" + echo $pid > $SERVER_PID + sleep 1 +fi +exit 1 diff --git a/sbin/ext/dss-data-api-server b/sbin/ext/dss-data-api-server new file mode 100644 index 0000000000..4f9010b9bf --- /dev/null +++ b/sbin/ext/dss-data-api-server @@ -0,0 +1,83 @@ +#!/bin/bash + +# get log directory +cd `dirname $0` +cd .. +INSTALL_HOME=`pwd` + +# set DSS_HOME +if [ "$DSS_HOME" = "" ]; then + export DSS_HOME=$INSTALL_HOME +fi + +# set DSS_CONF_DIR +if [ "$DSS_CONF_DIR" = "" ]; then + export DSS_CONF_DIR=$DSS_HOME/conf +fi + +SERVER_SUFFIX="dss-data-api-server" +## set log +if [ "$DSS_LOG_DIR" = "" ]; then + export DSS_LOG_DIR="$DSS_HOME/logs" +fi +export SERVER_LOG_PATH=$DSS_LOG_DIR +if [ ! -w "$SERVER_LOG_PATH" ] ; then + mkdir -p "$SERVER_LOG_PATH" +fi + +if test -z "$SERVER_HEAP_SIZE" +then + export SERVER_HEAP_SIZE="512M" +fi + +DEBUG_PORT= +if [ "$DEBUG_PORT" ]; +then + export DEBUG_CMD="-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=$DEBUG_PORT" +fi + +if test -z "$SERVER_JAVA_OPTS" +then + export SERVER_JAVA_OPTS="-DserviceName=$SERVER_SUFFIX -Xmx$SERVER_HEAP_SIZE -XX:+UseG1GC -Xloggc:$SERVER_LOG_PATH/$SERVER_SUFFIX.gc $DEBUG_CMD" +fi + +export SERVER_CLASS=com.webank.wedatasphere.dss.data.api.server.DSSDataApiApplication + +## conf dir +export SERVER_CONF_PATH=$DSS_CONF_DIR:$DSS_CONF_DIR/$SERVER_SUFFIX + +## commons lib +export DSS_COMMONS_LIB="$DSS_HOME/lib/dss-commons" + + +if [ ! -r "$DSS_COMMONS_LIB" ] ; then + echo "dss commons lib not exists $DSS_COMMONS_LIB" + exit 1 +fi + +## server lib +export SERVER_LIB=$DSS_HOME/lib/dss-data-api/$SERVER_SUFFIX + + +if [ ! -r "$SERVER_LIB" ] ; then + echo "server lib not exists $SERVER_LIB" + exit 1 +fi + +export SERVER_PID=$DSS_HOME/pid/${SERVER_SUFFIX}.pid +## set class path +export SERVER_CLASS_PATH=$SERVER_CONF_PATH:$DSS_COMMONS_LIB/*:$SERVER_LIB/* + +nohup java $SERVER_JAVA_OPTS -cp $SERVER_CLASS_PATH $SERVER_CLASS 2>&1 > $SERVER_LOG_PATH/$SERVER_SUFFIX.out & +pid=$! + +sleep 2 +if [[ -z "${pid}" ]]; then + echo "server $SERVER_SUFFIX start failed!" + exit 1 +else + echo "server $SERVER_SUFFIX start succeeded!" + echo $pid > $SERVER_PID + sleep 1 +fi +exit 1 diff --git a/sbin/ext/dss-data-governance-server b/sbin/ext/dss-data-governance-server new file mode 100644 index 0000000000..8b259fe749 --- /dev/null +++ b/sbin/ext/dss-data-governance-server @@ -0,0 +1,85 @@ +#!/bin/bash +# +# description: ecm start cmd +# +# get log directory +cd `dirname $0` +cd .. +INSTALL_HOME=`pwd` + +# set DSS_HOME +if [ "$DSS_HOME" = "" ]; then + export DSS_HOME=$INSTALL_HOME +fi + +# set DSS_CONF_DIR +if [ "$DSS_CONF_DIR" = "" ]; then + export DSS_CONF_DIR=$DSS_HOME/conf +fi + +SERVER_SUFFIX="dss-data-governance-server" +## set log +if [ "$DSS_LOG_DIR" = "" ]; then + export DSS_LOG_DIR="$DSS_HOME/logs" +fi +export SERVER_LOG_PATH=$DSS_LOG_DIR +if [ ! -w "$SERVER_LOG_PATH" ] ; then + mkdir -p "$SERVER_LOG_PATH" +fi + +if test -z "$SERVER_HEAP_SIZE" +then + export SERVER_HEAP_SIZE="512M" +fi + +DEBUG_PORT= +if [ "$DEBUG_PORT" ]; +then + export DEBUG_CMD="-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=$DEBUG_PORT" +fi + +if test -z "$SERVER_JAVA_OPTS" +then + export SERVER_JAVA_OPTS="-DserviceName=$SERVER_SUFFIX -Xmx$SERVER_HEAP_SIZE -XX:+UseG1GC -Xloggc:$SERVER_LOG_PATH/$SERVER_SUFFIX.gc $DEBUG_CMD" +fi + +export SERVER_CLASS=com.webank.wedatasphere.dss.data.governance.DSSDataGovernanceApplication + +## conf dir +export SERVER_CONF_PATH=$DSS_CONF_DIR:$DSS_CONF_DIR/$SERVER_SUFFIX + +## commons lib +export DSS_COMMONS_LIB="$DSS_HOME/lib/dss-commons" + + +if [ ! -r "$DSS_COMMONS_LIB" ] ; then + echo "dss commons lib not exists $DSS_COMMONS_LIB" + exit 1 +fi + +## server lib +export SERVER_LIB=$DSS_HOME/lib/dss-data-governance/$SERVER_SUFFIX + + +if [ ! -r "$SERVER_LIB" ] ; then + echo "server lib not exists $SERVER_LIB" + exit 1 +fi + +export SERVER_PID=$DSS_HOME/pid/${SERVER_SUFFIX}.pid +## set class path +export SERVER_CLASS_PATH=$SERVER_CONF_PATH:$DSS_COMMONS_LIB/*:$SERVER_LIB/* + +nohup java $SERVER_JAVA_OPTS -cp $SERVER_CLASS_PATH $SERVER_CLASS 2>&1 > $SERVER_LOG_PATH/$SERVER_SUFFIX.out & +pid=$! + +sleep 2 +if [[ -z "${pid}" ]]; then + echo "server $SERVER_SUFFIX start failed!" + exit 1 +else + echo "server $SERVER_SUFFIX start succeeded!" + echo $pid > $SERVER_PID + sleep 1 +fi +exit 1 diff --git a/sbin/ext/dss-flow-execution-server b/sbin/ext/dss-flow-execution-server new file mode 100644 index 0000000000..4b652c945d --- /dev/null +++ b/sbin/ext/dss-flow-execution-server @@ -0,0 +1,83 @@ +#!/bin/bash + +# get log directory +cd `dirname $0` +cd .. +INSTALL_HOME=`pwd` + +# set DSS_HOME +if [ "$DSS_HOME" = "" ]; then + export DSS_HOME=$INSTALL_HOME +fi + +# set DSS_CONF_DIR +if [ "$DSS_CONF_DIR" = "" ]; then + export DSS_CONF_DIR=$DSS_HOME/conf +fi + +SERVER_SUFFIX="dss-flow-execution-server" +## set log +if [ "$DSS_LOG_DIR" = "" ]; then + export DSS_LOG_DIR="$DSS_HOME/logs" +fi +export SERVER_LOG_PATH=$DSS_LOG_DIR +if [ ! -w "$SERVER_LOG_PATH" ] ; then + mkdir -p "$SERVER_LOG_PATH" +fi + +if test -z "$SERVER_HEAP_SIZE" +then + export SERVER_HEAP_SIZE="512M" +fi + +DEBUG_PORT= +if [ "$DEBUG_PORT" ]; +then + export DEBUG_CMD="-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=$DEBUG_PORT" +fi + +if test -z "$SERVER_JAVA_OPTS" +then + export SERVER_JAVA_OPTS="-DserviceName=$SERVER_SUFFIX -Xmx$SERVER_HEAP_SIZE -XX:+UseG1GC -Xloggc:$SERVER_LOG_PATH/$SERVER_SUFFIX.gc $DEBUG_CMD" +fi + +export SERVER_CLASS=com.webank.wedatasphere.dss.flow.execution.entrance.DSSFowExecutionServerApplication + +## conf dir +export SERVER_CONF_PATH=$DSS_CONF_DIR:$DSS_CONF_DIR/$SERVER_SUFFIX + +## commons lib +export DSS_COMMONS_LIB="$DSS_HOME/lib/dss-commons" + + +if [ ! -r "$DSS_COMMONS_LIB" ] ; then + echo "dss commons lib not exists $DSS_COMMONS_LIB" + exit 1 +fi + +## server lib +export SERVER_LIB=$DSS_HOME/lib/dss-orchestrator/$SERVER_SUFFIX + + +if [ ! -r "$SERVER_LIB" ] ; then + echo "server lib not exists $SERVER_LIB" + exit 1 +fi + +export SERVER_PID=$DSS_HOME/pid/${SERVER_SUFFIX}.pid +## set class path +export SERVER_CLASS_PATH=$SERVER_CONF_PATH:$DSS_COMMONS_LIB/*:$SERVER_LIB/* + +nohup java $SERVER_JAVA_OPTS -cp $SERVER_CLASS_PATH $SERVER_CLASS 2>&1 > $SERVER_LOG_PATH/$SERVER_SUFFIX.out & +pid=$! + +sleep 2 +if [[ -z "${pid}" ]]; then + echo "server $SERVER_SUFFIX start failed!" + exit 1 +else + echo "server $SERVER_SUFFIX start succeeded!" + echo $pid > $SERVER_PID + sleep 1 +fi +exit 1 diff --git a/sbin/ext/dss-framework-orchestrator-server b/sbin/ext/dss-framework-orchestrator-server new file mode 100644 index 0000000000..a1af76c970 --- /dev/null +++ b/sbin/ext/dss-framework-orchestrator-server @@ -0,0 +1,85 @@ +#!/bin/bash +# +# description: ecm start cmd + +# get log directory +cd `dirname $0` +cd .. +INSTALL_HOME=`pwd` + +# set DSS_HOME +if [ "$DSS_HOME" = "" ]; then + export DSS_HOME=$INSTALL_HOME +fi + +# set DSS_CONF_DIR +if [ "$DSS_CONF_DIR" = "" ]; then + export DSS_CONF_DIR=$DSS_HOME/conf +fi + +SERVER_SUFFIX="dss-framework-orchestrator-server" +## set log +if [ "$DSS_LOG_DIR" = "" ]; then + export DSS_LOG_DIR="$DSS_HOME/logs" +fi +export SERVER_LOG_PATH=$DSS_LOG_DIR +if [ ! -w "$SERVER_LOG_PATH" ] ; then + mkdir -p "$SERVER_LOG_PATH" +fi + +if test -z "$SERVER_HEAP_SIZE" +then + export SERVER_HEAP_SIZE="512M" +fi + +DEBUG_PORT= +if [ "$DEBUG_PORT" ]; +then + export DEBUG_CMD="-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=$DEBUG_PORT" +fi + +if test -z "$SERVER_JAVA_OPTS" +then + export SERVER_JAVA_OPTS="-DserviceName=$SERVER_SUFFIX -Xmx$SERVER_HEAP_SIZE -XX:+UseG1GC -Xloggc:$SERVER_LOG_PATH/$SERVER_SUFFIX.gc $DEBUG_CMD" +fi + +export SERVER_CLASS=com.webank.wedatasphere.dss.orchestrator.server.DSSOrchestratorServerApplication + +## conf dir +export SERVER_CONF_PATH=$DSS_CONF_DIR:$DSS_CONF_DIR/$SERVER_SUFFIX + +## commons lib +export DSS_COMMONS_LIB="$DSS_HOME/lib/dss-commons" + + +if [ ! -r "$DSS_COMMONS_LIB" ] ; then + echo "dss commons lib not exists $DSS_COMMONS_LIB" + exit 1 +fi + +## server lib +export SERVER_LIB=$DSS_HOME/lib/dss-framework/$SERVER_SUFFIX + + +if [ ! -r "$SERVER_LIB" ] ; then + echo "server lib not exists $SERVER_LIB" + exit 1 +fi + +export SERVER_PID=$DSS_HOME/pid/${SERVER_SUFFIX}.pid +## set class path +export SERVER_CLASS_PATH=$SERVER_CONF_PATH:$DSS_COMMONS_LIB/*:$SERVER_LIB/* + +nohup java $SERVER_JAVA_OPTS -cp $SERVER_CLASS_PATH $SERVER_CLASS 2>&1 > $SERVER_LOG_PATH/$SERVER_SUFFIX.out & +pid=$! + +sleep 2 +if [[ -z "${pid}" ]]; then + echo "server $SERVER_SUFFIX start failed!" + exit 1 +else + echo "server $SERVER_SUFFIX start succeeded!" + echo $pid > $SERVER_PID + sleep 1 +fi +exit 1 diff --git a/sbin/ext/dss-framework-project-server b/sbin/ext/dss-framework-project-server new file mode 100644 index 0000000000..ac969bb67d --- /dev/null +++ b/sbin/ext/dss-framework-project-server @@ -0,0 +1,83 @@ +#!/bin/bash + +# get log directory +cd `dirname $0` +cd .. +INSTALL_HOME=`pwd` + +# set DSS_HOME +if [ "$DSS_HOME" = "" ]; then + export DSS_HOME=$INSTALL_HOME +fi + +# set DSS_CONF_DIR +if [ "$DSS_CONF_DIR" = "" ]; then + export DSS_CONF_DIR=$DSS_HOME/conf +fi + +SERVER_SUFFIX="dss-framework-project-server" +## set log +if [ "$DSS_LOG_DIR" = "" ]; then + export DSS_LOG_DIR="$DSS_HOME/logs" +fi +export SERVER_LOG_PATH=$DSS_LOG_DIR +if [ ! -w "$SERVER_LOG_PATH" ] ; then + mkdir -p "$SERVER_LOG_PATH" +fi + +if test -z "$SERVER_HEAP_SIZE" +then + export SERVER_HEAP_SIZE="512M" +fi + +DEBUG_PORT= +if [ "$DEBUG_PORT" ]; +then + export DEBUG_CMD="-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=$DEBUG_PORT" +fi + +if test -z "$SERVER_JAVA_OPTS" +then + export SERVER_JAVA_OPTS="-DserviceName=$SERVER_SUFFIX -Xmx$SERVER_HEAP_SIZE -XX:+UseG1GC -Xloggc:$SERVER_LOG_PATH/$SERVER_SUFFIX.gc $DEBUG_CMD" +fi + +export SERVER_CLASS=com.webank.wedatasphere.dss.framework.project.server.DSSProjectServerApplication + +## conf dir +export SERVER_CONF_PATH=$DSS_CONF_DIR:$DSS_CONF_DIR/$SERVER_SUFFIX + +## commons lib +export DSS_COMMONS_LIB="$DSS_HOME/lib/dss-commons" + + +if [ ! -r "$DSS_COMMONS_LIB" ] ; then + echo "dss commons lib not exists $DSS_COMMONS_LIB" + exit 1 +fi + +## server lib +export SERVER_LIB=$DSS_HOME/lib/dss-framework/$SERVER_SUFFIX + + +if [ ! -r "$SERVER_LIB" ] ; then + echo "server lib not exists $SERVER_LIB" + exit 1 +fi + +export SERVER_PID=$DSS_HOME/pid/${SERVER_SUFFIX}.pid +## set class path +export SERVER_CLASS_PATH=$SERVER_CONF_PATH:$DSS_COMMONS_LIB/*:$SERVER_LIB/* + +nohup java $SERVER_JAVA_OPTS -cp $SERVER_CLASS_PATH $SERVER_CLASS 2>&1 > $SERVER_LOG_PATH/$SERVER_SUFFIX.out & +pid=$! + +sleep 2 +if [[ -z "${pid}" ]]; then + echo "server $SERVER_SUFFIX start failed!" + exit 1 +else + echo "server $SERVER_SUFFIX start succeeded!" + echo $pid > $SERVER_PID + sleep 1 +fi +exit 1 diff --git a/sbin/ext/dss-guide-server b/sbin/ext/dss-guide-server new file mode 100644 index 0000000000..a6eb968088 --- /dev/null +++ b/sbin/ext/dss-guide-server @@ -0,0 +1,85 @@ +#!/bin/bash +# +# description: ecm start cmd +# +# get log directory +cd `dirname $0` +cd .. +INSTALL_HOME=`pwd` + +# set DSS_HOME +if [ "$DSS_HOME" = "" ]; then + export DSS_HOME=$INSTALL_HOME +fi + +# set DSS_CONF_DIR +if [ "$DSS_CONF_DIR" = "" ]; then + export DSS_CONF_DIR=$DSS_HOME/conf +fi + +SERVER_SUFFIX="dss-guide-server" +## set log +if [ "$DSS_LOG_DIR" = "" ]; then + export DSS_LOG_DIR="$DSS_HOME/logs" +fi +export SERVER_LOG_PATH=$DSS_LOG_DIR +if [ ! -w "$SERVER_LOG_PATH" ] ; then + mkdir -p "$SERVER_LOG_PATH" +fi + +if test -z "$SERVER_HEAP_SIZE" +then + export SERVER_HEAP_SIZE="512M" +fi + +DEBUG_PORT= +if [ "$DEBUG_PORT" ]; +then + export DEBUG_CMD="-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=$DEBUG_PORT" +fi + +if test -z "$SERVER_JAVA_OPTS" +then + export SERVER_JAVA_OPTS="-DserviceName=$SERVER_SUFFIX -Xmx$SERVER_HEAP_SIZE -XX:+UseG1GC -Xloggc:$SERVER_LOG_PATH/$SERVER_SUFFIX.gc $DEBUG_CMD" +fi + +export SERVER_CLASS=com.webank.wedatasphere.dss.guide.server.DSSGuideApplication + +## conf dir +export SERVER_CONF_PATH=$DSS_CONF_DIR:$DSS_CONF_DIR/$SERVER_SUFFIX + +## commons lib +export DSS_COMMONS_LIB="$DSS_HOME/lib/dss-commons" + + +if [ ! -r "$DSS_COMMONS_LIB" ] ; then + echo "dss commons lib not exists $DSS_COMMONS_LIB" + exit 1 +fi + +## server lib +export SERVER_LIB=$DSS_HOME/lib/dss-guide/$SERVER_SUFFIX + + +if [ ! -r "$SERVER_LIB" ] ; then + echo "server lib not exists $SERVER_LIB" + exit 1 +fi + +export SERVER_PID=$DSS_HOME/pid/${SERVER_SUFFIX}.pid +## set class path +export SERVER_CLASS_PATH=$SERVER_CONF_PATH:$DSS_COMMONS_LIB/*:$SERVER_LIB/* + +nohup java $SERVER_JAVA_OPTS -cp $SERVER_CLASS_PATH $SERVER_CLASS 2>&1 > $SERVER_LOG_PATH/$SERVER_SUFFIX.out & +pid=$! + +sleep 2 +if [[ -z "${pid}" ]]; then + echo "server $SERVER_SUFFIX start failed!" + exit 1 +else + echo "server $SERVER_SUFFIX start succeeded!" + echo $pid > $SERVER_PID + sleep 1 +fi +exit 1 diff --git a/sbin/ext/dss-scriptis-server b/sbin/ext/dss-scriptis-server new file mode 100644 index 0000000000..ac4e479247 --- /dev/null +++ b/sbin/ext/dss-scriptis-server @@ -0,0 +1,83 @@ +#!/bin/bash + +# get log directory +cd `dirname $0` +cd .. +INSTALL_HOME=`pwd` + +# set DSS_HOME +if [ "$DSS_HOME" = "" ]; then + export DSS_HOME=$INSTALL_HOME +fi + +# set DSS_CONF_DIR +if [ "$DSS_CONF_DIR" = "" ]; then + export DSS_CONF_DIR=$DSS_HOME/conf +fi + +SERVER_SUFFIX="dss-scriptis-server" +## set log +if [ "$DSS_LOG_DIR" = "" ]; then + export DSS_LOG_DIR="$DSS_HOME/logs" +fi +export SERVER_LOG_PATH=$DSS_LOG_DIR +if [ ! -w "$SERVER_LOG_PATH" ] ; then + mkdir -p "$SERVER_LOG_PATH" +fi + +if test -z "$SERVER_HEAP_SIZE" +then + export SERVER_HEAP_SIZE="512M" +fi + +DEBUG_PORT= +if [ "$DEBUG_PORT" ]; +then + export DEBUG_CMD="-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=$DEBUG_PORT" +fi + +if test -z "$SERVER_JAVA_OPTS" +then + export SERVER_JAVA_OPTS="-DserviceName=$SERVER_SUFFIX -Xmx$SERVER_HEAP_SIZE -XX:+UseG1GC -Xloggc:$SERVER_LOG_PATH/$SERVER_SUFFIX.gc $DEBUG_CMD" +fi + +export SERVER_CLASS=com.webank.wedatasphere.dss.scriptis.DSSScriptisServerApplication + +## conf dir +export SERVER_CONF_PATH=$DSS_CONF_DIR:$DSS_CONF_DIR/$SERVER_SUFFIX + +## commons lib +export DSS_COMMONS_LIB="$DSS_HOME/lib/dss-commons" + + +if [ ! -r "$DSS_COMMONS_LIB" ] ; then + echo "dss commons lib not exists $DSS_COMMONS_LIB" + exit 1 +fi + +## server lib +export SERVER_LIB=$DSS_HOME/lib/dss-apps/$SERVER_SUFFIX + + +if [ ! -r "$SERVER_LIB" ] ; then + echo "server lib not exists $SERVER_LIB" + exit 1 +fi + +export SERVER_PID=$DSS_HOME/pid/${SERVER_SUFFIX}.pid +## set class path +export SERVER_CLASS_PATH=$SERVER_CONF_PATH:$DSS_COMMONS_LIB/*:$SERVER_LIB/* + +nohup java $SERVER_JAVA_OPTS -cp $SERVER_CLASS_PATH $SERVER_CLASS 2>&1 > $SERVER_LOG_PATH/$SERVER_SUFFIX.out & +pid=$! + +sleep 2 +if [[ -z "${pid}" ]]; then + echo "server $SERVER_SUFFIX start failed!" + exit 1 +else + echo "server $SERVER_SUFFIX start succeeded!" + echo $pid > $SERVER_PID + sleep 1 +fi +exit 1 diff --git a/sbin/ext/dss-workflow-server b/sbin/ext/dss-workflow-server new file mode 100644 index 0000000000..12b3c0be96 --- /dev/null +++ b/sbin/ext/dss-workflow-server @@ -0,0 +1,86 @@ +#!/bin/bash +# +# description: ecm start cmd +# + +# get log directory +cd `dirname $0` +cd .. +INSTALL_HOME=`pwd` + +# set DSS_HOME +if [ "$DSS_HOME" = "" ]; then + export DSS_HOME=$INSTALL_HOME +fi + +# set DSS_CONF_DIR +if [ "$DSS_CONF_DIR" = "" ]; then + export DSS_CONF_DIR=$DSS_HOME/conf +fi + +SERVER_SUFFIX="dss-workflow-server" +## set log +if [ "$DSS_LOG_DIR" = "" ]; then + export DSS_LOG_DIR="$DSS_HOME/logs" +fi +export SERVER_LOG_PATH=$DSS_LOG_DIR +if [ ! -w "$SERVER_LOG_PATH" ] ; then + mkdir -p "$SERVER_LOG_PATH" +fi + +if test -z "$SERVER_HEAP_SIZE" +then + export SERVER_HEAP_SIZE="512M" +fi + +DEBUG_PORT= +if [ "$DEBUG_PORT" ]; +then + export DEBUG_CMD="-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=$DEBUG_PORT" +fi + +if test -z "$SERVER_JAVA_OPTS" +then + export SERVER_JAVA_OPTS="-DserviceName=$SERVER_SUFFIX -Xmx$SERVER_HEAP_SIZE -XX:+UseG1GC -Xloggc:$SERVER_LOG_PATH/$SERVER_SUFFIX.gc $DEBUG_CMD" +fi + +export SERVER_CLASS=com.webank.wedatasphere.dss.workflow.DSSWorkflowServerApplication + +## conf dir +export SERVER_CONF_PATH=$DSS_CONF_DIR:$DSS_CONF_DIR/$SERVER_SUFFIX + +## commons lib +export DSS_COMMONS_LIB="$DSS_HOME/lib/dss-commons" + + +if [ ! -r "$DSS_COMMONS_LIB" ] ; then + echo "dss commons lib not exists $DSS_COMMONS_LIB" + exit 1 +fi + +## server lib +export SERVER_LIB=$DSS_HOME/lib/dss-orchestrator/$SERVER_SUFFIX + + +if [ ! -r "$SERVER_LIB" ] ; then + echo "server lib not exists $SERVER_LIB" + exit 1 +fi + +export SERVER_PID=$DSS_HOME/pid/${SERVER_SUFFIX}.pid +## set class path +export SERVER_CLASS_PATH=$SERVER_CONF_PATH:$DSS_COMMONS_LIB/*:$SERVER_LIB/* + +nohup java $SERVER_JAVA_OPTS -cp $SERVER_CLASS_PATH $SERVER_CLASS 2>&1 > $SERVER_LOG_PATH/$SERVER_SUFFIX.out & +pid=$! + +sleep 2 +if [[ -z "${pid}" ]]; then + echo "server $SERVER_SUFFIX start failed!" + exit 1 +else + echo "server $SERVER_SUFFIX start succeeded!" + echo $pid > $SERVER_PID + sleep 1 +fi +exit 1 diff --git a/sbin/k8s/dss-apiservice-server.sh b/sbin/k8s/dss-apiservice-server.sh new file mode 100644 index 0000000000..9629b24013 --- /dev/null +++ b/sbin/k8s/dss-apiservice-server.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +SERVER_SUFFIX="dss-apiservice-server" + + +export SERVER_CONF_PATH=/opt/dss/$SERVER_SUFFIX/conf +export SERVER_LOG_PATH=/opt/dss/$SERVER_SUFFIX/logs + +if [ ! -w "$SERVER_LOG_PATH" ] ; then + mkdir -p "$SERVER_LOG_PATH" +fi + +export SERVER_JAVA_OPTS="-DserviceName=$SERVER_SUFFIX -Xmx512M -XX:+UseG1GC -Xloggc:$SERVER_LOG_PATH/linkis.log" + +export SERVER_CLASS=com.webank.wedatasphere.dss.apiservice.DSSApiServiceServerApplication + + +export DSS_COMMONS_LIB=/opt/dss/dss-commons + +export SERVER_LIB=/opt/dss/$SERVER_SUFFIX/lib + + +export SERVER_CLASS_PATH=$SERVER_CONF_PATH:$DSS_COMMONS_LIB/*:$SERVER_LIB/* + +java $SERVER_JAVA_OPTS -cp $SERVER_CLASS_PATH $SERVER_CLASS \ No newline at end of file diff --git a/sbin/k8s/dss-data-api-server.sh b/sbin/k8s/dss-data-api-server.sh new file mode 100644 index 0000000000..ef6e440839 --- /dev/null +++ b/sbin/k8s/dss-data-api-server.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +SERVER_SUFFIX="dss-data-api-server" + + +export SERVER_CONF_PATH=/opt/dss/$SERVER_SUFFIX/conf +export SERVER_LOG_PATH=/opt/dss/$SERVER_SUFFIX/logs + +if [ ! -w "$SERVER_LOG_PATH" ] ; then + mkdir -p "$SERVER_LOG_PATH" +fi + +export SERVER_JAVA_OPTS="-DserviceName=$SERVER_SUFFIX -Xmx2048M -XX:+UseG1GC -Xloggc:$SERVER_LOG_PATH/linkis.log" + +export SERVER_CLASS=com.webank.wedatasphere.dss.data.api.server.DSSDataApiApplication + + +export DSS_COMMONS_LIB=/opt/dss/dss-commons + +export SERVER_LIB=/opt/dss/$SERVER_SUFFIX/lib + + +export SERVER_CLASS_PATH=$SERVER_CONF_PATH:$DSS_COMMONS_LIB/*:$SERVER_LIB/* + +java $SERVER_JAVA_OPTS -cp $SERVER_CLASS_PATH $SERVER_CLASS diff --git a/sbin/k8s/dss-data-governance-server.sh b/sbin/k8s/dss-data-governance-server.sh new file mode 100644 index 0000000000..76aca5370c --- /dev/null +++ b/sbin/k8s/dss-data-governance-server.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +SERVER_SUFFIX="dss-data-governance-server" + + +export SERVER_CONF_PATH=/opt/dss/$SERVER_SUFFIX/conf +export SERVER_LOG_PATH=/opt/dss/$SERVER_SUFFIX/logs + +if [ ! -w "$SERVER_LOG_PATH" ] ; then + mkdir -p "$SERVER_LOG_PATH" +fi + +export SERVER_JAVA_OPTS="-DserviceName=$SERVER_SUFFIX -Xmx2048M -XX:+UseG1GC -Xloggc:$SERVER_LOG_PATH/linkis.log" + +export SERVER_CLASS=com.webank.wedatasphere.dss.data.governance.DSSDataGovernanceApplication + + +export DSS_COMMONS_LIB=/opt/dss/dss-commons + +export SERVER_LIB=/opt/dss/$SERVER_SUFFIX/lib + + +export SERVER_CLASS_PATH=$SERVER_CONF_PATH:$DSS_COMMONS_LIB/*:$SERVER_LIB/* + +java $SERVER_JAVA_OPTS -cp $SERVER_CLASS_PATH $SERVER_CLASS diff --git a/sbin/k8s/dss-flow-execution-server.sh b/sbin/k8s/dss-flow-execution-server.sh new file mode 100644 index 0000000000..f6d70127fa --- /dev/null +++ b/sbin/k8s/dss-flow-execution-server.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +SERVER_SUFFIX="dss-flow-execution-server" + + +export SERVER_CONF_PATH=/opt/dss/$SERVER_SUFFIX/conf +export SERVER_LOG_PATH=/opt/dss/$SERVER_SUFFIX/logs + +if [ ! -w "$SERVER_LOG_PATH" ] ; then + mkdir -p "$SERVER_LOG_PATH" +fi + +export SERVER_JAVA_OPTS="-DserviceName=$SERVER_SUFFIX -Xmx512M -XX:+UseG1GC -Xloggc:$SERVER_LOG_PATH/linkis.log" + +export SERVER_CLASS=com.webank.wedatasphere.dss.flow.execution.entrance.DSSFowExecutionServerApplication + + +export DSS_COMMONS_LIB=/opt/dss/dss-commons + +export SERVER_LIB=/opt/dss/$SERVER_SUFFIX/lib + + +export SERVER_CLASS_PATH=$SERVER_CONF_PATH:$DSS_COMMONS_LIB/*:$SERVER_LIB/* + +java $SERVER_JAVA_OPTS -cp $SERVER_CLASS_PATH $SERVER_CLASS \ No newline at end of file diff --git a/sbin/k8s/dss-guide-server.sh b/sbin/k8s/dss-guide-server.sh new file mode 100644 index 0000000000..3bbff2b77d --- /dev/null +++ b/sbin/k8s/dss-guide-server.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +SERVER_SUFFIX="dss-guide-server" + + +export SERVER_CONF_PATH=/opt/dss/$SERVER_SUFFIX/conf +export SERVER_LOG_PATH=/opt/dss/$SERVER_SUFFIX/logs + +if [ ! -w "$SERVER_LOG_PATH" ] ; then + mkdir -p "$SERVER_LOG_PATH" +fi + +export SERVER_JAVA_OPTS="-DserviceName=$SERVER_SUFFIX -Xmx2048M -XX:+UseG1GC -Xloggc:$SERVER_LOG_PATH/linkis.log" + +export SERVER_CLASS=com.webank.wedatasphere.dss.guide.server.DSSGuideApplication + + +export DSS_COMMONS_LIB=/opt/dss/dss-commons + +export SERVER_LIB=/opt/dss/$SERVER_SUFFIX/lib + + +export SERVER_CLASS_PATH=$SERVER_CONF_PATH:$DSS_COMMONS_LIB/*:$SERVER_LIB/* + +java $SERVER_JAVA_OPTS -cp $SERVER_CLASS_PATH $SERVER_CLASS diff --git a/sbin/k8s/dss-workflow-server.sh b/sbin/k8s/dss-workflow-server.sh new file mode 100644 index 0000000000..ae9a627153 --- /dev/null +++ b/sbin/k8s/dss-workflow-server.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +SERVER_SUFFIX="dss-workflow-server" + + +export SERVER_CONF_PATH=/opt/dss/$SERVER_SUFFIX/conf +export SERVER_LOG_PATH=/opt/dss/$SERVER_SUFFIX/logs + +if [ ! -w "$SERVER_LOG_PATH" ] ; then + mkdir -p "$SERVER_LOG_PATH" +fi + +export SERVER_JAVA_OPTS="-DserviceName=$SERVER_SUFFIX -Xmx512M -XX:+UseG1GC -Xloggc:$SERVER_LOG_PATH/linkis.log" + +export SERVER_CLASS=com.webank.wedatasphere.dss.workflow.DSSWorkflowServerApplication + + +export DSS_COMMONS_LIB=/opt/dss/dss-commons + +export SERVER_LIB=/opt/dss/$SERVER_SUFFIX/lib + + +export SERVER_CLASS_PATH=$SERVER_CONF_PATH:$DSS_COMMONS_LIB/*:$SERVER_LIB/* + +java $SERVER_JAVA_OPTS -cp $SERVER_CLASS_PATH $SERVER_CLASS diff --git a/web/README-DEV.md b/web/README-DEV.md index c75b99752d..6b3ad0dbb4 100644 --- a/web/README-DEV.md +++ b/web/README-DEV.md @@ -32,7 +32,6 @@ - 子应用可以配置自己的layout需要在应用router模块导出配置subAppRoutes - 各应用需要使用iView作为UI库，并提供路由，国际化等配置写入config.json -- 各应用间不要相互直接依赖，确有依赖通过lerna管理 - 可复用组件，资源需要合理放置，packages/shared 共享组件方法，修改需要注意影响 - 各应用路由应以应用名做统一前缀 - 各应用之间需要事件通信，应当在config.json 里声明对应module文件路径 @@ -53,11 +52,7 @@ http://localhost:8080/#/demoHome ``` # 安装依赖 -lerna bootstrap -# 添加依赖通过lerna add 添加 -lerna add [@version] -# 指定组件添加依赖 -lerna add [@version] --@scope=@dataspherestudio/workspace +npm i # 开发启动DSS npm run serve # 运行部分模块子应用，支持通过module组合。如科管版本： diff --git a/web/README.md b/web/README.md new file mode 100644 index 0000000000..f6b4f90e83 --- /dev/null +++ b/web/README.md @@ -0,0 +1,45 @@ +Scriptis +============ + +[![License](https://img.shields.io/badge/license-Apache%202-4EB1BA.svg)](https://www.apache.org/licenses/LICENSE-2.0.html) + +English | [Chinese](docs/zh_CN/README.md) + +## Introduction + +Scriptis is for interactive data analysis with script development(SQL, Pyspark, HiveQL), task submission(Spark, Hive), UDF, function, resource management and intelligent diagnosis. + +![running](docs/en_US/images/readme/running.gif) + +## Features + +* Script editor: Support multi-language, auto-completion, syntax highlighting and SQL syntax error-correction. + +* Computation engines: Based on Linkis, Scriptis connects with multiple computation engines such as Spark, Hive, Python, etc. + +* Runtime functionality: Complete job life cycle display and intelligent diagnosis. + +* Result set: Multiple result sets support, customized result set alias and one-click visualization. + +* Database Services: Functionalities for database management and files(CVS, Excel) import/export to/from tables. + +* Context: UDFs, custom variables and functions management and sharing. + +* Console: Customized settings for engine parameters, task/engine management and resource isolation/display. + + +## QuickStart + +Read the Quick Start [Quick Start](/docs/en_US/ch3/Scriptis_Quick_Start.md) + +## Comparison with similar scheduler systems +![Comparison](/docs/en_US/images/readme/Comparison.png) + +## Community +If you desire immediate response, please kindly raise issues to us or scan the below QR code by WeChat and QQ to join our group: + +![WeChatQQ](/docs/en_US/images/wechatQQ.png) + +## License + +Scriptis is under the Apache 2.0 license. See the [LICENSE]((http://www.apache.org/licenses/LICENSE-2.0)) file for details diff --git a/web/docs/en_US/README.md b/web/docs/en_US/README.md new file mode 100644 index 0000000000..6ab30f1ad5 --- /dev/null +++ b/web/docs/en_US/README.md @@ -0,0 +1,34 @@ +## Introduction + +Scriptis is for interactive data analysis with script development(SQL, Pyspark, HiveQL), task submission(Spark, Hive), UDF, function, resource management and intelligent diagnosis. + +## Features + +* Script editor: Support multi-language, auto-completion, syntax highlighting and SQL syntax error-correction. + +* Computation engines: Based on Linkis, Scriptis connects with multiple computation engines such as Spark, Hive, Python, etc. + +* Runtime functionality: Complete job life cycle display and intelligent diagnosis. + +* Result set: Multiple result sets support, customized result set alias and one-click visualization. + +* Database Services: Functionalities for database management and files(CVS, Excel) import/export to/from tables. + +* Context: UDFs, custom variables and functions management and sharing. + +* Console: Customized settings for engine parameters, task/engine management and resource isolation/display. + + +## Comparison with similar scheduler systems +![Comparison](/docs/en_US/images/readme/Comparison.png) + +## Document +* [Front-end_deployment_documentation](ch1/Front-end_deployment_documentation.md) +* [Compilation](ch2/Compilation.md) +* [Scriptis_Quick_Start](ch3/Scriptis_Quick_Start.md) +* [Scriptis_Manual](ch4/Scriptis_Manual.md) + + +## License + +Scriptis is under the Apache 2.0 license. See the [LICENSE]((http://www.apache.org/licenses/LICENSE-2.0)) file for details \ No newline at end of file diff --git a/web/docs/en_US/SUMMARY.md b/web/docs/en_US/SUMMARY.md new file mode 100644 index 0000000000..2b829a4a8a --- /dev/null +++ b/web/docs/en_US/SUMMARY.md @@ -0,0 +1,10 @@ +# Summary +* [Introduction](README.md) +* [ch1 Front-end_deployment_documentation]() + * [Front-end_deployment_documentation](ch1/Front-end_deployment_documentation.md) +* [ch2 Compilation ]() + * [Compilation](ch2/Compilation.md) +* [ch3 Scriptis_Quick_Start]() + * [Scriptis_Quick_Start](ch3/Scriptis_Quick_Start.md) +* [ch4 Scriptis_Manual ]() + * [Scriptis_Manual](ch4/Scriptis_Manual.md) diff --git a/web/docs/en_US/ch1/Front-end_deployment_documentation.md b/web/docs/en_US/ch1/Front-end_deployment_documentation.md new file mode 100644 index 0000000000..5fa0ac685a --- /dev/null +++ b/web/docs/en_US/ch1/Front-end_deployment_documentation.md @@ -0,0 +1,114 @@ +Scriptis is a data analysis tool based on Linkis. Before deploying Scriptis, you need to deploy Linkis first. For the Linkis deploy document, see: [Linkis DeployDoc](https://github.com/WeBankFinTech/Linkis/blob/master/docs/en_US/ch1/deploy.md) + +## 1 Preparation + +1. Select the corresponding installation package to download. + +2. Unzip the downloaded installation package in the installation directory: unzip wedatasphere-scriptis-0.7.0-dist.zip. + +## 2 Deploy + +​ There are two deployment methods, automated and manual deployment. + +### 2.1 Automated deployment + +Go to the frontend directory ```wedatasphere-scriptis``` and edit ```vi config.sh ``` to change the interface address of the frontend and backend port. backend port interface address is the gateway address of linkis. + +### (3) Modify and save the configuration file created above + +``` +# Configuring front-end ports +scriptis_port="8088" + +# URL of the backend linkis gateway +linkis_url="http://localhost:20401" + +# Scriptis ip address +scriptis_ipaddr=$(ip addr | awk '/^[0-9]+: / {}; /inet.*global/ {print gensub(/(.*)\/(.*)/, "\\1", "g", $2)}') +``` + +After the modification, run the following command in the directory: ```sudo sh install.sh > install.log 2>&1``` + +Next, you can access ```http://scriptis_ipaddr:scriptis_port``` directly via Chrome, scriptis_port is the port configured in config.sh and scriptis_ipaddr is the IP of the machine that used for installation. + +If encounter access failure, please check install.log and find out the errors. + +### 2.2 Manual deployment + +1. Install Nginx: ```sudo yum install nginx -y``` + +2. Modify the configuration file:```sudo vi /etc/nginx/conf.d/scriptis.conf``` + + Add the following: + +``` +server { + listen 8080;# Access Port + server_name localhost; + #charset koi8-r; + #access_log /var/log/nginx/host.access.log main; + location / { + root /appcom/Install/scriptis/ROOT; # directory where package decompressed + #in the fronted + index index.html index.html; + } + location /ws {#webSocket configure spport + proxy_pass http://192.168.xxx.xxx:9001;#IP port of the linkis gateway service + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + } + location /api { + proxy_pass http://192.168.xxx.xxx:9001;#IP port of the linkis gateway service + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header x_real_ipP $remote_addr; + proxy_set_header remote_addr $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_http_version 1.1; + proxy_connect_timeout 4s; + proxy_read_timeout 600s; + proxy_send_timeout 12s; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection upgrade; + } + #error_page 404 /404.html; + # redirect server error pages to the static page /50x.html + # + error_page 500 502 503 504 /50x.html; + location = /50x.html { + root /usr/share/nginx/html; + } + } +``` + +3. Copy the frontend package to the corresponding directory: ```/appcom/Install/scriptis/ROOT; # directory where package decompressed in the frontend``` +4. Start service: ```sudo systemctl restart nginx``` +5. You can directly access ```http://nginx_ip:nginx_port``` via Chrome after execution. + +## 3 FAQs + +(1) limitations on the size of files that being uploaded + +``` +sudo vi /etc/nginx/nginx.conf +``` + +Change the uploading size: + +``` +client_max_body_size 200m +``` + +(2) Interface timeout + +``` +sudo vi /etc/nginx/conf.d/scriptis.conf +``` + +Change the interface timeout: + +``` +proxy_read_timeout 600s +``` + diff --git a/web/docs/en_US/ch2/Compilation.md b/web/docs/en_US/ch2/Compilation.md new file mode 100644 index 0000000000..1f22f0a29e --- /dev/null +++ b/web/docs/en_US/ch2/Compilation.md @@ -0,0 +1,102 @@ +# Compilation + +## Getting Started + +### Prerequisites + +Install Node.js on your computer. Download Link: [http://nodejs.cn/download/](http://nodejs.cn/download/). Recommend using the latest stable version. + +**Only do this step at the first time.** + +### Installation + +Run the following commands in terminal: + +``` +git clone https://github.com/WeBankFinTech/Scriptis.git +cd DataSphereStudio/web +npm install +``` + + Commands explanation: + +1. Pull remote repository to local: `git clone https://github.com/WeBankFinTech/Scriptis.git` + +2. Change to the root directory of the project: `cd DataSphereStudio/web` + +3. Install all dependencies required for the project: `npm install` + +**Only do this step at the first time.** + +### Configuration + +You need to make some configurations in your code, such as port address of backend server and socket address of backend server in .env.development file in root directory. + +``` +// Port address of backend server +VUE_APP_MN_CONFIG_PREFIX=http://yourIp:yourPort/yourPath +// Socket address +VUE_APP_MN_CONFIG_SOCKET=/yourSocketPath +``` + +You can refer to the official documentation of vue-cli for detailed explanation. [Modes and environment variables](https://cli.vuejs.org/guide/mode-and-env.html#modes) + +### Building project + +You can run the following command in terminal to build the project: + +``` +npm run build +``` + +A folder named "dist" would appear in your project's root directory if the command has run successfully and you can directly put "dist" to your static server. + +### How to run + +You would need to run the following command in terminal if you want to run project on your local browser and see corresponding effects after making changes to the code. + +``` +npm run serve +``` + +Access the application in browser (Chrome recommended) via link: [http://localhost:8080/](http://localhost:8080/) . + +Changes you make to the code would dynamically reflect on the +effects shown on browser when using the method described above to run project. + +**Notes: Since frontend and backend are developed separately, when running on local browser you need to allow cross domain access in order to access the port of backend server.** + +e.g. Chrome browser: + +Configuration in Windows: + +1. Close all browser windows. + +2. Create a shortcut of chrome, right-click to choose "properties" , then go to "Shortcut" tab find "Target" and add`--args --disable-web-security --user-data-dir=C:\MyChromeDevUserData` to it . +3. Use shortcut to open the browser. + +Configuration in MacOS: + +Run the following command. (You need to replace "yourname" in the path. If it's not working, check the path of MyChromeDevUserData on your machine and copy its path to the place right after "--user-data-dir=") + +``` +open -n /Applications/Google\ Chrome.app/ --args --disable-web-security --user-data-dir=/Users/yourname/MyChromeDevUserData/ +``` + +### FAQ + +#### Failed installation when running npm install + +Try to use Taobao npm mirror: + +``` +npm install -g cnpm --registry=https://registry.npm.taobao.org +``` + +Next, run the following command instead of npm install: + +``` +cnpm install +``` + +Note that you can still use `npm run serve` and `npm run build` to run and build project. \ No newline at end of file diff --git a/web/docs/en_US/ch3/Scriptis_Quick_Start.md b/web/docs/en_US/ch3/Scriptis_Quick_Start.md new file mode 100644 index 0000000000..4d1dcf207b --- /dev/null +++ b/web/docs/en_US/ch3/Scriptis_Quick_Start.md @@ -0,0 +1,35 @@ + +## Quick Login +     To be convenient to users，the default login username is consistent with the Linux user which is used for deployment. For example, if user 'hadoop' is used for deployment linkis, the user can use 'hadoop' as the username, and 'hadoop' as the password to login. Note that the password is the same with the username. +First enter the address of the front-end container: 192.168.xx.xx:8888, and then enter the username/password:hadoop/hadoop + +![01](../images/ch3/01.png) +__Hint：__ Other users must use their configured LDAP passwords to login. + +## Queue Configuration +     scriptis is backed by Spark running on yarn. To use Spark properly, users must declare an available queue in their configurations. Click Control Panel --> Settings --> Yarn Queue, and then enter the name of the yarn queue to which you have the permission. +![02](../images/ch3/02.png) + +## Create New Script +      +Finished with queue configuration, you may try to create a SQL script snippet for data query. Click WorkSpace --> Personal Directory, and then right-click to create a new SQL script. +1. Create +![03](../images/ch3/03.png) +2. Create SQL script +![04](../images/ch3/04.png) +3. Type SQL code snippet +![05](../images/ch3/05.png) + + +## Click & Run +      +Click the 'Run' button above the script area to execute the script. The status of the execution can be monitored by logs. +![01](../images/ch3/06.png) + +## Result Display +      +For the first-time execution, please wait for a slightly long time after clicking the 'Run' button, as the background service is firing a new Engine for you. Result sets will be displayed after successful executions. Users can download or export the result sets. +![01](../images/ch3/07.png) + +For more detailed manual please see：[Scriptis Manual](../ch4/Scriptis_Manual.md) + diff --git a/web/docs/en_US/ch4/Scriptis_Manual.md b/web/docs/en_US/ch4/Scriptis_Manual.md new file mode 100644 index 0000000000..af000bb51f --- /dev/null +++ b/web/docs/en_US/ch4/Scriptis_Manual.md @@ -0,0 +1,183 @@ +## 1 Functions + +Scriptis mainly has the following features: + +1. Workspace: Used to store scripts, data and log files. Support creating sql, hive, scala, python, pyspark scripts. +2. Dataset Module: Display datasets and tables based on user permission. +3. UDF Module: UDF are functions that can be used in sql and hql scripts. This module has the capability of managing, sharing and loading functions. +4. Function Module: Composed of personal, system and shared user-define functions. These functions can be used in python, pyspark, scala scripts. +5. HDFS Module: Personal directory of user's HDFS (distributed filesystem), used to store large files. +6. Script Module: Capable of editing, running and stopping scripts. Customizing variable configurations and shortcut keys are also supported. +7. Results: Include displaying, downloading and exporting the results. +8. Script History: Displaying the running history of the scripts. +9. Console: Users can access settings, global history, resource manager, global variables and FAQs here. +10. Bottom right pop up box: Include task, engine and queue managers. + +These functions are described in detail below. + +## 2 Workspace + +Workspace is a file directory that a user have full permission to. At here, a user could do various operations such as managing files. The recommending directory structure is: script, data, log and res, since it is quite clear and thus easy for users to check and manage. The major functions of workspace are listed below: + +1. Right-clicks on workspace, a user can select copying path, creating a directory, creating a script or refreshing. + +![ide05](../images/ch4/ide05.png) + +2. Locates on the top of this module, there is a search box for quick searching. + +![ide06](../images/ch4/ide06.png) + +3. Support creating following kinds of scripts: + +- sql: Correspond to SparkSQL in Spark engine, syntax guide: https://docs.databricks.com/spark/latest/spark-sql/index.html + +- hql: Correspond to Hive engine: syntax guide: https://cwiki.apache.org/confluence/display/Hive/LanguageManual + +- Scala: Correspond to scala in Spark engine, syntax guide: https://docs.databricks.com/spark/latest/dataframes-datasets/introduction-to-dataframes-scala.html + +- JDBC: sql standard syntax, not supported yet. + +- Python: Standalone python engine, compatible with python + +- PythonSpark: Correspond to python in Spark engine, syntax guide: https://docs.databricks.com/spark/latest/dataframes-datasets/introduction-to-dataframes-python.html + + ![ide07](../images/ch4/ide07.png) + +4. Right-click on script folder and files under it, users can choose to rename it, delete it, open it on the right side or export to hive (csv, txt, excel files) and hdfs. + + ![ide08](../images/ch4/ide08.png) + +## 3 Dataset Module + +Dataset module has the following functions. + +1. Get the information of datasets, tables and fields. + +![ide09](../images/ch4/ide09.png) + +2. Right-click on a table and select query table option can quickly generate a temporary hive script for data lookup. + +![ide10](../images/ch4/ide10.png) + +3. Right-click on a table and select describe table option can display the detailed information of this table and its corresponding fields and partitions. + +![ide11](../images/ch4/ide11.png) + +4. Right-click on a table, select export table option can generate a corresponding csv or excel file. + +![ide12](../images/ch4/ide12.png) + +## 4 UDF Module + +This module not only makes it easy for user to classify and display UDF, but also enables users to manage and share UDF. The major functions are listed below: + +#### 4.1 Displaying and loading functions + +Default top-level directory: + +- BDAP function: Provided by platform and can be used in sql, pyspark, scala and hive (written with sql) scripts. + +- System function: Functions that system provides and loaded by default. Can be used in sql, pyspark, scala and hive (written with sql) scripts. + +- Individual function: Self-define functions, include general functions and Spark exclusive functions. + +- Sharing function: Functions created by administrator and then shared to other users. + +- Apart from system functions, other types of functions must be loaded before using and a user must kill the started session after checking the functions. + + In addition, if a function is checked and loaded, it would correspondingly shown in auto-complete options. + + ![ide13](../images/ch4/ide13.png) + +#### 4.2 Creating a new UDF + +It is quite easy to create a new UDF as long as you've finished the code. The steps are as follows: + +1. To create a general UDF, a user needs to compile the corresponding Jar package first. General means either hql in Hive or sql in Spark applies here. + +2. To create a Spark exclusive UDF, a user needs to create a corresponding python or scala script. Besides, to ensure the correctness, it is better to test the scripts. + +3. Add this UDF to Scriptis: + + General UDF: Choose general then select the path in workspace for its Jar package. Next, fill in the full class path of UDF and add formatting as well as description: + +![ide14](../images/ch4/ide14.png) + + Spark exclusive UDF -- written in scala: Check Spark then select the corresponding scala script and fill in the registration format. (Function name in script): + +![ide15](../images/ch4/ide15.png) + + Spark exclusive UDF -- written in scala: Check Spark then select the corresponding python script and fill in the registration format. (Function name in script): + +![ide16](../images/ch4/ide16.png) + +#### 4.3 Way to write Scala and Python functions: + +For a PythonUDF, a user only needs to define a function, and the scripts have to correspond with this function. + +``` +def hello(id): + return str(id) + ":hello" +``` + +The way to create a ScalaUDF is quite similar to creating a Python UDF, a user only needs to define a function: + +``` +def helloWord(str: String): String = "hello, " + str +``` + +Note: Python UDF and Scala UDF can only applied in scripts that corresponding to the Spark engine. + +## 5 Function Module + +Function module is similar to UDF module, the only difference between them is that one is UDF and the other is self-defined function. Also notes that, functions defined by python can only be used in python and pyspark. Similarly, functions defined by scala can only be used in scala. + +## 6 Script Module + +The functions of this module are mainly integrated in the script edit box: + +1. Script editing: Support basic keyword highlighting, code formatting, code merging, auto-completion and shortcuts etc. + +2. Running and stopping: Users can choose to run only a segment of code or the entire script. By clicking stop button, users can terminate the running scripts whenever they want. + +3. Script edit box has configuration options for defining user-define functions that take effects within the script. + +![ide18](../images/ch4/ide18.png) + +## 7 Results + +This module has the following functions: + +1. For now, it supports showing results in a table, clicking the header to sort, double-clicking to copy the field name and all these functions are restricted to showing up to 5000 lines of records. More functions would be supported in future, such as displaying the selected columns and filed types. + +2. Visual analysis: Click on visual analysis button to visualize the result through VSBI. (Soon to be released) + +3. Downloading: Users can directly download the results as csv and excel files to local browser. Only support downloading 5000 lines for now. + +4. Exporting: Results can be exported to the workspace (shared directory of BDAP) in either csv or excel format and would not be restricted to 5000 lines if you choose full export at first. To use full export, add a comment in front of sql: `--set wds.linkis.engine.no.limit.allow=true` + +5. Go to Console--Configuration--Pipeline--Import and Export settings--Result export type to choose whether export results in csv format or excel format. + +## 8 Script History + +Script history shows all the running information of the script. A user can quickly find the log and running results of a script that was run before and therefore, avoid running the same script repeatedly. + +![ide23](../images/ch4/ide23.png) + +## 9 Console + +Console has the following functions: + +1. Settings: Include general settings (such as setting up queues) and data development related engine settings: spark, hive, python, pipeline, etc. + +![ide25](../images/ch4/ide25.png) + +2. Global variables: A global variable is a custom variable that can be applied to all scripts. If its name is same as the name of a variable in a script, that variable would take effect. + +3. Other functions: Global history, resource manager, FAQs. + +## 10 Bottom right pop up box + +Similar to the Windows task manager, users can quickly view and manage tasks, engines and queue resources here. + +![ide24](../images/ch4/ide24.png) \ No newline at end of file diff --git a/web/docs/en_US/images/ch3/01.png b/web/docs/en_US/images/ch3/01.png new file mode 100644 index 0000000000..f45529eb30 Binary files /dev/null and b/web/docs/en_US/images/ch3/01.png differ diff --git a/web/docs/en_US/images/ch3/02.png b/web/docs/en_US/images/ch3/02.png new file mode 100644 index 0000000000..65c7f14cb1 Binary files /dev/null and b/web/docs/en_US/images/ch3/02.png differ diff --git a/web/docs/en_US/images/ch3/03.png b/web/docs/en_US/images/ch3/03.png new file mode 100644 index 0000000000..b1fa65b15f Binary files /dev/null and b/web/docs/en_US/images/ch3/03.png differ diff --git a/web/docs/en_US/images/ch3/04.png b/web/docs/en_US/images/ch3/04.png new file mode 100644 index 0000000000..a768f716d5 Binary files /dev/null and b/web/docs/en_US/images/ch3/04.png differ diff --git a/web/docs/en_US/images/ch3/05.png b/web/docs/en_US/images/ch3/05.png new file mode 100644 index 0000000000..3909c97bf1 Binary files /dev/null and b/web/docs/en_US/images/ch3/05.png differ diff --git a/web/docs/en_US/images/ch3/06.png b/web/docs/en_US/images/ch3/06.png new file mode 100644 index 0000000000..8297ce05cd Binary files /dev/null and b/web/docs/en_US/images/ch3/06.png differ diff --git a/web/docs/en_US/images/ch3/07.png b/web/docs/en_US/images/ch3/07.png new file mode 100644 index 0000000000..065021bfc9 Binary files /dev/null and b/web/docs/en_US/images/ch3/07.png differ diff --git a/web/docs/en_US/images/ch4/ide05.png b/web/docs/en_US/images/ch4/ide05.png new file mode 100644 index 0000000000..10f3d91a31 Binary files /dev/null and b/web/docs/en_US/images/ch4/ide05.png differ diff --git a/web/docs/en_US/images/ch4/ide06.png b/web/docs/en_US/images/ch4/ide06.png new file mode 100644 index 0000000000..1fb5b2cd66 Binary files /dev/null and b/web/docs/en_US/images/ch4/ide06.png differ diff --git a/web/docs/en_US/images/ch4/ide07.png b/web/docs/en_US/images/ch4/ide07.png new file mode 100644 index 0000000000..fa414ab054 Binary files /dev/null and b/web/docs/en_US/images/ch4/ide07.png differ diff --git a/web/docs/en_US/images/ch4/ide08.png b/web/docs/en_US/images/ch4/ide08.png new file mode 100644 index 0000000000..5b2321f7c7 Binary files /dev/null and b/web/docs/en_US/images/ch4/ide08.png differ diff --git a/web/docs/en_US/images/ch4/ide09.png b/web/docs/en_US/images/ch4/ide09.png new file mode 100644 index 0000000000..47dc296908 Binary files /dev/null and b/web/docs/en_US/images/ch4/ide09.png differ diff --git a/web/docs/en_US/images/ch4/ide10.png b/web/docs/en_US/images/ch4/ide10.png new file mode 100644 index 0000000000..4141452a57 Binary files /dev/null and b/web/docs/en_US/images/ch4/ide10.png differ diff --git a/web/docs/en_US/images/ch4/ide11.png b/web/docs/en_US/images/ch4/ide11.png new file mode 100644 index 0000000000..d4fb1ec8ee Binary files /dev/null and b/web/docs/en_US/images/ch4/ide11.png differ diff --git a/web/docs/en_US/images/ch4/ide12.png b/web/docs/en_US/images/ch4/ide12.png new file mode 100644 index 0000000000..a615495ae4 Binary files /dev/null and b/web/docs/en_US/images/ch4/ide12.png differ diff --git a/web/docs/en_US/images/ch4/ide13.png b/web/docs/en_US/images/ch4/ide13.png new file mode 100644 index 0000000000..ff4b6d3c07 Binary files /dev/null and b/web/docs/en_US/images/ch4/ide13.png differ diff --git a/web/docs/en_US/images/ch4/ide14.png b/web/docs/en_US/images/ch4/ide14.png new file mode 100644 index 0000000000..9ca07e562b Binary files /dev/null and b/web/docs/en_US/images/ch4/ide14.png differ diff --git a/web/docs/en_US/images/ch4/ide15.png b/web/docs/en_US/images/ch4/ide15.png new file mode 100644 index 0000000000..740027f135 Binary files /dev/null and b/web/docs/en_US/images/ch4/ide15.png differ diff --git a/web/docs/en_US/images/ch4/ide16.png b/web/docs/en_US/images/ch4/ide16.png new file mode 100644 index 0000000000..a4a11ada2d Binary files /dev/null and b/web/docs/en_US/images/ch4/ide16.png differ diff --git a/web/docs/en_US/images/ch4/ide17.png b/web/docs/en_US/images/ch4/ide17.png new file mode 100644 index 0000000000..02576e3f94 Binary files /dev/null and b/web/docs/en_US/images/ch4/ide17.png differ diff --git a/web/docs/en_US/images/ch4/ide18.png b/web/docs/en_US/images/ch4/ide18.png new file mode 100644 index 0000000000..a69059d21e Binary files /dev/null and b/web/docs/en_US/images/ch4/ide18.png differ diff --git a/web/docs/en_US/images/ch4/ide19.png b/web/docs/en_US/images/ch4/ide19.png new file mode 100644 index 0000000000..49359d5465 Binary files /dev/null and b/web/docs/en_US/images/ch4/ide19.png differ diff --git a/web/docs/en_US/images/ch4/ide20.png b/web/docs/en_US/images/ch4/ide20.png new file mode 100644 index 0000000000..12b7caf0f4 Binary files /dev/null and b/web/docs/en_US/images/ch4/ide20.png differ diff --git a/web/docs/en_US/images/ch4/ide21.png b/web/docs/en_US/images/ch4/ide21.png new file mode 100644 index 0000000000..74bafffc23 Binary files /dev/null and b/web/docs/en_US/images/ch4/ide21.png differ diff --git a/web/docs/en_US/images/ch4/ide22.png b/web/docs/en_US/images/ch4/ide22.png new file mode 100644 index 0000000000..a767e9a211 Binary files /dev/null and b/web/docs/en_US/images/ch4/ide22.png differ diff --git a/web/docs/en_US/images/ch4/ide23.png b/web/docs/en_US/images/ch4/ide23.png new file mode 100644 index 0000000000..bfb5eef3dd Binary files /dev/null and b/web/docs/en_US/images/ch4/ide23.png differ diff --git a/web/docs/en_US/images/ch4/ide24.png b/web/docs/en_US/images/ch4/ide24.png new file mode 100644 index 0000000000..19c922ecc3 Binary files /dev/null and b/web/docs/en_US/images/ch4/ide24.png differ diff --git a/web/docs/en_US/images/ch4/ide25.png b/web/docs/en_US/images/ch4/ide25.png new file mode 100644 index 0000000000..be473089c4 Binary files /dev/null and b/web/docs/en_US/images/ch4/ide25.png differ diff --git a/web/docs/en_US/images/readme/Comparison.png b/web/docs/en_US/images/readme/Comparison.png new file mode 100644 index 0000000000..df3144cc61 Binary files /dev/null and b/web/docs/en_US/images/readme/Comparison.png differ diff --git a/web/docs/en_US/images/readme/cs.gif b/web/docs/en_US/images/readme/cs.gif new file mode 100644 index 0000000000..d735e1ba6c Binary files /dev/null and b/web/docs/en_US/images/readme/cs.gif differ diff --git a/web/docs/en_US/images/readme/datasource.gif b/web/docs/en_US/images/readme/datasource.gif new file mode 100644 index 0000000000..555a208e35 Binary files /dev/null and b/web/docs/en_US/images/readme/datasource.gif differ diff --git a/web/docs/en_US/images/readme/results.gif b/web/docs/en_US/images/readme/results.gif new file mode 100644 index 0000000000..cfef7cebb7 Binary files /dev/null and b/web/docs/en_US/images/readme/results.gif differ diff --git a/web/docs/en_US/images/readme/running.gif b/web/docs/en_US/images/readme/running.gif new file mode 100644 index 0000000000..9d310bc880 Binary files /dev/null and b/web/docs/en_US/images/readme/running.gif differ diff --git a/web/docs/en_US/images/readme/script.gif b/web/docs/en_US/images/readme/script.gif new file mode 100644 index 0000000000..39b729d0a3 Binary files /dev/null and b/web/docs/en_US/images/readme/script.gif differ diff --git a/web/docs/en_US/images/readme/setup.gif b/web/docs/en_US/images/readme/setup.gif new file mode 100644 index 0000000000..7515448fba Binary files /dev/null and b/web/docs/en_US/images/readme/setup.gif differ diff --git "a/web/docs/en_US/images/readme/\304\267\303\224\304\205\304\214.png" "b/web/docs/en_US/images/readme/\304\267\303\224\304\205\304\214.png" new file mode 100644 index 0000000000..d32188e5f2 Binary files /dev/null and "b/web/docs/en_US/images/readme/\304\267\303\224\304\205\304\214.png" differ diff --git a/web/docs/en_US/images/wechat.jpg b/web/docs/en_US/images/wechat.jpg new file mode 100644 index 0000000000..1e0e913e76 Binary files /dev/null and b/web/docs/en_US/images/wechat.jpg differ diff --git a/web/docs/en_US/images/wechatQQ.png b/web/docs/en_US/images/wechatQQ.png new file mode 100644 index 0000000000..01ec041d7f Binary files /dev/null and b/web/docs/en_US/images/wechatQQ.png differ diff --git a/web/docs/zh_CN/README.md b/web/docs/zh_CN/README.md new file mode 100644 index 0000000000..5567745a3f --- /dev/null +++ b/web/docs/zh_CN/README.md @@ -0,0 +1,56 @@ +Scriptis +============ + +[![License](https://img.shields.io/badge/license-Apache%202-4EB1BA.svg)](https://www.apache.org/licenses/LICENSE-2.0.html) + +[English](../../README.md) | Chinese + +## 引言： + +Scriptis是一款支持在线写SQL、Pyspark、HiveQL等脚本，提交给**Linkis**（Linkis是什么？[点我了解](https://github.com/WeBankFinTech/Linkis/blob/master/docs/zh_CN/README.md)）执行的数据分析Web工具，且支持UDF、函数、资源管控和智能诊断等企业级特性。 + +---- + +## 核心特点： + +**脚本编辑**：多语言、自动补全、语法高亮、SQL语法纠错； + +![脚本编辑](images/readme/script.gif) + +**计算引擎**：基于Linkis对接多计算引擎：Spark、Hive、TiSpark等； + + +**运行时功能**：Job全生命周期展示、错误码； + +![运行时功能](images/readme/running.gif) + +**上下文**：UDF和函数管理、函数共享、自定义变量； + +![上下文](images/readme/cs.gif) + +**结果集**：多结果集支持、结果集别名自定义、结果集一键发布到报表系统； + +![结果集](images/readme/results.gif) + +**数据库**：数据库管理功能、文件（CSV/Excel）快速导入导出到表； + +![数据库](images/readme/datasource.gif) + +**管理台**：多租户资源管控、引擎参数个性化配置、任务和会话管理。 + +![管理台](images/readme/setup.gif) +---- + +### 同类系统对比： +![对比图](images/readme/对比.png) + +## 社区 +如果您想得到最快的响应，请给我们提issue，或者您也可以扫码进群： + +![WeChatQQ](/docs/en_US/images/wechatQQ.png) + +### 文档： +* [部署手册](ch1/前台部署文档.md) +* [前台编译手册](ch2/编译文档.md) +* [快速使用手册](ch3/scriptis快速使用文档.md) +* [使用手册](ch4/Scriptis使用手册.md) diff --git a/web/docs/zh_CN/SUMMARY.md b/web/docs/zh_CN/SUMMARY.md new file mode 100644 index 0000000000..b19f779aa9 --- /dev/null +++ b/web/docs/zh_CN/SUMMARY.md @@ -0,0 +1,11 @@ +# Summary + +* [Introduction](README.md) +* [第一章 前台部署]() + * [前台部署手册](ch1/前台部署文档.md) +* [第二章 前台编译 ]() + * [前台编译手册](ch2/编译文档.md) +* [第三章 快速使用手册 ]() + * [scriptis快速使用手册](ch3/scriptis快速使用文档.md) +* [第四章 使用手册 ]() + * [scriptis快速使用手册](ch4/Scriptis使用手册.md) \ No newline at end of file diff --git "a/web/docs/zh_CN/ch1/\345\211\215\345\217\260\351\203\250\347\275\262\346\226\207\346\241\243.md" "b/web/docs/zh_CN/ch1/\345\211\215\345\217\260\351\203\250\347\275\262\346\226\207\346\241\243.md" new file mode 100644 index 0000000000..cd5a68d5d0 --- /dev/null +++ "b/web/docs/zh_CN/ch1/\345\211\215\345\217\260\351\203\250\347\275\262\346\226\207\346\241\243.md" @@ -0,0 +1,109 @@ +Scriptis是基于Linkis开发的数据分析工具，部署Scriptis前需要先将Linkis进行部署，Linkis的部署手册见：[Linkis部署手册](https://github.com/WeBankFinTech/Linkis/blob/master/docs/zh_CN/ch1/deploy.md) + +## 1、准备工作 + +1. 点击release 选择对应的安装包进行下载 +2. 将下载下来的安装包在安装目录进行解压：unzip wedatasphere-scriptis-0.7.0-dist.zip + +## 2、部署 +    分为两种部署方式，自动化部署和手动部署 + +### 2.1 自动化部署 +    进入前端目录```wedatasphere-scriptis``` 在该目录下编辑 ```vi config.sh ``` +更改前端端口和后端接口地址,后端接口地址为linkis的gateway地址 +```$xslt +# Configuring front-end ports +scriptis_port="8088" + +# URL of the backend linkis gateway +linkis_url="http://localhost:20401" + +# Scriptis ip address +scriptis_ipaddr=$(ip addr | awk '/^[0-9]+: / {}; /inet.*global/ {print gensub(/(.*)\/(.*)/, "\\1", "g", $2)}') +``` + +修改完后在该目录下执行：```sudo sh install.sh > install.log 2>&1``` + +执行完后可以直接通过在谷歌浏览器访问：```http://scriptis_ipaddr:scriptis_port``` 其中scriptis_port为config.sh里面配置的端口，scriptis_ipaddr为安装机器的IP + +如果访问失败：可以通过查看 install.log的日志查看哪一步出错 + +### 2.2 手动部署 +1.安装Nginx:```sudo yum install nginx -y``` + +2.修改配置文件：sudo vi /etc/nginx/conf.d/scriptis.conf +添加如下内容： +``` +server { + listen 8080;# 访问端口 + server_name localhost; + #charset koi8-r; + #access_log /var/log/nginx/host.access.log main; + location / { + root /appcom/Install/scriptis/ROOT; # 前端包解压的目录 + index index.html index.html; + } + location /ws {#webSocket配置支持 + proxy_pass http://192.168.xxx.xxx:9001;#linkis-gateway服务的ip端口 + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + } + location /api { + proxy_pass http://192.168.xxx.xxx:9001; # linkis-gateway服务的ip端口 + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header x_real_ipP $remote_addr; + proxy_set_header remote_addr $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_http_version 1.1; + proxy_connect_timeout 4s; + proxy_read_timeout 600s; + proxy_send_timeout 12s; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection upgrade; + } + #error_page 404 /404.html; + # redirect server error pages to the static page /50x.html + # + error_page 500 502 503 504 /50x.html; + location = /50x.html { + root /usr/share/nginx/html; + } + } + +``` + +3.将前端包拷贝到对应的目录:```/appcom/Install/scriptis/ROOT; # 前端包解压的目录 ``` + +4.启动服务```sudo systemctl restart nginx``` + +5.执行完后可以直接通过在谷歌浏览器访问：```http://nginx_ip:nginx_port``` + +## 3、常见问题 + +(1)上传文件大小限制 + +``` +sudo vi /etc/nginx/nginx.conf +``` + +更改上传大小 + +``` +client_max_body_size 200m +``` + + (2)接口超时 + +``` +sudo vi /etc/nginx/conf.d/scriptis.conf +``` + + +更改接口超时时间 + +``` +proxy_read_timeout 600s +``` + diff --git "a/web/docs/zh_CN/ch2/\302\261\327\220\327\202\327\233\326\276\326\264\302\265\302\265.md" "b/web/docs/zh_CN/ch2/\302\261\327\220\327\202\327\233\326\276\326\264\302\265\302\265.md" new file mode 100644 index 0000000000..ee447703af --- /dev/null +++ "b/web/docs/zh_CN/ch2/\302\261\327\220\327\202\327\233\326\276\326\264\302\265\302\265.md" @@ -0,0 +1,86 @@ +# 编译文档中文版 + +## 启动流程 + +### 一、安装Node.js +将Node.js下载到电脑本地，安装即可。下载地址：[http://nodejs.cn/download/](http://nodejs.cn/download/) （建议使用最新的稳定版本） +**该步骤仅第一次使用时需要执行。** + +### 二、安装项目 +在终端命令行中执行以下指令： + +``` +git clone ${ipAddress} +cd DataSphereStudio/web +npm install +``` + +指令简介： +1. 将项目包从远程仓库拉取到电脑本地：git clone ${ipAddress} +2. 进入项目包根目录：cd DataSphereStudio/web +3. 安装项目所需依赖：npm install + +**该步骤仅第一次使用时需要执行。** + +### 三、配置 +您需要在代码中进行一些配置，如后端接口地址，后端socket地址等，如根目录下的.env.development文件： + +``` +// 后端接口地址 +VUE_APP_MN_CONFIG_PREFIX=http://yourIp:yourPort/yourPath +// 后端socket地址 +VUE_APP_MN_CONFIG_SOCKET=/yourSocketPath +``` + +配置的具体解释可参考vue-cli官方文档：[环境变量和模式](https://cli.vuejs.org/zh/guide/mode-and-env.html#%E7%8E%AF%E5%A2%83%E5%8F%98%E9%87%8F%E5%92%8C%E6%A8%A1%E5%BC%8F) + +### 打包项目 +您可以通过在终端命令行执行以下指令对项目进行打包，生成压缩后的代码： + +``` +npm run build +``` + +该指令成功执行后，项目根目录下会出现一个名叫 “dist” 的文件夹，该文件夹即为打包好的代码。您可以直接将该文件夹放进您的静态服务器中。 + +### 运行项目 +如果您想在本地浏览器上运行该项目并且改动代码查看效果，需要在终端命令行中执行以下指令： + +``` +npm run serve +``` + +在浏览器中（建议Chrome浏览器）通过链接访问应用：[http://localhost:8080/](http://localhost:8080/) . +当您使用该方式运行项目时，您对代码的改动产生的效果，会动态体现在浏览器上。 + +**注意：因为项目采用前后端分离开发，所以在本地浏览器上运行时，需要对浏览器进行设置跨域才能访问后端接口：** + +比如chrome浏览器： +windows系统下的配置方式： +1. 关闭所有的chrome浏览器。 +2. 新建一个chrome快捷方式，右键“属性”，“快捷方式”选项卡里选择“目标”，添加  --args --disable-web-security --user-data-dir=C:\MyChromeDevUserData +3. 通过快捷方式打开chrome浏览器 +mac系统下的配置方式： +在终端命令行执行以下命令(需要替换路径中的yourname，若还不生效请检查您机器上MyChromeDevUserData文件夹的位置并将路径复制到下面指令的“--user-data-dir=”后面) + +``` +open -n /Applications/Google\ Chrome.app/ --args --disable-web-security --user-data-dir=/Users/yourname/MyChromeDevUserData/ +``` + + +### 常见问题 + +#### npm install无法成功 +如果遇到该情况，可以使用国内的淘宝npm镜像： + +``` +npm install -g cnpm --registry=https://registry.npm.taobao.org +``` + +接着，通过执行以下指令代替npm install指令 + +``` +cnpm install +``` + +注意，项目启动和打包时，仍然可以使用npm run build和npm run serve指令 \ No newline at end of file diff --git "a/web/docs/zh_CN/ch3/scriptis\345\277\253\351\200\237\344\275\277\347\224\250\346\226\207\346\241\243.md" "b/web/docs/zh_CN/ch3/scriptis\345\277\253\351\200\237\344\275\277\347\224\250\346\226\207\346\241\243.md" new file mode 100644 index 0000000000..f8aff5f664 --- /dev/null +++ "b/web/docs/zh_CN/ch3/scriptis\345\277\253\351\200\237\344\275\277\347\224\250\346\226\207\346\241\243.md" @@ -0,0 +1,33 @@ +Scriptis快速使用文档 +============ + +## 快速登录 +     为了方便用户使用，系统默认通过使用Linkis的部署用户名进行登录，比如是hadoop部署的可以直接通过 用户：hadoop，密码：hadoop(密码就是用户名)来进行登录。 +首先输入前端容器地址：192.168.xx.xx:8888 接着输入用户名密码：hadoop/hadoop +![01](../images/ch3/01.png) +__备注：__ 其他用户登录必须使用配置的LDAP的密码进行登录 + +## 设置队列 +     scriptis的Spark引擎是通过开发-->设置-->yarn队列名-->输入有权限的yarn 队列名 +![02](../images/ch3/02.png) + +## 创建脚本 +     设置完队列后，就可以建立一个sql脚本使用SparkSQL进行数据查询，用户只需要输入sql。点击工作空间-个人目录-右键创建sql脚本 + +1. 创建 +![03](../images/ch3/03.png) +2. 建立sql脚本 +![04](../images/ch3/04.png) +3. 输入sql脚本 +![05](../images/ch3/05.png) + +## 点击运行 +     点击脚本的运行按钮则可以运行脚本，并可以通过日志获悉运行状况 +![01](../images/ch3/06.png) + +## 结果展示 +     点击运行后，等待一段时间，第一次运行可能会稍微久点，因为需要启动引擎。运行完成后会将结果进行展示，用户可以对结果进行下载，导出等操作 +![01](../images/ch3/07.png) + +__详细的使用手册见：__ [使用手册文档](../ch4/Scriptis使用手册.md) + diff --git "a/web/docs/zh_CN/ch4/Scriptis\344\275\277\347\224\250\346\211\213\345\206\214.md" "b/web/docs/zh_CN/ch4/Scriptis\344\275\277\347\224\250\346\211\213\345\206\214.md" new file mode 100644 index 0000000000..05c5df60ae --- /dev/null +++ "b/web/docs/zh_CN/ch4/Scriptis\344\275\277\347\224\250\346\211\213\345\206\214.md" @@ -0,0 +1,130 @@ +## 1 功能简介 + + +scirptis主要拆分为一下功能： +1. 工作空间：用户存储用户的脚本，数据，日志等文件；支持建立：sql，hive，scala，python，pyspark等类型的脚本； +2. 数据库模块：展示用户拥有权限的数据库和表信息，以及表导入导出； +3. UDF模块：UDF是可以在sql和hql脚本中使用的函数，包括函数管理，勾选加载，共享等； +4. 方法模块：方法是用户自己，系统，共享的自定义函数，支持在python，pyspark，scala脚本中使用； +5. HDFS模块：用户的HDFS（分布式文件系统）个人目录，用于存放大数据文件； +6. 脚本模块：包括脚本编辑，运行，停止，自定义变量配置，快捷键； +7. 运行结果：包括结果展示，下载，导出； +8. 脚本历史：脚本历史展示了该脚本历史运行信息； +9. 管理台：主要包含设置，全局历史，资源管理器，全局变量，常见问题； +10. 右下角弹出框：包含任务管理器，引擎管理器，队列管理器； +下面对这些功能进行详细介绍。 + +## 2 工作空间 + +工作空间是一个文件目录，用户对该目录拥有所有的权限可以进行文件管理操作等。建议的目录结构是：script,data,log,res四个目录，目录结构清晰方便用户进行查看和管理。工作空间主要功能如下： +1. 工作空间右键主要包含复制路径，新建目录，新建脚本，刷新 +![ide05](../images/ch4/ide05.png) +2. 顶上搜索功能，支持对文件快速定位 +![ide06](../images/ch4/ide06.png) +3. 新建脚本功能支持建立以下脚本 + +* sql:对应Spark引擎的SparkSQL，语法指导手册：[sql语法指导手册](https://docs.databricks.com/spark/latest/spark-sql/index.html) +* hql：对应的是Hive引擎，Hive语法指导手册：[Hive语法指导手册](https://cwiki.apache.org/confluence/display/Hive/LanguageManual) +* Scala：对应的是Spark引擎的Scala语法，语法指导手册：[Spark Scala指导手册](https://docs.databricks.com/spark/latest/dataframes-datasets/introduction-to-dataframes-scala.html) +* Python：单机Python引擎，兼容Python语法 +* PythonSpark：对应的是Spark引擎的Python语法，语法指导手册：[pySpark指导手册](https://docs.databricks.com/spark/latest/dataframes-datasets/introduction-to-dataframes-python.html) + ![ide07](../images/ch4/ide07.png) +* 脚本右键功能，脚本右键主要有打卡到侧边，复制路径，重命名，删除，导入到hive（csv,txt,excel类型文件），导入到hdfs等功能 + ![ide08](../images/ch4/ide08.png) + +## 3 数据库模块 + +数据库模块主要有以下功能： +1. 刷库，刷表，刷字段信息 +![ide09](../images/ch4/ide09.png) +2. 表右键功能--查询表：快捷生产临时hive脚本进行数据查看 +![ide10](../images/ch4/ide10.png) +3. 表右键功能--查看表结构：展示表的字段详细信息，表详情信息，表分区信息等： +![ide11](../images/ch4/ide11.png) +4. 表右键功能--导出表：可以导出为csv和excel等文件 +![ide12](../images/ch4/ide12.png) + +## 4 UDF模块 + +UDF功能是方便用户对UDF进行分类展示，以及用户可以对个人函数进行管理，共享等。主要包含以下功能： + +#### 4.1UDF展示和加载功能 + +默认顶层目录是： + +* 平台函数：平台提供的UDF函数，可以在sql，pysaprk，scala，hive脚本的sql语言里面进行使用 +* 系统函数：系统默认已经提供并自动加载了的函数，可以在sql，pysaprk，scala，hive脚本的sql语言里面进行使用 +* 个人函数：个人自己新建立的UDF，包含通用函数，Spark专用函数。 +* 共享函数：科室管理员建立的UDF函数，并共享给其他人使用 + 除了系统函数外，其他类别的函数都必须点击加载后才能进行使用，而且点击勾选后一定要先kill掉已经启动的会话。 + 另外勾选加载的UDF，在脚本中也会进行自动补全提示： + ![ide13](../images/ch4/ide13.png) + +#### 4.2UDF新增函数功能： + +创建一个UDF很简单，主要在于做好前期的UDF代码的准备。步骤主要如下： +1. 建立通用UDF，需要先编译好相应的Jar包。这里的通用是指Hive的hql和Spark的sql都可以用 +2. 建立Spark专用的UDF，需要先建立好相应的py脚本或者scala脚本，为了使用的正确性，最好测试下相应的脚本 +3. 在开发建立该UDF: +通用UDF：需要勾选通用，并选择对应的jar包路径，填写UDF的全类路径，填写使用格式和描述： +![ide14](../images/ch4/ide14.png) +Spark专用--scala脚本：需要勾选Spark，并选择对应的scala脚本，填写注册格式（填写相应脚本中的函数名）： +![ide15](../images/ch4/ide15.png) +Spark专用--Python脚本：需要勾选Spark，并选择对应的python脚本，填写注册格式（填写相应脚本中的函数名）： +![ide16](../images/ch4/ide16.png) + +#### 4.3Scala和Python函数写法： + +PythonUDF建立方式，只需要定义一个方法即可，对应的脚本只能与该方法有关的内容： + +``` +def hello(id): + return str(id) + ":hello" +``` + +ScalaUDF 建立方式和Python类似，定义一个方法即可： + +``` +def helloWorld(str: String): String = "hello, " + str +``` + +注意：PythonUDF和ScalaUDF只能在Spark引擎对应的脚本中使用 + +## 5 方法模块： + +方法模块和UDF模块功能类似，只是一个是UDF一个自定义函数。另外需要注意的是：python脚本定义的函数只能在python和pysaprk中使用，scala脚本定义的函数只能在scala中进行使用。 + +## 6 脚本模块 + +脚本模块就是脚本编辑框包含的功能，主要有： +1. 脚本编辑：基础的关键字高亮功能，代码格式化，代码合并，自动补全，快捷健等功能。 +2. 运行和停止：用户可以选择代码进行执行，也可以直接点击执行运行整个脚本。脚本运行起来后可以通过停止按钮进行停止。 +3. 脚本编辑框有配置选项用于定义用户的自定义变量，该脚本内生效 +![ide18](../images/ch4/ide18.png) + +## 7 运行结果 + +运行结果包括以下功能： +1. 表格结果展示，点击表头可以进行排序，双击复制字段名等，功能会继续丰富，包括选择列展示，展示字段类型。只支持展示5000行 +2. 可视化分析：点击可视化分析，可以通过VSBI对产生的结果集进行可视化展示（待开源） +3. 下载：结果集支持直接下载到浏览器本地，支持CSV和excel，只支持下载5000行 +4. 导出：结果集导出可以支持导出到工作空间（BDAP的共享目录），结果集导出也支持CSV和Excel格式，并且通过打开全量导出，可以超过5000行的限制。全量导出开关，在执行的sql前面加上注释：`--set wds.linkis.engine.no.limit.allow=true` + +## 8 脚本历史 + +脚本历史展示了该脚本文件的历史运行信息，可以快速找到之前运行的脚本的日志，结果。可以有效减少多次运行相同的脚本： +![ide23](../images/ch4/ide23.png) + +## 9 管理台 + +管理台主要包含一下功能： +1. 设置：包含通用设置（如设置队列），数据开发相关引擎的设置：spark，hive，python，pipeline等 +![ide25](../images/ch4/ide25.png) +2. 全局变量：全局变量是可以用于所有脚本的自定义变量，如果变量重名则脚本中配置的生效。 +3. 其他功能：全局历史，资源管理器，常见问题 + +## 10 右下角弹出框 + +右下角弹出框点击后是类似于Windows的任务管理器类似的功能，用户可以快速的对任务管，引擎，队列资源进行查看和管理： +![ide24](../images/ch4/ide24.png) + diff --git a/web/docs/zh_CN/images/ch3/01.png b/web/docs/zh_CN/images/ch3/01.png new file mode 100644 index 0000000000..f45529eb30 Binary files /dev/null and b/web/docs/zh_CN/images/ch3/01.png differ diff --git a/web/docs/zh_CN/images/ch3/02.png b/web/docs/zh_CN/images/ch3/02.png new file mode 100644 index 0000000000..65c7f14cb1 Binary files /dev/null and b/web/docs/zh_CN/images/ch3/02.png differ diff --git a/web/docs/zh_CN/images/ch3/03.png b/web/docs/zh_CN/images/ch3/03.png new file mode 100644 index 0000000000..b1fa65b15f Binary files /dev/null and b/web/docs/zh_CN/images/ch3/03.png differ diff --git a/web/docs/zh_CN/images/ch3/04.png b/web/docs/zh_CN/images/ch3/04.png new file mode 100644 index 0000000000..a768f716d5 Binary files /dev/null and b/web/docs/zh_CN/images/ch3/04.png differ diff --git a/web/docs/zh_CN/images/ch3/05.png b/web/docs/zh_CN/images/ch3/05.png new file mode 100644 index 0000000000..3909c97bf1 Binary files /dev/null and b/web/docs/zh_CN/images/ch3/05.png differ diff --git a/web/docs/zh_CN/images/ch3/06.png b/web/docs/zh_CN/images/ch3/06.png new file mode 100644 index 0000000000..8297ce05cd Binary files /dev/null and b/web/docs/zh_CN/images/ch3/06.png differ diff --git a/web/docs/zh_CN/images/ch3/07.png b/web/docs/zh_CN/images/ch3/07.png new file mode 100644 index 0000000000..065021bfc9 Binary files /dev/null and b/web/docs/zh_CN/images/ch3/07.png differ diff --git a/web/docs/zh_CN/images/ch4/ide05.png b/web/docs/zh_CN/images/ch4/ide05.png new file mode 100644 index 0000000000..10f3d91a31 Binary files /dev/null and b/web/docs/zh_CN/images/ch4/ide05.png differ diff --git a/web/docs/zh_CN/images/ch4/ide06.png b/web/docs/zh_CN/images/ch4/ide06.png new file mode 100644 index 0000000000..1fb5b2cd66 Binary files /dev/null and b/web/docs/zh_CN/images/ch4/ide06.png differ diff --git a/web/docs/zh_CN/images/ch4/ide07.png b/web/docs/zh_CN/images/ch4/ide07.png new file mode 100644 index 0000000000..fa414ab054 Binary files /dev/null and b/web/docs/zh_CN/images/ch4/ide07.png differ diff --git a/web/docs/zh_CN/images/ch4/ide08.png b/web/docs/zh_CN/images/ch4/ide08.png new file mode 100644 index 0000000000..5b2321f7c7 Binary files /dev/null and b/web/docs/zh_CN/images/ch4/ide08.png differ diff --git a/web/docs/zh_CN/images/ch4/ide09.png b/web/docs/zh_CN/images/ch4/ide09.png new file mode 100644 index 0000000000..47dc296908 Binary files /dev/null and b/web/docs/zh_CN/images/ch4/ide09.png differ diff --git a/web/docs/zh_CN/images/ch4/ide10.png b/web/docs/zh_CN/images/ch4/ide10.png new file mode 100644 index 0000000000..4141452a57 Binary files /dev/null and b/web/docs/zh_CN/images/ch4/ide10.png differ diff --git a/web/docs/zh_CN/images/ch4/ide11.png b/web/docs/zh_CN/images/ch4/ide11.png new file mode 100644 index 0000000000..d4fb1ec8ee Binary files /dev/null and b/web/docs/zh_CN/images/ch4/ide11.png differ diff --git a/web/docs/zh_CN/images/ch4/ide12.png b/web/docs/zh_CN/images/ch4/ide12.png new file mode 100644 index 0000000000..a615495ae4 Binary files /dev/null and b/web/docs/zh_CN/images/ch4/ide12.png differ diff --git a/web/docs/zh_CN/images/ch4/ide13.png b/web/docs/zh_CN/images/ch4/ide13.png new file mode 100644 index 0000000000..ff4b6d3c07 Binary files /dev/null and b/web/docs/zh_CN/images/ch4/ide13.png differ diff --git a/web/docs/zh_CN/images/ch4/ide14.png b/web/docs/zh_CN/images/ch4/ide14.png new file mode 100644 index 0000000000..9ca07e562b Binary files /dev/null and b/web/docs/zh_CN/images/ch4/ide14.png differ diff --git a/web/docs/zh_CN/images/ch4/ide15.png b/web/docs/zh_CN/images/ch4/ide15.png new file mode 100644 index 0000000000..740027f135 Binary files /dev/null and b/web/docs/zh_CN/images/ch4/ide15.png differ diff --git a/web/docs/zh_CN/images/ch4/ide16.png b/web/docs/zh_CN/images/ch4/ide16.png new file mode 100644 index 0000000000..a4a11ada2d Binary files /dev/null and b/web/docs/zh_CN/images/ch4/ide16.png differ diff --git a/web/docs/zh_CN/images/ch4/ide17.png b/web/docs/zh_CN/images/ch4/ide17.png new file mode 100644 index 0000000000..02576e3f94 Binary files /dev/null and b/web/docs/zh_CN/images/ch4/ide17.png differ diff --git a/web/docs/zh_CN/images/ch4/ide18.png b/web/docs/zh_CN/images/ch4/ide18.png new file mode 100644 index 0000000000..a69059d21e Binary files /dev/null and b/web/docs/zh_CN/images/ch4/ide18.png differ diff --git a/web/docs/zh_CN/images/ch4/ide19.png b/web/docs/zh_CN/images/ch4/ide19.png new file mode 100644 index 0000000000..49359d5465 Binary files /dev/null and b/web/docs/zh_CN/images/ch4/ide19.png differ diff --git a/web/docs/zh_CN/images/ch4/ide20.png b/web/docs/zh_CN/images/ch4/ide20.png new file mode 100644 index 0000000000..12b7caf0f4 Binary files /dev/null and b/web/docs/zh_CN/images/ch4/ide20.png differ diff --git a/web/docs/zh_CN/images/ch4/ide21.png b/web/docs/zh_CN/images/ch4/ide21.png new file mode 100644 index 0000000000..74bafffc23 Binary files /dev/null and b/web/docs/zh_CN/images/ch4/ide21.png differ diff --git a/web/docs/zh_CN/images/ch4/ide22.png b/web/docs/zh_CN/images/ch4/ide22.png new file mode 100644 index 0000000000..a767e9a211 Binary files /dev/null and b/web/docs/zh_CN/images/ch4/ide22.png differ diff --git a/web/docs/zh_CN/images/ch4/ide23.png b/web/docs/zh_CN/images/ch4/ide23.png new file mode 100644 index 0000000000..bfb5eef3dd Binary files /dev/null and b/web/docs/zh_CN/images/ch4/ide23.png differ diff --git a/web/docs/zh_CN/images/ch4/ide24.png b/web/docs/zh_CN/images/ch4/ide24.png new file mode 100644 index 0000000000..19c922ecc3 Binary files /dev/null and b/web/docs/zh_CN/images/ch4/ide24.png differ diff --git a/web/docs/zh_CN/images/ch4/ide25.png b/web/docs/zh_CN/images/ch4/ide25.png new file mode 100644 index 0000000000..be473089c4 Binary files /dev/null and b/web/docs/zh_CN/images/ch4/ide25.png differ diff --git a/web/docs/zh_CN/images/readme/cs.gif b/web/docs/zh_CN/images/readme/cs.gif new file mode 100644 index 0000000000..4df76a1648 Binary files /dev/null and b/web/docs/zh_CN/images/readme/cs.gif differ diff --git a/web/docs/zh_CN/images/readme/datasource.gif b/web/docs/zh_CN/images/readme/datasource.gif new file mode 100644 index 0000000000..eedea96f5d Binary files /dev/null and b/web/docs/zh_CN/images/readme/datasource.gif differ diff --git a/web/docs/zh_CN/images/readme/results.gif b/web/docs/zh_CN/images/readme/results.gif new file mode 100644 index 0000000000..c95943c957 Binary files /dev/null and b/web/docs/zh_CN/images/readme/results.gif differ diff --git a/web/docs/zh_CN/images/readme/running.gif b/web/docs/zh_CN/images/readme/running.gif new file mode 100644 index 0000000000..81ff07240d Binary files /dev/null and b/web/docs/zh_CN/images/readme/running.gif differ diff --git a/web/docs/zh_CN/images/readme/script.gif b/web/docs/zh_CN/images/readme/script.gif new file mode 100644 index 0000000000..a503694bd6 Binary files /dev/null and b/web/docs/zh_CN/images/readme/script.gif differ diff --git a/web/docs/zh_CN/images/readme/setup.gif b/web/docs/zh_CN/images/readme/setup.gif new file mode 100644 index 0000000000..dabb635f8a Binary files /dev/null and b/web/docs/zh_CN/images/readme/setup.gif differ diff --git "a/web/docs/zh_CN/images/readme/\304\267\303\224\304\205\304\214.png" "b/web/docs/zh_CN/images/readme/\304\267\303\224\304\205\304\214.png" new file mode 100644 index 0000000000..d32188e5f2 Binary files /dev/null and "b/web/docs/zh_CN/images/readme/\304\267\303\224\304\205\304\214.png" differ diff --git a/web/package-lock.json b/web/package-lock.json index b73e82c403..0565f78872 100644 --- a/web/package-lock.json +++ b/web/package-lock.json @@ -1,13 +1,28 @@ { "name": "dataspherestudio", - "version": "1.1.6", + "version": "1.1.20", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "dataspherestudio", - "version": "1.1.6", + "version": "1.1.20", "hasInstallScript": true, + "workspaces": [ + "packages/apiServices", + "packages/cyeditor", + "packages/dataGovernance", + "packages/dss", + "packages/editor", + "packages/editorLsp", + "packages/exts", + "packages/scheduleCenter", + "packages/dolphinScheduler", + "packages/scriptis", + "packages/shared", + "packages/workflows", + "packages/workspace" + ], "dependencies": { "axios": "0.21.1", "babel-polyfill": "6.26.0", @@ -20,6 +35,7 @@ "md5": "2.3.0", "moment": "2.29.1", "monaco-editor": "0.19.3", + "monaco-languageclient": "^0.13.0", "qs": "6.9.4", "svgo": "1.3.0", "vue": "2.6.12", @@ -47,8 +63,8 @@ "less-loader": "6.1.0", "lint-staged": "8.2.1", "monaco-editor-webpack-plugin": "1.8.2", - "node-sass": "6.0.1", "patch-package": "6.2.2", + "sass": "1.49.0", "sass-loader": "10.2.1", "speed-measure-webpack-plugin": "1.5.0", "svg-sprite-loader": "5.0.0", @@ -1336,7 +1352,6 @@ "version": "7.17.9", "resolved": "https://registry.npmmirror.com/@babel/runtime/-/runtime-7.17.9.tgz", "integrity": "sha512-lSiBBvodq29uShpWGNbgFdKYNiFDo5/HIYsaCEY9ff4sb10x9jizo2+pRrSyF4jKZCXqgzuqBOQKbUm90gQwJg==", - "dev": true, "dependencies": { "regenerator-runtime": "^0.13.4" }, @@ -1428,6 +1443,54 @@ "node": ">=6.9.0" } }, + "node_modules/@dataspherestudio/apiServices": { + "resolved": "packages/apiServices", + "link": true + }, + "node_modules/@dataspherestudio/cyeditor": { + "resolved": "packages/cyeditor", + "link": true + }, + "node_modules/@dataspherestudio/dataGovernance": { + "resolved": "packages/dataGovernance", + "link": true + }, + "node_modules/@dataspherestudio/dolphinScheduler": { + "resolved": "packages/dolphinScheduler", + "link": true + }, + "node_modules/@dataspherestudio/dss": { + "resolved": "packages/dss", + "link": true + }, + "node_modules/@dataspherestudio/editor": { + "resolved": "packages/editor", + "link": true + }, + "node_modules/@dataspherestudio/editor-lsp": { + "resolved": "packages/editorLsp", + "link": true + }, + "node_modules/@dataspherestudio/exts": { + "resolved": "packages/exts", + "link": true + }, + "node_modules/@dataspherestudio/scriptis": { + "resolved": "packages/scriptis", + "link": true + }, + "node_modules/@dataspherestudio/shared": { + "resolved": "packages/shared", + "link": true + }, + "node_modules/@dataspherestudio/workflows": { + "resolved": "packages/workflows", + "link": true + }, + "node_modules/@dataspherestudio/workspace": { + "resolved": "packages/workspace", + "link": true + }, "node_modules/@gar/promisify": { "version": "1.1.3", "resolved": "https://registry.npmmirror.com/@gar/promisify/-/promisify-1.1.3.tgz", @@ -6009,6 +6072,24 @@ "@octokit/openapi-types": "^11.2.0" } }, + "node_modules/@riophae/vue-treeselect": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/@riophae/vue-treeselect/-/vue-treeselect-0.4.0.tgz", + "integrity": "sha512-J4atYmBqXQmiPFK/0B5sXKjtnGc21mBJEiyKIDZwk0Q9XuynVFX6IJ4EpaLmUgL5Tve7HAS7wkiGGSti6Uaxcg==", + "dependencies": { + "@babel/runtime": "^7.3.1", + "babel-helper-vue-jsx-merge-props": "^2.0.3", + "easings-css": "^1.0.0", + "fuzzysearch": "^1.0.3", + "is-promise": "^2.1.0", + "lodash": "^4.0.0", + "material-colors": "^1.2.6", + "watch-size": "^2.0.0" + }, + "peerDependencies": { + "vue": "^2.2.0" + } + }, "node_modules/@samverschueren/stream-to-observable": { "version": "0.3.1", "resolved": "https://registry.npmmirror.com/@samverschueren/stream-to-observable/-/stream-to-observable-0.3.1.tgz", @@ -7172,6 +7253,8 @@ "resolved": "https://registry.npmmirror.com/amdefine/-/amdefine-1.0.1.tgz", "integrity": "sha512-S2Hw0TtNkMJhIabBwIojKL9YHO5T0n5eNqWJ7Lrlel/zDbftQpxpapi8tZs3X1HWa+u+QeydGmzzNU0m09+Rcg==", "dev": true, + "optional": true, + "peer": true, "engines": { "node": ">=0.4.2" } @@ -7245,7 +7328,7 @@ "version": "3.1.2", "resolved": "https://registry.npmmirror.com/anymatch/-/anymatch-3.1.2.tgz", "integrity": "sha512-P43ePfOAIupkguHUycrc4qJ9kz8ZiuOUijaETwX7THt0Y/GNK7v0aa8rY816xWjZ7rJdA5XdMcpVFTKMq+RvWg==", - "optional": true, + "devOptional": true, "dependencies": { "normalize-path": "^3.0.0", "picomatch": "^2.0.4" @@ -7549,6 +7632,8 @@ "resolved": "https://registry.npmmirror.com/async-foreach/-/async-foreach-0.1.3.tgz", "integrity": "sha512-VUeSMD8nEGBWaZK4lizI1sf3yEC7pnAQ/mrI7pC2fBz2s/tq5jWWEngTwaf0Gruu/OoXRGLGg1XFqpYBiGTYJA==", "dev": true, + "optional": true, + "peer": true, "engines": { "node": "*" } @@ -7730,6 +7815,11 @@ "eslint": ">= 4.12.1" } }, + "node_modules/babel-helper-vue-jsx-merge-props": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/babel-helper-vue-jsx-merge-props/-/babel-helper-vue-jsx-merge-props-2.0.3.tgz", + "integrity": "sha512-gsLiKK7Qrb7zYJNgiXKpXblxbV5ffSwR0f5whkPAaBAR4fhi6bwRZxX9wBlIc5M/v8CCkXUbXZL4N/nSE97cqg==" + }, "node_modules/babel-loader": { "version": "8.2.5", "resolved": "https://registry.npmmirror.com/babel-loader/-/babel-loader-8.2.5.tgz", @@ -7955,6 +8045,12 @@ "@babel/core": "^7.0.0-0" } }, + "node_modules/babel-plugin-transform-remove-console": { + "version": "6.9.4", + "resolved": "https://registry.npmjs.org/babel-plugin-transform-remove-console/-/babel-plugin-transform-remove-console-6.9.4.tgz", + "integrity": "sha512-88blrUrMX3SPiGkT1GnvVY8E/7A+k6oj3MNvUtTIxJflFzXTw1bHkuJ/y039ouhFMp2prRn5cQGzokViYi1dsg==", + "dev": true + }, "node_modules/babel-polyfill": { "version": "6.26.0", "resolved": "https://registry.npmmirror.com/babel-polyfill/-/babel-polyfill-6.26.0.tgz", @@ -8110,7 +8206,7 @@ "version": "2.2.0", "resolved": "https://registry.npmmirror.com/binary-extensions/-/binary-extensions-2.2.0.tgz", "integrity": "sha512-jDctJ/IVQbZoJykoeHbhXpOlNBqGNcwXJKJog42E5HDPUwQTSdjCHdihjj0DlnheQ7blbT6dHOafNAiS8ooQKA==", - "optional": true, + "devOptional": true, "engines": { "node": ">=8" } @@ -8230,6 +8326,14 @@ "resolved": "https://registry.npmmirror.com/boolbase/-/boolbase-1.0.0.tgz", "integrity": "sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww==" }, + "node_modules/bootstrap": { + "version": "3.4.1", + "resolved": "https://registry.npmjs.org/bootstrap/-/bootstrap-3.4.1.tgz", + "integrity": "sha512-yN5oZVmRCwe5aKwzRj6736nSmKDX7pLYwsXiCj/EYmo16hODaBiT4En5btW/jhBF/seV+XMx3aYwukYC3A49DA==", + "engines": { + "node": ">=6" + } + }, "node_modules/brace-expansion": { "version": "1.1.11", "resolved": "https://registry.npmmirror.com/brace-expansion/-/brace-expansion-1.1.11.tgz", @@ -8815,7 +8919,7 @@ "version": "3.5.3", "resolved": "https://registry.npmmirror.com/chokidar/-/chokidar-3.5.3.tgz", "integrity": "sha512-Dr3sfKRP6oTcjf2JmUmFJfeVMvXBdegxB0iVQ5eb2V10uFJUCAS8OByZdVAyVb8xXNz3GjjTgj9kLWsZTqE6kw==", - "optional": true, + "devOptional": true, "dependencies": { "anymatch": "~3.1.2", "braces": "~3.0.2", @@ -8836,7 +8940,7 @@ "version": "3.0.2", "resolved": "https://registry.npmmirror.com/braces/-/braces-3.0.2.tgz", "integrity": "sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==", - "optional": true, + "devOptional": true, "dependencies": { "fill-range": "^7.0.1" }, @@ -8848,7 +8952,7 @@ "version": "7.0.1", "resolved": "https://registry.npmmirror.com/fill-range/-/fill-range-7.0.1.tgz", "integrity": "sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==", - "optional": true, + "devOptional": true, "dependencies": { "to-regex-range": "^5.0.1" }, @@ -8860,7 +8964,7 @@ "version": "7.0.0", "resolved": "https://registry.npmmirror.com/is-number/-/is-number-7.0.0.tgz", "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==", - "optional": true, + "devOptional": true, "engines": { "node": ">=0.12.0" } @@ -8869,7 +8973,7 @@ "version": "5.0.1", "resolved": "https://registry.npmmirror.com/to-regex-range/-/to-regex-range-5.0.1.tgz", "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==", - "optional": true, + "devOptional": true, "dependencies": { "is-number": "^7.0.0" }, @@ -9159,6 +9263,16 @@ "dev": true, "optional": true }, + "node_modules/clipboard": { + "version": "2.0.8", + "resolved": "https://registry.npmjs.org/clipboard/-/clipboard-2.0.8.tgz", + "integrity": "sha512-Y6WO0unAIQp5bLmk1zdThRhgJt/x3ks6f30s3oE3H1mgIEU33XyQjEf8gsf6DxC7NPX8Y1SsNWjUjL/ywLnnbQ==", + "dependencies": { + "good-listener": "^1.2.2", + "select": "^1.1.2", + "tiny-emitter": "^2.0.0" + } + }, "node_modules/clipboardy": { "version": "2.3.0", "resolved": "https://registry.npmmirror.com/clipboardy/-/clipboardy-2.3.0.tgz", @@ -11092,6 +11206,11 @@ "node": ">=4" } }, + "node_modules/cssfilter": { + "version": "0.0.10", + "resolved": "https://registry.npmjs.org/cssfilter/-/cssfilter-0.0.10.tgz", + "integrity": "sha512-FAaLDaplstoRsDR8XGYH51znUN0UY7nMc6Z9/fvE8EXGwvJE9hu7W2vHwx1+bd6gCYnln9nLbzxFTrcO9YQDZw==" + }, "node_modules/cssnano": { "version": "4.1.11", "resolved": "https://registry.npmmirror.com/cssnano/-/cssnano-4.1.11.tgz", @@ -11231,6 +11350,39 @@ "resolved": "https://registry.npmmirror.com/cyclist/-/cyclist-1.0.1.tgz", "integrity": "sha512-NJGVKPS81XejHcLhaLJS7plab0fK3slPh11mESeeDq2W4ZI5kUKK/LRRdVDvjJseojbPB7ZwjnyOybg3Igea/A==" }, + "node_modules/cytoscape": { + "version": "3.27.0", + "resolved": "https://registry.npmjs.org/cytoscape/-/cytoscape-3.27.0.tgz", + "integrity": "sha512-pPZJilfX9BxESwujODz5pydeGi+FBrXq1rcaB1mfhFXXFJ9GjE6CNndAk+8jPzoXGD+16LtSS4xlYEIUiW4Abg==", + "dependencies": { + "heap": "^0.2.6", + "lodash": "^4.17.21" + }, + "engines": { + "node": ">=0.10" + } + }, + "node_modules/cytoscape-klay": { + "version": "3.1.4", + "resolved": "https://registry.npmjs.org/cytoscape-klay/-/cytoscape-klay-3.1.4.tgz", + "integrity": "sha512-VwPj0VR25GPfy6qXVQRi/MYlZM/zkdvRhHlgqbM//lSvstgM6fhp3ik/uM8Wr8nlhskfqz/M1fIDmR6UckbS2A==", + "dependencies": { + "klayjs": "^0.4.1" + }, + "peerDependencies": { + "cytoscape": "^3.2.0" + } + }, + "node_modules/cytoscape/node_modules/lodash": { + "version": "4.17.21", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", + "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==" + }, + "node_modules/d3": { + "version": "3.5.17", + "resolved": "https://registry.npmjs.org/d3/-/d3-3.5.17.tgz", + "integrity": "sha512-yFk/2idb8OHPKkbAL8QaOaqENNoMhIaSHZerk3oQsECwkObkCpJyjYwCe+OHiq6UEdhe1m8ZGARRRO3ljFjlKg==" + }, "node_modules/d3-dispatch": { "version": "2.0.0", "resolved": "https://registry.npmmirror.com/d3-dispatch/-/d3-dispatch-2.0.0.tgz", @@ -11301,6 +11453,11 @@ "node": "*" } }, + "node_modules/dayjs": { + "version": "1.10.7", + "resolved": "https://registry.npmjs.org/dayjs/-/dayjs-1.10.7.tgz", + "integrity": "sha512-P6twpd70BcPK34K26uJ1KT3wlhpuOAPoMwJzpsIWUxHZ7wpmbdZL/hQqBDfz7hGurYSa5PhzdhDHtt319hL3ig==" + }, "node_modules/de-indent": { "version": "1.0.2", "resolved": "https://registry.npmmirror.com/de-indent/-/de-indent-1.0.2.tgz", @@ -11672,6 +11829,11 @@ "node": ">=0.4.0" } }, + "node_modules/delegate": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/delegate/-/delegate-3.2.0.tgz", + "integrity": "sha512-IofjkYBZaZivn0V8nnsMJGBr4jVLxHDheKSW88PyxS5QC4Vo9ZbZVvhzlSxY87fVq3STR6r+4cGepyHkcWOQSw==" + }, "node_modules/delegates": { "version": "1.0.0", "resolved": "https://registry.npmmirror.com/delegates/-/delegates-1.0.0.tgz", @@ -11727,6 +11889,14 @@ "integrity": "sha512-T0NIuQpnTvFDATNuHN5roPwSBG83rFsuO+MXXH9/3N1eFbn4wcPjttvjMLEPWJ0RGUYgQE7cGgS3tNxbqCGM7g==", "dev": true }, + "node_modules/dexie": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/dexie/-/dexie-3.0.4.tgz", + "integrity": "sha512-rwS3k8BBstjGFJAS/yjYrsRZucqitnrP3NBhqghl9ihWcWABlT3I5f2dZV9T3pmXGAP7G9p1q00g/axcsuOT8A==", + "engines": { + "node": ">=6.0" + } + }, "node_modules/dezalgo": { "version": "1.0.4", "resolved": "https://registry.npmmirror.com/dezalgo/-/dezalgo-1.0.4.tgz", @@ -11903,6 +12073,11 @@ "integrity": "sha512-YXQl1DSa4/PQyRfgrv6aoNjhasp/p4qs9FjJ4q4cQk+8m4r6k4ZSiEyytKG8f8W9gi8WsQtIObNmKd+tMzNTmA==", "dev": true }, + "node_modules/dt-sql-parser": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/dt-sql-parser/-/dt-sql-parser-1.2.1.tgz", + "integrity": "sha512-bp6Rtm8N3m2Z3tavtKGo8J5ffaPQdPodYV2xGi9f6IsWbHzOkpNBxHlORcWtnSLtA/JvZcvImQxgbkdaGiv5cw==" + }, "node_modules/duplexer": { "version": "0.1.2", "resolved": "https://registry.npmmirror.com/duplexer/-/duplexer-0.1.2.tgz", @@ -11920,6 +12095,11 @@ "stream-shift": "^1.0.0" } }, + "node_modules/easings-css": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/easings-css/-/easings-css-1.0.0.tgz", + "integrity": "sha512-7Uq7NdazNfVtr0RNmPAys8it0zKCuaqxJStYKEl72D3j4gbvXhhaM7iWNbqhA4C94ygCye6VuyhzBRQC4szeBg==" + }, "node_modules/easy-stack": { "version": "1.0.1", "resolved": "https://registry.npmmirror.com/easy-stack/-/easy-stack-1.0.1.tgz", @@ -14225,6 +14405,11 @@ "integrity": "sha512-xckBUXyTIqT97tq2x2AMb+g163b5JFysYk0x4qxNFwbfQkmNZoiRHb6sPzI9/QV33WeuvVYBUIiD4NzNIyqaRQ==", "dev": true }, + "node_modules/fuzzysearch": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/fuzzysearch/-/fuzzysearch-1.0.3.tgz", + "integrity": "sha512-s+kNWQuI3mo9OALw0HJ6YGmMbLqEufCh2nX/zzV5CrICQ/y4AwPxM+6TIiF9ItFCHXFCyM/BfCCmN57NTIJuPg==" + }, "node_modules/g-status": { "version": "2.0.2", "resolved": "https://registry.npmmirror.com/g-status/-/g-status-2.0.2.tgz", @@ -14316,6 +14501,8 @@ "resolved": "https://registry.npmmirror.com/gaze/-/gaze-1.1.3.tgz", "integrity": "sha512-BRdNm8hbWzFzWHERTrejLqwHDfS4GibPoq5wjTPIoJHoBtKGPg3xAFfxmM+9ztbXelxcf2hwQcaz1PtmFeue8g==", "dev": true, + "optional": true, + "peer": true, "dependencies": { "globule": "^1.0.0" }, @@ -14660,8 +14847,7 @@ "node_modules/glob-to-regexp": { "version": "0.3.0", "resolved": "https://registry.npmmirror.com/glob-to-regexp/-/glob-to-regexp-0.3.0.tgz", - "integrity": "sha512-Iozmtbqv0noj0uDDqoL0zNq0VBEfK2YFoMAZoxJe4cwphvLR+JskfF30QhXHOR4m3KrE6NLRYw+U9MRXvifyig==", - "dev": true + "integrity": "sha512-Iozmtbqv0noj0uDDqoL0zNq0VBEfK2YFoMAZoxJe4cwphvLR+JskfF30QhXHOR4m3KrE6NLRYw+U9MRXvifyig==" }, "node_modules/glob2base": { "version": "0.0.12", @@ -14717,6 +14903,8 @@ "resolved": "https://registry.npmmirror.com/globule/-/globule-1.3.3.tgz", "integrity": "sha512-mb1aYtDbIjTu4ShMB85m3UzjX9BVKe9WCzsnfMSZk+K5GpIbBOexgg4PPCt5eHDEG5/ZQAUX2Kct02zfiPLsKg==", "dev": true, + "optional": true, + "peer": true, "dependencies": { "glob": "~7.1.1", "lodash": "~4.17.10", @@ -14731,6 +14919,8 @@ "resolved": "https://registry.npmmirror.com/glob/-/glob-7.1.7.tgz", "integrity": "sha512-OvD9ENzPLbegENnYP5UUfJIirTg4+XwMWGaQfQTY0JenxNvvIKP3U3/tAQSPIu/lHxXYSZmpXlUHeqAIdKzBLQ==", "dev": true, + "optional": true, + "peer": true, "dependencies": { "fs.realpath": "^1.0.0", "inflight": "^1.0.4", @@ -14748,6 +14938,8 @@ "resolved": "https://registry.npmmirror.com/minimatch/-/minimatch-3.0.8.tgz", "integrity": "sha512-6FsRAQsxQ61mw+qP1ZzbL9Bc78x2p5OqNgNpnoAFLTrX8n5Kxph0CsnhmKKNXTWjXqU5L0pGPR7hYk+XWZr60Q==", "dev": true, + "optional": true, + "peer": true, "dependencies": { "brace-expansion": "^1.1.7" }, @@ -14755,6 +14947,14 @@ "node": "*" } }, + "node_modules/good-listener": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/good-listener/-/good-listener-1.2.2.tgz", + "integrity": "sha512-goW1b+d9q/HIwbVYZzZ6SsTr4IgE+WA44A0GmPIQstuOrgsFcT7VEJ48nmr9GaRtNu0XTKacFLGnBPAM6Afouw==", + "dependencies": { + "delegate": "^3.1.2" + } + }, "node_modules/graceful-fs": { "version": "4.2.10", "resolved": "https://registry.npmmirror.com/graceful-fs/-/graceful-fs-4.2.10.tgz", @@ -15018,12 +15218,26 @@ "he": "bin/he" } }, + "node_modules/heap": { + "version": "0.2.7", + "resolved": "https://registry.npmjs.org/heap/-/heap-0.2.7.tgz", + "integrity": "sha512-2bsegYkkHO+h/9MGbn6KWcE45cHZgPANo5LXF7EvWdT0yT2EguSVO1nDgU5c8+ZOPwp2vMNa7YFsJhVcDR9Sdg==" + }, "node_modules/hex-color-regex": { "version": "1.1.0", "resolved": "https://registry.npmmirror.com/hex-color-regex/-/hex-color-regex-1.1.0.tgz", "integrity": "sha512-l9sfDFsuqtOqKDsQdqrMRk0U85RZc0RtOR9yPI7mRVOa4FsR/BVnZ0shmQRM96Ji99kYZP/7hn1cedc1+ApsTQ==", "dev": true }, + "node_modules/highlight.js": { + "version": "9.18.3", + "resolved": "https://registry.npmjs.org/highlight.js/-/highlight.js-9.18.3.tgz", + "integrity": "sha512-zBZAmhSupHIl5sITeMqIJnYCDfAEc3Gdkqj65wC1lpI468MMQeeQkhcIAvk+RylAkxrCcI9xy9piHiXeQ1BdzQ==", + "deprecated": "Version no longer supported. Upgrade to @latest", + "engines": { + "node": "*" + } + }, "node_modules/hmac-drbg": { "version": "1.0.1", "resolved": "https://registry.npmmirror.com/hmac-drbg/-/hmac-drbg-1.0.1.tgz", @@ -15553,6 +15767,12 @@ "node": ">=0.10.0" } }, + "node_modules/immutable": { + "version": "4.3.6", + "resolved": "https://registry.npmjs.org/immutable/-/immutable-4.3.6.tgz", + "integrity": "sha512-Ju0+lEMyzMVZarkTn/gqRpdqd5dOPaz1mCZ0SH3JV6iFw81PldE/PEB1hWVEA288HPt4WXW8O7AWxB10M+03QQ==", + "dev": true + }, "node_modules/import-cwd": { "version": "2.1.0", "resolved": "https://registry.npmmirror.com/import-cwd/-/import-cwd-2.1.0.tgz", @@ -15962,7 +16182,7 @@ "version": "2.1.0", "resolved": "https://registry.npmmirror.com/is-binary-path/-/is-binary-path-2.1.0.tgz", "integrity": "sha512-ZMERYes6pDydyuGidse7OsHxtbI7WVeUEozgR/g7rd0xUimYNlvZRE/K2MgZTjWy725IfelLeVcEM97mmtRGXw==", - "optional": true, + "devOptional": true, "dependencies": { "binary-extensions": "^2.0.0" }, @@ -16306,8 +16526,7 @@ "node_modules/is-promise": { "version": "2.2.2", "resolved": "https://registry.npmmirror.com/is-promise/-/is-promise-2.2.2.tgz", - "integrity": "sha512-+lP4/6lKUBfQjZ2pdxThZvLUAafmZb8OAxFb8XXtiQmS35INgr85hdOGoEs124ez1FCnZJt6jau/T+alh58QFQ==", - "dev": true + "integrity": "sha512-+lP4/6lKUBfQjZ2pdxThZvLUAafmZb8OAxFb8XXtiQmS35INgr85hdOGoEs124ez1FCnZJt6jau/T+alh58QFQ==" }, "node_modules/is-regex": { "version": "1.1.4", @@ -16489,8 +16708,7 @@ "node_modules/jquery": { "version": "3.6.0", "resolved": "https://registry.npmmirror.com/jquery/-/jquery-3.6.0.tgz", - "integrity": "sha512-JVzAR/AjBvVt2BmYhxRCSYysDsPcssdmTFnzyLEts9qNwmjmu4JTAMYubEfwVOSwpQ1I1sKKFcxhZCI2buerfw==", - "peer": true + "integrity": "sha512-JVzAR/AjBvVt2BmYhxRCSYysDsPcssdmTFnzyLEts9qNwmjmu4JTAMYubEfwVOSwpQ1I1sKKFcxhZCI2buerfw==" }, "node_modules/js-base64": { "version": "2.6.4", @@ -16691,6 +16909,11 @@ "graceful-fs": "^4.1.11" } }, + "node_modules/klayjs": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/klayjs/-/klayjs-0.4.1.tgz", + "integrity": "sha512-WUNxuO7O79TEkxCj6OIaK5TJBkaWaR/IKNTakgV9PwDn+mrr63MLHed34AcE2yTaDntgO6l0zGFIzhcoTeroTA==" + }, "node_modules/klona": { "version": "2.0.5", "resolved": "https://registry.npmmirror.com/klona/-/klona-2.0.5.tgz", @@ -18013,12 +18236,25 @@ "node": ">=4" } }, + "node_modules/material-colors": { + "version": "1.2.6", + "resolved": "https://registry.npmjs.org/material-colors/-/material-colors-1.2.6.tgz", + "integrity": "sha512-6qE4B9deFBIa9YSpOc9O0Sgc43zTeVYbgDT5veRKSlB2+ZuHNoVVxA1L/ckMUayV9Ay9y7Z/SZCLcGteW9i7bg==" + }, "node_modules/math-random": { "version": "1.0.4", "resolved": "https://registry.npmmirror.com/math-random/-/math-random-1.0.4.tgz", "integrity": "sha512-rUxjysqif/BZQH2yhd5Aaq7vXMSx9NdEsQcyA07uEzIvxgI7zIr33gGsh+RU0/XjmQpCW7RsVof1vlkvQVCK5A==", "dev": true }, + "node_modules/mavon-editor": { + "version": "2.10.4", + "resolved": "https://registry.npmjs.org/mavon-editor/-/mavon-editor-2.10.4.tgz", + "integrity": "sha512-CFsBLkgt/KZBDg+SJYe2fyYv4zClY149PiwpH0rDAiiP4ae1XNs0GC8nBsoTeipsHcebDLN1QMkt3bUsnMDjQw==", + "dependencies": { + "xss": "^1.0.6" + } + }, "node_modules/md5": { "version": "2.3.0", "resolved": "https://registry.npmmirror.com/md5/-/md5-2.3.0.tgz", @@ -18829,6 +19065,25 @@ "node": "*" } }, + "node_modules/moment-timezone": { + "version": "0.5.45", + "resolved": "https://registry.npmjs.org/moment-timezone/-/moment-timezone-0.5.45.tgz", + "integrity": "sha512-HIWmqA86KcmCAhnMAN0wuDOARV/525R2+lOLotuGFzn4HO+FH+/645z2wx0Dt3iDv6/p61SIvKnDstISainhLQ==", + "dependencies": { + "moment": "^2.29.4" + }, + "engines": { + "node": "*" + } + }, + "node_modules/moment-timezone/node_modules/moment": { + "version": "2.30.1", + "resolved": "https://registry.npmjs.org/moment/-/moment-2.30.1.tgz", + "integrity": "sha512-uEmtNhbDOrWPFS+hdjFCBfy9f2YoyzRpwcl+DqpC6taX21FzsTLQVbMV/W7PzNSX6x/bhC1zA3c2UQ5NzH6how==", + "engines": { + "node": "*" + } + }, "node_modules/monaco-editor": { "version": "0.19.3", "resolved": "https://registry.npmmirror.com/monaco-editor/-/monaco-editor-0.19.3.tgz", @@ -18847,6 +19102,20 @@ "webpack": "^4.5.0" } }, + "node_modules/monaco-languageclient": { + "version": "0.13.0", + "resolved": "https://registry.npmjs.org/monaco-languageclient/-/monaco-languageclient-0.13.0.tgz", + "integrity": "sha512-aCwd33dTitwV5QwY56rpYHwzEGXei8TZ+yvZcvP3gEMd6Mizr8m3pOuoknDi2SUfLuNAHS6+ulvLgZlNQB5awg==", + "dependencies": { + "glob-to-regexp": "^0.3.0", + "vscode-jsonrpc": "^5.0.0", + "vscode-languageclient": "^6.0.0", + "vscode-uri": "^2.1.1" + }, + "engines": { + "vscode": "^1.41.0" + } + }, "node_modules/move-concurrently": { "version": "1.0.1", "resolved": "https://registry.npmmirror.com/move-concurrently/-/move-concurrently-1.0.1.tgz", @@ -18972,7 +19241,7 @@ "version": "2.15.0", "resolved": "https://registry.npmmirror.com/nan/-/nan-2.15.0.tgz", "integrity": "sha512-8ZtvEnA2c5aYCZYd1cvgdnU6cqwixRoYg70xPLWUws5ORTa/lnw+u4amixRS/Ac5U5mQVgp9pnlSUnbNWFaWZQ==", - "devOptional": true + "optional": true }, "node_modules/nanomatch": { "version": "1.2.13", @@ -19183,6 +19452,8 @@ "integrity": "sha512-f+Rbqt92Ful9gX0cGtdYwjTrWAaGURgaK5rZCWOgCNyGWusFYHhbqCCBoFBeat+HKETOU02AyTxNhJV0YZf2jQ==", "dev": true, "hasInstallScript": true, + "optional": true, + "peer": true, "dependencies": { "async-foreach": "^0.1.3", "chalk": "^1.1.1", @@ -19212,6 +19483,8 @@ "resolved": "https://registry.npmmirror.com/ansi-regex/-/ansi-regex-2.1.1.tgz", "integrity": "sha512-TIGnTpdo+E3+pCyAluZvtED5p5wCqLdezCyhPZzKPcxvFplEt4i+W7OONCKgeZFT3+y5NZZfOOS/Bdcanm1MYA==", "dev": true, + "optional": true, + "peer": true, "engines": { "node": ">=0.10.0" } @@ -19221,6 +19494,8 @@ "resolved": "https://registry.npmmirror.com/ansi-styles/-/ansi-styles-2.2.1.tgz", "integrity": "sha512-kmCevFghRiWM7HB5zTPULl4r9bVFSWjz62MhqizDGUrq2NWuNMQyuv4tHHoKJHs69M/MF64lEcHdYIocrdWQYA==", "dev": true, + "optional": true, + "peer": true, "engines": { "node": ">=0.10.0" } @@ -19230,6 +19505,8 @@ "resolved": "https://registry.npmmirror.com/chalk/-/chalk-1.1.3.tgz", "integrity": "sha512-U3lRVLMSlsCfjqYPbLyVv11M9CPW4I728d6TCKMAOJueEeB9/8o+eSsMnxPJD+Q+K909sdESg7C+tIkoH6on1A==", "dev": true, + "optional": true, + "peer": true, "dependencies": { "ansi-styles": "^2.2.1", "escape-string-regexp": "^1.0.2", @@ -19246,6 +19523,8 @@ "resolved": "https://registry.npmmirror.com/chownr/-/chownr-2.0.0.tgz", "integrity": "sha512-bIomtDF5KGpdogkLd9VspvFzk9KfpyyGlS8YFVZl7TGPBHL5snIOnxeshwVgPteQ9b4Eydl+pVbIyE1DcvCWgQ==", "dev": true, + "optional": true, + "peer": true, "engines": { "node": ">=10" } @@ -19255,6 +19534,8 @@ "resolved": "https://registry.npmmirror.com/cross-spawn/-/cross-spawn-7.0.3.tgz", "integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==", "dev": true, + "optional": true, + "peer": true, "dependencies": { "path-key": "^3.1.0", "shebang-command": "^2.0.0", @@ -19269,6 +19550,8 @@ "resolved": "https://registry.npmmirror.com/find-up/-/find-up-4.1.0.tgz", "integrity": "sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==", "dev": true, + "optional": true, + "peer": true, "dependencies": { "locate-path": "^5.0.0", "path-exists": "^4.0.0" @@ -19282,6 +19565,8 @@ "resolved": "https://registry.npmmirror.com/fs-minipass/-/fs-minipass-2.1.0.tgz", "integrity": "sha512-V/JgOLFCS+R6Vcq0slCuaeWEdNC3ouDlJMNIsacH2VtALiu9mV4LPrHc5cDl8k5aw6J8jwgWWpiTo5RYhmIzvg==", "dev": true, + "optional": true, + "peer": true, "dependencies": { "minipass": "^3.0.0" }, @@ -19294,6 +19579,8 @@ "resolved": "https://registry.npmmirror.com/get-stdin/-/get-stdin-4.0.1.tgz", "integrity": "sha512-F5aQMywwJ2n85s4hJPTT9RPxGmubonuB10MNYo17/xph174n2MIR33HRguhzVag10O/npM7SPk73LMZNP+FaWw==", "dev": true, + "optional": true, + "peer": true, "engines": { "node": ">=0.10.0" } @@ -19303,6 +19590,8 @@ "resolved": "https://registry.npmmirror.com/hosted-git-info/-/hosted-git-info-4.1.0.tgz", "integrity": "sha512-kyCuEOWjJqZuDbRHzL8V93NzQhwIB71oFWSyzVo+KPZI+pnQPPxucdkrOZvkLRnrf5URsQM+IJ09Dw29cRALIA==", "dev": true, + "optional": true, + "peer": true, "dependencies": { "lru-cache": "^6.0.0" }, @@ -19315,6 +19604,8 @@ "resolved": "https://registry.npmmirror.com/locate-path/-/locate-path-5.0.0.tgz", "integrity": "sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g==", "dev": true, + "optional": true, + "peer": true, "dependencies": { "p-locate": "^4.1.0" }, @@ -19327,6 +19618,8 @@ "resolved": "https://registry.npmmirror.com/lru-cache/-/lru-cache-6.0.0.tgz", "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", "dev": true, + "optional": true, + "peer": true, "dependencies": { "yallist": "^4.0.0" }, @@ -19339,6 +19632,8 @@ "resolved": "https://registry.npmmirror.com/meow/-/meow-9.0.0.tgz", "integrity": "sha512-+obSblOQmRhcyBt62furQqRAQpNyWXo8BuQ5bN7dG8wmwQ+vwHKp/rCFD4CrTP8CsDQD1sjoZ94K417XEUk8IQ==", "dev": true, + "optional": true, + "peer": true, "dependencies": { "@types/minimist": "^1.2.0", "camelcase-keys": "^6.2.2", @@ -19362,6 +19657,8 @@ "resolved": "https://registry.npmmirror.com/minipass/-/minipass-3.3.4.tgz", "integrity": "sha512-I9WPbWHCGu8W+6k1ZiGpPu0GkoKBeorkfKNuAFBNS1HNFJvke82sxvI5bzcCNpWPorkOO5QQ+zomzzwRxejXiw==", "dev": true, + "optional": true, + "peer": true, "dependencies": { "yallist": "^4.0.0" }, @@ -19374,6 +19671,8 @@ "resolved": "https://registry.npmmirror.com/minizlib/-/minizlib-2.1.2.tgz", "integrity": "sha512-bAxsR8BVfj60DWXHE3u30oHzfl4G7khkSuPW+qvpd7jFRHm7dLxOjUk1EHACJ/hxLY8phGJ0YhYHZo7jil7Qdg==", "dev": true, + "optional": true, + "peer": true, "dependencies": { "minipass": "^3.0.0", "yallist": "^4.0.0" @@ -19387,6 +19686,8 @@ "resolved": "https://registry.npmmirror.com/mkdirp/-/mkdirp-1.0.4.tgz", "integrity": "sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw==", "dev": true, + "optional": true, + "peer": true, "bin": { "mkdirp": "bin/cmd.js" }, @@ -19399,6 +19700,8 @@ "resolved": "https://registry.npmmirror.com/node-gyp/-/node-gyp-7.1.2.tgz", "integrity": "sha512-CbpcIo7C3eMu3dL1c3d0xw449fHIGALIJsRP4DDPHpyiW8vcriNY7ubh9TE4zEKfSxscY7PjeFnshE7h75ynjQ==", "dev": true, + "optional": true, + "peer": true, "dependencies": { "env-paths": "^2.2.0", "glob": "^7.1.4", @@ -19423,6 +19726,8 @@ "resolved": "https://registry.npmmirror.com/nopt/-/nopt-5.0.0.tgz", "integrity": "sha512-Tbj67rffqceeLpcRXrT7vKAN8CwfPeIBgM7E6iBkmKLV7bEMwpGgYLGv0jACUsECaa/vuxP0IjEont6umdMgtQ==", "dev": true, + "optional": true, + "peer": true, "dependencies": { "abbrev": "1" }, @@ -19438,6 +19743,8 @@ "resolved": "https://registry.npmmirror.com/normalize-package-data/-/normalize-package-data-3.0.3.tgz", "integrity": "sha512-p2W1sgqij3zMMyRC067Dg16bfzVH+w7hyegmpIvZ4JNjqtGOVAIvLmjBx3yP7YTe9vKJgkoNOPjwQGogDoMXFA==", "dev": true, + "optional": true, + "peer": true, "dependencies": { "hosted-git-info": "^4.0.1", "is-core-module": "^2.5.0", @@ -19453,6 +19760,8 @@ "resolved": "https://registry.npmmirror.com/p-locate/-/p-locate-4.1.0.tgz", "integrity": "sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A==", "dev": true, + "optional": true, + "peer": true, "dependencies": { "p-limit": "^2.2.0" }, @@ -19465,6 +19774,8 @@ "resolved": "https://registry.npmmirror.com/path-exists/-/path-exists-4.0.0.tgz", "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==", "dev": true, + "optional": true, + "peer": true, "engines": { "node": ">=8" } @@ -19474,6 +19785,8 @@ "resolved": "https://registry.npmmirror.com/path-key/-/path-key-3.1.1.tgz", "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", "dev": true, + "optional": true, + "peer": true, "engines": { "node": ">=8" } @@ -19483,6 +19796,8 @@ "resolved": "https://registry.npmmirror.com/read-pkg-up/-/read-pkg-up-7.0.1.tgz", "integrity": "sha512-zK0TB7Xd6JpCLmlLmufqykGE+/TlOePD6qKClNW7hHDKFh/J7/7gCWGR7joEQEW1bKq3a3yUZSObOoWLFQ4ohg==", "dev": true, + "optional": true, + "peer": true, "dependencies": { "find-up": "^4.1.0", "read-pkg": "^5.2.0", @@ -19497,6 +19812,8 @@ "resolved": "https://registry.npmmirror.com/type-fest/-/type-fest-0.8.1.tgz", "integrity": "sha512-4dbzIzqvjtgiM5rw1k5rEHtBANKmdudhGyBEajN01fEyhaAIhsoKNy6y7+IN93IfpFtwY9iqi7kD+xwKhQsNJA==", "dev": true, + "optional": true, + "peer": true, "engines": { "node": ">=8" } @@ -19506,6 +19823,8 @@ "resolved": "https://registry.npmmirror.com/rimraf/-/rimraf-3.0.2.tgz", "integrity": "sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==", "dev": true, + "optional": true, + "peer": true, "dependencies": { "glob": "^7.1.3" }, @@ -19518,6 +19837,8 @@ "resolved": "https://registry.npmmirror.com/semver/-/semver-7.3.8.tgz", "integrity": "sha512-NB1ctGL5rlHrPJtFDVIVzTyQylMLu9N9VICA6HSFJo8MCGVTMW6gfpicwKmmK/dAjTOrqu5l63JJOpDSrAis3A==", "dev": true, + "optional": true, + "peer": true, "dependencies": { "lru-cache": "^6.0.0" }, @@ -19533,6 +19854,8 @@ "resolved": "https://registry.npmmirror.com/shebang-command/-/shebang-command-2.0.0.tgz", "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", "dev": true, + "optional": true, + "peer": true, "dependencies": { "shebang-regex": "^3.0.0" }, @@ -19545,6 +19868,8 @@ "resolved": "https://registry.npmmirror.com/shebang-regex/-/shebang-regex-3.0.0.tgz", "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", "dev": true, + "optional": true, + "peer": true, "engines": { "node": ">=8" } @@ -19554,6 +19879,8 @@ "resolved": "https://registry.npmmirror.com/strip-ansi/-/strip-ansi-3.0.1.tgz", "integrity": "sha512-VhumSSbBqDTP8p2ZLKj40UjBCV4+v8bUSEpUb4KjRgWk9pbqGF4REFj6KEagidb2f/M6AzC0EmFyDNGaw9OCzg==", "dev": true, + "optional": true, + "peer": true, "dependencies": { "ansi-regex": "^2.0.0" }, @@ -19566,6 +19893,8 @@ "resolved": "https://registry.npmmirror.com/supports-color/-/supports-color-2.0.0.tgz", "integrity": "sha512-KKNVtd6pCYgPIKU4cp2733HWYCpplQhddZLBUryaAHou723x+FRzQ5Df824Fj+IyyuiQTRoub4SnIFfIcrp70g==", "dev": true, + "optional": true, + "peer": true, "engines": { "node": ">=0.8.0" } @@ -19575,6 +19904,8 @@ "resolved": "https://registry.npmmirror.com/tar/-/tar-6.1.12.tgz", "integrity": "sha512-jU4TdemS31uABHd+Lt5WEYJuzn+TJTCBLljvIAHZOz6M9Os5pJ4dD+vRFLxPa/n3T0iEFzpi+0x1UfuDZYbRMw==", "dev": true, + "optional": true, + "peer": true, "dependencies": { "chownr": "^2.0.0", "fs-minipass": "^2.0.0", @@ -19592,6 +19923,8 @@ "resolved": "https://registry.npmmirror.com/type-fest/-/type-fest-0.18.1.tgz", "integrity": "sha512-OIAYXk8+ISY+qTOwkHtKqzAuxchoMiD9Udx+FSGQDuiRR+PJKJHc2NJAXlbhkGwTt/4/nKZxELY1w3ReWOL8mw==", "dev": true, + "optional": true, + "peer": true, "engines": { "node": ">=10" } @@ -19601,6 +19934,8 @@ "resolved": "https://registry.npmmirror.com/which/-/which-2.0.2.tgz", "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", "dev": true, + "optional": true, + "peer": true, "dependencies": { "isexe": "^2.0.0" }, @@ -19615,7 +19950,9 @@ "version": "4.0.0", "resolved": "https://registry.npmmirror.com/yallist/-/yallist-4.0.0.tgz", "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", - "dev": true + "dev": true, + "optional": true, + "peer": true }, "node_modules/nopt": { "version": "4.0.3", @@ -22553,7 +22890,7 @@ "version": "3.6.0", "resolved": "https://registry.npmmirror.com/readdirp/-/readdirp-3.6.0.tgz", "integrity": "sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA==", - "optional": true, + "devOptional": true, "dependencies": { "picomatch": "^2.2.1" }, @@ -22561,6 +22898,11 @@ "node": ">=8.10.0" } }, + "node_modules/reconnecting-websocket": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/reconnecting-websocket/-/reconnecting-websocket-4.4.0.tgz", + "integrity": "sha512-D2E33ceRPga0NvTDhJmphEgJ7FUYF0v4lr1ki0csq06OdlxKfugGzN0dSkxM/NfqCxYELK4KcaTOUOjTV6Dcng==" + }, "node_modules/redent": { "version": "3.0.0", "resolved": "https://registry.npmmirror.com/redent/-/redent-3.0.0.tgz", @@ -22607,8 +22949,7 @@ "node_modules/regenerator-runtime": { "version": "0.13.9", "resolved": "https://registry.npmmirror.com/regenerator-runtime/-/regenerator-runtime-0.13.9.tgz", - "integrity": "sha512-p3VT+cOEgxFsRRA9X4lkI1E+k2/CtnKtU4gcxyaCUreilL/vqI6CdZ3wxVUx3UOUg+gnUOQQcRI7BmSI656MYA==", - "dev": true + "integrity": "sha512-p3VT+cOEgxFsRRA9X4lkI1E+k2/CtnKtU4gcxyaCUreilL/vqI6CdZ3wxVUx3UOUg+gnUOQQcRI7BmSI656MYA==" }, "node_modules/regenerator-transform": { "version": "0.15.0", @@ -23170,11 +23511,30 @@ "resolved": "https://registry.npmmirror.com/safer-buffer/-/safer-buffer-2.1.2.tgz", "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" }, + "node_modules/sass": { + "version": "1.49.0", + "resolved": "https://registry.npmjs.org/sass/-/sass-1.49.0.tgz", + "integrity": "sha512-TVwVdNDj6p6b4QymJtNtRS2YtLJ/CqZriGg0eIAbAKMlN8Xy6kbv33FsEZSF7FufFFM705SQviHjjThfaQ4VNw==", + "dev": true, + "dependencies": { + "chokidar": ">=3.0.0 <4.0.0", + "immutable": "^4.0.0", + "source-map-js": ">=0.6.2 <2.0.0" + }, + "bin": { + "sass": "sass.js" + }, + "engines": { + "node": ">=8.9.0" + } + }, "node_modules/sass-graph": { "version": "2.2.5", "resolved": "https://registry.npmmirror.com/sass-graph/-/sass-graph-2.2.5.tgz", "integrity": "sha512-VFWDAHOe6mRuT4mZRd4eKE+d8Uedrk6Xnh7Sh9b4NGufQLQjOrvf/MQoOdx+0s92L89FeyUUNfU597j/3uNpag==", "dev": true, + "optional": true, + "peer": true, "dependencies": { "glob": "^7.0.0", "lodash": "^4.0.0", @@ -23189,13 +23549,17 @@ "version": "7.0.3", "resolved": "https://registry.npmmirror.com/emoji-regex/-/emoji-regex-7.0.3.tgz", "integrity": "sha512-CwBLREIQ7LvYFB0WyRvwhq5N5qPhc6PMjD6bYggFlI5YyDgl+0vxq5VHbMOFqLg7hfWzmu8T5Z1QofhmTIhItA==", - "dev": true + "dev": true, + "optional": true, + "peer": true }, "node_modules/sass-graph/node_modules/string-width": { "version": "3.1.0", "resolved": "https://registry.npmmirror.com/string-width/-/string-width-3.1.0.tgz", "integrity": "sha512-vafcv6KjVZKSgz06oM/H6GDBrAtz8vdhQakGjFIvNrHA6y3HCF1CInLy+QLq8dTJPQ1b+KDUqDFctkdRW44e1w==", "dev": true, + "optional": true, + "peer": true, "dependencies": { "emoji-regex": "^7.0.1", "is-fullwidth-code-point": "^2.0.0", @@ -23210,6 +23574,8 @@ "resolved": "https://registry.npmmirror.com/yargs/-/yargs-13.3.2.tgz", "integrity": "sha512-AX3Zw5iPruN5ie6xGRIDgqkT+ZhnRlZMLMHAs8tg7nRruy2Nb+i5o9bwghAogtM08q1dpr2LVoS8KSTMYpWXUw==", "dev": true, + "optional": true, + "peer": true, "dependencies": { "cliui": "^5.0.0", "find-up": "^3.0.0", @@ -23228,6 +23594,8 @@ "resolved": "https://registry.npmmirror.com/yargs-parser/-/yargs-parser-13.1.2.tgz", "integrity": "sha512-3lbsNRf/j+A4QuSZfDRA7HRSfWrzO0YjqTJd5kjAq37Zep1CEgaYmrH9Q3GwPiB9cHyd1Y1UwggGhJGoxipbzg==", "dev": true, + "optional": true, + "peer": true, "dependencies": { "camelcase": "^5.0.0", "decamelize": "^1.2.0" @@ -23362,6 +23730,8 @@ "resolved": "https://registry.npmmirror.com/scss-tokenizer/-/scss-tokenizer-0.2.3.tgz", "integrity": "sha512-dYE8LhncfBUar6POCxMTm0Ln+erjeczqEvCJib5/7XNkdw1FkUGgwMPY360FY0FgPWQxHWCx29Jl3oejyGLM9Q==", "dev": true, + "optional": true, + "peer": true, "dependencies": { "js-base64": "^2.1.8", "source-map": "^0.4.2" @@ -23372,6 +23742,8 @@ "resolved": "https://registry.npmmirror.com/source-map/-/source-map-0.4.4.tgz", "integrity": "sha512-Y8nIfcb1s/7DcobUz1yOO1GSp7gyL+D9zLHDehT7iRESqGSxjJ448Sg7rvfgsRJCnKLdSl11uGf0s9X80cH0/A==", "dev": true, + "optional": true, + "peer": true, "dependencies": { "amdefine": ">=0.0.4" }, @@ -23379,6 +23751,11 @@ "node": ">=0.8.0" } }, + "node_modules/select": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/select/-/select-1.1.2.tgz", + "integrity": "sha512-OwpTSOfy6xSs1+pwcNrv0RBMOzI39Lp3qQKUTPVVPRjCdNa5JH/oPRiqsesIskK8TVgmRiHwO4KXlV2Li9dANA==" + }, "node_modules/select-hose": { "version": "2.0.0", "resolved": "https://registry.npmmirror.com/select-hose/-/select-hose-2.0.0.tgz", @@ -23968,6 +24345,11 @@ "node": ">=0.10.0" } }, + "node_modules/sortablejs": { + "version": "1.10.2", + "resolved": "https://registry.npmjs.org/sortablejs/-/sortablejs-1.10.2.tgz", + "integrity": "sha512-YkPGufevysvfwn5rfdlGyrGjt7/CRHwvRPogD/lC+TnvcN29jDpCifKP+rBqf+LRldfXSTh+0CGLcSg0VIxq3A==" + }, "node_modules/source-list-map": { "version": "2.0.1", "resolved": "https://registry.npmmirror.com/source-list-map/-/source-list-map-2.0.1.tgz", @@ -23981,6 +24363,15 @@ "node": ">=0.10.0" } }, + "node_modules/source-map-js": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.0.tgz", + "integrity": "sha512-itJW8lvSA0TXEphiRoawsCksnlf8SyvmFzIhltqAHluXd88pkCd+cXJVHTDwdCr0IzwptSm035IHQktUu1QUMg==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/source-map-resolve": { "version": "0.5.3", "resolved": "https://registry.npmmirror.com/source-map-resolve/-/source-map-resolve-0.5.3.tgz", @@ -24369,6 +24760,8 @@ "resolved": "https://registry.npmmirror.com/stdout-stream/-/stdout-stream-1.4.1.tgz", "integrity": "sha512-j4emi03KXqJWcIeF8eIXkjMFN1Cmb8gUlDYGeBALLPo5qdyTfA9bOtl8m33lRoC+vFMkP3gl0WsDr6+gzxbbTA==", "dev": true, + "optional": true, + "peer": true, "dependencies": { "readable-stream": "^2.0.1" } @@ -25332,6 +25725,11 @@ "integrity": "sha512-qsdtZH+vMoCARQtyod4imc2nIJwg9Cc7lPRrw9CzF8ZKR0khdr8+2nX80PBhET3tcyTtJDxAffGh2rXH4tyU8A==", "dev": true }, + "node_modules/tiny-emitter": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/tiny-emitter/-/tiny-emitter-2.1.0.tgz", + "integrity": "sha512-NB6Dk1A9xgQPMoGqC5CVXn123gWyte215ONT5Pp5a0yt4nlEoO1ZWeCwpncaekPHXO60i47ihFnZPiRPjRMq4Q==" + }, "node_modules/tinycolor2": { "version": "1.4.2", "resolved": "https://registry.npmmirror.com/tinycolor2/-/tinycolor2-1.4.2.tgz", @@ -25474,6 +25872,8 @@ "resolved": "https://registry.npmmirror.com/true-case-path/-/true-case-path-1.0.3.tgz", "integrity": "sha512-m6s2OdQe5wgpFMC+pAJ+q9djG82O2jcHPOI6RNg1yy9rCYR+WD6Nbpl32fDpfC56nirdRy+opFa/Vk7HYhqaew==", "dev": true, + "optional": true, + "peer": true, "dependencies": { "glob": "^7.1.2" } @@ -26018,6 +26418,61 @@ "resolved": "https://registry.npmmirror.com/vm-browserify/-/vm-browserify-1.1.2.tgz", "integrity": "sha512-2ham8XPWTONajOR0ohOKOHXkm3+gaBmGut3SRuu75xLd/RRaY6vqgh8NBYYk7+RW3u5AtzPQZG8F10LHkl0lAQ==" }, + "node_modules/vscode-jsonrpc": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/vscode-jsonrpc/-/vscode-jsonrpc-5.0.1.tgz", + "integrity": "sha512-JvONPptw3GAQGXlVV2utDcHx0BiY34FupW/kI6mZ5x06ER5DdPG/tXWMVHjTNULF5uKPOUUD0SaXg5QaubJL0A==", + "engines": { + "node": ">=8.0.0 || >=10.0.0" + } + }, + "node_modules/vscode-languageclient": { + "version": "6.1.4", + "resolved": "https://registry.npmjs.org/vscode-languageclient/-/vscode-languageclient-6.1.4.tgz", + "integrity": "sha512-EUOU+bJu6axmt0RFNo3nrglQLPXMfanbYViJee3Fbn2VuQoX0ZOI4uTYhSRvYLP2vfwTP/juV62P/mksCdTZMA==", + "dependencies": { + "semver": "^6.3.0", + "vscode-languageserver-protocol": "3.15.3" + }, + "engines": { + "vscode": "^1.41.0" + } + }, + "node_modules/vscode-languageclient/node_modules/semver": { + "version": "6.3.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", + "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==", + "bin": { + "semver": "bin/semver.js" + } + }, + "node_modules/vscode-languageserver-protocol": { + "version": "3.15.3", + "resolved": "https://registry.npmjs.org/vscode-languageserver-protocol/-/vscode-languageserver-protocol-3.15.3.tgz", + "integrity": "sha512-zrMuwHOAQRhjDSnflWdJG+O2ztMWss8GqUUB8dXLR/FPenwkiBNkMIJJYfSN6sgskvsF0rHAoBowNQfbyZnnvw==", + "dependencies": { + "vscode-jsonrpc": "^5.0.1", + "vscode-languageserver-types": "3.15.1" + } + }, + "node_modules/vscode-languageserver-types": { + "version": "3.15.1", + "resolved": "https://registry.npmjs.org/vscode-languageserver-types/-/vscode-languageserver-types-3.15.1.tgz", + "integrity": "sha512-+a9MPUQrNGRrGU630OGbYVQ+11iOIovjCkqxajPa9w57Sd5ruK8WQNsslzpa0x/QJqC8kRc2DUxWjIFwoNm4ZQ==" + }, + "node_modules/vscode-uri": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/vscode-uri/-/vscode-uri-2.1.2.tgz", + "integrity": "sha512-8TEXQxlldWAuIODdukIb+TR5s+9Ds40eSJrw+1iDDA9IFORPjMELarNQE3myz5XIkWWpdprmJjm1/SxMlWOC8A==" + }, + "node_modules/vscode-ws-jsonrpc": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/vscode-ws-jsonrpc/-/vscode-ws-jsonrpc-0.2.0.tgz", + "integrity": "sha512-NE9HNRgPjCaPyTJvIudcpyIWPImxwRDtuTX16yks7SAiZgSXigxAiZOvSvVBGmD1G/OMfrFo6BblOtjVR9DdVA==", + "dependencies": { + "vscode-jsonrpc": "^5.0.0" + } + }, "node_modules/vue": { "version": "2.6.12", "resolved": "https://registry.npmmirror.com/vue/-/vue-2.6.12.tgz", @@ -26152,6 +26607,14 @@ "integrity": "sha512-4gDntzrifFnCEvyoO8PqyJDmguXgVPxKiIxrBKjIowvL9l+N66196+72XVYR8BBf1Uv1Fgt3bGevJ+sEmxfZzw==", "dev": true }, + "node_modules/vuedraggable": { + "version": "2.24.3", + "resolved": "https://registry.npmjs.org/vuedraggable/-/vuedraggable-2.24.3.tgz", + "integrity": "sha512-6/HDXi92GzB+Hcs9fC6PAAozK1RLt1ewPTLjK0anTYguXLAeySDmcnqE8IC0xa7shvSzRjQXq3/+dsZ7ETGF3g==", + "dependencies": { + "sortablejs": "1.10.2" + } + }, "node_modules/vuescroll": { "version": "4.16.1", "resolved": "https://registry.npmmirror.com/vuescroll/-/vuescroll-4.16.1.tgz", @@ -26160,6 +26623,11 @@ "vue": "^2.0.0" } }, + "node_modules/watch-size": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/watch-size/-/watch-size-2.0.0.tgz", + "integrity": "sha512-M92R89dNoTPWyCD+HuUEDdhaDnh9jxPGOwlDc0u51jAgmjUvzqaEMynXSr3BaWs+QdHYk4KzibPy1TFtjLmOZQ==" + }, "node_modules/watchpack": { "version": "1.7.5", "resolved": "https://registry.npmmirror.com/watchpack/-/watchpack-1.7.5.tgz", @@ -27225,6 +27693,21 @@ "async-limiter": "~1.0.0" } }, + "node_modules/xss": { + "version": "1.0.15", + "resolved": "https://registry.npmjs.org/xss/-/xss-1.0.15.tgz", + "integrity": "sha512-FVdlVVC67WOIPvfOwhoMETV72f6GbW7aOabBC3WxN/oUdoEMDyLz4OgRv5/gck2ZeNqEQu+Tb0kloovXOfpYVg==", + "dependencies": { + "commander": "^2.20.3", + "cssfilter": "0.0.10" + }, + "bin": { + "xss": "bin/xss" + }, + "engines": { + "node": ">= 0.10.0" + } + }, "node_modules/xtend": { "version": "4.0.2", "resolved": "https://registry.npmmirror.com/xtend/-/xtend-4.0.2.tgz", @@ -27474,6 +27957,115 @@ "version": "4.3.2", "resolved": "https://registry.npmmirror.com/zrender/-/zrender-4.3.2.tgz", "integrity": "sha512-bIusJLS8c4DkIcdiK+s13HiQ/zjQQVgpNohtd8d94Y2DnJqgM1yjh/jpDb8DoL6hd7r8Awagw8e3qK/oLaWr3g==" + }, + "packages/apiServices": { + "name": "@dataspherestudio/apiServices", + "version": "1.1.19", + "dependencies": { + "@dataspherestudio/shared": "^1.1.19" + } + }, + "packages/cyeditor": { + "name": "@dataspherestudio/cyeditor", + "version": "0.0.1", + "dependencies": { + "cytoscape": "3.27.0", + "cytoscape-klay": "^3.1.4" + }, + "devDependencies": {} + }, + "packages/dataGovernance": { + "name": "@dataspherestudio/dataGovernance", + "version": "1.1.19", + "dependencies": { + "@dataspherestudio/shared": "^1.1.19", + "clipboard": "2.0.8", + "jquery": "3.6.0" + } + }, + "packages/dolphinScheduler": { + "name": "@dataspherestudio/dolphinScheduler", + "version": "1.1.19", + "dependencies": { + "@dataspherestudio/shared": "^1.1.19", + "bootstrap": "^3.4.1", + "d3": "3.5.17", + "dayjs": "1.10.7", + "moment-timezone": "^0.5.34" + } + }, + "packages/dss": { + "name": "@dataspherestudio/dss", + "version": "1.1.19", + "dependencies": { + "@dataspherestudio/scriptis": "^1.1.19", + "@dataspherestudio/shared": "^1.1.19" + }, + "devDependencies": { + "@vue/cli-plugin-babel": "3.12.1", + "@vue/cli-plugin-eslint": "3.12.1", + "@vue/cli-service": "3.12.1", + "@vue/eslint-config-standard": "4.0.0", + "babel-plugin-transform-remove-console": "6.9.4" + } + }, + "packages/editor": { + "name": "@dataspherestudio/editor", + "version": "1.1.19", + "dependencies": { + "@dataspherestudio/shared": "^1.1.19", + "dt-sql-parser": "1.2.1" + } + }, + "packages/editorLsp": { + "name": "@dataspherestudio/editor-lsp", + "version": "1.1.19", + "dependencies": { + "@dataspherestudio/shared": "^1.1.19", + "reconnecting-websocket": "4.4.0", + "vscode-ws-jsonrpc": "0.2.0" + } + }, + "packages/exts": { + "name": "@dataspherestudio/exts", + "version": "1.1.19", + "dependencies": { + "@dataspherestudio/shared": "^1.1.19" + } + }, + "packages/scriptis": { + "name": "@dataspherestudio/scriptis", + "version": "1.1.19", + "dependencies": { + "@dataspherestudio/shared": "^1.1.19", + "vuedraggable": "2.24.3" + } + }, + "packages/shared": { + "name": "@dataspherestudio/shared", + "version": "1.1.19", + "dependencies": { + "dexie": "3.0.4" + } + }, + "packages/workflows": { + "name": "@dataspherestudio/workflows", + "version": "1.1.19", + "dependencies": { + "@dataspherestudio/cyeditor": "^0.0.1", + "@dataspherestudio/scriptis": "^1.1.19", + "@dataspherestudio/shared": "^1.1.19" + } + }, + "packages/workspace": { + "name": "@dataspherestudio/workspace", + "version": "1.1.19", + "dependencies": { + "@dataspherestudio/shared": "^1.1.19", + "@riophae/vue-treeselect": "0.4.0", + "highlight.js": "9.18.3", + "mavon-editor": "2.10.4" + } } }, "dependencies": { @@ -28379,7 +28971,6 @@ "version": "7.17.9", "resolved": "https://registry.npmmirror.com/@babel/runtime/-/runtime-7.17.9.tgz", "integrity": "sha512-lSiBBvodq29uShpWGNbgFdKYNiFDo5/HIYsaCEY9ff4sb10x9jizo2+pRrSyF4jKZCXqgzuqBOQKbUm90gQwJg==", - "dev": true, "requires": { "regenerator-runtime": "^0.13.4" } @@ -28450,6 +29041,100 @@ "to-fast-properties": "^2.0.0" } }, + "@dataspherestudio/apiServices": { + "version": "file:packages/apiServices", + "requires": { + "@dataspherestudio/shared": "^1.1.19" + } + }, + "@dataspherestudio/cyeditor": { + "version": "file:packages/cyeditor", + "requires": { + "cytoscape": "3.27.0", + "cytoscape-klay": "^3.1.4" + } + }, + "@dataspherestudio/dataGovernance": { + "version": "file:packages/dataGovernance", + "requires": { + "@dataspherestudio/shared": "^1.1.19", + "clipboard": "2.0.8", + "jquery": "3.6.0" + } + }, + "@dataspherestudio/dolphinScheduler": { + "version": "file:packages/dolphinScheduler", + "requires": { + "@dataspherestudio/shared": "^1.1.19", + "bootstrap": "^3.4.1", + "d3": "3.5.17", + "dayjs": "1.10.7", + "moment-timezone": "^0.5.34" + } + }, + "@dataspherestudio/dss": { + "version": "file:packages/dss", + "requires": { + "@dataspherestudio/scriptis": "^1.1.19", + "@dataspherestudio/shared": "^1.1.19", + "@vue/cli-plugin-babel": "3.12.1", + "@vue/cli-plugin-eslint": "3.12.1", + "@vue/cli-service": "3.12.1", + "@vue/eslint-config-standard": "4.0.0", + "babel-plugin-transform-remove-console": "6.9.4" + } + }, + "@dataspherestudio/editor": { + "version": "file:packages/editor", + "requires": { + "@dataspherestudio/shared": "^1.1.19", + "dt-sql-parser": "1.2.1" + } + }, + "@dataspherestudio/editor-lsp": { + "version": "file:packages/editorLsp", + "requires": { + "@dataspherestudio/shared": "^1.1.19", + "reconnecting-websocket": "4.4.0", + "vscode-ws-jsonrpc": "0.2.0" + } + }, + "@dataspherestudio/exts": { + "version": "file:packages/exts", + "requires": { + "@dataspherestudio/shared": "^1.1.19" + } + }, + "@dataspherestudio/scriptis": { + "version": "file:packages/scriptis", + "requires": { + "@dataspherestudio/shared": "^1.1.19", + "vuedraggable": "2.24.3" + } + }, + "@dataspherestudio/shared": { + "version": "file:packages/shared", + "requires": { + "dexie": "3.0.4" + } + }, + "@dataspherestudio/workflows": { + "version": "file:packages/workflows", + "requires": { + "@dataspherestudio/cyeditor": "^0.0.1", + "@dataspherestudio/scriptis": "^1.1.19", + "@dataspherestudio/shared": "^1.1.19" + } + }, + "@dataspherestudio/workspace": { + "version": "file:packages/workspace", + "requires": { + "@dataspherestudio/shared": "^1.1.19", + "@riophae/vue-treeselect": "0.4.0", + "highlight.js": "9.18.3", + "mavon-editor": "2.10.4" + } + }, "@gar/promisify": { "version": "1.1.3", "resolved": "https://registry.npmmirror.com/@gar/promisify/-/promisify-1.1.3.tgz", @@ -32074,6 +32759,21 @@ "@octokit/openapi-types": "^11.2.0" } }, + "@riophae/vue-treeselect": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/@riophae/vue-treeselect/-/vue-treeselect-0.4.0.tgz", + "integrity": "sha512-J4atYmBqXQmiPFK/0B5sXKjtnGc21mBJEiyKIDZwk0Q9XuynVFX6IJ4EpaLmUgL5Tve7HAS7wkiGGSti6Uaxcg==", + "requires": { + "@babel/runtime": "^7.3.1", + "babel-helper-vue-jsx-merge-props": "^2.0.3", + "easings-css": "^1.0.0", + "fuzzysearch": "^1.0.3", + "is-promise": "^2.1.0", + "lodash": "^4.0.0", + "material-colors": "^1.2.6", + "watch-size": "^2.0.0" + } + }, "@samverschueren/stream-to-observable": { "version": "0.3.1", "resolved": "https://registry.npmmirror.com/@samverschueren/stream-to-observable/-/stream-to-observable-0.3.1.tgz", @@ -33071,7 +33771,9 @@ "version": "1.0.1", "resolved": "https://registry.npmmirror.com/amdefine/-/amdefine-1.0.1.tgz", "integrity": "sha512-S2Hw0TtNkMJhIabBwIojKL9YHO5T0n5eNqWJ7Lrlel/zDbftQpxpapi8tZs3X1HWa+u+QeydGmzzNU0m09+Rcg==", - "dev": true + "dev": true, + "optional": true, + "peer": true }, "ansi-colors": { "version": "3.2.4", @@ -33121,7 +33823,7 @@ "version": "3.1.2", "resolved": "https://registry.npmmirror.com/anymatch/-/anymatch-3.1.2.tgz", "integrity": "sha512-P43ePfOAIupkguHUycrc4qJ9kz8ZiuOUijaETwX7THt0Y/GNK7v0aa8rY816xWjZ7rJdA5XdMcpVFTKMq+RvWg==", - "optional": true, + "devOptional": true, "requires": { "normalize-path": "^3.0.0", "picomatch": "^2.0.4" @@ -33379,7 +34081,9 @@ "version": "0.1.3", "resolved": "https://registry.npmmirror.com/async-foreach/-/async-foreach-0.1.3.tgz", "integrity": "sha512-VUeSMD8nEGBWaZK4lizI1sf3yEC7pnAQ/mrI7pC2fBz2s/tq5jWWEngTwaf0Gruu/OoXRGLGg1XFqpYBiGTYJA==", - "dev": true + "dev": true, + "optional": true, + "peer": true }, "async-limiter": { "version": "1.0.1", @@ -33525,6 +34229,11 @@ "resolve": "^1.12.0" } }, + "babel-helper-vue-jsx-merge-props": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/babel-helper-vue-jsx-merge-props/-/babel-helper-vue-jsx-merge-props-2.0.3.tgz", + "integrity": "sha512-gsLiKK7Qrb7zYJNgiXKpXblxbV5ffSwR0f5whkPAaBAR4fhi6bwRZxX9wBlIc5M/v8CCkXUbXZL4N/nSE97cqg==" + }, "babel-loader": { "version": "8.2.5", "resolved": "https://registry.npmmirror.com/babel-loader/-/babel-loader-8.2.5.tgz", @@ -33696,6 +34405,12 @@ "@babel/helper-define-polyfill-provider": "^0.3.1" } }, + "babel-plugin-transform-remove-console": { + "version": "6.9.4", + "resolved": "https://registry.npmjs.org/babel-plugin-transform-remove-console/-/babel-plugin-transform-remove-console-6.9.4.tgz", + "integrity": "sha512-88blrUrMX3SPiGkT1GnvVY8E/7A+k6oj3MNvUtTIxJflFzXTw1bHkuJ/y039ouhFMp2prRn5cQGzokViYi1dsg==", + "dev": true + }, "babel-polyfill": { "version": "6.26.0", "resolved": "https://registry.npmmirror.com/babel-polyfill/-/babel-polyfill-6.26.0.tgz", @@ -33836,7 +34551,7 @@ "version": "2.2.0", "resolved": "https://registry.npmmirror.com/binary-extensions/-/binary-extensions-2.2.0.tgz", "integrity": "sha512-jDctJ/IVQbZoJykoeHbhXpOlNBqGNcwXJKJog42E5HDPUwQTSdjCHdihjj0DlnheQ7blbT6dHOafNAiS8ooQKA==", - "optional": true + "devOptional": true }, "bindings": { "version": "1.5.0", @@ -33949,6 +34664,11 @@ "resolved": "https://registry.npmmirror.com/boolbase/-/boolbase-1.0.0.tgz", "integrity": "sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww==" }, + "bootstrap": { + "version": "3.4.1", + "resolved": "https://registry.npmjs.org/bootstrap/-/bootstrap-3.4.1.tgz", + "integrity": "sha512-yN5oZVmRCwe5aKwzRj6736nSmKDX7pLYwsXiCj/EYmo16hODaBiT4En5btW/jhBF/seV+XMx3aYwukYC3A49DA==" + }, "brace-expansion": { "version": "1.1.11", "resolved": "https://registry.npmmirror.com/brace-expansion/-/brace-expansion-1.1.11.tgz", @@ -34468,7 +35188,7 @@ "version": "3.5.3", "resolved": "https://registry.npmmirror.com/chokidar/-/chokidar-3.5.3.tgz", "integrity": "sha512-Dr3sfKRP6oTcjf2JmUmFJfeVMvXBdegxB0iVQ5eb2V10uFJUCAS8OByZdVAyVb8xXNz3GjjTgj9kLWsZTqE6kw==", - "optional": true, + "devOptional": true, "requires": { "anymatch": "~3.1.2", "braces": "~3.0.2", @@ -34484,7 +35204,7 @@ "version": "3.0.2", "resolved": "https://registry.npmmirror.com/braces/-/braces-3.0.2.tgz", "integrity": "sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==", - "optional": true, + "devOptional": true, "requires": { "fill-range": "^7.0.1" } @@ -34493,7 +35213,7 @@ "version": "7.0.1", "resolved": "https://registry.npmmirror.com/fill-range/-/fill-range-7.0.1.tgz", "integrity": "sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==", - "optional": true, + "devOptional": true, "requires": { "to-regex-range": "^5.0.1" } @@ -34502,13 +35222,13 @@ "version": "7.0.0", "resolved": "https://registry.npmmirror.com/is-number/-/is-number-7.0.0.tgz", "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==", - "optional": true + "devOptional": true }, "to-regex-range": { "version": "5.0.1", "resolved": "https://registry.npmmirror.com/to-regex-range/-/to-regex-range-5.0.1.tgz", "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==", - "optional": true, + "devOptional": true, "requires": { "is-number": "^7.0.0" } @@ -34737,6 +35457,16 @@ "dev": true, "optional": true }, + "clipboard": { + "version": "2.0.8", + "resolved": "https://registry.npmjs.org/clipboard/-/clipboard-2.0.8.tgz", + "integrity": "sha512-Y6WO0unAIQp5bLmk1zdThRhgJt/x3ks6f30s3oE3H1mgIEU33XyQjEf8gsf6DxC7NPX8Y1SsNWjUjL/ywLnnbQ==", + "requires": { + "good-listener": "^1.2.2", + "select": "^1.1.2", + "tiny-emitter": "^2.0.0" + } + }, "clipboardy": { "version": "2.3.0", "resolved": "https://registry.npmmirror.com/clipboardy/-/clipboardy-2.3.0.tgz", @@ -36346,6 +37076,11 @@ "integrity": "sha512-/Tb/JcjK111nNScGob5MNtsntNM1aCNUDipB/TkwZFhyDrrE47SOx/18wF2bbjgc3ZzCSKW1T5nt5EbFoAz/Vg==", "dev": true }, + "cssfilter": { + "version": "0.0.10", + "resolved": "https://registry.npmjs.org/cssfilter/-/cssfilter-0.0.10.tgz", + "integrity": "sha512-FAaLDaplstoRsDR8XGYH51znUN0UY7nMc6Z9/fvE8EXGwvJE9hu7W2vHwx1+bd6gCYnln9nLbzxFTrcO9YQDZw==" + }, "cssnano": { "version": "4.1.11", "resolved": "https://registry.npmmirror.com/cssnano/-/cssnano-4.1.11.tgz", @@ -36463,6 +37198,35 @@ "resolved": "https://registry.npmmirror.com/cyclist/-/cyclist-1.0.1.tgz", "integrity": "sha512-NJGVKPS81XejHcLhaLJS7plab0fK3slPh11mESeeDq2W4ZI5kUKK/LRRdVDvjJseojbPB7ZwjnyOybg3Igea/A==" }, + "cytoscape": { + "version": "3.27.0", + "resolved": "https://registry.npmjs.org/cytoscape/-/cytoscape-3.27.0.tgz", + "integrity": "sha512-pPZJilfX9BxESwujODz5pydeGi+FBrXq1rcaB1mfhFXXFJ9GjE6CNndAk+8jPzoXGD+16LtSS4xlYEIUiW4Abg==", + "requires": { + "heap": "^0.2.6", + "lodash": "^4.17.21" + }, + "dependencies": { + "lodash": { + "version": "4.17.21", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", + "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==" + } + } + }, + "cytoscape-klay": { + "version": "3.1.4", + "resolved": "https://registry.npmjs.org/cytoscape-klay/-/cytoscape-klay-3.1.4.tgz", + "integrity": "sha512-VwPj0VR25GPfy6qXVQRi/MYlZM/zkdvRhHlgqbM//lSvstgM6fhp3ik/uM8Wr8nlhskfqz/M1fIDmR6UckbS2A==", + "requires": { + "klayjs": "^0.4.1" + } + }, + "d3": { + "version": "3.5.17", + "resolved": "https://registry.npmjs.org/d3/-/d3-3.5.17.tgz", + "integrity": "sha512-yFk/2idb8OHPKkbAL8QaOaqENNoMhIaSHZerk3oQsECwkObkCpJyjYwCe+OHiq6UEdhe1m8ZGARRRO3ljFjlKg==" + }, "d3-dispatch": { "version": "2.0.0", "resolved": "https://registry.npmmirror.com/d3-dispatch/-/d3-dispatch-2.0.0.tgz", @@ -36524,6 +37288,11 @@ "integrity": "sha512-jyCETtSl3VMZMWeRo7iY1FL19ges1t55hMo5yaam4Jrsm5EPL89UQkoQRyiI+Yf4k8r2ZpdngkV8hr1lIdjb3Q==", "dev": true }, + "dayjs": { + "version": "1.10.7", + "resolved": "https://registry.npmjs.org/dayjs/-/dayjs-1.10.7.tgz", + "integrity": "sha512-P6twpd70BcPK34K26uJ1KT3wlhpuOAPoMwJzpsIWUxHZ7wpmbdZL/hQqBDfz7hGurYSa5PhzdhDHtt319hL3ig==" + }, "de-indent": { "version": "1.0.2", "resolved": "https://registry.npmmirror.com/de-indent/-/de-indent-1.0.2.tgz", @@ -36818,6 +37587,11 @@ "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==", "dev": true }, + "delegate": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/delegate/-/delegate-3.2.0.tgz", + "integrity": "sha512-IofjkYBZaZivn0V8nnsMJGBr4jVLxHDheKSW88PyxS5QC4Vo9ZbZVvhzlSxY87fVq3STR6r+4cGepyHkcWOQSw==" + }, "delegates": { "version": "1.0.0", "resolved": "https://registry.npmmirror.com/delegates/-/delegates-1.0.0.tgz", @@ -36863,6 +37637,11 @@ "integrity": "sha512-T0NIuQpnTvFDATNuHN5roPwSBG83rFsuO+MXXH9/3N1eFbn4wcPjttvjMLEPWJ0RGUYgQE7cGgS3tNxbqCGM7g==", "dev": true }, + "dexie": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/dexie/-/dexie-3.0.4.tgz", + "integrity": "sha512-rwS3k8BBstjGFJAS/yjYrsRZucqitnrP3NBhqghl9ihWcWABlT3I5f2dZV9T3pmXGAP7G9p1q00g/axcsuOT8A==" + }, "dezalgo": { "version": "1.0.4", "resolved": "https://registry.npmmirror.com/dezalgo/-/dezalgo-1.0.4.tgz", @@ -37026,6 +37805,11 @@ "integrity": "sha512-YXQl1DSa4/PQyRfgrv6aoNjhasp/p4qs9FjJ4q4cQk+8m4r6k4ZSiEyytKG8f8W9gi8WsQtIObNmKd+tMzNTmA==", "dev": true }, + "dt-sql-parser": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/dt-sql-parser/-/dt-sql-parser-1.2.1.tgz", + "integrity": "sha512-bp6Rtm8N3m2Z3tavtKGo8J5ffaPQdPodYV2xGi9f6IsWbHzOkpNBxHlORcWtnSLtA/JvZcvImQxgbkdaGiv5cw==" + }, "duplexer": { "version": "0.1.2", "resolved": "https://registry.npmmirror.com/duplexer/-/duplexer-0.1.2.tgz", @@ -37043,6 +37827,11 @@ "stream-shift": "^1.0.0" } }, + "easings-css": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/easings-css/-/easings-css-1.0.0.tgz", + "integrity": "sha512-7Uq7NdazNfVtr0RNmPAys8it0zKCuaqxJStYKEl72D3j4gbvXhhaM7iWNbqhA4C94ygCye6VuyhzBRQC4szeBg==" + }, "easy-stack": { "version": "1.0.1", "resolved": "https://registry.npmmirror.com/easy-stack/-/easy-stack-1.0.1.tgz", @@ -38896,6 +39685,11 @@ "integrity": "sha512-xckBUXyTIqT97tq2x2AMb+g163b5JFysYk0x4qxNFwbfQkmNZoiRHb6sPzI9/QV33WeuvVYBUIiD4NzNIyqaRQ==", "dev": true }, + "fuzzysearch": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/fuzzysearch/-/fuzzysearch-1.0.3.tgz", + "integrity": "sha512-s+kNWQuI3mo9OALw0HJ6YGmMbLqEufCh2nX/zzV5CrICQ/y4AwPxM+6TIiF9ItFCHXFCyM/BfCCmN57NTIJuPg==" + }, "g-status": { "version": "2.0.2", "resolved": "https://registry.npmmirror.com/g-status/-/g-status-2.0.2.tgz", @@ -38973,6 +39767,8 @@ "resolved": "https://registry.npmmirror.com/gaze/-/gaze-1.1.3.tgz", "integrity": "sha512-BRdNm8hbWzFzWHERTrejLqwHDfS4GibPoq5wjTPIoJHoBtKGPg3xAFfxmM+9ztbXelxcf2hwQcaz1PtmFeue8g==", "dev": true, + "optional": true, + "peer": true, "requires": { "globule": "^1.0.0" } @@ -39252,8 +40048,7 @@ "glob-to-regexp": { "version": "0.3.0", "resolved": "https://registry.npmmirror.com/glob-to-regexp/-/glob-to-regexp-0.3.0.tgz", - "integrity": "sha512-Iozmtbqv0noj0uDDqoL0zNq0VBEfK2YFoMAZoxJe4cwphvLR+JskfF30QhXHOR4m3KrE6NLRYw+U9MRXvifyig==", - "dev": true + "integrity": "sha512-Iozmtbqv0noj0uDDqoL0zNq0VBEfK2YFoMAZoxJe4cwphvLR+JskfF30QhXHOR4m3KrE6NLRYw+U9MRXvifyig==" }, "glob2base": { "version": "0.0.12", @@ -39299,6 +40094,8 @@ "resolved": "https://registry.npmmirror.com/globule/-/globule-1.3.3.tgz", "integrity": "sha512-mb1aYtDbIjTu4ShMB85m3UzjX9BVKe9WCzsnfMSZk+K5GpIbBOexgg4PPCt5eHDEG5/ZQAUX2Kct02zfiPLsKg==", "dev": true, + "optional": true, + "peer": true, "requires": { "glob": "~7.1.1", "lodash": "~4.17.10", @@ -39310,6 +40107,8 @@ "resolved": "https://registry.npmmirror.com/glob/-/glob-7.1.7.tgz", "integrity": "sha512-OvD9ENzPLbegENnYP5UUfJIirTg4+XwMWGaQfQTY0JenxNvvIKP3U3/tAQSPIu/lHxXYSZmpXlUHeqAIdKzBLQ==", "dev": true, + "optional": true, + "peer": true, "requires": { "fs.realpath": "^1.0.0", "inflight": "^1.0.4", @@ -39324,12 +40123,22 @@ "resolved": "https://registry.npmmirror.com/minimatch/-/minimatch-3.0.8.tgz", "integrity": "sha512-6FsRAQsxQ61mw+qP1ZzbL9Bc78x2p5OqNgNpnoAFLTrX8n5Kxph0CsnhmKKNXTWjXqU5L0pGPR7hYk+XWZr60Q==", "dev": true, + "optional": true, + "peer": true, "requires": { "brace-expansion": "^1.1.7" } } } }, + "good-listener": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/good-listener/-/good-listener-1.2.2.tgz", + "integrity": "sha512-goW1b+d9q/HIwbVYZzZ6SsTr4IgE+WA44A0GmPIQstuOrgsFcT7VEJ48nmr9GaRtNu0XTKacFLGnBPAM6Afouw==", + "requires": { + "delegate": "^3.1.2" + } + }, "graceful-fs": { "version": "4.2.10", "resolved": "https://registry.npmmirror.com/graceful-fs/-/graceful-fs-4.2.10.tgz", @@ -39541,12 +40350,22 @@ "integrity": "sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw==", "dev": true }, + "heap": { + "version": "0.2.7", + "resolved": "https://registry.npmjs.org/heap/-/heap-0.2.7.tgz", + "integrity": "sha512-2bsegYkkHO+h/9MGbn6KWcE45cHZgPANo5LXF7EvWdT0yT2EguSVO1nDgU5c8+ZOPwp2vMNa7YFsJhVcDR9Sdg==" + }, "hex-color-regex": { "version": "1.1.0", "resolved": "https://registry.npmmirror.com/hex-color-regex/-/hex-color-regex-1.1.0.tgz", "integrity": "sha512-l9sfDFsuqtOqKDsQdqrMRk0U85RZc0RtOR9yPI7mRVOa4FsR/BVnZ0shmQRM96Ji99kYZP/7hn1cedc1+ApsTQ==", "dev": true }, + "highlight.js": { + "version": "9.18.3", + "resolved": "https://registry.npmjs.org/highlight.js/-/highlight.js-9.18.3.tgz", + "integrity": "sha512-zBZAmhSupHIl5sITeMqIJnYCDfAEc3Gdkqj65wC1lpI468MMQeeQkhcIAvk+RylAkxrCcI9xy9piHiXeQ1BdzQ==" + }, "hmac-drbg": { "version": "1.0.1", "resolved": "https://registry.npmmirror.com/hmac-drbg/-/hmac-drbg-1.0.1.tgz", @@ -39993,6 +40812,12 @@ "integrity": "sha512-6TDAlDPZxUFCv+fuOkIoXT/V/f3Qbq8e37p+YOiYrUv3v9cc3/6x78VdfPgFVaB9dZYeLUfKgHRebpkm/oP2VQ==", "dev": true }, + "immutable": { + "version": "4.3.6", + "resolved": "https://registry.npmjs.org/immutable/-/immutable-4.3.6.tgz", + "integrity": "sha512-Ju0+lEMyzMVZarkTn/gqRpdqd5dOPaz1mCZ0SH3JV6iFw81PldE/PEB1hWVEA288HPt4WXW8O7AWxB10M+03QQ==", + "dev": true + }, "import-cwd": { "version": "2.1.0", "resolved": "https://registry.npmmirror.com/import-cwd/-/import-cwd-2.1.0.tgz", @@ -40330,7 +41155,7 @@ "version": "2.1.0", "resolved": "https://registry.npmmirror.com/is-binary-path/-/is-binary-path-2.1.0.tgz", "integrity": "sha512-ZMERYes6pDydyuGidse7OsHxtbI7WVeUEozgR/g7rd0xUimYNlvZRE/K2MgZTjWy725IfelLeVcEM97mmtRGXw==", - "optional": true, + "devOptional": true, "requires": { "binary-extensions": "^2.0.0" } @@ -40587,8 +41412,7 @@ "is-promise": { "version": "2.2.2", "resolved": "https://registry.npmmirror.com/is-promise/-/is-promise-2.2.2.tgz", - "integrity": "sha512-+lP4/6lKUBfQjZ2pdxThZvLUAafmZb8OAxFb8XXtiQmS35INgr85hdOGoEs124ez1FCnZJt6jau/T+alh58QFQ==", - "dev": true + "integrity": "sha512-+lP4/6lKUBfQjZ2pdxThZvLUAafmZb8OAxFb8XXtiQmS35INgr85hdOGoEs124ez1FCnZJt6jau/T+alh58QFQ==" }, "is-regex": { "version": "1.1.4", @@ -40735,8 +41559,7 @@ "jquery": { "version": "3.6.0", "resolved": "https://registry.npmmirror.com/jquery/-/jquery-3.6.0.tgz", - "integrity": "sha512-JVzAR/AjBvVt2BmYhxRCSYysDsPcssdmTFnzyLEts9qNwmjmu4JTAMYubEfwVOSwpQ1I1sKKFcxhZCI2buerfw==", - "peer": true + "integrity": "sha512-JVzAR/AjBvVt2BmYhxRCSYysDsPcssdmTFnzyLEts9qNwmjmu4JTAMYubEfwVOSwpQ1I1sKKFcxhZCI2buerfw==" }, "js-base64": { "version": "2.6.4", @@ -40901,6 +41724,11 @@ "graceful-fs": "^4.1.11" } }, + "klayjs": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/klayjs/-/klayjs-0.4.1.tgz", + "integrity": "sha512-WUNxuO7O79TEkxCj6OIaK5TJBkaWaR/IKNTakgV9PwDn+mrr63MLHed34AcE2yTaDntgO6l0zGFIzhcoTeroTA==" + }, "klona": { "version": "2.0.5", "resolved": "https://registry.npmmirror.com/klona/-/klona-2.0.5.tgz", @@ -41966,12 +42794,25 @@ "escape-string-regexp": "^1.0.4" } }, + "material-colors": { + "version": "1.2.6", + "resolved": "https://registry.npmjs.org/material-colors/-/material-colors-1.2.6.tgz", + "integrity": "sha512-6qE4B9deFBIa9YSpOc9O0Sgc43zTeVYbgDT5veRKSlB2+ZuHNoVVxA1L/ckMUayV9Ay9y7Z/SZCLcGteW9i7bg==" + }, "math-random": { "version": "1.0.4", "resolved": "https://registry.npmmirror.com/math-random/-/math-random-1.0.4.tgz", "integrity": "sha512-rUxjysqif/BZQH2yhd5Aaq7vXMSx9NdEsQcyA07uEzIvxgI7zIr33gGsh+RU0/XjmQpCW7RsVof1vlkvQVCK5A==", "dev": true }, + "mavon-editor": { + "version": "2.10.4", + "resolved": "https://registry.npmjs.org/mavon-editor/-/mavon-editor-2.10.4.tgz", + "integrity": "sha512-CFsBLkgt/KZBDg+SJYe2fyYv4zClY149PiwpH0rDAiiP4ae1XNs0GC8nBsoTeipsHcebDLN1QMkt3bUsnMDjQw==", + "requires": { + "xss": "^1.0.6" + } + }, "md5": { "version": "2.3.0", "resolved": "https://registry.npmmirror.com/md5/-/md5-2.3.0.tgz", @@ -42643,6 +43484,21 @@ "resolved": "https://registry.npmmirror.com/moment/-/moment-2.29.1.tgz", "integrity": "sha512-kHmoybcPV8Sqy59DwNDY3Jefr64lK/by/da0ViFcuA4DH0vQg5Q6Ze5VimxkfQNSC+Mls/Kx53s7TjP1RhFEDQ==" }, + "moment-timezone": { + "version": "0.5.45", + "resolved": "https://registry.npmjs.org/moment-timezone/-/moment-timezone-0.5.45.tgz", + "integrity": "sha512-HIWmqA86KcmCAhnMAN0wuDOARV/525R2+lOLotuGFzn4HO+FH+/645z2wx0Dt3iDv6/p61SIvKnDstISainhLQ==", + "requires": { + "moment": "^2.29.4" + }, + "dependencies": { + "moment": { + "version": "2.30.1", + "resolved": "https://registry.npmjs.org/moment/-/moment-2.30.1.tgz", + "integrity": "sha512-uEmtNhbDOrWPFS+hdjFCBfy9f2YoyzRpwcl+DqpC6taX21FzsTLQVbMV/W7PzNSX6x/bhC1zA3c2UQ5NzH6how==" + } + } + }, "monaco-editor": { "version": "0.19.3", "resolved": "https://registry.npmmirror.com/monaco-editor/-/monaco-editor-0.19.3.tgz", @@ -42657,6 +43513,17 @@ "loader-utils": "^1.2.3" } }, + "monaco-languageclient": { + "version": "0.13.0", + "resolved": "https://registry.npmjs.org/monaco-languageclient/-/monaco-languageclient-0.13.0.tgz", + "integrity": "sha512-aCwd33dTitwV5QwY56rpYHwzEGXei8TZ+yvZcvP3gEMd6Mizr8m3pOuoknDi2SUfLuNAHS6+ulvLgZlNQB5awg==", + "requires": { + "glob-to-regexp": "^0.3.0", + "vscode-jsonrpc": "^5.0.0", + "vscode-languageclient": "^6.0.0", + "vscode-uri": "^2.1.1" + } + }, "move-concurrently": { "version": "1.0.1", "resolved": "https://registry.npmmirror.com/move-concurrently/-/move-concurrently-1.0.1.tgz", @@ -42768,7 +43635,7 @@ "version": "2.15.0", "resolved": "https://registry.npmmirror.com/nan/-/nan-2.15.0.tgz", "integrity": "sha512-8ZtvEnA2c5aYCZYd1cvgdnU6cqwixRoYg70xPLWUws5ORTa/lnw+u4amixRS/Ac5U5mQVgp9pnlSUnbNWFaWZQ==", - "devOptional": true + "optional": true }, "nanomatch": { "version": "1.2.13", @@ -42950,6 +43817,8 @@ "resolved": "https://registry.npmmirror.com/node-sass/-/node-sass-6.0.1.tgz", "integrity": "sha512-f+Rbqt92Ful9gX0cGtdYwjTrWAaGURgaK5rZCWOgCNyGWusFYHhbqCCBoFBeat+HKETOU02AyTxNhJV0YZf2jQ==", "dev": true, + "optional": true, + "peer": true, "requires": { "async-foreach": "^0.1.3", "chalk": "^1.1.1", @@ -42972,19 +43841,25 @@ "version": "2.1.1", "resolved": "https://registry.npmmirror.com/ansi-regex/-/ansi-regex-2.1.1.tgz", "integrity": "sha512-TIGnTpdo+E3+pCyAluZvtED5p5wCqLdezCyhPZzKPcxvFplEt4i+W7OONCKgeZFT3+y5NZZfOOS/Bdcanm1MYA==", - "dev": true + "dev": true, + "optional": true, + "peer": true }, "ansi-styles": { "version": "2.2.1", "resolved": "https://registry.npmmirror.com/ansi-styles/-/ansi-styles-2.2.1.tgz", "integrity": "sha512-kmCevFghRiWM7HB5zTPULl4r9bVFSWjz62MhqizDGUrq2NWuNMQyuv4tHHoKJHs69M/MF64lEcHdYIocrdWQYA==", - "dev": true + "dev": true, + "optional": true, + "peer": true }, "chalk": { "version": "1.1.3", "resolved": "https://registry.npmmirror.com/chalk/-/chalk-1.1.3.tgz", "integrity": "sha512-U3lRVLMSlsCfjqYPbLyVv11M9CPW4I728d6TCKMAOJueEeB9/8o+eSsMnxPJD+Q+K909sdESg7C+tIkoH6on1A==", "dev": true, + "optional": true, + "peer": true, "requires": { "ansi-styles": "^2.2.1", "escape-string-regexp": "^1.0.2", @@ -42997,13 +43872,17 @@ "version": "2.0.0", "resolved": "https://registry.npmmirror.com/chownr/-/chownr-2.0.0.tgz", "integrity": "sha512-bIomtDF5KGpdogkLd9VspvFzk9KfpyyGlS8YFVZl7TGPBHL5snIOnxeshwVgPteQ9b4Eydl+pVbIyE1DcvCWgQ==", - "dev": true + "dev": true, + "optional": true, + "peer": true }, "cross-spawn": { "version": "7.0.3", "resolved": "https://registry.npmmirror.com/cross-spawn/-/cross-spawn-7.0.3.tgz", "integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==", "dev": true, + "optional": true, + "peer": true, "requires": { "path-key": "^3.1.0", "shebang-command": "^2.0.0", @@ -43015,6 +43894,8 @@ "resolved": "https://registry.npmmirror.com/find-up/-/find-up-4.1.0.tgz", "integrity": "sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==", "dev": true, + "optional": true, + "peer": true, "requires": { "locate-path": "^5.0.0", "path-exists": "^4.0.0" @@ -43025,6 +43906,8 @@ "resolved": "https://registry.npmmirror.com/fs-minipass/-/fs-minipass-2.1.0.tgz", "integrity": "sha512-V/JgOLFCS+R6Vcq0slCuaeWEdNC3ouDlJMNIsacH2VtALiu9mV4LPrHc5cDl8k5aw6J8jwgWWpiTo5RYhmIzvg==", "dev": true, + "optional": true, + "peer": true, "requires": { "minipass": "^3.0.0" } @@ -43033,13 +43916,17 @@ "version": "4.0.1", "resolved": "https://registry.npmmirror.com/get-stdin/-/get-stdin-4.0.1.tgz", "integrity": "sha512-F5aQMywwJ2n85s4hJPTT9RPxGmubonuB10MNYo17/xph174n2MIR33HRguhzVag10O/npM7SPk73LMZNP+FaWw==", - "dev": true + "dev": true, + "optional": true, + "peer": true }, "hosted-git-info": { "version": "4.1.0", "resolved": "https://registry.npmmirror.com/hosted-git-info/-/hosted-git-info-4.1.0.tgz", "integrity": "sha512-kyCuEOWjJqZuDbRHzL8V93NzQhwIB71oFWSyzVo+KPZI+pnQPPxucdkrOZvkLRnrf5URsQM+IJ09Dw29cRALIA==", "dev": true, + "optional": true, + "peer": true, "requires": { "lru-cache": "^6.0.0" } @@ -43049,6 +43936,8 @@ "resolved": "https://registry.npmmirror.com/locate-path/-/locate-path-5.0.0.tgz", "integrity": "sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g==", "dev": true, + "optional": true, + "peer": true, "requires": { "p-locate": "^4.1.0" } @@ -43058,6 +43947,8 @@ "resolved": "https://registry.npmmirror.com/lru-cache/-/lru-cache-6.0.0.tgz", "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", "dev": true, + "optional": true, + "peer": true, "requires": { "yallist": "^4.0.0" } @@ -43067,6 +43958,8 @@ "resolved": "https://registry.npmmirror.com/meow/-/meow-9.0.0.tgz", "integrity": "sha512-+obSblOQmRhcyBt62furQqRAQpNyWXo8BuQ5bN7dG8wmwQ+vwHKp/rCFD4CrTP8CsDQD1sjoZ94K417XEUk8IQ==", "dev": true, + "optional": true, + "peer": true, "requires": { "@types/minimist": "^1.2.0", "camelcase-keys": "^6.2.2", @@ -43087,6 +43980,8 @@ "resolved": "https://registry.npmmirror.com/minipass/-/minipass-3.3.4.tgz", "integrity": "sha512-I9WPbWHCGu8W+6k1ZiGpPu0GkoKBeorkfKNuAFBNS1HNFJvke82sxvI5bzcCNpWPorkOO5QQ+zomzzwRxejXiw==", "dev": true, + "optional": true, + "peer": true, "requires": { "yallist": "^4.0.0" } @@ -43096,6 +43991,8 @@ "resolved": "https://registry.npmmirror.com/minizlib/-/minizlib-2.1.2.tgz", "integrity": "sha512-bAxsR8BVfj60DWXHE3u30oHzfl4G7khkSuPW+qvpd7jFRHm7dLxOjUk1EHACJ/hxLY8phGJ0YhYHZo7jil7Qdg==", "dev": true, + "optional": true, + "peer": true, "requires": { "minipass": "^3.0.0", "yallist": "^4.0.0" @@ -43105,13 +44002,17 @@ "version": "1.0.4", "resolved": "https://registry.npmmirror.com/mkdirp/-/mkdirp-1.0.4.tgz", "integrity": "sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw==", - "dev": true + "dev": true, + "optional": true, + "peer": true }, "node-gyp": { "version": "7.1.2", "resolved": "https://registry.npmmirror.com/node-gyp/-/node-gyp-7.1.2.tgz", "integrity": "sha512-CbpcIo7C3eMu3dL1c3d0xw449fHIGALIJsRP4DDPHpyiW8vcriNY7ubh9TE4zEKfSxscY7PjeFnshE7h75ynjQ==", "dev": true, + "optional": true, + "peer": true, "requires": { "env-paths": "^2.2.0", "glob": "^7.1.4", @@ -43130,6 +44031,8 @@ "resolved": "https://registry.npmmirror.com/nopt/-/nopt-5.0.0.tgz", "integrity": "sha512-Tbj67rffqceeLpcRXrT7vKAN8CwfPeIBgM7E6iBkmKLV7bEMwpGgYLGv0jACUsECaa/vuxP0IjEont6umdMgtQ==", "dev": true, + "optional": true, + "peer": true, "requires": { "abbrev": "1" } @@ -43139,6 +44042,8 @@ "resolved": "https://registry.npmmirror.com/normalize-package-data/-/normalize-package-data-3.0.3.tgz", "integrity": "sha512-p2W1sgqij3zMMyRC067Dg16bfzVH+w7hyegmpIvZ4JNjqtGOVAIvLmjBx3yP7YTe9vKJgkoNOPjwQGogDoMXFA==", "dev": true, + "optional": true, + "peer": true, "requires": { "hosted-git-info": "^4.0.1", "is-core-module": "^2.5.0", @@ -43151,6 +44056,8 @@ "resolved": "https://registry.npmmirror.com/p-locate/-/p-locate-4.1.0.tgz", "integrity": "sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A==", "dev": true, + "optional": true, + "peer": true, "requires": { "p-limit": "^2.2.0" } @@ -43159,19 +44066,25 @@ "version": "4.0.0", "resolved": "https://registry.npmmirror.com/path-exists/-/path-exists-4.0.0.tgz", "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==", - "dev": true + "dev": true, + "optional": true, + "peer": true }, "path-key": { "version": "3.1.1", "resolved": "https://registry.npmmirror.com/path-key/-/path-key-3.1.1.tgz", "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", - "dev": true + "dev": true, + "optional": true, + "peer": true }, "read-pkg-up": { "version": "7.0.1", "resolved": "https://registry.npmmirror.com/read-pkg-up/-/read-pkg-up-7.0.1.tgz", "integrity": "sha512-zK0TB7Xd6JpCLmlLmufqykGE+/TlOePD6qKClNW7hHDKFh/J7/7gCWGR7joEQEW1bKq3a3yUZSObOoWLFQ4ohg==", "dev": true, + "optional": true, + "peer": true, "requires": { "find-up": "^4.1.0", "read-pkg": "^5.2.0", @@ -43182,7 +44095,9 @@ "version": "0.8.1", "resolved": "https://registry.npmmirror.com/type-fest/-/type-fest-0.8.1.tgz", "integrity": "sha512-4dbzIzqvjtgiM5rw1k5rEHtBANKmdudhGyBEajN01fEyhaAIhsoKNy6y7+IN93IfpFtwY9iqi7kD+xwKhQsNJA==", - "dev": true + "dev": true, + "optional": true, + "peer": true } } }, @@ -43191,6 +44106,8 @@ "resolved": "https://registry.npmmirror.com/rimraf/-/rimraf-3.0.2.tgz", "integrity": "sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==", "dev": true, + "optional": true, + "peer": true, "requires": { "glob": "^7.1.3" } @@ -43200,6 +44117,8 @@ "resolved": "https://registry.npmmirror.com/semver/-/semver-7.3.8.tgz", "integrity": "sha512-NB1ctGL5rlHrPJtFDVIVzTyQylMLu9N9VICA6HSFJo8MCGVTMW6gfpicwKmmK/dAjTOrqu5l63JJOpDSrAis3A==", "dev": true, + "optional": true, + "peer": true, "requires": { "lru-cache": "^6.0.0" } @@ -43209,6 +44128,8 @@ "resolved": "https://registry.npmmirror.com/shebang-command/-/shebang-command-2.0.0.tgz", "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", "dev": true, + "optional": true, + "peer": true, "requires": { "shebang-regex": "^3.0.0" } @@ -43217,13 +44138,17 @@ "version": "3.0.0", "resolved": "https://registry.npmmirror.com/shebang-regex/-/shebang-regex-3.0.0.tgz", "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", - "dev": true + "dev": true, + "optional": true, + "peer": true }, "strip-ansi": { "version": "3.0.1", "resolved": "https://registry.npmmirror.com/strip-ansi/-/strip-ansi-3.0.1.tgz", "integrity": "sha512-VhumSSbBqDTP8p2ZLKj40UjBCV4+v8bUSEpUb4KjRgWk9pbqGF4REFj6KEagidb2f/M6AzC0EmFyDNGaw9OCzg==", "dev": true, + "optional": true, + "peer": true, "requires": { "ansi-regex": "^2.0.0" } @@ -43232,13 +44157,17 @@ "version": "2.0.0", "resolved": "https://registry.npmmirror.com/supports-color/-/supports-color-2.0.0.tgz", "integrity": "sha512-KKNVtd6pCYgPIKU4cp2733HWYCpplQhddZLBUryaAHou723x+FRzQ5Df824Fj+IyyuiQTRoub4SnIFfIcrp70g==", - "dev": true + "dev": true, + "optional": true, + "peer": true }, "tar": { "version": "6.1.12", "resolved": "https://registry.npmmirror.com/tar/-/tar-6.1.12.tgz", "integrity": "sha512-jU4TdemS31uABHd+Lt5WEYJuzn+TJTCBLljvIAHZOz6M9Os5pJ4dD+vRFLxPa/n3T0iEFzpi+0x1UfuDZYbRMw==", "dev": true, + "optional": true, + "peer": true, "requires": { "chownr": "^2.0.0", "fs-minipass": "^2.0.0", @@ -43252,13 +44181,17 @@ "version": "0.18.1", "resolved": "https://registry.npmmirror.com/type-fest/-/type-fest-0.18.1.tgz", "integrity": "sha512-OIAYXk8+ISY+qTOwkHtKqzAuxchoMiD9Udx+FSGQDuiRR+PJKJHc2NJAXlbhkGwTt/4/nKZxELY1w3ReWOL8mw==", - "dev": true + "dev": true, + "optional": true, + "peer": true }, "which": { "version": "2.0.2", "resolved": "https://registry.npmmirror.com/which/-/which-2.0.2.tgz", "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", "dev": true, + "optional": true, + "peer": true, "requires": { "isexe": "^2.0.0" } @@ -43267,7 +44200,9 @@ "version": "4.0.0", "resolved": "https://registry.npmmirror.com/yallist/-/yallist-4.0.0.tgz", "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", - "dev": true + "dev": true, + "optional": true, + "peer": true } } }, @@ -45722,11 +46657,16 @@ "version": "3.6.0", "resolved": "https://registry.npmmirror.com/readdirp/-/readdirp-3.6.0.tgz", "integrity": "sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA==", - "optional": true, + "devOptional": true, "requires": { "picomatch": "^2.2.1" } }, + "reconnecting-websocket": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/reconnecting-websocket/-/reconnecting-websocket-4.4.0.tgz", + "integrity": "sha512-D2E33ceRPga0NvTDhJmphEgJ7FUYF0v4lr1ki0csq06OdlxKfugGzN0dSkxM/NfqCxYELK4KcaTOUOjTV6Dcng==" + }, "redent": { "version": "3.0.0", "resolved": "https://registry.npmmirror.com/redent/-/redent-3.0.0.tgz", @@ -45766,8 +46706,7 @@ "regenerator-runtime": { "version": "0.13.9", "resolved": "https://registry.npmmirror.com/regenerator-runtime/-/regenerator-runtime-0.13.9.tgz", - "integrity": "sha512-p3VT+cOEgxFsRRA9X4lkI1E+k2/CtnKtU4gcxyaCUreilL/vqI6CdZ3wxVUx3UOUg+gnUOQQcRI7BmSI656MYA==", - "dev": true + "integrity": "sha512-p3VT+cOEgxFsRRA9X4lkI1E+k2/CtnKtU4gcxyaCUreilL/vqI6CdZ3wxVUx3UOUg+gnUOQQcRI7BmSI656MYA==" }, "regenerator-transform": { "version": "0.15.0", @@ -46233,11 +47172,24 @@ "resolved": "https://registry.npmmirror.com/safer-buffer/-/safer-buffer-2.1.2.tgz", "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" }, + "sass": { + "version": "1.49.0", + "resolved": "https://registry.npmjs.org/sass/-/sass-1.49.0.tgz", + "integrity": "sha512-TVwVdNDj6p6b4QymJtNtRS2YtLJ/CqZriGg0eIAbAKMlN8Xy6kbv33FsEZSF7FufFFM705SQviHjjThfaQ4VNw==", + "dev": true, + "requires": { + "chokidar": ">=3.0.0 <4.0.0", + "immutable": "^4.0.0", + "source-map-js": ">=0.6.2 <2.0.0" + } + }, "sass-graph": { "version": "2.2.5", "resolved": "https://registry.npmmirror.com/sass-graph/-/sass-graph-2.2.5.tgz", "integrity": "sha512-VFWDAHOe6mRuT4mZRd4eKE+d8Uedrk6Xnh7Sh9b4NGufQLQjOrvf/MQoOdx+0s92L89FeyUUNfU597j/3uNpag==", "dev": true, + "optional": true, + "peer": true, "requires": { "glob": "^7.0.0", "lodash": "^4.0.0", @@ -46249,13 +47201,17 @@ "version": "7.0.3", "resolved": "https://registry.npmmirror.com/emoji-regex/-/emoji-regex-7.0.3.tgz", "integrity": "sha512-CwBLREIQ7LvYFB0WyRvwhq5N5qPhc6PMjD6bYggFlI5YyDgl+0vxq5VHbMOFqLg7hfWzmu8T5Z1QofhmTIhItA==", - "dev": true + "dev": true, + "optional": true, + "peer": true }, "string-width": { "version": "3.1.0", "resolved": "https://registry.npmmirror.com/string-width/-/string-width-3.1.0.tgz", "integrity": "sha512-vafcv6KjVZKSgz06oM/H6GDBrAtz8vdhQakGjFIvNrHA6y3HCF1CInLy+QLq8dTJPQ1b+KDUqDFctkdRW44e1w==", "dev": true, + "optional": true, + "peer": true, "requires": { "emoji-regex": "^7.0.1", "is-fullwidth-code-point": "^2.0.0", @@ -46267,6 +47223,8 @@ "resolved": "https://registry.npmmirror.com/yargs/-/yargs-13.3.2.tgz", "integrity": "sha512-AX3Zw5iPruN5ie6xGRIDgqkT+ZhnRlZMLMHAs8tg7nRruy2Nb+i5o9bwghAogtM08q1dpr2LVoS8KSTMYpWXUw==", "dev": true, + "optional": true, + "peer": true, "requires": { "cliui": "^5.0.0", "find-up": "^3.0.0", @@ -46285,6 +47243,8 @@ "resolved": "https://registry.npmmirror.com/yargs-parser/-/yargs-parser-13.1.2.tgz", "integrity": "sha512-3lbsNRf/j+A4QuSZfDRA7HRSfWrzO0YjqTJd5kjAq37Zep1CEgaYmrH9Q3GwPiB9cHyd1Y1UwggGhJGoxipbzg==", "dev": true, + "optional": true, + "peer": true, "requires": { "camelcase": "^5.0.0", "decamelize": "^1.2.0" @@ -46379,6 +47339,8 @@ "resolved": "https://registry.npmmirror.com/scss-tokenizer/-/scss-tokenizer-0.2.3.tgz", "integrity": "sha512-dYE8LhncfBUar6POCxMTm0Ln+erjeczqEvCJib5/7XNkdw1FkUGgwMPY360FY0FgPWQxHWCx29Jl3oejyGLM9Q==", "dev": true, + "optional": true, + "peer": true, "requires": { "js-base64": "^2.1.8", "source-map": "^0.4.2" @@ -46389,12 +47351,19 @@ "resolved": "https://registry.npmmirror.com/source-map/-/source-map-0.4.4.tgz", "integrity": "sha512-Y8nIfcb1s/7DcobUz1yOO1GSp7gyL+D9zLHDehT7iRESqGSxjJ448Sg7rvfgsRJCnKLdSl11uGf0s9X80cH0/A==", "dev": true, + "optional": true, + "peer": true, "requires": { "amdefine": ">=0.0.4" } } } }, + "select": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/select/-/select-1.1.2.tgz", + "integrity": "sha512-OwpTSOfy6xSs1+pwcNrv0RBMOzI39Lp3qQKUTPVVPRjCdNa5JH/oPRiqsesIskK8TVgmRiHwO4KXlV2Li9dANA==" + }, "select-hose": { "version": "2.0.0", "resolved": "https://registry.npmmirror.com/select-hose/-/select-hose-2.0.0.tgz", @@ -46886,6 +47855,11 @@ "is-plain-obj": "^1.0.0" } }, + "sortablejs": { + "version": "1.10.2", + "resolved": "https://registry.npmjs.org/sortablejs/-/sortablejs-1.10.2.tgz", + "integrity": "sha512-YkPGufevysvfwn5rfdlGyrGjt7/CRHwvRPogD/lC+TnvcN29jDpCifKP+rBqf+LRldfXSTh+0CGLcSg0VIxq3A==" + }, "source-list-map": { "version": "2.0.1", "resolved": "https://registry.npmmirror.com/source-list-map/-/source-list-map-2.0.1.tgz", @@ -46896,6 +47870,12 @@ "resolved": "https://registry.npmmirror.com/source-map/-/source-map-0.5.7.tgz", "integrity": "sha512-LbrmJOMUSdEVxIKvdcJzQC+nQhe8FUZQTXQy6+I75skNgn3OoQ0DZA8YnFa7gp8tqtL3KPf1kmo0R5DoApeSGQ==" }, + "source-map-js": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.0.tgz", + "integrity": "sha512-itJW8lvSA0TXEphiRoawsCksnlf8SyvmFzIhltqAHluXd88pkCd+cXJVHTDwdCr0IzwptSm035IHQktUu1QUMg==", + "dev": true + }, "source-map-resolve": { "version": "0.5.3", "resolved": "https://registry.npmmirror.com/source-map-resolve/-/source-map-resolve-0.5.3.tgz", @@ -47215,6 +48195,8 @@ "resolved": "https://registry.npmmirror.com/stdout-stream/-/stdout-stream-1.4.1.tgz", "integrity": "sha512-j4emi03KXqJWcIeF8eIXkjMFN1Cmb8gUlDYGeBALLPo5qdyTfA9bOtl8m33lRoC+vFMkP3gl0WsDr6+gzxbbTA==", "dev": true, + "optional": true, + "peer": true, "requires": { "readable-stream": "^2.0.1" } @@ -48006,6 +48988,11 @@ "integrity": "sha512-qsdtZH+vMoCARQtyod4imc2nIJwg9Cc7lPRrw9CzF8ZKR0khdr8+2nX80PBhET3tcyTtJDxAffGh2rXH4tyU8A==", "dev": true }, + "tiny-emitter": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/tiny-emitter/-/tiny-emitter-2.1.0.tgz", + "integrity": "sha512-NB6Dk1A9xgQPMoGqC5CVXn123gWyte215ONT5Pp5a0yt4nlEoO1ZWeCwpncaekPHXO60i47ihFnZPiRPjRMq4Q==" + }, "tinycolor2": { "version": "1.4.2", "resolved": "https://registry.npmmirror.com/tinycolor2/-/tinycolor2-1.4.2.tgz", @@ -48117,6 +49104,8 @@ "resolved": "https://registry.npmmirror.com/true-case-path/-/true-case-path-1.0.3.tgz", "integrity": "sha512-m6s2OdQe5wgpFMC+pAJ+q9djG82O2jcHPOI6RNg1yy9rCYR+WD6Nbpl32fDpfC56nirdRy+opFa/Vk7HYhqaew==", "dev": true, + "optional": true, + "peer": true, "requires": { "glob": "^7.1.2" } @@ -48582,6 +49571,54 @@ "resolved": "https://registry.npmmirror.com/vm-browserify/-/vm-browserify-1.1.2.tgz", "integrity": "sha512-2ham8XPWTONajOR0ohOKOHXkm3+gaBmGut3SRuu75xLd/RRaY6vqgh8NBYYk7+RW3u5AtzPQZG8F10LHkl0lAQ==" }, + "vscode-jsonrpc": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/vscode-jsonrpc/-/vscode-jsonrpc-5.0.1.tgz", + "integrity": "sha512-JvONPptw3GAQGXlVV2utDcHx0BiY34FupW/kI6mZ5x06ER5DdPG/tXWMVHjTNULF5uKPOUUD0SaXg5QaubJL0A==" + }, + "vscode-languageclient": { + "version": "6.1.4", + "resolved": "https://registry.npmjs.org/vscode-languageclient/-/vscode-languageclient-6.1.4.tgz", + "integrity": "sha512-EUOU+bJu6axmt0RFNo3nrglQLPXMfanbYViJee3Fbn2VuQoX0ZOI4uTYhSRvYLP2vfwTP/juV62P/mksCdTZMA==", + "requires": { + "semver": "^6.3.0", + "vscode-languageserver-protocol": "3.15.3" + }, + "dependencies": { + "semver": { + "version": "6.3.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", + "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==" + } + } + }, + "vscode-languageserver-protocol": { + "version": "3.15.3", + "resolved": "https://registry.npmjs.org/vscode-languageserver-protocol/-/vscode-languageserver-protocol-3.15.3.tgz", + "integrity": "sha512-zrMuwHOAQRhjDSnflWdJG+O2ztMWss8GqUUB8dXLR/FPenwkiBNkMIJJYfSN6sgskvsF0rHAoBowNQfbyZnnvw==", + "requires": { + "vscode-jsonrpc": "^5.0.1", + "vscode-languageserver-types": "3.15.1" + } + }, + "vscode-languageserver-types": { + "version": "3.15.1", + "resolved": "https://registry.npmjs.org/vscode-languageserver-types/-/vscode-languageserver-types-3.15.1.tgz", + "integrity": "sha512-+a9MPUQrNGRrGU630OGbYVQ+11iOIovjCkqxajPa9w57Sd5ruK8WQNsslzpa0x/QJqC8kRc2DUxWjIFwoNm4ZQ==" + }, + "vscode-uri": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/vscode-uri/-/vscode-uri-2.1.2.tgz", + "integrity": "sha512-8TEXQxlldWAuIODdukIb+TR5s+9Ds40eSJrw+1iDDA9IFORPjMELarNQE3myz5XIkWWpdprmJjm1/SxMlWOC8A==" + }, + "vscode-ws-jsonrpc": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/vscode-ws-jsonrpc/-/vscode-ws-jsonrpc-0.2.0.tgz", + "integrity": "sha512-NE9HNRgPjCaPyTJvIudcpyIWPImxwRDtuTX16yks7SAiZgSXigxAiZOvSvVBGmD1G/OMfrFo6BblOtjVR9DdVA==", + "requires": { + "vscode-jsonrpc": "^5.0.0" + } + }, "vue": { "version": "2.6.12", "resolved": "https://registry.npmmirror.com/vue/-/vue-2.6.12.tgz", @@ -48697,12 +49734,25 @@ "integrity": "sha512-4gDntzrifFnCEvyoO8PqyJDmguXgVPxKiIxrBKjIowvL9l+N66196+72XVYR8BBf1Uv1Fgt3bGevJ+sEmxfZzw==", "dev": true }, + "vuedraggable": { + "version": "2.24.3", + "resolved": "https://registry.npmjs.org/vuedraggable/-/vuedraggable-2.24.3.tgz", + "integrity": "sha512-6/HDXi92GzB+Hcs9fC6PAAozK1RLt1ewPTLjK0anTYguXLAeySDmcnqE8IC0xa7shvSzRjQXq3/+dsZ7ETGF3g==", + "requires": { + "sortablejs": "1.10.2" + } + }, "vuescroll": { "version": "4.16.1", "resolved": "https://registry.npmmirror.com/vuescroll/-/vuescroll-4.16.1.tgz", "integrity": "sha512-7fRsG2Yw5Z07LUz/IIu9barpmYiN9q+ZTC+CrVamvCbmsxyhz8mU1OuYFbfORysaUskioNMxTGDo+HOzeDfSyQ==", "requires": {} }, + "watch-size": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/watch-size/-/watch-size-2.0.0.tgz", + "integrity": "sha512-M92R89dNoTPWyCD+HuUEDdhaDnh9jxPGOwlDc0u51jAgmjUvzqaEMynXSr3BaWs+QdHYk4KzibPy1TFtjLmOZQ==" + }, "watchpack": { "version": "1.7.5", "resolved": "https://registry.npmmirror.com/watchpack/-/watchpack-1.7.5.tgz", @@ -49573,6 +50623,15 @@ "async-limiter": "~1.0.0" } }, + "xss": { + "version": "1.0.15", + "resolved": "https://registry.npmjs.org/xss/-/xss-1.0.15.tgz", + "integrity": "sha512-FVdlVVC67WOIPvfOwhoMETV72f6GbW7aOabBC3WxN/oUdoEMDyLz4OgRv5/gck2ZeNqEQu+Tb0kloovXOfpYVg==", + "requires": { + "commander": "^2.20.3", + "cssfilter": "0.0.10" + } + }, "xtend": { "version": "4.0.2", "resolved": "https://registry.npmmirror.com/xtend/-/xtend-4.0.2.tgz", diff --git a/web/package.json b/web/package.json index ed9ef95339..be84dc4c5c 100644 --- a/web/package.json +++ b/web/package.json @@ -38,6 +38,7 @@ "md5": "2.3.0", "moment": "2.29.4", "monaco-editor": "0.19.3", + "monaco-languageclient": "^0.13.0", "qs": "6.9.4", "svgo": "1.3.0", "vue": "2.6.12", @@ -65,27 +66,28 @@ "less-loader": "6.1.0", "lint-staged": "8.2.1", "monaco-editor-webpack-plugin": "1.8.2", - "node-sass": "6.0.1", + "patch-package": "6.2.2", + "sass": "1.49.0", "sass-loader": "10.2.1", "speed-measure-webpack-plugin": "1.5.0", "svg-sprite-loader": "5.0.0", - "patch-package": "6.2.2", "vue-cli-plugin-mockjs": "0.1.3", "vue-template-compiler": "2.6.12", "webpack-virtual-modules": "0.3.2" }, "workspaces": [ - "packages/apiServices", - "packages/cyeditor", - "packages/dataGovernance", - "packages/dss", - "packages/editor", - "packages/editorLsp", - "packages/exts", - "packages/scheduleCenter", - "packages/scriptis", - "packages/shared", - "packages/workflows", - "packages/workspace" + "packages/apiServices", + "packages/cyeditor", + "packages/dataGovernance", + "packages/dss", + "packages/editor", + "packages/editorLsp", + "packages/exts", + "packages/scheduleCenter", + "packages/dolphinScheduler", + "packages/scriptis", + "packages/shared", + "packages/workflows", + "packages/workspace" ] } diff --git a/web/packages/apiServices/i18n/en.json b/web/packages/apiServices/i18n/en.json index ea6c18c296..66e6387447 100644 --- a/web/packages/apiServices/i18n/en.json +++ b/web/packages/apiServices/i18n/en.json @@ -55,7 +55,8 @@ "emter": "Enter", "startDate": "Please choose the start date", "dateAndTime": "Select Date And Time", - "inputSubmitter": "Input Submitter" + "inputSubmitter": "Input Submitter", + "limitStrLength": "The length is limited to {maxLength} characters" }, "label": { "apiName": "API Name：", @@ -158,7 +159,9 @@ }, "defaultValue": "Default Value", "displayName": "Display Name", - "describe": "Describe" + "describe": "Describe", + "setLength": "Length setting", + "setLengthTips": "The actual maximum length of the parameter can be entered depending on the browser version, request protocol, and data type. It may occur that the actual maximum input length of the field is less than the set maximum length" }, "rule": { "nameRule": "Please input api name!", diff --git a/web/packages/apiServices/i18n/zh.json b/web/packages/apiServices/i18n/zh.json index 96998a7b82..35f4e38d24 100644 --- a/web/packages/apiServices/i18n/zh.json +++ b/web/packages/apiServices/i18n/zh.json @@ -55,7 +55,8 @@ "emter": "请输入", "startDate": "请选择起始日期", "dateAndTime": "请选择日期和时间", - "inputSubmitter": "输入提交人" + "inputSubmitter": "输入提交人", + "limitStrLength": "长度限制在{maxLength}字符内" }, "label": { "apiName": "API名称：", @@ -158,7 +159,9 @@ }, "defaultValue": "默认值", "displayName": "展示名", - "describe": "描述" + "describe": "描述", + "setLength": "长度设置", + "setLengthTips": "参数实际可输入最大长度受浏览器版本，请求协议，数据类型等因素影响。可能会出现字段实际输入最大长度小于设置的最大长度的情况" }, "rule": { "nameRule": "请填写API名称", diff --git a/web/packages/apiServices/module/apiServices/index.vue b/web/packages/apiServices/module/apiServices/index.vue index 5e4d08bf93..21cc84d1ff 100644 --- a/web/packages/apiServices/module/apiServices/index.vue +++ b/web/packages/apiServices/module/apiServices/index.vue @@ -465,6 +465,7 @@ export default { let dropdownList = [ {id: 1, title: this.$t('message.apiServices.apiTable.operation.manager')}, {id: 2, title: this.$t('message.apiServices.apiTable.operation.enable')}, + {id: 4, title: this.$t('message.apiServices.delete')} ]; if(e.status === 1) { // 当状态为1时为开启状态，文本显示禁用 @@ -472,8 +473,6 @@ export default { } else { dropdownList[1].title = this.$t('message.apiServices.apiTable.operation.enable') } - // 判断是否显示删除按钮 - if(e.creator === storage.get('userInfo')) dropdownList.push({id: 4, title: this.$t('message.apiServices.delete')}); return dropdownList }, // 加工状态的显示文本和颜色 diff --git a/web/packages/apiServices/module/apiServicesExecute/index.vue b/web/packages/apiServices/module/apiServicesExecute/index.vue index 0214330429..c16728697d 100644 --- a/web/packages/apiServices/module/apiServicesExecute/index.vue +++ b/web/packages/apiServices/module/apiServicesExecute/index.vue @@ -67,7 +67,6 @@ export default { apiData: null, excuteLoading: false, tip: {}, - height: 500, tabslist: [], currentTaskId: '', historyList: [] @@ -85,11 +84,7 @@ export default { noClose: true, taskID: '' } - const result = await api.fetch('/dss/apiservice/getApiHistory', { - apiId: this.apiData.id, - apiVersionId: this.apiData.latestVersionId - }, 'get') - this.historyList = (result.data || []).map(item => ({ execID: item.strongerExecId, ...item })) + await this.fetchHistory() if (this.historyList[0] && ['Inited','Scheduled', 'Running'].includes(this.historyList[0].status)) { homeTabItem = { ...homeTabItem, @@ -100,16 +95,15 @@ export default { } this.tabslist.push(homeTabItem) }, + async fetchHistory() { + const result = await api.fetch('/dss/apiservice/getApiHistory', { + apiId: this.apiData.id, + apiVersionId: this.apiData.latestVersionId + }, 'get') + this.historyList = (result.data || []).map(item => ({ execID: item.strongerExecId, ...item })) + }, updateHistory(params) { - const data = { - ...this.historyList[0], - fileName: this.apiData.aliasName || this.apiData.name, - ...params - } - if (params.runningTime) { - data.costTime = params.runningTime - } - this.historyList.splice(0, 1, data) + this.fetchHistory() }, // 获取api相关数据 getExecutePath() { @@ -237,16 +231,16 @@ export default { flex-wrap: nowrap; justify-content: flex-start; align-items: center; - height: 45px; + height: 36px; @include bg-color($light-base-color, $dark-base-color); width: calc(100% - 45px); overflow: hidden; &.work-list-tab { - overflow-x: auto; + overflow-x: overlay; overflow-y: hidden; padding-left: 16px; &::-webkit-scrollbar { - height: 6px; + height: 4px; } &::-webkit-scrollbar-thumb { box-shadow: inset 0 0 2px rgba(0, 0, 0, 0.2); @@ -257,11 +251,6 @@ export default { box-shadow: inset 0 0 2px rgba(0, 0, 0, 0.2); border-radius: 3px; } - .list-group > span { - white-space: nowrap; - display: block; - height: 0; - } } .workbench-tab-item { text-align: center; diff --git a/web/packages/apiServices/module/apiServicesExecute/result.vue b/web/packages/apiServices/module/apiServicesExecute/result.vue index 024b9d1c11..4dc734822b 100644 --- a/web/packages/apiServices/module/apiServicesExecute/result.vue +++ b/web/packages/apiServices/module/apiServicesExecute/result.vue @@ -1,44 +1,69 @@ @@ -102,7 +102,8 @@ export default { navHeight: 0, showSetting: false, proxyUserName: '', - copilotEntryComponent: copilotWebComponent ? copilotWebComponent.copilotEntryComponent : null + copilotEntryComponent: null, + dataSourcetype: 'Hive', }; }, //组建内的守卫 @@ -155,9 +156,17 @@ export default { async created() { let baseInfo = storage.get('baseInfo', 'local') || {} const globalRes = await this.getGlobalLimit() + let copilotEnable = false + try { + const res = await this.getCopilotEnabled() + copilotEnable = !!res.inWhitelist + } catch (error) { + // + } baseInfo = { ...baseInfo, - ...globalRes.globalLimits + ...globalRes.globalLimits, + copilotEnable } storage.set('baseInfo', baseInfo, 'local') // languageServerDefaultEnable = true 默认启用language server @@ -190,43 +199,46 @@ export default { this.proxyUserName = baseInfo.proxyUserName } this.showSetting = !!baseInfo.proxyEnable; - }, 1500) - plugin.on('copilot_web_listener_createAndInster', (regs) => { - if (this.leftModule.key !== 2) { - this.chooseLeftModule(this.leftSideNavList[0]) + this.copilotEntryComponent = null; + plugin.emitHook('copilot_web_listener_event_remove') + if (baseInfo.copilotEnable) { + this.copilotEntryComponent = copilotWebComponent ? copilotWebComponent.copilotEntryComponent : null; + if (this.copilotEntryComponent) { + plugin.on('copilot_web_listener_createAndInster', (regs) => { + if (this.leftModule.key !== 2) { + this.chooseLeftModule(this.leftSideNavList[0]) + } + this.$nextTick(() => { + plugin.emit('copilot_web_listener_create', regs) + }) + }) + } } - this.$nextTick(() => { - plugin.emit('copilot_web_listener_create', regs) - }) - }) + }, 1500) }, beforeDestroy() { - // 监听窗口变化，获取浏览器宽高 - this.$Notice.close('show-db-table-many-tip') window.removeEventListener('resize', this.getHeight); plugin.emitHook('copilot_web_listener_event_remove') }, methods: { + changeScriptTab({ type }) { + this.dataSourcetype = type + }, getGlobalLimit() { return api.fetch(`/dss/scriptis/globalLimits`, {}, { method: 'get', cacheOptions: {time: 3000} }) }, + getCopilotEnabled() { + return api.fetch(`/copilot/user/isinwhitelist`, {}, { + method: 'get', + cacheOptions: {time: 3000} + }) + }, getHeight() { this.resize(window.innerHeight); }, - showTip(length) { - if (length > 30000) { - this.$Notice.close('show-db-table-many-tip') - this.$Notice.warning({ - duration: 0, - name: 'show-db-table-many-tip', - title: this.$t('message.scripts.propmpt'), - desc: this.$t('message.scripts.largedatatip') - }) - } - }, init() { this.chooseLeftModule(this.leftSideNavList[0]); this.resize(window.innerHeight); diff --git a/web/packages/shared/common/config/const.js b/web/packages/shared/common/config/const.js index 7c0dfcd153..1cceca3116 100644 --- a/web/packages/shared/common/config/const.js +++ b/web/packages/shared/common/config/const.js @@ -11,5 +11,8 @@ const ORCHESTRATORMODES = { CONSTSTORCHESTRATOR: 'pom_consist_orchestrator'// 组合工作流 } -export { DEVPROCESS, ORCHESTRATORMODES } +// 脚本执行终态 +const EXECUTE_COMPLETE_TYPE = ['Succeed', 'Failed', 'Cancelled', 'Timeout']; + +export { DEVPROCESS, ORCHESTRATORMODES, EXECUTE_COMPLETE_TYPE } diff --git a/web/packages/shared/common/config/scriptis.js b/web/packages/shared/common/config/scriptis.js index 103e81e21d..f174150cb0 100644 --- a/web/packages/shared/common/config/scriptis.js +++ b/web/packages/shared/common/config/scriptis.js @@ -353,7 +353,8 @@ export default Object.freeze([{ color: '#3573A6', isCanBeNew: true, label: 'Nebula', - isCanBeOpen: true + isCanBeOpen: true, + flowType: 'nebula', }, { rule: /(表详情)|(Table\sdetails)/, @@ -370,4 +371,10 @@ export default Object.freeze([{ executable: false, isCanBeOpen: true }, +{ + rule: /(图空间信息)|(边类型信息)|(标签信息)/, + executable: false, + scriptType: 'Nebula', + isCanBeOpen: true +}, ]) diff --git a/web/packages/shared/common/helper/aicompletion.js b/web/packages/shared/common/helper/aicompletion.js new file mode 100644 index 0000000000..2fa5455f2f --- /dev/null +++ b/web/packages/shared/common/helper/aicompletion.js @@ -0,0 +1,66 @@ +import api from '../service/api'; +let timer +export function sendLLMRequest(data, position) { + window.$APP_CONF.aisuggestion = false; + clearTimeout(timer); + if ((!data.segments.prefix && !data.segments.suffix)) { + return Promise.resolve({ + items: [] + }) + } + return new Promise((resolve, reject) => { + timer = setTimeout(() => { + return api.fetch(`/copilot/codecompletion`, + { ...data }, + { + method: 'post', + timeout: 5000, + removeCache: true + }).then((res) => { + if (res && res.choice) { + window.inlineCompletions = (res.choice || []).map(it => { + it.position = { + lineNumber: position.lineNumber, + column: position.column, + } + it.dbId = res.dbId + // it.text = it.text.replace(/^\s+/, '') + return it; + }); + resolve({ + items: window.inlineCompletions.map(it => { + return { + "text": it.text, + } + }) + }) + + } else { + resolve({ + items: [] + }) + } + }).catch(() => { + resolve({ + items: [] + }) + }) + }, 1000) + + }) +} + + +export function sendAccteptRequest(data) { + api.fetch(`/copilot/codeadoption`, + { adopt: true, id: data.dbId }, + { + method: 'post', + }) + .then(res => { + console.log(res) + }) + .catch(err => { + console.log(err) + }) +} \ No newline at end of file diff --git a/web/packages/shared/common/i18n/en.json b/web/packages/shared/common/i18n/en.json index b2a7d79e27..fb120245ee 100644 --- a/web/packages/shared/common/i18n/en.json +++ b/web/packages/shared/common/i18n/en.json @@ -229,6 +229,7 @@ "ShuttingDown": "Shutting Down", "LX": "By category", "AZT": "By status", + "AYY": "By Creator", "DDJK": "Waiting for API response!", "QDYQWFJS": "Cannot shut down an engine with starting status!", "WXZYQ": "No engine selected!", @@ -414,6 +415,7 @@ "HZXM": "Projects", "projectName": "Project Name", "inputProjectName": "Input Project Name", + "workspaceTarget": "Target Workspace", "gotoScriptis": "Visit Scriptis", "createProject": "Create Project", "editorProject": "Edit Project", diff --git a/web/packages/shared/common/i18n/zh.json b/web/packages/shared/common/i18n/zh.json index 3652986480..eea09f23d6 100644 --- a/web/packages/shared/common/i18n/zh.json +++ b/web/packages/shared/common/i18n/zh.json @@ -126,7 +126,7 @@ "queuememory": "队列内存使用率", "queuecpu": "队列CPU使用率", "watingList": "前面还有 {num} 个job在等待", - "showCompareDetail": "查看字段对比详情", + "showCompareDetail": "查看检测异常详情", "inconsistentFieldsDetail": "字段数据类型不一致详情", "showSolution": "查看解决方案", "columns": { @@ -211,8 +211,8 @@ "DKZDHH": "自动换行", "newdbcomplition": "库表联想-Beta", "dbcomplition": "库表联想-Release", - "closeLanguageServe": "关闭语言服务器开关[Beta]", - "openLanguageServe": "打开语言服务器开关[Beta]", + "closeLanguageServe": "关闭语言服务器[Beta]", + "openLanguageServe": "打开语言服务器[Beta]", "GBZDHH": "关闭自动换行", "GYFSYGWYF": "该语法属于高危语法，请谨慎操作！" }, @@ -230,6 +230,7 @@ "ShuttingDown": "关闭中", "LX": "按类别", "AZT": "按状态", + "AYY": "按应用", "DDJK": "请等待接口返回！", "QDYQWFJS": "启动状态引擎无法结束！", "WXZYQ": "未选中引擎！", @@ -414,6 +415,7 @@ "HZXM": "合作项目", "projectName": "项目名称", "inputProjectName": "请输入项目名称", + "workspaceTarget": "工作空间", "gotoScriptis": "进入Scriptis", "createProject": "创建项目", "editorProject": "修改项目", @@ -484,6 +486,7 @@ "Username": "用户名", "Password": "密码", "inputPassword": "请输入密码", + "inputPasswordNext": "PIN+TOKEN(测试环境使用静态密码)", "Prompt": "提示", "noPrompt": "不再提示", "adjustcolwidth": "当前列宽已调整，是否重置列宽？", @@ -632,4 +635,4 @@ } } } -} +} \ No newline at end of file diff --git a/web/packages/shared/common/service/api.js b/web/packages/shared/common/service/api.js index e305ffe999..6cfc387377 100644 --- a/web/packages/shared/common/service/api.js +++ b/web/packages/shared/common/service/api.js @@ -24,34 +24,35 @@ import { Message, Notice } from 'iview'; -import cache from './apiCache'; +import cacheReq, { cache } from './apiCache'; import qs from './querystring' import storage from "./storage" import i18n from '../i18n' -// 什么一个数组用于存储每个请求的取消函数和标识 -let pending = []; -let cancelConfig = null; -let CancelToken = axios.CancelToken; -let removePending = (config) => { - for (let p = 0; p < pending.length; p++) { - const params = JSON.stringify(config.params); - // 如果存在则执行取消操作 - if (pending[p].u === config.url + '&' + config.method + '&' + params) { - // pending[p].f();// 执行取消操作 - pending.splice(p, 1); // 移除记录 - } +// isCancel-取消标识 用于判断请求是不是被AbortController取消的 +const { isCancel } = axios +const cacheRequest = {} + +// 删除缓存队列中的请求 +function removeCacheRequest(reqKey) { + if (cacheRequest[reqKey]) { + console.log(`[API] abort cache request: ${reqKey}`) + // 通过AbortController实例上的abort来进行请求的取消 + cacheRequest[reqKey].abort() + delete cacheRequest[reqKey] } -}; +} -let cutReq = (config) => { - for (let p = 0; p < pending.length; p++) { - const params = JSON.stringify(config.params); - if (pending[p].u === config.url + '&' + config.method + '&' + params) { - return true; - } + +let getErrorMsg = (message) => { + let msg; + if (message === 'Request failed with status code 414') { + msg = 'URL长度超出最大限制'; + } else if (message === 'Request failed with status code 413') { + msg = '请求体内容超出最大限制'; } -}; + return msg; +} const instance = axios.create({ baseURL: process.env.VUE_APP_MN_CONFIG_PREFIX || `${location.protocol}//${window.location.host}/api/rest_j/v1/`, @@ -68,6 +69,9 @@ instance.interceptors.request.use((config) => { config.metadata = { startTime: Date.now() } + if (/\/user\/login/.test(config.url)) { + cache.reset(); + } if (/\/application\//.test(config.url)) { config.url = `http://${window.location.host}` + config.url } @@ -90,26 +94,22 @@ instance.interceptors.request.use((config) => { config.headers['Content-Type'] = 'application/x-www-form-urlencoded' config.data = qs(config.data) } - - let flag = cutReq(config); - // 当上一次相同请求未完成时，无法进行第二次相同请求 - if (flag === true) { - removePending(config); - return config; - } else { - const params = JSON.stringify(config.params); - // 用于正常请求出现错误时移除 - cancelConfig = config; - config.cancelToken = new CancelToken((c) => { - // 添加标识和取消函数 - pending.push({ - u: config.url + '&' + config.method + '&' + params, - f: c, - }); - }); - return config; + // removeCache - 是config里配置的是否清除相同请求的标识，不传则默认是不需要清除 + // 此处根据实际需求来 如果需要全部清除相同的请求功能 则默认为true 或者增加白名单 + const { url, method, removeCache = false } = config + if (removeCache) { + // 请求地址和请求方式组成唯一标识，将这个标识作为取消函数的key，保存到请求队列中 + const reqKey = `${url}&${method}` + // 如果config传了需要清除重复请求的removeCache，则如果存在重复请求，删除之前的请求 + removeCacheRequest(reqKey) + // 将请求加入请求队列，通过AbortController来进行手动取消 + const controller = new AbortController() + config.signal = controller.signal + cacheRequest[reqKey] = controller } + return config; }, (error) => { + console.log(error) Promise.reject(error); }); @@ -117,23 +117,35 @@ instance.interceptors.response.use((response) => { response.config.metadata.endTime = Date.now() const duration = response.config.metadata.endTime - response.config.metadata.startTime if (window.$Wa && duration > 2000) window.$Wa.log(`接口耗时 ${duration}: ${response.config.url}`); - // 在一个ajax响应成功后再执行取消操作，把已完成的请求从pending中移除 - removePending(response.config); - return response; + // 请求成功，从队列中移除 + const { url, method, removeCache = false } = response.config; + if (removeCache) removeCacheRequest(`${url}&${method}`) + return response }, (error) => { - error.config.metadata.endTime = Date.now() - const duration = error.config.metadata.endTime - error.config.metadata.sartTime - if (window.$Wa && duration > 2000) window.$Wa.log(`接口耗时 ${duration}: ${error.config.url}`); + if (isCancel(error)) { + // 通过CancelToken取消的请求不做任何处理 + console.warn('重复请求，自动拦截并取消') + return + } + if (error.config) { + error.config.metadata.endTime = Date.now() + const duration = error.config.metadata.endTime - error.config.metadata.sartTime + if (window.$Wa && duration > 2000) window.$Wa.log(`接口耗时 ${duration}: ${error.config.url}`); + } // 出现接口异常或者超时时的判断 - if ((error.message && error.message.indexOf('timeout') >= 0) || (error.request && error.request.status !== 200)) { - for (let p in pending) { - if (pending[p].u === cancelConfig.url + '&' + cancelConfig.method + '&' + JSON.stringify(cancelConfig.params)) { - pending.splice(p, 1); // 移除记录 - } + if (error.code === 'ERR_BAD_REQUEST' && getErrorMsg(error.message)) { + if (!error.response) error.response = {}; + error.response.data = { + message: getErrorMsg(error.message), + method: '', + status: 1, + data: { solution: null } } + return error.response; + } else if ((error.message && error.message.indexOf('timeout') >= 0) || (error.request && error.request.status !== 200)) { // 优先返回后台返回的错误信息，其次是接口返回 return error.response || error; - } else if (axios.Cancel) { + } else if (axios.isCancel(error)) { // 如果是pengding状态，弹出提示！ return { // data: { message: '接口请求中！请稍后……' }, @@ -151,9 +163,9 @@ const api = { return window.location.replace(res.data.SSOURL); } if (window.location.href.indexOf('/#/login') < 0) { - window.location.href = '/#/login' + return window.location.href = '/#/login' } - if (!showLoginTips && res.data.method.indexOf('jobhistory/governanceStationAdmin') < 0) { + if (!showLoginTips) { showLoginTips = true setTimeout(() => { showLoginTips = false @@ -343,7 +355,7 @@ const param = function (url, data, option) { } // cacheOptions接口数据缓存 {time} time为0则请求之后缓存在内存里的数据不清理 if (option.cacheOptions) { - option.adapter = cache(option.cacheOptions) + option.adapter = cacheReq(option.cacheOptions) } option.url = url; @@ -363,8 +375,7 @@ const showErrMsg = function (error) { msg = i18n.t('message.common.apierrmsg') } let isHoverNotice = {} - const checkPath = !error.response || error.response.config.url.indexOf('dss/guide/solution/reportProblem') < 0 - if (window.$APP_CONF && window.$APP_CONF.error_report && checkPath) { + if (window.$APP_CONF && window.$APP_CONF.error_report && error.response) { const noticeName = 'err_' + Date.now() isHoverNotice[noticeName] = false Notice.error({ @@ -431,7 +442,7 @@ const showErrMsg = function (error) { }) setTimeout(() => { document.querySelectorAll('.g-err-msg-div').forEach(ele => { - const erritem =ele.dataset.erritem + const erritem = ele.dataset.erritem ele.parentElement.parentElement.style.textAlign = 'left' ele.parentElement.style.display = 'block' ele.parentElement.style.padding = 0 @@ -446,14 +457,20 @@ const showErrMsg = function (error) { } else { Notice.error({ desc: msg, - title: i18n.t('message.common.errTitle') , + title: i18n.t('message.common.errTitle'), duration: 4 }); } } // 对于某些特殊接口，接口报错时不做展示 const isShowErr = (url) => { - const noShowUrlList = ['/validator/code-precheck']; + const noShowUrlList = [ + '/dss/datapipe/datasource/getSchemaBaseInfo', + '/validator/code-precheck', + '/copilot/codecompletion', + '/copilot/codeadoption', + '/copilot/user/isinwhitelist', + '/dss/guide/solution/reportProblem']; if (noShowUrlList.includes(url)) { return false; } diff --git a/web/packages/shared/common/service/apiCache.js b/web/packages/shared/common/service/apiCache.js index 52e152b875..7c86155f0d 100644 --- a/web/packages/shared/common/service/apiCache.js +++ b/web/packages/shared/common/service/apiCache.js @@ -11,6 +11,9 @@ export const cache = { }, clear(key) { delete this.data[key] + }, + reset() { + this.data = {} } } @@ -41,7 +44,7 @@ export default (options = {}) => async config => { if (!responsePromise) { responsePromise = (async () => { try { - const response = await axios.defaults.adapter(config) + const response = await axios.getAdapter(['xhr', 'http'])(config) return Promise.resolve(response) } catch (reason) { cache.clear(index) diff --git a/web/packages/shared/common/service/execute.js b/web/packages/shared/common/service/execute.js index e06fd28fb7..59c49a5ba4 100644 --- a/web/packages/shared/common/service/execute.js +++ b/web/packages/shared/common/service/execute.js @@ -20,7 +20,9 @@ import storage from '@dataspherestudio/shared/common/helper/storage'; import Vue from 'vue'; import axios from 'axios'; import { Message } from 'iview'; -import i18n from '@dataspherestudio/shared/common/i18n' +import i18n from '@dataspherestudio/shared/common/i18n'; +import { EXECUTE_COMPLETE_TYPE } from '@dataspherestudio/shared/common/config/const'; + /** * 记录轮询接口请求日志 @@ -333,7 +335,7 @@ Execute.prototype.queryLog = function() { try { const fromLine = this.fromLine // dpms: /#/product/100199/bug/detail/222584 - if ( this.openLog && ['Succeed', 'Failed', 'Cancelled', 'Timeout'].indexOf(this.status) !== -1) { + if ( this.openLog && EXECUTE_COMPLETE_TYPE.indexOf(this.status) !== -1) { if (this.resultsetInfo) { return api.fetch('/filesystem/openLog', { path: this.resultsetInfo.logPath @@ -475,48 +477,11 @@ Execute.prototype.getFirstResult = function() { taskID: this.taskID, status: this.status, }); - let params = { - path: this.currentResultPath, - pageSize, - } - // 如果是api执行需要带上taskId - if (this.getResultUrl !== 'filesystem') { - params.taskId = this.taskID - } - const url = `/${this.getResultUrl}/openFile`; - const pageSize = 5000; - api.fetch(url, params, 'get') - .then((rst) => { - if (rst.display_prohibited) { - this.trigger('result', { - metadata: [], - fileContent: [], - type: rst.type, - totalLine: rst.totalLine, - hugeData: true, - tipMsg: localStorage.getItem("locale") === "en" ? rst.en_msg : rst.zh_msg - }); - } else if (rst.metadata && rst.metadata.length >= 500) { - this.trigger('result', { - metadata: [], - fileContent: [], - type: rst.type, - totalLine: rst.totalLine, - hugeData: true - }); - } else { - this.trigger('result', rst); - } - - const log = `**result tips: ${i18n.t('message.common.execute.success.getResultList')}`; - this.trigger('log', log); - this.trigger('steps', 'Completed'); - this.run = false; - }) - .catch((err) => { - this.trigger('steps', 'FailedToGetResultFirst'); - logError(err, this); - }); + this.trigger('result', {}); + const log = `**result tips: ${i18n.t('message.common.execute.success.getResultList')}`; + this.trigger('log', log); + this.trigger('steps', 'Completed'); + this.run = false; } }; diff --git a/web/packages/shared/common/service/layoutMixin.js b/web/packages/shared/common/service/layoutMixin.js index 6f92d71e32..4242a26c39 100644 --- a/web/packages/shared/common/service/layoutMixin.js +++ b/web/packages/shared/common/service/layoutMixin.js @@ -10,9 +10,11 @@ export default { }, mounted() { document.addEventListener('copy', this.copyAction, false); + document.addEventListener('cut', this.copyAction, false); }, beforeDestroy() { document.removeEventListener('copy', this.copyAction, false); + document.removeEventListener('cut', this.copyAction, false); }, methods: { copyAction(event) { diff --git a/web/packages/shared/common/style/overwrite.scss b/web/packages/shared/common/style/overwrite.scss index f35698e487..cd685ba6cb 100644 --- a/web/packages/shared/common/style/overwrite.scss +++ b/web/packages/shared/common/style/overwrite.scss @@ -474,6 +474,11 @@ .progress-label { color: $dark-text-color !important; } + .designer div.node-disabled { + background-color: #515a6e; + color: #cfd0d34d; + border-radius: 4px; + } .designer div.node-selected { background: rgba(222, 232, 244, 0.62); box-shadow: 1px 4px 6px 1px rgba(106, 141, 185, 0.65); diff --git a/web/packages/shared/components/consoleComponent/console.vue b/web/packages/shared/components/consoleComponent/console.vue index 2a1a5192e9..8890cba96f 100644 --- a/web/packages/shared/components/consoleComponent/console.vue +++ b/web/packages/shared/components/consoleComponent/console.vue @@ -34,6 +34,17 @@ {{ $t('message.common.tabs.history') }} +
+ 列分页：500列 / 页，共{{ script.result.totalColumn }}列 + +
@@ -47,7 +58,6 @@ { - this.showPanelTab('result'); const storeResult = { 'headRows': ret.metadata, 'bodyRows': ret.fileContent, 'total': ret.totalLine, 'type': ret.type, + 'totalColumn': ret.totalColumn, 'cache': { offsetX: 0, offsetY: 0, @@ -268,6 +294,7 @@ export default { if (this.script.progress.current) { this.script.progress.current = 1; } + this.showPanelTab('result'); }); this.execute.on('progress', ({ progress, progressInfo, waitingSize }) => { // 这里progressInfo可能只是个空数组，或者数据第一个数据是一个空对象 @@ -378,7 +405,7 @@ export default { }) }, resetQuery() { - this.showPanelTab(this.isHistory ? 'result' : 'progress'); + this.showPanelTab(this.isHistory ? '' : 'progress'); this.isLogShow = false; this.localLog = { log: { all: '', error: '', warning: '', info: '' }, @@ -424,75 +451,8 @@ export default { }, changeResultSet(data, cb) { const resultSet = isUndefined(data.currentSet) ? this.script.resultSet : data.currentSet; - const findResult = this.script.resultList[resultSet]; - const resultPath = findResult && findResult.path; - const hasResult = Object.prototype.hasOwnProperty.call(this.script.resultList[resultSet], 'result'); - if (!hasResult) { - const pageSize = 5000; - const url = `/${this.getResultUrl}/openFile`; - let params = { - path: resultPath, - pageSize, - } - // 如果是api执行需要带上taskId - if (this.getResultUrl !== 'filesystem') { - params.taskId = this.work.taskID - } - api.fetch(url, { - ...params - }, 'get') - .then((ret) => { - let result = {} - if (ret.display_prohibited) { - result = { - 'headRows': [], - 'bodyRows': [], - 'total': ret.totalLine, - 'type': ret.type, - 'path': resultPath, - hugeData: true, - tipMsg: localStorage.getItem("locale") === "en" ? ret.en_msg : ret.zh_msg - }; - } else if (ret.metadata && ret.metadata.length >= 500) { - result = { - 'headRows': [], - 'bodyRows': [], - 'total': ret.totalLine, - 'type': ret.type, - 'path': resultPath, - hugeData: true - }; - } else { - result = { - 'headRows': ret.metadata, - 'bodyRows': ret.fileContent, - // 如果totalLine是null，就显示为0 - 'total': ret.totalLine ? ret.totalLine : 0, - // 如果内容为null,就显示暂无数据 - 'type': ret.fileContent ? ret.type : 0, - 'cache': { - offsetX: 0, - offsetY: 0, - }, - 'path': resultPath, - 'current': 1, - 'size': 20, - }; - } - this.$set(this.script.resultList[resultSet], 'result', result); - this.$set(this.script, 'resultSet', resultSet); - this.script.result = result; - this.script.resultSet = resultSet; - this.script.showPanel = 'result'; - cb(); - }).catch(() => { - cb(); - }); - } else { - this.script.result = this.script.resultList[resultSet].result; - this.$set(this.script, 'resultSet', resultSet); - this.script.resultSet = resultSet; - this.script.showPanel = 'result'; + this.script.resultSet = resultSet; + if (cb) { cb(); } }, @@ -548,7 +508,10 @@ export default { }); return tmpLogs; } - } + }, + beforeDestroy() { + window.removeEventListener('resize', this.resultResize) + }, } diff --git a/web/packages/shared/components/consoleComponent/history.vue b/web/packages/shared/components/consoleComponent/history.vue index 05888d122c..02d0169375 100644 --- a/web/packages/shared/components/consoleComponent/history.vue +++ b/web/packages/shared/components/consoleComponent/history.vue @@ -30,7 +30,8 @@ import util from '@dataspherestudio/shared/common/util'; import storage from '@dataspherestudio/shared/common/helper/storage'; import table from '@dataspherestudio/shared/components/virtualTable'; import mixin from '@dataspherestudio/shared/common/service/mixin'; -const EXECUTE_COMPLETE_TYPE = ['Succeed', 'Failed', 'Cancelled', 'Timeout']; +import { EXECUTE_COMPLETE_TYPE } from '@dataspherestudio/shared/common/config/const'; + export default { components: { historyTable: table.historyTable, diff --git a/web/packages/shared/components/consoleComponent/progress.vue b/web/packages/shared/components/consoleComponent/progress.vue index ce16c8112d..9e6f7b646c 100644 --- a/web/packages/shared/components/consoleComponent/progress.vue +++ b/web/packages/shared/components/consoleComponent/progress.vue @@ -29,10 +29,16 @@
{{codePreCheckInfo.checkData[0].errorMessage}} - +
- +
+ +
+
+ + {{paramsInfo.content}} +
diff --git a/web/packages/shared/components/consoleComponent/result.vue b/web/packages/shared/components/consoleComponent/result.vue index 9178e1efb3..d6d2f3b963 100644 --- a/web/packages/shared/components/consoleComponent/result.vue +++ b/web/packages/shared/components/consoleComponent/result.vue @@ -48,6 +48,7 @@ > diff --git a/web/packages/shared/components/projectForm/index.vue b/web/packages/shared/components/projectForm/index.vue index 24dc96e576..ec910bb3e0 100644 --- a/web/packages/shared/components/projectForm/index.vue +++ b/web/packages/shared/components/projectForm/index.vue @@ -1,6 +1,7 @@