From 1b9dc10e30356316816c0962ad52fc17a9dffa5a Mon Sep 17 00:00:00 2001 From: Lynwee <1507509064@qq.com> Date: Thu, 19 Sep 2024 12:13:26 +0800 Subject: [PATCH 01/16] feat: check pipelines' tokens before executing/creating pipeline (#8055) * feat(pipeline): check pipelines' tokens before executing/creating pipeline * fix(framework): fix lint errors * fix(framework): fix lint errors * feat(github): check multi tokens when creating pipelines --- backend/core/plugin/plugin_api.go | 4 + backend/core/plugin/plugin_blueprint.go | 10 +++ .../helpers/pluginhelper/api/misc_helpers.go | 10 +++ backend/plugins/ae/impl/impl.go | 5 ++ backend/plugins/azuredevops_go/impl/impl.go | 5 ++ backend/plugins/bamboo/impl/impl.go | 5 ++ backend/plugins/bitbucket/impl/impl.go | 5 ++ backend/plugins/bitbucket_server/impl/impl.go | 5 ++ backend/plugins/circleci/impl/impl.go | 5 ++ backend/plugins/customize/impl/impl.go | 4 + backend/plugins/dbt/impl/impl.go | 4 + backend/plugins/dora/impl/impl.go | 5 +- backend/plugins/feishu/impl/impl.go | 5 ++ backend/plugins/gitee/impl/impl.go | 5 ++ backend/plugins/gitextractor/impl/impl.go | 4 + backend/plugins/github/api/shared.go | 42 +++++++++++ backend/plugins/github/impl/impl.go | 4 + backend/plugins/gitlab/impl/impl.go | 5 ++ backend/plugins/icla/impl/impl.go | 4 + backend/plugins/issue_trace/impl/enricher.go | 5 +- backend/plugins/jenkins/impl/impl.go | 5 ++ backend/plugins/jira/impl/impl.go | 5 ++ backend/plugins/linker/impl/impl.go | 4 + backend/plugins/opsgenie/impl/impl.go | 5 ++ backend/plugins/org/impl/impl.go | 22 ++++++ backend/plugins/org/tasks/check_token.go | 75 +++++++++++++++++++ backend/plugins/org/tasks/task_data.go | 10 ++- backend/plugins/pagerduty/impl/impl.go | 5 ++ backend/plugins/refdiff/impl/impl.go | 4 + backend/plugins/slack/impl/impl.go | 5 ++ backend/plugins/sonarqube/impl/impl.go | 5 ++ backend/plugins/starrocks/impl/impl.go | 4 + backend/plugins/tapd/impl/impl.go | 5 ++ backend/plugins/teambition/impl/impl.go | 5 ++ backend/plugins/trello/impl/impl.go | 5 ++ backend/plugins/webhook/impl/impl.go | 4 + backend/plugins/zentao/impl/impl.go | 5 ++ backend/server/services/blueprint.go | 69 +++++++++++++++++ .../services/blueprint_makeplan_v200.go | 15 ++++ backend/server/services/project.go | 2 +- .../services/remote/models/plugin_remote.go | 1 + .../services/remote/plugin/plugin_impl.go | 5 ++ 42 files changed, 401 insertions(+), 5 deletions(-) create mode 100644 backend/plugins/github/api/shared.go create mode 100644 backend/plugins/org/tasks/check_token.go diff --git a/backend/core/plugin/plugin_api.go b/backend/core/plugin/plugin_api.go index f9195d2eed4..ea41a4895e8 100644 --- a/backend/core/plugin/plugin_api.go +++ b/backend/core/plugin/plugin_api.go @@ -73,6 +73,10 @@ type PluginApi interface { ApiResources() map[string]map[string]ApiResourceHandler } +type PluginTestConnectionAPI interface { + TestConnection(id uint64) errors.Error +} + const wrapResponseError = "WRAP_RESPONSE_ERROR" func WrapTestConnectionErrResp(basicRes context.BasicRes, err errors.Error) errors.Error { diff --git a/backend/core/plugin/plugin_blueprint.go b/backend/core/plugin/plugin_blueprint.go index b722bb9e2cf..a24513564b1 100644 --- a/backend/core/plugin/plugin_blueprint.go +++ b/backend/core/plugin/plugin_blueprint.go @@ -93,6 +93,16 @@ type ProjectMapper interface { MapProject(projectName string, scopes []Scope) (models.PipelinePlan, errors.Error) } +type ProjectTokenCheckerConnection struct { + PluginName string + ConnectionId uint64 +} + +// ProjectTokenChecker is implemented by the plugin org, which generate a task tp check all connection's tokens +type ProjectTokenChecker interface { + MakePipeline(skipCollectors bool, projectName string, scopes []ProjectTokenCheckerConnection) (models.PipelinePlan, errors.Error) +} + // CompositeDataSourcePluginBlueprintV200 is for unit test type CompositeDataSourcePluginBlueprintV200 interface { PluginMeta diff --git a/backend/helpers/pluginhelper/api/misc_helpers.go b/backend/helpers/pluginhelper/api/misc_helpers.go index e7315ceab5a..3dcf8778c7f 100644 --- a/backend/helpers/pluginhelper/api/misc_helpers.go +++ b/backend/helpers/pluginhelper/api/misc_helpers.go @@ -18,9 +18,11 @@ limitations under the License. package api import ( + "fmt" "github.com/apache/incubator-devlake/core/dal" "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/models" + "github.com/apache/incubator-devlake/core/plugin" ) // CallDB wraps DB calls with this signature, and handles the case if the struct is wrapped in a models.DynamicTabler. @@ -31,3 +33,11 @@ func CallDB(f func(any, ...dal.Clause) errors.Error, x any, clauses ...dal.Claus } return f(x, clauses...) } + +func GenerateTestingConnectionApiResourceInput(connectionID uint64) *plugin.ApiResourceInput { + return &plugin.ApiResourceInput{ + Params: map[string]string{ + "connectionId": fmt.Sprintf("%d", connectionID), + }, + } +} diff --git a/backend/plugins/ae/impl/impl.go b/backend/plugins/ae/impl/impl.go index 389857e1f30..7356d9e0e2c 100644 --- a/backend/plugins/ae/impl/impl.go +++ b/backend/plugins/ae/impl/impl.go @@ -130,6 +130,11 @@ func (p AE) MigrationScripts() []plugin.MigrationScript { return migrationscripts.All() } +func (p AE) TestConnection(id uint64) errors.Error { + _, err := api.TestExistingConnection(helper.GenerateTestingConnectionApiResourceInput(id)) + return err +} + func (p AE) ApiResources() map[string]map[string]plugin.ApiResourceHandler { return map[string]map[string]plugin.ApiResourceHandler{ "test": { diff --git a/backend/plugins/azuredevops_go/impl/impl.go b/backend/plugins/azuredevops_go/impl/impl.go index c567e49f57c..2aecfde5105 100644 --- a/backend/plugins/azuredevops_go/impl/impl.go +++ b/backend/plugins/azuredevops_go/impl/impl.go @@ -255,6 +255,11 @@ func (p Azuredevops) ApiResources() map[string]map[string]plugin.ApiResourceHand } } +func (p Azuredevops) TestConnection(id uint64) errors.Error { + _, err := api.TestExistingConnection(helper.GenerateTestingConnectionApiResourceInput(id)) + return err +} + func (p Azuredevops) MakeDataSourcePipelinePlanV200( connectionId uint64, scopes []*coreModels.BlueprintScope, diff --git a/backend/plugins/bamboo/impl/impl.go b/backend/plugins/bamboo/impl/impl.go index b90331df1ca..b7737770a72 100644 --- a/backend/plugins/bamboo/impl/impl.go +++ b/backend/plugins/bamboo/impl/impl.go @@ -216,6 +216,11 @@ func (p Bamboo) MigrationScripts() []plugin.MigrationScript { return migrationscripts.All() } +func (p Bamboo) TestConnection(id uint64) errors.Error { + _, err := api.TestExistingConnection(helper.GenerateTestingConnectionApiResourceInput(id)) + return err +} + func (p Bamboo) ApiResources() map[string]map[string]plugin.ApiResourceHandler { return map[string]map[string]plugin.ApiResourceHandler{ "test": { diff --git a/backend/plugins/bitbucket/impl/impl.go b/backend/plugins/bitbucket/impl/impl.go index dd917790c71..4ff5bc5e9e7 100644 --- a/backend/plugins/bitbucket/impl/impl.go +++ b/backend/plugins/bitbucket/impl/impl.go @@ -211,6 +211,11 @@ func (p Bitbucket) MakeDataSourcePipelinePlanV200( return api.MakeDataSourcePipelinePlanV200(p.SubTaskMetas(), connectionId, scopes, skipCollectors) } +func (p Bitbucket) TestConnection(id uint64) errors.Error { + _, err := api.TestExistingConnection(helper.GenerateTestingConnectionApiResourceInput(id)) + return err +} + func (p Bitbucket) ApiResources() map[string]map[string]plugin.ApiResourceHandler { return map[string]map[string]plugin.ApiResourceHandler{ "test": { diff --git a/backend/plugins/bitbucket_server/impl/impl.go b/backend/plugins/bitbucket_server/impl/impl.go index 18b4a46795a..fe446d76ffa 100644 --- a/backend/plugins/bitbucket_server/impl/impl.go +++ b/backend/plugins/bitbucket_server/impl/impl.go @@ -170,6 +170,11 @@ func (p BitbucketServer) MakeDataSourcePipelinePlanV200( return api.MakeDataSourcePipelinePlanV200(p.SubTaskMetas(), connectionId, scopes, skipCollectors) } +func (p BitbucketServer) TestConnection(id uint64) errors.Error { + _, err := api.TestExistingConnection(helper.GenerateTestingConnectionApiResourceInput(id)) + return err +} + func (p BitbucketServer) ApiResources() map[string]map[string]plugin.ApiResourceHandler { return map[string]map[string]plugin.ApiResourceHandler{ "connections/:connectionId/test": { diff --git a/backend/plugins/circleci/impl/impl.go b/backend/plugins/circleci/impl/impl.go index 8cca5895d9b..c08766a6d72 100644 --- a/backend/plugins/circleci/impl/impl.go +++ b/backend/plugins/circleci/impl/impl.go @@ -176,6 +176,11 @@ func (p Circleci) MigrationScripts() []plugin.MigrationScript { return migrationscripts.All() } +func (p Circleci) TestConnection(id uint64) errors.Error { + _, err := api.TestExistingConnection(helper.GenerateTestingConnectionApiResourceInput(id)) + return err +} + func (p Circleci) ApiResources() map[string]map[string]plugin.ApiResourceHandler { return map[string]map[string]plugin.ApiResourceHandler{ "test": { diff --git a/backend/plugins/customize/impl/impl.go b/backend/plugins/customize/impl/impl.go index facb0515d97..59b967aa518 100644 --- a/backend/plugins/customize/impl/impl.go +++ b/backend/plugins/customize/impl/impl.go @@ -110,3 +110,7 @@ func (p Customize) ApiResources() map[string]map[string]plugin.ApiResourceHandle }, } } + +func (p Customize) TestConnection(id uint64) errors.Error { + return nil +} diff --git a/backend/plugins/dbt/impl/impl.go b/backend/plugins/dbt/impl/impl.go index 88eafa9f633..55b7c2918ed 100644 --- a/backend/plugins/dbt/impl/impl.go +++ b/backend/plugins/dbt/impl/impl.go @@ -74,3 +74,7 @@ func (p Dbt) RootPkgPath() string { func (p Dbt) Name() string { return "dbt" } + +func (p Dbt) TestConnection(id uint64) errors.Error { + return nil +} diff --git a/backend/plugins/dora/impl/impl.go b/backend/plugins/dora/impl/impl.go index 24609739811..0d41a94ebad 100644 --- a/backend/plugins/dora/impl/impl.go +++ b/backend/plugins/dora/impl/impl.go @@ -19,7 +19,6 @@ package impl import ( "encoding/json" - "github.com/apache/incubator-devlake/core/dal" "github.com/apache/incubator-devlake/core/errors" coreModels "github.com/apache/incubator-devlake/core/models" @@ -168,3 +167,7 @@ func (p Dora) MakeMetricPluginPipelinePlanV200(projectName string, options json. } return plan, nil } + +func (p Dora) TestConnection(id uint64) errors.Error { + return nil +} diff --git a/backend/plugins/feishu/impl/impl.go b/backend/plugins/feishu/impl/impl.go index 92e0f1571c5..3158b5f1fef 100644 --- a/backend/plugins/feishu/impl/impl.go +++ b/backend/plugins/feishu/impl/impl.go @@ -132,6 +132,11 @@ func (p Feishu) MigrationScripts() []plugin.MigrationScript { return migrationscripts.All() } +func (p Feishu) TestConnection(id uint64) errors.Error { + _, err := api.TestExistingConnection(helper.GenerateTestingConnectionApiResourceInput(id)) + return err +} + func (p Feishu) ApiResources() map[string]map[string]plugin.ApiResourceHandler { return map[string]map[string]plugin.ApiResourceHandler{ "test": { diff --git a/backend/plugins/gitee/impl/impl.go b/backend/plugins/gitee/impl/impl.go index 35be389e79b..b422fbafc05 100644 --- a/backend/plugins/gitee/impl/impl.go +++ b/backend/plugins/gitee/impl/impl.go @@ -187,6 +187,11 @@ func (p Gitee) MigrationScripts() []plugin.MigrationScript { return migrationscripts.All() } +func (p Gitee) TestConnection(id uint64) errors.Error { + _, err := api.TestExistingConnection(helper.GenerateTestingConnectionApiResourceInput(id)) + return err +} + func (p Gitee) ApiResources() map[string]map[string]plugin.ApiResourceHandler { return map[string]map[string]plugin.ApiResourceHandler{ "test": { diff --git a/backend/plugins/gitextractor/impl/impl.go b/backend/plugins/gitextractor/impl/impl.go index 468e5173409..45c84ab6e17 100644 --- a/backend/plugins/gitextractor/impl/impl.go +++ b/backend/plugins/gitextractor/impl/impl.go @@ -122,3 +122,7 @@ func (p GitExtractor) Close(taskCtx plugin.TaskContext) errors.Error { func (p GitExtractor) RootPkgPath() string { return "github.com/apache/incubator-devlake/plugins/gitextractor" } + +func (p GitExtractor) TestConnection(id uint64) errors.Error { + return nil +} diff --git a/backend/plugins/github/api/shared.go b/backend/plugins/github/api/shared.go new file mode 100644 index 00000000000..61bb398015e --- /dev/null +++ b/backend/plugins/github/api/shared.go @@ -0,0 +1,42 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "context" + "fmt" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" +) + +func TestExistingConnectionForTokenCheck(input *plugin.ApiResourceInput) errors.Error { + connection, err := dsHelper.ConnApi.GetMergedConnection(input) + if err != nil { + return err + } + testConnectionResult, testConnectionErr := testExistingConnection(context.TODO(), connection.GithubConn) + if testConnectionErr != nil { + return testConnectionErr + } + for _, token := range testConnectionResult.Tokens { + if !token.Success { + return errors.Default.New(fmt.Sprintf("token %s failed with msg: %s", token.Token, token.Message)) + } + } + return nil +} diff --git a/backend/plugins/github/impl/impl.go b/backend/plugins/github/impl/impl.go index 52a772cf603..e030923706f 100644 --- a/backend/plugins/github/impl/impl.go +++ b/backend/plugins/github/impl/impl.go @@ -191,6 +191,10 @@ func (p Github) MigrationScripts() []plugin.MigrationScript { return migrationscripts.All() } +func (p Github) TestConnection(id uint64) errors.Error { + return api.TestExistingConnectionForTokenCheck(helper.GenerateTestingConnectionApiResourceInput(id)) +} + func (p Github) ApiResources() map[string]map[string]plugin.ApiResourceHandler { return map[string]map[string]plugin.ApiResourceHandler{ "test": { diff --git a/backend/plugins/gitlab/impl/impl.go b/backend/plugins/gitlab/impl/impl.go index d292db691ed..46f63b52f9c 100644 --- a/backend/plugins/gitlab/impl/impl.go +++ b/backend/plugins/gitlab/impl/impl.go @@ -244,6 +244,11 @@ func (p Gitlab) MigrationScripts() []plugin.MigrationScript { return migrationscripts.All() } +func (p Gitlab) TestConnection(id uint64) errors.Error { + _, err := api.TestExistingConnection(helper.GenerateTestingConnectionApiResourceInput(id)) + return err +} + func (p Gitlab) ApiResources() map[string]map[string]plugin.ApiResourceHandler { return map[string]map[string]plugin.ApiResourceHandler{ "test": { diff --git a/backend/plugins/icla/impl/impl.go b/backend/plugins/icla/impl/impl.go index 1f327f31318..2430741267e 100644 --- a/backend/plugins/icla/impl/impl.go +++ b/backend/plugins/icla/impl/impl.go @@ -104,6 +104,10 @@ func (p Icla) ApiResources() map[string]map[string]plugin.ApiResourceHandler { return nil } +func (p Icla) TestConnection(id uint64) errors.Error { + return nil +} + func (p Icla) Close(taskCtx plugin.TaskContext) errors.Error { data, ok := taskCtx.GetData().(*tasks.IclaTaskData) if !ok { diff --git a/backend/plugins/issue_trace/impl/enricher.go b/backend/plugins/issue_trace/impl/enricher.go index 1506f2563dd..aeca45aded0 100644 --- a/backend/plugins/issue_trace/impl/enricher.go +++ b/backend/plugins/issue_trace/impl/enricher.go @@ -19,7 +19,6 @@ package impl import ( "encoding/json" - "github.com/apache/incubator-devlake/core/context" "github.com/apache/incubator-devlake/core/dal" "github.com/apache/incubator-devlake/core/errors" @@ -151,6 +150,10 @@ func (p IssueTrace) GetTablesInfo() []dal.Tabler { } } +func (p IssueTrace) TestConnection(id uint64) errors.Error { + return nil +} + func (p IssueTrace) MakeMetricPluginPipelinePlanV200(projectName string, options json.RawMessage) (coreModels.PipelinePlan, errors.Error) { op := &tasks.Options{} if options != nil && string(options) != "\"\"" { diff --git a/backend/plugins/jenkins/impl/impl.go b/backend/plugins/jenkins/impl/impl.go index 2a686037fd4..5a7f478b634 100644 --- a/backend/plugins/jenkins/impl/impl.go +++ b/backend/plugins/jenkins/impl/impl.go @@ -177,6 +177,11 @@ func (p Jenkins) MakeDataSourcePipelinePlanV200( return api.MakeDataSourcePipelinePlanV200(p.SubTaskMetas(), connectionId, scopes, skipCollectors) } +func (p Jenkins) TestConnection(id uint64) errors.Error { + _, err := api.TestExistingConnection(helper.GenerateTestingConnectionApiResourceInput(id)) + return err +} + func (p Jenkins) ApiResources() map[string]map[string]plugin.ApiResourceHandler { return map[string]map[string]plugin.ApiResourceHandler{ "test": { diff --git a/backend/plugins/jira/impl/impl.go b/backend/plugins/jira/impl/impl.go index b7a62da20e1..51e623f32f9 100644 --- a/backend/plugins/jira/impl/impl.go +++ b/backend/plugins/jira/impl/impl.go @@ -285,6 +285,11 @@ func (p Jira) MigrationScripts() []plugin.MigrationScript { return migrationscripts.All() } +func (p Jira) TestConnection(id uint64) errors.Error { + _, err := api.TestExistingConnection(helper.GenerateTestingConnectionApiResourceInput(id)) + return err +} + func (p Jira) ApiResources() map[string]map[string]plugin.ApiResourceHandler { return map[string]map[string]plugin.ApiResourceHandler{ "test": { diff --git a/backend/plugins/linker/impl/impl.go b/backend/plugins/linker/impl/impl.go index 917b22df9aa..27b9a6cef20 100644 --- a/backend/plugins/linker/impl/impl.go +++ b/backend/plugins/linker/impl/impl.go @@ -125,3 +125,7 @@ func (p Linker) MakeMetricPluginPipelinePlanV200(projectName string, options jso } return plan, nil } + +func (p Linker) TestConnection(id uint64) errors.Error { + return nil +} diff --git a/backend/plugins/opsgenie/impl/impl.go b/backend/plugins/opsgenie/impl/impl.go index 5785d53ff8a..7da8552e630 100644 --- a/backend/plugins/opsgenie/impl/impl.go +++ b/backend/plugins/opsgenie/impl/impl.go @@ -155,6 +155,11 @@ func (p Opsgenie) Close(taskCtx plugin.TaskContext) errors.Error { return nil } +func (p Opsgenie) TestConnection(id uint64) errors.Error { + _, err := api.TestExistingConnection(helper.GenerateTestingConnectionApiResourceInput(id)) + return err +} + func (p Opsgenie) ApiResources() map[string]map[string]plugin.ApiResourceHandler { return map[string]map[string]plugin.ApiResourceHandler{ "test": { diff --git a/backend/plugins/org/impl/impl.go b/backend/plugins/org/impl/impl.go index cd4ffa7fda1..f0e6c4a0a0a 100644 --- a/backend/plugins/org/impl/impl.go +++ b/backend/plugins/org/impl/impl.go @@ -59,11 +59,33 @@ func (p Org) Name() string { func (p Org) SubTaskMetas() []plugin.SubTaskMeta { return []plugin.SubTaskMeta{ + tasks.TaskCheckTokenMeta, tasks.ConnectUserAccountsExactMeta, tasks.SetProjectMappingMeta, } } +func (p Org) MakePipeline(skipCollectors bool, projectName string, connections []plugin.ProjectTokenCheckerConnection) (coreModels.PipelinePlan, errors.Error) { + var plan coreModels.PipelinePlan + var stage coreModels.PipelineStage + + // construct task options for Org + options := make(map[string]interface{}) + if !skipCollectors { + options["projectConnections"] = connections + } + + stage = append(stage, &coreModels.PipelineTask{ + Plugin: p.Name(), + Subtasks: []string{ + tasks.TaskCheckTokenMeta.Name, + }, + Options: options, + }) + plan = append(plan, stage) + return plan, nil +} + func (p Org) MapProject(projectName string, scopes []plugin.Scope) (coreModels.PipelinePlan, errors.Error) { var plan coreModels.PipelinePlan var stage coreModels.PipelineStage diff --git a/backend/plugins/org/tasks/check_token.go b/backend/plugins/org/tasks/check_token.go new file mode 100644 index 00000000000..f147fe8e644 --- /dev/null +++ b/backend/plugins/org/tasks/check_token.go @@ -0,0 +1,75 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "fmt" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "golang.org/x/sync/errgroup" +) + +var TaskCheckTokenMeta = plugin.SubTaskMeta{ + Name: "checkTokens", + EntryPoint: checkProjectTokens, + EnabledByDefault: true, + Description: "set project mapping", + DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS}, +} + +func checkProjectTokens(taskCtx plugin.SubTaskContext) errors.Error { + logger := taskCtx.GetLogger() + taskData := taskCtx.GetData().(*TaskData) + connections := taskData.Options.ProjectConnections + logger.Debug("connections %+v", connections) + if len(connections) == 0 { + return nil + } + + g := new(errgroup.Group) + for _, connection := range connections { + conn := connection + logger.Debug("check conn: %+v", conn) + g.Go(func() error { + if err := checkConnectionToken(conn.ConnectionId, conn.PluginName); err != nil { + return err + } + return nil + }) + } + if err := g.Wait(); err != nil { + return errors.Convert(err) + } + return nil +} + +func checkConnectionToken(connectionID uint64, pluginName string) errors.Error { + pluginEntry, err := plugin.GetPlugin(pluginName) + if err != nil { + return err + } + if v, ok := pluginEntry.(plugin.PluginTestConnectionAPI); ok { + if err := v.TestConnection(connectionID); err != nil { + return err + } + return nil + } else { + msg := fmt.Sprintf("plugin: %s doesn't impl test connection api", pluginName) + return errors.Default.New(msg) + } +} diff --git a/backend/plugins/org/tasks/task_data.go b/backend/plugins/org/tasks/task_data.go index ca2a9b3002d..c92954740a9 100644 --- a/backend/plugins/org/tasks/task_data.go +++ b/backend/plugins/org/tasks/task_data.go @@ -20,8 +20,14 @@ package tasks import "github.com/apache/incubator-devlake/core/plugin" type Options struct { - ConnectionId uint64 `json:"connectionId"` - ProjectMappings []ProjectMapping `json:"projectMappings"` + ConnectionId uint64 `json:"connectionId"` + ProjectMappings []ProjectMapping `json:"projectMappings"` + ProjectConnections []ProjectConnection `json:"projectConnections"` +} + +type ProjectConnection struct { + PluginName string + ConnectionId uint64 } // ProjectMapping represents the relations between project and scopes diff --git a/backend/plugins/pagerduty/impl/impl.go b/backend/plugins/pagerduty/impl/impl.go index b237b30e491..827df870126 100644 --- a/backend/plugins/pagerduty/impl/impl.go +++ b/backend/plugins/pagerduty/impl/impl.go @@ -138,6 +138,11 @@ func (p PagerDuty) MigrationScripts() []plugin.MigrationScript { return migrationscripts.All() } +func (p PagerDuty) TestConnection(id uint64) errors.Error { + _, err := api.TestExistingConnection(helper.GenerateTestingConnectionApiResourceInput(id)) + return err +} + func (p PagerDuty) ApiResources() map[string]map[string]plugin.ApiResourceHandler { return map[string]map[string]plugin.ApiResourceHandler{ "test": { diff --git a/backend/plugins/refdiff/impl/impl.go b/backend/plugins/refdiff/impl/impl.go index b5572f6436a..e660b142497 100644 --- a/backend/plugins/refdiff/impl/impl.go +++ b/backend/plugins/refdiff/impl/impl.go @@ -110,3 +110,7 @@ func (p RefDiff) RootPkgPath() string { func (p RefDiff) ApiResources() map[string]map[string]plugin.ApiResourceHandler { return nil } + +func (p RefDiff) TestConnection(id uint64) errors.Error { + return nil +} diff --git a/backend/plugins/slack/impl/impl.go b/backend/plugins/slack/impl/impl.go index 8b3de9142dc..9ac894373a5 100644 --- a/backend/plugins/slack/impl/impl.go +++ b/backend/plugins/slack/impl/impl.go @@ -131,6 +131,11 @@ func (p Slack) MigrationScripts() []plugin.MigrationScript { return migrationscripts.All() } +func (p Slack) TestConnection(id uint64) errors.Error { + _, err := api.TestExistingConnection(helper.GenerateTestingConnectionApiResourceInput(id)) + return err +} + func (p Slack) ApiResources() map[string]map[string]plugin.ApiResourceHandler { return map[string]map[string]plugin.ApiResourceHandler{ "test": { diff --git a/backend/plugins/sonarqube/impl/impl.go b/backend/plugins/sonarqube/impl/impl.go index b6d2140324e..f0da8981282 100644 --- a/backend/plugins/sonarqube/impl/impl.go +++ b/backend/plugins/sonarqube/impl/impl.go @@ -170,6 +170,11 @@ func (p Sonarqube) MigrationScripts() []plugin.MigrationScript { return migrationscripts.All() } +func (p Sonarqube) TestConnection(id uint64) errors.Error { + _, err := api.TestExistingConnection(helper.GenerateTestingConnectionApiResourceInput(id)) + return err +} + func (p Sonarqube) ApiResources() map[string]map[string]plugin.ApiResourceHandler { return map[string]map[string]plugin.ApiResourceHandler{ "test": { diff --git a/backend/plugins/starrocks/impl/impl.go b/backend/plugins/starrocks/impl/impl.go index 511b8c7ec02..de036bace19 100644 --- a/backend/plugins/starrocks/impl/impl.go +++ b/backend/plugins/starrocks/impl/impl.go @@ -67,3 +67,7 @@ func (s StarRocks) Name() string { func (s StarRocks) RootPkgPath() string { return "github.com/merico-dev/lake/plugins/starrocks" } + +func (s StarRocks) TestConnection(id uint64) errors.Error { + return nil +} diff --git a/backend/plugins/tapd/impl/impl.go b/backend/plugins/tapd/impl/impl.go index 2885a43379a..2e133d821d5 100644 --- a/backend/plugins/tapd/impl/impl.go +++ b/backend/plugins/tapd/impl/impl.go @@ -265,6 +265,11 @@ func (p Tapd) MigrationScripts() []plugin.MigrationScript { return migrationscripts.All() } +func (p Tapd) TestConnection(id uint64) errors.Error { + _, err := api.TestExistingConnection(helper.GenerateTestingConnectionApiResourceInput(id)) + return err +} + func (p Tapd) ApiResources() map[string]map[string]plugin.ApiResourceHandler { return map[string]map[string]plugin.ApiResourceHandler{ "test": { diff --git a/backend/plugins/teambition/impl/impl.go b/backend/plugins/teambition/impl/impl.go index 662cace0fd7..728602d865c 100644 --- a/backend/plugins/teambition/impl/impl.go +++ b/backend/plugins/teambition/impl/impl.go @@ -169,6 +169,11 @@ func (p Teambition) MigrationScripts() []plugin.MigrationScript { return migrationscripts.All() } +func (p Teambition) TestConnection(id uint64) errors.Error { + _, err := api.TestExistingConnection(helper.GenerateTestingConnectionApiResourceInput(id)) + return err +} + func (p Teambition) ApiResources() map[string]map[string]plugin.ApiResourceHandler { return map[string]map[string]plugin.ApiResourceHandler{ "test": { diff --git a/backend/plugins/trello/impl/impl.go b/backend/plugins/trello/impl/impl.go index b3a0f088d0d..fdcb10fdc24 100644 --- a/backend/plugins/trello/impl/impl.go +++ b/backend/plugins/trello/impl/impl.go @@ -151,6 +151,11 @@ func (p Trello) ScopeConfig() dal.Tabler { return &models.TrelloScopeConfig{} } +func (p Trello) TestConnection(id uint64) errors.Error { + _, err := api.TestExistingConnection(helper.GenerateTestingConnectionApiResourceInput(id)) + return err +} + func (p Trello) ApiResources() map[string]map[string]plugin.ApiResourceHandler { return map[string]map[string]plugin.ApiResourceHandler{ "test": { diff --git a/backend/plugins/webhook/impl/impl.go b/backend/plugins/webhook/impl/impl.go index ada30478399..90e1f01986b 100644 --- a/backend/plugins/webhook/impl/impl.go +++ b/backend/plugins/webhook/impl/impl.go @@ -77,6 +77,10 @@ func (p Webhook) MigrationScripts() []plugin.MigrationScript { return migrationscripts.All() } +func (p Webhook) TestConnection(id uint64) errors.Error { + return nil +} + func (p Webhook) ApiResources() map[string]map[string]plugin.ApiResourceHandler { return map[string]map[string]plugin.ApiResourceHandler{ "connections": { diff --git a/backend/plugins/zentao/impl/impl.go b/backend/plugins/zentao/impl/impl.go index 5aa74c96606..55a98165d4e 100644 --- a/backend/plugins/zentao/impl/impl.go +++ b/backend/plugins/zentao/impl/impl.go @@ -277,6 +277,11 @@ func (p Zentao) MigrationScripts() []plugin.MigrationScript { return migrationscripts.All() } +func (p Zentao) TestConnection(id uint64) errors.Error { + _, err := api.TestExistingConnection(helper.GenerateTestingConnectionApiResourceInput(id)) + return err +} + func (p Zentao) ApiResources() map[string]map[string]plugin.ApiResourceHandler { return map[string]map[string]plugin.ApiResourceHandler{ "test": { diff --git a/backend/server/services/blueprint.go b/backend/server/services/blueprint.go index 4d106674d24..ff88211ddc2 100644 --- a/backend/server/services/blueprint.go +++ b/backend/server/services/blueprint.go @@ -20,6 +20,9 @@ package services import ( "encoding/json" "fmt" + "github.com/apache/incubator-devlake/core/log" + "github.com/apache/incubator-devlake/core/plugin" + "golang.org/x/sync/errgroup" "strings" "sync" @@ -419,6 +422,11 @@ func TriggerBlueprint(id uint64, triggerSyncPolicy *models.TriggerSyncPolicy, sh } blueprint.SkipCollectors = triggerSyncPolicy.SkipCollectors blueprint.FullSync = triggerSyncPolicy.FullSync + + if err := checkBlueprintTokens(blueprint, triggerSyncPolicy); err != nil { + return nil, errors.Default.Wrap(err, "check blue print tokens") + } + pipeline, err := createPipelineByBlueprint(blueprint, &models.SyncPolicy{ SkipOnFail: false, TimeAfter: nil, @@ -434,3 +442,64 @@ func TriggerBlueprint(id uint64, triggerSyncPolicy *models.TriggerSyncPolicy, sh } return pipeline, nil } + +func needToCheckToken(triggerSyncPolicy *models.TriggerSyncPolicy) bool { + // case1: retransform + if triggerSyncPolicy.SkipCollectors && !triggerSyncPolicy.FullSync { + return false + } + // case2: collect data: triggerSyncPolicy.SkipCollectors == false && triggerSyncPolicy.FullSync == false + // case3: collect data with fullsync: triggerSyncPolicy.SkipCollectors == false && triggerSyncPolicy.FullSync == true + // case4: others + return true +} + +func checkBlueprintTokens(blueprint *models.Blueprint, triggerSyncPolicy *models.TriggerSyncPolicy) errors.Error { + if blueprint == nil { + return errors.Default.New("blueprint is nil") + } + if triggerSyncPolicy == nil { + return errors.Default.New("triggerSyncPolicy is nil") + } + + if !needToCheckToken(triggerSyncPolicy) { + return nil + } + + if len(blueprint.Connections) == 0 { + return nil + } + + g := new(errgroup.Group) + for _, connection := range blueprint.Connections { + conn := *connection + g.Go(func() error { + if err := checkConnectionToken(logger, conn); err != nil { + return err + } + return nil + }) + } + if err := g.Wait(); err != nil { + return errors.Convert(err) + } + + return nil +} + +func checkConnectionToken(logger log.Logger, connection models.BlueprintConnection) errors.Error { + pluginEntry, err := plugin.GetPlugin(connection.PluginName) + if err != nil { + return err + } + if v, ok := pluginEntry.(plugin.PluginTestConnectionAPI); ok { + if err := v.TestConnection(connection.ConnectionId); err != nil { + logger.Error(err, "plugin: %s, id: %d", connection.PluginName, connection.ConnectionId) + return err + } + return nil + } else { + msg := fmt.Sprintf("plugin: %s doesn't impl test connection api", connection.PluginName) + return errors.Default.New(msg) + } +} diff --git a/backend/server/services/blueprint_makeplan_v200.go b/backend/server/services/blueprint_makeplan_v200.go index 727e843de70..960465069bb 100644 --- a/backend/server/services/blueprint_makeplan_v200.go +++ b/backend/server/services/blueprint_makeplan_v200.go @@ -121,11 +121,25 @@ func GeneratePlanJsonV200( } } var planForProjectMapping coreModels.PipelinePlan + var planForProjectTokenChecker coreModels.PipelinePlan if projectName != "" { p, err := plugin.GetPlugin("org") if err != nil { return nil, errors.Default.Wrap(err, "get plugin org") } + if pluginBp, ok := p.(plugin.ProjectTokenChecker); ok { + var simpleConns []plugin.ProjectTokenCheckerConnection + for _, connection := range connections { + simpleConns = append(simpleConns, plugin.ProjectTokenCheckerConnection{ + PluginName: connection.PluginName, + ConnectionId: connection.ConnectionId, + }) + } + planForProjectTokenChecker, err = pluginBp.MakePipeline(skipCollectors, projectName, simpleConns) + if err != nil { + return nil, errors.Default.Wrap(err, "org token checker make pipeline") + } + } if pluginBp, ok := p.(plugin.ProjectMapper); ok { planForProjectMapping, err = pluginBp.MapProject(projectName, scopes) if err != nil { @@ -134,6 +148,7 @@ func GeneratePlanJsonV200( } } plan := SequentializePipelinePlans( + planForProjectTokenChecker, planForProjectMapping, ParallelizePipelinePlans(sourcePlans...), ParallelizePipelinePlans(metricPlans...), diff --git a/backend/server/services/project.go b/backend/server/services/project.go index 98c9600a076..614f10680ef 100644 --- a/backend/server/services/project.go +++ b/backend/server/services/project.go @@ -219,7 +219,7 @@ func PatchProject(name string, body map[string]interface{}) (*models.ApiOutputPr return nil, err } - // allowed to changed the name + // allowed to change the name if projectInput.Name == "" { projectInput.Name = name } diff --git a/backend/server/services/remote/models/plugin_remote.go b/backend/server/services/remote/models/plugin_remote.go index 98828363053..3c97d8021fb 100644 --- a/backend/server/services/remote/models/plugin_remote.go +++ b/backend/server/services/remote/models/plugin_remote.go @@ -30,4 +30,5 @@ type RemotePlugin interface { plugin.PluginModel plugin.PluginMigration plugin.PluginSource + plugin.PluginTestConnectionAPI } diff --git a/backend/server/services/remote/plugin/plugin_impl.go b/backend/server/services/remote/plugin/plugin_impl.go index 6242b160053..ea7e912c53e 100644 --- a/backend/server/services/remote/plugin/plugin_impl.go +++ b/backend/server/services/remote/plugin/plugin_impl.go @@ -230,6 +230,11 @@ func (p *remotePluginImpl) ApiResources() map[string]map[string]plugin.ApiResour return p.resources } +func (p *remotePluginImpl) TestConnection(id uint64) errors.Error { + _, err := p.resources["connections/:connectionId/test"]["POST"](api.GenerateTestingConnectionApiResourceInput(id)) + return err +} + func (p *remotePluginImpl) OpenApiSpec() string { return p.openApiSpec } From 9cdd3b410428861ac8bd2af4de7fa70766eacef9 Mon Sep 17 00:00:00 2001 From: Lynwee <1507509064@qq.com> Date: Wed, 25 Sep 2024 10:29:38 +0800 Subject: [PATCH 02/16] feat(project): add token check result in project check API (#8099) --- backend/core/models/project.go | 22 +++++++++++++++++++++- backend/server/api/project/project.go | 9 +++++++++ backend/server/services/project.go | 27 +++++++++++++++++++++++++++ 3 files changed, 57 insertions(+), 1 deletion(-) diff --git a/backend/core/models/project.go b/backend/core/models/project.go index 99c61785d88..5227d2a1c64 100644 --- a/backend/core/models/project.go +++ b/backend/core/models/project.go @@ -73,9 +73,29 @@ type ApiOutputProject struct { } type ApiProjectCheck struct { - Exist bool `json:"exist" mapstructure:"exist"` + Exist bool `json:"exist" mapstructure:"exist"` + Tokens *ApiProjectCheckToken `json:"tokens,omitempty" mapstructure:"tokens"` } +type SuccessAndMessage struct { + Success bool `json:"success" mapstructure:"success"` + Message string `json:"message" mapstructure:"message"` +} + +// ApiProjectCheckToken +// +// { +// "plugin_name": +// { +// "connection_id": +// { +// "success": true, +// "message": "" +// } +// } +// } +type ApiProjectCheckToken = map[string]map[int]SuccessAndMessage + type Store struct { StoreKey string `gorm:"primaryKey;type:varchar(255)"` StoreValue json.RawMessage `gorm:"type:json;serializer:json"` diff --git a/backend/server/api/project/project.go b/backend/server/api/project/project.go index e826c71d170..b4a14fff5b4 100644 --- a/backend/server/api/project/project.go +++ b/backend/server/api/project/project.go @@ -73,6 +73,15 @@ func GetProjectCheck(c *gin.Context) { projectOutputCheck.Exist = true } + if c.Query("check_token") == "1" { + checkTokenResult, err := services.CheckProjectTokens(projectName) + if err != nil { + shared.ApiOutputError(c, errors.Default.Wrap(err, "error check project tokens")) + return + } + projectOutputCheck.Tokens = checkTokenResult + } + shared.ApiOutputSuccess(c, projectOutputCheck, http.StatusOK) // //shared.ApiOutputSuccess(c, projectOutputCheck, http.StatusOK) } diff --git a/backend/server/services/project.go b/backend/server/services/project.go index 614f10680ef..2e5c28aa842 100644 --- a/backend/server/services/project.go +++ b/backend/server/services/project.go @@ -193,6 +193,33 @@ func GetProject(name string) (*models.ApiOutputProject, errors.Error) { return makeProjectOutput(project, false) } +func CheckProjectTokens(name string) (*models.ApiProjectCheckToken, errors.Error) { + blueprint, err := GetBlueprintByProjectName(name) + if err != nil { + return nil, err + } + ret := make(map[string]map[int]models.SuccessAndMessage) + for _, connection := range blueprint.Connections { + pluginName := connection.PluginName + connectionId := int(connection.ConnectionId) + if _, ok := ret[pluginName]; !ok { + ret[pluginName] = make(map[int]models.SuccessAndMessage) + } + connectionTokenResult := models.SuccessAndMessage{ + Success: true, + Message: "success", + } + if err := checkConnectionToken(logger, *connection); err != nil { + ret[pluginName][connectionId] = models.SuccessAndMessage{ + Success: false, + Message: err.Error(), + } + } + ret[pluginName][connectionId] = connectionTokenResult + } + return &ret, nil +} + // PatchProject FIXME ... func PatchProject(name string, body map[string]interface{}) (*models.ApiOutputProject, errors.Error) { projectInput := &models.ApiInputProject{} From 8796873d698a9d41bdfcbc894aa62a1469928989 Mon Sep 17 00:00:00 2001 From: Lynwee <1507509064@qq.com> Date: Wed, 25 Sep 2024 11:11:54 +0800 Subject: [PATCH 03/16] Dev 1 (#8100) * feat(project): add token check result in project check API * feat(project): update token check API response --- backend/core/models/project.go | 22 ++++++---------------- backend/server/services/project.go | 25 ++++++++++++------------- 2 files changed, 18 insertions(+), 29 deletions(-) diff --git a/backend/core/models/project.go b/backend/core/models/project.go index 5227d2a1c64..6acafe27fbd 100644 --- a/backend/core/models/project.go +++ b/backend/core/models/project.go @@ -77,24 +77,14 @@ type ApiProjectCheck struct { Tokens *ApiProjectCheckToken `json:"tokens,omitempty" mapstructure:"tokens"` } -type SuccessAndMessage struct { - Success bool `json:"success" mapstructure:"success"` - Message string `json:"message" mapstructure:"message"` +type TokenResultSuccessAndMessage struct { + PluginName string `json:"pluginName" mapstructure:"pluginName"` + ConnectionID uint64 `json:"connectionId" mapstructure:"connectionId"` + Success bool `json:"success" mapstructure:"success"` + Message string `json:"message" mapstructure:"message"` } -// ApiProjectCheckToken -// -// { -// "plugin_name": -// { -// "connection_id": -// { -// "success": true, -// "message": "" -// } -// } -// } -type ApiProjectCheckToken = map[string]map[int]SuccessAndMessage +type ApiProjectCheckToken []TokenResultSuccessAndMessage type Store struct { StoreKey string `gorm:"primaryKey;type:varchar(255)"` diff --git a/backend/server/services/project.go b/backend/server/services/project.go index 2e5c28aa842..96afb6bf4f7 100644 --- a/backend/server/services/project.go +++ b/backend/server/services/project.go @@ -198,25 +198,24 @@ func CheckProjectTokens(name string) (*models.ApiProjectCheckToken, errors.Error if err != nil { return nil, err } - ret := make(map[string]map[int]models.SuccessAndMessage) + var ret models.ApiProjectCheckToken for _, connection := range blueprint.Connections { pluginName := connection.PluginName - connectionId := int(connection.ConnectionId) - if _, ok := ret[pluginName]; !ok { - ret[pluginName] = make(map[int]models.SuccessAndMessage) - } - connectionTokenResult := models.SuccessAndMessage{ - Success: true, - Message: "success", + connectionId := connection.ConnectionId + connectionTokenResult := models.TokenResultSuccessAndMessage{ + PluginName: pluginName, + ConnectionID: connectionId, + Success: true, + Message: "success", } if err := checkConnectionToken(logger, *connection); err != nil { - ret[pluginName][connectionId] = models.SuccessAndMessage{ - Success: false, - Message: err.Error(), - } + connectionTokenResult.Success = false + connectionTokenResult.Message = err.Error() + } - ret[pluginName][connectionId] = connectionTokenResult + ret = append(ret, connectionTokenResult) } + return &ret, nil } From c5daba38f369e3ba9f0d57be8d56f3ef4523474c Mon Sep 17 00:00:00 2001 From: Lynwee <1507509064@qq.com> Date: Wed, 25 Sep 2024 11:22:24 +0800 Subject: [PATCH 04/16] Dev 1 (#8101) * feat(project): add token check result in project check API * feat(project): update token check API response * fix(project): fix api doc * fix(project): update project check api doc --- backend/server/api/project/project.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/backend/server/api/project/project.go b/backend/server/api/project/project.go index b4a14fff5b4..08e70c626f5 100644 --- a/backend/server/api/project/project.go +++ b/backend/server/api/project/project.go @@ -53,12 +53,12 @@ func GetProject(c *gin.Context) { shared.ApiOutputSuccess(c, projectOutput, http.StatusOK) } -// @Summary Get project exist check -// @Description Get project exist check +// @Summary Get project related check +// @Description Get project related check info, such existence, token validity // @Tags framework/projects // @Accept application/json // @Param projectName path string true "project name" -// @Success 200 {object} models.ApiOutputProject +// @Success 200 {object} models.ApiProjectCheck // @Failure 400 {string} errcode.Error "Bad Request" // @Failure 500 {string} errcode.Error "Internal Error" // @Router /projects/{projectName}/check [get] From 89096fa9b90659cdc4a7cf0ef26d4481b88f1b7e Mon Sep 17 00:00:00 2001 From: d4x1 <1507509064@qq.com> Date: Wed, 25 Sep 2024 11:28:39 +0800 Subject: [PATCH 05/16] fix(project): update check project api doc --- backend/server/api/project/project.go | 1 + 1 file changed, 1 insertion(+) diff --git a/backend/server/api/project/project.go b/backend/server/api/project/project.go index 08e70c626f5..43687da7324 100644 --- a/backend/server/api/project/project.go +++ b/backend/server/api/project/project.go @@ -58,6 +58,7 @@ func GetProject(c *gin.Context) { // @Tags framework/projects // @Accept application/json // @Param projectName path string true "project name" +// @Param check_token query int false "need to check token validity or not" // @Success 200 {object} models.ApiProjectCheck // @Failure 400 {string} errcode.Error "Bad Request" // @Failure 500 {string} errcode.Error "Internal Error" From 4a393edc2c0e0cd00aa03b031756240f994f968d Mon Sep 17 00:00:00 2001 From: mintsweet <0x1304570@gmail.com> Date: Wed, 25 Sep 2024 15:54:48 +1200 Subject: [PATCH 06/16] feat: add connection token check before collect data --- config-ui/src/api/project/index.ts | 11 ++- .../routes/blueprint/detail/status-panel.tsx | 75 +++++++++++++++++-- 2 files changed, 80 insertions(+), 6 deletions(-) diff --git a/config-ui/src/api/project/index.ts b/config-ui/src/api/project/index.ts index 8e79ee5fa2f..bc27b5e00d1 100644 --- a/config-ui/src/api/project/index.ts +++ b/config-ui/src/api/project/index.ts @@ -24,7 +24,16 @@ export const list = (data: Pagination & { keyword?: string }): Promise<{ count: export const get = (name: string): Promise => request(`/projects/${encodeURIComponent(name)}`); -export const checkName = (name: string) => request(`/projects/${encodeURIComponent(name)}/check`); +export const check = ( + name: string, + data?: { check_token: 1 }, +): Promise<{ + exist: boolean; + tokens: Array<{ pluginName: string; connectionId: ID; success: boolean }>; +}> => + request(`/projects/${encodeURIComponent(name)}/check`, { + data, + }); export const create = (data: Pick) => request('/projects', { diff --git a/config-ui/src/routes/blueprint/detail/status-panel.tsx b/config-ui/src/routes/blueprint/detail/status-panel.tsx index c3de3c2ef2d..fd2606a9086 100644 --- a/config-ui/src/routes/blueprint/detail/status-panel.tsx +++ b/config-ui/src/routes/blueprint/detail/status-panel.tsx @@ -17,14 +17,15 @@ */ import { useState, useMemo } from 'react'; -import { useNavigate } from 'react-router-dom'; -import { MoreOutlined, DeleteOutlined } from '@ant-design/icons'; -import { Card, Modal, Switch, Button, Tooltip, Dropdown, Flex, Space } from 'antd'; +import { useNavigate, Link } from 'react-router-dom'; +import { MoreOutlined, DeleteOutlined, WarningOutlined } from '@ant-design/icons'; +import { theme, Card, Modal, Switch, Button, Tooltip, Dropdown, Flex, Space } from 'antd'; import API from '@/api'; import { Message } from '@/components'; import { getCron } from '@/config'; -import { useRefreshData } from '@/hooks'; +import { selectAllConnections } from '@/features/connections'; +import { useAppSelector, useRefreshData } from '@/hooks'; import { PipelineInfo, PipelineTasks, PipelineTable } from '@/routes/pipeline'; import { IBlueprint } from '@/types'; import { formatTime, operator } from '@/utils'; @@ -39,13 +40,22 @@ interface Props { } export const StatusPanel = ({ from, blueprint, pipelineId, onRefresh }: Props) => { - const [type, setType] = useState<'delete' | 'fullSync'>(); + const [type, setType] = useState<'delete' | 'fullSync' | 'checkTokenFailed'>(); const [page, setPage] = useState(1); const [pageSize] = useState(10); const [operating, setOperating] = useState(false); + const [connectionFailed, setConnectionFailed] = useState< + Array<{ unique: string; name: string; plugin: string; connectionId: ID }> + >([]); const navigate = useNavigate(); + const { + token: { orange5 }, + } = theme.useToken(); + + const connections = useAppSelector(selectAllConnections); + const cron = useMemo(() => getCron(blueprint.isManual, blueprint.cronConfig), [blueprint]); const { ready, data } = useRefreshData( @@ -64,6 +74,32 @@ export const StatusPanel = ({ from, blueprint, pipelineId, onRefresh }: Props) = skipCollectors?: boolean; fullSync?: boolean; }) => { + if (!skipCollectors && from === FromEnum.project) { + const [success, res] = await operator(() => API.project.check(blueprint.projectName, { check_token: 1 }), { + hideToast: true, + setOperating, + }); + + if (success && res.tokens.length) { + const connectionFailed = res.tokens + .filter((token: any) => !token.success) + .map((it: any) => { + const unique = `${it.pluginName}-${it.connectionId}`; + const connection = connections.find((c) => c.unique === unique); + return { + unique, + name: connection?.name ?? '', + plugin: it.pluginName, + connectionId: it.connectionId, + }; + }); + + setType('checkTokenFailed'); + setConnectionFailed(connectionFailed); + return; + } + } + const [success] = await operator(() => API.blueprint.trigger(blueprint.id, { skipCollectors, fullSync }), { setOperating, formatMessage: () => 'Trigger blueprint successful.', @@ -245,6 +281,35 @@ export const StatusPanel = ({ from, blueprint, pipelineId, onRefresh }: Props) = )} + + {type === 'checkTokenFailed' && ( + + + Invalid Token(s) Detected + + } + width={820} + footer={null} + onCancel={() => { + handleResetType(); + setConnectionFailed([]); + }} + > +

There are invalid tokens in the following connections. Please update them before re-syncing the data.

+
    + {connectionFailed.map((it) => ( +
  • + + {it.name} + +
    • + ))} +
+ + )} ); }; From dc705a62cf064af93a20cc5d60d65f73ada60d84 Mon Sep 17 00:00:00 2001 From: Lynwee <1507509064@qq.com> Date: Wed, 25 Sep 2024 13:22:41 +0800 Subject: [PATCH 07/16] feat(blueprint): add new api to check connection tokens (#8102) --- backend/core/models/blueprint.go | 9 +++++++ backend/core/models/project.go | 12 +--------- backend/server/api/blueprints/blueprints.go | 24 +++++++++++++++++++ backend/server/api/project/project.go | 12 ---------- backend/server/api/router.go | 1 + backend/server/services/blueprint.go | 24 +++++++++++++++++++ backend/server/services/project.go | 26 --------------------- 7 files changed, 59 insertions(+), 49 deletions(-) diff --git a/backend/core/models/blueprint.go b/backend/core/models/blueprint.go index f30adec9f05..9dc46882204 100644 --- a/backend/core/models/blueprint.go +++ b/backend/core/models/blueprint.go @@ -92,3 +92,12 @@ type SyncPolicy struct { TimeAfter *time.Time `json:"timeAfter"` TriggerSyncPolicy } + +type ConnectionTokenCheckResult struct { + PluginName string `json:"pluginName" mapstructure:"pluginName"` + ConnectionID uint64 `json:"connectionId" mapstructure:"connectionId"` + Success bool `json:"success" mapstructure:"success"` + Message string `json:"message" mapstructure:"message"` +} + +type ApiBlueprintConnectionTokenCheck []ConnectionTokenCheckResult diff --git a/backend/core/models/project.go b/backend/core/models/project.go index 6acafe27fbd..99c61785d88 100644 --- a/backend/core/models/project.go +++ b/backend/core/models/project.go @@ -73,19 +73,9 @@ type ApiOutputProject struct { } type ApiProjectCheck struct { - Exist bool `json:"exist" mapstructure:"exist"` - Tokens *ApiProjectCheckToken `json:"tokens,omitempty" mapstructure:"tokens"` + Exist bool `json:"exist" mapstructure:"exist"` } -type TokenResultSuccessAndMessage struct { - PluginName string `json:"pluginName" mapstructure:"pluginName"` - ConnectionID uint64 `json:"connectionId" mapstructure:"connectionId"` - Success bool `json:"success" mapstructure:"success"` - Message string `json:"message" mapstructure:"message"` -} - -type ApiProjectCheckToken []TokenResultSuccessAndMessage - type Store struct { StoreKey string `gorm:"primaryKey;type:varchar(255)"` StoreValue json.RawMessage `gorm:"type:json;serializer:json"` diff --git a/backend/server/api/blueprints/blueprints.go b/backend/server/api/blueprints/blueprints.go index 841027424cc..2e6e315cb91 100644 --- a/backend/server/api/blueprints/blueprints.go +++ b/backend/server/api/blueprints/blueprints.go @@ -18,6 +18,7 @@ limitations under the License. package blueprints import ( + "github.com/spf13/cast" "net/http" "strconv" @@ -227,6 +228,29 @@ func GetBlueprintPipelines(c *gin.Context) { shared.ApiOutputSuccess(c, shared.ResponsePipelines{Pipelines: pipelines, Count: count}, http.StatusOK) } +// @Summary get connection token check by blueprint id +// @Description get connection token check by blueprint id +// @Tags framework/blueprints +// @Accept application/json +// @Param blueprintId path int true "blueprint id" +// @Success 200 {object} models.ApiBlueprintConnectionTokenCheck +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /blueprints/{blueprintId}/connections-token-check [get] +func GetBlueprintConnectionTokenCheck(c *gin.Context) { + blueprintId, err := cast.ToUint64E(c.Param("blueprintId")) + if err != nil { + shared.ApiOutputError(c, errors.BadInput.Wrap(err, "bad blueprintID format supplied")) + return + } + resp, err := services.CheckBlueprintConnectionTokens(blueprintId) + if err != nil { + shared.ApiOutputError(c, errors.Default.Wrap(err, "error getting blue print connection token check")) + return + } + shared.ApiOutputSuccess(c, resp, http.StatusOK) +} + // @Summary delete blueprint by id // @Description delete blueprint by id // @Tags framework/blueprints diff --git a/backend/server/api/project/project.go b/backend/server/api/project/project.go index 43687da7324..fd5c14ebb5a 100644 --- a/backend/server/api/project/project.go +++ b/backend/server/api/project/project.go @@ -58,14 +58,12 @@ func GetProject(c *gin.Context) { // @Tags framework/projects // @Accept application/json // @Param projectName path string true "project name" -// @Param check_token query int false "need to check token validity or not" // @Success 200 {object} models.ApiProjectCheck // @Failure 400 {string} errcode.Error "Bad Request" // @Failure 500 {string} errcode.Error "Internal Error" // @Router /projects/{projectName}/check [get] func GetProjectCheck(c *gin.Context) { projectName := c.Param("projectName") - projectOutputCheck := &models.ApiProjectCheck{} _, err := services.GetProject(projectName) if err != nil { @@ -73,16 +71,6 @@ func GetProjectCheck(c *gin.Context) { } else { projectOutputCheck.Exist = true } - - if c.Query("check_token") == "1" { - checkTokenResult, err := services.CheckProjectTokens(projectName) - if err != nil { - shared.ApiOutputError(c, errors.Default.Wrap(err, "error check project tokens")) - return - } - projectOutputCheck.Tokens = checkTokenResult - } - shared.ApiOutputSuccess(c, projectOutputCheck, http.StatusOK) // //shared.ApiOutputSuccess(c, projectOutputCheck, http.StatusOK) } diff --git a/backend/server/api/router.go b/backend/server/api/router.go index 721517cf59a..226eb3035d3 100644 --- a/backend/server/api/router.go +++ b/backend/server/api/router.go @@ -59,6 +59,7 @@ func RegisterRouter(r *gin.Engine, basicRes context.BasicRes) { r.GET("/blueprints/:blueprintId", blueprints.Get) r.POST("/blueprints/:blueprintId/trigger", blueprints.Trigger) r.GET("/blueprints/:blueprintId/pipelines", blueprints.GetBlueprintPipelines) + r.GET("/blueprints/:blueprintId/connections-token-check", blueprints.GetBlueprintConnectionTokenCheck) r.POST("/tasks/:taskId/rerun", task.PostRerun) diff --git a/backend/server/services/blueprint.go b/backend/server/services/blueprint.go index ff88211ddc2..aced9eeeace 100644 --- a/backend/server/services/blueprint.go +++ b/backend/server/services/blueprint.go @@ -117,6 +117,30 @@ func GetBlueprint(blueprintId uint64, shouldSanitize bool) (*models.Blueprint, e return blueprint, nil } +func CheckBlueprintConnectionTokens(blueprintId uint64) (*models.ApiBlueprintConnectionTokenCheck, errors.Error) { + blueprint, err := GetBlueprint(blueprintId, false) + if err != nil { + return nil, err + } + var ret models.ApiBlueprintConnectionTokenCheck + for _, connection := range blueprint.Connections { + pluginName := connection.PluginName + connectionId := connection.ConnectionId + connectionTokenResult := models.ConnectionTokenCheckResult{ + PluginName: pluginName, + ConnectionID: connectionId, + Success: true, + Message: "success", + } + if err := checkConnectionToken(logger, *connection); err != nil { + connectionTokenResult.Success = false + connectionTokenResult.Message = err.Error() + } + ret = append(ret, connectionTokenResult) + } + return &ret, nil +} + // GetBlueprintByProjectName returns the detail of a given ProjectName func GetBlueprintByProjectName(projectName string) (*models.Blueprint, errors.Error) { if projectName == "" { diff --git a/backend/server/services/project.go b/backend/server/services/project.go index 96afb6bf4f7..614f10680ef 100644 --- a/backend/server/services/project.go +++ b/backend/server/services/project.go @@ -193,32 +193,6 @@ func GetProject(name string) (*models.ApiOutputProject, errors.Error) { return makeProjectOutput(project, false) } -func CheckProjectTokens(name string) (*models.ApiProjectCheckToken, errors.Error) { - blueprint, err := GetBlueprintByProjectName(name) - if err != nil { - return nil, err - } - var ret models.ApiProjectCheckToken - for _, connection := range blueprint.Connections { - pluginName := connection.PluginName - connectionId := connection.ConnectionId - connectionTokenResult := models.TokenResultSuccessAndMessage{ - PluginName: pluginName, - ConnectionID: connectionId, - Success: true, - Message: "success", - } - if err := checkConnectionToken(logger, *connection); err != nil { - connectionTokenResult.Success = false - connectionTokenResult.Message = err.Error() - - } - ret = append(ret, connectionTokenResult) - } - - return &ret, nil -} - // PatchProject FIXME ... func PatchProject(name string, body map[string]interface{}) (*models.ApiOutputProject, errors.Error) { projectInput := &models.ApiInputProject{} From 8c09e33402a49ade5a7e7bec383f200e4c1417c5 Mon Sep 17 00:00:00 2001 From: mintsweet <0x1304570@gmail.com> Date: Wed, 25 Sep 2024 18:29:09 +1200 Subject: [PATCH 08/16] fix: adjust the api for connections token check --- config-ui/src/api/blueprint/index.ts | 2 ++ config-ui/src/api/project/index.ts | 11 +---------- .../src/routes/blueprint/detail/status-panel.tsx | 10 +++++----- 3 files changed, 8 insertions(+), 15 deletions(-) diff --git a/config-ui/src/api/blueprint/index.ts b/config-ui/src/api/blueprint/index.ts index 3876d3e20da..99460c088ba 100644 --- a/config-ui/src/api/blueprint/index.ts +++ b/config-ui/src/api/blueprint/index.ts @@ -43,3 +43,5 @@ type TriggerQuery = { export const trigger = (id: ID, data: TriggerQuery = { skipCollectors: false, fullSync: false }) => request(`/blueprints/${id}/trigger`, { method: 'post', data }); + +export const connectionsTokenCheck = (id: ID) => request(`/blueprints/${id}/connections-token-check`); diff --git a/config-ui/src/api/project/index.ts b/config-ui/src/api/project/index.ts index bc27b5e00d1..8e79ee5fa2f 100644 --- a/config-ui/src/api/project/index.ts +++ b/config-ui/src/api/project/index.ts @@ -24,16 +24,7 @@ export const list = (data: Pagination & { keyword?: string }): Promise<{ count: export const get = (name: string): Promise => request(`/projects/${encodeURIComponent(name)}`); -export const check = ( - name: string, - data?: { check_token: 1 }, -): Promise<{ - exist: boolean; - tokens: Array<{ pluginName: string; connectionId: ID; success: boolean }>; -}> => - request(`/projects/${encodeURIComponent(name)}/check`, { - data, - }); +export const checkName = (name: string) => request(`/projects/${encodeURIComponent(name)}/check`); export const create = (data: Pick) => request('/projects', { diff --git a/config-ui/src/routes/blueprint/detail/status-panel.tsx b/config-ui/src/routes/blueprint/detail/status-panel.tsx index fd2606a9086..48707144e48 100644 --- a/config-ui/src/routes/blueprint/detail/status-panel.tsx +++ b/config-ui/src/routes/blueprint/detail/status-panel.tsx @@ -74,15 +74,15 @@ export const StatusPanel = ({ from, blueprint, pipelineId, onRefresh }: Props) = skipCollectors?: boolean; fullSync?: boolean; }) => { - if (!skipCollectors && from === FromEnum.project) { - const [success, res] = await operator(() => API.project.check(blueprint.projectName, { check_token: 1 }), { + if (!skipCollectors) { + const [success, res] = await operator(() => API.blueprint.connectionsTokenCheck(blueprint.id), { hideToast: true, setOperating, }); - if (success && res.tokens.length) { - const connectionFailed = res.tokens - .filter((token: any) => !token.success) + if (success && res.length) { + const connectionFailed = res + .filter((it: any) => !it.success) .map((it: any) => { const unique = `${it.pluginName}-${it.connectionId}`; const connection = connections.find((c) => c.unique === unique); From 9163679d2d0d37423a357eff64f9168247adf52d Mon Sep 17 00:00:00 2001 From: mintsweet <0x1304570@gmail.com> Date: Fri, 27 Sep 2024 14:13:20 +1200 Subject: [PATCH 09/16] feat: add a component connection-form-modal --- .../connection-form/connection-form-modal.tsx | 59 +++++++++++++++++++ .../{index.tsx => connection-form.tsx} | 0 .../components/connection-form/index.ts | 20 +++++++ .../components/connection-form/styled.ts | 35 +++++++++++ 4 files changed, 114 insertions(+) create mode 100644 config-ui/src/plugins/components/connection-form/connection-form-modal.tsx rename config-ui/src/plugins/components/connection-form/{index.tsx => connection-form.tsx} (100%) create mode 100644 config-ui/src/plugins/components/connection-form/index.ts create mode 100644 config-ui/src/plugins/components/connection-form/styled.ts diff --git a/config-ui/src/plugins/components/connection-form/connection-form-modal.tsx b/config-ui/src/plugins/components/connection-form/connection-form-modal.tsx new file mode 100644 index 00000000000..a6c689be5f0 --- /dev/null +++ b/config-ui/src/plugins/components/connection-form/connection-form-modal.tsx @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +import { useMemo } from 'react'; +import { theme, Modal } from 'antd'; + +import { getPluginConfig } from '@/plugins'; + +import { ConnectionForm } from '../connection-form'; + +import * as S from './styled'; + +interface Props { + plugin: string; + connectionId: ID; + open: boolean; + onCancel: () => void; +} + +export const ConnectionFormModal = ({ plugin, connectionId, open, onCancel }: Props) => { + const pluginConfig = useMemo(() => getPluginConfig(plugin), [plugin]); + + const { + token: { colorPrimary }, + } = theme.useToken(); + + return ( + + {pluginConfig.icon({ color: colorPrimary })} + Manage Connections: {pluginConfig.name} + + } + footer={null} + onCancel={onCancel} + > + + + ); +}; diff --git a/config-ui/src/plugins/components/connection-form/index.tsx b/config-ui/src/plugins/components/connection-form/connection-form.tsx similarity index 100% rename from config-ui/src/plugins/components/connection-form/index.tsx rename to config-ui/src/plugins/components/connection-form/connection-form.tsx diff --git a/config-ui/src/plugins/components/connection-form/index.ts b/config-ui/src/plugins/components/connection-form/index.ts new file mode 100644 index 00000000000..14f1daa900e --- /dev/null +++ b/config-ui/src/plugins/components/connection-form/index.ts @@ -0,0 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +export * from './connection-form-modal'; +export * from './connection-form'; diff --git a/config-ui/src/plugins/components/connection-form/styled.ts b/config-ui/src/plugins/components/connection-form/styled.ts new file mode 100644 index 00000000000..6036f510667 --- /dev/null +++ b/config-ui/src/plugins/components/connection-form/styled.ts @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +import styled from 'styled-components'; + +export const ModalTitle = styled.div` + display: flex; + align-items: center; + + .icon { + display: inline-flex; + margin-right: 8px; + width: 24px; + + & > svg { + width: 100%; + height: 100%; + } + } +`; From be5ac5b3608e50ce2acd6a6bb7e41e768dd633f9 Mon Sep 17 00:00:00 2001 From: mintsweet <0x1304570@gmail.com> Date: Fri, 27 Sep 2024 14:13:37 +1200 Subject: [PATCH 10/16] feat: add a component connection-name --- .../components/connection-name/index.tsx | 50 +++++++++++++++++++ .../components/connection-name/styled.ts | 46 +++++++++++++++++ config-ui/src/plugins/components/index.ts | 1 + 3 files changed, 97 insertions(+) create mode 100644 config-ui/src/plugins/components/connection-name/index.tsx create mode 100644 config-ui/src/plugins/components/connection-name/styled.ts diff --git a/config-ui/src/plugins/components/connection-name/index.tsx b/config-ui/src/plugins/components/connection-name/index.tsx new file mode 100644 index 00000000000..0edffbb1a4d --- /dev/null +++ b/config-ui/src/plugins/components/connection-name/index.tsx @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +import { theme } from 'antd'; + +import { selectConnection, selectWebhook } from '@/features/connections'; +import { useAppSelector } from '@/hooks'; +import { getPluginConfig } from '@/plugins'; + +import * as S from './styled'; + +interface Props { + plugin: string; + connectionId: ID; + onClick?: () => void; +} + +export const ConnectionName = ({ plugin, connectionId, onClick }: Props) => { + const { + token: { colorPrimary }, + } = theme.useToken(); + + const connection = useAppSelector((state) => selectConnection(state, `${plugin}-${connectionId}`)); + const webhook = useAppSelector((state) => selectWebhook(state, connectionId)); + const config = getPluginConfig(plugin); + + const name = connection ? connection.name : webhook ? webhook.name : `${plugin}/connection/${connectionId}`; + + return ( + + {config.icon({ color: colorPrimary })} + {name} + + ); +}; diff --git a/config-ui/src/plugins/components/connection-name/styled.ts b/config-ui/src/plugins/components/connection-name/styled.ts new file mode 100644 index 00000000000..d8d2c0a44f0 --- /dev/null +++ b/config-ui/src/plugins/components/connection-name/styled.ts @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +import styled from 'styled-components'; + +export const Wrapper = styled.div` + display: flex; + align-items: center; + + span + span { + margin-left: 4px; + } +`; + +export const Icon = styled.span` + display: inline-block; + width: 24px; + height: 24px; + + & > svg { + width: 100%; + height: 100%; + } +`; + +export const Name = styled.span` + max-width: 240px; + white-space: nowrap; + text-overflow: ellipsis; + overflow: hidden; +`; diff --git a/config-ui/src/plugins/components/index.ts b/config-ui/src/plugins/components/index.ts index 4cbfa01e5d7..99285c8415c 100644 --- a/config-ui/src/plugins/components/index.ts +++ b/config-ui/src/plugins/components/index.ts @@ -19,6 +19,7 @@ export * from './check-matched-items'; export * from './connection-form'; export * from './connection-list'; +export * from './connection-name'; export * from './connection-select'; export * from './connection-status'; export * from './data-scope-remote'; From 175f3d62fa4d26b14185b12f827209536da154d6 Mon Sep 17 00:00:00 2001 From: mintsweet <0x1304570@gmail.com> Date: Fri, 27 Sep 2024 14:14:23 +1200 Subject: [PATCH 11/16] fix: show connection form modal when connection token check failed --- .../components/connection-check/index.tsx | 39 +++++++++++++++++++ .../blueprint/detail/components/index.ts | 1 + .../routes/blueprint/detail/status-panel.tsx | 27 +++++-------- 3 files changed, 49 insertions(+), 18 deletions(-) create mode 100644 config-ui/src/routes/blueprint/detail/components/connection-check/index.tsx diff --git a/config-ui/src/routes/blueprint/detail/components/connection-check/index.tsx b/config-ui/src/routes/blueprint/detail/components/connection-check/index.tsx new file mode 100644 index 00000000000..4ab322c5ae2 --- /dev/null +++ b/config-ui/src/routes/blueprint/detail/components/connection-check/index.tsx @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +import { useState } from 'react'; +import { Flex } from 'antd'; + +import { ConnectionName, ConnectionFormModal } from '@/plugins'; + +interface Props { + plugin: string; + connectionId: ID; +} + +export const ConnectionCheck = ({ plugin, connectionId }: Props) => { + const [open, setOpen] = useState(false); + + return ( + + - + setOpen(true)} /> + setOpen(false)} /> + + ); +}; diff --git a/config-ui/src/routes/blueprint/detail/components/index.ts b/config-ui/src/routes/blueprint/detail/components/index.ts index ab8f3f607d0..5ce0fcf9802 100644 --- a/config-ui/src/routes/blueprint/detail/components/index.ts +++ b/config-ui/src/routes/blueprint/detail/components/index.ts @@ -18,4 +18,5 @@ export * from './add-connection-dialog'; export * from './advanced-editor'; +export * from './connection-check'; export * from './update-policy-dialog'; diff --git a/config-ui/src/routes/blueprint/detail/status-panel.tsx b/config-ui/src/routes/blueprint/detail/status-panel.tsx index 48707144e48..316a0e970df 100644 --- a/config-ui/src/routes/blueprint/detail/status-panel.tsx +++ b/config-ui/src/routes/blueprint/detail/status-panel.tsx @@ -17,21 +17,22 @@ */ import { useState, useMemo } from 'react'; -import { useNavigate, Link } from 'react-router-dom'; +import { useNavigate } from 'react-router-dom'; import { MoreOutlined, DeleteOutlined, WarningOutlined } from '@ant-design/icons'; import { theme, Card, Modal, Switch, Button, Tooltip, Dropdown, Flex, Space } from 'antd'; import API from '@/api'; import { Message } from '@/components'; import { getCron } from '@/config'; -import { selectAllConnections } from '@/features/connections'; -import { useAppSelector, useRefreshData } from '@/hooks'; +import { useRefreshData } from '@/hooks'; import { PipelineInfo, PipelineTasks, PipelineTable } from '@/routes/pipeline'; import { IBlueprint } from '@/types'; import { formatTime, operator } from '@/utils'; import { FromEnum } from '../types'; +import { ConnectionCheck } from './components'; + interface Props { from: FromEnum; blueprint: IBlueprint; @@ -44,9 +45,7 @@ export const StatusPanel = ({ from, blueprint, pipelineId, onRefresh }: Props) = const [page, setPage] = useState(1); const [pageSize] = useState(10); const [operating, setOperating] = useState(false); - const [connectionFailed, setConnectionFailed] = useState< - Array<{ unique: string; name: string; plugin: string; connectionId: ID }> - >([]); + const [connectionFailed, setConnectionFailed] = useState>([]); const navigate = useNavigate(); @@ -54,8 +53,6 @@ export const StatusPanel = ({ from, blueprint, pipelineId, onRefresh }: Props) = token: { orange5 }, } = theme.useToken(); - const connections = useAppSelector(selectAllConnections); - const cron = useMemo(() => getCron(blueprint.isManual, blueprint.cronConfig), [blueprint]); const { ready, data } = useRefreshData( @@ -84,11 +81,7 @@ export const StatusPanel = ({ from, blueprint, pipelineId, onRefresh }: Props) = const connectionFailed = res .filter((it: any) => !it.success) .map((it: any) => { - const unique = `${it.pluginName}-${it.connectionId}`; - const connection = connections.find((c) => c.unique === unique); return { - unique, - name: connection?.name ?? '', plugin: it.pluginName, connectionId: it.connectionId, }; @@ -299,12 +292,10 @@ export const StatusPanel = ({ from, blueprint, pipelineId, onRefresh }: Props) = }} >

There are invalid tokens in the following connections. Please update them before re-syncing the data.

-
    - {connectionFailed.map((it) => ( -
  • - - {it.name} - +
      + {connectionFailed.map(({ plugin, connectionId }) => ( +
    • +
      • ))}
    From 1fa474510b149074d3713fa9cdb1398452eddc6d Mon Sep 17 00:00:00 2001 From: mintsweet <0x1304570@gmail.com> Date: Fri, 27 Sep 2024 14:42:14 +1200 Subject: [PATCH 12/16] refactor: improve the component connection-name --- .../features/connections/components/index.ts | 19 ----- .../features/connections/components/name.tsx | 78 ------------------- config-ui/src/features/connections/index.ts | 1 - .../connection-form/connection-form-modal.tsx | 32 +++----- .../components/connection-form/styled.ts | 35 --------- .../components/connection-list/index.tsx | 40 +--------- .../components/connection-name/index.tsx | 37 +++++++-- .../blueprint/detail/configuration-panel.tsx | 3 +- config-ui/src/routes/blueprint/home/index.tsx | 2 +- .../src/routes/connection/connection.tsx | 15 +--- .../src/routes/connection/connections.tsx | 29 ++----- config-ui/src/routes/connection/styled.ts | 16 ---- config-ui/src/routes/project/home/index.tsx | 2 +- 13 files changed, 57 insertions(+), 252 deletions(-) delete mode 100644 config-ui/src/features/connections/components/index.ts delete mode 100644 config-ui/src/features/connections/components/name.tsx delete mode 100644 config-ui/src/plugins/components/connection-form/styled.ts diff --git a/config-ui/src/features/connections/components/index.ts b/config-ui/src/features/connections/components/index.ts deleted file mode 100644 index 113c17b4abd..00000000000 --- a/config-ui/src/features/connections/components/index.ts +++ /dev/null @@ -1,19 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -export * from './name'; diff --git a/config-ui/src/features/connections/components/name.tsx b/config-ui/src/features/connections/components/name.tsx deleted file mode 100644 index c1c2f6351e3..00000000000 --- a/config-ui/src/features/connections/components/name.tsx +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -import { theme } from 'antd'; -import styled from 'styled-components'; - -import { useAppSelector } from '@/hooks'; -import { getPluginConfig } from '@/plugins'; - -import { selectConnection, selectWebhook } from '../slice'; - -const Wrapper = styled.div` - display: flex; - align-items: center; - - span + span { - margin-left: 4px; - } - - .icon { - display: inline-block; - width: 24px; - height: 24px; - - & > svg { - width: 100%; - height: 100%; - } - } - - .name { - max-width: 240px; - white-space: nowrap; - text-overflow: ellipsis; - overflow: hidden; - } -`; - -interface Props { - plugin: string; - connectionId: ID; -} - -export const ConnectionName = ({ plugin, connectionId }: Props) => { - const { - token: { colorPrimary }, - } = theme.useToken(); - - const connection = useAppSelector((state) => selectConnection(state, `${plugin}-${connectionId}`)); - const webhook = useAppSelector((state) => selectWebhook(state, connectionId)); - const config = getPluginConfig(plugin); - - const name = connection ? connection.name : webhook ? webhook.name : `${plugin}/connection/${connectionId}`; - - return ( - - {config.icon({ color: colorPrimary })} - - {name} - - - ); -}; diff --git a/config-ui/src/features/connections/index.ts b/config-ui/src/features/connections/index.ts index 8cf36b47644..513ab48a7f8 100644 --- a/config-ui/src/features/connections/index.ts +++ b/config-ui/src/features/connections/index.ts @@ -16,5 +16,4 @@ * */ -export * from './components'; export * from './slice'; diff --git a/config-ui/src/plugins/components/connection-form/connection-form-modal.tsx b/config-ui/src/plugins/components/connection-form/connection-form-modal.tsx index a6c689be5f0..4eccb09ae80 100644 --- a/config-ui/src/plugins/components/connection-form/connection-form-modal.tsx +++ b/config-ui/src/plugins/components/connection-form/connection-form-modal.tsx @@ -16,44 +16,36 @@ * */ -import { useMemo } from 'react'; -import { theme, Modal } from 'antd'; - -import { getPluginConfig } from '@/plugins'; +import { Modal } from 'antd'; +import { ConnectionName } from '../connection-name'; import { ConnectionForm } from '../connection-form'; -import * as S from './styled'; - interface Props { plugin: string; - connectionId: ID; + connectionId?: ID; open: boolean; onCancel: () => void; + onSuccess?: (id: ID) => void; } -export const ConnectionFormModal = ({ plugin, connectionId, open, onCancel }: Props) => { - const pluginConfig = useMemo(() => getPluginConfig(plugin), [plugin]); - - const { - token: { colorPrimary }, - } = theme.useToken(); - +export const ConnectionFormModal = ({ plugin, connectionId, open, onCancel, onSuccess }: Props) => { return ( - {pluginConfig.icon({ color: colorPrimary })} - Manage Connections: {pluginConfig.name} - + `Manage Connections: ${pluginName}`} + /> } footer={null} - onCancel={onCancel} + onCancel={() => onCancel()} > - + ); }; diff --git a/config-ui/src/plugins/components/connection-form/styled.ts b/config-ui/src/plugins/components/connection-form/styled.ts deleted file mode 100644 index 6036f510667..00000000000 --- a/config-ui/src/plugins/components/connection-form/styled.ts +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -import styled from 'styled-components'; - -export const ModalTitle = styled.div` - display: flex; - align-items: center; - - .icon { - display: inline-flex; - margin-right: 8px; - width: 24px; - - & > svg { - width: 100%; - height: 100%; - } - } -`; diff --git a/config-ui/src/plugins/components/connection-list/index.tsx b/config-ui/src/plugins/components/connection-list/index.tsx index 4f06306397b..4ab9cd43f8b 100644 --- a/config-ui/src/plugins/components/connection-list/index.tsx +++ b/config-ui/src/plugins/components/connection-list/index.tsx @@ -16,35 +16,18 @@ * */ -import { useState, useMemo } from 'react'; +import { useState } from 'react'; import { useNavigate } from 'react-router-dom'; import { EyeOutlined, EditOutlined, DeleteOutlined, PlusOutlined } from '@ant-design/icons'; import { theme, Table, Button, Modal, message } from 'antd'; -import styled from 'styled-components'; import { selectConnections, removeConnection } from '@/features/connections'; import { Message } from '@/components'; import { useAppDispatch, useAppSelector } from '@/hooks'; -import { getPluginConfig, ConnectionStatus, ConnectionForm } from '@/plugins'; +import { ConnectionStatus, ConnectionFormModal } from '@/plugins'; import { WebHookConnection } from '@/plugins/register/webhook'; import { operator } from '@/utils'; -const ModalTitle = styled.div` - display: flex; - align-items: center; - - .icon { - display: inline-flex; - margin-right: 8px; - width: 24px; - - & > svg { - width: 100%; - height: 100%; - } - } -`; - interface Props { plugin: string; onCreate: () => void; @@ -57,8 +40,6 @@ export const ConnectionList = ({ plugin, onCreate }: Props) => { const [conflict, setConflict] = useState([]); const [errorMsg, setErrorMsg] = useState(''); - const pluginConfig = useMemo(() => getPluginConfig(plugin), [plugin]); - const { token: { colorPrimary }, } = theme.useToken(); @@ -159,22 +140,7 @@ export const ConnectionList = ({ plugin, onCreate }: Props) => { Create a New Connection {modalType === 'update' && ( - - {pluginConfig.icon({ color: colorPrimary })} - Manage Connections: {pluginConfig.name} - - } - footer={null} - onCancel={hanldeHideModal} - > - - + )} {modalType === 'delete' && ( string; onClick?: () => void; } -export const ConnectionName = ({ plugin, connectionId, onClick }: Props) => { +const Name = ({ plugin, connectionId }: Required>) => { + const connection = useAppSelector((state) => selectConnection(state, `${plugin}-${connectionId}`)); + const webhook = useAppSelector((state) => selectWebhook(state, connectionId)); + + return ( + {connection ? connection.name : webhook ? webhook.name : `${plugin}/connection/${connectionId}`} + ); +}; + +export const ConnectionName = ({ plugin, connectionId, customName, onClick }: Props) => { const { token: { colorPrimary }, } = theme.useToken(); - - const connection = useAppSelector((state) => selectConnection(state, `${plugin}-${connectionId}`)); - const webhook = useAppSelector((state) => selectWebhook(state, connectionId)); const config = getPluginConfig(plugin); - const name = connection ? connection.name : webhook ? webhook.name : `${plugin}/connection/${connectionId}`; + if (!connectionId) { + return ( + + {config.icon({ color: colorPrimary })} + {config.name} + + ); + } + + if (customName) { + return ( + + {config.icon({ color: colorPrimary })} + {customName(config.name)} + + ); + } return ( {config.icon({ color: colorPrimary })} - {name} + ); }; diff --git a/config-ui/src/routes/blueprint/detail/configuration-panel.tsx b/config-ui/src/routes/blueprint/detail/configuration-panel.tsx index 479214d1401..d0045416812 100644 --- a/config-ui/src/routes/blueprint/detail/configuration-panel.tsx +++ b/config-ui/src/routes/blueprint/detail/configuration-panel.tsx @@ -24,8 +24,7 @@ import { Flex, Table, Button } from 'antd'; import API from '@/api'; import { NoData } from '@/components'; import { getCron } from '@/config'; -import { ConnectionName } from '@/features/connections'; -import { getPluginConfig } from '@/plugins'; +import { getPluginConfig, ConnectionName } from '@/plugins'; import { IBlueprint, IBPMode } from '@/types'; import { formatTime, operator } from '@/utils'; diff --git a/config-ui/src/routes/blueprint/home/index.tsx b/config-ui/src/routes/blueprint/home/index.tsx index f8f3c2a44cf..82876123ba7 100644 --- a/config-ui/src/routes/blueprint/home/index.tsx +++ b/config-ui/src/routes/blueprint/home/index.tsx @@ -25,8 +25,8 @@ import dayjs from 'dayjs'; import API from '@/api'; import { PageHeader, Block, TextTooltip, IconButton } from '@/components'; import { getCronOptions, cronPresets, getCron } from '@/config'; -import { ConnectionName } from '@/features/connections'; import { useRefreshData } from '@/hooks'; +import { ConnectionName } from '@/plugins'; import { IBlueprint, IBPMode } from '@/types'; import { formatTime, operator } from '@/utils'; diff --git a/config-ui/src/routes/connection/connection.tsx b/config-ui/src/routes/connection/connection.tsx index 4670a1839a1..61bdfac79c2 100644 --- a/config-ui/src/routes/connection/connection.tsx +++ b/config-ui/src/routes/connection/connection.tsx @@ -28,6 +28,7 @@ import { selectConnection } from '@/features/connections'; import { useAppSelector, useRefreshData } from '@/hooks'; import { ConnectionStatus, + ConnectionName, DataScopeRemote, getPluginConfig, getPluginScopeId, @@ -37,8 +38,6 @@ import { import { IConnection } from '@/types'; import { operator } from '@/utils'; -import * as S from './styled'; - const brandName = import.meta.env.DEVLAKE_BRAND_NAME ?? 'DevLake'; export const Connection = () => { @@ -312,12 +311,7 @@ export const Connection = () => { centered style={{ width: 820 }} footer={null} - title={ - - {pluginConfig.icon({ color: colorPrimary })} - Add Data Scope: {name} - - } + title={ `Add Data Scope`} />} onCancel={handleHideDialog} > { centered footer={null} title={ - - {pluginConfig.icon({ color: colorPrimary })} - Associate Scope Config - + `Associate Scope Config`} /> } onCancel={handleHideDialog} > diff --git a/config-ui/src/routes/connection/connections.tsx b/config-ui/src/routes/connection/connections.tsx index 1148d994a9d..5cd68006a6f 100644 --- a/config-ui/src/routes/connection/connections.tsx +++ b/config-ui/src/routes/connection/connections.tsx @@ -23,7 +23,7 @@ import { chunk } from 'lodash'; import { selectPlugins, selectAllConnections, selectWebhooks } from '@/features/connections'; import { useAppSelector } from '@/hooks'; -import { getPluginConfig, ConnectionList, ConnectionForm } from '@/plugins'; +import { getPluginConfig, ConnectionName, ConnectionList, ConnectionFormModal } from '@/plugins'; import * as S from './styled'; @@ -155,12 +155,7 @@ export const Connections = () => { open width={820} centered - title={ - - {pluginConfig.icon({ color: colorPrimary })} - Manage Connections: {pluginConfig.name} - - } + title={ `Manage Connections: ${pluginName}`} />} footer={null} onCancel={handleHideDialog} > @@ -168,24 +163,12 @@ export const Connections = () => { )} {type === 'form' && pluginConfig && ( - - {pluginConfig.icon({ color: colorPrimary })} - Manage Connections: {pluginConfig.name} - - } - footer={null} onCancel={handleHideDialog} - > - handleSuccessAfter(pluginConfig.plugin, id)} - /> - + onSuccess={(id) => handleSuccessAfter(pluginConfig.plugin, id)} + /> )} ); diff --git a/config-ui/src/routes/connection/styled.ts b/config-ui/src/routes/connection/styled.ts index 6aa45c94808..066113c3a9c 100644 --- a/config-ui/src/routes/connection/styled.ts +++ b/config-ui/src/routes/connection/styled.ts @@ -111,19 +111,3 @@ export const Wrapper = styled.div<{ theme: string }>` } } `; - -export const ModalTitle = styled.div` - display: flex; - align-items: center; - - .icon { - display: inline-flex; - margin-right: 8px; - width: 24px; - - & > svg { - width: 100%; - height: 100%; - } - } -`; diff --git a/config-ui/src/routes/project/home/index.tsx b/config-ui/src/routes/project/home/index.tsx index 60ea6abf56e..eac2c22e68b 100644 --- a/config-ui/src/routes/project/home/index.tsx +++ b/config-ui/src/routes/project/home/index.tsx @@ -24,11 +24,11 @@ import { Flex, Table, Button, Modal, Input } from 'antd'; import API from '@/api'; import { PageHeader, Block, IconButton } from '@/components'; import { getCron } from '@/config'; -import { ConnectionName } from '@/features/connections'; import { useRefreshData } from '@/hooks'; import { OnboardTour } from '@/routes/onboard/components'; import { formatTime, operator } from '@/utils'; import { PipelineStatus } from '@/routes/pipeline'; +import { ConnectionName } from '@/plugins'; import { IBlueprint } from '@/types'; export const ProjectHomePage = () => { From 4942c81fa2e209c372905395f027be5b06c7dc9d Mon Sep 17 00:00:00 2001 From: mintsweet <0x1304570@gmail.com> Date: Fri, 27 Sep 2024 15:06:06 +1200 Subject: [PATCH 13/16] fix: error check connection token failed list --- config-ui/src/routes/blueprint/detail/status-panel.tsx | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/config-ui/src/routes/blueprint/detail/status-panel.tsx b/config-ui/src/routes/blueprint/detail/status-panel.tsx index 316a0e970df..8b23cec2bb2 100644 --- a/config-ui/src/routes/blueprint/detail/status-panel.tsx +++ b/config-ui/src/routes/blueprint/detail/status-panel.tsx @@ -87,9 +87,11 @@ export const StatusPanel = ({ from, blueprint, pipelineId, onRefresh }: Props) = }; }); - setType('checkTokenFailed'); - setConnectionFailed(connectionFailed); - return; + if (connectionFailed.length) { + setType('checkTokenFailed'); + setConnectionFailed(connectionFailed); + return; + } } } From 8bf8f8e962125af41c4d2564163b15a23e89e8f6 Mon Sep 17 00:00:00 2001 From: mintsweet <0x1304570@gmail.com> Date: Sun, 29 Sep 2024 15:41:21 +1300 Subject: [PATCH 14/16] refactor: extract the function in the panel to bp detail --- .../blueprint/detail/blueprint-detail.tsx | 140 ++++++++++++++++-- .../blueprint/detail/configuration-panel.tsx | 54 ++----- .../routes/blueprint/detail/status-panel.tsx | 134 ++--------------- 3 files changed, 152 insertions(+), 176 deletions(-) diff --git a/config-ui/src/routes/blueprint/detail/blueprint-detail.tsx b/config-ui/src/routes/blueprint/detail/blueprint-detail.tsx index 3cc5321f97a..c46d19fd150 100644 --- a/config-ui/src/routes/blueprint/detail/blueprint-detail.tsx +++ b/config-ui/src/routes/blueprint/detail/blueprint-detail.tsx @@ -16,20 +16,39 @@ * */ -import { useEffect, useState } from 'react'; -import { useLocation } from 'react-router-dom'; -import { Tabs } from 'antd'; +import { useState, useEffect, useReducer } from 'react'; +import { useLocation, useNavigate } from 'react-router-dom'; +import { WarningOutlined } from '@ant-design/icons'; +import { theme, Tabs, Modal } from 'antd'; import API from '@/api'; import { PageLoading } from '@/components'; import { useRefreshData } from '@/hooks'; +import { operator } from '@/utils'; import { FromEnum } from '../types'; +import { ConnectionCheck } from './components'; import { ConfigurationPanel } from './configuration-panel'; import { StatusPanel } from './status-panel'; import * as S from './styled'; +type ConnectionFailed = { + open: boolean; + failedList?: Array<{ plugin: string; connectionId: ID }>; +}; + +function reducer(state: ConnectionFailed, action: { type: string; failedList?: ConnectionFailed['failedList'] }) { + switch (action.type) { + case 'open': + return { open: true, failedList: action.failedList }; + case 'close': + return { open: false, failedList: [] }; + default: + return state; + } +} + interface Props { id: ID; from: FromEnum; @@ -38,8 +57,18 @@ interface Props { export const BlueprintDetail = ({ id, from }: Props) => { const [version, setVersion] = useState(1); const [activeKey, setActiveKey] = useState('status'); + const [operating, setOperating] = useState(false); + + const [{ open, failedList }, dispatch] = useReducer(reducer, { + open: false, + }); const { state } = useLocation(); + const navigate = useNavigate(); + + const { + token: { orange5 }, + } = theme.useToken(); useEffect(() => { setActiveKey(state?.activeKey ?? 'status'); @@ -50,12 +79,71 @@ export const BlueprintDetail = ({ id, from }: Props) => { return [bpRes, pipelineRes.pipelines[0]]; }, [version]); - const handlRefresh = () => { - setVersion((v) => v + 1); + const handleDelete = async () => { + const [success] = await operator(() => API.blueprint.remove(blueprint.id), { + setOperating, + formatMessage: () => 'Delete blueprint successful.', + }); + + if (success) { + navigate('/advanced/blueprints'); + } }; - const handleChangeActiveKey = (activeKey: string) => { - setActiveKey(activeKey); + const handleUpdate = async (payload: any) => { + const [success] = await operator( + () => + API.blueprint.update(blueprint.id, { + ...blueprint, + ...payload, + }), + { + setOperating, + formatMessage: () => + from === FromEnum.project ? 'Update project successful.' : 'Update blueprint successful.', + }, + ); + + if (success) { + setVersion((v) => v + 1); + } + }; + + const handleTrigger = async (payload?: { skipCollectors?: boolean; fullSync?: boolean }) => { + const { skipCollectors, fullSync } = payload ?? { skipCollectors: false, fullSync: false }; + + if (!skipCollectors) { + const [success, res] = await operator(() => API.blueprint.connectionsTokenCheck(blueprint.id), { + hideToast: true, + setOperating, + }); + + if (success && res.length) { + const connectionFailed = res + .filter((it: any) => !it.success) + .map((it: any) => { + return { + plugin: it.pluginName, + connectionId: it.connectionId, + }; + }); + + if (connectionFailed.length) { + dispatch({ type: 'open', failedList: connectionFailed }); + return; + } + } + } + + const [success] = await operator(() => API.blueprint.trigger(blueprint.id, { skipCollectors, fullSync }), { + setOperating, + formatMessage: () => 'Trigger blueprint successful.', + }); + + if (success) { + setVersion((v) => v + 1); + setActiveKey('status'); + } }; if (!ready || !data) { @@ -72,7 +160,15 @@ export const BlueprintDetail = ({ id, from }: Props) => { key: 'status', label: 'Status', children: ( - + ), }, { @@ -82,15 +178,37 @@ export const BlueprintDetail = ({ id, from }: Props) => { ), }, ]} activeKey={activeKey} - onChange={handleChangeActiveKey} + onChange={setActiveKey} /> + + + Invalid Token(s) Detected + + } + width={820} + footer={null} + onCancel={() => dispatch({ type: 'close' })} + > +

    There are invalid tokens in the following connections. Please update them before re-syncing the data.

    +
      + {(failedList ?? []).map(({ plugin, connectionId }) => ( +
    • + +
      • + ))} +
    +     ); }; diff --git a/config-ui/src/routes/blueprint/detail/configuration-panel.tsx b/config-ui/src/routes/blueprint/detail/configuration-panel.tsx index d0045416812..095c7f0ead9 100644 --- a/config-ui/src/routes/blueprint/detail/configuration-panel.tsx +++ b/config-ui/src/routes/blueprint/detail/configuration-panel.tsx @@ -21,12 +21,11 @@ import { Link } from 'react-router-dom'; import { FormOutlined, PlusOutlined } from '@ant-design/icons'; import { Flex, Table, Button } from 'antd'; -import API from '@/api'; import { NoData } from '@/components'; import { getCron } from '@/config'; import { getPluginConfig, ConnectionName } from '@/plugins'; import { IBlueprint, IBPMode } from '@/types'; -import { formatTime, operator } from '@/utils'; +import { formatTime } from '@/utils'; import { FromEnum } from '../types'; import { validRawPlan } from '../utils'; @@ -37,14 +36,14 @@ import * as S from './styled'; interface Props { from: FromEnum; blueprint: IBlueprint; - onRefresh: () => void; - onChangeTab: (tab: string) => void; + operating: boolean; + onUpdate: (payload: any) => void; + onTrigger: (payload?: { skipCollectors?: boolean; fullSync?: boolean }) => void; } -export const ConfigurationPanel = ({ from, blueprint, onRefresh, onChangeTab }: Props) => { +export const ConfigurationPanel = ({ from, blueprint, operating, onUpdate, onTrigger }: Props) => { const [type, setType] = useState<'policy' | 'add-connection'>(); const [rawPlan, setRawPlan] = useState(''); - const [operating, setOperating] = useState(false); useEffect(() => { setRawPlan(JSON.stringify(blueprint.plan, null, ' ')); @@ -79,41 +78,6 @@ export const ConfigurationPanel = ({ from, blueprint, onRefresh, onChangeTab }: setType('add-connection'); }; - const handleUpdate = async (payload: any) => { - const [success] = await operator( - () => - API.blueprint.update(blueprint.id, { - ...blueprint, - ...payload, - }), - { - setOperating, - formatMessage: () => - from === FromEnum.project ? 'Update project successful.' : 'Update blueprint successful.', - }, - ); - - if (success) { - onRefresh(); - handleCancel(); - } - }; - - const handleRun = async () => { - const [success] = await operator( - () => API.blueprint.trigger(blueprint.id, { skipCollectors: false, fullSync: false }), - { - setOperating, - formatMessage: () => 'Trigger blueprint successful.', - }, - ); - - if (success) { - onRefresh(); - onChangeTab('status'); - } - }; - return (
    @@ -212,7 +176,7 @@ export const ConfigurationPanel = ({ from, blueprint, onRefresh, onChangeTab }: ))} - @@ -228,7 +192,7 @@ export const ConfigurationPanel = ({ from, blueprint, onRefresh, onChangeTab }: - - handleUpdate({ enable })} + onChange={(enable) => onUpdate({ enable })} /> Blueprint Enabled @@ -252,7 +173,7 @@ export const StatusPanel = ({ from, blueprint, pipelineId, onRefresh }: Props) = loading: operating, }} onCancel={handleResetType} - onOk={handleDelete} + onOk={onDelete} > )} - - {type === 'checkTokenFailed' && ( - - - Invalid Token(s) Detected - - } - width={820} - footer={null} - onCancel={() => { - handleResetType(); - setConnectionFailed([]); - }} - > -

    There are invalid tokens in the following connections. Please update them before re-syncing the data.

    -
      - {connectionFailed.map(({ plugin, connectionId }) => ( -
    • - -
      • - ))} -
    -     - )} ); }; From 11d7f94533115863202fddea9b09ea792871919c Mon Sep 17 00:00:00 2001 From: mintsweet <0x1304570@gmail.com> Date: Sun, 29 Sep 2024 15:44:50 +1300 Subject: [PATCH 15/16] fix: not call onCancel function when updating connection --- .../components/connection-form/connection-form-modal.tsx | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/config-ui/src/plugins/components/connection-form/connection-form-modal.tsx b/config-ui/src/plugins/components/connection-form/connection-form-modal.tsx index 4eccb09ae80..cf14c156435 100644 --- a/config-ui/src/plugins/components/connection-form/connection-form-modal.tsx +++ b/config-ui/src/plugins/components/connection-form/connection-form-modal.tsx @@ -30,6 +30,11 @@ interface Props { } export const ConnectionFormModal = ({ plugin, connectionId, open, onCancel, onSuccess }: Props) => { + const handleSuccess = (id: ID) => { + onSuccess?.(id); + onCancel(); + }; + return ( onCancel()} > - + ); }; From 542c15523114af73b119f0ba67fe6d6190f083c6 Mon Sep 17 00:00:00 2001 From: mintsweet <0x1304570@gmail.com> Date: Sun, 29 Sep 2024 17:26:42 +1300 Subject: [PATCH 16/16] fix: search local items error --- config-ui/package.json | 2 +- .../components/data-scope-remote/search-local.tsx | 11 ++++++++++- config-ui/yarn.lock | 10 +++++----- 3 files changed, 16 insertions(+), 7 deletions(-) diff --git a/config-ui/package.json b/config-ui/package.json index 7e1305d7240..d765f38340c 100644 --- a/config-ui/package.json +++ b/config-ui/package.json @@ -25,7 +25,7 @@ "@ant-design/icons": "^5.4.0", "@fontsource/roboto": "^5.0.14", "@mints/hooks": "^1.0.0-beta.9", - "@mints/miller-columns": "^2.0.0-beta.10", + "@mints/miller-columns": "^2.0.0-beta.11", "@mui/icons-material": "^5.16.7", "@mui/material": "^5.16.7", "@mui/styled-engine-sc": "^6.0.0-alpha.18", diff --git a/config-ui/src/plugins/components/data-scope-remote/search-local.tsx b/config-ui/src/plugins/components/data-scope-remote/search-local.tsx index 961ca68e918..73e4bc1d050 100644 --- a/config-ui/src/plugins/components/data-scope-remote/search-local.tsx +++ b/config-ui/src/plugins/components/data-scope-remote/search-local.tsx @@ -203,7 +203,16 @@ export const SearchLocal = ({ mode, plugin, connectionId, config, disabledScope, it.title.includes(searchDebounce) && !it.canExpand) : scope} + items={ + searchDebounce + ? scope + .filter((it) => it.title.includes(searchDebounce) && !it.canExpand) + .map((it) => ({ + ...it, + parentId: null, + })) + : scope + } /> )} diff --git a/config-ui/yarn.lock b/config-ui/yarn.lock index c7db434e486..ca316b4a431 100644 --- a/config-ui/yarn.lock +++ b/config-ui/yarn.lock @@ -2087,9 +2087,9 @@ __metadata: languageName: node linkType: hard -"@mints/miller-columns@npm:^2.0.0-beta.10": - version: 2.0.0-beta.10 - resolution: "@mints/miller-columns@npm:2.0.0-beta.10" +"@mints/miller-columns@npm:^2.0.0-beta.11": + version: 2.0.0-beta.11 + resolution: "@mints/miller-columns@npm:2.0.0-beta.11" dependencies: "@fontsource/roboto": ^5.0.14 "@mui/material": ^5.16.7 @@ -2108,7 +2108,7 @@ __metadata: react-dom: ^18.2.0 react-infinite-scroll-component: ^6.1.0 styled-components: ^6.1.12 - checksum: 94c3ba41210f2ccbddc413ef6c26e5615f11c5a3aac7954484d838d1b05713543def70b6714e79cfc396efea522f553b2d648b36af994bf66456a440836a1386 + checksum: a2408f665aae4f037c265b18cac976b7b77ec2216e93ae24fdb8f77603ff7411efbb5398f30a6a1128e2df4e31b8ebb3b99efb9a0f2f502fec96394bde1f2259 languageName: node linkType: hard @@ -3911,7 +3911,7 @@ __metadata: "@ant-design/icons": ^5.4.0 "@fontsource/roboto": ^5.0.14 "@mints/hooks": ^1.0.0-beta.9 - "@mints/miller-columns": ^2.0.0-beta.10 + "@mints/miller-columns": ^2.0.0-beta.11 "@mui/icons-material": ^5.16.7 "@mui/material": ^5.16.7 "@mui/styled-engine-sc": ^6.0.0-alpha.18