From a2c60c3334283e30b7633e6efcaf1db166a1ea4c Mon Sep 17 00:00:00 2001 From: Martyn van Dijke Date: Thu, 12 Sep 2024 09:41:28 +0200 Subject: [PATCH 1/3] feat: introduce generic webook --- backend/plugins/webhook/api/generic.go | 109 +++++++++++++++++++++++++ backend/plugins/webhook/impl/impl.go | 3 + 2 files changed, 112 insertions(+) create mode 100644 backend/plugins/webhook/api/generic.go diff --git a/backend/plugins/webhook/api/generic.go b/backend/plugins/webhook/api/generic.go new file mode 100644 index 00000000000..489cf076c29 --- /dev/null +++ b/backend/plugins/webhook/api/generic.go @@ -0,0 +1,109 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "net/http" + "time" + + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/log" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/dbhelper" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/webhook/models" + "github.com/go-playground/validator/v10" +) + +type WebhookGenericReq struct { + Url string `mapstructure:"url"` + IssueKey string `mapstructure:"issueKey" validate:"required"` + Title string `mapstructure:"title" validate:"required"` + Description string `mapstructure:"description"` + Name string `mapstructure:"name"` + Json string `mapstructure:"json"` + + CreatedDate *time.Time `mapstructure:"createdDate"` + StartedDate *time.Time `mapstructure:"startedDate" validate:"required"` + FinishedDate *time.Time `mapstructure:"finishedDate" validate:"required"` +} + +func PostGeneric(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + connection := &models.WebhookConnection{} + err := connectionHelper.First(connection, input.Params) + if err != nil { + return nil, err + } + // get request + request := &WebhookGenericReq{} + + err = api.DecodeMapStruct(input.Body, request, true) + if err != nil { + return &plugin.ApiResourceOutput{Body: err.Error(), Status: http.StatusBadRequest}, nil + } + + // validate + vld = validator.New() + err = errors.Convert(vld.Struct(request)) + if err != nil { + return nil, errors.BadInput.Wrap(vld.Struct(request), `input json error`) + } + txHelper := dbhelper.NewTxHelper(basicRes, &err) + defer txHelper.End() + tx := txHelper.Begin() + if err := Create(connection, request, tx, logger); err != nil { + logger.Error(err, "create deployments") + return nil, err + } + + return &plugin.ApiResourceOutput{Body: nil, Status: http.StatusOK}, nil +} + +func Create(connection *models.WebhookConnection, request *WebhookGenericReq, tx dal.Transaction, logger log.Logger) errors.Error { + // validation + if request == nil { + return errors.BadInput.New("request body is nil") + } + if len(request.Json) == 0 { + return errors.BadInput.New("json payload is empty") + } + if request.CreatedDate == nil { + request.CreatedDate = request.StartedDate + } + if request.FinishedDate == nil { + now := time.Now() + request.FinishedDate = &now + } + createdDate := time.Now() + if request.CreatedDate != nil { + createdDate = *request.CreatedDate + } else if request.StartedDate != nil { + createdDate = *request.StartedDate + } + if request.CreatedDate == nil { + request.CreatedDate = &createdDate + } + + if err := tx.CreateOrUpdate(request.Json); err != nil { + logger.Error(err, "failed to generic json data to disk") + return err + } + + return nil +} diff --git a/backend/plugins/webhook/impl/impl.go b/backend/plugins/webhook/impl/impl.go index ada30478399..54d52c34030 100644 --- a/backend/plugins/webhook/impl/impl.go +++ b/backend/plugins/webhook/impl/impl.go @@ -100,6 +100,9 @@ func (p Webhook) ApiResources() map[string]map[string]plugin.ApiResourceHandler ":connectionId/deployments": { "POST": api.PostDeployments, }, + ":connectionId/generic": { + "POST": api.PostGeneric, + }, ":connectionId/issues": { "POST": api.PostIssue, }, From 617056b500197717b6c88a97926079a5b6b175cf Mon Sep 17 00:00:00 2001 From: Martyn van Dijke Date: Thu, 12 Sep 2024 14:38:11 +0200 Subject: [PATCH 2/3] try to store it in db --- backend/plugins/webhook/api/connection.go | 14 +++-- backend/plugins/webhook/api/generic.go | 70 +++++++++++++++-------- backend/plugins/webhook/impl/impl.go | 2 +- 3 files changed, 55 insertions(+), 31 deletions(-) diff --git a/backend/plugins/webhook/api/connection.go b/backend/plugins/webhook/api/connection.go index acb8a76a593..988ca496e83 100644 --- a/backend/plugins/webhook/api/connection.go +++ b/backend/plugins/webhook/api/connection.go @@ -139,12 +139,13 @@ func DeleteConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput type WebhookConnectionResponse struct { models.WebhookConnection - PostIssuesEndpoint string `json:"postIssuesEndpoint"` - CloseIssuesEndpoint string `json:"closeIssuesEndpoint"` - PostPipelineTaskEndpoint string `json:"postPipelineTaskEndpoint"` - PostPipelineDeployTaskEndpoint string `json:"postPipelineDeployTaskEndpoint"` - ClosePipelineEndpoint string `json:"closePipelineEndpoint"` - ApiKey *coreModels.ApiKey `json:"apiKey,omitempty"` + PostIssuesEndpoint string `json:"postIssuesEndpoint"` + CloseIssuesEndpoint string `json:"closeIssuesEndpoint"` + PostPipelineTaskEndpoint string `json:"postPipelineTaskEndpoint"` + PostPipelineDeployTaskEndpoint string `json:"postPipelineDeployTaskEndpoint"` + PostPipelineGenericTaskEndpoint string `json:"postPipelineGenericTaskEndpoint"` + ClosePipelineEndpoint string `json:"closePipelineEndpoint"` + ApiKey *coreModels.ApiKey `json:"apiKey,omitempty"` } // ListConnections @@ -197,6 +198,7 @@ func formatConnection(connection *models.WebhookConnection, withApiKeyInfo bool) response.CloseIssuesEndpoint = fmt.Sprintf(`/rest/plugins/webhook/connections/%d/issue/:issueKey/close`, connection.ID) response.PostPipelineTaskEndpoint = fmt.Sprintf(`/rest/plugins/webhook/connections/%d/cicd_tasks`, connection.ID) response.PostPipelineDeployTaskEndpoint = fmt.Sprintf(`/rest/plugins/webhook/connections/%d/deployments`, connection.ID) + response.PostPipelineGenericTaskEndpoint = fmt.Sprintf(`/rest/plugins/webhook/connections/%d/generic`, connection.ID) response.ClosePipelineEndpoint = fmt.Sprintf(`/rest/plugins/webhook/connections/%d/cicd_pipeline/:pipelineName/finish`, connection.ID) if withApiKeyInfo { db := basicRes.GetDal() diff --git a/backend/plugins/webhook/api/generic.go b/backend/plugins/webhook/api/generic.go index 489cf076c29..6d1a67ae461 100644 --- a/backend/plugins/webhook/api/generic.go +++ b/backend/plugins/webhook/api/generic.go @@ -18,6 +18,8 @@ limitations under the License. package api import ( + "encoding/json" + "fmt" "net/http" "time" @@ -32,20 +34,24 @@ import ( ) type WebhookGenericReq struct { - Url string `mapstructure:"url"` - IssueKey string `mapstructure:"issueKey" validate:"required"` - Title string `mapstructure:"title" validate:"required"` - Description string `mapstructure:"description"` - Name string `mapstructure:"name"` - Json string `mapstructure:"json"` - - CreatedDate *time.Time `mapstructure:"createdDate"` - StartedDate *time.Time `mapstructure:"startedDate" validate:"required"` - FinishedDate *time.Time `mapstructure:"finishedDate" validate:"required"` + Title string `mapstructure:"title" validate:"required"` + Description string `mapstructure:"description"` + Json map[string]interface{} `mapstructure:"json"` + + CreatedDate *time.Time `mapstructure:"createdDate"` +} + +type Generic struct { + Id int + Title string + Description string + CreatedDate *time.Time + data string } func PostGeneric(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { connection := &models.WebhookConnection{} + err := connectionHelper.First(connection, input.Params) if err != nil { return nil, err @@ -64,18 +70,19 @@ func PostGeneric(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, err if err != nil { return nil, errors.BadInput.Wrap(vld.Struct(request), `input json error`) } + txHelper := dbhelper.NewTxHelper(basicRes, &err) defer txHelper.End() tx := txHelper.Begin() if err := Create(connection, request, tx, logger); err != nil { - logger.Error(err, "create deployments") + logger.Error(err, "create generic") return nil, err } return &plugin.ApiResourceOutput{Body: nil, Status: http.StatusOK}, nil } -func Create(connection *models.WebhookConnection, request *WebhookGenericReq, tx dal.Transaction, logger log.Logger) errors.Error { +func Create(connection *models.WebhookConnection, request *WebhookGenericReq, db dal.Transaction, logger log.Logger) errors.Error { // validation if request == nil { return errors.BadInput.New("request body is nil") @@ -83,27 +90,42 @@ func Create(connection *models.WebhookConnection, request *WebhookGenericReq, tx if len(request.Json) == 0 { return errors.BadInput.New("json payload is empty") } - if request.CreatedDate == nil { - request.CreatedDate = request.StartedDate - } - if request.FinishedDate == nil { - now := time.Now() - request.FinishedDate = &now - } createdDate := time.Now() if request.CreatedDate != nil { createdDate = *request.CreatedDate - } else if request.StartedDate != nil { - createdDate = *request.StartedDate } if request.CreatedDate == nil { request.CreatedDate = &createdDate } - if err := tx.CreateOrUpdate(request.Json); err != nil { - logger.Error(err, "failed to generic json data to disk") - return err + fmt.Println(request.Json) + + jsonString, err := json.Marshal(request.Json) + fmt.Println(jsonString) + if err != nil { + logger.Error(err, "Error marshaling JSON:") + } + + generic := new(Generic) + generic.Title = request.Title + generic.Description = request.Description + generic.CreatedDate = request.CreatedDate + generic.data = string(jsonString) + + if !db.HasTable(generic) { + db.AutoMigrate(generic) + } + db.AutoMigrate(generic) + err = db.Create(generic) + if err != nil { + return nil } + db.All(generic) + + // if err := tx.CreateOrUpdate(generic); err != nil { + // logger.Error(err, "failed to generic json data to disk") + // return err + // } return nil } diff --git a/backend/plugins/webhook/impl/impl.go b/backend/plugins/webhook/impl/impl.go index 54d52c34030..3de809460ba 100644 --- a/backend/plugins/webhook/impl/impl.go +++ b/backend/plugins/webhook/impl/impl.go @@ -100,7 +100,7 @@ func (p Webhook) ApiResources() map[string]map[string]plugin.ApiResourceHandler ":connectionId/deployments": { "POST": api.PostDeployments, }, - ":connectionId/generic": { + "connections/:connectionId/generic": { "POST": api.PostGeneric, }, ":connectionId/issues": { From bce6f9d38a7fb7de3467f2dafa2e43caa4536d85 Mon Sep 17 00:00:00 2001 From: Martyn van Dijke Date: Thu, 12 Sep 2024 16:30:57 +0200 Subject: [PATCH 3/3] fix: update storage --- backend/plugins/webhook/api/generic.go | 35 ++++++-------------------- 1 file changed, 8 insertions(+), 27 deletions(-) diff --git a/backend/plugins/webhook/api/generic.go b/backend/plugins/webhook/api/generic.go index 6d1a67ae461..e81be7337b3 100644 --- a/backend/plugins/webhook/api/generic.go +++ b/backend/plugins/webhook/api/generic.go @@ -18,7 +18,6 @@ limitations under the License. package api import ( - "encoding/json" "fmt" "net/http" "time" @@ -36,17 +35,16 @@ import ( type WebhookGenericReq struct { Title string `mapstructure:"title" validate:"required"` Description string `mapstructure:"description"` - Json map[string]interface{} `mapstructure:"json"` + Data map[string]interface{} `mapstructure:"json"` CreatedDate *time.Time `mapstructure:"createdDate"` } type Generic struct { - Id int Title string Description string CreatedDate *time.Time - data string + Data string } func PostGeneric(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { @@ -74,7 +72,7 @@ func PostGeneric(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, err txHelper := dbhelper.NewTxHelper(basicRes, &err) defer txHelper.End() tx := txHelper.Begin() - if err := Create(connection, request, tx, logger); err != nil { + if err := SaveGeneric(connection, request, tx, logger); err != nil { logger.Error(err, "create generic") return nil, err } @@ -82,12 +80,12 @@ func PostGeneric(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, err return &plugin.ApiResourceOutput{Body: nil, Status: http.StatusOK}, nil } -func Create(connection *models.WebhookConnection, request *WebhookGenericReq, db dal.Transaction, logger log.Logger) errors.Error { +func SaveGeneric(connection *models.WebhookConnection, request *WebhookGenericReq, db dal.Transaction, logger log.Logger) errors.Error { // validation if request == nil { return errors.BadInput.New("request body is nil") } - if len(request.Json) == 0 { + if len(request.Data) == 0 { return errors.BadInput.New("json payload is empty") } createdDate := time.Now() @@ -98,34 +96,17 @@ func Create(connection *models.WebhookConnection, request *WebhookGenericReq, db request.CreatedDate = &createdDate } - fmt.Println(request.Json) - - jsonString, err := json.Marshal(request.Json) - fmt.Println(jsonString) - if err != nil { - logger.Error(err, "Error marshaling JSON:") - } - generic := new(Generic) generic.Title = request.Title generic.Description = request.Description generic.CreatedDate = request.CreatedDate - generic.data = string(jsonString) + generic.Data = fmt.Sprintf("%s", request.Data) - if !db.HasTable(generic) { - db.AutoMigrate(generic) - } db.AutoMigrate(generic) - err = db.Create(generic) + err := db.CreateOrUpdate(generic) if err != nil { - return nil + return err } - db.All(generic) - - // if err := tx.CreateOrUpdate(generic); err != nil { - // logger.Error(err, "failed to generic json data to disk") - // return err - // } return nil }