Merge branch 'main' into chore/fix-jira-scope-config

apache · Dec 18, 2024 · c053014 · c053014
2 parents 28bf2d3 + bebd054
commit c053014
Show file tree

Hide file tree

Showing 5 changed files with 68 additions and 31 deletions.
diff --git a/backend/core/runner/run_task.go b/backend/core/runner/run_task.go
@@ -335,7 +335,7 @@ func RunPluginSubTasks(
 				logger.Error(err, "")
 				where := dal.Where("task_id = ? and name = ?", task.ID, subtaskCtx.GetName())
 				if err := basicRes.GetDal().UpdateColumns(subtask, []dal.DalSet{
-					{ColumnName: "is_failed", Value: 1},
+					{ColumnName: "is_failed", Value: true},
 					{ColumnName: "message", Value: err.Error()},
 				}, where); err != nil {
 					basicRes.GetLogger().Error(err, "error writing subtask %v status to DB", subtaskCtx.GetName())

diff --git a/backend/helpers/pluginhelper/api/ds_remote_api_proxy_api.go b/backend/helpers/pluginhelper/api/ds_remote_api_proxy_api.go
@@ -39,11 +39,7 @@ type DsRemoteApiProxyHelper[C plugin.ToolLayerApiConnection] struct {
 }
 
 // NewDsRemoteApiProxyHelper creates a new DsRemoteApiProxyHelper
-func NewDsRemoteApiProxyHelper[
-	C plugin.ToolLayerApiConnection,
-](
-	modelApiHelper *ModelApiHelper[C],
-) *DsRemoteApiProxyHelper[C] {
+func NewDsRemoteApiProxyHelper[C plugin.ToolLayerApiConnection](modelApiHelper *ModelApiHelper[C]) *DsRemoteApiProxyHelper[C] {
 	return &DsRemoteApiProxyHelper[C]{
 		ModelApiHelper:       modelApiHelper,
 		logger:               modelApiHelper.basicRes.GetLogger().Nested("remote_api_helper"),

diff --git a/backend/plugins/github_graphql/tasks/job_collector.go b/backend/plugins/github_graphql/tasks/job_collector.go
@@ -51,7 +51,11 @@ type GraphqlQueryCheckSuite struct {
 		// equal to Job in rest
 		CheckRuns struct {
 			TotalCount int
-			Nodes      []struct {
+			PageInfo   struct {
+				EndCursor   string `graphql:"endCursor"`
+				HasNextPage bool   `graphql:"hasNextPage"`
+			}
+			Nodes []struct {
 				Id          string
 				Name        string
 				DetailsUrl  string
@@ -79,7 +83,7 @@ type GraphqlQueryCheckSuite struct {
 					}
 				} `graphql:"steps(first: 50)"`
 			}
-		} `graphql:"checkRuns(first: 50)"`
+		} `graphql:"checkRuns(first: $pageSize, after: $skipCursor)"`
 	} `graphql:"... on CheckSuite"`
 }
 
@@ -95,7 +99,45 @@ var CollectJobsMeta = plugin.SubTaskMeta{
 	DomainTypes:      []string{plugin.DOMAIN_TYPE_CICD},
 }
 
-var _ plugin.SubTaskEntryPoint = CollectAccount
+var _ plugin.SubTaskEntryPoint = CollectJobs
+
+func getPageInfo(query interface{}, args *helper.GraphqlCollectorArgs) (*helper.GraphqlQueryPageInfo, error) {
+	queryWrapper := query.(*GraphqlQueryCheckRunWrapper)
+	hasNextPage := false
+	endCursor := ""
+	for _, node := range queryWrapper.Node {
+		if node.CheckSuite.CheckRuns.PageInfo.HasNextPage {
+			hasNextPage = true
+			endCursor = node.CheckSuite.CheckRuns.PageInfo.EndCursor
+			break
+		}
+	}
+	return &helper.GraphqlQueryPageInfo{
+		EndCursor:   endCursor,
+		HasNextPage: hasNextPage,
+	}, nil
+}
+
+func buildQuery(reqData *helper.GraphqlRequestData) (interface{}, map[string]interface{}, error) {
+	query := &GraphqlQueryCheckRunWrapper{}
+	if reqData == nil {
+		return query, map[string]interface{}{}, nil
+	}
+	workflowRuns := reqData.Input.([]interface{})
+	checkSuiteIds := []map[string]interface{}{}
+	for _, iWorkflowRuns := range workflowRuns {
+		workflowRun := iWorkflowRuns.(*SimpleWorkflowRun)
+		checkSuiteIds = append(checkSuiteIds, map[string]interface{}{
+			`id`: graphql.ID(workflowRun.CheckSuiteNodeID),
+		})
+	}
+	variables := map[string]interface{}{
+		"node":       checkSuiteIds,
+		"pageSize":   graphql.Int(reqData.Pager.Size),
+		"skipCursor": (*graphql.String)(reqData.Pager.SkipCursor),
+	}
+	return query, variables, nil
+}
 
 func CollectJobs(taskCtx plugin.SubTaskContext) errors.Error {
 	db := taskCtx.GetDal()
@@ -137,26 +179,10 @@ func CollectJobs(taskCtx plugin.SubTaskContext) errors.Error {
 
 	err = apiCollector.InitGraphQLCollector(helper.GraphqlCollectorArgs{
 		Input:         iterator,
-		InputStep:     20,
+		InputStep:     10,
 		GraphqlClient: data.GraphqlClient,
-		BuildQuery: func(reqData *helper.GraphqlRequestData) (interface{}, map[string]interface{}, error) {
-			query := &GraphqlQueryCheckRunWrapper{}
-			if reqData == nil {
-				return query, map[string]interface{}{}, nil
-			}
-			workflowRuns := reqData.Input.([]interface{})
-			checkSuiteIds := []map[string]interface{}{}
-			for _, iWorkflowRuns := range workflowRuns {
-				workflowRun := iWorkflowRuns.(*SimpleWorkflowRun)
-				checkSuiteIds = append(checkSuiteIds, map[string]interface{}{
-					`id`: graphql.ID(workflowRun.CheckSuiteNodeID),
-				})
-			}
-			variables := map[string]interface{}{
-				"node": checkSuiteIds,
-			}
-			return query, variables, nil
-		},
+		BuildQuery:    buildQuery,
+		GetPageInfo:   getPageInfo,
 		ResponseParser: func(queryWrapper any) (messages []json.RawMessage, err errors.Error) {
 			query := queryWrapper.(*GraphqlQueryCheckRunWrapper)
 			for _, node := range query.Node {
@@ -168,12 +194,13 @@ func CollectJobs(taskCtx plugin.SubTaskContext) errors.Error {
 					if apiCollector.GetSince() != nil && !apiCollector.GetSince().Before(*updatedAt) {
 						return messages, helper.ErrFinishCollect
 					}
-					messages = append(messages, errors.Must1(json.Marshal(node)))
+					messages = append(messages, errors.Must1(json.Marshal(checkRun)))
 				}
 			}
 			return
 		},
 		IgnoreQueryErrors: true,
+		PageSize:          20,
 	})
 	if err != nil {
 		return err

diff --git a/backend/plugins/tapd/tasks/bug_converter.go b/backend/plugins/tapd/tasks/bug_converter.go
@@ -18,6 +18,7 @@ limitations under the License.
 package tasks
 
 import (
+	"math"
 	"reflect"
 	"strconv"
 	"time"
@@ -88,8 +89,12 @@ func ConvertBug(taskCtx plugin.SubTaskContext) errors.Error {
 				results = append(results, issueAssignee)
 			}
 			if domainL.ResolutionDate != nil && domainL.CreatedDate != nil {
-				temp := uint(domainL.ResolutionDate.Sub(*domainL.CreatedDate).Minutes())
-				domainL.LeadTimeMinutes = &temp
+				durationInMinutes := domainL.ResolutionDate.Sub(*domainL.CreatedDate).Minutes()
+				// we have found some issues' ResolutionDate is earlier than CreatedDate in tapd.
+				if durationInMinutes > 0 && durationInMinutes < math.MaxUint {
+					temp := uint(durationInMinutes)
+					domainL.LeadTimeMinutes = &temp
+				}
 			}
 			boardIssue := &ticket.BoardIssue{
 				BoardId: getWorkspaceIdGen().Generate(toolL.ConnectionId, toolL.WorkspaceId),

diff --git a/backend/plugins/zentao/models/connection.go b/backend/plugins/zentao/models/connection.go
@@ -73,6 +73,11 @@ type ZentaoConn struct {
 	DbMaxConns     int    `json:"dbMaxConns" mapstructure:"dbMaxConns"`
 }
 
+func (connection ZentaoConn) GetHash() string {
+	// zentao's token will expire after about 24min, so api client cannot be cached.
+	return ""
+}
+
 func (connection ZentaoConn) Sanitize() ZentaoConn {
 	connection.Password = ""
 	if connection.DbUrl != "" {
@@ -106,6 +111,10 @@ func (connection ZentaoConn) SanitizeDbUrl() string {
 	return dbUrl
 }
 
+func (connection ZentaoConnection) GetHash() string {
+	return connection.ZentaoConn.GetHash()
+}
+
 func (connection ZentaoConnection) Sanitize() ZentaoConnection {
 	connection.ZentaoConn = connection.ZentaoConn.Sanitize()
 	return connection