Merge branch 'main' into feat#6101

.github/workflows/grafana-dashboards-check.yml

@@ -0,0 +1,37 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: check-grafana-dashboards
+
+on:
+  pull_request:
+    branches:
+      - main
+      - release-*
+
+jobs:
+  check-grafana-dashboards:
+    name: check grafana dashboards
+    runs-on: ubuntu-20.04
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v2
+      - name: Check grafana dashboards whether using mysql uid
+        run: |
+          if grep '"type": "mysql"' grafana/dashboards/*; then
+              echo "There are dashboards which use mysql uid as datasource"
+              exit 1
+          fi

backend/helpers/pluginhelper/api/api_collector_with_state.go

@@ -113,7 +113,7 @@ func NewStatefulApiCollector(args RawDataSubTaskArgs) (*ApiCollectorStateManager
 // InitCollector init the embedded collector
 func (m *ApiCollectorStateManager) InitCollector(args ApiCollectorArgs) errors.Error {
 	args.RawDataSubTaskArgs = m.RawDataSubTaskArgs
-	args.Incremental = m.IsIncreamtal
+	args.Incremental = args.Incremental || m.IsIncreamtal
 	apiCollector, err := NewApiCollector(args)
 	if err != nil {
 		return err

backend/helpers/pluginhelper/api/scope_generic_helper.go

@@ -18,7 +18,6 @@ limitations under the License.
 package api
 
 import (
-	"encoding/json"
 	"fmt"
 	"reflect"
 	"strconv"
@@ -36,7 +35,6 @@ import (
 	"github.com/apache/incubator-devlake/helpers/dbhelper"
 	serviceHelper "github.com/apache/incubator-devlake/helpers/pluginhelper/services"
 	"github.com/go-playground/validator/v10"
-	"github.com/mitchellh/mapstructure"
 )
 
 type NoScopeConfig struct{}
@@ -64,8 +62,9 @@ type (
 	// Alias, for swagger purposes
 	ScopeRefDoc                                            = serviceHelper.BlueprintProjectPairs
 	ScopeRes[Scope plugin.ToolLayerScope, ScopeConfig any] struct {
-		Scope                    Scope                    `mapstructure:",squash"` // ideally we need this field to be embedded in the struct
-		ScopeResDoc[ScopeConfig] `mapstructure:",squash"` // however, only this type of embeding is supported as of golang 1.20
+		Scope       Scope               `mapstructure:"scope,omitempty" json:"scope,omitempty"`
+		ScopeConfig *ScopeConfig        `mapstructure:"scopeConfig,omitempty" json:"scopeConfig,omitempty"`
+		Blueprints  []*models.Blueprint `mapstructure:"blueprints,omitempty" json:"blueprints,omitempty"`
 	}
 	ScopeListRes[Scope plugin.ToolLayerScope, ScopeConfig any] struct {
 		Scopes []*ScopeRes[Scope, ScopeConfig] `mapstructure:"scopes" json:"scopes"`
@@ -601,22 +600,6 @@ func (gs *GenericScopeApiHelper[Conn, Scope, ScopeConfig]) transactionalDelete(t
 	return nil
 }
 
-// Implement MarshalJSON method to flatten all fields
-func (sr *ScopeRes[T, Y]) MarshalJSON() ([]byte, error) {
-	var flatMap map[string]interface{}
-	err := mapstructure.Decode(sr, &flatMap)
-	if err != nil {
-		return nil, err
-	}
-	// Encode the flattened map to JSON
-	result, err := json.Marshal(flatMap)
-	if err != nil {
-		return nil, err
-	}
-
-	return result, nil
-}
-
 func (gs *GenericScopeApiHelper[Conn, Scope, ScopeConfig]) getAffectedTables(pluginName string) ([]string, errors.Error) {
 	var tables []string
 	meta, err := plugin.GetPlugin(pluginName)

backend/plugins/github/tasks/cicd_job_convertor.go

@@ -99,7 +99,7 @@ func ConvertJobs(taskCtx plugin.SubTaskContext) (err errors.Error) {
 				Result: devops.GetResult(&devops.ResultRule{
 					Failed:  []string{"failure"},
 					Success: []string{"success"},
-					Skipped: []string{"skipped"},
+					Skipped: []string{"skipped", "NEUTRAL"},
 				}, line.Conclusion),
 				Status: devops.GetStatus(&devops.StatusRule[string]{
 					Done:    []string{"completed", "COMPLETED"},

backend/plugins/github/tasks/cicd_run_collector.go

@@ -20,18 +20,14 @@ package tasks
 import (
 	"encoding/json"
 	"fmt"
-	"io"
 	"net/http"
 	"net/url"
-	"reflect"
 	"time"
 
-	"github.com/apache/incubator-devlake/core/dal"
 	"github.com/apache/incubator-devlake/core/errors"
 	"github.com/apache/incubator-devlake/core/models/common"
 	"github.com/apache/incubator-devlake/core/plugin"
 	helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
-	"github.com/apache/incubator-devlake/plugins/github/models"
 )
 
 func init() {
@@ -67,7 +63,6 @@ var CollectRunsMeta = plugin.SubTaskMeta{
 
 func CollectRuns(taskCtx plugin.SubTaskContext) errors.Error {
 	data := taskCtx.GetData().(*GithubTaskData)
-	db := taskCtx.GetDal()
 	collector, err := helper.NewStatefulApiCollectorForFinalizableEntity(helper.FinalizableApiCollectorArgs{
 		RawDataSubTaskArgs: helper.RawDataSubTaskArgs{
 			Ctx: taskCtx,
@@ -85,6 +80,7 @@ func CollectRuns(taskCtx plugin.SubTaskContext) errors.Error {
 				UrlTemplate: "repos/{{ .Params.Name }}/actions/runs",
 				Query: func(reqData *helper.RequestData, createdAfter *time.Time) (url.Values, errors.Error) {
 					query := url.Values{}
+					query.Set("status", "completed")
 					query.Set("page", fmt.Sprintf("%v", reqData.Pager.Page))
 					query.Set("per_page", fmt.Sprintf("%v", reqData.Pager.Size))
 					return query, nil
@@ -110,36 +106,6 @@ func CollectRuns(taskCtx plugin.SubTaskContext) errors.Error {
 				return pj.CreatedAt.ToTime(), nil
 			},
 		},
-		CollectUnfinishedDetails: &helper.FinalizableApiCollectorDetailArgs{
-			BuildInputIterator: func() (helper.Iterator, errors.Error) {
-				// load unfinished runs from the database
-				cursor, err := db.Cursor(
-					dal.Select("id"),
-					dal.From(&models.GithubRun{}),
-					dal.Where(
-						"repo_id = ? AND connection_id = ? AND status IN ('ACTION_REQUIRED', 'STALE', 'IN_PROGRESS', 'QUEUED', 'REQUESTED', 'WAITING', 'PENDING')",
-						data.Options.GithubId, data.Options.ConnectionId,
-					),
-				)
-				if err != nil {
-					return nil, err
-				}
-				return helper.NewDalCursorIterator(db, cursor, reflect.TypeOf(SimpleGithubApiJob{}))
-			},
-
-			FinalizableApiCollectorCommonArgs: helper.FinalizableApiCollectorCommonArgs{
-				UrlTemplate: "repos/{{ .Params.Name }}/actions/runs/{{ .Input.ID }}",
-				ResponseParser: func(res *http.Response) ([]json.RawMessage, errors.Error) {
-					body, err := io.ReadAll(res.Body)
-					if err != nil {
-						return nil, errors.Convert(err)
-					}
-					res.Body.Close()
-					return []json.RawMessage{body}, nil
-				},
-				AfterResponse: ignoreHTTPStatus404,
-			},
-		},
 	})
 
 	if err != nil {

backend/plugins/jenkins/tasks/build_collector.go

@@ -20,17 +20,13 @@ package tasks
 import (
 	"encoding/json"
 	"fmt"
-	"io"
 	"net/http"
 	"net/url"
-	"reflect"
 	"time"
 
-	"github.com/apache/incubator-devlake/core/dal"
 	"github.com/apache/incubator-devlake/core/errors"
 	"github.com/apache/incubator-devlake/core/plugin"
 	helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
-	"github.com/apache/incubator-devlake/plugins/jenkins/models"
 )
 
 const RAW_BUILD_TABLE = "jenkins_api_builds"
@@ -55,7 +51,6 @@ type SimpleJenkinsApiBuild struct {
 
 func CollectApiBuilds(taskCtx plugin.SubTaskContext) errors.Error {
 	data := taskCtx.GetData().(*JenkinsTaskData)
-	db := taskCtx.GetDal()
 	collector, err := helper.NewStatefulApiCollectorForFinalizableEntity(helper.FinalizableApiCollectorArgs{
 		RawDataSubTaskArgs: helper.RawDataSubTaskArgs{
 			Params: JenkinsApiParams{
@@ -87,7 +82,20 @@ func CollectApiBuilds(taskCtx plugin.SubTaskContext) errors.Error {
 					if err != nil {
 						return nil, err
 					}
-					return data.Builds, nil
+
+					builds := make([]json.RawMessage, 0, len(data.Builds))
+					for _, build := range data.Builds {
+						var buildObj map[string]interface{}
+						err := json.Unmarshal(build, &buildObj)
+						if err != nil {
+							return nil, errors.Convert(err)
+						}
+						if buildObj["result"] != nil {
+							builds = append(builds, build)
+						}
+					}
+
+					return builds, nil
 				},
 			},
 			GetCreated: func(item json.RawMessage) (time.Time, errors.Error) {
@@ -101,34 +109,6 @@ func CollectApiBuilds(taskCtx plugin.SubTaskContext) errors.Error {
 				return time.Unix(seconds, nanos), nil
 			},
 		},
-		CollectUnfinishedDetails: &helper.FinalizableApiCollectorDetailArgs{
-			BuildInputIterator: func() (helper.Iterator, errors.Error) {
-				cursor, err := db.Cursor(
-					dal.Select("number"),
-					dal.From(&models.JenkinsBuild{}),
-					dal.Where(
-						"full_name = ? AND connection_id = ? AND result != 'SUCCESS' AND result != 'FAILURE'",
-						data.Options.JobFullName, data.Options.ConnectionId,
-					),
-				)
-				if err != nil {
-					return nil, err
-				}
-				return helper.NewDalCursorIterator(db, cursor, reflect.TypeOf(SimpleJenkinsApiBuild{}))
-			},
-			FinalizableApiCollectorCommonArgs: helper.FinalizableApiCollectorCommonArgs{
-				UrlTemplate: fmt.Sprintf("%sjob/%s/{{ .Input.Number }}/api/json?tree=number,url,result,timestamp,id,duration,estimatedDuration,building",
-					data.Options.JobPath, data.Options.JobName),
-				ResponseParser: func(res *http.Response) ([]json.RawMessage, errors.Error) {
-					body, err := io.ReadAll(res.Body)
-					if err != nil {
-						return nil, errors.Convert(err)
-					}
-					res.Body.Close()
-					return []json.RawMessage{body}, nil
-				},
-			},
-		},
 	})
 
 	if err != nil {

backend/plugins/pagerduty/api/remote.go

@@ -39,6 +39,7 @@ type RemoteScopesChild struct {
 	Id       string      `json:"id"`
 	Name     string      `json:"name"`
 	Data     interface{} `json:"data"`
+	FullName string      `json:"fullName"` // temporary fix, to adopt remote scope helper
 }
 
 type RemoteScopesOutput struct {
@@ -127,9 +128,10 @@ func RemoteScopes(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, er
 	// append service to output
 	for _, service := range response.Services {
 		child := RemoteScopesChild{
-			Type: TypeScope,
-			Id:   service.Id,
-			Name: service.Name,
+			Type:     TypeScope,
+			Id:       service.Id,
+			Name:     service.Name,
+			FullName: service.Name,
 			Data: models.Service{
 				Url:  service.HtmlUrl,
 				Id:   service.Id,

backend/server/services/remote/plugin/scope_api.go

@@ -115,7 +115,7 @@ func convertScopeResponse(scopes ...*api.ScopeRes[models.DynamicScopeModel, mode
 	responses := make([]map[string]any, len(scopes))
 	for i, scope := range scopes {
 		resMap := map[string]any{}
-		err := models.MapTo(scope.ScopeResDoc, &resMap)
+		err := models.MapTo(scope, &resMap)
 		if err != nil {
 			return nil, err
 		}

config-ui/nginx.conf

@@ -2,6 +2,7 @@ server {
   listen 4000;
   server_name localhost;
   absolute_redirect off;
+  client_max_body_size: 20m;
 
 ${SERVER_CONF}
 

config-ui/src/routes/api-keys/api.ts → config-ui/src/api/api-key/index.ts

@@ -17,15 +17,14 @@
  */
 
 import { request } from '@/utils';
-
 import * as T from './types';
 
-export const getApiKeys = (params?: Pagination): Promise<{ count: number; apikeys: T.Key[] }> =>
+export const list = (data?: Pagination): Promise<{ count: number; apikeys: T.Key[] }> =>
   request('/api-keys', {
-    data: params,
+    data,
   });
 
-export const createApiKey = (data: Pick<T.Key, 'name' | 'expiredAt' | 'allowedPath'>): Promise<T.Key> =>
+export const create = (data: Pick<T.Key, 'name' | 'expiredAt' | 'allowedPath'>): Promise<T.Key> =>
   request('/api-keys', {
     method: 'POST',
     data: {
@@ -34,7 +33,9 @@ export const createApiKey = (data: Pick<T.Key, 'name' | 'expiredAt' | 'allowedPa
     },
   });
 
-export const deleteApiKey = (id: string): Promise<void> =>
+export const remove = (id: string): Promise<void> =>
   request(`/api-keys/${id}`, {
     method: 'DELETE',
   });
+
+export const renew = (id: ID) => request(`/api-keys/${id}`, { method: 'put' });

config-ui/src/routes/error/components/needs-db-migrate/api.ts → config-ui/src/api/api-key/types.ts

@@ -16,6 +16,9 @@
  *
  */
 
-import { request } from '@/utils';
-
-export const migrate = () => request('/proceed-db-migration');
+export type Key = {
+  name: string;
+  expiredAt?: string;
+  allowedPath: string;
+  creator: string;
+};

config-ui/src/api/blueprint/index.ts

@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+import { request } from '@/utils';
+
+import * as T from './types';
+
+export const list = (data: Pagination & { type: string }): Promise<{ count: number; blueprints: T.Blueprint[] }> =>
+  request('/blueprints', { data });
+
+export const get = (id: ID): Promise<T.Blueprint> => request(`/blueprints/${id}`);
+
+export const create = (data: any) =>
+  request('/blueprints', {
+    method: 'post',
+    data,
+  });
+
+export const remove = (id: ID) => request(`/blueprints/${id}`, { method: 'delete' });
+
+export const update = (id: ID, data: T.Blueprint) => request(`/blueprints/${id}`, { method: 'patch', data });
+
+export const pipelines = (id: ID) => request(`/blueprints/${id}/pipelines`);
+
+export const trigger = (id: ID, data: T.TriggerQuery) => request(`/blueprints/${id}/trigger`, { method: 'post', data });