Merge branch 'apache:main' into feat-opsgenie-plugin

.github/workflows/grafana-dashboards-check.yml

@@ -0,0 +1,37 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: check-grafana-dashboards
+
+on:
+  pull_request:
+    branches:
+      - main
+      - release-*
+
+jobs:
+  check-grafana-dashboards:
+    name: check grafana dashboards
+    runs-on: ubuntu-20.04
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v2
+      - name: Check grafana dashboards whether using mysql uid
+        run: |
+          if grep '"type": "mysql"' grafana/dashboards/*; then
+              echo "There are dashboards which use mysql uid as datasource"
+              exit 1
+          fi

Makefile

@@ -71,25 +71,19 @@ swag:
 build-plugin:
 	make build-plugin -C backend
 
-build-worker:
-	make build-worker -C backend
-
 build-server:
 	make build-server -C backend
 
 build: build-plugin build-server
 
-all: build build-worker
+all: build
 
 tap-models:
 	make tap-models -C backend
 
 run:
 	make run -C backend
 
-worker:
-	make worker -C backend
-
 dev:
 	make dev -C backend
 

backend/Makefile

@@ -57,13 +57,6 @@ build-plugin:
 		PLUGIN=$(PLUGIN) sh scripts/compile-plugins.sh; \
 	fi
 
-build-worker:
-	if [ "$(DEBUG)" = "true" ]; then \
-		go build -gcflags='all=-N -l' -ldflags "-X 'github.com/apache/incubator-devlake/core/version.Version=$(VERSION)'" -o bin/lake-worker ./worker/; \
-	else \
-		go build -ldflags "-X 'github.com/apache/incubator-devlake/core/version.Version=$(VERSION)'" -o bin/lake-worker ./worker/; \
-	fi
-
 build-server: swag
 	if [ "$(DEBUG)" = "true" ]; then \
 		go build -gcflags='all=-N -l' -ldflags "-X 'github.com/apache/incubator-devlake/core/version.Version=$(VERSION)'" -o bin/lake ./server/; \
@@ -78,14 +71,11 @@ build-python: #don't mix this with the other build commands
 
 build: build-plugin build-server
 
-all: build build-worker
+all: build
 
 run:
 	go run server/main.go
 
-worker:
-	go run worker/*.go
-
 dev: build-plugin build-python run
 
 godev:

backend/core/config/config_viper.go

@@ -102,7 +102,6 @@ func getEnvPath() string {
 func setDefaultValue(v *viper.Viper) {
 	v.SetDefault("PORT", "8080")
 	v.SetDefault("PLUGIN_DIR", "bin/plugins")
-	v.SetDefault("TEMPORAL_TASK_QUEUE", "DEVLAKE_TASK_QUEUE")
 	v.SetDefault("REMOTE_PLUGIN_DIR", "python/plugins")
 	v.SetDefault("SWAGGER_DOCS_DIR", "resources/swagger")
 }

backend/core/models/common/base.go

@@ -18,6 +18,7 @@ limitations under the License.
 package common
 
 import (
+	"github.com/apache/incubator-devlake/core/dal"
 	"time"
 )
 
@@ -49,12 +50,12 @@ type Updater struct {
 // embedded fields for tool layer tables
 type RawDataOrigin struct {
 	// can be used for flushing outdated records from table
-	RawDataParams string `gorm:"column:_raw_data_params;type:varchar(255);index" json:"_raw_data_params"`
-	RawDataTable  string `gorm:"column:_raw_data_table;type:varchar(255)" json:"_raw_data_table"`
+	RawDataParams string `gorm:"column:_raw_data_params;type:varchar(255);index" json:"_raw_data_params" mapstructure:"rawDataParams"`
+	RawDataTable  string `gorm:"column:_raw_data_table;type:varchar(255)" json:"_raw_data_table" mapstructure:"rawDataTable"`
 	// can be used for debugging
-	RawDataId uint64 `gorm:"column:_raw_data_id" json:"_raw_data_id"`
+	RawDataId uint64 `gorm:"column:_raw_data_id" json:"_raw_data_id" mapstructure:"rawDataId"`
 	// we can store record index into this field, which is helpful for debugging
-	RawDataRemark string `gorm:"column:_raw_data_remark" json:"_raw_data_remark"`
+	RawDataRemark string `gorm:"column:_raw_data_remark" json:"_raw_data_remark" mapstructure:"rawDataRemark"`
 }
 
 type GetRawDataOrigin interface {
@@ -68,7 +69,7 @@ func (c *RawDataOrigin) GetRawDataOrigin() *RawDataOrigin {
 type NoPKModel struct {
 	CreatedAt     time.Time `json:"createdAt" mapstructure:"createdAt"`
 	UpdatedAt     time.Time `json:"updatedAt" mapstructure:"updatedAt"`
-	RawDataOrigin `swaggerignore:"true"`
+	RawDataOrigin `swaggerignore:"true" mapstructure:",squash"`
 }
 
 func NewNoPKModel() NoPKModel {
@@ -80,7 +81,7 @@ func NewNoPKModel() NoPKModel {
 }
 
 type Scope struct {
-	NoPKModel     `json:"-" mapstructure:"-"`
+	NoPKModel     `mapstructure:",squash"`
 	ConnectionId  uint64 `json:"connectionId" gorm:"primaryKey" validate:"required" mapstructure:"connectionId,omitempty"`
 	ScopeConfigId uint64 `json:"scopeConfigId,omitempty" mapstructure:"scopeConfigId,omitempty"`
 }
@@ -98,7 +99,7 @@ type ScopeConfig struct {
 	Model
 	Entities     []string `gorm:"type:json;serializer:json" json:"entities" mapstructure:"entities"`
 	ConnectionId uint64   `json:"connectionId" gorm:"index" validate:"required" mapstructure:"connectionId,omitempty"`
-	Name         string   `mapstructure:"name" json:"name" gorm:"type:varchar(255)" validate:"required"`
+	//Name         string   `mapstructure:"name" json:"name" gorm:"type:varchar(255)" validate:"required"`
 }
 
 func (s ScopeConfig) ScopeConfigConnectionId() uint64 {
@@ -107,3 +108,8 @@ func (s ScopeConfig) ScopeConfigConnectionId() uint64 {
 func (s ScopeConfig) ScopeConfigId() uint64 {
 	return s.ID
 }
+
+type ScopeConfigOperator[T dal.Tabler] interface {
+	*T
+	SetConnectionId(t *T, connectionId uint64)
+}

backend/core/models/common/string_float64.go

@@ -0,0 +1,85 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package common
+
+import (
+	"database/sql/driver"
+	"encoding/json"
+	"fmt"
+	"github.com/spf13/cast"
+)
+
+type StringFloat64 struct {
+	v float64
+	t string
+}
+
+func (f *StringFloat64) MarshalJSON() ([]byte, error) {
+	return json.Marshal(f.String())
+}
+
+func (f *StringFloat64) String() string {
+	if f.t == "string" {
+		return fmt.Sprintf("\"%v\"", f.v)
+	}
+	return fmt.Sprintf("%v", f.v)
+}
+
+func (f *StringFloat64) UnmarshalJSON(data []byte) error {
+	var i interface{}
+	if err := json.Unmarshal(data, &i); err != nil {
+		return err
+	}
+	switch i.(type) {
+	case float64:
+		f.t = "float64"
+	case string:
+		f.t = "string"
+	}
+	value, err := cast.ToFloat64E(i)
+	if err != nil {
+		return err
+	}
+	f.v = value
+	return nil
+}
+
+func (f *StringFloat64) Value() (driver.Value, error) {
+	if f == nil {
+		return nil, nil
+	}
+	return f.v, nil
+}
+
+func (f *StringFloat64) Scan(v interface{}) error {
+	switch value := v.(type) {
+	case float64:
+		*f = StringFloat64{
+			v: value,
+			t: "float64",
+		}
+	case string:
+		*f = StringFloat64{
+			v: cast.ToFloat64(value),
+			t: "string",
+		}
+	default:
+		return fmt.Errorf("%+v is an unknown type, with value: %v", v, value)
+	}
+	return nil
+}

backend/core/models/domainlayer/didgen/domain_id_generator.go

@@ -19,10 +19,11 @@ package didgen
 
 import (
 	"fmt"
+	"reflect"
+
 	"github.com/apache/incubator-devlake/core/errors"
 	plugin "github.com/apache/incubator-devlake/core/plugin"
 	"github.com/apache/incubator-devlake/impls/dalgorm"
-	"reflect"
 )
 
 type DomainIdGenerator struct {
@@ -65,15 +66,19 @@ func NewDomainIdGenerator(entityPtr interface{}) *DomainIdGenerator {
 }
 
 func (g *DomainIdGenerator) Generate(pkValues ...interface{}) string {
+	if len(pkValues) != len(g.pk) {
+		panic(fmt.Errorf("primary key values do not match, expected %v, got %v", len(g.pk), len(pkValues)))
+	}
 	id := g.prefix
-	for i, pkValue := range pkValues {
+	for i, pkField := range g.pk {
 		// append pk
+		pkValue := pkValues[i]
 		id += ":" + fmt.Sprintf("%v", pkValue)
 		// type checking
 		pkValueType := reflect.TypeOf(pkValue)
 		if pkValueType == wildcardType {
 			break
-		} else if pkValueType != g.pk[i].Type {
+		} else if pkValueType != pkField.Type {
 			panic(errors.Default.New(fmt.Sprintf("primary key type does not match: %s is %s type, and it should be %s type",
 				g.pk[i].Name,
 				pkValueType.Name(),

backend/core/plugin/hub.go

@@ -19,8 +19,10 @@ package plugin
 
 import (
 	"fmt"
-	"github.com/apache/incubator-devlake/core/errors"
 	"strings"
+
+	"github.com/apache/incubator-devlake/core/context"
+	"github.com/apache/incubator-devlake/core/errors"
 )
 
 // Allowing plugin to know each other
@@ -69,3 +71,14 @@ func FindPluginNameBySubPkgPath(subPkgPath string) (string, errors.Error) {
 	}
 	return "", errors.Default.New(fmt.Sprintf("Unable to find plugin for subPkgPath %s", subPkgPath))
 }
+
+func InitPlugins(basicRes context.BasicRes) {
+	for pluginName, pluginMeta := range plugins {
+		if pluginEntry, ok := pluginMeta.(PluginInit); ok {
+			err := pluginEntry.Init(basicRes)
+			if err != nil {
+				panic(fmt.Errorf("failed to initialize plugin %v due to %w", pluginName, err))
+			}
+		}
+	}
+}

backend/core/runner/loader.go

@@ -68,12 +68,6 @@ func LoadGoPlugins(basicRes context.BasicRes) errors.Error {
 			if !ok {
 				return errors.Default.New(fmt.Sprintf("%s PluginEntry must implement PluginMeta interface", pluginName))
 			}
-			if pluginEntry, ok := symPluginEntry.(plugin.PluginInit); ok {
-				err = pluginEntry.Init(basicRes)
-				if err != nil {
-					return err
-				}
-			}
 			err = plugin.RegisterPlugin(pluginName, pluginMeta)
 			if err != nil {
 				return err