Merge branch 'main' into feat-7243

backend/core/models/domainlayer/devops/cicd_deployment_commit.go

@@ -51,6 +51,10 @@ func (cicdDeploymentCommit CicdDeploymentCommit) TableName() string {
 }
 
 func (cicdDeploymentCommit CicdDeploymentCommit) ToDeployment() *CICDDeployment {
+	return cicdDeploymentCommit.ToDeploymentWithCustomDisplayTitle(cicdDeploymentCommit.DisplayTitle)
+}
+
+func (cicdDeploymentCommit CicdDeploymentCommit) ToDeploymentWithCustomDisplayTitle(displayTitle string) *CICDDeployment {
 	return &CICDDeployment{
 		DomainEntity: domainlayer.DomainEntity{
 			Id:        cicdDeploymentCommit.CicdDeploymentId,
@@ -72,7 +76,7 @@ func (cicdDeploymentCommit CicdDeploymentCommit) ToDeployment() *CICDDeployment
 		},
 		QueuedDurationSec: cicdDeploymentCommit.QueuedDurationSec,
 		DurationSec:       cicdDeploymentCommit.DurationSec,
-		DisplayTitle:      cicdDeploymentCommit.DisplayTitle,
+		DisplayTitle:      displayTitle,
 		Url:               cicdDeploymentCommit.Url,
 	}
 }

backend/core/models/domainlayer/devops/cicd_release.go

@@ -0,0 +1,53 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package devops
+
+import (
+	"github.com/apache/incubator-devlake/core/models/domainlayer"
+	"time"
+)
+
+type CicdRelease struct {
+	domainlayer.DomainEntity
+	PublishedAt time.Time `json:"publishedAt"`
+
+	CicdScopeId string `gorm:"index;type:varchar(255)"`
+
+	Name         string `gorm:"type:varchar(255)"`
+	DisplayTitle string `gorm:"type:varchar(255)"`
+	Description  string `json:"description"`
+	URL          string `json:"url"`
+
+	IsDraft      bool `json:"isDraft"`
+	IsLatest     bool `json:"isLatest"`
+	IsPrerelease bool `json:"isPrerelease"`
+
+	AuthorID string `json:"id" gorm:"type:varchar(255)"`
+
+	RepoId string `gorm:"type:varchar(255)"`
+	//RepoUrl string `gorm:"index;not null"`
+
+	TagName   string `json:"tagName"`
+	CommitSha string `gorm:"uniqueIndex;type:varchar(255)"`
+	//CommitMsg string
+	//RefName   string `gorm:"type:varchar(255)"`
+}
+
+func (CicdRelease) TableName() string {
+	return "cicd_releases"
+}

backend/core/models/domainlayer/domaininfo/domaininfo.go

@@ -73,6 +73,7 @@ func GetDomainTablesInfo() []dal.Tabler {
 		&devops.CiCDPipelineCommit{},
 		&devops.CicdScope{},
 		&devops.CICDDeployment{},
+		&devops.CicdRelease{},
 		// didgen no table
 		// ticket
 		&ticket.Board{},

backend/core/models/migrationscripts/20240514_add_cicd_release.go

@@ -0,0 +1,69 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package migrationscripts
+
+import (
+	"github.com/apache/incubator-devlake/core/models/migrationscripts/archived"
+	"time"
+
+	"github.com/apache/incubator-devlake/core/context"
+	"github.com/apache/incubator-devlake/core/errors"
+	"github.com/apache/incubator-devlake/core/plugin"
+)
+
+var _ plugin.MigrationScript = (*addCicdRelease)(nil)
+
+type cicdRelease20240514 struct {
+	archived.DomainEntity
+	PublishedAt time.Time `json:"publishedAt"`
+
+	CicdScopeId string `gorm:"index;type:varchar(255)"`
+
+	Name         string `gorm:"type:varchar(255)"`
+	DisplayTitle string `gorm:"type:varchar(255)"`
+	Description  string `json:"description"`
+	URL          string `json:"url"`
+
+	IsDraft      bool `json:"isDraft"`
+	IsLatest     bool `json:"isLatest"`
+	IsPrerelease bool `json:"isPrerelease"`
+
+	AuthorID string `json:"id" gorm:"type:varchar(255)"`
+
+	RepoId string `gorm:"type:varchar(255)"`
+
+	TagName string `json:"tagName"`
+}
+
+func (cicdRelease20240514) TableName() string {
+	return "cicd_releases"
+}
+
+type addCicdRelease struct{}
+
+func (*addCicdRelease) Up(basicRes context.BasicRes) errors.Error {
+	return basicRes.GetDal().AutoMigrate(cicdRelease20240514{})
+}
+
+func (*addCicdRelease) Version() uint64 {
+	return 20240514181200
+}
+
+func (*addCicdRelease) Name() string {
+	return "add cicd_releases table"
+}

backend/core/models/migrationscripts/20240515_add_commit_sha_to_cicd_release.go

@@ -0,0 +1,49 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package migrationscripts
+
+import (
+	"github.com/apache/incubator-devlake/core/context"
+	"github.com/apache/incubator-devlake/core/errors"
+	"github.com/apache/incubator-devlake/core/plugin"
+	"github.com/apache/incubator-devlake/helpers/migrationhelper"
+)
+
+var _ plugin.MigrationScript = (*addCommitShaToCicdRelease)(nil)
+
+type cicdRelease20240515 struct {
+	CommitSha string
+}
+
+func (cicdRelease20240515) TableName() string {
+	return "cicd_releases"
+}
+
+type addCommitShaToCicdRelease struct{}
+
+func (*addCommitShaToCicdRelease) Up(basicRes context.BasicRes) errors.Error {
+	return migrationhelper.AutoMigrateTables(basicRes, &cicdRelease20240515{})
+}
+
+func (*addCommitShaToCicdRelease) Version() uint64 {
+	return 20240515194901
+}
+
+func (*addCommitShaToCicdRelease) Name() string {
+	return "add commit_sha to cicd_releases"
+}

backend/core/models/migrationscripts/register.go

@@ -113,5 +113,7 @@ func All() []plugin.MigrationScript {
 		new(addSubtaskField),
 		new(addDisplayTitleAndUrl),
 		new(addSubtaskStates),
+		new(addCicdRelease),
+		new(addCommitShaToCicdRelease),
 	}
 }

backend/core/models/project.go

@@ -62,6 +62,7 @@ type ApiInputProject struct {
 	BaseProject `mapstructure:",squash"`
 	Enable      *bool         `json:"enable" mapstructure:"enable"`
 	Metrics     []*BaseMetric `json:"metrics" mapstructure:"metrics"`
+	Blueprint   *Blueprint    `json:"blueprint" mapstructure:"blueprint"`
 }
 
 type ApiOutputProject struct {

backend/helpers/e2ehelper/data_flow_tester.go

@@ -227,7 +227,10 @@ func (t *DataFlowTester) Subtask(subtaskMeta plugin.SubTaskMeta, taskData interf
 
 // SubtaskContext creates a subtask context
 func (t *DataFlowTester) SubtaskContext(taskData interface{}) plugin.SubTaskContext {
-	return contextimpl.NewStandaloneSubTaskContext(context.Background(), runner.CreateBasicRes(t.Cfg, t.Log, t.Db), t.Name, taskData, t.Name)
+	syncPolicy := &models.SyncPolicy{
+		FullSync: true,
+	}
+	return contextimpl.NewStandaloneSubTaskContext(context.Background(), runner.CreateBasicRes(t.Cfg, t.Log, t.Db), t.Name, taskData, t.Name, syncPolicy)
 }
 
 func filterColumn(column dal.ColumnMeta, opts TableOptions) bool {

backend/helpers/pluginhelper/api/api_client_test.go

@@ -33,7 +33,7 @@ func TestApiClientBlackList(t *testing.T) {
 		Err       errors.Error
 	}{
 		{
-			Name:    "Internal IP Addresses",
+			Name:    "Blocked Internal IP Addresses",
 			Pattern: "10.0.0.1/16",
 			Endpoints: []string{
 				"https://10.0.0.1",
@@ -44,7 +44,7 @@ func TestApiClientBlackList(t *testing.T) {
 			Err: ErrHostNotAllowed,
 		},
 		{
-			Name:    "Internal IP Addresses",
+			Name:    "Allowed Public IP Addresses",
 			Pattern: "10.0.0.1/16",
 			Endpoints: []string{
 				"http://10.1.0.1",
@@ -54,6 +54,38 @@ func TestApiClientBlackList(t *testing.T) {
 			},
 			Err: nil,
 		},
+		{
+			Name:    "Blocked IPv6 loopback",
+			Pattern: "::1/128",
+			Endpoints: []string{
+				"http://[::1]",
+			},
+			Err: ErrHostNotAllowed,
+		},
+		{
+			Name:    "Blocked IPv6 Unique Local Addresses IP Addresses",
+			Pattern: "fc00::/7",
+			Endpoints: []string{
+				"http://[fdf8:f53b:82e4::53]",
+			},
+			Err: ErrHostNotAllowed,
+		},
+		{
+			Name:    "Blocked IPv6 Link-Local IP Addresses",
+			Pattern: "fe80::/10",
+			Endpoints: []string{
+				"http://[fe80::200:5aee:feaa:20a2]",
+			},
+			Err: ErrHostNotAllowed,
+		},
+		{
+			Name:    "Allowed IPv6 Public IP Addresses",
+			Pattern: "fe80::/10",
+			Endpoints: []string{
+				"http://[2001:0002:6c::430]",
+			},
+			Err: nil,
+		},
 	} {
 		t.Run(tc.Name, func(t *testing.T) {
 			for _, endpoint := range tc.Endpoints {

backend/helpers/srvhelper/model_service_helper.go

@@ -103,7 +103,7 @@ func (srv *ModelSrvHelper[M]) Create(model *M) errors.Error {
 	err = srv.db.Create(model)
 	if err != nil {
 		if srv.db.IsDuplicationError(err) {
-			return errors.Conflict.Wrap(err, fmt.Sprintf("%s already exists", srv.modelName))
+			return errors.Default.New("The name of the current scope config is duplicated. Please modify it before saving.")
 		}
 		return err
 	}
@@ -115,7 +115,7 @@ func (srv *ModelSrvHelper[M]) Update(model *M) errors.Error {
 	err := srv.ValidateModel(model)
 	if err != nil {
 		if srv.db.IsDuplicationError(err) {
-			return errors.Conflict.Wrap(err, fmt.Sprintf("%s already exists", srv.modelName))
+			return errors.Default.New("The name of the current scope config is duplicated. Please modify it before saving.")
 		}
 		return err
 	}

backend/impls/context/default_subtask_context.go

@@ -22,6 +22,7 @@ import (
 	"time"
 
 	"github.com/apache/incubator-devlake/core/context"
+	"github.com/apache/incubator-devlake/core/models"
 	"github.com/apache/incubator-devlake/core/plugin"
 )
 
@@ -69,10 +70,13 @@ func NewStandaloneSubTaskContext(
 	name string,
 	data interface{},
 	pluginName string,
+	syncPolicy *models.SyncPolicy,
 ) plugin.SubTaskContext {
+	taskContext := &DefaultTaskContext{defaultExecContext: newDefaultExecContext(ctx, basicRes, pluginName, data, nil)}
+	taskContext.SetSyncPolicy(syncPolicy)
 	return &DefaultSubTaskContext{
 		newDefaultExecContext(ctx, basicRes, name, data, nil),
-		&DefaultTaskContext{defaultExecContext: newDefaultExecContext(ctx, basicRes, pluginName, data, nil)},
+		taskContext,
 		time.Time{},
 	}
 }

backend/plugins/gitextractor/parser/clone_gitcli.go

@@ -34,8 +34,7 @@ import (
 )
 
 var _ RepoCloner = (*GitcliCloner)(nil)
-var ErrShallowInfoProcessing = errors.BadInput.New("No data found for the selected time range. Please revise the 'Time Range' on your Project/Blueprint/Configuration page or in the API parameter.")
-var ErrNoDataOnIncrementalMode = errors.NotModified.New("No data found since the previous run.")
+var ErrNoData = errors.NotModified.New("No data to be collected")
 
 type GitcliCloner struct {
 	logger       log.Logger
@@ -84,10 +83,6 @@ func (g *GitcliCloner) CloneRepo(ctx plugin.SubTaskContext, localDir string) err
 	}
 	err = g.execCloneCommand(cmd)
 	if err != nil {
-		// it is likely that nothing to collect on incrmental mode
-		if errors.Is(err, ErrShallowInfoProcessing) && g.stateManager != nil && g.stateManager.IsIncremental() {
-			return ErrNoDataOnIncrementalMode
-		}
 		return err
 	}
 	// deepen the commits by 1 more step to avoid https://github.com/apache/incubator-devlake/issues/7426
@@ -222,7 +217,7 @@ func (g *GitcliCloner) execCloneCommand(cmd *exec.Cmd) errors.Error {
 		g.logger.Error(err, "git exited with error\n%s", combinedOutput.String())
 		if strings.Contains(combinedOutput.String(), "stderr: fatal: error processing shallow info: 4") ||
 			strings.Contains(combinedOutput.String(), "stderr: fatal: the remote end hung up unexpectedly") {
-			return ErrShallowInfoProcessing
+			return ErrNoData
 		}
 		return errors.Default.New("git exit error")
 	}

backend/plugins/gitextractor/tasks/repo_cloner.go

@@ -56,13 +56,10 @@ func CloneGitRepo(subTaskCtx plugin.SubTaskContext) errors.Error {
 	repoCloner := parser.NewGitcliCloner(subTaskCtx)
 	err = repoCloner.CloneRepo(subTaskCtx, localDir)
 	if err != nil {
-		if errors.Is(err, parser.ErrNoDataOnIncrementalMode) {
+		if errors.Is(err, parser.ErrNoData) {
 			taskData.SkipAllSubtasks = true
 			return nil
 		}
-		if errors.Is(err, parser.ErrShallowInfoProcessing) {
-			return nil
-		}
 		return err
 	}
 

backend/plugins/github/impl/impl.go

@@ -104,6 +104,7 @@ func (p Github) GetTablesInfo() []dal.Tabler {
 		&models.GithubIssueAssignee{},
 		&models.GithubScopeConfig{},
 		&models.GithubDeployment{},
+		&models.GithubRelease{},
 	}
 }