webhooks: only pull project for which request was received (#793)

If a webhook request's project matches a wildcard, pull only that
project instead of pulling the entire wildcard (which can be expensive
for wildcards matching many projects).

Author: sysedwinistrator

pkg/controller/controller.go

@@ -77,6 +77,7 @@ func (c *Controller) registerTasks() {
 		schemas.TaskTypePullEnvironmentsFromProject:  c.TaskHandlerPullEnvironmentsFromProject,
 		schemas.TaskTypePullEnvironmentsFromProjects: c.TaskHandlerPullEnvironmentsFromProjects,
 		schemas.TaskTypePullMetrics:                  c.TaskHandlerPullMetrics,
+		schemas.TaskTypePullProject:                  c.TaskHandlerPullProject,
 		schemas.TaskTypePullProjectsFromWildcard:     c.TaskHandlerPullProjectsFromWildcard,
 		schemas.TaskTypePullProjectsFromWildcards:    c.TaskHandlerPullProjectsFromWildcards,
 		schemas.TaskTypePullRefMetrics:               c.TaskHandlerPullRefMetrics,

pkg/controller/projects.go

@@ -9,6 +9,37 @@ import (
 	"github.com/mvisonneau/gitlab-ci-pipelines-exporter/pkg/schemas"
 )
 
+// PullProject ..
+func (c *Controller) PullProject(ctx context.Context, name string) error {
+	gp, err := c.Gitlab.GetProject(ctx, name)
+	if err != nil {
+		return err
+	}
+	p := schemas.NewProject(gp.PathWithNamespace)
+
+	projectExists, err := c.Store.ProjectExists(ctx, p.Key())
+	if err != nil {
+		return err
+	}
+
+	if !projectExists {
+		log.WithFields(log.Fields{
+			"project-name": p.Name,
+		}).Info("discovered new project")
+
+		if err := c.Store.SetProject(ctx, p); err != nil {
+			log.WithContext(ctx).
+				WithError(err).
+				Error()
+		}
+
+		c.ScheduleTask(ctx, schemas.TaskTypePullRefsFromProject, string(p.Key()), p)
+		c.ScheduleTask(ctx, schemas.TaskTypePullEnvironmentsFromProject, string(p.Key()), p)
+	}
+
+	return nil
+}
+
 // PullProjectsFromWildcard ..
 func (c *Controller) PullProjectsFromWildcard(ctx context.Context, w config.Wildcard) error {
 	foundProjects, err := c.Gitlab.ListProjects(ctx, w)

pkg/controller/scheduler.go

@@ -71,6 +71,13 @@ func NewTaskController(ctx context.Context, r *redis.Client, maximumJobsQueueSiz
 	return
 }
 
+// TaskHandlerPullProject ..
+func (c *Controller) TaskHandlerPullProject(ctx context.Context, name string) error {
+	defer c.unqueueTask(ctx, schemas.TaskTypePullProject, name)
+
+	return c.PullProject(ctx, name)
+}
+
 // TaskHandlerPullProjectsFromWildcard ..
 func (c *Controller) TaskHandlerPullProjectsFromWildcard(ctx context.Context, id string, w config.Wildcard) error {
 	defer c.unqueueTask(ctx, schemas.TaskTypePullProjectsFromWildcard, id)

pkg/controller/webhooks.go

@@ -155,9 +155,9 @@ func (c *Controller) triggerRefMetricsPull(ctx context.Context, ref schemas.Ref)
 
 		// Perhaps the project is discoverable through a wildcard
 		if !projectExists && len(c.Config.Wildcards) > 0 {
-			for id, w := range c.Config.Wildcards {
+			for _, w := range c.Config.Wildcards {
 				// If in all our wildcards we have one which can potentially match the project ref
-				// received, we trigger a scan
+				// received, we trigger a pull of the project
 				matches, err := isRefMatchingWilcard(w, ref)
 				if err != nil {
 					log.WithContext(ctx).
@@ -168,8 +168,8 @@ func (c *Controller) triggerRefMetricsPull(ctx context.Context, ref schemas.Ref)
 				}
 
 				if matches {
-					c.ScheduleTask(context.TODO(), schemas.TaskTypePullProjectsFromWildcard, strconv.Itoa(id), strconv.Itoa(id), w)
-					log.WithFields(logFields).Info("project ref not currently exported but its configuration matches a wildcard, triggering a pull of the projects from this wildcard")
+					c.ScheduleTask(context.TODO(), schemas.TaskTypePullProject, ref.Project.Name)
+					log.WithFields(logFields).Info("project ref not currently exported but its configuration matches a wildcard, triggering a pull of the project")
 				} else {
 					log.WithFields(logFields).Debug("project ref not matching wildcard, skipping..")
 				}
@@ -269,9 +269,9 @@ func (c *Controller) triggerEnvironmentMetricsPull(ctx context.Context, env sche
 
 		// Perhaps the project is discoverable through a wildcard
 		if !projectExists && len(c.Config.Wildcards) > 0 {
-			for id, w := range c.Config.Wildcards {
+			for _, w := range c.Config.Wildcards {
 				// If in all our wildcards we have one which can potentially match the env
-				// received, we trigger a scan
+				// received, we trigger a pull of the project
 				matches, err := isEnvMatchingWilcard(w, env)
 				if err != nil {
 					log.WithContext(ctx).
@@ -282,8 +282,8 @@ func (c *Controller) triggerEnvironmentMetricsPull(ctx context.Context, env sche
 				}
 
 				if matches {
-					c.ScheduleTask(ctx, schemas.TaskTypePullProjectsFromWildcard, strconv.Itoa(id), strconv.Itoa(id), w)
-					log.WithFields(logFields).Info("project environment not currently exported but its configuration matches a wildcard, triggering a pull of the projects from this wildcard")
+					c.ScheduleTask(context.TODO(), schemas.TaskTypePullProject, env.ProjectName)
+					log.WithFields(logFields).Info("project environment not currently exported but its configuration matches a wildcard, triggering a pull of the project")
 				} else {
 					log.WithFields(logFields).Debug("project ref not matching wildcard, skipping..")
 				}

pkg/schemas/tasks.go

@@ -4,6 +4,9 @@ package schemas
 type TaskType string
 
 const (
+	// TaskTypePullProject ..
+	TaskTypePullProject TaskType = "PullProject"
+
 	// TaskTypePullProjectsFromWildcard ..
 	TaskTypePullProjectsFromWildcard TaskType = "PullProjectsFromWildcard"