From 85d8635dbed855b48a7cb16659728906022c2d16 Mon Sep 17 00:00:00 2001 From: Juan Antonio Osorio Date: Thu, 21 Nov 2024 14:21:13 +0200 Subject: [PATCH] Kick off data sources service This service will contain all the business logic to deal with data sources. Signed-off-by: Juan Antonio Osorio --- database/mock/store.go | 150 +++++++++ database/query/datasources.sql | 73 ++++ internal/datasources/factory.go | 5 + internal/datasources/service/helpers.go | 67 ++++ internal/datasources/service/mock/service.go | 163 +++++++++ internal/datasources/service/options.go | 73 ++++ internal/datasources/service/service.go | 253 ++++++++++++++ internal/datasources/service/service_test.go | 287 ++++++++++++++++ internal/datasources/service/tx.go | 81 +++++ internal/db/datasources.sql.go | 329 +++++++++++++++++++ internal/db/querier.go | 29 ++ 11 files changed, 1510 insertions(+) create mode 100644 database/query/datasources.sql create mode 100644 internal/datasources/service/helpers.go create mode 100644 internal/datasources/service/mock/service.go create mode 100644 internal/datasources/service/options.go create mode 100644 internal/datasources/service/service.go create mode 100644 internal/datasources/service/service_test.go create mode 100644 internal/datasources/service/tx.go create mode 100644 internal/db/datasources.sql.go diff --git a/database/mock/store.go b/database/mock/store.go index 5f428fa277..bcd29aa613 100644 --- a/database/mock/store.go +++ b/database/mock/store.go @@ -44,6 +44,21 @@ func (m *MockStore) EXPECT() *MockStoreMockRecorder { return m.recorder } +// AddDataSourceFunction mocks base method. +func (m *MockStore) AddDataSourceFunction(ctx context.Context, arg db.AddDataSourceFunctionParams) (db.DataSourcesFunction, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "AddDataSourceFunction", ctx, arg) + ret0, _ := ret[0].(db.DataSourcesFunction) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// AddDataSourceFunction indicates an expected call of AddDataSourceFunction. +func (mr *MockStoreMockRecorder) AddDataSourceFunction(ctx, arg any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddDataSourceFunction", reflect.TypeOf((*MockStore)(nil).AddDataSourceFunction), ctx, arg) +} + // BeginTransaction mocks base method. func (m *MockStore) BeginTransaction() (*sql.Tx, error) { m.ctrl.T.Helper() @@ -192,6 +207,21 @@ func (mr *MockStoreMockRecorder) CountUsers(ctx any) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CountUsers", reflect.TypeOf((*MockStore)(nil).CountUsers), ctx) } +// CreateDataSource mocks base method. +func (m *MockStore) CreateDataSource(ctx context.Context, arg db.CreateDataSourceParams) (db.DataSource, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CreateDataSource", ctx, arg) + ret0, _ := ret[0].(db.DataSource) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// CreateDataSource indicates an expected call of CreateDataSource. +func (mr *MockStoreMockRecorder) CreateDataSource(ctx, arg any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateDataSource", reflect.TypeOf((*MockStore)(nil).CreateDataSource), ctx, arg) +} + // CreateEntitlements mocks base method. func (m *MockStore) CreateEntitlements(ctx context.Context, arg db.CreateEntitlementsParams) error { m.ctrl.T.Helper() @@ -459,6 +489,36 @@ func (mr *MockStoreMockRecorder) DeleteArtifact(ctx, id any) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteArtifact", reflect.TypeOf((*MockStore)(nil).DeleteArtifact), ctx, id) } +// DeleteDataSource mocks base method. +func (m *MockStore) DeleteDataSource(ctx context.Context, arg db.DeleteDataSourceParams) (db.DataSource, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteDataSource", ctx, arg) + ret0, _ := ret[0].(db.DataSource) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// DeleteDataSource indicates an expected call of DeleteDataSource. +func (mr *MockStoreMockRecorder) DeleteDataSource(ctx, arg any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteDataSource", reflect.TypeOf((*MockStore)(nil).DeleteDataSource), ctx, arg) +} + +// DeleteDataSourceFunction mocks base method. +func (m *MockStore) DeleteDataSourceFunction(ctx context.Context, arg db.DeleteDataSourceFunctionParams) (db.DataSourcesFunction, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteDataSourceFunction", ctx, arg) + ret0, _ := ret[0].(db.DataSourcesFunction) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// DeleteDataSourceFunction indicates an expected call of DeleteDataSourceFunction. +func (mr *MockStoreMockRecorder) DeleteDataSourceFunction(ctx, arg any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteDataSourceFunction", reflect.TypeOf((*MockStore)(nil).DeleteDataSourceFunction), ctx, arg) +} + // DeleteEntity mocks base method. func (m *MockStore) DeleteEntity(ctx context.Context, arg db.DeleteEntityParams) error { m.ctrl.T.Helper() @@ -910,6 +970,36 @@ func (mr *MockStoreMockRecorder) GetChildrenProjects(ctx, id any) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetChildrenProjects", reflect.TypeOf((*MockStore)(nil).GetChildrenProjects), ctx, id) } +// GetDataSource mocks base method. +func (m *MockStore) GetDataSource(ctx context.Context, arg db.GetDataSourceParams) (db.DataSource, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetDataSource", ctx, arg) + ret0, _ := ret[0].(db.DataSource) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetDataSource indicates an expected call of GetDataSource. +func (mr *MockStoreMockRecorder) GetDataSource(ctx, arg any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetDataSource", reflect.TypeOf((*MockStore)(nil).GetDataSource), ctx, arg) +} + +// GetDataSourceByName mocks base method. +func (m *MockStore) GetDataSourceByName(ctx context.Context, arg db.GetDataSourceByNameParams) (db.DataSource, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetDataSourceByName", ctx, arg) + ret0, _ := ret[0].(db.DataSource) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetDataSourceByName indicates an expected call of GetDataSourceByName. +func (mr *MockStoreMockRecorder) GetDataSourceByName(ctx, arg any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetDataSourceByName", reflect.TypeOf((*MockStore)(nil).GetDataSourceByName), ctx, arg) +} + // GetEntitiesByProjectHierarchy mocks base method. func (m *MockStore) GetEntitiesByProjectHierarchy(ctx context.Context, projects []uuid.UUID) ([]db.EntityInstance, error) { m.ctrl.T.Helper() @@ -1867,6 +1957,36 @@ func (mr *MockStoreMockRecorder) ListArtifactsByRepoID(ctx, repositoryID any) *g return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListArtifactsByRepoID", reflect.TypeOf((*MockStore)(nil).ListArtifactsByRepoID), ctx, repositoryID) } +// ListDataSourceFunctions mocks base method. +func (m *MockStore) ListDataSourceFunctions(ctx context.Context, dataSourceID uuid.UUID) ([]db.DataSourcesFunction, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ListDataSourceFunctions", ctx, dataSourceID) + ret0, _ := ret[0].([]db.DataSourcesFunction) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ListDataSourceFunctions indicates an expected call of ListDataSourceFunctions. +func (mr *MockStoreMockRecorder) ListDataSourceFunctions(ctx, dataSourceID any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListDataSourceFunctions", reflect.TypeOf((*MockStore)(nil).ListDataSourceFunctions), ctx, dataSourceID) +} + +// ListDataSources mocks base method. +func (m *MockStore) ListDataSources(ctx context.Context, projects []uuid.UUID) ([]db.DataSource, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ListDataSources", ctx, projects) + ret0, _ := ret[0].([]db.DataSource) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ListDataSources indicates an expected call of ListDataSources. +func (mr *MockStoreMockRecorder) ListDataSources(ctx, projects any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListDataSources", reflect.TypeOf((*MockStore)(nil).ListDataSources), ctx, projects) +} + // ListEvaluationHistory mocks base method. func (m *MockStore) ListEvaluationHistory(ctx context.Context, arg db.ListEvaluationHistoryParams) ([]db.ListEvaluationHistoryRow, error) { m.ctrl.T.Helper() @@ -2194,6 +2314,36 @@ func (mr *MockStoreMockRecorder) SetSubscriptionBundleVersion(ctx, arg any) *gom return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetSubscriptionBundleVersion", reflect.TypeOf((*MockStore)(nil).SetSubscriptionBundleVersion), ctx, arg) } +// UpdateDataSource mocks base method. +func (m *MockStore) UpdateDataSource(ctx context.Context, arg db.UpdateDataSourceParams) (db.DataSource, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "UpdateDataSource", ctx, arg) + ret0, _ := ret[0].(db.DataSource) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// UpdateDataSource indicates an expected call of UpdateDataSource. +func (mr *MockStoreMockRecorder) UpdateDataSource(ctx, arg any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateDataSource", reflect.TypeOf((*MockStore)(nil).UpdateDataSource), ctx, arg) +} + +// UpdateDataSourceFunction mocks base method. +func (m *MockStore) UpdateDataSourceFunction(ctx context.Context, arg db.UpdateDataSourceFunctionParams) (db.DataSourcesFunction, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "UpdateDataSourceFunction", ctx, arg) + ret0, _ := ret[0].(db.DataSourcesFunction) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// UpdateDataSourceFunction indicates an expected call of UpdateDataSourceFunction. +func (mr *MockStoreMockRecorder) UpdateDataSourceFunction(ctx, arg any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateDataSourceFunction", reflect.TypeOf((*MockStore)(nil).UpdateDataSourceFunction), ctx, arg) +} + // UpdateEncryptedSecret mocks base method. func (m *MockStore) UpdateEncryptedSecret(ctx context.Context, arg db.UpdateEncryptedSecretParams) error { m.ctrl.T.Helper() diff --git a/database/query/datasources.sql b/database/query/datasources.sql new file mode 100644 index 0000000000..e8cbcdb375 --- /dev/null +++ b/database/query/datasources.sql @@ -0,0 +1,73 @@ +-- CreateDataSource creates a new datasource in a given project. + +-- name: CreateDataSource :one +INSERT INTO data_sources (project_id, name, display_name) +VALUES ($1, $2, $3) RETURNING *; + +-- AddDataSourceFunction adds a function to a datasource. + +-- name: AddDataSourceFunction :one +INSERT INTO data_sources_functions (data_source_id, name, type, definition) +VALUES ($1, $2, $3, $4) RETURNING *; + +-- UpdateDataSource updates a datasource in a given project. + +-- name: UpdateDataSource :one +UPDATE data_sources +SET display_name = $3 +WHERE id = $1 AND project_id = $2 +RETURNING *; + +-- UpdateDataSourceFunction updates a function in a datasource. We're +-- only able to update the type and definition of the function. + +-- name: UpdateDataSourceFunction :one +UPDATE data_sources_functions +SET type = $3, definition = $4, updated_at = NOW() +WHERE data_source_id = $1 AND name = $2 +RETURNING *; + +-- name: DeleteDataSource :one +DELETE FROM data_sources +WHERE id = $1 AND project_id = $2 +RETURNING *; + +-- name: DeleteDataSourceFunction :one +DELETE FROM data_sources_functions +WHERE data_source_id = $1 AND name = $2 +RETURNING *; + +-- GetDataSource retrieves a datasource by its id and a project hierarchy. +-- +-- Note that to get a datasource for a given project, one can simply +-- pass one project id in the project_id array. + +-- name: GetDataSource :one +SELECT * FROM data_sources +WHERE id = $1 AND project_id = ANY(sqlc.arg(projects)::uuid[]); + +-- GetDataSourceByName retrieves a datasource by its name and +-- a project hierarchy. +-- +-- Note that to get a datasource for a given project, one can simply +-- pass one project id in the project_id array. + +-- name: GetDataSourceByName :one +SELECT * FROM data_sources +WHERE name = $1 AND project_id = ANY(sqlc.arg(projects)::uuid[]); + +-- ListDataSources retrieves all datasources for project hierarchy. +-- +-- Note that to get a datasource for a given project, one can simply +-- pass one project id in the project_id array. + +-- name: ListDataSources :many +SELECT * FROM data_sources +WHERE project_id = ANY(sqlc.arg(projects)::uuid[]); + +-- ListDataSourceFunctions retrieves all functions for a datasource. + +-- name: ListDataSourceFunctions :many +SELECT * FROM data_sources_functions +WHERE data_source_id = $1; + diff --git a/internal/datasources/factory.go b/internal/datasources/factory.go index e38e30493e..5c7a265430 100644 --- a/internal/datasources/factory.go +++ b/internal/datasources/factory.go @@ -12,6 +12,11 @@ import ( v1datasources "github.com/mindersec/minder/pkg/datasources/v1" ) +const ( + // DataSourceDriverRest is the driver type for a REST data source. + DataSourceDriverRest = "rest" +) + // BuildFromProtobuf is a factory function that builds a new data source based on the given // data source type. func BuildFromProtobuf(ds *minderv1.DataSource) (v1datasources.DataSource, error) { diff --git a/internal/datasources/service/helpers.go b/internal/datasources/service/helpers.go new file mode 100644 index 0000000000..16eb715685 --- /dev/null +++ b/internal/datasources/service/helpers.go @@ -0,0 +1,67 @@ +// SPDX-FileCopyrightText: Copyright 2024 The Minder Authors +// SPDX-License-Identifier: Apache-2.0 + +package service + +import ( + "context" + "fmt" + + "github.com/google/uuid" + + "github.com/mindersec/minder/internal/db" + minderv1 "github.com/mindersec/minder/pkg/api/protobuf/go/minder/v1" +) + +func (d *dataSourceService) getDataSourceSomehow( + ctx context.Context, + project uuid.UUID, + opts *ReadOptions, + theSomehow func(ctx context.Context, qtx db.ExtendQuerier, projs []uuid.UUID) (db.DataSource, error), +) (*minderv1.DataSource, error) { + stx, err := d.txBuilder(d, opts) + if err != nil { + return nil, fmt.Errorf("failed to start transaction: %w", err) + } + + //nolint:gosec // we'll log this error later. + defer stx.Rollback() + + tx := stx.Q() + + projs, err := listRelevantProjects(ctx, tx, project, opts.canSearchHierarchical()) + if err != nil { + return nil, fmt.Errorf("failed to list relevant projects: %w", err) + } + + ds, err := theSomehow(ctx, tx, projs) + if err != nil { + return nil, fmt.Errorf("failed to get data source by name: %w", err) + } + + dsfuncs, err := tx.ListDataSourceFunctions(ctx, ds.ID) + if err != nil { + return nil, fmt.Errorf("failed to get data source functions: %w", err) + } + + if err := stx.Commit(); err != nil { + return nil, fmt.Errorf("failed to commit transaction: %w", err) + } + + return dataSourceDBToProtobuf(ds, dsfuncs) +} + +func listRelevantProjects( + ctx context.Context, tx db.ExtendQuerier, project uuid.UUID, hierarchical bool, +) ([]uuid.UUID, error) { + if hierarchical { + projs, err := tx.GetParentProjects(ctx, project) + if err != nil { + return nil, err + } + + return projs, nil + } + + return []uuid.UUID{project}, nil +} diff --git a/internal/datasources/service/mock/service.go b/internal/datasources/service/mock/service.go new file mode 100644 index 0000000000..2d58c3587c --- /dev/null +++ b/internal/datasources/service/mock/service.go @@ -0,0 +1,163 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: ./service.go +// +// Generated by this command: +// +// mockgen -package mock_service -destination=./mock/service.go -source=./service.go +// + +// Package mock_service is a generated GoMock package. +package mock_service + +import ( + context "context" + reflect "reflect" + + uuid "github.com/google/uuid" + service "github.com/mindersec/minder/internal/datasources/service" + v1 "github.com/mindersec/minder/pkg/api/protobuf/go/minder/v1" + v10 "github.com/mindersec/minder/pkg/datasources/v1" + gomock "go.uber.org/mock/gomock" +) + +// MockDataSourcesService is a mock of DataSourcesService interface. +type MockDataSourcesService struct { + ctrl *gomock.Controller + recorder *MockDataSourcesServiceMockRecorder + isgomock struct{} +} + +// MockDataSourcesServiceMockRecorder is the mock recorder for MockDataSourcesService. +type MockDataSourcesServiceMockRecorder struct { + mock *MockDataSourcesService +} + +// NewMockDataSourcesService creates a new mock instance. +func NewMockDataSourcesService(ctrl *gomock.Controller) *MockDataSourcesService { + mock := &MockDataSourcesService{ctrl: ctrl} + mock.recorder = &MockDataSourcesServiceMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockDataSourcesService) EXPECT() *MockDataSourcesServiceMockRecorder { + return m.recorder +} + +// BuildDataSourceRegistry mocks base method. +func (m *MockDataSourcesService) BuildDataSourceRegistry(ctx context.Context, rt *v1.RuleType) (*v10.DataSourceRegistry, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "BuildDataSourceRegistry", ctx, rt) + ret0, _ := ret[0].(*v10.DataSourceRegistry) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// BuildDataSourceRegistry indicates an expected call of BuildDataSourceRegistry. +func (mr *MockDataSourcesServiceMockRecorder) BuildDataSourceRegistry(ctx, rt any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "BuildDataSourceRegistry", reflect.TypeOf((*MockDataSourcesService)(nil).BuildDataSourceRegistry), ctx, rt) +} + +// Create mocks base method. +func (m *MockDataSourcesService) Create(ctx context.Context, ds *v1.DataSource, opts *service.Options) (*v1.DataSource, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Create", ctx, ds, opts) + ret0, _ := ret[0].(*v1.DataSource) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Create indicates an expected call of Create. +func (mr *MockDataSourcesServiceMockRecorder) Create(ctx, ds, opts any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Create", reflect.TypeOf((*MockDataSourcesService)(nil).Create), ctx, ds, opts) +} + +// Delete mocks base method. +func (m *MockDataSourcesService) Delete(ctx context.Context, id, project uuid.UUID, opts *service.Options) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Delete", ctx, id, project, opts) + ret0, _ := ret[0].(error) + return ret0 +} + +// Delete indicates an expected call of Delete. +func (mr *MockDataSourcesServiceMockRecorder) Delete(ctx, id, project, opts any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Delete", reflect.TypeOf((*MockDataSourcesService)(nil).Delete), ctx, id, project, opts) +} + +// GetByID mocks base method. +func (m *MockDataSourcesService) GetByID(ctx context.Context, id, project uuid.UUID, opts *service.ReadOptions) (*v1.DataSource, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetByID", ctx, id, project, opts) + ret0, _ := ret[0].(*v1.DataSource) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetByID indicates an expected call of GetByID. +func (mr *MockDataSourcesServiceMockRecorder) GetByID(ctx, id, project, opts any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetByID", reflect.TypeOf((*MockDataSourcesService)(nil).GetByID), ctx, id, project, opts) +} + +// GetByName mocks base method. +func (m *MockDataSourcesService) GetByName(ctx context.Context, name string, project uuid.UUID, opts *service.ReadOptions) (*v1.DataSource, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetByName", ctx, name, project, opts) + ret0, _ := ret[0].(*v1.DataSource) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetByName indicates an expected call of GetByName. +func (mr *MockDataSourcesServiceMockRecorder) GetByName(ctx, name, project, opts any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetByName", reflect.TypeOf((*MockDataSourcesService)(nil).GetByName), ctx, name, project, opts) +} + +// List mocks base method. +func (m *MockDataSourcesService) List(ctx context.Context, project uuid.UUID, opts *service.ReadOptions) ([]*v1.DataSource, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "List", ctx, project, opts) + ret0, _ := ret[0].([]*v1.DataSource) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// List indicates an expected call of List. +func (mr *MockDataSourcesServiceMockRecorder) List(ctx, project, opts any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "List", reflect.TypeOf((*MockDataSourcesService)(nil).List), ctx, project, opts) +} + +// Update mocks base method. +func (m *MockDataSourcesService) Update(ctx context.Context, ds *v1.DataSource, opts *service.Options) (*v1.DataSource, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Update", ctx, ds, opts) + ret0, _ := ret[0].(*v1.DataSource) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Update indicates an expected call of Update. +func (mr *MockDataSourcesServiceMockRecorder) Update(ctx, ds, opts any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Update", reflect.TypeOf((*MockDataSourcesService)(nil).Update), ctx, ds, opts) +} + +// ValidateRuleTypeReferences mocks base method. +func (m *MockDataSourcesService) ValidateRuleTypeReferences(ctx context.Context, rt *v1.RuleType) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ValidateRuleTypeReferences", ctx, rt) + ret0, _ := ret[0].(error) + return ret0 +} + +// ValidateRuleTypeReferences indicates an expected call of ValidateRuleTypeReferences. +func (mr *MockDataSourcesServiceMockRecorder) ValidateRuleTypeReferences(ctx, rt any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ValidateRuleTypeReferences", reflect.TypeOf((*MockDataSourcesService)(nil).ValidateRuleTypeReferences), ctx, rt) +} diff --git a/internal/datasources/service/options.go b/internal/datasources/service/options.go new file mode 100644 index 0000000000..eeb3692778 --- /dev/null +++ b/internal/datasources/service/options.go @@ -0,0 +1,73 @@ +// SPDX-FileCopyrightText: Copyright 2024 The Minder Authors +// SPDX-License-Identifier: Apache-2.0 + +package service + +import "github.com/mindersec/minder/internal/db" + +// Options is a struct that contains the options for a service call +type Options struct { + qtx db.ExtendQuerier +} + +// OptionsBuilder is a function that returns a new Options struct +func OptionsBuilder() *Options { + return &Options{} +} + +// WithTransaction is a function that sets the transaction field in the Options struct +func (o *Options) WithTransaction(qtx db.ExtendQuerier) *Options { + if o == nil { + return nil + } + o.qtx = qtx + return o +} + +func (o *Options) getTransaction() db.ExtendQuerier { + if o == nil { + return nil + } + return o.qtx +} + +type txGetter interface { + getTransaction() db.ExtendQuerier +} + +// ReadOptions is a struct that contains the options for a read service call +// This extends the Options struct and adds a hierarchical field. +type ReadOptions struct { + Options + hierarchical bool +} + +// ReadBuilder is a function that returns a new ReadOptions struct +func ReadBuilder() *ReadOptions { + return &ReadOptions{} +} + +// Hierarchical allows the service to search in the project hierarchy +func (o *ReadOptions) Hierarchical() *ReadOptions { + if o == nil { + return nil + } + o.hierarchical = true + return o +} + +// WithTransaction is a function that sets the transaction field in the Options struct +func (o *ReadOptions) WithTransaction(qtx db.ExtendQuerier) *ReadOptions { + if o == nil { + return nil + } + o.qtx = qtx + return o +} + +func (o *ReadOptions) canSearchHierarchical() bool { + if o == nil { + return false + } + return o.hierarchical +} diff --git a/internal/datasources/service/service.go b/internal/datasources/service/service.go new file mode 100644 index 0000000000..b3ea3ae0a8 --- /dev/null +++ b/internal/datasources/service/service.go @@ -0,0 +1,253 @@ +// SPDX-FileCopyrightText: Copyright 2024 The Minder Authors +// SPDX-License-Identifier: Apache-2.0 + +// Package service encodes the business logic for dealing with data sources. +package service + +import ( + "context" + "database/sql" + "errors" + "fmt" + + "github.com/google/uuid" + "google.golang.org/grpc/codes" + "google.golang.org/protobuf/encoding/protojson" + + "github.com/mindersec/minder/internal/datasources" + "github.com/mindersec/minder/internal/db" + "github.com/mindersec/minder/internal/util" + minderv1 "github.com/mindersec/minder/pkg/api/protobuf/go/minder/v1" + v1datasources "github.com/mindersec/minder/pkg/datasources/v1" +) + +//go:generate go run go.uber.org/mock/mockgen -package mock_$GOPACKAGE -destination=./mock/$GOFILE -source=./$GOFILE + +// DataSourcesService is an interface that defines the methods for the data sources service. +type DataSourcesService interface { + // GetByName returns a data source by name. + GetByName(ctx context.Context, name string, project uuid.UUID, opts *ReadOptions) (*minderv1.DataSource, error) + + // GetByID returns a data source by ID. + GetByID(ctx context.Context, id uuid.UUID, project uuid.UUID, opts *ReadOptions) (*minderv1.DataSource, error) + + // List lists all data sources in the given project. + List(ctx context.Context, project uuid.UUID, opts *ReadOptions) ([]*minderv1.DataSource, error) + + // Create creates a new data source. + Create(ctx context.Context, ds *minderv1.DataSource, opts *Options) (*minderv1.DataSource, error) + + // Update updates an existing data source. + Update(ctx context.Context, ds *minderv1.DataSource, opts *Options) (*minderv1.DataSource, error) + + // Delete deletes a data source in the given project. + // + // Note that one cannot delete a data source that is in use by a rule type. + Delete(ctx context.Context, id uuid.UUID, project uuid.UUID, opts *Options) error + + // ValidateRuleTypeReferences takes the data source declarations in + // a rule type and validates that the data sources are available + // in the project hierarchy. + // + // Note that the rule type already contains project information. + ValidateRuleTypeReferences(ctx context.Context, rt *minderv1.RuleType) error + + // BuildDataSourceRegistry bundles up all data sources referenced in the rule type + // into a registry. + BuildDataSourceRegistry(ctx context.Context, rt *minderv1.RuleType) (*v1datasources.DataSourceRegistry, error) +} + +type dataSourceService struct { + store db.Store + + // This is a function that will begin a transaction for the service. + // We make this a function so that we can mock it in tests. + txBuilder func(d *dataSourceService, opts txGetter) (serviceTX, error) +} + +// NewDataSourceService creates a new data source service. +func NewDataSourceService(store db.Store) *dataSourceService { + return &dataSourceService{ + store: store, + txBuilder: beginTx, + } +} + +// WithTransactionBuilder sets the transaction builder for the data source service. +// +// Note this is mostly just useful for testing. +func (d *dataSourceService) WithTransactionBuilder(txBuilder func(d *dataSourceService, opts txGetter) (serviceTX, error)) { + d.txBuilder = txBuilder +} + +// Ensure that dataSourceService implements DataSourcesService. +var _ DataSourcesService = (*dataSourceService)(nil) + +func (d *dataSourceService) GetByName( + ctx context.Context, name string, project uuid.UUID, opts *ReadOptions) (*minderv1.DataSource, error) { + return d.getDataSourceSomehow( + ctx, project, opts, func(ctx context.Context, tx db.ExtendQuerier, projs []uuid.UUID, + ) (db.DataSource, error) { + ds, err := tx.GetDataSourceByName(ctx, db.GetDataSourceByNameParams{ + Name: name, + Projects: projs, + }) + if err != nil { + if errors.Is(err, sql.ErrNoRows) { + return db.DataSource{}, util.UserVisibleError(codes.NotFound, + "data source of name %s not found", name) + } + return db.DataSource{}, fmt.Errorf("failed to get data source by name: %w", err) + } + + return ds, nil + }) +} + +func (d *dataSourceService) GetByID( + ctx context.Context, id uuid.UUID, project uuid.UUID, opts *ReadOptions) (*minderv1.DataSource, error) { + return d.getDataSourceSomehow( + ctx, project, opts, func(ctx context.Context, tx db.ExtendQuerier, projs []uuid.UUID, + ) (db.DataSource, error) { + ds, err := tx.GetDataSource(ctx, db.GetDataSourceParams{ + ID: id, + Projects: projs, + }) + if errors.Is(err, sql.ErrNoRows) { + return db.DataSource{}, util.UserVisibleError(codes.NotFound, + "data source of id %s not found", id.String()) + } + if err != nil { + return db.DataSource{}, fmt.Errorf("failed to get data source by name: %w", err) + } + + return ds, nil + }) +} + +func (d *dataSourceService) List( + ctx context.Context, project uuid.UUID, opts *ReadOptions) ([]*minderv1.DataSource, error) { + stx, err := d.txBuilder(d, opts) + if err != nil { + return nil, fmt.Errorf("failed to start transaction: %w", err) + } + + //nolint:gosec // we'll log this error later. + defer stx.Rollback() + + tx := stx.Q() + + projs, err := listRelevantProjects(ctx, tx, project, opts.canSearchHierarchical()) + if err != nil { + return nil, fmt.Errorf("failed to list relevant projects: %w", err) + } + + dss, err := tx.ListDataSources(ctx, projs) + if err != nil { + return nil, fmt.Errorf("failed to list data sources: %w", err) + } + + outDS := make([]*minderv1.DataSource, len(dss)) + + for _, ds := range dss { + dsfuncs, err := tx.ListDataSourceFunctions(ctx, ds.ID) + if err != nil { + return nil, fmt.Errorf("failed to list data source functions: %w", err) + } + + dsProtobuf, err := dataSourceDBToProtobuf(ds, dsfuncs) + if err != nil { + return nil, fmt.Errorf("failed to convert data source to protobuf: %w", err) + } + + outDS = append(outDS, dsProtobuf) + } + + if err := stx.Commit(); err != nil { + return nil, fmt.Errorf("failed to commit transaction: %w", err) + } + + return outDS, nil +} + +// nolint:revive // there is a TODO +func (d *dataSourceService) Create( + ctx context.Context, ds *minderv1.DataSource, opts *Options) (*minderv1.DataSource, error) { + //TODO implement me + panic("implement me") +} + +// nolint:revive // there is a TODO +func (d *dataSourceService) Update( + ctx context.Context, ds *minderv1.DataSource, opts *Options) (*minderv1.DataSource, error) { + //TODO implement me + panic("implement me") +} + +// nolint:revive // there is a TODO +func (d *dataSourceService) Delete( + ctx context.Context, id uuid.UUID, project uuid.UUID, opts *Options) error { + //TODO implement me + panic("implement me") +} + +// nolint:revive // there is a TODO +func (d *dataSourceService) ValidateRuleTypeReferences( + ctx context.Context, rt *minderv1.RuleType) error { + //TODO implement me + panic("implement me") +} + +// nolint:revive // there is a TODO +func (d *dataSourceService) BuildDataSourceRegistry( + ctx context.Context, rt *minderv1.RuleType) (*v1datasources.DataSourceRegistry, error) { + //TODO implement me + panic("implement me") +} + +func dataSourceDBToProtobuf(ds db.DataSource, dsfuncs []db.DataSourcesFunction) (*minderv1.DataSource, error) { + outds := &minderv1.DataSource{ + Version: minderv1.VersionV1, + Type: string(minderv1.DataSourceResource), + Id: ds.ID.String(), + Name: ds.Name, + Context: &minderv1.ContextV2{ + ProjectId: ds.ProjectID.String(), + }, + } + + if len(dsfuncs) == 0 { + return nil, errors.New("data source is invalid and has no defintions") + } + + // All data source types should be equal... so we'll just take the first one. + dsfType := dsfuncs[0].Type + + switch dsfType { + case datasources.DataSourceDriverRest: + return dataSourceRestDBToProtobuf(outds, dsfuncs) + default: + return nil, fmt.Errorf("unknown data source type: %s", dsfType) + } +} + +func dataSourceRestDBToProtobuf(ds *minderv1.DataSource, dsfuncs []db.DataSourcesFunction) (*minderv1.DataSource, error) { + // At this point we have already validated that we have at least one function. + ds.Driver = &minderv1.DataSource_Rest{ + Rest: &minderv1.RestDataSource{ + Def: make(map[string]*minderv1.RestDataSource_Def, len(dsfuncs)), + }, + } + + for _, dsf := range dsfuncs { + key := dsf.Name + dsfToParse := &minderv1.RestDataSource_Def{} + if err := protojson.Unmarshal(dsf.Definition, dsfToParse); err != nil { + return nil, fmt.Errorf("failed to unmarshal data source definition for %s: %w", key, err) + } + + ds.GetRest().Def[key] = dsfToParse + } + + return ds, nil +} diff --git a/internal/datasources/service/service_test.go b/internal/datasources/service/service_test.go new file mode 100644 index 0000000000..b6728b8fc3 --- /dev/null +++ b/internal/datasources/service/service_test.go @@ -0,0 +1,287 @@ +// SPDX-FileCopyrightText: Copyright 2024 The Minder Authors +// SPDX-License-Identifier: Apache-2.0 + +package service + +import ( + "context" + "database/sql" + "fmt" + "testing" + + "github.com/google/uuid" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" + "google.golang.org/protobuf/encoding/protojson" + "google.golang.org/protobuf/types/known/structpb" + + mockdb "github.com/mindersec/minder/database/mock" + "github.com/mindersec/minder/internal/datasources" + "github.com/mindersec/minder/internal/db" + minderv1 "github.com/mindersec/minder/pkg/api/protobuf/go/minder/v1" +) + +func TestGetByName(t *testing.T) { + t.Parallel() + + type args struct { + name string + project uuid.UUID + opts *ReadOptions + } + tests := []struct { + name string + args args + setup func(mockDB *mockdb.MockStore) + want *minderv1.DataSource + wantErr bool + }{ + { + name: "DataSource found", + args: args{ + name: "test_name", + project: uuid.New(), + opts: &ReadOptions{}, + }, + setup: func(mockDB *mockdb.MockStore) { + dsID := uuid.New() + mockDB.EXPECT().GetDataSourceByName(gomock.Any(), gomock.Any()).Return(db.DataSource{ + ID: dsID, + Name: "test_name", + ProjectID: uuid.New(), + }, nil) + + is, err := structpb.NewStruct(map[string]any{ + "type": "object", + "properties": map[string]any{ + "test": "string", + }, + }) + require.NoError(t, err, "failed to create struct") + + mockDB.EXPECT().ListDataSourceFunctions(gomock.Any(), dsID). + Return([]db.DataSourcesFunction{ + { + ID: uuid.New(), + DataSourceID: dsID, + Name: "test_function", + Type: string(datasources.DataSourceDriverRest), + Definition: restDriverToJson(t, &minderv1.RestDataSource_Def{ + Endpoint: "http://example.com", + InputSchema: is, + }), + }, + }, nil) + }, + want: &minderv1.DataSource{ + Name: "test_name", + }, + wantErr: false, + }, + { + name: "DataSource not found", + args: args{ + name: "non_existent", + project: uuid.New(), + opts: &ReadOptions{}, + }, + setup: func(mockDB *mockdb.MockStore) { + mockDB.EXPECT().GetDataSourceByName(gomock.Any(), gomock.Any()). + Return(db.DataSource{}, sql.ErrNoRows) + }, + want: nil, + wantErr: true, + }, + { + name: "Database error", + args: args{ + name: "test_name", + project: uuid.New(), + opts: &ReadOptions{}, + }, + setup: func(mockDB *mockdb.MockStore) { + mockDB.EXPECT().GetDataSourceByName(gomock.Any(), gomock.Any()). + Return(db.DataSource{}, fmt.Errorf("database error")) + }, + want: nil, + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + // Setup + mockStore := mockdb.NewMockStore(ctrl) + + svc := NewDataSourceService(mockStore) + svc.txBuilder = func(_ *dataSourceService, _ txGetter) (serviceTX, error) { + return &fakeTxBuilder{ + store: mockStore, + }, nil + } + tt.setup(mockStore) + + got, err := svc.GetByName(context.Background(), tt.args.name, tt.args.project, tt.args.opts) + if tt.wantErr { + assert.Error(t, err) + return + } + + require.NoError(t, err) + assert.Equal(t, tt.want.Name, got.Name) + assert.NotNilf(t, got.Driver, "driver is nil") + }) + } +} + +func TestGetByID(t *testing.T) { + t.Parallel() + + type args struct { + id uuid.UUID + project uuid.UUID + opts *ReadOptions + } + tests := []struct { + name string + args args + setup func(id uuid.UUID, mockDB *mockdb.MockStore) + want *minderv1.DataSource + wantErr bool + }{ + { + name: "DataSource found", + args: args{ + id: uuid.New(), + project: uuid.New(), + opts: &ReadOptions{}, + }, + setup: func(id uuid.UUID, mockDB *mockdb.MockStore) { + mockDB.EXPECT().GetDataSource(gomock.Any(), gomock.Any()).Return(db.DataSource{ + ID: id, + Name: "test_name", + ProjectID: uuid.New(), + }, nil) + + is, err := structpb.NewStruct(map[string]any{ + "type": "object", + "properties": map[string]any{ + "test": "string", + }, + }) + require.NoError(t, err, "failed to create struct") + + mockDB.EXPECT().ListDataSourceFunctions(gomock.Any(), id). + Return([]db.DataSourcesFunction{ + { + ID: uuid.New(), + DataSourceID: id, + Name: "test_function", + Type: string(datasources.DataSourceDriverRest), + Definition: restDriverToJson(t, &minderv1.RestDataSource_Def{ + Endpoint: "http://example.com", + InputSchema: is, + }), + }, + }, nil) + }, + want: &minderv1.DataSource{ + Name: "test_name", + }, + wantErr: false, + }, + { + name: "DataSource not found", + args: args{ + id: uuid.New(), + project: uuid.New(), + opts: &ReadOptions{}, + }, + setup: func(_ uuid.UUID, mockDB *mockdb.MockStore) { + mockDB.EXPECT().GetDataSource(gomock.Any(), gomock.Any()). + Return(db.DataSource{}, sql.ErrNoRows) + }, + want: nil, + wantErr: true, + }, + { + name: "Database error", + args: args{ + id: uuid.New(), + project: uuid.New(), + opts: &ReadOptions{}, + }, + setup: func(_ uuid.UUID, mockDB *mockdb.MockStore) { + mockDB.EXPECT().GetDataSource(gomock.Any(), gomock.Any()). + Return(db.DataSource{}, fmt.Errorf("database error")) + }, + want: nil, + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + // Setup + mockStore := mockdb.NewMockStore(ctrl) + + svc := NewDataSourceService(mockStore) + svc.txBuilder = func(_ *dataSourceService, _ txGetter) (serviceTX, error) { + return &fakeTxBuilder{ + store: mockStore, + }, nil + } + tt.setup(tt.args.id, mockStore) + + got, err := svc.GetByID(context.Background(), tt.args.id, tt.args.project, tt.args.opts) + if tt.wantErr { + assert.Error(t, err) + return + } + + require.NoError(t, err) + assert.Equal(t, tt.want.Name, got.Name) + assert.NotNilf(t, got.Driver, "driver is nil") + }) + } +} + +type fakeTxBuilder struct { + store db.Store + errorOnCommit bool +} + +func (f *fakeTxBuilder) Q() db.ExtendQuerier { + return f.store +} + +func (f *fakeTxBuilder) Commit() error { + if f.errorOnCommit { + return fmt.Errorf("error on commit") + } + return nil +} + +func (_ *fakeTxBuilder) Rollback() error { + return nil +} + +func restDriverToJson(t *testing.T, rs *minderv1.RestDataSource_Def) []byte { + t.Helper() + + out, err := protojson.Marshal(rs) + require.NoError(t, err) + + return out +} diff --git a/internal/datasources/service/tx.go b/internal/datasources/service/tx.go new file mode 100644 index 0000000000..e9c6e26367 --- /dev/null +++ b/internal/datasources/service/tx.go @@ -0,0 +1,81 @@ +// SPDX-FileCopyrightText: Copyright 2024 The Minder Authors +// SPDX-License-Identifier: Apache-2.0 + +package service + +import ( + "database/sql" + + "github.com/mindersec/minder/internal/db" +) + +// serviceTX is an interface that defines the methods for a service transaction. +// +// This service may be used with a pre-built transaction or without one. +// thus, we need to be able to handle both cases. +type serviceTX interface { + Q() db.ExtendQuerier + Commit() error + Rollback() error +} + +// This is a handy helper function to optionally begin a transaction if we've already +// got one. +func beginTx(d *dataSourceService, opts txGetter) (serviceTX, error) { + if opts.getTransaction() != nil { + return &externalTX{q: opts.getTransaction()}, nil + } + + return beginInternalTx(d, opts) +} + +// builds a new transaction for the service +func beginInternalTx(d *dataSourceService, opts txGetter) (serviceTX, error) { + tx, err := d.store.BeginTransaction() + if err != nil { + return nil, err + } + + // If this is a read-only operation we can set the transaction to read-only + // We can know this by casting to the *ReadOptions struct + if _, ok := opts.(*ReadOptions); ok { + if _, err := tx.Query("SET TRANSACTION READ ONLY"); err != nil { + return nil, err + } + } + + return &internalTX{tx: tx, q: d.store.GetQuerierWithTransaction(tx)}, nil +} + +type externalTX struct { + q db.ExtendQuerier +} + +func (e *externalTX) Q() db.ExtendQuerier { + return e.q +} + +func (_ *externalTX) Commit() error { + return nil +} + +func (_ *externalTX) Rollback() error { + return nil +} + +type internalTX struct { + tx *sql.Tx + q db.ExtendQuerier +} + +func (i *internalTX) Q() db.ExtendQuerier { + return i.q +} + +func (i *internalTX) Commit() error { + return i.tx.Commit() +} + +func (i *internalTX) Rollback() error { + return i.tx.Rollback() +} diff --git a/internal/db/datasources.sql.go b/internal/db/datasources.sql.go new file mode 100644 index 0000000000..578c6df849 --- /dev/null +++ b/internal/db/datasources.sql.go @@ -0,0 +1,329 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.27.0 +// source: datasources.sql + +package db + +import ( + "context" + "encoding/json" + + "github.com/google/uuid" + "github.com/lib/pq" +) + +const addDataSourceFunction = `-- name: AddDataSourceFunction :one + +INSERT INTO data_sources_functions (data_source_id, name, type, definition) +VALUES ($1, $2, $3, $4) RETURNING id, name, type, data_source_id, definition, created_at, updated_at +` + +type AddDataSourceFunctionParams struct { + DataSourceID uuid.UUID `json:"data_source_id"` + Name string `json:"name"` + Type string `json:"type"` + Definition json.RawMessage `json:"definition"` +} + +// AddDataSourceFunction adds a function to a datasource. +func (q *Queries) AddDataSourceFunction(ctx context.Context, arg AddDataSourceFunctionParams) (DataSourcesFunction, error) { + row := q.db.QueryRowContext(ctx, addDataSourceFunction, + arg.DataSourceID, + arg.Name, + arg.Type, + arg.Definition, + ) + var i DataSourcesFunction + err := row.Scan( + &i.ID, + &i.Name, + &i.Type, + &i.DataSourceID, + &i.Definition, + &i.CreatedAt, + &i.UpdatedAt, + ) + return i, err +} + +const createDataSource = `-- name: CreateDataSource :one + +INSERT INTO data_sources (project_id, name, display_name) +VALUES ($1, $2, $3) RETURNING id, name, display_name, project_id, created_at, updated_at +` + +type CreateDataSourceParams struct { + ProjectID uuid.UUID `json:"project_id"` + Name string `json:"name"` + DisplayName string `json:"display_name"` +} + +// CreateDataSource creates a new datasource in a given project. +func (q *Queries) CreateDataSource(ctx context.Context, arg CreateDataSourceParams) (DataSource, error) { + row := q.db.QueryRowContext(ctx, createDataSource, arg.ProjectID, arg.Name, arg.DisplayName) + var i DataSource + err := row.Scan( + &i.ID, + &i.Name, + &i.DisplayName, + &i.ProjectID, + &i.CreatedAt, + &i.UpdatedAt, + ) + return i, err +} + +const deleteDataSource = `-- name: DeleteDataSource :one +DELETE FROM data_sources +WHERE id = $1 AND project_id = $2 +RETURNING id, name, display_name, project_id, created_at, updated_at +` + +type DeleteDataSourceParams struct { + ID uuid.UUID `json:"id"` + ProjectID uuid.UUID `json:"project_id"` +} + +func (q *Queries) DeleteDataSource(ctx context.Context, arg DeleteDataSourceParams) (DataSource, error) { + row := q.db.QueryRowContext(ctx, deleteDataSource, arg.ID, arg.ProjectID) + var i DataSource + err := row.Scan( + &i.ID, + &i.Name, + &i.DisplayName, + &i.ProjectID, + &i.CreatedAt, + &i.UpdatedAt, + ) + return i, err +} + +const deleteDataSourceFunction = `-- name: DeleteDataSourceFunction :one +DELETE FROM data_sources_functions +WHERE data_source_id = $1 AND name = $2 +RETURNING id, name, type, data_source_id, definition, created_at, updated_at +` + +type DeleteDataSourceFunctionParams struct { + DataSourceID uuid.UUID `json:"data_source_id"` + Name string `json:"name"` +} + +func (q *Queries) DeleteDataSourceFunction(ctx context.Context, arg DeleteDataSourceFunctionParams) (DataSourcesFunction, error) { + row := q.db.QueryRowContext(ctx, deleteDataSourceFunction, arg.DataSourceID, arg.Name) + var i DataSourcesFunction + err := row.Scan( + &i.ID, + &i.Name, + &i.Type, + &i.DataSourceID, + &i.Definition, + &i.CreatedAt, + &i.UpdatedAt, + ) + return i, err +} + +const getDataSource = `-- name: GetDataSource :one + +SELECT id, name, display_name, project_id, created_at, updated_at FROM data_sources +WHERE id = $1 AND project_id = ANY($2::uuid[]) +` + +type GetDataSourceParams struct { + ID uuid.UUID `json:"id"` + Projects []uuid.UUID `json:"projects"` +} + +// GetDataSource retrieves a datasource by its id and a project hierarchy. +// +// Note that to get a datasource for a given project, one can simply +// pass one project id in the project_id array. +func (q *Queries) GetDataSource(ctx context.Context, arg GetDataSourceParams) (DataSource, error) { + row := q.db.QueryRowContext(ctx, getDataSource, arg.ID, pq.Array(arg.Projects)) + var i DataSource + err := row.Scan( + &i.ID, + &i.Name, + &i.DisplayName, + &i.ProjectID, + &i.CreatedAt, + &i.UpdatedAt, + ) + return i, err +} + +const getDataSourceByName = `-- name: GetDataSourceByName :one + +SELECT id, name, display_name, project_id, created_at, updated_at FROM data_sources +WHERE name = $1 AND project_id = ANY($2::uuid[]) +` + +type GetDataSourceByNameParams struct { + Name string `json:"name"` + Projects []uuid.UUID `json:"projects"` +} + +// GetDataSourceByName retrieves a datasource by its name and +// a project hierarchy. +// +// Note that to get a datasource for a given project, one can simply +// pass one project id in the project_id array. +func (q *Queries) GetDataSourceByName(ctx context.Context, arg GetDataSourceByNameParams) (DataSource, error) { + row := q.db.QueryRowContext(ctx, getDataSourceByName, arg.Name, pq.Array(arg.Projects)) + var i DataSource + err := row.Scan( + &i.ID, + &i.Name, + &i.DisplayName, + &i.ProjectID, + &i.CreatedAt, + &i.UpdatedAt, + ) + return i, err +} + +const listDataSourceFunctions = `-- name: ListDataSourceFunctions :many + +SELECT id, name, type, data_source_id, definition, created_at, updated_at FROM data_sources_functions +WHERE data_source_id = $1 +` + +// ListDataSourceFunctions retrieves all functions for a datasource. +func (q *Queries) ListDataSourceFunctions(ctx context.Context, dataSourceID uuid.UUID) ([]DataSourcesFunction, error) { + rows, err := q.db.QueryContext(ctx, listDataSourceFunctions, dataSourceID) + if err != nil { + return nil, err + } + defer rows.Close() + items := []DataSourcesFunction{} + for rows.Next() { + var i DataSourcesFunction + if err := rows.Scan( + &i.ID, + &i.Name, + &i.Type, + &i.DataSourceID, + &i.Definition, + &i.CreatedAt, + &i.UpdatedAt, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const listDataSources = `-- name: ListDataSources :many + +SELECT id, name, display_name, project_id, created_at, updated_at FROM data_sources +WHERE project_id = ANY($1::uuid[]) +` + +// ListDataSources retrieves all datasources for project hierarchy. +// +// Note that to get a datasource for a given project, one can simply +// pass one project id in the project_id array. +func (q *Queries) ListDataSources(ctx context.Context, projects []uuid.UUID) ([]DataSource, error) { + rows, err := q.db.QueryContext(ctx, listDataSources, pq.Array(projects)) + if err != nil { + return nil, err + } + defer rows.Close() + items := []DataSource{} + for rows.Next() { + var i DataSource + if err := rows.Scan( + &i.ID, + &i.Name, + &i.DisplayName, + &i.ProjectID, + &i.CreatedAt, + &i.UpdatedAt, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const updateDataSource = `-- name: UpdateDataSource :one + +UPDATE data_sources +SET display_name = $3 +WHERE id = $1 AND project_id = $2 +RETURNING id, name, display_name, project_id, created_at, updated_at +` + +type UpdateDataSourceParams struct { + ID uuid.UUID `json:"id"` + ProjectID uuid.UUID `json:"project_id"` + DisplayName string `json:"display_name"` +} + +// UpdateDataSource updates a datasource in a given project. +func (q *Queries) UpdateDataSource(ctx context.Context, arg UpdateDataSourceParams) (DataSource, error) { + row := q.db.QueryRowContext(ctx, updateDataSource, arg.ID, arg.ProjectID, arg.DisplayName) + var i DataSource + err := row.Scan( + &i.ID, + &i.Name, + &i.DisplayName, + &i.ProjectID, + &i.CreatedAt, + &i.UpdatedAt, + ) + return i, err +} + +const updateDataSourceFunction = `-- name: UpdateDataSourceFunction :one + +UPDATE data_sources_functions +SET type = $3, definition = $4, updated_at = NOW() +WHERE data_source_id = $1 AND name = $2 +RETURNING id, name, type, data_source_id, definition, created_at, updated_at +` + +type UpdateDataSourceFunctionParams struct { + DataSourceID uuid.UUID `json:"data_source_id"` + Name string `json:"name"` + Type string `json:"type"` + Definition json.RawMessage `json:"definition"` +} + +// UpdateDataSourceFunction updates a function in a datasource. We're +// only able to update the type and definition of the function. +func (q *Queries) UpdateDataSourceFunction(ctx context.Context, arg UpdateDataSourceFunctionParams) (DataSourcesFunction, error) { + row := q.db.QueryRowContext(ctx, updateDataSourceFunction, + arg.DataSourceID, + arg.Name, + arg.Type, + arg.Definition, + ) + var i DataSourcesFunction + err := row.Scan( + &i.ID, + &i.Name, + &i.Type, + &i.DataSourceID, + &i.Definition, + &i.CreatedAt, + &i.UpdatedAt, + ) + return i, err +} diff --git a/internal/db/querier.go b/internal/db/querier.go index dc9d33477b..840f5581cb 100644 --- a/internal/db/querier.go +++ b/internal/db/querier.go @@ -13,6 +13,8 @@ import ( ) type Querier interface { + // AddDataSourceFunction adds a function to a datasource. + AddDataSourceFunction(ctx context.Context, arg AddDataSourceFunctionParams) (DataSourcesFunction, error) BulkGetProfilesByID(ctx context.Context, profileIds []uuid.UUID) ([]BulkGetProfilesByIDRow, error) CountProfilesByEntityType(ctx context.Context) ([]CountProfilesByEntityTypeRow, error) CountProfilesByName(ctx context.Context, name string) (int64, error) @@ -20,6 +22,8 @@ type Querier interface { CountRepositories(ctx context.Context) (int64, error) CountRepositoriesByProjectID(ctx context.Context, projectID uuid.UUID) (int64, error) CountUsers(ctx context.Context) (int64, error) + // CreateDataSource creates a new datasource in a given project. + CreateDataSource(ctx context.Context, arg CreateDataSourceParams) (DataSource, error) CreateEntitlements(ctx context.Context, arg CreateEntitlementsParams) error // CreateEntity adds an entry to the entity_instances table so it can be tracked by Minder. CreateEntity(ctx context.Context, arg CreateEntityParams) (EntityInstance, error) @@ -47,6 +51,8 @@ type Querier interface { CreateUser(ctx context.Context, identitySubject string) (User, error) DeleteAllPropertiesForEntity(ctx context.Context, entityID uuid.UUID) error DeleteArtifact(ctx context.Context, id uuid.UUID) error + DeleteDataSource(ctx context.Context, arg DeleteDataSourceParams) (DataSource, error) + DeleteDataSourceFunction(ctx context.Context, arg DeleteDataSourceFunctionParams) (DataSourcesFunction, error) // DeleteEntity removes an entity from the entity_instances table for a project. DeleteEntity(ctx context.Context, arg DeleteEntityParams) error DeleteEvaluationHistoryByIDs(ctx context.Context, evaluationids []uuid.UUID) (int64, error) @@ -84,6 +90,17 @@ type Querier interface { GetArtifactByName(ctx context.Context, arg GetArtifactByNameParams) (Artifact, error) GetBundle(ctx context.Context, arg GetBundleParams) (Bundle, error) GetChildrenProjects(ctx context.Context, id uuid.UUID) ([]GetChildrenProjectsRow, error) + // GetDataSource retrieves a datasource by its id and a project hierarchy. + // + // Note that to get a datasource for a given project, one can simply + // pass one project id in the project_id array. + GetDataSource(ctx context.Context, arg GetDataSourceParams) (DataSource, error) + // GetDataSourceByName retrieves a datasource by its name and + // a project hierarchy. + // + // Note that to get a datasource for a given project, one can simply + // pass one project id in the project_id array. + GetDataSourceByName(ctx context.Context, arg GetDataSourceByNameParams) (DataSource, error) // GetEntitiesByProjectHierarchy retrieves all entities for a project or hierarchy of projects. GetEntitiesByProjectHierarchy(ctx context.Context, projects []uuid.UUID) ([]EntityInstance, error) // GetEntitiesByProvider retrieves all entities of a given provider. @@ -177,6 +194,13 @@ type Querier interface { InsertRemediationEvent(ctx context.Context, arg InsertRemediationEventParams) error ListAllRootProjects(ctx context.Context) ([]Project, error) ListArtifactsByRepoID(ctx context.Context, repositoryID uuid.NullUUID) ([]Artifact, error) + // ListDataSourceFunctions retrieves all functions for a datasource. + ListDataSourceFunctions(ctx context.Context, dataSourceID uuid.UUID) ([]DataSourcesFunction, error) + // ListDataSources retrieves all datasources for project hierarchy. + // + // Note that to get a datasource for a given project, one can simply + // pass one project id in the project_id array. + ListDataSources(ctx context.Context, projects []uuid.UUID) ([]DataSource, error) ListEvaluationHistory(ctx context.Context, arg ListEvaluationHistoryParams) ([]ListEvaluationHistoryRow, error) ListEvaluationHistoryStaleRecords(ctx context.Context, arg ListEvaluationHistoryStaleRecordsParams) ([]ListEvaluationHistoryStaleRecordsRow, error) ListFlushCache(ctx context.Context) ([]FlushCache, error) @@ -225,6 +249,11 @@ type Querier interface { ReleaseLock(ctx context.Context, arg ReleaseLockParams) error RepositoryExistsAfterID(ctx context.Context, id uuid.UUID) (bool, error) SetSubscriptionBundleVersion(ctx context.Context, arg SetSubscriptionBundleVersionParams) error + // UpdateDataSource updates a datasource in a given project. + UpdateDataSource(ctx context.Context, arg UpdateDataSourceParams) (DataSource, error) + // UpdateDataSourceFunction updates a function in a datasource. We're + // only able to update the type and definition of the function. + UpdateDataSourceFunction(ctx context.Context, arg UpdateDataSourceFunctionParams) (DataSourcesFunction, error) UpdateEncryptedSecret(ctx context.Context, arg UpdateEncryptedSecretParams) error // UpdateInvitationRole updates an invitation by its code. This is intended to be // called by a user who has issued an invitation and then decided to change the