diff --git a/models/meshmodel/core/v1alpha1/category.go b/models/meshmodel/core/v1alpha1/category.go new file mode 100644 index 00000000..492588d5 --- /dev/null +++ b/models/meshmodel/core/v1alpha1/category.go @@ -0,0 +1,81 @@ +package v1alpha1 + +import ( + "encoding/json" + "sync" + + "github.com/google/uuid" + "github.com/layer5io/meshkit/database" + "gorm.io/gorm" +) + +var categoryCreationLock sync.Mutex //Each model will perform a check and if the category already doesn't exist, it will create a category. This lock will make sure that there are no race conditions. + +// swagger:response Category +type Category struct { + ID uuid.UUID `json:"-"` + Name string `json:"name"` + Metadata map[string]interface{} `json:"metadata" yaml:"metadata"` +} +type CategoryDB struct { + ID uuid.UUID `json:"-"` + Name string `json:"categoryName" gorm:"categoryName"` + Metadata []byte `json:"categoryMetadata" gorm:"categoryMetadata"` +} +type CategoryFilter struct { + Name string + OrderOn string + Sort string //asc or desc. Default behavior is asc + Limit int //If 0 or unspecified then all records are returned and limit is not used + Offset int +} + +const DefaultCategory = "Miscellaneous" + +// Create the filter from map[string]interface{} +func (cf *CategoryFilter) Create(m map[string]interface{}) { + if m == nil { + return + } + cf.Name = m["name"].(string) +} +func CreateCategory(db *database.Handler, cat Category) (uuid.UUID, error) { + if cat.Name == "" { + cat.Name = DefaultCategory + } + byt, err := json.Marshal(cat) + if err != nil { + return uuid.UUID{}, err + } + catID := uuid.NewSHA1(uuid.UUID{}, byt) + var category CategoryDB + categoryCreationLock.Lock() + defer categoryCreationLock.Unlock() + err = db.First(&category, "id = ?", catID).Error + if err != nil && err != gorm.ErrRecordNotFound { + return uuid.UUID{}, err + } + if err == gorm.ErrRecordNotFound { //The category is already not present and needs to be inserted + cat.ID = catID + catdb := cat.GetCategoryDB(db) + err = db.Create(&catdb).Error + if err != nil { + return uuid.UUID{}, err + } + return catdb.ID, nil + } + return category.ID, nil +} + +func (cdb *CategoryDB) GetCategory(db *database.Handler) (cat Category) { + cat.ID = cdb.ID + cat.Name = cdb.Name + _ = json.Unmarshal(cdb.Metadata, &cat.Metadata) + return +} +func (c *Category) GetCategoryDB(db *database.Handler) (catdb CategoryDB) { + catdb.ID = c.ID + catdb.Name = c.Name + catdb.Metadata, _ = json.Marshal(c.Metadata) + return +} diff --git a/models/meshmodel/core/v1alpha1/component.go b/models/meshmodel/core/v1alpha1/component.go index 076d3de9..dd2979c8 100644 --- a/models/meshmodel/core/v1alpha1/component.go +++ b/models/meshmodel/core/v1alpha1/component.go @@ -8,7 +8,6 @@ import ( "github.com/google/uuid" "github.com/layer5io/meshkit/database" "github.com/layer5io/meshkit/models/meshmodel/core/types" - "gorm.io/gorm" "gorm.io/gorm/clause" ) @@ -69,49 +68,33 @@ func emptySchemaCheck(schema string) (valid bool) { } func CreateComponent(db *database.Handler, c ComponentDefinition) (uuid.UUID, error) { c.ID = uuid.New() - tempModelID := uuid.New() - byt, err := json.Marshal(c.Model) + mid, err := CreateModel(db, c.Model) if err != nil { return uuid.UUID{}, err } if !emptySchemaCheck(c.Schema) { c.Metadata["hasInvalidSchema"] = true } - modelID := uuid.NewSHA1(uuid.UUID{}, byt) - var model Model - modelCreationLock.Lock() - err = db.First(&model, "id = ?", modelID).Error - if err != nil && err != gorm.ErrRecordNotFound { - return uuid.UUID{}, err - } - if model.ID == tempModelID || err == gorm.ErrRecordNotFound { //The model is already not present and needs to be inserted - model = c.Model - model.ID = modelID - err = db.Create(&model).Error - if err != nil { - modelCreationLock.Unlock() - return uuid.UUID{}, err - } - } - modelCreationLock.Unlock() cdb := c.GetComponentDefinitionDB() - cdb.ModelID = model.ID + cdb.ModelID = mid err = db.Create(&cdb).Error return c.ID, err } func GetMeshModelComponents(db *database.Handler, f ComponentFilter) (c []ComponentDefinition) { type componentDefinitionWithModel struct { ComponentDefinitionDB - Model + ModelDB + CategoryDB } var componentDefinitionsWithModel []componentDefinitionWithModel finder := db.Model(&ComponentDefinitionDB{}). - Select("component_definition_dbs.*, models.*"). - Joins("JOIN models ON component_definition_dbs.model_id = models.id") // + Select("component_definition_dbs.*, model_dbs.*,category_dbs.*"). + Joins("JOIN model_dbs ON component_definition_dbs.model_id = model_dbs.id"). + Joins("JOIN category_dbs ON model_dbs.category_id = category_dbs.id") // if f.Greedy { if f.Name != "" && f.DisplayName != "" { - finder = finder.Where("component_definition_dbs.kind LIKE ? OR component_definition_dbs.display_name LIKE ?", f.Name+"%", f.DisplayName+"%") + finder = finder.Where("component_definition_dbs.kind LIKE ? OR display_name LIKE ?", f.Name+"%", f.DisplayName+"%") } else if f.Name != "" { finder = finder.Where("component_definition_dbs.kind LIKE ?", f.Name+"%") } else if f.DisplayName != "" { @@ -126,14 +109,16 @@ func GetMeshModelComponents(db *database.Handler, f ComponentFilter) (c []Compon } } if f.ModelName != "" && f.ModelName != "all" { - finder = finder.Where("models.name = ?", f.ModelName) + finder = finder.Where("model_dbs.name = ?", f.ModelName) } if f.APIVersion != "" { finder = finder.Where("component_definition_dbs.api_version = ?", f.APIVersion) } - + if f.CategoryName != "" { + finder = finder.Where("category_dbs.name = ?", f.CategoryName) + } if f.Version != "" { - finder = finder.Where("models.version = ?", f.Version) + finder = finder.Where("model_dbs.version = ?", f.Version) } if f.OrderOn != "" { if f.Sort == "desc" { @@ -155,23 +140,25 @@ func GetMeshModelComponents(db *database.Handler, f ComponentFilter) (c []Compon if f.Trim { cm.Schema = "" } - c = append(c, cm.ComponentDefinitionDB.GetComponentDefinition(cm.Model)) + c = append(c, cm.ComponentDefinitionDB.GetComponentDefinition(cm.ModelDB.GetModel(cm.CategoryDB.GetCategory(db)))) } + return c } type ComponentFilter struct { - Name string - APIVersion string - Greedy bool //when set to true - instead of an exact match, name will be prefix matched - Trim bool //when set to true - the schema is not returned - DisplayName string - ModelName string - Version string - Sort string //asc or desc. Default behavior is asc - OrderOn string - Limit int //If 0 or unspecified then all records are returned and limit is not used - Offset int + Name string + APIVersion string + Greedy bool //when set to true - instead of an exact match, name will be prefix matched + Trim bool //when set to true - the schema is not returned + DisplayName string + ModelName string + CategoryName string + Version string + Sort string //asc or desc. Default behavior is asc + OrderOn string + Limit int //If 0 or unspecified then all records are returned and limit is not used + Offset int } // Create the filter from map[string]interface{} diff --git a/models/meshmodel/core/v1alpha1/models.go b/models/meshmodel/core/v1alpha1/models.go index 279e725f..e79eeb81 100644 --- a/models/meshmodel/core/v1alpha1/models.go +++ b/models/meshmodel/core/v1alpha1/models.go @@ -1,9 +1,13 @@ package v1alpha1 import ( + "encoding/json" + "fmt" "sync" "github.com/google/uuid" + "github.com/layer5io/meshkit/database" + "gorm.io/gorm" ) var modelCreationLock sync.Mutex //Each component/relationship will perform a check and if the model already doesn't exist, it will create a model. This lock will make sure that there are no race conditions. @@ -19,20 +23,79 @@ type ModelFilter struct { Offset int } +// Create the filter from map[string]interface{} +func (cf *ModelFilter) Create(m map[string]interface{}) { + if m == nil { + return + } + cf.Name = m["name"].(string) +} + // swagger:response Model type Model struct { + ID uuid.UUID `json:"-"` + Name string `json:"name"` + Version string `json:"version"` + DisplayName string `json:"displayName" gorm:"modelDisplayName"` + Category Category `json:"category"` + Metadata map[string]interface{} `json:"metadata" yaml:"modelMetadata"` +} +type ModelDB struct { ID uuid.UUID `json:"-"` - Name string `json:"name"` + CategoryID uuid.UUID `json:"-" gorm:"categoryID"` + Name string `json:"modelName" gorm:"modelName"` Version string `json:"version"` DisplayName string `json:"modelDisplayName" gorm:"modelDisplayName"` - Category string `json:"category"` SubCategory string `json:"subCategory" gorm:"subCategory"` + Metadata []byte `json:"modelMetadata" gorm:"modelMetadata"` } -// Create the filter from map[string]interface{} -func (cf *ModelFilter) Create(m map[string]interface{}) { - if m == nil { - return +func CreateModel(db *database.Handler, cmodel Model) (uuid.UUID, error) { + byt, err := json.Marshal(cmodel) + if err != nil { + return uuid.UUID{}, err } - cf.Name = m["name"].(string) + modelID := uuid.NewSHA1(uuid.UUID{}, byt) + var model ModelDB + if cmodel.Name == "" { + return uuid.UUID{}, fmt.Errorf("empty or invalid model name passed") + } + modelCreationLock.Lock() + defer modelCreationLock.Unlock() + err = db.First(&model, "id = ?", modelID).Error + if err != nil && err != gorm.ErrRecordNotFound { + return uuid.UUID{}, err + } + if err == gorm.ErrRecordNotFound { //The model is already not present and needs to be inserted + id, err := CreateCategory(db, cmodel.Category) + if err != nil { + return uuid.UUID{}, err + } + cmodel.ID = modelID + mdb := cmodel.GetModelDB() + mdb.CategoryID = id + err = db.Create(&mdb).Error + if err != nil { + return uuid.UUID{}, err + } + return mdb.ID, nil + } + return model.ID, nil +} +func (cmd *ModelDB) GetModel(cat Category) (c Model) { + c.ID = cmd.ID + c.Category = cat + c.DisplayName = cmd.DisplayName + c.Name = cmd.Name + c.Version = cmd.Version + _ = json.Unmarshal(cmd.Metadata, &c.Metadata) + return +} +func (c *Model) GetModelDB() (cmd ModelDB) { + cmd.ID = c.ID + cmd.DisplayName = c.DisplayName + cmd.Name = c.Name + cmd.Version = c.Version + cmd.Metadata, _ = json.Marshal(c.Metadata) + return } diff --git a/models/meshmodel/core/v1alpha1/policy.go b/models/meshmodel/core/v1alpha1/policy.go index ecc9efbb..f616cb4e 100644 --- a/models/meshmodel/core/v1alpha1/policy.go +++ b/models/meshmodel/core/v1alpha1/policy.go @@ -8,7 +8,6 @@ import ( "github.com/google/uuid" "github.com/layer5io/meshkit/database" "github.com/layer5io/meshkit/models/meshmodel/core/types" - "gorm.io/gorm" ) type PolicyDefinition struct { @@ -59,7 +58,7 @@ func GetMeshModelPolicy(db *database.Handler, f PolicyFilter) (pl []PolicyDefini var componentDefinitionsWithModel []componentDefinitionWithModel finder := db.Model(&PolicyDefinitionDB{}). Select("policy_definition_dbs.*, models.*"). - Joins("JOIN models ON models.id = policy_definition_dbs.model_id") + Joins("JOIN model_dbs ON model_dbs.id = policy_definition_dbs.model_id") if f.Kind != "" { finder = finder.Where("policy_definition_dbs.kind = ?", f.Kind) } @@ -67,7 +66,7 @@ func GetMeshModelPolicy(db *database.Handler, f PolicyFilter) (pl []PolicyDefini finder = finder.Where("policy_definition_dbs.sub_type = ?", f.SubType) } if f.ModelName != "" { - finder = finder.Where("models.name = ?", f.ModelName) + finder = finder.Where("model_dbs.name = ?", f.ModelName) } err := finder.Scan(&componentDefinitionsWithModel).Error if err != nil { @@ -94,30 +93,12 @@ func (pdb *PolicyDefinitionDB) GetPolicyDefinition(m Model) (p PolicyDefinition) func CreatePolicy(db *database.Handler, p PolicyDefinition) (uuid.UUID, error) { p.ID = uuid.New() - tempModelID := uuid.New() - byt, err := json.Marshal(p.Model) + mid, err := CreateModel(db, p.Model) if err != nil { return uuid.UUID{}, err } - modelID := uuid.NewSHA1(uuid.UUID{}, byt) - var model Model - modelCreationLock.Lock() - err = db.First(&model, "id = ?", modelID).Error - if err != nil && err != gorm.ErrRecordNotFound { - return uuid.UUID{}, err - } - if model.ID == tempModelID || err == gorm.ErrRecordNotFound { - model = p.Model - model.ID = modelID - err = db.Create(&model).Error - if err != nil { - modelCreationLock.Unlock() - return uuid.UUID{}, err - } - } - modelCreationLock.Unlock() pdb := p.GetPolicyDefinitionDB() - pdb.ModelID = model.ID + pdb.ModelID = mid err = db.Create(&pdb).Error if err != nil { return uuid.UUID{}, err diff --git a/models/meshmodel/core/v1alpha1/relationship.go b/models/meshmodel/core/v1alpha1/relationship.go index de3af366..f9a85cb8 100644 --- a/models/meshmodel/core/v1alpha1/relationship.go +++ b/models/meshmodel/core/v1alpha1/relationship.go @@ -8,7 +8,6 @@ import ( "github.com/google/uuid" "github.com/layer5io/meshkit/database" "github.com/layer5io/meshkit/models/meshmodel/core/types" - "gorm.io/gorm" "gorm.io/gorm/clause" ) @@ -62,12 +61,14 @@ func (rf *RelationshipFilter) Create(m map[string]interface{}) { func GetMeshModelRelationship(db *database.Handler, f RelationshipFilter) (r []RelationshipDefinition) { type componentDefinitionWithModel struct { RelationshipDefinitionDB - Model + ModelDB + CategoryDB } var componentDefinitionsWithModel []componentDefinitionWithModel finder := db.Model(&RelationshipDefinitionDB{}). - Select("relationship_definition_dbs.*, models.*"). - Joins("JOIN models ON relationship_definition_dbs.model_id = models.id") // + Select("relationship_definition_dbs.*, model_dbs.*"). + Joins("JOIN model_dbs ON relationship_definition_dbs.model_id = model_dbs.id"). // + Joins("JOIN category_dbs ON model_dbs.category_id = category_dbs.id") // if f.Kind != "" { if f.Greedy { finder = finder.Where("relationship_definition_dbs.kind LIKE ?", f.Kind) @@ -79,10 +80,10 @@ func GetMeshModelRelationship(db *database.Handler, f RelationshipFilter) (r []R finder = finder.Where("relationship_definition_dbs.sub_type = ?", f.SubType) } if f.ModelName != "" { - finder = finder.Where("models.name = ?", f.ModelName) + finder = finder.Where("model_dbs.name = ?", f.ModelName) } if f.Version != "" { - finder = finder.Where("models.version = ?", f.Version) + finder = finder.Where("model_dbs.version = ?", f.Version) } if f.OrderOn != "" { if f.Sort == "desc" { @@ -101,7 +102,7 @@ func GetMeshModelRelationship(db *database.Handler, f RelationshipFilter) (r []R fmt.Println(err.Error()) //for debugging } for _, cm := range componentDefinitionsWithModel { - r = append(r, cm.RelationshipDefinitionDB.GetRelationshipDefinition(cm.Model)) + r = append(r, cm.RelationshipDefinitionDB.GetRelationshipDefinition(cm.ModelDB.GetModel(cm.CategoryDB.GetCategory(db)))) } return r } @@ -132,30 +133,12 @@ func (r RelationshipDefinition) GetID() uuid.UUID { func CreateRelationship(db *database.Handler, r RelationshipDefinition) (uuid.UUID, error) { r.ID = uuid.New() - tempModelID := uuid.New() - byt, err := json.Marshal(r.Model) + mid, err := CreateModel(db, r.Model) if err != nil { return uuid.UUID{}, err } - modelID := uuid.NewSHA1(uuid.UUID{}, byt) - var model Model - modelCreationLock.Lock() - err = db.First(&model, "id = ?", modelID).Error - if err != nil && err != gorm.ErrRecordNotFound { - return uuid.UUID{}, err - } - if model.ID == tempModelID || err == gorm.ErrRecordNotFound { //The model is already not present and needs to be inserted - model = r.Model - model.ID = modelID - err = db.Create(&model).Error - if err != nil { - modelCreationLock.Unlock() - return uuid.UUID{}, err - } - } - modelCreationLock.Unlock() rdb := r.GetRelationshipDefinitionDB() - rdb.ModelID = model.ID + rdb.ModelID = mid err = db.Create(&rdb).Error if err != nil { return uuid.UUID{}, err diff --git a/models/meshmodel/registry.go b/models/meshmodel/registry.go index 684e9886..a6f63f8a 100644 --- a/models/meshmodel/registry.go +++ b/models/meshmodel/registry.go @@ -73,7 +73,8 @@ func NewRegistryManager(db *database.Handler) (*RegistryManager, error) { &v1alpha1.ComponentDefinitionDB{}, &v1alpha1.RelationshipDefinitionDB{}, &v1alpha1.PolicyDefinitionDB{}, - &v1alpha1.Model{}, + &v1alpha1.ModelDB{}, + &v1alpha1.CategoryDB{}, ) if err != nil { return nil, err @@ -85,7 +86,8 @@ func (rm *RegistryManager) Cleanup() { &Registry{}, &Host{}, &v1alpha1.ComponentDefinitionDB{}, - &v1alpha1.Model{}, + &v1alpha1.ModelDB{}, + &v1alpha1.CategoryDB{}, &v1alpha1.RelationshipDefinitionDB{}, ) } @@ -182,31 +184,72 @@ func (rm *RegistryManager) GetEntities(f types.Filter) []Entity { return nil } } -func (rm *RegistryManager) GetModels(f types.Filter) []v1alpha1.Model { - var mod []v1alpha1.Model - finder := rm.db.Model(&mod) +func (rm *RegistryManager) GetModels(db *database.Handler, f types.Filter) []v1alpha1.Model { + var m []v1alpha1.Model + type modelWithCategories struct { + v1alpha1.ModelDB + v1alpha1.CategoryDB + } + + var modelWithCategoriess []modelWithCategories + finder := db.Model(&v1alpha1.ModelDB{}). + Select("model_dbs.*, category_dbs.*"). + Joins("JOIN category_dbs ON model_dbs.category_id = category_dbs.id") // if mf, ok := f.(*v1alpha1.ModelFilter); ok { if mf.Greedy { if mf.Name != "" && mf.DisplayName != "" { - finder = finder.Where("name LIKE ? OR display_name LIKE ?", mf.Name+"%", mf.DisplayName+"%") + finder = finder.Where("model_dbs.name LIKE ? OR model_dbs.display_name LIKE ?", mf.Name+"%", mf.DisplayName+"%") } else if mf.Name != "" { - finder = finder.Where("name LIKE ?", mf.Name+"%") + finder = finder.Where("model_dbs.name LIKE ?", mf.Name+"%") } else if mf.DisplayName != "" { - finder = finder.Where("display_name LIKE ?", mf.DisplayName+"%") + finder = finder.Where("model_dbs.display_name LIKE ?", mf.DisplayName+"%") } } else { if mf.Name != "" { - finder = finder.Where("name = ?", mf.Name) + finder = finder.Where("model_dbs.name = ?", mf.Name) } if mf.DisplayName != "" { - finder = finder.Where("display_name = ?", mf.DisplayName) + finder = finder.Where("model_dbs.display_name = ?", mf.DisplayName) } } if mf.Version != "" { - finder = finder.Where("version = ?", mf.Version) + finder = finder.Where("model_dbs.version = ?", mf.Version) } if mf.Category != "" { - finder = finder.Where("category = ?", mf.Category) + finder = finder.Where("category_dbs.name = ?", mf.Category) + } + if mf.OrderOn != "" { + if mf.Sort == "desc" { + finder = finder.Order(clause.OrderByColumn{Column: clause.Column{Name: mf.OrderOn}, Desc: true}) + } else { + finder = finder.Order(mf.OrderOn) + } + } + if mf.Limit != 0 { + finder = finder.Limit(mf.Limit) + } + if mf.Offset != 0 { + finder = finder.Offset(mf.Offset) + } + } + err := finder. + Scan(&modelWithCategoriess).Error + if err != nil { + fmt.Println(modelWithCategoriess) + fmt.Println(err.Error()) //for debugging + } + for _, modelDB := range modelWithCategoriess { + m = append(m, modelDB.ModelDB.GetModel(modelDB.GetCategory(db))) + } + return m +} +func (rm *RegistryManager) GetCategories(db *database.Handler, f types.Filter) []v1alpha1.Category { + var catdb []v1alpha1.CategoryDB + var cat []v1alpha1.Category + finder := rm.db.Model(&catdb) + if mf, ok := f.(*v1alpha1.CategoryFilter); ok { + if mf.Name != "" { + finder = finder.Where("name = ?", mf.Name) } if mf.OrderOn != "" { if mf.Sort == "desc" { @@ -222,8 +265,11 @@ func (rm *RegistryManager) GetModels(f types.Filter) []v1alpha1.Model { finder = finder.Offset(mf.Offset) } } - _ = finder.Find(&mod).Error - return mod + _ = finder.Find(&catdb).Error + for _, c := range catdb { + cat = append(cat, c.GetCategory(db)) + } + return cat } func (rm *RegistryManager) GetRegistrant(e Entity) Host { eID := e.GetID() diff --git a/models/oam/core/v1alpha1/application_component.go b/models/oam/core/v1alpha1/application_component.go index 1d8d08d2..3d746413 100644 --- a/models/oam/core/v1alpha1/application_component.go +++ b/models/oam/core/v1alpha1/application_component.go @@ -58,6 +58,6 @@ func GetAnnotationsForWorkload(w v1alpha1.ComponentDefinition) map[string]string res[fmt.Sprintf("%s.k8s.APIVersion", MesheryAnnotationPrefix)] = w.APIVersion res[fmt.Sprintf("%s.k8s.Kind", MesheryAnnotationPrefix)] = w.Kind res[fmt.Sprintf("%s.model.version", MesheryAnnotationPrefix)] = w.Model.Version - res[fmt.Sprintf("%s.model.category", MesheryAnnotationPrefix)] = w.Model.Category + res[fmt.Sprintf("%s.model.category", MesheryAnnotationPrefix)] = w.Model.Category.Name return res } diff --git a/utils/artifacthub/package.go b/utils/artifacthub/package.go index 394fc599..8855e8b9 100644 --- a/utils/artifacthub/package.go +++ b/utils/artifacthub/package.go @@ -47,6 +47,9 @@ func (pkg AhPackage) GenerateComponents() ([]v1alpha1.ComponentDefinition, error } comp.Model.Version = pkg.Version comp.Model.Name = pkg.Name + comp.Model.Category = v1alpha1.Category{ + Name: "", + } comp.Model.DisplayName = manifests.FormatToReadableString(comp.Model.Name) components = append(components, comp) }