From 0d372affc2ca1ab54145ae4c8e0a1ade6ec0ef24 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 20 Nov 2025 17:17:07 +0000 Subject: [PATCH 1/7] Initial plan From e0a7578ef8f6bd837607b915ef38ee83560b6e7e Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 20 Nov 2025 17:26:02 +0000 Subject: [PATCH 2/7] Update artifacthub sourceURL logic to use actual package URL Co-authored-by: aabidsofi19 <65964225+aabidsofi19@users.noreply.github.com> --- generators/artifacthub/package_manager.go | 16 ++++- registry/model.go | 83 +++++++++++++++++------ 2 files changed, 78 insertions(+), 21 deletions(-) diff --git a/generators/artifacthub/package_manager.go b/generators/artifacthub/package_manager.go index a6615899..afc01e6b 100644 --- a/generators/artifacthub/package_manager.go +++ b/generators/artifacthub/package_manager.go @@ -2,6 +2,7 @@ package artifacthub import ( "fmt" + "strings" "github.com/meshery/meshkit/generators/models" ) @@ -12,7 +13,20 @@ type ArtifactHubPackageManager struct { } func (ahpm ArtifactHubPackageManager) GetPackage() (models.Package, error) { - // get relevant packages + // Check if SourceURL is an actual URL (from a previous generation) + // If so, create a package directly from it instead of searching + if ahpm.SourceURL != "" && (strings.HasPrefix(ahpm.SourceURL, "http://") || strings.HasPrefix(ahpm.SourceURL, "https://") || strings.HasPrefix(ahpm.SourceURL, "oci://")) { + // SourceURL is an actual URL, use it directly + pkg := AhPackage{ + Name: ahpm.PackageName, + ChartUrl: ahpm.SourceURL, + } + // Try to extract version from the URL or fetch it + _ = pkg.UpdatePackageData() // This might fail but that's okay, ChartUrl is already set + return pkg, nil + } + + // get relevant packages by searching with package name pkgs, err := GetAhPackagesWithName(ahpm.PackageName) if err != nil { return nil, err diff --git a/registry/model.go b/registry/model.go index 6a84abed..c188bf04 100644 --- a/registry/model.go +++ b/registry/model.go @@ -623,17 +623,22 @@ func GenerateComponentsFromPkg(pkg models.Package, compDirPath string, defVersio return 0, 0, err } lengthOfComps := len(comps) + + // Set the source_uri in the model metadata to the actual package URL + if modelDef.Metadata == nil { + modelDef.Metadata = &_model.ModelDefinition_Metadata{} + } + if modelDef.Metadata.AdditionalProperties == nil { + modelDef.Metadata.AdditionalProperties = make(map[string]interface{}) + } + // Get the actual source URL from the package + actualSourceURL := pkg.GetSourceURL() + if actualSourceURL != "" { + modelDef.Metadata.AdditionalProperties["source_uri"] = actualSourceURL + } + for _, comp := range comps { comp.Version = defVersion - if modelDef.Metadata == nil { - modelDef.Metadata = &_model.ModelDefinition_Metadata{} - } - if modelDef.Metadata.AdditionalProperties == nil { - modelDef.Metadata.AdditionalProperties = make(map[string]interface{}) - } - if comp.Model != nil && comp.Model.Metadata != nil && comp.Model.Metadata.AdditionalProperties != nil { - modelDef.Metadata.AdditionalProperties["source_uri"] = comp.Model.Metadata.AdditionalProperties["source_uri"] - } comp.Model = &modelDef AssignDefaultsForCompDefs(&comp, &modelDef) @@ -865,7 +870,34 @@ func InvokeGenerationFromSheet(wg *sync.WaitGroup, path string, modelsheetID, co return } - generator, err := generators.NewGenerator(model.Registrant, model.SourceURL, model.Model) + // Check if we have an existing model definition with a source_uri to use for updates + sourceURL := model.SourceURL + if utils.ReplaceSpacesAndConvertToLowercase(model.Registrant) == "artifacthub" { + // Try to read existing model definition to get the actual package URL + existingModelPath := filepath.Join(path, model.Model) + if entries, err := os.ReadDir(existingModelPath); err == nil && len(entries) > 0 { + // Find the latest version directory + for _, entry := range entries { + if entry.IsDir() { + modelDefPath := filepath.Join(existingModelPath, entry.Name(), defVersion, "model.json") + if existingData, err := os.ReadFile(modelDefPath); err == nil { + var existingModelDef _model.ModelDefinition + if err := encoding.Unmarshal(existingData, &existingModelDef); err == nil { + if existingModelDef.Metadata != nil && existingModelDef.Metadata.AdditionalProperties != nil { + if existingSourceURI, ok := existingModelDef.Metadata.AdditionalProperties["source_uri"].(string); ok && existingSourceURI != "" { + // Use the existing source_uri for updates + sourceURL = existingSourceURI + break + } + } + } + } + } + } + } + } + + generator, err := generators.NewGenerator(model.Registrant, sourceURL, model.Model) if err != nil { err = ErrGenerateModel(err, model.Model) LogError.Error(err) @@ -912,21 +944,25 @@ func InvokeGenerationFromSheet(wg *sync.WaitGroup, path string, modelsheetID, co if alreadyExist { totalAvailableModels-- } + + // Set the source_uri in the model metadata to the actual package URL + if modelDef.Metadata == nil { + modelDef.Metadata = &_model.ModelDefinition_Metadata{} + } + if modelDef.Metadata.AdditionalProperties == nil { + modelDef.Metadata.AdditionalProperties = make(map[string]interface{}) + } + // Get the actual source URL from the package + actualSourceURL := pkg.GetSourceURL() + if actualSourceURL != "" { + modelDef.Metadata.AdditionalProperties["source_uri"] = actualSourceURL + } + for _, comp := range comps { comp.Version = defVersion // Assign the component status corresponding to model status. // i.e., If model is enabled, comps are also "enabled". Ultimately, all individual comps will have the ability to control their status. // The status "enabled" indicates that the component will be registered inside the registry. - if modelDef.Metadata == nil { - modelDef.Metadata = &_model.ModelDefinition_Metadata{} - } - if modelDef.Metadata.AdditionalProperties == nil { - modelDef.Metadata.AdditionalProperties = make(map[string]interface{}) - } - - if comp.Model != nil && comp.Model.Metadata != nil && comp.Model.Metadata.AdditionalProperties != nil { - modelDef.Metadata.AdditionalProperties["source_uri"] = comp.Model.Metadata.AdditionalProperties["source_uri"] - } comp.Model = modelDef AssignDefaultsForCompDefs(&comp, modelDef) @@ -940,6 +976,13 @@ func InvokeGenerationFromSheet(wg *sync.WaitGroup, path string, modelsheetID, co return } } + + // Write the updated model definition with the correct source_uri back to the file system + modelFilePath := filepath.Join(modelDirPath, "model.json") + if err := modelDef.WriteModelDefinition(modelFilePath, "json"); err != nil { + LogError.Error(ErrGenerateModel(err, model.Model)) + } + if !alreadyExist { if len(comps) == 0 { err = ErrGenerateModel(fmt.Errorf("no components found for model"), model.Model) From d27096a4a296457f76f2fcabe40769172f8cbae4 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 20 Nov 2025 17:28:55 +0000 Subject: [PATCH 3/7] Add test for direct URL usage in ArtifactHub package manager Co-authored-by: aabidsofi19 <65964225+aabidsofi19@users.noreply.github.com> --- generators/artifacthub/package_test.go | 52 ++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/generators/artifacthub/package_test.go b/generators/artifacthub/package_test.go index 34b24938..b515c8c8 100644 --- a/generators/artifacthub/package_test.go +++ b/generators/artifacthub/package_test.go @@ -71,3 +71,55 @@ func TestGetChartUrl(t *testing.T) { }) } } + +// TestGetPackageWithDirectURL tests that when SourceURL is an actual URL, +// it is used directly instead of searching +func TestGetPackageWithDirectURL(t *testing.T) { + tests := []struct { + name string + packageName string + sourceURL string + wantErr bool + }{ + { + name: "Direct HTTP URL should be used", + packageName: "consul", + sourceURL: "https://charts.bitnami.com/bitnami/consul-1.0.0.tgz", + wantErr: false, + }, + { + name: "Direct HTTPS URL should be used", + packageName: "test-package", + sourceURL: "https://example.com/charts/test-1.0.0.tgz", + wantErr: false, + }, + { + name: "OCI URL should be used", + packageName: "test-oci", + sourceURL: "oci://registry.example.com/charts/test", + wantErr: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + pm := ArtifactHubPackageManager{ + PackageName: tt.packageName, + SourceURL: tt.sourceURL, + } + + pkg, err := pm.GetPackage() + if (err != nil) != tt.wantErr { + t.Errorf("GetPackage() error = %v, wantErr %v", err, tt.wantErr) + return + } + + if err == nil { + // Verify the package has the correct source URL + if pkg.GetSourceURL() != tt.sourceURL { + t.Errorf("GetSourceURL() = %v, want %v", pkg.GetSourceURL(), tt.sourceURL) + } + } + }) + } +} From 2d3f1ad4cf01f674341d7c5a00ef4fccf7f9eacc Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 21 Nov 2025 12:07:35 +0000 Subject: [PATCH 4/7] Update CSV model's SourceURL to actual package URL for spreadsheet persistence Co-authored-by: aabidsofi19 <65964225+aabidsofi19@users.noreply.github.com> --- registry/model.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/registry/model.go b/registry/model.go index c188bf04..7dee0b94 100644 --- a/registry/model.go +++ b/registry/model.go @@ -956,6 +956,8 @@ func InvokeGenerationFromSheet(wg *sync.WaitGroup, path string, modelsheetID, co actualSourceURL := pkg.GetSourceURL() if actualSourceURL != "" { modelDef.Metadata.AdditionalProperties["source_uri"] = actualSourceURL + // Update the CSV model's SourceURL to the actual package URL for spreadsheet updates + model.SourceURL = actualSourceURL } for _, comp := range comps { From ce0d4e34e0a9b615682fdcad0e91e20a71f31257 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 21 Nov 2025 12:16:12 +0000 Subject: [PATCH 5/7] Remove redundant disk reading logic - use CSV SourceURL directly Co-authored-by: aabidsofi19 <65964225+aabidsofi19@users.noreply.github.com> --- registry/model.go | 30 ++---------------------------- 1 file changed, 2 insertions(+), 28 deletions(-) diff --git a/registry/model.go b/registry/model.go index 7dee0b94..e2708949 100644 --- a/registry/model.go +++ b/registry/model.go @@ -870,34 +870,8 @@ func InvokeGenerationFromSheet(wg *sync.WaitGroup, path string, modelsheetID, co return } - // Check if we have an existing model definition with a source_uri to use for updates - sourceURL := model.SourceURL - if utils.ReplaceSpacesAndConvertToLowercase(model.Registrant) == "artifacthub" { - // Try to read existing model definition to get the actual package URL - existingModelPath := filepath.Join(path, model.Model) - if entries, err := os.ReadDir(existingModelPath); err == nil && len(entries) > 0 { - // Find the latest version directory - for _, entry := range entries { - if entry.IsDir() { - modelDefPath := filepath.Join(existingModelPath, entry.Name(), defVersion, "model.json") - if existingData, err := os.ReadFile(modelDefPath); err == nil { - var existingModelDef _model.ModelDefinition - if err := encoding.Unmarshal(existingData, &existingModelDef); err == nil { - if existingModelDef.Metadata != nil && existingModelDef.Metadata.AdditionalProperties != nil { - if existingSourceURI, ok := existingModelDef.Metadata.AdditionalProperties["source_uri"].(string); ok && existingSourceURI != "" { - // Use the existing source_uri for updates - sourceURL = existingSourceURI - break - } - } - } - } - } - } - } - } - - generator, err := generators.NewGenerator(model.Registrant, sourceURL, model.Model) + // Use the SourceURL from the CSV, which now contains the actual package URL after first generation + generator, err := generators.NewGenerator(model.Registrant, model.SourceURL, model.Model) if err != nil { err = ErrGenerateModel(err, model.Model) LogError.Error(err) From 2c258b1a9e67f2c32b87839f4500d9b5c227c72a Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 21 Nov 2025 12:20:05 +0000 Subject: [PATCH 6/7] Add documentation for ArtifactHub generator flow Co-authored-by: aabidsofi19 <65964225+aabidsofi19@users.noreply.github.com> --- generators/readme.md | 66 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) diff --git a/generators/readme.md b/generators/readme.md index e69de29b..c2c71c91 100644 --- a/generators/readme.md +++ b/generators/readme.md @@ -0,0 +1,66 @@ +# Generators + +Generators are responsible for creating Meshery components from various sources like ArtifactHub and GitHub. + +## ArtifactHub Generator + +The ArtifactHub generator creates Meshery components by discovering Helm charts from [ArtifactHub](https://artifacthub.io/). + +### How It Works + +#### First Generation (New Model) + +When a model is generated for the first time with `registrant: artifacthub`: + +1. **CSV Input**: The model's `SourceURL` field contains a search query (e.g., `"consul"`) +2. **Search ArtifactHub**: The generator searches ArtifactHub for packages matching the name +3. **Package Selection**: The best package is selected based on ranking (verified publisher, CNCF, official, etc.) +4. **Package Resolution**: The selected package's actual chart URL is obtained (e.g., `"https://charts.bitnami.com/bitnami/consul-1.0.0.tgz"`) +5. **Component Generation**: Components are generated from the Helm chart's CRDs +6. **Persistence**: + - Model metadata `source_uri` is set to the actual package URL + - CSV model's `SourceURL` is updated to the actual package URL + - Model definition is written to filesystem + - Updated CSV is sent to spreadsheet with the resolved URL + +#### Subsequent Updates (Existing Model) + +When the model is updated in subsequent runs: + +1. **CSV Input**: The model's `SourceURL` field contains the actual package URL (e.g., `"https://charts.bitnami.com/bitnami/consul-1.0.0.tgz"`) +2. **Direct URL Usage**: The generator detects the URL and uses it directly (no search needed) +3. **Package Fetch**: The package is fetched from the URL +4. **Component Generation**: Components are generated from the Helm chart's CRDs + +### Benefits + +- **Consistency**: The actual package URL is stored and used everywhere +- **Efficiency**: Subsequent updates don't need to search ArtifactHub again +- **Reliability**: Using direct URLs ensures we get the same package every time +- **Backward Compatible**: Still supports search queries for new models + +### URL Detection + +The generator automatically detects if `SourceURL` is: +- **A search query**: Any string that doesn't start with `http://`, `https://`, or `oci://` +- **A direct URL**: Any string starting with `http://`, `https://`, or `oci://` + +### Code Structure + +``` +generators/ +├── artifacthub/ +│ ├── package_manager.go # Main entry point, detects URL vs search query +│ ├── scanner.go # Searches ArtifactHub API +│ ├── ranker.go # Ranks packages by quality metrics +│ └── package.go # Handles package data and component generation +└── generator.go # Factory for creating generators +``` + +## GitHub Generator + +The GitHub generator creates Meshery components from GitHub repositories containing Kubernetes manifests or Helm charts. + +### Usage + +Set `registrant: github` in the model CSV and provide a GitHub URL in the `SourceURL` field. From 56606e1302dca0aab26c98e2d9a58471b29d79cb Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 24 Nov 2025 13:31:14 +0000 Subject: [PATCH 7/7] Remove duplicate model write - use existing writeModelDefToFileSystem with source_uri support Co-authored-by: aabidsofi19 <65964225+aabidsofi19@users.noreply.github.com> --- registry/model.go | 37 +++++++++++++++---------------------- 1 file changed, 15 insertions(+), 22 deletions(-) diff --git a/registry/model.go b/registry/model.go index e2708949..8ad0e764 100644 --- a/registry/model.go +++ b/registry/model.go @@ -98,7 +98,7 @@ type ModelCSV struct { } var modelMetadataValues = []string{ - "primaryColor", "secondaryColor", "svgColor", "svgWhite", "svgComplete", "styleOverrides", "capabilities", "isAnnotation", "shape", + "primaryColor", "secondaryColor", "svgColor", "svgWhite", "svgComplete", "styleOverrides", "capabilities", "isAnnotation", "shape", "sourceURL", } // keep @@ -162,6 +162,11 @@ func (m *ModelCSV) UpdateModelDefinition(modelDef *_model.ModelDefinition) error isAnnotation = true } metadata.IsAnnotation = &isAnnotation + case "sourceURL": + // Store SourceURL as source_uri in metadata for tracking the actual package URL + if m.SourceURL != "" { + metadata.AdditionalProperties["source_uri"] = m.SourceURL + } default: // For keys that do not have a direct mapping, store them in AdditionalProperties metadata.AdditionalProperties[key] = modelMetadata[key] @@ -909,6 +914,15 @@ func InvokeGenerationFromSheet(wg *sync.WaitGroup, path string, modelsheetID, co LogError.Error(err) return } + + // Get the actual source URL from the package and update the CSV model's SourceURL + // This needs to happen before writeModelDefToFileSystem so the model is written with the correct source_uri + actualSourceURL := pkg.GetSourceURL() + if actualSourceURL != "" { + // Update the CSV model's SourceURL to the actual package URL for spreadsheet updates + model.SourceURL = actualSourceURL + } + modelDef, alreadyExist, err := writeModelDefToFileSystem(&model, version, modelDirPath) if err != nil { err = ErrGenerateModel(err, model.Model) @@ -918,21 +932,6 @@ func InvokeGenerationFromSheet(wg *sync.WaitGroup, path string, modelsheetID, co if alreadyExist { totalAvailableModels-- } - - // Set the source_uri in the model metadata to the actual package URL - if modelDef.Metadata == nil { - modelDef.Metadata = &_model.ModelDefinition_Metadata{} - } - if modelDef.Metadata.AdditionalProperties == nil { - modelDef.Metadata.AdditionalProperties = make(map[string]interface{}) - } - // Get the actual source URL from the package - actualSourceURL := pkg.GetSourceURL() - if actualSourceURL != "" { - modelDef.Metadata.AdditionalProperties["source_uri"] = actualSourceURL - // Update the CSV model's SourceURL to the actual package URL for spreadsheet updates - model.SourceURL = actualSourceURL - } for _, comp := range comps { comp.Version = defVersion @@ -953,12 +952,6 @@ func InvokeGenerationFromSheet(wg *sync.WaitGroup, path string, modelsheetID, co } } - // Write the updated model definition with the correct source_uri back to the file system - modelFilePath := filepath.Join(modelDirPath, "model.json") - if err := modelDef.WriteModelDefinition(modelFilePath, "json"); err != nil { - LogError.Error(ErrGenerateModel(err, model.Model)) - } - if !alreadyExist { if len(comps) == 0 { err = ErrGenerateModel(fmt.Errorf("no components found for model"), model.Model)