HMS-4934: Parse module metadata

go.mod

@@ -13,6 +13,7 @@ require (
 require (
 	github.com/cloudflare/circl v1.3.9 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/mitchellh/mapstructure v1.5.0 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/stretchr/objx v0.5.2 // indirect
 	golang.org/x/crypto v0.24.0 // indirect

go.sum

@@ -10,6 +10,8 @@ github.com/h2non/filetype v1.1.3 h1:FKkx9QbD7HR/zjK1Ia5XiBsq9zdLi5Kf3zGyFTAFkGg=
 github.com/h2non/filetype v1.1.3/go.mod h1:319b3zT68BvV+WRj7cwy856M2ehB3HqNOt6sy1HndBY=
 github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA=
 github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
+github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
+github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY=

pkg/yum/mocks/repomd.xml

@@ -35,4 +35,9 @@
 <checksum type="sha256">1a3f4adf9a598d5badaaef70e67a0f02198c68ca118f5543a91c3fd8ca95c6aa</checksum>
 <timestamp>1299190192</timestamp>
 </data>
+<data type="modules">
+    <location href="repodata/module.yaml.zst"/>
+    <checksum type="sha256">4307ecf77fe1abaf567a15336c5141d813ae223602d2bc4cd606b94fd9269fd4</checksum>
+    <timestamp>1299190192</timestamp>
+</data>
 </repomd>

pkg/yum/module_stream.go

@@ -0,0 +1,131 @@
+package yum
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+
+	"github.com/mitchellh/mapstructure"
+	"gopkg.in/yaml.v3"
+)
+
+// Better userfacing struct
+type ModuleStream struct {
+	Name    string
+	Streams []Stream
+}
+
+type Stream struct {
+	Name        string                 `mapstructure:"name"`
+	Stream      string                 `mapstructure:"stream"`
+	Version     string                 `mapstructure:"version"`
+	Context     string                 `mapstructure:"context"`
+	Arch        string                 `mapstructure:"arch"`
+	Summary     string                 `mapstructure:"summary"`
+	Description string                 `mapstructure:"description"`
+	Artifacts   Artifacts              `mapstructure:"artifacts"`
+	Profiles    map[string]RpmProfiles `mapstructure:"profiles"`
+}
+
+type RpmProfiles struct {
+	Rpms []string `mapstructure:"rpms"`
+}
+
+type Artifacts struct {
+	Rpms []string `mapstructure:"rpms"`
+}
+
+type ModuleMD struct {
+	Document string `mapstructure:"document"`
+	Version  int    `mapstructure:"version"`
+	Data     Stream `yaml:"data"`
+}
+
+// ModuleMDs Returns the modulemd documents from the "modules" metadata in the given yum repository
+func (r *Repository) ModuleMDs(ctx context.Context) ([]ModuleMD, int, error) {
+	var modulesURL *string
+	var err error
+	var resp *http.Response
+	var moduleMDs []ModuleMD
+
+	if r.moduleMDs != nil {
+		return r.moduleMDs, 200, nil
+	}
+
+	if _, _, err := r.Repomd(ctx); err != nil {
+		return nil, 0, fmt.Errorf("error parsing repomd.xml: %w", err)
+	}
+
+	if modulesURL, err = r.getModulesURL(); err != nil {
+		return nil, 0, fmt.Errorf("error parsing modules md URL: %w", err)
+	}
+
+	if modulesURL != nil {
+		req, err := http.NewRequestWithContext(ctx, http.MethodGet, *modulesURL, nil)
+		if err != nil {
+			return nil, 0, fmt.Errorf("error creating request: %w", err)
+		}
+
+		if resp, err = r.settings.Client.Do(req); err != nil {
+			return nil, erroredStatusCode(resp), fmt.Errorf("GET error for file %v: %w", modulesURL, err)
+		}
+		defer resp.Body.Close()
+
+		if moduleMDs, err = parseModuleMDs(resp.Body); err != nil {
+			return nil, resp.StatusCode, fmt.Errorf("error parsing comps.xml: %w", err)
+		}
+
+		return moduleMDs, resp.StatusCode, nil
+	}
+	r.moduleMDs = moduleMDs
+	return moduleMDs, 0, err
+}
+
+// parses modulemd objects from a given io reader
+// modules yaml files include different types of documents which is hard to parse
+// this implements a two step process:
+//
+//	Parse each document into a map, with the value of interface, and then
+//	use mapstructure to parse the interface into a ModuleMD struct
+func parseModuleMDs(body io.ReadCloser) ([]ModuleMD, error) {
+	moduleMDs := make([]ModuleMD, 0)
+
+	reader, err := ExtractIfCompressed(body)
+	if err != nil {
+		return moduleMDs, fmt.Errorf("error extracting compressed streams: %w", err)
+	}
+
+	decoder := yaml.NewDecoder(reader)
+	for {
+		var doc map[string]interface{}
+
+		// Decode the next document
+		err := decoder.Decode(&doc)
+		if err != nil {
+			if errors.Is(err, io.EOF) {
+				break
+			}
+			return nil, fmt.Errorf("error decoding streams: %w", err)
+		}
+		// Only care about modulemds right now
+		if doc["document"] == "modulemd" {
+			var module ModuleMD
+			config := &mapstructure.DecoderConfig{
+				WeaklyTypedInput: true,
+				Result:           &module,
+			}
+			mapDecode, err := mapstructure.NewDecoder(config)
+			if err != nil {
+				return moduleMDs, fmt.Errorf("error creating map decoder: %w", err)
+			}
+			err = mapDecode.Decode(doc)
+			if err != nil {
+				return nil, fmt.Errorf("error decoding map: %w", err)
+			}
+			moduleMDs = append(moduleMDs, module)
+		}
+	}
+	return moduleMDs, nil
+}

pkg/yum/module_stream_test.go

@@ -0,0 +1,47 @@
+package yum
+
+import (
+	_ "embed"
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestParseModuleMDs(t *testing.T) {
+	f, err := os.Open("mocks/module.yaml.zst")
+	assert.NoError(t, err)
+
+	parsed, err := parseModuleMDs(f)
+	assert.NoError(t, err)
+	assert.Equal(t, 11, len(parsed))
+	assert.NotEmpty(t, parsed[0].Data.Name)
+	assert.NotEmpty(t, parsed[0].Data.Artifacts.Rpms)
+}
+
+func TestParseRhel8Modules(t *testing.T) {
+	f, err := os.Open("mocks/rhel8.modules.yaml.gz")
+	assert.NoError(t, err)
+	defer f.Close()
+	require.NoError(t, err)
+
+	modules, err := parseModuleMDs(f)
+	require.NoError(t, err)
+
+	assert.Len(t, modules, 862)
+
+	assert.NotEmpty(t, modules)
+	found := false
+	for _, module := range modules {
+		if module.Data.Name == "ruby" && module.Data.Stream == "2.5" {
+			found = true
+			assert.NotEmpty(t, module.Data.Artifacts.Rpms)
+			assert.NotEmpty(t, module.Data.Profiles)
+			value, ok := module.Data.Profiles["common"]
+			assert.True(t, ok)
+			assert.Equal(t, []string{"ruby"}, value.Rpms)
+		}
+	}
+	assert.True(t, found)
+}

pkg/yum/repository.go

@@ -2,7 +2,6 @@ package yum
 
 import (
 	"bufio"
-	"bytes"
 	"compress/gzip"
 	"context"
 	"encoding/xml"
@@ -97,6 +96,7 @@ type YumRepository interface {
 	Packages(ctx context.Context) (packages []Package, statusCode int, err error)
 	Repomd(ctx context.Context) (repomd *Repomd, statusCode int, err error)
 	Signature(ctx context.Context) (repomdSignature *string, statusCode int, err error)
+	ModuleMDs(ctx context.Context) ([]ModuleMD, int, error)
 	Comps(ctx context.Context) (comps *Comps, statusCode int, err error)
 	PackageGroups(ctx context.Context) (packageGroups []PackageGroup, statusCode int, err error)
 	Environments(ctx context.Context) (environments []Environment, statusCode int, err error)
@@ -105,10 +105,11 @@ type YumRepository interface {
 
 type Repository struct {
 	settings        YummySettings
-	packages        []Package // Packages repository contains
-	repomdSignature *string   // Signature of the repository
-	repomd          *Repomd   // Repomd of the repository
-	comps           *Comps    // Comps of the repository
+	packages        []Package  // Packages repository contains
+	repomdSignature *string    // Signature of the repository
+	repomd          *Repomd    // Repomd of the repository
+	comps           *Comps     // Comps of the repository
+	moduleMDs       []ModuleMD // Module md documents of the repository, used to compute moduleStreams
 }
 
 func NewRepository(settings YummySettings) (Repository, error) {
@@ -374,6 +375,29 @@ func (r *Repository) getCompsURL() (*string, error) {
 	return Ptr(url.String()), nil
 }
 
+func (r *Repository) getModulesURL() (*string, error) {
+	var compsLocation string
+
+	for _, data := range r.repomd.Data {
+		if data.Type == "modules_gz" {
+			compsLocation = data.Location.Href
+		} else if data.Type == "modules" {
+			compsLocation = data.Location.Href
+		}
+	}
+
+	if compsLocation == "" {
+		return nil, nil
+	}
+
+	URL, err := url.Parse(*r.settings.URL)
+	if err != nil {
+		return nil, err
+	}
+	URL.Path = path.Join(URL.Path, compsLocation)
+	return Ptr(URL.String()), nil
+}
+
 func (r *Repository) getSignatureURL() (string, error) {
 	url, err := r.getRepomdURL()
 	if err == nil {
@@ -442,32 +466,11 @@ func ParseCompsXML(body io.ReadCloser, url *string) (Comps, error) {
 	packageGroups := []PackageGroup{}
 	environments := []Environment{}
 
-	byteValue, err := io.ReadAll(body)
-	if err != nil {
-		return comps, fmt.Errorf("io.reader read failure: %w", err)
-	}
-
 	// determine the file type from the header
-	bufferedReader := bufio.NewReader(bytes.NewReader(byteValue))
-	header, err := bufferedReader.Peek(20)
+	reader, err := ExtractIfCompressed(body)
 	if err != nil {
 		return comps, err
 	}
-	fileType, err := filetype.Match(header)
-	if err != nil {
-		return comps, err
-	}
-
-	// handle compressed comps
-	if fileType == matchers.TypeGz || fileType == matchers.TypeZstd || fileType == matchers.TypeXz {
-		reader, err = ParseCompressedData(bytes.NewReader(byteValue))
-		if err != nil {
-			return comps, err
-		}
-		// handle uncompressed comps
-	} else {
-		reader = bytes.NewReader(byteValue)
-	}
 
 	decoder := xml.NewDecoder(reader)
 

pkg/yum/repository_test.go

@@ -26,6 +26,9 @@ var compsXML []byte
 //go:embed "mocks/repomd.xml.asc"
 var signatureXML []byte
 
+//go:embed "mocks/module.yaml.zst"
+var moduleYamlZst []byte
+
 func TestConfigure(t *testing.T) {
 	firstURL := "http://first.example.com"
 	firstClient := &http.Client{}
@@ -133,6 +136,10 @@ func TestFetchRepomd(t *testing.T) {
 				Type:     "updateinfo",
 				Location: Location{Href: "repodata/updateinfo.xml.gz"},
 			},
+			{
+				Type:     "modules",
+				Location: Location{Href: "repodata/module.yaml.zst"},
+			},
 		},
 		Revision:     "1308257578",
 		RepomdString: &repomdStringMock,
@@ -361,6 +368,7 @@ func server() *httptest.Server {
 	mux.HandleFunc("/repodata/primary.xml.gz", servePrimaryXML)
 	mux.HandleFunc("/repodata/comps.xml", serveCompsXML)
 	mux.HandleFunc("/repodata/repomd.xml.asc", serveSignatureXML)
+	mux.HandleFunc("/repodata/module.yaml.zst", serveModulesMd)
 	mux.HandleFunc("/gpgkey.pub", serveGPGKey)
 	return httptest.NewServer(mux)
 }
@@ -388,3 +396,10 @@ func serveSignatureXML(w http.ResponseWriter, r *http.Request) {
 	body := signatureXML
 	_, _ = w.Write(body)
 }
+
+func serveModulesMd(w http.ResponseWriter, r *http.Request) {
+	w.Header().Add("Content-Type", "text/xml")
+
+	body := moduleYamlZst
+	_, _ = w.Write(body)
+}

pkg/yum/utils.go

@@ -1,6 +1,38 @@
 package yum
 
+import (
+	"bufio"
+	"io"
+
+	"github.com/h2non/filetype"
+	"github.com/h2non/filetype/matchers"
+)
+
 // Converts any struct to a pointer to that struct
 func Ptr[T any](item T) *T {
 	return &item
 }
+
+func ExtractIfCompressed(reader io.ReadCloser) (extractedReader io.Reader, err error) {
+	bufferedReader := bufio.NewReader(reader)
+	header, err := bufferedReader.Peek(20)
+	if err != nil {
+		return nil, err
+	}
+	fileType, err := filetype.Match(header)
+	if err != nil {
+		return nil, err
+	}
+
+	// handle compressed file
+	if fileType == matchers.TypeGz || fileType == matchers.TypeZstd || fileType == matchers.TypeXz {
+		extractedReader, err = ParseCompressedData(bufferedReader)
+		if err != nil {
+			return nil, err
+		}
+		return extractedReader, nil
+	} else {
+		// handle uncompressed comps
+		return bufferedReader, nil
+	}
+}