diff --git a/go.mod b/go.mod
index 5ab5509..5998467 100644
--- a/go.mod
+++ b/go.mod
@@ -13,6 +13,7 @@ require (
require (
github.com/cloudflare/circl v1.3.9 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
+ github.com/mitchellh/mapstructure v1.5.0 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/stretchr/objx v0.5.2 // indirect
golang.org/x/crypto v0.24.0 // indirect
diff --git a/go.sum b/go.sum
index 091a6de..d093501 100644
--- a/go.sum
+++ b/go.sum
@@ -10,6 +10,8 @@ github.com/h2non/filetype v1.1.3 h1:FKkx9QbD7HR/zjK1Ia5XiBsq9zdLi5Kf3zGyFTAFkGg=
github.com/h2non/filetype v1.1.3/go.mod h1:319b3zT68BvV+WRj7cwy856M2ehB3HqNOt6sy1HndBY=
github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA=
github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
+github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
+github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY=
diff --git a/pkg/yum/mocks/module.yaml.zst b/pkg/yum/mocks/module.yaml.zst
new file mode 100644
index 0000000..e5ac50c
Binary files /dev/null and b/pkg/yum/mocks/module.yaml.zst differ
diff --git a/pkg/yum/mocks/repomd.xml b/pkg/yum/mocks/repomd.xml
index de67a56..556a195 100644
--- a/pkg/yum/mocks/repomd.xml
+++ b/pkg/yum/mocks/repomd.xml
@@ -35,4 +35,9 @@
1a3f4adf9a598d5badaaef70e67a0f02198c68ca118f5543a91c3fd8ca95c6aa
1299190192
+
+
+ 4307ecf77fe1abaf567a15336c5141d813ae223602d2bc4cd606b94fd9269fd4
+ 1299190192
+
\ No newline at end of file
diff --git a/pkg/yum/mocks/rhel8.modules.yaml.gz b/pkg/yum/mocks/rhel8.modules.yaml.gz
new file mode 100644
index 0000000..92dd53c
Binary files /dev/null and b/pkg/yum/mocks/rhel8.modules.yaml.gz differ
diff --git a/pkg/yum/module_stream.go b/pkg/yum/module_stream.go
new file mode 100644
index 0000000..df1231e
--- /dev/null
+++ b/pkg/yum/module_stream.go
@@ -0,0 +1,131 @@
+package yum
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "io"
+ "net/http"
+
+ "github.com/mitchellh/mapstructure"
+ "gopkg.in/yaml.v3"
+)
+
+// Better userfacing struct
+type ModuleStream struct {
+ Name string
+ Streams []Stream
+}
+
+type Stream struct {
+ Name string `mapstructure:"name"`
+ Stream string `mapstructure:"stream"`
+ Version string `mapstructure:"version"`
+ Context string `mapstructure:"context"`
+ Arch string `mapstructure:"arch"`
+ Summary string `mapstructure:"summary"`
+ Description string `mapstructure:"description"`
+ Artifacts Artifacts `mapstructure:"artifacts"`
+ Profiles map[string]RpmProfiles `mapstructure:"profiles"`
+}
+
+type RpmProfiles struct {
+ Rpms []string `mapstructure:"rpms"`
+}
+
+type Artifacts struct {
+ Rpms []string `mapstructure:"rpms"`
+}
+
+type ModuleMD struct {
+ Document string `mapstructure:"document"`
+ Version int `mapstructure:"version"`
+ Data Stream `yaml:"data"`
+}
+
+// ModuleMDs Returns the modulemd documents from the "modules" metadata in the given yum repository
+func (r *Repository) ModuleMDs(ctx context.Context) ([]ModuleMD, int, error) {
+ var modulesURL *string
+ var err error
+ var resp *http.Response
+ var moduleMDs []ModuleMD
+
+ if r.moduleMDs != nil {
+ return r.moduleMDs, 200, nil
+ }
+
+ if _, _, err := r.Repomd(ctx); err != nil {
+ return nil, 0, fmt.Errorf("error parsing repomd.xml: %w", err)
+ }
+
+ if modulesURL, err = r.getModulesURL(); err != nil {
+ return nil, 0, fmt.Errorf("error parsing modules md URL: %w", err)
+ }
+
+ if modulesURL != nil {
+ req, err := http.NewRequestWithContext(ctx, http.MethodGet, *modulesURL, nil)
+ if err != nil {
+ return nil, 0, fmt.Errorf("error creating request: %w", err)
+ }
+
+ if resp, err = r.settings.Client.Do(req); err != nil {
+ return nil, erroredStatusCode(resp), fmt.Errorf("GET error for file %v: %w", modulesURL, err)
+ }
+ defer resp.Body.Close()
+
+ if moduleMDs, err = parseModuleMDs(resp.Body); err != nil {
+ return nil, resp.StatusCode, fmt.Errorf("error parsing comps.xml: %w", err)
+ }
+
+ return moduleMDs, resp.StatusCode, nil
+ }
+ r.moduleMDs = moduleMDs
+ return moduleMDs, 0, err
+}
+
+// parses modulemd objects from a given io reader
+// modules yaml files include different types of documents which is hard to parse
+// this implements a two step process:
+//
+// Parse each document into a map, with the value of interface, and then
+// use mapstructure to parse the interface into a ModuleMD struct
+func parseModuleMDs(body io.ReadCloser) ([]ModuleMD, error) {
+ moduleMDs := make([]ModuleMD, 0)
+
+ reader, err := ExtractIfCompressed(body)
+ if err != nil {
+ return moduleMDs, fmt.Errorf("error extracting compressed streams: %w", err)
+ }
+
+ decoder := yaml.NewDecoder(reader)
+ for {
+ var doc map[string]interface{}
+
+ // Decode the next document
+ err := decoder.Decode(&doc)
+ if err != nil {
+ if errors.Is(err, io.EOF) {
+ break
+ }
+ return nil, fmt.Errorf("error decoding streams: %w", err)
+ }
+ // Only care about modulemds right now
+ if doc["document"] == "modulemd" {
+ var module ModuleMD
+ config := &mapstructure.DecoderConfig{
+ WeaklyTypedInput: true,
+ Result: &module,
+ }
+ mapDecode, err := mapstructure.NewDecoder(config)
+ if err != nil {
+ return moduleMDs, fmt.Errorf("error creating map decoder: %w", err)
+ }
+ err = mapDecode.Decode(doc)
+ if err != nil {
+ return nil, fmt.Errorf("error decoding map: %w", err)
+ }
+ moduleMDs = append(moduleMDs, module)
+ }
+ }
+ return moduleMDs, nil
+}
diff --git a/pkg/yum/module_stream_test.go b/pkg/yum/module_stream_test.go
new file mode 100644
index 0000000..3c75a95
--- /dev/null
+++ b/pkg/yum/module_stream_test.go
@@ -0,0 +1,47 @@
+package yum
+
+import (
+ _ "embed"
+ "os"
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+func TestParseModuleMDs(t *testing.T) {
+ f, err := os.Open("mocks/module.yaml.zst")
+ assert.NoError(t, err)
+
+ parsed, err := parseModuleMDs(f)
+ assert.NoError(t, err)
+ assert.Equal(t, 11, len(parsed))
+ assert.NotEmpty(t, parsed[0].Data.Name)
+ assert.NotEmpty(t, parsed[0].Data.Artifacts.Rpms)
+}
+
+func TestParseRhel8Modules(t *testing.T) {
+ f, err := os.Open("mocks/rhel8.modules.yaml.gz")
+ assert.NoError(t, err)
+ defer f.Close()
+ require.NoError(t, err)
+
+ modules, err := parseModuleMDs(f)
+ require.NoError(t, err)
+
+ assert.Len(t, modules, 862)
+
+ assert.NotEmpty(t, modules)
+ found := false
+ for _, module := range modules {
+ if module.Data.Name == "ruby" && module.Data.Stream == "2.5" {
+ found = true
+ assert.NotEmpty(t, module.Data.Artifacts.Rpms)
+ assert.NotEmpty(t, module.Data.Profiles)
+ value, ok := module.Data.Profiles["common"]
+ assert.True(t, ok)
+ assert.Equal(t, []string{"ruby"}, value.Rpms)
+ }
+ }
+ assert.True(t, found)
+}
diff --git a/pkg/yum/repository.go b/pkg/yum/repository.go
index 3f73945..82249ea 100644
--- a/pkg/yum/repository.go
+++ b/pkg/yum/repository.go
@@ -2,7 +2,6 @@ package yum
import (
"bufio"
- "bytes"
"compress/gzip"
"context"
"encoding/xml"
@@ -97,6 +96,7 @@ type YumRepository interface {
Packages(ctx context.Context) (packages []Package, statusCode int, err error)
Repomd(ctx context.Context) (repomd *Repomd, statusCode int, err error)
Signature(ctx context.Context) (repomdSignature *string, statusCode int, err error)
+ ModuleMDs(ctx context.Context) ([]ModuleMD, int, error)
Comps(ctx context.Context) (comps *Comps, statusCode int, err error)
PackageGroups(ctx context.Context) (packageGroups []PackageGroup, statusCode int, err error)
Environments(ctx context.Context) (environments []Environment, statusCode int, err error)
@@ -105,10 +105,11 @@ type YumRepository interface {
type Repository struct {
settings YummySettings
- packages []Package // Packages repository contains
- repomdSignature *string // Signature of the repository
- repomd *Repomd // Repomd of the repository
- comps *Comps // Comps of the repository
+ packages []Package // Packages repository contains
+ repomdSignature *string // Signature of the repository
+ repomd *Repomd // Repomd of the repository
+ comps *Comps // Comps of the repository
+ moduleMDs []ModuleMD // Module md documents of the repository, used to compute moduleStreams
}
func NewRepository(settings YummySettings) (Repository, error) {
@@ -374,6 +375,29 @@ func (r *Repository) getCompsURL() (*string, error) {
return Ptr(url.String()), nil
}
+func (r *Repository) getModulesURL() (*string, error) {
+ var compsLocation string
+
+ for _, data := range r.repomd.Data {
+ if data.Type == "modules_gz" {
+ compsLocation = data.Location.Href
+ } else if data.Type == "modules" {
+ compsLocation = data.Location.Href
+ }
+ }
+
+ if compsLocation == "" {
+ return nil, nil
+ }
+
+ URL, err := url.Parse(*r.settings.URL)
+ if err != nil {
+ return nil, err
+ }
+ URL.Path = path.Join(URL.Path, compsLocation)
+ return Ptr(URL.String()), nil
+}
+
func (r *Repository) getSignatureURL() (string, error) {
url, err := r.getRepomdURL()
if err == nil {
@@ -442,32 +466,11 @@ func ParseCompsXML(body io.ReadCloser, url *string) (Comps, error) {
packageGroups := []PackageGroup{}
environments := []Environment{}
- byteValue, err := io.ReadAll(body)
- if err != nil {
- return comps, fmt.Errorf("io.reader read failure: %w", err)
- }
-
// determine the file type from the header
- bufferedReader := bufio.NewReader(bytes.NewReader(byteValue))
- header, err := bufferedReader.Peek(20)
+ reader, err := ExtractIfCompressed(body)
if err != nil {
return comps, err
}
- fileType, err := filetype.Match(header)
- if err != nil {
- return comps, err
- }
-
- // handle compressed comps
- if fileType == matchers.TypeGz || fileType == matchers.TypeZstd || fileType == matchers.TypeXz {
- reader, err = ParseCompressedData(bytes.NewReader(byteValue))
- if err != nil {
- return comps, err
- }
- // handle uncompressed comps
- } else {
- reader = bytes.NewReader(byteValue)
- }
decoder := xml.NewDecoder(reader)
diff --git a/pkg/yum/repository_test.go b/pkg/yum/repository_test.go
index 906b135..bf61eb2 100644
--- a/pkg/yum/repository_test.go
+++ b/pkg/yum/repository_test.go
@@ -26,6 +26,9 @@ var compsXML []byte
//go:embed "mocks/repomd.xml.asc"
var signatureXML []byte
+//go:embed "mocks/module.yaml.zst"
+var moduleYamlZst []byte
+
func TestConfigure(t *testing.T) {
firstURL := "http://first.example.com"
firstClient := &http.Client{}
@@ -133,6 +136,10 @@ func TestFetchRepomd(t *testing.T) {
Type: "updateinfo",
Location: Location{Href: "repodata/updateinfo.xml.gz"},
},
+ {
+ Type: "modules",
+ Location: Location{Href: "repodata/module.yaml.zst"},
+ },
},
Revision: "1308257578",
RepomdString: &repomdStringMock,
@@ -361,6 +368,7 @@ func server() *httptest.Server {
mux.HandleFunc("/repodata/primary.xml.gz", servePrimaryXML)
mux.HandleFunc("/repodata/comps.xml", serveCompsXML)
mux.HandleFunc("/repodata/repomd.xml.asc", serveSignatureXML)
+ mux.HandleFunc("/repodata/module.yaml.zst", serveModulesMd)
mux.HandleFunc("/gpgkey.pub", serveGPGKey)
return httptest.NewServer(mux)
}
@@ -388,3 +396,10 @@ func serveSignatureXML(w http.ResponseWriter, r *http.Request) {
body := signatureXML
_, _ = w.Write(body)
}
+
+func serveModulesMd(w http.ResponseWriter, r *http.Request) {
+ w.Header().Add("Content-Type", "text/xml")
+
+ body := moduleYamlZst
+ _, _ = w.Write(body)
+}
diff --git a/pkg/yum/utils.go b/pkg/yum/utils.go
index 5aff24e..dcb4213 100644
--- a/pkg/yum/utils.go
+++ b/pkg/yum/utils.go
@@ -1,6 +1,38 @@
package yum
+import (
+ "bufio"
+ "io"
+
+ "github.com/h2non/filetype"
+ "github.com/h2non/filetype/matchers"
+)
+
// Converts any struct to a pointer to that struct
func Ptr[T any](item T) *T {
return &item
}
+
+func ExtractIfCompressed(reader io.ReadCloser) (extractedReader io.Reader, err error) {
+ bufferedReader := bufio.NewReader(reader)
+ header, err := bufferedReader.Peek(20)
+ if err != nil {
+ return nil, err
+ }
+ fileType, err := filetype.Match(header)
+ if err != nil {
+ return nil, err
+ }
+
+ // handle compressed file
+ if fileType == matchers.TypeGz || fileType == matchers.TypeZstd || fileType == matchers.TypeXz {
+ extractedReader, err = ParseCompressedData(bufferedReader)
+ if err != nil {
+ return nil, err
+ }
+ return extractedReader, nil
+ } else {
+ // handle uncompressed comps
+ return bufferedReader, nil
+ }
+}
diff --git a/pkg/yum/yum_repository_mock.go b/pkg/yum/yum_repository_mock.go
index 9a35a79..b4e7c40 100644
--- a/pkg/yum/yum_repository_mock.go
+++ b/pkg/yum/yum_repository_mock.go
@@ -1,4 +1,4 @@
-// Code generated by mockery v2.36.1. DO NOT EDIT.
+// Code generated by mockery v2.50.0. DO NOT EDIT.
package yum
@@ -13,7 +13,7 @@ type MockYumRepository struct {
mock.Mock
}
-// Clear provides a mock function with given fields:
+// Clear provides a mock function with no fields
func (_m *MockYumRepository) Clear() {
_m.Called()
}
@@ -22,6 +22,10 @@ func (_m *MockYumRepository) Clear() {
func (_m *MockYumRepository) Comps(ctx context.Context) (*Comps, int, error) {
ret := _m.Called(ctx)
+ if len(ret) == 0 {
+ panic("no return value specified for Comps")
+ }
+
var r0 *Comps
var r1 int
var r2 error
@@ -60,6 +64,10 @@ func (_m *MockYumRepository) Configure(settings YummySettings) {
func (_m *MockYumRepository) Environments(ctx context.Context) ([]Environment, int, error) {
ret := _m.Called(ctx)
+ if len(ret) == 0 {
+ panic("no return value specified for Environments")
+ }
+
var r0 []Environment
var r1 int
var r2 error
@@ -89,10 +97,51 @@ func (_m *MockYumRepository) Environments(ctx context.Context) ([]Environment, i
return r0, r1, r2
}
+// ModuleMDs provides a mock function with given fields: ctx
+func (_m *MockYumRepository) ModuleMDs(ctx context.Context) ([]ModuleMD, int, error) {
+ ret := _m.Called(ctx)
+
+ if len(ret) == 0 {
+ panic("no return value specified for ModuleMDs")
+ }
+
+ var r0 []ModuleMD
+ var r1 int
+ var r2 error
+ if rf, ok := ret.Get(0).(func(context.Context) ([]ModuleMD, int, error)); ok {
+ return rf(ctx)
+ }
+ if rf, ok := ret.Get(0).(func(context.Context) []ModuleMD); ok {
+ r0 = rf(ctx)
+ } else {
+ if ret.Get(0) != nil {
+ r0 = ret.Get(0).([]ModuleMD)
+ }
+ }
+
+ if rf, ok := ret.Get(1).(func(context.Context) int); ok {
+ r1 = rf(ctx)
+ } else {
+ r1 = ret.Get(1).(int)
+ }
+
+ if rf, ok := ret.Get(2).(func(context.Context) error); ok {
+ r2 = rf(ctx)
+ } else {
+ r2 = ret.Error(2)
+ }
+
+ return r0, r1, r2
+}
+
// PackageGroups provides a mock function with given fields: ctx
func (_m *MockYumRepository) PackageGroups(ctx context.Context) ([]PackageGroup, int, error) {
ret := _m.Called(ctx)
+ if len(ret) == 0 {
+ panic("no return value specified for PackageGroups")
+ }
+
var r0 []PackageGroup
var r1 int
var r2 error
@@ -126,6 +175,10 @@ func (_m *MockYumRepository) PackageGroups(ctx context.Context) ([]PackageGroup,
func (_m *MockYumRepository) Packages(ctx context.Context) ([]Package, int, error) {
ret := _m.Called(ctx)
+ if len(ret) == 0 {
+ panic("no return value specified for Packages")
+ }
+
var r0 []Package
var r1 int
var r2 error
@@ -159,6 +212,10 @@ func (_m *MockYumRepository) Packages(ctx context.Context) ([]Package, int, erro
func (_m *MockYumRepository) Repomd(ctx context.Context) (*Repomd, int, error) {
ret := _m.Called(ctx)
+ if len(ret) == 0 {
+ panic("no return value specified for Repomd")
+ }
+
var r0 *Repomd
var r1 int
var r2 error
@@ -192,6 +249,10 @@ func (_m *MockYumRepository) Repomd(ctx context.Context) (*Repomd, int, error) {
func (_m *MockYumRepository) Signature(ctx context.Context) (*string, int, error) {
ret := _m.Called(ctx)
+ if len(ret) == 0 {
+ panic("no return value specified for Signature")
+ }
+
var r0 *string
var r1 int
var r2 error