diff --git a/internal/cmd/fetcher/main.go b/internal/cmd/fetcher/main.go index 0789e6198..667fa0469 100644 --- a/internal/cmd/fetcher/main.go +++ b/internal/cmd/fetcher/main.go @@ -13,7 +13,6 @@ import ( "os" "os/exec" "path/filepath" - "slices" "strings" "time" @@ -94,7 +93,8 @@ func (f *pluginFilter) includes(org, name string) bool { // Fetcher is an interface for fetching plugin versions from external sources. type Fetcher interface { - Fetch(ctx context.Context, config *source.Config) (string, error) + // FetchVersions return all non-prerelease versions for the specified source. + FetchVersions(ctx context.Context, src *source.Source) ([]string, error) } func main() { @@ -646,7 +646,7 @@ func fetchPendingCreations( versionTime func(ctx context.Context, path string) (time.Time, error), ) (map[string]*pluginToCreate, error) { filter := newPluginFilter(includes) - latestVersions := make(map[string]string, len(configs)) + versionsByCacheKey := make(map[string][]string, len(configs)) pendingCreations := make(map[string]*pluginToCreate) for _, config := range configs { @@ -670,35 +670,29 @@ func fetchPendingCreations( continue } } - newVersion := latestVersions[config.CacheKey()] - if newVersion == "" { + versions, ok := versionsByCacheKey[config.CacheKey()] + if !ok { var err error - newVersion, err = fetcher.Fetch(ctx, config) + versions, err = fetcher.FetchVersions(ctx, &config.Source) if err != nil { - if errors.Is(err, fetchclient.ErrSemverPrerelease) { - logger.InfoContext(ctx, "skipping source", slog.String("filename", config.Filename), slog.Any("error", err)) - continue - } - return nil, err + return nil, fmt.Errorf("%s: %w", config.Filename, err) } - latestVersions[config.CacheKey()] = newVersion + versionsByCacheKey[config.CacheKey()] = versions } - // Some plugins share the same source but specify different ignore versions. - // Ensure we continue to only fetch the latest version once but still respect ignores. - if slices.Contains(config.Source.IgnoreVersions, newVersion) { - logger.InfoContext(ctx, "skipping source", slog.String("filename", config.Filename), slog.String("version", newVersion)) - continue + newVersion, err := config.Source.LatestVersion(versions) + if err != nil { + return nil, fmt.Errorf("%s: %w", config.Filename, err) } // Convert to absolute path to match plugin.Walk behavior (which converts paths via filepath.Abs) pluginDir, err := filepath.Abs(filepath.Dir(config.Filename)) if err != nil { return nil, err } - ok, err := checkDirExists(filepath.Join(pluginDir, newVersion)) + exists, err := checkDirExists(filepath.Join(pluginDir, newVersion)) if err != nil { return nil, err } - if ok { + if exists { continue } previousVersion, err := getLatestVersionFromDir(pluginDir) diff --git a/internal/cmd/fetcher/main_test.go b/internal/cmd/fetcher/main_test.go index 4beccbfe0..eb7f65e93 100644 --- a/internal/cmd/fetcher/main_test.go +++ b/internal/cmd/fetcher/main_test.go @@ -203,9 +203,9 @@ func TestRunDependencyOrdering(t *testing.T) { // Mock fetcher that returns new versions for our test plugins // Cache keys are formatted as "github-owner-repository" fetcher := &mockFetcher{ - versions: map[string]string{ - "github-test-base-plugin": "v2.0.0", - "github-test-consumer-plugin": "v2.0.0", + versions: map[string][]string{ + "github-test-base-plugin": {"v1.0.0", "v2.0.0"}, + "github-test-consumer-plugin": {"v1.0.0", "v2.0.0"}, }, } @@ -244,9 +244,9 @@ func TestRunUpdateFrequency(t *testing.T) { repository: base-plugin ` fetcher := &mockFetcher{ - versions: map[string]string{ - "github-test-base-plugin": "v2.0.0", - "github-test-consumer-plugin": "v2.0.0", + versions: map[string][]string{ + "github-test-base-plugin": {"v1.0.0", "v2.0.0"}, + "github-test-consumer-plugin": {"v1.0.0", "v2.0.0"}, }, } @@ -294,16 +294,14 @@ func TestRunUpdateFrequency(t *testing.T) { // mockFetcher returns predetermined versions for testing. type mockFetcher struct { - versions map[string]string // maps cache key (e.g., "github-owner-repo") -> version to return + versions map[string][]string // maps cache key (e.g., "github-owner-repo") -> versions to return } -func (m *mockFetcher) Fetch(_ context.Context, config *source.Config) (string, error) { - key := config.CacheKey() - if version, ok := m.versions[key]; ok { - return version, nil +func (m *mockFetcher) FetchVersions(_ context.Context, src *source.Source) ([]string, error) { + if versions, ok := m.versions[src.CacheKey()]; ok { + return versions, nil } - // Return a default version if not in map - return "v1.0.0", nil + return []string{"v1.0.0"}, nil } // setupTestRepository creates a complete test repository structure with: @@ -614,3 +612,45 @@ func newTestContainer(t *testing.T, root string) appext.Container { logger := slog.New(slog.NewTextHandler(testWriter{t}, &slog.HandlerOptions{Level: slog.LevelDebug})) return appext.NewContainer(nameContainer, logger, appext.LogLevelDebug, appext.LogFormatText) } + +func TestFetchPendingCreationsMaxVersionSharedSource(t *testing.T) { + t.Parallel() + pluginsDir := filepath.Join(t.TempDir(), "plugins") + writePlugin := func(name string, sourceYAML string) { + pluginDir := filepath.Join(pluginsDir, "test", name) + require.NoError(t, os.MkdirAll(filepath.Join(pluginDir, "v1.0.0"), 0755)) + require.NoError(t, os.WriteFile(filepath.Join(pluginDir, "source.yaml"), []byte(sourceYAML), 0644)) + } + writePlugin("unbounded", `source: + github: + owner: test + repository: shared +`) + writePlugin("bounded", `source: + github: + owner: test + repository: shared + max_version: 2.0.0 +`) + configs, err := source.GatherConfigs(pluginsDir) + require.NoError(t, err) + fetcher := &mockFetcher{ + versions: map[string][]string{ + "github-test-shared": {"v1.0.0", "v1.1.0", "v2.0.0"}, + }, + } + logger := slog.New(slog.DiscardHandler) + pending, err := fetchPendingCreations(t.Context(), logger, fetcher, configs, nil, nil) + require.NoError(t, err) + require.Len(t, pending, 2) + for _, config := range configs { + pluginDir, err := filepath.Abs(filepath.Dir(config.Filename)) + require.NoError(t, err) + require.Contains(t, pending, pluginDir) + want := "v2.0.0" + if config.Source.MaxVersion != "" { + want = "v1.1.0" + } + assert.Equal(t, want, pending[pluginDir].newVersion, pluginDir) + } +} diff --git a/internal/fetchclient/fetchclient.go b/internal/fetchclient/fetchclient.go index 3f94e9fc9..b6c71b0d5 100644 --- a/internal/fetchclient/fetchclient.go +++ b/internal/fetchclient/fetchclient.go @@ -6,14 +6,17 @@ import ( "encoding/xml" "errors" "fmt" + "io" "net/http" "net/url" "os" + "slices" "strings" - "buf.build/go/standard/xslices" "github.com/google/go-github/v72/github" "github.com/hashicorp/go-retryablehttp" + "golang.org/x/mod/modfile" + "golang.org/x/mod/module" "golang.org/x/mod/semver" "github.com/bufbuild/plugins/internal/source" @@ -30,16 +33,12 @@ const ( pypiURL = "https://pypi.org/simple" ) -var ( - // ErrSemverPrerelease is returned when a version is a pre-release. - ErrSemverPrerelease = errors.New("pre-release versions are not supported") -) - -// Client is a client used to fetch latest package version. +// Client is a client used to fetch package versions. type Client struct { - httpClient *http.Client - ghClient *github.Client - pypiBaseURL string + httpClient *http.Client + ghClient *github.Client + goProxyBaseURL string + pypiBaseURL string } // New returns a new client. @@ -50,9 +49,10 @@ func New() *Client { ghClient = ghClient.WithAuthToken(ghToken) } return &Client{ - httpClient: httpClient, - ghClient: ghClient, - pypiBaseURL: pypiURL, + httpClient: httpClient, + ghClient: ghClient, + goProxyBaseURL: goProxyURL, + pypiBaseURL: pypiURL, } } @@ -63,62 +63,41 @@ func NewHTTPClient() *http.Client { return retryableClient.StandardClient() } -// Fetch fetches new versions based on the given config and returns a valid semver version -// that can be used with the Go semver package. The version is guaranteed to contain a "v" prefix. -func (c *Client) Fetch(ctx context.Context, config *source.Config) (string, error) { - version, err := c.fetch(ctx, config) +// FetchVersions returns every stable version published by the source's +// upstream. Each version is valid semver with a "v" prefix. Prereleases and +// versions that are not valid semver are omitted. +func (c *Client) FetchVersions(ctx context.Context, src *source.Source) ([]string, error) { + versions, err := c.fetchVersions(ctx, src) if err != nil { - return "", fmt.Errorf("%s: %w", config.Source.Name(), err) - } - // We must ensure that the version is prefixed with "v" for the semver package. - if !strings.HasPrefix(version, "v") { - version = "v" + version + return nil, fmt.Errorf("%s: %w", src.Name(), err) } - if !semver.IsValid(version) { - return "", fmt.Errorf("%s: invalid semver: %s", config.Source.Name(), version) - } - if semver.Prerelease(version) != "" { - return "", fmt.Errorf("%s: %w: %s", config.Source.Name(), ErrSemverPrerelease, version) + if len(versions) == 0 { + return nil, fmt.Errorf("%s: no versions found", src.Name()) } - return version, nil + return versions, nil } -func (c *Client) fetch(ctx context.Context, config *source.Config) (string, error) { - ignoreVersions := xslices.ToStructMap(config.Source.IgnoreVersions) - maxVersion := config.Source.MaxVersion - if maxVersion != "" { - if !strings.HasPrefix(maxVersion, "v") { - maxVersion = "v" + maxVersion - } - if !semver.IsValid(maxVersion) { - return "", fmt.Errorf("%s: max_version is not a valid semver: %s", config.Filename, config.Source.MaxVersion) - } - } +func (c *Client) fetchVersions(ctx context.Context, src *source.Source) ([]string, error) { switch { - case config.Source.GitHub != nil: - return c.fetchGithub(ctx, config.Source.GitHub.Owner, config.Source.GitHub.Repository, ignoreVersions, maxVersion) - case config.Source.DartFlutter != nil: - return c.fetchDartFlutter(ctx, config.Source.DartFlutter.Name, ignoreVersions, maxVersion) - case config.Source.GoProxy != nil: - return c.fetchGoProxy(ctx, config.Source.GoProxy.Name, ignoreVersions, maxVersion) - case config.Source.NPMRegistry != nil: - return c.fetchNPMRegistry(ctx, config.Source.NPMRegistry.Name, ignoreVersions, maxVersion) - case config.Source.Maven != nil: - return c.fetchMaven(ctx, config.Source.Maven.Group, config.Source.Maven.Name, ignoreVersions, maxVersion) - case config.Source.Crates != nil: - return c.fetchCrate(ctx, config.Source.Crates.CrateName, ignoreVersions, maxVersion) - case config.Source.PyPI != nil: - return c.fetchPyPI(ctx, config.Source.PyPI.Name, ignoreVersions, maxVersion) - } - return "", errors.New("failed to match a source") + case src.GitHub != nil: + return c.fetchGithub(ctx, src.GitHub.Owner, src.GitHub.Repository) + case src.DartFlutter != nil: + return c.fetchDartFlutter(ctx, src.DartFlutter.Name) + case src.GoProxy != nil: + return c.fetchGoProxy(ctx, src.GoProxy.Name) + case src.NPMRegistry != nil: + return c.fetchNPMRegistry(ctx, src.NPMRegistry.Name) + case src.Maven != nil: + return c.fetchMaven(ctx, src.Maven.Group, src.Maven.Name) + case src.Crates != nil: + return c.fetchCrate(ctx, src.Crates.CrateName) + case src.PyPI != nil: + return c.fetchPyPI(ctx, src.PyPI.Name) + } + return nil, errors.New("failed to match a source") } -func (c *Client) fetchDartFlutter( - ctx context.Context, - name string, - ignoreVersions map[string]struct{}, - maxVersion string, -) (string, error) { +func (c *Client) fetchDartFlutter(ctx context.Context, name string) (_ []string, retErr error) { request, err := http.NewRequestWithContext( ctx, http.MethodGet, @@ -126,55 +105,37 @@ func (c *Client) fetchDartFlutter( nil, ) if err != nil { - return "", err + return nil, err } response, err := c.httpClient.Do(request) if err != nil { - return "", err + return nil, err } - defer response.Body.Close() + defer func() { + retErr = errors.Join(retErr, response.Body.Close()) + }() if response.StatusCode != http.StatusOK { - return "", fmt.Errorf("received status code %d retrieving %q", response.StatusCode, request.URL.String()) + return nil, fmt.Errorf("received status code %d retrieving %q", response.StatusCode, request.URL.String()) } var data struct { - Latest struct { - Version string `json:"version"` - } `json:"latest"` Versions []struct { Version string `json:"version"` } `json:"versions"` } if err := json.NewDecoder(response.Body).Decode(&data); err != nil { - return "", err - } - if len(ignoreVersions) == 0 && maxVersion == "" { - return data.Latest.Version, nil + return nil, err } - var latestVersion string + versions := make([]string, 0, len(data.Versions)) for _, version := range data.Versions { - version, ok := ensureSemverPrefix(version.Version) - if !ok { - continue - } - if _, ok := ignoreVersions[version]; ok { - continue - } - if maxVersion != "" && semver.Compare(version, maxVersion) >= 0 { - continue - } - if latestVersion == "" || semver.Compare(latestVersion, version) < 0 { - latestVersion = version + if v, ok := ensureSemverPrefix(version.Version); ok { + versions = append(versions, v) } } - // Shouldn't be possible unless we've ignored all versions - if latestVersion == "" { - return "", fmt.Errorf("failed to calculate latest version for dart source %s", name) - } - return latestVersion, nil + return versions, nil } -func (c *Client) fetchCrate(ctx context.Context, name string, ignoreVersions map[string]struct{}, maxVersion string) (string, error) { +func (c *Client) fetchCrate(ctx context.Context, name string) (_ []string, retErr error) { request, err := http.NewRequestWithContext( ctx, http.MethodGet, @@ -182,18 +143,20 @@ func (c *Client) fetchCrate(ctx context.Context, name string, ignoreVersions map nil, ) if err != nil { - return "", err + return nil, err } // See https://github.com/bufbuild/plugins/issues/252 for more information. // We must be careful with this API and respect the crawling policy. request.Header.Set("User-Agent", "bufbuild (github.com/bufbuild/plugins)") response, err := c.httpClient.Do(request) if err != nil { - return "", err + return nil, err } - defer response.Body.Close() + defer func() { + retErr = errors.Join(retErr, response.Body.Close()) + }() if response.StatusCode != http.StatusOK { - return "", fmt.Errorf("received status code %d retrieving %q", response.StatusCode, request.URL.String()) + return nil, fmt.Errorf("received status code %d retrieving %q", response.StatusCode, request.URL.String()) } var data struct { @@ -203,7 +166,7 @@ func (c *Client) fetchCrate(ctx context.Context, name string, ignoreVersions map } `json:"versions"` } if err := json.NewDecoder(response.Body).Decode(&data); err != nil { - return "", err + return nil, err } versions := make([]string, 0, len(data.Versions)) for _, version := range data.Versions { @@ -212,60 +175,73 @@ func (c *Client) fetchCrate(ctx context.Context, name string, ignoreVersions map // from the server's index. continue } - v, ok := ensureSemverPrefix(version.Num) - if !ok { - continue - } - if _, ok := ignoreVersions[v]; ok { - continue - } - if maxVersion != "" && semver.Compare(v, maxVersion) >= 0 { - continue + if v, ok := ensureSemverPrefix(version.Num); ok { + versions = append(versions, v) } - versions = append(versions, v) - } - if len(versions) == 0 { - return "", errors.New("no versions found") } - semver.Sort(versions) - return versions[len(versions)-1], nil + return versions, nil } -func (c *Client) fetchGoProxy(ctx context.Context, name string, ignoreVersions map[string]struct{}, maxVersion string) (string, error) { - if len(ignoreVersions) > 0 { - return "", errors.New("ignore_versions not supported yet for go sources") +func (c *Client) fetchGoProxy(ctx context.Context, name string) ([]string, error) { + modulePath := strings.TrimPrefix(name, "/") + escapedPath, err := module.EscapePath(modulePath) + if err != nil { + return nil, err } - if maxVersion != "" { - return "", errors.New("max_version not supported yet for go sources") + list, err := c.getGoProxy(ctx, escapedPath+"/@v/list") + if err != nil { + return nil, err } - request, err := http.NewRequestWithContext( - ctx, - http.MethodGet, - fmt.Sprintf("%s/%s/@latest", goProxyURL, strings.TrimPrefix(name, "/")), - nil, - ) + var versions []string + for line := range strings.Lines(string(list)) { + if v, ok := ensureSemverPrefix(strings.TrimSpace(line)); ok { + versions = append(versions, v) + } + } + if len(versions) == 0 { + return nil, nil + } + // Like the go command, honor retractions declared by the highest version. + highestVersion := slices.MaxFunc(versions, semver.Compare) + escapedVersion, err := module.EscapeVersion(highestVersion) if err != nil { - return "", err + return nil, err } - response, err := c.httpClient.Do(request) + modPath := escapedPath + "/@v/" + escapedVersion + ".mod" + modData, err := c.getGoProxy(ctx, modPath) if err != nil { - return "", err + return nil, err } - defer response.Body.Close() - if response.StatusCode != http.StatusOK { - return "", fmt.Errorf("received status code %d retrieving %q", response.StatusCode, request.URL.String()) + modFile, err := modfile.ParseLax(modPath, modData, nil) + if err != nil { + return nil, fmt.Errorf("failed to parse %s: %w", modPath, err) } + return slices.DeleteFunc(versions, func(version string) bool { + return slices.ContainsFunc(modFile.Retract, func(retract *modfile.Retract) bool { + return semver.Compare(version, retract.Low) >= 0 && semver.Compare(version, retract.High) <= 0 + }) + }), nil +} - var data struct { - Version string `json:"Version"` //nolint:tagliatelle +func (c *Client) getGoProxy(ctx context.Context, path string) (_ []byte, retErr error) { + request, err := http.NewRequestWithContext(ctx, http.MethodGet, c.goProxyBaseURL+"/"+path, nil) + if err != nil { + return nil, err } - if err := json.NewDecoder(response.Body).Decode(&data); err != nil { - return "", err + response, err := c.httpClient.Do(request) + if err != nil { + return nil, err + } + defer func() { + retErr = errors.Join(retErr, response.Body.Close()) + }() + if response.StatusCode != http.StatusOK { + return nil, fmt.Errorf("received status code %d retrieving %q", response.StatusCode, request.URL.String()) } - return data.Version, nil + return io.ReadAll(response.Body) } -func (c *Client) fetchNPMRegistry(ctx context.Context, name string, ignoreVersions map[string]struct{}, maxVersion string) (string, error) { +func (c *Client) fetchNPMRegistry(ctx context.Context, name string) (_ []string, retErr error) { request, err := http.NewRequestWithContext( ctx, http.MethodGet, @@ -273,68 +249,53 @@ func (c *Client) fetchNPMRegistry(ctx context.Context, name string, ignoreVersio nil, ) if err != nil { - return "", err + return nil, err } response, err := c.httpClient.Do(request) if err != nil { - return "", err + return nil, err } - defer response.Body.Close() + defer func() { + retErr = errors.Join(retErr, response.Body.Close()) + }() if response.StatusCode != http.StatusOK { - return "", fmt.Errorf("received status code %d retrieving %q", response.StatusCode, request.URL.String()) + return nil, fmt.Errorf("received status code %d retrieving %q", response.StatusCode, request.URL.String()) } var data struct { Versions map[string]any `json:"versions"` } if err := json.NewDecoder(response.Body).Decode(&data); err != nil { - return "", err + return nil, err } - latestVersion := "" + versions := make([]string, 0, len(data.Versions)) for version := range data.Versions { - semverVersion, ok := ensureSemverPrefix(version) - if !ok { - continue - } - if _, ignored := ignoreVersions[semverVersion]; ignored { - continue - } - if maxVersion != "" && semver.Compare(semverVersion, maxVersion) >= 0 { - continue - } - if latestVersion == "" || semver.Compare(latestVersion, semverVersion) < 0 { - latestVersion = semverVersion + if v, ok := ensureSemverPrefix(version); ok { + versions = append(versions, v) } } - if latestVersion == "" { - return "", errors.New("no versions found") - } - return latestVersion, nil + return versions, nil } -func (c *Client) fetchMaven( - ctx context.Context, - group string, - name string, - ignoreVersions map[string]struct{}, - maxVersion string, -) (string, error) { +func (c *Client) fetchMaven(ctx context.Context, group string, name string) (_ []string, retErr error) { groupComponents := strings.Split(group, ".") targetURL, err := url.JoinPath(mavenURL, append(groupComponents, name, "maven-metadata.xml")...) if err != nil { - return "", err + return nil, err } request, err := http.NewRequestWithContext(ctx, http.MethodGet, targetURL, nil) if err != nil { - return "", err + return nil, err } response, err := c.httpClient.Do(request) if err != nil { - return "", err + return nil, err } - defer response.Body.Close() + defer func() { + retErr = errors.Join(retErr, response.Body.Close()) + }() if response.StatusCode != http.StatusOK { - return "", fmt.Errorf("received status code %d retrieving %q", response.StatusCode, request.URL.String()) + return nil, fmt.Errorf("received status code %d retrieving %q", response.StatusCode, request.URL.String()) } var metadata struct { GroupID string `xml:"groupId"` @@ -347,38 +308,18 @@ func (c *Client) fetchMaven( } `xml:"versioning"` } if err := xml.NewDecoder(response.Body).Decode(&metadata); err != nil { - return "", err + return nil, err } - latestVersion := "" + versions := make([]string, 0, len(metadata.Versioning.Versions)) for _, version := range metadata.Versioning.Versions { - v, ok := ensureSemverPrefix(version) - if !ok { - continue - } - v = semver.Canonical(v) - if _, ok := ignoreVersions[v]; ok { - continue - } - if maxVersion != "" && semver.Compare(v, maxVersion) >= 0 { - continue - } - if latestVersion == "" || semver.Compare(latestVersion, v) < 0 { - latestVersion = v + if v, ok := ensureSemverPrefix(version); ok { + versions = append(versions, semver.Canonical(v)) } } - if latestVersion == "" { - return "", errors.New("failed to determine latest version from metadata") - } - return latestVersion, nil + return versions, nil } -func (c *Client) fetchGithub( - ctx context.Context, - owner string, - repository string, - ignoreVersions map[string]struct{}, - maxVersion string, -) (string, error) { +func (c *Client) fetchGithub(ctx context.Context, owner string, repository string) ([]string, error) { // With the GitHub API we have a few options: // // ✅ 1. list all git tags @@ -395,19 +336,13 @@ func (c *Client) fetchGithub( PerPage: 100, }) if err != nil { - return "", err + return nil, err } for _, tag := range tags { if tag.Name == nil { continue } if v, ok := ensureSemverPrefix(*tag.Name); ok { - if _, ok := ignoreVersions[v]; ok { - continue - } - if maxVersion != "" && semver.Compare(v, maxVersion) >= 0 { - continue - } versions = append(versions, v) } } @@ -416,14 +351,10 @@ func (c *Client) fetchGithub( break } } - if len(versions) == 0 { - return "", errors.New("no versions found") - } - semver.Sort(versions) - return versions[len(versions)-1], nil + return versions, nil } -func (c *Client) fetchPyPI(ctx context.Context, name string, ignoreVersions map[string]struct{}, maxVersion string) (string, error) { +func (c *Client) fetchPyPI(ctx context.Context, name string) (_ []string, retErr error) { request, err := http.NewRequestWithContext( ctx, http.MethodGet, @@ -431,49 +362,39 @@ func (c *Client) fetchPyPI(ctx context.Context, name string, ignoreVersions map[ nil, ) if err != nil { - return "", err + return nil, err } request.Header.Set("Accept", "application/vnd.pypi.simple.v1+json") response, err := c.httpClient.Do(request) if err != nil { - return "", err + return nil, err } - defer response.Body.Close() + defer func() { + retErr = errors.Join(retErr, response.Body.Close()) + }() if response.StatusCode != http.StatusOK { - return "", fmt.Errorf("received status code %d retrieving %q", response.StatusCode, request.URL.String()) + return nil, fmt.Errorf("received status code %d retrieving %q", response.StatusCode, request.URL.String()) } var data struct { Versions []string `json:"versions"` } if err := json.NewDecoder(response.Body).Decode(&data); err != nil { - return "", err + return nil, err } - var versions []string + versions := make([]string, 0, len(data.Versions)) for _, version := range data.Versions { - v, ok := ensureSemverPrefix(version) - if !ok { - continue - } - if _, ok := ignoreVersions[v]; ok { - continue + if v, ok := ensureSemverPrefix(version); ok { + versions = append(versions, v) } - if maxVersion != "" && semver.Compare(v, maxVersion) >= 0 { - continue - } - versions = append(versions, v) - } - if len(versions) == 0 { - return "", errors.New("no versions found") } - semver.Sort(versions) - return versions[len(versions)-1], nil + return versions, nil } // ensureSemverPrefix checks if the given version is valid semver, optionally // prefixing with "v". The output version is not guaranteed to be the same -// as input. This function returns false if the version is not valid semver or -// is a prerelease. +// as input. This function returns false if the version is not valid semver, is +// a prerelease, or has build metadata. func ensureSemverPrefix(version string) (string, bool) { if !strings.HasPrefix(version, "v") { version = "v" + version @@ -481,7 +402,7 @@ func ensureSemverPrefix(version string) (string, bool) { if !semver.IsValid(version) { return "", false } - if semver.Prerelease(version) != "" { + if semver.Prerelease(version) != "" || semver.Build(version) != "" { return "", false } return version, true diff --git a/internal/fetchclient/fetchclient_test.go b/internal/fetchclient/fetchclient_test.go index 8e91d1f62..6cb4c15fd 100644 --- a/internal/fetchclient/fetchclient_test.go +++ b/internal/fetchclient/fetchclient_test.go @@ -2,8 +2,10 @@ package fetchclient import ( "encoding/json" + "io" "net/http" "net/http/httptest" + "strings" "testing" "github.com/stretchr/testify/assert" @@ -14,39 +16,24 @@ func TestFetchPyPI(t *testing.T) { t.Parallel() tests := []struct { - name string - versions []string - ignoreVersions map[string]struct{} - maxVersion string - wantVersion string - wantErr string + name string + versions []string + wantVersions []string }{ { - name: "returns latest semver version", - versions: []string{"3.5.0", "3.6.0", "5.0.0", "1.0"}, - wantVersion: "v5.0.0", + name: "returns all semver versions with v prefix", + versions: []string{"3.5.0", "3.6.0", "5.0.0", "1.0"}, + wantVersions: []string{"v3.5.0", "v3.6.0", "v5.0.0", "v1.0"}, }, { - name: "skips pre-release versions", - versions: []string{"1.2.5", "2.0.0b7"}, - wantVersion: "v1.2.5", + name: "skips pre-release versions", + versions: []string{"1.2.5", "2.0.0b7", "2.0.0rc1"}, + wantVersions: []string{"v1.2.5"}, }, { - name: "respects ignore_versions", - versions: []string{"3.6.0", "5.0.0"}, - ignoreVersions: map[string]struct{}{"v5.0.0": {}}, - wantVersion: "v3.6.0", - }, - { - name: "respects max_version exclusive upper bound", - versions: []string{"3.6.0", "5.0.0"}, - maxVersion: "v5.0.0", - wantVersion: "v3.6.0", - }, - { - name: "error when no valid versions remain", - versions: []string{"2.0.0b7", "2.0.0rc1"}, - wantErr: "no versions found", + name: "skips versions with build metadata", + versions: []string{"1.2.5", "1.2.5+1", "1.3.0+build.7"}, + wantVersions: []string{"v1.2.5"}, }, } @@ -67,17 +54,89 @@ func TestFetchPyPI(t *testing.T) { httpClient: srv.Client(), pypiBaseURL: srv.URL, } - ignoreVersions := tt.ignoreVersions - if ignoreVersions == nil { - ignoreVersions = map[string]struct{}{} + got, err := c.fetchPyPI(t.Context(), "mypy-protobuf") + require.NoError(t, err) + assert.Equal(t, tt.wantVersions, got) + }) + } +} + +func TestFetchGoProxy(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + list string + mod string + wantVersions []string + wantErr string + }{ + { + name: "returns tagged versions", + list: "v1.1.0\nv1.0.0\n", + mod: "module github.com/Example/mod\n", + wantVersions: []string{"v1.1.0", "v1.0.0"}, + }, + { + name: "skips prerelease and incompatible versions", + list: "v1.0.0\nv1.1.0-rc.1\nv2.0.0+incompatible\n", + mod: "module github.com/Example/mod\n", + wantVersions: []string{"v1.0.0"}, + }, + { + name: "skips versions retracted by the highest version", + list: "v1.0.0\nv1.1.0\nv1.2.0\nv1.3.0\nv1.4.0\n", + mod: `module github.com/Example/mod + +retract ( + v1.4.0 // retracts itself + [v1.1.0, v1.2.0] +) +`, + wantVersions: []string{"v1.0.0", "v1.3.0"}, + }, + { + name: "empty list", + list: "", + wantVersions: nil, + }, + { + name: "invalid go.mod", + list: "v1.0.0\n", + mod: "retract (\n", + wantErr: "failed to parse", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var body string + switch { + case r.URL.Path == "/github.com/!example/mod/@v/list": + body = tt.list + case strings.HasPrefix(r.URL.Path, "/github.com/!example/mod/@v/") && strings.HasSuffix(r.URL.Path, ".mod"): + body = tt.mod + default: + http.NotFound(w, r) + return + } + _, _ = io.WriteString(w, body) + })) + t.Cleanup(srv.Close) + + c := &Client{ + httpClient: srv.Client(), + goProxyBaseURL: srv.URL, } - got, err := c.fetchPyPI(t.Context(), "mypy-protobuf", ignoreVersions, tt.maxVersion) + got, err := c.fetchGoProxy(t.Context(), "github.com/Example/mod") if tt.wantErr != "" { require.ErrorContains(t, err, tt.wantErr) return } require.NoError(t, err) - assert.Equal(t, tt.wantVersion, got) + assert.Equal(t, tt.wantVersions, got) }) } } diff --git a/internal/source/config.go b/internal/source/config.go index c08201681..9011da53f 100644 --- a/internal/source/config.go +++ b/internal/source/config.go @@ -1,8 +1,13 @@ package source import ( + "errors" + "fmt" "io" + "slices" + "strings" + "golang.org/x/mod/semver" "gopkg.in/yaml.v3" ) @@ -26,6 +31,9 @@ func NewConfig(reader io.Reader) (*Config, error) { if err := decoder.Decode(&config); err != nil { return nil, err } + if err := config.Source.normalizeVersions(); err != nil { + return nil, err + } return config, nil } @@ -79,6 +87,30 @@ func (s *Source) Name() string { return "unknown" } +// LatestVersion returns the highest of the given versions that is not listed +// in IgnoreVersions and is below MaxVersion (if set). Versions must be valid +// semver with a "v" prefix. +func (s *Source) LatestVersion(versions []string) (string, error) { + var latestVersion string + for _, version := range versions { + if s.MaxVersion != "" && semver.Compare(version, s.MaxVersion) >= 0 { + continue + } + if slices.ContainsFunc(s.IgnoreVersions, func(ignoreVersion string) bool { + return semver.Compare(version, ignoreVersion) == 0 + }) { + continue + } + if latestVersion == "" || semver.Compare(version, latestVersion) > 0 { + latestVersion = version + } + } + if latestVersion == "" { + return "", errors.New("no versions satisfy ignore_versions and max_version") + } + return latestVersion, nil +} + func (s *Source) CacheKey() string { name := s.Name() switch { @@ -100,6 +132,33 @@ func (s *Source) CacheKey() string { return name } +// normalizeVersions adds a "v" prefix to MaxVersion and IgnoreVersions and +// validates that they are semver. +func (s *Source) normalizeVersions() error { + if s.MaxVersion != "" { + maxVersion := ensureVPrefix(s.MaxVersion) + if !semver.IsValid(maxVersion) { + return fmt.Errorf("max_version is not a valid semver: %s", s.MaxVersion) + } + s.MaxVersion = maxVersion + } + for i, ignoreVersion := range s.IgnoreVersions { + normalized := ensureVPrefix(ignoreVersion) + if !semver.IsValid(normalized) { + return fmt.Errorf("ignore_versions entry is not a valid semver: %q", ignoreVersion) + } + s.IgnoreVersions[i] = normalized + } + return nil +} + +func ensureVPrefix(version string) string { + if strings.HasPrefix(version, "v") { + return version + } + return "v" + version +} + // CratesConfig is the crates.io API configuration. type CratesConfig struct { CrateName string `yaml:"crate_name"` diff --git a/internal/source/config_test.go b/internal/source/config_test.go index a247311f3..91fd7a70a 100644 --- a/internal/source/config_test.go +++ b/internal/source/config_test.go @@ -23,3 +23,146 @@ func TestConfigWithUpdateFrequency(t *testing.T) { assert.Equal(t, Duration(30*24*time.Hour), *config.Source.UpdateFrequency) assert.Equal(t, "test", config.Source.GitHub.Owner) } + +func TestNewConfigNormalizesVersions(t *testing.T) { + t.Parallel() + tests := []struct { + name string + sourceYAML string + wantMaxVersion string + wantIgnoreVersions []string + wantErr string + }{ + { + name: "adds v prefix", + sourceYAML: `source: + github: + owner: test + repository: test-repo + max_version: 2.0.0 + ignore_versions: + - 1.2.3 + - v1.2.4 +`, + wantMaxVersion: "v2.0.0", + wantIgnoreVersions: []string{"v1.2.3", "v1.2.4"}, + }, + { + name: "no constraints", + sourceYAML: `source: + github: + owner: test + repository: test-repo +`, + }, + { + name: "invalid max version", + sourceYAML: `source: + github: + owner: test + repository: test-repo + max_version: two +`, + wantErr: "max_version is not a valid semver: two", + }, + { + name: "invalid ignore version", + sourceYAML: `source: + github: + owner: test + repository: test-repo + ignore_versions: + - v1.2.x +`, + wantErr: `ignore_versions entry is not a valid semver: "v1.2.x"`, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + config, err := NewConfig(strings.NewReader(test.sourceYAML)) + if test.wantErr != "" { + require.ErrorContains(t, err, test.wantErr) + return + } + require.NoError(t, err) + assert.Equal(t, test.wantMaxVersion, config.Source.MaxVersion) + assert.Equal(t, test.wantIgnoreVersions, config.Source.IgnoreVersions) + }) + } +} + +func TestSourceLatestVersion(t *testing.T) { + t.Parallel() + tests := []struct { + name string + constraints string + versions []string + want string + wantErr string + }{ + { + name: "returns highest version", + versions: []string{"v1.10.0", "v1.9.0", "v2.0.0", "v1.2"}, + want: "v2.0.0", + }, + { + name: "skips ignored versions", + constraints: ` ignore_versions: + - v2.0.0 +`, + versions: []string{"v1.0.0", "v1.1.0", "v2.0.0"}, + want: "v1.1.0", + }, + { + name: "ignored versions match without patch", + constraints: ` ignore_versions: + - "3.0" +`, + versions: []string{"v2.0.0", "v3.0.0"}, + want: "v2.0.0", + }, + { + name: "max version is exclusive", + constraints: ` max_version: 2.0.0 +`, + versions: []string{"v1.21.0", "v2.0.0", "v2.1.0"}, + want: "v1.21.0", + }, + { + name: "ignore and max version combined", + constraints: ` max_version: v2.0.0 + ignore_versions: + - v1.1.0 +`, + versions: []string{"v1.0.0", "v1.1.0", "v2.0.0"}, + want: "v1.0.0", + }, + { + name: "no versions satisfy constraints", + constraints: ` max_version: v2.0.0 +`, + versions: []string{"v2.0.0"}, + wantErr: "no versions satisfy", + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + sourceYAML := `source: + github: + owner: test + repository: test-repo +` + test.constraints + config, err := NewConfig(strings.NewReader(sourceYAML)) + require.NoError(t, err) + got, err := config.Source.LatestVersion(test.versions) + if test.wantErr != "" { + require.ErrorContains(t, err, test.wantErr) + return + } + require.NoError(t, err) + assert.Equal(t, test.want, got) + }) + } +} diff --git a/internal/source/source.go b/internal/source/source.go index b87c502a9..3af8868e5 100644 --- a/internal/source/source.go +++ b/internal/source/source.go @@ -2,6 +2,7 @@ package source import ( "errors" + "fmt" "io/fs" "os" "path/filepath" @@ -38,7 +39,7 @@ func loadConfigFile(filename string) (_ *Config, retErr error) { }() config, err := NewConfig(file) if err != nil { - return nil, err + return nil, fmt.Errorf("%s: %w", filename, err) } config.Filename = filename return config, nil