diff --git a/cmd/example/main.go b/cmd/example/main.go new file mode 100644 index 0000000..972cd57 --- /dev/null +++ b/cmd/example/main.go @@ -0,0 +1,101 @@ +// Copyright 2024 Stacklok, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//nolint:revive +package main + +import ( + "context" + "flag" + "fmt" + "os" + + v2client "github.com/stacklok/trusty-sdk-go/pkg/v2/client" + v2types "github.com/stacklok/trusty-sdk-go/pkg/v2/types" +) + +func main() { + var endpoint, pname string + flag.StringVar(&endpoint, "endpoint", "", "Trusty API endpoint to call") + flag.StringVar(&pname, "pname", "", "Package name") + flag.Parse() + + ctx := context.Background() + client := v2client.New() + + switch endpoint { + case "summary": + if err := summary(ctx, client, pname); err != nil { + fmt.Fprintf(os.Stderr, "error calling endpoint: %s\n", err) + os.Exit(1) + } + case "pkg-meta": + if err := pkg(ctx, client, pname); err != nil { + fmt.Fprintf(os.Stderr, "error calling endpoint: %s\n", err) + os.Exit(1) + } + case "alternatives": + if err := alternatives(ctx, client, pname); err != nil { + fmt.Fprintf(os.Stderr, "error calling endpoint: %s\n", err) + os.Exit(1) + } + case "": + fmt.Fprintf(os.Stderr, "endpoint is mandatory\n") + os.Exit(1) + default: + fmt.Fprintf(os.Stderr, "invalid method: %s\n", endpoint) + os.Exit(1) + } +} + +func summary(ctx context.Context, client v2client.Trusty, pname string) error { + res, err := client.Summary(ctx, &v2types.Dependency{ + PackageName: pname, + }) + if err != nil { + return err + } + + fmt.Printf("%+v\n", res) + return nil +} + +func pkg(ctx context.Context, client v2client.Trusty, pname string) error { + res, err := client.PackageMetadata(ctx, &v2types.Dependency{ + PackageName: pname, + }) + if err != nil { + return err + } + + fmt.Printf("%+v\n", res) + fmt.Printf("STATUS: %+v\n", *res.Status) + fmt.Printf("MALICIOUS: %+v\n", res.Malicious) + for _, contributor := range res.Contributors { + fmt.Printf("CONTRIBUTOR: %+v\n", contributor) + } + return nil +} + +func alternatives(ctx context.Context, client v2client.Trusty, pname string) error { + res, err := client.Alternatives(ctx, &v2types.Dependency{ + PackageName: pname, + }) + if err != nil { + return err + } + + fmt.Printf("%+v\n", res) + return nil +} diff --git a/go.mod b/go.mod index 95c6b2a..954cfae 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,7 @@ toolchain go1.23.1 require ( github.com/BurntSushi/toml v1.4.0 github.com/google/go-github/v66 v66.0.0 + github.com/google/uuid v1.6.0 github.com/package-url/packageurl-go v0.1.3 github.com/stretchr/testify v1.9.0 golang.org/x/oauth2 v0.23.0 diff --git a/go.sum b/go.sum index 99231f4..6c6b030 100644 --- a/go.sum +++ b/go.sum @@ -10,6 +10,8 @@ github.com/google/go-github/v66 v66.0.0 h1:ADJsaXj9UotwdgK8/iFZtv7MLc8E8WBl62WLd github.com/google/go-github/v66 v66.0.0/go.mod h1:+4SO9Zkuyf8ytMj0csN1NR/5OTR+MfqPp8P8dVlcvY4= github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/nozzle/throttler v0.0.0-20180817012639-2ea982251481 h1:Up6+btDp321ZG5/zdSLo48H9Iaq0UQGthrhWC6pCxzE= github.com/nozzle/throttler v0.0.0-20180817012639-2ea982251481/go.mod h1:yKZQO8QE2bHlgozqWDiRVqTFlLQSj30K/6SAK8EeYFw= github.com/package-url/packageurl-go v0.1.3 h1:4juMED3hHiz0set3Vq3KeQ75KD1avthoXLtmE3I0PLs= diff --git a/internal/client/client.go b/internal/client/client.go new file mode 100644 index 0000000..a53ef13 --- /dev/null +++ b/internal/client/client.go @@ -0,0 +1,450 @@ +// Copyright 2024 Stacklok, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package client provides a rest client to talk to the Trusty API. +package client + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "net/url" + "os" + "strings" + "time" + + packageurl "github.com/package-url/packageurl-go" + khttp "sigs.k8s.io/release-utils/http" + + v1types "github.com/stacklok/trusty-sdk-go/pkg/v1/types" + v2types "github.com/stacklok/trusty-sdk-go/pkg/v2/types" +) + +const ( + defaultEndpoint = "https://api.trustypkg.dev" + endpointEnvVar = "TRUSTY_ENDPOINT" + reportPath = "v1/report" +) + +// Options configures the Trusty API client +type Options struct { + HttpClient netClient + + // Workers is the number of parallel request the client makes to the API + Workers int + + // BaseURL of the Trusty API + BaseURL string + + // WaitForIngestion causes the http client to wait and retry if Trusty + // responds with a successful request but with a "pending" or "scoring" status + WaitForIngestion bool + + // ErrOnFailedIngestion makes the client return an error on a Report call + // when the ingestion failed internally withing trusty. If false, the + // report data willbe returned but the application needs to check the + // ingestion status and handle it. + ErrOnFailedIngestion bool + + // IngestionRetryWait is the number of seconds that the client will wait for + // package ingestion before retrying. + IngestionRetryWait int + + // IngestionMaxRetries is the maximum number of requests the client will + // send while waiting for ingestion to finish + IngestionMaxRetries int +} + +// DefaultOptions is the default Trusty client options set +var DefaultOptions = Options{ + Workers: 2, + BaseURL: defaultEndpoint, + WaitForIngestion: true, + IngestionRetryWait: 5, +} + +type netClient interface { + GetRequestGroup([]string) ([]*http.Response, []error) + GetRequest(string) (*http.Response, error) +} + +// New returns a new Trusty REST client +func New() *Trusty { + opts := DefaultOptions + opts.HttpClient = khttp.NewAgent().WithMaxParallel(opts.Workers).WithFailOnHTTPError(true) + if ep := os.Getenv(endpointEnvVar); ep != "" { + opts.BaseURL = ep + } + return NewWithOptions(opts) +} + +// NewWithOptions returns a new client with the specified options set +func NewWithOptions(opts Options) *Trusty { + if opts.BaseURL == "" { + opts.BaseURL = DefaultOptions.BaseURL + } + + if opts.Workers == 0 { + opts.Workers = DefaultOptions.Workers + } + + if opts.HttpClient == nil { + opts.HttpClient = khttp.NewAgent().WithMaxParallel(opts.Workers).WithFailOnHTTPError(true) + } + + return &Trusty{ + Options: opts, + } +} + +func urlFromEndpointAndPaths( + baseUrl, endpoint string, params map[string]string, +) (*url.URL, error) { + u, err := url.Parse(baseUrl) + if err != nil { + return nil, fmt.Errorf("failed to parse endpoint: %w", err) + } + u = u.JoinPath(endpoint) + + // Add query parameters for package_name and package_type + q := u.Query() + for k, v := range params { + q.Set(k, v) + } + u.RawQuery = q.Encode() + + return u, nil +} + +// Trusty is the main trusty client +type Trusty struct { + Options Options +} + +// GroupReport queries the Trusty API in parallel for a group of dependencies. +func (t *Trusty) GroupReport(_ context.Context, deps []*v1types.Dependency) ([]*v1types.Reply, error) { + urls := []string{} + for _, dep := range deps { + u, err := t.PackageEndpoint(dep) + if err != nil { + return nil, fmt.Errorf("unable to get endpoint for: %q: %w", dep.Name, err) + } + urls = append(urls, u) + } + + responses, errs := t.Options.HttpClient.GetRequestGroup(urls) + if err := errors.Join(errs...); err != nil { + return nil, fmt.Errorf("fetching data from Trusty: %w", err) + } + + // Parse the replies + resps := make([]*v1types.Reply, len(responses)) + for i := range responses { + defer responses[i].Body.Close() + dec := json.NewDecoder(responses[i].Body) + resps[i] = &v1types.Reply{} + if err := dec.Decode(resps[i]); err != nil { + return nil, fmt.Errorf("could not unmarshal response #%d: %w", i, err) + } + } + return resps, nil +} + +// PurlEndpoint returns the API endpoint url to query for data about a purl +func (t *Trusty) PurlEndpoint(purl string) (string, error) { + dep, err := t.PurlToDependency(purl) + if err != nil { + return "", fmt.Errorf("getting dependency from %q", purl) + } + ep, err := t.PackageEndpoint(dep) + if err != nil { + return "", fmt.Errorf("getting package endpoint: %w", err) + } + return ep, nil +} + +// PackageEndpoint takes a dependency and returns the Trusty endpoint to +// query data about it. +func (t *Trusty) PackageEndpoint(dep *v1types.Dependency) (string, error) { + // Check dependency data: + errs := []error{} + if dep.Name == "" { + errs = append(errs, fmt.Errorf("dependency has no name defined")) + } + if dep.Ecosystem.AsString() == "" { + errs = append(errs, fmt.Errorf("dependency has no ecosystem set")) + } + + if err := errors.Join(errs...); err != nil { + return "", err + } + + u, err := url.Parse(t.Options.BaseURL + "/" + reportPath) + if err != nil { + return "", fmt.Errorf("failed to parse endpoint: %w", err) + } + + params := map[string]string{ + "package_name": dep.Name, + "package_type": strings.ToLower(dep.Ecosystem.AsString()), + } + + // Add query parameters for package_name and package_type + q := u.Query() + for k, v := range params { + q.Set(k, v) + } + u.RawQuery = q.Encode() + + return u.String(), nil +} + +// PurlToEcosystem returns a trusty ecosystem constant from a Package URL's type +func (_ *Trusty) PurlToEcosystem(purl string) v1types.Ecosystem { + switch { + case strings.HasPrefix(purl, "pkg:golang"): + return v1types.ECOSYSTEM_GO + case strings.HasPrefix(purl, "pkg:npm"): + return v1types.ECOSYSTEM_NPM + case strings.HasPrefix(purl, "pkg:pypi"): + return v1types.ECOSYSTEM_PYPI + default: + return v1types.Ecosystem(0) + } +} + +// PurlToDependency takes a string with a package url +func (t *Trusty) PurlToDependency(purlString string) (*v1types.Dependency, error) { + e := t.PurlToEcosystem(purlString) + if e == 0 { + // Ecosystem nil or not supported + return nil, fmt.Errorf("ecosystem not supported") + } + + purl, err := packageurl.FromString(purlString) + if err != nil { + return nil, fmt.Errorf("unable to parse package url: %w", err) + } + name := purl.Name + if purl.Namespace != "" { + name = purl.Namespace + "/" + purl.Name + } + return &v1types.Dependency{ + Ecosystem: e, + Name: name, + Version: purl.Version, + }, nil +} + +// Report returns a dependency report with all the data that Trusty has +// available for a package. +func (t *Trusty) Report(_ context.Context, dep *v1types.Dependency) (*v1types.Reply, error) { + u, err := t.PackageEndpoint(dep) + if err != nil { + return nil, fmt.Errorf("computing package endpoint: %w", err) + } + + var r v1types.Reply + tries := 0 + for { + resp, err := t.Options.HttpClient.GetRequest(u) + if err != nil { + return nil, fmt.Errorf("could not send request: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("received non-200 response: %d", resp.StatusCode) + } + + dec := json.NewDecoder(resp.Body) + if err := dec.Decode(&r); err != nil { + return nil, fmt.Errorf("could not unmarshal response: %w", err) + } + fmt.Printf("Attempt #%d to fetch package, status: %s", tries, r.PackageData.Status) + + shouldRetry, err := evalRetry(r.PackageData.Status, t.Options) + if err != nil { + return nil, err + } + + if !shouldRetry { + break + } + + tries++ + if tries > t.Options.IngestionMaxRetries { + return nil, fmt.Errorf("time out reached waiting for package ingestion") + } + time.Sleep(time.Duration(t.Options.IngestionRetryWait) * time.Second) + } + + return &r, err +} + +func evalRetry(status string, opts Options) (shouldRetry bool, err error) { + // First, error if the ingestion status is invalid + if status != v1types.IngestStatusFailed && status != v1types.IngestStatusComplete && + status != v1types.IngestStatusPending && status != v1types.IngestStatusScoring { + + return false, fmt.Errorf("unexpected ingestion status when querying package") + } + + if status == v1types.IngestStatusFailed && opts.ErrOnFailedIngestion { + return false, fmt.Errorf("upstream error ingesting package data") + } + + // Package ingestion is ready + if status == v1types.IngestStatusComplete { + return false, nil + } + + // Client configured to return raw response (even when package is not ready) + if !opts.WaitForIngestion || status == v1types.IngestStatusFailed { + return false, nil + } + + return true, nil +} + +const ( + // v2 paths + v2SummaryPath = "v2/summary" + v2PkgPath = "v2/pkg" + v2Alternatives = "v2/alternatives" +) + +// Summary fetches a summary of Security Signal information +// for the package. +func (t *Trusty) Summary( + _ context.Context, + dep *v2types.Dependency, +) (*v2types.PackageSummaryAnnotation, error) { + if dep.PackageName == "" { + return nil, fmt.Errorf("dependency has no name defined") + } + + u, err := urlFor(t.Options.BaseURL, v2SummaryPath) + if err != nil { + return nil, fmt.Errorf("failed to parse endpoint: %w", err) + } + + // Add query parameters for package_name, package_type, and + // package_version. + q := u.Query() + q.Set("package_name", dep.PackageName) + if dep.PackageType != nil && *dep.PackageType != "" { + q.Set("package_type", strings.ToLower(*dep.PackageType)) + } + if dep.PackageVersion != nil && *dep.PackageVersion != "" { + q.Set("package_version", *dep.PackageVersion) + } + u.RawQuery = q.Encode() + + return doRequest[v2types.PackageSummaryAnnotation](t.Options.HttpClient, u.String()) +} + +// PackageMetadata fetched the metadata for a package. +// +// This includes the package's name, version, description, and +// other metadata about contributors. +func (t *Trusty) PackageMetadata( + _ context.Context, + dep *v2types.Dependency, +) (*v2types.TrustyPackageData, error) { + if dep.PackageName == "" { + return nil, fmt.Errorf("dependency has no name defined") + } + + u, err := urlFor(t.Options.BaseURL, v2PkgPath) + if err != nil { + return nil, fmt.Errorf("failed to parse endpoint: %w", err) + } + + // Add query parameters for package_name, package_type, and + // package_version. + q := u.Query() + q.Set("package_name", dep.PackageName) + if dep.PackageType != nil && *dep.PackageType != "" { + q.Set("package_type", strings.ToLower(*dep.PackageType)) + } + if dep.PackageVersion != nil && *dep.PackageVersion != "" { + q.Set("package_version", *dep.PackageVersion) + } + u.RawQuery = q.Encode() + + return doRequest[v2types.TrustyPackageData](t.Options.HttpClient, u.String()) +} + +// Alternatives fetches packages that can be used in place of the +// given one. +func (t *Trusty) Alternatives( + _ context.Context, + dep *v2types.Dependency, +) (*v2types.PackageAlternatives, error) { + if dep.PackageName == "" { + return nil, fmt.Errorf("dependency has no name defined") + } + + u, err := urlFor(t.Options.BaseURL, v2Alternatives) + if err != nil { + return nil, fmt.Errorf("failed to parse endpoint: %w", err) + } + + // Add query parameters for package_name, package_type, and + // package_version. + q := u.Query() + q.Set("package_name", dep.PackageName) + if dep.PackageType != nil && *dep.PackageType != "" { + q.Set("package_type", strings.ToLower(*dep.PackageType)) + } + if dep.PackageVersion != nil && *dep.PackageVersion != "" { + q.Set("package_version", *dep.PackageVersion) + } + u.RawQuery = q.Encode() + + return doRequest[v2types.PackageAlternatives](t.Options.HttpClient, u.String()) +} + +// doRequest only wraps (1) an HTTP GET issued to the given URL using +// the given client, and (2) result deserialization. +func doRequest[T any](client netClient, fullurl string) (*T, error) { + resp, err := client.GetRequest(fullurl) + if err != nil { + return nil, fmt.Errorf("could not send request: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("received non-200 response: %d", resp.StatusCode) + } + + var res T + dec := json.NewDecoder(resp.Body) + if err := dec.Decode(&res); err != nil { + return nil, fmt.Errorf("could not unmarshal response: %w", err) + } + + return &res, nil +} + +func urlFor(baseURL, path string) (*url.URL, error) { + ustr, err := url.JoinPath(baseURL, path) + if err != nil { + return nil, err + } + return url.Parse(ustr) +} diff --git a/pkg/v1/client/client_test.go b/internal/client/client_test.go similarity index 90% rename from pkg/v1/client/client_test.go rename to internal/client/client_test.go index 199235c..e18175a 100644 --- a/pkg/v1/client/client_test.go +++ b/internal/client/client_test.go @@ -25,7 +25,7 @@ import ( "github.com/stretchr/testify/require" - "github.com/stacklok/trusty-sdk-go/pkg/v1/types" + v1types "github.com/stacklok/trusty-sdk-go/pkg/v1/types" ) func newFakeClient() *fakeClient { @@ -131,7 +131,7 @@ func TestReport(t *testing.T) { t.Parallel() respBody := `{"package_name":"requestts","package_type":"pypi", "package_data": { "status":"complete"} }` - testdep := &types.Dependency{ + testdep := &v1types.Dependency{ Name: "requestts", Ecosystem: 1, } @@ -142,9 +142,9 @@ func TestReport(t *testing.T) { for _, tc := range []struct { name string - dep *types.Dependency + dep *v1types.Dependency prepare func(*fakeClient) - expected *types.Reply + expected *v1types.Reply mustErr bool options *Options }{ @@ -157,14 +157,14 @@ func TestReport(t *testing.T) { Body: buildReader(respBody), }) }, - expected: &types.Reply{ + expected: &v1types.Reply{ PackageName: "requestts", PackageType: "pypi", }, }, { name: "no-dep-name", - dep: &types.Dependency{ + dep: &v1types.Dependency{ Ecosystem: 1, }, prepare: func(_ *fakeClient) {}, @@ -172,7 +172,7 @@ func TestReport(t *testing.T) { }, { name: "no-dep-ecosystem", - dep: &types.Dependency{ + dep: &v1types.Dependency{ Name: "test", }, prepare: func(_ *fakeClient) {}, @@ -288,25 +288,25 @@ func TestGroupReport(t *testing.T) { respBody1 := `{"package_name":"requestts","package_type":"pypi"}` respBody2 := `{"package_name":"tensorflow","package_type":"pypi"}` - testdep1 := &types.Dependency{ + testdep1 := &v1types.Dependency{ Name: "requestts", Ecosystem: 1, } - testdep2 := &types.Dependency{ + testdep2 := &v1types.Dependency{ Name: "tensorflow", Ecosystem: 1, } for _, tc := range []struct { name string - deps []*types.Dependency + deps []*v1types.Dependency prepare func(*fakeClient) - expected []*types.Reply + expected []*v1types.Reply mustErr bool }{ { name: "normal", - deps: []*types.Dependency{testdep1, testdep2}, + deps: []*v1types.Dependency{testdep1, testdep2}, prepare: func(fc *fakeClient) { fc.resps = append(fc.resps, &http.Response{ @@ -319,7 +319,7 @@ func TestGroupReport(t *testing.T) { }, ) }, - expected: []*types.Reply{ + expected: []*v1types.Reply{ { PackageName: "requestts", PackageType: "pypi", @@ -333,7 +333,7 @@ func TestGroupReport(t *testing.T) { { name: "no-dep-name", - deps: []*types.Dependency{ + deps: []*v1types.Dependency{ {Ecosystem: 1}, testdep1, }, prepare: func(_ *fakeClient) {}, @@ -341,7 +341,7 @@ func TestGroupReport(t *testing.T) { }, { name: "no-dep-ecosystem", - deps: []*types.Dependency{ + deps: []*v1types.Dependency{ {Name: "test"}, testdep1, }, prepare: func(_ *fakeClient) {}, @@ -349,7 +349,7 @@ func TestGroupReport(t *testing.T) { }, { name: "http-fails", - deps: []*types.Dependency{testdep1}, + deps: []*v1types.Dependency{testdep1}, prepare: func(fc *fakeClient) { fc.errs = append(fc.errs, fmt.Errorf("fake error")) }, @@ -357,7 +357,7 @@ func TestGroupReport(t *testing.T) { }, { name: "http-non-200", - deps: []*types.Dependency{testdep1, testdep2}, + deps: []*v1types.Dependency{testdep1, testdep2}, prepare: func(fc *fakeClient) { fc.resps = append( fc.resps, &http.Response{ @@ -378,7 +378,7 @@ func TestGroupReport(t *testing.T) { }, { name: "bad-response-json", - deps: []*types.Dependency{testdep1, testdep2}, + deps: []*v1types.Dependency{testdep1, testdep2}, prepare: func(fc *fakeClient) { fc.resps = append(fc.resps, &http.Response{ @@ -470,31 +470,31 @@ func TestPurlToDependency(t *testing.T) { for _, tc := range []struct { name string purl string - expected *types.Dependency + expected *v1types.Dependency mustErr bool }{ { name: "golang", purl: "pkg:golang/github.com/k8s.io/release@v1.0.8", - expected: &types.Dependency{Name: "github.com/k8s.io/release", Version: "v1.0.8", Ecosystem: types.ECOSYSTEM_GO}, + expected: &v1types.Dependency{Name: "github.com/k8s.io/release", Version: "v1.0.8", Ecosystem: v1types.ECOSYSTEM_GO}, mustErr: false, }, { name: "pypi", purl: "pkg:pypi/requests@v1.2.3", - expected: &types.Dependency{Name: "requests", Version: "v1.2.3", Ecosystem: types.ECOSYSTEM_PYPI}, + expected: &v1types.Dependency{Name: "requests", Version: "v1.2.3", Ecosystem: v1types.ECOSYSTEM_PYPI}, mustErr: false, }, { name: "npm", purl: "pkg:npm/%40react-stately/color@3.7.0", - expected: &types.Dependency{Name: "@react-stately/color", Version: "3.7.0", Ecosystem: types.ECOSYSTEM_NPM}, + expected: &v1types.Dependency{Name: "@react-stately/color", Version: "3.7.0", Ecosystem: v1types.ECOSYSTEM_NPM}, mustErr: false, }, { name: "no-version", purl: "pkg:npm/%40react-stately/color", - expected: &types.Dependency{Name: "@react-stately/color", Version: "", Ecosystem: types.ECOSYSTEM_NPM}, + expected: &v1types.Dependency{Name: "@react-stately/color", Version: "", Ecosystem: v1types.ECOSYSTEM_NPM}, mustErr: false, }, { diff --git a/pkg/v1/client/client.go b/pkg/v1/client/client.go index a1336c1..ce73d22 100644 --- a/pkg/v1/client/client.go +++ b/pkg/v1/client/client.go @@ -17,304 +17,45 @@ package client import ( "context" - "encoding/json" - "errors" - "fmt" - "net/http" - "net/url" - "os" - "strings" - "time" - - packageurl "github.com/package-url/packageurl-go" - khttp "sigs.k8s.io/release-utils/http" + internalclient "github.com/stacklok/trusty-sdk-go/internal/client" "github.com/stacklok/trusty-sdk-go/pkg/v1/types" ) -const ( - defaultEndpoint = "https://gh.trustypkg.dev" - endpointEnvVar = "TRUSTY_ENDPOINT" - reportPath = "v1/report" -) - // Options configures the Trusty API client -type Options struct { - HttpClient netClient - - // Workers is the number of parallel request the client makes to the API - Workers int - - // BaseURL of the Trusty API - BaseURL string - - // WaitForIngestion causes the http client to wait and retry if Trusty - // responds with a successful request but with a "pending" or "scoring" status - WaitForIngestion bool - - // ErrOnFailedIngestion makes the client return an error on a Report call - // when the ingestion failed internally withing trusty. If false, the - // report data willbe returned but the application needs to check the - // ingestion status and handle it. - ErrOnFailedIngestion bool - - // IngestionRetryWait is the number of seconds that the client will wait for - // package ingestion before retrying. - IngestionRetryWait int - - // IngestionMaxRetries is the maximum number of requests the client will - // send while waiting for ingestion to finish - IngestionMaxRetries int -} +type Options = internalclient.Options // DefaultOptions is the default Trusty client options set -var DefaultOptions = Options{ - Workers: 2, - BaseURL: defaultEndpoint, - WaitForIngestion: true, - IngestionRetryWait: 5, -} - -type netClient interface { - GetRequestGroup([]string) ([]*http.Response, []error) - GetRequest(string) (*http.Response, error) +var DefaultOptions = internalclient.DefaultOptions + +// Trusty is a client on v1 Trusty APIs. +type Trusty interface { + // Report returns a dependency report with all the data that + // Trusty has available for a package. + Report(context.Context, *types.Dependency) (*types.Reply, error) + // GroupReport queries the Trusty API in parallel for a group + // of dependencies. + GroupReport(context.Context, []*types.Dependency) ([]*types.Reply, error) + + // PurlEndpoint returns the API endpoint url to query for data + // about a purl. + PurlEndpoint(string) (string, error) + // PackageEndpoint takes a dependency and returns the Trusty + // endpoint to query data about it. + PackageEndpoint(*types.Dependency) (string, error) + // PurlToEcosystem returns a trusty ecosystem constant from a + // Package URL's type. + PurlToEcosystem(string) types.Ecosystem + // PurlToDependency takes a string with a package url. + PurlToDependency(string) (*types.Dependency, error) } // New returns a new Trusty REST client -func New() *Trusty { - opts := DefaultOptions - opts.HttpClient = khttp.NewAgent().WithMaxParallel(opts.Workers).WithFailOnHTTPError(true) - if ep := os.Getenv(endpointEnvVar); ep != "" { - opts.BaseURL = ep - } - return NewWithOptions(opts) +func New() Trusty { + return internalclient.New() } // NewWithOptions returns a new client with the specified options set -func NewWithOptions(opts Options) *Trusty { - if opts.BaseURL == "" { - opts.BaseURL = DefaultOptions.BaseURL - } - - if opts.Workers == 0 { - opts.Workers = DefaultOptions.Workers - } - - if opts.HttpClient == nil { - opts.HttpClient = khttp.NewAgent().WithMaxParallel(opts.Workers).WithFailOnHTTPError(true) - } - - return &Trusty{ - Options: opts, - } -} - -func urlFromEndpointAndPaths( - baseUrl, endpoint string, params map[string]string, -) (*url.URL, error) { - u, err := url.Parse(baseUrl) - if err != nil { - return nil, fmt.Errorf("failed to parse endpoint: %w", err) - } - u = u.JoinPath(endpoint) - - // Add query parameters for package_name and package_type - q := u.Query() - for k, v := range params { - q.Set(k, v) - } - u.RawQuery = q.Encode() - - return u, nil -} - -// Trusty is the main trusty client -type Trusty struct { - Options Options -} - -// GroupReport queries the Trusty API in parallel for a group of dependencies. -func (t *Trusty) GroupReport(_ context.Context, deps []*types.Dependency) ([]*types.Reply, error) { - urls := []string{} - for _, dep := range deps { - u, err := t.PackageEndpoint(dep) - if err != nil { - return nil, fmt.Errorf("unable to get endpoint for: %q: %w", dep.Name, err) - } - urls = append(urls, u) - } - - responses, errs := t.Options.HttpClient.GetRequestGroup(urls) - if err := errors.Join(errs...); err != nil { - return nil, fmt.Errorf("fetching data from Trusty: %w", err) - } - - // Parse the replies - resps := make([]*types.Reply, len(responses)) - for i := range responses { - defer responses[i].Body.Close() - dec := json.NewDecoder(responses[i].Body) - resps[i] = &types.Reply{} - if err := dec.Decode(resps[i]); err != nil { - return nil, fmt.Errorf("could not unmarshal response #%d: %w", i, err) - } - } - return resps, nil -} - -// PurlEndpoint returns the API endpoint url to query for data about a purl -func (t *Trusty) PurlEndpoint(purl string) (string, error) { - dep, err := t.PurlToDependency(purl) - if err != nil { - return "", fmt.Errorf("getting dependency from %q", purl) - } - ep, err := t.PackageEndpoint(dep) - if err != nil { - return "", fmt.Errorf("getting package endpoint: %w", err) - } - return ep, nil -} - -// PackageEndpoint takes a dependency and returns the Trusty endpoint to -// query data about it. -func (t *Trusty) PackageEndpoint(dep *types.Dependency) (string, error) { - // Check dependency data: - errs := []error{} - if dep.Name == "" { - errs = append(errs, fmt.Errorf("dependency has no name defined")) - } - if dep.Ecosystem.AsString() == "" { - errs = append(errs, fmt.Errorf("dependency has no ecosystem set")) - } - - if err := errors.Join(errs...); err != nil { - return "", err - } - - u, err := url.Parse(t.Options.BaseURL + "/" + reportPath) - if err != nil { - return "", fmt.Errorf("failed to parse endpoint: %w", err) - } - - params := map[string]string{ - "package_name": dep.Name, - "package_type": strings.ToLower(dep.Ecosystem.AsString()), - } - - // Add query parameters for package_name and package_type - q := u.Query() - for k, v := range params { - q.Set(k, v) - } - u.RawQuery = q.Encode() - - return u.String(), nil -} - -// PurlToEcosystem returns a trusty ecosystem constant from a Package URL's type -func (_ *Trusty) PurlToEcosystem(purl string) types.Ecosystem { - switch { - case strings.HasPrefix(purl, "pkg:golang"): - return types.ECOSYSTEM_GO - case strings.HasPrefix(purl, "pkg:npm"): - return types.ECOSYSTEM_NPM - case strings.HasPrefix(purl, "pkg:pypi"): - return types.ECOSYSTEM_PYPI - default: - return types.Ecosystem(0) - } -} - -// PurlToDependency takes a string with a package url -func (t *Trusty) PurlToDependency(purlString string) (*types.Dependency, error) { - e := t.PurlToEcosystem(purlString) - if e == 0 { - // Ecosystem nil or not supported - return nil, fmt.Errorf("ecosystem not supported") - } - - purl, err := packageurl.FromString(purlString) - if err != nil { - return nil, fmt.Errorf("unable to parse package url: %w", err) - } - name := purl.Name - if purl.Namespace != "" { - name = purl.Namespace + "/" + purl.Name - } - return &types.Dependency{ - Ecosystem: e, - Name: name, - Version: purl.Version, - }, nil -} - -// Report returns a dependency report with all the data that Trusty has -// available for a package. -func (t *Trusty) Report(_ context.Context, dep *types.Dependency) (*types.Reply, error) { - u, err := t.PackageEndpoint(dep) - if err != nil { - return nil, fmt.Errorf("computing package endpoint: %w", err) - } - - var r types.Reply - tries := 0 - for { - resp, err := t.Options.HttpClient.GetRequest(u) - if err != nil { - return nil, fmt.Errorf("could not send request: %w", err) - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("received non-200 response: %d", resp.StatusCode) - } - - dec := json.NewDecoder(resp.Body) - if err := dec.Decode(&r); err != nil { - return nil, fmt.Errorf("could not unmarshal response: %w", err) - } - fmt.Printf("Attempt #%d to fetch package, status: %s", tries, r.PackageData.Status) - - shouldRetry, err := evalRetry(r.PackageData.Status, t.Options) - if err != nil { - return nil, err - } - - if !shouldRetry { - break - } - - tries++ - if tries > t.Options.IngestionMaxRetries { - return nil, fmt.Errorf("time out reached waiting for package ingestion") - } - time.Sleep(time.Duration(t.Options.IngestionRetryWait) * time.Second) - } - - return &r, err -} - -func evalRetry(status string, opts Options) (shouldRetry bool, err error) { - // First, error if the ingestion status is invalid - if status != types.IngestStatusFailed && status != types.IngestStatusComplete && - status != types.IngestStatusPending && status != types.IngestStatusScoring { - - return false, fmt.Errorf("unexpected ingestion status when querying package") - } - - if status == types.IngestStatusFailed && opts.ErrOnFailedIngestion { - return false, fmt.Errorf("upstream error ingesting package data") - } - - // Package ingestion is ready - if status == types.IngestStatusComplete { - return false, nil - } - - // Client configured to return raw response (even when package is not ready) - if !opts.WaitForIngestion || status == types.IngestStatusFailed { - return false, nil - } - - return true, nil +func NewWithOptions(opts Options) Trusty { + return internalclient.NewWithOptions(opts) } diff --git a/pkg/v2/client/client.go b/pkg/v2/client/client.go new file mode 100644 index 0000000..97d2551 --- /dev/null +++ b/pkg/v2/client/client.go @@ -0,0 +1,46 @@ +// Copyright 2024 Stacklok, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package client provides a rest client to talk to the Trusty API v2. +package client + +import ( + "context" + + internalclient "github.com/stacklok/trusty-sdk-go/internal/client" + types "github.com/stacklok/trusty-sdk-go/pkg/v2/types" +) + +// Options configures the Trusty API client +type Options = internalclient.Options + +// DefaultOptions is the default Trusty client options set +var DefaultOptions = internalclient.DefaultOptions + +// Trusty is a client on v2 Trusty APIs. +type Trusty interface { + Summary(context.Context, *types.Dependency) (*types.PackageSummaryAnnotation, error) + PackageMetadata(context.Context, *types.Dependency) (*types.TrustyPackageData, error) + Alternatives(context.Context, *types.Dependency) (*types.PackageAlternatives, error) +} + +// New returns a new Trusty REST client +func New() Trusty { + return internalclient.New() +} + +// NewWithOptions returns a new client with the specified options set +func NewWithOptions(opts Options) Trusty { + return internalclient.NewWithOptions(opts) +} diff --git a/pkg/v2/types/types.go b/pkg/v2/types/types.go new file mode 100644 index 0000000..45f218b --- /dev/null +++ b/pkg/v2/types/types.go @@ -0,0 +1,320 @@ +// +// Copyright 2024 Stacklok, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package types is the collection of main data types used by the +// Trusty libraries +package types + +import ( + "encoding/json" + "fmt" + "time" + + "github.com/google/uuid" +) + +// Dependency represents request arguments for various endpoints. +type Dependency struct { + PackageName string + PackageType *string + PackageVersion *string +} + +// PackageSummaryAnnotation represents a package annotation. +type PackageSummaryAnnotation struct { + Score *float64 `json:"score"` + Description SummaryDescription `json:"description"` + Status *Status `json:"status"` + // The following field is currently not straightroward to + // parse because it lacks timezone information, + // i.e. 2024-11-14T11:24:09.119788. + // + // It is not a huge gap at the moment, so I'd rather add it + // back once we agree on the format. + // + // UpdatedAt *time.Time `json:"updated_at"` +} + +// Status represents that processing status of a package. It might be +// `"in_progress"` if the package was never seen previously, and +// changes to `"complete"` once processed. +type Status string + +var ( + // StatusInProgress represents a package being processed. + StatusInProgress Status = "in_progress" + // StatusComplete represents an already processed package. + StatusComplete Status = "complete" +) + +//nolint:revive +func (t *Status) UnmarshalJSON(data []byte) error { + var tmp string + if err := json.Unmarshal(data, &tmp); err != nil { + return err + } + + switch tmp { + case "in_progress": + *t = StatusInProgress + case "complete": + *t = StatusComplete + default: + return fmt.Errorf("invalid status type: %s", tmp) + } + + return nil +} + +// SummaryDescription is the response body of a `GET /v2/summary` +type SummaryDescription struct { + From string `json:"from"` + Provenance float64 `json:"provenance"` + TrustSummary float64 `json:"trust-summary"` + TypoSquatting float64 `json:"typosquatting"` + ActivityUser float64 `json:"activity_user"` + ActivityRepo float64 `json:"activity_repo"` + Activity float64 `json:"activity"` + TrustActivity float64 `json:"trust-activity"` + Malicious bool `json:"malicious"` + ProvenanceType *ProvenanceType `json:"provenance_type"` +} + +// ProvenanceType is the type of provenance information that Trusty +// was able to gather. +type ProvenanceType string + +var ( + // ProvenanceTypeVerifiedProvenance represents a fully + // verified provenance information. + ProvenanceTypeVerifiedProvenance ProvenanceType = "verified_provenance" + // ProvenanceTypeHistoricalProvenance represents a verified + // historical provenance information. + ProvenanceTypeHistoricalProvenance ProvenanceType = "historical_provenance_match" + // ProvenanceTypeUnknown represents no provenance information. + ProvenanceTypeUnknown ProvenanceType = "unknown" + // ProvenanceTypeMismatched represents conflicting provenance + // information. + ProvenanceTypeMismatched ProvenanceType = "mismatched" +) + +//nolint:revive +func (t *ProvenanceType) UnmarshalJSON(data []byte) error { + var tmp string + if err := json.Unmarshal(data, &tmp); err != nil { + return err + } + + switch tmp { + case "verified_provenance": + *t = ProvenanceTypeVerifiedProvenance + case "historical_provenance_match": + *t = ProvenanceTypeHistoricalProvenance + case "unknown": + *t = ProvenanceTypeUnknown + case "mismatched": + *t = ProvenanceTypeMismatched + default: + return fmt.Errorf("invalid provenance type: %s", tmp) + } + + return nil +} + +// PackageType represents a package's ecosystem. +type PackageType string + +// Implementation note: we do not implement `UnmarshalJSON` for +// `PackageType` because we want to get new package types seamlessly +// as they're added to Trusty. The downside of this is that sdk users +// must match new types manually until we add the case to the list. + +var ( + // PackageTypePypi is the ecosystem of Python packages. + PackageTypePypi PackageType = "pypi" + // PackageTypeNpm is the ecosystem of JavaScript packages. + PackageTypeNpm PackageType = "npm" + // PackageTypeCrates is the ecosystem of Rust packages. + PackageTypeCrates PackageType = "crates" + // PackageTypeMaven is the ecosystem of Java packages. + PackageTypeMaven PackageType = "maven" + // PackageTypeGo is the ecosystem of Go packages. + PackageTypeGo PackageType = "go" +) + +// PackageStatus represents a package's status in the package +// repository. +type PackageStatus string + +var ( + // PackageStatusPending represents status pending + PackageStatusPending PackageStatus = "pending" + // PackageStatusInitial represents status initial + PackageStatusInitial PackageStatus = "initial" + // PackageStatusNeighbours represents status neoghbours + PackageStatusNeighbours PackageStatus = "neighbours" + // PackageStatusComplete represents status complete + PackageStatusComplete PackageStatus = "complete" + // PackageStatusFailed represents status failed + PackageStatusFailed PackageStatus = "failed" + // PackageStatusScoring represents status scoring + PackageStatusScoring PackageStatus = "scoring" + // PackageStatusPropagate represents status propagate + PackageStatusPropagate PackageStatus = "propagate" + // PackageStatusDeleted represents status deleted + PackageStatusDeleted PackageStatus = "deleted" +) + +//nolint:revive +func (t *PackageStatus) UnmarshalJSON(data []byte) error { + var tmp string + if err := json.Unmarshal(data, &tmp); err != nil { + return err + } + + switch tmp { + case "pending": + *t = PackageStatusPending + case "initial": + *t = PackageStatusInitial + case "neighbours": + *t = PackageStatusNeighbours + case "complete": + *t = PackageStatusComplete + case "failed": + *t = PackageStatusFailed + case "scoring": + *t = PackageStatusScoring + case "propagate": + *t = PackageStatusPropagate + case "deleted": + *t = PackageStatusDeleted + default: + return fmt.Errorf("invalid package status type: %s", tmp) + } + + return nil +} + +// TrustyPackageData is the full package information as returned from +// the `GET /v2/pkg` API call. In contrast to `PackageBrief` this +// structure captures the complete data that trusty knows about the +// described package. +type TrustyPackageData struct { + ID *uuid.UUID `json:"id"` + Status *PackageStatus `json:"status"` + StatusCode *string `json:"status_code"` + Name string `json:"name"` + Type PackageType `json:"type"` + Version *string `json:"version"` + // The following field is currently not straightroward to + // parse because it lacks timezone information, + // i.e. 2024-11-14T11:24:09.119788. + // + // It is not a huge gap at the moment, so I'd rather add it + // back once we agree on the format. + // + // VersionDate *time.Time `json:"version_date"` + Author *string `json:"author"` + AuthorEmail *string `json:"author_email"` + Description *string `json:"packag_description"` + RepoDescription *string `json:"repo_description"` + Origin *string `json:"origin"` + StarGazersCount *int `json:"stargazers_count"` + WatchersCount *int `json:"watchers_count"` + HomePage *string `json:"home_page"` + HasIssues *bool `json:"has_issues"` + HasProjects *bool `json:"has_projects"` + HasDownloads *bool `json:"has_downloads"` + ForksCount *int `json:"forks_count"` + Archived *bool `json:"archived"` + IsDeprecated *bool `json:"is_deprecated"` + Disabled *bool `json:"disabled"` + OpenIssuesCount *int `json:"open_issues_count"` + Visibility *string `json:"visibility"` + DefaultBranch *string `json:"default_branch"` + RepositoryID *uuid.UUID `json:"repository_id"` + RepositoryName *string `json:"repository_name"` + ContributorCount *int `json:"contributor_count"` + PublicRepos *int `json:"public_repos"` + PublicGists *int `json:"public_gists"` + Followers *int `json:"followers"` + Following *int `json:"following"` + Owner *User `json:"owner"` + Contributors []*User `json:"contributors"` + // The following field is currently not straightroward to + // parse because it lacks timezone information, + // i.e. 2024-11-14T11:24:09.119788. + // + // It is not a huge gap at the moment, so I'd rather add it + // back once we agree on the format. + // + // LastUpdate *time.Time `json:"last_update"` + Scores interface{} `json:"scores"` + Malicious *PackageMaliciousPayload `json:"malicious"` + HasTriggeredReingestion *bool `json:"has_triggered_reingestion"` +} + +// User represents an individual or bot that acts on repositories in +// some way. +type User struct { + Id uuid.UUID `json:"id"` + Author *string `json:"author"` + Author_email *string `json:"author_email"` + Login *string `json:"login"` + Avatar_url *string `json:"avatar_url"` + Gravatar_id *string `json:"gravatar_id"` + Url *string `json:"url"` + Html_url *string `json:"html_url"` + Company *string `json:"company"` + Blog *string `json:"blog"` + Location *string `json:"location"` + Email *string `json:"email"` + Hireable bool `json:"hireable"` + Twitter_username *string `json:"twitter_username"` + Public_repos *int `json:"public_repos"` + Public_gists *int `json:"public_gists"` + Followers *int `json:"followers"` + Following *int `json:"following"` + Scores interface{} `json:"scores"` +} + +// PackageMaliciousPayload represents the payload details for a +// malicious package. +type PackageMaliciousPayload struct { + Summary string `json:"summary"` + Details *string `json:"details"` + Published *time.Time `json:"published"` + Modified *time.Time `json:"modified"` + Source *string `json:"source"` +} + +// PackageAlternatives is a list of alternative packages to the one +// requested. +type PackageAlternatives struct { + Status Status `json:"status"` // in_progress or complete + Packages []*PackageBasicInfo `json:"packages"` +} + +// PackageBasicInfo contains basic information about a package. +type PackageBasicInfo struct { + ID *uuid.UUID `json:"id"` + PackageName string `json:"package_name"` + PackageType *PackageType `json:"package_type"` + PackageVersion *string `json:"package_version"` + RepoDescription *string `json:"repo_description"` + Score *float64 `json:"score"` + IsMalicious bool `json:"is_malicious"` +}