lookup/validate tags using Github API

Author: dmgk

apis/apis.go

@@ -4,10 +4,33 @@ import (
 	"fmt"
 	"io/ioutil"
 	"net/http"
+	"os"
+	"strings"
 )
 
-func get(url string) ([]byte, error) {
-	resp, err := http.Get(url)
+const (
+	OfflineKey           = "M2T_OFFLINE"
+	GithubCredentialsKey = "M2T_GITHUB"
+	// GitlabsCredentialsKey = "M2T_GITLAB"
+)
+
+func get(url string, credsKey string) ([]byte, error) {
+	req, err := http.NewRequest("GET", url, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	if credsKey != "" {
+		creds := os.Getenv(credsKey)
+		if creds != "" {
+			credsSlice := strings.Split(creds, ":")
+			if len(credsSlice) == 2 {
+				req.SetBasicAuth(credsSlice[0], credsSlice[1])
+			}
+		}
+	}
+
+	resp, err := http.DefaultClient.Do(req)
 	if err != nil {
 		return nil, fmt.Errorf("GET %s: %v", url, err)
 	}

apis/github.go

@@ -2,27 +2,71 @@ package apis
 
 import (
 	"encoding/json"
+	"errors"
 	"fmt"
 	"net/url"
+	"strings"
 )
 
 type GithubCommit struct {
-	ID string `json:"sha"`
+	SHA string `json:"sha"`
 }
 
-func GetGithubCommit(account, project, ref string) (*GithubCommit, error) {
+type GithubRef struct {
+	Ref string `json:"ref"`
+}
+
+var githubRateLimitError = fmt.Sprintf(`Github API rate limit exceeded. Please either:
+- set %s environment variable to your Github "username:personal_access_token"
+  to let modules2tuple call Github API using basic authentication.
+  To create a new token, navigate to https://github.com/settings/tokens/new
+  (leave all checkboxes unchecked, modules2tuple doesn't need any access to your account)
+- set %s=1 or pass "-offline" flag to module2tuple to disable network access`, OfflineKey, GithubCredentialsKey)
+
+func GetGithubCommit(account, project, tag string) (string, error) {
+	projectID := fmt.Sprintf("%s/%s", url.PathEscape(account), url.PathEscape(project))
+	url := fmt.Sprintf("https://api.github.com/repos/%s/commits/%s", projectID, tag)
+
+	resp, err := get(url, GithubCredentialsKey)
+	if err != nil {
+		if strings.Contains(err.Error(), "API rate limit exceeded") {
+			return "", errors.New(githubRateLimitError)
+		}
+		return "", fmt.Errorf("error getting commit %s for %s/%s: %v", tag, account, project, err)
+	}
+
+	var res GithubCommit
+	if err := json.Unmarshal(resp, &res); err != nil {
+		return "", fmt.Errorf("error unmarshalling: %v, resp: %v", err, string(resp))
+	}
+
+	return res.SHA, nil
+}
+
+func LookupGithubTag(account, project, tag string) (string, error) {
 	projectID := fmt.Sprintf("%s/%s", url.PathEscape(account), url.PathEscape(project))
-	url := fmt.Sprintf("https://api.github.com/repos/%s/commits/%s", projectID, ref)
+	url := fmt.Sprintf("https://api.github.com/repos/%s/git/refs/tags", projectID)
 
-	resp, err := get(url)
+	resp, err := get(url, GithubCredentialsKey)
 	if err != nil {
-		return nil, fmt.Errorf("error getting commit %s for %s/%s: %v", ref, account, project, err)
+		if strings.Contains(err.Error(), "API rate limit exceeded") {
+			return "", errors.New(githubRateLimitError)
+		}
+		return "", fmt.Errorf("error getting refs for %s/%s: %v", account, project, err)
+	}
+
+	var res []GithubRef
+	if err := json.Unmarshal(resp, &res); err != nil {
+		return "", fmt.Errorf("error unmarshalling: %v, resp: %v", err, string(resp))
 	}
 
-	var ret GithubCommit
-	if err := json.Unmarshal(resp, &ret); err != nil {
-		return nil, fmt.Errorf("error unmarshalling: %v, resp: %v", err, string(resp))
+	// Github API returns tags sorted by creation time, earliest first.
+	// Iterate through them in reverse order to find the most recent matching tag.
+	for i := len(res) - 1; i >= 0; i-- {
+		if strings.HasSuffix(res[i].Ref, "/"+tag) {
+			return strings.TrimPrefix(res[i].Ref, "refs/tags/"), nil
+		}
 	}
 
-	return &ret, nil
+	return "", fmt.Errorf("tag %v doesn't seem to exist in %s/%s", tag, account, project)
 }

apis/github_test.go

@@ -6,19 +6,40 @@ import "testing"
 
 func TestGetGithubCommit(t *testing.T) {
 	examples := []struct {
-		account, project, ref, ID string
+		account, project, ref, hash string
 	}{
 		{"dmgk", "modules2tuple", "v1.9.0", "fc09878b93db35aafc74311f7ea6684ac08a3b83"},
 		{"dmgk", "modules2tuple", "a0cdb416ca2c", "a0cdb416ca2cbf6d3dad67a97f4fdcfac954503e"},
 	}
 
 	for i, x := range examples {
-		c, err := GetGithubCommit(x.account, x.project, x.ref)
+		hash, err := GetGithubCommit(x.account, x.project, x.ref)
 		if err != nil {
 			t.Fatal(err)
 		}
-		if x.ID != c.ID {
-			t.Errorf("expected commit ID %s, got %s (example %d)", x.ID, c.ID, i)
+		if x.hash != hash {
+			t.Errorf("expected commit hash %s, got %s (example %d)", x.hash, hash, i)
+		}
+	}
+}
+
+func TestLookupGithubTag(t *testing.T) {
+	examples := []struct {
+		account, project, given, expected string
+	}{
+		{"hashicorp", "vault", "v1.0.4", "api/v1.0.4"},
+		{"hashicorp", "vault", "v1.3.4", "v1.3.4"},
+		// this repo has earlier mathing tag "codec/codecgen/v1.1.7"
+		{"ugorji", "go", "v1.1.7", "v1.1.7"},
+	}
+
+	for i, x := range examples {
+		tag, err := LookupGithubTag(x.account, x.project, x.given)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if x.expected != tag {
+			t.Errorf("expected tag %s, got %s (example %d)", x.expected, tag, i)
 		}
 	}
 }

apis/gitlab.go

@@ -7,22 +7,22 @@ import (
 )
 
 type GitlabCommit struct {
-	ID string `json:"id"`
+	SHA string `json:"id"`
 }
 
-func GetGitlabCommit(site, account, project, commit string) (*GitlabCommit, error) {
+func GetGitlabCommit(site, account, project, commit string) (string, error) {
 	projectID := url.PathEscape(fmt.Sprintf("%s/%s", account, project))
 	url := fmt.Sprintf("%s/api/v4/projects/%s/repository/commits/%s", site, projectID, commit)
 
-	resp, err := get(url)
+	resp, err := get(url, "")
 	if err != nil {
-		return nil, fmt.Errorf("error getting commit %s for %s/%s: %v", commit, account, project, err)
+		return "", fmt.Errorf("error getting commit %s for %s/%s: %v", commit, account, project, err)
 	}
 
-	var ret GitlabCommit
-	if err := json.Unmarshal(resp, &ret); err != nil {
-		return nil, fmt.Errorf("error unmarshalling: %v, resp: %v", err, string(resp))
+	var res GitlabCommit
+	if err := json.Unmarshal(resp, &res); err != nil {
+		return "", fmt.Errorf("error unmarshalling: %v, resp: %v", err, string(resp))
 	}
 
-	return &ret, nil
+	return res.SHA, nil
 }

apis/gitlab_test.go

@@ -6,19 +6,19 @@ import "testing"
 
 func TestGetGitlabCommit(t *testing.T) {
 	examples := []struct {
-		site, account, project, ref, ID string
+		site, account, project, ref, hash string
 	}{
 		{"https://gitlab.com", "gitlab-org", "gitaly-proto", "v1.32.0", "f4db5d05d437abe1154d7308ca044d3577b5ccba"},
 		{"https://gitlab.com", "gitlab-org", "labkit", "0c3fc7cdd57c", "0c3fc7cdd57c57da5ab474aa72b6640d2bdc9ebb"},
 	}
 
 	for i, x := range examples {
-		c, err := GetGitlabCommit(x.site, x.account, x.project, x.ref)
+		hash, err := GetGitlabCommit(x.site, x.account, x.project, x.ref)
 		if err != nil {
 			t.Fatal(err)
 		}
-		if x.ID != c.ID {
-			t.Errorf("expected commit ID %s, got %s (example %d)", x.ID, c.ID, i)
+		if x.hash != hash {
+			t.Errorf("expected commit hash %s, got %s (example %d)", x.hash, hash, i)
 		}
 	}
 }

main.go

@@ -7,6 +7,7 @@ import (
 	"path"
 	"text/template"
 
+	"github.com/dmgk/modules2tuple/apis"
 	"github.com/dmgk/modules2tuple/tuple"
 )
 
@@ -25,14 +26,19 @@ func main() {
 		os.Exit(1)
 	}
 
+	var haveTuples bool
 	parser := tuple.NewParser(flagPackagePrefix, flagOffline)
 	tuples, errors := parser.Load(args[0])
 	if len(tuples) != 0 {
 		fmt.Print(tuples)
+		haveTuples = true
 	}
 	if errors != nil {
-		fmt.Println()
+		if haveTuples {
+			fmt.Println()
+		}
 		fmt.Print(errors)
+		fmt.Println()
 	}
 }
 
@@ -53,7 +59,7 @@ this commit ID translation can be disabled with -offline flag.
 `))
 
 var (
-	flagOffline       = false
+	flagOffline       = os.Getenv(apis.OfflineKey) != ""
 	flagPackagePrefix = "vendor"
 	flagVersion       = false
 )

tuple/parser.go

@@ -50,15 +50,28 @@ func (p *Parser) Read(r io.Reader) (Tuples, error) {
 						return
 					}
 					if !p.offline {
-						// Call Gitlab API to translate go.mod short commit IDs and tags
-						// to the full 40-character commit IDs as required by bsd.sites.mk
-						if _, ok := t.Source.(GL); ok {
-							c, err := apis.GetGitlabCommit(t.Source.Site(), t.Account, t.Project, t.Tag)
+						switch t.Source.(type) {
+						case GH:
+							if strings.HasPrefix(t.Tag, "v") {
+								// Call Gihub API to check tags. Go seem to be able to magically
+								// translate tags like "v1.0.4" to the "api/v1.0.4" which is really used
+								// by upstream. We'll try to do the same.
+								tag, err := apis.LookupGithubTag(t.Account, t.Project, t.Tag)
+								if err != nil {
+									ch <- err
+									return
+								}
+								t.Tag = tag
+							}
+						case GL:
+							// Call Gitlab API to translate go.mod short commit IDs and tags
+							// to the full 40-character commit IDs as required by bsd.sites.mk
+							hash, err := apis.GetGitlabCommit(t.Source.Site(), t.Account, t.Project, t.Tag)
 							if err != nil {
 								ch <- err
 								return
 							}
-							t.Tag = c.ID
+							t.Tag = hash
 						}
 					}
 					ch <- t

tuple/tuple.go

@@ -82,14 +82,14 @@ func (tt Tuples) EnsureUniqueGithubProjectAndTag() error {
 		if t.Account != prevTuple.Account {
 			// different Account, but the same Project and Tag
 			if t.Project == prevTuple.Project && t.Tag == prevTuple.Tag {
-				c, err := apis.GetGithubCommit(t.Account, t.Project, t.Tag)
+				hash, err := apis.GetGithubCommit(t.Account, t.Project, t.Tag)
 				if err != nil {
 					return DuplicateProjectAndTag(t.String())
 				}
-				if len(c.ID) < 12 {
-					return errors.New("unexpectedly short commit ID")
+				if len(hash) < 12 {
+					return errors.New("unexpectedly short Githib commit hash")
 				}
-				t.Tag = c.ID[:12]
+				t.Tag = hash[:12]
 			}
 		}
 	}

tuple/tuple_online_test.go

@@ -22,6 +22,6 @@ func TestUniqueProjectAndTag(t *testing.T) {
 	}
 	out := tt.String()
 	if out != expected {
-		t.Errorf("expected output %s, got %s", expected, out)
+		t.Errorf("expected output\n%s, got\n%s", expected, out)
 	}
 }