From d646ef370fe1ec45a8113f0fb8c70435c63058d0 Mon Sep 17 00:00:00 2001 From: Amit Saha Date: Thu, 28 Sep 2023 07:25:55 +1000 Subject: [PATCH 1/8] refactor repositories related code --- bitbucket.go | 5 --- github.go | 93 +++++++++++++++++++++++------------------------- repositories.go | 5 +++ show_coverage.sh | 4 +++ 4 files changed, 54 insertions(+), 53 deletions(-) create mode 100755 show_coverage.sh diff --git a/bitbucket.go b/bitbucket.go index 01a4053..a7f3401 100644 --- a/bitbucket.go +++ b/bitbucket.go @@ -1,7 +1,6 @@ package main import ( - "log" "strings" bitbucket "github.com/ktrysmt/go-bitbucket" @@ -14,10 +13,6 @@ func getBitbucketRepositories( ignoreFork bool, ) ([]*Repository, error) { - if client == nil { - log.Fatalf("Couldn't acquire a client to talk to %s", service) - } - var repositories []*Repository var cloneURL string diff --git a/github.go b/github.go index 2b481c4..320af85 100644 --- a/github.go +++ b/github.go @@ -2,7 +2,6 @@ package main import ( "context" - "log" "strings" "github.com/google/go-github/v34/github" @@ -15,73 +14,71 @@ func getGithubRepositories( ignoreFork bool, ) ([]*Repository, error) { - if client == nil { - log.Fatalf("Couldn't acquire a client to talk to %s", service) - } - var repositories []*Repository var cloneURL string - ctx := context.Background() + var ghRepository []*github.Repository + + ctx := context.TODO() - if githubRepoType == "starred" { + switch githubRepoType { + case "starred": options := github.ActivityListStarredOptions{} for { stars, resp, err := client.(*github.Client).Activity.ListStarred(ctx, "", &options) - if err == nil { - for _, star := range stars { - if *star.Repository.Fork && ignoreFork { - continue - } - namespace := strings.Split(*star.Repository.FullName, "/")[0] - if useHTTPSClone != nil && *useHTTPSClone { - cloneURL = *star.Repository.CloneURL - } else { - cloneURL = *star.Repository.SSHURL - } - repositories = append(repositories, &Repository{CloneURL: cloneURL, Name: *star.Repository.Name, Namespace: namespace, Private: *star.Repository.Private}) - } - } else { + if err != nil { return nil, err } + for _, star := range stars { + ghRepository = append(ghRepository, star.Repository) + } if resp.NextPage == 0 { break } options.ListOptions.Page = resp.NextPage } - return repositories, nil - } + default: + options := github.RepositoryListOptions{Type: githubRepoType} - options := github.RepositoryListOptions{Type: githubRepoType} - githubNamespaceWhitelistLength := len(githubNamespaceWhitelist) - - for { - repos, resp, err := client.(*github.Client).Repositories.List(ctx, "", &options) - if err == nil { + for { + repos, resp, err := client.(*github.Client).Repositories.List(ctx, "", &options) + if err != nil { + return nil, err + } for _, repo := range repos { - if *repo.Fork && ignoreFork { - continue - } - namespace := strings.Split(*repo.FullName, "/")[0] + ghRepository = append(ghRepository, repo) + } + if resp.NextPage == 0 { + break + } + options.ListOptions.Page = resp.NextPage + } + } - if githubNamespaceWhitelistLength > 0 && !contains(githubNamespaceWhitelist, namespace) { - continue - } + githubNamespaceWhitelistLength := len(githubNamespaceWhitelist) + for _, repo := range ghRepository { + if *repo.Fork && ignoreFork { + continue + } - if useHTTPSClone != nil && *useHTTPSClone { - cloneURL = *repo.CloneURL - } else { - cloneURL = *repo.SSHURL - } - repositories = append(repositories, &Repository{CloneURL: cloneURL, Name: *repo.Name, Namespace: namespace, Private: *repo.Private}) - } - } else { - return nil, err + namespace := strings.Split(*repo.FullName, "/")[0] + if githubNamespaceWhitelistLength > 0 && !contains(githubNamespaceWhitelist, namespace) { + continue } - if resp.NextPage == 0 { - break + ghRepository = append(ghRepository, repo) + cloneURL = *repo.SSHURL + if useHTTPSClone != nil && *useHTTPSClone { + cloneURL = *repo.CloneURL } - options.ListOptions.Page = resp.NextPage + repositories = append( + repositories, + &Repository{ + CloneURL: cloneURL, + Name: *repo.Name, + Namespace: namespace, + Private: *repo.Private, + }, + ) } return repositories, nil } diff --git a/repositories.go b/repositories.go index 93d48c5..21d4fe7 100644 --- a/repositories.go +++ b/repositories.go @@ -1,6 +1,7 @@ package main import ( + "fmt" "net/http" ) @@ -39,6 +40,10 @@ func getRepositories( var repositories []*Repository var err error + if client == nil { + return nil, fmt.Errorf("Couldn't acquire a client to talk to %s", service) + } + switch service { case "github": repositories, err = getGithubRepositories( diff --git a/show_coverage.sh b/show_coverage.sh new file mode 100755 index 0000000..d3a7465 --- /dev/null +++ b/show_coverage.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +go test -coverprofile cover.out +go tool cover -html=cover.out From d3e6938e1f27dcd391688e2a25bf5b1e4d33cfdb Mon Sep 17 00:00:00 2001 From: Amit Saha Date: Thu, 28 Sep 2023 07:45:31 +1000 Subject: [PATCH 2/8] Refactor the client creation code --- client.go | 46 +++++++++++++--------------------------------- client_test.go | 12 ++++++------ config.go | 17 ++++++++++------- main.go | 2 +- options.go | 12 ++++++++++++ user_data.go | 8 +++++++- 6 files changed, 49 insertions(+), 48 deletions(-) diff --git a/client.go b/client.go index 1ad993e..b5062b1 100644 --- a/client.go +++ b/client.go @@ -71,22 +71,11 @@ func getToken(service string) (string, error) { return string(i.Data), nil } -func newClient(service string, gitHostURL string) interface{} { - var gitHostURLParsed *url.URL +func newClient(service string, gitHostURLParsed *url.URL) interface{} { var err error - // If a git host URL has been passed in, we assume it's - // a gitlab installation - if len(gitHostURL) != 0 { - gitHostURLParsed, err = url.Parse(gitHostURL) - if err != nil { - log.Fatalf("Invalid gitlab URL: %s", gitHostURL) - } - api, _ := url.Parse("api/v4/") - gitHostURLParsed = gitHostURLParsed.ResolveReference(api) - } - - if service == "github" { + switch service { + case "github": githubToken := os.Getenv("GITHUB_TOKEN") if githubToken == "" { githubToken, err = getToken("GITHUB") @@ -95,11 +84,10 @@ func newClient(service string, gitHostURL string) interface{} { } if githubToken == "" { log.Fatal("GitHub token not available") - } else { - err := saveToken("GITHUB", githubToken) - if err != nil { - log.Fatal("Error saving token") - } + } + err := saveToken("GITHUB", githubToken) + if err != nil { + log.Fatal("Error saving token") } } gitHostToken = githubToken @@ -112,9 +100,8 @@ func newClient(service string, gitHostURL string) interface{} { client.BaseURL = gitHostURLParsed } return client - } - if service == "gitlab" { + case "gitlab": gitlabToken := os.Getenv("GITLAB_TOKEN") if gitlabToken == "" { log.Fatal("GITLAB_TOKEN environment variable not set") @@ -130,27 +117,20 @@ func newClient(service string, gitHostURL string) interface{} { log.Fatalf("Error creating gitlab client: %v", err) } return client - } - if service == "bitbucket" { + case "bitbucket": bitbucketUsername := os.Getenv("BITBUCKET_USERNAME") - if bitbucketUsername == "" { - log.Fatal("BITBUCKET_USERNAME environment variable not set") - } - bitbucketPassword := os.Getenv("BITBUCKET_PASSWORD") - if bitbucketPassword == "" { - log.Fatal("BITBUCKET_PASSWORD environment variable not set") + if bitbucketUsername == "" || bitbucketPassword == "" { + log.Fatal("BITBUCKET_USERNAME and BITBUCKET_PASSWORD environment variables must be set") } - gitHostToken = bitbucketPassword - client := bitbucket.NewBasicAuth(bitbucketUsername, bitbucketPassword) if gitHostURLParsed != nil { client.SetApiBaseURL(gitHostURLParsed.String()) } return client + default: + return nil } - - return nil } diff --git a/client_test.go b/client_test.go index d45506a..1d099a8 100644 --- a/client_test.go +++ b/client_test.go @@ -19,33 +19,33 @@ func TestNewClient(t *testing.T) { expectedGitHostBaseURL := customGitHost.ResolveReference(api) // Client for github.com - client := newClient("github", "") + client := newClient("github", nil) client = client.(*github.Client) // Client for Enterprise Github - client = newClient("github", customGitHost.String()) + client = newClient("github", customGitHost) gotBaseURL := client.(*github.Client).BaseURL if gotBaseURL.String() != expectedGitHostBaseURL.String() { t.Errorf("Expected BaseURL to be: %v, Got: %v\n", expectedGitHostBaseURL, gotBaseURL) } // Client for gitlab.com - client = newClient("gitlab", "") + client = newClient("gitlab", nil) client = client.(*gitlab.Client) // Client for custom gitlab installation - client = newClient("gitlab", customGitHost.String()) + client = newClient("gitlab", customGitHost) gotBaseURL = client.(*gitlab.Client).BaseURL() if gotBaseURL.String() != expectedGitHostBaseURL.String() { t.Errorf("Expected BaseURL to be: %v, Got: %v\n", expectedGitHostBaseURL, gotBaseURL) } // Client for bitbucket.com - client = newClient("bitbucket", "") + client = newClient("bitbucket", nil) client = client.(*bitbucket.Client) // Not yet supported - client = newClient("notyetsupported", "") + client = newClient("notyetsupported", nil) if client != nil { t.Errorf("Expected nil") } diff --git a/config.go b/config.go index 278663e..5c06335 100644 --- a/config.go +++ b/config.go @@ -1,13 +1,16 @@ package main +import "net/url" + type appConfig struct { - service string - gitHostURL string - backupDir string - ignorePrivate bool - ignoreFork bool - useHTTPSClone bool - bare bool + service string + gitHostURL string + gitHostURLParsed *url.URL + backupDir string + ignorePrivate bool + ignoreFork bool + useHTTPSClone bool + bare bool githubRepoType string githubNamespaceWhitelist []string diff --git a/main.go b/main.go index 640a17e..b447ac1 100644 --- a/main.go +++ b/main.go @@ -34,7 +34,7 @@ func main() { log.Fatal(err) } - client := newClient(c.service, c.gitHostURL) + client := newClient(c.service, c.gitHostURLParsed) var executionErr error // TODO implement validation of options so that we don't diff --git a/options.go b/options.go index d6b86cc..5908d4b 100644 --- a/options.go +++ b/options.go @@ -3,6 +3,7 @@ package main import ( "errors" "flag" + "net/url" "strings" ) @@ -85,5 +86,16 @@ func validateConfig(c *appConfig) error { if !validGitlabProjectMembership(c.gitlabProjectMembershipType) { return errors.New("Please specify a valid gitlab project membership - all/owner/member") } + + if len(c.gitHostURL) != 0 { + gitHostURLParsed, err := url.Parse(c.gitHostURL) + if err != nil { + return err + } + api, _ := url.Parse("api/v4/") + gitHostURLParsed = gitHostURLParsed.ResolveReference(api) + c.gitHostURLParsed = gitHostURLParsed + } + return nil } diff --git a/user_data.go b/user_data.go index f72251f..b85ffaf 100644 --- a/user_data.go +++ b/user_data.go @@ -8,6 +8,7 @@ import ( "io/ioutil" "log" "net/http" + "net/url" "os" "path" "strings" @@ -233,7 +234,12 @@ type GithubUserMigrationDeleteResult struct { // DeleteGithubUserMigration deletes an existing migration func DeleteGithubUserMigration(id *int64) GithubUserMigrationDeleteResult { - client := newClient("github", "https://github.com") + u, err := url.Parse("https://github.com") + if err != nil { + panic(err) + } + + client := newClient("github", u) ctx := context.Background() response, err := client.(*github.Client).Migrations.DeleteUserMigration(ctx, *id) From 46aa73deef83906207a99e09bb8e99e5e5daa20f Mon Sep 17 00:00:00 2001 From: Amit Saha Date: Thu, 28 Sep 2023 08:02:17 +1000 Subject: [PATCH 3/8] wip client refactor --- client.go | 16 +++++++++++++++- client_test.go | 2 +- config.go | 31 ++++++++++++++++++++++--------- main.go | 2 +- options.go | 23 ----------------------- 5 files changed, 39 insertions(+), 35 deletions(-) diff --git a/client.go b/client.go index b5062b1..11798f0 100644 --- a/client.go +++ b/client.go @@ -71,8 +71,17 @@ func getToken(service string) (string, error) { return string(i.Data), nil } -func newClient(service string, gitHostURLParsed *url.URL) interface{} { +func newClient(service string, c *appConfig) interface{} { var err error + var gitHostURLParsed *url.URL + + if len(c.gitHostURL) != 0 { + gitHostURLParsed, err = url.Parse(c.gitHostURL) + if err != nil { + log.Fatal(err) + } + gitHostURLParsed = gitHostURLParsed + } switch service { case "github": @@ -102,6 +111,11 @@ func newClient(service string, gitHostURLParsed *url.URL) interface{} { return client case "gitlab": + + if gitHostURLParsed != nil { + api, _ := url.Parse("api/v4/") + gitHostURLParsed = gitHostURLParsed.ResolveReference(api) + } gitlabToken := os.Getenv("GITLAB_TOKEN") if gitlabToken == "" { log.Fatal("GITLAB_TOKEN environment variable not set") diff --git a/client_test.go b/client_test.go index 1d099a8..2af8123 100644 --- a/client_test.go +++ b/client_test.go @@ -25,7 +25,7 @@ func TestNewClient(t *testing.T) { // Client for Enterprise Github client = newClient("github", customGitHost) gotBaseURL := client.(*github.Client).BaseURL - if gotBaseURL.String() != expectedGitHostBaseURL.String() { + if gotBaseURL.String() != customGitHost.String() { t.Errorf("Expected BaseURL to be: %v, Got: %v\n", expectedGitHostBaseURL, gotBaseURL) } diff --git a/config.go b/config.go index 5c06335..40c58f5 100644 --- a/config.go +++ b/config.go @@ -1,16 +1,17 @@ package main -import "net/url" +import ( + "errors" +) type appConfig struct { - service string - gitHostURL string - gitHostURLParsed *url.URL - backupDir string - ignorePrivate bool - ignoreFork bool - useHTTPSClone bool - bare bool + service string + gitHostURL string + backupDir string + ignorePrivate bool + ignoreFork bool + useHTTPSClone bool + bare bool githubRepoType string githubNamespaceWhitelist []string @@ -23,3 +24,15 @@ type appConfig struct { gitlabProjectVisibility string gitlabProjectMembershipType string } + +func validateConfig(c *appConfig) error { + if _, ok := knownServices[c.service]; !ok { + return errors.New("Please specify the git service type: github, gitlab, bitbucket") + } + + if !validGitlabProjectMembership(c.gitlabProjectMembershipType) { + return errors.New("Please specify a valid gitlab project membership - all/owner/member") + } + + return nil +} diff --git a/main.go b/main.go index b447ac1..d8e268a 100644 --- a/main.go +++ b/main.go @@ -34,7 +34,7 @@ func main() { log.Fatal(err) } - client := newClient(c.service, c.gitHostURLParsed) + client := newClient(c.service, c) var executionErr error // TODO implement validation of options so that we don't diff --git a/options.go b/options.go index 5908d4b..13182a8 100644 --- a/options.go +++ b/options.go @@ -3,7 +3,6 @@ package main import ( "errors" "flag" - "net/url" "strings" ) @@ -77,25 +76,3 @@ func initConfig(args []string) (*appConfig, error) { c.backupDir = setupBackupDir(&c.backupDir, &c.service, &c.gitHostURL) return &c, nil } - -func validateConfig(c *appConfig) error { - if _, ok := knownServices[c.service]; !ok { - return errors.New("Please specify the git service type: github, gitlab, bitbucket") - } - - if !validGitlabProjectMembership(c.gitlabProjectMembershipType) { - return errors.New("Please specify a valid gitlab project membership - all/owner/member") - } - - if len(c.gitHostURL) != 0 { - gitHostURLParsed, err := url.Parse(c.gitHostURL) - if err != nil { - return err - } - api, _ := url.Parse("api/v4/") - gitHostURLParsed = gitHostURLParsed.ResolveReference(api) - c.gitHostURLParsed = gitHostURLParsed - } - - return nil -} From 9368e7d57d8aad339855207dd7b04155e3cc18e9 Mon Sep 17 00:00:00 2001 From: Amit Saha Date: Sat, 14 Oct 2023 23:46:58 +1100 Subject: [PATCH 4/8] fix tests and remove Delete* function as it is unused --- client.go | 3 +-- client_test.go | 29 +++++++++++++++++++++-------- user_data.go | 26 -------------------------- 3 files changed, 22 insertions(+), 36 deletions(-) diff --git a/client.go b/client.go index 11798f0..f1a06fd 100644 --- a/client.go +++ b/client.go @@ -75,12 +75,11 @@ func newClient(service string, c *appConfig) interface{} { var err error var gitHostURLParsed *url.URL - if len(c.gitHostURL) != 0 { + if c != nil && len(c.gitHostURL) != 0 { gitHostURLParsed, err = url.Parse(c.gitHostURL) if err != nil { log.Fatal(err) } - gitHostURLParsed = gitHostURLParsed } switch service { diff --git a/client_test.go b/client_test.go index 2af8123..57cd656 100644 --- a/client_test.go +++ b/client_test.go @@ -13,35 +13,48 @@ func TestNewClient(t *testing.T) { setupRepositoryTests() defer teardownRepositoryTests() - customGitHost, _ := url.Parse("https://git.mycompany.com") + var defaultGithubConfig appConfig + defaultGithubConfig.gitHostURL = "https://github.com" + + var defaultGitlabConfig appConfig + defaultGitlabConfig.gitHostURL = "https://gitlab.com" + + var defaultBitbucketConfig appConfig + defaultBitbucketConfig.gitHostURL = "https://bitbucket.org" + + var customGithostConfig appConfig + customGitHost := "https://git.mycompany.com" + customGithostConfig.gitHostURL = customGitHost + // http://stackoverflow.com/questions/23051339/how-to-avoid-end-of-url-slash-being-removed-when-resolvereference-in-go api, _ := url.Parse("api/v4/") - expectedGitHostBaseURL := customGitHost.ResolveReference(api) + customGitHostParsed, _ := url.Parse(customGitHost) + expectedGitHostBaseURL := customGitHostParsed.ResolveReference(api) // Client for github.com - client := newClient("github", nil) + client := newClient("github", &defaultGithubConfig) client = client.(*github.Client) // Client for Enterprise Github - client = newClient("github", customGitHost) + client = newClient("github", &customGithostConfig) gotBaseURL := client.(*github.Client).BaseURL - if gotBaseURL.String() != customGitHost.String() { + if gotBaseURL.String() != customGitHostParsed.String() { t.Errorf("Expected BaseURL to be: %v, Got: %v\n", expectedGitHostBaseURL, gotBaseURL) } // Client for gitlab.com - client = newClient("gitlab", nil) + client = newClient("gitlab", &defaultGitlabConfig) client = client.(*gitlab.Client) // Client for custom gitlab installation - client = newClient("gitlab", customGitHost) + client = newClient("gitlab", &customGithostConfig) gotBaseURL = client.(*gitlab.Client).BaseURL() if gotBaseURL.String() != expectedGitHostBaseURL.String() { t.Errorf("Expected BaseURL to be: %v, Got: %v\n", expectedGitHostBaseURL, gotBaseURL) } // Client for bitbucket.com - client = newClient("bitbucket", nil) + client = newClient("bitbucket", &defaultBitbucketConfig) client = client.(*bitbucket.Client) // Not yet supported diff --git a/user_data.go b/user_data.go index b85ffaf..be0f942 100644 --- a/user_data.go +++ b/user_data.go @@ -8,7 +8,6 @@ import ( "io/ioutil" "log" "net/http" - "net/url" "os" "path" "strings" @@ -232,31 +231,6 @@ type GithubUserMigrationDeleteResult struct { GhResponseBody string `json:"mesage"` } -// DeleteGithubUserMigration deletes an existing migration -func DeleteGithubUserMigration(id *int64) GithubUserMigrationDeleteResult { - u, err := url.Parse("https://github.com") - if err != nil { - panic(err) - } - - client := newClient("github", u) - ctx := context.Background() - response, err := client.(*github.Client).Migrations.DeleteUserMigration(ctx, *id) - - result := GithubUserMigrationDeleteResult{} - result.GhStatusCode = response.StatusCode - if err != nil { - result.GhResponseBody = err.Error() - return result - } - data, err := ioutil.ReadAll(response.Body) - if err != nil { - panic(err) - } - result.GhResponseBody = string(data) - return result -} - func getGithubUserOwnedOrgs(ctx context.Context, client interface{}) ([]*github.Organization, error) { var ownedOrgs []*github.Organization From e788ba5dd94a7a23f526c9dcc00dd9d3ebf349c7 Mon Sep 17 00:00:00 2001 From: Amit Saha Date: Sun, 15 Oct 2023 00:03:17 +1100 Subject: [PATCH 5/8] refactor newClient --- client.go | 4 ++-- client_test.go | 59 ++++++++++++++++++++++++++++++-------------------- main.go | 2 +- 3 files changed, 39 insertions(+), 26 deletions(-) diff --git a/client.go b/client.go index f1a06fd..76778ff 100644 --- a/client.go +++ b/client.go @@ -71,7 +71,7 @@ func getToken(service string) (string, error) { return string(i.Data), nil } -func newClient(service string, c *appConfig) interface{} { +func newClient(c *appConfig) interface{} { var err error var gitHostURLParsed *url.URL @@ -82,7 +82,7 @@ func newClient(service string, c *appConfig) interface{} { } } - switch service { + switch c.service { case "github": githubToken := os.Getenv("GITHUB_TOKEN") if githubToken == "" { diff --git a/client_test.go b/client_test.go index 57cd656..53481ef 100644 --- a/client_test.go +++ b/client_test.go @@ -13,52 +13,65 @@ func TestNewClient(t *testing.T) { setupRepositoryTests() defer teardownRepositoryTests() - var defaultGithubConfig appConfig - defaultGithubConfig.gitHostURL = "https://github.com" - - var defaultGitlabConfig appConfig - defaultGitlabConfig.gitHostURL = "https://gitlab.com" - - var defaultBitbucketConfig appConfig - defaultBitbucketConfig.gitHostURL = "https://bitbucket.org" + defaultGithubConfig := appConfig{ + service: "github", + gitHostURL: "https://github.com", + } + customGithubConfig := appConfig{ + service: "github", + gitHostURL: "https://git.mycompany.com", + } - var customGithostConfig appConfig - customGitHost := "https://git.mycompany.com" - customGithostConfig.gitHostURL = customGitHost + defaultGitlabConfig := appConfig{ + service: "gitlab", + gitHostURL: "https://gitlab.com", + } + customGitlabConfig := appConfig{ + service: "gitlab", + gitHostURL: "https://git.mycompany.com", + } - // http://stackoverflow.com/questions/23051339/how-to-avoid-end-of-url-slash-being-removed-when-resolvereference-in-go - api, _ := url.Parse("api/v4/") - customGitHostParsed, _ := url.Parse(customGitHost) - expectedGitHostBaseURL := customGitHostParsed.ResolveReference(api) + defaultBitbucketConfig := appConfig{ + service: "bitbucket", + gitHostURL: "https://bitbucket.org", + } // Client for github.com - client := newClient("github", &defaultGithubConfig) + client := newClient(&defaultGithubConfig) client = client.(*github.Client) // Client for Enterprise Github - client = newClient("github", &customGithostConfig) + client = newClient(&customGithubConfig) gotBaseURL := client.(*github.Client).BaseURL - if gotBaseURL.String() != customGitHostParsed.String() { - t.Errorf("Expected BaseURL to be: %v, Got: %v\n", expectedGitHostBaseURL, gotBaseURL) + if gotBaseURL.String() != customGithubConfig.gitHostURL { + t.Errorf("Expected BaseURL to be: %v, Got: %v\n", customGithubConfig.gitHostURL, gotBaseURL) } // Client for gitlab.com - client = newClient("gitlab", &defaultGitlabConfig) + client = newClient(&defaultGitlabConfig) client = client.(*gitlab.Client) + // http://stackoverflow.com/questions/23051339/how-to-avoid-end-of-url-slash-being-removed-when-resolvereference-in-go + api, _ := url.Parse("api/v4/") + customGitHostParsed, _ := url.Parse(customGitlabConfig.gitHostURL) + expectedGitHostBaseURL := customGitHostParsed.ResolveReference(api) + // Client for custom gitlab installation - client = newClient("gitlab", &customGithostConfig) + client = newClient(&customGitlabConfig) gotBaseURL = client.(*gitlab.Client).BaseURL() if gotBaseURL.String() != expectedGitHostBaseURL.String() { t.Errorf("Expected BaseURL to be: %v, Got: %v\n", expectedGitHostBaseURL, gotBaseURL) } // Client for bitbucket.com - client = newClient("bitbucket", &defaultBitbucketConfig) + client = newClient(&defaultBitbucketConfig) client = client.(*bitbucket.Client) // Not yet supported - client = newClient("notyetsupported", nil) + unsupportedServiceConfig := appConfig{ + service: "notyetsupported", + } + client = newClient(&unsupportedServiceConfig) if client != nil { t.Errorf("Expected nil") } diff --git a/main.go b/main.go index d8e268a..c79afb2 100644 --- a/main.go +++ b/main.go @@ -34,7 +34,7 @@ func main() { log.Fatal(err) } - client := newClient(c.service, c) + client := newClient(c) var executionErr error // TODO implement validation of options so that we don't From 4f8976c5bb1f49462863a86f9bb6b0f0be24bb09 Mon Sep 17 00:00:00 2001 From: Amit Saha Date: Sun, 15 Oct 2023 00:39:29 +1100 Subject: [PATCH 6/8] More client code refactor --- client.go | 51 ++++++++++++++++++++++++++------------------------ client_test.go | 40 +++++++++++++++++++++++++++++---------- main.go | 6 +++++- 3 files changed, 62 insertions(+), 35 deletions(-) diff --git a/client.go b/client.go index 76778ff..2c0b83b 100644 --- a/client.go +++ b/client.go @@ -2,8 +2,8 @@ package main import ( "context" + "errors" "fmt" - "log" "net/http" "net/url" "os" @@ -21,14 +21,14 @@ import ( var keyringServiceName = "gitbackup-cli" var gitbackupClientID = "7b56a77c7dfba0800524" -func startOAuthFlow() string { +func startOAuthFlow() (*string, error) { clientID := gitbackupClientID scopes := []string{"repo", "user", "admin:org"} httpClient := http.DefaultClient code, err := device.RequestCode(httpClient, "https://github.com/login/device/code", clientID, scopes) if err != nil { - panic(err) + return nil, err } fmt.Printf("Copy code: %s\n", code.UserCode) @@ -36,13 +36,13 @@ func startOAuthFlow() string { accessToken, err := device.PollToken(httpClient, "https://github.com/login/oauth/access_token", clientID, code) if err != nil { - panic(err) + return nil, err } - return accessToken.Token + return &accessToken.Token, nil } -func saveToken(service string, token string) error { +func saveToken(service string, token *string) error { ring, err := keyring.Open(keyring.Config{ ServiceName: keyringServiceName, }) @@ -52,7 +52,7 @@ func saveToken(service string, token string) error { err = ring.Set(keyring.Item{ Key: service + "_TOKEN", - Data: []byte(token), + Data: []byte(*token), }) return err } @@ -71,14 +71,14 @@ func getToken(service string) (string, error) { return string(i.Data), nil } -func newClient(c *appConfig) interface{} { +func newClient(c *appConfig) (interface{}, error) { var err error var gitHostURLParsed *url.URL if c != nil && len(c.gitHostURL) != 0 { gitHostURLParsed, err = url.Parse(c.gitHostURL) if err != nil { - log.Fatal(err) + return nil, err } } @@ -88,14 +88,17 @@ func newClient(c *appConfig) interface{} { if githubToken == "" { githubToken, err = getToken("GITHUB") if err != nil { - githubToken = startOAuthFlow() - } - if githubToken == "" { - log.Fatal("GitHub token not available") - } - err := saveToken("GITHUB", githubToken) - if err != nil { - log.Fatal("Error saving token") + githubToken, err := startOAuthFlow() + if err != nil { + return nil, err + } + if githubToken == nil { + return nil, fmt.Errorf("GitHub token not available") + } + err = saveToken("GITHUB", githubToken) + if err != nil { + return nil, fmt.Errorf("Error saving token") + } } } gitHostToken = githubToken @@ -107,7 +110,7 @@ func newClient(c *appConfig) interface{} { if gitHostURLParsed != nil { client.BaseURL = gitHostURLParsed } - return client + return client, nil case "gitlab": @@ -117,7 +120,7 @@ func newClient(c *appConfig) interface{} { } gitlabToken := os.Getenv("GITLAB_TOKEN") if gitlabToken == "" { - log.Fatal("GITLAB_TOKEN environment variable not set") + return nil, fmt.Errorf("GITLAB_TOKEN environment variable not set") } gitHostToken = gitlabToken @@ -127,23 +130,23 @@ func newClient(c *appConfig) interface{} { } client, err := gitlab.NewClient(gitlabToken, baseUrlOption) if err != nil { - log.Fatalf("Error creating gitlab client: %v", err) + return nil, fmt.Errorf("Error creating gitlab client: %v", err) } - return client + return client, nil case "bitbucket": bitbucketUsername := os.Getenv("BITBUCKET_USERNAME") bitbucketPassword := os.Getenv("BITBUCKET_PASSWORD") if bitbucketUsername == "" || bitbucketPassword == "" { - log.Fatal("BITBUCKET_USERNAME and BITBUCKET_PASSWORD environment variables must be set") + return nil, fmt.Errorf("BITBUCKET_USERNAME and BITBUCKET_PASSWORD environment variables must be set") } gitHostToken = bitbucketPassword client := bitbucket.NewBasicAuth(bitbucketUsername, bitbucketPassword) if gitHostURLParsed != nil { client.SetApiBaseURL(gitHostURLParsed.String()) } - return client + return client, nil default: - return nil + return nil, errors.New("invalid service") } } diff --git a/client_test.go b/client_test.go index 53481ef..78b1385 100644 --- a/client_test.go +++ b/client_test.go @@ -4,6 +4,8 @@ import ( "net/url" "testing" + "strings" + "github.com/google/go-github/v34/github" "github.com/ktrysmt/go-bitbucket" gitlab "github.com/xanzy/go-gitlab" @@ -37,18 +39,29 @@ func TestNewClient(t *testing.T) { } // Client for github.com - client := newClient(&defaultGithubConfig) + client, err := newClient(&defaultGithubConfig) + if err != nil { + t.Fatal(err) + } client = client.(*github.Client) // Client for Enterprise Github - client = newClient(&customGithubConfig) + client, err = newClient(&customGithubConfig) + if err != nil { + t.Fatal(err) + } + gotBaseURL := client.(*github.Client).BaseURL if gotBaseURL.String() != customGithubConfig.gitHostURL { t.Errorf("Expected BaseURL to be: %v, Got: %v\n", customGithubConfig.gitHostURL, gotBaseURL) } // Client for gitlab.com - client = newClient(&defaultGitlabConfig) + client, err = newClient(&defaultGitlabConfig) + if err != nil { + t.Fatal(err) + } + client = client.(*gitlab.Client) // http://stackoverflow.com/questions/23051339/how-to-avoid-end-of-url-slash-being-removed-when-resolvereference-in-go @@ -57,23 +70,30 @@ func TestNewClient(t *testing.T) { expectedGitHostBaseURL := customGitHostParsed.ResolveReference(api) // Client for custom gitlab installation - client = newClient(&customGitlabConfig) + client, err = newClient(&customGitlabConfig) + if err != nil { + t.Fatal(err) + } + gotBaseURL = client.(*gitlab.Client).BaseURL() if gotBaseURL.String() != expectedGitHostBaseURL.String() { t.Errorf("Expected BaseURL to be: %v, Got: %v\n", expectedGitHostBaseURL, gotBaseURL) } - // Client for bitbucket.com - client = newClient(&defaultBitbucketConfig) + // Client for bitbucket.org + client, err = newClient(&defaultBitbucketConfig) + if err != nil { + t.Fatal(err) + } + client = client.(*bitbucket.Client) // Not yet supported unsupportedServiceConfig := appConfig{ service: "notyetsupported", } - client = newClient(&unsupportedServiceConfig) - if client != nil { - t.Errorf("Expected nil") + client, err = newClient(&unsupportedServiceConfig) + if !strings.Contains(err.Error(), "invalid service") { + t.Fatal(err) } - } diff --git a/main.go b/main.go index c79afb2..52990f6 100644 --- a/main.go +++ b/main.go @@ -34,7 +34,11 @@ func main() { log.Fatal(err) } - client := newClient(c) + client, err := newClient(c) + if err != nil { + log.Fatal(err) + } + var executionErr error // TODO implement validation of options so that we don't From 5161af676bbb7b37ed543679847ee6f9a08529f1 Mon Sep 17 00:00:00 2001 From: Amit Saha Date: Sun, 15 Oct 2023 00:47:26 +1100 Subject: [PATCH 7/8] client refactor --- client.go | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/client.go b/client.go index 2c0b83b..4a117d4 100644 --- a/client.go +++ b/client.go @@ -42,7 +42,7 @@ func startOAuthFlow() (*string, error) { return &accessToken.Token, nil } -func saveToken(service string, token *string) error { +func saveTokenToKeyring(service string, token *string) error { ring, err := keyring.Open(keyring.Config{ ServiceName: keyringServiceName, }) @@ -57,7 +57,7 @@ func saveToken(service string, token *string) error { return err } -func getToken(service string) (string, error) { +func getTokenFromKeyring(service string) (string, error) { ring, err := keyring.Open(keyring.Config{ ServiceName: keyringServiceName, }) @@ -86,16 +86,13 @@ func newClient(c *appConfig) (interface{}, error) { case "github": githubToken := os.Getenv("GITHUB_TOKEN") if githubToken == "" { - githubToken, err = getToken("GITHUB") + githubToken, err = getTokenFromKeyring("GITHUB") if err != nil { githubToken, err := startOAuthFlow() - if err != nil { - return nil, err - } - if githubToken == nil { - return nil, fmt.Errorf("GitHub token not available") + if githubToken == nil || err != nil { + return nil, fmt.Errorf("GitHub token not available %w", err) } - err = saveToken("GITHUB", githubToken) + err = saveTokenToKeyring("GITHUB", githubToken) if err != nil { return nil, fmt.Errorf("Error saving token") } From 9fe0994314de8671abae374100e074278576b9a9 Mon Sep 17 00:00:00 2001 From: Amit Saha Date: Sun, 15 Oct 2023 01:01:24 +1100 Subject: [PATCH 8/8] refactor backup --- backup.go | 86 ++++++++++++++++++++++++++++--------------------------- 1 file changed, 44 insertions(+), 42 deletions(-) diff --git a/backup.go b/backup.go index e63be6f..4d9e328 100644 --- a/backup.go +++ b/backup.go @@ -17,59 +17,61 @@ var appFS = afero.NewOsFs() var gitCommand = "git" var gethomeDir = homedir.Dir +func updateExistingClone(repoDir string, bare bool, repo *Repository) ([]byte, error) { + log.Printf("%s exists, updating. \n", repo.Name) + var cmd *exec.Cmd + if bare { + cmd = execCommand(gitCommand, "-C", repoDir, "remote", "update", "--prune") + } else { + cmd = execCommand(gitCommand, "-C", repoDir, "pull") + } + return cmd.CombinedOutput() +} + +func newClone(repoDir string, bare bool, repo *Repository, useHTTPSClone *bool) ([]byte, error) { + + log.Printf("Cloning %s\n", repo.Name) + log.Printf("%#v\n", repo) + + if repo.Private && ignorePrivate != nil && *ignorePrivate { + log.Printf("Skipping %s as it is a private repo.\n", repo.Name) + return nil, nil + } + + if useHTTPSClone != nil && *useHTTPSClone { + // Add username and token to the clone URL + // https://gitlab.com/amitsaha/testproject1 => https://amitsaha:token@gitlab.com/amitsaha/testproject1 + u, err := url.Parse(repo.CloneURL) + if err != nil { + log.Fatalf("Invalid clone URL: %v\n", err) + } + repo.CloneURL = u.Scheme + "://" + gitHostUsername + ":" + gitHostToken + "@" + u.Host + u.Path + } + + var cmd *exec.Cmd + if bare { + cmd = execCommand(gitCommand, "clone", "--mirror", repo.CloneURL, repoDir) + } else { + cmd = execCommand(gitCommand, "clone", repo.CloneURL, repoDir) + } + return cmd.CombinedOutput() +} + // Check if we have a copy of the repo already, if // we do, we update the repo, else we do a fresh clone func backUp(backupDir string, repo *Repository, bare bool, wg *sync.WaitGroup) ([]byte, error) { defer wg.Done() - var dirName string + dirName := repo.Name if bare { - dirName = repo.Name + ".git" - } else { - dirName = repo.Name + dirName += ".git" } repoDir := path.Join(backupDir, repo.Namespace, dirName) - _, err := appFS.Stat(repoDir) - - var stdoutStderr []byte if err == nil { - log.Printf("%s exists, updating. \n", repo.Name) - var cmd *exec.Cmd - if bare { - cmd = execCommand(gitCommand, "-C", repoDir, "remote", "update", "--prune") - } else { - cmd = execCommand(gitCommand, "-C", repoDir, "pull") - } - stdoutStderr, err = cmd.CombinedOutput() - } else { - log.Printf("Cloning %s\n", repo.Name) - log.Printf("%#v\n", repo) - - if repo.Private && ignorePrivate != nil && *ignorePrivate { - log.Printf("Skipping %s as it is a private repo.\n", repo.Name) - return stdoutStderr, nil - } - - if useHTTPSClone != nil && *useHTTPSClone { - // Add username and token to the clone URL - // https://gitlab.com/amitsaha/testproject1 => https://amitsaha:token@gitlab.com/amitsaha/testproject1 - u, err := url.Parse(repo.CloneURL) - if err != nil { - log.Fatalf("Invalid clone URL: %v\n", err) - } - repo.CloneURL = u.Scheme + "://" + gitHostUsername + ":" + gitHostToken + "@" + u.Host + u.Path - } - - var cmd *exec.Cmd - if bare { - cmd = execCommand(gitCommand, "clone", "--mirror", repo.CloneURL, repoDir) - } else { - cmd = execCommand(gitCommand, "clone", repo.CloneURL, repoDir) - } - stdoutStderr, err = cmd.CombinedOutput() + return updateExistingClone(repoDir, bare, repo) } - return stdoutStderr, err + return newClone(repoDir, bare, repo, useHTTPSClone) } func setupBackupDir(backupDir, service, githostURL *string) string {