GitLab: Allow cloning starred repositories (#70)

Add support for cloning only starred repositories in GitLab

Author: amitsaha

README.md

@@ -104,31 +104,35 @@ Typing ``-help`` will display the command line options that `gitbackup` recogniz
 $ gitbackup -help
 Usage of ./gitbackup:
   -backupdir string
-    	Backup directory
+        Backup directory
   -bare
-    	Clone bare repositories
+        Clone bare repositories
   -githost.url string
-    	DNS of the custom Git host
+        DNS of the custom Git host
   -github.createUserMigration
-    	Download user data
+        Download user data
+  -github.createUserMigrationRetry
+        Retry creating the GitHub user migration if we get an error (default true)
+  -github.createUserMigrationRetryMax int
+        Number of retries to attempt for creating GitHub user migration (default 5)
   -github.listUserMigrations
-    	List available user migrations
+        List available user migrations
   -github.repoType string
-    	Repo types to backup (all, owner, member, starred) (default "all")
+        Repo types to backup (all, owner, member, starred) (default "all")
   -github.waitForUserMigration
-    	Wait for migration to complete (default true)
+        Wait for migration to complete (default true)
   -gitlab.projectMembershipType string
-    	Project type to clone (all, owner, member) (default "all")
+        Project type to clone (all, owner, member, starred) (default "all")
   -gitlab.projectVisibility string
-    	Visibility level of Projects to clone (internal, public, private) (default "internal")
+        Visibility level of Projects to clone (internal, public, private) (default "internal")
   -ignore-fork
-    	Ignore repositories which are forks
+        Ignore repositories which are forks
   -ignore-private
-    	Ignore private repositories/projects
+        Ignore private repositories/projects
   -service string
-    	Git Hosted Service Name (github/gitlab/bitbucket)
+        Git Hosted Service Name (github/gitlab/bitbucket)
   -use-https-clone
-    	Use HTTPS for cloning instead of SSH
+        Use HTTPS for cloning instead of SSH
 ```
 
 #### Backing up your GitHub repositories
@@ -196,6 +200,14 @@ $ GITLAB_TOKEN=secret$token gitbackup \
     -gitlab.projectMembershipType member
 ```
 
+To backup GitLub repositories you have starred:
+
+```lang=bash
+$ GITLAB_TOKEN=secret$token gitbackup -service gitlab \
+  -gitlab.projectMembershipType starred \
+  -gitlab.projectVisibility public
+```
+
 #### GitHub Enterprise or custom GitLab installation
 
 To specify a custom GitHub enterprise or GitLab location, specify the ``service`` as well as the

backup.go

@@ -63,7 +63,6 @@ func backUp(backupDir string, repo *Repository, bare bool, wg *sync.WaitGroup) (
 		}
 		stdoutStderr, err = cmd.CombinedOutput()
 	}
-
 	return stdoutStderr, err
 }
 

helpers.go

@@ -44,7 +44,7 @@ func getUsername(client interface{}, service string) string {
 }
 
 func validGitlabProjectMembership(membership string) bool {
-	validMemberships := []string{"all", "owner", "member"}
+	validMemberships := []string{"all", "owner", "member", "starred"}
 	for _, m := range validMemberships {
 		if membership == m {
 			return true

main.go

@@ -58,16 +58,16 @@ func main() {
 	githubWaitForMigrationComplete := flag.Bool("github.waitForUserMigration", true, "Wait for migration to complete")
 
 	// Gitlab specific flags
-	gitlabRepoVisibility := flag.String("gitlab.projectVisibility", "internal", "Visibility level of Projects to clone (internal, public, private)")
-	gitlabProjectMembership := flag.String("gitlab.projectMembershipType", "all", "Project type to clone (all, owner, member)")
+	gitlabProjectVisibility := flag.String("gitlab.projectVisibility", "internal", "Visibility level of Projects to clone (internal, public, private)")
+	gitlabProjectMembershipType := flag.String("gitlab.projectMembershipType", "all", "Project type to clone (all, owner, member, starred)")
 
 	flag.Parse()
 
 	if _, ok := knownServices[*service]; !ok {
 		log.Fatal("Please specify the git service type: github, gitlab, bitbucket")
 	}
 
-	if !validGitlabProjectMembership(*gitlabProjectMembership) {
+	if !validGitlabProjectMembership(*gitlabProjectMembershipType) {
 		log.Fatal("Please specify a valid gitlab project membership - all/owner/member")
 	}
 
@@ -111,7 +111,7 @@ func main() {
 
 	} else if *githubCreateUserMigration {
 
-		repos, err := getRepositories(client, *service, *githubRepoType, *gitlabRepoVisibility, *gitlabProjectMembership, *ignoreFork)
+		repos, err := getRepositories(client, *service, *githubRepoType, *gitlabProjectVisibility, *gitlabProjectMembershipType, *ignoreFork)
 		if err != nil {
 			log.Fatalf("Error getting list of repositories: %v", err)
 		}
@@ -161,7 +161,10 @@ func main() {
 		if len(gitHostUsername) == 0 && !*ignorePrivate && *useHTTPSClone {
 			log.Fatal("Your Git host's username is needed for backing up private repositories via HTTPS")
 		}
-		repos, err := getRepositories(client, *service, *githubRepoType, *gitlabRepoVisibility, *gitlabProjectMembership, *ignoreFork)
+		repos, err := getRepositories(
+			client, *service, *githubRepoType,
+			*gitlabProjectVisibility, *gitlabProjectMembershipType, *ignoreFork,
+		)
 		if err != nil {
 			log.Fatal(err)
 		} else {

repositories.go

@@ -37,7 +37,11 @@ type Repository struct {
 	Private   bool
 }
 
-func getRepositories(client interface{}, service string, githubRepoType string, gitlabRepoVisibility string, gitlabProjectType string, ignoreFork bool) ([]*Repository, error) {
+func getRepositories(client interface{},
+	service string, githubRepoType string,
+	gitlabProjectVisibility string, gitlabProjectMembershipType string,
+	ignoreFork bool,
+) ([]*Repository, error) {
 
 	if client == nil {
 		log.Fatalf("Couldn't acquire a client to talk to %s", service)
@@ -104,25 +108,26 @@ func getRepositories(client interface{}, service string, githubRepoType string,
 
 	if service == "gitlab" {
 		var visibility gitlab.VisibilityValue
-		var options gitlab.ListProjectsOptions
-
-		var owned bool
-		var memberOf bool
-
-		if gitlabProjectType == "owner" {
-			owned = true
-		}
-		if gitlabProjectType == "member" {
-			memberOf = true
+		var boolTrue bool = true
+
+		gitlabListOptions := gitlab.ListProjectsOptions{}
+
+		switch gitlabProjectMembershipType {
+
+		case "owner":
+			gitlabListOptions.Owned = &boolTrue
+		case "member":
+			gitlabListOptions.Membership = &boolTrue
+		case "starred":
+			gitlabListOptions.Starred = &boolTrue
+		case "all":
+			gitlabListOptions.Owned = &boolTrue
+			gitlabListOptions.Membership = &boolTrue
+			gitlabListOptions.Starred = &boolTrue
 		}
 
-		if gitlabProjectType == "all" {
-			owned = true
-			memberOf = true
-		}
-
-		if gitlabRepoVisibility != "all" {
-			switch gitlabRepoVisibility {
+		if gitlabProjectVisibility != "all" {
+			switch gitlabProjectVisibility {
 			case "public":
 				visibility = gitlab.PublicVisibility
 			case "private":
@@ -132,30 +137,27 @@ func getRepositories(client interface{}, service string, githubRepoType string,
 			case "default":
 				visibility = gitlab.InternalVisibility
 			}
-			options = gitlab.ListProjectsOptions{Visibility: &visibility, Membership: &memberOf, Owned: &owned}
-		} else {
-			options = gitlab.ListProjectsOptions{Membership: &memberOf, Owned: &owned}
+			gitlabListOptions.Visibility = &visibility
 		}
 
 		for {
-			repos, resp, err := client.(*gitlab.Client).Projects.ListProjects(&options)
-			if err == nil {
-				for _, repo := range repos {
-					namespace := strings.Split(repo.PathWithNamespace, "/")[0]
-					if useHTTPSClone != nil && *useHTTPSClone {
-						cloneURL = repo.WebURL
-					} else {
-						cloneURL = repo.SSHURLToRepo
-					}
-					repositories = append(repositories, &Repository{CloneURL: cloneURL, Name: repo.Name, Namespace: namespace, Private: repo.Public})
-				}
-			} else {
+			repos, resp, err := client.(*gitlab.Client).Projects.ListProjects(&gitlabListOptions)
+			if err != nil {
 				return nil, err
 			}
+			for _, repo := range repos {
+				namespace := strings.Split(repo.PathWithNamespace, "/")[0]
+				if useHTTPSClone != nil && *useHTTPSClone {
+					cloneURL = repo.WebURL
+				} else {
+					cloneURL = repo.SSHURLToRepo
+				}
+				repositories = append(repositories, &Repository{CloneURL: cloneURL, Name: repo.Name, Namespace: namespace, Private: repo.Public})
+			}
 			if resp.NextPage == 0 {
 				break
 			}
-			options.ListOptions.Page = resp.NextPage
+			gitlabListOptions.ListOptions.Page = resp.NextPage
 		}
 	}
 
@@ -207,6 +209,5 @@ func getRepositories(client interface{}, service string, githubRepoType string,
 			}
 		}
 	}
-
 	return repositories, nil
 }

repositories_test.go

@@ -138,6 +138,36 @@ func TestGetGitLabRepositories(t *testing.T) {
 	}
 }
 
+func TestGetStarredGitLabRepositories(t *testing.T) {
+	setup()
+	defer teardown()
+
+	mux.HandleFunc("/api/v4/projects", func(w http.ResponseWriter, r *http.Request) {
+		log.Printf("%#v\n", r.URL.Query())
+		if len(r.URL.Query().Get("starred")) != 0 {
+			fmt.Fprint(w, `[{"path_with_namespace": "test/starred-repo-r1", "id":1, "ssh_url_to_repo": "https://gitlab.com/u/r1", "name": "starred-repo-r1"}]`)
+			return
+		}
+		fmt.Fprintf(w, `[]`)
+	})
+
+	repos, err := getRepositories(GitLabClient, "gitlab", "", "", "starred", false)
+	if err != nil {
+		t.Fatalf("%v", err)
+	}
+	var expected []*Repository
+	expected = append(expected, &Repository{Namespace: "test", CloneURL: "https://gitlab.com/u/r1", Name: "starred-repo-r1"})
+
+	if !reflect.DeepEqual(repos, expected) {
+		if len(repos) != len(expected) {
+			t.Fatalf("Expected: %#v, Got: %v", expected, repos)
+		}
+		for i := 0; i < len(expected); i++ {
+			t.Errorf("Expected %+v, Got %+v", expected[i], repos[i])
+		}
+	}
+}
+
 func TestGetBitbucketRepositories(t *testing.T) {
 	setup()
 	defer teardown()