GitHub: Add users/orgs whitelist flag (#88)

Author: jcm300

Diff for: README.md

@@ -117,6 +117,8 @@ Usage of ./gitbackup:
         Number of retries to attempt for creating GitHub user migration (default 5)
   -github.listUserMigrations
         List available user migrations
+  -github.namespaceWhitelist string
+        Organizations/Users from where we should clone (separate each value by a comma: 'user1,org2')
   -github.repoType string
         Repo types to backup (all, owner, member, starred) (default "all")
   -github.waitForUserMigration
@@ -161,6 +163,12 @@ Separately, to backup GitHub repositories you have starred:
 $ GITHUB_TOKEN=secret$token gitbackup -service github -github.repoType starred
 ```
 
+Additionally, to backup only the GitHub repositories under 'user1' and 'org3':
+
+```lang=bash
+$ GITHUB_TOKEN=secret$token gitbackup -service github -github.namespaceWhitelist "user1,org3"
+```
+
 #### Backing up your GitLab repositories
 
 To backup all projects you either own or are a member of which have their [visibility](https://docs.gitlab.com/ce/api/projects.html#project-visibility-level) set to

Diff for: helpers.go

@@ -52,3 +52,12 @@ func validGitlabProjectMembership(membership string) bool {
 	}
 	return false
 }
+
+func contains(list []string, x string) bool {
+	for _, item := range list {
+		if item == x {
+			return true
+		}
+	}
+	return false
+}

Diff for: main.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"log"
 	"sync"
+	"strings"
 	"time"
 
 	"github.com/google/go-github/v34/github"
@@ -46,6 +47,7 @@ func main() {
 
 	// GitHub specific flags
 	githubRepoType := flag.String("github.repoType", "all", "Repo types to backup (all, owner, member, starred)")
+	githubNamespaceWhitelistString := flag.String("github.namespaceWhitelist", "", "Organizations/Users from where we should clone (separate each value by a comma: 'user1,org2')")
 
 	githubCreateUserMigration := flag.Bool("github.createUserMigration", false, "Download user data")
 	githubCreateUserMigrationRetry := flag.Bool("github.createUserMigrationRetry", true, "Retry creating the GitHub user migration if we get an error")
@@ -59,6 +61,13 @@ func main() {
 
 	flag.Parse()
 
+	// Split namespaces
+	githubNamespaceWhitelist := []string{}
+
+	if len(*githubNamespaceWhitelistString) > 0 {
+		githubNamespaceWhitelist = strings.Split(*githubNamespaceWhitelistString, ",")
+	}
+
 	if _, ok := knownServices[*service]; !ok {
 		log.Fatal("Please specify the git service type: github, gitlab, bitbucket")
 	}
@@ -96,7 +105,7 @@ func main() {
 
 	} else if *githubCreateUserMigration {
 
-		repos, err := getRepositories(client, *service, *githubRepoType, *gitlabProjectVisibility, *gitlabProjectMembershipType, *ignoreFork)
+		repos, err := getRepositories(client, *service, *githubRepoType, githubNamespaceWhitelist, *gitlabProjectVisibility, *gitlabProjectMembershipType, *ignoreFork)
 		if err != nil {
 			log.Fatalf("Error getting list of repositories: %v", err)
 		}
@@ -147,7 +156,7 @@ func main() {
 			log.Fatal("Your Git host's username is needed for backing up private repositories via HTTPS")
 		}
 		repos, err := getRepositories(
-			client, *service, *githubRepoType,
+			client, *service, *githubRepoType, githubNamespaceWhitelist,
 			*gitlabProjectVisibility, *gitlabProjectMembershipType, *ignoreFork,
 		)
 		if err != nil {

Diff for: repositories.go

@@ -38,7 +38,7 @@ type Repository struct {
 }
 
 func getRepositories(client interface{},
-	service string, githubRepoType string,
+	service string, githubRepoType string, githubNamespaceWhitelist []string,
 	gitlabProjectVisibility string, gitlabProjectMembershipType string,
 	ignoreFork bool,
 ) ([]*Repository, error) {
@@ -80,6 +80,8 @@ func getRepositories(client interface{},
 			}
 		} else {
 			options := github.RepositoryListOptions{Type: githubRepoType}
+			githubNamespaceWhitelistLength := len(githubNamespaceWhitelist)
+
 			for {
 				repos, resp, err := client.(*github.Client).Repositories.List(ctx, "", &options)
 				if err == nil {
@@ -88,6 +90,11 @@ func getRepositories(client interface{},
 							continue
 						}
 						namespace := strings.Split(*repo.FullName, "/")[0]
+
+						if githubNamespaceWhitelistLength > 0 && !contains(githubNamespaceWhitelist, namespace) {
+							continue
+						}
+
 						if useHTTPSClone != nil && *useHTTPSClone {
 							cloneURL = *repo.CloneURL
 						} else {

Diff for: repositories_test.go

@@ -67,7 +67,7 @@ func TestGetPublicGitHubRepositories(t *testing.T) {
 		fmt.Fprint(w, `[{"full_name": "test/r1", "id":1, "ssh_url": "https://github.com/u/r1", "name": "r1", "private": false, "fork": false}]`)
 	})
 
-	repos, err := getRepositories(GitHubClient, "github", "all", "", "", false)
+	repos, err := getRepositories(GitHubClient, "github", "all", []string{}, "", "", false)
 	if err != nil {
 		t.Fatalf("%v", err)
 	}
@@ -86,7 +86,7 @@ func TestGetPrivateGitHubRepositories(t *testing.T) {
 		fmt.Fprint(w, `[{"full_name": "test/r1", "id":1, "ssh_url": "https://github.com/u/r1", "name": "r1", "private": true, "fork": false}]`)
 	})
 
-	repos, err := getRepositories(GitHubClient, "github", "all", "", "", false)
+	repos, err := getRepositories(GitHubClient, "github", "all", []string{}, "", "", false)
 	if err != nil {
 		t.Fatalf("%v", err)
 	}
@@ -105,7 +105,7 @@ func TestGetStarredGitHubRepositories(t *testing.T) {
 		fmt.Fprint(w, `[{"repo":{"full_name": "test/r1", "id":1, "ssh_url": "https://github.com/u/r1", "name": "r1", "private": true, "fork": false}}]`)
 	})
 
-	repos, err := getRepositories(GitHubClient, "github", "starred", "", "", false)
+	repos, err := getRepositories(GitHubClient, "github", "starred", []string{}, "", "", false)
 	if err != nil {
 		t.Fatalf("%v", err)
 	}
@@ -116,6 +116,31 @@ func TestGetStarredGitHubRepositories(t *testing.T) {
 	}
 }
 
+func TestGetWhitelistGitHubRepositories(t *testing.T) {
+	setupRepositoryTests()
+	defer teardownRepositoryTests()
+
+	mux.HandleFunc("/user/repos", func(w http.ResponseWriter, r *http.Request) {
+		fmt.Fprint(w, `[
+			{"full_name": "test/r1", "id":1, "ssh_url": "https://github.com/u/r1", "name": "r1", "private": false, "fork": false},
+			{"full_name": "user1/r1", "id":1, "ssh_url": "https://github.com/u/r1", "name": "r1", "private": false, "fork": false},
+			{"full_name": "org1/r1", "id":1, "ssh_url": "https://github.com/u/r1", "name": "r1", "private": false, "fork": false}
+		]`)
+	})
+
+	repos, err := getRepositories(GitHubClient, "github", "all", []string{"test", "user1"}, "", "", false)
+	if err != nil {
+		t.Fatalf("%v", err)
+	}
+	var expected []*Repository
+	expected = append(expected, &Repository{Namespace: "test", CloneURL: "https://github.com/u/r1", Name: "r1", Private: false})
+	expected = append(expected, &Repository{Namespace: "user1", CloneURL: "https://github.com/u/r1", Name: "r1", Private: false})
+
+	if !reflect.DeepEqual(repos, expected) {
+		t.Errorf("Expected %+v, Got %+v", expected, repos)
+	}
+}
+
 func TestGetGitLabRepositories(t *testing.T) {
 	setupRepositoryTests()
 	defer teardownRepositoryTests()
@@ -124,7 +149,7 @@ func TestGetGitLabRepositories(t *testing.T) {
 		fmt.Fprint(w, `[{"path_with_namespace": "test/r1", "id":1, "ssh_url_to_repo": "https://gitlab.com/u/r1", "name": "r1"}]`)
 	})
 
-	repos, err := getRepositories(GitLabClient, "gitlab", "internal", "", "", false)
+	repos, err := getRepositories(GitLabClient, "gitlab", "internal", []string{}, "", "", false)
 	if err != nil {
 		t.Fatalf("%v", err)
 	}
@@ -149,7 +174,7 @@ func TestGetGitLabPrivateRepositories(t *testing.T) {
 	})
 
 	repos, err := getRepositories(GitLabClient, "gitlab",
-		"private", "", "", false)
+		"private", []string{}, "", "", false)
 	if err != nil {
 		t.Fatalf("%v", err)
 	}
@@ -176,7 +201,7 @@ func TestGetStarredGitLabRepositories(t *testing.T) {
 		fmt.Fprintf(w, `[]`)
 	})
 
-	repos, err := getRepositories(GitLabClient, "gitlab", "", "", "starred", false)
+	repos, err := getRepositories(GitLabClient, "gitlab", "", []string{}, "", "starred", false)
 	if err != nil {
 		t.Fatalf("%v", err)
 	}
@@ -205,7 +230,7 @@ func TestGetBitbucketRepositories(t *testing.T) {
 		fmt.Fprint(w, `{"pagelen": 10, "page": 1, "size": 1, "values": [{"full_name":"abc/def", "slug":"def", "is_private":true, "links":{"clone":[{"name":"https", "href":"https://[email protected]/abc/def.git"}, {"name":"ssh", "href":"[email protected]:abc/def.git"}]}}]}`)
 	})
 
-	repos, err := getRepositories(BitbucketClient, "bitbucket", "", "", "", false)
+	repos, err := getRepositories(BitbucketClient, "bitbucket", "", []string{}, "", "", false)
 	if err != nil {
 		t.Fatalf("%v", err)
 	}