diff --git a/README.md b/README.md index c08a4d9..f383b7a 100644 --- a/README.md +++ b/README.md @@ -9,8 +9,8 @@ GitLab (including custom GitLab installations). pull requests or other data associated with a git repository. This may or may not be in the future scope of this tool. -If you are following along my Linux Journal article, please obtain the version of the source tagged -with [lj-0.1](https://github.com/amitsaha/gitbackup/releases/tag/lj-0.1). +If you are following along my Linux Journal article (published in 2017), please obtain the version of the +source tagged with [lj-0.1](https://github.com/amitsaha/gitbackup/releases/tag/lj-0.1). ## Installling `gitbackup` @@ -23,6 +23,28 @@ and architecture and copy the binary somewhere in your ``$PATH``. It is recommen backing up GitHub repositories and [GitLab personal access token](https://gitlab.com/profile/personal_access_tokens) for GitLab. You can supply the token to ``gitbackup`` using ``GITHUB_TOKEN`` and ``GITLAB_TOKEN`` environment variables respectively. +### OAuth Scopes required + +#### GitHub + +- `repo`: Reading repositories, including private repositories +- `user - read:user`: Reading the authenticated user details. This is needed for retrieving username which is needed for retrieving private repositories. + +#### GitLab + +- `api`: Grants complete read/write access to the API, including all groups and projects. +For some reason, `read_user` and `read_repository` is not sufficient. + +### Security and credentials + +When you provide the tokens via environment variables, they remain accessible in your shell history +and via the processes' environment for the lifetime of the process. By default, SSH authentication +is used to clone your repositories. If `use-https-clone` is specified, private repositories +are cloned via `https` basic auth and the token provided will be stored in the repositories' +`.git/config`. + +### Examples + Typing ``-help`` will display the command line options that `gitbackup` recognizes: ``` @@ -38,8 +60,12 @@ Usage of ./bin/gitbackup: Project type to clone (all, owner, member) (default "all") -gitlab.projectVisibility string Visibility level of Projects to clone (internal, public, private) (default "internal") + -ignore-private + Ignore private repositories/projects -service string - Git Hosted Service Name (github/gitlab) + Git Hosted Service Name (github/gitlab) + -use-https-clone + Use HTTPS for cloning instead of SSH ``` ### Backing up your GitHub repositories @@ -123,7 +149,6 @@ Similarly, it will create a ``gitlab.com`` directory, if you are backing up repo If you have specified a Git Host URL, it will create a directory structure ``data/host-url/``. - ## Building If you have Golang 1.12.x+ installed, you can clone the repository and: diff --git a/backup.go b/backup.go index 58b00be..af4d7fb 100644 --- a/backup.go +++ b/backup.go @@ -30,8 +30,20 @@ func backUp(backupDir string, repo *Repository, wg *sync.WaitGroup) ([]byte, err cmd := execCommand(gitCommand, "-C", repoDir, "pull") stdoutStderr, err = cmd.CombinedOutput() } else { - log.Printf("Cloning %s \n", repo.Name) - cmd := execCommand(gitCommand, "clone", repo.GitURL, repoDir) + log.Printf("Cloning %s\n", repo.Name) + log.Printf("%#v\n", repo) + + if repo.Private && useHTTPSClone != nil && *useHTTPSClone && ignorePrivate != nil && !*ignorePrivate { + // Add username and token to the clone URL + // https://gitlab.com/amitsaha/testproject1 => https://amitsaha:token@gitlab.com/amitsaha/testproject1 + u, err := url.Parse(repo.CloneURL) + if err != nil { + log.Fatalf("Invalid clone URL: %v\n", err) + } + repo.CloneURL = u.Scheme + "://" + gitHostUsername + ":" + gitHostToken + "@" + u.Host + u.Path + } + + cmd := execCommand(gitCommand, "clone", repo.CloneURL, repoDir) stdoutStderr, err = cmd.CombinedOutput() } diff --git a/backup_test.go b/backup_test.go index fb7af51..e5917fc 100644 --- a/backup_test.go +++ b/backup_test.go @@ -2,12 +2,13 @@ package main import ( "fmt" - "github.com/spf13/afero" "os" "os/exec" "path" "sync" "testing" + + "github.com/spf13/afero" ) func fakePullCommand(command string, args ...string) (cmd *exec.Cmd) { @@ -28,7 +29,7 @@ func fakeCloneCommand(command string, args ...string) (cmd *exec.Cmd) { func TestBackup(t *testing.T) { var wg sync.WaitGroup - repo := Repository{Name: "testrepo", GitURL: "git://foo.com/foo"} + repo := Repository{Name: "testrepo", CloneURL: "git://foo.com/foo"} backupDir := "/tmp/backupdir" // Memory FS diff --git a/client.go b/client.go index 1997566..8462694 100644 --- a/client.go +++ b/client.go @@ -30,6 +30,7 @@ func newClient(service string, gitHostURL string) interface{} { if githubToken == "" { log.Fatal("GITHUB_TOKEN environment variable not set") } + gitHostToken = githubToken ts := oauth2.StaticTokenSource( &oauth2.Token{AccessToken: githubToken}, ) @@ -46,6 +47,7 @@ func newClient(service string, gitHostURL string) interface{} { if gitlabToken == "" { log.Fatal("GITLAB_TOKEN environment variable not set") } + gitHostToken = gitlabToken client := gitlab.NewClient(nil, gitlabToken) if gitHostURLParsed != nil { client.SetBaseURL(gitHostURLParsed.String()) diff --git a/helpers.go b/helpers.go new file mode 100644 index 0000000..a5a140e --- /dev/null +++ b/helpers.go @@ -0,0 +1,35 @@ +package main + +import ( + "context" + "log" + + "github.com/google/go-github/github" + gitlab "github.com/xanzy/go-gitlab" +) + +func getUsername(client interface{}, service string) string { + + if client == nil { + log.Fatalf("Couldn't acquire a client to talk to %s", service) + } + + if service == "github" { + ctx := context.Background() + user, _, err := client.(*github.Client).Users.Get(ctx, "") + if err != nil { + log.Fatal("Error retrieving username", err.Error()) + } + return *user.Name + } + + if service == "gitlab" { + user, _, err := client.(*gitlab.Client).Users.CurrentUser() + if err != nil { + log.Fatal("Error retrieving username", err.Error()) + } + return user.Username + } + + return "" +} diff --git a/main.go b/main.go index 9043ac4..cbb7b16 100644 --- a/main.go +++ b/main.go @@ -10,6 +10,11 @@ import ( // concurrent git clones var MaxConcurrentClones = 20 +var gitHostToken string +var useHTTPSClone *bool +var ignorePrivate *bool +var gitHostUsername string + func main() { // Used for waiting for all the goroutines to finish before exiting @@ -26,6 +31,8 @@ func main() { service := flag.String("service", "", "Git Hosted Service Name (github/gitlab)") githostURL := flag.String("githost.url", "", "DNS of the custom Git host") backupDir := flag.String("backupdir", "", "Backup directory") + ignorePrivate = flag.Bool("ignore-private", false, "Ignore private repositories/projects") + useHTTPSClone = flag.Bool("use-https-clone", false, "Use HTTPS for cloning instead of SSH") // GitHub specific flags githubRepoType := flag.String("github.repoType", "all", "Repo types to backup (all, owner, member)") @@ -42,6 +49,12 @@ func main() { *backupDir = setupBackupDir(*backupDir, *service, *githostURL) tokens := make(chan bool, MaxConcurrentClones) client := newClient(*service, *githostURL) + + gitHostUsername = getUsername(client, *service) + + if len(gitHostUsername) == 0 && !*ignorePrivate && *useHTTPSClone { + log.Fatal("Your Git host's username is needed for backing up private repositories via HTTPS") + } repos, err := getRepositories(client, *service, *githubRepoType, *gitlabRepoVisibility, *gitlabProjectMembership) if err != nil { log.Fatal(err) diff --git a/repositories.go b/repositories.go index 4ed5da2..b1c3ede 100644 --- a/repositories.go +++ b/repositories.go @@ -30,9 +30,10 @@ type Response struct { // Repository is a container for the details for a repository // we will backup type Repository struct { - GitURL string + CloneURL string Name string Namespace string + Private bool } func getRepositories(client interface{}, service string, githubRepoType string, gitlabRepoVisibility string, gitlabProjectType string) ([]*Repository, error) { @@ -42,6 +43,7 @@ func getRepositories(client interface{}, service string, githubRepoType string, } var repositories []*Repository + var cloneURL string if service == "github" { ctx := context.Background() @@ -51,7 +53,12 @@ func getRepositories(client interface{}, service string, githubRepoType string, if err == nil { for _, repo := range repos { namespace := strings.Split(*repo.FullName, "/")[0] - repositories = append(repositories, &Repository{GitURL: *repo.GitURL, Name: *repo.Name, Namespace: namespace}) + if useHTTPSClone != nil && *useHTTPSClone { + cloneURL = *repo.CloneURL + } else { + cloneURL = *repo.SSHURL + } + repositories = append(repositories, &Repository{CloneURL: cloneURL, Name: *repo.Name, Namespace: namespace, Private: *repo.Private}) } } else { return nil, err @@ -103,7 +110,12 @@ func getRepositories(client interface{}, service string, githubRepoType string, if err == nil { for _, repo := range repos { namespace := strings.Split(repo.PathWithNamespace, "/")[0] - repositories = append(repositories, &Repository{GitURL: repo.SSHURLToRepo, Name: repo.Name, Namespace: namespace}) + if useHTTPSClone != nil && *useHTTPSClone { + cloneURL = repo.WebURL + } else { + cloneURL = repo.SSHURLToRepo + } + repositories = append(repositories, &Repository{CloneURL: cloneURL, Name: repo.Name, Namespace: namespace, Private: repo.Public}) } } else { return nil, err diff --git a/repositories_test.go b/repositories_test.go index 42157cd..69ad2c4 100644 --- a/repositories_test.go +++ b/repositories_test.go @@ -11,7 +11,7 @@ import ( "testing" "github.com/google/go-github/github" - "github.com/xanzy/go-gitlab" + gitlab "github.com/xanzy/go-gitlab" ) var ( @@ -57,7 +57,7 @@ func TestGetGitHubRepositories(t *testing.T) { defer teardown() mux.HandleFunc("/user/repos", func(w http.ResponseWriter, r *http.Request) { - fmt.Fprint(w, `[{"full_name": "test/r1", "id":1, "git_url": "git://github.com/u/r1", "name": "r1"}]`) + fmt.Fprint(w, `[{"full_name": "test/r1", "id":1, "ssh_url": "https://github.com/u/r1", "name": "r1", "private": false}]`) }) repos, err := getRepositories(GitHubClient, "github", "all", "", "") @@ -65,7 +65,7 @@ func TestGetGitHubRepositories(t *testing.T) { t.Fatalf("%v", err) } var expected []*Repository - expected = append(expected, &Repository{Namespace: "test", GitURL: "git://github.com/u/r1", Name: "r1"}) + expected = append(expected, &Repository{Namespace: "test", CloneURL: "https://github.com/u/r1", Name: "r1", Private: false}) if !reflect.DeepEqual(repos, expected) { t.Errorf("Expected %+v, Got %+v", expected, repos) } @@ -76,15 +76,15 @@ func TestGetGitLabRepositories(t *testing.T) { defer teardown() mux.HandleFunc("/api/v4/projects", func(w http.ResponseWriter, r *http.Request) { - fmt.Fprint(w, `[{"path_with_namespace": "test/r1", "id":1, "ssh_url_to_repo": "git://gitlab.com/u/r1", "name": "r1"}]`) + fmt.Fprint(w, `[{"path_with_namespace": "test/r1", "id":1, "ssh_url_to_repo": "https://gitlab.com/u/r1", "name": "r1"}]`) }) - repos, err := getRepositories(GitLabClient, "gitlab", "internal", "","") + repos, err := getRepositories(GitLabClient, "gitlab", "internal", "", "") if err != nil { t.Fatalf("%v", err) } var expected []*Repository - expected = append(expected, &Repository{Namespace: "test", GitURL: "git://gitlab.com/u/r1", Name: "r1"}) + expected = append(expected, &Repository{Namespace: "test", CloneURL: "https://gitlab.com/u/r1", Name: "r1"}) if !reflect.DeepEqual(repos, expected) { for i := 0; i < len(repos); i++ { t.Errorf("Expected %+v, Got %+v", expected[i], repos[i])