Skip to content

Commit 2c7432c

Browse files
amitsahaamitsaha
authored
Merge pull request #64 from amitsaha/issue_62_retry
GitHub - User data retry logic update
2 parents fa5e695 + 1a886d3 commit 2c7432c

File tree

5 files changed

+142
-19
lines changed

5 files changed

+142
-19
lines changed

‎go.mod

+1
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ require (
77
github.com/cli/oauth v0.8.0
88
github.com/google/go-github/v34 v34.0.0
99
github.com/ktrysmt/go-bitbucket v0.9.1
10+
github.com/migueleliasweb/go-github-mock v0.0.5 // indirect
1011
github.com/mitchellh/go-homedir v1.1.0
1112
github.com/spf13/afero v1.2.2
1213
github.com/xanzy/go-gitlab v0.16.1

‎go.sum

+14
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
22
github.com/99designs/keyring v1.1.6 h1:kVDC2uCgVwecxCk+9zoCt2uEL6dt+dfVzMvGgnVcIuM=
33
github.com/99designs/keyring v1.1.6/go.mod h1:16e0ds7LGQQcT59QqkTg72Hh5ShM51Byv5PEmW6uoRU=
4+
github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs=
5+
github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0=
46
github.com/cli/browser v1.0.0/go.mod h1:IEWkHYbLjkhtjwwWlwTHW2lGxeS5gezEQBMLTwDHf5Q=
57
github.com/cli/oauth v0.8.0 h1:YTFgPXSTvvDUFti3tR4o6q7Oll2SnQ9ztLwCAn4/IOA=
68
github.com/cli/oauth v0.8.0/go.mod h1:qd/FX8ZBD6n1sVNQO3aIdRxeu5LGw9WhKnYhIIoC2A4=
@@ -12,16 +14,25 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
1214
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
1315
github.com/dvsekhvalnov/jose2go v0.0.0-20200901110807-248326c1351b h1:HBah4D48ypg3J7Np4N+HY/ZR76fx3HEUGxDU6Uk39oQ=
1416
github.com/dvsekhvalnov/jose2go v0.0.0-20200901110807-248326c1351b/go.mod h1:7BvyPhdbLxMXIYTFPLsyJRFMsKmOZnQmzh6Gb+uquuM=
17+
github.com/go-kit/log v0.2.0 h1:7i2K3eKTos3Vc0enKCfnVcgHh2olr/MyfboYq7cAcFw=
18+
github.com/go-kit/log v0.2.0/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0=
19+
github.com/go-logfmt/logfmt v0.5.1 h1:otpy5pqBCBZ1ng9RQ0dPu4PN7ba75Y/aA+UpowDyNVA=
20+
github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs=
1521
github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 h1:ZpnhV/YsD2/4cESfV5+Hoeu/iUR3ruzNvZ+yQfO03a0=
1622
github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2/go.mod h1:bBOAhwG1umN6/6ZUMtDFBMQR8jRg9O75tm9K00oMsK4=
1723
github.com/golang/protobuf v1.0.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
1824
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
1925
github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs=
2026
github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
27+
github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
2128
github.com/google/go-github/v34 v34.0.0 h1:/siYFImY8KwGc5QD1gaPf+f8QX6tLwxNIco2RkYxoFA=
2229
github.com/google/go-github/v34 v34.0.0/go.mod h1:w/2qlrXUfty+lbyO6tatnzIw97v1CM+/jZcwXMDiPQQ=
30+
github.com/google/go-github/v37 v37.0.0 h1:rCspN8/6kB1BAJWZfuafvHhyfIo5fkAulaP/3bOQ/tM=
31+
github.com/google/go-github/v37 v37.0.0/go.mod h1:LM7in3NmXDrX58GbEHy7FtNLbI2JijX93RnMKvWG3m4=
2332
github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk=
2433
github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
34+
github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI=
35+
github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
2536
github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c h1:6rhixN/i8ZofjG1Y75iExal34USq5p+wiN1tpie8IrU=
2637
github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c/go.mod h1:NMPJylDgVpX0MLRlPy15sqSwOFv/U1GZ2m21JhFfek0=
2738
github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88 h1:uC1QfSlInpQF+M0ao65imhwqKnz3Q2z/d8PWZRMQvDM=
@@ -41,6 +52,8 @@ github.com/mattn/go-colorable v0.0.9 h1:UVL0vNpWh04HeJXV0KLcaT7r06gOH2l4OW6ddYRU
4152
github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
4253
github.com/mattn/go-isatty v0.0.3 h1:ns/ykhmWi7G9O+8a448SecJU3nSMBXJfqQkl0upE1jI=
4354
github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
55+
github.com/migueleliasweb/go-github-mock v0.0.5 h1:oCUwIPIknszT0DkjGT3VfILe1FgUDaNgEnj4w8mTZZA=
56+
github.com/migueleliasweb/go-github-mock v0.0.5/go.mod h1:gTpcHVcrBxK35OOQP3aGrgQypxvEoFTvtR0VGaEs2VM=
4457
github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
4558
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
4659
github.com/mitchellh/mapstructure v0.0.0-20180220230111-00c29f56e238 h1:+MZW2uvHgN8kYvksEN3f7eFL2wpzk0GxmlFsMybWc7E=
@@ -85,6 +98,7 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
8598
golang.org/x/text v0.3.5 h1:i6eZZ+zk0SOf0xgBpEpPD18qWcJda6q1sxt3S0kzyUQ=
8699
golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
87100
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
101+
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
88102
google.golang.org/appengine v1.0.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
89103
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
90104
google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=

‎main.go

+8-13
Original file line numberDiff line numberDiff line change
@@ -7,14 +7,15 @@ import (
77
"log"
88
"net/url"
99
"sync"
10-
"time"
1110

1211
"github.com/google/go-github/v34/github"
1312
)
1413

1514
// MaxConcurrentClones is the upper limit of the maximum number of
1615
// concurrent git clones
17-
var MaxConcurrentClones = 20
16+
const MaxConcurrentClones = 20
17+
18+
const defaultMaxUserMigrationRetry = 5
1819

1920
var gitHostToken string
2021
var useHTTPSClone *bool
@@ -50,7 +51,8 @@ func main() {
5051
githubRepoType := flag.String("github.repoType", "all", "Repo types to backup (all, owner, member, starred)")
5152

5253
githubCreateUserMigration := flag.Bool("github.createUserMigration", false, "Download user data")
53-
githubCreateUserMigrationRetry := flag.Bool("github.createUserMigrationRetry", true, "Retry creating the user migration if we get an error")
54+
githubCreateUserMigrationRetry := flag.Bool("github.createUserMigrationRetry", true, "Retry creating the GitHub user migration if we get an error")
55+
githubCreateUserMigrationRetryMax := flag.Int("github.createUserMigrationRetryMax", defaultMaxUserMigrationRetry, "Number of retries to attempt for creating GitHub user migration")
5456
githubListUserMigrations := flag.Bool("github.listUserMigrations", false, "List available user migrations")
5557
githubWaitForMigrationComplete := flag.Bool("github.waitForUserMigration", true, "Wait for migration to complete")
5658

@@ -114,18 +116,11 @@ func main() {
114116
}
115117

116118
log.Printf("Creating a user migration for %d repos", len(repos))
117-
m, err := createGithubUserMigration(context.Background(), client, repos)
119+
m, err := createGithubUserMigration(context.Background(), client, repos, *githubCreateUserMigrationRetry, *githubCreateUserMigrationRetryMax)
118120
if err != nil {
119-
if !*githubCreateUserMigrationRetry {
120-
log.Fatalf("Error creating migration: %v", err)
121-
}
122-
log.Printf("Got error when creating migration: %v. Retrying after sleeping for 300 seconds.", err)
123-
time.Sleep(300 * time.Second)
124-
m, err = createGithubUserMigration(context.Background(), client, repos)
125-
if err != nil {
126-
log.Fatalf("Error creating migration: %v", err)
127-
}
121+
log.Fatalf("Error creating migration: %v", err)
128122
}
123+
129124
if *githubWaitForMigrationComplete {
130125
downloadGithubUserMigrationData(client, *backupDir, m.ID)
131126
}

‎user_data.go

+21-6
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,11 @@ import (
1515
"github.com/google/go-github/v34/github"
1616
)
1717

18-
func createGithubUserMigration(ctx context.Context, client interface{}, repos []*Repository) (*github.UserMigration, error) {
18+
func createGithubUserMigration(ctx context.Context, client interface{}, repos []*Repository, retry bool, maxNumRetries int) (*github.UserMigration, error) {
19+
var m *github.UserMigration
20+
var err error
21+
var resp *github.Response
22+
1923
migrationOpts := github.UserMigrationOptions{
2024
LockRepositories: false,
2125
ExcludeAttachments: false,
@@ -25,11 +29,22 @@ func createGithubUserMigration(ctx context.Context, client interface{}, repos []
2529
repoPaths = append(repoPaths, fmt.Sprintf("%s/%s", repo.Namespace, repo.Name))
2630
}
2731

28-
m, resp, err := client.(*github.Client).Migrations.StartUserMigration(ctx, repoPaths, &migrationOpts)
29-
if err != nil {
30-
defer resp.Body.Close()
31-
data, _ := ioutil.ReadAll(resp.Body)
32-
log.Printf("%v", string(data))
32+
numAttempts := 1
33+
if retry {
34+
numAttempts += maxNumRetries
35+
}
36+
37+
var errResponse []byte
38+
for i := 1; i <= numAttempts; i++ {
39+
m, resp, err = client.(*github.Client).Migrations.StartUserMigration(ctx, repoPaths, &migrationOpts)
40+
if err == nil {
41+
return m, nil
42+
}
43+
if resp != nil {
44+
errResponse, _ = ioutil.ReadAll(resp.Body)
45+
resp.Body.Close()
46+
}
47+
log.Printf("Attempt #%d: Error creating user migration: %v", i, string(errResponse))
3348
}
3449
return m, err
3550
}

‎user_migration_test.go

+98
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,98 @@
1+
package main
2+
3+
import (
4+
"context"
5+
"net/http"
6+
"sync"
7+
"testing"
8+
9+
"github.com/google/go-github/v34/github"
10+
githubmock "github.com/migueleliasweb/go-github-mock/src/mock"
11+
)
12+
13+
type requestCounter struct {
14+
mutex sync.Mutex
15+
cnt int
16+
originalTransport http.RoundTripper
17+
}
18+
19+
func (c *requestCounter) RoundTrip(r *http.Request) (*http.Response, error) {
20+
c.mutex.Lock()
21+
defer c.mutex.Unlock()
22+
c.cnt += 1
23+
resp, err := c.originalTransport.RoundTrip(r)
24+
return resp, err
25+
}
26+
27+
func TestCreateGitHubUserMigrationRetryMax(t *testing.T) {
28+
expectedNumAttempts := defaultMaxUserMigrationRetry + 1
29+
30+
mockedHTTPClient := githubmock.NewMockedHTTPClient(
31+
githubmock.WithRequestMatchHandler(
32+
githubmock.PostUserMigrations,
33+
http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
34+
githubmock.WriteError(
35+
w,
36+
http.StatusBadGateway,
37+
"github 502",
38+
)
39+
}),
40+
),
41+
)
42+
requestCounter := requestCounter{}
43+
requestCounter.originalTransport = mockedHTTPClient.Transport
44+
mockedHTTPClient.Transport = &requestCounter
45+
46+
c := github.NewClient(mockedHTTPClient)
47+
48+
ctx := context.Background()
49+
50+
_, _ = createGithubUserMigration(ctx, c, nil, true, defaultMaxUserMigrationRetry)
51+
if requestCounter.cnt != expectedNumAttempts {
52+
t.Fatalf("Expected:%d attempts, got: %d\n", expectedNumAttempts, requestCounter.cnt)
53+
}
54+
}
55+
56+
func TestCreateGitHubUserMigrationFailOnceThenSucceed(t *testing.T) {
57+
expectedNumAttempts := 2
58+
mockRepoName := "mock-repo-1"
59+
60+
mockedHTTPClient := githubmock.NewMockedHTTPClient(
61+
githubmock.WithRequestMatch(
62+
githubmock.PostUserMigrations,
63+
"rubbish_1",
64+
github.UserMigration{
65+
Repositories: []*github.Repository{
66+
{
67+
Name: &mockRepoName,
68+
},
69+
},
70+
},
71+
),
72+
)
73+
requestCounter := requestCounter{}
74+
requestCounter.originalTransport = mockedHTTPClient.Transport
75+
mockedHTTPClient.Transport = &requestCounter
76+
77+
c := github.NewClient(mockedHTTPClient)
78+
ctx := context.Background()
79+
80+
reposToMigrate := []*Repository{
81+
{
82+
Name: "mock-repo-1",
83+
Namespace: "test-user-1",
84+
},
85+
}
86+
87+
m, err := createGithubUserMigration(ctx, c, reposToMigrate, true, defaultMaxUserMigrationRetry)
88+
if err != nil {
89+
t.Fatal(err)
90+
}
91+
92+
if len(m.Repositories) != len(reposToMigrate) {
93+
t.Fatalf("Expected %d repos in the migration. Got: %d", len(reposToMigrate), len(m.Repositories))
94+
}
95+
if requestCounter.cnt != expectedNumAttempts {
96+
t.Fatalf("Expected to send %d requests, sent: %d\n", defaultMaxUserMigrationRetry+1, requestCounter.cnt)
97+
}
98+
}

0 commit comments

Comments
 (0)