add migration code to download from one instance and upload to another

cmd/integrationtest/command.go

@@ -40,11 +40,12 @@ func NewIntegrationTestCmd() *cobra.Command {
 			keepPod, _ := cmd.Flags().GetBool("keepPod")
 			sidecar, _ := cmd.Flags().GetBool("sidecar")
 			indyProxyUrl, _ := cmd.Flags().GetString("indyProxyUrl")
+			migrateTargetIndy, _ := cmd.Flags().GetString("migrateTargetIndy")
 			metaCheckRepo := ""
 			if len(args) >= 5 {
 				metaCheckRepo = args[4]
 			}
-			integrationtest.Run(args[0], args[1], args[2], args[3], metaCheckRepo, clearCache, dryRun, keepPod, sidecar, indyProxyUrl)
+			integrationtest.Run(args[0], args[1], args[2], args[3], metaCheckRepo, clearCache, dryRun, keepPod, sidecar, indyProxyUrl, migrateTargetIndy)
 		},
 	}
 
@@ -53,6 +54,7 @@ func NewIntegrationTestCmd() *cobra.Command {
 	exec.Flags().BoolP("keepPod", "k", false, "Keep the pod after test to debug.")
 	exec.Flags().BoolP("sidecar", "s", false, "Send requests through sidecar.")
 	exec.Flags().StringP("indyProxyUrl", "p", "", "Indy generic proxy url.")
+	exec.Flags().StringP("migrateTargetIndy", "m", "", "Migrate download artifacts to new Indy url")
 	return exec
 }
 

pkg/buildtest/run.go

@@ -7,13 +7,16 @@ import (
 	"path"
 	"strings"
 
+	"github.com/commonjava/indy-tests/pkg/promotetest"
+
 	common "github.com/commonjava/indy-tests/pkg/common"
 )
 
 const (
 	TMP_DOWNLOAD_DIR = "/tmp/download"
 	TMP_UPLOAD_DIR   = "/tmp/upload"
 	PROXY_           = "proxy-"
+	//migrateTargetIndyHost = "indy-gateway-indy--test.apps.gpc.ocp-hub.prod.psi.redhat.com"
 )
 
 func Run(originalIndy, foloId, replacement, targetIndy, packageType string, processNum int) {
@@ -23,11 +26,11 @@ func Run(originalIndy, foloId, replacement, targetIndy, packageType string, proc
 	}
 	foloTrackContent := common.GetFoloRecord(origIndy, foloId)
 	newBuildName := common.GenerateRandomBuildName()
-	DoRun(originalIndy, targetIndy, "", packageType, newBuildName, foloTrackContent, nil, processNum, false, false)
+	DoRun(originalIndy, targetIndy, "", "", packageType, newBuildName, foloTrackContent, nil, processNum, false, false)
 }
 
 // Create the repo structure and do the download/upload
-func DoRun(originalIndy, targetIndy, indyProxyUrl, packageType, newBuildName string, foloTrackContent common.TrackedContent,
+func DoRun(originalIndy, targetIndy, indyProxyUrl, migrateTargetIndy, packageType, newBuildName string, foloTrackContent common.TrackedContent,
 	additionalRepos []string,
 	processNum int, clearCache, dryRun bool) bool {
 
@@ -40,6 +43,13 @@ func DoRun(originalIndy, targetIndy, indyProxyUrl, packageType, newBuildName str
 		os.Exit(1)
 	}
 
+	migrateEnabled := (migrateTargetIndy != "")
+	if migrateEnabled {
+		migrateTargetIndyHost, _ := common.ValidateTargetIndyOrExit(migrateTargetIndy)
+		fmt.Printf("Migrate to host %s", migrateTargetIndyHost)
+		prepareIndyRepos("http://"+migrateTargetIndyHost, newBuildName, *buildMeta, additionalRepos, dryRun)
+	}
+
 	trackingId := foloTrackContent.TrackingKey.Id
 	downloadDir, uploadDir := prepareDownUploadDirectories(trackingId, clearCache)
 
@@ -64,12 +74,57 @@ func DoRun(originalIndy, targetIndy, indyProxyUrl, packageType, newBuildName str
 		}
 		return success
 	}
+
+	migrateFunc := func(md5str, originalArtiURL, targetArtiURL string, migrateTargetArtiURL string) bool {
+		fileLoc := path.Join(downloadDir, path.Base(targetArtiURL))
+		if dryRun {
+			fmt.Printf("Dry run download, url: %s\n", targetArtiURL)
+			return true
+		}
+		success := false
+		success, _ = common.DownloadFile(targetArtiURL, fileLoc)
+		if success {
+			common.Md5Check(fileLoc, md5str)
+			if dryRun {
+				fmt.Printf("Dry run upload, url: %s\n", migrateTargetArtiURL)
+				return true
+			}
+			common.UploadFile(migrateTargetArtiURL, fileLoc)
+		}
+		return success
+	}
+
 	broken := false
 	if len(downloads) > 0 {
 		fmt.Println("Start handling downloads artifacts.")
 		fmt.Printf("==========================================\n\n")
-		if processNum > 1 {
+		if processNum > 1 && !migrateEnabled {
 			broken = !common.ConcurrentRun(processNum, downloads, downloadFunc)
+		} else if migrateEnabled {
+			migrateTargetIndyHost, _ := common.ValidateTargetIndyOrExit(migrateTargetIndy)
+			migrateUploads := prepareDownloadEntriesByFolo(migrateTargetIndyHost, newBuildName, packageType, foloTrackContent, additionalRepos, proxyEnabled)
+			paths := []string{}
+			rhpaths := []string{}
+			for i, down := range downloads {
+				fmt.Print(i)
+				broken = !migrateFunc(down[0], down[1], down[2], migrateUploads[i][2])
+				if broken {
+					break
+				} else {
+					if strings.Contains(i, "redhat"){
+						rhpaths = append(rhpaths, i)
+					} else {
+						paths = append(paths, i)
+					}
+				}
+			}
+			targetStore := packageType+":hosted:shared-imports"
+			rhTargetStore := packageType+":hosted:pnc-builds"
+			sourceStore := packageType+":hosted:"+newBuildName
+			promotetest.MigratePromote("http://"+migrateTargetIndyHost, newBuildName, sourceStore, targetStore, paths, false)
+			if rhpaths != nil{
+				promotetest.MigratePromote("http://"+migrateTargetIndyHost, newBuildName, sourceStore, rhTargetStore, rhpaths, false)
+			}
 		} else {
 			for _, down := range downloads {
 				broken = !downloadFunc(down[0], down[1], down[2])

pkg/integrationtest/run.go

@@ -56,7 +56,7 @@ const (
  * j. Retrieve the metadata files from step #f again, check that the new version is gone
  * k. Clean up. Delete the build group G and the hosted repo A. Delete folo record.
  */
-func Run(indyBaseUrl, datasetRepoUrl, buildId, promoteTargetStore, metaCheckRepo string, clearCache, dryRun, keepPod, sidecar bool, indyProxyUrl string) {
+func Run(indyBaseUrl, datasetRepoUrl, buildId, promoteTargetStore, metaCheckRepo string, clearCache, dryRun, keepPod, sidecar bool, indyProxyUrl string, migrateTargetIndy string) {
 	if indyProxyUrl != "" {
 		fmt.Println("Enable generic proxy: " + indyProxyUrl)
 	}
@@ -92,7 +92,7 @@ func Run(indyBaseUrl, datasetRepoUrl, buildId, promoteTargetStore, metaCheckRepo
 	originalIndy := getOriginalIndyBaseUrl(foloTrackContent.Uploads[0].LocalUrl)
 	buildName := common.GenerateRandomBuildName()
 	prev := t
-	buildSuccess := buildtest.DoRun(originalIndy, indyBaseUrl, indyProxyUrl, packageType, buildName, foloTrackContent, additionalRepos, DEFAULT_ROUTINES, clearCache, dryRun)
+	buildSuccess := buildtest.DoRun(originalIndy, indyBaseUrl, indyProxyUrl, migrateTargetIndy, packageType, buildName, foloTrackContent, additionalRepos, DEFAULT_ROUTINES, clearCache, dryRun)
 	t = time.Now()
 	fmt.Printf("Create mock group(%s) and download/upload SUCCESS, elapsed(s): %f\n", buildName, t.Sub(prev).Seconds())
 

pkg/promotetest/run.go

@@ -46,3 +46,7 @@ func DoRun(indyBaseUrl, foloTrackId, sourceStore, targetStore, newVersionNum str
 
 	return promote(indyBaseUrl, foloTrackId, sourceStore, targetStore, paths, dryRun)
 }
+
+func MigratePromote(indyURL, trackingId, source, target string, paths []string, dryRun bool) (string, int, bool) {
+	return promote(indyURL, trackingId, source, target, paths, dryRun)
+}