diff --git a/main.go b/main.go index c481e51..96ad0cd 100644 --- a/main.go +++ b/main.go @@ -103,6 +103,16 @@ func main() { Usage: "s3 path prefix (optional)", EnvVar: "S3_PREFIX", }, + cli.StringFlag{ + Name: "s3-endpoint", + Usage: "s3 endpoint (optional)", + EnvVar: "S3_ENDPOINT", + }, + cli.BoolFlag{ + Name: "s3-path-style", + Usage: "s3 path style (optional)", + EnvVar: "S3_PATH_STYLE", + }, cli.BoolTFlag{ Name: "debug", Usage: "enable debug mode", @@ -343,9 +353,12 @@ func main() { return err } - bucket := c.GlobalString("s3-bucket") - prefix := c.GlobalString("s3-prefix") - return migrate.MigrateLogsS3(source, bucket, prefix) + return migrate.MigrateLogsS3( + source, + c.GlobalString("s3-bucket"), + c.GlobalString("s3-prefix"), + c.GlobalString("s3-endpoint"), + c.GlobalBool("s3-path-style")) }, }, { diff --git a/migrate/logs.go b/migrate/logs.go index 16e0c3e..48814c2 100644 --- a/migrate/logs.go +++ b/migrate/logs.go @@ -3,8 +3,10 @@ package migrate import ( "bytes" "database/sql" + "errors" "fmt" "path" + "strings" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/session" @@ -64,7 +66,10 @@ func MigrateLogs(source, target *sql.DB) error { } // MigrateLogsS3 migrates the steps from the V0 database to S3. -func MigrateLogsS3(source *sql.DB, bucket, prefix string) error { +func MigrateLogsS3(source *sql.DB, bucket, prefix, endpoint string, pathStyle bool) error { + if bucket == "" { + return errors.New("bucket is required") + } stepsV0 := []*StepV0{} // 1. load all stages from the V0 database. @@ -75,12 +80,18 @@ func MigrateLogsS3(source *sql.DB, bucket, prefix string) error { logrus.Infof("migrating %d logs", len(stepsV0)) + disableSSL := false + + if endpoint != "" { + disableSSL = !strings.HasPrefix(endpoint, "https://") + } + // 2. create the s3 client sess := session.Must( session.NewSession(&aws.Config{ - // Endpoint: aws.String(endpoint), - // DisableSSL: aws.Bool(disableSSL), - // S3ForcePathStyle: aws.Bool(pathStyle), + Endpoint: aws.String(endpoint), + DisableSSL: aws.Bool(disableSSL), + S3ForcePathStyle: aws.Bool(pathStyle), }), ) @@ -106,7 +117,11 @@ func MigrateLogsS3(source *sql.DB, bucket, prefix string) error { } _, err = uploader.Upload(input) if err != nil { - logrus.WithError(err).Errorln("migration failed") + logrus.WithFields(logrus.Fields{ + "proc_id": logsV0.ProcID, + "step_id": stepV0.ID, + "bucket": bucket, + }).WithError(err).Errorln("migration failed") return err } }