Add options to logs s3

main.go

@@ -103,6 +103,16 @@ func main() {
 			Usage:  "s3 path prefix (optional)",
 			EnvVar: "S3_PREFIX",
 		},
+		cli.StringFlag{
+			Name:   "s3-endpoint",
+			Usage:  "s3 endpoint (optional)",
+			EnvVar: "S3_ENDPOINT",
+		},
+		cli.BoolFlag{
+			Name:   "s3-path-style",
+			Usage:  "s3 path style (optional)",
+			EnvVar: "S3_PATH_STYLE",
+		},
 		cli.BoolTFlag{
 			Name:   "debug",
 			Usage:  "enable debug mode",
@@ -343,9 +353,12 @@ func main() {
 					return err
 				}
 
-				bucket := c.GlobalString("s3-bucket")
-				prefix := c.GlobalString("s3-prefix")
-				return migrate.MigrateLogsS3(source, bucket, prefix)
+				return migrate.MigrateLogsS3(
+					source,
+					c.GlobalString("s3-bucket"),
+					c.GlobalString("s3-prefix"),
+					c.GlobalString("s3-endpoint"),
+					c.GlobalBool("s3-path-style"))
 			},
 		},
 		{

migrate/logs.go

@@ -3,8 +3,10 @@ package migrate
 import (
 	"bytes"
 	"database/sql"
+	"errors"
 	"fmt"
 	"path"
+	"strings"
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/session"
@@ -64,7 +66,10 @@ func MigrateLogs(source, target *sql.DB) error {
 }
 
 // MigrateLogsS3 migrates the steps from the V0 database to S3.
-func MigrateLogsS3(source *sql.DB, bucket, prefix string) error {
+func MigrateLogsS3(source *sql.DB, bucket, prefix, endpoint string, pathStyle bool) error {
+	if bucket == "" {
+		return errors.New("bucket is required")
+	}
 	stepsV0 := []*StepV0{}
 
 	// 1. load all stages from the V0 database.
@@ -75,12 +80,18 @@ func MigrateLogsS3(source *sql.DB, bucket, prefix string) error {
 
 	logrus.Infof("migrating %d logs", len(stepsV0))
 
+	disableSSL := false
+
+	if endpoint != "" {
+		disableSSL = !strings.HasPrefix(endpoint, "https://")
+	}
+
 	// 2. create the s3 client
 	sess := session.Must(
 		session.NewSession(&aws.Config{
-			// Endpoint:         aws.String(endpoint),
-			// DisableSSL:       aws.Bool(disableSSL),
-			// S3ForcePathStyle: aws.Bool(pathStyle),
+			Endpoint:         aws.String(endpoint),
+			DisableSSL:       aws.Bool(disableSSL),
+			S3ForcePathStyle: aws.Bool(pathStyle),
 		}),
 	)
 
@@ -106,7 +117,11 @@ func MigrateLogsS3(source *sql.DB, bucket, prefix string) error {
 		}
 		_, err = uploader.Upload(input)
 		if err != nil {
-			logrus.WithError(err).Errorln("migration failed")
+			logrus.WithFields(logrus.Fields{
+				"proc_id": logsV0.ProcID,
+				"step_id": stepV0.ID,
+				"bucket":  bucket,
+			}).WithError(err).Errorln("migration failed")
 			return err
 		}
 	}