sync to s3

Author: kenerwin88

backend/cmd/sync-s3/main.go

@@ -0,0 +1,68 @@
+package main
+
+import (
+	"context"
+	"log"
+	"os"
+	"path/filepath"
+	"bufio"
+	"strings"
+
+	"go.uber.org/zap"
+	"pixelmap.io/backend/internal/ingestor"
+)
+
+func loadEnv() {
+	envPath := ".env"
+	if _, err := os.Stat(envPath); os.IsNotExist(err) {
+		envPath = filepath.Join("..", ".env")
+	}
+	
+	file, err := os.Open(envPath)
+	if err != nil {
+		log.Printf("Warning: Could not load .env file: %v", err)
+		return
+	}
+	defer file.Close()
+
+	scanner := bufio.NewScanner(file)
+	for scanner.Scan() {
+		line := scanner.Text()
+		if line == "" || strings.HasPrefix(line, "#") {
+			continue
+		}
+		
+		parts := strings.SplitN(line, "=", 2)
+		if len(parts) == 2 {
+			key := strings.TrimSpace(parts[0])
+			value := strings.TrimSpace(parts[1])
+			value = strings.Trim(value, `"'`)
+			os.Setenv(key, value)
+		}
+	}
+}
+
+func main() {
+	// Load environment variables
+	loadEnv()
+
+	// Create logger
+	logger, _ := zap.NewProduction()
+	defer logger.Sync()
+
+	// Create S3 syncer
+	s3Syncer, err := ingestor.NewS3Syncer(logger, "cache")
+	if err != nil {
+		log.Fatal("Failed to create S3 syncer:", err)
+	}
+
+	log.Println("Starting S3 sync...")
+	
+	// Sync to S3
+	ctx := context.Background()
+	if err := s3Syncer.SyncWithS3(ctx); err != nil {
+		log.Fatal("Failed to sync to S3:", err)
+	}
+
+	log.Println("S3 sync complete!")
+}

backend/sync-to-s3.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+
+# Load AWS credentials from .env if needed
+if [ -f .env ]; then
+  export $(cat .env | grep -v '^#' | xargs)
+fi
+
+# S3 bucket name (from the Go code we saw)
+BUCKET="pixelmap.art"
+
+echo "Syncing cache directory to S3 bucket: $BUCKET"
+
+# Sync the entire cache directory to S3
+# --delete removes files from S3 that don't exist locally
+# Remove --delete if you want to keep old files
+aws s3 sync cache/ s3://$BUCKET/ \
+  --exclude ".DS_Store" \
+  --exclude "*.log" \
+  --acl public-read
+
+echo "Sync complete!"
+
+# Optionally, you can sync just specific directories:
+# aws s3 sync cache/tile/ s3://$BUCKET/tile/ --acl public-read
+# aws s3 cp cache/tiledata.json s3://$BUCKET/tiledata.json --acl public-read