fix: add deployment cleanup on startup

server/lib/cleanup.go

@@ -0,0 +1,142 @@
+package lib
+
+import (
+	"time"
+
+	"github.com/corecollectives/mist/models"
+	"github.com/corecollectives/mist/queue"
+	"github.com/rs/zerolog/log"
+)
+
+func CleanupOnStartup() error {
+	err := cleanupUpdates()
+	if err != nil {
+		return err
+	}
+	err = cleanupDeployments()
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func cleanupDeployments() error {
+	deployments, err := models.GetIncompleteDeployments()
+	if err != nil {
+		return err
+	}
+	if len(deployments) == 0 {
+		return nil
+	}
+
+	log.Info().
+		Int("count", len(deployments)).
+		Msg("Found incomplete deployments on startup, cleaning up")
+
+	errorMsg := "system died before deployment could complete"
+	for _, dep := range deployments {
+		if dep.Status == "deploying" || dep.Status == "building" {
+			log.Warn().
+				Int64("deployment_id", dep.ID).
+				Str("status", string(dep.Status)).
+				Msg("Marking interrupted deployment as failed")
+
+			err = models.UpdateDeploymentStatus(dep.ID, "failed", "failed", 0, &errorMsg)
+			if err != nil {
+				log.Error().Err(err).Int64("deployment_id", dep.ID).Msg("Failed to mark deployment as failed")
+				return err
+			}
+		} else if dep.Status == "pending" {
+			log.Info().
+				Int64("deployment_id", dep.ID).
+				Msg("Re-queuing pending deployment")
+
+			err = queue.GetQueue().AddJob(dep.ID)
+			if err != nil {
+				log.Error().Err(err).Int64("deployment_id", dep.ID).Msg("Failed to re-queue pending deployment")
+				return err
+			}
+		} else {
+			continue
+		}
+	}
+
+	log.Info().Msg("Deployment cleanup completed successfully")
+	return nil
+}
+
+func cleanupUpdates() error {
+	logs, err := models.GetUpdateLogs(1)
+	if err != nil {
+		return err
+	}
+
+	if len(logs) == 0 {
+		return nil
+	}
+
+	latestLog := logs[0]
+
+	if latestLog.Status != "in_progress" {
+		return nil
+	}
+
+	log.Info().
+		Int64("update_log_id", latestLog.ID).
+		Str("from_version", latestLog.VersionFrom).
+		Str("to_version", latestLog.VersionTo).
+		Str("age", time.Since(latestLog.StartedAt).String()).
+		Msg("Found in-progress update on startup, checking status")
+
+	currentVersion, err := models.GetSystemSetting("version")
+	if err != nil {
+		log.Error().Err(err).Msg("Failed to get current version for update completion check")
+		return err
+	}
+
+	if currentVersion == "" {
+		currentVersion = "1.0.0"
+	}
+
+	if currentVersion == latestLog.VersionTo {
+		log.Info().
+			Int64("update_log_id", latestLog.ID).
+			Str("version", currentVersion).
+			Msg("Completing successful update that was interrupted by service restart")
+
+		completionLog := latestLog.Logs + "\n✅ Update completed successfully (verified on restart)\n"
+		err = models.UpdateUpdateLogStatus(latestLog.ID, "success", completionLog, nil)
+		if err != nil {
+			log.Error().Err(err).Int64("update_log_id", latestLog.ID).Msg("Failed to complete pending update")
+			return err
+		}
+
+		log.Info().
+			Int64("update_log_id", latestLog.ID).
+			Str("from", latestLog.VersionFrom).
+			Str("to", latestLog.VersionTo).
+			Msg("Successfully completed pending update")
+		return nil
+	}
+
+	log.Warn().
+		Int64("update_log_id", latestLog.ID).
+		Str("expected_version", latestLog.VersionTo).
+		Str("current_version", currentVersion).
+		Str("age", time.Since(latestLog.StartedAt).String()).
+		Msg("Update appears to have failed (version mismatch detected on startup)")
+
+	errMsg := "Update process was interrupted and version does not match target"
+	failureLog := latestLog.Logs + "\n❌ " + errMsg + "\n"
+	err = models.UpdateUpdateLogStatus(latestLog.ID, "failed", failureLog, &errMsg)
+	if err != nil {
+		log.Error().Err(err).Int64("update_log_id", latestLog.ID).Msg("Failed to mark failed update")
+		return err
+	}
+
+	log.Info().
+		Int64("update_log_id", latestLog.ID).
+		Msg("Marked failed update as failed")
+
+	return nil
+}

server/main.go

@@ -3,6 +3,7 @@ package main
 import (
 	"github.com/corecollectives/mist/api"
 	"github.com/corecollectives/mist/db"
+	"github.com/corecollectives/mist/lib"
 	"github.com/corecollectives/mist/models"
 	"github.com/corecollectives/mist/queue"
 	"github.com/corecollectives/mist/store"
@@ -27,8 +28,9 @@ func main() {
 	// particular update in the db, and it gets stuck in 'in_progress' which leads disability in doing
 	// updates, so on each startup we need to check if the last update was successfull or not and change
 	// the status in the db accordingly, even if the update failed atleast we can retry it
-	if err := models.CheckAndCompletePendingUpdates(); err != nil {
-		log.Warn().Err(err).Msg("Failed to check pending updates")
+	// similarly if a deployment is running and the system goes down due to overload or any other thing, it get stuck to "progress" this function cleans that too
+	if err := lib.CleanupOnStartup(); err != nil {
+		log.Warn().Err(err).Msg("Failed to check pending updates and deployments")
 	}
 
 	err = store.InitStore()

server/models/deployment.go

@@ -287,3 +287,42 @@ func UpdateContainerInfo(depID int64, containerID, containerName, imageTag strin
 	_, err := db.Exec(query, containerID, containerName, imageTag, depID)
 	return err
 }
+
+func GetIncompleteDeployments() ([]Deployment, error) {
+	query := `
+	SELECT id, app_id, commit_hash, commit_message, commit_author, triggered_by, 
+	       deployment_number, container_id, container_name, image_tag,
+	       logs, build_logs_path, status, stage, progress, error_message, 
+	       created_at, started_at, finished_at, duration, is_active, rolled_back_from
+	FROM deployments
+	WHERE status = 'bulding' OR status = 'deploying' ?
+	ORDER BY created_at DESC
+	`
+
+	rows, err := db.Query(query)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+
+	var deployments []Deployment
+	for rows.Next() {
+		var d Deployment
+		if err := rows.Scan(
+			&d.ID, &d.AppID, &d.CommitHash, &d.CommitMessage, &d.CommitAuthor,
+			&d.TriggeredBy, &d.DeploymentNumber, &d.ContainerID, &d.ContainerName,
+			&d.ImageTag, &d.Logs, &d.BuildLogsPath, &d.Status, &d.Stage, &d.Progress,
+			&d.ErrorMessage, &d.CreatedAt, &d.StartedAt, &d.FinishedAt, &d.Duration,
+			&d.IsActive, &d.RolledBackFrom,
+		); err != nil {
+			return nil, err
+		}
+		deployments = append(deployments, d)
+	}
+
+	if err := rows.Err(); err != nil {
+		return nil, err
+	}
+
+	return deployments, nil
+}

server/models/updateLog.go

@@ -217,79 +217,3 @@ func GetUpdateLogsAsString() (string, error) {
 
 	return builder.String(), nil
 }
-
-func CheckAndCompletePendingUpdates() error {
-	logs, err := GetUpdateLogs(1)
-	if err != nil {
-		return err
-	}
-
-	if len(logs) == 0 {
-		return nil
-	}
-
-	latestLog := logs[0]
-
-	if latestLog.Status != "in_progress" {
-		return nil
-	}
-
-	log.Info().
-		Int64("update_log_id", latestLog.ID).
-		Str("from_version", latestLog.VersionFrom).
-		Str("to_version", latestLog.VersionTo).
-		Str("age", time.Since(latestLog.StartedAt).String()).
-		Msg("Found in-progress update on startup, checking status")
-
-	currentVersion, err := GetSystemSetting("version")
-	if err != nil {
-		log.Error().Err(err).Msg("Failed to get current version for update completion check")
-		return err
-	}
-
-	if currentVersion == "" {
-		currentVersion = "1.0.0"
-	}
-
-	if currentVersion == latestLog.VersionTo {
-		log.Info().
-			Int64("update_log_id", latestLog.ID).
-			Str("version", currentVersion).
-			Msg("Completing successful update that was interrupted by service restart")
-
-		completionLog := latestLog.Logs + "\n✅ Update completed successfully (verified on restart)\n"
-		err = UpdateUpdateLogStatus(latestLog.ID, "success", completionLog, nil)
-		if err != nil {
-			log.Error().Err(err).Int64("update_log_id", latestLog.ID).Msg("Failed to complete pending update")
-			return err
-		}
-
-		log.Info().
-			Int64("update_log_id", latestLog.ID).
-			Str("from", latestLog.VersionFrom).
-			Str("to", latestLog.VersionTo).
-			Msg("Successfully completed pending update")
-		return nil
-	}
-
-	log.Warn().
-		Int64("update_log_id", latestLog.ID).
-		Str("expected_version", latestLog.VersionTo).
-		Str("current_version", currentVersion).
-		Str("age", time.Since(latestLog.StartedAt).String()).
-		Msg("Update appears to have failed (version mismatch detected on startup)")
-
-	errMsg := "Update process was interrupted and version does not match target"
-	failureLog := latestLog.Logs + "\n❌ " + errMsg + "\n"
-	err = UpdateUpdateLogStatus(latestLog.ID, "failed", failureLog, &errMsg)
-	if err != nil {
-		log.Error().Err(err).Int64("update_log_id", latestLog.ID).Msg("Failed to mark failed update")
-		return err
-	}
-
-	log.Info().
-		Int64("update_log_id", latestLog.ID).
-		Msg("Marked failed update as failed")
-
-	return nil
-}