diff --git a/server/lib/cleanup.go b/server/lib/cleanup.go new file mode 100644 index 0000000..c0214e1 --- /dev/null +++ b/server/lib/cleanup.go @@ -0,0 +1,142 @@ +package lib + +import ( + "time" + + "github.com/corecollectives/mist/models" + "github.com/corecollectives/mist/queue" + "github.com/rs/zerolog/log" +) + +func CleanupOnStartup() error { + err := cleanupUpdates() + if err != nil { + return err + } + err = cleanupDeployments() + if err != nil { + return err + } + return nil +} + +func cleanupDeployments() error { + deployments, err := models.GetIncompleteDeployments() + if err != nil { + return err + } + if len(deployments) == 0 { + return nil + } + + log.Info(). + Int("count", len(deployments)). + Msg("Found incomplete deployments on startup, cleaning up") + + errorMsg := "system died before deployment could complete" + for _, dep := range deployments { + if dep.Status == "deploying" || dep.Status == "building" { + log.Warn(). + Int64("deployment_id", dep.ID). + Str("status", string(dep.Status)). + Msg("Marking interrupted deployment as failed") + + err = models.UpdateDeploymentStatus(dep.ID, "failed", "failed", 0, &errorMsg) + if err != nil { + log.Error().Err(err).Int64("deployment_id", dep.ID).Msg("Failed to mark deployment as failed") + return err + } + } else if dep.Status == "pending" { + log.Info(). + Int64("deployment_id", dep.ID). + Msg("Re-queuing pending deployment") + + err = queue.GetQueue().AddJob(dep.ID) + if err != nil { + log.Error().Err(err).Int64("deployment_id", dep.ID).Msg("Failed to re-queue pending deployment") + return err + } + } else { + continue + } + } + + log.Info().Msg("Deployment cleanup completed successfully") + return nil +} + +func cleanupUpdates() error { + logs, err := models.GetUpdateLogs(1) + if err != nil { + return err + } + + if len(logs) == 0 { + return nil + } + + latestLog := logs[0] + + if latestLog.Status != "in_progress" { + return nil + } + + log.Info(). + Int64("update_log_id", latestLog.ID). + Str("from_version", latestLog.VersionFrom). + Str("to_version", latestLog.VersionTo). + Str("age", time.Since(latestLog.StartedAt).String()). + Msg("Found in-progress update on startup, checking status") + + currentVersion, err := models.GetSystemSetting("version") + if err != nil { + log.Error().Err(err).Msg("Failed to get current version for update completion check") + return err + } + + if currentVersion == "" { + currentVersion = "1.0.0" + } + + if currentVersion == latestLog.VersionTo { + log.Info(). + Int64("update_log_id", latestLog.ID). + Str("version", currentVersion). + Msg("Completing successful update that was interrupted by service restart") + + completionLog := latestLog.Logs + "\n✅ Update completed successfully (verified on restart)\n" + err = models.UpdateUpdateLogStatus(latestLog.ID, "success", completionLog, nil) + if err != nil { + log.Error().Err(err).Int64("update_log_id", latestLog.ID).Msg("Failed to complete pending update") + return err + } + + log.Info(). + Int64("update_log_id", latestLog.ID). + Str("from", latestLog.VersionFrom). + Str("to", latestLog.VersionTo). + Msg("Successfully completed pending update") + return nil + } + + log.Warn(). + Int64("update_log_id", latestLog.ID). + Str("expected_version", latestLog.VersionTo). + Str("current_version", currentVersion). + Str("age", time.Since(latestLog.StartedAt).String()). + Msg("Update appears to have failed (version mismatch detected on startup)") + + errMsg := "Update process was interrupted and version does not match target" + failureLog := latestLog.Logs + "\n❌ " + errMsg + "\n" + err = models.UpdateUpdateLogStatus(latestLog.ID, "failed", failureLog, &errMsg) + if err != nil { + log.Error().Err(err).Int64("update_log_id", latestLog.ID).Msg("Failed to mark failed update") + return err + } + + log.Info(). + Int64("update_log_id", latestLog.ID). + Msg("Marked failed update as failed") + + return nil +} diff --git a/server/main.go b/server/main.go index 69517e7..3f1ae02 100644 --- a/server/main.go +++ b/server/main.go @@ -3,6 +3,7 @@ package main import ( "github.com/corecollectives/mist/api" "github.com/corecollectives/mist/db" + "github.com/corecollectives/mist/lib" "github.com/corecollectives/mist/models" "github.com/corecollectives/mist/queue" "github.com/corecollectives/mist/store" @@ -27,8 +28,9 @@ func main() { // particular update in the db, and it gets stuck in 'in_progress' which leads disability in doing // updates, so on each startup we need to check if the last update was successfull or not and change // the status in the db accordingly, even if the update failed atleast we can retry it - if err := models.CheckAndCompletePendingUpdates(); err != nil { - log.Warn().Err(err).Msg("Failed to check pending updates") + // similarly if a deployment is running and the system goes down due to overload or any other thing, it get stuck to "progress" this function cleans that too + if err := lib.CleanupOnStartup(); err != nil { + log.Warn().Err(err).Msg("Failed to check pending updates and deployments") } err = store.InitStore() diff --git a/server/models/deployment.go b/server/models/deployment.go index 1572cba..85fc5da 100644 --- a/server/models/deployment.go +++ b/server/models/deployment.go @@ -287,3 +287,42 @@ func UpdateContainerInfo(depID int64, containerID, containerName, imageTag strin _, err := db.Exec(query, containerID, containerName, imageTag, depID) return err } + +func GetIncompleteDeployments() ([]Deployment, error) { + query := ` + SELECT id, app_id, commit_hash, commit_message, commit_author, triggered_by, + deployment_number, container_id, container_name, image_tag, + logs, build_logs_path, status, stage, progress, error_message, + created_at, started_at, finished_at, duration, is_active, rolled_back_from + FROM deployments + WHERE status = 'bulding' OR status = 'deploying' ? + ORDER BY created_at DESC + ` + + rows, err := db.Query(query) + if err != nil { + return nil, err + } + defer rows.Close() + + var deployments []Deployment + for rows.Next() { + var d Deployment + if err := rows.Scan( + &d.ID, &d.AppID, &d.CommitHash, &d.CommitMessage, &d.CommitAuthor, + &d.TriggeredBy, &d.DeploymentNumber, &d.ContainerID, &d.ContainerName, + &d.ImageTag, &d.Logs, &d.BuildLogsPath, &d.Status, &d.Stage, &d.Progress, + &d.ErrorMessage, &d.CreatedAt, &d.StartedAt, &d.FinishedAt, &d.Duration, + &d.IsActive, &d.RolledBackFrom, + ); err != nil { + return nil, err + } + deployments = append(deployments, d) + } + + if err := rows.Err(); err != nil { + return nil, err + } + + return deployments, nil +} diff --git a/server/models/updateLog.go b/server/models/updateLog.go index 181b3a6..0169779 100644 --- a/server/models/updateLog.go +++ b/server/models/updateLog.go @@ -217,79 +217,3 @@ func GetUpdateLogsAsString() (string, error) { return builder.String(), nil } - -func CheckAndCompletePendingUpdates() error { - logs, err := GetUpdateLogs(1) - if err != nil { - return err - } - - if len(logs) == 0 { - return nil - } - - latestLog := logs[0] - - if latestLog.Status != "in_progress" { - return nil - } - - log.Info(). - Int64("update_log_id", latestLog.ID). - Str("from_version", latestLog.VersionFrom). - Str("to_version", latestLog.VersionTo). - Str("age", time.Since(latestLog.StartedAt).String()). - Msg("Found in-progress update on startup, checking status") - - currentVersion, err := GetSystemSetting("version") - if err != nil { - log.Error().Err(err).Msg("Failed to get current version for update completion check") - return err - } - - if currentVersion == "" { - currentVersion = "1.0.0" - } - - if currentVersion == latestLog.VersionTo { - log.Info(). - Int64("update_log_id", latestLog.ID). - Str("version", currentVersion). - Msg("Completing successful update that was interrupted by service restart") - - completionLog := latestLog.Logs + "\n✅ Update completed successfully (verified on restart)\n" - err = UpdateUpdateLogStatus(latestLog.ID, "success", completionLog, nil) - if err != nil { - log.Error().Err(err).Int64("update_log_id", latestLog.ID).Msg("Failed to complete pending update") - return err - } - - log.Info(). - Int64("update_log_id", latestLog.ID). - Str("from", latestLog.VersionFrom). - Str("to", latestLog.VersionTo). - Msg("Successfully completed pending update") - return nil - } - - log.Warn(). - Int64("update_log_id", latestLog.ID). - Str("expected_version", latestLog.VersionTo). - Str("current_version", currentVersion). - Str("age", time.Since(latestLog.StartedAt).String()). - Msg("Update appears to have failed (version mismatch detected on startup)") - - errMsg := "Update process was interrupted and version does not match target" - failureLog := latestLog.Logs + "\n❌ " + errMsg + "\n" - err = UpdateUpdateLogStatus(latestLog.ID, "failed", failureLog, &errMsg) - if err != nil { - log.Error().Err(err).Int64("update_log_id", latestLog.ID).Msg("Failed to mark failed update") - return err - } - - log.Info(). - Int64("update_log_id", latestLog.ID). - Msg("Marked failed update as failed") - - return nil -}