Skip to content

DEV-1044 fix(server): fix oom and behave better when rollbacks are required #120

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
uptickmetachu merged 3 commits into from
Feb 14, 2025
Merged

DEV-1044 fix(server): fix oom and behave better when rollbacks are required #120

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
f4e890b
chore: bump gitops resources limits
uptickmetachu Feb 14, 2025
fcd9680
fix: remove gitops resource limits
uptickmetachu Feb 14, 2025
762cdb3
fix(helm): if a roll back is required; rollback before upgrading
uptickmetachu Feb 14, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 0 additions & 2 deletions charts/gitops/templates/deployment.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,8 +40,6 @@ spec:
resources:
requests:
memory: "750Mi"
limits:
memory: "1500Mi"

envFrom:
- configMapRef:
Expand Down
76 changes: 52 additions & 24 deletions gitops_server/workers/deployer/deploy.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@

from gitops.common.app import App
from gitops_server import settings
from gitops_server.types import AppDefinitions, UpdateAppResult
from gitops_server.types import AppDefinitions, RunOutput, UpdateAppResult
from gitops_server.utils import get_repo_name_from_url, github, run, slack
from gitops_server.utils.git import temp_repo

Expand Down Expand Up @@ -133,6 +133,17 @@ async def uninstall_app(self, app: App) -> UpdateAppResult:
)
return update_result

async def rollback_deployment(self, app: App) -> None:
with tracer.start_as_current_span("rollback_deployment", attributes={"app": app.name}):
logger.warning(
"Rolling back %s deployment due to previous failed helm install",
app.name,
)
await run(
f"helm rollback --namespace={app.namespace} {app.name}",
suppress_errors=True,
)

async def update_app_deployment(self, app: App) -> UpdateAppResult | None:
app.set_value("deployment.labels.gitops/deploy_id", self.deploy_id)
app.set_value("deployment.labels.gitops/status", github.STATUSES.in_progress)
Expand All @@ -155,6 +166,40 @@ async def update_app_deployment(self, app: App) -> UpdateAppResult | None:
os.fsync(cfg.fileno())

with tracer.start_as_current_span("helm_upgrade"):

async def upgrade_helm_git() -> RunOutput:
result = await run(
"helm secrets upgrade --create-namespace"
f" --history-max {MAX_HELM_HISTORY}"
" --install"
" --timeout=600s"
f"{' --set skip_migrations=true' if self.skip_migrations else ''}"
f" -f {cfg.name}"
f" --namespace={app.namespace}"
f" {app.name}"
f" {chart_folder_path}",
suppress_errors=True,
)
return result

result = await upgrade_helm_git()
if result["exit_code"] != 0 and "is in progress" in result["output"]:
await self.rollback_deployment(app)
result = await upgrade_helm_git()

elif app.chart.type == "helm":
span.set_attribute("gitops.chart.type", "helm")
with tempfile.NamedTemporaryFile(suffix=".yml") as cfg:
cfg.write(json.dumps(app.values).encode())
cfg.flush()
os.fsync(cfg.fileno())
chart_version_arguments = f" --version={app.chart.version}" if app.chart.version else ""
with tracer.start_as_current_span("helm_repo_add"):
await run(f"helm repo add {app.chart.helm_repo} {app.chart.helm_repo_url}")

with tracer.start_as_current_span("helm_upgrade"):

async def upgrade_helm_chart() -> RunOutput:
result = await run(
"helm secrets upgrade --create-namespace"
f" --history-max {MAX_HELM_HISTORY}"
Expand All @@ -164,32 +209,15 @@ async def update_app_deployment(self, app: App) -> UpdateAppResult | None:
f" -f {cfg.name}"
f" --namespace={app.namespace}"
f" {app.name}"
f" {chart_folder_path}",
f" {app.chart.helm_chart} {chart_version_arguments}",
suppress_errors=True,
)
elif app.chart.type == "helm":
span.set_attribute("gitops.chart.type", "helm")
with tempfile.NamedTemporaryFile(suffix=".yml") as cfg:
cfg.write(json.dumps(app.values).encode())
cfg.flush()
os.fsync(cfg.fileno())
chart_version_arguments = f" --version={app.chart.version}" if app.chart.version else ""
with tracer.start_as_current_span("helm_repo_add"):
await run(f"helm repo add {app.chart.helm_repo} {app.chart.helm_repo_url}")
return result

with tracer.start_as_current_span("helm_upgrade"):
result = await run(
"helm secrets upgrade --create-namespace"
f" --history-max {MAX_HELM_HISTORY}"
" --install"
" --timeout=600s"
f"{' --set skip_migrations=true' if self.skip_migrations else ''}"
f" -f {cfg.name}"
f" --namespace={app.namespace}"
f" {app.name}"
f" {app.chart.helm_chart} {chart_version_arguments}",
suppress_errors=True,
)
result = await upgrade_helm_chart()
if result["exit_code"] != 0 and "is in progress" in result["output"]:
await self.rollback_deployment(app)
result = await upgrade_helm_chart()
else:
logger.warning("Local is not implemented yet")
return None
Expand Down
Loading