From f4e890b0d382266d34ab4311806a627dbb6dad65 Mon Sep 17 00:00:00 2001 From: William Chu Date: Fri, 14 Feb 2025 14:10:25 +1100 Subject: [PATCH 1/3] chore: bump gitops resources limits --- charts/gitops/templates/deployment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/gitops/templates/deployment.yml b/charts/gitops/templates/deployment.yml index 1cb1a69..fb48ae8 100644 --- a/charts/gitops/templates/deployment.yml +++ b/charts/gitops/templates/deployment.yml @@ -41,7 +41,7 @@ spec: requests: memory: "750Mi" limits: - memory: "1500Mi" + memory: "2500Mi" envFrom: - configMapRef: From fcd9680f6e94e5cb7a286c269aeaa1626c95a181 Mon Sep 17 00:00:00 2001 From: William Chu Date: Fri, 14 Feb 2025 14:10:42 +1100 Subject: [PATCH 2/3] fix: remove gitops resource limits --- charts/gitops/templates/deployment.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/charts/gitops/templates/deployment.yml b/charts/gitops/templates/deployment.yml index fb48ae8..cda4714 100644 --- a/charts/gitops/templates/deployment.yml +++ b/charts/gitops/templates/deployment.yml @@ -40,8 +40,6 @@ spec: resources: requests: memory: "750Mi" - limits: - memory: "2500Mi" envFrom: - configMapRef: From 762cdb388162c6b8645ec01f0d1c94c20d948b9f Mon Sep 17 00:00:00 2001 From: William Chu Date: Fri, 14 Feb 2025 14:45:20 +1100 Subject: [PATCH 3/3] fix(helm): if a roll back is required; rollback before upgrading --- gitops_server/workers/deployer/deploy.py | 76 ++++++++++++++++-------- 1 file changed, 52 insertions(+), 24 deletions(-) diff --git a/gitops_server/workers/deployer/deploy.py b/gitops_server/workers/deployer/deploy.py index 9db7aa1..361ca8d 100644 --- a/gitops_server/workers/deployer/deploy.py +++ b/gitops_server/workers/deployer/deploy.py @@ -11,7 +11,7 @@ from gitops.common.app import App from gitops_server import settings -from gitops_server.types import AppDefinitions, UpdateAppResult +from gitops_server.types import AppDefinitions, RunOutput, UpdateAppResult from gitops_server.utils import get_repo_name_from_url, github, run, slack from gitops_server.utils.git import temp_repo @@ -133,6 +133,17 @@ async def uninstall_app(self, app: App) -> UpdateAppResult: ) return update_result + async def rollback_deployment(self, app: App) -> None: + with tracer.start_as_current_span("rollback_deployment", attributes={"app": app.name}): + logger.warning( + "Rolling back %s deployment due to previous failed helm install", + app.name, + ) + await run( + f"helm rollback --namespace={app.namespace} {app.name}", + suppress_errors=True, + ) + async def update_app_deployment(self, app: App) -> UpdateAppResult | None: app.set_value("deployment.labels.gitops/deploy_id", self.deploy_id) app.set_value("deployment.labels.gitops/status", github.STATUSES.in_progress) @@ -155,6 +166,40 @@ async def update_app_deployment(self, app: App) -> UpdateAppResult | None: os.fsync(cfg.fileno()) with tracer.start_as_current_span("helm_upgrade"): + + async def upgrade_helm_git() -> RunOutput: + result = await run( + "helm secrets upgrade --create-namespace" + f" --history-max {MAX_HELM_HISTORY}" + " --install" + " --timeout=600s" + f"{' --set skip_migrations=true' if self.skip_migrations else ''}" + f" -f {cfg.name}" + f" --namespace={app.namespace}" + f" {app.name}" + f" {chart_folder_path}", + suppress_errors=True, + ) + return result + + result = await upgrade_helm_git() + if result["exit_code"] != 0 and "is in progress" in result["output"]: + await self.rollback_deployment(app) + result = await upgrade_helm_git() + + elif app.chart.type == "helm": + span.set_attribute("gitops.chart.type", "helm") + with tempfile.NamedTemporaryFile(suffix=".yml") as cfg: + cfg.write(json.dumps(app.values).encode()) + cfg.flush() + os.fsync(cfg.fileno()) + chart_version_arguments = f" --version={app.chart.version}" if app.chart.version else "" + with tracer.start_as_current_span("helm_repo_add"): + await run(f"helm repo add {app.chart.helm_repo} {app.chart.helm_repo_url}") + + with tracer.start_as_current_span("helm_upgrade"): + + async def upgrade_helm_chart() -> RunOutput: result = await run( "helm secrets upgrade --create-namespace" f" --history-max {MAX_HELM_HISTORY}" @@ -164,32 +209,15 @@ async def update_app_deployment(self, app: App) -> UpdateAppResult | None: f" -f {cfg.name}" f" --namespace={app.namespace}" f" {app.name}" - f" {chart_folder_path}", + f" {app.chart.helm_chart} {chart_version_arguments}", suppress_errors=True, ) - elif app.chart.type == "helm": - span.set_attribute("gitops.chart.type", "helm") - with tempfile.NamedTemporaryFile(suffix=".yml") as cfg: - cfg.write(json.dumps(app.values).encode()) - cfg.flush() - os.fsync(cfg.fileno()) - chart_version_arguments = f" --version={app.chart.version}" if app.chart.version else "" - with tracer.start_as_current_span("helm_repo_add"): - await run(f"helm repo add {app.chart.helm_repo} {app.chart.helm_repo_url}") + return result - with tracer.start_as_current_span("helm_upgrade"): - result = await run( - "helm secrets upgrade --create-namespace" - f" --history-max {MAX_HELM_HISTORY}" - " --install" - " --timeout=600s" - f"{' --set skip_migrations=true' if self.skip_migrations else ''}" - f" -f {cfg.name}" - f" --namespace={app.namespace}" - f" {app.name}" - f" {app.chart.helm_chart} {chart_version_arguments}", - suppress_errors=True, - ) + result = await upgrade_helm_chart() + if result["exit_code"] != 0 and "is in progress" in result["output"]: + await self.rollback_deployment(app) + result = await upgrade_helm_chart() else: logger.warning("Local is not implemented yet") return None