Skip to content

Commit e0a1a8d

Browse files
dannyzakendannyzaken
authored
Merge pull request #1678 from dannyzaken/danny-fixes
fixes to db capacity alerts
2 parents 01497d5 + 9d98e31 commit e0a1a8d

File tree

2 files changed

+65
-121
lines changed

2 files changed

+65
-121
lines changed

‎deploy/internal/prometheus-rules.yaml‎

Lines changed: 32 additions & 60 deletions
Original file line numberDiff line numberDiff line change
@@ -247,63 +247,35 @@ spec:
247247
severity: critical
248248
- name: noobaa-db-alert.rules
249249
rules:
250-
- alert: NooBaaDatabaseReachingCapacity
251-
annotations:
252-
description: The NooBaa database on pod noobaa-db-pg-cluster-1 is using 80% of its PVC requested size.
253-
message: NooBaa NooBaa database on pod noobaa-db-pg-cluster-1 is using 80% of its PVC capacity.
254-
severity_level: warning
255-
storage_type: NooBaa
256-
expr: |
257-
(
258-
cnpg_pg_database_size_bytes{datname="nbcore", namespace="openshift-storage", pod="noobaa-db-pg-cluster-1"}
259-
/on(namespace) group_left(persistentvolumeclaim)
260-
kube_persistentvolumeclaim_resource_requests_storage_bytes{namespace="openshift-storage", persistentvolumeclaim="noobaa-db-pg-cluster-1"}
261-
) * 100 > 80
262-
for: 5m
263-
labels:
264-
severity: warning
265-
- alert: NooBaaDatabaseReachingCapacity
266-
annotations:
267-
description: The NooBaa database on pod noobaa-db-pg-cluster-2 is using 80% of its PVC requested size.
268-
message: NooBaa NooBaa database on pod noobaa-db-pg-cluster-2 is using 80% of its PVC capacity.
269-
severity_level: warning
270-
storage_type: NooBaa
271-
expr: |
272-
(
273-
cnpg_pg_database_size_bytes{datname="nbcore", namespace="openshift-storage", pod="noobaa-db-pg-cluster-2"}
274-
/on(namespace) group_left(persistentvolumeclaim)
275-
kube_persistentvolumeclaim_resource_requests_storage_bytes{namespace="openshift-storage", persistentvolumeclaim="noobaa-db-pg-cluster-2"}
276-
) * 100 > 80
277-
for: 5m
278-
labels:
279-
severity: warning
280-
- alert: NooBaaDatabaseStorageFull
281-
annotations:
282-
description: The NooBaa database on pod noobaa-db-pg-cluster-1 is using over 90% of its PVC requested size. Increase the DB size as soon as possible.
283-
message: NooBaa NooBaa database on pod noobaa-db-pg-cluster-1 is using over 90% of its PVC capacity.
284-
severity_level: critical
285-
storage_type: NooBaa
286-
expr: |
287-
(
288-
cnpg_pg_database_size_bytes{datname="nbcore", namespace="openshift-storage", pod="noobaa-db-pg-cluster-1"}
289-
/on(namespace) group_left(persistentvolumeclaim)
290-
kube_persistentvolumeclaim_resource_requests_storage_bytes{namespace="openshift-storage", persistentvolumeclaim="noobaa-db-pg-cluster-1"}
291-
) * 100 > 90
292-
for: 5m
293-
labels:
294-
severity: critical
295-
- alert: NooBaaDatabaseStorageFull
296-
annotations:
297-
description: The NooBaa database on pod noobaa-db-pg-cluster-2 is using over 90% of its PVC requested size. Increase the DB size as soon as possible.
298-
message: NooBaa NooBaa database on pod noobaa-db-pg-cluster-2 is using over 90% of its PVC capacity.
299-
severity_level: critical
300-
storage_type: NooBaa
301-
expr: |
302-
(
303-
cnpg_pg_database_size_bytes{datname="nbcore", namespace="openshift-storage", pod="noobaa-db-pg-cluster-2"}
304-
/on(namespace) group_left(persistentvolumeclaim)
305-
kube_persistentvolumeclaim_resource_requests_storage_bytes{namespace="openshift-storage", persistentvolumeclaim="noobaa-db-pg-cluster-2"}
306-
) * 100 > 90
307-
for: 5m
308-
labels:
309-
severity: critical
250+
- alert: NooBaaDatabaseReachingCapacity
251+
annotations:
252+
description: The NooBaa database on pod {{ $labels.pod }} has reached 80% of its PVC capacity.
253+
message: The NooBaa database on pod {{ $labels.pod }} is consuming 80% of its PVC capacity. Plan to increase the PVC size soon to prevent service impact.
254+
severity_level: warning
255+
storage_type: NooBaa
256+
expr: |
257+
((sum by (pod) (cnpg_collector_pg_wal{value="size"})
258+
+ sum by (pod) (cnpg_pg_database_size_bytes{datname="nbcore"}))
259+
/
260+
sum by (pod) (
261+
label_replace(kube_persistentvolumeclaim_resource_requests_storage_bytes{namespace="openshift-storage"}, "pod", "$1", "persistentvolumeclaim", "(.*)"
262+
))) * 100 > 80
263+
for: 5m
264+
labels:
265+
severity: warning
266+
- alert: NooBaaDatabaseStorageFull
267+
annotations:
268+
description: The NooBaa database on pod {{ $labels.pod }} has exceeded 90% of its PVC capacity. Immediate action is required
269+
message: The NooBaa database on pod {{ $labels.pod }} has exceeded 90% of its PVC capacity. Expand the PVC size now to avoid imminent service disruption.
270+
severity_level: critical
271+
storage_type: NooBaa
272+
expr: |
273+
((sum by (pod) (cnpg_collector_pg_wal{value="size"})
274+
+ sum by (pod) (cnpg_pg_database_size_bytes{datname="nbcore"}))
275+
/
276+
sum by (pod) (
277+
label_replace(kube_persistentvolumeclaim_resource_requests_storage_bytes{namespace="openshift-storage"}, "pod", "$1", "persistentvolumeclaim", "(.*)"
278+
))) * 100 > 90
279+
for: 1m
280+
labels:
281+
severity: critical

‎pkg/bundle/deploy.go‎

Lines changed: 33 additions & 61 deletions
Original file line numberDiff line numberDiff line change
@@ -4537,7 +4537,7 @@ spec:
45374537
claimName: noobaa-pv-claim
45384538
`
45394539

4540-
const Sha256_deploy_internal_prometheus_rules_yaml = "3d136d9c9891c9d3bdfdab4d8b5104ab31329b0740744a2e0bbcb34fa26bebf2"
4540+
const Sha256_deploy_internal_prometheus_rules_yaml = "9dba8cfe7b655d3467b091531c95e6d34e8bd179f36ece6eaf3cff8ef73df23d"
45414541

45424542
const File_deploy_internal_prometheus_rules_yaml = `apiVersion: monitoring.coreos.com/v1
45434543
kind: PrometheusRule
@@ -4788,66 +4788,38 @@ spec:
47884788
severity: critical
47894789
- name: noobaa-db-alert.rules
47904790
rules:
4791-
- alert: NooBaaDatabaseReachingCapacity
4792-
annotations:
4793-
description: The NooBaa database on pod noobaa-db-pg-cluster-1 is using 80% of its PVC requested size.
4794-
message: NooBaa NooBaa database on pod noobaa-db-pg-cluster-1 is using 80% of its PVC capacity.
4795-
severity_level: warning
4796-
storage_type: NooBaa
4797-
expr: |
4798-
(
4799-
cnpg_pg_database_size_bytes{datname="nbcore", namespace="openshift-storage", pod="noobaa-db-pg-cluster-1"}
4800-
/on(namespace) group_left(persistentvolumeclaim)
4801-
kube_persistentvolumeclaim_resource_requests_storage_bytes{namespace="openshift-storage", persistentvolumeclaim="noobaa-db-pg-cluster-1"}
4802-
) * 100 > 80
4803-
for: 5m
4804-
labels:
4805-
severity: warning
4806-
- alert: NooBaaDatabaseReachingCapacity
4807-
annotations:
4808-
description: The NooBaa database on pod noobaa-db-pg-cluster-2 is using 80% of its PVC requested size.
4809-
message: NooBaa NooBaa database on pod noobaa-db-pg-cluster-2 is using 80% of its PVC capacity.
4810-
severity_level: warning
4811-
storage_type: NooBaa
4812-
expr: |
4813-
(
4814-
cnpg_pg_database_size_bytes{datname="nbcore", namespace="openshift-storage", pod="noobaa-db-pg-cluster-2"}
4815-
/on(namespace) group_left(persistentvolumeclaim)
4816-
kube_persistentvolumeclaim_resource_requests_storage_bytes{namespace="openshift-storage", persistentvolumeclaim="noobaa-db-pg-cluster-2"}
4817-
) * 100 > 80
4818-
for: 5m
4819-
labels:
4820-
severity: warning
4821-
- alert: NooBaaDatabaseStorageFull
4822-
annotations:
4823-
description: The NooBaa database on pod noobaa-db-pg-cluster-1 is using over 90% of its PVC requested size. Increase the DB size as soon as possible.
4824-
message: NooBaa NooBaa database on pod noobaa-db-pg-cluster-1 is using over 90% of its PVC capacity.
4825-
severity_level: critical
4826-
storage_type: NooBaa
4827-
expr: |
4828-
(
4829-
cnpg_pg_database_size_bytes{datname="nbcore", namespace="openshift-storage", pod="noobaa-db-pg-cluster-1"}
4830-
/on(namespace) group_left(persistentvolumeclaim)
4831-
kube_persistentvolumeclaim_resource_requests_storage_bytes{namespace="openshift-storage", persistentvolumeclaim="noobaa-db-pg-cluster-1"}
4832-
) * 100 > 90
4833-
for: 5m
4834-
labels:
4835-
severity: critical
4836-
- alert: NooBaaDatabaseStorageFull
4837-
annotations:
4838-
description: The NooBaa database on pod noobaa-db-pg-cluster-2 is using over 90% of its PVC requested size. Increase the DB size as soon as possible.
4839-
message: NooBaa NooBaa database on pod noobaa-db-pg-cluster-2 is using over 90% of its PVC capacity.
4840-
severity_level: critical
4841-
storage_type: NooBaa
4842-
expr: |
4843-
(
4844-
cnpg_pg_database_size_bytes{datname="nbcore", namespace="openshift-storage", pod="noobaa-db-pg-cluster-2"}
4845-
/on(namespace) group_left(persistentvolumeclaim)
4846-
kube_persistentvolumeclaim_resource_requests_storage_bytes{namespace="openshift-storage", persistentvolumeclaim="noobaa-db-pg-cluster-2"}
4847-
) * 100 > 90
4848-
for: 5m
4849-
labels:
4850-
severity: critical
4791+
- alert: NooBaaDatabaseReachingCapacity
4792+
annotations:
4793+
description: The NooBaa database on pod {{ $labels.pod }} has reached 80% of its PVC capacity.
4794+
message: The NooBaa database on pod {{ $labels.pod }} is consuming 80% of its PVC capacity. Plan to increase the PVC size soon to prevent service impact.
4795+
severity_level: warning
4796+
storage_type: NooBaa
4797+
expr: |
4798+
((sum by (pod) (cnpg_collector_pg_wal{value="size"})
4799+
+ sum by (pod) (cnpg_pg_database_size_bytes{datname="nbcore"}))
4800+
/
4801+
sum by (pod) (
4802+
label_replace(kube_persistentvolumeclaim_resource_requests_storage_bytes{namespace="openshift-storage"}, "pod", "$1", "persistentvolumeclaim", "(.*)"
4803+
))) * 100 > 80
4804+
for: 5m
4805+
labels:
4806+
severity: warning
4807+
- alert: NooBaaDatabaseStorageFull
4808+
annotations:
4809+
description: The NooBaa database on pod {{ $labels.pod }} has exceeded 90% of its PVC capacity. Immediate action is required
4810+
message: The NooBaa database on pod {{ $labels.pod }} has exceeded 90% of its PVC capacity. Expand the PVC size now to avoid imminent service disruption.
4811+
severity_level: critical
4812+
storage_type: NooBaa
4813+
expr: |
4814+
((sum by (pod) (cnpg_collector_pg_wal{value="size"})
4815+
+ sum by (pod) (cnpg_pg_database_size_bytes{datname="nbcore"}))
4816+
/
4817+
sum by (pod) (
4818+
label_replace(kube_persistentvolumeclaim_resource_requests_storage_bytes{namespace="openshift-storage"}, "pod", "$1", "persistentvolumeclaim", "(.*)"
4819+
))) * 100 > 90
4820+
for: 1m
4821+
labels:
4822+
severity: critical
48514823
`
48524824

48534825
const Sha256_deploy_internal_pvc_agent_yaml = "c76fd98867e2e098204377899568a6e1e60062ece903c7bcbeb3444193ec13f8"

0 commit comments

Comments
 (0)