78 lines
3.1 KiB
YAML
78 lines
3.1 KiB
YAML
apiVersion: operator.victoriametrics.com/v1beta1
|
|
kind: VMRule
|
|
metadata:
|
|
name: backup-alerts
|
|
namespace: observability
|
|
spec:
|
|
groups:
|
|
- name: backup-schedule-staleness
|
|
rules:
|
|
- alert: BackupCronJobNotScheduled
|
|
expr: |
|
|
time() - kube_cronjob_status_last_schedule_time{cronjob=~"forgejo-s3-backup|secrets-backup", namespace="gitea"}
|
|
> 26 * 3600
|
|
for: 5m
|
|
labels:
|
|
severity: critical
|
|
cronjob: "{{ $labels.cronjob }}"
|
|
annotations:
|
|
value: "{{ $value | humanizeDuration }}"
|
|
description: >-
|
|
CronJob {{ $labels.namespace }}/{{ $labels.cronjob }} has not been
|
|
scheduled for over 26 hours in cluster {{ $labels.cluster_environment }}.
|
|
Last schedule was {{ $value | humanizeDuration }} ago.
|
|
summary: "Backup CronJob {{ $labels.cronjob }} is stale"
|
|
|
|
- alert: BackupCronJobNeverScheduled
|
|
expr: |
|
|
kube_cronjob_status_last_schedule_time{cronjob=~"forgejo-s3-backup|secrets-backup", namespace="gitea"}
|
|
== 0
|
|
for: 30m
|
|
labels:
|
|
severity: critical
|
|
cronjob: "{{ $labels.cronjob }}"
|
|
annotations:
|
|
description: >-
|
|
CronJob {{ $labels.namespace }}/{{ $labels.cronjob }} has never been
|
|
scheduled in cluster {{ $labels.cluster_environment }}.
|
|
summary: "Backup CronJob {{ $labels.cronjob }} never ran"
|
|
|
|
- name: backup-job-failures
|
|
rules:
|
|
- alert: BackupJobFailed
|
|
expr: |
|
|
max by(cluster_environment, namespace, job_name) (
|
|
kube_job_status_failed{job_name=~"forgejo-s3-backup-.*|secrets-backup-.*", namespace="gitea"}
|
|
) > 0
|
|
for: 30s
|
|
labels:
|
|
severity: critical
|
|
job_name: "{{ $labels.job_name }}"
|
|
annotations:
|
|
value: "{{ $value }}"
|
|
description: >-
|
|
Backup job {{ $labels.namespace }}/{{ $labels.job_name }} has
|
|
{{ $value }} failed pod(s) in cluster {{ $labels.cluster_environment }}.
|
|
summary: "Backup job {{ $labels.job_name }} failed"
|
|
|
|
- name: backup-job-duration
|
|
rules:
|
|
- alert: BackupJobTooSlow
|
|
expr: |
|
|
(
|
|
time() - kube_job_status_start_time{job_name=~"forgejo-s3-backup-.*|secrets-backup-.*", namespace="gitea"}
|
|
) > 300
|
|
and
|
|
kube_job_status_active{job_name=~"forgejo-s3-backup-.*|secrets-backup-.*", namespace="gitea"} > 0
|
|
for: 1m
|
|
labels:
|
|
severity: major
|
|
job_name: "{{ $labels.job_name }}"
|
|
annotations:
|
|
value: "{{ $value | humanizeDuration }}"
|
|
description: >-
|
|
Backup job {{ $labels.namespace }}/{{ $labels.job_name }} has been
|
|
running for {{ $value | humanizeDuration }} (threshold: 5m)
|
|
in cluster {{ $labels.cluster_environment }}. This may indicate a
|
|
hung process or connectivity issue.
|
|
summary: "Backup job {{ $labels.job_name }} running too long"
|