From 0316eefa43c725484f31a0d6248f4a3a0175c737 Mon Sep 17 00:00:00 2001
From: Daniel Sy <Daniel.Sy@telekom.de>
Date: Fri, 19 Jun 2026 12:42:04 +0200
Subject: [PATCH] =?UTF-8?q?fix(observability):=20=F0=9F=90=9B=20disable=20?=
 =?UTF-8?q?false-positive=20control-plane=20alerts=20and=20fix=20empty=20c?=
 =?UTF-8?q?luster=5Fenvironment=20label?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Hub defaultRules groups kubernetesSystemControllerManager, kubeScheduler, and
kubernetesSystemScheduler used wrong key 'enabled: false' — chart expects 'create: false'.
This caused KubeControllerManagerDown/KubeSchedulerDown to fire as false positives
because OTC CCE managed k8s does not expose control plane for scraping.

Dev local vmagent had empty externalLabels, so backup-alert rules evaluated by local
vmalert had no cluster_environment label on kube_job_status_failed metrics. Added
cluster_environment=dev to match what the vm-client-stack vmagent adds for hub shipping.
---
 .../stacks/observability/victoria-k8s-stack/values.yaml     | 3 ++-
 .../stacks/observability/victoria-k8s-stack/values.yaml     | 6 +++---
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/otc/dev.t09.de/stacks/observability/victoria-k8s-stack/values.yaml b/otc/dev.t09.de/stacks/observability/victoria-k8s-stack/values.yaml
index 9751113..e7bffbc 100644
--- a/otc/dev.t09.de/stacks/observability/victoria-k8s-stack/values.yaml
+++ b/otc/dev.t09.de/stacks/observability/victoria-k8s-stack/values.yaml
@@ -708,7 +708,8 @@ vmagent:
     port: "8429"
     selectAllByDefault: true
     scrapeInterval: 20s
-    externalLabels: {}
+    externalLabels:
+      cluster_environment: "dev"
       # For multi-cluster setups it is useful to use "cluster" label to identify the metrics source.
       # For example:
       # cluster: cluster-name
diff --git a/otc/observability.buildth.ing/stacks/observability/victoria-k8s-stack/values.yaml b/otc/observability.buildth.ing/stacks/observability/victoria-k8s-stack/values.yaml
index 5bb9361..c535829 100644
--- a/otc/observability.buildth.ing/stacks/observability/victoria-k8s-stack/values.yaml
+++ b/otc/observability.buildth.ing/stacks/observability/victoria-k8s-stack/values.yaml
@@ -201,13 +201,13 @@ defaultRules:
       enabled: true
       rules: {}
     kubernetesSystemControllerManager:
-      enabled: false
+      create: false
       rules: {}
     kubeScheduler:
-      enabled: false
+      create: false
       rules: {}
     kubernetesSystemScheduler:
-      enabled: false
+      create: false
       rules: {}
     kubeStateMetrics:
       enabled: true