From da0ccbd1b54bd847ea2e5a933658dbc566a33dfd Mon Sep 17 00:00:00 2001
From: Martin McCaffery <martin.mccaffery@think-ahead.tech>
Date: Mon, 1 Jun 2026 16:47:31 +0100
Subject: [PATCH] fix(observability): enable ArgoCD/CoreDNS scraping, add
 cluster label, fix node dashboard

---
 .../stacks/core/argocd/values.yaml            | 24 +++++++++++++++++++
 .../manifests/argocd-scrape.yaml              | 13 ++++++++++
 .../manifests/coredns-scrape.yaml             | 14 +++++++++++
 .../victoria-k8s-stack/values.yaml            | 11 +++++----
 4 files changed, 58 insertions(+), 4 deletions(-)
 create mode 100644 otc/observability.buildth.ing/stacks/observability/victoria-k8s-stack/manifests/argocd-scrape.yaml
 create mode 100644 otc/observability.buildth.ing/stacks/observability/victoria-k8s-stack/manifests/coredns-scrape.yaml

diff --git a/otc/observability.buildth.ing/stacks/core/argocd/values.yaml b/otc/observability.buildth.ing/stacks/core/argocd/values.yaml
index 04b0f2b..182d8ca 100644
--- a/otc/observability.buildth.ing/stacks/core/argocd/values.yaml
+++ b/otc/observability.buildth.ing/stacks/core/argocd/values.yaml
@@ -35,6 +35,30 @@ configs:
   tls:
     certificates:
 
+controller:
+  metrics:
+    enabled: true
+    serviceMonitor:
+      enabled: false
+
+server:
+  metrics:
+    enabled: true
+    serviceMonitor:
+      enabled: false
+
+repoServer:
+  metrics:
+    enabled: true
+    serviceMonitor:
+      enabled: false
+
+applicationSet:
+  metrics:
+    enabled: true
+    serviceMonitor:
+      enabled: false
+
 notifications:
   enabled: false
 
diff --git a/otc/observability.buildth.ing/stacks/observability/victoria-k8s-stack/manifests/argocd-scrape.yaml b/otc/observability.buildth.ing/stacks/observability/victoria-k8s-stack/manifests/argocd-scrape.yaml
new file mode 100644
index 0000000..d72e88d
--- /dev/null
+++ b/otc/observability.buildth.ing/stacks/observability/victoria-k8s-stack/manifests/argocd-scrape.yaml
@@ -0,0 +1,13 @@
+apiVersion: operator.victoriametrics.com/v1beta1
+kind: VMServiceScrape
+metadata:
+  name: argocd
+spec:
+  namespaceSelector:
+    matchNames:
+      - argocd
+  selector:
+    matchLabels:
+      app.kubernetes.io/part-of: argocd
+  endpoints:
+    - port: metrics
diff --git a/otc/observability.buildth.ing/stacks/observability/victoria-k8s-stack/manifests/coredns-scrape.yaml b/otc/observability.buildth.ing/stacks/observability/victoria-k8s-stack/manifests/coredns-scrape.yaml
new file mode 100644
index 0000000..c7e379e
--- /dev/null
+++ b/otc/observability.buildth.ing/stacks/observability/victoria-k8s-stack/manifests/coredns-scrape.yaml
@@ -0,0 +1,14 @@
+apiVersion: operator.victoriametrics.com/v1beta1
+kind: VMPodScrape
+metadata:
+  name: coredns
+spec:
+  namespaceSelector:
+    matchNames:
+      - kube-system
+  selector:
+    matchLabels:
+      k8s-app: coredns
+  podMetricsEndpoints:
+    - targetPort: 9153
+      path: /metrics
diff --git a/otc/observability.buildth.ing/stacks/observability/victoria-k8s-stack/values.yaml b/otc/observability.buildth.ing/stacks/observability/victoria-k8s-stack/values.yaml
index cdb96a9..9b6aba9 100644
--- a/otc/observability.buildth.ing/stacks/observability/victoria-k8s-stack/values.yaml
+++ b/otc/observability.buildth.ing/stacks/observability/victoria-k8s-stack/values.yaml
@@ -711,10 +711,8 @@ vmagent:
     port: "8429"
     selectAllByDefault: true
     scrapeInterval: 20s
-    externalLabels: {}
-      # For multi-cluster setups it is useful to use "cluster" label to identify the metrics source.
-      # For example:
-      # cluster: cluster-name
+    externalLabels:
+      cluster: observability
     extraArgs:
       promscrape.streamParse: "true"
       # Do not store original labels in vmagent's memory by default. This reduces the amount of memory used by vmagent
@@ -921,6 +919,11 @@ prometheus-node-exporter:
             - action: drop
               source_labels: [mountpoint]
               regex: "/var/lib/kubelet/pods.+"
+            - action: replace
+              source_labels: [__name__, instance]
+              regex: "node_uname_info;([^:]+):.+"
+              target_label: nodename
+              replacement: "$1"
 # -- kube-state-metrics dependency chart configuration. For possible values check [here](https://github.com/prometheus-community/helm-charts/blob/main/charts/kube-state-metrics/values.yaml)
 kube-state-metrics:
   enabled: true