K3S-HOME/observability
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: update dashboards and OTel collector for proper metrics/logs

Browse Source 
- certmanager.json: use Thanos datasource, fix variable regex
- argocd.json: use Thanos datasource via $datasource variable
- logs.json: update to use OTel labels (k8s_namespace_name, k8s_container_name)
- collector.yaml: add loki.resource.labels hint for proper Loki label mapping

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
Mayne0213
2026-01-10 03:36:37 +09:00
parent de81ca68c9
commit 518b5c31ef
4 changed files with 109 additions and 96 deletions
Download Patch File Download Diff File Expand all files Collapse all files

108
grafana/dashboards/argocd.json
View File

@@ -67,7 +67,7 @@
"panels": [
{
"collapsed": false,
"datasource": "Prometheus",
"datasource": "$datasource",
"fieldConfig": {
"defaults": {},
"overrides": []
@@ -84,7 +84,7 @@
"type": "row"
},
{
"datasource": "Prometheus",
"datasource": "$datasource",
"gridPos": {
"h": 4,
"w": 2,
@@ -103,7 +103,7 @@
},
{
"cacheTimeout": null,
"datasource": "Prometheus",
"datasource": "$datasource",
"fieldConfig": {
"defaults": {
"color": {
@@ -166,7 +166,7 @@
},
{
"cacheTimeout": null,
"datasource": "Prometheus",
"datasource": "$datasource",
"fieldConfig": {
"defaults": {
"color": {
@@ -232,7 +232,7 @@
},
{
"cacheTimeout": null,
"datasource": "Prometheus",
"datasource": "$datasource",
"fieldConfig": {
"defaults": {
"color": {
@@ -300,7 +300,7 @@
},
{
"cacheTimeout": null,
"datasource": "Prometheus",
"datasource": "$datasource",
"fieldConfig": {
"defaults": {
"color": {
@@ -366,7 +366,7 @@
},
{
"cacheTimeout": null,
"datasource": "Prometheus",
"datasource": "$datasource",
"fieldConfig": {
"defaults": {
"color": {
@@ -444,7 +444,7 @@
"cacheTimeout": null,
"dashLength": 10,
"dashes": false,
"datasource": "Prometheus",
"datasource": "$datasource",
"decimals": null,
"fieldConfig": {
"defaults": {
@@ -548,7 +548,7 @@
},
{
"collapsed": true,
"datasource": "Prometheus",
"datasource": "$datasource",
"fieldConfig": {
"defaults": {},
"overrides": []
@@ -574,7 +574,7 @@
"cacheTimeout": null,
"dashLength": 10,
"dashes": false,
"datasource": "Prometheus",
"datasource": "$datasource",
"decimals": null,
"fill": 1,
"fillGradient": 0,
@@ -685,7 +685,7 @@
"cacheTimeout": null,
"dashLength": 10,
"dashes": false,
"datasource": "Prometheus",
"datasource": "$datasource",
"decimals": null,
"fill": 1,
"fillGradient": 0,
@@ -787,7 +787,7 @@
},
{
"collapsed": true,
"datasource": "Prometheus",
"datasource": "$datasource",
"fieldConfig": {
"defaults": {},
"overrides": []
@@ -805,7 +805,7 @@
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "Prometheus",
"datasource": "$datasource",
"decimals": null,
"fill": 1,
"fillGradient": 0,
@@ -906,7 +906,7 @@
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "Prometheus",
"datasource": "$datasource",
"decimals": null,
"fill": 1,
"fillGradient": 0,
@@ -1006,7 +1006,7 @@
},
{
"collapsed": true,
"datasource": "Prometheus",
"datasource": "$datasource",
"fieldConfig": {
"defaults": {},
"overrides": []
@@ -1024,7 +1024,7 @@
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "Prometheus",
"datasource": "$datasource",
"fill": 1,
"fillGradient": 0,
"gridPos": {
@@ -1128,7 +1128,7 @@
"mode": "spectrum"
},
"dataFormat": "tsbuckets",
"datasource": "Prometheus",
"datasource": "$datasource",
"gridPos": {
"h": 7,
"w": 24,
@@ -1187,7 +1187,7 @@
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "Prometheus",
"datasource": "$datasource",
"fill": 1,
"fillGradient": 0,
"gridPos": {
@@ -1286,7 +1286,7 @@
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "Prometheus",
"datasource": "$datasource",
"fill": 1,
"fillGradient": 0,
"gridPos": {
@@ -1380,7 +1380,7 @@
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "Prometheus",
"datasource": "$datasource",
"decimals": null,
"fill": 1,
"fillGradient": 0,
@@ -1476,7 +1476,7 @@
},
{
"collapsed": true,
"datasource": "Prometheus",
"datasource": "$datasource",
"fieldConfig": {
"defaults": {},
"overrides": []
@@ -1494,7 +1494,7 @@
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "Prometheus",
"datasource": "$datasource",
"fill": 1,
"fillGradient": 0,
"gridPos": {
@@ -1589,7 +1589,7 @@
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "Prometheus",
"datasource": "$datasource",
"fill": 1,
"fillGradient": 0,
"gridPos": {
@@ -1687,7 +1687,7 @@
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "Prometheus",
"datasource": "$datasource",
"fill": 1,
"fillGradient": 0,
"gridPos": {
@@ -1785,7 +1785,7 @@
},
{
"collapsed": true,
"datasource": "Prometheus",
"datasource": "$datasource",
"fieldConfig": {
"defaults": {},
"overrides": []
@@ -1803,7 +1803,7 @@
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "Prometheus",
"datasource": "$datasource",
"fill": 1,
"fillGradient": 0,
"gridPos": {
@@ -1897,7 +1897,7 @@
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "Prometheus",
"datasource": "$datasource",
"fill": 1,
"fillGradient": 0,
"gridPos": {
@@ -1992,7 +1992,7 @@
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "Prometheus",
"datasource": "$datasource",
"fill": 1,
"fillGradient": 0,
"gridPos": {
@@ -2087,7 +2087,7 @@
},
{
"collapsed": true,
"datasource": "Prometheus",
"datasource": "$datasource",
"fieldConfig": {
"defaults": {},
"overrides": []
@@ -2105,7 +2105,7 @@
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "Prometheus",
"datasource": "$datasource",
"fill": 1,
"fillGradient": 0,
"gridPos": {
@@ -2195,7 +2195,7 @@
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "Prometheus",
"datasource": "$datasource",
"fill": 1,
"fillGradient": 0,
"gridPos": {
@@ -2293,7 +2293,7 @@
"mode": "spectrum"
},
"dataFormat": "tsbuckets",
"datasource": "Prometheus",
"datasource": "$datasource",
"gridPos": {
"h": 8,
"w": 12,
@@ -2357,7 +2357,7 @@
"mode": "spectrum"
},
"dataFormat": "tsbuckets",
"datasource": "Prometheus",
"datasource": "$datasource",
"gridPos": {
"h": 8,
"w": 12,
@@ -2413,7 +2413,7 @@
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "Prometheus",
"datasource": "$datasource",
"fill": 1,
"fillGradient": 0,
"gridPos": {
@@ -2504,7 +2504,7 @@
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "Prometheus",
"datasource": "$datasource",
"fill": 1,
"fillGradient": 0,
"gridPos": {
@@ -2596,7 +2596,7 @@
},
{
"collapsed": true,
"datasource": "Prometheus",
"datasource": "$datasource",
"fieldConfig": {
"defaults": {},
"overrides": []
@@ -2614,7 +2614,7 @@
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "Prometheus",
"datasource": "$datasource",
"fill": 1,
"gridPos": {
"h": 8,
@@ -2700,7 +2700,7 @@
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "Prometheus",
"datasource": "$datasource",
"fill": 1,
"gridPos": {
"h": 9,
@@ -2786,7 +2786,7 @@
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "Prometheus",
"datasource": "$datasource",
"fill": 1,
"gridPos": {
"h": 9,
@@ -2887,7 +2887,7 @@
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "Prometheus",
"datasource": "$datasource",
"decimals": null,
"fill": 1,
"gridPos": {
@@ -2980,7 +2980,7 @@
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "Prometheus",
"datasource": "$datasource",
"fill": 1,
"gridPos": {
"h": 9,
@@ -3070,7 +3070,7 @@
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "Prometheus",
"datasource": "$datasource",
"fill": 1,
"gridPos": {
"h": 9,
@@ -3160,7 +3160,7 @@
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "Prometheus",
"datasource": "$datasource",
"fill": 1,
"gridPos": {
"h": 9,
@@ -3257,7 +3257,7 @@
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "Prometheus",
"datasource": "$datasource",
"fill": 1,
"gridPos": {
"h": 9,
@@ -3346,7 +3346,7 @@
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "Prometheus",
"datasource": "$datasource",
"fill": 1,
"gridPos": {
"h": 9,
@@ -3435,7 +3435,7 @@
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "Prometheus",
"datasource": "$datasource",
"fill": 1,
"gridPos": {
"h": 9,
@@ -3524,7 +3524,7 @@
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "Prometheus",
"datasource": "$datasource",
"fill": 1,
"gridPos": {
"h": 9,
@@ -3614,7 +3614,7 @@
},
{
"collapsed": true,
"datasource": "Prometheus",
"datasource": "$datasource",
"fieldConfig": {
"defaults": {},
"overrides": []
@@ -3632,7 +3632,7 @@
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "Prometheus",
"datasource": "$datasource",
"fill": 1,
"fillGradient": 0,
"gridPos": {
@@ -3727,8 +3727,8 @@
{
"current": {
"selected": false,
"text": "Prometheus",
"value": "Prometheus"
"text": "Thanos",
"value": "Thanos"
},
"description": null,
"error": null,
@@ -3740,14 +3740,14 @@
"options": [],
"query": "prometheus",
"refresh": 1,
"regex": "",
"regex": "/Thanos/",
"skipUrlSync": false,
"type": "datasource"
},
{
"allValue": ".*",
"current": {},
"datasource": "Prometheus",
"datasource": "$datasource",
"definition": "label_values(kube_pod_info, namespace)",
"description": null,
"error": null,
@@ -3874,7 +3874,7 @@
{
"allValue": ".*",
"current": {},
"datasource": "Prometheus",
"datasource": "$datasource",
"definition": "label_values(argocd_cluster_info, server)",
"description": null,
"error": null,

36
grafana/dashboards/certmanager.json
View File

@@ -68,7 +68,7 @@
},
{
"columns": [],
"datasource": "Prometheus",
"datasource": "$Datasource",
"fontSize": "100%",
"gridPos": {
"h": 8,
@@ -148,7 +148,7 @@
"cacheTimeout": null,
"dashLength": 10,
"dashes": false,
"datasource": "Prometheus",
"datasource": "$Datasource",
"fill": 0,
"fillGradient": 0,
"gridPos": {
@@ -240,7 +240,7 @@
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "Prometheus",
"datasource": "$Datasource",
"fill": 0,
"fillGradient": 0,
"gridPos": {
@@ -344,7 +344,7 @@
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "Prometheus",
"datasource": "$Datasource",
"fill": 1,
"fillGradient": 0,
"gridPos": {
@@ -434,7 +434,7 @@
"rgba(237, 129, 40, 0.89)",
"#d44a3a"
],
"datasource": "Prometheus",
"datasource": "$Datasource",
"format": "none",
"gauge": {
"maxValue": 100,
@@ -514,7 +514,7 @@
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "Prometheus",
"datasource": "$Datasource",
"fill": 1,
"fillGradient": 0,
"gridPos": {
@@ -600,7 +600,7 @@
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "Prometheus",
"datasource": "$Datasource",
"fill": 1,
"fillGradient": 0,
"gridPos": {
@@ -686,7 +686,7 @@
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "Prometheus",
"datasource": "$Datasource",
"fill": 1,
"fillGradient": 0,
"gridPos": {
@@ -772,7 +772,7 @@
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "Prometheus",
"datasource": "$Datasource",
"fill": 1,
"fillGradient": 0,
"gridPos": {
@@ -858,7 +858,7 @@
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "Prometheus",
"datasource": "$Datasource",
"fill": 1,
"fillGradient": 0,
"gridPos": {
@@ -944,7 +944,7 @@
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "Prometheus",
"datasource": "$Datasource",
"fill": 1,
"fillGradient": 0,
"gridPos": {
@@ -1030,7 +1030,7 @@
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "Prometheus",
"datasource": "$Datasource",
"fill": 1,
"fillGradient": 0,
"gridPos": {
@@ -1116,7 +1116,7 @@
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "Prometheus",
"datasource": "$Datasource",
"fill": 1,
"fillGradient": 0,
"gridPos": {
@@ -1208,8 +1208,8 @@
"list": [
{
"current": {
"text": "k8s-qa",
"value": "k8s-qa"
"text": "Thanos",
"value": "Thanos"
},
"hide": 0,
"includeAll": false,
@@ -1219,14 +1219,14 @@
"options": [],
"query": "prometheus",
"refresh": 1,
"regex": "/k8s.*/",
"regex": "/Thanos/",
"skipUrlSync": false,
"type": "datasource"
},
{
"allValue": ".*",
"current": {},
"datasource": "Prometheus",
"datasource": "$Datasource",
"definition": "label_values(exported_namespace)",
"hide": 0,
"includeAll": true,
@@ -1248,7 +1248,7 @@
{
"allValue": ".*",
"current": {},
"datasource": "Prometheus",
"datasource": "$Datasource",
"definition": "label_values(certmanager_certificate_expiration_timestamp_seconds{exported_namespace=~\"$Namespace\"},name) ",
"hide": 0,
"includeAll": true,

52
grafana/dashboards/logs.json
View File

@@ -38,7 +38,6 @@
"annotations": {
"list": [
{
"$$hashKey": "object:75",
"builtIn": 1,
"datasource": "-- Grafana --",
"enable": true,
@@ -50,14 +49,12 @@
]
},
"description": "Log Viewer Dashboard for Loki",
"editable": false,
"editable": true,
"gnetId": 13639,
"graphTooltip": 0,
"id": 7,
"iteration": 1608932746420,
"links": [
{
"$$hashKey": "object:59",
"icon": "bolt",
"includeVars": true,
"keepTime": true,
@@ -65,10 +62,9 @@
"targetBlank": true,
"title": "View In Explore",
"type": "link",
"url": "/explore?orgId=1&left=[\"now-1h\",\"now\",\"Loki\",{\"expr\":\"{job=\\\"$app\\\"}\"},{\"ui\":[true,true,true,\"none\"]}]"
"url": "/explore?orgId=1&left=[\"now-1h\",\"now\",\"Loki\",{\"expr\":\"{k8s_namespace_name=\\\"$namespace\\\", k8s_container_name=\\\"$container\\\"}\"},{\"ui\":[true,true,true,\"none\"]}]"
},
{
"$$hashKey": "object:61",
"icon": "external link",
"tags": [],
"targetBlank": true,
@@ -124,7 +120,7 @@
"steppedLine": false,
"targets": [
{
"expr": "sum(count_over_time({job=\"$app\"} |= \"$search\" [$__interval]))",
"expr": "sum(count_over_time({k8s_namespace_name=\"$namespace\", k8s_container_name=~\"$container\"} |= \"$search\" [$__interval]))",
"legendFormat": "",
"refId": "A"
}
@@ -149,7 +145,6 @@
},
"yaxes": [
{
"$$hashKey": "object:168",
"format": "short",
"label": null,
"logBase": 1,
@@ -158,7 +153,6 @@
"show": false
},
{
"$$hashKey": "object:169",
"format": "short",
"label": null,
"logBase": 1,
@@ -196,7 +190,7 @@
},
"targets": [
{
"expr": "{job=\"$app\"} |= \"$search\" | logfmt",
"expr": "{k8s_namespace_name=\"$namespace\", k8s_container_name=~\"$container\"} |= \"$search\"",
"hide": false,
"legendFormat": "",
"refId": "A"
@@ -216,26 +210,38 @@
"templating": {
"list": [
{
"allValue": null,
"current": {},
"datasource": "Loki",
"definition": "label_values(job)",
"definition": "label_values(k8s_namespace_name)",
"hide": 0,
"includeAll": false,
"label": "App",
"label": "Namespace",
"multi": false,
"name": "app",
"name": "namespace",
"options": [],
"query": "label_values(job)",
"query": "label_values(k8s_namespace_name)",
"refresh": 1,
"regex": "",
"skipUrlSync": false,
"sort": 0,
"tagValuesQuery": "",
"tags": [],
"tagsQuery": "",
"type": "query",
"useTags": false
"sort": 1,
"type": "query"
},
{
"current": {},
"datasource": "Loki",
"definition": "label_values({k8s_namespace_name=\"$namespace\"}, k8s_container_name)",
"hide": 0,
"includeAll": true,
"label": "Container",
"multi": false,
"name": "container",
"options": [],
"query": "label_values({k8s_namespace_name=\"$namespace\"}, k8s_container_name)",
"refresh": 1,
"regex": "",
"skipUrlSync": false,
"sort": 1,
"type": "query"
},
{
"current": {
@@ -244,7 +250,7 @@
"value": ""
},
"hide": 0,
"label": "String Match",
"label": "Search",
"name": "search",
"options": [
{
@@ -280,5 +286,5 @@
"timezone": "",
"title": "Logs",
"uid": "sadlil-loki-apps-dashboard",
"version": 1
"version": 2
}

9
opentelemetry-collector/manifests/collector.yaml
View File

@@ -142,6 +142,13 @@ spec:
- type: move
from: attributes.log
to: body
# Loki label hints - tell Loki exporter which attributes to use as labels
- type: add
field: resource["loki.resource.labels"]
value: "k8s.namespace.name, k8s.pod.name, k8s.container.name, k8s.node.name"
- type: add
field: attributes["loki.attribute.labels"]
value: "log.iostream"
# Prometheus receiver - self metrics only
prometheus:
@@ -202,7 +209,7 @@ spec:
loki:
endpoint: http://loki.loki.svc.cluster.local:3100/loki/api/v1/push
default_labels_enabled:
exporter: true
exporter: false
level: true
debug: