Mayne0213
9583be9b46
- to JSON and use sidecar ConfigMaps - Export 14 dashboards to JSON files - Use kustomize configMapGenerator for dashboard ConfigMaps - Enable Grafana sidecar to load dashboards from ConfigMaps - Keep Longhorn and Traefik Official from grafana.com
14040 lines
395 KiB
JSON
14040 lines
395 KiB
JSON
{
|
|
"annotations": {
|
|
"list": [
|
|
{
|
|
"$$hashKey": "object:7",
|
|
"builtIn": 1,
|
|
"datasource": {
|
|
"type": "datasource",
|
|
"uid": "grafana"
|
|
},
|
|
"enable": true,
|
|
"hide": true,
|
|
"iconColor": "rgba(0, 211, 255, 1)",
|
|
"name": "Annotations & Alerts",
|
|
"target": {
|
|
"limit": 100,
|
|
"matchAny": false,
|
|
"tags": [],
|
|
"type": "dashboard"
|
|
},
|
|
"type": "dashboard"
|
|
}
|
|
]
|
|
},
|
|
"description": "Loki Metrics Dashboard",
|
|
"editable": false,
|
|
"fiscalYearStartMonth": 0,
|
|
"gnetId": 17781,
|
|
"graphTooltip": 1,
|
|
"id": 29,
|
|
"links": [],
|
|
"liveNow": false,
|
|
"panels": [
|
|
{
|
|
"collapsed": false,
|
|
"gridPos": {
|
|
"h": 1,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 0
|
|
},
|
|
"id": 102,
|
|
"panels": [],
|
|
"title": "Loki General Info",
|
|
"type": "row"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "thresholds"
|
|
},
|
|
"links": [],
|
|
"mappings": [
|
|
{
|
|
"options": {
|
|
"null": {
|
|
"index": 0,
|
|
"text": "N/A"
|
|
}
|
|
},
|
|
"type": "value"
|
|
}
|
|
],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
}
|
|
]
|
|
},
|
|
"unit": "short"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 3,
|
|
"w": 3,
|
|
"x": 0,
|
|
"y": 1
|
|
},
|
|
"id": 59,
|
|
"interval": "$interval",
|
|
"maxDataPoints": 1,
|
|
"options": {
|
|
"colorMode": "background",
|
|
"graphMode": "none",
|
|
"justifyMode": "center",
|
|
"orientation": "auto",
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"last"
|
|
],
|
|
"fields": "/.*/",
|
|
"values": false
|
|
},
|
|
"textMode": "auto"
|
|
},
|
|
"pluginVersion": "9.3.2",
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"exemplar": false,
|
|
"expr": "loki_build_info{job=\"$jobs\",instance=\"$host:$instance\"}",
|
|
"format": "table",
|
|
"instant": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "__auto",
|
|
"range": true,
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Loki Version",
|
|
"transformations": [
|
|
{
|
|
"id": "organize",
|
|
"options": {
|
|
"excludeByName": {
|
|
"Time": true,
|
|
"Value": true,
|
|
"__name__": true,
|
|
"goversion": true,
|
|
"instance": true,
|
|
"job": true,
|
|
"version": true
|
|
},
|
|
"indexByName": {
|
|
"Time": 0,
|
|
"Value": 8,
|
|
"__name__": 1,
|
|
"branch": 2,
|
|
"goversion": 4,
|
|
"instance": 5,
|
|
"job": 6,
|
|
"revision": 3,
|
|
"version": 7
|
|
},
|
|
"renameByName": {
|
|
"branch": "Loki Branch",
|
|
"goversion": "",
|
|
"instance": "",
|
|
"revision": "Loki Revision",
|
|
"version": ""
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"type": "stat"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"description": "",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"decimals": 0,
|
|
"links": [],
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
}
|
|
]
|
|
},
|
|
"unit": "short"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 3,
|
|
"w": 3,
|
|
"x": 3,
|
|
"y": 1
|
|
},
|
|
"id": 10,
|
|
"interval": "$interval",
|
|
"options": {
|
|
"colorMode": "value",
|
|
"graphMode": "none",
|
|
"justifyMode": "center",
|
|
"orientation": "auto",
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"last"
|
|
],
|
|
"fields": "",
|
|
"values": false
|
|
},
|
|
"textMode": "auto"
|
|
},
|
|
"pluginVersion": "9.3.2",
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "sum(loki_internal_log_messages_total{job=\"$jobs\",instance=\"$host:$instance\"})",
|
|
"instant": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Internal Log Messages",
|
|
"type": "stat"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"links": [],
|
|
"mappings": [
|
|
{
|
|
"options": {
|
|
"null": {
|
|
"index": 0,
|
|
"text": "N/A"
|
|
}
|
|
},
|
|
"type": "value"
|
|
}
|
|
],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
}
|
|
]
|
|
},
|
|
"unit": "short"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 3,
|
|
"w": 3,
|
|
"x": 6,
|
|
"y": 1
|
|
},
|
|
"id": 80,
|
|
"interval": "$interval",
|
|
"options": {
|
|
"colorMode": "value",
|
|
"graphMode": "none",
|
|
"justifyMode": "center",
|
|
"orientation": "auto",
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"last"
|
|
],
|
|
"fields": "",
|
|
"values": false
|
|
},
|
|
"textMode": "auto"
|
|
},
|
|
"pluginVersion": "9.3.2",
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "deprecated_flags_inuse_total{job=\"$jobs\",instance=\"$host:$instance\"}",
|
|
"instant": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "Deprecated",
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "loki_experimental_features_in_use_total{job=\"$jobs\",instance=\"$host:$instance\"}",
|
|
"hide": false,
|
|
"instant": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "Experimental",
|
|
"refId": "B"
|
|
}
|
|
],
|
|
"title": "Features Currently Set",
|
|
"type": "stat"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"links": [],
|
|
"mappings": [
|
|
{
|
|
"options": {
|
|
"null": {
|
|
"index": 0,
|
|
"text": "N/A"
|
|
}
|
|
},
|
|
"type": "value"
|
|
}
|
|
],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
},
|
|
{
|
|
"color": "semi-dark-red",
|
|
"value": 1
|
|
}
|
|
]
|
|
},
|
|
"unit": "short"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 3,
|
|
"w": 3,
|
|
"x": 9,
|
|
"y": 1
|
|
},
|
|
"id": 24,
|
|
"interval": "$interval",
|
|
"options": {
|
|
"colorMode": "value",
|
|
"graphMode": "area",
|
|
"justifyMode": "center",
|
|
"orientation": "auto",
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"last"
|
|
],
|
|
"fields": "",
|
|
"values": false
|
|
},
|
|
"textMode": "auto"
|
|
},
|
|
"pluginVersion": "9.3.2",
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "loki_panic_total{job=\"$jobs\",instance=\"$host:$instance\"}",
|
|
"instant": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "The total number of panics triggered",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Panics",
|
|
"type": "stat"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"links": [],
|
|
"mappings": [
|
|
{
|
|
"options": {
|
|
"null": {
|
|
"index": 0,
|
|
"text": "N/A"
|
|
}
|
|
},
|
|
"type": "value"
|
|
}
|
|
],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
},
|
|
{
|
|
"color": "semi-dark-red",
|
|
"value": 0
|
|
},
|
|
{
|
|
"color": "green",
|
|
"value": 1
|
|
}
|
|
]
|
|
},
|
|
"unit": "short"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 3,
|
|
"w": 3,
|
|
"x": 12,
|
|
"y": 1
|
|
},
|
|
"id": 112,
|
|
"interval": "$interval",
|
|
"options": {
|
|
"colorMode": "value",
|
|
"graphMode": "area",
|
|
"justifyMode": "center",
|
|
"orientation": "auto",
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"last"
|
|
],
|
|
"fields": "",
|
|
"values": false
|
|
},
|
|
"textMode": "auto"
|
|
},
|
|
"pluginVersion": "9.3.2",
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "loki_distributor_replication_factor{job=\"$jobs\",instance=\"$host:$instance\"}",
|
|
"instant": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "The configured replication factor",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Replication Faktor",
|
|
"type": "stat"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "thresholds"
|
|
},
|
|
"links": [],
|
|
"mappings": [
|
|
{
|
|
"options": {
|
|
"null": {
|
|
"index": 0,
|
|
"text": "N/A"
|
|
}
|
|
},
|
|
"type": "value"
|
|
}
|
|
],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
}
|
|
]
|
|
},
|
|
"unit": "short"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 3,
|
|
"w": 3,
|
|
"x": 15,
|
|
"y": 1
|
|
},
|
|
"id": 81,
|
|
"interval": "$interval",
|
|
"maxDataPoints": 1,
|
|
"options": {
|
|
"colorMode": "background",
|
|
"graphMode": "none",
|
|
"justifyMode": "center",
|
|
"orientation": "auto",
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"last"
|
|
],
|
|
"fields": "/.*/",
|
|
"values": false
|
|
},
|
|
"textMode": "auto"
|
|
},
|
|
"pluginVersion": "9.3.2",
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"exemplar": false,
|
|
"expr": "go_info{job=\"$jobs\",instance=\"$host:$instance\"}",
|
|
"format": "table",
|
|
"instant": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "__auto",
|
|
"range": true,
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Go Version",
|
|
"transformations": [
|
|
{
|
|
"id": "organize",
|
|
"options": {
|
|
"excludeByName": {
|
|
"Time": true,
|
|
"Value": true,
|
|
"__name__": true,
|
|
"branch": true,
|
|
"instance": true,
|
|
"job": true,
|
|
"revision": true,
|
|
"version": false
|
|
},
|
|
"indexByName": {
|
|
"Time": 0,
|
|
"Value": 8,
|
|
"__name__": 1,
|
|
"branch": 2,
|
|
"goversion": 4,
|
|
"instance": 5,
|
|
"job": 6,
|
|
"revision": 3,
|
|
"version": 7
|
|
},
|
|
"renameByName": {
|
|
"branch": "",
|
|
"goversion": "Go Version",
|
|
"instance": "",
|
|
"revision": "",
|
|
"version": "Go Version"
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"type": "stat"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"description": "",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "thresholds"
|
|
},
|
|
"links": [],
|
|
"mappings": [
|
|
{
|
|
"options": {
|
|
"null": {
|
|
"index": 0,
|
|
"text": "N/A"
|
|
}
|
|
},
|
|
"type": "value"
|
|
}
|
|
],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
},
|
|
{
|
|
"color": "semi-dark-red",
|
|
"value": 500
|
|
}
|
|
]
|
|
},
|
|
"unit": "short"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 3,
|
|
"w": 3,
|
|
"x": 18,
|
|
"y": 1
|
|
},
|
|
"id": 86,
|
|
"options": {
|
|
"colorMode": "value",
|
|
"graphMode": "area",
|
|
"justifyMode": "auto",
|
|
"orientation": "auto",
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"last"
|
|
],
|
|
"fields": "",
|
|
"values": false
|
|
},
|
|
"textMode": "auto"
|
|
},
|
|
"pluginVersion": "9.3.2",
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"exemplar": false,
|
|
"expr": "sum(go_goroutines{job=\"$jobs\",instance=\"$host:$instance\"})",
|
|
"format": "table",
|
|
"instant": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "Number of goroutines that currently exist",
|
|
"range": true,
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Total Goroutines",
|
|
"transformations": [],
|
|
"type": "stat"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"description": "",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "thresholds"
|
|
},
|
|
"links": [],
|
|
"mappings": [
|
|
{
|
|
"options": {
|
|
"null": {
|
|
"index": 0,
|
|
"text": "N/A"
|
|
}
|
|
},
|
|
"type": "value"
|
|
}
|
|
],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
},
|
|
{
|
|
"color": "semi-dark-red",
|
|
"value": 25
|
|
}
|
|
]
|
|
},
|
|
"unit": "short"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 3,
|
|
"w": 3,
|
|
"x": 21,
|
|
"y": 1
|
|
},
|
|
"id": 87,
|
|
"options": {
|
|
"colorMode": "value",
|
|
"graphMode": "area",
|
|
"justifyMode": "auto",
|
|
"orientation": "auto",
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"last"
|
|
],
|
|
"fields": "",
|
|
"values": false
|
|
},
|
|
"textMode": "auto"
|
|
},
|
|
"pluginVersion": "9.3.2",
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"exemplar": false,
|
|
"expr": "sum(go_threads{job=\"$jobs\",instance=\"$host:$instance\"})",
|
|
"format": "table",
|
|
"instant": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "Number of OS threads created",
|
|
"range": true,
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Total Threads",
|
|
"transformations": [],
|
|
"type": "stat"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"links": [],
|
|
"mappings": [
|
|
{
|
|
"options": {
|
|
"null": {
|
|
"index": 0,
|
|
"text": "N/A"
|
|
}
|
|
},
|
|
"type": "value"
|
|
}
|
|
],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
}
|
|
]
|
|
},
|
|
"unit": "cps"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 6,
|
|
"x": 0,
|
|
"y": 4
|
|
},
|
|
"id": 235,
|
|
"interval": "$interval",
|
|
"options": {
|
|
"colorMode": "value",
|
|
"graphMode": "area",
|
|
"justifyMode": "center",
|
|
"orientation": "auto",
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"last"
|
|
],
|
|
"fields": "",
|
|
"values": false
|
|
},
|
|
"textMode": "auto"
|
|
},
|
|
"pluginVersion": "9.3.2",
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "sum(rate(loki_distributor_lines_received_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by(tenant)",
|
|
"hide": false,
|
|
"instant": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "{{ tenant }}",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Distributor Lines",
|
|
"transformations": [
|
|
{
|
|
"id": "calculateField",
|
|
"options": {}
|
|
},
|
|
{
|
|
"id": "organize",
|
|
"options": {
|
|
"excludeByName": {},
|
|
"indexByName": {},
|
|
"renameByName": {
|
|
"fake": "Default Tenant"
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"type": "stat"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"links": [],
|
|
"mappings": [
|
|
{
|
|
"options": {
|
|
"null": {
|
|
"index": 0,
|
|
"text": "N/A"
|
|
}
|
|
},
|
|
"type": "value"
|
|
}
|
|
],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
}
|
|
]
|
|
},
|
|
"unit": "binBps"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 6,
|
|
"x": 6,
|
|
"y": 4
|
|
},
|
|
"id": 234,
|
|
"interval": "$interval",
|
|
"options": {
|
|
"colorMode": "value",
|
|
"graphMode": "area",
|
|
"justifyMode": "center",
|
|
"orientation": "auto",
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"last"
|
|
],
|
|
"fields": "",
|
|
"values": false
|
|
},
|
|
"textMode": "auto"
|
|
},
|
|
"pluginVersion": "9.3.2",
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "sum(rate(loki_distributor_bytes_received_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by(tenant)",
|
|
"hide": false,
|
|
"instant": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "{{ tenant }}",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Distributor Volume",
|
|
"transformations": [
|
|
{
|
|
"id": "calculateField",
|
|
"options": {
|
|
"mode": "reduceRow",
|
|
"reduce": {
|
|
"include": [],
|
|
"reducer": "sum"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"id": "organize",
|
|
"options": {
|
|
"excludeByName": {},
|
|
"indexByName": {},
|
|
"renameByName": {
|
|
"Received / fake": "Received / Default Tenant",
|
|
"fake": "Default Tenant"
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"type": "stat"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"links": [],
|
|
"mappings": [
|
|
{
|
|
"options": {
|
|
"null": {
|
|
"index": 0,
|
|
"text": "N/A"
|
|
}
|
|
},
|
|
"type": "value"
|
|
}
|
|
],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
}
|
|
]
|
|
},
|
|
"unit": "ops"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 6,
|
|
"x": 12,
|
|
"y": 4
|
|
},
|
|
"id": 230,
|
|
"interval": "$interval",
|
|
"options": {
|
|
"colorMode": "value",
|
|
"graphMode": "area",
|
|
"justifyMode": "center",
|
|
"orientation": "auto",
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"last"
|
|
],
|
|
"fields": "",
|
|
"values": false
|
|
},
|
|
"textMode": "auto"
|
|
},
|
|
"pluginVersion": "9.3.2",
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "rate(loki_ingester_chunks_created_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])",
|
|
"hide": false,
|
|
"instant": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "Create",
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "sum(rate(loki_ingester_chunks_stored_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (tenant) ",
|
|
"hide": false,
|
|
"instant": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "Store / {{ tenant }}",
|
|
"refId": "B"
|
|
}
|
|
],
|
|
"title": "Ingester Chunk Operations",
|
|
"transformations": [
|
|
{
|
|
"id": "calculateField",
|
|
"options": {}
|
|
},
|
|
{
|
|
"id": "organize",
|
|
"options": {
|
|
"excludeByName": {},
|
|
"indexByName": {},
|
|
"renameByName": {
|
|
"Store / fake": "Store / Default Tenant"
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"type": "stat"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"links": [],
|
|
"mappings": [
|
|
{
|
|
"options": {
|
|
"null": {
|
|
"index": 0,
|
|
"text": "N/A"
|
|
}
|
|
},
|
|
"type": "value"
|
|
}
|
|
],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
}
|
|
]
|
|
},
|
|
"unit": "binBps"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 6,
|
|
"x": 18,
|
|
"y": 4
|
|
},
|
|
"id": 188,
|
|
"interval": "$interval",
|
|
"options": {
|
|
"colorMode": "value",
|
|
"graphMode": "area",
|
|
"justifyMode": "auto",
|
|
"orientation": "auto",
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"last"
|
|
],
|
|
"fields": "",
|
|
"values": false
|
|
},
|
|
"textMode": "auto"
|
|
},
|
|
"pluginVersion": "9.3.2",
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "sum(rate(loki_ingester_chunk_stored_bytes_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (tenant)",
|
|
"instant": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "{{ tenant }}",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Ingester Chunks Volume",
|
|
"transformations": [
|
|
{
|
|
"id": "calculateField",
|
|
"options": {}
|
|
},
|
|
{
|
|
"id": "organize",
|
|
"options": {
|
|
"excludeByName": {},
|
|
"indexByName": {},
|
|
"renameByName": {
|
|
"fake": "Default Tenant"
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"type": "stat"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"links": [],
|
|
"mappings": [
|
|
{
|
|
"options": {
|
|
"null": {
|
|
"index": 0,
|
|
"text": "N/A"
|
|
}
|
|
},
|
|
"type": "value"
|
|
}
|
|
],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
}
|
|
]
|
|
},
|
|
"unit": "short"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 3,
|
|
"w": 3,
|
|
"x": 0,
|
|
"y": 10
|
|
},
|
|
"id": 23,
|
|
"interval": "$interval",
|
|
"options": {
|
|
"colorMode": "value",
|
|
"graphMode": "area",
|
|
"justifyMode": "center",
|
|
"orientation": "auto",
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"last"
|
|
],
|
|
"fields": "",
|
|
"values": false
|
|
},
|
|
"textMode": "auto"
|
|
},
|
|
"pluginVersion": "9.3.2",
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "sum(loki_store_series_total{job=\"$jobs\",instance=\"$host:$instance\"}) by(status)",
|
|
"instant": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "{{ status }}",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Store Series",
|
|
"transformations": [
|
|
{
|
|
"id": "calculateField",
|
|
"options": {}
|
|
},
|
|
{
|
|
"id": "organize",
|
|
"options": {
|
|
"excludeByName": {},
|
|
"indexByName": {},
|
|
"renameByName": {
|
|
"discarded": "Discarded",
|
|
"matched": "Matched"
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"type": "stat"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"links": [],
|
|
"mappings": [
|
|
{
|
|
"options": {
|
|
"null": {
|
|
"index": 0,
|
|
"text": "N/A"
|
|
}
|
|
},
|
|
"type": "value"
|
|
}
|
|
],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
}
|
|
]
|
|
},
|
|
"unit": "short"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 3,
|
|
"w": 3,
|
|
"x": 3,
|
|
"y": 10
|
|
},
|
|
"id": 131,
|
|
"interval": "$interval",
|
|
"options": {
|
|
"colorMode": "value",
|
|
"graphMode": "area",
|
|
"justifyMode": "center",
|
|
"orientation": "auto",
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"last"
|
|
],
|
|
"fields": "",
|
|
"values": false
|
|
},
|
|
"textMode": "auto"
|
|
},
|
|
"pluginVersion": "9.3.2",
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "sum(loki_store_chunks_downloaded_total{job=\"$jobs\",instance=\"$host:$instance\"}) by(status)",
|
|
"instant": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "{{ status }}",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Store Chunks Downloaded",
|
|
"transformations": [
|
|
{
|
|
"id": "calculateField",
|
|
"options": {}
|
|
},
|
|
{
|
|
"id": "organize",
|
|
"options": {
|
|
"excludeByName": {},
|
|
"indexByName": {},
|
|
"renameByName": {
|
|
"discarded": "Discarded",
|
|
"matched": "Matched"
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"type": "stat"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"links": [],
|
|
"mappings": [
|
|
{
|
|
"options": {
|
|
"null": {
|
|
"index": 0,
|
|
"text": "N/A"
|
|
}
|
|
},
|
|
"type": "value"
|
|
}
|
|
],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
}
|
|
]
|
|
},
|
|
"unit": "short"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 3,
|
|
"w": 3,
|
|
"x": 6,
|
|
"y": 10
|
|
},
|
|
"id": 137,
|
|
"interval": "$interval",
|
|
"options": {
|
|
"colorMode": "value",
|
|
"graphMode": "area",
|
|
"justifyMode": "center",
|
|
"orientation": "auto",
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"last"
|
|
],
|
|
"fields": "",
|
|
"values": false
|
|
},
|
|
"textMode": "auto"
|
|
},
|
|
"pluginVersion": "9.3.2",
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "sum(loki_index_chunk_refs_total{job=\"$jobs\",instance=\"$host:$instance\"}) by(status)",
|
|
"instant": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "{{ status }}",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Index Chunks Downloaded",
|
|
"transformations": [
|
|
{
|
|
"id": "calculateField",
|
|
"options": {}
|
|
},
|
|
{
|
|
"id": "organize",
|
|
"options": {
|
|
"excludeByName": {},
|
|
"indexByName": {},
|
|
"renameByName": {
|
|
"discarded": "Discarded",
|
|
"matched": "Matched"
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"type": "stat"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"links": [],
|
|
"mappings": [
|
|
{
|
|
"options": {
|
|
"null": {
|
|
"index": 0,
|
|
"text": "N/A"
|
|
}
|
|
},
|
|
"type": "value"
|
|
}
|
|
],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
}
|
|
]
|
|
},
|
|
"unit": "short"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 3,
|
|
"w": 3,
|
|
"x": 12,
|
|
"y": 10
|
|
},
|
|
"id": 191,
|
|
"interval": "$interval",
|
|
"options": {
|
|
"colorMode": "value",
|
|
"graphMode": "area",
|
|
"justifyMode": "center",
|
|
"orientation": "auto",
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"last"
|
|
],
|
|
"fields": "",
|
|
"values": false
|
|
},
|
|
"textMode": "auto"
|
|
},
|
|
"pluginVersion": "9.3.2",
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "loki_ingester_memory_chunks{job=\"$jobs\",instance=\"$host:$instance\"}",
|
|
"hide": false,
|
|
"instant": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "Chunks in memory",
|
|
"refId": "B"
|
|
}
|
|
],
|
|
"title": "Ingester Chunks In Memory",
|
|
"transformations": [],
|
|
"type": "stat"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"links": [],
|
|
"mappings": [
|
|
{
|
|
"options": {
|
|
"null": {
|
|
"index": 0,
|
|
"text": "N/A"
|
|
}
|
|
},
|
|
"type": "value"
|
|
}
|
|
],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
}
|
|
]
|
|
},
|
|
"unit": "short"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 3,
|
|
"w": 3,
|
|
"x": 15,
|
|
"y": 10
|
|
},
|
|
"id": 192,
|
|
"interval": "$interval",
|
|
"options": {
|
|
"colorMode": "value",
|
|
"graphMode": "area",
|
|
"justifyMode": "center",
|
|
"orientation": "auto",
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"last"
|
|
],
|
|
"fields": "",
|
|
"values": false
|
|
},
|
|
"textMode": "auto"
|
|
},
|
|
"pluginVersion": "9.3.2",
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "sum(loki_ingester_memory_streams{job=\"$jobs\",instance=\"$host:$instance\"}) by(tenant)",
|
|
"instant": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "{{ tenant }}",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Ingester Streams in Memory",
|
|
"transformations": [
|
|
{
|
|
"id": "calculateField",
|
|
"options": {}
|
|
},
|
|
{
|
|
"id": "organize",
|
|
"options": {
|
|
"excludeByName": {},
|
|
"indexByName": {
|
|
"Time": 0,
|
|
"Total": 2,
|
|
"fake": 1
|
|
},
|
|
"renameByName": {
|
|
"discarded": "Discarded",
|
|
"fake": "Default Tenant",
|
|
"matched": "Matched"
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"type": "stat"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"links": [],
|
|
"mappings": [
|
|
{
|
|
"options": {
|
|
"null": {
|
|
"index": 0,
|
|
"text": "N/A"
|
|
}
|
|
},
|
|
"type": "value"
|
|
}
|
|
],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
}
|
|
]
|
|
},
|
|
"unit": "ops"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 6,
|
|
"x": 18,
|
|
"y": 10
|
|
},
|
|
"id": 233,
|
|
"interval": "$interval",
|
|
"options": {
|
|
"colorMode": "value",
|
|
"graphMode": "area",
|
|
"justifyMode": "center",
|
|
"orientation": "auto",
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"last"
|
|
],
|
|
"fields": "",
|
|
"values": false
|
|
},
|
|
"textMode": "auto"
|
|
},
|
|
"pluginVersion": "9.3.2",
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "sum(rate(loki_ingester_streams_created_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (tenant)",
|
|
"hide": false,
|
|
"instant": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "Create / {{ tenant }}",
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "sum(rate(loki_ingester_streams_removed_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (tenant)",
|
|
"hide": false,
|
|
"instant": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "Delete / {{ tenant }}",
|
|
"refId": "B"
|
|
}
|
|
],
|
|
"title": "Ingester Stream Operations",
|
|
"transformations": [
|
|
{
|
|
"id": "calculateField",
|
|
"options": {}
|
|
},
|
|
{
|
|
"id": "organize",
|
|
"options": {
|
|
"excludeByName": {},
|
|
"indexByName": {},
|
|
"renameByName": {
|
|
"Create / fake": "Create / Default Tenant",
|
|
"Delete / fake": "Delete / Default Tenant",
|
|
"Store / fake": "Store / Default Tenant"
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"type": "stat"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"links": [],
|
|
"mappings": [
|
|
{
|
|
"options": {
|
|
"null": {
|
|
"index": 0,
|
|
"text": "N/A"
|
|
}
|
|
},
|
|
"type": "value"
|
|
}
|
|
],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
}
|
|
]
|
|
},
|
|
"unit": "ops"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 3,
|
|
"w": 6,
|
|
"x": 0,
|
|
"y": 13
|
|
},
|
|
"id": 236,
|
|
"interval": "$interval",
|
|
"options": {
|
|
"colorMode": "value",
|
|
"graphMode": "area",
|
|
"justifyMode": "center",
|
|
"orientation": "auto",
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"last"
|
|
],
|
|
"fields": "",
|
|
"values": false
|
|
},
|
|
"textMode": "auto"
|
|
},
|
|
"pluginVersion": "9.3.2",
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "sum(rate(loki_chunk_store_fetched_chunks_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (user)",
|
|
"hide": false,
|
|
"instant": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "{{ user}}",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Store Operations Fetched",
|
|
"transformations": [
|
|
{
|
|
"id": "calculateField",
|
|
"options": {}
|
|
},
|
|
{
|
|
"id": "organize",
|
|
"options": {
|
|
"excludeByName": {},
|
|
"indexByName": {},
|
|
"renameByName": {
|
|
"Create / fake": "Create / Default Tenant",
|
|
"Delete / fake": "Delete / Default Tenant",
|
|
"Fetch / fake": "Fetch / Default Tenant",
|
|
"Store / fake": "Store / Default Tenant",
|
|
"fake": "Default Tenant"
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"type": "stat"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"links": [],
|
|
"mappings": [
|
|
{
|
|
"options": {
|
|
"null": {
|
|
"index": 0,
|
|
"text": "N/A"
|
|
}
|
|
},
|
|
"type": "value"
|
|
}
|
|
],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
}
|
|
]
|
|
},
|
|
"unit": "ops"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 3,
|
|
"w": 6,
|
|
"x": 6,
|
|
"y": 13
|
|
},
|
|
"id": 237,
|
|
"interval": "$interval",
|
|
"options": {
|
|
"colorMode": "value",
|
|
"graphMode": "area",
|
|
"justifyMode": "center",
|
|
"orientation": "auto",
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"last"
|
|
],
|
|
"fields": "",
|
|
"values": false
|
|
},
|
|
"textMode": "auto"
|
|
},
|
|
"pluginVersion": "9.3.2",
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "sum(rate(loki_chunk_store_stored_chunks_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (user)",
|
|
"hide": false,
|
|
"instant": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "{{ user}}",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Store Operations Stored",
|
|
"transformations": [
|
|
{
|
|
"id": "calculateField",
|
|
"options": {}
|
|
},
|
|
{
|
|
"id": "organize",
|
|
"options": {
|
|
"excludeByName": {},
|
|
"indexByName": {},
|
|
"renameByName": {
|
|
"Create / fake": "Create / Default Tenant",
|
|
"Delete / fake": "Delete / Default Tenant",
|
|
"Fetch / fake": "Default Tenant",
|
|
"Store / fake": "Store / Default Tenant",
|
|
"fake": "Default Tenant"
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"type": "stat"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"links": [],
|
|
"mappings": [
|
|
{
|
|
"options": {
|
|
"null": {
|
|
"index": 0,
|
|
"text": "N/A"
|
|
}
|
|
},
|
|
"type": "value"
|
|
}
|
|
],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
}
|
|
]
|
|
},
|
|
"unit": "bytes"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 3,
|
|
"w": 3,
|
|
"x": 12,
|
|
"y": 13
|
|
},
|
|
"id": 231,
|
|
"interval": "$interval",
|
|
"options": {
|
|
"colorMode": "value",
|
|
"graphMode": "area",
|
|
"justifyMode": "center",
|
|
"orientation": "auto",
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"last"
|
|
],
|
|
"fields": "",
|
|
"values": false
|
|
},
|
|
"textMode": "auto"
|
|
},
|
|
"pluginVersion": "9.3.2",
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "histogram_quantile(0.95, sum(rate(loki_ingester_chunk_size_bytes_bucket{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (le))",
|
|
"hide": false,
|
|
"instant": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "Chunk Size",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Ingester Chunk Size",
|
|
"transformations": [],
|
|
"type": "stat"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"links": [],
|
|
"mappings": [
|
|
{
|
|
"options": {
|
|
"null": {
|
|
"index": 0,
|
|
"text": "N/A"
|
|
}
|
|
},
|
|
"type": "value"
|
|
}
|
|
],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
}
|
|
]
|
|
},
|
|
"unit": "percent"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 3,
|
|
"w": 3,
|
|
"x": 15,
|
|
"y": 13
|
|
},
|
|
"id": 232,
|
|
"interval": "$interval",
|
|
"options": {
|
|
"colorMode": "value",
|
|
"graphMode": "area",
|
|
"justifyMode": "center",
|
|
"orientation": "auto",
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"last"
|
|
],
|
|
"fields": "",
|
|
"values": false
|
|
},
|
|
"textMode": "auto"
|
|
},
|
|
"pluginVersion": "9.3.2",
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "histogram_quantile(0.95, sum(rate(loki_ingester_chunk_compression_ratio_bucket{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (le))",
|
|
"hide": false,
|
|
"instant": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "Compression Ratio",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Ingester Chunk Compression",
|
|
"transformations": [],
|
|
"type": "stat"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"links": [],
|
|
"mappings": [
|
|
{
|
|
"options": {
|
|
"null": {
|
|
"index": 0,
|
|
"text": "N/A"
|
|
}
|
|
},
|
|
"type": "value"
|
|
}
|
|
],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
}
|
|
]
|
|
},
|
|
"unit": "binBps"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 3,
|
|
"w": 6,
|
|
"x": 0,
|
|
"y": 16
|
|
},
|
|
"id": 238,
|
|
"interval": "$interval",
|
|
"options": {
|
|
"colorMode": "value",
|
|
"graphMode": "area",
|
|
"justifyMode": "center",
|
|
"orientation": "auto",
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"last"
|
|
],
|
|
"fields": "",
|
|
"values": false
|
|
},
|
|
"textMode": "auto"
|
|
},
|
|
"pluginVersion": "9.3.2",
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "rate(loki_chunk_store_fetched_chunk_bytes_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])",
|
|
"hide": false,
|
|
"instant": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "{{ user}}",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Store Volume Fetched",
|
|
"transformations": [
|
|
{
|
|
"id": "calculateField",
|
|
"options": {}
|
|
},
|
|
{
|
|
"id": "organize",
|
|
"options": {
|
|
"excludeByName": {},
|
|
"indexByName": {},
|
|
"renameByName": {
|
|
"Create / fake": "Create / Default Tenant",
|
|
"Delete / fake": "Delete / Default Tenant",
|
|
"Fetch / fake": "Fetch / Default Tenant",
|
|
"Store / fake": "Store / Default Tenant",
|
|
"fake": "Default Tenant"
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"type": "stat"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"links": [],
|
|
"mappings": [
|
|
{
|
|
"options": {
|
|
"null": {
|
|
"index": 0,
|
|
"text": "N/A"
|
|
}
|
|
},
|
|
"type": "value"
|
|
}
|
|
],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
}
|
|
]
|
|
},
|
|
"unit": "binBps"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 3,
|
|
"w": 6,
|
|
"x": 6,
|
|
"y": 16
|
|
},
|
|
"id": 239,
|
|
"interval": "$interval",
|
|
"options": {
|
|
"colorMode": "value",
|
|
"graphMode": "area",
|
|
"justifyMode": "center",
|
|
"orientation": "auto",
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"last"
|
|
],
|
|
"fields": "",
|
|
"values": false
|
|
},
|
|
"textMode": "auto"
|
|
},
|
|
"pluginVersion": "9.3.2",
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "rate(loki_chunk_store_stored_chunk_bytes_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])",
|
|
"hide": false,
|
|
"instant": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "{{ user}}",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Store Volume Stored",
|
|
"transformations": [
|
|
{
|
|
"id": "calculateField",
|
|
"options": {}
|
|
},
|
|
{
|
|
"id": "organize",
|
|
"options": {
|
|
"excludeByName": {},
|
|
"indexByName": {},
|
|
"renameByName": {
|
|
"Create / fake": "Create / Default Tenant",
|
|
"Delete / fake": "Delete / Default Tenant",
|
|
"Fetch / fake": "Fetch / Default Tenant",
|
|
"Store / fake": "Store / Default Tenant",
|
|
"fake": "Default Tenant"
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"type": "stat"
|
|
},
|
|
{
|
|
"collapsed": true,
|
|
"gridPos": {
|
|
"h": 1,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 19
|
|
},
|
|
"id": 143,
|
|
"panels": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"description": "",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "Time",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 10,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "auto",
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
}
|
|
]
|
|
},
|
|
"unit": "s"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 2
|
|
},
|
|
"id": 151,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
"min",
|
|
"max",
|
|
"mean",
|
|
"last"
|
|
],
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "single",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "histogram_quantile(0.95, sum(rate(loki_request_duration_seconds_bucket{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (le,method,route,status_code))",
|
|
"hide": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "{{ method }} / {{ route }} / {{ status_code }}",
|
|
"range": true,
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Loki API - Request Duration",
|
|
"transformations": [],
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"description": "",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "Requests",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 10,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "auto",
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
}
|
|
]
|
|
},
|
|
"unit": "cps"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 12,
|
|
"y": 2
|
|
},
|
|
"id": 128,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
"min",
|
|
"max",
|
|
"mean",
|
|
"last"
|
|
],
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "single",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "sum(rate(loki_inflight_requests{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (method, route)",
|
|
"hide": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "{{ method }} / {{ route }}",
|
|
"range": true,
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Loki API - Request Current",
|
|
"transformations": [],
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"description": "",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "Size",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 10,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "auto",
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
}
|
|
]
|
|
},
|
|
"unit": "bytes"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 10
|
|
},
|
|
"id": 153,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
"min",
|
|
"max",
|
|
"mean",
|
|
"last"
|
|
],
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "single",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "histogram_quantile(0.95, sum(rate(loki_response_message_bytes_bucket{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (le,method,route))",
|
|
"hide": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "{{ method }} / {{ route }}",
|
|
"range": true,
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Loki API - Response Size",
|
|
"transformations": [],
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"description": "",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "Size",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 10,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "auto",
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
}
|
|
]
|
|
},
|
|
"unit": "bytes"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 12,
|
|
"y": 10
|
|
},
|
|
"id": 152,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
"min",
|
|
"max",
|
|
"mean",
|
|
"last"
|
|
],
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "single",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "histogram_quantile(0.95, sum(rate(loki_request_message_bytes_bucket{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (le,method,route))",
|
|
"hide": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "{{ method }} / {{ route }}",
|
|
"range": true,
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Loki API - Request Size",
|
|
"transformations": [],
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"description": "",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "Connections",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 10,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "auto",
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
}
|
|
]
|
|
},
|
|
"unit": "none"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 18
|
|
},
|
|
"id": 96,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
"min",
|
|
"max",
|
|
"mean",
|
|
"last"
|
|
],
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "single",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "loki_tcp_connections{job=\"$jobs\",instance=\"$host:$instance\"}",
|
|
"hide": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "{{ protocol }}",
|
|
"range": true,
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Loki - TCP Connections",
|
|
"transformations": [
|
|
{
|
|
"id": "calculateField",
|
|
"options": {
|
|
"mode": "reduceRow",
|
|
"reduce": {
|
|
"include": [
|
|
"grpc",
|
|
"http"
|
|
],
|
|
"reducer": "sum"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"id": "organize",
|
|
"options": {
|
|
"excludeByName": {},
|
|
"indexByName": {
|
|
"Time": 1,
|
|
"Total": 0,
|
|
"grpc": 2,
|
|
"http": 3
|
|
},
|
|
"renameByName": {
|
|
"grpc": "gRPC",
|
|
"http": "HTTP"
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"type": "timeseries"
|
|
}
|
|
],
|
|
"title": "Loki API / Connections Metrics",
|
|
"type": "row"
|
|
},
|
|
{
|
|
"collapsed": true,
|
|
"gridPos": {
|
|
"h": 1,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 20
|
|
},
|
|
"id": 114,
|
|
"panels": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"description": "",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "Cache Hits",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 10,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "auto",
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
}
|
|
]
|
|
},
|
|
"unit": "cps"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 3
|
|
},
|
|
"id": 115,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
"min",
|
|
"max",
|
|
"mean",
|
|
"last"
|
|
],
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "single",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "sum(rate(loki_cache_hits{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by(name)",
|
|
"hide": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "__auto",
|
|
"range": true,
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Loki Cache - Hits",
|
|
"transformations": [],
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"description": "",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "Keys",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 10,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "auto",
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
}
|
|
]
|
|
},
|
|
"unit": "cps"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 12,
|
|
"y": 3
|
|
},
|
|
"id": 116,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
"min",
|
|
"max",
|
|
"mean",
|
|
"last"
|
|
],
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "single",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "sum(rate(loki_cache_fetched_keys{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by(name)",
|
|
"hide": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "__auto",
|
|
"range": true,
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Loki Cache - Fetched Keys",
|
|
"transformations": [],
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"description": "",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "Time",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 10,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "auto",
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
}
|
|
]
|
|
},
|
|
"unit": "s"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 11
|
|
},
|
|
"id": 118,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
"min",
|
|
"max",
|
|
"mean",
|
|
"last"
|
|
],
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "single",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "histogram_quantile(0.95, sum(rate(loki_cache_request_duration_seconds_bucket{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (le,name,method))",
|
|
"hide": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "{{ name }} / {{ method }}",
|
|
"range": true,
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Loki Cache - Request Duration",
|
|
"transformations": [
|
|
{
|
|
"id": "renameByRegex",
|
|
"options": {
|
|
"regex": "(.*) / (.*)\\.(.*)",
|
|
"renamePattern": "$1 / $3"
|
|
}
|
|
}
|
|
],
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"description": "",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "Size",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 10,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "auto",
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
}
|
|
]
|
|
},
|
|
"unit": "bytes"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 12,
|
|
"y": 11
|
|
},
|
|
"id": 119,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
"min",
|
|
"max",
|
|
"mean",
|
|
"last"
|
|
],
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "single",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "histogram_quantile(0.95, sum(rate(loki_cache_value_size_bytes_bucket{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (le,name,method))",
|
|
"hide": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "{{ name }} / {{ method }}",
|
|
"range": true,
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Loki Cache - Value Size",
|
|
"transformations": [],
|
|
"type": "timeseries"
|
|
}
|
|
],
|
|
"title": "Loki Cache Metrics",
|
|
"type": "row"
|
|
},
|
|
{
|
|
"collapsed": true,
|
|
"gridPos": {
|
|
"h": 1,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 21
|
|
},
|
|
"id": 108,
|
|
"panels": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"description": "",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "Bytes",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 10,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "auto",
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
}
|
|
]
|
|
},
|
|
"unit": "binBps"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 4
|
|
},
|
|
"id": 109,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
"min",
|
|
"max",
|
|
"mean",
|
|
"last"
|
|
],
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "single",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "sum(rate(loki_distributor_bytes_received_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by(tenant)",
|
|
"hide": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "__auto",
|
|
"range": true,
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Loki Distributor - Received Bytes",
|
|
"transformations": [
|
|
{
|
|
"id": "organize",
|
|
"options": {
|
|
"excludeByName": {},
|
|
"indexByName": {},
|
|
"renameByName": {
|
|
"fake": "Default Tenant"
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"description": "",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "Lines",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 10,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "auto",
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
}
|
|
]
|
|
},
|
|
"unit": "cps"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 12,
|
|
"y": 4
|
|
},
|
|
"id": 110,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
"min",
|
|
"max",
|
|
"mean",
|
|
"last"
|
|
],
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "single",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "sum(rate(loki_distributor_lines_received_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by(tenant)",
|
|
"hide": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "__auto",
|
|
"range": true,
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Loki Distributor - Received Lines",
|
|
"transformations": [
|
|
{
|
|
"id": "organize",
|
|
"options": {
|
|
"excludeByName": {},
|
|
"indexByName": {},
|
|
"renameByName": {
|
|
"fake": "Default Tenant"
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"description": "",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "Appends",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 10,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "auto",
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
}
|
|
]
|
|
},
|
|
"unit": "cps"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 12
|
|
},
|
|
"id": 111,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
"min",
|
|
"max",
|
|
"mean",
|
|
"last"
|
|
],
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "single",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "sum(rate(loki_distributor_ingester_appends_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by(ingester)",
|
|
"hide": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "__auto",
|
|
"range": true,
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Loki Distributor - Batch Appents Sent To Ingester",
|
|
"transformations": [
|
|
{
|
|
"id": "organize",
|
|
"options": {
|
|
"excludeByName": {},
|
|
"indexByName": {},
|
|
"renameByName": {
|
|
"fake": "Default Tenant"
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"type": "timeseries"
|
|
}
|
|
],
|
|
"title": "Loki Distributor Metrics",
|
|
"type": "row"
|
|
},
|
|
{
|
|
"collapsed": true,
|
|
"gridPos": {
|
|
"h": 1,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 22
|
|
},
|
|
"id": 127,
|
|
"panels": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"description": "",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "Time",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 10,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "auto",
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
}
|
|
]
|
|
},
|
|
"unit": "s"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 5
|
|
},
|
|
"id": 130,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
"min",
|
|
"max",
|
|
"mean",
|
|
"last"
|
|
],
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "single",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "histogram_quantile(0.95, sum(rate(loki_index_request_duration_seconds_bucket{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (le,component,operation,status_code))",
|
|
"hide": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "{{ component }} / {{ operation }} / {{ status_code }}",
|
|
"range": true,
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Loki Index - Request Duration",
|
|
"transformations": [
|
|
{
|
|
"id": "renameByRegex",
|
|
"options": {
|
|
"regex": "(.*) / (.*)\\.(.*)",
|
|
"renamePattern": "$1 / $3"
|
|
}
|
|
}
|
|
],
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"description": "",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "Chunks",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 10,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "auto",
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
}
|
|
]
|
|
},
|
|
"unit": "cps"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 12,
|
|
"y": 5
|
|
},
|
|
"id": 138,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
"min",
|
|
"max",
|
|
"mean",
|
|
"last"
|
|
],
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "single",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "sum(rate(loki_index_chunk_refs_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (status)",
|
|
"hide": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "{{ status }}",
|
|
"range": true,
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Loki Index - Chunks Downloaded",
|
|
"transformations": [
|
|
{
|
|
"id": "calculateField",
|
|
"options": {
|
|
"mode": "reduceRow",
|
|
"reduce": {
|
|
"include": [
|
|
"debug",
|
|
"error",
|
|
"info",
|
|
"warn"
|
|
],
|
|
"reducer": "sum"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"id": "organize",
|
|
"options": {
|
|
"excludeByName": {},
|
|
"indexByName": {
|
|
"Time": 1,
|
|
"Total": 0,
|
|
"grpc": 2,
|
|
"http": 3
|
|
},
|
|
"renameByName": {
|
|
"debug": "Debug",
|
|
"discarded": "Discarded",
|
|
"error": "Error",
|
|
"grpc": "gRPC",
|
|
"http": "HTTP",
|
|
"info": "Info",
|
|
"matched": "Matched",
|
|
"warn": "Warn"
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"description": "",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "Time",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 10,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "auto",
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
}
|
|
]
|
|
},
|
|
"unit": "s"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 13
|
|
},
|
|
"id": 201,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
"min",
|
|
"max",
|
|
"mean",
|
|
"last"
|
|
],
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "single",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "histogram_quantile(0.95, sum(rate(loki_kv_request_duration_seconds_bucket{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (le,kv_name,operation,role,status_code,type))",
|
|
"hide": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "{{ kv_name }} / {{ type }} / {{ operation }} / {{ status_code }}",
|
|
"range": true,
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Loki KV - Request Duration",
|
|
"transformations": [],
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"description": "",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "Time",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 10,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "auto",
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
}
|
|
]
|
|
},
|
|
"unit": "s"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 12,
|
|
"y": 13
|
|
},
|
|
"id": 202,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
"min",
|
|
"max",
|
|
"mean",
|
|
"last"
|
|
],
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "single",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "histogram_quantile(0.95, sum(rate(cortex_kv_request_duration_seconds_bucket{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (le,kv_name,operation,role,status_code,type))",
|
|
"hide": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "{{ kv_name }} / {{ type }} / {{ operation }} / {{ status_code }}",
|
|
"range": true,
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Cortex KV - Request Duration",
|
|
"transformations": [],
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"description": "",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "Time",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 10,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "auto",
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
}
|
|
]
|
|
},
|
|
"unit": "s"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 21
|
|
},
|
|
"id": 228,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
"min",
|
|
"max",
|
|
"mean",
|
|
"last"
|
|
],
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "single",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "histogram_quantile(0.95, sum(rate(cortex_consul_request_duration_seconds_bucket{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (le,kv_name,operation,status_code))",
|
|
"hide": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "{{ kv_name }} / {{ operation }} / {{ status_code }}",
|
|
"range": true,
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Cortex KV - Request Duration",
|
|
"transformations": [],
|
|
"type": "timeseries"
|
|
}
|
|
],
|
|
"title": "Loki Index Metrics",
|
|
"type": "row"
|
|
},
|
|
{
|
|
"collapsed": true,
|
|
"gridPos": {
|
|
"h": 1,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 23
|
|
},
|
|
"id": 174,
|
|
"panels": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"description": "",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "Time",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 10,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "auto",
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
}
|
|
]
|
|
},
|
|
"unit": "s"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 6
|
|
},
|
|
"id": 189,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
"min",
|
|
"max",
|
|
"mean",
|
|
"last"
|
|
],
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "single",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "histogram_quantile(0.95, sum(rate(loki_ingester_client_request_duration_seconds_bucket{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (le,operation,status_code))",
|
|
"hide": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "{{ operation }} / {{ status_code }}",
|
|
"range": true,
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Loki Ingester - Client Request Duration",
|
|
"transformations": [],
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"description": "",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "Operations",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 10,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "auto",
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
}
|
|
]
|
|
},
|
|
"unit": "none"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 12,
|
|
"y": 6
|
|
},
|
|
"id": 185,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
"min",
|
|
"max",
|
|
"mean",
|
|
"last"
|
|
],
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "single",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "rate(loki_ingester_chunks_created_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])",
|
|
"hide": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "Chunk creations ",
|
|
"range": true,
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "sum(rate(loki_ingester_chunks_flushed_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (reason)",
|
|
"hide": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "Chunk flushes / {{ reason }}",
|
|
"range": true,
|
|
"refId": "B"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "sum(rate(loki_ingester_chunks_stored_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (tenant) ",
|
|
"hide": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "Chunks stored / {{ tenant }}",
|
|
"range": true,
|
|
"refId": "C"
|
|
}
|
|
],
|
|
"title": "Loki Ingester - Chunk Operations",
|
|
"transformations": [
|
|
{
|
|
"id": "renameByRegex",
|
|
"options": {
|
|
"regex": "(.*)fake$",
|
|
"renamePattern": "$1Default Tenant"
|
|
}
|
|
}
|
|
],
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"description": "",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "Checkpoints",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 10,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "auto",
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
}
|
|
]
|
|
},
|
|
"unit": "none"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 14
|
|
},
|
|
"id": 175,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
"min",
|
|
"max",
|
|
"mean",
|
|
"last"
|
|
],
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "single",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "rate(loki_ingester_checkpoint_creations_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])",
|
|
"hide": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "Checkpoint creations ",
|
|
"range": true,
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "rate(loki_ingester_checkpoint_creations_failed_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])",
|
|
"hide": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "Checkpoint creation failures",
|
|
"range": true,
|
|
"refId": "B"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "rate(loki_ingester_checkpoint_deletions_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])",
|
|
"hide": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "Checkpoint deletions",
|
|
"range": true,
|
|
"refId": "C"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "rate(loki_ingester_checkpoint_deletions_failed_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])",
|
|
"hide": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "Checkpoint deletion failures",
|
|
"range": true,
|
|
"refId": "D"
|
|
}
|
|
],
|
|
"title": "Loki Ingester - Checkpoints",
|
|
"transformations": [
|
|
{
|
|
"id": "renameByRegex",
|
|
"options": {
|
|
"regex": "(.*)fake$",
|
|
"renamePattern": "$1Default Tenant"
|
|
}
|
|
}
|
|
],
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"description": "",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "Bytes",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 10,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "auto",
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
}
|
|
]
|
|
},
|
|
"unit": "binBps"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 12,
|
|
"y": 14
|
|
},
|
|
"id": 177,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
"min",
|
|
"max",
|
|
"mean",
|
|
"last"
|
|
],
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "single",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "rate(loki_ingester_checkpoint_logged_bytes_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])",
|
|
"hide": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "Bytes written to disk for checkpointing",
|
|
"range": true,
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Loki Ingester - Checkpoints Volume",
|
|
"transformations": [
|
|
{
|
|
"id": "renameByRegex",
|
|
"options": {
|
|
"regex": "(.*)fake$",
|
|
"renamePattern": "$1Default Tenant"
|
|
}
|
|
}
|
|
],
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"description": "",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "Time",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 10,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "auto",
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
}
|
|
]
|
|
},
|
|
"unit": "s"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 22
|
|
},
|
|
"id": 183,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
"min",
|
|
"max",
|
|
"mean",
|
|
"last"
|
|
],
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "single",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "loki_ingester_checkpoint_duration_seconds{job=\"$jobs\",instance=\"$host:$instance\",quantile=\"0.99\"}",
|
|
"hide": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "Time taken to create a checkpoint (0.99 quantile)",
|
|
"range": true,
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "loki_ingester_checkpoint_duration_seconds{job=\"$jobs\",instance=\"$host:$instance\",quantile=\"0.9\"}",
|
|
"hide": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "Time taken to create a checkpoint (0.9 quantile)",
|
|
"range": true,
|
|
"refId": "B"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "loki_ingester_checkpoint_duration_seconds{job=\"$jobs\",instance=\"$host:$instance\",quantile=\"0.5\"}",
|
|
"hide": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "Time taken to create a checkpoint (0.5 quantile)",
|
|
"range": true,
|
|
"refId": "C"
|
|
}
|
|
],
|
|
"title": "Loki Ingester - Checkpoint Duration",
|
|
"transformations": [],
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"description": "",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "Blocks",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 10,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "auto",
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
}
|
|
]
|
|
},
|
|
"unit": "none"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 12,
|
|
"y": 22
|
|
},
|
|
"id": 176,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
"min",
|
|
"max",
|
|
"mean",
|
|
"last"
|
|
],
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "single",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "histogram_quantile(0.95, sum(rate(loki_ingester_blocks_per_chunk_bucket{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (le))",
|
|
"hide": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "Blocks per Chunk",
|
|
"range": true,
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Loki Ingester - Blocks Per Chunk",
|
|
"transformations": [],
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"description": "",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "%",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 10,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "auto",
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
}
|
|
]
|
|
},
|
|
"unit": "percent"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 30
|
|
},
|
|
"id": 180,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
"min",
|
|
"max",
|
|
"mean",
|
|
"last"
|
|
],
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "single",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "histogram_quantile(0.95, sum(rate(loki_ingester_chunk_compression_ratio_bucket{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (le))",
|
|
"hide": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "Compression ratio of chunks (when stored)",
|
|
"range": true,
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Loki Ingester - Chunk Compression Ratio",
|
|
"transformations": [],
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"description": "",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "Count",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 10,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "auto",
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
}
|
|
]
|
|
},
|
|
"unit": "none"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 12,
|
|
"y": 30
|
|
},
|
|
"id": 190,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
"min",
|
|
"max",
|
|
"mean",
|
|
"last"
|
|
],
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "single",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "loki_ingester_memory_chunks{job=\"$jobs\",instance=\"$host:$instance\"}",
|
|
"hide": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "Chunks in memory",
|
|
"range": true,
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Loki Ingester - Chunks In Memory",
|
|
"transformations": [],
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"description": "",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "Bytes",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 10,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "auto",
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
}
|
|
]
|
|
},
|
|
"unit": "bytes"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 38
|
|
},
|
|
"id": 184,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
"min",
|
|
"max",
|
|
"mean",
|
|
"last"
|
|
],
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "single",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "histogram_quantile(0.95, sum(rate(loki_ingester_chunk_size_bytes_bucket{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (le))",
|
|
"hide": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "Distribution of stored chunk sizes (when stored)",
|
|
"range": true,
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Loki Ingester - Chunk Size",
|
|
"transformations": [],
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"description": "",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "Time",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 10,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "auto",
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
}
|
|
]
|
|
},
|
|
"unit": "s"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 12,
|
|
"y": 38
|
|
},
|
|
"id": 178,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
"min",
|
|
"max",
|
|
"mean",
|
|
"last"
|
|
],
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "single",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "histogram_quantile(0.95, sum(rate(loki_ingester_chunk_age_seconds_bucket{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (le))",
|
|
"hide": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "Distribution of chunk ages (when stored)",
|
|
"range": true,
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Loki Ingester - Chunk Age",
|
|
"transformations": [],
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"description": "",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "%",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 10,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "auto",
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
}
|
|
]
|
|
},
|
|
"unit": "percentunit"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 46
|
|
},
|
|
"id": 186,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
"min",
|
|
"max",
|
|
"mean",
|
|
"last"
|
|
],
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "single",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "histogram_quantile(0.95, sum(rate(loki_ingester_chunk_utilization_bucket{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (le))",
|
|
"hide": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "Distribution of stored chunk utilization (when stored)",
|
|
"range": true,
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Loki Ingester - Chunk Usage",
|
|
"transformations": [],
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"description": "",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "Bytes",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 10,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "auto",
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
}
|
|
]
|
|
},
|
|
"unit": "bytes"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 12,
|
|
"y": 46
|
|
},
|
|
"id": 187,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
"min",
|
|
"max",
|
|
"mean",
|
|
"last"
|
|
],
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "single",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "sum(rate(loki_ingester_chunk_stored_bytes_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (tenant)",
|
|
"hide": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "Bytes stored in chunks / {{ tenant }}",
|
|
"range": true,
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Loki Ingester - Chunk Stored Volume",
|
|
"transformations": [
|
|
{
|
|
"id": "renameByRegex",
|
|
"options": {
|
|
"regex": "(.*)fake$",
|
|
"renamePattern": "$1Default Tenant"
|
|
}
|
|
}
|
|
],
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"description": "",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "Samples",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 10,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "auto",
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
}
|
|
]
|
|
},
|
|
"unit": "none"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 54
|
|
},
|
|
"id": 194,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
"min",
|
|
"max",
|
|
"mean",
|
|
"last"
|
|
],
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "single",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "histogram_quantile(0.95, sum(rate(loki_ingester_samples_per_chunk_bucket{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (le))",
|
|
"hide": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "Samples per chunk",
|
|
"range": true,
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Loki Ingester - Samples Per Chunk",
|
|
"transformations": [],
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"description": "",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "Time",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 10,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "auto",
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
}
|
|
]
|
|
},
|
|
"unit": "h"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 12,
|
|
"y": 54
|
|
},
|
|
"id": 179,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
"min",
|
|
"max",
|
|
"mean",
|
|
"last"
|
|
],
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "single",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "histogram_quantile(0.95, sum(rate(loki_ingester_chunk_bounds_hours_bucket{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (le))",
|
|
"hide": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "Distribution of chunk end-start durations",
|
|
"range": true,
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Loki Ingester - Chunk Bounds (Hours)",
|
|
"transformations": [],
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"description": "",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "Time",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 10,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "auto",
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
}
|
|
]
|
|
},
|
|
"unit": "s"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 62
|
|
},
|
|
"id": 181,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
"min",
|
|
"max",
|
|
"mean",
|
|
"last"
|
|
],
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "single",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "histogram_quantile(0.95, sum(rate(loki_ingester_chunk_encode_time_seconds_bucket{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (le))",
|
|
"hide": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "Distribution of chunk encode times",
|
|
"range": true,
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Loki Ingester - Chunk Encoding Time",
|
|
"transformations": [],
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"description": "",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "Lines",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 10,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "auto",
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
}
|
|
]
|
|
},
|
|
"unit": "none"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 12,
|
|
"y": 62
|
|
},
|
|
"id": 182,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
"min",
|
|
"max",
|
|
"mean",
|
|
"last"
|
|
],
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "single",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "histogram_quantile(0.95, sum(rate(loki_ingester_chunk_entries_bucket{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (le))",
|
|
"hide": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "Distribution of stored lines per chunk (when stored)",
|
|
"range": true,
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Loki Ingester - Lines Per Chunk",
|
|
"transformations": [],
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"description": "",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "Operations",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 10,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "auto",
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
}
|
|
]
|
|
},
|
|
"unit": "none"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 70
|
|
},
|
|
"id": 196,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
"min",
|
|
"max",
|
|
"mean",
|
|
"last"
|
|
],
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "single",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "sum(rate(loki_ingester_streams_created_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (tenant)",
|
|
"hide": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "Stream creations / {{ tenant }}",
|
|
"range": true,
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "sum(rate(loki_ingester_streams_removed_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (tenant)",
|
|
"hide": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "Stream deletetions / {{ tenant }}",
|
|
"range": true,
|
|
"refId": "B"
|
|
}
|
|
],
|
|
"title": "Loki Ingester - Stream Operations",
|
|
"transformations": [
|
|
{
|
|
"id": "renameByRegex",
|
|
"options": {
|
|
"regex": "(.*)fake$",
|
|
"renamePattern": "$1Default Tenant"
|
|
}
|
|
}
|
|
],
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"description": "",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "Streams",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 10,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "auto",
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
}
|
|
]
|
|
},
|
|
"unit": "none"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 12,
|
|
"y": 70
|
|
},
|
|
"id": 193,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
"min",
|
|
"max",
|
|
"mean",
|
|
"last"
|
|
],
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "single",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "sum(loki_ingester_memory_streams{job=\"$jobs\",instance=\"$host:$instance\"}) by (tenant)",
|
|
"hide": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "Streams in memory / {{ tenant }}",
|
|
"range": true,
|
|
"refId": "B"
|
|
}
|
|
],
|
|
"title": "Loki Ingester - Streams In Memory",
|
|
"transformations": [
|
|
{
|
|
"id": "renameByRegex",
|
|
"options": {
|
|
"regex": "(.*)fake$",
|
|
"renamePattern": "$1Default Tenant"
|
|
}
|
|
}
|
|
],
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"description": "",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "Records",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 10,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "auto",
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
}
|
|
]
|
|
},
|
|
"unit": "none"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 78
|
|
},
|
|
"id": 197,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
"min",
|
|
"max",
|
|
"mean",
|
|
"last"
|
|
],
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "single",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "rate(loki_ingester_wal_records_logged_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])",
|
|
"hide": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "WAL records logged",
|
|
"range": true,
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "rate(loki_ingester_wal_duplicate_entries_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])",
|
|
"hide": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "Discarded entries during WAL replay due to existing in checkpoints",
|
|
"range": true,
|
|
"refId": "B"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "rate(loki_ingester_wal_disk_full_failures_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])",
|
|
"hide": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "WAL write failures due to full disk",
|
|
"range": true,
|
|
"refId": "C"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "sum(rate(loki_ingester_wal_discarded_samples_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (reason)",
|
|
"hide": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "WAL segment entries discarded during replay / {{ reason }}",
|
|
"range": true,
|
|
"refId": "D"
|
|
}
|
|
],
|
|
"title": "Loki Ingester - WAL Records",
|
|
"transformations": [
|
|
{
|
|
"id": "renameByRegex",
|
|
"options": {
|
|
"regex": "(.*)fake$",
|
|
"renamePattern": "$1Default Tenant"
|
|
}
|
|
}
|
|
],
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"description": "",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "Bytes",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 10,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "auto",
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
}
|
|
]
|
|
},
|
|
"unit": "bytes"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 12,
|
|
"y": 78
|
|
},
|
|
"id": 200,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
"min",
|
|
"max",
|
|
"mean",
|
|
"last"
|
|
],
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "single",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "loki_ingester_wal_bytes_in_use{job=\"$jobs\",instance=\"$host:$instance\"}",
|
|
"hide": false,
|
|
"legendFormat": "Currently in use by the WAL recovery process",
|
|
"range": true,
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "rate(loki_ingester_wal_logged_bytes_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])",
|
|
"hide": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "Written to disk for WAL records",
|
|
"range": true,
|
|
"refId": "B"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "sum(rate(loki_ingester_wal_discarded_bytes_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (reason)",
|
|
"hide": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "WAL segment bytes discarded during replay / {{ reason }}",
|
|
"range": true,
|
|
"refId": "C"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "sum(rate(loki_ingester_wal_discarded_bytes_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (reason)",
|
|
"hide": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "Recovered from the WAL",
|
|
"range": true,
|
|
"refId": "D"
|
|
}
|
|
],
|
|
"title": "Loki Ingester - WAL Volume",
|
|
"transformations": [
|
|
{
|
|
"id": "renameByRegex",
|
|
"options": {
|
|
"regex": "(.*)fake$",
|
|
"renamePattern": "$1Default Tenant"
|
|
}
|
|
}
|
|
],
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"description": "",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "Operations",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 10,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "auto",
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
}
|
|
]
|
|
},
|
|
"unit": "none"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 86
|
|
},
|
|
"id": 198,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
"min",
|
|
"max",
|
|
"mean",
|
|
"last"
|
|
],
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "single",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "rate(loki_ingester_wal_recovered_entries_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])",
|
|
"hide": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "Entries recovered from the WAL",
|
|
"range": true,
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "rate(loki_ingester_wal_recovered_chunks_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])",
|
|
"hide": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "Chunks recovered from the WAL checkpoints",
|
|
"range": true,
|
|
"refId": "B"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "rate(loki_ingester_wal_recovered_streams_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])",
|
|
"hide": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "Streams recovered from the WAL",
|
|
"range": true,
|
|
"refId": "C"
|
|
}
|
|
],
|
|
"title": "Loki Ingester - WAL Recovery Operations",
|
|
"transformations": [
|
|
{
|
|
"id": "renameByRegex",
|
|
"options": {
|
|
"regex": "(.*)fake$",
|
|
"renamePattern": "$1Default Tenant"
|
|
}
|
|
}
|
|
],
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"description": "",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "Count",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 10,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "auto",
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
}
|
|
]
|
|
},
|
|
"unit": "none"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 12,
|
|
"y": 86
|
|
},
|
|
"id": 199,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
"min",
|
|
"max",
|
|
"mean",
|
|
"last"
|
|
],
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "single",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "loki_ingester_wal_replay_active{job=\"$jobs\",instance=\"$host:$instance\"}",
|
|
"hide": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "WAL is replaying",
|
|
"range": true,
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "loki_ingester_wal_replay_flushing{job=\"$jobs\",instance=\"$host:$instance\"}",
|
|
"hide": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "WAL replay in flushing phase due to backpressure",
|
|
"range": true,
|
|
"refId": "B"
|
|
}
|
|
],
|
|
"title": "Loki Ingester - WAL Replay Status",
|
|
"transformations": [
|
|
{
|
|
"id": "renameByRegex",
|
|
"options": {
|
|
"regex": "(.*)fake$",
|
|
"renamePattern": "$1Default Tenant"
|
|
}
|
|
}
|
|
],
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"description": "",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "Length",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 10,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "auto",
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
}
|
|
]
|
|
},
|
|
"unit": "none"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 94
|
|
},
|
|
"id": 218,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
"min",
|
|
"max",
|
|
"mean",
|
|
"last"
|
|
],
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "single",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "cortex_ingester_flush_queue_length{job=\"$jobs\",instance=\"$host:$instance\"}",
|
|
"hide": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "Series pending in the flush queue",
|
|
"range": true,
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Cortex Ingester - Flush Queue",
|
|
"transformations": [],
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"description": "",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "Clients",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 10,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "auto",
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
}
|
|
]
|
|
},
|
|
"unit": "none"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 12,
|
|
"y": 94
|
|
},
|
|
"id": 229,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
"min",
|
|
"max",
|
|
"mean",
|
|
"last"
|
|
],
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "single",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "cortex_distributor_ingester_clients{job=\"$jobs\",instance=\"$host:$instance\"}",
|
|
"hide": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "Ingester clients",
|
|
"range": true,
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Cortex Ingester - Clients",
|
|
"transformations": [],
|
|
"type": "timeseries"
|
|
}
|
|
],
|
|
"title": "Loki Ingester Metrics",
|
|
"type": "row"
|
|
},
|
|
{
|
|
"collapsed": true,
|
|
"gridPos": {
|
|
"h": 1,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 24
|
|
},
|
|
"id": 121,
|
|
"panels": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"description": "",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "Messages",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 10,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "auto",
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
}
|
|
]
|
|
},
|
|
"unit": "cps"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 7
|
|
},
|
|
"id": 124,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
"min",
|
|
"max",
|
|
"mean",
|
|
"last"
|
|
],
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "single",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "sum(rate(loki_internal_log_messages_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (level)",
|
|
"hide": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "{{ protocol }}",
|
|
"range": true,
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Loki - Internal Log Messages",
|
|
"transformations": [
|
|
{
|
|
"id": "calculateField",
|
|
"options": {
|
|
"mode": "reduceRow",
|
|
"reduce": {
|
|
"include": [
|
|
"debug",
|
|
"error",
|
|
"info",
|
|
"warn"
|
|
],
|
|
"reducer": "sum"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"id": "organize",
|
|
"options": {
|
|
"excludeByName": {},
|
|
"indexByName": {
|
|
"Time": 1,
|
|
"Total": 0,
|
|
"grpc": 2,
|
|
"http": 3
|
|
},
|
|
"renameByName": {
|
|
"debug": "Debug",
|
|
"error": "Error",
|
|
"grpc": "gRPC",
|
|
"http": "HTTP",
|
|
"info": "Info",
|
|
"warn": "Warn"
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"description": "",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "Flushes",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 10,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "auto",
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
}
|
|
]
|
|
},
|
|
"unit": "none"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 12,
|
|
"y": 7
|
|
},
|
|
"id": 217,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
"min",
|
|
"max",
|
|
"mean",
|
|
"last"
|
|
],
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "single",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "histogram_quantile(0.95, sum(rate(loki_log_flushes_bucket{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (le))",
|
|
"hide": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "Log flushes - line-buffered logger",
|
|
"range": true,
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Loki Line - Log Flushes",
|
|
"transformations": [],
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"description": "",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"custom": {
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
}
|
|
},
|
|
"links": []
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 15
|
|
},
|
|
"id": 122,
|
|
"maxDataPoints": 50,
|
|
"options": {
|
|
"calculate": false,
|
|
"cellGap": 2,
|
|
"cellValues": {
|
|
"unit": "bytes"
|
|
},
|
|
"color": {
|
|
"exponent": 0.5,
|
|
"fill": "red",
|
|
"mode": "scheme",
|
|
"reverse": false,
|
|
"scale": "exponential",
|
|
"scheme": "BrBG",
|
|
"steps": 64
|
|
},
|
|
"exemplars": {
|
|
"color": "rgba(255,0,255,0.7)"
|
|
},
|
|
"filterValues": {
|
|
"le": 1E-9
|
|
},
|
|
"legend": {
|
|
"show": true
|
|
},
|
|
"rowsFrame": {
|
|
"layout": "auto"
|
|
},
|
|
"tooltip": {
|
|
"show": true,
|
|
"yHistogram": false
|
|
},
|
|
"yAxis": {
|
|
"axisLabel": "Bucket Size",
|
|
"axisPlacement": "left",
|
|
"reverse": false,
|
|
"unit": "bytes"
|
|
}
|
|
},
|
|
"pluginVersion": "9.3.2",
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"exemplar": false,
|
|
"expr": "rate(loki_bytes_per_line_bucket{job=\"$jobs\", instance=\"$host:$instance\"}[$__rate_interval])",
|
|
"format": "heatmap",
|
|
"instant": false,
|
|
"interval": "",
|
|
"legendFormat": "{{ le }}",
|
|
"range": true,
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Loki Line - Line Size Distribution",
|
|
"transformations": [
|
|
{
|
|
"id": "organize",
|
|
"options": {
|
|
"excludeByName": {},
|
|
"indexByName": {},
|
|
"renameByName": {
|
|
"+Inf": "Unlimited",
|
|
"0.00010239999999999998": "10e-5",
|
|
"0.0010485759999999998": "10e-4",
|
|
"0.010485759999999998": "10e-3",
|
|
"0.10485759999999998": "10e-2",
|
|
"1.0239999999999999e-05": "10e-6",
|
|
"1.0239999999999999e-06": "10e-7",
|
|
"9.999999999999998e-08": "10e-8",
|
|
"9.999999999999999e-09": "10e-9",
|
|
"9.999999999999999e-10": "10e-10"
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"type": "heatmap"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"description": "",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "Size",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 10,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "auto",
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
}
|
|
]
|
|
},
|
|
"unit": "bytes"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 12,
|
|
"y": 15
|
|
},
|
|
"id": 123,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
"min",
|
|
"max",
|
|
"mean",
|
|
"last"
|
|
],
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "single",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "histogram_quantile(0.95, sum(rate(loki_bytes_per_line_bucket{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (le))",
|
|
"hide": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "Number of bytes per line",
|
|
"range": true,
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Loki Line - Line Size",
|
|
"transformations": [
|
|
{
|
|
"id": "renameByRegex",
|
|
"options": {
|
|
"regex": "(.*) / (.*)\\.(.*)",
|
|
"renamePattern": "$1 / $3"
|
|
}
|
|
}
|
|
],
|
|
"type": "timeseries"
|
|
}
|
|
],
|
|
"title": "Loki Line Metrics",
|
|
"type": "row"
|
|
},
|
|
{
|
|
"collapsed": true,
|
|
"gridPos": {
|
|
"h": 1,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 25
|
|
},
|
|
"id": 140,
|
|
"panels": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"description": "",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "Lines",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 10,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "auto",
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
}
|
|
]
|
|
},
|
|
"unit": "cps"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 8
|
|
},
|
|
"id": 148,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
"min",
|
|
"max",
|
|
"mean",
|
|
"last"
|
|
],
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "single",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "rate(loki_logql_querystats_ingester_sent_lines_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])",
|
|
"hide": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "Lines",
|
|
"range": true,
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Loki LogQL - Lines Received From Ingester",
|
|
"transformations": [],
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"description": "",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "Duplicates",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 10,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "auto",
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
}
|
|
]
|
|
},
|
|
"unit": "cps"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 12,
|
|
"y": 8
|
|
},
|
|
"id": 150,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
"min",
|
|
"max",
|
|
"mean",
|
|
"last"
|
|
],
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "single",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "rate(loki_logql_querystats_duplicates_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])",
|
|
"hide": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "Lines",
|
|
"range": true,
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Loki LogQL - Duplicates Found",
|
|
"transformations": [],
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"description": "",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "Time",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 10,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "auto",
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
}
|
|
]
|
|
},
|
|
"unit": "s"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 16
|
|
},
|
|
"id": 125,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
"min",
|
|
"max",
|
|
"mean",
|
|
"last"
|
|
],
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "single",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "histogram_quantile(0.95, sum(rate(logql_query_duration_seconds_bucket{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (le,query_type))",
|
|
"hide": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "{{ query_type }}",
|
|
"range": true,
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Loki LogQL - Query Duration",
|
|
"transformations": [],
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"description": "",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "Zime",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 10,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "auto",
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
}
|
|
]
|
|
},
|
|
"unit": "s"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 12,
|
|
"y": 16
|
|
},
|
|
"id": 147,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
"min",
|
|
"max",
|
|
"mean",
|
|
"last"
|
|
],
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "single",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "histogram_quantile(0.95, sum(rate(loki_logql_querystats_latency_seconds_bucket{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (le,type,status_code))",
|
|
"hide": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "{{ type }} / {{ status_code }}",
|
|
"range": true,
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Loki LogQL - Query Latency",
|
|
"transformations": [],
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"description": "",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "Size",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 10,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "auto",
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
}
|
|
]
|
|
},
|
|
"unit": "binBps"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 24
|
|
},
|
|
"id": 141,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
"min",
|
|
"max",
|
|
"mean",
|
|
"last"
|
|
],
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "single",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "histogram_quantile(0.95, sum(rate(loki_logql_querystats_bytes_processed_per_seconds_bucket{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (le,type,status_code))",
|
|
"hide": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "{{ type }} / {{ status_code }}",
|
|
"range": true,
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Loki LogQL - Query Volume Processed",
|
|
"transformations": [],
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"description": "",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "Chunks",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 10,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "auto",
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
}
|
|
]
|
|
},
|
|
"unit": "cps"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 12,
|
|
"y": 24
|
|
},
|
|
"id": 149,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
"min",
|
|
"max",
|
|
"mean",
|
|
"last"
|
|
],
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "single",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "sum(rate(loki_logql_querystats_downloaded_chunk_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by(type,status_code)",
|
|
"hide": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "{{ type }} / {{ status_code }}",
|
|
"range": true,
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Loki LogQL - Downloaded Chunks",
|
|
"transformations": [],
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"description": "",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "Zime",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 10,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "auto",
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
}
|
|
]
|
|
},
|
|
"unit": "s"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 32
|
|
},
|
|
"id": 146,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
"min",
|
|
"max",
|
|
"mean",
|
|
"last"
|
|
],
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "single",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "histogram_quantile(0.95, sum(rate(loki_logql_querystats_chunk_download_latency_seconds_bucket{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (le,type,status_code))",
|
|
"hide": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "{{ type }} / {{ status_code }}",
|
|
"range": true,
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Loki LogQL - Chunk Download Latency",
|
|
"transformations": [],
|
|
"type": "timeseries"
|
|
}
|
|
],
|
|
"title": "Loki LogQL Metrics",
|
|
"type": "row"
|
|
},
|
|
{
|
|
"collapsed": true,
|
|
"gridPos": {
|
|
"h": 1,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 26
|
|
},
|
|
"id": 204,
|
|
"panels": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"description": "",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "Time",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 10,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "auto",
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
}
|
|
]
|
|
},
|
|
"unit": "s"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 9
|
|
},
|
|
"id": 206,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
"min",
|
|
"max",
|
|
"mean",
|
|
"last"
|
|
],
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "single",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "histogram_quantile(0.95, sum(rate(loki_querier_query_frontend_request_duration_seconds_bucket{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (le,operation,status_code))",
|
|
"hide": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "{{ operation }} / {{ status_code }}",
|
|
"range": true,
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Loki Querier - Frontend Request Duration",
|
|
"transformations": [],
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"description": "",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "Count",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 10,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "auto",
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
}
|
|
]
|
|
},
|
|
"unit": "none"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 12,
|
|
"y": 9
|
|
},
|
|
"id": 210,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
"min",
|
|
"max",
|
|
"mean",
|
|
"last"
|
|
],
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "single",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "rate(loki_query_frontend_log_result_cache_hit_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])",
|
|
"hide": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "Cache hits for the frontend cache",
|
|
"range": true,
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "rate(loki_query_frontend_log_result_cache_miss_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])",
|
|
"hide": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "Cache misses for the frontend cache",
|
|
"range": true,
|
|
"refId": "B"
|
|
}
|
|
],
|
|
"title": "Loki Querier - Frontend Cache",
|
|
"transformations": [],
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"description": "",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "Time",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 10,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "auto",
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
}
|
|
]
|
|
},
|
|
"unit": "s"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 17
|
|
},
|
|
"id": 211,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
"min",
|
|
"max",
|
|
"mean",
|
|
"last"
|
|
],
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "single",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "histogram_quantile(0.95, sum(rate(loki_query_frontend_partitions_bucket{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (le))",
|
|
"hide": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "Time-based partitions (sub-requests) per request",
|
|
"range": true,
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Loki Querier - Frontend Partitions",
|
|
"transformations": [],
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"description": "",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "Count",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 10,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "auto",
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
}
|
|
]
|
|
},
|
|
"unit": "none"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 12,
|
|
"y": 17
|
|
},
|
|
"id": 212,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
"min",
|
|
"max",
|
|
"mean",
|
|
"last"
|
|
],
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "single",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "histogram_quantile(0.95, sum(rate(loki_query_frontend_shard_factor_bucket{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (le,mapper))",
|
|
"hide": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "Downstream queries per request / {{ mapper }}",
|
|
"range": true,
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Loki Querier - Frontend Shard Factor",
|
|
"transformations": [],
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"description": "",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "Count",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 10,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "auto",
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
}
|
|
]
|
|
},
|
|
"unit": "none"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 25
|
|
},
|
|
"id": 213,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
"min",
|
|
"max",
|
|
"mean",
|
|
"last"
|
|
],
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "single",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "sum(rate(loki_query_frontend_sharding_parsed_queries_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (le,mapper,type)",
|
|
"hide": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "Parsed queries by evaluation type / {{ mapper }} / {{ type }}",
|
|
"range": true,
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "histogram_quantile(0.95, sum(rate(loki_query_frontend_shards_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (le,mapper,type))",
|
|
"hide": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "Downstream queries by expression type / {{ mapper }} / {{ type }}",
|
|
"range": true,
|
|
"refId": "B"
|
|
}
|
|
],
|
|
"title": "Loki Querier - Frontend Shards",
|
|
"transformations": [],
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"description": "",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "Count",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 10,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "auto",
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
}
|
|
]
|
|
},
|
|
"unit": "none"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 12,
|
|
"y": 25
|
|
},
|
|
"id": 209,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
"min",
|
|
"max",
|
|
"mean",
|
|
"last"
|
|
],
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "single",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "loki_querier_query_frontend_clients{job=\"$jobs\",instance=\"$host:$instance\"}",
|
|
"hide": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "Clients connected to query-frontend",
|
|
"range": true,
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "loki_querier_tail_active{job=\"$jobs\",instance=\"$host:$instance\"}",
|
|
"hide": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "Active tailers",
|
|
"range": true,
|
|
"refId": "B"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "loki_querier_tail_active_streams{job=\"$jobs\",instance=\"$host:$instance\"}",
|
|
"hide": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "Active streams being tailed",
|
|
"range": true,
|
|
"refId": "C"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "loki_querier_worker_concurrency{job=\"$jobs\",instance=\"$host:$instance\"}",
|
|
"hide": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "Concurrent querier workers",
|
|
"range": true,
|
|
"refId": "D"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "loki_querier_worker_inflight_queries{job=\"$jobs\",instance=\"$host:$instance\"}",
|
|
"hide": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "Queries being processed by the querier workers",
|
|
"range": true,
|
|
"refId": "E"
|
|
}
|
|
],
|
|
"title": "Loki Querier - Index Cache Clients / Workers",
|
|
"transformations": [],
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"description": "",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "Operations",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 10,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "auto",
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
}
|
|
]
|
|
},
|
|
"unit": "none"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 33
|
|
},
|
|
"id": 208,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
"min",
|
|
"max",
|
|
"mean",
|
|
"last"
|
|
],
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "single",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "rate(loki_querier_index_cache_gets_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])",
|
|
"hide": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "Gets for the index cache",
|
|
"range": true,
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "rate(loki_querier_index_cache_puts_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])",
|
|
"hide": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "Puts for the index cache",
|
|
"range": true,
|
|
"refId": "B"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "rate(loki_querier_index_cache_hits_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])",
|
|
"hide": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "Cache hits for the index cache",
|
|
"range": true,
|
|
"refId": "C"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "rate(loki_querier_index_cache_corruptions_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])",
|
|
"hide": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "Cache corruptions for the index cache",
|
|
"range": true,
|
|
"refId": "D"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "rate(loki_querier_index_cache_encode_errors_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])",
|
|
"hide": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "Errors for the index cache while encoding the body",
|
|
"range": true,
|
|
"refId": "E"
|
|
}
|
|
],
|
|
"title": "Loki Querier - Index Cache Operations",
|
|
"transformations": [],
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"description": "",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "Connections",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 10,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "auto",
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
}
|
|
]
|
|
},
|
|
"unit": "none"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 12,
|
|
"y": 33
|
|
},
|
|
"id": 219,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
"min",
|
|
"max",
|
|
"mean",
|
|
"last"
|
|
],
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "single",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "cortex_query_frontend_connected_schedulers{job=\"$jobs\",instance=\"$host:$instance\"}",
|
|
"hide": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "Schedulers this frontend is connected to",
|
|
"range": true,
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "cortex_query_scheduler_connected_frontend_clients{job=\"$jobs\",instance=\"$host:$instance\"}",
|
|
"hide": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "Query-frontend worker clients",
|
|
"range": true,
|
|
"refId": "B"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "cortex_query_scheduler_connected_querier_clients{job=\"$jobs\",instance=\"$host:$instance\"}",
|
|
"hide": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "Querier worker clients",
|
|
"range": true,
|
|
"refId": "C"
|
|
}
|
|
],
|
|
"title": "Cortex Query - Frontend Connections",
|
|
"transformations": [],
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"description": "",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "Operations",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 10,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "auto",
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
}
|
|
]
|
|
},
|
|
"unit": "cps"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 41
|
|
},
|
|
"id": 215,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
"min",
|
|
"max",
|
|
"mean",
|
|
"last"
|
|
],
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "single",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "sum(rate(querier_cache_added_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (cache)",
|
|
"hide": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "Entries added to the cache / {{ cache }}",
|
|
"range": true,
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "sum(rate(querier_cache_added_new_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (cache)",
|
|
"hide": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "New entries added to the cache / {{ cache }}",
|
|
"range": true,
|
|
"refId": "B"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "sum(rate(querier_cache_evicted_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (cache,reason)",
|
|
"hide": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "Entries evicted from the cache / {{ cache }} / {{ reason }}",
|
|
"range": true,
|
|
"refId": "C"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "sum(rate(querier_cache_gets_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (cache)",
|
|
"hide": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "Get calls to the cache / {{ cache }}",
|
|
"range": true,
|
|
"refId": "D"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "sum(rate(querier_cache_misses_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (cache)",
|
|
"hide": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "Missed calls to the cache / {{ cache }}",
|
|
"range": true,
|
|
"refId": "E"
|
|
}
|
|
],
|
|
"title": "Loki Query Cache - Throughput",
|
|
"transformations": [],
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"description": "",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "Count",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 10,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "auto",
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
}
|
|
]
|
|
},
|
|
"unit": "none"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 12,
|
|
"y": 41
|
|
},
|
|
"id": 214,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
"min",
|
|
"max",
|
|
"mean",
|
|
"last"
|
|
],
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "single",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "sum(querier_cache_entries{job=\"$jobs\",instance=\"$host:$instance\"}) by (cache)",
|
|
"hide": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "Cache Entries / {{ cache }}",
|
|
"range": true,
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Loki Query Cache - Entries",
|
|
"transformations": [],
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"description": "",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "Bytes",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 10,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "auto",
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
}
|
|
]
|
|
},
|
|
"unit": "bytes"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 49
|
|
},
|
|
"id": 216,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
"min",
|
|
"max",
|
|
"mean",
|
|
"last"
|
|
],
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "single",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "sum(querier_cache_memory_bytes{job=\"$jobs\",instance=\"$host:$instance\"}) by (cache)",
|
|
"hide": false,
|
|
"legendFormat": "Current cache size / {{ cache }}",
|
|
"range": true,
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Loki Query Cache - Size",
|
|
"transformations": [
|
|
{
|
|
"id": "renameByRegex",
|
|
"options": {
|
|
"regex": "(.*)fake$",
|
|
"renamePattern": "$1Default Tenant"
|
|
}
|
|
}
|
|
],
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"description": "",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "Count",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 10,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "auto",
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
}
|
|
]
|
|
},
|
|
"unit": "none"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 12,
|
|
"y": 49
|
|
},
|
|
"id": 207,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
"min",
|
|
"max",
|
|
"mean",
|
|
"last"
|
|
],
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "single",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "histogram_quantile(0.95, sum(rate(cortex_query_frontend_retries_bucket{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (le))",
|
|
"hide": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "Request retries",
|
|
"range": true,
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Cortex Query - Frontend Retries",
|
|
"transformations": [],
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"description": "",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "Time",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 10,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "auto",
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
}
|
|
]
|
|
},
|
|
"unit": "s"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 57
|
|
},
|
|
"id": 205,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
"min",
|
|
"max",
|
|
"mean",
|
|
"last"
|
|
],
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "single",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "histogram_quantile(0.95, sum(rate(cortex_frontend_query_range_duration_seconds_bucket{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (le,method,status_code))",
|
|
"hide": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "{{ method }} / {{ status_code }}",
|
|
"range": true,
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Cortex Query - Frontend Range Request Duration",
|
|
"transformations": [],
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"description": "",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "Connections",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 10,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "auto",
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
}
|
|
]
|
|
},
|
|
"unit": "none"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 12,
|
|
"y": 57
|
|
},
|
|
"id": 220,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
"min",
|
|
"max",
|
|
"mean",
|
|
"last"
|
|
],
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "single",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "cortex_query_frontend_queries_in_progress{job=\"$jobs\",instance=\"$host:$instance\"}",
|
|
"hide": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "Queries in progress",
|
|
"range": true,
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Cortex Query - Frontend Operations",
|
|
"transformations": [],
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"description": "",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "Requests",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 10,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "auto",
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
}
|
|
]
|
|
},
|
|
"unit": "none"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 65
|
|
},
|
|
"id": 221,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
"min",
|
|
"max",
|
|
"mean",
|
|
"last"
|
|
],
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "single",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "cortex_query_scheduler_inflight_requests{job=\"$jobs\",instance=\"$host:$instance\",quantile=\"0.99\"}",
|
|
"hide": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "Number of inflight requests (0.99 quantile)",
|
|
"range": true,
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "cortex_query_scheduler_inflight_requests{job=\"$jobs\",instance=\"$host:$instance\",quantile=\"0.95\"}",
|
|
"hide": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "Number of inflight requests (0.95 quantile)",
|
|
"range": true,
|
|
"refId": "B"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "cortex_query_scheduler_inflight_requests{job=\"$jobs\",instance=\"$host:$instance\",quantile=\"0.9\"}",
|
|
"hide": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "Number of inflight requests (0.9 quantile)",
|
|
"range": true,
|
|
"refId": "C"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "cortex_query_scheduler_inflight_requests{job=\"$jobs\",instance=\"$host:$instance\",quantile=\"0.8\"}",
|
|
"hide": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "Number of inflight requests (0.8 quantile)",
|
|
"range": true,
|
|
"refId": "D"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "cortex_query_scheduler_inflight_requests{job=\"$jobs\",instance=\"$host:$instance\",quantile=\"0.75\"}",
|
|
"hide": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "Number of inflight requests (0.75 quantile)",
|
|
"range": true,
|
|
"refId": "E"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "cortex_query_scheduler_inflight_requests{job=\"$jobs\",instance=\"$host:$instance\",quantile=\"0.5\"}",
|
|
"hide": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "Number of inflight requests (0.5 quantile)",
|
|
"range": true,
|
|
"refId": "F"
|
|
}
|
|
],
|
|
"title": "Cortex Query - Scheduler Inflight Requests",
|
|
"transformations": [],
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"description": "",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "Time",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 10,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "auto",
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
}
|
|
]
|
|
},
|
|
"unit": "s"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 12,
|
|
"y": 65
|
|
},
|
|
"id": 222,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
"min",
|
|
"max",
|
|
"mean",
|
|
"last"
|
|
],
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "single",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "histogram_quantile(0.95, sum(rate(cortex_query_scheduler_queue_duration_seconds_bucket{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (le))",
|
|
"hide": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "Time spend by requests in queue before getting picked up by a querier",
|
|
"range": true,
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Cortex Query - Scheduler Queue Duration",
|
|
"transformations": [],
|
|
"type": "timeseries"
|
|
}
|
|
],
|
|
"title": "Loki Querier",
|
|
"type": "row"
|
|
},
|
|
{
|
|
"collapsed": true,
|
|
"gridPos": {
|
|
"h": 1,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 27
|
|
},
|
|
"id": 224,
|
|
"panels": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"description": "",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "Heartbeats",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 10,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "auto",
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
}
|
|
]
|
|
},
|
|
"unit": "cps"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 10
|
|
},
|
|
"id": 225,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
"min",
|
|
"max",
|
|
"mean",
|
|
"last"
|
|
],
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "single",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "sum(rate(ring_member_heartbeats_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (name)",
|
|
"hide": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "Heartbeats sent / {{ name }}",
|
|
"range": true,
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Loki Ring - Heathbeats",
|
|
"transformations": [],
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"description": "",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "Tokens",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 10,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "auto",
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
}
|
|
]
|
|
},
|
|
"unit": "none"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 12,
|
|
"y": 10
|
|
},
|
|
"id": 226,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
"min",
|
|
"max",
|
|
"mean",
|
|
"last"
|
|
],
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "single",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "sum(ring_member_tokens_owned{job=\"$jobs\",instance=\"$host:$instance\"}) by (name)",
|
|
"hide": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "Tokens owned / {{ name }}",
|
|
"range": true,
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "sum(ring_member_tokens_to_own{job=\"$jobs\",instance=\"$host:$instance\"}) by (name)",
|
|
"hide": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "Tokens to be owned / {{ name }}",
|
|
"range": true,
|
|
"refId": "B"
|
|
}
|
|
],
|
|
"title": "Loki Ring - Tokens",
|
|
"transformations": [],
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"description": "",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "Heartbeats",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 10,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "auto",
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
}
|
|
]
|
|
},
|
|
"unit": "cps"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 18
|
|
},
|
|
"id": 227,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
"min",
|
|
"max",
|
|
"mean",
|
|
"last"
|
|
],
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "single",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "sum(rate(cortex_member_consul_heartbeats_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (name)",
|
|
"hide": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "Heartbeats sent / {{ name }}",
|
|
"range": true,
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Cortex Ring Member - Heathbeats",
|
|
"transformations": [],
|
|
"type": "timeseries"
|
|
}
|
|
],
|
|
"title": "Loki Ring Metrics",
|
|
"type": "row"
|
|
},
|
|
{
|
|
"collapsed": true,
|
|
"gridPos": {
|
|
"h": 1,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 28
|
|
},
|
|
"id": 133,
|
|
"panels": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"description": "",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "Chunks",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 10,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "auto",
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
}
|
|
]
|
|
},
|
|
"unit": "cps"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 11
|
|
},
|
|
"id": 135,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
"min",
|
|
"max",
|
|
"mean",
|
|
"last"
|
|
],
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "single",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "sum(rate(loki_store_series_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (status)",
|
|
"hide": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "{{ status }}",
|
|
"range": true,
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Loki Store - Series Referenced",
|
|
"transformations": [
|
|
{
|
|
"id": "calculateField",
|
|
"options": {
|
|
"mode": "reduceRow",
|
|
"reduce": {
|
|
"include": [
|
|
"debug",
|
|
"error",
|
|
"info",
|
|
"warn"
|
|
],
|
|
"reducer": "sum"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"id": "organize",
|
|
"options": {
|
|
"excludeByName": {},
|
|
"indexByName": {
|
|
"Time": 1,
|
|
"Total": 0,
|
|
"grpc": 2,
|
|
"http": 3
|
|
},
|
|
"renameByName": {
|
|
"debug": "Debug",
|
|
"discarded": "Discarded",
|
|
"error": "Error",
|
|
"grpc": "gRPC",
|
|
"http": "HTTP",
|
|
"info": "Info",
|
|
"matched": "Matched",
|
|
"warn": "Warn"
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"description": "",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "Chunks",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 10,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "auto",
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
}
|
|
]
|
|
},
|
|
"unit": "cps"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 12,
|
|
"y": 11
|
|
},
|
|
"id": 134,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
"min",
|
|
"max",
|
|
"mean",
|
|
"last"
|
|
],
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "single",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "sum(rate(loki_store_chunks_downloaded_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (status)",
|
|
"hide": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "{{ status }}",
|
|
"range": true,
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Loki Store - Chunks Downloaded",
|
|
"transformations": [
|
|
{
|
|
"id": "calculateField",
|
|
"options": {
|
|
"mode": "reduceRow",
|
|
"reduce": {
|
|
"include": [
|
|
"debug",
|
|
"error",
|
|
"info",
|
|
"warn"
|
|
],
|
|
"reducer": "sum"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"id": "organize",
|
|
"options": {
|
|
"excludeByName": {},
|
|
"indexByName": {
|
|
"Time": 1,
|
|
"Total": 0,
|
|
"grpc": 2,
|
|
"http": 3
|
|
},
|
|
"renameByName": {
|
|
"debug": "Debug",
|
|
"discarded": "Discarded",
|
|
"error": "Error",
|
|
"grpc": "gRPC",
|
|
"http": "HTTP",
|
|
"info": "Info",
|
|
"matched": "Matched",
|
|
"warn": "Warn"
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"description": "",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "Batch Size",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 10,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "auto",
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
}
|
|
]
|
|
},
|
|
"unit": "none"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 19
|
|
},
|
|
"id": 136,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
"min",
|
|
"max",
|
|
"mean",
|
|
"last"
|
|
],
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "single",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "histogram_quantile(0.95, sum(rate(loki_store_chunks_per_batch_bucket{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (le,status))",
|
|
"hide": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "{{ status }}",
|
|
"range": true,
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Loki Store - Chunk Batch Size",
|
|
"transformations": [],
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"description": "",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "Count",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 10,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "auto",
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
}
|
|
]
|
|
},
|
|
"unit": "cps"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 12,
|
|
"y": 19
|
|
},
|
|
"id": 165,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
"min",
|
|
"max",
|
|
"mean",
|
|
"last"
|
|
],
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "single",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "rate(loki_chunk_fetcher_cache_dequeued_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])",
|
|
"hide": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "Dequeue from buffer",
|
|
"range": true,
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "rate(loki_chunk_fetcher_cache_enqueued_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])",
|
|
"hide": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "Enqueue to buffer",
|
|
"range": true,
|
|
"refId": "B"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "rate(loki_chunk_fetcher_cache_skipped_buffer_full_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])",
|
|
"hide": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "Skipped due to buffer full",
|
|
"range": true,
|
|
"refId": "C"
|
|
}
|
|
],
|
|
"title": "Loki Store - Chunk Fetcher Cache",
|
|
"transformations": [],
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"description": "",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "Chunks",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 10,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "auto",
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
}
|
|
]
|
|
},
|
|
"unit": "cps"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 27
|
|
},
|
|
"id": 166,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
"min",
|
|
"max",
|
|
"mean",
|
|
"last"
|
|
],
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "single",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "sum(rate(loki_chunk_store_fetched_chunks_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (user)",
|
|
"hide": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "Fetched / {{ user }}",
|
|
"range": true,
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "sum(rate(loki_chunk_store_stored_chunks_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (user)",
|
|
"hide": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "Stored / {{ user }}",
|
|
"range": true,
|
|
"refId": "B"
|
|
}
|
|
],
|
|
"title": "Loki Store - Fetched / Stored Chunks",
|
|
"transformations": [
|
|
{
|
|
"id": "renameByRegex",
|
|
"options": {
|
|
"regex": "(.*)fake$",
|
|
"renamePattern": "$1Default Tenant"
|
|
}
|
|
}
|
|
],
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"description": "",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "Size",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 10,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "auto",
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
}
|
|
]
|
|
},
|
|
"unit": "binBps"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 12,
|
|
"y": 27
|
|
},
|
|
"id": 167,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
"min",
|
|
"max",
|
|
"mean",
|
|
"last"
|
|
],
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "single",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "rate(loki_chunk_store_fetched_chunk_bytes_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])",
|
|
"hide": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "Fetched / {{ user }}",
|
|
"range": true,
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "rate(loki_chunk_store_stored_chunk_bytes_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])",
|
|
"hide": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "Stored / {{ user }}",
|
|
"range": true,
|
|
"refId": "B"
|
|
}
|
|
],
|
|
"title": "Loki Store - Fetched / Stored Chunk Volume",
|
|
"transformations": [
|
|
{
|
|
"id": "renameByRegex",
|
|
"options": {
|
|
"regex": "(.*)fake$",
|
|
"renamePattern": "$1Default Tenant"
|
|
}
|
|
}
|
|
],
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"description": "",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"custom": {
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
}
|
|
},
|
|
"links": []
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 35
|
|
},
|
|
"id": 163,
|
|
"maxDataPoints": 50,
|
|
"options": {
|
|
"calculate": false,
|
|
"cellGap": 2,
|
|
"cellValues": {
|
|
"unit": "cps"
|
|
},
|
|
"color": {
|
|
"exponent": 0.5,
|
|
"fill": "red",
|
|
"mode": "scheme",
|
|
"reverse": false,
|
|
"scale": "exponential",
|
|
"scheme": "RdBu",
|
|
"steps": 64
|
|
},
|
|
"exemplars": {
|
|
"color": "rgba(255,0,255,0.7)"
|
|
},
|
|
"filterValues": {
|
|
"le": 0
|
|
},
|
|
"legend": {
|
|
"show": true
|
|
},
|
|
"rowsFrame": {
|
|
"layout": "auto"
|
|
},
|
|
"tooltip": {
|
|
"show": true,
|
|
"yHistogram": false
|
|
},
|
|
"yAxis": {
|
|
"axisLabel": "Bucket Entries",
|
|
"axisPlacement": "left",
|
|
"reverse": false,
|
|
"unit": "none"
|
|
}
|
|
},
|
|
"pluginVersion": "9.3.2",
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"exemplar": false,
|
|
"expr": "rate(loki_chunk_store_chunks_per_query_bucket{job=\"$jobs\", instance=\"$host:$instance\"}[$__rate_interval])",
|
|
"format": "heatmap",
|
|
"instant": false,
|
|
"interval": "",
|
|
"legendFormat": "{{ le }}",
|
|
"range": true,
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Loki Store - Chunks Per Query Distribution",
|
|
"transformations": [
|
|
{
|
|
"id": "organize",
|
|
"options": {
|
|
"excludeByName": {},
|
|
"indexByName": {},
|
|
"renameByName": {
|
|
"+Inf": "Unlimited",
|
|
"0.00010239999999999998": "10e-5",
|
|
"0.0010485759999999998": "10e-4",
|
|
"0.010485759999999998": "10e-3",
|
|
"0.10485759999999998": "10e-2",
|
|
"1.0239999999999999e-05": "10e-6",
|
|
"1.0239999999999999e-06": "10e-7",
|
|
"13568.999999999998": "13569",
|
|
"144.99999999999997": "145",
|
|
"1536.9999999999998": "1537",
|
|
"24.999999999999996": "25",
|
|
"27264.999999999996": "27265",
|
|
"320.99999999999994": "321",
|
|
"3200.9999999999995": "3201",
|
|
"64.99999999999999": "65",
|
|
"6528.999999999999": "6529",
|
|
"704.9999999999999": "705",
|
|
"8.999999999999998": "9",
|
|
"9.999999999999998e-08": "10e-8",
|
|
"9.999999999999999e-09": "10e-9",
|
|
"9.999999999999999e-10": "10e-10"
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"type": "heatmap"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"description": "",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "Bucket Entries",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 10,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "auto",
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
}
|
|
]
|
|
},
|
|
"unit": "cps"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 12,
|
|
"y": 35
|
|
},
|
|
"id": 156,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
"min",
|
|
"max",
|
|
"mean",
|
|
"last"
|
|
],
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "single",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "histogram_quantile(0.95, sum(rate(loki_chunk_store_chunks_per_query_bucket{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (le))",
|
|
"hide": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "{{ le }}",
|
|
"range": true,
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Loki Store - Chunks Per Query",
|
|
"transformations": [],
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"description": "",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"custom": {
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
}
|
|
},
|
|
"links": []
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 43
|
|
},
|
|
"id": 154,
|
|
"maxDataPoints": 50,
|
|
"options": {
|
|
"calculate": false,
|
|
"cellGap": 2,
|
|
"cellValues": {
|
|
"unit": "cps"
|
|
},
|
|
"color": {
|
|
"exponent": 0.5,
|
|
"fill": "red",
|
|
"mode": "scheme",
|
|
"reverse": false,
|
|
"scale": "exponential",
|
|
"scheme": "RdBu",
|
|
"steps": 64
|
|
},
|
|
"exemplars": {
|
|
"color": "rgba(255,0,255,0.7)"
|
|
},
|
|
"filterValues": {
|
|
"le": 0
|
|
},
|
|
"legend": {
|
|
"show": true
|
|
},
|
|
"rowsFrame": {
|
|
"layout": "auto"
|
|
},
|
|
"tooltip": {
|
|
"show": true,
|
|
"yHistogram": false
|
|
},
|
|
"yAxis": {
|
|
"axisLabel": "Bucket Entries",
|
|
"axisPlacement": "left",
|
|
"reverse": false,
|
|
"unit": "none"
|
|
}
|
|
},
|
|
"pluginVersion": "9.3.2",
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"exemplar": false,
|
|
"expr": "rate(loki_chunk_store_index_entries_per_chunk_bucket{job=\"$jobs\", instance=\"$host:$instance\"}[$__rate_interval])",
|
|
"format": "heatmap",
|
|
"instant": false,
|
|
"interval": "",
|
|
"legendFormat": "{{ le }}",
|
|
"range": true,
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Loki Store - Index Entries Per Chunk Distribution",
|
|
"transformations": [
|
|
{
|
|
"id": "organize",
|
|
"options": {
|
|
"excludeByName": {},
|
|
"indexByName": {},
|
|
"renameByName": {
|
|
"+Inf": "Unlimited",
|
|
"0.00010239999999999998": "10e-5",
|
|
"0.0010485759999999998": "10e-4",
|
|
"0.010485759999999998": "10e-3",
|
|
"0.10485759999999998": "10e-2",
|
|
"1.0239999999999999e-05": "10e-6",
|
|
"1.0239999999999999e-06": "10e-7",
|
|
"13568.999999999998": "13569",
|
|
"144.99999999999997": "145",
|
|
"1536.9999999999998": "1537",
|
|
"24.999999999999996": "25",
|
|
"27264.999999999996": "27265",
|
|
"320.99999999999994": "321",
|
|
"3200.9999999999995": "3201",
|
|
"64.99999999999999": "65",
|
|
"6528.999999999999": "6529",
|
|
"704.9999999999999": "705",
|
|
"8.999999999999998": "9",
|
|
"9.999999999999998e-08": "10e-8",
|
|
"9.999999999999999e-09": "10e-9",
|
|
"9.999999999999999e-10": "10e-10"
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"type": "heatmap"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"description": "",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "Bucket Entries",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 10,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "auto",
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
}
|
|
]
|
|
},
|
|
"unit": "cps"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 12,
|
|
"y": 43
|
|
},
|
|
"id": 164,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
"min",
|
|
"max",
|
|
"mean",
|
|
"last"
|
|
],
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "single",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "histogram_quantile(0.95, sum(rate(loki_chunk_store_index_entries_per_chunk_bucket{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (le))",
|
|
"hide": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "{{ le }}",
|
|
"range": true,
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Loki Store - Index Entries Per Chunk",
|
|
"transformations": [],
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"description": "",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"custom": {
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
}
|
|
},
|
|
"links": []
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 51
|
|
},
|
|
"id": 159,
|
|
"maxDataPoints": 50,
|
|
"options": {
|
|
"calculate": false,
|
|
"cellGap": 2,
|
|
"cellValues": {
|
|
"unit": "cps"
|
|
},
|
|
"color": {
|
|
"exponent": 0.5,
|
|
"fill": "red",
|
|
"mode": "scheme",
|
|
"reverse": false,
|
|
"scale": "exponential",
|
|
"scheme": "RdBu",
|
|
"steps": 64
|
|
},
|
|
"exemplars": {
|
|
"color": "rgba(255,0,255,0.7)"
|
|
},
|
|
"filterValues": {
|
|
"le": 0
|
|
},
|
|
"legend": {
|
|
"show": true
|
|
},
|
|
"rowsFrame": {
|
|
"layout": "auto"
|
|
},
|
|
"tooltip": {
|
|
"show": true,
|
|
"yHistogram": false
|
|
},
|
|
"yAxis": {
|
|
"axisLabel": "Bucket Entries",
|
|
"axisPlacement": "left",
|
|
"reverse": false,
|
|
"unit": "none"
|
|
}
|
|
},
|
|
"pluginVersion": "9.3.2",
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"exemplar": false,
|
|
"expr": "rate(loki_chunk_store_index_lookups_per_query_bucket{job=\"$jobs\", instance=\"$host:$instance\"}[$__rate_interval])",
|
|
"format": "heatmap",
|
|
"instant": false,
|
|
"interval": "",
|
|
"legendFormat": "{{ le }}",
|
|
"range": true,
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Loki Store - Index Lookups Per Query Distribution",
|
|
"transformations": [
|
|
{
|
|
"id": "organize",
|
|
"options": {
|
|
"excludeByName": {},
|
|
"indexByName": {},
|
|
"renameByName": {
|
|
"+Inf": "Unlimited",
|
|
"0.00010239999999999998": "10e-5",
|
|
"0.0010485759999999998": "10e-4",
|
|
"0.010485759999999998": "10e-3",
|
|
"0.10485759999999998": "10e-2",
|
|
"1.0239999999999999e-05": "10e-6",
|
|
"1.0239999999999999e-06": "10e-7",
|
|
"13568.999999999998": "13569",
|
|
"144.99999999999997": "145",
|
|
"1536.9999999999998": "1537",
|
|
"24.999999999999996": "25",
|
|
"27264.999999999996": "27265",
|
|
"320.99999999999994": "321",
|
|
"3200.9999999999995": "3201",
|
|
"64.99999999999999": "65",
|
|
"6528.999999999999": "6529",
|
|
"704.9999999999999": "705",
|
|
"8.999999999999998": "9",
|
|
"9.999999999999998e-08": "10e-8",
|
|
"9.999999999999999e-09": "10e-9",
|
|
"9.999999999999999e-10": "10e-10"
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"type": "heatmap"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"description": "",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "Bucket Entries",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 10,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "auto",
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
}
|
|
]
|
|
},
|
|
"unit": "cps"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 12,
|
|
"y": 51
|
|
},
|
|
"id": 160,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
"min",
|
|
"max",
|
|
"mean",
|
|
"last"
|
|
],
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "single",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "histogram_quantile(0.95, sum(rate(loki_chunk_store_index_lookups_per_query_bucket{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (le))",
|
|
"hide": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "{{ le }}",
|
|
"range": true,
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Loki Store - Index Lookups Per Query",
|
|
"transformations": [],
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"description": "",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"custom": {
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
}
|
|
},
|
|
"links": []
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 59
|
|
},
|
|
"id": 157,
|
|
"maxDataPoints": 50,
|
|
"options": {
|
|
"calculate": false,
|
|
"cellGap": 2,
|
|
"cellValues": {
|
|
"unit": "cps"
|
|
},
|
|
"color": {
|
|
"exponent": 0.5,
|
|
"fill": "red",
|
|
"mode": "scheme",
|
|
"reverse": false,
|
|
"scale": "exponential",
|
|
"scheme": "RdBu",
|
|
"steps": 64
|
|
},
|
|
"exemplars": {
|
|
"color": "rgba(255,0,255,0.7)"
|
|
},
|
|
"filterValues": {
|
|
"le": 0
|
|
},
|
|
"legend": {
|
|
"show": true
|
|
},
|
|
"rowsFrame": {
|
|
"layout": "auto"
|
|
},
|
|
"tooltip": {
|
|
"show": true,
|
|
"yHistogram": false
|
|
},
|
|
"yAxis": {
|
|
"axisLabel": "Bucket Entries",
|
|
"axisPlacement": "left",
|
|
"reverse": false,
|
|
"unit": "none"
|
|
}
|
|
},
|
|
"pluginVersion": "9.3.2",
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"exemplar": false,
|
|
"expr": "rate(loki_chunk_store_series_pre_intersection_per_query_bucket{job=\"$jobs\", instance=\"$host:$instance\"}[$__rate_interval])",
|
|
"format": "heatmap",
|
|
"instant": false,
|
|
"interval": "",
|
|
"legendFormat": "{{ le }}",
|
|
"range": true,
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Loki Store - Series (Pre Intersection) Per Chunk Distribution",
|
|
"transformations": [
|
|
{
|
|
"id": "organize",
|
|
"options": {
|
|
"excludeByName": {},
|
|
"indexByName": {},
|
|
"renameByName": {
|
|
"+Inf": "Unlimited",
|
|
"0.00010239999999999998": "10e-5",
|
|
"0.0010485759999999998": "10e-4",
|
|
"0.010485759999999998": "10e-3",
|
|
"0.10485759999999998": "10e-2",
|
|
"1.0239999999999999e-05": "10e-6",
|
|
"1.0239999999999999e-06": "10e-7",
|
|
"13568.999999999998": "13569",
|
|
"144.99999999999997": "145",
|
|
"1536.9999999999998": "1537",
|
|
"24.999999999999996": "25",
|
|
"27264.999999999996": "27265",
|
|
"320.99999999999994": "321",
|
|
"3200.9999999999995": "3201",
|
|
"64.99999999999999": "65",
|
|
"6528.999999999999": "6529",
|
|
"704.9999999999999": "705",
|
|
"8.999999999999998": "9",
|
|
"9.999999999999998e-08": "10e-8",
|
|
"9.999999999999999e-09": "10e-9",
|
|
"9.999999999999999e-10": "10e-10"
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"type": "heatmap"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"description": "",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "Bucket Entries",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 10,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "auto",
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
}
|
|
]
|
|
},
|
|
"unit": "cps"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 12,
|
|
"y": 59
|
|
},
|
|
"id": 158,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
"min",
|
|
"max",
|
|
"mean",
|
|
"last"
|
|
],
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "single",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "histogram_quantile(0.95, sum(rate(loki_chunk_store_series_pre_intersection_per_query_bucket{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (le))",
|
|
"hide": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "{{ le }}",
|
|
"range": true,
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Loki Store - Series (Pre Intersection) Per Chunk",
|
|
"transformations": [],
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"description": "",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"custom": {
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
}
|
|
},
|
|
"links": []
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 67
|
|
},
|
|
"id": 161,
|
|
"maxDataPoints": 50,
|
|
"options": {
|
|
"calculate": false,
|
|
"cellGap": 2,
|
|
"cellValues": {
|
|
"unit": "cps"
|
|
},
|
|
"color": {
|
|
"exponent": 0.5,
|
|
"fill": "red",
|
|
"mode": "scheme",
|
|
"reverse": false,
|
|
"scale": "exponential",
|
|
"scheme": "RdBu",
|
|
"steps": 64
|
|
},
|
|
"exemplars": {
|
|
"color": "rgba(255,0,255,0.7)"
|
|
},
|
|
"filterValues": {
|
|
"le": 0
|
|
},
|
|
"legend": {
|
|
"show": true
|
|
},
|
|
"rowsFrame": {
|
|
"layout": "auto"
|
|
},
|
|
"tooltip": {
|
|
"show": true,
|
|
"yHistogram": false
|
|
},
|
|
"yAxis": {
|
|
"axisLabel": "Bucket Entries",
|
|
"axisPlacement": "left",
|
|
"reverse": false,
|
|
"unit": "none"
|
|
}
|
|
},
|
|
"pluginVersion": "9.3.2",
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"exemplar": false,
|
|
"expr": "rate(loki_chunk_store_series_post_intersection_per_query_bucket{job=\"$jobs\", instance=\"$host:$instance\"}[$__rate_interval])",
|
|
"format": "heatmap",
|
|
"instant": false,
|
|
"interval": "",
|
|
"legendFormat": "{{ le }}",
|
|
"range": true,
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Loki Store - Series (Post Intersection) Per Chunk Distribution",
|
|
"transformations": [
|
|
{
|
|
"id": "organize",
|
|
"options": {
|
|
"excludeByName": {},
|
|
"indexByName": {},
|
|
"renameByName": {
|
|
"+Inf": "Unlimited",
|
|
"0.00010239999999999998": "10e-5",
|
|
"0.0010485759999999998": "10e-4",
|
|
"0.010485759999999998": "10e-3",
|
|
"0.10485759999999998": "10e-2",
|
|
"1.0239999999999999e-05": "10e-6",
|
|
"1.0239999999999999e-06": "10e-7",
|
|
"13568.999999999998": "13569",
|
|
"144.99999999999997": "145",
|
|
"1536.9999999999998": "1537",
|
|
"24.999999999999996": "25",
|
|
"27264.999999999996": "27265",
|
|
"320.99999999999994": "321",
|
|
"3200.9999999999995": "3201",
|
|
"64.99999999999999": "65",
|
|
"6528.999999999999": "6529",
|
|
"704.9999999999999": "705",
|
|
"8.999999999999998": "9",
|
|
"9.999999999999998e-08": "10e-8",
|
|
"9.999999999999999e-09": "10e-9",
|
|
"9.999999999999999e-10": "10e-10"
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"type": "heatmap"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"description": "",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "Bucket Entries",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 10,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "auto",
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
}
|
|
]
|
|
},
|
|
"unit": "cps"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 12,
|
|
"y": 67
|
|
},
|
|
"id": 162,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
"min",
|
|
"max",
|
|
"mean",
|
|
"last"
|
|
],
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "single",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "histogram_quantile(0.95, sum(rate(loki_chunk_store_series_post_intersection_per_query_bucket{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (le))",
|
|
"hide": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "{{ le }}",
|
|
"range": true,
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Loki Store - Series (Post Intersection) Per Chunk",
|
|
"transformations": [],
|
|
"type": "timeseries"
|
|
}
|
|
],
|
|
"title": "Loki Store Metrics",
|
|
"type": "row"
|
|
},
|
|
{
|
|
"collapsed": true,
|
|
"gridPos": {
|
|
"h": 1,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 29
|
|
},
|
|
"id": 170,
|
|
"panels": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"description": "",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "Errors",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 10,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "auto",
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
}
|
|
]
|
|
},
|
|
"unit": "none"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 12
|
|
},
|
|
"id": 171,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
"min",
|
|
"max",
|
|
"mean",
|
|
"last"
|
|
],
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "single",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "loki_table_manager_create_failures{job=\"$jobs\",instance=\"$host:$instance\"}",
|
|
"hide": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "Table create failures",
|
|
"range": true,
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "loki_table_manager_delete_failures{job=\"$jobs\",instance=\"$host:$instance\"}",
|
|
"hide": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "Table delete failures",
|
|
"range": true,
|
|
"refId": "B"
|
|
}
|
|
],
|
|
"title": "Loki Table Manager - Errors",
|
|
"transformations": [
|
|
{
|
|
"id": "renameByRegex",
|
|
"options": {
|
|
"regex": "(.*)fake$",
|
|
"renamePattern": "$1Default Tenant"
|
|
}
|
|
}
|
|
],
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"description": "",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "Time",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 10,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "auto",
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
}
|
|
]
|
|
},
|
|
"unit": "s"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 12,
|
|
"y": 12
|
|
},
|
|
"id": 172,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
"min",
|
|
"max",
|
|
"mean",
|
|
"last"
|
|
],
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "single",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "histogram_quantile(0.95, sum(rate(loki_table_manager_sync_duration_seconds_bucket{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (le,operation,status_code))",
|
|
"hide": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "{{ operation }} / {{ status_code }}",
|
|
"range": true,
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Loki Table Manager - Sync Duration",
|
|
"transformations": [
|
|
{
|
|
"id": "renameByRegex",
|
|
"options": {
|
|
"regex": "(.*)\\.(.*)",
|
|
"renamePattern": "$1 $2"
|
|
}
|
|
}
|
|
],
|
|
"type": "timeseries"
|
|
}
|
|
],
|
|
"title": "Loki Table Manager Metrics",
|
|
"type": "row"
|
|
},
|
|
{
|
|
"collapsed": true,
|
|
"gridPos": {
|
|
"h": 1,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 30
|
|
},
|
|
"id": 83,
|
|
"panels": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"description": "",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "Count",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 10,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "auto",
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
}
|
|
]
|
|
},
|
|
"unit": "none"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 13
|
|
},
|
|
"id": 168,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
"min",
|
|
"max",
|
|
"mean",
|
|
"last"
|
|
],
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "single",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "go_goroutines{job=\"$jobs\",instance=\"$host:$instance\"}",
|
|
"hide": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "Goroutines",
|
|
"range": true,
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "go_threads{job=\"$jobs\",instance=\"$host:$instance\"}",
|
|
"hide": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "Threads",
|
|
"range": true,
|
|
"refId": "B"
|
|
}
|
|
],
|
|
"title": "Go - Goroutines / Threads",
|
|
"transformations": [
|
|
{
|
|
"id": "renameByRegex",
|
|
"options": {
|
|
"regex": "(.*)fake$",
|
|
"renamePattern": "$1Default Tenant"
|
|
}
|
|
}
|
|
],
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"description": "",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "Avg. Latency",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 10,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "auto",
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
}
|
|
]
|
|
},
|
|
"unit": "s"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 12,
|
|
"y": 13
|
|
},
|
|
"id": 91,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
"min",
|
|
"max",
|
|
"mean",
|
|
"last"
|
|
],
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "single",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"exemplar": false,
|
|
"expr": "rate(go_sched_latencies_seconds_sum{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])/rate(go_sched_latencies_seconds_count{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])",
|
|
"hide": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "Average Goroutine latency",
|
|
"range": true,
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Go - Goroutines Latency",
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"description": "",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"custom": {
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
}
|
|
},
|
|
"links": []
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 21
|
|
},
|
|
"id": 94,
|
|
"maxDataPoints": 50,
|
|
"options": {
|
|
"calculate": false,
|
|
"cellGap": 2,
|
|
"cellValues": {
|
|
"unit": "cps"
|
|
},
|
|
"color": {
|
|
"exponent": 0.5,
|
|
"fill": "red",
|
|
"mode": "scheme",
|
|
"reverse": false,
|
|
"scale": "exponential",
|
|
"scheme": "RdBu",
|
|
"steps": 64
|
|
},
|
|
"exemplars": {
|
|
"color": "rgba(255,0,255,0.7)"
|
|
},
|
|
"filterValues": {
|
|
"le": 1E-9
|
|
},
|
|
"legend": {
|
|
"show": true
|
|
},
|
|
"rowsFrame": {
|
|
"layout": "auto"
|
|
},
|
|
"tooltip": {
|
|
"show": true,
|
|
"yHistogram": false
|
|
},
|
|
"yAxis": {
|
|
"axisLabel": "Bucket Latency",
|
|
"axisPlacement": "left",
|
|
"reverse": false,
|
|
"unit": "s"
|
|
}
|
|
},
|
|
"pluginVersion": "9.3.2",
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"exemplar": false,
|
|
"expr": "rate(go_sched_latencies_seconds_bucket{job=\"$jobs\", instance=\"$host:$instance\"}[$__rate_interval])",
|
|
"format": "heatmap",
|
|
"instant": false,
|
|
"interval": "",
|
|
"legendFormat": "{{ le }}",
|
|
"range": true,
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Go - Goroutines Latency Distribution",
|
|
"transformations": [
|
|
{
|
|
"id": "organize",
|
|
"options": {
|
|
"excludeByName": {},
|
|
"indexByName": {},
|
|
"renameByName": {
|
|
"+Inf": "Unlimited",
|
|
"0.00010239999999999998": "10e-5",
|
|
"0.0010485759999999998": "10e-4",
|
|
"0.010485759999999998": "10e-3",
|
|
"0.10485759999999998": "10e-2",
|
|
"1.0239999999999999e-05": "10e-6",
|
|
"1.0239999999999999e-06": "10e-7",
|
|
"13568.999999999998": "13569",
|
|
"144.99999999999997": "145",
|
|
"1536.9999999999998": "1537",
|
|
"24.999999999999996": "25",
|
|
"27264.999999999996": "27265",
|
|
"320.99999999999994": "321",
|
|
"3200.9999999999995": "3201",
|
|
"64.99999999999999": "65",
|
|
"6528.999999999999": "6529",
|
|
"704.9999999999999": "705",
|
|
"8.999999999999998": "9",
|
|
"9.999999999999998e-08": "10e-8",
|
|
"9.999999999999999e-09": "10e-9",
|
|
"9.999999999999999e-10": "10e-10"
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"type": "heatmap"
|
|
}
|
|
],
|
|
"title": "Go General Info",
|
|
"type": "row"
|
|
},
|
|
{
|
|
"collapsed": true,
|
|
"gridPos": {
|
|
"h": 1,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 31
|
|
},
|
|
"id": 104,
|
|
"panels": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"description": "",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "Bytes",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 10,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "auto",
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
},
|
|
{
|
|
"color": "red",
|
|
"value": 80
|
|
}
|
|
]
|
|
},
|
|
"unit": "bytes"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 14
|
|
},
|
|
"id": 92,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
"min",
|
|
"max",
|
|
"mean",
|
|
"last"
|
|
],
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "single",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "go_memory_classes_heap_free_bytes{job=\"$jobs\",instance=\"$host:$instance\"}",
|
|
"interval": "$interval",
|
|
"legendFormat": "Free memory",
|
|
"range": true,
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "go_memory_classes_heap_objects_bytes{job=\"$jobs\",instance=\"$host:$instance\"}",
|
|
"hide": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "Allocated objects memory",
|
|
"range": true,
|
|
"refId": "B"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "go_memory_classes_heap_released_bytes{job=\"$jobs\",instance=\"$host:$instance\"}",
|
|
"hide": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "Released memory",
|
|
"range": true,
|
|
"refId": "C"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "go_memory_classes_heap_stacks_bytes{job=\"$jobs\",instance=\"$host:$instance\"}",
|
|
"hide": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "Stack reserved memory",
|
|
"range": true,
|
|
"refId": "D"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "go_memory_classes_heap_unused_bytes{job=\"$jobs\",instance=\"$host:$instance\"}",
|
|
"hide": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "Reserved unused memory",
|
|
"range": true,
|
|
"refId": "E"
|
|
}
|
|
],
|
|
"title": "Go - Memory Heap",
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"description": "",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "Bytes",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 10,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "auto",
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
},
|
|
{
|
|
"color": "red",
|
|
"value": 80
|
|
}
|
|
]
|
|
},
|
|
"unit": "bytes"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 12,
|
|
"y": 14
|
|
},
|
|
"id": 93,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
"min",
|
|
"max",
|
|
"mean",
|
|
"last"
|
|
],
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "single",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "go_memory_classes_metadata_mcache_free_bytes{job=\"$jobs\",instance=\"$host:$instance\"}",
|
|
"interval": "$interval",
|
|
"legendFormat": "Free mcache memory",
|
|
"range": true,
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "go_memory_classes_metadata_mcache_inuse_bytes{job=\"$jobs\",instance=\"$host:$instance\"}",
|
|
"hide": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "Used mcache memory",
|
|
"range": true,
|
|
"refId": "B"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "go_memory_classes_metadata_mspan_free_bytes{job=\"$jobs\",instance=\"$host:$instance\"}",
|
|
"hide": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "Free mspan memory",
|
|
"range": true,
|
|
"refId": "C"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "go_memory_classes_metadata_mspan_inuse_bytes{job=\"$jobs\",instance=\"$host:$instance\"}",
|
|
"hide": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "Used mspan memory",
|
|
"range": true,
|
|
"refId": "D"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "go_memory_classes_metadata_other_bytes{job=\"$jobs\",instance=\"$host:$instance\"}",
|
|
"hide": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "Used other metadata memory",
|
|
"range": true,
|
|
"refId": "E"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "go_memory_classes_other_bytes{job=\"$jobs\",instance=\"$host:$instance\"}",
|
|
"hide": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "Used other memory",
|
|
"range": true,
|
|
"refId": "F"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "go_memory_classes_profiling_buckets_bytes{job=\"$jobs\",instance=\"$host:$instance\"}",
|
|
"hide": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "Used profiling bucket memory",
|
|
"range": true,
|
|
"refId": "G"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "go_memory_classes_total_bytes{job=\"$jobs\",instance=\"$host:$instance\"}",
|
|
"hide": true,
|
|
"interval": "$interval",
|
|
"legendFormat": "Used total memory",
|
|
"range": true,
|
|
"refId": "H"
|
|
}
|
|
],
|
|
"title": "Go - Memory Metadata",
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"description": "",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "Avg. Object Size",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 10,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "auto",
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
}
|
|
]
|
|
},
|
|
"unit": "bytes"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 22
|
|
},
|
|
"id": 95,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
"min",
|
|
"max",
|
|
"mean",
|
|
"last"
|
|
],
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "single",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"exemplar": false,
|
|
"expr": "rate(go_gc_heap_allocs_bytes_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])/rate(go_gc_heap_allocs_objects_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])",
|
|
"hide": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "Average object size allocated to the heap by the application",
|
|
"range": true,
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"exemplar": false,
|
|
"expr": "rate(go_gc_heap_frees_bytes_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])/rate(go_gc_heap_frees_objects_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])",
|
|
"hide": false,
|
|
"instant": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "Average object size freed from the heap by the GC",
|
|
"range": true,
|
|
"refId": "B"
|
|
}
|
|
],
|
|
"title": "Go - Heap Object Allocation / Free Sizes",
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"description": "",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"custom": {
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
}
|
|
},
|
|
"links": []
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 12,
|
|
"y": 22
|
|
},
|
|
"id": 89,
|
|
"maxDataPoints": 50,
|
|
"options": {
|
|
"calculate": false,
|
|
"cellGap": 2,
|
|
"cellValues": {
|
|
"unit": "cps"
|
|
},
|
|
"color": {
|
|
"exponent": 0.5,
|
|
"fill": "red",
|
|
"mode": "scheme",
|
|
"reverse": false,
|
|
"scale": "exponential",
|
|
"scheme": "PuBuGn",
|
|
"steps": 64
|
|
},
|
|
"exemplars": {
|
|
"color": "rgba(255,0,255,0.7)"
|
|
},
|
|
"filterValues": {
|
|
"le": 1E-9
|
|
},
|
|
"legend": {
|
|
"show": true
|
|
},
|
|
"rowsFrame": {
|
|
"layout": "auto"
|
|
},
|
|
"tooltip": {
|
|
"show": true,
|
|
"yHistogram": false
|
|
},
|
|
"yAxis": {
|
|
"axisLabel": "Bucket Size",
|
|
"axisPlacement": "left",
|
|
"reverse": false,
|
|
"unit": "bytes"
|
|
}
|
|
},
|
|
"pluginVersion": "9.3.2",
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"exemplar": false,
|
|
"expr": "rate(go_gc_heap_frees_by_size_bytes_bucket{job=\"$jobs\", instance=\"$host:$instance\"}[$__rate_interval])",
|
|
"format": "heatmap",
|
|
"instant": false,
|
|
"interval": "",
|
|
"legendFormat": "{{ le }}",
|
|
"range": true,
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Go - Heap Free Distribution",
|
|
"transformations": [
|
|
{
|
|
"id": "organize",
|
|
"options": {
|
|
"excludeByName": {},
|
|
"indexByName": {},
|
|
"renameByName": {
|
|
"+Inf": "Unlimited",
|
|
"13568.999999999998": "13569",
|
|
"144.99999999999997": "145",
|
|
"1536.9999999999998": "1537",
|
|
"24.999999999999996": "25",
|
|
"27264.999999999996": "27265",
|
|
"320.99999999999994": "321",
|
|
"3200.9999999999995": "3201",
|
|
"64.99999999999999": "65",
|
|
"6528.999999999999": "6529",
|
|
"704.9999999999999": "705",
|
|
"8.999999999999998": "9"
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"type": "heatmap"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"description": "",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"custom": {
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
}
|
|
},
|
|
"links": []
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 30
|
|
},
|
|
"id": 88,
|
|
"maxDataPoints": 50,
|
|
"options": {
|
|
"calculate": false,
|
|
"cellGap": 2,
|
|
"cellValues": {
|
|
"unit": "cps"
|
|
},
|
|
"color": {
|
|
"exponent": 0.5,
|
|
"fill": "red",
|
|
"mode": "scheme",
|
|
"reverse": false,
|
|
"scale": "exponential",
|
|
"scheme": "RdBu",
|
|
"steps": 64
|
|
},
|
|
"exemplars": {
|
|
"color": "rgba(255,0,255,0.7)"
|
|
},
|
|
"filterValues": {
|
|
"le": 1E-9
|
|
},
|
|
"legend": {
|
|
"show": true
|
|
},
|
|
"rowsFrame": {
|
|
"layout": "auto"
|
|
},
|
|
"tooltip": {
|
|
"show": true,
|
|
"yHistogram": false
|
|
},
|
|
"yAxis": {
|
|
"axisLabel": "Bucket Size",
|
|
"axisPlacement": "left",
|
|
"reverse": false,
|
|
"unit": "bytes"
|
|
}
|
|
},
|
|
"pluginVersion": "9.3.2",
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"exemplar": false,
|
|
"expr": "rate(go_gc_heap_allocs_by_size_bytes_bucket{job=\"$jobs\", instance=\"$host:$instance\"}[$__rate_interval])",
|
|
"format": "heatmap",
|
|
"instant": false,
|
|
"interval": "",
|
|
"legendFormat": "{{ le }}",
|
|
"range": true,
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Go - Heap Allocation Distribution",
|
|
"transformations": [
|
|
{
|
|
"id": "organize",
|
|
"options": {
|
|
"excludeByName": {},
|
|
"indexByName": {},
|
|
"renameByName": {
|
|
"+Inf": "Unlimited",
|
|
"13568.999999999998": "13569",
|
|
"144.99999999999997": "145",
|
|
"1536.9999999999998": "1537",
|
|
"24.999999999999996": "25",
|
|
"27264.999999999996": "27265",
|
|
"320.99999999999994": "321",
|
|
"3200.9999999999995": "3201",
|
|
"64.99999999999999": "65",
|
|
"6528.999999999999": "6529",
|
|
"704.9999999999999": "705",
|
|
"8.999999999999998": "9"
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"type": "heatmap"
|
|
}
|
|
],
|
|
"title": "Go Memory Metrics",
|
|
"type": "row"
|
|
},
|
|
{
|
|
"collapsed": true,
|
|
"gridPos": {
|
|
"h": 1,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 32
|
|
},
|
|
"id": 106,
|
|
"panels": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"description": "",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "Cycles",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 10,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "auto",
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
}
|
|
]
|
|
},
|
|
"unit": "cps"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 15
|
|
},
|
|
"id": 84,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
"min",
|
|
"max",
|
|
"mean",
|
|
"last"
|
|
],
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "single",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "rate(go_gc_cycles_total_gc_cycles_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])",
|
|
"hide": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "Count of all completed GC cycles",
|
|
"range": true,
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "rate(go_gc_cycles_automatic_gc_cycles_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])",
|
|
"hide": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "Count of completed GC cycles generated by the Go runtime",
|
|
"range": true,
|
|
"refId": "B"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "rate(go_gc_cycles_forced_gc_cycles_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])",
|
|
"interval": "$interval",
|
|
"legendFormat": "Count of completed GC cycles forced by the application.",
|
|
"range": true,
|
|
"refId": "C"
|
|
}
|
|
],
|
|
"title": "Go - Garbage Collection Cycles",
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"description": "",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "Avg. Pause Duration",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 10,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "auto",
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
}
|
|
]
|
|
},
|
|
"unit": "s"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 12,
|
|
"y": 15
|
|
},
|
|
"id": 85,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
"min",
|
|
"max",
|
|
"mean",
|
|
"last"
|
|
],
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "single",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "rate(go_gc_duration_seconds_sum{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])/rate(go_gc_duration_seconds_count{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])",
|
|
"hide": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "Average GC cycle pause duration",
|
|
"range": true,
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "rate(go_gc_pauses_seconds_sum{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])/rate(go_gc_pauses_seconds_count{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])",
|
|
"hide": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "Average GC cycle halt duration",
|
|
"range": true,
|
|
"refId": "B"
|
|
}
|
|
],
|
|
"title": "Go - Garbage Collection Pause Duration",
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"description": "",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"custom": {
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
}
|
|
},
|
|
"links": []
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 23
|
|
},
|
|
"id": 90,
|
|
"maxDataPoints": 50,
|
|
"options": {
|
|
"calculate": false,
|
|
"cellGap": 2,
|
|
"cellValues": {
|
|
"unit": "cps"
|
|
},
|
|
"color": {
|
|
"exponent": 0.5,
|
|
"fill": "red",
|
|
"mode": "scheme",
|
|
"reverse": false,
|
|
"scale": "exponential",
|
|
"scheme": "BrBG",
|
|
"steps": 64
|
|
},
|
|
"exemplars": {
|
|
"color": "rgba(255,0,255,0.7)"
|
|
},
|
|
"filterValues": {
|
|
"le": 1E-9
|
|
},
|
|
"legend": {
|
|
"show": true
|
|
},
|
|
"rowsFrame": {
|
|
"layout": "auto"
|
|
},
|
|
"tooltip": {
|
|
"show": true,
|
|
"yHistogram": false
|
|
},
|
|
"yAxis": {
|
|
"axisLabel": "Bucket Duration",
|
|
"axisPlacement": "left",
|
|
"reverse": false,
|
|
"unit": "s"
|
|
}
|
|
},
|
|
"pluginVersion": "9.3.2",
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"exemplar": false,
|
|
"expr": "rate(go_gc_pauses_seconds_bucket{job=\"$jobs\", instance=\"$host:$instance\"}[$__rate_interval])",
|
|
"format": "heatmap",
|
|
"instant": false,
|
|
"interval": "",
|
|
"legendFormat": "{{ le }}",
|
|
"range": true,
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Go - Garbage Collection Pauses Distribution",
|
|
"transformations": [
|
|
{
|
|
"id": "organize",
|
|
"options": {
|
|
"excludeByName": {},
|
|
"indexByName": {},
|
|
"renameByName": {
|
|
"+Inf": "Unlimited",
|
|
"0.00010239999999999998": "10e-5",
|
|
"0.0010485759999999998": "10e-4",
|
|
"0.010485759999999998": "10e-3",
|
|
"0.10485759999999998": "10e-2",
|
|
"1.0239999999999999e-05": "10e-6",
|
|
"1.0239999999999999e-06": "10e-7",
|
|
"9.999999999999998e-08": "10e-8",
|
|
"9.999999999999999e-09": "10e-9",
|
|
"9.999999999999999e-10": "10e-10"
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"type": "heatmap"
|
|
}
|
|
],
|
|
"title": "Go GC Metrics",
|
|
"type": "row"
|
|
},
|
|
{
|
|
"collapsed": true,
|
|
"gridPos": {
|
|
"h": 1,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 33
|
|
},
|
|
"id": 75,
|
|
"panels": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"description": "",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "Seconds",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 10,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "auto",
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
},
|
|
{
|
|
"color": "red",
|
|
"value": 80
|
|
}
|
|
]
|
|
},
|
|
"unit": "s"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 16
|
|
},
|
|
"id": 78,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
"min",
|
|
"max",
|
|
"mean",
|
|
"last"
|
|
],
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "single",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "rate(process_cpu_seconds_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])",
|
|
"hide": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "Total user and system CPU time spent in seconds",
|
|
"range": true,
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Process - CPU seconds",
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"description": "",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "Bytes",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 10,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "auto",
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
},
|
|
{
|
|
"color": "red",
|
|
"value": 80
|
|
}
|
|
]
|
|
},
|
|
"unit": "bytes"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 12,
|
|
"y": 16
|
|
},
|
|
"id": 79,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
"min",
|
|
"max",
|
|
"mean",
|
|
"last"
|
|
],
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "single",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "process_virtual_memory_bytes{job=\"$jobs\",instance=\"$host:$instance\"}",
|
|
"interval": "$interval",
|
|
"legendFormat": "Virtual memory size",
|
|
"range": true,
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "process_resident_memory_bytes{job=\"$jobs\",instance=\"$host:$instance\"}",
|
|
"hide": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "Resident memory size",
|
|
"range": true,
|
|
"refId": "B"
|
|
}
|
|
],
|
|
"title": "Process - Memory",
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"description": "",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "Descriptors",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 0,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "auto",
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
},
|
|
{
|
|
"color": "red",
|
|
"value": 80
|
|
}
|
|
]
|
|
},
|
|
"unit": "none"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 24
|
|
},
|
|
"id": 77,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
"min",
|
|
"max",
|
|
"mean",
|
|
"last"
|
|
],
|
|
"displayMode": "table",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "single",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "process_max_fds{job=\"$jobs\",instance=\"$host:$instance\"}",
|
|
"hide": false,
|
|
"interval": "$interval",
|
|
"legendFormat": "Maximum number of open file descriptors",
|
|
"range": true,
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "process_open_fds{job=\"$jobs\",instance=\"$host:$instance\"}",
|
|
"interval": "$interval",
|
|
"legendFormat": "Number of open file descriptors",
|
|
"range": true,
|
|
"refId": "B"
|
|
}
|
|
],
|
|
"title": "Process - File descriptors",
|
|
"type": "timeseries"
|
|
}
|
|
],
|
|
"title": "Process Info",
|
|
"type": "row"
|
|
}
|
|
],
|
|
"refresh": "1m",
|
|
"schemaVersion": 37,
|
|
"style": "dark",
|
|
"tags": [
|
|
"loki"
|
|
],
|
|
"templating": {
|
|
"list": [
|
|
{
|
|
"current": {
|
|
"selected": false,
|
|
"text": "default",
|
|
"value": "default"
|
|
},
|
|
"hide": 2,
|
|
"includeAll": false,
|
|
"label": "Datasource",
|
|
"multi": false,
|
|
"name": "DS_PROMETHEUS",
|
|
"options": [],
|
|
"query": "prometheus",
|
|
"refresh": 1,
|
|
"regex": "",
|
|
"skipUrlSync": false,
|
|
"type": "datasource"
|
|
},
|
|
{
|
|
"current": {},
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"definition": "label_values(loki_build_info,job)",
|
|
"hide": 0,
|
|
"includeAll": false,
|
|
"label": "Job",
|
|
"multi": false,
|
|
"name": "jobs",
|
|
"options": [],
|
|
"query": {
|
|
"query": "label_values(loki_build_info,job)",
|
|
"refId": "StandardVariableQuery"
|
|
},
|
|
"refresh": 1,
|
|
"regex": "",
|
|
"skipUrlSync": false,
|
|
"sort": 1,
|
|
"type": "query",
|
|
"useTags": false
|
|
},
|
|
{
|
|
"current": {},
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"definition": "up{job=~\"$jobs\"}",
|
|
"hide": 0,
|
|
"includeAll": false,
|
|
"label": "Host",
|
|
"multi": false,
|
|
"name": "host",
|
|
"options": [],
|
|
"query": {
|
|
"query": "up{job=~\"$jobs\"}",
|
|
"refId": "StandardVariableQuery"
|
|
},
|
|
"refresh": 1,
|
|
"regex": ".*instance=\"(.*?):.*\".*",
|
|
"skipUrlSync": false,
|
|
"sort": 1,
|
|
"type": "query"
|
|
},
|
|
{
|
|
"current": {},
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "PBFA97CFB590B2093"
|
|
},
|
|
"definition": "up{job=~\"$jobs\"}",
|
|
"hide": 0,
|
|
"includeAll": false,
|
|
"label": "Instance",
|
|
"multi": false,
|
|
"name": "instance",
|
|
"options": [],
|
|
"query": {
|
|
"query": "up{job=~\"$jobs\"}",
|
|
"refId": "StandardVariableQuery"
|
|
},
|
|
"refresh": 1,
|
|
"regex": ".*instance=\".*:(.*?)\".*",
|
|
"skipUrlSync": false,
|
|
"sort": 3,
|
|
"type": "query"
|
|
},
|
|
{
|
|
"auto": true,
|
|
"auto_count": 30,
|
|
"auto_min": "10s",
|
|
"current": {
|
|
"selected": false,
|
|
"text": "30s",
|
|
"value": "30s"
|
|
},
|
|
"hide": 0,
|
|
"label": "Interval",
|
|
"name": "interval",
|
|
"options": [
|
|
{
|
|
"selected": false,
|
|
"text": "auto",
|
|
"value": "$__auto_interval_interval"
|
|
},
|
|
{
|
|
"selected": true,
|
|
"text": "30s",
|
|
"value": "30s"
|
|
},
|
|
{
|
|
"selected": false,
|
|
"text": "1m",
|
|
"value": "1m"
|
|
},
|
|
{
|
|
"selected": false,
|
|
"text": "5m",
|
|
"value": "5m"
|
|
},
|
|
{
|
|
"selected": false,
|
|
"text": "1h",
|
|
"value": "1h"
|
|
},
|
|
{
|
|
"selected": false,
|
|
"text": "6h",
|
|
"value": "6h"
|
|
},
|
|
{
|
|
"selected": false,
|
|
"text": "1d",
|
|
"value": "1d"
|
|
}
|
|
],
|
|
"query": "30s,1m,5m,1h,6h,1d",
|
|
"refresh": 2,
|
|
"skipUrlSync": false,
|
|
"type": "interval"
|
|
}
|
|
]
|
|
},
|
|
"time": {
|
|
"from": "now-24h",
|
|
"to": "now"
|
|
},
|
|
"timepicker": {
|
|
"refresh_intervals": [
|
|
"5s",
|
|
"10s",
|
|
"30s",
|
|
"1m",
|
|
"5m",
|
|
"15m",
|
|
"30m",
|
|
"1h",
|
|
"2h",
|
|
"1d"
|
|
],
|
|
"time_options": [
|
|
"5m",
|
|
"15m",
|
|
"1h",
|
|
"6h",
|
|
"12h",
|
|
"24h",
|
|
"2d",
|
|
"7d",
|
|
"30d"
|
|
]
|
|
},
|
|
"timezone": "browser",
|
|
"title": "Loki Metrics Dashboard",
|
|
"uid": "f12e71ae-a681-4a80-83d8-3593a91d6598",
|
|
"version": 1,
|
|
"weekStart": ""
|
|
}
|