{ "annotations": { "list": [ { "$$hashKey": "object:7", "builtIn": 1, "datasource": { "type": "datasource", "uid": "grafana" }, "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", "name": "Annotations & Alerts", "target": { "limit": 100, "matchAny": false, "tags": [], "type": "dashboard" }, "type": "dashboard" } ] }, "description": "Loki Metrics Dashboard", "editable": false, "fiscalYearStartMonth": 0, "gnetId": 17781, "graphTooltip": 1, "id": 29, "links": [], "liveNow": false, "panels": [ { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, "id": 102, "panels": [], "title": "Loki General Info", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "links": [], "mappings": [ { "options": { "null": { "index": 0, "text": "N/A" } }, "type": "value" } ], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 3, "w": 3, "x": 0, "y": 1 }, "id": 59, "interval": "$interval", "maxDataPoints": 1, "options": { "colorMode": "background", "graphMode": "none", "justifyMode": "center", "orientation": "auto", "reduceOptions": { "calcs": [ "last" ], "fields": "/.*/", "values": false }, "textMode": "auto" }, "pluginVersion": "9.3.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "exemplar": false, "expr": "loki_build_info{job=\"$jobs\",instance=\"$host:$instance\"}", "format": "table", "instant": false, "interval": "$interval", "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "Loki Version", "transformations": [ { "id": "organize", "options": { "excludeByName": { "Time": true, "Value": true, "__name__": true, "goversion": true, "instance": true, "job": true, "version": true }, "indexByName": { "Time": 0, "Value": 8, "__name__": 1, "branch": 2, "goversion": 4, "instance": 5, "job": 6, "revision": 3, "version": 7 }, "renameByName": { "branch": "Loki Branch", "goversion": "", "instance": "", "revision": "Loki Revision", "version": "" } } } ], "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "", "fieldConfig": { "defaults": { "decimals": 0, "links": [], "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 3, "w": 3, "x": 3, "y": 1 }, "id": 10, "interval": "$interval", "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "center", "orientation": "auto", "reduceOptions": { "calcs": [ "last" ], "fields": "", "values": false }, "textMode": "auto" }, "pluginVersion": "9.3.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "sum(loki_internal_log_messages_total{job=\"$jobs\",instance=\"$host:$instance\"})", "instant": false, "interval": "$interval", "legendFormat": "", "refId": "A" } ], "title": "Internal Log Messages", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "fieldConfig": { "defaults": { "links": [], "mappings": [ { "options": { "null": { "index": 0, "text": "N/A" } }, "type": "value" } ], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 3, "w": 3, "x": 6, "y": 1 }, "id": 80, "interval": "$interval", "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "center", "orientation": "auto", "reduceOptions": { "calcs": [ "last" ], "fields": "", "values": false }, "textMode": "auto" }, "pluginVersion": "9.3.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "deprecated_flags_inuse_total{job=\"$jobs\",instance=\"$host:$instance\"}", "instant": false, "interval": "$interval", "legendFormat": "Deprecated", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "loki_experimental_features_in_use_total{job=\"$jobs\",instance=\"$host:$instance\"}", "hide": false, "instant": false, "interval": "$interval", "legendFormat": "Experimental", "refId": "B" } ], "title": "Features Currently Set", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "fieldConfig": { "defaults": { "links": [], "mappings": [ { "options": { "null": { "index": 0, "text": "N/A" } }, "type": "value" } ], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "semi-dark-red", "value": 1 } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 3, "w": 3, "x": 9, "y": 1 }, "id": 24, "interval": "$interval", "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "center", "orientation": "auto", "reduceOptions": { "calcs": [ "last" ], "fields": "", "values": false }, "textMode": "auto" }, "pluginVersion": "9.3.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "loki_panic_total{job=\"$jobs\",instance=\"$host:$instance\"}", "instant": false, "interval": "$interval", "legendFormat": "The total number of panics triggered", "refId": "A" } ], "title": "Panics", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "fieldConfig": { "defaults": { "links": [], "mappings": [ { "options": { "null": { "index": 0, "text": "N/A" } }, "type": "value" } ], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "semi-dark-red", "value": 0 }, { "color": "green", "value": 1 } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 3, "w": 3, "x": 12, "y": 1 }, "id": 112, "interval": "$interval", "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "center", "orientation": "auto", "reduceOptions": { "calcs": [ "last" ], "fields": "", "values": false }, "textMode": "auto" }, "pluginVersion": "9.3.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "loki_distributor_replication_factor{job=\"$jobs\",instance=\"$host:$instance\"}", "instant": false, "interval": "$interval", "legendFormat": "The configured replication factor", "refId": "A" } ], "title": "Replication Faktor", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "links": [], "mappings": [ { "options": { "null": { "index": 0, "text": "N/A" } }, "type": "value" } ], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 3, "w": 3, "x": 15, "y": 1 }, "id": 81, "interval": "$interval", "maxDataPoints": 1, "options": { "colorMode": "background", "graphMode": "none", "justifyMode": "center", "orientation": "auto", "reduceOptions": { "calcs": [ "last" ], "fields": "/.*/", "values": false }, "textMode": "auto" }, "pluginVersion": "9.3.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "exemplar": false, "expr": "go_info{job=\"$jobs\",instance=\"$host:$instance\"}", "format": "table", "instant": false, "interval": "$interval", "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "Go Version", "transformations": [ { "id": "organize", "options": { "excludeByName": { "Time": true, "Value": true, "__name__": true, "branch": true, "instance": true, "job": true, "revision": true, "version": false }, "indexByName": { "Time": 0, "Value": 8, "__name__": 1, "branch": 2, "goversion": 4, "instance": 5, "job": 6, "revision": 3, "version": 7 }, "renameByName": { "branch": "", "goversion": "Go Version", "instance": "", "revision": "", "version": "Go Version" } } } ], "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "links": [], "mappings": [ { "options": { "null": { "index": 0, "text": "N/A" } }, "type": "value" } ], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "semi-dark-red", "value": 500 } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 3, "w": 3, "x": 18, "y": 1 }, "id": 86, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "last" ], "fields": "", "values": false }, "textMode": "auto" }, "pluginVersion": "9.3.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "exemplar": false, "expr": "sum(go_goroutines{job=\"$jobs\",instance=\"$host:$instance\"})", "format": "table", "instant": false, "interval": "$interval", "legendFormat": "Number of goroutines that currently exist", "range": true, "refId": "A" } ], "title": "Total Goroutines", "transformations": [], "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "links": [], "mappings": [ { "options": { "null": { "index": 0, "text": "N/A" } }, "type": "value" } ], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "semi-dark-red", "value": 25 } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 3, "w": 3, "x": 21, "y": 1 }, "id": 87, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "last" ], "fields": "", "values": false }, "textMode": "auto" }, "pluginVersion": "9.3.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "exemplar": false, "expr": "sum(go_threads{job=\"$jobs\",instance=\"$host:$instance\"})", "format": "table", "instant": false, "interval": "$interval", "legendFormat": "Number of OS threads created", "range": true, "refId": "A" } ], "title": "Total Threads", "transformations": [], "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "fieldConfig": { "defaults": { "links": [], "mappings": [ { "options": { "null": { "index": 0, "text": "N/A" } }, "type": "value" } ], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "cps" }, "overrides": [] }, "gridPos": { "h": 6, "w": 6, "x": 0, "y": 4 }, "id": 235, "interval": "$interval", "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "center", "orientation": "auto", "reduceOptions": { "calcs": [ "last" ], "fields": "", "values": false }, "textMode": "auto" }, "pluginVersion": "9.3.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "sum(rate(loki_distributor_lines_received_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by(tenant)", "hide": false, "instant": false, "interval": "$interval", "legendFormat": "{{ tenant }}", "refId": "A" } ], "title": "Distributor Lines", "transformations": [ { "id": "calculateField", "options": {} }, { "id": "organize", "options": { "excludeByName": {}, "indexByName": {}, "renameByName": { "fake": "Default Tenant" } } } ], "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "fieldConfig": { "defaults": { "links": [], "mappings": [ { "options": { "null": { "index": 0, "text": "N/A" } }, "type": "value" } ], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "binBps" }, "overrides": [] }, "gridPos": { "h": 6, "w": 6, "x": 6, "y": 4 }, "id": 234, "interval": "$interval", "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "center", "orientation": "auto", "reduceOptions": { "calcs": [ "last" ], "fields": "", "values": false }, "textMode": "auto" }, "pluginVersion": "9.3.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "sum(rate(loki_distributor_bytes_received_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by(tenant)", "hide": false, "instant": false, "interval": "$interval", "legendFormat": "{{ tenant }}", "refId": "A" } ], "title": "Distributor Volume", "transformations": [ { "id": "calculateField", "options": { "mode": "reduceRow", "reduce": { "include": [], "reducer": "sum" } } }, { "id": "organize", "options": { "excludeByName": {}, "indexByName": {}, "renameByName": { "Received / fake": "Received / Default Tenant", "fake": "Default Tenant" } } } ], "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "fieldConfig": { "defaults": { "links": [], "mappings": [ { "options": { "null": { "index": 0, "text": "N/A" } }, "type": "value" } ], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "ops" }, "overrides": [] }, "gridPos": { "h": 6, "w": 6, "x": 12, "y": 4 }, "id": 230, "interval": "$interval", "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "center", "orientation": "auto", "reduceOptions": { "calcs": [ "last" ], "fields": "", "values": false }, "textMode": "auto" }, "pluginVersion": "9.3.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "rate(loki_ingester_chunks_created_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])", "hide": false, "instant": false, "interval": "$interval", "legendFormat": "Create", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "sum(rate(loki_ingester_chunks_stored_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (tenant) ", "hide": false, "instant": false, "interval": "$interval", "legendFormat": "Store / {{ tenant }}", "refId": "B" } ], "title": "Ingester Chunk Operations", "transformations": [ { "id": "calculateField", "options": {} }, { "id": "organize", "options": { "excludeByName": {}, "indexByName": {}, "renameByName": { "Store / fake": "Store / Default Tenant" } } } ], "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "fieldConfig": { "defaults": { "links": [], "mappings": [ { "options": { "null": { "index": 0, "text": "N/A" } }, "type": "value" } ], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "binBps" }, "overrides": [] }, "gridPos": { "h": 6, "w": 6, "x": 18, "y": 4 }, "id": 188, "interval": "$interval", "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "last" ], "fields": "", "values": false }, "textMode": "auto" }, "pluginVersion": "9.3.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "sum(rate(loki_ingester_chunk_stored_bytes_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (tenant)", "instant": false, "interval": "$interval", "legendFormat": "{{ tenant }}", "refId": "A" } ], "title": "Ingester Chunks Volume", "transformations": [ { "id": "calculateField", "options": {} }, { "id": "organize", "options": { "excludeByName": {}, "indexByName": {}, "renameByName": { "fake": "Default Tenant" } } } ], "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "fieldConfig": { "defaults": { "links": [], "mappings": [ { "options": { "null": { "index": 0, "text": "N/A" } }, "type": "value" } ], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 3, "w": 3, "x": 0, "y": 10 }, "id": 23, "interval": "$interval", "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "center", "orientation": "auto", "reduceOptions": { "calcs": [ "last" ], "fields": "", "values": false }, "textMode": "auto" }, "pluginVersion": "9.3.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "sum(loki_store_series_total{job=\"$jobs\",instance=\"$host:$instance\"}) by(status)", "instant": false, "interval": "$interval", "legendFormat": "{{ status }}", "refId": "A" } ], "title": "Store Series", "transformations": [ { "id": "calculateField", "options": {} }, { "id": "organize", "options": { "excludeByName": {}, "indexByName": {}, "renameByName": { "discarded": "Discarded", "matched": "Matched" } } } ], "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "fieldConfig": { "defaults": { "links": [], "mappings": [ { "options": { "null": { "index": 0, "text": "N/A" } }, "type": "value" } ], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 3, "w": 3, "x": 3, "y": 10 }, "id": 131, "interval": "$interval", "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "center", "orientation": "auto", "reduceOptions": { "calcs": [ "last" ], "fields": "", "values": false }, "textMode": "auto" }, "pluginVersion": "9.3.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "sum(loki_store_chunks_downloaded_total{job=\"$jobs\",instance=\"$host:$instance\"}) by(status)", "instant": false, "interval": "$interval", "legendFormat": "{{ status }}", "refId": "A" } ], "title": "Store Chunks Downloaded", "transformations": [ { "id": "calculateField", "options": {} }, { "id": "organize", "options": { "excludeByName": {}, "indexByName": {}, "renameByName": { "discarded": "Discarded", "matched": "Matched" } } } ], "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "fieldConfig": { "defaults": { "links": [], "mappings": [ { "options": { "null": { "index": 0, "text": "N/A" } }, "type": "value" } ], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 3, "w": 3, "x": 6, "y": 10 }, "id": 137, "interval": "$interval", "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "center", "orientation": "auto", "reduceOptions": { "calcs": [ "last" ], "fields": "", "values": false }, "textMode": "auto" }, "pluginVersion": "9.3.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "sum(loki_index_chunk_refs_total{job=\"$jobs\",instance=\"$host:$instance\"}) by(status)", "instant": false, "interval": "$interval", "legendFormat": "{{ status }}", "refId": "A" } ], "title": "Index Chunks Downloaded", "transformations": [ { "id": "calculateField", "options": {} }, { "id": "organize", "options": { "excludeByName": {}, "indexByName": {}, "renameByName": { "discarded": "Discarded", "matched": "Matched" } } } ], "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "fieldConfig": { "defaults": { "links": [], "mappings": [ { "options": { "null": { "index": 0, "text": "N/A" } }, "type": "value" } ], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 3, "w": 3, "x": 12, "y": 10 }, "id": 191, "interval": "$interval", "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "center", "orientation": "auto", "reduceOptions": { "calcs": [ "last" ], "fields": "", "values": false }, "textMode": "auto" }, "pluginVersion": "9.3.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "loki_ingester_memory_chunks{job=\"$jobs\",instance=\"$host:$instance\"}", "hide": false, "instant": false, "interval": "$interval", "legendFormat": "Chunks in memory", "refId": "B" } ], "title": "Ingester Chunks In Memory", "transformations": [], "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "fieldConfig": { "defaults": { "links": [], "mappings": [ { "options": { "null": { "index": 0, "text": "N/A" } }, "type": "value" } ], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 3, "w": 3, "x": 15, "y": 10 }, "id": 192, "interval": "$interval", "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "center", "orientation": "auto", "reduceOptions": { "calcs": [ "last" ], "fields": "", "values": false }, "textMode": "auto" }, "pluginVersion": "9.3.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "sum(loki_ingester_memory_streams{job=\"$jobs\",instance=\"$host:$instance\"}) by(tenant)", "instant": false, "interval": "$interval", "legendFormat": "{{ tenant }}", "refId": "A" } ], "title": "Ingester Streams in Memory", "transformations": [ { "id": "calculateField", "options": {} }, { "id": "organize", "options": { "excludeByName": {}, "indexByName": { "Time": 0, "Total": 2, "fake": 1 }, "renameByName": { "discarded": "Discarded", "fake": "Default Tenant", "matched": "Matched" } } } ], "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "fieldConfig": { "defaults": { "links": [], "mappings": [ { "options": { "null": { "index": 0, "text": "N/A" } }, "type": "value" } ], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "ops" }, "overrides": [] }, "gridPos": { "h": 6, "w": 6, "x": 18, "y": 10 }, "id": 233, "interval": "$interval", "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "center", "orientation": "auto", "reduceOptions": { "calcs": [ "last" ], "fields": "", "values": false }, "textMode": "auto" }, "pluginVersion": "9.3.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "sum(rate(loki_ingester_streams_created_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (tenant)", "hide": false, "instant": false, "interval": "$interval", "legendFormat": "Create / {{ tenant }}", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "sum(rate(loki_ingester_streams_removed_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (tenant)", "hide": false, "instant": false, "interval": "$interval", "legendFormat": "Delete / {{ tenant }}", "refId": "B" } ], "title": "Ingester Stream Operations", "transformations": [ { "id": "calculateField", "options": {} }, { "id": "organize", "options": { "excludeByName": {}, "indexByName": {}, "renameByName": { "Create / fake": "Create / Default Tenant", "Delete / fake": "Delete / Default Tenant", "Store / fake": "Store / Default Tenant" } } } ], "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "fieldConfig": { "defaults": { "links": [], "mappings": [ { "options": { "null": { "index": 0, "text": "N/A" } }, "type": "value" } ], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "ops" }, "overrides": [] }, "gridPos": { "h": 3, "w": 6, "x": 0, "y": 13 }, "id": 236, "interval": "$interval", "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "center", "orientation": "auto", "reduceOptions": { "calcs": [ "last" ], "fields": "", "values": false }, "textMode": "auto" }, "pluginVersion": "9.3.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "sum(rate(loki_chunk_store_fetched_chunks_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (user)", "hide": false, "instant": false, "interval": "$interval", "legendFormat": "{{ user}}", "refId": "A" } ], "title": "Store Operations Fetched", "transformations": [ { "id": "calculateField", "options": {} }, { "id": "organize", "options": { "excludeByName": {}, "indexByName": {}, "renameByName": { "Create / fake": "Create / Default Tenant", "Delete / fake": "Delete / Default Tenant", "Fetch / fake": "Fetch / Default Tenant", "Store / fake": "Store / Default Tenant", "fake": "Default Tenant" } } } ], "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "fieldConfig": { "defaults": { "links": [], "mappings": [ { "options": { "null": { "index": 0, "text": "N/A" } }, "type": "value" } ], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "ops" }, "overrides": [] }, "gridPos": { "h": 3, "w": 6, "x": 6, "y": 13 }, "id": 237, "interval": "$interval", "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "center", "orientation": "auto", "reduceOptions": { "calcs": [ "last" ], "fields": "", "values": false }, "textMode": "auto" }, "pluginVersion": "9.3.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "sum(rate(loki_chunk_store_stored_chunks_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (user)", "hide": false, "instant": false, "interval": "$interval", "legendFormat": "{{ user}}", "refId": "A" } ], "title": "Store Operations Stored", "transformations": [ { "id": "calculateField", "options": {} }, { "id": "organize", "options": { "excludeByName": {}, "indexByName": {}, "renameByName": { "Create / fake": "Create / Default Tenant", "Delete / fake": "Delete / Default Tenant", "Fetch / fake": "Default Tenant", "Store / fake": "Store / Default Tenant", "fake": "Default Tenant" } } } ], "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "fieldConfig": { "defaults": { "links": [], "mappings": [ { "options": { "null": { "index": 0, "text": "N/A" } }, "type": "value" } ], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "bytes" }, "overrides": [] }, "gridPos": { "h": 3, "w": 3, "x": 12, "y": 13 }, "id": 231, "interval": "$interval", "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "center", "orientation": "auto", "reduceOptions": { "calcs": [ "last" ], "fields": "", "values": false }, "textMode": "auto" }, "pluginVersion": "9.3.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "histogram_quantile(0.95, sum(rate(loki_ingester_chunk_size_bytes_bucket{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (le))", "hide": false, "instant": false, "interval": "$interval", "legendFormat": "Chunk Size", "refId": "A" } ], "title": "Ingester Chunk Size", "transformations": [], "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "fieldConfig": { "defaults": { "links": [], "mappings": [ { "options": { "null": { "index": 0, "text": "N/A" } }, "type": "value" } ], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "percent" }, "overrides": [] }, "gridPos": { "h": 3, "w": 3, "x": 15, "y": 13 }, "id": 232, "interval": "$interval", "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "center", "orientation": "auto", "reduceOptions": { "calcs": [ "last" ], "fields": "", "values": false }, "textMode": "auto" }, "pluginVersion": "9.3.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "histogram_quantile(0.95, sum(rate(loki_ingester_chunk_compression_ratio_bucket{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (le))", "hide": false, "instant": false, "interval": "$interval", "legendFormat": "Compression Ratio", "refId": "A" } ], "title": "Ingester Chunk Compression", "transformations": [], "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "fieldConfig": { "defaults": { "links": [], "mappings": [ { "options": { "null": { "index": 0, "text": "N/A" } }, "type": "value" } ], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "binBps" }, "overrides": [] }, "gridPos": { "h": 3, "w": 6, "x": 0, "y": 16 }, "id": 238, "interval": "$interval", "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "center", "orientation": "auto", "reduceOptions": { "calcs": [ "last" ], "fields": "", "values": false }, "textMode": "auto" }, "pluginVersion": "9.3.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "rate(loki_chunk_store_fetched_chunk_bytes_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])", "hide": false, "instant": false, "interval": "$interval", "legendFormat": "{{ user}}", "refId": "A" } ], "title": "Store Volume Fetched", "transformations": [ { "id": "calculateField", "options": {} }, { "id": "organize", "options": { "excludeByName": {}, "indexByName": {}, "renameByName": { "Create / fake": "Create / Default Tenant", "Delete / fake": "Delete / Default Tenant", "Fetch / fake": "Fetch / Default Tenant", "Store / fake": "Store / Default Tenant", "fake": "Default Tenant" } } } ], "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "fieldConfig": { "defaults": { "links": [], "mappings": [ { "options": { "null": { "index": 0, "text": "N/A" } }, "type": "value" } ], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "binBps" }, "overrides": [] }, "gridPos": { "h": 3, "w": 6, "x": 6, "y": 16 }, "id": 239, "interval": "$interval", "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "center", "orientation": "auto", "reduceOptions": { "calcs": [ "last" ], "fields": "", "values": false }, "textMode": "auto" }, "pluginVersion": "9.3.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "rate(loki_chunk_store_stored_chunk_bytes_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])", "hide": false, "instant": false, "interval": "$interval", "legendFormat": "{{ user}}", "refId": "A" } ], "title": "Store Volume Stored", "transformations": [ { "id": "calculateField", "options": {} }, { "id": "organize", "options": { "excludeByName": {}, "indexByName": {}, "renameByName": { "Create / fake": "Create / Default Tenant", "Delete / fake": "Delete / Default Tenant", "Fetch / fake": "Fetch / Default Tenant", "Store / fake": "Store / Default Tenant", "fake": "Default Tenant" } } } ], "type": "stat" }, { "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 19 }, "id": 143, "panels": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "Time", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 2 }, "id": 151, "options": { "legend": { "calcs": [ "min", "max", "mean", "last" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "histogram_quantile(0.95, sum(rate(loki_request_duration_seconds_bucket{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (le,method,route,status_code))", "hide": false, "interval": "$interval", "legendFormat": "{{ method }} / {{ route }} / {{ status_code }}", "range": true, "refId": "A" } ], "title": "Loki API - Request Duration", "transformations": [], "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "Requests", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "cps" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 2 }, "id": 128, "options": { "legend": { "calcs": [ "min", "max", "mean", "last" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "sum(rate(loki_inflight_requests{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (method, route)", "hide": false, "interval": "$interval", "legendFormat": "{{ method }} / {{ route }}", "range": true, "refId": "A" } ], "title": "Loki API - Request Current", "transformations": [], "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "Size", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "bytes" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 10 }, "id": 153, "options": { "legend": { "calcs": [ "min", "max", "mean", "last" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "histogram_quantile(0.95, sum(rate(loki_response_message_bytes_bucket{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (le,method,route))", "hide": false, "interval": "$interval", "legendFormat": "{{ method }} / {{ route }}", "range": true, "refId": "A" } ], "title": "Loki API - Response Size", "transformations": [], "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "Size", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "bytes" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 10 }, "id": 152, "options": { "legend": { "calcs": [ "min", "max", "mean", "last" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "histogram_quantile(0.95, sum(rate(loki_request_message_bytes_bucket{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (le,method,route))", "hide": false, "interval": "$interval", "legendFormat": "{{ method }} / {{ route }}", "range": true, "refId": "A" } ], "title": "Loki API - Request Size", "transformations": [], "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "Connections", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 18 }, "id": 96, "options": { "legend": { "calcs": [ "min", "max", "mean", "last" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "loki_tcp_connections{job=\"$jobs\",instance=\"$host:$instance\"}", "hide": false, "interval": "$interval", "legendFormat": "{{ protocol }}", "range": true, "refId": "A" } ], "title": "Loki - TCP Connections", "transformations": [ { "id": "calculateField", "options": { "mode": "reduceRow", "reduce": { "include": [ "grpc", "http" ], "reducer": "sum" } } }, { "id": "organize", "options": { "excludeByName": {}, "indexByName": { "Time": 1, "Total": 0, "grpc": 2, "http": 3 }, "renameByName": { "grpc": "gRPC", "http": "HTTP" } } } ], "type": "timeseries" } ], "title": "Loki API / Connections Metrics", "type": "row" }, { "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 20 }, "id": 114, "panels": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "Cache Hits", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "cps" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 3 }, "id": 115, "options": { "legend": { "calcs": [ "min", "max", "mean", "last" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "sum(rate(loki_cache_hits{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by(name)", "hide": false, "interval": "$interval", "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "Loki Cache - Hits", "transformations": [], "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "Keys", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "cps" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 3 }, "id": 116, "options": { "legend": { "calcs": [ "min", "max", "mean", "last" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "sum(rate(loki_cache_fetched_keys{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by(name)", "hide": false, "interval": "$interval", "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "Loki Cache - Fetched Keys", "transformations": [], "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "Time", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 11 }, "id": 118, "options": { "legend": { "calcs": [ "min", "max", "mean", "last" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "histogram_quantile(0.95, sum(rate(loki_cache_request_duration_seconds_bucket{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (le,name,method))", "hide": false, "interval": "$interval", "legendFormat": "{{ name }} / {{ method }}", "range": true, "refId": "A" } ], "title": "Loki Cache - Request Duration", "transformations": [ { "id": "renameByRegex", "options": { "regex": "(.*) / (.*)\\.(.*)", "renamePattern": "$1 / $3" } } ], "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "Size", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "bytes" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 11 }, "id": 119, "options": { "legend": { "calcs": [ "min", "max", "mean", "last" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "histogram_quantile(0.95, sum(rate(loki_cache_value_size_bytes_bucket{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (le,name,method))", "hide": false, "interval": "$interval", "legendFormat": "{{ name }} / {{ method }}", "range": true, "refId": "A" } ], "title": "Loki Cache - Value Size", "transformations": [], "type": "timeseries" } ], "title": "Loki Cache Metrics", "type": "row" }, { "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 21 }, "id": 108, "panels": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "Bytes", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "binBps" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 4 }, "id": 109, "options": { "legend": { "calcs": [ "min", "max", "mean", "last" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "sum(rate(loki_distributor_bytes_received_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by(tenant)", "hide": false, "interval": "$interval", "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "Loki Distributor - Received Bytes", "transformations": [ { "id": "organize", "options": { "excludeByName": {}, "indexByName": {}, "renameByName": { "fake": "Default Tenant" } } } ], "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "Lines", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "cps" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 4 }, "id": 110, "options": { "legend": { "calcs": [ "min", "max", "mean", "last" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "sum(rate(loki_distributor_lines_received_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by(tenant)", "hide": false, "interval": "$interval", "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "Loki Distributor - Received Lines", "transformations": [ { "id": "organize", "options": { "excludeByName": {}, "indexByName": {}, "renameByName": { "fake": "Default Tenant" } } } ], "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "Appends", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "cps" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 12 }, "id": 111, "options": { "legend": { "calcs": [ "min", "max", "mean", "last" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "sum(rate(loki_distributor_ingester_appends_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by(ingester)", "hide": false, "interval": "$interval", "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "Loki Distributor - Batch Appents Sent To Ingester", "transformations": [ { "id": "organize", "options": { "excludeByName": {}, "indexByName": {}, "renameByName": { "fake": "Default Tenant" } } } ], "type": "timeseries" } ], "title": "Loki Distributor Metrics", "type": "row" }, { "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 22 }, "id": 127, "panels": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "Time", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 5 }, "id": 130, "options": { "legend": { "calcs": [ "min", "max", "mean", "last" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "histogram_quantile(0.95, sum(rate(loki_index_request_duration_seconds_bucket{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (le,component,operation,status_code))", "hide": false, "interval": "$interval", "legendFormat": "{{ component }} / {{ operation }} / {{ status_code }}", "range": true, "refId": "A" } ], "title": "Loki Index - Request Duration", "transformations": [ { "id": "renameByRegex", "options": { "regex": "(.*) / (.*)\\.(.*)", "renamePattern": "$1 / $3" } } ], "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "Chunks", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "cps" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 5 }, "id": 138, "options": { "legend": { "calcs": [ "min", "max", "mean", "last" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "sum(rate(loki_index_chunk_refs_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (status)", "hide": false, "interval": "$interval", "legendFormat": "{{ status }}", "range": true, "refId": "A" } ], "title": "Loki Index - Chunks Downloaded", "transformations": [ { "id": "calculateField", "options": { "mode": "reduceRow", "reduce": { "include": [ "debug", "error", "info", "warn" ], "reducer": "sum" } } }, { "id": "organize", "options": { "excludeByName": {}, "indexByName": { "Time": 1, "Total": 0, "grpc": 2, "http": 3 }, "renameByName": { "debug": "Debug", "discarded": "Discarded", "error": "Error", "grpc": "gRPC", "http": "HTTP", "info": "Info", "matched": "Matched", "warn": "Warn" } } } ], "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "Time", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 13 }, "id": 201, "options": { "legend": { "calcs": [ "min", "max", "mean", "last" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "histogram_quantile(0.95, sum(rate(loki_kv_request_duration_seconds_bucket{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (le,kv_name,operation,role,status_code,type))", "hide": false, "interval": "$interval", "legendFormat": "{{ kv_name }} / {{ type }} / {{ operation }} / {{ status_code }}", "range": true, "refId": "A" } ], "title": "Loki KV - Request Duration", "transformations": [], "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "Time", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 13 }, "id": 202, "options": { "legend": { "calcs": [ "min", "max", "mean", "last" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "histogram_quantile(0.95, sum(rate(cortex_kv_request_duration_seconds_bucket{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (le,kv_name,operation,role,status_code,type))", "hide": false, "interval": "$interval", "legendFormat": "{{ kv_name }} / {{ type }} / {{ operation }} / {{ status_code }}", "range": true, "refId": "A" } ], "title": "Cortex KV - Request Duration", "transformations": [], "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "Time", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 21 }, "id": 228, "options": { "legend": { "calcs": [ "min", "max", "mean", "last" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "histogram_quantile(0.95, sum(rate(cortex_consul_request_duration_seconds_bucket{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (le,kv_name,operation,status_code))", "hide": false, "interval": "$interval", "legendFormat": "{{ kv_name }} / {{ operation }} / {{ status_code }}", "range": true, "refId": "A" } ], "title": "Cortex KV - Request Duration", "transformations": [], "type": "timeseries" } ], "title": "Loki Index Metrics", "type": "row" }, { "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 23 }, "id": 174, "panels": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "Time", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 6 }, "id": 189, "options": { "legend": { "calcs": [ "min", "max", "mean", "last" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "histogram_quantile(0.95, sum(rate(loki_ingester_client_request_duration_seconds_bucket{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (le,operation,status_code))", "hide": false, "interval": "$interval", "legendFormat": "{{ operation }} / {{ status_code }}", "range": true, "refId": "A" } ], "title": "Loki Ingester - Client Request Duration", "transformations": [], "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "Operations", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 6 }, "id": 185, "options": { "legend": { "calcs": [ "min", "max", "mean", "last" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "rate(loki_ingester_chunks_created_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])", "hide": false, "interval": "$interval", "legendFormat": "Chunk creations ", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "sum(rate(loki_ingester_chunks_flushed_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (reason)", "hide": false, "interval": "$interval", "legendFormat": "Chunk flushes / {{ reason }}", "range": true, "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "sum(rate(loki_ingester_chunks_stored_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (tenant) ", "hide": false, "interval": "$interval", "legendFormat": "Chunks stored / {{ tenant }}", "range": true, "refId": "C" } ], "title": "Loki Ingester - Chunk Operations", "transformations": [ { "id": "renameByRegex", "options": { "regex": "(.*)fake$", "renamePattern": "$1Default Tenant" } } ], "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "Checkpoints", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 14 }, "id": 175, "options": { "legend": { "calcs": [ "min", "max", "mean", "last" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "rate(loki_ingester_checkpoint_creations_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])", "hide": false, "interval": "$interval", "legendFormat": "Checkpoint creations ", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "rate(loki_ingester_checkpoint_creations_failed_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])", "hide": false, "interval": "$interval", "legendFormat": "Checkpoint creation failures", "range": true, "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "rate(loki_ingester_checkpoint_deletions_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])", "hide": false, "interval": "$interval", "legendFormat": "Checkpoint deletions", "range": true, "refId": "C" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "rate(loki_ingester_checkpoint_deletions_failed_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])", "hide": false, "interval": "$interval", "legendFormat": "Checkpoint deletion failures", "range": true, "refId": "D" } ], "title": "Loki Ingester - Checkpoints", "transformations": [ { "id": "renameByRegex", "options": { "regex": "(.*)fake$", "renamePattern": "$1Default Tenant" } } ], "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "Bytes", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "binBps" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 14 }, "id": 177, "options": { "legend": { "calcs": [ "min", "max", "mean", "last" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "rate(loki_ingester_checkpoint_logged_bytes_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])", "hide": false, "interval": "$interval", "legendFormat": "Bytes written to disk for checkpointing", "range": true, "refId": "A" } ], "title": "Loki Ingester - Checkpoints Volume", "transformations": [ { "id": "renameByRegex", "options": { "regex": "(.*)fake$", "renamePattern": "$1Default Tenant" } } ], "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "Time", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 22 }, "id": 183, "options": { "legend": { "calcs": [ "min", "max", "mean", "last" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "loki_ingester_checkpoint_duration_seconds{job=\"$jobs\",instance=\"$host:$instance\",quantile=\"0.99\"}", "hide": false, "interval": "$interval", "legendFormat": "Time taken to create a checkpoint (0.99 quantile)", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "loki_ingester_checkpoint_duration_seconds{job=\"$jobs\",instance=\"$host:$instance\",quantile=\"0.9\"}", "hide": false, "interval": "$interval", "legendFormat": "Time taken to create a checkpoint (0.9 quantile)", "range": true, "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "loki_ingester_checkpoint_duration_seconds{job=\"$jobs\",instance=\"$host:$instance\",quantile=\"0.5\"}", "hide": false, "interval": "$interval", "legendFormat": "Time taken to create a checkpoint (0.5 quantile)", "range": true, "refId": "C" } ], "title": "Loki Ingester - Checkpoint Duration", "transformations": [], "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "Blocks", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 22 }, "id": 176, "options": { "legend": { "calcs": [ "min", "max", "mean", "last" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "histogram_quantile(0.95, sum(rate(loki_ingester_blocks_per_chunk_bucket{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (le))", "hide": false, "interval": "$interval", "legendFormat": "Blocks per Chunk", "range": true, "refId": "A" } ], "title": "Loki Ingester - Blocks Per Chunk", "transformations": [], "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "%", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "percent" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 30 }, "id": 180, "options": { "legend": { "calcs": [ "min", "max", "mean", "last" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "histogram_quantile(0.95, sum(rate(loki_ingester_chunk_compression_ratio_bucket{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (le))", "hide": false, "interval": "$interval", "legendFormat": "Compression ratio of chunks (when stored)", "range": true, "refId": "A" } ], "title": "Loki Ingester - Chunk Compression Ratio", "transformations": [], "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "Count", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 30 }, "id": 190, "options": { "legend": { "calcs": [ "min", "max", "mean", "last" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "loki_ingester_memory_chunks{job=\"$jobs\",instance=\"$host:$instance\"}", "hide": false, "interval": "$interval", "legendFormat": "Chunks in memory", "range": true, "refId": "A" } ], "title": "Loki Ingester - Chunks In Memory", "transformations": [], "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "Bytes", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "bytes" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 38 }, "id": 184, "options": { "legend": { "calcs": [ "min", "max", "mean", "last" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "histogram_quantile(0.95, sum(rate(loki_ingester_chunk_size_bytes_bucket{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (le))", "hide": false, "interval": "$interval", "legendFormat": "Distribution of stored chunk sizes (when stored)", "range": true, "refId": "A" } ], "title": "Loki Ingester - Chunk Size", "transformations": [], "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "Time", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 38 }, "id": 178, "options": { "legend": { "calcs": [ "min", "max", "mean", "last" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "histogram_quantile(0.95, sum(rate(loki_ingester_chunk_age_seconds_bucket{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (le))", "hide": false, "interval": "$interval", "legendFormat": "Distribution of chunk ages (when stored)", "range": true, "refId": "A" } ], "title": "Loki Ingester - Chunk Age", "transformations": [], "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "%", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 46 }, "id": 186, "options": { "legend": { "calcs": [ "min", "max", "mean", "last" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "histogram_quantile(0.95, sum(rate(loki_ingester_chunk_utilization_bucket{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (le))", "hide": false, "interval": "$interval", "legendFormat": "Distribution of stored chunk utilization (when stored)", "range": true, "refId": "A" } ], "title": "Loki Ingester - Chunk Usage", "transformations": [], "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "Bytes", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "bytes" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 46 }, "id": 187, "options": { "legend": { "calcs": [ "min", "max", "mean", "last" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "sum(rate(loki_ingester_chunk_stored_bytes_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (tenant)", "hide": false, "interval": "$interval", "legendFormat": "Bytes stored in chunks / {{ tenant }}", "range": true, "refId": "A" } ], "title": "Loki Ingester - Chunk Stored Volume", "transformations": [ { "id": "renameByRegex", "options": { "regex": "(.*)fake$", "renamePattern": "$1Default Tenant" } } ], "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "Samples", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 54 }, "id": 194, "options": { "legend": { "calcs": [ "min", "max", "mean", "last" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "histogram_quantile(0.95, sum(rate(loki_ingester_samples_per_chunk_bucket{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (le))", "hide": false, "interval": "$interval", "legendFormat": "Samples per chunk", "range": true, "refId": "A" } ], "title": "Loki Ingester - Samples Per Chunk", "transformations": [], "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "Time", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "h" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 54 }, "id": 179, "options": { "legend": { "calcs": [ "min", "max", "mean", "last" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "histogram_quantile(0.95, sum(rate(loki_ingester_chunk_bounds_hours_bucket{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (le))", "hide": false, "interval": "$interval", "legendFormat": "Distribution of chunk end-start durations", "range": true, "refId": "A" } ], "title": "Loki Ingester - Chunk Bounds (Hours)", "transformations": [], "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "Time", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 62 }, "id": 181, "options": { "legend": { "calcs": [ "min", "max", "mean", "last" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "histogram_quantile(0.95, sum(rate(loki_ingester_chunk_encode_time_seconds_bucket{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (le))", "hide": false, "interval": "$interval", "legendFormat": "Distribution of chunk encode times", "range": true, "refId": "A" } ], "title": "Loki Ingester - Chunk Encoding Time", "transformations": [], "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "Lines", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 62 }, "id": 182, "options": { "legend": { "calcs": [ "min", "max", "mean", "last" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "histogram_quantile(0.95, sum(rate(loki_ingester_chunk_entries_bucket{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (le))", "hide": false, "interval": "$interval", "legendFormat": "Distribution of stored lines per chunk (when stored)", "range": true, "refId": "A" } ], "title": "Loki Ingester - Lines Per Chunk", "transformations": [], "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "Operations", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 70 }, "id": 196, "options": { "legend": { "calcs": [ "min", "max", "mean", "last" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "sum(rate(loki_ingester_streams_created_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (tenant)", "hide": false, "interval": "$interval", "legendFormat": "Stream creations / {{ tenant }}", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "sum(rate(loki_ingester_streams_removed_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (tenant)", "hide": false, "interval": "$interval", "legendFormat": "Stream deletetions / {{ tenant }}", "range": true, "refId": "B" } ], "title": "Loki Ingester - Stream Operations", "transformations": [ { "id": "renameByRegex", "options": { "regex": "(.*)fake$", "renamePattern": "$1Default Tenant" } } ], "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "Streams", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 70 }, "id": 193, "options": { "legend": { "calcs": [ "min", "max", "mean", "last" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "sum(loki_ingester_memory_streams{job=\"$jobs\",instance=\"$host:$instance\"}) by (tenant)", "hide": false, "interval": "$interval", "legendFormat": "Streams in memory / {{ tenant }}", "range": true, "refId": "B" } ], "title": "Loki Ingester - Streams In Memory", "transformations": [ { "id": "renameByRegex", "options": { "regex": "(.*)fake$", "renamePattern": "$1Default Tenant" } } ], "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "Records", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 78 }, "id": 197, "options": { "legend": { "calcs": [ "min", "max", "mean", "last" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "rate(loki_ingester_wal_records_logged_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])", "hide": false, "interval": "$interval", "legendFormat": "WAL records logged", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "rate(loki_ingester_wal_duplicate_entries_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])", "hide": false, "interval": "$interval", "legendFormat": "Discarded entries during WAL replay due to existing in checkpoints", "range": true, "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "rate(loki_ingester_wal_disk_full_failures_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])", "hide": false, "interval": "$interval", "legendFormat": "WAL write failures due to full disk", "range": true, "refId": "C" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "sum(rate(loki_ingester_wal_discarded_samples_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (reason)", "hide": false, "interval": "$interval", "legendFormat": "WAL segment entries discarded during replay / {{ reason }}", "range": true, "refId": "D" } ], "title": "Loki Ingester - WAL Records", "transformations": [ { "id": "renameByRegex", "options": { "regex": "(.*)fake$", "renamePattern": "$1Default Tenant" } } ], "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "Bytes", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "bytes" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 78 }, "id": 200, "options": { "legend": { "calcs": [ "min", "max", "mean", "last" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "loki_ingester_wal_bytes_in_use{job=\"$jobs\",instance=\"$host:$instance\"}", "hide": false, "legendFormat": "Currently in use by the WAL recovery process", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "rate(loki_ingester_wal_logged_bytes_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])", "hide": false, "interval": "$interval", "legendFormat": "Written to disk for WAL records", "range": true, "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "sum(rate(loki_ingester_wal_discarded_bytes_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (reason)", "hide": false, "interval": "$interval", "legendFormat": "WAL segment bytes discarded during replay / {{ reason }}", "range": true, "refId": "C" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "sum(rate(loki_ingester_wal_discarded_bytes_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (reason)", "hide": false, "interval": "$interval", "legendFormat": "Recovered from the WAL", "range": true, "refId": "D" } ], "title": "Loki Ingester - WAL Volume", "transformations": [ { "id": "renameByRegex", "options": { "regex": "(.*)fake$", "renamePattern": "$1Default Tenant" } } ], "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "Operations", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 86 }, "id": 198, "options": { "legend": { "calcs": [ "min", "max", "mean", "last" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "rate(loki_ingester_wal_recovered_entries_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])", "hide": false, "interval": "$interval", "legendFormat": "Entries recovered from the WAL", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "rate(loki_ingester_wal_recovered_chunks_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])", "hide": false, "interval": "$interval", "legendFormat": "Chunks recovered from the WAL checkpoints", "range": true, "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "rate(loki_ingester_wal_recovered_streams_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])", "hide": false, "interval": "$interval", "legendFormat": "Streams recovered from the WAL", "range": true, "refId": "C" } ], "title": "Loki Ingester - WAL Recovery Operations", "transformations": [ { "id": "renameByRegex", "options": { "regex": "(.*)fake$", "renamePattern": "$1Default Tenant" } } ], "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "Count", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 86 }, "id": 199, "options": { "legend": { "calcs": [ "min", "max", "mean", "last" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "loki_ingester_wal_replay_active{job=\"$jobs\",instance=\"$host:$instance\"}", "hide": false, "interval": "$interval", "legendFormat": "WAL is replaying", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "loki_ingester_wal_replay_flushing{job=\"$jobs\",instance=\"$host:$instance\"}", "hide": false, "interval": "$interval", "legendFormat": "WAL replay in flushing phase due to backpressure", "range": true, "refId": "B" } ], "title": "Loki Ingester - WAL Replay Status", "transformations": [ { "id": "renameByRegex", "options": { "regex": "(.*)fake$", "renamePattern": "$1Default Tenant" } } ], "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "Length", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 94 }, "id": 218, "options": { "legend": { "calcs": [ "min", "max", "mean", "last" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "cortex_ingester_flush_queue_length{job=\"$jobs\",instance=\"$host:$instance\"}", "hide": false, "interval": "$interval", "legendFormat": "Series pending in the flush queue", "range": true, "refId": "A" } ], "title": "Cortex Ingester - Flush Queue", "transformations": [], "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "Clients", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 94 }, "id": 229, "options": { "legend": { "calcs": [ "min", "max", "mean", "last" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "cortex_distributor_ingester_clients{job=\"$jobs\",instance=\"$host:$instance\"}", "hide": false, "interval": "$interval", "legendFormat": "Ingester clients", "range": true, "refId": "A" } ], "title": "Cortex Ingester - Clients", "transformations": [], "type": "timeseries" } ], "title": "Loki Ingester Metrics", "type": "row" }, { "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 24 }, "id": 121, "panels": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "Messages", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "cps" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 7 }, "id": 124, "options": { "legend": { "calcs": [ "min", "max", "mean", "last" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "sum(rate(loki_internal_log_messages_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (level)", "hide": false, "interval": "$interval", "legendFormat": "{{ protocol }}", "range": true, "refId": "A" } ], "title": "Loki - Internal Log Messages", "transformations": [ { "id": "calculateField", "options": { "mode": "reduceRow", "reduce": { "include": [ "debug", "error", "info", "warn" ], "reducer": "sum" } } }, { "id": "organize", "options": { "excludeByName": {}, "indexByName": { "Time": 1, "Total": 0, "grpc": 2, "http": 3 }, "renameByName": { "debug": "Debug", "error": "Error", "grpc": "gRPC", "http": "HTTP", "info": "Info", "warn": "Warn" } } } ], "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "Flushes", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 7 }, "id": 217, "options": { "legend": { "calcs": [ "min", "max", "mean", "last" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "histogram_quantile(0.95, sum(rate(loki_log_flushes_bucket{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (le))", "hide": false, "interval": "$interval", "legendFormat": "Log flushes - line-buffered logger", "range": true, "refId": "A" } ], "title": "Loki Line - Log Flushes", "transformations": [], "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "", "fieldConfig": { "defaults": { "custom": { "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "scaleDistribution": { "type": "linear" } }, "links": [] }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 15 }, "id": 122, "maxDataPoints": 50, "options": { "calculate": false, "cellGap": 2, "cellValues": { "unit": "bytes" }, "color": { "exponent": 0.5, "fill": "red", "mode": "scheme", "reverse": false, "scale": "exponential", "scheme": "BrBG", "steps": 64 }, "exemplars": { "color": "rgba(255,0,255,0.7)" }, "filterValues": { "le": 1E-9 }, "legend": { "show": true }, "rowsFrame": { "layout": "auto" }, "tooltip": { "show": true, "yHistogram": false }, "yAxis": { "axisLabel": "Bucket Size", "axisPlacement": "left", "reverse": false, "unit": "bytes" } }, "pluginVersion": "9.3.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "exemplar": false, "expr": "rate(loki_bytes_per_line_bucket{job=\"$jobs\", instance=\"$host:$instance\"}[$__rate_interval])", "format": "heatmap", "instant": false, "interval": "", "legendFormat": "{{ le }}", "range": true, "refId": "A" } ], "title": "Loki Line - Line Size Distribution", "transformations": [ { "id": "organize", "options": { "excludeByName": {}, "indexByName": {}, "renameByName": { "+Inf": "Unlimited", "0.00010239999999999998": "10e-5", "0.0010485759999999998": "10e-4", "0.010485759999999998": "10e-3", "0.10485759999999998": "10e-2", "1.0239999999999999e-05": "10e-6", "1.0239999999999999e-06": "10e-7", "9.999999999999998e-08": "10e-8", "9.999999999999999e-09": "10e-9", "9.999999999999999e-10": "10e-10" } } } ], "type": "heatmap" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "Size", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "bytes" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 15 }, "id": 123, "options": { "legend": { "calcs": [ "min", "max", "mean", "last" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "histogram_quantile(0.95, sum(rate(loki_bytes_per_line_bucket{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (le))", "hide": false, "interval": "$interval", "legendFormat": "Number of bytes per line", "range": true, "refId": "A" } ], "title": "Loki Line - Line Size", "transformations": [ { "id": "renameByRegex", "options": { "regex": "(.*) / (.*)\\.(.*)", "renamePattern": "$1 / $3" } } ], "type": "timeseries" } ], "title": "Loki Line Metrics", "type": "row" }, { "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 25 }, "id": 140, "panels": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "Lines", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "cps" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 8 }, "id": 148, "options": { "legend": { "calcs": [ "min", "max", "mean", "last" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "rate(loki_logql_querystats_ingester_sent_lines_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])", "hide": false, "interval": "$interval", "legendFormat": "Lines", "range": true, "refId": "A" } ], "title": "Loki LogQL - Lines Received From Ingester", "transformations": [], "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "Duplicates", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "cps" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 8 }, "id": 150, "options": { "legend": { "calcs": [ "min", "max", "mean", "last" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "rate(loki_logql_querystats_duplicates_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])", "hide": false, "interval": "$interval", "legendFormat": "Lines", "range": true, "refId": "A" } ], "title": "Loki LogQL - Duplicates Found", "transformations": [], "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "Time", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 16 }, "id": 125, "options": { "legend": { "calcs": [ "min", "max", "mean", "last" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "histogram_quantile(0.95, sum(rate(logql_query_duration_seconds_bucket{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (le,query_type))", "hide": false, "interval": "$interval", "legendFormat": "{{ query_type }}", "range": true, "refId": "A" } ], "title": "Loki LogQL - Query Duration", "transformations": [], "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "Zime", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 16 }, "id": 147, "options": { "legend": { "calcs": [ "min", "max", "mean", "last" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "histogram_quantile(0.95, sum(rate(loki_logql_querystats_latency_seconds_bucket{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (le,type,status_code))", "hide": false, "interval": "$interval", "legendFormat": "{{ type }} / {{ status_code }}", "range": true, "refId": "A" } ], "title": "Loki LogQL - Query Latency", "transformations": [], "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "Size", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "binBps" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 24 }, "id": 141, "options": { "legend": { "calcs": [ "min", "max", "mean", "last" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "histogram_quantile(0.95, sum(rate(loki_logql_querystats_bytes_processed_per_seconds_bucket{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (le,type,status_code))", "hide": false, "interval": "$interval", "legendFormat": "{{ type }} / {{ status_code }}", "range": true, "refId": "A" } ], "title": "Loki LogQL - Query Volume Processed", "transformations": [], "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "Chunks", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "cps" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 24 }, "id": 149, "options": { "legend": { "calcs": [ "min", "max", "mean", "last" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "sum(rate(loki_logql_querystats_downloaded_chunk_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by(type,status_code)", "hide": false, "interval": "$interval", "legendFormat": "{{ type }} / {{ status_code }}", "range": true, "refId": "A" } ], "title": "Loki LogQL - Downloaded Chunks", "transformations": [], "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "Zime", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 32 }, "id": 146, "options": { "legend": { "calcs": [ "min", "max", "mean", "last" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "histogram_quantile(0.95, sum(rate(loki_logql_querystats_chunk_download_latency_seconds_bucket{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (le,type,status_code))", "hide": false, "interval": "$interval", "legendFormat": "{{ type }} / {{ status_code }}", "range": true, "refId": "A" } ], "title": "Loki LogQL - Chunk Download Latency", "transformations": [], "type": "timeseries" } ], "title": "Loki LogQL Metrics", "type": "row" }, { "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 26 }, "id": 204, "panels": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "Time", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 9 }, "id": 206, "options": { "legend": { "calcs": [ "min", "max", "mean", "last" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "histogram_quantile(0.95, sum(rate(loki_querier_query_frontend_request_duration_seconds_bucket{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (le,operation,status_code))", "hide": false, "interval": "$interval", "legendFormat": "{{ operation }} / {{ status_code }}", "range": true, "refId": "A" } ], "title": "Loki Querier - Frontend Request Duration", "transformations": [], "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "Count", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 9 }, "id": 210, "options": { "legend": { "calcs": [ "min", "max", "mean", "last" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "rate(loki_query_frontend_log_result_cache_hit_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])", "hide": false, "interval": "$interval", "legendFormat": "Cache hits for the frontend cache", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "rate(loki_query_frontend_log_result_cache_miss_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])", "hide": false, "interval": "$interval", "legendFormat": "Cache misses for the frontend cache", "range": true, "refId": "B" } ], "title": "Loki Querier - Frontend Cache", "transformations": [], "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "Time", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 17 }, "id": 211, "options": { "legend": { "calcs": [ "min", "max", "mean", "last" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "histogram_quantile(0.95, sum(rate(loki_query_frontend_partitions_bucket{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (le))", "hide": false, "interval": "$interval", "legendFormat": "Time-based partitions (sub-requests) per request", "range": true, "refId": "A" } ], "title": "Loki Querier - Frontend Partitions", "transformations": [], "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "Count", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 17 }, "id": 212, "options": { "legend": { "calcs": [ "min", "max", "mean", "last" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "histogram_quantile(0.95, sum(rate(loki_query_frontend_shard_factor_bucket{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (le,mapper))", "hide": false, "interval": "$interval", "legendFormat": "Downstream queries per request / {{ mapper }}", "range": true, "refId": "A" } ], "title": "Loki Querier - Frontend Shard Factor", "transformations": [], "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "Count", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 25 }, "id": 213, "options": { "legend": { "calcs": [ "min", "max", "mean", "last" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "sum(rate(loki_query_frontend_sharding_parsed_queries_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (le,mapper,type)", "hide": false, "interval": "$interval", "legendFormat": "Parsed queries by evaluation type / {{ mapper }} / {{ type }}", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "histogram_quantile(0.95, sum(rate(loki_query_frontend_shards_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (le,mapper,type))", "hide": false, "interval": "$interval", "legendFormat": "Downstream queries by expression type / {{ mapper }} / {{ type }}", "range": true, "refId": "B" } ], "title": "Loki Querier - Frontend Shards", "transformations": [], "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "Count", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 25 }, "id": 209, "options": { "legend": { "calcs": [ "min", "max", "mean", "last" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "loki_querier_query_frontend_clients{job=\"$jobs\",instance=\"$host:$instance\"}", "hide": false, "interval": "$interval", "legendFormat": "Clients connected to query-frontend", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "loki_querier_tail_active{job=\"$jobs\",instance=\"$host:$instance\"}", "hide": false, "interval": "$interval", "legendFormat": "Active tailers", "range": true, "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "loki_querier_tail_active_streams{job=\"$jobs\",instance=\"$host:$instance\"}", "hide": false, "interval": "$interval", "legendFormat": "Active streams being tailed", "range": true, "refId": "C" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "loki_querier_worker_concurrency{job=\"$jobs\",instance=\"$host:$instance\"}", "hide": false, "interval": "$interval", "legendFormat": "Concurrent querier workers", "range": true, "refId": "D" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "loki_querier_worker_inflight_queries{job=\"$jobs\",instance=\"$host:$instance\"}", "hide": false, "interval": "$interval", "legendFormat": "Queries being processed by the querier workers", "range": true, "refId": "E" } ], "title": "Loki Querier - Index Cache Clients / Workers", "transformations": [], "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "Operations", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 33 }, "id": 208, "options": { "legend": { "calcs": [ "min", "max", "mean", "last" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "rate(loki_querier_index_cache_gets_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])", "hide": false, "interval": "$interval", "legendFormat": "Gets for the index cache", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "rate(loki_querier_index_cache_puts_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])", "hide": false, "interval": "$interval", "legendFormat": "Puts for the index cache", "range": true, "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "rate(loki_querier_index_cache_hits_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])", "hide": false, "interval": "$interval", "legendFormat": "Cache hits for the index cache", "range": true, "refId": "C" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "rate(loki_querier_index_cache_corruptions_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])", "hide": false, "interval": "$interval", "legendFormat": "Cache corruptions for the index cache", "range": true, "refId": "D" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "rate(loki_querier_index_cache_encode_errors_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])", "hide": false, "interval": "$interval", "legendFormat": "Errors for the index cache while encoding the body", "range": true, "refId": "E" } ], "title": "Loki Querier - Index Cache Operations", "transformations": [], "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "Connections", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 33 }, "id": 219, "options": { "legend": { "calcs": [ "min", "max", "mean", "last" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "cortex_query_frontend_connected_schedulers{job=\"$jobs\",instance=\"$host:$instance\"}", "hide": false, "interval": "$interval", "legendFormat": "Schedulers this frontend is connected to", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "cortex_query_scheduler_connected_frontend_clients{job=\"$jobs\",instance=\"$host:$instance\"}", "hide": false, "interval": "$interval", "legendFormat": "Query-frontend worker clients", "range": true, "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "cortex_query_scheduler_connected_querier_clients{job=\"$jobs\",instance=\"$host:$instance\"}", "hide": false, "interval": "$interval", "legendFormat": "Querier worker clients", "range": true, "refId": "C" } ], "title": "Cortex Query - Frontend Connections", "transformations": [], "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "Operations", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "cps" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 41 }, "id": 215, "options": { "legend": { "calcs": [ "min", "max", "mean", "last" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "sum(rate(querier_cache_added_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (cache)", "hide": false, "interval": "$interval", "legendFormat": "Entries added to the cache / {{ cache }}", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "sum(rate(querier_cache_added_new_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (cache)", "hide": false, "interval": "$interval", "legendFormat": "New entries added to the cache / {{ cache }}", "range": true, "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "sum(rate(querier_cache_evicted_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (cache,reason)", "hide": false, "interval": "$interval", "legendFormat": "Entries evicted from the cache / {{ cache }} / {{ reason }}", "range": true, "refId": "C" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "sum(rate(querier_cache_gets_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (cache)", "hide": false, "interval": "$interval", "legendFormat": "Get calls to the cache / {{ cache }}", "range": true, "refId": "D" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "sum(rate(querier_cache_misses_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (cache)", "hide": false, "interval": "$interval", "legendFormat": "Missed calls to the cache / {{ cache }}", "range": true, "refId": "E" } ], "title": "Loki Query Cache - Throughput", "transformations": [], "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "Count", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 41 }, "id": 214, "options": { "legend": { "calcs": [ "min", "max", "mean", "last" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "sum(querier_cache_entries{job=\"$jobs\",instance=\"$host:$instance\"}) by (cache)", "hide": false, "interval": "$interval", "legendFormat": "Cache Entries / {{ cache }}", "range": true, "refId": "A" } ], "title": "Loki Query Cache - Entries", "transformations": [], "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "Bytes", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "bytes" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 49 }, "id": 216, "options": { "legend": { "calcs": [ "min", "max", "mean", "last" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "sum(querier_cache_memory_bytes{job=\"$jobs\",instance=\"$host:$instance\"}) by (cache)", "hide": false, "legendFormat": "Current cache size / {{ cache }}", "range": true, "refId": "A" } ], "title": "Loki Query Cache - Size", "transformations": [ { "id": "renameByRegex", "options": { "regex": "(.*)fake$", "renamePattern": "$1Default Tenant" } } ], "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "Count", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 49 }, "id": 207, "options": { "legend": { "calcs": [ "min", "max", "mean", "last" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "histogram_quantile(0.95, sum(rate(cortex_query_frontend_retries_bucket{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (le))", "hide": false, "interval": "$interval", "legendFormat": "Request retries", "range": true, "refId": "A" } ], "title": "Cortex Query - Frontend Retries", "transformations": [], "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "Time", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 57 }, "id": 205, "options": { "legend": { "calcs": [ "min", "max", "mean", "last" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "histogram_quantile(0.95, sum(rate(cortex_frontend_query_range_duration_seconds_bucket{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (le,method,status_code))", "hide": false, "interval": "$interval", "legendFormat": "{{ method }} / {{ status_code }}", "range": true, "refId": "A" } ], "title": "Cortex Query - Frontend Range Request Duration", "transformations": [], "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "Connections", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 57 }, "id": 220, "options": { "legend": { "calcs": [ "min", "max", "mean", "last" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "cortex_query_frontend_queries_in_progress{job=\"$jobs\",instance=\"$host:$instance\"}", "hide": false, "interval": "$interval", "legendFormat": "Queries in progress", "range": true, "refId": "A" } ], "title": "Cortex Query - Frontend Operations", "transformations": [], "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "Requests", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 65 }, "id": 221, "options": { "legend": { "calcs": [ "min", "max", "mean", "last" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "cortex_query_scheduler_inflight_requests{job=\"$jobs\",instance=\"$host:$instance\",quantile=\"0.99\"}", "hide": false, "interval": "$interval", "legendFormat": "Number of inflight requests (0.99 quantile)", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "cortex_query_scheduler_inflight_requests{job=\"$jobs\",instance=\"$host:$instance\",quantile=\"0.95\"}", "hide": false, "interval": "$interval", "legendFormat": "Number of inflight requests (0.95 quantile)", "range": true, "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "cortex_query_scheduler_inflight_requests{job=\"$jobs\",instance=\"$host:$instance\",quantile=\"0.9\"}", "hide": false, "interval": "$interval", "legendFormat": "Number of inflight requests (0.9 quantile)", "range": true, "refId": "C" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "cortex_query_scheduler_inflight_requests{job=\"$jobs\",instance=\"$host:$instance\",quantile=\"0.8\"}", "hide": false, "interval": "$interval", "legendFormat": "Number of inflight requests (0.8 quantile)", "range": true, "refId": "D" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "cortex_query_scheduler_inflight_requests{job=\"$jobs\",instance=\"$host:$instance\",quantile=\"0.75\"}", "hide": false, "interval": "$interval", "legendFormat": "Number of inflight requests (0.75 quantile)", "range": true, "refId": "E" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "cortex_query_scheduler_inflight_requests{job=\"$jobs\",instance=\"$host:$instance\",quantile=\"0.5\"}", "hide": false, "interval": "$interval", "legendFormat": "Number of inflight requests (0.5 quantile)", "range": true, "refId": "F" } ], "title": "Cortex Query - Scheduler Inflight Requests", "transformations": [], "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "Time", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 65 }, "id": 222, "options": { "legend": { "calcs": [ "min", "max", "mean", "last" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "histogram_quantile(0.95, sum(rate(cortex_query_scheduler_queue_duration_seconds_bucket{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (le))", "hide": false, "interval": "$interval", "legendFormat": "Time spend by requests in queue before getting picked up by a querier", "range": true, "refId": "A" } ], "title": "Cortex Query - Scheduler Queue Duration", "transformations": [], "type": "timeseries" } ], "title": "Loki Querier", "type": "row" }, { "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 27 }, "id": 224, "panels": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "Heartbeats", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "cps" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 10 }, "id": 225, "options": { "legend": { "calcs": [ "min", "max", "mean", "last" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "sum(rate(ring_member_heartbeats_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (name)", "hide": false, "interval": "$interval", "legendFormat": "Heartbeats sent / {{ name }}", "range": true, "refId": "A" } ], "title": "Loki Ring - Heathbeats", "transformations": [], "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "Tokens", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 10 }, "id": 226, "options": { "legend": { "calcs": [ "min", "max", "mean", "last" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "sum(ring_member_tokens_owned{job=\"$jobs\",instance=\"$host:$instance\"}) by (name)", "hide": false, "interval": "$interval", "legendFormat": "Tokens owned / {{ name }}", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "sum(ring_member_tokens_to_own{job=\"$jobs\",instance=\"$host:$instance\"}) by (name)", "hide": false, "interval": "$interval", "legendFormat": "Tokens to be owned / {{ name }}", "range": true, "refId": "B" } ], "title": "Loki Ring - Tokens", "transformations": [], "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "Heartbeats", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "cps" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 18 }, "id": 227, "options": { "legend": { "calcs": [ "min", "max", "mean", "last" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "sum(rate(cortex_member_consul_heartbeats_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (name)", "hide": false, "interval": "$interval", "legendFormat": "Heartbeats sent / {{ name }}", "range": true, "refId": "A" } ], "title": "Cortex Ring Member - Heathbeats", "transformations": [], "type": "timeseries" } ], "title": "Loki Ring Metrics", "type": "row" }, { "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 28 }, "id": 133, "panels": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "Chunks", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "cps" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 11 }, "id": 135, "options": { "legend": { "calcs": [ "min", "max", "mean", "last" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "sum(rate(loki_store_series_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (status)", "hide": false, "interval": "$interval", "legendFormat": "{{ status }}", "range": true, "refId": "A" } ], "title": "Loki Store - Series Referenced", "transformations": [ { "id": "calculateField", "options": { "mode": "reduceRow", "reduce": { "include": [ "debug", "error", "info", "warn" ], "reducer": "sum" } } }, { "id": "organize", "options": { "excludeByName": {}, "indexByName": { "Time": 1, "Total": 0, "grpc": 2, "http": 3 }, "renameByName": { "debug": "Debug", "discarded": "Discarded", "error": "Error", "grpc": "gRPC", "http": "HTTP", "info": "Info", "matched": "Matched", "warn": "Warn" } } } ], "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "Chunks", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "cps" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 11 }, "id": 134, "options": { "legend": { "calcs": [ "min", "max", "mean", "last" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "sum(rate(loki_store_chunks_downloaded_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (status)", "hide": false, "interval": "$interval", "legendFormat": "{{ status }}", "range": true, "refId": "A" } ], "title": "Loki Store - Chunks Downloaded", "transformations": [ { "id": "calculateField", "options": { "mode": "reduceRow", "reduce": { "include": [ "debug", "error", "info", "warn" ], "reducer": "sum" } } }, { "id": "organize", "options": { "excludeByName": {}, "indexByName": { "Time": 1, "Total": 0, "grpc": 2, "http": 3 }, "renameByName": { "debug": "Debug", "discarded": "Discarded", "error": "Error", "grpc": "gRPC", "http": "HTTP", "info": "Info", "matched": "Matched", "warn": "Warn" } } } ], "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "Batch Size", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 19 }, "id": 136, "options": { "legend": { "calcs": [ "min", "max", "mean", "last" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "histogram_quantile(0.95, sum(rate(loki_store_chunks_per_batch_bucket{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (le,status))", "hide": false, "interval": "$interval", "legendFormat": "{{ status }}", "range": true, "refId": "A" } ], "title": "Loki Store - Chunk Batch Size", "transformations": [], "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "Count", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "cps" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 19 }, "id": 165, "options": { "legend": { "calcs": [ "min", "max", "mean", "last" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "rate(loki_chunk_fetcher_cache_dequeued_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])", "hide": false, "interval": "$interval", "legendFormat": "Dequeue from buffer", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "rate(loki_chunk_fetcher_cache_enqueued_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])", "hide": false, "interval": "$interval", "legendFormat": "Enqueue to buffer", "range": true, "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "rate(loki_chunk_fetcher_cache_skipped_buffer_full_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])", "hide": false, "interval": "$interval", "legendFormat": "Skipped due to buffer full", "range": true, "refId": "C" } ], "title": "Loki Store - Chunk Fetcher Cache", "transformations": [], "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "Chunks", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "cps" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 27 }, "id": 166, "options": { "legend": { "calcs": [ "min", "max", "mean", "last" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "sum(rate(loki_chunk_store_fetched_chunks_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (user)", "hide": false, "interval": "$interval", "legendFormat": "Fetched / {{ user }}", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "sum(rate(loki_chunk_store_stored_chunks_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (user)", "hide": false, "interval": "$interval", "legendFormat": "Stored / {{ user }}", "range": true, "refId": "B" } ], "title": "Loki Store - Fetched / Stored Chunks", "transformations": [ { "id": "renameByRegex", "options": { "regex": "(.*)fake$", "renamePattern": "$1Default Tenant" } } ], "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "Size", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "binBps" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 27 }, "id": 167, "options": { "legend": { "calcs": [ "min", "max", "mean", "last" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "rate(loki_chunk_store_fetched_chunk_bytes_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])", "hide": false, "interval": "$interval", "legendFormat": "Fetched / {{ user }}", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "rate(loki_chunk_store_stored_chunk_bytes_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])", "hide": false, "interval": "$interval", "legendFormat": "Stored / {{ user }}", "range": true, "refId": "B" } ], "title": "Loki Store - Fetched / Stored Chunk Volume", "transformations": [ { "id": "renameByRegex", "options": { "regex": "(.*)fake$", "renamePattern": "$1Default Tenant" } } ], "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "", "fieldConfig": { "defaults": { "custom": { "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "scaleDistribution": { "type": "linear" } }, "links": [] }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 35 }, "id": 163, "maxDataPoints": 50, "options": { "calculate": false, "cellGap": 2, "cellValues": { "unit": "cps" }, "color": { "exponent": 0.5, "fill": "red", "mode": "scheme", "reverse": false, "scale": "exponential", "scheme": "RdBu", "steps": 64 }, "exemplars": { "color": "rgba(255,0,255,0.7)" }, "filterValues": { "le": 0 }, "legend": { "show": true }, "rowsFrame": { "layout": "auto" }, "tooltip": { "show": true, "yHistogram": false }, "yAxis": { "axisLabel": "Bucket Entries", "axisPlacement": "left", "reverse": false, "unit": "none" } }, "pluginVersion": "9.3.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "exemplar": false, "expr": "rate(loki_chunk_store_chunks_per_query_bucket{job=\"$jobs\", instance=\"$host:$instance\"}[$__rate_interval])", "format": "heatmap", "instant": false, "interval": "", "legendFormat": "{{ le }}", "range": true, "refId": "A" } ], "title": "Loki Store - Chunks Per Query Distribution", "transformations": [ { "id": "organize", "options": { "excludeByName": {}, "indexByName": {}, "renameByName": { "+Inf": "Unlimited", "0.00010239999999999998": "10e-5", "0.0010485759999999998": "10e-4", "0.010485759999999998": "10e-3", "0.10485759999999998": "10e-2", "1.0239999999999999e-05": "10e-6", "1.0239999999999999e-06": "10e-7", "13568.999999999998": "13569", "144.99999999999997": "145", "1536.9999999999998": "1537", "24.999999999999996": "25", "27264.999999999996": "27265", "320.99999999999994": "321", "3200.9999999999995": "3201", "64.99999999999999": "65", "6528.999999999999": "6529", "704.9999999999999": "705", "8.999999999999998": "9", "9.999999999999998e-08": "10e-8", "9.999999999999999e-09": "10e-9", "9.999999999999999e-10": "10e-10" } } } ], "type": "heatmap" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "Bucket Entries", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "cps" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 35 }, "id": 156, "options": { "legend": { "calcs": [ "min", "max", "mean", "last" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "histogram_quantile(0.95, sum(rate(loki_chunk_store_chunks_per_query_bucket{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (le))", "hide": false, "interval": "$interval", "legendFormat": "{{ le }}", "range": true, "refId": "A" } ], "title": "Loki Store - Chunks Per Query", "transformations": [], "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "", "fieldConfig": { "defaults": { "custom": { "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "scaleDistribution": { "type": "linear" } }, "links": [] }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 43 }, "id": 154, "maxDataPoints": 50, "options": { "calculate": false, "cellGap": 2, "cellValues": { "unit": "cps" }, "color": { "exponent": 0.5, "fill": "red", "mode": "scheme", "reverse": false, "scale": "exponential", "scheme": "RdBu", "steps": 64 }, "exemplars": { "color": "rgba(255,0,255,0.7)" }, "filterValues": { "le": 0 }, "legend": { "show": true }, "rowsFrame": { "layout": "auto" }, "tooltip": { "show": true, "yHistogram": false }, "yAxis": { "axisLabel": "Bucket Entries", "axisPlacement": "left", "reverse": false, "unit": "none" } }, "pluginVersion": "9.3.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "exemplar": false, "expr": "rate(loki_chunk_store_index_entries_per_chunk_bucket{job=\"$jobs\", instance=\"$host:$instance\"}[$__rate_interval])", "format": "heatmap", "instant": false, "interval": "", "legendFormat": "{{ le }}", "range": true, "refId": "A" } ], "title": "Loki Store - Index Entries Per Chunk Distribution", "transformations": [ { "id": "organize", "options": { "excludeByName": {}, "indexByName": {}, "renameByName": { "+Inf": "Unlimited", "0.00010239999999999998": "10e-5", "0.0010485759999999998": "10e-4", "0.010485759999999998": "10e-3", "0.10485759999999998": "10e-2", "1.0239999999999999e-05": "10e-6", "1.0239999999999999e-06": "10e-7", "13568.999999999998": "13569", "144.99999999999997": "145", "1536.9999999999998": "1537", "24.999999999999996": "25", "27264.999999999996": "27265", "320.99999999999994": "321", "3200.9999999999995": "3201", "64.99999999999999": "65", "6528.999999999999": "6529", "704.9999999999999": "705", "8.999999999999998": "9", "9.999999999999998e-08": "10e-8", "9.999999999999999e-09": "10e-9", "9.999999999999999e-10": "10e-10" } } } ], "type": "heatmap" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "Bucket Entries", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "cps" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 43 }, "id": 164, "options": { "legend": { "calcs": [ "min", "max", "mean", "last" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "histogram_quantile(0.95, sum(rate(loki_chunk_store_index_entries_per_chunk_bucket{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (le))", "hide": false, "interval": "$interval", "legendFormat": "{{ le }}", "range": true, "refId": "A" } ], "title": "Loki Store - Index Entries Per Chunk", "transformations": [], "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "", "fieldConfig": { "defaults": { "custom": { "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "scaleDistribution": { "type": "linear" } }, "links": [] }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 51 }, "id": 159, "maxDataPoints": 50, "options": { "calculate": false, "cellGap": 2, "cellValues": { "unit": "cps" }, "color": { "exponent": 0.5, "fill": "red", "mode": "scheme", "reverse": false, "scale": "exponential", "scheme": "RdBu", "steps": 64 }, "exemplars": { "color": "rgba(255,0,255,0.7)" }, "filterValues": { "le": 0 }, "legend": { "show": true }, "rowsFrame": { "layout": "auto" }, "tooltip": { "show": true, "yHistogram": false }, "yAxis": { "axisLabel": "Bucket Entries", "axisPlacement": "left", "reverse": false, "unit": "none" } }, "pluginVersion": "9.3.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "exemplar": false, "expr": "rate(loki_chunk_store_index_lookups_per_query_bucket{job=\"$jobs\", instance=\"$host:$instance\"}[$__rate_interval])", "format": "heatmap", "instant": false, "interval": "", "legendFormat": "{{ le }}", "range": true, "refId": "A" } ], "title": "Loki Store - Index Lookups Per Query Distribution", "transformations": [ { "id": "organize", "options": { "excludeByName": {}, "indexByName": {}, "renameByName": { "+Inf": "Unlimited", "0.00010239999999999998": "10e-5", "0.0010485759999999998": "10e-4", "0.010485759999999998": "10e-3", "0.10485759999999998": "10e-2", "1.0239999999999999e-05": "10e-6", "1.0239999999999999e-06": "10e-7", "13568.999999999998": "13569", "144.99999999999997": "145", "1536.9999999999998": "1537", "24.999999999999996": "25", "27264.999999999996": "27265", "320.99999999999994": "321", "3200.9999999999995": "3201", "64.99999999999999": "65", "6528.999999999999": "6529", "704.9999999999999": "705", "8.999999999999998": "9", "9.999999999999998e-08": "10e-8", "9.999999999999999e-09": "10e-9", "9.999999999999999e-10": "10e-10" } } } ], "type": "heatmap" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "Bucket Entries", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "cps" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 51 }, "id": 160, "options": { "legend": { "calcs": [ "min", "max", "mean", "last" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "histogram_quantile(0.95, sum(rate(loki_chunk_store_index_lookups_per_query_bucket{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (le))", "hide": false, "interval": "$interval", "legendFormat": "{{ le }}", "range": true, "refId": "A" } ], "title": "Loki Store - Index Lookups Per Query", "transformations": [], "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "", "fieldConfig": { "defaults": { "custom": { "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "scaleDistribution": { "type": "linear" } }, "links": [] }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 59 }, "id": 157, "maxDataPoints": 50, "options": { "calculate": false, "cellGap": 2, "cellValues": { "unit": "cps" }, "color": { "exponent": 0.5, "fill": "red", "mode": "scheme", "reverse": false, "scale": "exponential", "scheme": "RdBu", "steps": 64 }, "exemplars": { "color": "rgba(255,0,255,0.7)" }, "filterValues": { "le": 0 }, "legend": { "show": true }, "rowsFrame": { "layout": "auto" }, "tooltip": { "show": true, "yHistogram": false }, "yAxis": { "axisLabel": "Bucket Entries", "axisPlacement": "left", "reverse": false, "unit": "none" } }, "pluginVersion": "9.3.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "exemplar": false, "expr": "rate(loki_chunk_store_series_pre_intersection_per_query_bucket{job=\"$jobs\", instance=\"$host:$instance\"}[$__rate_interval])", "format": "heatmap", "instant": false, "interval": "", "legendFormat": "{{ le }}", "range": true, "refId": "A" } ], "title": "Loki Store - Series (Pre Intersection) Per Chunk Distribution", "transformations": [ { "id": "organize", "options": { "excludeByName": {}, "indexByName": {}, "renameByName": { "+Inf": "Unlimited", "0.00010239999999999998": "10e-5", "0.0010485759999999998": "10e-4", "0.010485759999999998": "10e-3", "0.10485759999999998": "10e-2", "1.0239999999999999e-05": "10e-6", "1.0239999999999999e-06": "10e-7", "13568.999999999998": "13569", "144.99999999999997": "145", "1536.9999999999998": "1537", "24.999999999999996": "25", "27264.999999999996": "27265", "320.99999999999994": "321", "3200.9999999999995": "3201", "64.99999999999999": "65", "6528.999999999999": "6529", "704.9999999999999": "705", "8.999999999999998": "9", "9.999999999999998e-08": "10e-8", "9.999999999999999e-09": "10e-9", "9.999999999999999e-10": "10e-10" } } } ], "type": "heatmap" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "Bucket Entries", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "cps" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 59 }, "id": 158, "options": { "legend": { "calcs": [ "min", "max", "mean", "last" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "histogram_quantile(0.95, sum(rate(loki_chunk_store_series_pre_intersection_per_query_bucket{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (le))", "hide": false, "interval": "$interval", "legendFormat": "{{ le }}", "range": true, "refId": "A" } ], "title": "Loki Store - Series (Pre Intersection) Per Chunk", "transformations": [], "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "", "fieldConfig": { "defaults": { "custom": { "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "scaleDistribution": { "type": "linear" } }, "links": [] }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 67 }, "id": 161, "maxDataPoints": 50, "options": { "calculate": false, "cellGap": 2, "cellValues": { "unit": "cps" }, "color": { "exponent": 0.5, "fill": "red", "mode": "scheme", "reverse": false, "scale": "exponential", "scheme": "RdBu", "steps": 64 }, "exemplars": { "color": "rgba(255,0,255,0.7)" }, "filterValues": { "le": 0 }, "legend": { "show": true }, "rowsFrame": { "layout": "auto" }, "tooltip": { "show": true, "yHistogram": false }, "yAxis": { "axisLabel": "Bucket Entries", "axisPlacement": "left", "reverse": false, "unit": "none" } }, "pluginVersion": "9.3.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "exemplar": false, "expr": "rate(loki_chunk_store_series_post_intersection_per_query_bucket{job=\"$jobs\", instance=\"$host:$instance\"}[$__rate_interval])", "format": "heatmap", "instant": false, "interval": "", "legendFormat": "{{ le }}", "range": true, "refId": "A" } ], "title": "Loki Store - Series (Post Intersection) Per Chunk Distribution", "transformations": [ { "id": "organize", "options": { "excludeByName": {}, "indexByName": {}, "renameByName": { "+Inf": "Unlimited", "0.00010239999999999998": "10e-5", "0.0010485759999999998": "10e-4", "0.010485759999999998": "10e-3", "0.10485759999999998": "10e-2", "1.0239999999999999e-05": "10e-6", "1.0239999999999999e-06": "10e-7", "13568.999999999998": "13569", "144.99999999999997": "145", "1536.9999999999998": "1537", "24.999999999999996": "25", "27264.999999999996": "27265", "320.99999999999994": "321", "3200.9999999999995": "3201", "64.99999999999999": "65", "6528.999999999999": "6529", "704.9999999999999": "705", "8.999999999999998": "9", "9.999999999999998e-08": "10e-8", "9.999999999999999e-09": "10e-9", "9.999999999999999e-10": "10e-10" } } } ], "type": "heatmap" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "Bucket Entries", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "cps" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 67 }, "id": 162, "options": { "legend": { "calcs": [ "min", "max", "mean", "last" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "histogram_quantile(0.95, sum(rate(loki_chunk_store_series_post_intersection_per_query_bucket{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (le))", "hide": false, "interval": "$interval", "legendFormat": "{{ le }}", "range": true, "refId": "A" } ], "title": "Loki Store - Series (Post Intersection) Per Chunk", "transformations": [], "type": "timeseries" } ], "title": "Loki Store Metrics", "type": "row" }, { "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 29 }, "id": 170, "panels": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "Errors", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 12 }, "id": 171, "options": { "legend": { "calcs": [ "min", "max", "mean", "last" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "loki_table_manager_create_failures{job=\"$jobs\",instance=\"$host:$instance\"}", "hide": false, "interval": "$interval", "legendFormat": "Table create failures", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "loki_table_manager_delete_failures{job=\"$jobs\",instance=\"$host:$instance\"}", "hide": false, "interval": "$interval", "legendFormat": "Table delete failures", "range": true, "refId": "B" } ], "title": "Loki Table Manager - Errors", "transformations": [ { "id": "renameByRegex", "options": { "regex": "(.*)fake$", "renamePattern": "$1Default Tenant" } } ], "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "Time", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 12 }, "id": 172, "options": { "legend": { "calcs": [ "min", "max", "mean", "last" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "histogram_quantile(0.95, sum(rate(loki_table_manager_sync_duration_seconds_bucket{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])) by (le,operation,status_code))", "hide": false, "interval": "$interval", "legendFormat": "{{ operation }} / {{ status_code }}", "range": true, "refId": "A" } ], "title": "Loki Table Manager - Sync Duration", "transformations": [ { "id": "renameByRegex", "options": { "regex": "(.*)\\.(.*)", "renamePattern": "$1 $2" } } ], "type": "timeseries" } ], "title": "Loki Table Manager Metrics", "type": "row" }, { "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 30 }, "id": 83, "panels": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "Count", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 13 }, "id": 168, "options": { "legend": { "calcs": [ "min", "max", "mean", "last" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "go_goroutines{job=\"$jobs\",instance=\"$host:$instance\"}", "hide": false, "interval": "$interval", "legendFormat": "Goroutines", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "go_threads{job=\"$jobs\",instance=\"$host:$instance\"}", "hide": false, "interval": "$interval", "legendFormat": "Threads", "range": true, "refId": "B" } ], "title": "Go - Goroutines / Threads", "transformations": [ { "id": "renameByRegex", "options": { "regex": "(.*)fake$", "renamePattern": "$1Default Tenant" } } ], "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "Avg. Latency", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 13 }, "id": 91, "options": { "legend": { "calcs": [ "min", "max", "mean", "last" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "exemplar": false, "expr": "rate(go_sched_latencies_seconds_sum{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])/rate(go_sched_latencies_seconds_count{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])", "hide": false, "interval": "$interval", "legendFormat": "Average Goroutine latency", "range": true, "refId": "A" } ], "title": "Go - Goroutines Latency", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "", "fieldConfig": { "defaults": { "custom": { "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "scaleDistribution": { "type": "linear" } }, "links": [] }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 21 }, "id": 94, "maxDataPoints": 50, "options": { "calculate": false, "cellGap": 2, "cellValues": { "unit": "cps" }, "color": { "exponent": 0.5, "fill": "red", "mode": "scheme", "reverse": false, "scale": "exponential", "scheme": "RdBu", "steps": 64 }, "exemplars": { "color": "rgba(255,0,255,0.7)" }, "filterValues": { "le": 1E-9 }, "legend": { "show": true }, "rowsFrame": { "layout": "auto" }, "tooltip": { "show": true, "yHistogram": false }, "yAxis": { "axisLabel": "Bucket Latency", "axisPlacement": "left", "reverse": false, "unit": "s" } }, "pluginVersion": "9.3.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "exemplar": false, "expr": "rate(go_sched_latencies_seconds_bucket{job=\"$jobs\", instance=\"$host:$instance\"}[$__rate_interval])", "format": "heatmap", "instant": false, "interval": "", "legendFormat": "{{ le }}", "range": true, "refId": "A" } ], "title": "Go - Goroutines Latency Distribution", "transformations": [ { "id": "organize", "options": { "excludeByName": {}, "indexByName": {}, "renameByName": { "+Inf": "Unlimited", "0.00010239999999999998": "10e-5", "0.0010485759999999998": "10e-4", "0.010485759999999998": "10e-3", "0.10485759999999998": "10e-2", "1.0239999999999999e-05": "10e-6", "1.0239999999999999e-06": "10e-7", "13568.999999999998": "13569", "144.99999999999997": "145", "1536.9999999999998": "1537", "24.999999999999996": "25", "27264.999999999996": "27265", "320.99999999999994": "321", "3200.9999999999995": "3201", "64.99999999999999": "65", "6528.999999999999": "6529", "704.9999999999999": "705", "8.999999999999998": "9", "9.999999999999998e-08": "10e-8", "9.999999999999999e-09": "10e-9", "9.999999999999999e-10": "10e-10" } } } ], "type": "heatmap" } ], "title": "Go General Info", "type": "row" }, { "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 31 }, "id": 104, "panels": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "Bytes", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "bytes" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 14 }, "id": 92, "options": { "legend": { "calcs": [ "min", "max", "mean", "last" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "go_memory_classes_heap_free_bytes{job=\"$jobs\",instance=\"$host:$instance\"}", "interval": "$interval", "legendFormat": "Free memory", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "go_memory_classes_heap_objects_bytes{job=\"$jobs\",instance=\"$host:$instance\"}", "hide": false, "interval": "$interval", "legendFormat": "Allocated objects memory", "range": true, "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "go_memory_classes_heap_released_bytes{job=\"$jobs\",instance=\"$host:$instance\"}", "hide": false, "interval": "$interval", "legendFormat": "Released memory", "range": true, "refId": "C" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "go_memory_classes_heap_stacks_bytes{job=\"$jobs\",instance=\"$host:$instance\"}", "hide": false, "interval": "$interval", "legendFormat": "Stack reserved memory", "range": true, "refId": "D" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "go_memory_classes_heap_unused_bytes{job=\"$jobs\",instance=\"$host:$instance\"}", "hide": false, "interval": "$interval", "legendFormat": "Reserved unused memory", "range": true, "refId": "E" } ], "title": "Go - Memory Heap", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "Bytes", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "bytes" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 14 }, "id": 93, "options": { "legend": { "calcs": [ "min", "max", "mean", "last" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "go_memory_classes_metadata_mcache_free_bytes{job=\"$jobs\",instance=\"$host:$instance\"}", "interval": "$interval", "legendFormat": "Free mcache memory", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "go_memory_classes_metadata_mcache_inuse_bytes{job=\"$jobs\",instance=\"$host:$instance\"}", "hide": false, "interval": "$interval", "legendFormat": "Used mcache memory", "range": true, "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "go_memory_classes_metadata_mspan_free_bytes{job=\"$jobs\",instance=\"$host:$instance\"}", "hide": false, "interval": "$interval", "legendFormat": "Free mspan memory", "range": true, "refId": "C" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "go_memory_classes_metadata_mspan_inuse_bytes{job=\"$jobs\",instance=\"$host:$instance\"}", "hide": false, "interval": "$interval", "legendFormat": "Used mspan memory", "range": true, "refId": "D" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "go_memory_classes_metadata_other_bytes{job=\"$jobs\",instance=\"$host:$instance\"}", "hide": false, "interval": "$interval", "legendFormat": "Used other metadata memory", "range": true, "refId": "E" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "go_memory_classes_other_bytes{job=\"$jobs\",instance=\"$host:$instance\"}", "hide": false, "interval": "$interval", "legendFormat": "Used other memory", "range": true, "refId": "F" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "go_memory_classes_profiling_buckets_bytes{job=\"$jobs\",instance=\"$host:$instance\"}", "hide": false, "interval": "$interval", "legendFormat": "Used profiling bucket memory", "range": true, "refId": "G" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "go_memory_classes_total_bytes{job=\"$jobs\",instance=\"$host:$instance\"}", "hide": true, "interval": "$interval", "legendFormat": "Used total memory", "range": true, "refId": "H" } ], "title": "Go - Memory Metadata", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "Avg. Object Size", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "bytes" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 22 }, "id": 95, "options": { "legend": { "calcs": [ "min", "max", "mean", "last" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "exemplar": false, "expr": "rate(go_gc_heap_allocs_bytes_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])/rate(go_gc_heap_allocs_objects_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])", "hide": false, "interval": "$interval", "legendFormat": "Average object size allocated to the heap by the application", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "exemplar": false, "expr": "rate(go_gc_heap_frees_bytes_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])/rate(go_gc_heap_frees_objects_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])", "hide": false, "instant": false, "interval": "$interval", "legendFormat": "Average object size freed from the heap by the GC", "range": true, "refId": "B" } ], "title": "Go - Heap Object Allocation / Free Sizes", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "", "fieldConfig": { "defaults": { "custom": { "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "scaleDistribution": { "type": "linear" } }, "links": [] }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 22 }, "id": 89, "maxDataPoints": 50, "options": { "calculate": false, "cellGap": 2, "cellValues": { "unit": "cps" }, "color": { "exponent": 0.5, "fill": "red", "mode": "scheme", "reverse": false, "scale": "exponential", "scheme": "PuBuGn", "steps": 64 }, "exemplars": { "color": "rgba(255,0,255,0.7)" }, "filterValues": { "le": 1E-9 }, "legend": { "show": true }, "rowsFrame": { "layout": "auto" }, "tooltip": { "show": true, "yHistogram": false }, "yAxis": { "axisLabel": "Bucket Size", "axisPlacement": "left", "reverse": false, "unit": "bytes" } }, "pluginVersion": "9.3.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "exemplar": false, "expr": "rate(go_gc_heap_frees_by_size_bytes_bucket{job=\"$jobs\", instance=\"$host:$instance\"}[$__rate_interval])", "format": "heatmap", "instant": false, "interval": "", "legendFormat": "{{ le }}", "range": true, "refId": "A" } ], "title": "Go - Heap Free Distribution", "transformations": [ { "id": "organize", "options": { "excludeByName": {}, "indexByName": {}, "renameByName": { "+Inf": "Unlimited", "13568.999999999998": "13569", "144.99999999999997": "145", "1536.9999999999998": "1537", "24.999999999999996": "25", "27264.999999999996": "27265", "320.99999999999994": "321", "3200.9999999999995": "3201", "64.99999999999999": "65", "6528.999999999999": "6529", "704.9999999999999": "705", "8.999999999999998": "9" } } } ], "type": "heatmap" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "", "fieldConfig": { "defaults": { "custom": { "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "scaleDistribution": { "type": "linear" } }, "links": [] }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 30 }, "id": 88, "maxDataPoints": 50, "options": { "calculate": false, "cellGap": 2, "cellValues": { "unit": "cps" }, "color": { "exponent": 0.5, "fill": "red", "mode": "scheme", "reverse": false, "scale": "exponential", "scheme": "RdBu", "steps": 64 }, "exemplars": { "color": "rgba(255,0,255,0.7)" }, "filterValues": { "le": 1E-9 }, "legend": { "show": true }, "rowsFrame": { "layout": "auto" }, "tooltip": { "show": true, "yHistogram": false }, "yAxis": { "axisLabel": "Bucket Size", "axisPlacement": "left", "reverse": false, "unit": "bytes" } }, "pluginVersion": "9.3.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "exemplar": false, "expr": "rate(go_gc_heap_allocs_by_size_bytes_bucket{job=\"$jobs\", instance=\"$host:$instance\"}[$__rate_interval])", "format": "heatmap", "instant": false, "interval": "", "legendFormat": "{{ le }}", "range": true, "refId": "A" } ], "title": "Go - Heap Allocation Distribution", "transformations": [ { "id": "organize", "options": { "excludeByName": {}, "indexByName": {}, "renameByName": { "+Inf": "Unlimited", "13568.999999999998": "13569", "144.99999999999997": "145", "1536.9999999999998": "1537", "24.999999999999996": "25", "27264.999999999996": "27265", "320.99999999999994": "321", "3200.9999999999995": "3201", "64.99999999999999": "65", "6528.999999999999": "6529", "704.9999999999999": "705", "8.999999999999998": "9" } } } ], "type": "heatmap" } ], "title": "Go Memory Metrics", "type": "row" }, { "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 32 }, "id": 106, "panels": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "Cycles", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "cps" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 15 }, "id": 84, "options": { "legend": { "calcs": [ "min", "max", "mean", "last" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "rate(go_gc_cycles_total_gc_cycles_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])", "hide": false, "interval": "$interval", "legendFormat": "Count of all completed GC cycles", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "rate(go_gc_cycles_automatic_gc_cycles_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])", "hide": false, "interval": "$interval", "legendFormat": "Count of completed GC cycles generated by the Go runtime", "range": true, "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "rate(go_gc_cycles_forced_gc_cycles_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])", "interval": "$interval", "legendFormat": "Count of completed GC cycles forced by the application.", "range": true, "refId": "C" } ], "title": "Go - Garbage Collection Cycles", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "Avg. Pause Duration", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 15 }, "id": 85, "options": { "legend": { "calcs": [ "min", "max", "mean", "last" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "rate(go_gc_duration_seconds_sum{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])/rate(go_gc_duration_seconds_count{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])", "hide": false, "interval": "$interval", "legendFormat": "Average GC cycle pause duration", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "rate(go_gc_pauses_seconds_sum{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])/rate(go_gc_pauses_seconds_count{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])", "hide": false, "interval": "$interval", "legendFormat": "Average GC cycle halt duration", "range": true, "refId": "B" } ], "title": "Go - Garbage Collection Pause Duration", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "", "fieldConfig": { "defaults": { "custom": { "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "scaleDistribution": { "type": "linear" } }, "links": [] }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 23 }, "id": 90, "maxDataPoints": 50, "options": { "calculate": false, "cellGap": 2, "cellValues": { "unit": "cps" }, "color": { "exponent": 0.5, "fill": "red", "mode": "scheme", "reverse": false, "scale": "exponential", "scheme": "BrBG", "steps": 64 }, "exemplars": { "color": "rgba(255,0,255,0.7)" }, "filterValues": { "le": 1E-9 }, "legend": { "show": true }, "rowsFrame": { "layout": "auto" }, "tooltip": { "show": true, "yHistogram": false }, "yAxis": { "axisLabel": "Bucket Duration", "axisPlacement": "left", "reverse": false, "unit": "s" } }, "pluginVersion": "9.3.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "exemplar": false, "expr": "rate(go_gc_pauses_seconds_bucket{job=\"$jobs\", instance=\"$host:$instance\"}[$__rate_interval])", "format": "heatmap", "instant": false, "interval": "", "legendFormat": "{{ le }}", "range": true, "refId": "A" } ], "title": "Go - Garbage Collection Pauses Distribution", "transformations": [ { "id": "organize", "options": { "excludeByName": {}, "indexByName": {}, "renameByName": { "+Inf": "Unlimited", "0.00010239999999999998": "10e-5", "0.0010485759999999998": "10e-4", "0.010485759999999998": "10e-3", "0.10485759999999998": "10e-2", "1.0239999999999999e-05": "10e-6", "1.0239999999999999e-06": "10e-7", "9.999999999999998e-08": "10e-8", "9.999999999999999e-09": "10e-9", "9.999999999999999e-10": "10e-10" } } } ], "type": "heatmap" } ], "title": "Go GC Metrics", "type": "row" }, { "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 33 }, "id": 75, "panels": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "Seconds", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 16 }, "id": 78, "options": { "legend": { "calcs": [ "min", "max", "mean", "last" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "rate(process_cpu_seconds_total{job=\"$jobs\",instance=\"$host:$instance\"}[$__rate_interval])", "hide": false, "interval": "$interval", "legendFormat": "Total user and system CPU time spent in seconds", "range": true, "refId": "A" } ], "title": "Process - CPU seconds", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "Bytes", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "bytes" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 16 }, "id": 79, "options": { "legend": { "calcs": [ "min", "max", "mean", "last" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "process_virtual_memory_bytes{job=\"$jobs\",instance=\"$host:$instance\"}", "interval": "$interval", "legendFormat": "Virtual memory size", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "process_resident_memory_bytes{job=\"$jobs\",instance=\"$host:$instance\"}", "hide": false, "interval": "$interval", "legendFormat": "Resident memory size", "range": true, "refId": "B" } ], "title": "Process - Memory", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "Descriptors", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": true, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 24 }, "id": 77, "options": { "legend": { "calcs": [ "min", "max", "mean", "last" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "process_max_fds{job=\"$jobs\",instance=\"$host:$instance\"}", "hide": false, "interval": "$interval", "legendFormat": "Maximum number of open file descriptors", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "process_open_fds{job=\"$jobs\",instance=\"$host:$instance\"}", "interval": "$interval", "legendFormat": "Number of open file descriptors", "range": true, "refId": "B" } ], "title": "Process - File descriptors", "type": "timeseries" } ], "title": "Process Info", "type": "row" } ], "refresh": "1m", "schemaVersion": 37, "style": "dark", "tags": [ "loki" ], "templating": { "list": [ { "current": { "selected": false, "text": "default", "value": "default" }, "hide": 2, "includeAll": false, "label": "Datasource", "multi": false, "name": "DS_PROMETHEUS", "options": [], "query": "prometheus", "refresh": 1, "regex": "", "skipUrlSync": false, "type": "datasource" }, { "current": {}, "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "definition": "label_values(loki_build_info,job)", "hide": 0, "includeAll": false, "label": "Job", "multi": false, "name": "jobs", "options": [], "query": { "query": "label_values(loki_build_info,job)", "refId": "StandardVariableQuery" }, "refresh": 1, "regex": "", "skipUrlSync": false, "sort": 1, "type": "query", "useTags": false }, { "current": {}, "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "definition": "up{job=~\"$jobs\"}", "hide": 0, "includeAll": false, "label": "Host", "multi": false, "name": "host", "options": [], "query": { "query": "up{job=~\"$jobs\"}", "refId": "StandardVariableQuery" }, "refresh": 1, "regex": ".*instance=\"(.*?):.*\".*", "skipUrlSync": false, "sort": 1, "type": "query" }, { "current": {}, "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "definition": "up{job=~\"$jobs\"}", "hide": 0, "includeAll": false, "label": "Instance", "multi": false, "name": "instance", "options": [], "query": { "query": "up{job=~\"$jobs\"}", "refId": "StandardVariableQuery" }, "refresh": 1, "regex": ".*instance=\".*:(.*?)\".*", "skipUrlSync": false, "sort": 3, "type": "query" }, { "auto": true, "auto_count": 30, "auto_min": "10s", "current": { "selected": false, "text": "30s", "value": "30s" }, "hide": 0, "label": "Interval", "name": "interval", "options": [ { "selected": false, "text": "auto", "value": "$__auto_interval_interval" }, { "selected": true, "text": "30s", "value": "30s" }, { "selected": false, "text": "1m", "value": "1m" }, { "selected": false, "text": "5m", "value": "5m" }, { "selected": false, "text": "1h", "value": "1h" }, { "selected": false, "text": "6h", "value": "6h" }, { "selected": false, "text": "1d", "value": "1d" } ], "query": "30s,1m,5m,1h,6h,1d", "refresh": 2, "skipUrlSync": false, "type": "interval" } ] }, "time": { "from": "now-24h", "to": "now" }, "timepicker": { "refresh_intervals": [ "5s", "10s", "30s", "1m", "5m", "15m", "30m", "1h", "2h", "1d" ], "time_options": [ "5m", "15m", "1h", "6h", "12h", "24h", "2d", "7d", "30d" ] }, "timezone": "browser", "title": "Loki Metrics Dashboard", "uid": "f12e71ae-a681-4a80-83d8-3593a91d6598", "version": 1, "weekStart": "" }