FIX(grafana): change OOM panel to stat type

- Replace timeseries with stat panel for OOM detection
- Show total count of OOMKilled pods instead of timeline
- Gauge metric not suitable for timeseries visualization

grafana/dashboards/main.json

@@ -2353,45 +2353,11 @@
         "type": "prometheus",
         "uid": "${datasource}"
       },
-      "description": "No data is generally a good thing here.",
+      "description": "Pods that were OOM killed (based on lastState).",
       "fieldConfig": {
         "defaults": {
           "color": {
-            "mode": "palette-classic"
+            "mode": "thresholds"
-          },
-          "custom": {
-            "axisBorderShow": false,
-            "axisCenteredZero": false,
-            "axisColorMode": "text",
-            "axisLabel": "",
-            "axisPlacement": "auto",
-            "barAlignment": 0,
-            "barWidthFactor": 0.6,
-            "drawStyle": "points",
-            "fillOpacity": 25,
-            "gradientMode": "opacity",
-            "hideFrom": {
-              "legend": false,
-              "tooltip": false,
-              "viz": false
-            },
-            "insertNulls": false,
-            "lineInterpolation": "smooth",
-            "lineWidth": 2,
-            "pointSize": 5,
-            "scaleDistribution": {
-              "type": "linear"
-            },
-            "showPoints": "never",
-            "showValues": false,
-            "spanNulls": false,
-            "stacking": {
-              "group": "A",
-              "mode": "none"
-            },
-            "thresholdsStyle": {
-              "mode": "off"
-            }
           },
           "mappings": [],
           "thresholds": {
@@ -2399,11 +2365,15 @@
             "steps": [
               {
                 "color": "green",
-                "value": 0
+                "value": null
+              },
+              {
+                "color": "yellow",
+                "value": 1
               },
               {
                 "color": "red",
-                "value": 80
+                "value": 3
               }
             ]
           },
@@ -2419,23 +2389,18 @@
       },
       "id": 87,
       "options": {
-        "legend": {
+        "colorMode": "value",
-          "calcs": [
+        "graphMode": "none",
-            "min",
+        "justifyMode": "auto",
-            "max",
+        "orientation": "horizontal",
-            "mean"
+        "reduceOptions": {
-          ],
+          "calcs": ["lastNotNull"],
-          "displayMode": "table",
+          "fields": "",
-          "placement": "right",
+          "values": false
-          "showLegend": true,
-          "sortBy": "Max",
-          "sortDesc": true
         },
-        "tooltip": {
+        "showPercentChange": false,
-          "hideZeros": false,
+        "textMode": "auto",
-          "mode": "multi",
+        "wideLayout": true
-          "sort": "none"
-        }
       },
       "pluginVersion": "12.3.1",
       "targets": [
@@ -2445,16 +2410,14 @@
             "uid": "${datasource}"
           },
           "editorMode": "code",
-          "exemplar": true,
+          "expr": "count(kube_pod_container_status_last_terminated_reason{cluster=\"$cluster\", reason=\"OOMKilled\"} == 1)",
-          "expr": "count by (namespace) (kube_pod_container_status_last_terminated_reason{cluster=\"$cluster\", reason=\"OOMKilled\"} == 1) or vector(0)",
+          "instant": true,
-          "interval": "",
+          "legendFormat": "OOMKilled Pods",
-          "legendFormat": "{{ namespace }}",
-          "range": true,
           "refId": "A"
         }
       ],
-      "title": "OOM Events by namespace",
+      "title": "OOMKilled Pods",
-      "type": "timeseries"
+      "type": "stat"
     },
     {
       "datasource": {