FEAT(thanos): add Thanos for Prometheus HA and long-term storage

- Add Thanos Query, Store Gateway, Compactor
- Enable Prometheus Sidecar with S3 (MinIO) storage
- Configure Prometheus replicas: 2 with pod anti-affinity
- Add ExternalSecrets for MinIO credentials
- Retention: raw 7d, 5m downsampled 30d, 1h downsampled 90d

prometheus/helm-values.yaml

@@ -37,11 +37,40 @@ kubelet:
 # Prometheus
 prometheus:
   enabled: true
-  
+
+  # Thanos Sidecar - for long-term storage and HA
+  thanosService:
+    enabled: true
+  thanosServiceMonitor:
+    enabled: true
+
   prometheusSpec:
+    # HA: 2 replicas on different worker nodes
+    replicas: 2
+    replicaExternalLabelName: prometheus_replica
+
+    # Pod anti-affinity for HA
+    affinity:
+      podAntiAffinity:
+        preferredDuringSchedulingIgnoredDuringExecution:
+          - weight: 100
+            podAffinityTerm:
+              labelSelector:
+                matchLabels:
+                  app.kubernetes.io/name: prometheus
+              topologyKey: kubernetes.io/hostname
+
     scrapeInterval: 60s       # 30s → 60s (메모리 절감)
     evaluationInterval: 60s   # 30s → 60s
-    retention: 3d             # 7d → 3d (메모리 절감)
+    retention: 3d             # Local retention (S3 has longer retention via Thanos)
+
+    # Thanos Sidecar configuration
+    thanos:
+      image: quay.io/thanos/thanos:v0.37.2
+      objectStorageConfig:
+        existingSecret:
+          name: thanos-objstore-secret
+          key: objstore.yml
 
     storageSpec:
       volumeClaimTemplate:

prometheus/manifests/secret.yaml

@@ -16,3 +16,36 @@ spec:
       remoteRef:
         key: postgresql
         property: PASSWORD
+---
+apiVersion: external-secrets.io/v1
+kind: ExternalSecret
+metadata:
+  name: thanos-objstore-secret
+  namespace: prometheus
+spec:
+  refreshInterval: 1h
+  secretStoreRef:
+    name: vault-backend
+    kind: ClusterSecretStore
+  target:
+    name: thanos-objstore-secret
+    template:
+      engineVersion: v2
+      data:
+        objstore.yml: |
+          type: S3
+          config:
+            bucket: thanos
+            endpoint: minio.minio.svc.cluster.local:9000
+            access_key: {{ .access_key }}
+            secret_key: {{ .secret_key }}
+            insecure: true
+  data:
+    - secretKey: access_key
+      remoteRef:
+        key: secret/minio
+        property: root-user
+    - secretKey: secret_key
+      remoteRef:
+        key: secret/minio
+        property: root-password