FEAT(observability): enable HA with replica 2 and soft anti-affinity

- Add replicaCount: 2 to goldilocks, vpa, alertmanager
- Add replicas: 2 to loki singleBinary
- Add soft pod anti-affinity for node distribution
- Keep kube-state-metrics at replica 1 to prevent duplicate metrics

FIX(loki): revert to replica 1 for Single Binary mode

- Single Binary mode cannot run more than 1 replica without object storage
- Remove affinity configuration for single replica
- Keep filesystem storage backend

vpa/helm-values.yaml

@@ -4,7 +4,7 @@
 # Recommender - recommends resource requests based on historical usage
 recommender:
   enabled: true
-  replicaCount: 1
+  replicaCount: 2
 
   resources:
     requests:
@@ -25,3 +25,15 @@ admissionController:
 
 # Optional: Configure default VPA behavior
 # metrics-server should be installed in the cluster
+
+# Affinity - Soft Anti-Affinity to spread pods across nodes
+recommender:
+  affinity:
+    podAntiAffinity:
+      preferredDuringSchedulingIgnoredDuringExecution:
+        - weight: 100
+          podAffinityTerm:
+            labelSelector:
+              matchLabels:
+                app: vpa-recommender
+            topologyKey: kubernetes.io/hostname