PERF(repo): move system pods to master node

- Add nodeSelector for master node placement
- Add tolerations for NoExecute taint
- kube-state-metrics: schedule on master
- goldilocks-controller: schedule on master, reduce to 1 replica
- vpa-recommender: schedule on master, remove anti-affinity
- Free worker node resources for applications

goldilocks/helm-values.yaml

@@ -47,7 +47,7 @@ dashboard:
 # Controller configuration
 controller:
   enabled: true
-  replicaCount: 2
+  replicaCount: 1
 
   resources:
     requests:
@@ -60,6 +60,15 @@ controller:
   # Set to false to only monitor namespaces with the label: goldilocks.fairwinds.com/enabled=true
   enableCostRecommendations: true
 
+  # Schedule on master node
+  nodeSelector:
+    node-role.kubernetes.io/master: ""
+
+  tolerations:
+    - key: node-role.kubernetes.io/master
+      operator: Exists
+      effect: NoExecute
+
 # VPA configuration (should already be installed)
 vpa:
   # Set to false since we're installing VPA separately

kube-state-metrics/helm-values.yaml

@@ -14,6 +14,15 @@ resources:
   limits:
     memory: 150Mi
 
+# Schedule on master node
+nodeSelector:
+  node-role.kubernetes.io/master: ""
+
+tolerations:
+  - key: node-role.kubernetes.io/master
+    operator: Exists
+    effect: NoExecute
+
 service:
   type: ClusterIP
   clusterIP: None

vpa/helm-values.yaml

@@ -13,15 +13,14 @@ recommender:
     limits:
       memory: 192Mi
 
-  affinity:
+  # Schedule on master node
-    podAntiAffinity:
+  nodeSelector:
-      preferredDuringSchedulingIgnoredDuringExecution:
+    node-role.kubernetes.io/master: ""
-        - weight: 100
+
-          podAffinityTerm:
+  tolerations:
-            labelSelector:
+    - key: node-role.kubernetes.io/master
-              matchLabels:
+      operator: Exists
-                app: vpa-recommender
+      effect: NoExecute
-            topologyKey: kubernetes.io/hostname
 
 # Updater - applies recommended resource requests to pods
 # Disabled because we're using updateMode: Off (recommendations only)