PERF(repo): move system pods to master node

- Add nodeSelector for master node placement
- Add tolerations for NoExecute taint
- kube-state-metrics: schedule on master
- goldilocks-controller: schedule on master, reduce to 1 replica
- vpa-recommender: schedule on master, remove anti-affinity
- Free worker node resources for applications

goldilocks/helm-values.yaml

@@ -47,7 +47,7 @@ dashboard:
 # Controller configuration
 controller:
   enabled: true
-  replicaCount: 2
+  replicaCount: 1
 
   resources:
     requests:
@@ -60,6 +60,15 @@ controller:
   # Set to false to only monitor namespaces with the label: goldilocks.fairwinds.com/enabled=true
   enableCostRecommendations: true
 
+  # Schedule on master node
+  nodeSelector:
+    node-role.kubernetes.io/master: ""
+
+  tolerations:
+    - key: node-role.kubernetes.io/master
+      operator: Exists
+      effect: NoExecute
+
 # VPA configuration (should already be installed)
 vpa:
   # Set to false since we're installing VPA separately