FEAT(tempo): configure S3 storage with MinIO

- Enable env var expansion in config
- Configure extraEnv for S3 credentials
- Fix OTel Collector image settings

tempo/argocd.yaml

@@ -0,0 +1,45 @@
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: tempo
+  namespace: argocd
+  finalizers:
+  - resources-finalizer.argocd.argoproj.io
+spec:
+  project: default
+  sources:
+  - repoURL: https://grafana.github.io/helm-charts
+    chart: tempo
+    targetRevision: 1.17.0
+    helm:
+      valueFiles:
+      - $values/tempo/helm-values.yaml
+  - repoURL: https://github.com/K3S-HOME/observability.git
+    targetRevision: main
+    ref: values
+  - repoURL: https://github.com/K3S-HOME/observability.git
+    targetRevision: main
+    path: tempo/manifests
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: tempo
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true
+      allowEmpty: false
+    syncOptions:
+    - CreateNamespace=true
+    - PrunePropagationPolicy=foreground
+    - PruneLast=true
+    retry:
+      limit: 5
+      backoff:
+        duration: 5s
+        factor: 2
+        maxDuration: 3m
+    managedNamespaceMetadata:
+      labels:
+        goldilocks.fairwinds.com/enabled: 'true'
+        minio-s3: enabled
+  revisionHistoryLimit: 10

tempo/helm-values.yaml

@@ -0,0 +1,99 @@
+# Tempo Helm Values
+# Chart: https://github.com/grafana/helm-charts/tree/main/charts/tempo
+#
+# Architecture:
+# - Single binary (monolithic) mode for small clusters
+# - MinIO S3 for trace storage
+# - OTLP receiver for OpenTelemetry data
+# - Integrates with Grafana for trace visualization
+
+# Run on master node for stability
+tolerations:
+  - key: node-role.kubernetes.io/control-plane
+    operator: Exists
+    effect: NoSchedule
+nodeSelector:
+  node-role.kubernetes.io/control-plane: "true"
+
+# =============================================================================
+# Resource Limits (optimized for small cluster)
+# =============================================================================
+resources:
+  requests:
+    cpu: 50m
+    memory: 128Mi
+  limits:
+    memory: 512Mi
+
+# =============================================================================
+# Tempo Configuration
+# =============================================================================
+tempo:
+  # Receivers - protocols Tempo accepts
+  receivers:
+    otlp:
+      protocols:
+        grpc:
+          endpoint: 0.0.0.0:4317
+        http:
+          endpoint: 0.0.0.0:4318
+
+  # Retention settings
+  retention: 72h  # Keep traces for 3 days
+
+  # Backend storage (MinIO S3)
+  # Uses environment variable expansion
+  storage:
+    trace:
+      backend: s3
+      s3:
+        bucket: tempo
+        endpoint: minio.minio.svc.cluster.local:9000
+        access_key: ${S3_ACCESS_KEY}
+        secret_key: ${S3_SECRET_KEY}
+        insecure: true
+
+  # Query settings
+  querier:
+    frontend_worker:
+      frontend_address: ""
+
+  # Metrics generator for trace-derived metrics
+  metricsGenerator:
+    enabled: true
+    remoteWriteUrl: http://prometheus-kube-prometheus-prometheus.prometheus.svc:9090/api/v1/write
+
+  # Environment variables from secret for S3 credentials
+  extraEnv:
+    - name: S3_ACCESS_KEY
+      valueFrom:
+        secretKeyRef:
+          name: tempo-s3-secret
+          key: S3_ACCESS_KEY
+    - name: S3_SECRET_KEY
+      valueFrom:
+        secretKeyRef:
+          name: tempo-s3-secret
+          key: S3_SECRET_KEY
+
+# =============================================================================
+# Persistence (local cache)
+# =============================================================================
+persistence:
+  enabled: true
+  storageClassName: local-path
+  size: 2Gi
+
+# =============================================================================
+# Service
+# =============================================================================
+service:
+  type: ClusterIP
+
+# =============================================================================
+# ServiceMonitor for Prometheus
+# =============================================================================
+serviceMonitor:
+  enabled: true
+  additionalLabels:
+    release: prometheus

tempo/kustomization.yaml

@@ -0,0 +1,5 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - argocd.yaml

tempo/manifests/secret.yaml

@@ -0,0 +1,26 @@
+apiVersion: external-secrets.io/v1
+kind: ExternalSecret
+metadata:
+  name: tempo-s3-secret
+  namespace: tempo
+spec:
+  refreshInterval: 1h
+  secretStoreRef:
+    name: vault-backend
+    kind: ClusterSecretStore
+  target:
+    name: tempo-s3-secret
+    template:
+      engineVersion: v2
+      data:
+        S3_ACCESS_KEY: "{{ .access_key }}"
+        S3_SECRET_KEY: "{{ .secret_key }}"
+  data:
+    - secretKey: access_key
+      remoteRef:
+        key: minio
+        property: ROOT_USER
+    - secretKey: secret_key
+      remoteRef:
+        key: minio
+        property: ROOT_PASSWORD