feat(monitoring): Tempo + OTel Collector for tripit tracing (ADR-0032 Phase 2)

Stand up the cluster's first trace store + OTLP ingress so tripit's OpenTelemetry
spans (Phase 1, already live in prod) export and correlate with logs:
- Grafana Tempo (single-binary, filesystem on proxmox-lvm 20Gi, 30d)
- OTel Collector (contrib; otlp -> redaction deny-list backstop -> batch -> tempo)
- Grafana: a Tempo datasource + an ADDITIVE trace_id->Tempo derivedField on the
  Loki datasource (no uid change, so existing dashboards are unaffected)
- tripit deployment: LOG_FORMAT=json + OTEL_EXPORTER_OTLP_ENDPOINT -> the Collector

Additive (new helm releases; Loki/Prometheus/Grafana untouched). Offline
'terraform validate' clean; full plan+apply runs in CI (locked git-crypt blocks a
local plan as non-admin).

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

stacks/monitoring/modules/monitoring/loki.tf

@@ -524,6 +524,19 @@ resource "kubernetes_config_map" "grafana_loki_datasource" {
         access    = "proxy"
         url       = "http://loki.monitoring.svc.cluster.local:3100"
         isDefault = false
+        jsonData = {
+          # Log -> trace (tripit ADR-0032): pull trace_id out of tripit's JSON log
+          # lines and deep-link to the trace in Tempo. Additive — does NOT set a
+          # uid, so existing dashboards' references to this datasource are
+          # unaffected.
+          derivedFields = [{
+            name            = "trace_id"
+            matcherRegex    = "\"trace_id\":\\s*\"([a-f0-9]{32})\""
+            url             = "$${__value.raw}"
+            datasourceUid   = "tempo"
+            urlDisplayLabel = "View trace in Tempo"
+          }]
+        }
       }]
     })
   }

stacks/monitoring/modules/monitoring/otel-collector.tf

@@ -0,0 +1,16 @@
+# OpenTelemetry Collector — the OTLP ingress + redaction backstop in front of
+# Tempo (tripit ADR-0032). Apps export OTLP here; it redacts deny-listed values,
+# buffers, and forwards to Tempo. Same helm-release pattern as loki.tf/tempo.tf.
+resource "helm_release" "otel_collector" {
+  namespace        = kubernetes_namespace.monitoring.metadata[0].name
+  create_namespace = true
+  name             = "otel-collector"
+
+  repository = "https://open-telemetry.github.io/opentelemetry-helm-charts"
+  chart      = "opentelemetry-collector"
+
+  values  = [file("${path.module}/otel-collector.yaml")]
+  timeout = 600
+
+  depends_on = [helm_release.tempo]
+}

stacks/monitoring/modules/monitoring/otel-collector.yaml

@@ -0,0 +1,79 @@
+# OpenTelemetry Collector — OTLP ingress + deny-list redaction backstop in front
+# of Tempo (tripit ADR-0032). Contrib image (the `redaction` processor is contrib-
+# only). Traces-only: the chart's default jaeger/zipkin/prometheus receivers and
+# metrics/logs pipelines are nulled out so the collector runs a single clean
+# otlp -> redaction -> batch -> tempo pipeline.
+mode: deployment
+replicaCount: 1
+
+image:
+  repository: otel/opentelemetry-collector-contrib
+
+command:
+  name: otelcol-contrib
+
+presets:
+  kubernetesAttributes:
+    enabled: false
+
+# Only the OTLP ports; drop the chart's default jaeger/zipkin/metrics ports.
+ports:
+  otlp:
+    enabled: true
+    containerPort: 4317
+    servicePort: 4317
+    protocol: TCP
+  otlp-http:
+    enabled: true
+    containerPort: 4318
+    servicePort: 4318
+    protocol: TCP
+  jaeger-compact: { enabled: false }
+  jaeger-thrift: { enabled: false }
+  jaeger-grpc: { enabled: false }
+  zipkin: { enabled: false }
+  metrics: { enabled: false }
+
+resources:
+  requests:
+    cpu: 50m
+    memory: 128Mi
+  limits:
+    memory: 512Mi
+
+config:
+  # Null the chart defaults we don't use (configured-but-unused components fail
+  # collector startup).
+  receivers:
+    jaeger: null
+    zipkin: null
+    prometheus: null
+    otlp:
+      protocols:
+        grpc:
+          endpoint: ${env:MY_POD_IP}:4317
+        http:
+          endpoint: ${env:MY_POD_IP}:4318
+  processors:
+    # Deny-list backstop (ADR-0032): drop credential-shaped attribute VALUES even
+    # if an app accidentally puts one on a span. In-app hygiene is primary.
+    redaction:
+      allow_all_keys: true
+      blocked_values:
+        - "(?i)bearer\\s+[a-z0-9._~+/=-]+"
+        - "eyJ[a-zA-Z0-9_-]{5,}\\.[a-zA-Z0-9_-]{5,}\\.[a-zA-Z0-9_-]+"
+        - "-----BEGIN[\\s\\S]*?-----END[^-]*-----"
+    batch: {}
+  exporters:
+    otlp/tempo:
+      endpoint: tempo.monitoring.svc.cluster.local:4317
+      tls:
+        insecure: true
+  service:
+    pipelines:
+      traces:
+        receivers: [otlp]
+        processors: [redaction, batch]
+        exporters: [otlp/tempo]
+      metrics: null
+      logs: null

stacks/monitoring/modules/monitoring/tempo.tf

@@ -0,0 +1,45 @@
+# Grafana Tempo — trace store for the TripIt observability stack (tripit ADR-0032,
+# infra plan docs/plans/2026-06-21-tripit-observability-tempo-otel.md). Phase 2:
+# the app already trace-correlates its logs on Loki (Phase 1); this adds the trace
+# UI + logs<->traces correlation. Additive to the monitoring stack — same
+# helm-release pattern as loki.tf.
+resource "helm_release" "tempo" {
+  namespace        = kubernetes_namespace.monitoring.metadata[0].name
+  create_namespace = true
+  name             = "tempo"
+
+  repository = "https://grafana.github.io/helm-charts"
+  chart      = "tempo" # single-binary (filesystem) — the Loki-scale, single-writer twin
+
+  values  = [file("${path.module}/tempo.yaml")]
+  timeout = 600
+}
+
+# Grafana Tempo datasource + trace->logs correlation (Tempo span -> its Loki logs
+# by trace_id). The reverse (Loki log -> Tempo trace) is the derivedField added to
+# the Loki datasource in loki.tf. Discovered by the Grafana sidecar via the
+# grafana_datasource label, same as the Loki datasource.
+resource "kubernetes_config_map" "grafana_tempo_datasource" {
+  metadata {
+    name      = "grafana-tempo-datasource"
+    namespace = kubernetes_namespace.monitoring.metadata[0].name
+    labels = {
+      grafana_datasource = "1"
+    }
+  }
+  data = {
+    "tempo-datasource.yaml" = yamlencode({
+      apiVersion = 1
+      datasources = [{
+        name      = "Tempo"
+        type      = "tempo"
+        access    = "proxy"
+        uid       = "tempo"
+        url       = "http://tempo.monitoring.svc.cluster.local:3100"
+        isDefault = false
+      }]
+    })
+  }
+
+  depends_on = [helm_release.tempo]
+}

stacks/monitoring/modules/monitoring/tempo.yaml

@@ -0,0 +1,36 @@
+# Grafana Tempo — single-binary trace store for the TripIt observability stack
+# (tripit ADR-0032). Mirrors Loki: filesystem storage on a proxmox-lvm PVC,
+# SingleBinary, ingests OTLP from the OTel Collector. Additive — independent of
+# Loki/Prometheus/Grafana.
+tempo:
+  retention: 720h # 30d, matching Loki
+  storage:
+    trace:
+      backend: local
+      local:
+        path: /var/tempo/traces
+      wal:
+        path: /var/tempo/wal
+  # OTLP ingest (from the OTel Collector). gRPC 4317 / HTTP 4318.
+  receivers:
+    otlp:
+      protocols:
+        grpc:
+          endpoint: 0.0.0.0:4317
+        http:
+          endpoint: 0.0.0.0:4318
+  # Tempo query/HTTP API — the Grafana datasource URL targets this (3100).
+  server:
+    http_listen_port: 3100
+
+persistence:
+  enabled: true
+  size: 20Gi
+  storageClassName: proxmox-lvm
+
+resources:
+  requests:
+    cpu: 100m
+    memory: 256Mi
+  limits:
+    memory: 1Gi

stacks/tripit/main.tf

@@ -131,6 +131,14 @@ locals {
     # not-yet-visited countries + real UK bank-holiday leave windows + fares).
     # `claude_agent` mode requires images >= the #23 slice (already deployed).
     RESEARCH_PROVIDER = "claude_agent"
+    # Observability (tripit ADR-0032, Phase 2): structured JSON logs (so Loki's
+    # trace_id derivedField links to Tempo) + OTLP trace export to the in-cluster
+    # OTel Collector -> Tempo. Both require images >= the observability slice
+    # (v0.81.0, already live), so they land after that rollout — same image-first
+    # hold-order as the provider modes around here. Empty endpoint = no export
+    # (Phase 1 default); set here flips Phase 2 on.
+    LOG_FORMAT                  = "json"
+    OTEL_EXPORTER_OTLP_ENDPOINT = "http://otel-collector-opentelemetry-collector.monitoring.svc.cluster.local:4318/v1/traces"
     # Stay cover photos (tripit issue #47, ADR-0017): auto-fetch each picked
     # city's Wikipedia lead image (keyless REST summary API, "City, Country"
     # first), downloaded into the app's STORAGE_DIR (never hotlinked) and