infra/stacks/frigate/main.tf

variable "tls_secret_name" {
  type      = string
  sensitive = true
}
variable "nfs_server" { type = string }


resource "kubernetes_namespace" "frigate" {
  metadata {
    name = "frigate"
    labels = {
      tier = local.tiers.gpu
    }
    # labels = {
    #   "istio-injection" : "enabled"
    # }
  }
  lifecycle {
    # KYVERNO_LIFECYCLE_V1: goldilocks-vpa-auto-mode ClusterPolicy stamps this label on every namespace
    ignore_changes = [metadata[0].labels["goldilocks.fairwinds.com/vpa-update-mode"]]
  }
}

module "tls_secret" {
  source          = "../../modules/kubernetes/setup_tls_secret"
  namespace       = kubernetes_namespace.frigate.metadata[0].name
  tls_secret_name = var.tls_secret_name
}

resource "kubernetes_persistent_volume_claim" "config_encrypted" {
  wait_until_bound = false
  metadata {
    name      = "frigate-config-encrypted"
    namespace = kubernetes_namespace.frigate.metadata[0].name
    annotations = {
      "resize.topolvm.io/threshold"     = "80%"
      "resize.topolvm.io/increase"      = "100%"
      "resize.topolvm.io/storage_limit" = "5Gi"
    }
  }
  spec {
    access_modes       = ["ReadWriteOnce"]
    storage_class_name = "proxmox-lvm-encrypted"
    resources {
      requests = {
        storage = "1Gi"
      }
    }
  }
}

module "nfs_media_host" {
  source     = "../../modules/kubernetes/nfs_volume"
  name       = "frigate-media-host"
  namespace  = kubernetes_namespace.frigate.metadata[0].name
  nfs_server = "192.168.1.127"
  nfs_path   = "/srv/nfs/frigate/media"
}

resource "kubernetes_deployment" "frigate" {
  metadata {
    name      = "frigate"
    namespace = kubernetes_namespace.frigate.metadata[0].name
    labels = {
      app  = "frigate"
      tier = local.tiers.gpu
    }
    annotations = {
      "reloader.stakater.com/search" = "true"
    }
  }
  spec {
    replicas = 1 # Temporarily disabled due to high power consumption
    strategy {
      type = "Recreate"
    }
    selector {
      match_labels = {
        app = "frigate"
      }
    }
    template {
      metadata {
        labels = {
          app = "frigate"
        }
      }
      spec {
        node_selector = {
          "nvidia.com/gpu.present" : "true"
        }
        toleration {
          key      = "nvidia.com/gpu"
          operator = "Equal"
          value    = "true"
          effect   = "NoSchedule"
        }
        container {
          # image = "ghcr.io/blakeblackshear/frigate:stable"
          # image = "ghcr.io/blakeblackshear/frigate:stable-tensorrt"
          image = "ghcr.io/blakeblackshear/frigate:0.17.0-beta1-tensorrt"
          name  = "frigate"

          resources {
            requests = {
              cpu    = "1500m"
              memory = "5Gi"
            }
            limits = {
              memory           = "10Gi"
              "nvidia.com/gpu" = "1"
            }
          }
          env {
            name  = "FRIGATE_RTSP_PASSWORD"
            value = "password"
          }
          port {
            container_port = 5000
          }
          port {
            container_port = 8554
          }
          port {
            container_port = 8555
            protocol       = "TCP"
          }
          port {
            container_port = 8555
            protocol       = "UDP"
          }
          volume_mount {
            name       = "config"
            mount_path = "/config"
          }
          volume_mount {
            name       = "dri"
            mount_path = "/dev/dri"
          }
          volume_mount {
            name       = "dshm"
            mount_path = "/dev/shm"
          }
          volume_mount {
            name       = "media"
            mount_path = "/media/frigate"
          }
          volume_mount {
            name       = "cache-tmpfs"
            mount_path = "/tmp/cache"
          }
          # Restart pod if GPU becomes unavailable, Frigate hangs, or
          # detector falls back to CPU (inference time spikes from ~20ms to 200ms+)
          liveness_probe {
            exec {
              command = ["sh", "-c", <<-EOT
                nvidia-smi > /dev/null 2>&1 || exit 1
                STATS=$(curl -sf --max-time 5 http://localhost:5000/api/stats) || exit 1
                echo "$STATS" | python3 -c "
import sys, json
stats = json.load(sys.stdin)
for name, det in stats.get('detectors', {}).items():
    speed = det.get('inference_speed', 0)
    if speed > 100:
        print(f'UNHEALTHY: detector {name} inference {speed}ms > 100ms threshold')
        sys.exit(1)
"
              EOT
              ]
            }
            initial_delay_seconds = 120
            period_seconds        = 60
            timeout_seconds       = 10
            failure_threshold     = 3
          }
          # TensorRT model loading can take several minutes
          startup_probe {
            http_get {
              path = "/api/version"
              port = 5000
            }
            period_seconds    = 10
            failure_threshold = 30 # up to 5 minutes for startup
          }
          security_context {
            privileged = true
          }
        }

        volume {
          name = "config"
          persistent_volume_claim {
            claim_name = kubernetes_persistent_volume_claim.config_encrypted.metadata[0].name
          }
        }
        volume {
          name = "dshm"
          empty_dir {
            medium     = "Memory"
            size_limit = "1Gi"
          }
        }
        volume {
          name = "media"
          persistent_volume_claim {
            claim_name = module.nfs_media_host.claim_name
          }
        }
        volume {
          name = "cache-tmpfs"
          empty_dir {
            medium     = "Memory"
            size_limit = "512Mi"
          }
        }
        volume {
          name = "dri"
          host_path {
            path = "/dev/dri"
            type = "Directory"
          }
        }
      }
    }
  }
  lifecycle {
    # KYVERNO_LIFECYCLE_V1: Kyverno admission webhook mutates dns_config with ndots=2
    ignore_changes = [spec[0].template[0].spec[0].dns_config]
  }
}

resource "kubernetes_service" "frigate" {
  metadata {
    name      = "frigate"
    namespace = kubernetes_namespace.frigate.metadata[0].name
    labels = {
      "app" = "frigate"
    }
  }

  spec {
    selector = {
      app = "frigate"
    }
    port {
      name        = "http"
      target_port = 5000
      port        = 80
      protocol    = "TCP"
    }
  }
}

resource "kubernetes_service" "frigate-rtsp" {
  metadata {
    name      = "frigate-rtsp"
    namespace = kubernetes_namespace.frigate.metadata[0].name
    labels = {
      "app" = "frigate"
    }
  }

  spec {
    type = "NodePort" # Should always live on node1 where the gpu is
    selector = {
      app = "frigate"
    }
    port {
      name        = "rtsp-tcp"
      target_port = 8554
      port        = 8554
      protocol    = "TCP"
      node_port   = 30554
    }
    port {
      name        = "rtsp-udp"
      target_port = 8554
      port        = 8554
      protocol    = "UDP"
      node_port   = 30554
    }
  }
}

module "ingress" {
  source          = "../../modules/kubernetes/ingress_factory"
  dns_type        = "proxied"
  namespace       = kubernetes_namespace.frigate.metadata[0].name
  name            = "frigate"
  tls_secret_name = var.tls_secret_name
  auth            = "required"
  extra_annotations = {
    "gethomepage.dev/enabled"      = "true"
    "gethomepage.dev/name"         = "Frigate"
    "gethomepage.dev/description"  = "NVR & object detection"
    "gethomepage.dev/icon"         = "frigate.png"
    "gethomepage.dev/group"        = "Media & Entertainment"
    "gethomepage.dev/pod-selector" = ""
    "gethomepage.dev/widget.type"  = "frigate"
    "gethomepage.dev/widget.url"   = "http://frigate.frigate.svc.cluster.local"
  }
}

module "ingress-internal" {
  source                  = "../../modules/kubernetes/ingress_factory"
  auth                    = "required"
  namespace               = kubernetes_namespace.frigate.metadata[0].name
  name                    = "frigate-lan"
  host                    = "frigate-lan"
  root_domain             = "viktorbarzin.lan"
  service_name            = "frigate"
  tls_secret_name         = var.tls_secret_name
  allow_local_access_only = true
  ssl_redirect            = false
  extra_annotations = {
    "gethomepage.dev/enabled" = "false"
  }
}