From 7e646e1c7c1a540a0630f44c4829205b6976fd1e Mon Sep 17 00:00:00 2001 From: Viktor Barzin Date: Sat, 20 Jun 2026 09:11:08 +0000 Subject: [PATCH] crowdsec: add cs-firewall-bouncer DaemonSet (direct-host nftables enforcement) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Drops banned source IPs in-kernel via nftables (hooks input+forward, so DNAT'd LoadBalancer traffic is caught before reaching Traefik) for DIRECT hosts — the direct-side replacement for the dead Traefik plugin, zero per-request hop. No published image exists, so an initContainer fetches the pinned official static binary (v0.0.34) onto a stock debian-slim base (nftables backend uses netlink directly, no nft CLI needed). hostNetwork + NET_ADMIN/NET_RAW (not privileged). Config (with api_key) in a Secret, Reloader-annotated. crowdsec ns is already in the Kyverno wave-1 exclude list, so the privileged/hostNetwork pod is admitted. Pinned to k8s-node2 (runs a Traefik pod) for one-node validation before the nodeSelector is removed to roll cluster-wide. Fail-open by element timeout if the bouncer stops. Co-Authored-By: Claude Opus 4.8 --- .../modules/crowdsec/firewall_bouncer.tf | 254 ++++++++++++++++++ 1 file changed, 254 insertions(+) create mode 100644 stacks/crowdsec/modules/crowdsec/firewall_bouncer.tf diff --git a/stacks/crowdsec/modules/crowdsec/firewall_bouncer.tf b/stacks/crowdsec/modules/crowdsec/firewall_bouncer.tf new file mode 100644 index 00000000..c161486d --- /dev/null +++ b/stacks/crowdsec/modules/crowdsec/firewall_bouncer.tf @@ -0,0 +1,254 @@ +# ============================================================================= +# cs-firewall-bouncer — in-kernel (nftables) enforcement DaemonSet +# ============================================================================= +# CrowdSec currently enforces NOTHING at the network layer: the Traefik Yaegi +# (lua) bouncer plugin is dead, so banned IPs still reach Traefik. For DIRECT +# (non-Cloudflare-proxied) hosts we drop banned source IPs IN-KERNEL via +# cs-firewall-bouncer's nftables backend — the packet is dropped before it ever +# reaches Traefik, costing zero per-request hops. +# +# Topology this respects (do NOT change without re-reading docs/architecture/networking.md): +# - Calico CNI + kube-proxy in IPTABLES mode (NOT eBPF). +# - Traefik is a LoadBalancer Service at 10.0.20.203, externalTrafficPolicy=Local +# (real client IP preserved — that's the whole point of the dedicated .203 IP). +# - LB traffic is DNAT'd to the Traefik POD, so the original source IP survives +# into the `forward` netfilter hook. The drop rule MUST therefore cover the +# `forward` hook, not only `input` (a pod-destined packet traverses forward, +# not input, on the node). Hence nftables_hooks: [input, forward] below. +# +# Packaging: cs-firewall-bouncer publishes NO container image. We pin the +# official release binary (v0.0.34, 2025-08-04 — latest stable) and fetch it at +# runtime: an initContainer (curlimages/curl — has curl + tar, alpine) downloads +# + extracts the static binary into an emptyDir; the main container +# (debian:bookworm-slim) runs it. The nftables backend talks netlink DIRECTLY +# via github.com/google/nftables (go.mod + pkg/nftables/nftables.go: no os/exec) +# and the docs confirm "mode nftables relies on github.com/google/nftables to +# create table, chain and set" — so NO `nft` userspace CLI is needed and a plain +# slim base image suffices. The binary is built CGO_ENABLED=0 / -extldflags +# -static (Makefile), so it runs on glibc (debian) or musl (alpine) alike. +# +# Source: https://github.com/crowdsecurity/cs-firewall-bouncer +# https://docs.crowdsec.net/u/bouncers/firewall/ +# +# nodeSelector pins this to ONE node (k8s-node2, which runs a Traefik pod) for first validation. +# !!! REMOVING THE nodeSelector ROLLS THIS DAEMONSET CLUSTER-WIDE !!! +# Do that ONLY after the one-node validation checklist passes (see commit/PR). + +locals { + # Pin a specific stable release. Bump deliberately (re-validate on one node first). + firewall_bouncer_version = "v0.0.34" + firewall_bouncer_tgz_url = "https://github.com/crowdsecurity/cs-firewall-bouncer/releases/download/${local.firewall_bouncer_version}/crowdsec-firewall-bouncer-linux-amd64.tgz" + firewall_bouncer_bin_path = "/opt/firewall-bouncer/crowdsec-firewall-bouncer" + firewall_bouncer_cfg_path = "/etc/crowdsec/bouncers/crowdsec-firewall-bouncer.yaml" + + # Rendered firewall-bouncer config. Lives in a Secret (NOT a ConfigMap) because + # it embeds api_key. Key names/structure verified against the reference config + # (config/crowdsec-firewall-bouncer.yaml @ v0.0.34): + # - `set-only` uses a HYPHEN (not set_only). + # - `nftables_hooks` is a TOP-LEVEL list (sibling of `nftables:`, underscore). + # - `deny_action` values are uppercase DROP / REJECT. + # - `log_mode: stdout` sends logs to the container's stdout (default is `file`). + # - api_url carries a trailing slash (matches the reference default). + firewall_bouncer_yaml = <<-YAML + mode: nftables + update_frequency: 10s + log_mode: stdout + log_level: info + api_url: http://crowdsec-service.crowdsec.svc.cluster.local:8080/ + api_key: ${var.firewall_bouncer_key} + insecure_skip_verify: false + disable_ipv6: false + deny_action: DROP + deny_log: true + nftables: + ipv4: + enabled: true + set-only: false + table: crowdsec + chain: crowdsec-chain + priority: -10 + ipv6: + enabled: true + set-only: false + table: crowdsec6 + chain: crowdsec6-chain + priority: -10 + nftables_hooks: + - input + - forward + YAML +} + +resource "kubernetes_secret" "firewall_bouncer_config" { + metadata { + name = "crowdsec-firewall-bouncer-config" + namespace = kubernetes_namespace.crowdsec.metadata[0].name + labels = { + "app.kubernetes.io/name" = "crowdsec-firewall-bouncer" + tier = var.tier + } + annotations = { + # Rotate the pods if the API key / config ever changes (the binary reads + # the config only at startup). + "reloader.stakater.com/match" = "true" + } + } + data = { + "crowdsec-firewall-bouncer.yaml" = local.firewall_bouncer_yaml + } + type = "Opaque" +} + +resource "kubernetes_daemon_set_v1" "firewall_bouncer" { + metadata { + name = "crowdsec-firewall-bouncer" + namespace = kubernetes_namespace.crowdsec.metadata[0].name + labels = { + "app.kubernetes.io/name" = "crowdsec-firewall-bouncer" + tier = var.tier + } + } + spec { + selector { + match_labels = { + "app.kubernetes.io/name" = "crowdsec-firewall-bouncer" + } + } + template { + metadata { + labels = { + "app.kubernetes.io/name" = "crowdsec-firewall-bouncer" + tier = var.tier + } + annotations = { + # Bounce pods when the config Secret changes (api_key rotation etc.). + "secret.reloader.stakater.com/reload" = kubernetes_secret.firewall_bouncer_config.metadata[0].name + } + } + spec { + priority_class_name = "tier-1-cluster" + + # Program the HOST's nftables ruleset (not the pod netns) — the bouncer's + # drop rules must live in the host network namespace where DNAT'd LB + # traffic transits the forward hook. + host_network = true + dns_policy = "ClusterFirstWithHostNet" + + # ---- FIRST-VALIDATION PIN ---------------------------------------------- + # Pinned to a SINGLE node so a mistake in the nftables rules can only + # affect one node. k8s-node2 is chosen because it currently runs a Traefik + # pod — required to validate the `forward`-hook drop on DNAT'd LoadBalancer + # traffic (under ETP=Local a node with no Traefik pod never sees that path, + # so the validation would be meaningless there). + # REMOVE this nodeSelector to roll the bouncer to EVERY node (the normal + # end state for a firewall bouncer) — but ONLY after the one-node + # validation checklist passes. + node_selector = { + "kubernetes.io/hostname" = "k8s-node2" + } + # ------------------------------------------------------------------------ + + # initContainer fetches + extracts the pinned release binary into the + # shared emptyDir. curlimages/curl is alpine and ships curl + tar. + init_container { + name = "fetch-bouncer" + image = "curlimages/curl:8.10.1" + command = [ + "sh", "-c", + <<-EOT + set -eu + echo "Downloading cs-firewall-bouncer ${local.firewall_bouncer_version}..." + curl -fsSL "${local.firewall_bouncer_tgz_url}" -o /tmp/fb.tgz + # Archive layout (verified @ v0.0.34): a single versioned top dir + # `crowdsec-firewall-bouncer-vX.Y.Z/` containing the binary plus + # config/, scripts/, install.sh. Strip that dir and extract ONLY the + # binary — the `*/crowdsec-firewall-bouncer` glob matches one path + # segment after the top dir, so config/...yaml is NOT pulled. + tar -xzf /tmp/fb.tgz -C /opt/firewall-bouncer --strip-components=1 \ + --wildcards '*/crowdsec-firewall-bouncer' + chmod +x ${local.firewall_bouncer_bin_path} + echo "Fetched: $(ls -l ${local.firewall_bouncer_bin_path})" + EOT + ] + volume_mount { + name = "binary" + mount_path = "/opt/firewall-bouncer" + } + resources { + requests = { + cpu = "10m" + memory = "32Mi" + } + limits = { + memory = "64Mi" + } + } + } + + container { + name = "firewall-bouncer" + image = "debian:bookworm-slim" + command = [ + local.firewall_bouncer_bin_path, + "-c", local.firewall_bouncer_cfg_path, + ] + + # nftables backend needs NET_ADMIN to program the host ruleset. NET_RAW + # is the proven-safe companion the reference container images add. We + # deliberately AVOID full `privileged: true` — these two caps are + # sufficient for the netlink nftables path (no iptables/ipset shell-out + # here). If validation shows rules are NOT being installed, the next + # thing to try is privileged:true (see checklist) — but start minimal. + security_context { + capabilities { + add = ["NET_ADMIN", "NET_RAW"] + } + } + + volume_mount { + name = "binary" + mount_path = "/opt/firewall-bouncer" + read_only = true + } + volume_mount { + name = "config" + mount_path = local.firewall_bouncer_cfg_path + sub_path = "crowdsec-firewall-bouncer.yaml" + read_only = true + } + + # No liveness probe: the bouncer runs as PID 1, so a crash — or a bad + # config that makes it exit non-zero at startup — surfaces on its own as + # a pod restart / CrashLoopBackOff. This avoids coupling pod liveness to + # a periodic LAPI round-trip (a brief LAPI blip must NOT bounce the pod). + + resources { + requests = { + cpu = "10m" + memory = "64Mi" + } + # crowdsec-quota enforces limits.memory on every pod in the ns. + limits = { + memory = "128Mi" + } + } + } + + volume { + name = "binary" + empty_dir {} + } + volume { + name = "config" + secret { + secret_name = kubernetes_secret.firewall_bouncer_config.metadata[0].name + } + } + } + } + } + lifecycle { + # KYVERNO_LIFECYCLE_V1: Kyverno admission webhook mutates dns_config with ndots=2 + ignore_changes = [spec[0].template[0].spec[0].dns_config] + } +}