crowdsec: add cs-firewall-bouncer DaemonSet (direct-host nftables enforcement)
Drops banned source IPs in-kernel via nftables (hooks input+forward, so DNAT'd LoadBalancer traffic is caught before reaching Traefik) for DIRECT hosts — the direct-side replacement for the dead Traefik plugin, zero per-request hop. No published image exists, so an initContainer fetches the pinned official static binary (v0.0.34) onto a stock debian-slim base (nftables backend uses netlink directly, no nft CLI needed). hostNetwork + NET_ADMIN/NET_RAW (not privileged). Config (with api_key) in a Secret, Reloader-annotated. crowdsec ns is already in the Kyverno wave-1 exclude list, so the privileged/hostNetwork pod is admitted. Pinned to k8s-node2 (runs a Traefik pod) for one-node validation before the nodeSelector is removed to roll cluster-wide. Fail-open by element timeout if the bouncer stops. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Viktor Barzin
parent
38675b7922
commit
7e646e1c7c
1 changed files with 254 additions and 0 deletions
254
stacks/crowdsec/modules/crowdsec/firewall_bouncer.tf
Normal file
254
stacks/crowdsec/modules/crowdsec/firewall_bouncer.tf
Normal file
|
|
@ -0,0 +1,254 @@
|
|||
# =============================================================================
|
||||
# cs-firewall-bouncer — in-kernel (nftables) enforcement DaemonSet
|
||||
# =============================================================================
|
||||
# CrowdSec currently enforces NOTHING at the network layer: the Traefik Yaegi
|
||||
# (lua) bouncer plugin is dead, so banned IPs still reach Traefik. For DIRECT
|
||||
# (non-Cloudflare-proxied) hosts we drop banned source IPs IN-KERNEL via
|
||||
# cs-firewall-bouncer's nftables backend — the packet is dropped before it ever
|
||||
# reaches Traefik, costing zero per-request hops.
|
||||
#
|
||||
# Topology this respects (do NOT change without re-reading docs/architecture/networking.md):
|
||||
# - Calico CNI + kube-proxy in IPTABLES mode (NOT eBPF).
|
||||
# - Traefik is a LoadBalancer Service at 10.0.20.203, externalTrafficPolicy=Local
|
||||
# (real client IP preserved — that's the whole point of the dedicated .203 IP).
|
||||
# - LB traffic is DNAT'd to the Traefik POD, so the original source IP survives
|
||||
# into the `forward` netfilter hook. The drop rule MUST therefore cover the
|
||||
# `forward` hook, not only `input` (a pod-destined packet traverses forward,
|
||||
# not input, on the node). Hence nftables_hooks: [input, forward] below.
|
||||
#
|
||||
# Packaging: cs-firewall-bouncer publishes NO container image. We pin the
|
||||
# official release binary (v0.0.34, 2025-08-04 — latest stable) and fetch it at
|
||||
# runtime: an initContainer (curlimages/curl — has curl + tar, alpine) downloads
|
||||
# + extracts the static binary into an emptyDir; the main container
|
||||
# (debian:bookworm-slim) runs it. The nftables backend talks netlink DIRECTLY
|
||||
# via github.com/google/nftables (go.mod + pkg/nftables/nftables.go: no os/exec)
|
||||
# and the docs confirm "mode nftables relies on github.com/google/nftables to
|
||||
# create table, chain and set" — so NO `nft` userspace CLI is needed and a plain
|
||||
# slim base image suffices. The binary is built CGO_ENABLED=0 / -extldflags
|
||||
# -static (Makefile), so it runs on glibc (debian) or musl (alpine) alike.
|
||||
#
|
||||
# Source: https://github.com/crowdsecurity/cs-firewall-bouncer
|
||||
# https://docs.crowdsec.net/u/bouncers/firewall/
|
||||
#
|
||||
# nodeSelector pins this to ONE node (k8s-node2, which runs a Traefik pod) for first validation.
|
||||
# !!! REMOVING THE nodeSelector ROLLS THIS DAEMONSET CLUSTER-WIDE !!!
|
||||
# Do that ONLY after the one-node validation checklist passes (see commit/PR).
|
||||
|
||||
locals {
|
||||
# Pin a specific stable release. Bump deliberately (re-validate on one node first).
|
||||
firewall_bouncer_version = "v0.0.34"
|
||||
firewall_bouncer_tgz_url = "https://github.com/crowdsecurity/cs-firewall-bouncer/releases/download/${local.firewall_bouncer_version}/crowdsec-firewall-bouncer-linux-amd64.tgz"
|
||||
firewall_bouncer_bin_path = "/opt/firewall-bouncer/crowdsec-firewall-bouncer"
|
||||
firewall_bouncer_cfg_path = "/etc/crowdsec/bouncers/crowdsec-firewall-bouncer.yaml"
|
||||
|
||||
# Rendered firewall-bouncer config. Lives in a Secret (NOT a ConfigMap) because
|
||||
# it embeds api_key. Key names/structure verified against the reference config
|
||||
# (config/crowdsec-firewall-bouncer.yaml @ v0.0.34):
|
||||
# - `set-only` uses a HYPHEN (not set_only).
|
||||
# - `nftables_hooks` is a TOP-LEVEL list (sibling of `nftables:`, underscore).
|
||||
# - `deny_action` values are uppercase DROP / REJECT.
|
||||
# - `log_mode: stdout` sends logs to the container's stdout (default is `file`).
|
||||
# - api_url carries a trailing slash (matches the reference default).
|
||||
firewall_bouncer_yaml = <<-YAML
|
||||
mode: nftables
|
||||
update_frequency: 10s
|
||||
log_mode: stdout
|
||||
log_level: info
|
||||
api_url: http://crowdsec-service.crowdsec.svc.cluster.local:8080/
|
||||
api_key: ${var.firewall_bouncer_key}
|
||||
insecure_skip_verify: false
|
||||
disable_ipv6: false
|
||||
deny_action: DROP
|
||||
deny_log: true
|
||||
nftables:
|
||||
ipv4:
|
||||
enabled: true
|
||||
set-only: false
|
||||
table: crowdsec
|
||||
chain: crowdsec-chain
|
||||
priority: -10
|
||||
ipv6:
|
||||
enabled: true
|
||||
set-only: false
|
||||
table: crowdsec6
|
||||
chain: crowdsec6-chain
|
||||
priority: -10
|
||||
nftables_hooks:
|
||||
- input
|
||||
- forward
|
||||
YAML
|
||||
}
|
||||
|
||||
resource "kubernetes_secret" "firewall_bouncer_config" {
|
||||
metadata {
|
||||
name = "crowdsec-firewall-bouncer-config"
|
||||
namespace = kubernetes_namespace.crowdsec.metadata[0].name
|
||||
labels = {
|
||||
"app.kubernetes.io/name" = "crowdsec-firewall-bouncer"
|
||||
tier = var.tier
|
||||
}
|
||||
annotations = {
|
||||
# Rotate the pods if the API key / config ever changes (the binary reads
|
||||
# the config only at startup).
|
||||
"reloader.stakater.com/match" = "true"
|
||||
}
|
||||
}
|
||||
data = {
|
||||
"crowdsec-firewall-bouncer.yaml" = local.firewall_bouncer_yaml
|
||||
}
|
||||
type = "Opaque"
|
||||
}
|
||||
|
||||
resource "kubernetes_daemon_set_v1" "firewall_bouncer" {
|
||||
metadata {
|
||||
name = "crowdsec-firewall-bouncer"
|
||||
namespace = kubernetes_namespace.crowdsec.metadata[0].name
|
||||
labels = {
|
||||
"app.kubernetes.io/name" = "crowdsec-firewall-bouncer"
|
||||
tier = var.tier
|
||||
}
|
||||
}
|
||||
spec {
|
||||
selector {
|
||||
match_labels = {
|
||||
"app.kubernetes.io/name" = "crowdsec-firewall-bouncer"
|
||||
}
|
||||
}
|
||||
template {
|
||||
metadata {
|
||||
labels = {
|
||||
"app.kubernetes.io/name" = "crowdsec-firewall-bouncer"
|
||||
tier = var.tier
|
||||
}
|
||||
annotations = {
|
||||
# Bounce pods when the config Secret changes (api_key rotation etc.).
|
||||
"secret.reloader.stakater.com/reload" = kubernetes_secret.firewall_bouncer_config.metadata[0].name
|
||||
}
|
||||
}
|
||||
spec {
|
||||
priority_class_name = "tier-1-cluster"
|
||||
|
||||
# Program the HOST's nftables ruleset (not the pod netns) — the bouncer's
|
||||
# drop rules must live in the host network namespace where DNAT'd LB
|
||||
# traffic transits the forward hook.
|
||||
host_network = true
|
||||
dns_policy = "ClusterFirstWithHostNet"
|
||||
|
||||
# ---- FIRST-VALIDATION PIN ----------------------------------------------
|
||||
# Pinned to a SINGLE node so a mistake in the nftables rules can only
|
||||
# affect one node. k8s-node2 is chosen because it currently runs a Traefik
|
||||
# pod — required to validate the `forward`-hook drop on DNAT'd LoadBalancer
|
||||
# traffic (under ETP=Local a node with no Traefik pod never sees that path,
|
||||
# so the validation would be meaningless there).
|
||||
# REMOVE this nodeSelector to roll the bouncer to EVERY node (the normal
|
||||
# end state for a firewall bouncer) — but ONLY after the one-node
|
||||
# validation checklist passes.
|
||||
node_selector = {
|
||||
"kubernetes.io/hostname" = "k8s-node2"
|
||||
}
|
||||
# ------------------------------------------------------------------------
|
||||
|
||||
# initContainer fetches + extracts the pinned release binary into the
|
||||
# shared emptyDir. curlimages/curl is alpine and ships curl + tar.
|
||||
init_container {
|
||||
name = "fetch-bouncer"
|
||||
image = "curlimages/curl:8.10.1"
|
||||
command = [
|
||||
"sh", "-c",
|
||||
<<-EOT
|
||||
set -eu
|
||||
echo "Downloading cs-firewall-bouncer ${local.firewall_bouncer_version}..."
|
||||
curl -fsSL "${local.firewall_bouncer_tgz_url}" -o /tmp/fb.tgz
|
||||
# Archive layout (verified @ v0.0.34): a single versioned top dir
|
||||
# `crowdsec-firewall-bouncer-vX.Y.Z/` containing the binary plus
|
||||
# config/, scripts/, install.sh. Strip that dir and extract ONLY the
|
||||
# binary — the `*/crowdsec-firewall-bouncer` glob matches one path
|
||||
# segment after the top dir, so config/...yaml is NOT pulled.
|
||||
tar -xzf /tmp/fb.tgz -C /opt/firewall-bouncer --strip-components=1 \
|
||||
--wildcards '*/crowdsec-firewall-bouncer'
|
||||
chmod +x ${local.firewall_bouncer_bin_path}
|
||||
echo "Fetched: $(ls -l ${local.firewall_bouncer_bin_path})"
|
||||
EOT
|
||||
]
|
||||
volume_mount {
|
||||
name = "binary"
|
||||
mount_path = "/opt/firewall-bouncer"
|
||||
}
|
||||
resources {
|
||||
requests = {
|
||||
cpu = "10m"
|
||||
memory = "32Mi"
|
||||
}
|
||||
limits = {
|
||||
memory = "64Mi"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
container {
|
||||
name = "firewall-bouncer"
|
||||
image = "debian:bookworm-slim"
|
||||
command = [
|
||||
local.firewall_bouncer_bin_path,
|
||||
"-c", local.firewall_bouncer_cfg_path,
|
||||
]
|
||||
|
||||
# nftables backend needs NET_ADMIN to program the host ruleset. NET_RAW
|
||||
# is the proven-safe companion the reference container images add. We
|
||||
# deliberately AVOID full `privileged: true` — these two caps are
|
||||
# sufficient for the netlink nftables path (no iptables/ipset shell-out
|
||||
# here). If validation shows rules are NOT being installed, the next
|
||||
# thing to try is privileged:true (see checklist) — but start minimal.
|
||||
security_context {
|
||||
capabilities {
|
||||
add = ["NET_ADMIN", "NET_RAW"]
|
||||
}
|
||||
}
|
||||
|
||||
volume_mount {
|
||||
name = "binary"
|
||||
mount_path = "/opt/firewall-bouncer"
|
||||
read_only = true
|
||||
}
|
||||
volume_mount {
|
||||
name = "config"
|
||||
mount_path = local.firewall_bouncer_cfg_path
|
||||
sub_path = "crowdsec-firewall-bouncer.yaml"
|
||||
read_only = true
|
||||
}
|
||||
|
||||
# No liveness probe: the bouncer runs as PID 1, so a crash — or a bad
|
||||
# config that makes it exit non-zero at startup — surfaces on its own as
|
||||
# a pod restart / CrashLoopBackOff. This avoids coupling pod liveness to
|
||||
# a periodic LAPI round-trip (a brief LAPI blip must NOT bounce the pod).
|
||||
|
||||
resources {
|
||||
requests = {
|
||||
cpu = "10m"
|
||||
memory = "64Mi"
|
||||
}
|
||||
# crowdsec-quota enforces limits.memory on every pod in the ns.
|
||||
limits = {
|
||||
memory = "128Mi"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
volume {
|
||||
name = "binary"
|
||||
empty_dir {}
|
||||
}
|
||||
volume {
|
||||
name = "config"
|
||||
secret {
|
||||
secret_name = kubernetes_secret.firewall_bouncer_config.metadata[0].name
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
lifecycle {
|
||||
# KYVERNO_LIFECYCLE_V1: Kyverno admission webhook mutates dns_config with ndots=2
|
||||
ignore_changes = [spec[0].template[0].spec[0].dns_config]
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue