[ci skip] Implement multi-user Kubernetes access with OIDC
- Add RBAC module (modules/kubernetes/rbac/) with admin, power-user, and namespace-owner roles, API server OIDC flags, and audit logging - Add self-service portal (modules/kubernetes/k8s-portal/) SvelteKit app with kubeconfig download and setup instructions - Configure Alloy to collect audit logs from kube-apiserver - Add Grafana dashboard for Kubernetes audit log visualization - Configure Authentik OIDC provider with groups scope mapping - Wire up k8s_users and ssh_private_key variables through module chain
This commit is contained in:
parent
9853b5edf7
commit
9bcdb9e59f
8 changed files with 783 additions and 0 deletions
10
main.tf
10
main.tf
|
|
@ -155,10 +155,16 @@ variable "affine_postgresql_password" { type = string }
|
||||||
variable "health_postgresql_password" { type = string }
|
variable "health_postgresql_password" { type = string }
|
||||||
variable "health_secret_key" { type = string }
|
variable "health_secret_key" { type = string }
|
||||||
variable "moltbot_ssh_key" { type = string }
|
variable "moltbot_ssh_key" { type = string }
|
||||||
|
variable "moltbot_skill_secrets" { type = map(string) }
|
||||||
variable "gemini_api_key" { type = string }
|
variable "gemini_api_key" { type = string }
|
||||||
variable "llama_api_key" { type = string }
|
variable "llama_api_key" { type = string }
|
||||||
variable "brave_api_key" { type = string }
|
variable "brave_api_key" { type = string }
|
||||||
|
|
||||||
|
variable "k8s_users" {
|
||||||
|
type = map(any)
|
||||||
|
default = {}
|
||||||
|
}
|
||||||
|
|
||||||
variable "kube_config_path" {
|
variable "kube_config_path" {
|
||||||
type = string
|
type = string
|
||||||
default = "~/.kube/config"
|
default = "~/.kube/config"
|
||||||
|
|
@ -695,9 +701,13 @@ module "kubernetes_cluster" {
|
||||||
health_postgresql_password = var.health_postgresql_password
|
health_postgresql_password = var.health_postgresql_password
|
||||||
health_secret_key = var.health_secret_key
|
health_secret_key = var.health_secret_key
|
||||||
moltbot_ssh_key = var.moltbot_ssh_key
|
moltbot_ssh_key = var.moltbot_ssh_key
|
||||||
|
moltbot_skill_secrets = var.moltbot_skill_secrets
|
||||||
gemini_api_key = var.gemini_api_key
|
gemini_api_key = var.gemini_api_key
|
||||||
llama_api_key = var.llama_api_key
|
llama_api_key = var.llama_api_key
|
||||||
brave_api_key = var.brave_api_key
|
brave_api_key = var.brave_api_key
|
||||||
|
|
||||||
|
k8s_users = var.k8s_users
|
||||||
|
ssh_private_key = var.ssh_private_key
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
|
||||||
105
modules/kubernetes/k8s-portal/main.tf
Normal file
105
modules/kubernetes/k8s-portal/main.tf
Normal file
|
|
@ -0,0 +1,105 @@
|
||||||
|
variable "tls_secret_name" {}
|
||||||
|
variable "tier" { type = string }
|
||||||
|
|
||||||
|
resource "kubernetes_namespace" "k8s_portal" {
|
||||||
|
metadata {
|
||||||
|
name = "k8s-portal"
|
||||||
|
labels = {
|
||||||
|
tier = var.tier
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
module "tls_secret" {
|
||||||
|
source = "../setup_tls_secret"
|
||||||
|
namespace = kubernetes_namespace.k8s_portal.metadata[0].name
|
||||||
|
tls_secret_name = var.tls_secret_name
|
||||||
|
}
|
||||||
|
|
||||||
|
resource "kubernetes_config_map" "k8s_portal_config" {
|
||||||
|
metadata {
|
||||||
|
name = "k8s-portal-config"
|
||||||
|
namespace = kubernetes_namespace.k8s_portal.metadata[0].name
|
||||||
|
}
|
||||||
|
|
||||||
|
data = {
|
||||||
|
# CA cert extracted from kubeconfig — will be populated with cluster CA cert
|
||||||
|
"ca.crt" = ""
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
resource "kubernetes_deployment" "k8s_portal" {
|
||||||
|
metadata {
|
||||||
|
name = "k8s-portal"
|
||||||
|
namespace = kubernetes_namespace.k8s_portal.metadata[0].name
|
||||||
|
labels = {
|
||||||
|
app = "k8s-portal"
|
||||||
|
tier = var.tier
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
spec {
|
||||||
|
replicas = 1
|
||||||
|
selector {
|
||||||
|
match_labels = {
|
||||||
|
app = "k8s-portal"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
template {
|
||||||
|
metadata {
|
||||||
|
labels = {
|
||||||
|
app = "k8s-portal"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
spec {
|
||||||
|
container {
|
||||||
|
name = "portal"
|
||||||
|
image = "viktorbarzin/k8s-portal:latest"
|
||||||
|
port {
|
||||||
|
container_port = 3000
|
||||||
|
}
|
||||||
|
|
||||||
|
volume_mount {
|
||||||
|
name = "config"
|
||||||
|
mount_path = "/config"
|
||||||
|
read_only = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
volume {
|
||||||
|
name = "config"
|
||||||
|
config_map {
|
||||||
|
name = kubernetes_config_map.k8s_portal_config.metadata[0].name
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
resource "kubernetes_service" "k8s_portal" {
|
||||||
|
metadata {
|
||||||
|
name = "k8s-portal"
|
||||||
|
namespace = kubernetes_namespace.k8s_portal.metadata[0].name
|
||||||
|
}
|
||||||
|
|
||||||
|
spec {
|
||||||
|
selector = {
|
||||||
|
app = "k8s-portal"
|
||||||
|
}
|
||||||
|
port {
|
||||||
|
port = 80
|
||||||
|
target_port = 3000
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
module "ingress" {
|
||||||
|
source = "../ingress_factory"
|
||||||
|
namespace = kubernetes_namespace.k8s_portal.metadata[0].name
|
||||||
|
name = "k8s-portal"
|
||||||
|
tls_secret_name = var.tls_secret_name
|
||||||
|
protected = true # Require Authentik login
|
||||||
|
}
|
||||||
|
|
@ -126,10 +126,21 @@ variable "affine_postgresql_password" { type = string }
|
||||||
variable "health_postgresql_password" { type = string }
|
variable "health_postgresql_password" { type = string }
|
||||||
variable "health_secret_key" { type = string }
|
variable "health_secret_key" { type = string }
|
||||||
variable "moltbot_ssh_key" { type = string }
|
variable "moltbot_ssh_key" { type = string }
|
||||||
|
variable "moltbot_skill_secrets" { type = map(string) }
|
||||||
variable "gemini_api_key" { type = string }
|
variable "gemini_api_key" { type = string }
|
||||||
variable "llama_api_key" { type = string }
|
variable "llama_api_key" { type = string }
|
||||||
variable "brave_api_key" { type = string }
|
variable "brave_api_key" { type = string }
|
||||||
|
|
||||||
|
variable "k8s_users" {
|
||||||
|
type = map(any)
|
||||||
|
default = {}
|
||||||
|
}
|
||||||
|
variable "ssh_private_key" {
|
||||||
|
type = string
|
||||||
|
default = ""
|
||||||
|
sensitive = true
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
variable "defcon_level" {
|
variable "defcon_level" {
|
||||||
type = number
|
type = number
|
||||||
|
|
@ -829,6 +840,22 @@ module "authentik" {
|
||||||
postgres_password = var.authentik_postgres_password
|
postgres_password = var.authentik_postgres_password
|
||||||
}
|
}
|
||||||
|
|
||||||
|
module "rbac" {
|
||||||
|
source = "./rbac"
|
||||||
|
for_each = contains(local.active_modules, "authentik") ? { rbac = true } : {}
|
||||||
|
tier = local.tiers.cluster
|
||||||
|
tls_secret_name = var.tls_secret_name
|
||||||
|
k8s_users = var.k8s_users
|
||||||
|
ssh_private_key = var.ssh_private_key
|
||||||
|
}
|
||||||
|
|
||||||
|
module "k8s-portal" {
|
||||||
|
source = "./k8s-portal"
|
||||||
|
for_each = contains(local.active_modules, "authentik") ? { portal = true } : {}
|
||||||
|
tier = local.tiers.edge
|
||||||
|
tls_secret_name = var.tls_secret_name
|
||||||
|
}
|
||||||
|
|
||||||
module "linkwarden" {
|
module "linkwarden" {
|
||||||
source = "./linkwarden"
|
source = "./linkwarden"
|
||||||
for_each = contains(local.active_modules, "linkwarden") ? { linkwarden = true } : {}
|
for_each = contains(local.active_modules, "linkwarden") ? { linkwarden = true } : {}
|
||||||
|
|
@ -1142,6 +1169,7 @@ module "moltbot" {
|
||||||
for_each = contains(local.active_modules, "moltbot") ? { moltbot = true } : {}
|
for_each = contains(local.active_modules, "moltbot") ? { moltbot = true } : {}
|
||||||
tls_secret_name = var.tls_secret_name
|
tls_secret_name = var.tls_secret_name
|
||||||
ssh_key = var.moltbot_ssh_key
|
ssh_key = var.moltbot_ssh_key
|
||||||
|
skill_secrets = var.moltbot_skill_secrets
|
||||||
gemini_api_key = var.gemini_api_key
|
gemini_api_key = var.gemini_api_key
|
||||||
llama_api_key = var.llama_api_key
|
llama_api_key = var.llama_api_key
|
||||||
brave_api_key = var.brave_api_key
|
brave_api_key = var.brave_api_key
|
||||||
|
|
|
||||||
|
|
@ -99,6 +99,25 @@ alloy:
|
||||||
forward_to = [loki.write.default.receiver]
|
forward_to = [loki.write.default.receiver]
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Kubernetes audit log collection from /var/log/kubernetes/audit.log
|
||||||
|
// Requires alloy.mounts.varlog=true to mount /var/log from the host
|
||||||
|
local.file_match "audit_logs" {
|
||||||
|
path_targets = [{
|
||||||
|
__path__ = "/var/log/kubernetes/audit.log",
|
||||||
|
job = "kubernetes-audit",
|
||||||
|
node = env("HOSTNAME"),
|
||||||
|
}]
|
||||||
|
}
|
||||||
|
|
||||||
|
loki.source.file "audit_logs" {
|
||||||
|
targets = local.file_match.audit_logs.targets
|
||||||
|
forward_to = [loki.write.default.receiver]
|
||||||
|
}
|
||||||
|
|
||||||
|
# Mount /var/log from the host for file-based log collection (audit logs)
|
||||||
|
mounts:
|
||||||
|
varlog: true
|
||||||
|
|
||||||
# Resource limits for DaemonSet pods
|
# Resource limits for DaemonSet pods
|
||||||
# Alloy tails logs from all containers on the node via K8s API and batches
|
# Alloy tails logs from all containers on the node via K8s API and batches
|
||||||
# them to Loki. Memory scales with number of active log streams (~30-50 per node).
|
# them to Loki. Memory scales with number of active log streams (~30-50 per node).
|
||||||
|
|
|
||||||
204
modules/kubernetes/monitoring/dashboards/k8s-audit.json
Normal file
204
modules/kubernetes/monitoring/dashboards/k8s-audit.json
Normal file
|
|
@ -0,0 +1,204 @@
|
||||||
|
{
|
||||||
|
"annotations": {
|
||||||
|
"list": [
|
||||||
|
{
|
||||||
|
"builtIn": 1,
|
||||||
|
"datasource": { "type": "datasource", "uid": "grafana" },
|
||||||
|
"enable": true,
|
||||||
|
"hide": true,
|
||||||
|
"iconColor": "rgba(0, 211, 255, 1)",
|
||||||
|
"name": "Annotations & Alerts",
|
||||||
|
"type": "dashboard"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"description": "Kubernetes API server audit logs from Loki",
|
||||||
|
"editable": true,
|
||||||
|
"fiscalYearStartMonth": 0,
|
||||||
|
"graphTooltip": 1,
|
||||||
|
"id": 0,
|
||||||
|
"links": [],
|
||||||
|
"panels": [
|
||||||
|
{
|
||||||
|
"collapsed": false,
|
||||||
|
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 },
|
||||||
|
"id": 100,
|
||||||
|
"panels": [],
|
||||||
|
"title": "Recent Activity",
|
||||||
|
"type": "row"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" },
|
||||||
|
"description": "Recent Kubernetes API actions from audit logs",
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"color": { "mode": "thresholds" },
|
||||||
|
"custom": {
|
||||||
|
"align": "auto",
|
||||||
|
"cellOptions": { "type": "auto" },
|
||||||
|
"inspect": false
|
||||||
|
},
|
||||||
|
"mappings": [],
|
||||||
|
"thresholds": {
|
||||||
|
"mode": "absolute",
|
||||||
|
"steps": [{ "color": "green", "value": null }]
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"overrides": []
|
||||||
|
},
|
||||||
|
"gridPos": { "h": 12, "w": 24, "x": 0, "y": 1 },
|
||||||
|
"id": 1,
|
||||||
|
"options": {
|
||||||
|
"cellHeight": "sm",
|
||||||
|
"footer": { "countRows": false, "fields": "", "reducer": ["sum"], "show": false },
|
||||||
|
"showHeader": true,
|
||||||
|
"sortBy": [{ "desc": true, "displayName": "Time" }]
|
||||||
|
},
|
||||||
|
"pluginVersion": "12.3.0",
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" },
|
||||||
|
"editorMode": "code",
|
||||||
|
"expr": "{job=\"kubernetes-audit\"} | json | line_format \"{{.user.username}} {{.verb}} {{.objectRef.resource}} {{.objectRef.namespace}}\"",
|
||||||
|
"legendFormat": "",
|
||||||
|
"queryType": "range",
|
||||||
|
"refId": "A"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"title": "Recent Actions",
|
||||||
|
"type": "table"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"collapsed": false,
|
||||||
|
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 13 },
|
||||||
|
"id": 101,
|
||||||
|
"panels": [],
|
||||||
|
"title": "Request Rates",
|
||||||
|
"type": "row"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" },
|
||||||
|
"description": "API request count by user over time",
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"color": { "mode": "palette-classic" },
|
||||||
|
"custom": {
|
||||||
|
"axisBorderShow": false,
|
||||||
|
"axisCenteredZero": false,
|
||||||
|
"axisColorMode": "text",
|
||||||
|
"axisPlacement": "auto",
|
||||||
|
"barAlignment": 0,
|
||||||
|
"barWidthFactor": 0.6,
|
||||||
|
"drawStyle": "line",
|
||||||
|
"fillOpacity": 20,
|
||||||
|
"gradientMode": "none",
|
||||||
|
"hideFrom": { "legend": false, "tooltip": false, "viz": false },
|
||||||
|
"insertNulls": false,
|
||||||
|
"lineInterpolation": "smooth",
|
||||||
|
"lineWidth": 2,
|
||||||
|
"pointSize": 5,
|
||||||
|
"scaleDistribution": { "type": "linear" },
|
||||||
|
"showPoints": "never",
|
||||||
|
"spanNulls": false,
|
||||||
|
"stacking": { "group": "A", "mode": "none" },
|
||||||
|
"thresholdsStyle": { "mode": "off" }
|
||||||
|
},
|
||||||
|
"mappings": [],
|
||||||
|
"thresholds": {
|
||||||
|
"mode": "absolute",
|
||||||
|
"steps": [{ "color": "green", "value": null }]
|
||||||
|
},
|
||||||
|
"unit": "short"
|
||||||
|
},
|
||||||
|
"overrides": []
|
||||||
|
},
|
||||||
|
"gridPos": { "h": 10, "w": 24, "x": 0, "y": 14 },
|
||||||
|
"id": 2,
|
||||||
|
"options": {
|
||||||
|
"legend": { "calcs": ["sum", "lastNotNull"], "displayMode": "table", "placement": "bottom", "showLegend": true },
|
||||||
|
"tooltip": { "mode": "multi", "sort": "desc" }
|
||||||
|
},
|
||||||
|
"pluginVersion": "12.3.0",
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" },
|
||||||
|
"editorMode": "code",
|
||||||
|
"expr": "sum by (user_username) (count_over_time({job=\"kubernetes-audit\"} | json [5m]))",
|
||||||
|
"legendFormat": "{{user_username}}",
|
||||||
|
"queryType": "range",
|
||||||
|
"refId": "A"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"title": "Request Count by User",
|
||||||
|
"type": "timeseries"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"collapsed": false,
|
||||||
|
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 24 },
|
||||||
|
"id": 102,
|
||||||
|
"panels": [],
|
||||||
|
"title": "Denied Requests",
|
||||||
|
"type": "row"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" },
|
||||||
|
"description": "API requests denied with HTTP 403+ status codes",
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"color": { "mode": "thresholds" },
|
||||||
|
"custom": {
|
||||||
|
"align": "auto",
|
||||||
|
"cellOptions": { "type": "auto" },
|
||||||
|
"inspect": false
|
||||||
|
},
|
||||||
|
"mappings": [],
|
||||||
|
"thresholds": {
|
||||||
|
"mode": "absolute",
|
||||||
|
"steps": [
|
||||||
|
{ "color": "green", "value": null },
|
||||||
|
{ "color": "red", "value": 403 }
|
||||||
|
]
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"overrides": []
|
||||||
|
},
|
||||||
|
"gridPos": { "h": 12, "w": 24, "x": 0, "y": 25 },
|
||||||
|
"id": 3,
|
||||||
|
"options": {
|
||||||
|
"cellHeight": "sm",
|
||||||
|
"footer": { "countRows": false, "fields": "", "reducer": ["sum"], "show": false },
|
||||||
|
"showHeader": true,
|
||||||
|
"sortBy": [{ "desc": true, "displayName": "Time" }]
|
||||||
|
},
|
||||||
|
"pluginVersion": "12.3.0",
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" },
|
||||||
|
"editorMode": "code",
|
||||||
|
"expr": "{job=\"kubernetes-audit\"} | json | responseStatus_code >= 403",
|
||||||
|
"legendFormat": "",
|
||||||
|
"queryType": "range",
|
||||||
|
"refId": "A"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"title": "Denied Requests (403+)",
|
||||||
|
"type": "table"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"preload": false,
|
||||||
|
"refresh": "30s",
|
||||||
|
"schemaVersion": 42,
|
||||||
|
"tags": ["kubernetes", "audit", "security"],
|
||||||
|
"templating": {
|
||||||
|
"list": []
|
||||||
|
},
|
||||||
|
"time": {
|
||||||
|
"from": "now-24h",
|
||||||
|
"to": "now"
|
||||||
|
},
|
||||||
|
"timepicker": {},
|
||||||
|
"timezone": "",
|
||||||
|
"title": "Kubernetes Audit Logs",
|
||||||
|
"uid": "k8s-audit",
|
||||||
|
"version": 1
|
||||||
|
}
|
||||||
55
modules/kubernetes/rbac/apiserver-oidc.tf
Normal file
55
modules/kubernetes/rbac/apiserver-oidc.tf
Normal file
|
|
@ -0,0 +1,55 @@
|
||||||
|
# Configure kube-apiserver for OIDC authentication
|
||||||
|
# This SSHs to k8s-master and adds OIDC flags to the static pod manifest.
|
||||||
|
# Kubelet auto-restarts the API server when the manifest changes.
|
||||||
|
|
||||||
|
variable "k8s_master_host" {
|
||||||
|
type = string
|
||||||
|
default = "10.0.20.100"
|
||||||
|
}
|
||||||
|
|
||||||
|
variable "ssh_private_key" {
|
||||||
|
type = string
|
||||||
|
sensitive = true
|
||||||
|
}
|
||||||
|
|
||||||
|
variable "oidc_issuer_url" {
|
||||||
|
type = string
|
||||||
|
default = "https://authentik.viktorbarzin.me/application/o/kubernetes/"
|
||||||
|
}
|
||||||
|
|
||||||
|
variable "oidc_client_id" {
|
||||||
|
type = string
|
||||||
|
default = "kubernetes"
|
||||||
|
}
|
||||||
|
|
||||||
|
resource "null_resource" "apiserver_oidc_config" {
|
||||||
|
connection {
|
||||||
|
type = "ssh"
|
||||||
|
user = "wizard"
|
||||||
|
host = var.k8s_master_host
|
||||||
|
private_key = var.ssh_private_key
|
||||||
|
}
|
||||||
|
|
||||||
|
provisioner "remote-exec" {
|
||||||
|
inline = [
|
||||||
|
# Check if OIDC flags already present
|
||||||
|
"if grep -q 'oidc-issuer-url' /etc/kubernetes/manifests/kube-apiserver.yaml; then echo 'OIDC flags already configured'; exit 0; fi",
|
||||||
|
|
||||||
|
# Backup the manifest
|
||||||
|
"sudo cp /etc/kubernetes/manifests/kube-apiserver.yaml /etc/kubernetes/manifests/kube-apiserver.yaml.bak",
|
||||||
|
|
||||||
|
# Add OIDC flags after the last --tls-private-key-file flag (safe insertion point)
|
||||||
|
"sudo sed -i '/- --tls-private-key-file/a\\ - --oidc-issuer-url=${var.oidc_issuer_url}\\n - --oidc-client-id=${var.oidc_client_id}\\n - --oidc-username-claim=email\\n - --oidc-groups-claim=groups' /etc/kubernetes/manifests/kube-apiserver.yaml",
|
||||||
|
|
||||||
|
# Wait for API server to restart (kubelet watches the manifest)
|
||||||
|
"echo 'Waiting for API server to restart...'",
|
||||||
|
"sleep 30",
|
||||||
|
"sudo kubectl --kubeconfig=/etc/kubernetes/admin.conf get nodes || echo 'API server still restarting, check manually'",
|
||||||
|
]
|
||||||
|
}
|
||||||
|
|
||||||
|
triggers = {
|
||||||
|
oidc_issuer_url = var.oidc_issuer_url
|
||||||
|
oidc_client_id = var.oidc_client_id
|
||||||
|
}
|
||||||
|
}
|
||||||
95
modules/kubernetes/rbac/audit-policy.tf
Normal file
95
modules/kubernetes/rbac/audit-policy.tf
Normal file
|
|
@ -0,0 +1,95 @@
|
||||||
|
# Deploy audit policy to k8s-master and configure kube-apiserver to use it.
|
||||||
|
# Audit logs are written to /var/log/kubernetes/audit.log on the master node.
|
||||||
|
# Alloy (log collector DaemonSet) will pick them up and ship to Loki.
|
||||||
|
|
||||||
|
resource "null_resource" "audit_policy" {
|
||||||
|
connection {
|
||||||
|
type = "ssh"
|
||||||
|
user = "wizard"
|
||||||
|
host = var.k8s_master_host
|
||||||
|
private_key = var.ssh_private_key
|
||||||
|
}
|
||||||
|
|
||||||
|
# Upload audit policy file
|
||||||
|
provisioner "file" {
|
||||||
|
content = yamlencode({
|
||||||
|
apiVersion = "audit.k8s.io/v1"
|
||||||
|
kind = "Policy"
|
||||||
|
rules = [
|
||||||
|
{
|
||||||
|
# Don't log requests to the API discovery endpoints (very noisy)
|
||||||
|
level = "None"
|
||||||
|
resources = [{
|
||||||
|
group = ""
|
||||||
|
resources = ["endpoints", "services", "services/status"]
|
||||||
|
}]
|
||||||
|
users = ["system:kube-proxy"]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
# Don't log watch requests (very noisy)
|
||||||
|
level = "None"
|
||||||
|
verbs = ["watch"]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
# Don't log health checks
|
||||||
|
level = "None"
|
||||||
|
nonResourceURLs = ["/healthz*", "/readyz*", "/livez*"]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
# Log secret access at Metadata level only (no request/response bodies)
|
||||||
|
level = "Metadata"
|
||||||
|
resources = [{
|
||||||
|
group = ""
|
||||||
|
resources = ["secrets"]
|
||||||
|
}]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
# Log all other mutating requests at RequestResponse level
|
||||||
|
level = "RequestResponse"
|
||||||
|
verbs = ["create", "update", "patch", "delete"]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
# Log read requests at Metadata level
|
||||||
|
level = "Metadata"
|
||||||
|
verbs = ["get", "list"]
|
||||||
|
},
|
||||||
|
]
|
||||||
|
})
|
||||||
|
destination = "/tmp/audit-policy.yaml"
|
||||||
|
}
|
||||||
|
|
||||||
|
provisioner "remote-exec" {
|
||||||
|
inline = [
|
||||||
|
# Move audit policy to proper location
|
||||||
|
"sudo mkdir -p /etc/kubernetes/policies",
|
||||||
|
"sudo mv /tmp/audit-policy.yaml /etc/kubernetes/policies/audit-policy.yaml",
|
||||||
|
"sudo chown root:root /etc/kubernetes/policies/audit-policy.yaml",
|
||||||
|
|
||||||
|
# Create audit log directory
|
||||||
|
"sudo mkdir -p /var/log/kubernetes",
|
||||||
|
|
||||||
|
# Check if audit flags already present
|
||||||
|
"if grep -q 'audit-policy-file' /etc/kubernetes/manifests/kube-apiserver.yaml; then echo 'Audit flags already configured'; exit 0; fi",
|
||||||
|
|
||||||
|
# Add audit flags to kube-apiserver manifest
|
||||||
|
"sudo sed -i '/- --oidc-groups-claim/a\\ - --audit-policy-file=/etc/kubernetes/policies/audit-policy.yaml\\n - --audit-log-path=/var/log/kubernetes/audit.log\\n - --audit-log-maxage=7\\n - --audit-log-maxbackup=3\\n - --audit-log-maxsize=100' /etc/kubernetes/manifests/kube-apiserver.yaml",
|
||||||
|
|
||||||
|
# Add volume mount for audit policy (hostPath)
|
||||||
|
# The kube-apiserver pod needs access to the policy file and log directory
|
||||||
|
"sudo sed -i '/volumes:/a\\ - hostPath:\\n path: /etc/kubernetes/policies\\n type: DirectoryOrCreate\\n name: audit-policy\\n - hostPath:\\n path: /var/log/kubernetes\\n type: DirectoryOrCreate\\n name: audit-log' /etc/kubernetes/manifests/kube-apiserver.yaml",
|
||||||
|
|
||||||
|
"sudo sed -i '/volumeMounts:/a\\ - mountPath: /etc/kubernetes/policies\\n name: audit-policy\\n readOnly: true\\n - mountPath: /var/log/kubernetes\\n name: audit-log' /etc/kubernetes/manifests/kube-apiserver.yaml",
|
||||||
|
|
||||||
|
# Wait for API server to restart
|
||||||
|
"echo 'Waiting for API server to restart with audit logging...'",
|
||||||
|
"sleep 30",
|
||||||
|
"sudo kubectl --kubeconfig=/etc/kubernetes/admin.conf get nodes || echo 'API server still restarting'",
|
||||||
|
]
|
||||||
|
}
|
||||||
|
|
||||||
|
triggers = {
|
||||||
|
policy_version = "v1" # Bump to re-apply
|
||||||
|
}
|
||||||
|
|
||||||
|
depends_on = [null_resource.apiserver_oidc_config]
|
||||||
|
}
|
||||||
267
modules/kubernetes/rbac/main.tf
Normal file
267
modules/kubernetes/rbac/main.tf
Normal file
|
|
@ -0,0 +1,267 @@
|
||||||
|
variable "tls_secret_name" {}
|
||||||
|
variable "tier" { type = string }
|
||||||
|
|
||||||
|
variable "k8s_users" {
|
||||||
|
type = map(object({
|
||||||
|
role = string # "admin", "power-user", "namespace-owner"
|
||||||
|
email = string # OIDC email claim
|
||||||
|
namespaces = optional(list(string), []) # for namespace-owners
|
||||||
|
quota = optional(object({
|
||||||
|
cpu_requests = optional(string, "2")
|
||||||
|
memory_requests = optional(string, "4Gi")
|
||||||
|
cpu_limits = optional(string, "4")
|
||||||
|
memory_limits = optional(string, "8Gi")
|
||||||
|
pods = optional(string, "20")
|
||||||
|
}), {})
|
||||||
|
}))
|
||||||
|
default = {}
|
||||||
|
}
|
||||||
|
|
||||||
|
# --- Admin role ---
|
||||||
|
# Binds to built-in cluster-admin ClusterRole
|
||||||
|
|
||||||
|
resource "kubernetes_cluster_role_binding" "admin_users" {
|
||||||
|
for_each = { for name, user in var.k8s_users : name => user if user.role == "admin" }
|
||||||
|
|
||||||
|
metadata {
|
||||||
|
name = "oidc-admin-${each.key}"
|
||||||
|
}
|
||||||
|
|
||||||
|
role_ref {
|
||||||
|
api_group = "rbac.authorization.k8s.io"
|
||||||
|
kind = "ClusterRole"
|
||||||
|
name = "cluster-admin"
|
||||||
|
}
|
||||||
|
|
||||||
|
subject {
|
||||||
|
kind = "User"
|
||||||
|
name = each.value.email
|
||||||
|
api_group = "rbac.authorization.k8s.io"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
# --- Power-user role ---
|
||||||
|
# Can manage workloads cluster-wide but cannot modify RBAC, nodes, or persistent volumes
|
||||||
|
|
||||||
|
resource "kubernetes_cluster_role" "power_user" {
|
||||||
|
metadata {
|
||||||
|
name = "oidc-power-user"
|
||||||
|
}
|
||||||
|
|
||||||
|
# Core resources
|
||||||
|
rule {
|
||||||
|
api_groups = [""]
|
||||||
|
resources = ["pods", "pods/log", "pods/exec", "services", "endpoints", "configmaps", "secrets", "persistentvolumeclaims", "events", "namespaces"]
|
||||||
|
verbs = ["get", "list", "watch"]
|
||||||
|
}
|
||||||
|
|
||||||
|
rule {
|
||||||
|
api_groups = [""]
|
||||||
|
resources = ["pods", "services", "configmaps", "secrets", "persistentvolumeclaims"]
|
||||||
|
verbs = ["create", "update", "patch", "delete"]
|
||||||
|
}
|
||||||
|
|
||||||
|
# Apps
|
||||||
|
rule {
|
||||||
|
api_groups = ["apps"]
|
||||||
|
resources = ["deployments", "statefulsets", "daemonsets", "replicasets"]
|
||||||
|
verbs = ["get", "list", "watch", "create", "update", "patch", "delete"]
|
||||||
|
}
|
||||||
|
|
||||||
|
# Batch
|
||||||
|
rule {
|
||||||
|
api_groups = ["batch"]
|
||||||
|
resources = ["jobs", "cronjobs"]
|
||||||
|
verbs = ["get", "list", "watch", "create", "update", "patch", "delete"]
|
||||||
|
}
|
||||||
|
|
||||||
|
# Networking
|
||||||
|
rule {
|
||||||
|
api_groups = ["networking.k8s.io"]
|
||||||
|
resources = ["ingresses", "networkpolicies"]
|
||||||
|
verbs = ["get", "list", "watch", "create", "update", "patch", "delete"]
|
||||||
|
}
|
||||||
|
|
||||||
|
# Autoscaling
|
||||||
|
rule {
|
||||||
|
api_groups = ["autoscaling"]
|
||||||
|
resources = ["horizontalpodautoscalers"]
|
||||||
|
verbs = ["get", "list", "watch", "create", "update", "patch", "delete"]
|
||||||
|
}
|
||||||
|
|
||||||
|
# Read-only on cluster-level resources
|
||||||
|
rule {
|
||||||
|
api_groups = [""]
|
||||||
|
resources = ["nodes"]
|
||||||
|
verbs = ["get", "list", "watch"]
|
||||||
|
}
|
||||||
|
|
||||||
|
rule {
|
||||||
|
api_groups = ["storage.k8s.io"]
|
||||||
|
resources = ["storageclasses"]
|
||||||
|
verbs = ["get", "list", "watch"]
|
||||||
|
}
|
||||||
|
|
||||||
|
rule {
|
||||||
|
api_groups = ["rbac.authorization.k8s.io"]
|
||||||
|
resources = ["clusterroles", "clusterrolebindings", "roles", "rolebindings"]
|
||||||
|
verbs = ["get", "list", "watch"]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
resource "kubernetes_cluster_role_binding" "power_users" {
|
||||||
|
for_each = { for name, user in var.k8s_users : name => user if user.role == "power-user" }
|
||||||
|
|
||||||
|
metadata {
|
||||||
|
name = "oidc-power-user-${each.key}"
|
||||||
|
}
|
||||||
|
|
||||||
|
role_ref {
|
||||||
|
api_group = "rbac.authorization.k8s.io"
|
||||||
|
kind = "ClusterRole"
|
||||||
|
name = kubernetes_cluster_role.power_user.metadata[0].name
|
||||||
|
}
|
||||||
|
|
||||||
|
subject {
|
||||||
|
kind = "User"
|
||||||
|
name = each.value.email
|
||||||
|
api_group = "rbac.authorization.k8s.io"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
# --- Namespace-owner role ---
|
||||||
|
# Full admin within assigned namespaces + read-only cluster-wide
|
||||||
|
|
||||||
|
locals {
|
||||||
|
# Flatten user->namespace pairs for iteration
|
||||||
|
namespace_owner_pairs = flatten([
|
||||||
|
for name, user in var.k8s_users : [
|
||||||
|
for ns in user.namespaces : {
|
||||||
|
user_key = name
|
||||||
|
namespace = ns
|
||||||
|
email = user.email
|
||||||
|
quota = user.quota
|
||||||
|
}
|
||||||
|
] if user.role == "namespace-owner"
|
||||||
|
])
|
||||||
|
}
|
||||||
|
|
||||||
|
resource "kubernetes_namespace" "user_namespaces" {
|
||||||
|
for_each = { for pair in local.namespace_owner_pairs : "${pair.user_key}-${pair.namespace}" => pair }
|
||||||
|
|
||||||
|
metadata {
|
||||||
|
name = each.value.namespace
|
||||||
|
labels = {
|
||||||
|
tier = var.tier
|
||||||
|
"k8s-portal/owner" = each.value.user_key
|
||||||
|
"k8s-portal/managed-by" = "rbac-module"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
resource "kubernetes_role_binding" "namespace_owner" {
|
||||||
|
for_each = { for pair in local.namespace_owner_pairs : "${pair.user_key}-${pair.namespace}" => pair }
|
||||||
|
|
||||||
|
metadata {
|
||||||
|
name = "namespace-owner-${each.value.user_key}"
|
||||||
|
namespace = each.value.namespace
|
||||||
|
}
|
||||||
|
|
||||||
|
role_ref {
|
||||||
|
api_group = "rbac.authorization.k8s.io"
|
||||||
|
kind = "ClusterRole"
|
||||||
|
name = "admin" # Built-in ClusterRole with full namespace access
|
||||||
|
}
|
||||||
|
|
||||||
|
subject {
|
||||||
|
kind = "User"
|
||||||
|
name = each.value.email
|
||||||
|
api_group = "rbac.authorization.k8s.io"
|
||||||
|
}
|
||||||
|
|
||||||
|
depends_on = [kubernetes_namespace.user_namespaces]
|
||||||
|
}
|
||||||
|
|
||||||
|
# Read-only cluster-wide access for namespace owners
|
||||||
|
resource "kubernetes_cluster_role" "namespace_owner_readonly" {
|
||||||
|
metadata {
|
||||||
|
name = "oidc-namespace-owner-readonly"
|
||||||
|
}
|
||||||
|
|
||||||
|
rule {
|
||||||
|
api_groups = [""]
|
||||||
|
resources = ["namespaces", "nodes"]
|
||||||
|
verbs = ["get", "list", "watch"]
|
||||||
|
}
|
||||||
|
|
||||||
|
rule {
|
||||||
|
api_groups = [""]
|
||||||
|
resources = ["pods", "services", "configmaps", "events"]
|
||||||
|
verbs = ["get", "list", "watch"]
|
||||||
|
}
|
||||||
|
|
||||||
|
rule {
|
||||||
|
api_groups = ["apps"]
|
||||||
|
resources = ["deployments", "statefulsets", "daemonsets"]
|
||||||
|
verbs = ["get", "list", "watch"]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
resource "kubernetes_cluster_role_binding" "namespace_owner_readonly" {
|
||||||
|
for_each = { for name, user in var.k8s_users : name => user if user.role == "namespace-owner" }
|
||||||
|
|
||||||
|
metadata {
|
||||||
|
name = "oidc-ns-owner-readonly-${each.key}"
|
||||||
|
}
|
||||||
|
|
||||||
|
role_ref {
|
||||||
|
api_group = "rbac.authorization.k8s.io"
|
||||||
|
kind = "ClusterRole"
|
||||||
|
name = kubernetes_cluster_role.namespace_owner_readonly.metadata[0].name
|
||||||
|
}
|
||||||
|
|
||||||
|
subject {
|
||||||
|
kind = "User"
|
||||||
|
name = each.value.email
|
||||||
|
api_group = "rbac.authorization.k8s.io"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
# Resource quotas per user namespace
|
||||||
|
resource "kubernetes_resource_quota" "user_namespace_quota" {
|
||||||
|
for_each = { for pair in local.namespace_owner_pairs : "${pair.user_key}-${pair.namespace}" => pair }
|
||||||
|
|
||||||
|
metadata {
|
||||||
|
name = "user-quota"
|
||||||
|
namespace = each.value.namespace
|
||||||
|
}
|
||||||
|
|
||||||
|
spec {
|
||||||
|
hard = {
|
||||||
|
"requests.cpu" = each.value.quota.cpu_requests
|
||||||
|
"requests.memory" = each.value.quota.memory_requests
|
||||||
|
"limits.cpu" = each.value.quota.cpu_limits
|
||||||
|
"limits.memory" = each.value.quota.memory_limits
|
||||||
|
"pods" = each.value.quota.pods
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
depends_on = [kubernetes_namespace.user_namespaces]
|
||||||
|
}
|
||||||
|
|
||||||
|
# ConfigMap with user-role mapping for the self-service portal
|
||||||
|
resource "kubernetes_config_map" "user_roles" {
|
||||||
|
metadata {
|
||||||
|
name = "k8s-user-roles"
|
||||||
|
namespace = "k8s-portal"
|
||||||
|
}
|
||||||
|
|
||||||
|
data = {
|
||||||
|
"users.json" = jsonencode({
|
||||||
|
for name, user in var.k8s_users : user.email => {
|
||||||
|
role = user.role
|
||||||
|
namespaces = user.namespaces
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
Loading…
Add table
Add a link
Reference in a new issue