From 0610ea30d466fd05ebd3749fefda259005b2bd4c Mon Sep 17 00:00:00 2001 From: Viktor Barzin Date: Sun, 15 Mar 2026 22:23:36 +0000 Subject: [PATCH] add generic multi-user cluster onboarding system Data-driven user onboarding: add a JSON entry to Vault KV k8s_users, apply vault + platform + woodpecker stacks, and everything is auto-generated. Vault stack: namespace creation, per-user Vault policies with secret isolation via identity entities/aliases, K8s deployer roles, CI policy update. Platform stack: domains field in k8s_users type, TLS secrets per user namespace, user domains merged into Cloudflare DNS, user-roles ConfigMap mounted in portal. Woodpecker stack: admin list auto-generated from k8s_users, WOODPECKER_OPEN=true. K8s-portal: dual-track onboarding (general/namespace-owner), namespace-owner dashboard with Vault/kubectl commands, setup script adds Vault+Terraform+Terragrunt, contributing page with CI pipeline template, versioned image tags in CI pipeline. New: stacks/_template/ with copyable stack template for namespace-owners. --- .woodpecker/k8s-portal.yml | 8 +- stacks/_template/main.tf.example | 90 ++++++++++++ stacks/_template/terragrunt.hcl | 8 ++ stacks/platform/main.tf | 8 +- .../k8s-portal/files/src/routes/+page.svelte | 31 +++- .../src/routes/contributing/+page.svelte | 53 +++++++ .../files/src/routes/onboarding/+page.svelte | 119 ++++++++++++---- .../files/src/routes/setup/script/+server.ts | 66 +++++++++ stacks/platform/modules/k8s-portal/main.tf | 21 ++- stacks/platform/modules/rbac/main.tf | 13 ++ stacks/vault/main.tf | 134 ++++++++++++++++++ stacks/woodpecker/main.tf | 15 ++ stacks/woodpecker/values.yaml | 4 +- 13 files changed, 530 insertions(+), 40 deletions(-) create mode 100644 stacks/_template/main.tf.example create mode 100644 stacks/_template/terragrunt.hcl diff --git a/.woodpecker/k8s-portal.yml b/.woodpecker/k8s-portal.yml index a540cf61..39c9ff17 100644 --- a/.woodpecker/k8s-portal.yml +++ b/.woodpecker/k8s-portal.yml @@ -24,23 +24,23 @@ steps: context: stacks/platform/modules/k8s-portal/files platforms: - linux/amd64 - auto_tag: true + tag: ["${CI_PIPELINE_NUMBER}", "latest"] cache_from: "viktorbarzin/k8s-portal:latest" cache_to: "type=inline" - name: deploy image: bitnami/kubectl:latest commands: - - "kubectl rollout restart deployment/k8s-portal -n k8s-portal" + - "kubectl set image deployment/k8s-portal portal=viktorbarzin/k8s-portal:${CI_PIPELINE_NUMBER} -n k8s-portal" - "kubectl rollout status deployment/k8s-portal -n k8s-portal --timeout=120s" - - "echo 'k8s-portal deployed successfully'" + - "echo 'k8s-portal deployed successfully (build ${CI_PIPELINE_NUMBER})'" - name: slack image: curlimages/curl commands: - | curl -s -X POST -H 'Content-type: application/json' \ - --data "{\"text\":\"K8s Portal: build + deploy ${CI_PIPELINE_STATUS}\"}" \ + --data "{\"text\":\"K8s Portal: build #${CI_PIPELINE_NUMBER} ${CI_PIPELINE_STATUS}\"}" \ "$SLACK_WEBHOOK" || true environment: SLACK_WEBHOOK: diff --git a/stacks/_template/main.tf.example b/stacks/_template/main.tf.example new file mode 100644 index 00000000..1a85b00a --- /dev/null +++ b/stacks/_template/main.tf.example @@ -0,0 +1,90 @@ +# ============================================================================= +# Stack Template — Copy this directory to stacks// and customize. +# Then submit a PR to the infra repo. +# ============================================================================= +# +# Prerequisites: +# 1. You are a namespace-owner in k8s_users (Vault KV secret/platform) +# 2. Your namespace already exists (created by vault stack) +# 3. You have Vault CLI access: vault login -method=oidc +# +# Steps: +# 1. cp -r stacks/_template stacks/myapp +# 2. mv stacks/myapp/main.tf.example stacks/myapp/main.tf +# 3. Search-replace below +# 4. Store secrets: vault kv put secret//myapp KEY=value +# 5. git checkout -b feat/myapp && git push +# 6. Open PR, get reviewed, merge +# 7. Admin runs: cd stacks/myapp && terragrunt apply +# ============================================================================= + +variable "tls_secret_name" { + type = string + sensitive = true +} + +# NOTE: Your namespace is auto-created by the vault stack from k8s_users. +# Only add a kubernetes_namespace resource if you need a SEPARATE namespace +# for this specific app (not your user namespace). + +module "tls_secret" { + source = "../../modules/kubernetes/setup_tls_secret" + namespace = "" # e.g., "anca" + tls_secret_name = var.tls_secret_name +} + +resource "kubernetes_deployment" "app" { + metadata { + name = "" + namespace = "" + } + spec { + replicas = 1 + selector { + match_labels = { app = "" } + } + template { + metadata { + labels = { app = "" } + } + spec { + container { + name = "" + image = "/:" + port { + container_port = 8080 # Change to your app's port + } + resources { + requests = { cpu = "10m", memory = "128Mi" } + limits = { memory = "128Mi" } + } + } + } + } + } + lifecycle { + ignore_changes = [spec[0].template[0].spec[0].dns_config] + } +} + +resource "kubernetes_service" "app" { + metadata { + name = "" + namespace = "" + } + spec { + selector = { app = "" } + port { + port = 80 + target_port = 8080 # Match container_port above + } + } +} + +module "ingress" { + source = "../../modules/kubernetes/ingress_factory" + namespace = "" + name = "" + tls_secret_name = var.tls_secret_name + protected = false # Set true to require Authentik login +} diff --git a/stacks/_template/terragrunt.hcl b/stacks/_template/terragrunt.hcl new file mode 100644 index 00000000..0d1c8e53 --- /dev/null +++ b/stacks/_template/terragrunt.hcl @@ -0,0 +1,8 @@ +include "root" { + path = find_in_parent_folders() +} + +dependency "platform" { + config_path = "../platform" + skip_outputs = true +} diff --git a/stacks/platform/main.tf b/stacks/platform/main.tf index 4c0793ab..85f7237c 100644 --- a/stacks/platform/main.tf +++ b/stacks/platform/main.tf @@ -68,6 +68,12 @@ locals { mailserver_aliases = jsondecode(data.vault_kv_secret_v2.secrets.data["mailserver_aliases"]) mailserver_opendkim_key = jsondecode(data.vault_kv_secret_v2.secrets.data["mailserver_opendkim_key"]) mailserver_sasl_passwd = jsondecode(data.vault_kv_secret_v2.secrets.data["mailserver_sasl_passwd"]) + + # User domains from namespace-owners for DNS/Cloudflare + user_domains = flatten([ + for name, user in local.k8s_users : user.domains + if user.role == "namespace-owner" + ]) } # ============================================================================= @@ -377,7 +383,7 @@ module "cloudflared" { cloudflare_zone_id = var.cloudflare_zone_id cloudflare_tunnel_id = var.cloudflare_tunnel_id public_ip = var.public_ip - cloudflare_proxied_names = var.cloudflare_proxied_names + cloudflare_proxied_names = concat(var.cloudflare_proxied_names, local.user_domains) cloudflare_non_proxied_names = var.cloudflare_non_proxied_names cloudflare_tunnel_token = data.vault_kv_secret_v2.secrets.data["cloudflare_tunnel_token"] } diff --git a/stacks/platform/modules/k8s-portal/files/src/routes/+page.svelte b/stacks/platform/modules/k8s-portal/files/src/routes/+page.svelte index 961011cd..2d13fa39 100644 --- a/stacks/platform/modules/k8s-portal/files/src/routes/+page.svelte +++ b/stacks/platform/modules/k8s-portal/files/src/routes/+page.svelte @@ -20,10 +20,39 @@ {/if} + {#if data.role === 'namespace-owner'} +
+

Your Namespace

+

Assigned namespaces: {data.namespaces.join(', ')}

+ +

Quick Commands

+ 
+# Check your pods
+kubectl get pods -n {data.namespaces[0]}
+
+# View quota usage
+kubectl describe resourcequota -n {data.namespaces[0]}
+
+# Log into Vault
+vault login -method=oidc
+
+# Store a secret
+vault kv put secret/{data.username}/myapp KEY=value
+
+# Get K8s deploy token
+vault write kubernetes/creds/{data.namespaces[0]}-deployer \
+  kubernetes_namespace={data.namespaces[0]}
+
+ {/if} +

Get Started

    -
  1. Complete the onboarding guide (VPN, kubectl, git)
    2. + {#if data.role === 'namespace-owner'} +
  2. Complete the namespace-owner onboarding guide
    3. + {:else} +
  3. Complete the onboarding guide (VPN, kubectl, git)
    4. + {/if}
  4. Install kubectl and kubelogin
  5. Download your kubeconfig
  6. Run kubectl get namespaces to verify access
    7. diff --git a/stacks/platform/modules/k8s-portal/files/src/routes/contributing/+page.svelte b/stacks/platform/modules/k8s-portal/files/src/routes/contributing/+page.svelte index 6f0d1903..2375a2cb 100644 --- a/stacks/platform/modules/k8s-portal/files/src/routes/contributing/+page.svelte +++ b/stacks/platform/modules/k8s-portal/files/src/routes/contributing/+page.svelte @@ -48,6 +48,59 @@

    Comment on your PR: "I need a database password for my-service." Viktor will add it to the encrypted secrets file and push to your branch.

    Then reference it in your Terraform: var.my_service_db_password

+ +
+

Namespace Owner Workflow

+

If you are a namespace owner, you can deploy your own apps:

+
    +
  1. Clone the infra repo: git clone https://github.com/ViktorBarzin/infra.git
    2. +
  2. Copy the template: cp -r stacks/_template stacks/your-app
    3. +
  3. Rename: mv stacks/your-app/main.tf.example stacks/your-app/main.tf
    4. +
  4. Edit main.tf — replace all <placeholders>
    5. +
  5. Store secrets in Vault: vault kv put secret/your-username/your-app KEY=value
    6. +
  6. Add your app domain to your domains list in Vault KV
    7. +
  7. Submit a PR, get it reviewed
    8. +
  8. After merge, admin runs terragrunt apply
    9. +
+
+ +
+

CI Pipeline Template

+

Create a .woodpecker.yml in your app's Forgejo repo:

+ 
{`steps:
+  - name: build
+    image: woodpeckerci/plugin-docker-buildx
+    settings:
+      repo: your-dockerhub-user/myapp
+      tag: ["\${CI_PIPELINE_NUMBER}", "latest"]
+      username:
+        from_secret: dockerhub-username
+      password:
+        from_secret: dockerhub-token
+      platforms: linux/amd64
+
+  - name: deploy
+    image: hashicorp/vault:1.18.1
+    commands:
+      - export VAULT_ADDR=http://vault-active.vault.svc.cluster.local:8200
+      - export VAULT_TOKEN=$(vault write -field=token auth/kubernetes/login
+          role=ci jwt=$(cat /var/run/secrets/kubernetes.io/serviceaccount/token))
+      - KUBE_TOKEN=$(vault write -field=service_account_token
+          kubernetes/creds/YOUR_NAMESPACE-deployer
+          kubernetes_namespace=YOUR_NAMESPACE)
+      - kubectl --server=https://kubernetes.default.svc
+          --token=$KUBE_TOKEN
+          --certificate-authority=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt
+          -n YOUR_NAMESPACE set image deployment/myapp
+          myapp=your-dockerhub-user/myapp:\${CI_PIPELINE_NUMBER}`}
+
+ +
+

Need a secret for your app?

+

As a namespace owner, you manage your own secrets in Vault:

+ 
vault kv put secret/your-username/your-app DB_PASSWORD=mysecret API_KEY=abc123
+

Then reference them in your Terraform using a data "vault_kv_secret_v2" block.

+
diff --git a/stacks/platform/modules/k8s-portal/files/src/routes/setup/script/+server.ts b/stacks/platform/modules/k8s-portal/files/src/routes/setup/script/+server.ts index 0f1b18e7..82419194 100644 --- a/stacks/platform/modules/k8s-portal/files/src/routes/setup/script/+server.ts +++ b/stacks/platform/modules/k8s-portal/files/src/routes/setup/script/+server.ts @@ -105,6 +105,44 @@ else echo "[OK] kubeseal installed" fi +# Install Vault CLI +if command -v vault &>/dev/null; then + echo "[OK] vault already installed" +else + echo "[..] Installing Vault CLI..." + VAULT_VERSION="1.18.1" + curl -fsSLO "https://releases.hashicorp.com/vault/\${VAULT_VERSION}/vault_\${VAULT_VERSION}_linux_amd64.zip" + unzip -o "vault_\${VAULT_VERSION}_linux_amd64.zip" vault -d /tmp + \$SUDO mv /tmp/vault "\$INSTALL_DIR/" + rm -f "vault_\${VAULT_VERSION}_linux_amd64.zip" + echo "[OK] vault installed" +fi + +# Install Terragrunt +if command -v terragrunt &>/dev/null; then + echo "[OK] terragrunt already installed" +else + echo "[..] Installing terragrunt..." + TG_VERSION=\$(curl -fsSL -o /dev/null -w "%{url_effective}" https://github.com/gruntwork-io/terragrunt/releases/latest | grep -o '[^/]*\$') + curl -fsSLO "https://github.com/gruntwork-io/terragrunt/releases/download/\${TG_VERSION}/terragrunt_linux_amd64" + chmod +x terragrunt_linux_amd64 + \$SUDO mv terragrunt_linux_amd64 "\$INSTALL_DIR/terragrunt" + echo "[OK] terragrunt installed" +fi + +# Install Terraform +if command -v terraform &>/dev/null; then + echo "[OK] terraform already installed" +else + echo "[..] Installing terraform..." + TF_VERSION="1.9.8" + curl -fsSLO "https://releases.hashicorp.com/terraform/\${TF_VERSION}/terraform_\${TF_VERSION}_linux_amd64.zip" + unzip -o "terraform_\${TF_VERSION}_linux_amd64.zip" terraform -d /tmp + \$SUDO mv /tmp/terraform "\$INSTALL_DIR/" + rm -f "terraform_\${TF_VERSION}_linux_amd64.zip" + echo "[OK] terraform installed" +fi + # Write kubeconfig mkdir -p ~/.kube cat > ~/.kube/config-home << 'KUBECONFIG_EOF' @@ -168,6 +206,34 @@ else echo "[OK] kubeseal installed" fi +# Install Vault CLI +if command -v vault &>/dev/null; then + echo "[OK] vault already installed" +else + echo "[..] Installing Vault CLI..." + brew tap hashicorp/tap + brew install hashicorp/tap/vault + echo "[OK] vault installed" +fi + +# Install Terragrunt +if command -v terragrunt &>/dev/null; then + echo "[OK] terragrunt already installed" +else + echo "[..] Installing terragrunt..." + brew install terragrunt + echo "[OK] terragrunt installed" +fi + +# Install Terraform +if command -v terraform &>/dev/null; then + echo "[OK] terraform already installed" +else + echo "[..] Installing terraform..." + brew install hashicorp/tap/terraform + echo "[OK] terraform installed" +fi + # Write kubeconfig mkdir -p ~/.kube cat > ~/.kube/config-home << 'KUBECONFIG_EOF' diff --git a/stacks/platform/modules/k8s-portal/main.tf b/stacks/platform/modules/k8s-portal/main.tf index b1265138..15d1ab08 100644 --- a/stacks/platform/modules/k8s-portal/main.tf +++ b/stacks/platform/modules/k8s-portal/main.tf @@ -66,7 +66,14 @@ resource "kubernetes_deployment" "k8s_portal" { volume_mount { name = "config" - mount_path = "/config" + mount_path = "/config/ca.crt" + sub_path = "ca.crt" + read_only = true + } + volume_mount { + name = "user-roles" + mount_path = "/config/users.json" + sub_path = "users.json" read_only = true } resources { @@ -86,6 +93,12 @@ resource "kubernetes_deployment" "k8s_portal" { name = kubernetes_config_map.k8s_portal_config.metadata[0].name } } + volume { + name = "user-roles" + config_map { + name = "k8s-user-roles" + } + } dns_config { option { name = "ndots" @@ -95,6 +108,12 @@ resource "kubernetes_deployment" "k8s_portal" { } } } + lifecycle { + ignore_changes = [ + spec[0].template[0].spec[0].dns_config, + spec[0].template[0].spec[0].container[0].image, # CI updates image tag + ] + } } resource "kubernetes_service" "k8s_portal" { diff --git a/stacks/platform/modules/rbac/main.tf b/stacks/platform/modules/rbac/main.tf index ef76292b..203be47b 100644 --- a/stacks/platform/modules/rbac/main.tf +++ b/stacks/platform/modules/rbac/main.tf @@ -6,6 +6,7 @@ variable "k8s_users" { role = string # "admin", "power-user", "namespace-owner" email = string # OIDC email claim namespaces = optional(list(string), []) # for namespace-owners + domains = optional(list(string), []) # subdomains for user apps quota = optional(object({ cpu_requests = optional(string, "2") memory_requests = optional(string, "4Gi") @@ -248,3 +249,15 @@ resource "kubernetes_config_map" "user_roles" { }) } } + +# TLS secret in each user namespace (so they can create HTTPS ingresses) +module "user_namespace_tls" { + for_each = nonsensitive(toset(flatten([ + for name, user in var.k8s_users : user.namespaces + if user.role == "namespace-owner" + ]))) + + source = "../../../../modules/kubernetes/setup_tls_secret" + namespace = each.value + tls_secret_name = var.tls_secret_name +} diff --git a/stacks/vault/main.tf b/stacks/vault/main.tf index d12dacd4..05636e9e 100644 --- a/stacks/vault/main.tf +++ b/stacks/vault/main.tf @@ -317,6 +317,10 @@ resource "vault_policy" "ci" { path "secret/metadata/*" { capabilities = ["list"] } + # Allow CI to get dynamic K8s deploy tokens for user namespaces + path "kubernetes/creds/*-deployer" { + capabilities = ["read"] + } EOT } @@ -654,3 +658,133 @@ resource "vault_kubernetes_secret_backend_role" "local_admin" { kubernetes_role_type = "ClusterRole" kubernetes_role_name = "cluster-admin" } + +# ============================================================================= +# Multi-User Namespace Onboarding +# ============================================================================= +# All resources below are auto-generated from the k8s_users map in Vault KV. +# Adding a new user requires only a JSON entry in secret/platform → k8s_users. + +data "vault_kv_secret_v2" "platform" { + mount = "secret" + name = "platform" + depends_on = [helm_release.vault] +} + +locals { + k8s_users = jsondecode(data.vault_kv_secret_v2.platform.data["k8s_users"]) + + # Flatten user -> namespace pairs for namespace-owners + namespace_owner_namespaces = flatten([ + for name, user in local.k8s_users : [ + for ns in user.namespaces : { + user_key = name + namespace = ns + email = user.email + } + ] if user.role == "namespace-owner" + ]) + + # Unique namespaces across all namespace-owners + user_namespaces = toset(flatten([ + for name, user in local.k8s_users : user.namespaces + if user.role == "namespace-owner" + ])) +} + +resource "kubernetes_namespace" "user_namespace" { + for_each = local.user_namespaces + + metadata { + name = each.value + labels = { + tier = "4-aux" + "resource-governance/custom-quota" = "true" + "managed-by" = "vault-user-onboarding" + } + } +} + +resource "vault_policy" "namespace_owner" { + for_each = nonsensitive({ + for name, user in local.k8s_users : name => user + if user.role == "namespace-owner" + }) + + name = "namespace-owner-${each.key}" + policy = <<-EOT + # Read/write own secrets + path "secret/data/${each.key}" { + capabilities = ["create", "read", "update", "delete", "list"] + } + path "secret/data/${each.key}/*" { + capabilities = ["create", "read", "update", "delete", "list"] + } + path "secret/metadata/${each.key}" { + capabilities = ["list", "read", "delete"] + } + path "secret/metadata/${each.key}/*" { + capabilities = ["list", "read", "delete"] + } + %{for ns in each.value.namespaces} + # Dynamic K8s credentials for ${ns} namespace + path "kubernetes/creds/${ns}-deployer" { + capabilities = ["read"] + } + %{endfor} + EOT +} + +resource "vault_identity_entity" "namespace_owner" { + for_each = nonsensitive({ + for name, user in local.k8s_users : name => user + if user.role == "namespace-owner" + }) + + name = each.key + policies = [vault_policy.namespace_owner[each.key].name] +} + +resource "vault_identity_entity_alias" "namespace_owner" { + for_each = nonsensitive({ + for name, user in local.k8s_users : name => user + if user.role == "namespace-owner" + }) + + name = each.value.email + mount_accessor = vault_jwt_auth_backend.oidc.accessor + canonical_id = vault_identity_entity.namespace_owner[each.key].id +} + +resource "kubernetes_role" "user_deployer" { + for_each = local.user_namespaces + + metadata { + name = "${each.value}-deployer" + namespace = each.value + } + rule { + api_groups = ["apps"] + resources = ["deployments"] + verbs = ["get", "list", "patch", "update"] + } + rule { + api_groups = [""] + resources = ["pods"] + verbs = ["get", "list"] + } + + depends_on = [kubernetes_namespace.user_namespace] +} + +resource "vault_kubernetes_secret_backend_role" "user_deployer" { + for_each = local.user_namespaces + + backend = vault_kubernetes_secret_backend.k8s.path + name = "${each.value}-deployer" + allowed_kubernetes_namespaces = [each.value] + token_default_ttl = 1800 + token_max_ttl = 3600 + kubernetes_role_type = "Role" + kubernetes_role_name = kubernetes_role.user_deployer[each.key].metadata[0].name +} diff --git a/stacks/woodpecker/main.tf b/stacks/woodpecker/main.tf index f94a1e1d..cad34908 100644 --- a/stacks/woodpecker/main.tf +++ b/stacks/woodpecker/main.tf @@ -11,6 +11,20 @@ data "vault_kv_secret_v2" "secrets" { name = "woodpecker" } +data "vault_kv_secret_v2" "platform" { + mount = "secret" + name = "platform" +} + +locals { + k8s_users = jsondecode(data.vault_kv_secret_v2.platform.data["k8s_users"]) + + # Build admin list: existing admin + all namespace-owner usernames + woodpecker_admins = join(",", concat( + ["ViktorBarzin"], + [for name, user in local.k8s_users : name if user.role == "namespace-owner"] + )) +} resource "kubernetes_namespace" "woodpecker" { metadata { @@ -206,6 +220,7 @@ resource "helm_release" "woodpecker" { forgejo_client_id = data.vault_kv_secret_v2.secrets.data["forgejo_client_id"] forgejo_client_secret = data.vault_kv_secret_v2.secrets.data["forgejo_client_secret"] forgejo_url = var.woodpecker_forgejo_url + woodpecker_admins = local.woodpecker_admins }) ] diff --git a/stacks/woodpecker/values.yaml b/stacks/woodpecker/values.yaml index f264d2de..c8247e0e 100644 --- a/stacks/woodpecker/values.yaml +++ b/stacks/woodpecker/values.yaml @@ -10,8 +10,8 @@ server: tag: "v3.13.0" env: WOODPECKER_HOST: "https://ci.viktorbarzin.me" - WOODPECKER_ADMIN: "ViktorBarzin" - WOODPECKER_OPEN: "false" + WOODPECKER_ADMIN: "${woodpecker_admins}" + WOODPECKER_OPEN: "true" WOODPECKER_GITHUB: "true" WOODPECKER_GITHUB_URL: "https://github.com" WOODPECKER_GITHUB_CLIENT: "${github_client_id}"