From cb1a34fd00a9ec411e7ff5b6d25a387cf5bc4605 Mon Sep 17 00:00:00 2001 From: Viktor Barzin Date: Sun, 24 May 2026 11:27:26 +0000 Subject: [PATCH] nextcloud: expose PVE NFS roots + /anca-elements via Files External MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Mounts the Proxmox host NFS exports (/srv/nfs and /srv/nfs-ssd) into the NC pod and surfaces them through occ files_external:create: - /PVE NFS Pool → /mnt/pve-nfs (admin group only) - /PVE NFS-SSD Pool → /mnt/pve-nfs-ssd (admin group only) - /anca-elements → /mnt/pve-nfs/anca-elements (admin, anca users) Mount visibility is controlled by occ files_external:applicable; no Files Access Control. ACL state is reconciled idempotently by a bootstrap Job that diffs desired vs current applicable_users / applicable_groups (via files_external:list --output=json). Bootstrap fixes vs initial design: - Sync loop used `[ -n "$U" ] && cmd` which returns 1 on empty input, triggering set -e on no-op re-runs. Switched to process substitution `< <(jq ...)` so empty diff -> loop body never runs -> 0 exit. - RBAC missed `watch` verb (kubectl wait spammed reflector errors). - Manifest used display-name "viktor" instead of NC username "admin" for the /anca-elements applicable list. Chart values: added two PV-backed volume mounts at /mnt/pve-nfs[+ssd] and pinned securityContext to fsGroup=33 with fsGroupChangePolicy: OnRootMismatch (chart default Always would recurse 600k+ files on every pod restart). --- stacks/nextcloud/.terraform.lock.hcl | 8 + stacks/nextcloud/chart_values.yaml | 19 ++ stacks/nextcloud/external_storage.tf | 293 +++++++++++++++++++++++++++ stacks/nextcloud/main.tf | 20 +- stacks/nextcloud/providers.tf | 12 ++ 5 files changed, 351 insertions(+), 1 deletion(-) create mode 100644 stacks/nextcloud/external_storage.tf diff --git a/stacks/nextcloud/.terraform.lock.hcl b/stacks/nextcloud/.terraform.lock.hcl index fabbc047..522ec0cc 100644 --- a/stacks/nextcloud/.terraform.lock.hcl +++ b/stacks/nextcloud/.terraform.lock.hcl @@ -24,6 +24,14 @@ provider "registry.terraform.io/cloudflare/cloudflare" { ] } +provider "registry.terraform.io/gavinbunney/kubectl" { + version = "1.19.0" + constraints = "~> 1.14" + hashes = [ + "h1:9QkxPjp0x5FZFfJbE+B7hBOoads9gmdfj9aYu5N4Sfc=", + ] +} + provider "registry.terraform.io/goauthentik/authentik" { version = "2024.12.1" constraints = "~> 2024.10" diff --git a/stacks/nextcloud/chart_values.yaml b/stacks/nextcloud/chart_values.yaml index f7eb302c..3b44d270 100644 --- a/stacks/nextcloud/chart_values.yaml +++ b/stacks/nextcloud/chart_values.yaml @@ -73,12 +73,22 @@ nextcloud: configMap: name: nextcloud-db-password-sync defaultMode: 0755 + - name: pve-nfs + persistentVolumeClaim: + claimName: nextcloud-pve-nfs-root + - name: pve-nfs-ssd + persistentVolumeClaim: + claimName: nextcloud-pve-nfs-ssd-root extraVolumeMounts: - name: apache-tuning mountPath: /etc/apache2/mods-available/mpm_prefork.conf subPath: mpm_prefork.conf - name: db-password-sync mountPath: /docker-entrypoint-hooks.d/before-starting + - name: pve-nfs + mountPath: /mnt/pve-nfs + - name: pve-nfs-ssd + mountPath: /mnt/pve-nfs-ssd internalDatabase: enabled: false @@ -134,6 +144,15 @@ podAnnotations: dependency.kyverno.io/wait-for: "mysql.dbaas:3306,redis-master.redis:6379" secret.reloader.stakater.com/reload: "nextcloud-db-creds" +# OnRootMismatch: kubelet only recursively chowns the volume to fsGroup if the +# root dir's GID doesn't already match. Without this, every pod restart triggers +# a ~30-min recursive chown of /srv/nfs and /srv/nfs-ssd (600k+ files) — the +# default policy "Always" recurses every time. Locks fsGroup=33 explicitly so +# this block fully replaces the chart's default {fsGroup: 33}. +securityContext: + fsGroup: 33 + fsGroupChangePolicy: OnRootMismatch + collabora: enabled: false # Using onlyoffice instead diff --git a/stacks/nextcloud/external_storage.tf b/stacks/nextcloud/external_storage.tf new file mode 100644 index 00000000..bcb4247b --- /dev/null +++ b/stacks/nextcloud/external_storage.tf @@ -0,0 +1,293 @@ +# Nextcloud Files External bootstrap — mount-per-archive + applicable_users model. +# Creates two admin-only root browser mounts (PVE NFS Pool, PVE NFS-SSD Pool) +# pointing at the NFS roots mounted at /mnt/pve-nfs and /mnt/pve-nfs-ssd inside +# the Nextcloud container, plus per-archive mounts visible only to the named +# users. Safe to re-run — the bootstrap Job is idempotent. +# +# ACL model (verified via context7 + NC docs): +# Mount visibility is controlled by `occ files_external:applicable`. +# A mount with no applicable users/groups is visible to ALL users — so we +# always set at least one applicable group (admin) or user list. +# +# occ commands used (syntax verified via context7): +# files_external:create local null::null --config "datadir=" +# files_external:list --output=json → array; each entry has numeric .mount_id, +# .applicable_users [], .applicable_groups [] +# files_external:applicable --add-user= +# files_external:applicable --remove-user= +# files_external:applicable --add-group= +# files_external:applicable --remove-group= +# +# Note: `files_external:applicable` has NO --output=json flag (write-only command). +# Current applicable state is read from files_external:list --output=json instead. +# +# NO Files Access Control. Drop the workflow-engine machinery entirely. + +# ── External storage manifest (JSON) ──────────────────────────────────────── + +resource "kubernetes_config_map_v1" "nextcloud_external_storage_manifest" { + metadata { + name = "nextcloud-external-storage-manifest" + namespace = kubernetes_namespace.nextcloud.metadata[0].name + } + + data = { + "manifest.json" = jsonencode({ + rootMounts = [ + { + mountPoint = "/PVE NFS Pool" + dataDir = "/mnt/pve-nfs" + applicableGroup = "admin" + }, + { + mountPoint = "/PVE NFS-SSD Pool" + dataDir = "/mnt/pve-nfs-ssd" + applicableGroup = "admin" + }, + ] + archiveMounts = [ + { + mountPoint = "/anca-elements" + dataDir = "/mnt/pve-nfs/anca-elements" + # NC usernames (not display names): admin is Viktor, anca is Anca. + applicableUsers = ["anca", "admin"] + applicableGroups = [] + }, + ] + }) + } +} + +# ── RBAC for the bootstrap Job ─────────────────────────────────────────────── + +resource "kubernetes_service_account" "nextcloud_external_storage_bootstrap" { + metadata { + name = "nextcloud-external-storage-bootstrap" + namespace = kubernetes_namespace.nextcloud.metadata[0].name + } +} + +resource "kubernetes_role" "nextcloud_external_storage_bootstrap" { + metadata { + name = "nextcloud-external-storage-bootstrap" + namespace = kubernetes_namespace.nextcloud.metadata[0].name + } + + rule { + api_groups = [""] + resources = ["pods"] + verbs = ["list", "get", "watch"] + } + + rule { + api_groups = [""] + resources = ["pods/exec"] + verbs = ["create"] + } +} + +resource "kubernetes_role_binding" "nextcloud_external_storage_bootstrap" { + metadata { + name = "nextcloud-external-storage-bootstrap" + namespace = kubernetes_namespace.nextcloud.metadata[0].name + } + + role_ref { + api_group = "rbac.authorization.k8s.io" + kind = "Role" + name = kubernetes_role.nextcloud_external_storage_bootstrap.metadata[0].name + } + + subject { + kind = "ServiceAccount" + name = kubernetes_service_account.nextcloud_external_storage_bootstrap.metadata[0].name + namespace = kubernetes_namespace.nextcloud.metadata[0].name + } +} + +# ── Bootstrap Job ──────────────────────────────────────────────────────────── + +resource "kubernetes_job_v1" "nextcloud_external_storage_bootstrap" { + metadata { + name = "nextcloud-external-storage-bootstrap" + namespace = kubernetes_namespace.nextcloud.metadata[0].name + } + + spec { + backoff_limit = 5 + ttl_seconds_after_finished = 600 + + template { + metadata {} + spec { + restart_policy = "OnFailure" + service_account_name = kubernetes_service_account.nextcloud_external_storage_bootstrap.metadata[0].name + + container { + name = "bootstrap" + image = "bitnami/kubectl:latest" + + # bitnami/kubectl (debian-12 base) ships jq — no apt-get needed. + # HCL heredoc: only $${...} needs escaping; bare $VAR and $(...) + # are passed through unchanged by HCL. No nested heredocs used. + command = ["/bin/bash", "-c", <<-EOF + set -euo pipefail + trap 'echo "[bootstrap] FAIL at line $LINENO — exit $?"' ERR + + MANIFEST=/manifest/manifest.json + NC_NS=nextcloud + NC_LABEL="app.kubernetes.io/name=nextcloud" + + # ── 1. Wait for NC pod to be Ready ────────────────────────────── + echo "[bootstrap] Waiting for NC pod Ready (timeout 10m)..." + kubectl wait -n "$NC_NS" pod \ + -l "$NC_LABEL" \ + --for=condition=Ready \ + --timeout=600s + echo "[bootstrap] Pod is Ready." + + # ── 2. Resolve pod name ───────────────────────────────────────── + NC_POD=$(kubectl get pods -n "$NC_NS" -l "$NC_LABEL" \ + -o jsonpath='{.items[0].metadata.name}') + echo "[bootstrap] Target pod: $NC_POD" + + # ── 3. occ helper — must run as www-data ──────────────────────── + nc_occ() { + kubectl exec -n "$NC_NS" "$NC_POD" -c nextcloud -- \ + runuser -u www-data -- php /var/www/html/occ "$@" + } + + # ── 4. Enable files_external (idempotent) ─────────────────────── + nc_occ app:enable files_external || true + # NO files_accesscontrol — that app is not used in this model. + + # ── 5. Helpers ────────────────────────────────────────────────── + + # get_mount_id + # Reads files_external:list --output=json (array of mount objects). + # Each object has a numeric "mount_id" and a string "mount_point". + get_mount_id() { + local MP="$1" + nc_occ files_external:list --output=json 2>/dev/null \ + | jq -r --arg mp "$MP" \ + '.[] | select(.mount_point == $mp) | .mount_id' \ + | head -1 + } + + # ensure_mount → echoes the numeric mount id + ensure_mount() { + local MP="$1" DIR="$2" + local MID + MID=$(get_mount_id "$MP") + if [ -z "$MID" ]; then + echo "[bootstrap] Creating mount '$MP' -> $DIR" >&2 + nc_occ files_external:create "$MP" local null::null \ + --config "datadir=$DIR" + MID=$(get_mount_id "$MP") + else + echo "[bootstrap] Mount '$MP' already exists (id=$MID)" >&2 + fi + echo "$MID" + } + + # sync_applicable + # Reads current applicable state from files_external:list --output=json + # (fields: applicable_users [], applicable_groups []). + # Diffs against desired sets; adds missing, removes extras. + # When no applicable users + no groups are set, NC treats the mount + # as visible to ALL — so desired sets must always be non-empty. + # + # Process substitution `< <(jq ...)` feeds the loops directly: when + # jq emits no rows (already-synced state), the body never runs and + # the loop returns 0 — avoiding a set -e exit on a no-op re-run. + sync_applicable() { + local MID="$1" DESIRED_USERS_JSON="$2" DESIRED_GROUPS_JSON="$3" + + # Read current state from files_external:list --output=json + local MOUNT_JSON + MOUNT_JSON=$(nc_occ files_external:list --output=json 2>/dev/null \ + | jq -c --argjson mid "$MID" '.[] | select(.mount_id == $mid)') + + local CURRENT_USERS_JSON CURRENT_GROUPS_JSON + CURRENT_USERS_JSON=$(echo "$MOUNT_JSON" \ + | jq -c '.applicable_users // []') + CURRENT_GROUPS_JSON=$(echo "$MOUNT_JSON" \ + | jq -c '.applicable_groups // []') + + while IFS= read -r U; do + nc_occ files_external:applicable "$MID" --add-user="$U" + done < <(jq -rn \ + --argjson d "$DESIRED_USERS_JSON" \ + --argjson c "$CURRENT_USERS_JSON" \ + '($d - $c)[]') + + while IFS= read -r U; do + nc_occ files_external:applicable "$MID" --remove-user="$U" + done < <(jq -rn \ + --argjson d "$DESIRED_USERS_JSON" \ + --argjson c "$CURRENT_USERS_JSON" \ + '($c - $d)[]') + + while IFS= read -r G; do + nc_occ files_external:applicable "$MID" --add-group="$G" + done < <(jq -rn \ + --argjson d "$DESIRED_GROUPS_JSON" \ + --argjson c "$CURRENT_GROUPS_JSON" \ + '($d - $c)[]') + + while IFS= read -r G; do + nc_occ files_external:applicable "$MID" --remove-group="$G" + done < <(jq -rn \ + --argjson d "$DESIRED_GROUPS_JSON" \ + --argjson c "$CURRENT_GROUPS_JSON" \ + '($c - $d)[]') + } + + # ── 6. Process root mounts (admin group only) ─────────────────── + ROOT_COUNT=$(jq '.rootMounts | length' "$MANIFEST") + for i in $(seq 0 $((ROOT_COUNT - 1))); do + MP=$(jq -r ".rootMounts[$i].mountPoint" "$MANIFEST") + DIR=$(jq -r ".rootMounts[$i].dataDir" "$MANIFEST") + GROUP=$(jq -r ".rootMounts[$i].applicableGroup" "$MANIFEST") + MID=$(ensure_mount "$MP" "$DIR") + sync_applicable "$MID" '[]' "[\"$GROUP\"]" + done + + # ── 7. Process archive mounts (per-user / per-group) ─────────── + ARCH_COUNT=$(jq '.archiveMounts | length' "$MANIFEST") + for i in $(seq 0 $((ARCH_COUNT - 1))); do + MP=$(jq -r ".archiveMounts[$i].mountPoint" "$MANIFEST") + DIR=$(jq -r ".archiveMounts[$i].dataDir" "$MANIFEST") + USERS_JSON=$(jq -c ".archiveMounts[$i].applicableUsers // []" "$MANIFEST") + GROUPS_JSON=$(jq -c ".archiveMounts[$i].applicableGroups // []" "$MANIFEST") + MID=$(ensure_mount "$MP" "$DIR") + sync_applicable "$MID" "$USERS_JSON" "$GROUPS_JSON" + done + + echo "[bootstrap] Bootstrap complete." + EOF + ] + + volume_mount { + name = "manifest" + mount_path = "/manifest" + } + } + + volume { + name = "manifest" + config_map { + name = kubernetes_config_map_v1.nextcloud_external_storage_manifest.metadata[0].name + } + } + } + } + } + + depends_on = [helm_release.nextcloud] + + lifecycle { + # KYVERNO_LIFECYCLE_V1: Kyverno admission webhook mutates dns_config with ndots=2 + ignore_changes = [spec[0].template[0].spec[0].dns_config] + } +} diff --git a/stacks/nextcloud/main.tf b/stacks/nextcloud/main.tf index da3de2c3..cd27572c 100644 --- a/stacks/nextcloud/main.tf +++ b/stacks/nextcloud/main.tf @@ -30,7 +30,7 @@ resource "kubernetes_namespace" "nextcloud" { tier = local.tiers.edge "resource-governance/custom-limitrange" = "true" "resource-governance/custom-quota" = "true" - "keel.sh/enrolled" = "true" + "keel.sh/enrolled" = "true" } } lifecycle { @@ -226,6 +226,24 @@ module "nfs_nextcloud_backup_host" { nfs_path = "/srv/nfs/nextcloud-backup" } +module "nfs_pve_root_host" { + source = "../../modules/kubernetes/nfs_volume" + name = "nextcloud-pve-nfs-root" + namespace = kubernetes_namespace.nextcloud.metadata[0].name + nfs_server = "192.168.1.127" + nfs_path = "/srv/nfs" + storage = "3000Gi" +} + +module "nfs_pve_ssd_root_host" { + source = "../../modules/kubernetes/nfs_volume" + name = "nextcloud-pve-nfs-ssd-root" + namespace = kubernetes_namespace.nextcloud.metadata[0].name + nfs_server = "192.168.1.127" + nfs_path = "/srv/nfs-ssd" + storage = "100Gi" +} + module "ingress" { source = "../../modules/kubernetes/ingress_factory" # Native WebDAV / CalDAV / CardDAV clients (Nextcloud desktop+mobile apps, diff --git a/stacks/nextcloud/providers.tf b/stacks/nextcloud/providers.tf index 012af700..d5469984 100644 --- a/stacks/nextcloud/providers.tf +++ b/stacks/nextcloud/providers.tf @@ -13,6 +13,13 @@ terraform { source = "goauthentik/authentik" version = "~> 2024.10" } + # kubectl (gavinbunney) — workaround for hashicorp/kubernetes + # `kubernetes_manifest` panics on Kyverno CRDs. See beads code-e2dp. + # Declared for all stacks but only used where opted-in. + kubectl = { + source = "gavinbunney/kubectl" + version = "~> 1.14" + } } } @@ -35,3 +42,8 @@ provider "vault" { address = "https://vault.viktorbarzin.me" skip_child_token = true } + +provider "kubectl" { + config_path = var.kube_config_path + load_config_file = true +}