6b7efcd2d6
Pipeline 88 imported the namespace but its refresh dropped the PVC, both services, the ingress and the tls secret from state (PG-backend state races on this new stack's first applies), so the apply again died on 'already exists' conflicts. State now holds namespace+deployment; adopt the missing five with import blocks (TF 1.5 errors on importing already-managed addresses, so only the missing set is listed). Stanzas come out once applied.
262 lines
8 KiB
HCL
262 lines
8 KiB
HCL
# Android emulator — shared in-cluster Android 16 (API 36) testing instance.
|
|
# Agents drive it over adb (10.0.20.200:5555); humans watch the screen at
|
|
# https://android-emulator.viktorbarzin.lan (noVNC). The SDK + system image +
|
|
# AVD live on the PVC; the container image is just JDK + cmdline-tools + libs
|
|
# (built manually from docker/, see README.md).
|
|
#
|
|
# Decision record: docs/adr/0001-android-emulator-in-cluster.md
|
|
# - privileged + /dev/kvm hostPath (namespace is on the Kyverno exclude list)
|
|
# - swiftshader rendering — deliberately NOT on the contended T4 GPU node
|
|
|
|
# Recovery stanzas (delete after they apply once): pipeline 85 created all of
|
|
# these but PG-backend state races during the new schema's first applies left
|
|
# them out of (or dropped them from) the recorded state, so plans kept trying
|
|
# to re-create live resources. House pattern: adopt via import blocks,
|
|
# plan-to-zero, then drop the stanzas. State currently holds namespace +
|
|
# deployment; these five are the ones missing (TF 1.5 errors on importing an
|
|
# already-managed address, so only the missing set is listed).
|
|
import {
|
|
to = kubernetes_persistent_volume_claim.sdk
|
|
id = "android-emulator/android-emulator-sdk"
|
|
}
|
|
import {
|
|
to = kubernetes_service.adb
|
|
id = "android-emulator/android-emulator-adb"
|
|
}
|
|
import {
|
|
to = kubernetes_service.novnc
|
|
id = "android-emulator/android-emulator"
|
|
}
|
|
import {
|
|
to = module.ingress-internal.kubernetes_ingress_v1.proxied-ingress
|
|
id = "android-emulator/android-emulator"
|
|
}
|
|
import {
|
|
to = module.tls_secret.kubernetes_secret.tls_secret
|
|
id = "android-emulator/tls-secret"
|
|
}
|
|
|
|
resource "kubernetes_namespace" "android-emulator" {
|
|
metadata {
|
|
name = "android-emulator"
|
|
labels = {
|
|
"istio-injection" : "disabled"
|
|
tier = local.tiers.cluster
|
|
}
|
|
}
|
|
lifecycle {
|
|
ignore_changes = [metadata[0].labels["goldilocks.fairwinds.com/vpa-update-mode"]]
|
|
}
|
|
}
|
|
|
|
module "tls_secret" {
|
|
source = "../../modules/kubernetes/setup_tls_secret"
|
|
namespace = kubernetes_namespace.android-emulator.metadata[0].name
|
|
tls_secret_name = var.tls_secret_name
|
|
}
|
|
|
|
# SDK + system image + AVD + snapshots. First boot downloads ~2.5GB (≈9GB
|
|
# unpacked) into here;
|
|
# subsequent pod restarts reuse it (boot in ~1 min instead of ~15).
|
|
# DELIBERATE deviation from the proxmox-lvm backup convention: no backup
|
|
# CronJob — everything on this PVC is a regenerable download/cache (wipe the
|
|
# PVC and the next boot rebuilds it; that's also the documented recovery path).
|
|
resource "kubernetes_persistent_volume_claim" "sdk" {
|
|
wait_until_bound = false
|
|
metadata {
|
|
name = "android-emulator-sdk"
|
|
namespace = kubernetes_namespace.android-emulator.metadata[0].name
|
|
annotations = {
|
|
"resize.topolvm.io/threshold" = "10%"
|
|
"resize.topolvm.io/increase" = "50%"
|
|
"resize.topolvm.io/storage_limit" = "60Gi"
|
|
}
|
|
}
|
|
spec {
|
|
access_modes = ["ReadWriteOnce"]
|
|
storage_class_name = "proxmox-lvm"
|
|
resources {
|
|
requests = {
|
|
storage = "30Gi"
|
|
}
|
|
}
|
|
}
|
|
lifecycle {
|
|
# Autoresizer grows requests.storage up to storage_limit; PVCs can't shrink.
|
|
ignore_changes = [spec[0].resources[0].requests]
|
|
}
|
|
}
|
|
|
|
resource "kubernetes_deployment" "android-emulator" {
|
|
metadata {
|
|
name = "android-emulator"
|
|
namespace = kubernetes_namespace.android-emulator.metadata[0].name
|
|
labels = {
|
|
app = "android-emulator"
|
|
}
|
|
}
|
|
spec {
|
|
replicas = 1
|
|
strategy {
|
|
type = "Recreate" # RWO PVC — old pod must release it first
|
|
}
|
|
selector {
|
|
match_labels = { app = "android-emulator" }
|
|
}
|
|
template {
|
|
metadata {
|
|
labels = { app = "android-emulator" }
|
|
}
|
|
spec {
|
|
image_pull_secrets {
|
|
name = "registry-credentials"
|
|
}
|
|
container {
|
|
name = "emulator"
|
|
image = "forgejo.viktorbarzin.me/viktor/android-emulator:${var.image_tag}"
|
|
|
|
security_context {
|
|
privileged = true # /dev/kvm access
|
|
}
|
|
|
|
port {
|
|
name = "adb"
|
|
container_port = 5555
|
|
}
|
|
port {
|
|
name = "novnc"
|
|
container_port = 6080
|
|
}
|
|
|
|
volume_mount {
|
|
name = "sdk"
|
|
mount_path = "/sdk"
|
|
}
|
|
volume_mount {
|
|
name = "kvm"
|
|
mount_path = "/dev/kvm"
|
|
}
|
|
|
|
resources {
|
|
# No CPU limit (cluster-wide rule — CFS throttling). Burstable on
|
|
# memory: the tier-1 quota caps requests.memory at 4Gi per
|
|
# namespace, so requests stay under it while the limit gives the
|
|
# emulator (qemu -memory 4096 + guest overhead + Xvfb/VNC + JVM
|
|
# sdkmanager on first boot) room to peak — same Burstable pattern
|
|
# tiers 3/4 use deliberately.
|
|
requests = {
|
|
cpu = "2"
|
|
memory = "3Gi"
|
|
}
|
|
limits = {
|
|
memory = "8Gi"
|
|
}
|
|
}
|
|
|
|
# First boot downloads the system image + cold-boots Android: allow
|
|
# up to ~30 min before the pod is declared failed.
|
|
startup_probe {
|
|
exec {
|
|
command = ["/bin/bash", "-c", "/sdk/platform-tools/adb shell getprop sys.boot_completed | grep -q 1"]
|
|
}
|
|
period_seconds = 20
|
|
failure_threshold = 90
|
|
}
|
|
readiness_probe {
|
|
exec {
|
|
command = ["/bin/bash", "-c", "/sdk/platform-tools/adb shell getprop sys.boot_completed | grep -q 1"]
|
|
}
|
|
period_seconds = 30
|
|
failure_threshold = 3
|
|
}
|
|
liveness_probe {
|
|
exec {
|
|
command = ["/bin/bash", "-c", "/sdk/platform-tools/adb shell getprop sys.boot_completed | grep -q 1"]
|
|
}
|
|
period_seconds = 60
|
|
failure_threshold = 10 # generous — don't reboot mid-test on a hiccup
|
|
}
|
|
}
|
|
|
|
volume {
|
|
name = "sdk"
|
|
persistent_volume_claim {
|
|
claim_name = kubernetes_persistent_volume_claim.sdk.metadata[0].name
|
|
}
|
|
}
|
|
volume {
|
|
name = "kvm"
|
|
host_path {
|
|
path = "/dev/kvm"
|
|
type = "CharDevice"
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
lifecycle {
|
|
ignore_changes = [spec[0].template[0].spec[0].dns_config] # KYVERNO_LIFECYCLE_V1
|
|
}
|
|
}
|
|
|
|
# adb endpoint for agents/devvms: `adb connect 10.0.20.200:5555`.
|
|
# Unauthenticated by nature — LAN-only via MetalLB, never exposed publicly.
|
|
resource "kubernetes_service" "adb" {
|
|
metadata {
|
|
name = "android-emulator-adb"
|
|
namespace = kubernetes_namespace.android-emulator.metadata[0].name
|
|
annotations = {
|
|
"metallb.universe.tf/loadBalancerIPs" = "10.0.20.200"
|
|
"metallb.io/allow-shared-ip" = "shared"
|
|
}
|
|
}
|
|
spec {
|
|
type = "LoadBalancer"
|
|
selector = {
|
|
app = "android-emulator"
|
|
}
|
|
port {
|
|
name = "adb"
|
|
port = 5555
|
|
target_port = 5555
|
|
}
|
|
}
|
|
}
|
|
|
|
resource "kubernetes_service" "novnc" {
|
|
metadata {
|
|
name = "android-emulator"
|
|
namespace = kubernetes_namespace.android-emulator.metadata[0].name
|
|
labels = {
|
|
app = "android-emulator"
|
|
}
|
|
}
|
|
spec {
|
|
selector = {
|
|
app = "android-emulator"
|
|
}
|
|
port {
|
|
name = "http"
|
|
port = 80
|
|
target_port = 6080
|
|
}
|
|
}
|
|
}
|
|
|
|
# Browser screen view (noVNC) — LAN only.
|
|
module "ingress-internal" {
|
|
source = "../../modules/kubernetes/ingress_factory"
|
|
# auth = "none": LAN-only (allow_local_access_only) noVNC screen view of the
|
|
# shared test emulator — no user data behind it; Authentik would break the
|
|
# websocket flow agents and users rely on.
|
|
auth = "none"
|
|
namespace = kubernetes_namespace.android-emulator.metadata[0].name
|
|
name = "android-emulator"
|
|
root_domain = "viktorbarzin.lan"
|
|
tls_secret_name = var.tls_secret_name
|
|
allow_local_access_only = true
|
|
ssl_redirect = false
|
|
extra_annotations = {
|
|
"gethomepage.dev/enabled" = "false"
|
|
}
|
|
}
|