add docker registry vm and allow multiple provisioning cmds in templates [ci skip]
This commit is contained in:
Viktor Barzin
parent
22a32c5de4
commit
b15246a2cb
5 changed files with 171 additions and 44 deletions
128
main.tf
128
main.tf
|
|
@ -36,7 +36,7 @@ variable "dbaas_pgadmin_password" {}
|
|||
variable "drone_github_client_id" {}
|
||||
variable "drone_github_client_secret" {}
|
||||
variable "drone_rpc_secret" {}
|
||||
# variable "dockerhub_password" {}
|
||||
variable "dockerhub_registry_password" {}
|
||||
variable "oauth2_proxy_client_id" {}
|
||||
variable "oauth2_proxy_client_secret" {}
|
||||
variable "oauth2_proxy_authenticated_emails" {}
|
||||
|
|
@ -119,18 +119,6 @@ variable "xray_reality_private_key" { type = string }
|
|||
variable "xray_reality_short_ids" { type = list(string) }
|
||||
|
||||
|
||||
# data "terraform_remote_state" "foo" {
|
||||
# backend = "kubernetes"
|
||||
# config = {
|
||||
# secret_suffix = "state"
|
||||
# namespace = "drone"
|
||||
# in_cluster_config = var.prod
|
||||
# host = "https://kubernetes:6443"
|
||||
# // load_config_file = true
|
||||
# }
|
||||
|
||||
# depends_on = [module.kubernetes_cluster]
|
||||
# }
|
||||
provider "kubernetes" {
|
||||
config_path = var.prod ? "" : "~/.kube/config"
|
||||
}
|
||||
|
|
@ -149,33 +137,107 @@ provider "proxmox" {
|
|||
}
|
||||
# TODO: add DEFCON levels
|
||||
|
||||
# resource "proxmox_virtual_environment_network_linux_vlan" "vlan1" {
|
||||
# node_name = "pve"
|
||||
# name = "ens160.99"
|
||||
|
||||
# comment = "VLAN 99"
|
||||
# }
|
||||
# Main module to init infra from
|
||||
|
||||
locals {
|
||||
vm_template_name = "ubuntu-2404-cloudinit-template"
|
||||
vm_cloud_init_snippet_name = "cloud_init.yaml"
|
||||
k8s_vm_template = "ubuntu-2404-cloudinit-k8s-template"
|
||||
k8s_cloud_init_snippet_name = "k8s_cloud_init.yaml"
|
||||
k8s_cloud_init_image_path = "/var/lib/vz/template/iso/noble-server-cloudimg-amd64-k8s.img"
|
||||
|
||||
non_k8s_vm_template = "ubuntu-2404-cloudinit-non-k8s-template"
|
||||
non_k8s_cloud_init_snippet_name = "non_k8s_cloud_init.yaml"
|
||||
non_k8s_cloud_init_image_path = "/var/lib/vz/template/iso/noble-server-cloudimg-amd64-non-k8s.img"
|
||||
|
||||
cloud_init_image_url = "https://cloud-images.ubuntu.com/noble/current/noble-server-cloudimg-amd64.img"
|
||||
}
|
||||
|
||||
# Main module to init infra from
|
||||
module "template-vm" {
|
||||
source = "./modules/create-template-vm"
|
||||
proxmox_host = var.proxmox_host
|
||||
proxmox_user = "root" # SSH user on Proxmox host
|
||||
cloud_image_url = "https://cloud-images.ubuntu.com/noble/current/noble-server-cloudimg-amd64.img"
|
||||
image_path = "/var/lib/vz/template/iso/noble-server-cloudimg-amd64.img"
|
||||
template_id = 8000
|
||||
template_name = local.vm_template_name
|
||||
module "k8s-node-template" {
|
||||
source = "./modules/create-template-vm"
|
||||
proxmox_host = var.proxmox_host
|
||||
proxmox_user = "root" # SSH user on Proxmox host
|
||||
|
||||
snippet_name = local.vm_cloud_init_snippet_name
|
||||
user_passwd = var.vm_wizard_password
|
||||
k8s_join_command = var.k8s_join_command
|
||||
cloud_image_url = local.cloud_init_image_url
|
||||
image_path = local.k8s_cloud_init_image_path
|
||||
template_id = 2000
|
||||
template_name = local.k8s_vm_template
|
||||
user_passwd = var.vm_wizard_password
|
||||
|
||||
is_k8s_template = true # provision cloud init file with k8s deps
|
||||
snippet_name = local.k8s_cloud_init_snippet_name
|
||||
# Add mirror registry
|
||||
containerd_config_update_command = "echo '[plugins.\"io.containerd.grpc.v1.cri\".registry.mirrors.\"docker.io\"]' >> /etc/containerd/config.toml && echo ' endpoint = [\"http://10.0.20.10:5000\"]' >> /etc/containerd/config.toml" # docker registry vm
|
||||
k8s_join_command = var.k8s_join_command
|
||||
}
|
||||
|
||||
module "non-k8s-node-template" {
|
||||
source = "./modules/create-template-vm"
|
||||
proxmox_host = var.proxmox_host
|
||||
proxmox_user = "root" # SSH user on Proxmox host
|
||||
|
||||
cloud_image_url = local.cloud_init_image_url
|
||||
image_path = local.non_k8s_cloud_init_image_path
|
||||
template_id = 1000
|
||||
template_name = local.non_k8s_vm_template
|
||||
user_passwd = var.vm_wizard_password
|
||||
|
||||
is_k8s_template = false # provision cloud init file without k8s deps
|
||||
snippet_name = local.non_k8s_cloud_init_snippet_name
|
||||
}
|
||||
|
||||
module "docker-registry-template" {
|
||||
source = "./modules/create-template-vm"
|
||||
|
||||
proxmox_host = var.proxmox_host
|
||||
proxmox_user = "root" # SSH user on Proxmox host
|
||||
|
||||
cloud_image_url = local.cloud_init_image_url
|
||||
image_path = local.non_k8s_cloud_init_image_path # keke
|
||||
template_id = 1001
|
||||
template_name = "docker-registry-template"
|
||||
|
||||
user_passwd = var.vm_wizard_password
|
||||
|
||||
is_k8s_template = false # provision cloud init file without k8s deps
|
||||
snippet_name = "docker-registry.yaml"
|
||||
|
||||
# Setup registry config and start container
|
||||
provision_cmds = [
|
||||
"mkdir -p /etc/docker-registry",
|
||||
format("echo %s | base64 -d > /etc/docker-registry/config.yml",
|
||||
base64encode(
|
||||
templatefile("./modules/docker-registry/config.yaml", {
|
||||
password = var.dockerhub_registry_password
|
||||
}
|
||||
)
|
||||
)
|
||||
),
|
||||
"docker run -p 5000:5000 -d --restart always --name registry -v /etc/docker-registry/config.yml:/etc/docker/registry/config.yml registry:2"
|
||||
]
|
||||
}
|
||||
|
||||
|
||||
module "docker-registry-vm" {
|
||||
source = "./modules/create-vm"
|
||||
vmid = 220
|
||||
|
||||
vm_cpus = 4
|
||||
vm_mem_mb = 4196
|
||||
vm_disk_size = "32G"
|
||||
|
||||
template_name = "docker-registry-template"
|
||||
vm_name = "docker-registry"
|
||||
cisnippet_name = "docker-registry.yaml"
|
||||
|
||||
vm_mac_address = "DE:AD:BE:EF:22:22" # mapped to 10.0.20.10 in dhcp
|
||||
bridge = "vmbr1"
|
||||
vlan_tag = "20"
|
||||
}
|
||||
|
||||
# module that provisions the proxmox host?
|
||||
# make dns stateless?
|
||||
# pfsense/truenas configs in code
|
||||
# etcd db backup in code
|
||||
|
||||
# module "k8s_node5" {
|
||||
# template_name = local.vm_template_name
|
||||
# source = "./modules/create-vm"
|
||||
|
|
|
|||
|
|
@ -24,32 +24,43 @@ packages:
|
|||
- apt-transport-https
|
||||
- ca-certificates
|
||||
- gpg
|
||||
- isc-dhcp-client
|
||||
# docker
|
||||
- docker-ce
|
||||
- docker-ce-cli
|
||||
- containerd.io
|
||||
- docker-buildx-plugin
|
||||
- docker-compose-plugin
|
||||
%{if is_k8s_template}
|
||||
# kubernetes
|
||||
- kubeadm
|
||||
- kubelet
|
||||
%{endif}
|
||||
|
||||
apt:
|
||||
sources:
|
||||
%{if is_k8s_template}
|
||||
kubernetes:
|
||||
source: "deb https://pkgs.k8s.io/core:/stable:/v1.32/deb/ /"
|
||||
keyid: "DE15B14486CD377B9E876E1A234654DA9A296436"
|
||||
filename: kubernetes.list
|
||||
docket:
|
||||
%{endif}
|
||||
docker:
|
||||
source: "deb https://download.docker.com/linux/ubuntu noble stable"
|
||||
keyid: "9DC858229FC7DD38854AE2D88D81803C0EBFCD88"
|
||||
filename: docker.list
|
||||
|
||||
runcmd:
|
||||
%{if is_k8s_template}
|
||||
- apt-mark hold kubelet kubeadm kubectl
|
||||
- systemctl stop kubelet
|
||||
- containerd config default | sudo tee /etc/containerd/config.toml
|
||||
- ${containerd_config_update_command}
|
||||
- systemctl restart containerd
|
||||
- ${k8s_join_command}
|
||||
- systemctl enable kubelet
|
||||
- systemctl start kubelet
|
||||
%{ endif }
|
||||
%{ for provision_cmd in provision_cmds ~}
|
||||
- ${provision_cmd}
|
||||
%{ endfor ~}
|
||||
|
|
|
|||
|
|
@ -13,6 +13,16 @@ variable "k8s_join_command" {
|
|||
type = string
|
||||
default = ""
|
||||
}
|
||||
variable "containerd_config_update_command" {
|
||||
type = string
|
||||
default = ""
|
||||
description = "Command to execute to update containerd config.toml; e.g add mirror"
|
||||
}
|
||||
variable "is_k8s_template" { type = bool }
|
||||
variable "provision_cmds" {
|
||||
type = list(string)
|
||||
default = []
|
||||
}
|
||||
|
||||
# SSH connection to Proxmox
|
||||
resource "null_resource" "create_template_remote" {
|
||||
|
|
@ -60,7 +70,15 @@ resource "null_resource" "upload_cloud_init" {
|
|||
|
||||
provisioner "file" {
|
||||
destination = "/var/lib/vz/snippets/${var.snippet_name}"
|
||||
content = templatefile("${path.module}/cloud_init.yaml", { authorized_ssh_key = file("~/.ssh/id_ed25519.pub"), passwd = var.user_passwd, k8s_join_command = var.k8s_join_command })
|
||||
content = templatefile("${path.module}/cloud_init.yaml", {
|
||||
is_k8s_template = var.is_k8s_template,
|
||||
authorized_ssh_key = file("~/.ssh/id_ed25519.pub"),
|
||||
passwd = var.user_passwd,
|
||||
provision_cmds = var.provision_cmds,
|
||||
k8s_join_command = var.k8s_join_command,
|
||||
containerd_config_update_command = var.containerd_config_update_command
|
||||
}
|
||||
)
|
||||
}
|
||||
|
||||
triggers = {
|
||||
|
|
|
|||
|
|
@ -1,20 +1,21 @@
|
|||
variable "vm_name" { type = string }
|
||||
variable "vmid" {
|
||||
type = number
|
||||
default = 0
|
||||
}
|
||||
variable "template_name" { type = string }
|
||||
variable "vm_name" { default = "terraform-test" }
|
||||
variable "vm_cpus" {
|
||||
type = number
|
||||
default = 4
|
||||
}
|
||||
|
||||
variable "vm_mem_mb" {
|
||||
type = number
|
||||
default = 8192
|
||||
}
|
||||
|
||||
variable "vm_disk_size" {
|
||||
type = string
|
||||
default = "64G"
|
||||
}
|
||||
|
||||
variable "vm_mac_address" {
|
||||
type = string
|
||||
default = null
|
||||
|
|
@ -29,11 +30,6 @@ variable "vlan_tag" {
|
|||
type = string
|
||||
default = null
|
||||
}
|
||||
variable "vmid" {
|
||||
type = number
|
||||
default = 0
|
||||
}
|
||||
|
||||
|
||||
resource "proxmox_vm_qemu" "cloudinit-vm" {
|
||||
vmid = var.vmid
|
||||
|
|
|
|||
40
modules/docker-registry/config.yaml
Normal file
40
modules/docker-registry/config.yaml
Normal file
|
|
@ -0,0 +1,40 @@
|
|||
version: 0.1
|
||||
log:
|
||||
fields:
|
||||
service: registry
|
||||
storage:
|
||||
cache:
|
||||
blobdescriptor: inmemory
|
||||
filesystem:
|
||||
rootdirectory: /var/lib/registry
|
||||
maxsize: 5GiB
|
||||
delete:
|
||||
enabled: true
|
||||
maintenance:
|
||||
uploadpurging:
|
||||
enabled: true
|
||||
age: 24h
|
||||
interval: 4h
|
||||
dryrun: false
|
||||
readonly:
|
||||
enabled: false
|
||||
http:
|
||||
addr: :5000
|
||||
headers:
|
||||
X-Content-Type-Options: [nosniff]
|
||||
debug:
|
||||
addr: ":5001"
|
||||
# Enable proxy on nodes - https://github.com/containerd/containerd/blob/main/docs/cri/registry.md
|
||||
# https://ops.tips/gists/retrieving-docker-registry-metrics-using-prometheus/
|
||||
prometheus:
|
||||
enabled: true
|
||||
path: "/metrics"
|
||||
health:
|
||||
storagedriver:
|
||||
enabled: true
|
||||
interval: 10s
|
||||
threshold: 3
|
||||
proxy:
|
||||
remoteurl: https://registry-1.docker.io
|
||||
username: vbarzin@gmail.com
|
||||
password: ${password}
|
||||
Loading…
Add table
Add a link
Reference in a new issue