move helper scripts in scripts dir [ci skip]

scripts/kill_ns.sh

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+set -e
+
+NAMESPACE=$1
+if [ -z "$NAMESPACE" ]; then
+	echo "Pass in parameter namespace"
+	exit 1
+fi
+kubectl proxy &
+kubectl get namespace $NAMESPACE -o json |jq '.spec = {"finalizers":[]}' > /tmp/kill_rogue_ns.json
+curl -k -H "Content-Type: application/json" -X PUT --data-binary @/tmp/kill_rogue_ns.json 127.0.0.1:8001/api/v1/namespaces/$NAMESPACE/finalize
+kill %1

scripts/node_registry_manager.sh

@@ -0,0 +1,95 @@
+#!/bin/bash
+
+# Simple and reliable containerd registry mirror manager
+# Usage: ./registry-mirror.sh [--add|--remove] [mirror_url]
+# Docs - https://github.com/containerd/containerd/blob/main/docs/cri/registry.md
+# To apply on all nodes (tail +3 skips master node):
+# for node in $(kubectl get nodes -o wide | awk '{print $6}' | tail -n +3); do cat node_registry_manager.sh | s wizard@$node "sudo bash -s -- --add http://192.168.1.10:5000"; done
+
+CONFIG_FILE="/etc/containerd/config.toml"
+BACKUP_FILE="/etc/containerd/config.toml.bak"
+
+# Validate environment
+[ -f "$CONFIG_FILE" ] || { echo "Error: $CONFIG_FILE not found" >&2; exit 1; }
+[ "$(id -u)" -eq 0 ] || { echo "Error: Requires root privileges" >&2; exit 1; }
+
+add_mirror() {
+    local mirror_url="$1"
+    
+    # Create backup
+    cp -p "$CONFIG_FILE" "$BACKUP_FILE"
+    
+    # Check if mirror already exists
+    if grep -q "endpoint = \[.*\"$mirror_url\".*\]" "$CONFIG_FILE"; then
+        echo "Mirror already exists: $mirror_url"
+        return 0
+    fi
+
+    # Check if docker.io section exists
+    if grep -q "^\[plugins\.\"io\.containerd\.grpc\.v1\.cri\"\.registry\.mirrors\.\"docker.io\"\]" "$CONFIG_FILE"; then
+        # Append to existing section
+        sed -i "/^\[plugins\."io\.containerd\.grpc\.v1\.cri"\.registry\.mirrors\."docker.io"\]/a \  endpoint = [\"$mirror_url\"]" "$CONFIG_FILE"
+    else
+        # Add new section after registry.mirrors
+        if grep -q "^\[plugins\."io\.containerd\.grpc\.v1\.cri"\.registry\.mirrors\]" "$CONFIG_FILE"; then
+            sed -i "/^\[plugins\."io\.containerd\.grpc\.v1\.cri"\.registry\.mirrors\]/a \\n[plugins.\"io.containerd.grpc.v1.cri\".registry.mirrors.\"docker.io\"]\n  endpoint = [\"$mirror_url\"]" "$CONFIG_FILE"
+        else
+            # Add complete new section
+            echo -e "\n[plugins.\"io.containerd.grpc.v1.cri\".registry.mirrors.\"docker.io\"]\n  endpoint = [\"$mirror_url\"]" >> "$CONFIG_FILE"
+        fi
+    fi
+    
+    echo "Added mirror: $mirror_url"
+}
+
+remove_mirror() {
+    local mirror_url="$1"
+    
+    # Create backup
+    cp -p "$CONFIG_FILE" "$BACKUP_FILE"
+    
+    # Remove the specific mirror URL
+    sed -i "/endpoint = \[.*\"$mirror_url\".*\]/d" "$CONFIG_FILE"
+    
+    # Clean up empty sections
+    sed -i '/^\[plugins\."io\.containerd\.grpc\.v1\.cri"\.registry\.mirrors\."docker.io"\]$/,/^\[/{//!d}' "$CONFIG_FILE"
+    sed -i '/^\[plugins\."io\.containerd\.grpc\.v1\.cri"\.registry\.mirrors\."docker.io"\]$/d' "$CONFIG_FILE"
+    
+    # Clean up multiple empty lines
+    sed -i '/^$/N;/^\n$/D' "$CONFIG_FILE"
+    
+    echo "Removed mirror: $mirror_url"
+}
+
+restart_containerd() {
+    echo "Restarting containerd..."
+    if systemctl restart containerd; then
+        echo "Successfully restarted containerd"
+        return 0
+    else
+        echo "Error: Failed to restart containerd" >&2
+        return 1
+    fi
+}
+
+case "$1" in
+    --add)
+        [ -z "$2" ] && { echo "Error: Mirror URL required" >&2; exit 1; }
+        add_mirror "$2"
+        restart_containerd || exit 1
+        ;;
+    --remove)
+        [ -z "$2" ] && { echo "Error: Mirror URL required" >&2; exit 1; }
+        remove_mirror "$2"
+        restart_containerd || exit 1
+        ;;
+    *)
+        echo "Usage: $0 [--add|--remove] [mirror_url]" >&2
+        echo "Examples:" >&2
+        echo "  Add mirror:    $0 --add https://registry.example.com" >&2
+        echo "  Remove mirror: $0 --remove https://registry.example.com" >&2
+        exit 1
+        ;;
+esac
+
+exit 0

scripts/renew_worker_certs.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+
+echo 'KUBELET_KUBEADM_ARGS="--container-runtime-endpoint=unix:///var/run/containerd/containerd.sock --pod-infra-container-image=k8s.gcr.io/pause:3.7 --rotate-certificates=true --rotate-server-certificates=true"' | sudo tee /var/lib/kubelet/kubeadm-flags.env
+
+sudo systemctl daemon-reload
+sudo systemctl restart kubelet
+
+# Aprprove all csrs:
+# for csr in $(kb get csr | grep Pending | awk '{print $1}'); do echo $csr; kb certificate approve $csr; done

scripts/server_safe_poweroff/deploy_to_nas.sh

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build -o /tmp/powercheck-armv8 . && rsync /tmp/powercheck-armv8 Administrator@nas:~/server-power-cycle/ && rm /tmp/powercheck-armv8
+rsync synology_main.sh Administrator@nas:~/server-power-cycle/

scripts/server_safe_poweroff/go.mod

@@ -0,0 +1,12 @@
+module viktorbarzin/server-lifecycle
+
+go 1.22.0
+
+toolchain go1.23.6
+
+require (
+	github.com/gosnmp/gosnmp v1.39.0
+	github.com/nightlyone/lockfile v1.0.0
+)
+
+require github.com/golang/glog v1.2.4 // indirect

scripts/server_safe_poweroff/go.sum

@@ -0,0 +1,14 @@
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/golang/glog v1.2.4 h1:CNNw5U8lSiiBk7druxtSHHTsRWcxKoac6kZKm2peBBc=
+github.com/golang/glog v1.2.4/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwmO+w=
+github.com/gosnmp/gosnmp v1.39.0 h1:mPJtSWFLkEemo2bz4fdNztZIFHYG86MC6c6veocq0ZE=
+github.com/gosnmp/gosnmp v1.39.0/go.mod h1:CxVS6bXqmWZlafUj9pZUnQX5e4fAltqPcijxWpCitDo=
+github.com/nightlyone/lockfile v1.0.0 h1:RHep2cFKK4PonZJDdEl4GmkabuhbsRMgk/k3uAmxBiA=
+github.com/nightlyone/lockfile v1.0.0/go.mod h1:rywoIealpdNse2r832aiD9jRk8ErCatROs6LzC841CI=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
+github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

scripts/server_safe_poweroff/idrac_utils.go

@@ -0,0 +1,125 @@
+package main
+
+import (
+	"bytes"
+	"crypto/tls"
+	"encoding/json"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net/http"
+
+	"github.com/golang/glog"
+)
+
+type PowerStateResponse struct {
+	PowerState string `json:"PowerState"`
+}
+type ResetType string
+
+const (
+	On               ResetType = "On"
+	GracefulShutdown ResetType = "GracefulShutdown"
+)
+
+func checkPowerState(idractCredentials idracCredentials) (string, error) {
+	// Construct the full URL for the Redfish Systems endpoint
+	redfishURL := fmt.Sprintf("%s/redfish/v1/Systems/System.Embedded.1", idractCredentials.url)
+
+	// Create an HTTP client
+	client := &http.Client{
+		Transport: &http.Transport{
+			TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
+		},
+	}
+
+	// Create a new GET request
+	req, err := http.NewRequest("GET", redfishURL, nil)
+	if err != nil {
+		return "", fmt.Errorf("failed to create request: %v", err)
+	}
+
+	// Set basic authentication
+	req.SetBasicAuth(idractCredentials.username, idractCredentials.password)
+
+	// Set the Accept header to request JSON
+	req.Header.Set("Accept", "application/json")
+
+	// Send the request
+	resp, err := client.Do(req)
+	if err != nil {
+		return "", fmt.Errorf("failed to send request: %v", err)
+	}
+	defer resp.Body.Close()
+
+	// Check the HTTP status code
+	if resp.StatusCode != http.StatusOK {
+		body, _ := io.ReadAll(resp.Body)
+		return "", fmt.Errorf("unexpected status code: %d, response: %s", resp.StatusCode, string(body))
+	}
+
+	// Read the response body
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return "", fmt.Errorf("failed to read response body: %v", err)
+	}
+
+	// return string(body), nil
+	// Parse the JSON response
+	var powerStateResponse PowerStateResponse
+	err = json.Unmarshal(body, &powerStateResponse)
+	if err != nil {
+		return "", fmt.Errorf("failed to parse JSON response: %v", err)
+	}
+
+	// Return the power state
+	return powerStateResponse.PowerState, nil
+}
+
+func performGracefulShutdown(idracCredentials idracCredentials) error {
+	return performResetType(idracCredentials, GracefulShutdown)
+}
+
+func performPowerOn(idracCredentials idracCredentials) error {
+	return performResetType(idracCredentials, On)
+}
+
+func performResetType(idracCredentials idracCredentials, resetType ResetType) error {
+	glog.Warningf("Starting graceful reset type %s!\n", resetType)
+	// Define the payload for the shutdown request
+	payload := map[string]string{
+		"ResetType": string(resetType), // Only ResetType is needed
+	}
+	payloadBytes, err := json.Marshal(payload)
+	if err != nil {
+		return fmt.Errorf("failed to marshal payload: %v", err)
+	}
+
+	// Create a new HTTP request
+	req, err := http.NewRequest("POST", idracCredentials.url, bytes.NewBuffer(payloadBytes))
+	if err != nil {
+		return fmt.Errorf("failed to create request: %v", err)
+	}
+
+	// Set headers
+	req.Header.Set("Content-Type", "application/json")
+	req.SetBasicAuth(idracCredentials.username, idracCredentials.password)
+
+	// Send the request
+	client := &http.Client{}
+	resp, err := client.Do(req)
+	if err != nil {
+		return fmt.Errorf("failed to send request: %v", err)
+	}
+	defer resp.Body.Close()
+
+	// Check the response status code
+	if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusAccepted {
+		body, _ := ioutil.ReadAll(resp.Body)
+		return fmt.Errorf("unexpected status code: %d, response: %s", resp.StatusCode, string(body))
+	}
+
+	glog.Infof("Reset type %s initiated successfully.\n")
+	return nil
+
+}

scripts/server_safe_poweroff/main.go

@@ -0,0 +1,107 @@
+package main
+
+import (
+	"flag"
+	"log"
+
+	"github.com/golang/glog"
+	"github.com/nightlyone/lockfile"
+)
+
+const upsMinutesRemainingThreshold = 20
+
+type idracCredentials = struct {
+	url      string
+	username string
+	password string
+}
+
+func main() {
+	idracUsername := flag.String("idracUsername", "root", "iDRAC username")
+	idracPassword := flag.String("idracPassword", "calvin", "iDRAC password")
+	idracHost := flag.String("idracHost", "192.168.1.4", "iDRAC host")
+	flag.Parse()
+	defer glog.Flush()
+	// lock, err := tryGetLock()
+	// if err != nil {
+	// 	glog.Fatalf("Failed to acquire lock:  %v", err)
+	// }
+	// defer lock.Unlock()
+
+	glog.Info("Checking server power state")
+	idracCredentials := idracCredentials{
+		url:      "https://" + *idracHost,
+		username: *idracUsername,
+		password: *idracPassword,
+	}
+	powerState, err := checkPowerState(idracCredentials)
+	if err != nil {
+		glog.Fatalf("Failed to check power state: %v", err)
+	}
+	glog.Infof("Server power state: %s", powerState)
+
+	glog.Info("Checking UPS state")
+	snmp := getSNMPClient()
+	// Connect to the SNMP agent
+	err = snmp.Connect()
+	if err != nil {
+		log.Fatalf("Failed to connect to UPS SNMP agent: %v", err)
+	}
+	defer snmp.Conn.Close()
+
+	upsState, err := getPowerState(snmp)
+	if err != nil {
+		glog.Fatalf("Failed to get UPS power state: %v", err)
+	}
+
+	if powerState == "On" {
+		handleWhenServerOn(upsState, idracCredentials)
+	} else if powerState == "Off" {
+		handleWhenServerOff(upsState, idracCredentials)
+	} else {
+		glog.Fatalf("Unknown server state %s", powerState)
+	}
+}
+func handleWhenServerOn(upsState UPSPowerState, idracCredentials idracCredentials) {
+	if upsState.inputVoltage > 0 {
+		glog.Infof("UPS is on AC power: %d. Nothing to do.\n", upsState.inputVoltage)
+		return
+	} else {
+		glog.Warningln("UPS is on Battery power")
+		if upsState.minutesRemaining < upsMinutesRemainingThreshold {
+			glog.Warningf("Minutes remaining is too low - %d Turning off server.", upsState.minutesRemaining)
+			// Perform a graceful shutdown of the server
+			performGracefulShutdown(idracCredentials)
+		} else {
+			glog.Warningf("Minutes remaining is %d. Server will not be shutdown yet.", upsState.minutesRemaining)
+			return
+		}
+	}
+}
+
+func handleWhenServerOff(upsState UPSPowerState, idracCredentials idracCredentials) {
+	if upsState.inputVoltage > 0 {
+		glog.Infof("UPS is on AC power: %d\n", upsState.inputVoltage)
+		if upsState.minutesRemaining < upsMinutesRemainingThreshold {
+			glog.Infof("UPS battery is still too low - %d minutes remaining. Not turning on server yet.\n", upsState.minutesRemaining)
+		} else {
+			glog.Infof("UPS is on AC power and battery has charged - %d minutes remaining. Turning on server...\n", upsState.minutesRemaining)
+			// Perform startup of the server
+			performPowerOn(idracCredentials)
+		}
+	} else {
+		glog.Warningln("UPS is still on battery power")
+		return
+	}
+}
+func tryGetLock() (*lockfile.Lockfile, error) {
+	lock, err := lockfile.New("/tmp/server_safe_poweroff.pid")
+	if err != nil {
+		log.Fatalf("Failed to create lock file: %v", err)
+	}
+	err = lock.TryLock()
+	if err != nil {
+		return nil, err
+	}
+	return &lock, nil
+}

scripts/server_safe_poweroff/synology_main.sh

@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+
+# This is used to run the main program on synology nas and log all messages to synology's log system
+
+cd /var/services/homes/Administrator/server-power-cycle
+echo "Starting powercheck"
+./powercheck-armv8 -log_dir=./logs
+
+echo "script completed successfully, logging to synlogy's logs"
+
+
+while IFS= read -r line; do
+# for line in $(cat ./logs/powercheck-armv8.INFO); do
+    msg=$(echo $line | grep -E '^[IWEF][0-9]{4} [0-9]{2}:[0-9]{2}:[0-9]{2}\.[0-9]{6}'| awk '{$1=$2=$3=$4=""; print $0}' | sed 's/^ *//')
+    #echo $line
+    echo $msg
+    if [[ -n $msg ]]; then
+        synologset1 sys info 0x11800000 "$msg"
+    fi
+done < "./logs/powercheck-armv8.INFO"
+
+# Cleanup logs
+find ./logs -type f -mtime +7 -exec rm {} \;

scripts/server_safe_poweroff/ups_utils.go

@@ -0,0 +1,46 @@
+package main
+
+import (
+	"time"
+
+	"github.com/golang/glog"
+	"github.com/gosnmp/gosnmp"
+)
+
+type UPSPowerState = struct {
+	inputVoltage     int
+	minutesRemaining uint
+}
+
+func getSNMPClient() *gosnmp.GoSNMP {
+
+	// Define SNMP connection parameters
+	target := "192.168.1.5"
+	community := "Public0"
+
+	// Create a new SNMP client
+	snmp := &gosnmp.GoSNMP{
+		Target:    target,
+		Port:      161, // Default SNMP port
+		Community: community,
+		Version:   gosnmp.Version2c, // Use SNMP v2c
+		Timeout:   time.Duration(5) * time.Second,
+	}
+	return snmp
+}
+func getPowerState(snmp *gosnmp.GoSNMP) (UPSPowerState, error) {
+	oids := []string{
+		// "1.3.6.1.2.1.33.1.2.2.0",     // seconds on battery
+		"1.3.6.1.2.1.33.1.3.3.1.3.1", // input voltage
+		"1.3.6.1.2.1.33.1.2.3.0",     // minutes remaining
+	}
+	// Perform an SNMP GET request to retrieve the values for the specified OIDs
+	result, err := snmp.Get(oids)
+	if err != nil {
+		glog.Fatalf("Failed to perform SNMP GET request: %v", err)
+	}
+
+	inputVoltage := (result.Variables[0].Value).(int)
+	minutesRemaining := result.Variables[1].Value.(uint)
+	return UPSPowerState{inputVoltage, minutesRemaining}, nil
+}

scripts/stop_storage_services.sh

@@ -0,0 +1,50 @@
+#!/usr/bin/env bash
+
+# Stop services that may become in a corrupted state if storage is suddenly disconnected
+
+
+set -euxo pipefail
+
+function scale() { kubectl scale deployment --replicas=$3 --namespace $1 $2; }
+
+### ============================
+### MAIN
+### ============================
+cmd="${1:-stop}"
+case "$cmd" in
+  stop)
+    scale calibre calibre 0
+    scale redis redis 0
+    scale uptime-kuma uptime-kuma 0
+    scale paperless-ngx paperless-ngx 0
+    scale vaultwarden vaultwarden 0
+    scale immich immich-postgresql 0
+    scale nextcloud nextcloud 0
+    scale monitoring prometheus-server 0
+
+    scale technitium technitium 0
+    scale dbaas mysql 0
+    scale dbaas postgresql 0
+    ;;
+  start)
+    scale dbaas mysql 1
+    scale dbaas postgresql 1
+    scale technitium technitium 1
+    scale immich immich-postgresql 1
+    scale nextcloud nextcloud 1
+    scale paperless-ngx paperless-ngx 1
+    scale monitoring prometheus-server 1
+    scale redis redis 1
+    scale uptime-kuma uptime-kuma 1
+    scale vaultwarden vaultwarden 1
+    scale calibre calibre 1
+    ;;
+    # echo "[!] Cleanup only removes links (not flushing all iptables to avoid surprises)."
+    # ip netns list | grep -qw "$NS_NAME" && sudo ip netns del "$NS_NAME" || true
+    # has_link "$HOST_VETH" && sudo ip link del "$HOST_VETH" || true
+    # ;;
+  *)
+    echo "Usage: $0 [stop|start]"
+    exit 1
+    ;;
+esac

scripts/update-istio-injection.sh

@@ -0,0 +1,28 @@
+#!/usr/bin/env bash
+set -e
+from=$1
+to=$2
+
+if [ -z "$from" ] || [ -z "$to" ]; then
+	echo 'pass 2 positional parameters - $from and $to'
+	exit 1
+fi
+
+commands=()
+# Update terraform modules
+for file in $(grep -rni "\"istio-injection\" : \"$from\"" . | grep -v '#' | awk '{print $1}' | cut -d':' -f1); do
+	echo $file
+	sed -i "s/istio-injection\" : \"$from\"/istio-injection\" : \"$to\"/" $file
+
+	ns=$(echo $file | cut -d'/' -f 4)
+	commands+=("kubectl -n $ns get deployments --no-headers | awk '{print \$1}' | xargs kubectl -n $ns rollout restart deployment")
+done
+
+# Apply changes
+terraform apply -auto-approve
+
+# Restart deployments
+for cmd in "${commands[@]}"; do
+	echo $cmd
+	bash -c "$cmd"
+done

scripts/update_k8s.sh

@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+
+# run for all nodes using :
+# for n in $(kbn | grep 'k8s-node' | awk '{print $1}'); do echo $n; kb drain $n --ignore-daemonsets --delete-emptydir-data; s wizard@$n 'bash -s' <update_k8s.sh; kb uncordon $n; done
+
+set -e
+export stable_version='1.32'  # change me
+export release="$stable_version.4"  # change me
+
+echo "Upgrading to $stable_version"
+
+echo "deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/v$stable_version/deb/ /" | sudo tee /etc/apt/sources.list.d/kubernetes.list
+sudo mkdir -p /etc/apt/keyrings
+curl -fsSL "https://pkgs.k8s.io/core:/stable:/v$stable_version/deb/Release.key" | sudo gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg --batch --yes
+
+sudo apt-mark unhold kubeadm kubelet kubectl
+sudo apt-get update 
+sudo apt-get install -y kubeadm="$release-*" 
+
+HOSTNAME=$(hostname)
+SEARCH_STR="master"
+
+if [[ "$HOSTNAME" == *"$SEARCH_STR"* ]]; then
+    echo "Upgrading master"
+    sudo kubeadm upgrade plan && sudo kubeadm upgrade apply v$release -y
+else
+    echo "Upgrading worker"
+    sudo kubeadm upgrade node 
+fi
+
+sudo apt-get install -y kubelet="$release-*" kubectl="$release-*"
+sudo apt-mark hold kubeadm kubelet kubectl
+
+
+sudo systemctl daemon-reload
+sudo systemctl restart kubelet

scripts/update_node.sh

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+
+# sudo apt update && sudo apt autoremove -y && sudo apt upgrade -y
+sudo do-release-upgrade
+sudo apt update && sudo apt autoremove -y && sudo apt upgrade -y
+
+
+