From b9ea9cbaa284752b4684fda30d15e5b5e6c648ff Mon Sep 17 00:00:00 2001 From: Viktor Barzin Date: Sun, 9 Mar 2025 19:15:29 +0000 Subject: [PATCH] add go module to check server power and turn it off is there is no pwoer and ups is low [ci skip] --- server_safe_poweroff/deploy_to_nas.sh | 3 + server_safe_poweroff/go.mod | 12 +++ server_safe_poweroff/go.sum | 14 +++ server_safe_poweroff/idrac_utils.go | 125 ++++++++++++++++++++++++++ server_safe_poweroff/main.go | 107 ++++++++++++++++++++++ server_safe_poweroff/synology_main.sh | 23 +++++ server_safe_poweroff/ups_utils.go | 46 ++++++++++ 7 files changed, 330 insertions(+) create mode 100755 server_safe_poweroff/deploy_to_nas.sh create mode 100644 server_safe_poweroff/go.mod create mode 100644 server_safe_poweroff/go.sum create mode 100644 server_safe_poweroff/idrac_utils.go create mode 100644 server_safe_poweroff/main.go create mode 100755 server_safe_poweroff/synology_main.sh create mode 100644 server_safe_poweroff/ups_utils.go diff --git a/server_safe_poweroff/deploy_to_nas.sh b/server_safe_poweroff/deploy_to_nas.sh new file mode 100755 index 00000000..2904fae3 --- /dev/null +++ b/server_safe_poweroff/deploy_to_nas.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env bash +CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build -o /tmp/powercheck-armv8 . && rsync /tmp/powercheck-armv8 Administrator@nas:~/server-power-cycle/ && rm /tmp/powercheck-armv8 +rsync synology_main.sh Administrator@nas:~/server-power-cycle/ diff --git a/server_safe_poweroff/go.mod b/server_safe_poweroff/go.mod new file mode 100644 index 00000000..2d708f74 --- /dev/null +++ b/server_safe_poweroff/go.mod @@ -0,0 +1,12 @@ +module viktorbarzin/server-lifecycle + +go 1.22.0 + +toolchain go1.23.6 + +require ( + github.com/gosnmp/gosnmp v1.39.0 + github.com/nightlyone/lockfile v1.0.0 +) + +require github.com/golang/glog v1.2.4 // indirect diff --git a/server_safe_poweroff/go.sum b/server_safe_poweroff/go.sum new file mode 100644 index 00000000..fd6b64ba --- /dev/null +++ b/server_safe_poweroff/go.sum @@ -0,0 +1,14 @@ +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/golang/glog v1.2.4 h1:CNNw5U8lSiiBk7druxtSHHTsRWcxKoac6kZKm2peBBc= +github.com/golang/glog v1.2.4/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwmO+w= +github.com/gosnmp/gosnmp v1.39.0 h1:mPJtSWFLkEemo2bz4fdNztZIFHYG86MC6c6veocq0ZE= +github.com/gosnmp/gosnmp v1.39.0/go.mod h1:CxVS6bXqmWZlafUj9pZUnQX5e4fAltqPcijxWpCitDo= +github.com/nightlyone/lockfile v1.0.0 h1:RHep2cFKK4PonZJDdEl4GmkabuhbsRMgk/k3uAmxBiA= +github.com/nightlyone/lockfile v1.0.0/go.mod h1:rywoIealpdNse2r832aiD9jRk8ErCatROs6LzC841CI= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/server_safe_poweroff/idrac_utils.go b/server_safe_poweroff/idrac_utils.go new file mode 100644 index 00000000..b07e7697 --- /dev/null +++ b/server_safe_poweroff/idrac_utils.go @@ -0,0 +1,125 @@ +package main + +import ( + "bytes" + "crypto/tls" + "encoding/json" + "fmt" + "io" + "io/ioutil" + "net/http" + + "github.com/golang/glog" +) + +type PowerStateResponse struct { + PowerState string `json:"PowerState"` +} +type ResetType string + +const ( + On ResetType = "On" + GracefulShutdown ResetType = "GracefulShutdown" +) + +func checkPowerState(idractCredentials idracCredentials) (string, error) { + // Construct the full URL for the Redfish Systems endpoint + redfishURL := fmt.Sprintf("%s/redfish/v1/Systems/System.Embedded.1", idractCredentials.url) + + // Create an HTTP client + client := &http.Client{ + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + }, + } + + // Create a new GET request + req, err := http.NewRequest("GET", redfishURL, nil) + if err != nil { + return "", fmt.Errorf("failed to create request: %v", err) + } + + // Set basic authentication + req.SetBasicAuth(idractCredentials.username, idractCredentials.password) + + // Set the Accept header to request JSON + req.Header.Set("Accept", "application/json") + + // Send the request + resp, err := client.Do(req) + if err != nil { + return "", fmt.Errorf("failed to send request: %v", err) + } + defer resp.Body.Close() + + // Check the HTTP status code + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(resp.Body) + return "", fmt.Errorf("unexpected status code: %d, response: %s", resp.StatusCode, string(body)) + } + + // Read the response body + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return "", fmt.Errorf("failed to read response body: %v", err) + } + + // return string(body), nil + // Parse the JSON response + var powerStateResponse PowerStateResponse + err = json.Unmarshal(body, &powerStateResponse) + if err != nil { + return "", fmt.Errorf("failed to parse JSON response: %v", err) + } + + // Return the power state + return powerStateResponse.PowerState, nil +} + +func performGracefulShutdown(idracCredentials idracCredentials) error { + return performResetType(idracCredentials, GracefulShutdown) +} + +func performPowerOn(idracCredentials idracCredentials) error { + return performResetType(idracCredentials, On) +} + +func performResetType(idracCredentials idracCredentials, resetType ResetType) error { + glog.Warningf("Starting graceful reset type %s!\n", resetType) + // Define the payload for the shutdown request + payload := map[string]string{ + "ResetType": string(resetType), // Only ResetType is needed + } + payloadBytes, err := json.Marshal(payload) + if err != nil { + return fmt.Errorf("failed to marshal payload: %v", err) + } + + // Create a new HTTP request + req, err := http.NewRequest("POST", idracCredentials.url, bytes.NewBuffer(payloadBytes)) + if err != nil { + return fmt.Errorf("failed to create request: %v", err) + } + + // Set headers + req.Header.Set("Content-Type", "application/json") + req.SetBasicAuth(idracCredentials.username, idracCredentials.password) + + // Send the request + client := &http.Client{} + resp, err := client.Do(req) + if err != nil { + return fmt.Errorf("failed to send request: %v", err) + } + defer resp.Body.Close() + + // Check the response status code + if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusAccepted { + body, _ := ioutil.ReadAll(resp.Body) + return fmt.Errorf("unexpected status code: %d, response: %s", resp.StatusCode, string(body)) + } + + glog.Infof("Reset type %s initiated successfully.\n") + return nil + +} diff --git a/server_safe_poweroff/main.go b/server_safe_poweroff/main.go new file mode 100644 index 00000000..3404fcbc --- /dev/null +++ b/server_safe_poweroff/main.go @@ -0,0 +1,107 @@ +package main + +import ( + "flag" + "log" + + "github.com/golang/glog" + "github.com/nightlyone/lockfile" +) + +const upsMinutesRemainingThreshold = 20 + +type idracCredentials = struct { + url string + username string + password string +} + +func main() { + idracUsername := flag.String("idracUsername", "root", "iDRAC username") + idracPassword := flag.String("idracPassword", "calvin", "iDRAC password") + idracHost := flag.String("idracHost", "192.168.1.4", "iDRAC host") + flag.Parse() + defer glog.Flush() + // lock, err := tryGetLock() + // if err != nil { + // glog.Fatalf("Failed to acquire lock: %v", err) + // } + // defer lock.Unlock() + + glog.Info("Checking server power state") + idracCredentials := idracCredentials{ + url: "https://" + *idracHost, + username: *idracUsername, + password: *idracPassword, + } + powerState, err := checkPowerState(idracCredentials) + if err != nil { + glog.Fatalf("Failed to check power state: %v", err) + } + glog.Infof("Server power state: %s", powerState) + + glog.Info("Checking UPS state") + snmp := getSNMPClient() + // Connect to the SNMP agent + err = snmp.Connect() + if err != nil { + log.Fatalf("Failed to connect to UPS SNMP agent: %v", err) + } + defer snmp.Conn.Close() + + upsState, err := getPowerState(snmp) + if err != nil { + glog.Fatalf("Failed to get UPS power state: %v", err) + } + + if powerState == "On" { + handleWhenServerOn(upsState, idracCredentials) + } else if powerState == "Off" { + handleWhenServerOff(upsState, idracCredentials) + } else { + glog.Fatalf("Unknown server state %s", powerState) + } +} +func handleWhenServerOn(upsState UPSPowerState, idracCredentials idracCredentials) { + if upsState.inputVoltage > 0 { + glog.Infof("UPS is on AC power: %d. Nothing to do.\n", upsState.inputVoltage) + return + } else { + glog.Warningln("UPS is on Battery power") + if upsState.minutesRemaining < upsMinutesRemainingThreshold { + glog.Warningf("Minutes remaining is too low - %d Turning off server.", upsState.minutesRemaining) + // Perform a graceful shutdown of the server + performGracefulShutdown(idracCredentials) + } else { + glog.Warningf("Minutes remaining is %d. Server will not be shutdown yet.", upsState.minutesRemaining) + return + } + } +} + +func handleWhenServerOff(upsState UPSPowerState, idracCredentials idracCredentials) { + if upsState.inputVoltage > 0 { + glog.Infof("UPS is on AC power: %d\n", upsState.inputVoltage) + if upsState.minutesRemaining < upsMinutesRemainingThreshold { + glog.Infof("UPS battery is still too low - %d minutes remaining. Not turning on server yet.\n", upsState.minutesRemaining) + } else { + glog.Infof("UPS is on AC power and battery has charged - %d minutes remaining. Turning on server...\n", upsState.minutesRemaining) + // Perform startup of the server + performPowerOn(idracCredentials) + } + } else { + glog.Warningln("UPS is still on battery power") + return + } +} +func tryGetLock() (*lockfile.Lockfile, error) { + lock, err := lockfile.New("/tmp/server_safe_poweroff.pid") + if err != nil { + log.Fatalf("Failed to create lock file: %v", err) + } + err = lock.TryLock() + if err != nil { + return nil, err + } + return &lock, nil +} diff --git a/server_safe_poweroff/synology_main.sh b/server_safe_poweroff/synology_main.sh new file mode 100755 index 00000000..054b5c71 --- /dev/null +++ b/server_safe_poweroff/synology_main.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash + +# This is used to run the main program on synology nas and log all messages to synology's log system + +cd /var/services/homes/Administrator/server-power-cycle +echo "Starting powercheck" +./powercheck-armv8 -log_dir=./logs + +echo "script completed successfully, logging to synlogy's logs" + + +while IFS= read -r line; do +# for line in $(cat ./logs/powercheck-armv8.INFO); do + msg=$(echo $line | grep -E '^[IWEF][0-9]{4} [0-9]{2}:[0-9]{2}:[0-9]{2}\.[0-9]{6}'| awk '{$1=$2=$3=$4=""; print $0}' | sed 's/^ *//') + #echo $line + echo $msg + if [[ -n $msg ]]; then + synologset1 sys info 0x11800000 "$msg" + fi +done < "./logs/powercheck-armv8.INFO" + +# Cleanup logs +find ./logs -type f -mtime +7 -exec rm {} \; diff --git a/server_safe_poweroff/ups_utils.go b/server_safe_poweroff/ups_utils.go new file mode 100644 index 00000000..3a78348d --- /dev/null +++ b/server_safe_poweroff/ups_utils.go @@ -0,0 +1,46 @@ +package main + +import ( + "time" + + "github.com/golang/glog" + "github.com/gosnmp/gosnmp" +) + +type UPSPowerState = struct { + inputVoltage int + minutesRemaining uint +} + +func getSNMPClient() *gosnmp.GoSNMP { + + // Define SNMP connection parameters + target := "192.168.1.5" + community := "Public0" + + // Create a new SNMP client + snmp := &gosnmp.GoSNMP{ + Target: target, + Port: 161, // Default SNMP port + Community: community, + Version: gosnmp.Version2c, // Use SNMP v2c + Timeout: time.Duration(5) * time.Second, + } + return snmp +} +func getPowerState(snmp *gosnmp.GoSNMP) (UPSPowerState, error) { + oids := []string{ + // "1.3.6.1.2.1.33.1.2.2.0", // seconds on battery + "1.3.6.1.2.1.33.1.3.3.1.3.1", // input voltage + "1.3.6.1.2.1.33.1.2.3.0", // minutes remaining + } + // Perform an SNMP GET request to retrieve the values for the specified OIDs + result, err := snmp.Get(oids) + if err != nil { + glog.Fatalf("Failed to perform SNMP GET request: %v", err) + } + + inputVoltage := (result.Variables[0].Value).(int) + minutesRemaining := result.Variables[1].Value.(uint) + return UPSPowerState{inputVoltage, minutesRemaining}, nil +}