add go module to check server power and turn it off is there is no pwoer and ups is low [ci skip]
This commit is contained in:
parent
2b2474c507
commit
b9ea9cbaa2
7 changed files with 330 additions and 0 deletions
3
server_safe_poweroff/deploy_to_nas.sh
Executable file
3
server_safe_poweroff/deploy_to_nas.sh
Executable file
|
|
@ -0,0 +1,3 @@
|
|||
#!/usr/bin/env bash
|
||||
CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build -o /tmp/powercheck-armv8 . && rsync /tmp/powercheck-armv8 Administrator@nas:~/server-power-cycle/ && rm /tmp/powercheck-armv8
|
||||
rsync synology_main.sh Administrator@nas:~/server-power-cycle/
|
||||
12
server_safe_poweroff/go.mod
Normal file
12
server_safe_poweroff/go.mod
Normal file
|
|
@ -0,0 +1,12 @@
|
|||
module viktorbarzin/server-lifecycle
|
||||
|
||||
go 1.22.0
|
||||
|
||||
toolchain go1.23.6
|
||||
|
||||
require (
|
||||
github.com/gosnmp/gosnmp v1.39.0
|
||||
github.com/nightlyone/lockfile v1.0.0
|
||||
)
|
||||
|
||||
require github.com/golang/glog v1.2.4 // indirect
|
||||
14
server_safe_poweroff/go.sum
Normal file
14
server_safe_poweroff/go.sum
Normal file
|
|
@ -0,0 +1,14 @@
|
|||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/golang/glog v1.2.4 h1:CNNw5U8lSiiBk7druxtSHHTsRWcxKoac6kZKm2peBBc=
|
||||
github.com/golang/glog v1.2.4/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwmO+w=
|
||||
github.com/gosnmp/gosnmp v1.39.0 h1:mPJtSWFLkEemo2bz4fdNztZIFHYG86MC6c6veocq0ZE=
|
||||
github.com/gosnmp/gosnmp v1.39.0/go.mod h1:CxVS6bXqmWZlafUj9pZUnQX5e4fAltqPcijxWpCitDo=
|
||||
github.com/nightlyone/lockfile v1.0.0 h1:RHep2cFKK4PonZJDdEl4GmkabuhbsRMgk/k3uAmxBiA=
|
||||
github.com/nightlyone/lockfile v1.0.0/go.mod h1:rywoIealpdNse2r832aiD9jRk8ErCatROs6LzC841CI=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
|
||||
github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
125
server_safe_poweroff/idrac_utils.go
Normal file
125
server_safe_poweroff/idrac_utils.go
Normal file
|
|
@ -0,0 +1,125 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/tls"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
|
||||
"github.com/golang/glog"
|
||||
)
|
||||
|
||||
type PowerStateResponse struct {
|
||||
PowerState string `json:"PowerState"`
|
||||
}
|
||||
type ResetType string
|
||||
|
||||
const (
|
||||
On ResetType = "On"
|
||||
GracefulShutdown ResetType = "GracefulShutdown"
|
||||
)
|
||||
|
||||
func checkPowerState(idractCredentials idracCredentials) (string, error) {
|
||||
// Construct the full URL for the Redfish Systems endpoint
|
||||
redfishURL := fmt.Sprintf("%s/redfish/v1/Systems/System.Embedded.1", idractCredentials.url)
|
||||
|
||||
// Create an HTTP client
|
||||
client := &http.Client{
|
||||
Transport: &http.Transport{
|
||||
TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
|
||||
},
|
||||
}
|
||||
|
||||
// Create a new GET request
|
||||
req, err := http.NewRequest("GET", redfishURL, nil)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to create request: %v", err)
|
||||
}
|
||||
|
||||
// Set basic authentication
|
||||
req.SetBasicAuth(idractCredentials.username, idractCredentials.password)
|
||||
|
||||
// Set the Accept header to request JSON
|
||||
req.Header.Set("Accept", "application/json")
|
||||
|
||||
// Send the request
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to send request: %v", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
// Check the HTTP status code
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
return "", fmt.Errorf("unexpected status code: %d, response: %s", resp.StatusCode, string(body))
|
||||
}
|
||||
|
||||
// Read the response body
|
||||
body, err := ioutil.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to read response body: %v", err)
|
||||
}
|
||||
|
||||
// return string(body), nil
|
||||
// Parse the JSON response
|
||||
var powerStateResponse PowerStateResponse
|
||||
err = json.Unmarshal(body, &powerStateResponse)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to parse JSON response: %v", err)
|
||||
}
|
||||
|
||||
// Return the power state
|
||||
return powerStateResponse.PowerState, nil
|
||||
}
|
||||
|
||||
func performGracefulShutdown(idracCredentials idracCredentials) error {
|
||||
return performResetType(idracCredentials, GracefulShutdown)
|
||||
}
|
||||
|
||||
func performPowerOn(idracCredentials idracCredentials) error {
|
||||
return performResetType(idracCredentials, On)
|
||||
}
|
||||
|
||||
func performResetType(idracCredentials idracCredentials, resetType ResetType) error {
|
||||
glog.Warningf("Starting graceful reset type %s!\n", resetType)
|
||||
// Define the payload for the shutdown request
|
||||
payload := map[string]string{
|
||||
"ResetType": string(resetType), // Only ResetType is needed
|
||||
}
|
||||
payloadBytes, err := json.Marshal(payload)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to marshal payload: %v", err)
|
||||
}
|
||||
|
||||
// Create a new HTTP request
|
||||
req, err := http.NewRequest("POST", idracCredentials.url, bytes.NewBuffer(payloadBytes))
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create request: %v", err)
|
||||
}
|
||||
|
||||
// Set headers
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.SetBasicAuth(idracCredentials.username, idracCredentials.password)
|
||||
|
||||
// Send the request
|
||||
client := &http.Client{}
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to send request: %v", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
// Check the response status code
|
||||
if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusAccepted {
|
||||
body, _ := ioutil.ReadAll(resp.Body)
|
||||
return fmt.Errorf("unexpected status code: %d, response: %s", resp.StatusCode, string(body))
|
||||
}
|
||||
|
||||
glog.Infof("Reset type %s initiated successfully.\n")
|
||||
return nil
|
||||
|
||||
}
|
||||
107
server_safe_poweroff/main.go
Normal file
107
server_safe_poweroff/main.go
Normal file
|
|
@ -0,0 +1,107 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"flag"
|
||||
"log"
|
||||
|
||||
"github.com/golang/glog"
|
||||
"github.com/nightlyone/lockfile"
|
||||
)
|
||||
|
||||
const upsMinutesRemainingThreshold = 20
|
||||
|
||||
type idracCredentials = struct {
|
||||
url string
|
||||
username string
|
||||
password string
|
||||
}
|
||||
|
||||
func main() {
|
||||
idracUsername := flag.String("idracUsername", "root", "iDRAC username")
|
||||
idracPassword := flag.String("idracPassword", "calvin", "iDRAC password")
|
||||
idracHost := flag.String("idracHost", "192.168.1.4", "iDRAC host")
|
||||
flag.Parse()
|
||||
defer glog.Flush()
|
||||
// lock, err := tryGetLock()
|
||||
// if err != nil {
|
||||
// glog.Fatalf("Failed to acquire lock: %v", err)
|
||||
// }
|
||||
// defer lock.Unlock()
|
||||
|
||||
glog.Info("Checking server power state")
|
||||
idracCredentials := idracCredentials{
|
||||
url: "https://" + *idracHost,
|
||||
username: *idracUsername,
|
||||
password: *idracPassword,
|
||||
}
|
||||
powerState, err := checkPowerState(idracCredentials)
|
||||
if err != nil {
|
||||
glog.Fatalf("Failed to check power state: %v", err)
|
||||
}
|
||||
glog.Infof("Server power state: %s", powerState)
|
||||
|
||||
glog.Info("Checking UPS state")
|
||||
snmp := getSNMPClient()
|
||||
// Connect to the SNMP agent
|
||||
err = snmp.Connect()
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to connect to UPS SNMP agent: %v", err)
|
||||
}
|
||||
defer snmp.Conn.Close()
|
||||
|
||||
upsState, err := getPowerState(snmp)
|
||||
if err != nil {
|
||||
glog.Fatalf("Failed to get UPS power state: %v", err)
|
||||
}
|
||||
|
||||
if powerState == "On" {
|
||||
handleWhenServerOn(upsState, idracCredentials)
|
||||
} else if powerState == "Off" {
|
||||
handleWhenServerOff(upsState, idracCredentials)
|
||||
} else {
|
||||
glog.Fatalf("Unknown server state %s", powerState)
|
||||
}
|
||||
}
|
||||
func handleWhenServerOn(upsState UPSPowerState, idracCredentials idracCredentials) {
|
||||
if upsState.inputVoltage > 0 {
|
||||
glog.Infof("UPS is on AC power: %d. Nothing to do.\n", upsState.inputVoltage)
|
||||
return
|
||||
} else {
|
||||
glog.Warningln("UPS is on Battery power")
|
||||
if upsState.minutesRemaining < upsMinutesRemainingThreshold {
|
||||
glog.Warningf("Minutes remaining is too low - %d Turning off server.", upsState.minutesRemaining)
|
||||
// Perform a graceful shutdown of the server
|
||||
performGracefulShutdown(idracCredentials)
|
||||
} else {
|
||||
glog.Warningf("Minutes remaining is %d. Server will not be shutdown yet.", upsState.minutesRemaining)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func handleWhenServerOff(upsState UPSPowerState, idracCredentials idracCredentials) {
|
||||
if upsState.inputVoltage > 0 {
|
||||
glog.Infof("UPS is on AC power: %d\n", upsState.inputVoltage)
|
||||
if upsState.minutesRemaining < upsMinutesRemainingThreshold {
|
||||
glog.Infof("UPS battery is still too low - %d minutes remaining. Not turning on server yet.\n", upsState.minutesRemaining)
|
||||
} else {
|
||||
glog.Infof("UPS is on AC power and battery has charged - %d minutes remaining. Turning on server...\n", upsState.minutesRemaining)
|
||||
// Perform startup of the server
|
||||
performPowerOn(idracCredentials)
|
||||
}
|
||||
} else {
|
||||
glog.Warningln("UPS is still on battery power")
|
||||
return
|
||||
}
|
||||
}
|
||||
func tryGetLock() (*lockfile.Lockfile, error) {
|
||||
lock, err := lockfile.New("/tmp/server_safe_poweroff.pid")
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to create lock file: %v", err)
|
||||
}
|
||||
err = lock.TryLock()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &lock, nil
|
||||
}
|
||||
23
server_safe_poweroff/synology_main.sh
Executable file
23
server_safe_poweroff/synology_main.sh
Executable file
|
|
@ -0,0 +1,23 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
# This is used to run the main program on synology nas and log all messages to synology's log system
|
||||
|
||||
cd /var/services/homes/Administrator/server-power-cycle
|
||||
echo "Starting powercheck"
|
||||
./powercheck-armv8 -log_dir=./logs
|
||||
|
||||
echo "script completed successfully, logging to synlogy's logs"
|
||||
|
||||
|
||||
while IFS= read -r line; do
|
||||
# for line in $(cat ./logs/powercheck-armv8.INFO); do
|
||||
msg=$(echo $line | grep -E '^[IWEF][0-9]{4} [0-9]{2}:[0-9]{2}:[0-9]{2}\.[0-9]{6}'| awk '{$1=$2=$3=$4=""; print $0}' | sed 's/^ *//')
|
||||
#echo $line
|
||||
echo $msg
|
||||
if [[ -n $msg ]]; then
|
||||
synologset1 sys info 0x11800000 "$msg"
|
||||
fi
|
||||
done < "./logs/powercheck-armv8.INFO"
|
||||
|
||||
# Cleanup logs
|
||||
find ./logs -type f -mtime +7 -exec rm {} \;
|
||||
46
server_safe_poweroff/ups_utils.go
Normal file
46
server_safe_poweroff/ups_utils.go
Normal file
|
|
@ -0,0 +1,46 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"github.com/golang/glog"
|
||||
"github.com/gosnmp/gosnmp"
|
||||
)
|
||||
|
||||
type UPSPowerState = struct {
|
||||
inputVoltage int
|
||||
minutesRemaining uint
|
||||
}
|
||||
|
||||
func getSNMPClient() *gosnmp.GoSNMP {
|
||||
|
||||
// Define SNMP connection parameters
|
||||
target := "192.168.1.5"
|
||||
community := "Public0"
|
||||
|
||||
// Create a new SNMP client
|
||||
snmp := &gosnmp.GoSNMP{
|
||||
Target: target,
|
||||
Port: 161, // Default SNMP port
|
||||
Community: community,
|
||||
Version: gosnmp.Version2c, // Use SNMP v2c
|
||||
Timeout: time.Duration(5) * time.Second,
|
||||
}
|
||||
return snmp
|
||||
}
|
||||
func getPowerState(snmp *gosnmp.GoSNMP) (UPSPowerState, error) {
|
||||
oids := []string{
|
||||
// "1.3.6.1.2.1.33.1.2.2.0", // seconds on battery
|
||||
"1.3.6.1.2.1.33.1.3.3.1.3.1", // input voltage
|
||||
"1.3.6.1.2.1.33.1.2.3.0", // minutes remaining
|
||||
}
|
||||
// Perform an SNMP GET request to retrieve the values for the specified OIDs
|
||||
result, err := snmp.Get(oids)
|
||||
if err != nil {
|
||||
glog.Fatalf("Failed to perform SNMP GET request: %v", err)
|
||||
}
|
||||
|
||||
inputVoltage := (result.Variables[0].Value).(int)
|
||||
minutesRemaining := result.Variables[1].Value.(uint)
|
||||
return UPSPowerState{inputVoltage, minutesRemaining}, nil
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue