viktor/infra
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

move grafana into separate file and tunr off persistence as we use external db now [ci skip]

Browse source
This commit is contained in:
Viktor Barzin 2025-12-28 20:05:27 +00:00
parent 5b33bbfcc8
commit 34f90c06dc
No known key found for this signature in database
GPG key ID: 4056458DBDBF8863
3 changed files with 108 additions and 33 deletions
Download patch file Download diff file Expand all files Collapse all files

70
modules/kubernetes/monitoring/dashboards/nvidia.json
View file

@ -19,9 +19,8 @@
"description": "This dashboard is to display the metrics from DCGM Exporter on a Kubernetes (1.13+) cluster", "description": "This dashboard is to display the metrics from DCGM Exporter on a Kubernetes (1.13+) cluster",
"editable": true, "editable": true,
"fiscalYearStartMonth": 0, "fiscalYearStartMonth": 0,
"gnetId": 12239,
"graphTooltip": 0, "graphTooltip": 0,
"id": 26, "id": 0,
"links": [], "links": [],
"panels": [ "panels": [
{ {
@ -41,6 +40,7 @@
"axisLabel": "", "axisLabel": "",
"axisPlacement": "auto", "axisPlacement": "auto",
"barAlignment": 0, "barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "line", "drawStyle": "line",
"fillOpacity": 10, "fillOpacity": 10,
"gradientMode": "none", "gradientMode": "none",
@ -57,6 +57,7 @@
"type": "linear" "type": "linear"
}, },
"showPoints": "never", "showPoints": "never",
"showValues": false,
"spanNulls": false, "spanNulls": false,
"stacking": { "stacking": {
"group": "A", "group": "A",
@ -72,7 +73,7 @@
"steps": [ "steps": [
{ {
"color": "green", "color": "green",
"value": null "value": 0
}, },
{ {
"color": "red", "color": "red",
@ -103,10 +104,12 @@
"showLegend": true "showLegend": true
}, },
"tooltip": { "tooltip": {
"hideZeros": false,
"mode": "multi", "mode": "multi",
"sort": "none" "sort": "none"
} }
}, },
"pluginVersion": "12.3.0",
"targets": [ "targets": [
{ {
"datasource": { "datasource": {
@ -142,7 +145,7 @@
"steps": [ "steps": [
{ {
"color": "green", "color": "green",
"value": null "value": 0
}, },
{ {
"color": "#EAB839", "color": "#EAB839",
@ -180,7 +183,7 @@
"showThresholdMarkers": true, "showThresholdMarkers": true,
"sizing": "auto" "sizing": "auto"
}, },
"pluginVersion": "11.1.0", "pluginVersion": "12.3.0",
"targets": [ "targets": [
{ {
"datasource": { "datasource": {
@ -215,6 +218,7 @@
"axisLabel": "", "axisLabel": "",
"axisPlacement": "auto", "axisPlacement": "auto",
"barAlignment": 0, "barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "line", "drawStyle": "line",
"fillOpacity": 10, "fillOpacity": 10,
"gradientMode": "none", "gradientMode": "none",
@ -231,6 +235,7 @@
"type": "linear" "type": "linear"
}, },
"showPoints": "never", "showPoints": "never",
"showValues": false,
"spanNulls": false, "spanNulls": false,
"stacking": { "stacking": {
"group": "A", "group": "A",
@ -246,7 +251,7 @@
"steps": [ "steps": [
{ {
"color": "green", "color": "green",
"value": null "value": 0
}, },
{ {
"color": "red", "color": "red",
@ -277,11 +282,12 @@
"showLegend": true "showLegend": true
}, },
"tooltip": { "tooltip": {
"hideZeros": false,
"mode": "multi", "mode": "multi",
"sort": "none" "sort": "none"
} }
}, },
"pluginVersion": "6.5.2", "pluginVersion": "12.3.0",
"targets": [ "targets": [
{ {
"datasource": { "datasource": {
@ -317,7 +323,7 @@
"steps": [ "steps": [
{ {
"color": "green", "color": "green",
"value": null "value": 0
}, },
{ {
"color": "#EAB839", "color": "#EAB839",
@ -355,7 +361,7 @@
"showThresholdMarkers": true, "showThresholdMarkers": true,
"sizing": "auto" "sizing": "auto"
}, },
"pluginVersion": "11.1.0", "pluginVersion": "12.3.0",
"targets": [ "targets": [
{ {
"datasource": { "datasource": {
@ -363,10 +369,12 @@
"uid": "PBFA97CFB590B2093" "uid": "PBFA97CFB590B2093"
}, },
"editorMode": "code", "editorMode": "code",
"exemplar": false,
"expr": "sum(nvidia_tesla_t4_DCGM_FI_DEV_POWER_USAGE)", "expr": "sum(nvidia_tesla_t4_DCGM_FI_DEV_POWER_USAGE)",
"instant": true,
"interval": "", "interval": "",
"legendFormat": "", "legendFormat": "",
"range": true, "range": false,
"refId": "A" "refId": "A"
} }
], ],
@ -390,6 +398,7 @@
"axisLabel": "", "axisLabel": "",
"axisPlacement": "auto", "axisPlacement": "auto",
"barAlignment": 0, "barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "line", "drawStyle": "line",
"fillOpacity": 10, "fillOpacity": 10,
"gradientMode": "none", "gradientMode": "none",
@ -406,6 +415,7 @@
"type": "linear" "type": "linear"
}, },
"showPoints": "never", "showPoints": "never",
"showValues": false,
"spanNulls": false, "spanNulls": false,
"stacking": { "stacking": {
"group": "A", "group": "A",
@ -423,7 +433,7 @@
"steps": [ "steps": [
{ {
"color": "green", "color": "green",
"value": null "value": 0
}, },
{ {
"color": "red", "color": "red",
@ -454,10 +464,12 @@
"showLegend": true "showLegend": true
}, },
"tooltip": { "tooltip": {
"hideZeros": false,
"mode": "multi", "mode": "multi",
"sort": "none" "sort": "none"
} }
}, },
"pluginVersion": "12.3.0",
"targets": [ "targets": [
{ {
"datasource": { "datasource": {
@ -492,6 +504,7 @@
"axisLabel": "", "axisLabel": "",
"axisPlacement": "auto", "axisPlacement": "auto",
"barAlignment": 0, "barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "line", "drawStyle": "line",
"fillOpacity": 10, "fillOpacity": 10,
"gradientMode": "none", "gradientMode": "none",
@ -508,6 +521,7 @@
"type": "linear" "type": "linear"
}, },
"showPoints": "never", "showPoints": "never",
"showValues": false,
"spanNulls": false, "spanNulls": false,
"stacking": { "stacking": {
"group": "A", "group": "A",
@ -523,7 +537,7 @@
"steps": [ "steps": [
{ {
"color": "green", "color": "green",
"value": null "value": 0
}, },
{ {
"color": "red", "color": "red",
@ -553,10 +567,12 @@
"showLegend": true "showLegend": true
}, },
"tooltip": { "tooltip": {
"hideZeros": false,
"mode": "multi", "mode": "multi",
"sort": "none" "sort": "none"
} }
}, },
"pluginVersion": "12.3.0",
"targets": [ "targets": [
{ {
"datasource": { "datasource": {
@ -591,6 +607,7 @@
"axisLabel": "", "axisLabel": "",
"axisPlacement": "auto", "axisPlacement": "auto",
"barAlignment": 0, "barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "line", "drawStyle": "line",
"fillOpacity": 10, "fillOpacity": 10,
"gradientMode": "none", "gradientMode": "none",
@ -607,6 +624,7 @@
"type": "linear" "type": "linear"
}, },
"showPoints": "never", "showPoints": "never",
"showValues": false,
"spanNulls": false, "spanNulls": false,
"stacking": { "stacking": {
"group": "A", "group": "A",
@ -622,7 +640,7 @@
"steps": [ "steps": [
{ {
"color": "green", "color": "green",
"value": null "value": 0
}, },
{ {
"color": "red", "color": "red",
@ -641,7 +659,6 @@
"y": 24 "y": 24
}, },
"id": 2, "id": 2,
"interval": "",
"options": { "options": {
"legend": { "legend": {
"calcs": [ "calcs": [
@ -654,10 +671,12 @@
"showLegend": true "showLegend": true
}, },
"tooltip": { "tooltip": {
"hideZeros": false,
"mode": "multi", "mode": "multi",
"sort": "none" "sort": "none"
} }
}, },
"pluginVersion": "12.3.0",
"targets": [ "targets": [
{ {
"datasource": { "datasource": {
@ -678,33 +697,20 @@
"type": "timeseries" "type": "timeseries"
} }
], ],
"preload": false,
"refresh": "auto", "refresh": "auto",
"schemaVersion": 39, "schemaVersion": 42,
"tags": [], "tags": [],
"templating": { "templating": {
"list": [] "list": []
}, },
"time": { "time": {
"from": "now-30m", "from": "now-3h",
"to": "now" "to": "now"
}, },
"timepicker": { "timepicker": {},
"refresh_intervals": [
"5s",
"10s",
"30s",
"1m",
"5m",
"15m",
"30m",
"1h",
"2h",
"1d"
]
},
"timezone": "", "timezone": "",
"title": "NVIDIA DCGM Exporter Dashboard", "title": "NVIDIA DCGM Exporter Dashboard",
"uid": "Oxed_c6Wz", "uid": "Oxed_c6Wz",
"version": 7, "version": 8
"weekStart": ""
} }

68
modules/kubernetes/monitoring/grafana.tf Normal file
View file

@ -0,0 +1,68 @@
# resource "kubernetes_persistent_volume" "prometheus_grafana_pv" {
# metadata {
# name = "grafana-pv"
# }
# spec {
# capacity = {
# "storage" = "2Gi"
# }
# access_modes = ["ReadWriteOnce"]
# persistent_volume_source {
# nfs {
# path = "/mnt/main/grafana"
# server = "10.0.10.15"
# }
# # iscsi {
# # target_portal = "iscsi.viktorbarzin.lan:3260"
# # iqn = "iqn.2020-12.lan.viktorbarzin:storage:monitoring:grafana"
# # lun = 0
# # fs_type = "ext4"
# # }
# }
# }
# }
resource "kubernetes_persistent_volume" "alertmanager_pv" {
metadata {
name = "alertmanager-pv"
}
spec {
capacity = {
"storage" = "2Gi"
}
access_modes = ["ReadWriteOnce"]
persistent_volume_source {
nfs {
path = "/mnt/main/alertmanager"
server = "10.0.10.15"
}
}
}
}
# resource "kubernetes_persistent_volume_claim" "grafana_pvc" {
# metadata {
# name = "grafana-pvc"
# namespace = "monitoring"
# }
# spec {
# access_modes = ["ReadWriteOnce"]
# resources {
# requests = {
# "storage" = "2Gi"
# }
# }
# }
# }
resource "helm_release" "grafana" {
namespace = "monitoring"
create_namespace = true
name = "grafana"
atomic = true
repository = "https://grafana.github.io/helm-charts"
chart = "grafana"
values = [templatefile("${path.module}/grafana_chart_values.yaml", { db_password = var.grafana_db_password })]
}

3
modules/kubernetes/monitoring/grafana_chart_values.yaml
View file

@ -1,7 +1,7 @@
deploymentStrategy: deploymentStrategy:
type: Recreate type: Recreate
persistence: persistence:
enabled: true enabled: false # using external mysql
existingClaim: "grafana-pvc" existingClaim: "grafana-pvc"
ingress: ingress:
enabled: "true" enabled: "true"
@ -39,6 +39,7 @@ dashboardProviders:
path: "/var/lib/grafana/dashboards/default" path: "/var/lib/grafana/dashboards/default"
env: env:
GF_DATABASE_PASSWORD: "${db_password}" GF_DATABASE_PASSWORD: "${db_password}"
GF_SERVER_ROOT_URL: https://grafana.viktorbarzin.me
grafana.ini: grafana.ini:
database: database: