monitoring: add Prometheus + Grafana stack (kube-prometheus-stack)
Deploys kube-prometheus-stack via helm-controller: Prometheus, Grafana (internal-only), node-exporter, kube-state-metrics. Adds prometheus-pve-exporter for Proxmox (token in an out-of-band Secret), scrape configs for external hosts (.48/.70/.71/.49), community dashboards (Node Exporter Full, Proxmox via Prometheus) and a custom Hardware Temperatures dashboard. Grafana liveness made tolerant for slow first-boot migrations. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
teddy
parent
a064f41250
commit
0a02da49a4
3 changed files with 275 additions and 0 deletions
91
monitoring/grafana-temps-dashboard.yaml
Normal file
91
monitoring/grafana-temps-dashboard.yaml
Normal file
|
|
@ -0,0 +1,91 @@
|
||||||
|
apiVersion: v1
|
||||||
|
kind: ConfigMap
|
||||||
|
metadata:
|
||||||
|
name: grafana-dashboard-temps
|
||||||
|
namespace: monitoring
|
||||||
|
labels:
|
||||||
|
grafana_dashboard: "1"
|
||||||
|
data:
|
||||||
|
hardware-temperatures.json: |
|
||||||
|
{
|
||||||
|
"annotations": {"list": []},
|
||||||
|
"editable": true,
|
||||||
|
"graphTooltip": 0,
|
||||||
|
"links": [],
|
||||||
|
"panels": [
|
||||||
|
{
|
||||||
|
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"color": {"mode": "thresholds"},
|
||||||
|
"unit": "celsius",
|
||||||
|
"thresholds": {"mode": "absolute", "steps": [
|
||||||
|
{"color": "green", "value": null},
|
||||||
|
{"color": "yellow", "value": 60},
|
||||||
|
{"color": "orange", "value": 75},
|
||||||
|
{"color": "red", "value": 85}
|
||||||
|
]}
|
||||||
|
},
|
||||||
|
"overrides": []
|
||||||
|
},
|
||||||
|
"gridPos": {"h": 8, "w": 12, "x": 0, "y": 0},
|
||||||
|
"id": 1,
|
||||||
|
"options": {
|
||||||
|
"colorMode": "background",
|
||||||
|
"graphMode": "area",
|
||||||
|
"orientation": "auto",
|
||||||
|
"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false},
|
||||||
|
"textMode": "auto"
|
||||||
|
},
|
||||||
|
"pluginVersion": "11.0.0",
|
||||||
|
"title": "Max temperature per host (now)",
|
||||||
|
"type": "stat",
|
||||||
|
"targets": [
|
||||||
|
{"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||||
|
"expr": "max by (instance) (node_hwmon_temp_celsius)",
|
||||||
|
"legendFormat": "{{instance}}", "refId": "A"}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"color": {"mode": "palette-classic"},
|
||||||
|
"unit": "celsius",
|
||||||
|
"custom": {"drawStyle": "line", "lineWidth": 1, "fillOpacity": 10, "showPoints": "never"},
|
||||||
|
"thresholds": {"mode": "absolute", "steps": [
|
||||||
|
{"color": "green", "value": null},
|
||||||
|
{"color": "red", "value": 85}
|
||||||
|
]}
|
||||||
|
},
|
||||||
|
"overrides": []
|
||||||
|
},
|
||||||
|
"gridPos": {"h": 8, "w": 12, "x": 12, "y": 0},
|
||||||
|
"id": 2,
|
||||||
|
"options": {
|
||||||
|
"legend": {"displayMode": "table", "placement": "bottom", "calcs": ["last", "max"]},
|
||||||
|
"tooltip": {"mode": "multi"}
|
||||||
|
},
|
||||||
|
"title": "All temperature sensors",
|
||||||
|
"type": "timeseries",
|
||||||
|
"targets": [
|
||||||
|
{"datasource": {"type": "prometheus", "uid": "${datasource}"},
|
||||||
|
"expr": "node_hwmon_temp_celsius",
|
||||||
|
"legendFormat": "{{instance}} / {{chip}} / {{sensor}}", "refId": "A"}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"refresh": "30s",
|
||||||
|
"schemaVersion": 39,
|
||||||
|
"tags": ["hardware", "temperature"],
|
||||||
|
"templating": {"list": [
|
||||||
|
{"current": {}, "hide": 0, "includeAll": false, "label": "Datasource",
|
||||||
|
"multi": false, "name": "datasource", "options": [], "query": "prometheus",
|
||||||
|
"refresh": 1, "type": "datasource"}
|
||||||
|
]},
|
||||||
|
"time": {"from": "now-6h", "to": "now"},
|
||||||
|
"timezone": "Europe/Oslo",
|
||||||
|
"title": "Hardware Temperatures",
|
||||||
|
"uid": "hw-temps",
|
||||||
|
"version": 1
|
||||||
|
}
|
||||||
131
monitoring/monitoring.yaml
Normal file
131
monitoring/monitoring.yaml
Normal file
|
|
@ -0,0 +1,131 @@
|
||||||
|
apiVersion: v1
|
||||||
|
kind: Namespace
|
||||||
|
metadata:
|
||||||
|
name: monitoring
|
||||||
|
---
|
||||||
|
# kube-prometheus-stack: Prometheus + Grafana + node-exporter + kube-state-metrics.
|
||||||
|
# Deployed via k3s helm-controller. Grafana is internal-only (ClusterIP) for now;
|
||||||
|
# an ingress stub is commented at the bottom for when you want grafana.roysland.net.
|
||||||
|
apiVersion: helm.cattle.io/v1
|
||||||
|
kind: HelmChart
|
||||||
|
metadata:
|
||||||
|
name: kube-prometheus-stack
|
||||||
|
namespace: kube-system
|
||||||
|
spec:
|
||||||
|
chart: kube-prometheus-stack
|
||||||
|
repo: https://prometheus-community.github.io/helm-charts
|
||||||
|
targetNamespace: monitoring
|
||||||
|
valuesContent: |-
|
||||||
|
# Alertmanager off for now (no notification channel configured yet).
|
||||||
|
alertmanager:
|
||||||
|
enabled: false
|
||||||
|
|
||||||
|
grafana:
|
||||||
|
adminPassword: "roysland-grafana-changeme"
|
||||||
|
defaultDashboardsTimezone: Europe/Oslo
|
||||||
|
service:
|
||||||
|
type: ClusterIP
|
||||||
|
# Grafana 13 first-boot migrations are slow on the local-path disk; be
|
||||||
|
# tolerant so the liveness probe doesn't kill it mid-migration.
|
||||||
|
livenessProbe:
|
||||||
|
initialDelaySeconds: 120
|
||||||
|
timeoutSeconds: 30
|
||||||
|
failureThreshold: 30
|
||||||
|
persistence:
|
||||||
|
enabled: true
|
||||||
|
storageClassName: local-path
|
||||||
|
size: 2Gi
|
||||||
|
# Sidecar loads any ConfigMap labelled grafana_dashboard across namespaces
|
||||||
|
# (used by the custom Temperatures dashboard).
|
||||||
|
sidecar:
|
||||||
|
dashboards:
|
||||||
|
enabled: true
|
||||||
|
searchNamespace: ALL
|
||||||
|
# Auto-provisioned community dashboards (downloaded from grafana.com on start).
|
||||||
|
dashboardProviders:
|
||||||
|
dashboardproviders.yaml:
|
||||||
|
apiVersion: 1
|
||||||
|
providers:
|
||||||
|
- name: default
|
||||||
|
orgId: 1
|
||||||
|
folder: ""
|
||||||
|
type: file
|
||||||
|
disableDeletion: false
|
||||||
|
editable: true
|
||||||
|
options:
|
||||||
|
path: /var/lib/grafana/dashboards/default
|
||||||
|
dashboards:
|
||||||
|
default:
|
||||||
|
node-exporter-full:
|
||||||
|
gnetId: 1860 # Node Exporter Full (CPU/RAM/disk/net + temps), pick host via $instance
|
||||||
|
revision: 45
|
||||||
|
datasource: Prometheus
|
||||||
|
proxmox:
|
||||||
|
gnetId: 10347 # Proxmox via Prometheus (pve-exporter)
|
||||||
|
revision: 5
|
||||||
|
datasource: Prometheus
|
||||||
|
|
||||||
|
# Node metrics for the k3s host itself.
|
||||||
|
nodeExporter:
|
||||||
|
enabled: true
|
||||||
|
kube-state-metrics:
|
||||||
|
enabled: true
|
||||||
|
|
||||||
|
prometheus:
|
||||||
|
prometheusSpec:
|
||||||
|
retention: 15d
|
||||||
|
scrapeInterval: 30s
|
||||||
|
# Persist metrics on the local-path PVC.
|
||||||
|
storageSpec:
|
||||||
|
volumeClaimTemplate:
|
||||||
|
spec:
|
||||||
|
storageClassName: local-path
|
||||||
|
accessModes: ["ReadWriteOnce"]
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
storage: 10Gi
|
||||||
|
# Pick up ServiceMonitors/PodMonitors from any namespace, not just chart-labelled ones.
|
||||||
|
serviceMonitorSelectorNilUsesHelmValues: false
|
||||||
|
podMonitorSelectorNilUsesHelmValues: false
|
||||||
|
# External targets: Proxmox host + the other VMs + test server.
|
||||||
|
additionalScrapeConfigs:
|
||||||
|
# node_exporter on the Proxmox host and the other servers (installed via ansible).
|
||||||
|
- job_name: node-external
|
||||||
|
static_configs:
|
||||||
|
- targets:
|
||||||
|
- 192.168.50.48:9100 # proxmox host (master)
|
||||||
|
- 192.168.50.70:9100 # minio
|
||||||
|
- 192.168.50.71:9100 # forgejo
|
||||||
|
- 192.168.50.49:9100 # test server
|
||||||
|
labels:
|
||||||
|
group: infra-hosts
|
||||||
|
# Proxmox VE stats via prometheus-pve-exporter (deployed in-cluster).
|
||||||
|
- job_name: proxmox-pve
|
||||||
|
metrics_path: /pve
|
||||||
|
params:
|
||||||
|
module: [default]
|
||||||
|
static_configs:
|
||||||
|
- targets:
|
||||||
|
- 192.168.50.48 # the PVE node to query
|
||||||
|
relabel_configs:
|
||||||
|
- source_labels: [__address__]
|
||||||
|
target_label: __param_target
|
||||||
|
- source_labels: [__param_target]
|
||||||
|
target_label: instance
|
||||||
|
- target_label: __address__
|
||||||
|
replacement: prometheus-pve-exporter.monitoring.svc.cluster.local:9221
|
||||||
|
# ---
|
||||||
|
# To expose Grafana later at grafana.roysland.net, set grafana.ingress in the
|
||||||
|
# values above:
|
||||||
|
# grafana:
|
||||||
|
# ingress:
|
||||||
|
# enabled: true
|
||||||
|
# ingressClassName: traefik
|
||||||
|
# annotations:
|
||||||
|
# cert-manager.io/cluster-issuer: letsencrypt
|
||||||
|
# traefik.ingress.kubernetes.io/router.middlewares: monitoring-https-redirect@kubernetescrd
|
||||||
|
# hosts: [grafana.roysland.net]
|
||||||
|
# tls:
|
||||||
|
# - hosts: [grafana.roysland.net]
|
||||||
|
# secretName: grafana-tls
|
||||||
|
# (and add a Middleware named https-redirect in the monitoring namespace)
|
||||||
53
monitoring/pve-exporter.yaml
Normal file
53
monitoring/pve-exporter.yaml
Normal file
|
|
@ -0,0 +1,53 @@
|
||||||
|
# prometheus-pve-exporter — exposes Proxmox VE metrics for Prometheus.
|
||||||
|
# The API token lives in the `pve-exporter` Secret, created out-of-band (NOT in git):
|
||||||
|
# kubectl -n monitoring create secret generic pve-exporter \
|
||||||
|
# --from-literal=PVE_USER=monitoring@pve \
|
||||||
|
# --from-literal=PVE_TOKEN_NAME=prometheus \
|
||||||
|
# --from-literal=PVE_TOKEN_VALUE=<token> \
|
||||||
|
# --from-literal=PVE_VERIFY_SSL=false
|
||||||
|
apiVersion: apps/v1
|
||||||
|
kind: Deployment
|
||||||
|
metadata:
|
||||||
|
name: prometheus-pve-exporter
|
||||||
|
namespace: monitoring
|
||||||
|
labels:
|
||||||
|
app: prometheus-pve-exporter
|
||||||
|
spec:
|
||||||
|
replicas: 1
|
||||||
|
selector:
|
||||||
|
matchLabels:
|
||||||
|
app: prometheus-pve-exporter
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
labels:
|
||||||
|
app: prometheus-pve-exporter
|
||||||
|
spec:
|
||||||
|
containers:
|
||||||
|
- name: pve-exporter
|
||||||
|
image: prompve/prometheus-pve-exporter:3.9.0
|
||||||
|
envFrom:
|
||||||
|
- secretRef:
|
||||||
|
name: pve-exporter
|
||||||
|
ports:
|
||||||
|
- containerPort: 9221
|
||||||
|
name: http
|
||||||
|
securityContext:
|
||||||
|
runAsNonRoot: true
|
||||||
|
runAsUser: 1000
|
||||||
|
allowPrivilegeEscalation: false
|
||||||
|
capabilities:
|
||||||
|
drop: ["ALL"]
|
||||||
|
seccompProfile:
|
||||||
|
type: RuntimeDefault
|
||||||
|
---
|
||||||
|
apiVersion: v1
|
||||||
|
kind: Service
|
||||||
|
metadata:
|
||||||
|
name: prometheus-pve-exporter
|
||||||
|
namespace: monitoring
|
||||||
|
spec:
|
||||||
|
selector:
|
||||||
|
app: prometheus-pve-exporter
|
||||||
|
ports:
|
||||||
|
- port: 9221
|
||||||
|
targetPort: 9221
|
||||||
Loading…
Add table
Add a link
Reference in a new issue