294 lines
10 KiB
YAML
294 lines
10 KiB
YAML
---
|
|
# Prometheus Alert Rules for Kubernetes
|
|
# Covers pods, nodes, deployments, and resource usage
|
|
|
|
groups:
|
|
- name: kubernetes_pods
|
|
interval: 30s
|
|
rules:
|
|
# Pod crash looping
|
|
- alert: PodCrashLooping
|
|
expr: |
|
|
rate(kube_pod_container_status_restarts_total[15m]) > 0
|
|
for: 5m
|
|
labels:
|
|
severity: warning
|
|
team: platform
|
|
component: kubernetes
|
|
annotations:
|
|
summary: "Pod is crash looping - {{ $labels.namespace }}/{{ $labels.pod }}"
|
|
description: |
|
|
Pod {{ $labels.namespace }}/{{ $labels.pod }} has restarted {{ $value }} times in the last 15 minutes.
|
|
|
|
Check pod logs:
|
|
kubectl logs -n {{ $labels.namespace }} {{ $labels.pod }} --previous
|
|
runbook_url: "https://runbooks.example.com/pod-crash-loop"
|
|
|
|
# Pod not ready
|
|
- alert: PodNotReady
|
|
expr: |
|
|
sum by (namespace, pod) (kube_pod_status_phase{phase!~"Running|Succeeded"}) > 0
|
|
for: 10m
|
|
labels:
|
|
severity: warning
|
|
team: platform
|
|
component: kubernetes
|
|
annotations:
|
|
summary: "Pod not ready - {{ $labels.namespace }}/{{ $labels.pod }}"
|
|
description: |
|
|
Pod {{ $labels.namespace }}/{{ $labels.pod }} is in {{ $labels.phase }} state for 10 minutes.
|
|
|
|
Investigate:
|
|
kubectl describe pod -n {{ $labels.namespace }} {{ $labels.pod }}
|
|
runbook_url: "https://runbooks.example.com/pod-not-ready"
|
|
|
|
# Pod OOMKilled
|
|
- alert: PodOOMKilled
|
|
expr: |
|
|
sum by (namespace, pod) (kube_pod_container_status_terminated_reason{reason="OOMKilled"}) > 0
|
|
for: 1m
|
|
labels:
|
|
severity: warning
|
|
team: platform
|
|
component: kubernetes
|
|
annotations:
|
|
summary: "Pod killed due to OOM - {{ $labels.namespace }}/{{ $labels.pod }}"
|
|
description: |
|
|
Pod {{ $labels.namespace }}/{{ $labels.pod }} was killed due to out-of-memory.
|
|
|
|
Increase memory limits or investigate memory leak.
|
|
runbook_url: "https://runbooks.example.com/oom-killed"
|
|
|
|
- name: kubernetes_deployments
|
|
interval: 30s
|
|
rules:
|
|
# Deployment replica mismatch
|
|
- alert: DeploymentReplicasMismatch
|
|
expr: |
|
|
kube_deployment_spec_replicas != kube_deployment_status_replicas_available
|
|
for: 15m
|
|
labels:
|
|
severity: warning
|
|
team: platform
|
|
component: kubernetes
|
|
annotations:
|
|
summary: "Deployment replicas mismatch - {{ $labels.namespace }}/{{ $labels.deployment }}"
|
|
description: |
|
|
Deployment {{ $labels.namespace }}/{{ $labels.deployment }} has been running with
|
|
fewer replicas than desired for 15 minutes.
|
|
|
|
Desired: {{ $value }}
|
|
Available: Check deployment status
|
|
runbook_url: "https://runbooks.example.com/replica-mismatch"
|
|
|
|
# Deployment rollout stuck
|
|
- alert: DeploymentRolloutStuck
|
|
expr: |
|
|
kube_deployment_status_condition{condition="Progressing", status="false"} > 0
|
|
for: 15m
|
|
labels:
|
|
severity: warning
|
|
team: platform
|
|
component: kubernetes
|
|
annotations:
|
|
summary: "Deployment rollout stuck - {{ $labels.namespace }}/{{ $labels.deployment }}"
|
|
description: |
|
|
Deployment {{ $labels.namespace }}/{{ $labels.deployment }} rollout is stuck.
|
|
|
|
Check rollout status:
|
|
kubectl rollout status deployment/{{ $labels.deployment }} -n {{ $labels.namespace }}
|
|
runbook_url: "https://runbooks.example.com/rollout-stuck"
|
|
|
|
- name: kubernetes_nodes
|
|
interval: 30s
|
|
rules:
|
|
# Node not ready
|
|
- alert: NodeNotReady
|
|
expr: |
|
|
kube_node_status_condition{condition="Ready",status="true"} == 0
|
|
for: 5m
|
|
labels:
|
|
severity: critical
|
|
team: platform
|
|
component: kubernetes
|
|
annotations:
|
|
summary: "Node not ready - {{ $labels.node }}"
|
|
description: |
|
|
Node {{ $labels.node }} has been NotReady for 5 minutes.
|
|
|
|
This will affect pod scheduling and availability.
|
|
|
|
Check node status:
|
|
kubectl describe node {{ $labels.node }}
|
|
runbook_url: "https://runbooks.example.com/node-not-ready"
|
|
|
|
# Node memory pressure
|
|
- alert: NodeMemoryPressure
|
|
expr: |
|
|
kube_node_status_condition{condition="MemoryPressure",status="true"} == 1
|
|
for: 5m
|
|
labels:
|
|
severity: warning
|
|
team: platform
|
|
component: kubernetes
|
|
annotations:
|
|
summary: "Node under memory pressure - {{ $labels.node }}"
|
|
description: |
|
|
Node {{ $labels.node }} is experiencing memory pressure.
|
|
|
|
Pods may be evicted. Consider scaling up or evicting low-priority pods.
|
|
runbook_url: "https://runbooks.example.com/memory-pressure"
|
|
|
|
# Node disk pressure
|
|
- alert: NodeDiskPressure
|
|
expr: |
|
|
kube_node_status_condition{condition="DiskPressure",status="true"} == 1
|
|
for: 5m
|
|
labels:
|
|
severity: warning
|
|
team: platform
|
|
component: kubernetes
|
|
annotations:
|
|
summary: "Node under disk pressure - {{ $labels.node }}"
|
|
description: |
|
|
Node {{ $labels.node }} is experiencing disk pressure.
|
|
|
|
Clean up disk space or add capacity.
|
|
runbook_url: "https://runbooks.example.com/disk-pressure"
|
|
|
|
# Node high CPU
|
|
- alert: NodeHighCPU
|
|
expr: |
|
|
(1 - avg(rate(node_cpu_seconds_total{mode="idle"}[5m])) by (instance)) * 100 > 80
|
|
for: 15m
|
|
labels:
|
|
severity: warning
|
|
team: platform
|
|
component: kubernetes
|
|
annotations:
|
|
summary: "Node high CPU usage - {{ $labels.instance }}"
|
|
description: |
|
|
Node {{ $labels.instance }} CPU usage is {{ $value | humanize }}%.
|
|
|
|
Check for resource-intensive pods or scale cluster.
|
|
runbook_url: "https://runbooks.example.com/node-high-cpu"
|
|
|
|
- name: kubernetes_resources
|
|
interval: 30s
|
|
rules:
|
|
# Container CPU throttling
|
|
- alert: ContainerCPUThrottling
|
|
expr: |
|
|
rate(container_cpu_cfs_throttled_seconds_total[5m]) > 0.5
|
|
for: 10m
|
|
labels:
|
|
severity: warning
|
|
team: platform
|
|
component: kubernetes
|
|
annotations:
|
|
summary: "Container CPU throttling - {{ $labels.namespace }}/{{ $labels.pod }}"
|
|
description: |
|
|
Container {{ $labels.container }} in pod {{ $labels.namespace }}/{{ $labels.pod }}
|
|
is being CPU throttled.
|
|
|
|
CPU throttling rate: {{ $value | humanize }}
|
|
|
|
Consider increasing CPU limits.
|
|
runbook_url: "https://runbooks.example.com/cpu-throttling"
|
|
|
|
# Container memory usage high
|
|
- alert: ContainerMemoryUsageHigh
|
|
expr: |
|
|
(container_memory_usage_bytes / container_spec_memory_limit_bytes) > 0.9
|
|
for: 10m
|
|
labels:
|
|
severity: warning
|
|
team: platform
|
|
component: kubernetes
|
|
annotations:
|
|
summary: "Container memory usage high - {{ $labels.namespace }}/{{ $labels.pod }}"
|
|
description: |
|
|
Container {{ $labels.container }} in pod {{ $labels.namespace }}/{{ $labels.pod }}
|
|
is using {{ $value | humanizePercentage }} of its memory limit.
|
|
|
|
Risk of OOMKill. Consider increasing memory limits.
|
|
runbook_url: "https://runbooks.example.com/high-memory"
|
|
|
|
- name: kubernetes_pv
|
|
interval: 30s
|
|
rules:
|
|
# PersistentVolume nearing full
|
|
- alert: PersistentVolumeFillingUp
|
|
expr: |
|
|
(kubelet_volume_stats_available_bytes / kubelet_volume_stats_capacity_bytes) < 0.15
|
|
for: 10m
|
|
labels:
|
|
severity: warning
|
|
team: platform
|
|
component: kubernetes
|
|
annotations:
|
|
summary: "PersistentVolume filling up - {{ $labels.namespace }}/{{ $labels.persistentvolumeclaim }}"
|
|
description: |
|
|
PersistentVolume {{ $labels.persistentvolumeclaim }} in namespace {{ $labels.namespace }}
|
|
is {{ $value | humanizePercentage }} full.
|
|
|
|
Available space is running low. Consider expanding volume.
|
|
runbook_url: "https://runbooks.example.com/pv-filling-up"
|
|
|
|
# PersistentVolume critically full
|
|
- alert: PersistentVolumeCriticallyFull
|
|
expr: |
|
|
(kubelet_volume_stats_available_bytes / kubelet_volume_stats_capacity_bytes) < 0.05
|
|
for: 5m
|
|
labels:
|
|
severity: critical
|
|
team: platform
|
|
component: kubernetes
|
|
annotations:
|
|
summary: "PersistentVolume critically full - {{ $labels.namespace }}/{{ $labels.persistentvolumeclaim }}"
|
|
description: |
|
|
PersistentVolume {{ $labels.persistentvolumeclaim }} in namespace {{ $labels.namespace }}
|
|
is {{ $value | humanizePercentage }} full.
|
|
|
|
Immediate action required to prevent application failures.
|
|
runbook_url: "https://runbooks.example.com/pv-critically-full"
|
|
|
|
- name: kubernetes_jobs
|
|
interval: 30s
|
|
rules:
|
|
# Job failed
|
|
- alert: JobFailed
|
|
expr: |
|
|
kube_job_status_failed > 0
|
|
for: 5m
|
|
labels:
|
|
severity: warning
|
|
team: platform
|
|
component: kubernetes
|
|
annotations:
|
|
summary: "Job failed - {{ $labels.namespace }}/{{ $labels.job_name }}"
|
|
description: |
|
|
Job {{ $labels.namespace }}/{{ $labels.job_name }} has failed.
|
|
|
|
Check job logs:
|
|
kubectl logs job/{{ $labels.job_name }} -n {{ $labels.namespace }}
|
|
runbook_url: "https://runbooks.example.com/job-failed"
|
|
|
|
# CronJob not running
|
|
- alert: CronJobNotRunning
|
|
expr: |
|
|
time() - kube_cronjob_status_last_schedule_time > 3600
|
|
for: 10m
|
|
labels:
|
|
severity: warning
|
|
team: platform
|
|
component: kubernetes
|
|
annotations:
|
|
summary: "CronJob not running - {{ $labels.namespace }}/{{ $labels.cronjob }}"
|
|
description: |
|
|
CronJob {{ $labels.namespace}}/{{ $labels.cronjob }} hasn't run in over an hour.
|
|
|
|
Check CronJob status:
|
|
kubectl describe cronjob {{ $labels.cronjob }} -n {{ $labels.namespace }}
|
|
runbook_url: "https://runbooks.example.com/cronjob-not-running"
|