Linux 服务器监控完全指南 2026：Prometheus + Grafana + Alertmanager 实战部署

1
# 目录结构：
2
# monitoring/
3
# ├── docker-compose.yml
4
# ├── prometheus/
5
# │   ├── prometheus.yml
6
# │   └── rules/
7
# │       └── alerts.yml
8
# └── alertmanager/
9
#     └── alertmanager.yml
10

11
services:
12
  prometheus:
13
    image: prom/prometheus:v2.53.0
14
    container_name: prometheus
15
    restart: unless-stopped
16
    ports:
17
      - "9090:9090"
18
    volumes:
19
      - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
20
      - ./prometheus/rules:/etc/prometheus/rules:ro
21
      - prometheus_data:/prometheus
22
    command:
23
      - "--config.file=/etc/prometheus/prometheus.yml"
24
      - "--storage.tsdb.path=/prometheus"
25
      - "--storage.tsdb.retention.time=30d"   # 数据保留30天
26
      - "--web.enable-lifecycle"               # 允许热重载配置
27
      - "--web.enable-admin-api"
28
    networks: [monitoring]
29

30
  grafana:
31
    image: grafana/grafana:11.1.0
32
    container_name: grafana
33
    restart: unless-stopped
34
    ports:
35
      - "3000:3000"
36
    environment:
37
      - GF_SECURITY_ADMIN_USER=admin
38
      - GF_SECURITY_ADMIN_PASSWORD=changeme123   # 部署后立即修改！
39
      - GF_USERS_ALLOW_SIGN_UP=false
40
      - GF_SERVER_DOMAIN=monitor.example.com
41
    volumes:
42
      - grafana_data:/var/lib/grafana
43
    networks: [monitoring]
44
    depends_on: [prometheus]
45

46
  alertmanager:
47
    image: prom/alertmanager:v0.27.0
48
    container_name: alertmanager
49
    restart: unless-stopped
50
    ports:
51
      - "9093:9093"
52
    volumes:
53
      - ./alertmanager/alertmanager.yml:/etc/alertmanager/alertmanager.yml:ro
54
      - alertmanager_data:/alertmanager
55
    command:
56
      - "--config.file=/etc/alertmanager/alertmanager.yml"
57
      - "--storage.path=/alertmanager"
58
    networks: [monitoring]
59

60
  node-exporter:
61
    image: prom/node-exporter:v1.8.1
62
    container_name: node-exporter
63
    restart: unless-stopped
64
    ports:
65
      - "9100:9100"
66
    volumes:
67
      - /proc:/host/proc:ro
68
      - /sys:/host/sys:ro
69
      - /:/rootfs:ro
70
    command:
71
      - "--path.procfs=/host/proc"
72
      - "--path.rootfs=/rootfs"
73
      - "--path.sysfs=/host/sys"
74
      - "--collector.filesystem.mount-points-exclude=^/(sys|proc|dev|host|etc)($$|/)"
75
    network_mode: host   # 直接使用宿主机网络，采集更准确
76

77
  cadvisor:
78
    image: gcr.io/cadvisor/cadvisor:v0.49.1
79
    container_name: cadvisor
80
    restart: unless-stopped
81
    ports:
82
      - "8080:8080"
83
    volumes:
84
      - /:/rootfs:ro
85
      - /var/run:/var/run:ro
86
      - /sys:/sys:ro
87
      - /var/lib/docker:/var/lib/docker:ro
88
    networks: [monitoring]
89

90
volumes:
91
  prometheus_data:
92
  grafana_data:
93
  alertmanager_data:
94

95
networks:
96
  monitoring:
97
    driver: bridge

1
global:
2
  scrape_interval:     15s    # 每15秒拉取一次指标
3
  evaluation_interval: 15s    # 每15秒评估一次告警规则
4
  scrape_timeout:      10s
5

6
# 告警规则文件
7
rule_files:
8
  - "rules/*.yml"
9

10
# Alertmanager 配置
11
alerting:
12
  alertmanagers:
13
    - static_configs:
14
        - targets: ["alertmanager:9093"]
15

16
# 采集目标
17
scrape_configs:
18
  # 监控 Prometheus 自身
19
  - job_name: "prometheus"
20
    static_configs:
21
      - targets: ["localhost:9090"]
22

23
  # 本机 Node Exporter
24
  - job_name: "node-local"
25
    static_configs:
26
      - targets: ["localhost:9100"]
27
        labels:
28
          instance: "monitor-server"
29
          env: "production"
30

31
  # 远程服务器（静态配置）
32
  - job_name: "node-remote"
33
    static_configs:
34
      - targets:
35
          - "192.168.1.10:9100"
36
          - "192.168.1.11:9100"
37
        labels:
38
          env: "production"
39
      - targets:
40
          - "192.168.2.10:9100"
41
        labels:
42
          env: "staging"
43

44
  # Docker 容器指标
45
  - job_name: "cadvisor"
46
    static_configs:
47
      - targets: ["cadvisor:8080"]
48

49
  # 应用服务（示例：FastAPI 暴露的 /metrics 端点）
50
  - job_name: "myapp"
51
    metrics_path: /metrics
52
    static_configs:
53
      - targets: ["myapp:8000"]
54
    relabel_configs:
55
      - source_labels: [__address__]
56
        target_label: instance

1
groups:
2
  - name: system.rules
3
    interval: 1m   # 覆盖全局 evaluation_interval
4
    rules:
5

6
      # ── CPU 告警 ────────────────────────────────────────────
7
      - alert: HighCPUUsage
8
        expr: |
9
          100 - (avg by (instance) (
10
            rate(node_cpu_seconds_total{mode="idle"}[5m])
11
          ) * 100) > 85
12
        for: 5m        # 持续5分钟才告警（避免尖刺误报）
13
        labels:
14
          severity: warning
15
        annotations:
16
          summary: "{{ $labels.instance }} CPU 使用率过高"
17
          description: "CPU 使用率 {{ $value | humanize }}%，已持续5分钟"
18

19
      - alert: CriticalCPUUsage
20
        expr: |
21
          100 - (avg by (instance) (
22
            rate(node_cpu_seconds_total{mode="idle"}[5m])
23
          ) * 100) > 95
24
        for: 2m
25
        labels:
26
          severity: critical
27
        annotations:
28
          summary: "🔴 {{ $labels.instance }} CPU 严重告警"
29
          description: "CPU 使用率 {{ $value | humanize }}%，需要立即处理"
30

31
      # ── 内存告警 ────────────────────────────────────────────
32
      - alert: HighMemoryUsage
33
        expr: |
34
          (node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes)
35
          / node_memory_MemTotal_bytes * 100 > 90
36
        for: 5m
37
        labels:
38
          severity: warning
39
        annotations:
40
          summary: "{{ $labels.instance }} 内存使用率过高"
41
          description: "内存使用率 {{ $value | humanize }}%"
42

43
      # ── 磁盘告警 ────────────────────────────────────────────
44
      - alert: DiskUsageHigh
45
        expr: |
46
          (node_filesystem_size_bytes{fstype!~"tmpfs|overlay"}
47
           - node_filesystem_avail_bytes{fstype!~"tmpfs|overlay"})
48
          / node_filesystem_size_bytes{fstype!~"tmpfs|overlay"} * 100 > 85
49
        for: 10m
50
        labels:
51
          severity: warning
52
        annotations:
53
          summary: "{{ $labels.instance }} 磁盘空间不足"
54
          description: "挂载点 {{ $labels.mountpoint }} 使用率 {{ $value | humanize }}%"
55

56
      - alert: DiskWillFillIn24h
57
        expr: |
58
          predict_linear(
59
            node_filesystem_avail_bytes{fstype!~"tmpfs|overlay"}[6h], 24 * 3600
60
          ) < 0
61
        for: 30m
62
        labels:
63
          severity: warning
64
        annotations:
65
          summary: "⚠️ {{ $labels.instance }} 磁盘预计24小时内耗尽"
66
          description: "按当前写入速率，{{ $labels.mountpoint }} 将在24小时内写满"
67

68
      # ── 主机存活 ────────────────────────────────────────────
69
      - alert: InstanceDown
70
        expr: up == 0
71
        for: 1m
72
        labels:
73
          severity: critical
74
        annotations:
75
          summary: "🔴 {{ $labels.instance }} 已下线"
76
          description: "Prometheus 无法连接到 {{ $labels.instance }}"
77

78
      # ── 系统负载 ────────────────────────────────────────────
79
      - alert: HighSystemLoad
80
        expr: node_load5 > count by (instance) (node_cpu_seconds_total{mode="idle"}) * 2
81
        for: 10m
82
        labels:
83
          severity: warning
84
        annotations:
85
          summary: "{{ $labels.instance }} 系统负载过高"
86
          description: "5分钟负载 {{ $value | humanize }}（CPU 核数 2倍以上）"

1
global:
2
  resolve_timeout: 5m
3
  # SMTP 配置（全局）
4
  smtp_smarthost: "smtp.example.com:587"
5
  smtp_from: "alertmanager@example.com"
6
  smtp_auth_username: "alertmanager@example.com"
7
  smtp_auth_password: "email_password"
8

9
# 消息模板
10
templates:
11
  - "/etc/alertmanager/templates/*.tmpl"
12

13
# 路由树
14
route:
15
  receiver: "default"      # 默认接收者
16
  group_by: ["alertname", "instance"]
17
  group_wait:      30s     # 等待同组告警聚合
18
  group_interval:  5m      # 同组告警再次发送间隔
19
  repeat_interval: 4h      # 相同告警重复发送间隔
20

21
  routes:
22
    # critical 告警发给紧急渠道
23
    - matchers:
24
        - severity = "critical"
25
      receiver: "wechat-critical"
26
      repeat_interval: 30m   # 严重告警每30分钟重复
27

28
    # warning 告警发给普通渠道
29
    - matchers:
30
        - severity = "warning"
31
      receiver: "telegram-warning"
32

33
# 接收者定义
34
receivers:
35
  - name: "default"
36
    email_configs:
37
      - to: "ops-team@example.com"
38
        send_resolved: true
39

40
  # 企业微信机器人
41
  - name: "wechat-critical"
42
    webhook_configs:
43
      - url: "http://wechat-webhook-adapter:5000/send"
44
        send_resolved: true
45

46
  # Telegram
47
  - name: "telegram-warning"
48
    telegram_configs:
49
      - bot_token: "your_telegram_bot_token"
50
        chat_id: -1001234567890   # 群组 chat_id（负数）
51
        message: |
52
          {{ range .Alerts }}
53
          *{{ .Labels.alertname }}* {{ if eq .Status "resolved" }}✅ 已恢复{{ else }}🔴 触发{{ end }}
54
          *实例*: {{ .Labels.instance }}
55
          *详情*: {{ .Annotations.description }}
56
          {{ end }}
57
        send_resolved: true
58

59
# 抑制规则（critical 触发时，抑制同实例的 warning）
60
inhibit_rules:
61
  - source_matchers:
62
      - severity = "critical"
63
    target_matchers:
64
      - severity = "warning"
65
    equal: ["instance"]