# 下载
ALERTMANAGER_VERSION="0.27.0"
wget https://github.com/prometheus/alertmanager/releases/download/v${ALERTMANAGER_VERSION}/alertmanager-${ALERTMANAGER_VERSION}.linux-amd64.tar.gz

# 解压安装
tar xvfz alertmanager-${ALERTMANAGER_VERSION}.linux-amd64.tar.gz
sudo cp alertmanager-${ALERTMANAGER_VERSION}.linux-amd64/alertmanager /usr/local/bin/
sudo cp alertmanager-${ALERTMANAGER_VERSION}.linux-amd64/amtool /usr/local/bin/

# Docker
docker run -d \
  --name=alertmanager \
  -p 9093:9093 \
  -v $(pwd)/alertmanager.yml:/etc/alertmanager/alertmanager.yml \
  prom/alertmanager:v0.27.0

# /etc/systemd/system/alertmanager.service
[Unit]
Description=Alertmanager
After=network.target

[Service]
Type=simple
User=prometheus
ExecStart=/usr/local/bin/alertmanager \
  --config.file=/etc/alertmanager/alertmanager.yml \
  --storage.path=/var/lib/alertmanager/ \
  --web.listen-address=0.0.0.0:9093 \
  --cluster.listen-address=0.0.0.0:9094
Restart=always

[Install]
WantedBy=multi-user.target

# /etc/alertmanager/alertmanager.yml
global:
  resolve_timeout: 5m
  smtp_from: '[email protected]'
  smtp_smarthost: 'smtp.example.com:587'
  smtp_auth_username: '[email protected]'
  smtp_auth_password: 'password'
  smtp_require_tls: true
  slack_api_url: 'https://hooks.slack.com/services/xxx/yyy/zzz'

# 模板
templates:
  - '/etc/alertmanager/templates/*.tmpl'

# 路由
route:
  receiver: 'default-receiver'
  group_by: ['alertname', 'cluster', 'service']
  group_wait: 30s
  group_interval: 5m
  repeat_interval: 4h
  
  routes:
    - match:
        severity: 'critical'
      receiver: 'critical-receiver'
      group_wait: 10s
      
    - match:
        severity: 'warning'
      receiver: 'warning-receiver'
      
    - match_re:
        service: '^(api|web)$'
      receiver: 'service-receiver'
      
    - match:
        severity: 'info'
      receiver: 'info-receiver'
      repeat_interval: 24h

# 抑制
inhibit_rules:
  - source_match:
      severity: 'critical'
    target_match:
      severity: 'warning'
    equal: ['alertname', 'cluster', 'service']

# 接收器
receivers:
  - name: 'default-receiver'
    email_configs:
      - to: '[email protected]'
        
  - name: 'critical-receiver'
    pagerduty_configs:
      - routing_key: 'xxx'
    email_configs:
      - to: '[email protected]'
        
  - name: 'warning-receiver'
    slack_configs:
      - channel: '#alerts-warning'
        title: '{{ template "slack.title" . }}'
        text: '{{ template "slack.text" . }}'
        
  - name: 'info-receiver'
    webhook_configs:
      - url: 'http://bot:8080/alert'
        
  - name: 'service-receiver'
    webhook_configs:
      - url: 'http://webhook:9095/alert'

inhibit_rules:
  # 当 critical 级别告警触发时，抑制同名的 warning 告警
  - source_match:
      severity: 'critical'
    target_match:
      severity: 'warning'
    equal: ['alertname', 'cluster', 'service']

  # 当节点宕机时，抑制该节点的所有其他告警
  - source_match:
      alertname: 'InstanceDown'
    target_match_re:
      alertname: '.+'
    equal: ['instance']

# 创建静默（2小时）
amtool silence add alertname="InstanceDown" instance="node1:9090" \
  --duration="2h" \
  --comment="Planned maintenance"

# 创建静默（特定时间范围）
amtool silence add alertname="HighCPU" \
  --start="2024-01-01T22:00:00Z" \
  --end="2024-01-02T06:00:00Z" \
  --comment="Scheduled maintenance window"

# 查看所有静默
amtool silence query

# 删除静默
amtool silence expire <silence-id>

# 查询当前活跃告警
amtool alert query

receivers:
  - name: 'email'
    email_configs:
      - to: '[email protected]'
        from: '[email protected]'
        smarthost: 'smtp.example.com:587'
        auth_username: 'user'
        auth_password: 'pass'
        headers:
          subject: '[{{ .Status | toUpper }}] {{ .CommonLabels.alertname }}'
        html: '{{ template "email.html" . }}'
        send_resolved: true

receivers:
  - name: 'slack'
    slack_configs:
      - channel: '#alerts'
        title: '{{ .CommonLabels.alertname }}'
        text: >-
          {{ range .Alerts }}
          *Alert:* {{ .Labels.alertname }}
          *Severity:* {{ .Labels.severity }}
          *Description:* {{ .Annotations.description }}
          *Instance:* {{ .Labels.instance }}
          {{ end }}
        send_resolved: true

receivers:
  - name: 'dingtalk'
    webhook_configs:
      - url: 'http://dingtalk-webhook:8060/dingtalk/ops/send'
        send_resolved: true

receivers:
  - name: 'pagerduty'
    pagerduty_configs:
      - routing_key: '<integration-key>'
        severity: '{{ .GroupLabels.severity }}'
        description: '{{ .CommonAnnotations.summary }}'

# 启动集群节点
alertmanager \
  --config.file=/etc/alertmanager/alertmanager.yml \
  --cluster.peer=alertmanager1:9094 \
  --cluster.peer=alertmanager2:9094 \
  --cluster.listen-address=0.0.0.0:9094