# 启动 vLLM 时启用指标（默认端口 8000 上的 /metrics 端点）
vllm serve Qwen/Qwen2.5-7B-Instruct \
    --served-model-name qwen-7b

# 指标端点自动可用
curl http://localhost:8000/metrics

# 查看所有指标
curl -s http://localhost:8000/metrics | head -100

# 过滤特定指标
curl -s http://localhost:8000/metrics | grep "vllm:num_requests"

# 格式化输出
curl -s http://localhost:8000/metrics | grep "vllm:gpu_cache"

# prometheus.yml
global:
  scrape_interval: 15s
  evaluation_interval: 15s

scrape_configs:
  - job_name: 'vllm'
    static_configs:
      - targets: ['vllm-server:8000']
    metrics_path: /metrics
    scrape_interval: 5s  # vLLM 建议更频繁的采集
    
  # 可选：GPU 指标（通过 DCGM Exporter）
  - job_name: 'nvidia-dcgm'
    static_configs:
      - targets: ['dcgm-exporter:9400']

# 启动 DCGM Exporter（NVIDIA 官方 GPU 监控工具）
docker run -d \
    --gpus all \
    --name dcgm-exporter \
    -p 9400:9400 \
    nvcr.io/nvidia/k8s/dcgm-exporter:3.3.8-3.6.0-ubuntu22.04

# 吞吐量（每秒生成的 token 数）
vllm:avg_generation_throughput_toks_per_s{model="qwen-7b"}

# 请求数
vllm:num_requests_running{model="qwen-7b"}
vllm:num_requests_waiting{model="qwen-7b"}

# GPU 缓存使用率
vllm:gpu_cache_usage_perc{model="qwen-7b"}

# 请求成功率
rate(vllm:request_success{model="qwen-7b"}[5m]) / 
(rate(vllm:request_success{model="qwen-7b"}[5m]) + rate(vllm:request_failure{model="qwen-7b"}[5m]))

# P99 推理延迟
histogram_quantile(0.99, rate(vllm:request_inference_time_seconds_bucket{model="qwen-7b"}[5m]))

# 平均队列等待时间
rate(vllm:request_queue_time_seconds_sum{model="qwen-7b"}[5m]) / 
rate(vllm:request_queue_time_seconds_count{model="qwen-7b"}[5m])

# GPU 利用率（DCGM）
DCGM_FI_DEV_GPU_UTIL{gpu="0"}

# GPU 温度
DCGM_FI_DEV_GPU_TEMP{gpu="0"}

# 设置日志级别
export VLLM_LOGGING_LEVEL=INFO  # DEBUG, INFO, WARNING, ERROR

# 启动时的配置
vllm serve model \
    --disable-log-requests  # 禁用逐请求日志（高并发时减少日志量）

# 自定义日志处理
import logging
import json
from datetime import datetime

class JSONFormatter(logging.Formatter):
    def format(self, record):
        log_data = {
            "timestamp": datetime.utcnow().isoformat(),
            "level": record.levelname,
            "message": record.getMessage(),
            "module": record.module,
        }
        return json.dumps(log_data, ensure_ascii=False)

# 配置
handler = logging.StreamHandler()
handler.setFormatter(JSONFormatter())
logger = logging.getLogger("vllm")
logger.addHandler(handler)

# vllm_alerts.yml
groups:
  - name: vllm_alerts
    rules:
      # GPU 缓存使用率过高
      - alert: VLLMHighCacheUsage
        expr: vllm:gpu_cache_usage_perc > 0.95
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "vLLM GPU 缓存使用率超过 95%"
          description: "模型 {{ $labels.model }} 的 GPU 缓存使用率已达 {{ $value | humanizePercentage }}"

      # 等待队列过长
      - alert: VLLMHighQueueLength
        expr: vllm:num_requests_waiting > 500
        for: 2m
        labels:
          severity: critical
        annotations:
          summary: "vLLM 等待队列过长"
          description: "模型 {{ $labels.model }} 有 {{ $value }} 个请求等待中"

      # 吞吐量下降
      - alert: VLLMLowThroughput
        expr: vllm:avg_generation_throughput_toks_per_s < 100
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "vLLM 吞吐量过低"
          description: "模型 {{ $labels.model }} 吞吐量仅 {{ $value }} tokens/s"

      # 请求失败率高
      - alert: VLLMHighFailureRate
        expr: rate(vllm:request_failure[5m]) / (rate(vllm:request_success[5m]) + rate(vllm:request_failure[5m])) > 0.05
        for: 3m
        labels:
          severity: critical
        annotations:
          summary: "vLLM 请求失败率超过 5%"

      # GPU 温度过高
      - alert: GPUHighTemperature
        expr: DCGM_FI_DEV_GPU_TEMP > 85
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "GPU 温度过高: {{ $value }}°C"

      # 推理延迟过高
      - alert: VLLMHighLatency
        expr: histogram_quantile(0.99, rate(vllm:request_inference_time_seconds_bucket[5m])) > 10
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "vLLM P99 推理延迟超过 10 秒"

# alertmanager.yml
route:
  group_by: ['alertname', 'severity']
  group_wait: 30s
  group_interval: 5m
  repeat_interval: 4h
  receiver: 'default'
  routes:
    - match:
        severity: critical
      receiver: 'pager'

receivers:
  - name: 'default'
    webhook_configs:
      - url: 'http://dingtalk-webhook:8060/dingtalk/ops/send'
  
  - name: 'pager'
    webhook_configs:
      - url: 'http://pagerduty-webhook/alert'

# custom_metrics.py
"""自定义应用层监控指标"""

from prometheus_client import Counter, Histogram, Gauge, start_http_server
import time

# 定义自定义指标
request_counter = Counter(
    'app_llm_requests_total',
    'Total LLM requests',
    ['endpoint', 'model', 'status']
)

request_latency = Histogram(
    'app_llm_request_latency_seconds',
    'LLM request latency',
    ['endpoint', 'model'],
    buckets=[0.1, 0.5, 1.0, 2.0, 5.0, 10.0, 30.0, 60.0]
)

active_users = Gauge(
    'app_active_users',
    'Number of active users'
)

# 中间件包装
def monitor_request(func):
    def wrapper(*args, **kwargs):
        start = time.time()
        try:
            result = func(*args, **kwargs)
            request_counter.labels(
                endpoint="chat",
                model="qwen-7b",
                status="success"
            ).inc()
            return result
        except Exception as e:
            request_counter.labels(
                endpoint="chat",
                model="qwen-7b",
                status="error"
            ).inc()
            raise
        finally:
            request_latency.labels(
                endpoint="chat",
                model="qwen-7b"
            ).observe(time.time() - start)
    return wrapper

# 启动指标服务器
start_http_server(9090)

# docker-compose.monitoring.yml
version: '3.8'

services:
  prometheus:
    image: prom/prometheus:latest
    ports:
      - "9090:9090"
    volumes:
      - ./prometheus.yml:/etc/prometheus/prometheus.yml
      - ./vllm_alerts.yml:/etc/prometheus/vllm_alerts.yml
    command:
      - '--config.file=/etc/prometheus/prometheus.yml'
      - '--storage.tsdb.retention.time=30d'

  grafana:
    image: grafana/grafana:latest
    ports:
      - "3000:3000"
    environment:
      - GF_SECURITY_ADMIN_PASSWORD=admin
    volumes:
      - grafana-data:/var/lib/grafana
      - ./grafana/dashboards:/etc/grafana/provisioning/dashboards
      - ./grafana/datasources:/etc/grafana/provisioning/datasources

  alertmanager:
    image: prom/alertmanager:latest
    ports:
      - "9093:9093"
    volumes:
      - ./alertmanager.yml:/etc/alertmanager/alertmanager.yml

  dcgm-exporter:
    image: nvcr.io/nvidia/k8s/dcgm-exporter:3.3.8-3.6.0-ubuntu22.04
    ports:
      - "9400:9400"
    deploy:
      resources:
        reservations:
          devices:
            - driver: nvidia
              count: all
              capabilities: [gpu]

volumes:
  grafana-data: