PromQL高级查询 #

# 子查询语法
<表达式>[<范围>:<分辨率>]

# 查询过去1小时内每分钟的最大值
max_over_time(http_requests_total[1h:1m])

# 查询过去1小时内每分钟的最小值
min_over_time(http_requests_total[1h:1m])

# 查询过去1小时内每分钟的平均值
avg_over_time(http_requests_total[1h:1m])

# 计算过去1小时内速率的最大值
max_over_time(rate(http_requests_total[5m])[1h:])

# 计算过去1小时内P99延迟的最大值
max_over_time(
  histogram_quantile(0.99, rate(http_request_duration_seconds_bucket[5m]))[1h:]
)

# 查询特定时间点的数据
http_requests_total @ 1700000000

# 查询特定时间点的范围数据
http_requests_total[5m] @ 1700000000

# 使用start()和end()
http_requests_total @ start()
http_requests_total @ end()

# 子查询中使用@
rate(http_requests_total[5m])[1h:] @ start()

# 计算同比
http_requests_total 
/ 
(http_requests_total @ (time() - 86400))

# 多条件正则匹配
http_requests_total{
    method=~"GET|POST",
    status=~"2..|3..",
    endpoint!~"/health|/metrics"
}

# 使用标签名正则匹配
{__name__=~"http_.*", job="api-server"}

# 排除多个标签值
http_requests_total{status!~"4..|5.."}

# 复杂组合
http_requests_total{
    method="GET",
    status=~"2.."
} or http_requests_total{
    method="POST",
    status="201"
}

# 服务可用性（SLA）
# 定义：成功请求 / 总请求 * 100

sum(rate(http_requests_total{status!~"5.."}[5m])) 
/ 
sum(rate(http_requests_total[5m])) * 100

# 按服务分组计算可用性
sum by (service) (rate(http_requests_total{status!~"5.."}[5m])) 
/ 
sum by (service) (rate(http_requests_total[5m])) * 100

# 计算过去30天的可用性
avg_over_time(
  sum(rate(http_requests_total{status!~"5.."}[5m])) 
  / 
  sum(rate(http_requests_total[5m])) * 100
[30d:5m])

# 错误预算
# 定义：允许的错误请求数 = 总请求 * (1 - SLA目标)

# 假设SLA目标为99.9%
sum(increase(http_requests_total[30d])) * (1 - 0.999)

# 已消耗的错误预算
sum(increase(http_requests_total{status=~"5.."}[30d]))

# 剩余错误预算
(
  sum(increase(http_requests_total[30d])) * (1 - 0.999) 
  - 
  sum(increase(http_requests_total{status=~"5.."}[30d]))
)

# 错误预算消耗率
sum(increase(http_requests_total{status=~"5.."}[30d])) 
/ 
(sum(increase(http_requests_total[30d])) * (1 - 0.999)) * 100

# 预测磁盘何时满
predict_linear(node_filesystem_avail_bytes{mountpoint="/"}[1h], 3600 * 24 * 7)

# 预测内存何时耗尽
predict_linear(node_memory_MemAvailable_bytes[1h], 3600 * 24)

# 计算存储增长速率
deriv(node_filesystem_size_bytes{mountpoint="/"}[1d])

# 预测请求增长
predict_linear(http_requests_total[1h], 3600 * 24 * 30)

# 基于标准差的异常检测
# 超过平均值+3倍标准差视为异常

http_requests_total 
> 
(
  avg_over_time(http_requests_total[1h]) 
  + 
  3 * stddev_over_time(http_requests_total[1h])
)

# 基于历史数据对比
# 当前值超过历史平均值2倍
rate(http_requests_total[5m]) 
> 
2 * avg_over_time(rate(http_requests_total[5m])[1d:5m])

# 检测突增
(
  rate(http_requests_total[5m]) 
  - 
  rate(http_requests_total[5m] offset 1h)
) 
/ 
rate(http_requests_total[5m] offset 1h) > 0.5

查询优化原则：

┌─────────────────────────────────────────────┐
│ 1. 减少时间序列数量                        │
├─────────────────────────────────────────────┤
│ • 使用标签过滤                              │
│ • 避免高基数标签                            │
│ • 使用聚合减少序列数                        │
└─────────────────────────────────────────────┘

┌─────────────────────────────────────────────┐
│ 2. 减少数据量                              │
├─────────────────────────────────────────────┤
│ • 缩短查询时间范围                          │
│ • 使用合理的step值                          │
│ • 避免不必要的范围查询                      │
└─────────────────────────────────────────────┘

┌─────────────────────────────────────────────┐
│ 3. 使用高效函数                            │
├─────────────────────────────────────────────┤
│ • rate()比increase()更高效                 │
│ • 避免嵌套复杂函数                          │
│ • 使用内置函数代替复杂计算                  │
└─────────────────────────────────────────────┘

# 不推荐：查询所有指标再过滤
sum(rate(http_requests_total[5m])) by (method)

# 推荐：先过滤再聚合
sum by (method) (rate(http_requests_total{job="api-server"}[5m]))

# 不推荐：大范围查询
rate(http_requests_total[30d])

# 推荐：合理范围
rate(http_requests_total[5m])

# 不推荐：嵌套复杂函数
histogram_quantile(0.99, 
  sum by (le) (
    rate(http_request_duration_seconds_bucket[5m])
  )
)

# 推荐：简化查询
histogram_quantile(0.99, 
  sum by (le) (rate(http_request_duration_seconds_bucket[5m]))
)

# 不推荐：多次查询同一指标
sum(rate(http_requests_total[5m])) + 
sum(rate(http_requests_total[5m])) * 2

# 推荐：使用变量
sum(rate(http_requests_total[5m])) * 3

# recording_rules.yml

groups:
  - name: http_requests
    interval: 30s
    rules:
      # 预计算QPS
      - record: job:http_requests:rate5m
        expr: sum by (job) (rate(http_requests_total[5m]))
      
      # 预计算错误率
      - record: job:http_errors:rate5m
        expr: sum by (job) (rate(http_requests_total{status=~"5.."}[5m]))
      
      # 预计算可用性
      - record: job:availability:ratio5m
        expr: |
          sum by (job) (rate(http_requests_total{status!~"5.."}[5m]))
          /
          sum by (job) (rate(http_requests_total[5m]))
      
      # 预计算P99延迟
      - record: job:http_request_duration:p99_5m
        expr: |
          histogram_quantile(0.99, 
            sum by (job, le) (rate(http_request_duration_seconds_bucket[5m]))
          )

# 7天趋势
rate(http_requests_total[5m]) 
/ 
rate(http_requests_total[5m] offset 7d)

# 环比增长
(
  sum(increase(http_requests_total[1d])) 
  - 
  sum(increase(http_requests_total[1d] offset 1d))
) 
/ 
sum(increase(http_requests_total[1d] offset 1d)) * 100

# 同比增长
(
  sum(increase(http_requests_total[1d])) 
  - 
  sum(increase(http_requests_total[1d] offset 7d]))
) 
/ 
sum(increase(http_requests_total[1d] offset 7d])) * 100

# 移动平均
avg_over_time(rate(http_requests_total[5m])[1h:])

# CPU容量
# 已使用CPU核心数
sum(rate(node_cpu_seconds_total{mode!="idle"}[5m])) by (instance)

# 总CPU核心数
count(node_cpu_seconds_total{mode="idle"}) by (instance)

# CPU使用率
sum(rate(node_cpu_seconds_total{mode!="idle"}[5m])) by (instance) 
/ 
count(node_cpu_seconds_total{mode="idle"}) by (instance) * 100

# 内存容量
# 已使用内存
sum(node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes) by (instance)

# 总内存
sum(node_memory_MemTotal_bytes) by (instance)

# 内存使用率
(node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes) 
/ 
node_memory_MemTotal_bytes * 100

# 响应时间分析
# 平均响应时间
rate(http_request_duration_seconds_sum[5m]) 
/ 
rate(http_request_duration_seconds_count[5m])

# P50响应时间
histogram_quantile(0.5, rate(http_request_duration_seconds_bucket[5m]))

# P90响应时间
histogram_quantile(0.9, rate(http_request_duration_seconds_bucket[5m]))

# P99响应时间
histogram_quantile(0.99, rate(http_request_duration_seconds_bucket[5m]))

# 响应时间分布
sum by (le) (rate(http_request_duration_seconds_bucket[5m]))

# 错误率分析
# 总错误率
sum(rate(http_requests_total{status=~"5.."}[5m])) 
/ 
sum(rate(http_requests_total[5m])) * 100

# 按状态码分组
sum by (status) (rate(http_requests_total{status=~"5.."}[5m]))

# 按服务分组
sum by (service) (rate(http_requests_total{status=~"5.."}[5m])) 
/ 
sum by (service) (rate(http_requests_total[5m])) * 100

# 错误趋势
sum(rate(http_requests_total{status=~"5.."}[5m])) 
/ 
sum(rate(http_requests_total[5m] offset 1h)) * 100

Grafana变量语法：

# 简单变量
$variable_name

# 标签值变量
label_values(http_requests_total, method)

# 查询变量
query_result(sum by (service) (rate(http_requests_total[5m])))

# 使用变量过滤
http_requests_total{method="$method"}
http_requests_total{service=~"$service"}

# 单值面板
sum(rate(http_requests_total[5m]))

# 时间序列面板
rate(http_requests_total[5m])

# 饼图
sum by (status) (rate(http_requests_total[5m]))

# 表格
sum by (instance, method) (rate(http_requests_total[5m]))

# 热力图
sum by (le) (rate(http_request_duration_seconds_bucket[5m]))

# 状态面板
up{job="node-exporter"}

# 使用有意义的变量名
# 不推荐
a = rate(http_requests_total[5m])

# 推荐
request_rate = rate(http_requests_total[5m])

# 使用注释说明复杂查询
# 计算过去5分钟的HTTP请求错误率
sum(rate(http_requests_total{status=~"5.."}[5m])) 
/ 
sum(rate(http_requests_total[5m])) * 100

# CPU使用率模板
100 - (avg by(instance) (irate(node_cpu_seconds_total{mode="idle"}[5m])) * 100)

# 内存使用率模板
(1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) * 100

# 磁盘使用率模板
(1 - (node_filesystem_avail_bytes{fstype!~"tmpfs|overlay"} / node_filesystem_size_bytes{fstype!~"tmpfs|overlay"})) * 100

# 网络流量模板
irate(node_network_receive_bytes_total{device!~"lo|veth.*"}[5m])
irate(node_network_transmit_bytes_total{device!~"lo|veth.*"}[5m])

# 错误率模板
sum(rate(http_requests_total{status=~"5.."}[5m])) / sum(rate(http_requests_total[5m])) * 100