Spanner监控告警 #

Spanner监控架构
├── Cloud Monitoring
│   ├── 指标收集
│   ├── 仪表盘
│   ├── 告警策略
│   └── 日志集成
│
├── 系统表
│   ├── 查询统计
│   ├── 表统计
│   └── 索引统计
│
└── 客户端监控
    ├── 延迟追踪
    ├── 错误追踪
    └── 自定义指标

监控指标分类:
├── 性能指标
│   ├── 延迟
│   ├── 吞吐量
│   └── 并发数
│
├── 资源指标
│   ├── CPU使用率
│   ├── 存储使用量
│   └── 会话使用率
│
├── 可用性指标
│   ├── 错误率
│   ├── 可用性
│   └── 重试次数
│
└── 业务指标
    ├── 事务数
    ├── 查询数
    └── 数据量

# 启用Cloud Monitoring API
gcloud services enable monitoring.googleapis.com

# 创建监控工作区
gcloud monitoring workspaces create my-workspace

# 使用gcloud创建仪表盘
gcloud monitoring dashboards create \
    --config-from-file=dashboard.json

{
  "displayName": "Spanner Dashboard",
  "gridLayout": {
    "widgets": [
      {
        "title": "CPU Utilization",
        "xyChart": {
          "dataSets": [{
            "timeSeriesQuery": {
              "timeSeriesFilter": {
                "filter": "resource.type=\"spanner_instance\" AND metric.type=\"spanner.googleapis.com/instance/cpu/utilization\""
              }
            }
          }]
        }
      }
    ]
  }
}

# 使用gcloud查看指标
gcloud monitoring time-series list \
    --filter='resource.type="spanner_instance"' \
    --filter='metric.type="spanner.googleapis.com/instance/cpu/utilization"'

# 创建CPU告警
gcloud alpha monitoring policies create \
    --display-name="Spanner CPU High" \
    --notification-channels=projects/my-project/notificationChannels/channel-id \
    --condition-display-name="CPU > 80%" \
    --condition-filter='resource.type="spanner_instance" AND metric.type="spanner.googleapis.com/instance/cpu/utilization"' \
    --condition-threshold-value=0.8 \
    --condition-threshold-duration=300s

# 创建邮件通知渠道
gcloud beta monitoring channels create \
    --display-name="Email Alert" \
    --type=email \
    --channel-labels=email_address=admin@example.com

# 创建Slack通知渠道
gcloud beta monitoring channels create \
    --display-name="Slack Alert" \
    --type=slack \
    --channel-labels=channel_name=#alerts

# 延迟告警
gcloud alpha monitoring policies create \
    --display-name="Spanner High Latency" \
    --condition-display-name="P99 Latency > 100ms" \
    --condition-filter='resource.type="spanner_instance" AND metric.type="spanner.googleapis.com/instance/latency"' \
    --condition-threshold-value=0.1

# 存储告警
gcloud alpha monitoring policies create \
    --display-name="Spanner Storage High" \
    --condition-display-name="Storage > 90%" \
    --condition-filter='resource.type="spanner_instance" AND metric.type="spanner.googleapis.com/instance/storage/used"' \
    --condition-threshold-value=0.9

# 错误率告警
gcloud alpha monitoring policies create \
    --display-name="Spanner Error Rate High" \
    --condition-display-name="Error Rate > 1%" \
    --condition-filter='resource.type="spanner_instance" AND metric.type="spanner.googleapis.com/instance/api/request_count"' \
    --condition-threshold-value=0.01

-- 查看最慢的查询
SELECT 
    text,
    execution_count,
    avg_latency_seconds,
    avg_rows,
    avg_bytes
FROM INFORMATION_SCHEMA.QUERY_STATS_TOP_HOUR
ORDER BY avg_latency_seconds DESC
LIMIT 10;

-- 查看最频繁的查询
SELECT 
    text,
    execution_count,
    avg_latency_seconds
FROM INFORMATION_SCHEMA.QUERY_STATS_TOP_HOUR
ORDER BY execution_count DESC
LIMIT 10;

-- 查看表大小
SELECT 
    table_name,
    row_count,
    size_bytes
FROM INFORMATION_SCHEMA.TABLE_STATS
ORDER BY size_bytes DESC;

-- 查看列统计
SELECT 
    table_name,
    column_name,
    total_rows,
    distinct_rows
FROM INFORMATION_SCHEMA.COLUMN_STATS;

-- 查看索引使用情况
SELECT 
    table_name,
    index_name,
    row_count,
    size_bytes
FROM INFORMATION_SCHEMA.INDEX_STATS
ORDER BY size_bytes DESC;

# 启用Cloud Logging
gcloud services enable logging.googleapis.com

# 查看Spanner日志
gcloud logging read "resource.type=spanner_instance" \
    --project=my-project

# 查询错误日志
gcloud logging read "resource.type=spanner_instance AND severity>=ERROR"

# 查询特定时间范围的日志
gcloud logging read "resource.type=spanner_instance AND timestamp>=\"2024-03-27T00:00:00Z\""

# 创建基于日志的告警
gcloud alpha logging sinks create spanner-errors \
    pubsub.googleapis.com/projects/my-project/topics/spanner-errors \
    --log-filter="resource.type=spanner_instance AND severity>=ERROR"

监控策略建议:
├── 监控关键指标
├── 设置合理告警阈值
├── 配置多级告警
├── 定期审查告警
└── 优化告警策略

告警策略建议:
├── 分级告警(紧急/警告/信息)
├── 避免告警风暴
├── 设置合理的静默时间
├── 配置多个通知渠道
└── 定期测试告警

仪表盘建议:
├── 包含关键指标
├── 分组展示相关指标
├── 添加注释说明
├── 定期更新优化
└── 分享给相关人员

1. 监控关键指标
   └── CPU、延迟、存储

2. 配置合理告警
   └── 阈值和通知渠道

3. 使用系统表分析
   └── 查询和表统计

4. 定期审查优化
   └── 更新监控策略

5. 建立应急响应
   └── 告警处理流程