Elasticsearch监控告警 #

监控层次
├── 集群监控
│   ├── 集群状态
│   ├── 节点状态
│   └── 分片状态
├── 节点监控
│   ├── 资源使用
│   ├── JVM状态
│   └── 线程池
├── 索引监控
│   ├── 文档数量
│   ├── 存储大小
│   └── 操作统计
└── 查询监控
    ├── 查询延迟
    ├── 查询吞吐
    └── 慢查询

xpack.monitoring.enabled: true
xpack.monitoring.collection.enabled: true

monitoring.ui.container.elasticsearch.enabled: true
elasticsearch.hosts: ["http://localhost:9200"]
elasticsearch.username: "kibana_system"
elasticsearch.password: "password"

Kibana监控指标
├── 集群概览
│   ├── 状态
│   ├── 节点数
│   └── 分片数
├── 节点指标
│   ├── CPU使用率
│   ├── 内存使用率
│   ├── 磁盘使用率
│   └── JVM堆使用率
├── 索引指标
│   ├── 索引速率
│   ├── 搜索速率
│   └── 文档数
└── 性能指标
    ├── 查询延迟
    └── 索引延迟

docker run -d \
  --name elasticsearch_exporter \
  -p 9114:9114 \
  -e ES_URI=http://localhost:9200 \
  quay.io/prometheuscommunity/elasticsearch-exporter:latest

scrape_configs:
  - job_name: 'elasticsearch'
    static_configs:
      - targets: ['localhost:9114']

{
  "rule_type_id": ".es-query",
  "name": "Cluster Health Alert",
  "params": {
    "index": ".monitoring-es-*",
    "query": {
      "bool": {
        "filter": [
          { "term": { "cluster_stats.status": "red" } }
        ]
      }
    }
  }
}

groups:
  - name: elasticsearch
    rules:
      - alert: ElasticsearchClusterRed
        expr: elasticsearch_cluster_health_status{color="red"} == 1
        for: 1m
        labels:
          severity: critical
        annotations:
          summary: "Elasticsearch cluster is red"
          
      - alert: ElasticsearchClusterYellow
        expr: elasticsearch_cluster_health_status{color="yellow"} == 1
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "Elasticsearch cluster is yellow"
          
      - alert: ElasticsearchHeapTooHigh
        expr: elasticsearch_jvm_memory_used_bytes{area="heap"} / elasticsearch_jvm_memory_max_bytes{area="heap"} > 0.85
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "Elasticsearch heap usage is high"
          
      - alert: ElasticsearchDiskSpaceLow
        expr: elasticsearch_fs_total_available_in_bytes / elasticsearch_fs_total_total_in_bytes < 0.15
        for: 5m
        labels:
          severity: critical
        annotations:
          summary: "Elasticsearch disk space is low"

PUT /_watcher/watch/cluster_health
{
  "trigger": {
    "schedule": {
      "interval": "1m"
    }
  },
  "input": {
    "http": {
      "request": {
        "host": "localhost",
        "port": 9200,
        "path": "/_cluster/health"
      }
    }
  },
  "condition": {
    "compare": {
      "ctx.payload.status": {
        "not_eq": "green"
      }
    }
  },
  "actions": {
    "email_admin": {
      "email": {
        "to": "admin@example.com",
        "subject": "Elasticsearch Cluster Alert",
        "body": "Cluster status is {{ctx.payload.status}}"
      }
    }
  }
}

path.logs: /var/log/elasticsearch

logger.level: info

filebeat.inputs:
  - type: log
    paths:
      - /var/log/elasticsearch/*.log
    fields:
      type: elasticsearch

output.elasticsearch:
  hosts: ["localhost:9200"]
  index: "elasticsearch-logs-%{+yyyy.MM.dd}"

监控策略
├── 分层监控
│   ├── 集群层
│   ├── 节点层
│   └── 应用层
├── 告警分级
│   ├── 严重：立即处理
│   ├── 警告：关注处理
│   └── 信息：记录观察
└── 定期审查
    └── 定期检查告警规则

告警最佳实践
├── 避免告警风暴
│   └── 合理设置阈值和持续时间
├── 告警可操作
│   └── 提供处理建议
├── 告警分级
│   └── 不同级别不同处理
└── 定期测试
    └── 确保告警有效

仪表盘建议
├── 集群概览
│   ├── 状态、节点数、分片数
│   └── 存储使用、查询吞吐
├── 节点详情
│   ├── CPU、内存、磁盘
│   └── JVM、线程池
├── 索引详情
│   ├── 文档数、存储大小
│   └── 操作统计
└── 性能趋势
    ├── 查询延迟趋势
    └── 索引速率趋势

GET /_cluster/health

GET /_cluster/allocation/explain

GET /_nodes/hot_threads

GET /_cat/pending_tasks?v