监控告警 #

# 获取所有指标
curl "http://localhost:8983/solr/admin/metrics"

# 获取特定指标
curl "http://localhost:8983/solr/admin/metrics?key=solr.jvm.memory.*"

# 获取Core指标
curl "http://localhost:8983/solr/mycore/admin/metrics"

# 获取集群状态
curl "http://localhost:8983/solr/admin/collections?action=CLUSTERSTATUS"

# 获取节点状态
curl "http://localhost:8983/solr/admin/collections?action=CLUSTERSTATUS&collection=mycollection"

# Ping检查
curl "http://localhost:8983/solr/mycore/admin/ping"

# 集群健康检查
curl "http://localhost:8983/solr/admin/collections?action=CLUSTERHEALTH"

{
  "responseHeader": {
    "status": 0
  },
  "cluster": {
    "collections": {
      "mycollection": {
        "shards": {
          "shard1": {
            "state": "active",
            "replicas": {
              "core_node1": {
                "state": "active",
                "leader": "true",
                "base_url": "http://node1:8983/solr"
              }
            }
          }
        }
      }
    }
  }
}

<metrics>
  <reporter name="prometheus" class="org.apache.solr.metrics.reporters.SolrPrometheusExporter">
    <str name="registry">solr.core</str>
    <str name="registry">solr.jvm</str>
    <str name="registry">solr.node</str>
  </reporter>
</metrics>

scrape_configs:
  - job_name: 'solr'
    static_configs:
      - targets: ['localhost:8983']
    metrics_path: '/solr/admin/metrics'
    params:
      key: ['solr.jvm', 'solr.core', 'solr.node']

<Configuration>
  <Appenders>
    <Console name="Console" target="SYSTEM_OUT">
      <PatternLayout pattern="%d{yyyy-MM-dd HH:mm:ss.SSS} [%t] %-5p %c{1}:%L - %m%n"/>
    </Console>
    <RollingFile name="RollingFile" fileName="logs/solr.log"
                 filePattern="logs/solr-%d{yyyy-MM-dd}-%i.log">
      <PatternLayout pattern="%d{yyyy-MM-dd HH:mm:ss.SSS} [%t] %-5p %c{1}:%L - %m%n"/>
      <Policies>
        <TimeBasedTriggeringPolicy interval="1"/>
        <SizeBasedTriggeringPolicy size="100MB"/>
      </Policies>
      <DefaultRolloverStrategy max="10"/>
    </RollingFile>
  </Appenders>
  <Loggers>
    <Root level="info">
      <AppenderRef ref="Console"/>
      <AppenderRef ref="RollingFile"/>
    </Root>
  </Loggers>
</Configuration>

# 动态修改日志级别
curl "http://localhost:8983/solr/admin/info/logging?set=root:WARN"

# 查看错误日志
grep ERROR logs/solr.log

# 查看慢查询
grep "QTime" logs/solr.log | awk -F'QTime=' '{print $2}' | awk -F'}' '{print $1}' | sort -n | tail -10

groups:
  - name: solr_alerts
    rules:
      - alert: SolrDown
        expr: up{job="solr"} == 0
        for: 1m
        labels:
          severity: critical
        annotations:
          summary: "Solr instance down"
          
      - alert: SolrHighMemory
        expr: solr_jvm_memory_heap_used_bytes / solr_jvm_memory_heap_max_bytes > 0.9
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "Solr high memory usage"
          
      - alert: SolrHighGC
        expr: rate(solr_jvm_gc_G1_Old_Generation_count_total[5m]) > 1
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "Solr high GC rate"
          
      - alert: SolrHighLatency
        expr: solr_core_handler_select_avg_time_per_request > 1000
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "Solr high query latency"
          
      - alert: SolrLowCacheHitRatio
        expr: solr_core_cache_queryResultCache_hitratio < 0.5
        for: 10m
        labels:
          severity: warning
        annotations:
          summary: "Solr low cache hit ratio"

global:
  smtp_smarthost: 'smtp.example.com:587'
  smtp_from: 'alertmanager@example.com'
  smtp_auth_username: 'alertmanager@example.com'
  smtp_auth_password: 'password'

route:
  receiver: 'team-email'
  routes:
    - match:
        severity: critical
      receiver: 'team-email-critical'

receivers:
  - name: 'team-email'
    email_configs:
      - to: 'team@example.com'
  - name: 'team-email-critical'
    email_configs:
      - to: 'team-critical@example.com'

#!/bin/bash
# health_check.sh

SOLR_URL="http://localhost:8983/solr"
COLLECTIONS=("mycollection" "othercollection")

for collection in "${COLLECTIONS[@]}"; do
  response=$(curl -s "${SOLR_URL}/${collection}/admin/ping")
  status=$(echo "$response" | jq -r '.status')
  
  if [ "$status" != "OK" ]; then
    echo "CRITICAL: ${collection} is not healthy"
    exit 2
  fi
done

echo "OK: All collections are healthy"
exit 0

#!/bin/bash
# collect_metrics.sh

SOLR_URL="http://localhost:8983/solr"

# 获取JVM内存
memory=$(curl -s "${SOLR_URL}/admin/metrics?key=solr.jvm.memory.heap.*" | jq '.metrics')

# 获取查询统计
queries=$(curl -s "${SOLR_URL}/admin/metrics?key=solr.core.*.handler./select.*" | jq '.metrics')

# 输出指标
echo "Memory: ${memory}"
echo "Queries: ${queries}"