容器监控与日志 #

# 查看所有运行容器
docker stats

# 查看指定容器
docker stats mycontainer

# 只显示一次
docker stats --no-stream

# 自定义格式
docker stats --format "table {{.Name}}\t{{.CPUPerc}}\t{{.MemUsage}}"

# 输出JSON格式
docker stats --no-stream --format "{{json .}}"

# 查看容器进程
docker top mycontainer

# 使用ps参数
docker top mycontainer aux
docker top mycontainer -ef

# 查看完整信息
docker inspect mycontainer

# 查看状态信息
docker inspect --format '{{json .State}}' mycontainer | jq

# 查看网络信息
docker inspect --format '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' mycontainer

# 查看资源限制
docker inspect --format '{{.HostConfig.Memory}}' mycontainer
docker inspect --format '{{.HostConfig.NanoCpus}}' mycontainer

# 监控所有事件
docker events

# 过滤容器事件
docker events --filter "type=container"

# 过滤特定事件
docker events --filter "event=start"
docker events --filter "event=stop"
docker events --filter "event=oom"

# 过滤特定容器
docker events --filter "container=mycontainer"

# 时间范围
docker events --since "1h"
docker events --since "2024-01-01" --until "2024-01-02"

# 格式化输出
docker events --format '{{.Time}} {{.Type}} {{.Action}} {{.Actor.Attributes.name}}'

# 查看日志
docker logs mycontainer

# 实时查看
docker logs -f mycontainer

# 查看最后N行
docker logs --tail 100 mycontainer

# 添加时间戳
docker logs -t mycontainer

# 时间范围过滤
docker logs --since "2024-01-01" mycontainer
docker logs --since "2h" mycontainer
docker logs --until "2024-01-02" mycontainer

# 组合使用
docker logs -f --tail 100 --since "1h" mycontainer

# 使用local驱动
docker run --log-driver local nginx

# 使用syslog驱动
docker run --log-driver syslog nginx

# 配置日志选项
docker run \
  --log-driver json-file \
  --log-opt max-size=10m \
  --log-opt max-file=3 \
  nginx

// /etc/docker/daemon.json
{
  "log-driver": "json-file",
  "log-opts": {
    "max-size": "10m",
    "max-file": "3",
    "labels": "production",
    "tag": "{{.Name}}/{{.ID}}"
  }
}

# 配置日志轮转
docker run \
  --log-driver json-file \
  --log-opt max-size=10m \
  --log-opt max-file=5 \
  nginx

# local驱动配置
docker run \
  --log-driver local \
  --log-opt max-size=10m \
  --log-opt max-file=5 \
  --log-opt compress=true \
  nginx

# 启动cAdvisor
docker run -d \
  --name=cadvisor \
  --volume=/:/rootfs:ro \
  --volume=/var/run:/var/run:ro \
  --volume=/sys:/sys:ro \
  --volume=/var/lib/docker/:/var/lib/docker:ro \
  --publish=8080:8080 \
  google/cadvisor:latest

# 访问 http://localhost:8080

# docker-compose.yml
version: '3.8'

services:
  prometheus:
    image: prom/prometheus
    ports:
      - "9090:9090"
    volumes:
      - ./prometheus.yml:/etc/prometheus/prometheus.yml
    command:
      - '--config.file=/etc/prometheus/prometheus.yml'

  grafana:
    image: grafana/grafana
    ports:
      - "3000:3000"
    environment:
      - GF_SECURITY_ADMIN_PASSWORD=admin
    volumes:
      - grafana-data:/var/lib/grafana

  cadvisor:
    image: google/cadvisor
    ports:
      - "8080:8080"
    volumes:
      - /:/rootfs:ro
      - /var/run:/var/run:ro
      - /sys:/sys:ro
      - /var/lib/docker/:/var/lib/docker:ro

volumes:
  grafana-data:

# 启动node exporter
docker run -d \
  --name=node-exporter \
  --net="host" \
  --pid="host" \
  -v "/:/host:ro,rslave" \
  quay.io/prometheus/node-exporter \
  --path.rootfs=/host

# docker-compose.yml
version: '3.8'

services:
  elasticsearch:
    image: elasticsearch:8.10.0
    environment:
      - discovery.type=single-node
      - xpack.security.enabled=false
    ports:
      - "9200:9200"
    volumes:
      - es-data:/usr/share/elasticsearch/data

  logstash:
    image: logstash:8.10.0
    ports:
      - "5044:5044"
    volumes:
      - ./logstash.conf:/usr/share/logstash/pipeline/logstash.conf
    depends_on:
      - elasticsearch

  kibana:
    image: kibana:8.10.0
    ports:
      - "5601:5601"
    environment:
      - ELASTICSEARCH_HOSTS=http://elasticsearch:9200
    depends_on:
      - elasticsearch

volumes:
  es-data:

# 使用Fluentd日志驱动
docker run \
  --log-driver=fluentd \
  --log-opt fluentd-address=localhost:24224 \
  --log-opt tag=docker.{{.Name}} \
  nginx

# 启动Fluentd
docker run -d \
  --name fluentd \
  -p 24224:24224 \
  -v /data/fluentd:/fluentd/log \
  fluent/fluentd

# docker-compose.yml
version: '3.8'

services:
  loki:
    image: grafana/loki:latest
    ports:
      - "3100:3100"
    volumes:
      - ./loki-config.yml:/etc/loki/local-config.yaml
    command: -config.file=/etc/loki/local-config.yaml

  promtail:
    image: grafana/promtail:latest
    volumes:
      - /var/log:/var/log
      - ./promtail-config.yml:/etc/promtail/config.yml
    command: -config.file=/etc/promtail/config.yml

  grafana:
    image: grafana/grafana:latest
    ports:
      - "3000:3000"
    environment:
      - GF_PATHS_PROVISIONING=/etc/grafana/provisioning

# Dockerfile
HEALTHCHECK --interval=30s --timeout=3s --start-period=10s --retries=3 \
    CMD curl -f http://localhost:8080/health || exit 1

# 查看健康状态
docker inspect --format '{{.State.Health.Status}}' mycontainer

# 查看健康检查历史
docker inspect --format '{{json .State.Health}}' mycontainer | jq

# 监控脚本
#!/bin/bash
while true; do
    status=$(docker inspect --format '{{.State.Health.Status}}' mycontainer)
    if [ "$status" != "healthy" ]; then
        echo "Container is $status"
        # 发送告警
    fi
    sleep 30
done

{
  "log-driver": "json-file",
  "log-opts": {
    "max-size": "10m",
    "max-file": "3"
  }
}

# 应用输出JSON格式日志
docker run -e LOG_FORMAT=json myapp

┌─────────────────────────────────────────────────────┐
│                   关键监控指标                       │
├─────────────────────────────────────────────────────┤
│                                                     │
│  容器指标:                                          │
│  - CPU使用率                                        │
│  - 内存使用率                                       │
│  - 网络IO                                           │
│  - 磁盘IO                                           │
│  - 进程数                                           │
│                                                     │
│  应用指标:                                          │
│  - 请求响应时间                                     │
│  - 请求错误率                                       │
│  - 请求吞吐量                                       │
│  - 健康检查状态                                     │
│                                                     │
│  系统指标:                                          │
│  - 主机CPU/内存                                     │
│  - 磁盘空间                                         │
│  - 网络连接数                                       │
│                                                     │
└─────────────────────────────────────────────────────┘

# Prometheus告警规则示例
groups:
  - name: container_alerts
    rules:
      - alert: ContainerHighCPU
        expr: rate(container_cpu_usage_seconds_total{name!=""}[5m]) > 0.8
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "Container {{ $labels.name }} high CPU usage"

      - alert: ContainerHighMemory
        expr: container_memory_usage_bytes / container_spec_memory_limit_bytes > 0.9
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "Container {{ $labels.name }} high memory usage"

      - alert: ContainerUnhealthy
        expr: container_health_status{status="unhealthy"} == 1
        for: 1m
        labels:
          severity: critical
        annotations:
          summary: "Container {{ $labels.name }} is unhealthy"

# 集中式日志收集架构
┌──────────┐     ┌──────────┐     ┌──────────┐
│ Container│     │ Container│     │ Container│
└────┬─────┘     └────┬─────┘     └────┬─────┘
     │                │                │
     └────────────────┼────────────────┘
                      │
                      ↓
               ┌──────────┐
               │ Log Agent│
               │(Fluentd) │
               └────┬─────┘
                    │
                    ↓
            ┌──────────────┐
            │ Log Storage  │
            │(Elasticsearch)│
            └──────┬───────┘
                   │
                   ↓
            ┌──────────────┐
            │ Log Viewer   │
            │  (Kibana)    │
            └──────────────┘