日志收集 #

日志类型
    │
    ├── 容器日志
    │   ├── stdout/stderr
    │   └── 应用日志文件
    │
    ├── 系统日志
    │   ├── kubelet日志
    │   └── 系统服务日志
    │
    └── 审计日志
        └── API Server审计日志

日志架构
    │
    ├── 节点级日志
    │   └── 容器运行时管理
    │
    ├── 集群级日志
    │   ├── 日志采集Agent
    │   └── 日志存储后端
    │
    └── 日志分析
        ├── 搜索查询
        └── 可视化展示

apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: elasticsearch
  namespace: logging
spec:
  serviceName: elasticsearch
  replicas: 1
  selector:
    matchLabels:
      app: elasticsearch
  template:
    metadata:
      labels:
        app: elasticsearch
    spec:
      containers:
      - name: elasticsearch
        image: docker.elastic.co/elasticsearch/elasticsearch:8.8.0
        env:
        - name: discovery.type
          value: single-node
        - name: ES_JAVA_OPTS
          value: "-Xms1g -Xmx1g"
        - name: xpack.security.enabled
          value: "false"
        ports:
        - containerPort: 9200
        volumeMounts:
        - name: data
          mountPath: /usr/share/elasticsearch/data
      volumes:
      - name: data
        persistentVolumeClaim:
          claimName: elasticsearch-pvc
---
apiVersion: v1
kind: Service
metadata:
  name: elasticsearch
  namespace: logging
spec:
  ports:
  - port: 9200
  selector:
    app: elasticsearch

apiVersion: v1
kind: ServiceAccount
metadata:
  name: fluentd
  namespace: logging
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: fluentd
rules:
- apiGroups: [""]
  resources: ["pods", "namespaces"]
  verbs: ["get", "list", "watch"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: fluentd
subjects:
- kind: ServiceAccount
  name: fluentd
  namespace: logging
roleRef:
  kind: ClusterRole
  name: fluentd
  apiGroup: rbac.authorization.k8s.io
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: fluentd
  namespace: logging
spec:
  selector:
    matchLabels:
      app: fluentd
  template:
    metadata:
      labels:
        app: fluentd
    spec:
      serviceAccountName: fluentd
      tolerations:
      - key: node-role.kubernetes.io/master
        effect: NoSchedule
      containers:
      - name: fluentd
        image: fluent/fluentd-kubernetes-daemonset:v1-debian-elasticsearch
        env:
        - name: FLUENT_ELASTICSEARCH_HOST
          value: "elasticsearch.logging.svc.cluster.local"
        - name: FLUENT_ELASTICSEARCH_PORT
          value: "9200"
        - name: FLUENT_ELASTICSEARCH_SCHEME
          value: "http"
        volumeMounts:
        - name: varlog
          mountPath: /var/log
        - name: varlibdockercontainers
          mountPath: /var/lib/docker/containers
          readOnly: true
      volumes:
      - name: varlog
        hostPath:
          path: /var/log
      - name: varlibdockercontainers
        hostPath:
          path: /var/lib/docker/containers

apiVersion: apps/v1
kind: Deployment
metadata:
  name: kibana
  namespace: logging
spec:
  replicas: 1
  selector:
    matchLabels:
      app: kibana
  template:
    metadata:
      labels:
        app: kibana
    spec:
      containers:
      - name: kibana
        image: docker.elastic.co/kibana/kibana:8.8.0
        env:
        - name: ELASTICSEARCH_HOSTS
          value: "http://elasticsearch:9200"
        ports:
        - containerPort: 5601
---
apiVersion: v1
kind: Service
metadata:
  name: kibana
  namespace: logging
spec:
  ports:
  - port: 80
    targetPort: 5601
  selector:
    app: kibana
  type: LoadBalancer

apiVersion: v1
kind: ConfigMap
metadata:
  name: fluent-bit-config
  namespace: logging
data:
  fluent-bit.conf: |
    [SERVICE]
        Flush         1
        Log_Level     info
        Daemon        off
        Parsers_File  parsers.conf
    
    [INPUT]
        Name              tail
        Tag               kube.*
        Path              /var/log/containers/*.log
        Parser            docker
        DB                /var/log/flb_kube.db
        Mem_Buf_Limit     5MB
        Skip_Long_Lines   On
        Refresh_Interval  10
    
    [FILTER]
        Name                kubernetes
        Match               kube.*
        Kube_URL            https://kubernetes.default.svc:443
        Kube_CA_File        /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
        Kube_Token_File     /var/run/secrets/kubernetes.io/serviceaccount/token
        Merge_Log           On
        K8S-Logging.Parser  On
        K8S-Logging.Exclude On
    
    [OUTPUT]
        Name            es
        Match           *
        Host            elasticsearch
        Port            9200
        Logstash_Format On
        Retry_Limit     False
  
  parsers.conf: |
    [PARSER]
        Name        docker
        Format      json
        Time_Key    time
        Time_Format %Y-%m-%dT%H:%M:%S.%L
        Time_Keep   On

apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: fluent-bit
  namespace: logging
spec:
  selector:
    matchLabels:
      app: fluent-bit
  template:
    metadata:
      labels:
        app: fluent-bit
    spec:
      containers:
      - name: fluent-bit
        image: fluent/fluent-bit:2.1.0
        volumeMounts:
        - name: varlog
          mountPath: /var/log
        - name: varlibdockercontainers
          mountPath: /var/lib/docker/containers
          readOnly: true
        - name: config
          mountPath: /fluent-bit/etc/
      volumes:
      - name: varlog
        hostPath:
          path: /var/log
      - name: varlibdockercontainers
        hostPath:
          path: /var/lib/docker/containers
      - name: config
        configMap:
          name: fluent-bit-config

apiVersion: v1
kind: Pod
metadata:
  name: app-pod
spec:
  containers:
  - name: app
    image: nginx
    command: ["sh", "-c", "echo 'Hello World' && sleep 3600"]

apiVersion: v1
kind: Pod
metadata:
  name: json-log-pod
  annotations:
    fluentbit.io/parser: json
spec:
  containers:
  - name: app
    image: myapp
    command: ["sh", "-c", "echo '{\"level\":\"info\",\"message\":\"Hello World\",\"timestamp\":\"2024-01-01T00:00:00Z\"}' && sleep 3600"]

# Java堆栈日志
apiVersion: v1
kind: ConfigMap
metadata:
  name: fluent-bit-parsers
data:
  parsers.conf: |
    [PARSER]
        Name        multiline
        Format      regex
        Regex       /^(?<time>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}) (?<level>\w+) (?<message>.*)/
        Time_Key    time
        Time_Format %Y-%m-%d %H:%M:%S

# 查看Pod日志
kubectl logs <pod-name>

# 实时查看日志
kubectl logs -f <pod-name>

# 查看最近N行日志
kubectl logs --tail=100 <pod-name>

# 查看指定时间日志
kubectl logs --since=1h <pod-name>

# 查看之前容器日志
kubectl logs --previous <pod-name>

# 多容器Pod指定容器
kubectl logs <pod-name> -c <container-name>

# 查询特定Pod日志
{
  "query": {
    "match": {
      "kubernetes.pod_name": "my-pod"
    }
  }
}

# 查询错误日志
{
  "query": {
    "match": {
      "log": "ERROR"
    }
  }
}

# 时间范围查询
{
  "query": {
    "range": {
      "@timestamp": {
        "gte": "now-1h",
        "lte": "now"
      }
    }
  }
}

日志规范建议
    │
    ├── 使用标准输出
    │   └── stdout/stderr
    │
    ├── JSON格式
    │   ├── level
    │   ├── message
    │   └── timestamp
    │
    ├── 避免敏感信息
    │   └── 脱敏处理
    │
    └── 合理日志级别
        ├── DEBUG
        ├── INFO
        ├── WARN
        └── ERROR

apiVersion: v1
kind: ConfigMap
metadata:
  name: app-config
data:
  LOG_LEVEL: "info"
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: app
spec:
  template:
    spec:
      containers:
      - name: app
        image: myapp
        env:
        - name: LOG_LEVEL
          valueFrom:
            configMapKeyRef:
              name: app-config
              key: LOG_LEVEL

# 容器日志轮转配置
# /etc/docker/daemon.json
{
  "log-driver": "json-file",
  "log-opts": {
    "max-size": "10m",
    "max-file": "3"
  }
}

# 查看Fluentd日志
kubectl logs -n logging -l app=fluentd

# 查看Elasticsearch状态
kubectl exec -n logging -it elasticsearch-0 -- curl localhost:9200/_cluster/health

# 检查日志采集
kubectl exec -n logging -it fluentd-xxx -- ls /var/log/containers/

# 测试Elasticsearch连接
kubectl run test --image=curlimages/curl --rm -it -- curl http://elasticsearch:9200