集群维护 #

集群维护内容
    │
    ├── 节点管理
    │   ├── 节点扩缩容
    │   ├── 节点维护
    │   └── 节点故障处理
    │
    ├── 证书管理
    │   ├── 证书更新
    │   └── 证书轮换
    │
    ├── 备份恢复
    │   ├── etcd备份
    │   └── 资源备份
    │
    └── 日常运维
        ├── 日志管理
        ├── 资源清理
        └── 性能优化

# 查看节点状态
kubectl get nodes

# 查看节点详情
kubectl describe node <node-name>

# 查看节点资源
kubectl top nodes

# 标记节点不可调度
kubectl cordon <node-name>

# 驱逐节点上的Pod
kubectl drain <node-name> --ignore-daemonsets --delete-emptydir-data

# 恢复节点调度
kubectl uncordon <node-name>

# 添加标签
kubectl label node <node-name> key=value

# 删除标签
kubectl label node <node-name> key-

# 查看标签
kubectl get nodes --show-labels

# 添加污点
kubectl taint nodes <node-name> key=value:NoSchedule

# 删除污点
kubectl taint nodes <node-name> key:NoSchedule-

# 查看污点
kubectl describe node <node-name> | grep Taints

# 查看所有证书有效期
kubeadm certs check-expiration

# 输出示例
CERTIFICATE                EXPIRES                  RESIDUAL TIME   EXTERNALLY MANAGED
admin.conf                 Jan 10, 2025 12:00 UTC   364d            no
apiserver                  Jan 10, 2025 12:00 UTC   364d            no
apiserver-etcd-client      Jan 10, 2025 12:00 UTC   364d            no

# 更新所有证书
kubeadm certs renew all

# 更新特定证书
kubeadm certs renew admin.conf
kubeadm certs renew apiserver

# 更新后重启控制平面
kubectl restart kube-apiserver
kubectl restart kube-controller-manager
kubectl restart kube-scheduler

# 更新admin.conf
kubeadm certs renew admin.conf

# 复制到用户目录
cp /etc/kubernetes/admin.conf ~/.kube/config

# 备份etcd
ETCDCTL_API=3 etcdctl snapshot save /backup/etcd-snapshot.db \
  --cacert=/etc/kubernetes/pki/etcd/ca.crt \
  --cert=/etc/kubernetes/pki/etcd/server.crt \
  --key=/etc/kubernetes/pki/etcd/server.key

# 查看备份状态
ETCDCTL_API=3 etcdctl snapshot status /backup/etcd-snapshot.db

# 停止etcd和kube-apiserver
systemctl stop kube-apiserver
systemctl stop etcd

# 恢复etcd数据
ETCDCTL_API=3 etcdctl snapshot restore /backup/etcd-snapshot.db \
  --data-dir=/var/lib/etcd-restore

# 修改etcd配置
mv /var/lib/etcd /var/lib/etcd-old
mv /var/lib/etcd-restore /var/lib/etcd

# 启动etcd和kube-apiserver
systemctl start etcd
systemctl start kube-apiserver

#!/bin/bash
BACKUP_DIR="/backup/etcd"
DATE=$(date +%Y%m%d_%H%M%S)

ETCDCTL_API=3 etcdctl snapshot save ${BACKUP_DIR}/etcd-${DATE}.db \
  --cacert=/etc/kubernetes/pki/etcd/ca.crt \
  --cert=/etc/kubernetes/pki/etcd/server.crt \
  --key=/etc/kubernetes/pki/etcd/server.key

# 保留最近7天备份
find ${BACKUP_DIR} -name "etcd-*.db" -mtime +7 -delete

# 备份所有命名空间资源
kubectl get all -A -o yaml > all-resources.yaml

# 备份特定命名空间
kubectl get all -n default -o yaml > default-resources.yaml

# 备份特定资源类型
kubectl get deployments -A -o yaml > deployments.yaml
kubectl get configmaps -A -o yaml > configmaps.yaml
kubectl get secrets -A -o yaml --dry-run=client > secrets.yaml

# 安装Velero
velero install --provider aws --bucket my-bucket --secret-file ./credentials-velero

# 创建备份
velero backup create my-backup

# 定期备份
velero schedule create daily-backup --schedule="0 2 * * *"

# 恢复备份
velero restore create --from-backup my-backup

# 查看组件日志
kubectl logs -n kube-system <pod-name>

# 查看kubelet日志
journalctl -u kubelet -f

# 查看容器日志
kubectl logs <pod-name> -c <container-name>

# 日志轮转配置
# /etc/docker/daemon.json
{
  "log-driver": "json-file",
  "log-opts": {
    "max-size": "10m",
    "max-file": "3"
  }
}

# 清理完成的Job
kubectl delete jobs --field-selector status.successful=1

# 清理失败的Pod
kubectl delete pods --field-selector status.phase=Failed

# 清理孤儿资源
kubectl delete pods --owner_references.uid=<uid>

# 清理未使用的镜像
docker image prune -a

# 清理未使用的容器
docker container prune

# 查看节点资源使用
kubectl top nodes

# 查看Pod资源使用
kubectl top pods -A

# 查看资源限制
kubectl describe node <node-name> | grep -A 5 "Allocated resources"

# 优化建议
# 1. 合理设置requests和limits
# 2. 清理未使用的资源
# 3. 优化镜像大小
# 4. 使用资源配额限制

# 查看节点状态
kubectl describe node <node-name>

# 查看节点事件
kubectl get events --field-selector involvedObject.name=<node-name>

# 检查kubelet状态
systemctl status kubelet

# 检查容器运行时
systemctl status containerd

# 检查网络插件
kubectl get pods -n kube-system -l k8s-app=calico-node

# 检查DNS
kubectl exec -it <pod> -- nslookup kubernetes

# 检查Service
kubectl get endpoints <service-name>

# 检查网络策略
kubectl get networkpolicy -A

# 检查PV状态
kubectl get pv

# 检查PVC状态
kubectl get pvc -A

# 检查存储类
kubectl get storageclass

# 检查CSI驱动
kubectl get pods -n kube-system -l app=csi-driver

# 查看组件状态
kubectl get componentstatuses

# 查看集群信息
kubectl cluster-info

# 查看资源使用
kubectl top nodes
kubectl top pods -A

# Prometheus告警规则
groups:
- name: cluster-alerts
  rules:
  - alert: NodeNotReady
    expr: kube_node_status_condition{condition="Ready",status="true"} == 0
    for: 5m
    labels:
      severity: critical
    annotations:
      summary: "Node {{ $labels.node }} is not ready"

日常维护检查
    │
    ├── 检查节点状态
    │   └── kubectl get nodes
    │
    ├── 检查Pod状态
    │   └── kubectl get pods -A
    │
    ├── 检查资源使用
    │   └── kubectl top nodes/pods
    │
    ├── 检查证书有效期
    │   └── kubeadm certs check-expiration
    │
    └── 检查备份状态
        └── 查看备份文件

维护窗口建议
    │
    ├── 低峰期维护
    │   └── 减少业务影响
    │
    ├── 提前通知
    │   └── 告知相关团队
    │
    ├── 准备回滚方案
    │   └── 出问题可快速恢复
    │
    └── 监控验证
        └── 维护后检查集群状态