备份恢复 #

备份类型：

┌─────────────────────────────────────────┐
│           备份方式                       │
├─────────────────────────────────────────┤
│                                         │
│  ┌─────────────────────────────────┐   │
│  │  全量备份                        │   │
│  │  - 完整数据副本                  │   │
│  │  - 恢复速度快                    │   │
│  │  - 占用空间大                    │   │
│  └─────────────────────────────────┘   │
│                                         │
│  ┌─────────────────────────────────┐   │
│  │  增量备份                        │   │
│  │  - 只备份变更数据                │   │
│  │  - 占用空间小                    │   │
│  │  - 恢复时间较长                  │   │
│  └─────────────────────────────────┘   │
│                                         │
│  ┌─────────────────────────────────┐   │
│  │  快照备份                        │   │
│  │  - 存储层快照                    │   │
│  │  - 速度快                        │   │
│  │  - 依赖存储支持                  │   │
│  └─────────────────────────────────┘   │
│                                         │
└─────────────────────────────────────────┘

wget https://github.com/zilliztech/milvus-backup/releases/download/v0.4.0/milvus-backup_Linux_x86_64.tar.gz

tar -zxvf milvus-backup_Linux_x86_64.tar.gz

chmod +x milvus-backup

mv milvus-backup /usr/local/bin/

milvus:
  address: localhost
  port: 19530
  user: root
  password: Milvus

minio:
  address: localhost
  port: 9000
  accessKeyID: minioadmin
  secretAccessKey: minioadmin
  useSSL: false
  bucketName: a-bucket
  rootPath: files

backup:
  backupPath: /backup/milvus
  maxSegmentGroupSize: 2GiB

milvus-backup create -n backup_20240101

milvus-backup create -n backup_20240101 --collections "documents,images"

milvus-backup create -n backup_20240101 --database "my_database"

milvus-backup list

milvus-backup list --database "my_database"

milvus-backup restore -n backup_20240101

milvus-backup restore -n backup_20240101 --collections "documents"

milvus-backup restore -n backup_20240101 -s _restored

milvus-backup delete -n backup_20240101

from pymilvus import Collection, connections
import json

def export_collection(collection_name, output_file):
    connections.connect("default", host="localhost", port="19530")
    collection = Collection(collection_name)
    collection.load()
    
    all_data = []
    offset = 0
    batch_size = 10000
    
    while True:
        results = collection.query(
            expr="",
            output_fields=["*"],
            offset=offset,
            limit=batch_size
        )
        
        if not results:
            break
        
        all_data.extend(results)
        offset += batch_size
    
    with open(output_file, 'w') as f:
        json.dump(all_data, f)
    
    print(f"导出 {len(all_data)} 条数据到 {output_file}")

export_collection("documents", "documents_backup.json")

from pymilvus import Collection, connections
import json

def import_collection(collection_name, input_file, batch_size=1000):
    connections.connect("default", host="localhost", port="19530")
    collection = Collection(collection_name)
    
    with open(input_file, 'r') as f:
        data = json.load(f)
    
    fields = [field.name for field in collection.schema.fields]
    
    for i in range(0, len(data), batch_size):
        batch = data[i:i+batch_size]
        
        batch_data = []
        for field in fields:
            batch_data.append([item.get(field) for item in batch])
        
        collection.insert(batch_data)
    
    collection.flush()
    print(f"导入 {len(data)} 条数据")

import_collection("documents_restored", "documents_backup.json")

mc alias set milvus-minio http://localhost:9000 minioadmin minioadmin

mc mirror milvus-minio/a-bucket /backup/minio/milvus-backup

mc cp --recursive milvus-minio/a-bucket /backup/minio/milvus-backup

mc mirror /backup/minio/milvus-backup milvus-minio/a-bucket

aws s3 sync s3://milvus-bucket /backup/s3/milvus-backup

aws s3 sync /backup/s3/milvus-backup s3://milvus-bucket

ETCDCTL_API=3 etcdctl snapshot save /backup/etcd/snapshot.db \
  --endpoints=http://localhost:2379

ETCDCTL_API=3 etcdctl snapshot status /backup/etcd/snapshot.db

ETCDCTL_API=3 etcdctl snapshot restore /backup/etcd/snapshot.db \
  --data-dir=/var/lib/etcd-restore

crontab -e

0 2 * * * /usr/local/bin/milvus-backup create -n backup_$(date +\%Y\%m\%d) >> /var/log/milvus-backup.log 2>&1

0 3 * * 0 /usr/bin/find /backup/milvus -type d -mtime +30 -exec rm -rf {} \;

apiVersion: batch/v1
kind: CronJob
metadata:
  name: milvus-backup
  namespace: milvus
spec:
  schedule: "0 2 * * *"
  jobTemplate:
    spec:
      template:
        spec:
          containers:
          - name: backup
            image: milvus-backup:latest
            command:
            - /bin/sh
            - -c
            - |
              milvus-backup create -n backup_$(date +%Y%m%d)
              find /backup -type d -mtime +7 -exec rm -rf {} \;
            volumeMounts:
            - name: backup-storage
              mountPath: /backup
          volumes:
          - name: backup-storage
            persistentVolumeClaim:
              claimName: backup-pvc
          restartPolicy: OnFailure

灾难恢复流程：

┌──────────┐     ┌──────────┐     ┌──────────┐
│ 停止服务  │────▶│ 恢复etcd  │────▶│ 恢复存储  │
└──────────┘     └──────────┘     └──────────┘
                        │
                        ▼
                 ┌──────────┐
                 │ 启动服务  │
                 └──────────┘
                        │
                        ▼
                 ┌──────────┐
                 │ 验证数据  │
                 └──────────┘

#!/bin/bash

echo "停止Milvus服务..."
kubectl scale deployment -n milvus --replicas=0 --all

echo "恢复etcd..."
ETCDCTL_API=3 etcdctl snapshot restore /backup/etcd/snapshot.db \
  --data-dir=/var/lib/etcd

echo "恢复MinIO..."
mc mirror /backup/minio/milvus-backup milvus-minio/a-bucket

echo "启动Milvus服务..."
kubectl scale deployment -n milvus --replicas=1 --all

echo "等待服务就绪..."
sleep 60

echo "验证数据..."
python3 verify_data.py

echo "恢复完成"

备份策略建议：

┌─────────────────────────────────────────┐
│           备份频率                       │
├─────────────────────────────────────────┤
│  全量备份: 每周一次                      │
│  增量备份: 每天一次                      │
│  快照备份: 每小时一次                    │
└─────────────────────────────────────────┘

┌─────────────────────────────────────────┐
│           保留周期                       │
├─────────────────────────────────────────┤
│  日备份: 保留7天                         │
│  周备份: 保留4周                         │
│  月备份: 保留12个月                      │
└─────────────────────────────────────────┘

def verify_backup(backup_name):
    from pymilvus import connections, utility, Collection
    
    connections.connect("default", host="localhost", port="19530")
    
    collections = utility.list_collections()
    
    for collection_name in collections:
        collection = Collection(collection_name)
        count = collection.num_entities
        print(f"{collection_name}: {count} 条数据")
    
    return True

verify_backup("backup_20240101")

备份检查清单：

□ 备份工具已安装配置
□ 定时备份任务已设置
□ 备份存储空间充足
□ 备份保留策略已配置
□ 恢复流程已测试
□ 备份验证脚本已准备
□ 灾难恢复计划已制定

groups:
- name: milvus-backup
  rules:
  - alert: BackupFailed
    expr: milvus_backup_status == 0
    for: 5m
    labels:
      severity: critical
    annotations:
      summary: "Milvus备份失败"
      description: "备份任务执行失败，请检查日志"