RocksDB故障排查 #

故障排查流程：
├── 收集信息 - 错误日志、监控数据
├── 分析现象 - 确定问题类型
├── 定位原因 - 找到根本原因
├── 解决问题 - 实施解决方案
└── 验证效果 - 确认问题解决

Error: IO error: While lock file: /path/to/db/LOCK: Resource temporarily unavailable

# 检查是否有进程占用
lsof /path/to/db/LOCK

# 确认无进程后删除锁文件
rm /path/to/db/LOCK

# 重新打开数据库

Error: Corruption: Corrupted WAL file

// 设置恢复模式
rocksdb::Options options;
options.wal_recovery_mode = rocksdb::WALRecoveryMode::kPointInTimeRecovery;

// 或跳过损坏记录
options.wal_recovery_mode = rocksdb::WALRecoveryMode::kSkipAnyCorruptedRecords;

Error: Corruption: MANIFEST corrupted

# 1. 备份当前数据
cp -r /path/to/db /path/to/db_backup

# 2. 尝试使用ldb修复
ldb repair /path/to/db

# 3. 如果修复失败，从备份恢复

// 检查MemTable状态
uint64_t num_memtables;
db->GetIntProperty("rocksdb.num-immutable-mem-table", &num_memtables);

// 检查L0文件数
uint64_t l0_files;
db->GetIntProperty("rocksdb.num-files-at-level0", &l0_files);

// 检查待Compaction
uint64_t pending_compaction;
db->GetIntProperty("rocksdb.estimate-pending-compaction-bytes", &pending_compaction);

// 1. 增加MemTable大小
options.write_buffer_size = 128 * 1024 * 1024;

// 2. 增加后台线程
options.max_background_jobs = 16;

// 3. 调整Compaction触发阈值
options.level0_file_num_compaction_trigger = 8;

// 4. 使用批量写入
rocksdb::WriteBatch batch;
batch.Put(key1, value1);
batch.Put(key2, value2);
db->Write(WriteOptions(), &batch);

Error: IO error: Write stopped

# 检查L0文件数
ldb dump --db=/path/to/db --count_only

# 检查配置
# level0_stop_writes_trigger

// 1. 手动触发Compaction
db->CompactRange(CompactRangeOptions(), nullptr, nullptr);

// 2. 调整停止写入阈值
options.level0_stop_writes_trigger = 50;

// 3. 增加后台Compaction线程
options.max_background_compactions = 8;

// 1. 限制Compaction速率
options.rate_limiter.reset(
    rocksdb::NewGenericRateLimiter(100 * 1024 * 1024)  // 100MB/s
);

// 2. 增加后台线程
options.max_background_flushes = 4;
options.max_background_compactions = 8;

// 3. 调整MemTable配置
options.max_write_buffer_number = 6;
options.min_write_buffer_number_to_merge = 2;

// 检查缓存命中率
auto stats = db->GetOptions().statistics;
uint64_t hits = stats->getTickerCount(rocksdb::BLOCK_CACHE_HIT);
uint64_t misses = stats->getTickerCount(rocksdb::BLOCK_CACHE_MISS);
double hit_rate = (double)hits / (hits + misses);

// 检查L0文件数
uint64_t l0_files;
db->GetIntProperty("rocksdb.num-files-at-level0", &l0_files);

// 1. 增加Block Cache
rocksdb::BlockBasedTableOptions table_options;
table_options.block_cache = rocksdb::NewLRUCache(4 * 1024 * 1024 * 1024);

// 2. 启用布隆过滤器
table_options.filter_policy.reset(rocksdb::NewBloomFilterPolicy(10));

// 3. 减少L0文件
options.level0_file_num_compaction_trigger = 4;

Error: Corruption: block checksum mismatch

# 1. 检查SST文件完整性
sst_dump --file=/path/to/file.sst --command=verify

# 2. 如果确认损坏，删除损坏文件
# 注意：会导致数据丢失
rm /path/to/corrupted_file.sst

# 3. 从备份恢复
ldb restore --db=/path/to/db --backup_dir=/path/to/backup

# 检查各层文件大小
for i in {0..6}; do
    echo "Level $i:"
    ls -lh /path/to/db/*_$i.sst 2>/dev/null | awk '{sum+=$5} END {print sum}'
done

# 检查快照数量
ldb dump --db=/path/to/db --count_only

// 1. 手动触发Compaction
db->CompactRange(CompactRangeOptions(), nullptr, nullptr);

// 2. 释放快照
db->ReleaseSnapshot(snapshot);

// 3. 启用压缩
options.compression = rocksdb::CompressionType::kZSTD;

Error: IO error: No space left on device

# 1. 紧急清理空间
rm /path/to/db/*.log.old  # 删除旧日志

# 2. 触发Compaction
ldb compact --db=/path/to/db

# 3. 如果无法恢复，迁移数据
# 停止写入，复制到更大磁盘

// 监控关键指标
void DiagnosePerformance(rocksdb::DB* db) {
    uint64_t value;
    
    // L0文件数
    db->GetIntProperty("rocksdb.num-files-at-level0", &value);
    std::cout << "L0 files: " << value << std::endl;
    
    // 待Compaction
    db->GetIntProperty("rocksdb.estimate-pending-compaction-bytes", &value);
    std::cout << "Pending compaction: " << value / 1024 / 1024 << " MB" << std::endl;
    
    // MemTable数量
    db->GetIntProperty("rocksdb.num-immutable-mem-table", &value);
    std::cout << "Immutable MemTables: " << value << std::endl;
}

// 1. 定期Compaction
db->CompactRange(CompactRangeOptions(), nullptr, nullptr);

// 2. 调整配置
options.max_background_jobs = 16;
options.level0_file_num_compaction_trigger = 4;

// 3. 监控并预警

void DiagnoseMemory(rocksdb::DB* db) {
    uint64_t value;
    
    db->GetIntProperty("rocksdb.cur-size-all-mem-tables", &value);
    std::cout << "MemTable: " << value / 1024 / 1024 << " MB" << std::endl;
    
    db->GetIntProperty("rocksdb.block-cache-usage", &value);
    std::cout << "Block Cache: " << value / 1024 / 1024 << " MB" << std::endl;
    
    db->GetIntProperty("rocksdb.estimate-table-readers-mem", &value);
    std::cout << "Table Readers: " << value / 1024 / 1024 << " MB" << std::endl;
}

// 1. 减小MemTable
options.write_buffer_size = 32 * 1024 * 1024;
options.max_write_buffer_number = 2;

// 2. 减小Block Cache
table_options.block_cache = rocksdb::NewLRUCache(256 * 1024 * 1024);

// 3. 使用高压缩比
options.compression = rocksdb::CompressionType::kZSTD;

# 查看RocksDB日志
tail -f /path/to/db/LOG

# 搜索错误
grep -i error /path/to/db/LOG
grep -i corruption /path/to/db/LOG
grep -i "write stall" /path/to/db/LOG

#!/bin/bash
# diagnose_rocksdb.sh

DB_PATH=$1

echo "=== RocksDB Diagnosis ==="

# 检查文件数量
echo "File counts:"
echo "  SST files: $(ls $DB_PATH/*.sst 2>/dev/null | wc -l)"
echo "  WAL files: $(ls $DB_PATH/*.log 2>/dev/null | wc -l)"

# 检查磁盘使用
echo ""
echo "Disk usage:"
du -sh $DB_PATH

# 检查最近日志
echo ""
echo "Recent errors:"
grep -i error $DB_PATH/LOG 2>/dev/null | tail -10

echo ""
echo "=== Diagnosis Complete ==="

# 每日检查
0 0 * * * /path/to/health_check.sh /path/to/db

# 每周Compaction
0 2 * * 0 ldb compact --db=/path/to/db

# 每月备份
0 3 1 * * ldb backup --db=/path/to/db --backup_dir=/path/to/backup