RocksDB写入优化 #

写入性能影响因素：

1. WAL写入
   - sync选项影响
   - 磁盘IO速度

2. MemTable
   - 大小和数量
   - Flush速度

3. Compaction
   - 触发频率
   - 后台线程数

4. 锁竞争
   - 并发写入
   - 资源争用

#include <rocksdb/options.h>

// 高可靠性写入
rocksdb::WriteOptions GetSyncWriteOptions() {
    rocksdb::WriteOptions options;
    options.sync = true;  // 每次写入sync到磁盘
    return options;
}

// 高性能写入
rocksdb::WriteOptions GetAsyncWriteOptions() {
    rocksdb::WriteOptions options;
    options.sync = false;  // 依赖OS刷盘
    return options;
}

// 禁用WAL
rocksdb::WriteOptions GetNoWALOptions() {
    rocksdb::WriteOptions options;
    options.disableWAL = true;  // 不写WAL
    return options;
}

sync选项性能对比：

sync = true:
- 每次写入调用fsync
- 延迟：1-10ms
- 吞吐：1K-10K ops/s

sync = false:
- 依赖OS刷盘
- 延迟：0.1-1ms
- 吞吐：10K-100K ops/s

disableWAL = true:
- 不写WAL
- 延迟：0.01-0.1ms
- 吞吐：100K-1M ops/s
- 风险：可能丢失数据

#include <rocksdb/options.h>

rocksdb::Options GetSeparateWALOptions() {
    rocksdb::Options options;
    
    // WAL放在高速磁盘
    options.wal_dir = "/fast_ssd/wal";
    
    // 数据放在大容量磁盘
    // db_dir = "/large_hdd/data";
    
    return options;
}

#include <rocksdb/options.h>

// 高吞吐配置
rocksdb::Options GetLargeMemTableOptions() {
    rocksdb::Options options;
    
    options.write_buffer_size = 128 * 1024 * 1024;  // 128MB
    options.max_write_buffer_number = 8;
    
    return options;
}

// 低延迟配置
rocksdb::Options GetSmallMemTableOptions() {
    rocksdb::Options options;
    
    options.write_buffer_size = 32 * 1024 * 1024;  // 32MB
    options.max_write_buffer_number = 3;
    
    return options;
}

MemTable大小影响：

大MemTable (128MB+):
- 优点：减少Flush频率，高吞吐
- 缺点：Flush时间长，内存占用高

小MemTable (32MB-):
- 优点：Flush快，低延迟
- 缺点：Flush频繁，影响吞吐

建议：
- 高吞吐场景：64-128MB
- 低延迟场景：32-64MB
- 内存受限：16-32MB

#include <rocksdb/options.h>

rocksdb::Options GetFlushOptions() {
    rocksdb::Options options;
    
    // 增加Flush线程
    options.max_background_flushes = 4;
    
    // 总后台任务数
    options.max_background_jobs = 16;
    
    // 调整触发阈值
    options.level0_file_num_compaction_trigger = 8;
    
    return options;
}

#include <rocksdb/db.h>
#include <rocksdb/write_batch.h>

// 单次写入
void SingleWrites(rocksdb::DB* db, const std::vector<std::string>& keys) {
    for (const auto& key : keys) {
        db->Put(rocksdb::WriteOptions(), key, "value");
    }
}

// 批量写入
void BatchWrites(rocksdb::DB* db, const std::vector<std::string>& keys) {
    rocksdb::WriteBatch batch;
    
    for (const auto& key : keys) {
        batch.Put(key, "value");
    }
    
    db->Write(rocksdb::WriteOptions(), &batch);
}

// 批量写入性能提升：10-100倍

#include <rocksdb/db.h>
#include <rocksdb/write_batch.h>

void OptimizedBatchWrite(rocksdb::DB* db, 
                         const std::vector<std::pair<std::string, std::string>>& data,
                         size_t batch_size = 1000) {
    
    rocksdb::WriteBatch batch;
    size_t count = 0;
    
    for (const auto& [key, value] : data) {
        batch.Put(key, value);
        count++;
        
        if (count >= batch_size) {
            db->Write(rocksdb::WriteOptions(), &batch);
            batch.Clear();
            count = 0;
        }
    }
    
    // 写入剩余数据
    if (count > 0) {
        db->Write(rocksdb::WriteOptions(), &batch);
    }
}

// 推荐批次大小：
// - 实时写入：100-1000
// - 批量导入：1000-10000

#include <rocksdb/options.h>

rocksdb::Options GetConcurrentWriteOptions() {
    rocksdb::Options options;
    
    // 允许并发MemTable写入
    options.allow_concurrent_memtable_write = true;
    
    // MemTable写入线程数
    options.max_write_buffer_number = 8;
    
    // 后台线程
    options.max_background_jobs = 16;
    
    return options;
}

#include <rocksdb/options.h>

rocksdb::WriteOptions GetLowPriorityOptions() {
    rocksdb::WriteOptions options;
    options.low_pri = true;  // 低优先级写入
    return options;
}

// 使用场景：
// - 后台批量导入
// - 非关键数据写入

#include <rocksdb/options.h>

rocksdb::Options GetCompactionOptimizedOptions() {
    rocksdb::Options options;
    
    // 增加后台线程
    options.max_background_compactions = 8;
    options.max_background_jobs = 16;
    
    // 调整触发阈值
    options.level0_file_num_compaction_trigger = 8;
    options.level0_slowdown_writes_trigger = 30;
    options.level0_stop_writes_trigger = 50;
    
    // 速率限制
    options.rate_limiter.reset(
        rocksdb::NewGenericRateLimiter(100 * 1024 * 1024)
    );
    
    return options;
}

#include <rocksdb/options.h>

rocksdb::Options GetTieredCompactionOptions() {
    rocksdb::Options options;
    
    // 使用Universal Compaction
    options.compaction_style = rocksdb::CompactionStyle::kCompactionStyleUniversal;
    
    // 配置参数
    options.level0_file_num_compaction_trigger = 4;
    options.level0_layer_size_compaction_trigger = 2;
    
    return options;
}

#include <rocksdb/options.h>
#include <rocksdb/table.h>

rocksdb::Options GetHighThroughputOptions() {
    rocksdb::Options options;
    
    // 大MemTable
    options.write_buffer_size = 128 * 1024 * 1024;
    options.max_write_buffer_number = 8;
    
    // 异步写入
    // WriteOptions.sync = false
    
    // 后台线程
    options.max_background_jobs = 16;
    
    // Compaction配置
    options.level0_file_num_compaction_trigger = 8;
    
    // 压缩
    options.compression = rocksdb::CompressionType::kLZ4Compression;
    
    return options;
}

#include <rocksdb/options.h>

rocksdb::Options GetLowLatencyOptions() {
    rocksdb::Options options;
    
    // 小MemTable
    options.write_buffer_size = 32 * 1024 * 1024;
    options.max_write_buffer_number = 3;
    
    // 快速Flush
    options.max_background_flushes = 4;
    
    // 快速Compaction
    options.level0_file_num_compaction_trigger = 2;
    
    return options;
}

#include <rocksdb/options.h>

rocksdb::Options GetBulkLoadOptions() {
    rocksdb::Options options;
    
    // 大MemTable
    options.write_buffer_size = 256 * 1024 * 1024;
    options.max_write_buffer_number = 8;
    
    // 禁用WAL
    // WriteOptions.disableWAL = true
    
    // 后台线程
    options.max_background_jobs = 16;
    
    // 使用FIFO Compaction
    options.compaction_style = rocksdb::CompactionStyle::kCompactionStyleFIFO;
    
    return options;
}

#include <rocksdb/db.h>
#include <rocksdb/statistics.h>
#include <iostream>

void PrintWriteStats(rocksdb::DB* db) {
    auto stats = db->GetOptions().statistics;
    
    // 写入次数
    uint64_t writes = stats->getTickerCount(rocksdb::NUMBER_KEYS_WRITTEN);
    std::cout << "Keys written: " << writes << std::endl;
    
    // WAL写入字节数
    uint64_t wal_bytes = stats->getTickerCount(rocksdb::WAL_FILE_BYTES);
    std::cout << "WAL bytes: " << wal_bytes / 1024 / 1024 << " MB" << std::endl;
    
    // WAL sync次数
    uint64_t wal_syncs = stats->getTickerCount(rocksdb::WAL_FILE_SYNCED);
    std::cout << "WAL syncs: " << wal_syncs << std::endl;
    
    // MemTable写入
    uint64_t memtable_bytes = stats->getTickerCount(rocksdb::BYTES_WRITTEN);
    std::cout << "Bytes written: " << memtable_bytes / 1024 / 1024 << " MB" << std::endl;
}

#include <rocksdb/statistics.h>

void PrintWriteLatency(rocksdb::DB* db) {
    auto stats = db->GetOptions().statistics;
    
    // 写入延迟直方图
    auto histogram = stats->getHistogramData(rocksdb::WRITE_STALL);
    
    std::cout << "Write stall:" << std::endl;
    std::cout << "  Count: " << histogram.count << std::endl;
    std::cout << "  Average: " << histogram.average << " us" << std::endl;
    std::cout << "  P50: " << histogram.median << " us" << std::endl;
    std::cout << "  P95: " << histogram.percentile95 << " us" << std::endl;
    std::cout << "  P99: " << histogram.percentile99 << " us" << std::endl;
}