数据预处理 #

% 创建含缺失值的数据
data = [1 2 NaN 4 5 NaN 7 8 9 10];

% 检测NaN
idx = isnan(data);          % 逻辑索引
count = sum(isnan(data));   % 缺失值数量

% 检测表中的缺失值
T = table([1; 2; NaN; 4], {'a'; 'b'; ''; 'd'});
missing_rows = ismissing(T);

% 删除含NaN的元素
clean_data = data(~isnan(data));

% 删除含NaN的行
data = [1 2 NaN; 4 5 6; NaN 8 9];
clean_data = data(~any(isnan(data), 2), :);

% 使用rmmissing
T = table([1; 2; NaN; 4], [10; NaN; 30; 40]);
T_clean = rmmissing(T);

% 用均值填充
data = [1 2 NaN 4 5 NaN 7 8];
data(isnan(data)) = mean(data, 'omitnan');

% 用中位数填充
data(isnan(data)) = median(data, 'omitnan');

% 用前一个值填充
data = fillmissing(data, 'previous');

% 用线性插值填充
data = fillmissing(data, 'linear');

% 用常数填充
data = fillmissing(data, 'constant', 0);

% 表数据填充
T = table([1; NaN; 3; NaN], [10; 20; NaN; 40]);
T_filled = fillmissing(T, 'linear');

% 线性插值
x = [1 2 NaN 4 5];
y = fillmissing(x, 'linear');

% 三次样条插值
y = fillmissing(x, 'spline');

% 移动平均
y = fillmissing(x, 'movmean', 3);

% 移动中位数
y = fillmissing(x, 'movmedian', 3);

% 生成数据
data = [randn(100, 1); 10; -8; 12];

% Z-score方法
z = (data - mean(data)) / std(data);
outliers = abs(z) > 3;

% IQR方法
Q1 = quantile(data, 0.25);
Q3 = quantile(data, 0.75);
IQR = Q3 - Q1;
lower = Q1 - 1.5 * IQR;
upper = Q3 + 1.5 * IQR;
outliers = data < lower | data > upper;

% 默认方法（MAD）
outliers = isoutlier(data);

% 指定方法
outliers = isoutlier(data, 'mean');    % 均值方法
outliers = isoutlier(data, 'median');  % 中位数方法
outliers = isoutlier(data, 'quartiles'); % 四分位方法

% 指定阈值
outliers = isoutlier(data, 'mean', 'ThresholdFactor', 3);

% 删除异常值
clean_data = data(~outliers);

% 用中位数替换
data(outliers) = median(data(~outliers));

% 用边界值替换
data(data > upper) = upper;
data(data < lower) = lower;

% 使用filloutliers
clean_data = filloutliers(data, 'linear');
clean_data = filloutliers(data, 'median');

data = randn(100, 1) * 10 + 50;

% 手动计算
z = (data - mean(data)) / std(data);

% 使用zscore函数
z = zscore(data);

% 多维数据
X = randn(100, 5);
Z = zscore(X);  % 每列标准化

data = rand(100, 1) * 100;

% 归一化到[0, 1]
normalized = (data - min(data)) / (max(data) - min(data));

% 归一化到[a, b]
a = 0;
b = 1;
normalized = (data - min(data)) / (max(data) - min(data)) * (b - a) + a;

% 使用normalize函数
normalized = normalize(data, 'range');

% 最大绝对值标准化
normalized = normalize(data, 'scale');

% L2范数标准化
normalized = normalize(data, 'norm', 2);

% 中位数绝对偏差标准化
normalized = normalize(data, 'medianiqr');

% 对数变换
log_data = log(data);
log_data = log1p(data);  % log(1+x)，避免log(0)

% 对数变换
data = rand(100, 1) * 1000;
log_data = log(data);
log10_data = log10(data);
log2_data = log2(data);

% Box-Cox变换
[transformed, lambda] = boxcox(data);

data = rand(100, 1) * 100;
sqrt_data = sqrt(data);

% 一阶差分
data = cumsum(randn(100, 1));
diff_data = diff(data);

% 二阶差分
diff2_data = diff(data, 2);

% 季节差分
seasonal_diff = data(13:end) - data(1:end-12);

data = sin(linspace(0, 10, 100))' + 0.2*randn(100, 1);

% 简单移动平均
smoothed = movmean(data, 5);

% 加权移动平均
weights = [1 2 3 2 1];
smoothed = movmean(data, weights);

% 中心移动平均
smoothed = movmean(data, 5, 'Endpoints', 'shrink');

% 移动中位数
smoothed = movmedian(data, 5);

% Savitzky-Golay滤波
smoothed = sgolayfilt(data, 3, 11);

% 高斯滤波
smoothed = gaussfilt(data, 2);

% smooth函数
smoothed = smooth(data, 0.1, 'rloess');

% 每隔n个点取一个
data = randn(1000, 1);
downsampled = data(1:10:end);

% 使用downsample函数
downsampled = downsample(data, 10);

% 平均下采样
n = 10;
downsampled = mean(reshape(data, n, []), 1)';

% 线性插值
x = 1:10;
y = sin(x);
x_new = linspace(1, 10, 100);
y_new = interp1(x, y, x_new, 'linear');

% 三次样条插值
y_new = interp1(x, y, x_new, 'spline');

% 使用upsample函数
upsampled = upsample(data, 10);

% 改变采样率
data = sin(linspace(0, 10, 100));
resampled = resample(data, 2, 1);  % 2倍上采样
resampled = resample(data, 1, 2);  % 2倍下采样

function clean_data = preprocess_data(data)
% PREPROCESS_DATA 完整数据预处理
    
    % 1. 处理缺失值
    if any(isnan(data))
        data = fillmissing(data, 'linear');
        fprintf('已填充缺失值\n');
    end
    
    % 2. 处理异常值
    outliers = isoutlier(data);
    if any(outliers)
        data = filloutliers(data, 'median');
        fprintf('已处理异常值: %d个\n', sum(outliers));
    end
    
    % 3. 平滑数据
    data = movmean(data, 5);
    
    % 4. 标准化
    clean_data = zscore(data);
    fprintf('数据已标准化\n');
end

function report = data_quality_report(data)
% DATA_QUALITY_REPORT 生成数据质量报告
    
    n = length(data);
    
    report.total_count = n;
    report.missing_count = sum(isnan(data));
    report.missing_rate = report.missing_count / n * 100;
    
    clean_data = data(~isnan(data));
    report.valid_count = length(clean_data);
    
    outliers = isoutlier(clean_data);
    report.outlier_count = sum(outliers);
    report.outlier_rate = report.outlier_count / report.valid_count * 100;
    
    report.mean = mean(clean_data);
    report.std = std(clean_data);
    report.min = min(clean_data);
    report.max = max(clean_data);
    
    % 打印报告
    fprintf('\n===== 数据质量报告 =====\n');
    fprintf('总数据量: %d\n', report.total_count);
    fprintf('缺失值: %d (%.2f%%)\n', report.missing_count, report.missing_rate);
    fprintf('异常值: %d (%.2f%%)\n', report.outlier_count, report.outlier_rate);
    fprintf('均值: %.4f\n', report.mean);
    fprintf('标准差: %.4f\n', report.std);
    fprintf('范围: [%.4f, %.4f]\n', report.min, report.max);
end