异常值处理

IQR 与 Z-score 异常值检测与处理。

核心代码

def handle_outliers(
    *,
    df: pd.DataFrame,
    params: dict[str, Any],
    column_descriptions: dict[str, str],
) -> tuple[pd.DataFrame, dict[str, Any], list[str], dict[str, str]]:
    """异常值检测、删除、缩尾和标记。"""
    columns = require_numeric_columns(df, normalize_string_list(params.get('columns')), label=_('异常值变量'))
    method = str(params.get('method') or 'iqr').strip().lower()
    action = str(params.get('action') or 'winsorize').strip().lower()
    result_df = df.copy()
    bounds: dict[str, dict[str, float | None]] = {}
    combined_mask = pd.Series(False, index=result_df.index)

    boxplot_stats: dict[str, dict[str, float | None]] = {}

    for column in columns:
        series = pd.to_numeric(result_df[column], errors='coerce')
        lower, upper = _outlier_bounds(series, method=method, params=params)
        column_mask = pd.Series(False, index=result_df.index)
        if lower is not None:
            column_mask = column_mask | (series < lower)
        if upper is not None:
            column_mask = column_mask | (series > upper)
        combined_mask = combined_mask | column_mask
        bounds[column] = {
            'lower': None if lower is None or np.isnan(lower) else float(lower),
            'upper': None if upper is None or np.isnan(upper) else float(upper),
            'outlier_rows': int(column_mask.sum()),
        }
        boxplot_stats[column] = _build_boxplot_stats(
            series=series,
            lower=lower,
            upper=upper,
            outlier_rows=int(column_mask.sum()),
        )

        if action == 'winsorize':
            if lower is not None:
                result_df.loc[series < lower, column] = lower
            if upper is not None:
                result_df.loc[series > upper, column] = upper
        elif action == 'indicator':
            indicator_name = ensure_new_column_absent(result_df, f'{column}_outlier')
            result_df[indicator_name] = column_mask.astype(int)
        elif action == 'remove':
            continue
        else:
            raise DataProcessingValidationError(_('异常值操作仅支持 remove、winsorize、indicator'))

    if action == 'remove':
        result_df = result_df.loc[~combined_mask].reset_index(drop=True)

    details = {
        'action': action,
        'method': method,
        'columns': columns,
        'bounds': bounds,
        'boxplot_stats': boxplot_stats,
        'affected_rows': int(combined_mask.sum()),
    }
    warnings = []
    if int(combined_mask.sum()) == 0:
        warnings.append(_('未检测到异常值'))
    return result_df, details, warnings, column_descriptions