数据清洗实战Data Cleaning

import hashlib
import polars as pl

raw = pl.read_csv("data/dirty_finance.csv")
clean = (
    raw.with_columns(
        pl.col("date").str.to_date(strict=False),
        pl.col("amount").str.replace_all(",", "").cast(pl.Float64, strict=False),
        pl.col("phone").str.replace(r"(\d{3})\d+(\d{2})", r"$1****$2"),
    )
    .unique(subset=["record_id"], keep="last")
    .filter(pl.col("date").is_not_null() & pl.col("amount").is_not_null())
    .sort("record_id")
)
clean.write_parquet("output/clean.parquet")
print(clean.shape, hashlib.sha256(open("output/clean.parquet", "rb").read()).hexdigest())