freshdata_ai_copilot_demo.py

"""FreshData AI Copilot: messy customer data -> clean, private, audit-ready.

The flagship end-to-end story. A deliberately messy customer export
(``examples/data/messy_customers.csv`` — duplicates, sentinel strings,
invalid ages, negative salaries, mixed date formats, variant city/plan
spellings, raw emails and phone numbers) goes through four stages:

1. **Analyze** — the experimental AI Copilot produces a deterministic,
   offline report: ranked problems, a PII warning, context-policy
   violations, an explainable cleaning plan, and copy-ready code.
2. **Mask** — PII is hashed before anything else happens, so no later
   stage (or model) ever sees a raw identifier.
3. **Clean** — business rules compiled from plain English are enforced by
   ``fd.clean``, with every action recorded in the audit trail.
4. **Verify** — near-duplicate categories are merged, the Data Trust
   Score is re-computed, and the policy is re-validated to prove the
   violations are gone.

Run:

    python examples/freshdata_ai_copilot_demo.py

No network access, API key, or optional dependency is required.
"""

from pathlib import Path

import pandas as pd

import freshdata as fd
from freshdata.enterprise import MaskingRule, compute_trust_score, merge_clusters
from freshdata.experimental.ai_copilot import analyze_dataset

DATA = Path(__file__).parent / "data" / "messy_customers.csv"

RULES_IN_ENGLISH = "age must be between 0 and 120. salary must be at least 0."


def banner(title: str) -> None:
    print(f"\n{'=' * 72}\n{title}\n{'=' * 72}")


def main() -> None:
    df = pd.read_csv(DATA)

    # ------------------------------------------------------------------ 1
    banner("1) ANALYZE — AI Copilot report (deterministic, offline)")
    report = analyze_dataset(
        df,
        goal="Prepare this customer dataset for analytics and ML",
        privacy="mask_pii_before_reasoning",
        context_policy={
            "email": "must_mask",
            "phone": "must_mask",
            "age": "must_be_between_0_and_120",
            "salary": "must_be_positive",
            "city": "normalize_spelling",
        },
        source_hint="examples/data/messy_customers.csv",
    )
    print(report.summary)
    print()
    print(report.cleaning_plan)

    banner("Copy-ready pipeline generated by the copilot")
    print(report.recommended_code)

    # The model context is the ONLY payload an LLM provider would ever see.
    # Prove no raw PII is in it (or anywhere else in the report):
    leaked = "alice.johnson@example.com" in report.to_json()
    print(f"\nraw PII anywhere in the report/model context? {leaked}")

    # ------------------------------------------------------------------ 2
    banner("2) MASK — hash PII before any other processing")
    trust_before = report.trust
    masked, privacy_report = fd.anonymize(
        df,
        rules=(
            MaskingRule(name="mask_email", columns=("email",), strategy="hash"),
            MaskingRule(name="mask_phone", columns=("phone",), strategy="hash"),
        ),
    )
    print(privacy_report.summary())
    print("masked sample:", masked.loc[0, "email"])

    # ------------------------------------------------------------------ 3
    banner("3) CLEAN — enforce plain-English rules with a full audit trail")
    policy = fd.compile_context(RULES_IN_ENGLISH, df=masked)
    print(policy.summary(), "\n")
    cleaned, clean_report = fd.clean(masked, policy=policy, return_report=True, outliers="clip")
    print(clean_report.summary())

    # ------------------------------------------------------------------ 4
    banner("4) VERIFY — normalize categories, re-score trust, re-validate")
    cleaned, cluster_results = merge_clusters(cleaned, columns=["city", "plan", "churned"])
    merged = sum(r.n_cells_merged for r in cluster_results)
    print(f"category spellings merged: {merged} cell(s) across city/plan/churned")
    print("city values now:", ", ".join(sorted(cleaned["city"].dropna().unique())[:8]), "…")

    trust_after = compute_trust_score(cleaned)
    remaining = fd.validate(cleaned, policy=policy)

    banner("BEFORE / AFTER")
    rows = [
        ("rows", len(df), len(cleaned)),
        ("missing cells", int(df.isna().sum().sum()), int(cleaned.isna().sum().sum())),
        ("duplicate rows", int(df.duplicated().sum()), int(cleaned.duplicated().sum())),
        ("policy violations", len(report.policy_violations), len(remaining.errors)),
        ("PII cells exposed", privacy_report.cells_changed, 0),
        (
            "Trust Score",
            f"{trust_before.overall:.1f} (grade {trust_before.grade})",
            f"{trust_after.overall:.1f} (grade {trust_after.grade})",
        ),
    ]
    width = max(len(r[0]) for r in rows)
    for label, before, after in rows:
        print(f"  {label:<{width}}   {before!s:>20}  ->  {after!s}")

    banner("AUDIT TRAIL — every decision, explained")
    for action in clean_report.actions[:6]:
        why = f"  (why: {action.rationale})" if action.rationale else ""
        print(f"  [{action.step}] {action.column or '<table>'}: {action.description}{why}")
    print(f"  … {len(clean_report.actions)} actions total; export with report.to_json()")
    print(
        f"\ncopilot audit: engine={report.audit['engine']}, "
        f"model_context sha256={report.audit['model_context_sha256'][:16]}…"
    )


if __name__ == "__main__":
    main()