Responsible AI at Scale ⚖️

# Responsible AI Audit with Fairlearn — Google Colab
!pip install fairlearn scikit-learn pandas matplotlib -q

import numpy as np
import pandas as pd
from sklearn.datasets import make_classification
from sklearn.model_selection import train_test_split
from sklearn.ensemble import GradientBoostingClassifier
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import StandardScaler
from sklearn.impute import SimpleImputer
from sklearn.metrics import accuracy_score, confusion_matrix
from fairlearn.metrics import (
    MetricFrame, demographic_parity_difference,
    equalized_odds_difference, false_positive_rate
)
from fairlearn.postprocessing import ThresholdOptimizer
import matplotlib.pyplot as plt

# ── Step 1: Generate biased dataset ──
np.random.seed(42)
n = 2000

# Rural group (40% of sample, worse features due to historical underbanking)
urban_mask = np.random.choice([0, 1], size=n, p=[0.4, 0.6])
X = np.column_stack([
    np.random.normal(50, 15, n) + urban_mask * 10,  # income higher in urban
    np.random.normal(200, 80, n) - urban_mask * 20, # loan amount
    np.random.normal(650, 80, n) + urban_mask * 40, # credit score biased up for urban
    np.random.normal(5, 3, n).clip(0),              # employment years
    np.random.randint(0, 4, n).astype(float)        # existing loans
])
# Ground truth default: mainly income and loan ratio, NOT location
true_default = ((X[:, 0] / X[:, 1]) < 0.3).astype(int)

df = pd.DataFrame(X, columns=['income', 'loan_amount', 'credit_score',
                                'employment_years', 'existing_loans'])
df['is_urban'] = urban_mask
df['default']  = true_default

X_train, X_test, y_train, y_test, grp_train, grp_test = train_test_split(
    df.drop(['is_urban', 'default'], axis=1),
    df['default'], df['is_urban'],
    test_size=0.3, random_state=42
)

# ── Step 2: Train baseline model ──
pipeline = Pipeline([
    ('imputer', SimpleImputer(strategy='median')),
    ('scaler',  StandardScaler()),
    ('model',   GradientBoostingClassifier(n_estimators=100, random_state=42))
])
pipeline.fit(X_train, y_train)
y_pred = pipeline.predict(X_test)

print("=== BASELINE MODEL AUDIT ===")
print(f"Overall accuracy: {accuracy_score(y_test, y_pred):.3f}")

# ── Step 3: Fairness metrics with MetricFrame ──
mf = MetricFrame(
    metrics={
        "accuracy":           accuracy_score,
        "false_positive_rate": false_positive_rate
    },
    y_true=y_test,
    y_pred=y_pred,
    sensitive_features=grp_test.map({0: "Rural", 1: "Urban"})
)

print("\nAccuracy by group:")
print(mf.by_group["accuracy"].to_string())
print("\nFalse Positive Rate by group (incorrectly flagged as default):")
print(mf.by_group["false_positive_rate"].to_string())
print(f"\nDemographic Parity Difference:  {demographic_parity_difference(y_test, y_pred, sensitive_features=grp_test):.3f}")
print(f"Equalised Odds Difference:       {equalized_odds_difference(y_test, y_pred, sensitive_features=grp_test):.3f}")
# 0 = perfectly fair | 1 = maximally unfair

# ── Step 4: Mitigate bias with ThresholdOptimizer ──
# Adjust decision thresholds per group to equalise FPR
mitigator = ThresholdOptimizer(
    estimator=pipeline,
    constraints="equalized_odds",
    objective="accuracy_score",
    predict_method="auto"
)
mitigator.fit(X_train, y_train, sensitive_features=grp_train)
y_pred_fair = mitigator.predict(X_test, sensitive_features=grp_test)

print("\n=== AFTER FAIRNESS MITIGATION ===")
mf_fair = MetricFrame(
    metrics={"accuracy": accuracy_score, "false_positive_rate": false_positive_rate},
    y_true=y_test, y_pred=y_pred_fair,
    sensitive_features=grp_test.map({0: "Rural", 1: "Urban"})
)
print(f"Overall accuracy: {accuracy_score(y_test, y_pred_fair):.3f} (may decrease slightly)")
print("\nAccuracy by group:")
print(mf_fair.by_group["accuracy"].to_string())
print("\nFalse Positive Rate by group (after mitigation):")
print(mf_fair.by_group["false_positive_rate"].to_string())
print(f"\nEqualised Odds Difference after mitigation: {equalized_odds_difference(y_test, y_pred_fair, sensitive_features=grp_test):.3f}")