Supervised LearningFoundations, Regression & Classification

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.datasets import fetch_california_housing

# ── Load Dataset ──────────────────────────────────────────
housing = fetch_california_housing(as_frame=True)
df = housing.frame

print("Dataset Shape:", df.shape)
print("\nFirst 5 rows:")
print(df.head())

print("\nStatistical Summary:")
print(df.describe())

# ── Check for Missing Values ──────────────────────────────
print("\nMissing Values:")
print(df.isnull().sum())

# ── Visualize Feature Correlations ───────────────────────
plt.figure(figsize=(12, 8))
sns.heatmap(df.corr(), annot=True, fmt=".2f", cmap="coolwarm", linewidths=0.5)
plt.title("Feature Correlation Heatmap", fontsize=16, pad=20)
plt.tight_layout()
plt.savefig("correlation_heatmap.png", dpi=150)
plt.show()

# ── Distribution of Target Variable ──────────────────────
plt.figure(figsize=(8, 5))
sns.histplot(df["MedHouseVal"], bins=50, kde=True, color="#4C72B0")
plt.title("Distribution of Median House Value")
plt.xlabel("Median House Value (×$100,000)")
plt.tight_layout()
plt.show()

from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler

# ── Separate Features and Target ─────────────────────────
X = df.drop("MedHouseVal", axis=1)
y = df["MedHouseVal"]

# ── Train/Validation/Test Split ──────────────────────────
# 70% train, 15% validation, 15% test
X_train, X_temp, y_train, y_temp = train_test_split(
    X, y, test_size=0.30, random_state=42
)
X_val, X_test, y_val, y_test = train_test_split(
    X_temp, y_temp, test_size=0.50, random_state=42
)

print(f"Training set:   {X_train.shape[0]} samples")
print(f"Validation set: {X_val.shape[0]} samples")
print(f"Test set:       {X_test.shape[0]} samples")

# ── Feature Scaling (StandardScaler) ─────────────────────
# Formula: z = (x - μ) / σ   (zero mean, unit variance)
scaler = StandardScaler()

# IMPORTANT: Fit ONLY on training data to prevent data leakage!
X_train_scaled = scaler.fit_transform(X_train)
X_val_scaled   = scaler.transform(X_val)     # Only transform, never fit
X_test_scaled  = scaler.transform(X_test)    # Only transform, never fit

print("\nScaling complete. Feature means (should be ~0):")
print(np.round(X_train_scaled.mean(axis=0), 3))

from sklearn.linear_model import LinearRegression, Ridge, Lasso
from sklearn.ensemble import RandomForestRegressor, GradientBoostingRegressor
from sklearn.metrics import mean_squared_error, r2_score, mean_absolute_error

def evaluate_model(name, model, X_tr, y_tr, X_v, y_v):
    """Train a model and print key regression metrics."""
    model.fit(X_tr, y_tr)
    preds = model.predict(X_v)

    rmse = np.sqrt(mean_squared_error(y_v, preds))
    mae  = mean_absolute_error(y_v, preds)
    r2   = r2_score(y_v, preds)

    print(f"\n── {name} ──")
    print(f"   RMSE : {rmse:.4f}")
    print(f"   MAE  : {mae:.4f}")
    print(f"   R²   : {r2:.4f}")
    return model, preds

# ── Define Models ─────────────────────────────────────────
models = [
    ("Linear Regression",       LinearRegression()),
    ("Ridge Regression (L2)",   Ridge(alpha=1.0)),
    ("Lasso Regression (L1)",   Lasso(alpha=0.01)),
    ("Random Forest",           RandomForestRegressor(n_estimators=100, random_state=42)),
    ("Gradient Boosting",       GradientBoostingRegressor(
        n_estimators=200, learning_rate=0.1, random_state=42
    )),
]

results = {}
for name, model in models:
    trained_model, preds = evaluate_model(
        name, model,
        X_train_scaled, y_train,
        X_val_scaled, y_val
    )
    results[name] = (trained_model, preds)

from sklearn.model_selection import GridSearchCV

# ── Define the search space ───────────────────────────────
param_grid = {
    "n_estimators":      [100, 200, 300],
    "max_depth":         [3, 5, 7, None],
    "min_samples_split": [2, 5, 10],
}

rf = RandomForestRegressor(random_state=42)

# 5-fold cross-validation with negative MSE as scoring
grid_search = GridSearchCV(
    estimator=rf,
    param_grid=param_grid,
    cv=5,
    scoring="neg_mean_squared_error",
    n_jobs=-1,     # Use all available CPU cores
    verbose=1,
)

grid_search.fit(X_train_scaled, y_train)
print("\n✅ Best Parameters Found:")
print(grid_search.best_params_)

best_rf = grid_search.best_estimator_
test_preds = best_rf.predict(X_test_scaled)
final_r2 = r2_score(y_test, test_preds)
final_rmse = np.sqrt(mean_squared_error(y_test, test_preds))

print(f"\n🏆 Final Test Set Performance (Best Random Forest):")
print(f"   RMSE : {final_rmse:.4f}")
print(f"   R²   : {final_r2:.4f}")

# ── Feature Importance Plot ───────────────────────────────
importances = pd.Series(best_rf.feature_importances_, index=X.columns)
importances.sort_values(ascending=True).plot(
    kind="barh", figsize=(10, 6),
    color="#2C7BB6", title="Feature Importances — Random Forest"
)
plt.tight_layout()
plt.savefig("feature_importances.png", dpi=150)
plt.show()

from sklearn.datasets import load_iris
from sklearn.svm import SVC
from sklearn.metrics import classification_report, confusion_matrix
from sklearn.model_selection import StratifiedKFold, cross_val_score

# ── Load Iris Dataset ──────────────────────────────────────
iris = load_iris(as_frame=True)
X_iris, y_iris = iris.data, iris.target

X_tr, X_te, y_tr, y_te = train_test_split(
    X_iris, y_iris, test_size=0.20, random_state=42, stratify=y_iris
)

sc = StandardScaler()
X_tr_sc = sc.fit_transform(X_tr)
X_te_sc = sc.transform(X_te)

# ── Train SVM with RBF Kernel ─────────────────────────────
svm = SVC(kernel="rbf", C=10.0, gamma="scale", probability=True)
svm.fit(X_tr_sc, y_tr)

y_pred = svm.predict(X_te_sc)

print("📊 Classification Report:")
print(classification_report(y_te, y_pred, target_names=iris.target_names))

# ── Confusion Matrix ──────────────────────────────────────
cm = confusion_matrix(y_te, y_pred)
plt.figure(figsize=(6, 5))
sns.heatmap(cm, annot=True, fmt="d", cmap="Blues",
    xticklabels=iris.target_names, yticklabels=iris.target_names)
plt.title("Confusion Matrix — SVM Classifier")
plt.ylabel("Actual"); plt.xlabel("Predicted")
plt.tight_layout()
plt.savefig("confusion_matrix.png", dpi=150)
plt.show()

# ── 10-Fold Stratified Cross-Validation ──────────────────
cv = StratifiedKFold(n_splits=10, shuffle=True, random_state=42)
cv_scores = cross_val_score(svm, sc.fit_transform(X_iris), y_iris, cv=cv, scoring="accuracy")
print(f"\n10-Fold CV Accuracy: {cv_scores.mean():.4f} ± {cv_scores.std():.4f}")