pip install deepchecks scikit-learn joblib

import joblib
import pandas as pd
from deepchecks.tabular import Dataset

# Load artifacts
prod_model = joblib.load("models/production.pkl")
candidate_model = joblib.load("models/candidate.pkl")

test_df = pd.read_parquet("data/held_out_test.parquet")

# Deepchecks Dataset wraps the DataFrame with schema metadata.
# cat_features must be specified for segment checks to work correctly.
# Per deepchecks-tests skill: omitting cat_features causes distribution
# checks to misfire.
test_ds = Dataset(
    test_df,
    label="target",
    cat_features=["region", "plan_tier"],
)

from sklearn.metrics import (
    accuracy_score,
    f1_score,
    roc_auc_score,
    root_mean_squared_error,
)

y_true = test_df["target"].values
X_test = test_df.drop(columns=["target"])

# Classification gate (swap for regression block below as needed)
prod_preds = prod_model.predict(X_test)
cand_preds = candidate_model.predict(X_test)
prod_proba = prod_model.predict_proba(X_test)[:, 1]
cand_proba = candidate_model.predict_proba(X_test)[:, 1]

metrics = {
    "accuracy": (
        accuracy_score(y_true, prod_preds),
        accuracy_score(y_true, cand_preds),
    ),
    "f1_weighted": (
        f1_score(y_true, prod_preds, average="weighted"),
        f1_score(y_true, cand_preds, average="weighted"),
    ),
    "roc_auc": (
        roc_auc_score(y_true, prod_proba),
        roc_auc_score(y_true, cand_proba),
    ),
}

# Regression variant (replace classification block above)
# metrics = {
#     "rmse": (
#         root_mean_squared_error(y_true, prod_model.predict(X_test)),
#         root_mean_squared_error(y_true, candidate_model.predict(X_test)),
#     ),
# }

import sys

# Load from config/gate_thresholds.yaml in practice; hardcoded here for clarity.
TOLERANCES = {
    "accuracy":    0.01,   # candidate may drop at most 1 pp
    "f1_weighted": 0.02,   # candidate may drop at most 2 pp
    "roc_auc":     0.01,   # candidate may drop at most 1 pp
    # "rmse":      0.05,   # candidate RMSE may rise at most 5 % of prod value
}

HIGHER_IS_BETTER = {"accuracy", "f1_weighted", "roc_auc"}

failures = []

for metric, (prod_val, cand_val) in metrics.items():
    tol = TOLERANCES[metric]
    if metric in HIGHER_IS_BETTER:
        regressed = (prod_val - cand_val) > tol
    else:
        regressed = (cand_val - prod_val) > tol * prod_val

    status = "FAIL" if regressed else "PASS"
    print(f"  {metric}: prod={prod_val:.4f}  cand={cand_val:.4f}  [{status}]")
    if regressed:
        failures.append(
            f"{metric}: candidate {cand_val:.4f} regressed vs prod {prod_val:.4f}"
            f" (tolerance {tol})"
        )

if failures:
    print("\nGate FAILED:")
    for f in failures:
        print(f"  {f}")
    sys.exit(1)

print("\nGlobal metric gate PASSED.")

from deepchecks.tabular.checks import WeakSegmentsPerformance

seg_check = WeakSegmentsPerformance(
    segment_minimum_size_ratio=0.05,  # ignore segments smaller than 5 %
)
# Gate: no segment may perform more than 15 % below the dataset average.
seg_check.add_condition_segments_relative_performance_greater_than(
    max_ratio_change=0.15
)

seg_result = seg_check.run(test_ds, candidate_model)
seg_result.save_as_html("segment_report_candidate.html")

if not seg_result.passed_conditions():
    print("Segment gate FAILED: candidate regresses on at least one slice.")
    sys.exit(1)

print("Segment gate PASSED.")

from deepchecks.tabular.checks import TrainTestPerformance
from deepchecks.tabular import Dataset

train_df = pd.read_parquet("data/train.parquet")
train_ds = Dataset(train_df, label="target", cat_features=["region", "plan_tier"])

ttp_check = TrainTestPerformance(
    scorers=["f1_macro", "recall_per_class", "precision_per_class"]
)
ttp_check.add_condition_train_test_relative_degradation_less_than(0.15)

ttp_result = ttp_check.run(train_ds, test_ds, candidate_model)
ttp_result.save_as_html("train_test_performance.html")

if not ttp_result.passed_conditions():
    print("Train-test degradation gate FAILED.")
    sys.exit(1)

print("Train-test degradation gate PASSED.")

jobs:
  model-regression-gate:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4

      - name: Install dependencies
        run: pip install deepchecks scikit-learn joblib

      - name: Download model artifacts
        run: |
          aws s3 cp s3://my-bucket/models/production.pkl models/production.pkl
          aws s3 cp s3://my-bucket/models/candidate.pkl  models/candidate.pkl

      - name: Run regression gate
        run: python ml/regression_gate.py

      - name: Upload reports
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: model-regression-reports
          path: "*.html"