Teacher-student models with feed

In [2]:

!uv pip install lightgbm scikit-learn pandas pyarrow matplotlib ucimlrepo --quiet

!uv pip install lightgbm scikit-learn pandas pyarrow matplotlib ucimlrepo --quiet

In [ ]:

import lightgbm as lgbm
from lefts.interface import leaf, feed, ensemble, split
from lefts.helpers import tabular_model

import polars as pl

from functools import partial
from ucimlrepo import fetch_ucirepo

import lightgbm as lgbm from lefts.interface import leaf, feed, ensemble, split from lefts.helpers import tabular_model import polars as pl from functools import partial from ucimlrepo import fetch_ucirepo

In [4]:

dataset = fetch_ucirepo(id=601)
data = pl.from_pandas(dataset.data.original)
features = dataset.data.features.columns
target = "Machine failure"

dataset = fetch_ucirepo(id=601) data = pl.from_pandas(dataset.data.original) features = dataset.data.features.columns target = "Machine failure"

In [5]:

# Convert String features to categoricals so that LGBM doesn't complain
for feature in data.columns:
    if data.schema[feature] == pl.String:
        data = data.with_columns(pl.col(feature).cast(pl.Enum(data[feature].unique())))

# Convert String features to categoricals so that LGBM doesn't complain for feature in data.columns: if data.schema[feature] == pl.String: data = data.with_columns(pl.col(feature).cast(pl.Enum(data[feature].unique())))

In [6]:

# Give ourselves a row index to keep track of train and test splits
data = data.with_row_index(name="row")

# Give ourselves a row index to keep track of train and test splits data = data.with_row_index(name="row")

In [ ]:

numeric_types = {pl.Int8, pl.Int16, pl.Int32, pl.Int64, pl.Float32, pl.Float64}
features_list = [f for f in features if data.schema[f] in numeric_types]

teacher_hyperparams = dict(
    n_estimators=500, max_depth=6, learning_rate=0.05, num_leaves=63, verbose=-1
)
student_hyperparams = dict(
    n_estimators=25, max_depth=3, learning_rate=0.05, num_leaves=15, verbose=-1
)

teacher = leaf(
    tabular_model(
        partial(lgbm.LGBMClassifier, **teacher_hyperparams),
        features=features_list,
        target=target,
    ),
    label="teacher",
)

student = leaf(
    tabular_model(
        partial(lgbm.LGBMRegressor, **student_hyperparams),
        features=features_list,
        target="teacher",
    ),
    label="student",
)

untutored = leaf(
    tabular_model(
        partial(lgbm.LGBMClassifier, **student_hyperparams),
        features=features_list,
        target=target,
    ),
    label="untutored_student",
)

numeric_types = {pl.Int8, pl.Int16, pl.Int32, pl.Int64, pl.Float32, pl.Float64} features_list = [f for f in features if data.schema[f] in numeric_types] teacher_hyperparams = dict( n_estimators=500, max_depth=6, learning_rate=0.05, num_leaves=63, verbose=-1 ) student_hyperparams = dict( n_estimators=25, max_depth=3, learning_rate=0.05, num_leaves=15, verbose=-1 ) teacher = leaf( tabular_model( partial(lgbm.LGBMClassifier, **teacher_hyperparams), features=features_list, target=target, ), label="teacher", ) student = leaf( tabular_model( partial(lgbm.LGBMRegressor, **student_hyperparams), features=features_list, target="teacher", ), label="student", ) untutored = leaf( tabular_model( partial(lgbm.LGBMClassifier, **student_hyperparams), features=features_list, target=target, ), label="untutored_student", )

In [10]:

distillation = feed("distillation", source=teacher, consumer=student)
all_models = ensemble("full", distillation, untutored)

distillation = feed("distillation", source=teacher, consumer=student) all_models = ensemble("full", distillation, untutored)

In [11]:

full_experiment = split(
    name="train_test_split",
    model=all_models,
    train_filter=pl.col("row") < pl.col("row").max() * 0.75,
    test_filter=pl.lit(True),  # Generate a prediction on every row so we can see
)

full_experiment = split( name="train_test_split", model=all_models, train_filter=pl.col("row") < pl.col("row").max() * 0.75, test_filter=pl.lit(True), # Generate a prediction on every row so we can see )

In [13]:

full_experiment.fit(data);

full_experiment.fit(data);

In [14]:

fitted = full_experiment.predict(data)
fitted = fitted.with_columns(student_predictions=pl.col("student").clip(0, 1))

fitted = full_experiment.predict(data) fitted = fitted.with_columns(student_predictions=pl.col("student").clip(0, 1))

In [15]:

import polars.selectors as cs

fitted.with_columns(
    *[
        (pl.col(c) - pl.col(target)).alias(f"residual_{c}")
        for c in ["teacher", "student_predictions", "untutored_student"]
    ]
).select(cs.contains("residual").pow(2).mean())

import polars.selectors as cs fitted.with_columns( *[ (pl.col(c) - pl.col(target)).alias(f"residual_{c}") for c in ["teacher", "student_predictions", "untutored_student"] ] ).select(cs.contains("residual").pow(2).mean())

Out[15]:

shape: (1, 3)

residual_teacher	residual_student_predictions	residual_untutored_student
f64	f64	f64
0.0027	0.021012	0.0266

In [15]: