LGBM rolling retrain with early stopping

In [1]:

!uv pip install lightgbm scikit-learn pandas pyarrow matplotlib --quiet

!uv pip install lightgbm scikit-learn pandas pyarrow matplotlib --quiet

In [2]:

import lightgbm as lgbm
from lefts.interface import leaf, lift

import polars as pl
import datetime as dt

import lightgbm as lgbm from lefts.interface import leaf, lift import polars as pl import datetime as dt

Load dataset¶

In [3]:

from sklearn.datasets import fetch_california_housing

raw = fetch_california_housing(as_frame=True)

month_starts = [dt.date(2020, m, 1) for m in range(1, 13)]

data = (
    pl.from_pandas(raw.frame)
    .rename({"MedHouseVal": "price"})
    .sample(n=2400, seed=42)
    .with_row_index("_i")
    .with_columns(date=pl.Series("date", [month_starts[i // 200] for i in range(2400)]))
    .drop("_i")
)

features = [c for c in data.columns if c not in ("price", "date")]
target = "price"

from sklearn.datasets import fetch_california_housing raw = fetch_california_housing(as_frame=True) month_starts = [dt.date(2020, m, 1) for m in range(1, 13)] data = ( pl.from_pandas(raw.frame) .rename({"MedHouseVal": "price"}) .sample(n=2400, seed=42) .with_row_index("_i") .with_columns(date=pl.Series("date", [month_starts[i // 200] for i in range(2400)])) .drop("_i") ) features = [c for c in data.columns if c not in ("price", "date")] target = "price"

Set up Model¶

In [5]:

class LGBMWrapper:
    def __init__(self, features, target, **kwargs):
        self.model = lgbm.LGBMRegressor(**kwargs)
        self.features = features
        self.target = target

    def fit(self, training_set: pl.DataFrame, validation_set: pl.DataFrame = None):

        X_train = training_set.select(*self.features).to_pandas()
        y_train = training_set[self.target].to_pandas()

        eval_set = [(X_train, y_train)]
        eval_names = ["training"]

        if validation_set is not None:
            X_val = validation_set.select(*self.features).to_pandas()
            y_val = validation_set[self.target].to_pandas()
            eval_set += [(X_val, y_val)]
            eval_names += ["validation"]

        self.model.fit(X_train, y_train, eval_set=eval_set, eval_names=eval_names)

    def predict(self, df: pl.DataFrame):
        X = df.select(*self.features).to_pandas()
        return self.model.predict(X)

class LGBMWrapper: def __init__(self, features, target, **kwargs): self.model = lgbm.LGBMRegressor(**kwargs) self.features = features self.target = target def fit(self, training_set: pl.DataFrame, validation_set: pl.DataFrame = None): X_train = training_set.select(*self.features).to_pandas() y_train = training_set[self.target].to_pandas() eval_set = [(X_train, y_train)] eval_names = ["training"] if validation_set is not None: X_val = validation_set.select(*self.features).to_pandas() y_val = validation_set[self.target].to_pandas() eval_set += [(X_val, y_val)] eval_names += ["validation"] self.model.fit(X_train, y_train, eval_set=eval_set, eval_names=eval_names) def predict(self, df: pl.DataFrame): X = df.select(*self.features).to_pandas() return self.model.predict(X)

In [6]:

from functools import partial

parameterised_model = partial(
    LGBMWrapper,
    features=features,
    target=target,
    num_leaves=31,
    max_depth=5,
    learning_rate=0.05,
    n_estimators=500,
    early_stopping_round=10,
)

from functools import partial parameterised_model = partial( LGBMWrapper, features=features, target=target, num_leaves=31, max_depth=5, learning_rate=0.05, n_estimators=500, early_stopping_round=10, )

In [7]:

model = leaf(label="lgbm", model_constructor=parameterised_model)

model = leaf(label="lgbm", model_constructor=parameterised_model)

In [8]:

# Quarterly retraining — each model trains on all data before its cutoff,
# validates on the following month, and predicts on the rest.
retrain_points = [dt.date(2020, 4, 1), dt.date(2020, 7, 1), dt.date(2020, 10, 1)]
validation_period_length = dt.timedelta(days=27)

rolling_train = lift(
    model=model,
    values=retrain_points,
    train_filter=lambda d: pl.col("date") < d,
    validation_filter=lambda d: pl.col("date").is_between(
        d, d + validation_period_length
    ),
    test_filter=lambda d: pl.col("date") > d + validation_period_length,
    name="RollingRetrain",
)

# Quarterly retraining — each model trains on all data before its cutoff, # validates on the following month, and predicts on the rest. retrain_points = [dt.date(2020, 4, 1), dt.date(2020, 7, 1), dt.date(2020, 10, 1)] validation_period_length = dt.timedelta(days=27) rolling_train = lift( model=model, values=retrain_points, train_filter=lambda d: pl.col("date") < d, validation_filter=lambda d: pl.col("date").is_between( d, d + validation_period_length ), test_filter=lambda d: pl.col("date") > d + validation_period_length, name="RollingRetrain", )

In [9]:

rolling_train.fit(data);

rolling_train.fit(data);

In [11]:

#  Access the eval curves from the last retrain fold
evals = rolling_train.fitted["lgbm[RollingRetrain=2020-10-01]"].model.evals_result_
pl.Series(evals["training"]["l2"]).to_pandas().plot(legend=True, label="training")
pl.Series(evals["validation"]["l2"]).to_pandas().plot(legend=True, label="validation")

# Access the eval curves from the last retrain fold evals = rolling_train.fitted["lgbm[RollingRetrain=2020-10-01]"].model.evals_result_ pl.Series(evals["training"]["l2"]).to_pandas().plot(legend=True, label="training") pl.Series(evals["validation"]["l2"]).to_pandas().plot(legend=True, label="validation")

Out[11]:

<Axes: >

In [12]:

# Predictions from each fold in separate columns
rolling_train.predict(data)

# Predictions from each fold in separate columns rolling_train.predict(data)

Out[12]:

shape: (2_400, 13)

MedInc	HouseAge	AveRooms	AveBedrms	Population	AveOccup	Latitude	Longitude	price	date	lgbm[RollingRetrain=2020-04-01]	lgbm[RollingRetrain=2020-07-01]	lgbm[RollingRetrain=2020-10-01]
f64	f64	f64	f64	f64	f64	f64	f64	f64	date	f64	f64	f64
5.6431	52.0	5.817352	1.073059	558.0	2.547945	37.85	-122.25	3.413	2020-01-01	null	null	null
3.8462	52.0	6.281853	1.081081	565.0	2.181467	37.85	-122.25	3.422	2020-01-01	null	null	null
4.0368	52.0	4.761658	1.103627	413.0	2.139896	37.85	-122.25	2.697	2020-01-01	null	null	null
1.6424	50.0	4.401691	1.040169	1131.0	2.391121	37.84	-122.28	1.089	2020-01-01	null	null	null
1.5045	43.0	4.589681	1.120393	1061.0	2.60688	37.82	-122.27	0.938	2020-01-01	null	null	null
…	…	…	…	…	…	…	…	…	…	…	…	…
2.7989	27.0	5.922179	1.099222	1583.0	3.079767	38.52	-121.98	1.267	2020-12-01	1.062776	1.369759	1.051092
3.4187	26.0	5.230769	0.942308	194.0	3.730769	38.83	-122.0	0.984	2020-12-01	1.124089	1.576896	1.06289
1.4934	26.0	5.157303	1.082397	761.0	2.850187	39.08	-121.56	0.483	2020-12-01	0.643526	1.046847	0.696813
2.4167	20.0	4.808917	0.936306	457.0	2.910828	39.0	-121.44	0.67	2020-12-01	0.887503	1.3	0.810171
2.3886	16.0	5.254717	1.162264	1387.0	2.616981	39.37	-121.24	0.894	2020-12-01	0.857328	1.251356	0.877626