NGBoost

Case 1: 線形データ・不均一分散

import numpy as np
import matplotlib.pyplot as plt
from scipy.stats import norm

x = np.linspace(0, 10, 1000)
sigma = np.sqrt(x)
y = norm.rvs(loc=x, scale=sigma, random_state=0)
X = x.reshape(-1, 1)

fig, ax = plt.subplots()
ax.scatter(x, y)
ax.plot(x, x, color="black", alpha=.5, label="mean")
ax.set(xlabel="x", ylabel="y")
ax.legend()
fig.show()

from ngboost import NGBRegressor
from sklearn.model_selection import train_test_split
from sklearn.metrics import mean_squared_error

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)

ngb = NGBRegressor().fit(X_train, y_train)
y_pred = ngb.predict(X_test)
y_dist = ngb.pred_dist(X_test)

print('Test MSE', mean_squared_error(y_pred, y_test))

# test Negative Log Likelihood
test_NLL = -y_dist.logpdf(y_test).mean()
print('Test NLL', test_NLL)

[iter 0] loss=2.7044 val_loss=0.0000 scale=1.0000 norm=3.0710

[iter 100] loss=2.1347 val_loss=0.0000 scale=2.0000 norm=3.3846

[iter 200] loss=1.9511 val_loss=0.0000 scale=2.0000 norm=3.1829

[iter 300] loss=1.8825 val_loss=0.0000 scale=2.0000 norm=3.1140

[iter 400] loss=1.8434 val_loss=0.0000 scale=2.0000 norm=3.0608

Test MSE 6.8069964328899895
Test NLL 2.4322070624337173

fig, ax = plt.subplots()
ax.scatter(x, y, alpha=.5)
ax.plot(x, x, color="black", alpha=.5, label="mean")
ax.set(xlabel="x", ylabel="y")
ax.legend()

X_test = np.sort(X_test, axis=0)
y_dist = ngb.pred_dist(X_test)

alphas = [0.05, 0.01]
colors = ["darkorange", "tomato"]
for alpha, color in zip(alphas, colors):
    upper = norm.ppf(q=1 - (alpha/2), loc=y_dist.params["loc"], scale=y_dist.params["scale"])
    lower = norm.ppf(q=(alpha/2), loc=y_dist.params["loc"], scale=y_dist.params["scale"])
    ax.plot(X_test[:, 0], upper, alpha=.9, color=color, linestyle="--", label=rf"$\alpha$={alpha}")
    ax.plot(X_test[:, 0], lower, alpha=.9, color=color, linestyle="--")

ax.legend()
fig.show()

Case 2: 非線形データ・不均一分散

import numpy as np
import matplotlib.pyplot as plt
from scipy.stats import norm

x = np.linspace(0, 5, 1000)
sigma = (np.sin(x / 1) + 2) * 5
z = 10 + x + x ** 2
y = norm.rvs(loc=z, scale=sigma, random_state=0)
X = x.reshape(-1, 1)

fig, ax = plt.subplots()
ax.scatter(x, y)
ax.plot(x, z, color="black", alpha=.5, label="mean")
ax.set(xlabel="x", ylabel="y")
ax.legend()
fig.show()

from ngboost import NGBRegressor
from sklearn.model_selection import train_test_split
from sklearn.metrics import mean_squared_error

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)

ngb = NGBRegressor().fit(X_train, y_train)
y_pred = ngb.predict(X_test)
y_dist = ngb.pred_dist(X_test)

print('Test MSE', mean_squared_error(y_pred, y_test))

# test Negative Log Likelihood
test_NLL = -y_dist.logpdf(y_test).mean()
print('Test NLL', test_NLL)

[iter 0] loss=4.0900 val_loss=0.0000 scale=1.0000 norm=11.9177

[iter 100] loss=3.7139 val_loss=0.0000 scale=2.0000 norm=16.4177

[iter 200] loss=3.5957 val_loss=0.0000 scale=2.0000 norm=15.7549

[iter 300] loss=3.5479 val_loss=0.0000 scale=2.0000 norm=15.3516

[iter 400] loss=3.5120 val_loss=0.0000 scale=2.0000 norm=14.9871

Test MSE 139.01759896825132
Test NLL 3.8621984388068915

fig, ax = plt.subplots()
ax.scatter(x, y, alpha=.5)
ax.plot(x, x, color="black", alpha=.5, label="mean")
ax.set(xlabel="x", ylabel="y")
ax.legend()

X_test = np.sort(X_test, axis=0)
y_dist = ngb.pred_dist(X_test)

alphas = [0.05, 0.01]
colors = ["darkorange", "tomato"]
for alpha, color in zip(alphas, colors):
    upper = norm.ppf(q=1 - (alpha/2), loc=y_dist.params["loc"], scale=y_dist.params["scale"])
    lower = norm.ppf(q=(alpha/2), loc=y_dist.params["loc"], scale=y_dist.params["scale"])
    ax.plot(X_test[:, 0], upper, alpha=.9, color=color, linestyle="--", label=rf"$\alpha$={alpha}")
    ax.plot(X_test[:, 0], lower, alpha=.9, color=color, linestyle="--")

ax.legend()
fig.show()