Influence Function 線形回帰での例

Influence Function 線形回帰での例#

参考：Influence Functionでインスタンスの重要度を解釈する - Dropout

\[\begin{split} \begin{align} L(z, \theta) &= (y - x^\top \theta)^2\\ \nabla_{\theta} L(z, \theta) &= - 2 (y - x^\top \theta) x = -2xy + 2 (x^\top \theta) x\\ R(\theta) &= (y - X\theta)^\top (y - X\theta) \\ \nabla_{\theta} R(\theta) &= -2 X^\top y + 2 (X^\top X) \theta \\ H = \nabla_{\theta}^2 R(\theta) &= 2 (X^\top X)\\ \hat{\theta} &= (X^\top X)^{-1} X^\top y \end{align} \end{split}\]

なので

\[\begin{split} \begin{align} \mathcal{I}_{up, params}(z) &= -H^{-1}_{\hat{\theta}} \nabla_{\theta} L(z, \hat{\theta})\\ &= -( 2 X^\top X )^{-1} ( - 2 (y - x^\top \hat\theta) x ) \\ \mathcal{I}_{up, loss}(z, z_{test}) &= - \nabla_{\theta} L(z_{test}, \hat{\theta})^\top H^{-1}_{\hat{\theta}} \nabla_{\theta} L(z, \hat{\theta}) \\ &= (2 (y_{test} - x_{test}^\top \hat\theta) x_{test} )^\top ( 2X^\top X )^{-1} ( - 2 (y - x^\top \hat\theta) x ) \end{align} \end{split}\]

def influence_params(x, y, X, theta):
    hessian = 2 * (X.T @ X)
    nabla_l_train = - 2 * (y - x.T @ theta) * x
    return np.linalg.inv(hessian) @ nabla_l_train

def influence_loss(x, y, x_test, y_test, X, theta):
    nabla_l_test = (y_test - x_test.T @ theta) * x_test
    hessian = 2 * (X.T @ X)
    nabla_l_train = - 2 * (y - x.T @ theta) * x
    return nabla_l_test.T @ np.linalg.inv(hessian) @ nabla_l_train

import numpy as np

# generate data
n, p = 100, 2
np.random.seed(0)
X = np.random.uniform(size=(n, p))
theta = np.random.uniform(size=p).round(1)
print(theta)
e = np.random.normal(scale=0.1, size=n)
y = X @ theta + e

[0.3 0.7]

class LinearRegression:

    def fit(self, X, y):
        self.theta_ = np.linalg.inv(X.T @ X) @ (X.T @ y)
        return self

    def predict(self, X):
        return X @ self.theta_

# データ全件でのモデル
model = LinearRegression().fit(X, y)
theta_hat = model.theta_
theta_hat

array([0.27816619, 0.71295385])

Influence (params)#

LOO#

\[ \hat{\theta}_{-z} - \hat{\theta} \]

# LOO

diffs = np.array([])
theta_wo_z = np.array([])
for i in range(n):
    X_wo_z = np.concatenate((X[:i, ], X[(i+1):, ]), axis=0)
    y_wo_z = np.concatenate((y[:i], y[(i+1):]), axis=0)
    assert X_wo_z.shape[0] == n - 1
    assert y_wo_z.shape[0] == n - 1
    theta_wo_z_i = LinearRegression().fit(X_wo_z, y_wo_z).theta_
    diff_i = np.linalg.norm(theta_wo_z_i - theta_hat)
    diffs = np.append(diffs, diff_i)
    theta_wo_z = np.concatenate([theta_wo_z, theta_wo_z_i])

theta_wo_z = theta_wo_z.reshape(n, -1)

# LOO (params)

diffs = np.array([])
for i in range(n):
    X_wo_z = np.concatenate((X[:i, ], X[(i+1):, ]), axis=0)
    y_wo_z = np.concatenate((y[:i], y[(i+1):]), axis=0)
    assert X_wo_z.shape[0] == n - 1
    assert y_wo_z.shape[0] == n - 1
    theta_wo_z_i = LinearRegression().fit(X_wo_z, y_wo_z).theta_
    diff_i = np.linalg.norm(theta_wo_z_i - theta_hat)  # ノルムがいいのかはわからんがL2ノルムにしてみる
    diffs = np.append(diffs, diff_i)

diffs_loo = diffs

idxs = np.argsort(diffs)[::-1]
X[idxs][:5]

array([[0.43758721, 0.891773  ],
       [0.0202184 , 0.83261985],
       [0.7936977 , 0.22392469],
       [0.65632959, 0.13818295],
       [0.0641475 , 0.69247212]])