Loss Function Comparison

Fan Gong 03/06/2019

This notebook aims to generate graphs to compare different kinds of loss function

1. Load Library

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt

2. Regression Loss Function Comparison

Here I use notation $r$ to represent the residule $y_i - f(x_i)$

class regression_loss:
    '''
    This class contains all the regression loss function
    '''
    
    def __init__(self, r):
        '''
        r: The residule of the regression model 
        '''
        self.r = r    
    
    def MSE(self):
        return self.r**2
    
    def MAE(self):
        return 2*abs(self.r)
    
    def Huber(self,para):
        huber = []
        for ri in self.r:
            if abs(ri) <= para:
                r_huber =  0.5 * ri**2
            else:
                r_huber = para* abs(ri) - 0.5 * para**2
            huber.append(r_huber)
        
        return huber
    
    def quantile(self,quantile):
        quantile_loss = []
        for ri in self.r:
            if ri<0:
                quantile_r = (1-quantile)*abs(ri)
            else:
                quantile_r = quantile*abs(ri)
            quantile_loss.append(quantile_r)
        return quantile_loss

r = np.arange(-3,3,0.1)

regression_loss = regression_loss(r)

fig = plt.figure(figsize=(12,8))
ax = fig.add_subplot(111)
ax.plot(r, regression_loss.MSE(), label = 'MSE', color = 'red')
ax.plot(r, regression_loss.MAE(), label = 'MAE', color = 'blue')
ax.plot(r, regression_loss.Huber(2), label = 'Huber(1)', color = 'green')
ax.plot(r, regression_loss.quantile(0.75), label = 'Quantile(0.25)', color = 'black')
ax.set_xlabel('Residule y-f(x)')
ax.set_ylabel('Loss')
ax.set_title('Regression Loss Function Comparison')
ax.legend()
plt.show()

3. Classification Loss Function Comparison

Here I use notation $m$ as the notation to represent the deviance $y_if(x_i)$

class classification_loss:
    '''
    This Class contains all the classification function
    '''
    
    def __init__(self, m):
        '''
        m: the deviance of the classification model
        '''
        self.m = m
    
    def misclassification_loss(self):
        return (self.m<0).astype(int)
    
    def binomial_deviance(self):
        return np.log(1+np.exp(-m))
    
    def exponential_loss(self):
        return np.exp(-m)
    
    def Hinge(self):
        loss = []
        for mi in m:
            hinge_m = max(0, 1-mi)
            loss.append(hinge_m)
        return loss

m = np.arange(-2,2,0.01)

classification_loss = classification_loss(m)

fig = plt.figure(figsize=(12,8))
ax = fig.add_subplot(111)
ax.plot(m, classification_loss.misclassification_loss(), label = '0-1 Loss', color = 'red')
ax.plot(m, classification_loss.binomial_deviance(), label = 'Binomial Deviance', color = 'blue')
ax.plot(m, classification_loss.exponential_loss(), label = 'Exponential Loss', color = 'green')
ax.plot(m, classification_loss.Hinge(), label = 'Hinge Loss', color = 'black')
ax.set_xlabel('Deviance yf(x)')
ax.set_ylabel('Loss')
ax.set_title('Classification Loss Function Comparison')
ax.legend()
plt.show()