IBM HR Data로 해보는 퇴사자 예측

from collections import OrderedDict

# Data preprocessing
from sklearn.preprocessing import OneHotEncoder, StandardScaler, LabelEncoder, RobustScaler
from sklearn.model_selection import train_test_split, GridSearchCV, learning_curve
from sklearn.metrics import accuracy_score, roc_curve, roc_auc_score, recall_score, f1_score, confusion_matrix, precision_score
from sklearn.pipeline import Pipeline
from sklearn.compose import ColumnTransformer
from sklearn.impute import SimpleImputer
from lightgbm import LGBMClassifier


# 모델 import
from sklearn.tree import DecisionTreeClassifier, export_graphviz
from sklearn.ensemble import RandomForestClassifier
from sklearn.linear_model import LogisticRegression

import xgboost as xgb
import statsmodels.formula.api as smf
import statsmodels.api as sm
import graphviz
import pydotplus

# EDA package
import pandas as pd
import numpy as np
import missingno
import matplotlib.pyplot as plt
import seaborn as sns
import warnings
import matplotlib as mpl


# warnings 끄기
warnings.filterwarnings('ignore')

# pandas display option view row & columns
pd.set_option('display.max_row', 500)
pd.set_option('display.max_columns', 100)
pd.set_option('display.max_colwidth', 1000)

# # matplotlib set
plt.rc('font', family='DejaVu Sans')  # For MacOS
plt.rc('axes', unicode_minus=False)

%matplotlib inline

data = pd.read_csv('WA_Fn-UseC_-HR-Employee-Attrition.csv')
data.tail()

data.info()

# Education
change_dict = {1: 'Below College', 2: 'College', 3: 'Bachelor', 4: 'Master', 5: 'Doctor'}
data.replace({'Education': change_dict}, inplace=True)
data['Education'].unique()

# EnvironmentSatisfaction
change_dict = {1: 'Low', 2: 'Medium', 3: 'High', 4: 'Very High'}
data.replace({'EnvironmentSatisfaction': change_dict}, inplace=True)
data['EnvironmentSatisfaction'].unique()

# JobInvolvement
change_dict = {1: 'Low', 2: 'Medium', 3: 'High', 4: 'Very High'}
data.replace({'JobInvolvement': change_dict}, inplace=True)
data['JobInvolvement'].unique()

# JobSatisfaction
change_dict = {1: 'Low', 2: 'Medium', 3: 'High', 4: 'Very High'}
data.replace({'JobSatisfaction': change_dict}, inplace=True)
data['JobSatisfaction'].unique()

# PerformanceRating
change_dict = {1: 'Low', 2: 'Good', 3: 'Excellent', 4: 'Outstanding'}
data.replace({'PerformanceRating': change_dict}, inplace=True)
data['PerformanceRating'].unique()

# RelationshipSatisfaction
change_dict = {1: 'Low', 2: 'Medium', 3: 'High', 4: 'Very High'}
data.replace({'RelationshipSatisfaction': change_dict}, inplace=True)
data['RelationshipSatisfaction'].unique()

# WorkLifeBalance
change_dict = {1: 'Bad', 2: 'Good', 3: 'Better', 4: 'Best'}
data.replace({'WorkLifeBalance': change_dict}, inplace=True)
data['WorkLifeBalance'].unique()

# JobLevel, StockOptionLevel, TrainingTimesLastYear, NumCompaniesWorked, TotalWorkingYears
data = data.astype({'JobLevel': object, 'StockOptionLevel': object, 'NumCompaniesWorked': object})
data

data.info()

missingno.matrix(data)
plt.show()

print('Attrition 비율')
print(f'{data.Attrition.value_counts().index[0]} : {round(data.Attrition.value_counts()[0] / len(data), 2) * 100}%')
print(f'{data.Attrition.value_counts().index[1]} : {round(data.Attrition.value_counts()[1] / len(data), 2) * 100}%')

plt.figure(figsize = (12,12))
sns.countplot(x = data['Attrition'])
plt.title('Attrition의 분포')
plt.show()

# category column
cate_cols = []
for column in data.columns:
    if data[column].dtype == object:
        cate_cols.append(column)
        print('=============================================================================================')
        print(f'{column} : {data[column].unique()}')
        print(f'{data[column].value_counts()}')
        print()
        
print()
print(f'object column의 갯수 : {len(cate_cols)} 개')

# category column 그래프로 보기
fig, ax = plt.subplots(5, 4, figsize=(28, 28), constrained_layout=True)
ax = ax.flatten()
fig.suptitle('Attrition과 Category Column들의 분포', fontsize=16)

for i in range(len(cate_cols)):
    sns.countplot(x=cate_cols[i], data=data,
                      hue='Attrition', ax=ax[i]).set(xlabel = None)
    ax[i].set(title = cate_cols[i])
    
    if data[cate_cols[1]].nunique() >= 3:
        ax[i].tick_params(labelrotation=20)
        
plt.show()

# continuous column
cont_cols = []
for column in data.columns:
    if data[column].dtype != object:
        cont_cols.append(column)
        print(f'{column} : {data[column].nunique()}')
        print('==============================')
print()
print(f'연속형 column의 갯수 : {len(cont_cols)} 개')

fig, ax = plt.subplots(4, 4, figsize=(28, 14), constrained_layout=True)
ax = ax.flatten()
fig.suptitle('Attrition과 continuous Column들의 분포', fontsize=16)

for i in range(len(cont_cols)):
    sns.distplot(data[data['Attrition'] == 'Yes'][cont_cols[i]], color='Red', ax=ax[i], hist = False).set(xlabel = None, ylabel = None)
    sns.distplot(data[data['Attrition'] == 'No'][cont_cols[i]], ax=ax[i],hist = False).set(xlabel = None, ylabel = None)
    ax[i].set(title = cont_cols[i])
    
plt.show()

# boxplot
fig, ax = plt.subplots(4, 4, figsize=(28, 14), constrained_layout=True)
ax = ax.flatten()
fig.suptitle('Boxplot of Attrition', fontsize=16)

for i in range(len(cont_cols)):
    ax[i].set(title = cont_cols[i])
    sns.boxplot(x=data['Attrition'], y = data[cont_cols[i]], ax=ax[i]).set(xlabel=None,  ylabel=None)
    
plt.show()

# EmployeeCount, StandardHours, Over18, EmployeeNumber 
print('Over18 :', data['Over18'].unique()[0])
print('EmployeeCount :', data['EmployeeCount'].unique()[0])
print('StandardHours :', data['StandardHours'].unique()[0])
print('EmployeeNumber :', data['EmployeeNumber'].unique()[0])
data.drop(['EmployeeCount', 'StandardHours', 'Over18', 'EmployeeNumber'], axis = 1, inplace = True)
print(data.shape)
data.tail()

# 상관계수 구하기
data_cp = data.copy()
data_cp = pd.get_dummies(data_cp, drop_first= True)
data_cp = data_cp[['Attrition_Yes'] + [column for column in data_cp.columns if column != 'Attrition_Yes']]
data_corr = data_cp.corr()
print(data_corr.shape)

plt.figure(figsize=(36, 18))
plt.title('Corr Heatmap')

# 실제 히트맵 그리는 코드
sns.set(font_scale=1.2)
sns.heatmap(data_corr, annot=True, annot_kws={
    "size": 90 / np.sqrt(len(data_corr))}, fmt='.2f', cmap='RdYlBu_r', linewidths=0.5,)
plt.savefig('corrmap.png')
plt.show()

temps = data_corr[(data_corr > 0.4) | (data_corr < -0.4)]
high_corr = []
for c in temps.columns: 
    temp = temps[c].dropna()

    if len(temp) == 1:
        continue
    high_corr.append([temp.name, temp.to_dict()])

fig, ax = plt.subplots(6,5, figsize = (36, 36), constrained_layout=True)
fig.suptitle('Corr', fontsize=16)
ax = ax.flatten()

for i, c in enumerate(high_corr):
    ordered_d = OrderedDict(sorted(high_corr[i][1].items(), key=lambda t:t[1], reverse=True))
    title = ordered_d.popitem(0)
    
    sns.barplot(x = list(ordered_d.keys()), y = list(ordered_d.values()), ax = ax[i])
    ax[i].set(title = title[0])
    
    if len(ordered_d.keys()) > 2:
        ax[i].tick_params(labelrotation=20)
plt.savefig('corrbar.png')
plt.show()

# 잘안보여서 일단 종속변수 상관관계만 확인
plt.title('Attrition of Corr')
data_cp.drop('Attrition_Yes', axis = 1).corrwith(data_cp.Attrition_Yes).sort_values().plot(kind='barh', figsize = (10, 24))
plt.show()

from statsmodels.stats.outliers_influence import variance_inflation_factor
# 피처마다의 VIF 계수를 출력합니다.
vif = pd.DataFrame()
vif["VIF Factor"] = [variance_inflation_factor(data_corr.values, i) for i in range(data_corr.shape[1])]
vif["features"] = data_corr.columns
vif.sort_values(by='VIF Factor', ascending = False)

# 비율 확인할 pivot 테이블 만드는 함수
def make_pivot(data, x, y, func):
    table = pd.pivot_table(data = data, values = 'Age', index = x, columns= y, aggfunc=func)
    table['total'] = table['No'] + table['Yes']
    table['Attrition_rate'] = table['Yes'] / table['total'] * 100
    return table

rate = make_pivot(data, 'DistanceFromHome', 'Attrition', func=len)

# plt.rc('font', family='AppleGothic') 
plt.figure(figsize=(12,8))
plt.title('DistanceFromHome', fontsize=16)
sns.barplot(rate.index, rate.Attrition_rate)
plt.show()

round(data[data.DistanceFromHome >= 22]['DistanceFromHome'].count() / len(data),2)

data['FarFromHome'] = np.where(data.DistanceFromHome >= 22 , 1, 0)
data['FarFromHome'] = data['FarFromHome'].astype(object)

plt.figure(figsize = (12,8))
plt.title('월급여에 따른 퇴사율')
sns.boxplot(x = 'Attrition', y = 'MonthlyIncome', data = data)
plt.show()

data[data['Attrition'] == 'Yes']['MonthlyIncome'].median()

data['LowMonthlyIncome'] = np.where(data.MonthlyIncome <= 3202 , 1, 0)
data['LowMonthlyIncome'] = data['LowMonthlyIncome'].astype(object)

rate = make_pivot(data, 'JobInvolvement', 'Attrition', func=len)

plt.figure(figsize=(12, 8))
plt.title('업무 환경에 따른 퇴사인원 비율', fontsize=16)
sns.barplot(rate.index, rate.Attrition_rate, order=['Low', 'Medium', 'High', 'Very High'])
plt.show()

rate = make_pivot(data, 'WorkLifeBalance', 'Attrition', func= len)

plt.figure(figsize = (12,8))
plt.title('워라벨 정도에 따른 퇴사 인원 비율')
sns.barplot(rate.index, rate.Attrition_rate, order=['Bad', 'Good', 'Better', 'Best'])
plt.show()

rate = make_pivot(data, 'Department', 'Attrition', func= len)
fig, ax = plt.subplots(1,2, figsize = (16,8))

sns.countplot(data['Department'], hue = data['Attrition'], ax = ax[0], order = ['Sales', 'Human Resources', 'Research & Development'])
ax[0].set(title = 'Attrition of Department')

sns.barplot(rate.index, rate.Attrition_rate, ax = ax[1], order = ['Sales', 'Human Resources', 'Research & Development'])
ax[1].set(title = 'Attrition of Department rate')

plt.show()

plt.figure(figsize=(12, 8))
plt.title('MonthlyIncome of Department')
sns.boxplot(x='Department', y='MonthlyIncome', data=data)
plt.show()

temp = data.groupby('Department', axis=0).agg(['min', 'median', 'mean', 'max', 'std'])['MonthlyIncome']
temp

rate = make_pivot(data, 'TotalWorkingYears', 'Attrition', func= len).fillna(0)

plt.figure(figsize = (12,8))
plt.title('Attrition of TotalWorkingYears rate')
sns.barplot(rate.index, rate.Attrition_rate)
plt.show()

data['LowWorkingYears'] = np.where(data.TotalWorkingYears <= 2 , 1, 0)
data['LowWorkingYears'] = data['LowWorkingYears'].astype(object)

plt.figure(figsize=(12,8))
plt.title('Age histogram')
data['Age'].plot(kind = 'hist')
plt.show()

# [(17.958, 26.4] : 0,  (26.4, 34.8] : 1, (34.8, 43.2] : 2, (43.2, 51.6] : 3,  (51.6, 60.0] : 4]
# (미포함, 포함) 임
data['Age_cut'] = pd.cut(data['Age'], 5, labels=[0, 1, 2, 3, 4]).astype(object)

plt.figure(figsize=(12,8))
sns.countplot(data.NumCompaniesWorked)
plt.show()

data[(data.NumCompaniesWorked == 0) & (data.TotalWorkingYears == data.YearsAtCompany)][['NumCompaniesWorked','TotalWorkingYears','YearsAtCompany']]

print(data[(data.NumCompaniesWorked == 0) & (data.TotalWorkingYears != data.YearsAtCompany)][['NumCompaniesWorked','TotalWorkingYears','YearsAtCompany']].shape)
data[(data.NumCompaniesWorked == 0) & (data.TotalWorkingYears != data.YearsAtCompany)][['NumCompaniesWorked','TotalWorkingYears','YearsAtCompany']].head()

data[(data.TotalWorkingYears == data.YearsAtCompany)][['NumCompaniesWorked','TotalWorkingYears','YearsAtCompany']].head()

data[(data.NumCompaniesWorked >= 2) & (data.TotalWorkingYears == data.YearsAtCompany)][['NumCompaniesWorked','TotalWorkingYears','YearsAtCompany']].head()

data[(data.NumCompaniesWorked >= 1) & ((data.TotalWorkingYears - data.YearsAtCompany) == 1)][['NumCompaniesWorked','TotalWorkingYears','YearsAtCompany']].head()

data.NumCompaniesWorked.replace(0, 1, inplace = True)
plt.figure(figsize=(12,8))
sns.countplot(data.NumCompaniesWorked)
plt.show()

data.NumCompaniesWorked = data.NumCompaniesWorked.astype(object)

point = {'Low' : 0, 'Medium' : 1, 'High' : 2 , 'Very High' : 3}
EnvironmentSatisfaction = data['EnvironmentSatisfaction'].map(point)
RelationshipSatisfaction = data['RelationshipSatisfaction'].map(point)
JobSatisfaction = data['JobSatisfaction'].map(point)
data['TotalSatisfaction'] = (EnvironmentSatisfaction + RelationshipSatisfaction + JobSatisfaction).astype(object)

plt.figure(figsize=(12, 8))
plt.title('Total Satisfaction')
sns.countplot(data['TotalSatisfaction'])
plt.show()

cont_cols = [i for i in data.columns if data[i].dtype != object]
fig, ax = plt.subplots(2, 7, figsize=(36, 12))
ax = ax.flatten()

for index, col in enumerate(cont_cols):
    sns.boxplot(data[col], ax=ax[index])

# 아웃라이어 확인 함수 생성
def outlier(data, col):
    q1 = np.percentile(data[col], 25) 
    q3 = np.percentile(data[col], 75)
    IQR = q3 - q1
    outlier_step = 1.5 * IQR
    return data[(data[col] < q1 - outlier_step) | (data[col] > q3 + outlier_step)]

monthlyincom_outlier = outlier(data, 'MonthlyIncome')
print(f'MonthlyIncome의 아웃라이어 갯수 {monthlyincom_outlier.shape[0]}개')
print(f'전체 데이터의 {round(monthlyincom_outlier.shape[0] / data.shape[0],2)}% 차지')
print(monthlyincom_outlier.Attrition.value_counts())
sns.countplot(monthlyincom_outlier.Attrition)
plt.show()

workingyear_outlier = outlier(data, 'TotalWorkingYears')
print(f'TotalWorkingYears의 아웃라이어 갯수 {workingyear_outlier.shape[0]}개')
print(f'전체 데이터의 {round(workingyear_outlier.shape[0] / data.shape[0],2)}% 차지')
print(workingyear_outlier.Attrition.value_counts())
sns.countplot(workingyear_outlier.Attrition)
plt.show()

traininigtimes_outlier = outlier(data, 'TrainingTimesLastYear')
print(f'TrainingTimesLastYear의 아웃라이어 갯수 {traininigtimes_outlier.shape[0]}개')
print(f'전체 데이터의 {round(traininigtimes_outlier.shape[0] / data.shape[0],2)}% 차지')
print(traininigtimes_outlier.Attrition.value_counts())
sns.countplot(traininigtimes_outlier.Attrition)
plt.show()

yearsatcompany_outlier = outlier(data, 'YearsAtCompany')
print(f'YearsAtCompany의 아웃라이어 갯수 {yearsatcompany_outlier.shape[0]}개')
print(f'전체 데이터의 {round(yearsatcompany_outlier.shape[0] / data.shape[0],2)}% 차지')
print(yearsatcompany_outlier.Attrition.value_counts())
sns.countplot(yearsatcompany_outlier.Attrition)
plt.show()

yearsincurrentrole_outlier = outlier(data, 'YearsInCurrentRole')
print(f'YearsInCurrentRole의 아웃라이어 갯수 {yearsincurrentrole_outlier.shape[0]}개')
print(f'전체 데이터의 {round(yearsincurrentrole_outlier.shape[0] / data.shape[0],2)}% 차지')
print(yearsincurrentrole_outlier.Attrition.value_counts())
sns.countplot(yearsincurrentrole_outlier.Attrition)
plt.show()

yearssincelastpromotion_outlier = outlier(data, 'YearsSinceLastPromotion')
print(f'YearsSinceLastPromotion의 아웃라이어 갯수 {yearssincelastpromotion_outlier.shape[0]}개')
print(f'전체 데이터의 {round(yearssincelastpromotion_outlier.shape[0] / data.shape[0],2)}% 차지')
print(yearssincelastpromotion_outlier.Attrition.value_counts())
sns.countplot(yearssincelastpromotion_outlier.Attrition)
plt.show()

yearswithcurrmanager_outlier = outlier(data, 'YearsWithCurrManager')
print(f'YearsWithCurrManager의 아웃라이어 갯수 {yearswithcurrmanager_outlier.shape[0]}개')
print(f'전체 데이터의 {round(yearswithcurrmanager_outlier.shape[0] / data.shape[0],2)}% 차지')
print(yearswithcurrmanager_outlier.Attrition.value_counts())
sns.countplot(yearswithcurrmanager_outlier.Attrition)
plt.show()

data_copy = data.copy()
outlier_df = pd.concat([yearsatcompany_outlier, yearsincurrentrole_outlier]).drop_duplicates()
data_copy.drop(index=outlier_df.index, inplace=True)
data_copy.drop(labels=['MonthlyIncome', 'Age', 'TotalWorkingYears', 'YearsAtCompany', 'Department',
                       'EnvironmentSatisfaction', 'RelationshipSatisfaction', 'JobSatisfaction'], axis=1, inplace=True)
data_copy.shape

numeric_features = [column for column in data_copy.columns if data_copy[column].dtype != object]
categorical_features = [column for column in data_copy.columns if data_copy[column].dtype == object]

data_copy[categorical_features] = data_copy[categorical_features].apply(LabelEncoder().fit_transform)
data_copy[numeric_features] = RobustScaler().fit_transform(data_copy[numeric_features])

X = data_copy.drop(labels= ['Attrition'], axis = 1)
y = data_copy.Attrition

X_train, X_test, y_train, y_test = train_test_split(
    X, y, stratify=y, random_state=87, test_size=0.2)

train_rate = round(y_train.sum() / len(y_train),2)
test_rate = round(y_test.sum() / len(y_test),2)
print(f'학습 데이터에서의 Target 비율 : {train_rate}')
print(f'테스트 데이터에서의 Target 비율 : {test_rate}')

# Plot learning curve
def plot_learning_curve(estimator, title, X, y, ylim=None, cv=None,
                        n_jobs=-1, train_sizes=np.linspace(.1, 1.0, 5), scoring = 'accuracy'):
    """
    Generate a simple plot of the test and traning learning curve.

    Parameters
    ----------
    estimator : object type that implements the "fit" and "predict" methods
        An object of that type which is cloned for each validation.

    title : string
        Title for the chart.

    X : array-like, shape (n_samples, n_features)
        Training vector, where n_samples is the number of samples and
        n_features is the number of features.

    y : array-like, shape (n_samples) or (n_samples, n_features), optional
        Target relative to X for classification or regression;
        None for unsupervised learning.

    ylim : tuple, shape (ymin, ymax), optional
        Defines minimum and maximum yvalues plotted.

    cv : integer, cross-validation generator, optional
        If an integer is passed, it is the number of folds (defaults to 3).
        Specific cross-validation objects can be passed, see
        sklearn.cross_validation module for the list of possible objects

    n_jobs : integer, optional
        Number of jobs to run in parallel (default 1).

    x1 = np.linspace(0, 10, 8, endpoint=True) produces
        8 evenly spaced points in the range 0 to 10
    """

    plt.figure(figsize = (12,8))
    plt.title(title)
    if ylim is not None:
        plt.ylim(*ylim)

    plt.xlabel("Training examples")
    plt.ylabel("Score")
    train_sizes, train_scores, test_scores = learning_curve(
        estimator, X, y, cv=cv, n_jobs=n_jobs, train_sizes=train_sizes, scoring=scoring)
    train_scores_mean = np.mean(train_scores, axis=1)
    train_scores_std = np.std(train_scores, axis=1)
    test_scores_mean = np.mean(test_scores, axis=1)
    test_scores_std = np.std(test_scores, axis=1)
    plt.grid()

    plt.fill_between(train_sizes, train_scores_mean - train_scores_std,
                     train_scores_mean + train_scores_std, alpha=0.1,
                     color="r")
    plt.fill_between(train_sizes, test_scores_mean - test_scores_std,
                     test_scores_mean + test_scores_std, alpha=0.1, color="g")
    plt.plot(train_sizes, train_scores_mean, 'o-', color="r",
             label="Training score")
    plt.plot(train_sizes, test_scores_mean, 'o-', color="g",
             label="Cross-validation score")

    plt.legend(loc="best")
    return plt.show()

def model_score(y_test, predict):
    print('Accuracy : ',round(accuracy_score(y_test, predict), 3))
    print('Recall : ',round(recall_score(y_test, predict), 3))
    print('Precision : ',round(precision_score(y_test, predict), 3))
    print('F1_Score : ',round(f1_score(y_test, predict), 3))
    print()
    print('Confusion_matrix :')
    print(confusion_matrix(y_test, predict))

clf_model = DecisionTreeClassifier(random_state=87)

params_grid = [{}]

gridsearch = GridSearchCV(
    estimator=clf_model, param_grid=params_grid, cv=5, n_jobs=-1, return_train_score=True)
gridsearch.fit(X_train, y_train)

result = []
result.append({'Estimator': str(gridsearch.best_estimator_).split('(')[0],
           'mean_train_score': float(gridsearch.cv_results_['mean_train_score']),
           'mean_test_score': float(gridsearch.cv_results_['mean_test_score'])})

clf_best_model = gridsearch.best_estimator_
clf_best_model.fit(X_train, y_train)
clf_predict = clf_best_model.predict(X_test)

model_score(y_test, clf_predict)
plot_learning_curve(clf_best_model, 'DecisionTree Learning Curve', X_train, y_train, cv=5)

important = clf_best_model.feature_importances_

pd.DataFrame(important, X_train.columns,columns=['important']).sort_values(by = 'important', ascending=False).plot(kind='bar', figsize = (36, 10), rot = 45, title ='Important Feature')
plt.show()

dot_data = export_graphviz(clf_best_model, out_file=None, 
                feature_names = X_train.columns,
                class_names = data.Attrition.unique(),
                max_depth = 5, # 표현하고 싶은 최대 depth
                precision = 3, # 소수점 표기 자릿수
                filled = True, # class별 color 채우기
                rounded=True, # 박스의 모양을 둥글게
               )
pydot_graph = pydotplus.graph_from_dot_data(dot_data)
pydot_graph.set_size("\"24\"")
gvz_graph = graphviz.Source(pydot_graph.to_string())
gvz_graph

rf_model = RandomForestClassifier(random_state=87)
params_grid = [{}]

gridsearch = GridSearchCV(
    estimator=rf_model, param_grid=params_grid, cv=5, n_jobs=-1, return_train_score=True)
gridsearch.fit(X_train, y_train)

result.append({'Estimator': str(gridsearch.best_estimator_).split('(')[0],
           'mean_train_score': float(gridsearch.cv_results_['mean_train_score']),
           'mean_test_score': float(gridsearch.cv_results_['mean_test_score'])})

rf_best_model = gridsearch.best_estimator_
rf_best_model.fit(X_train, y_train)
rf_predict = rf_best_model.predict(X_test)

model_score(y_test, rf_predict)
plot_learning_curve(rf_best_model, 'RandomForest Learning Curve', X_train, y_train, cv=5)

important = rf_best_model.feature_importances_

pd.DataFrame(important, X_train.columns,columns=['important']).sort_values(by = 'important', ascending=False).plot(kind='bar', figsize = (36, 10), rot = 45, title ='Important Feature')
plt.show()

dot_data = export_graphviz(rf_best_model[0], out_file=None, 
                feature_names = X_train.columns,
                class_names = data.Attrition.unique(),
                max_depth = 5, # 표현하고 싶은 최대 depth
                precision = 3, # 소수점 표기 자릿수
                filled = True, # class별 color 채우기
                rounded=True, # 박스의 모양을 둥글게
               )
pydot_graph = pydotplus.graph_from_dot_data(dot_data)
pydot_graph.set_size("\"24\"")
gvz_graph = graphviz.Source(pydot_graph.to_string())
gvz_graph

lr_model = LogisticRegression(random_state=87)
params_grid = [{}]

gridsearch = GridSearchCV(
    estimator=lr_model, param_grid=params_grid, cv=5, n_jobs=-1, return_train_score=True)
gridsearch.fit(X_train, y_train)

result.append({'Estimator': str(gridsearch.best_estimator_).split('(')[0],
           'mean_train_score': float(gridsearch.cv_results_['mean_train_score']),
           'mean_test_score': float(gridsearch.cv_results_['mean_test_score'])})

lr_best_model = gridsearch.best_estimator_
lr_best_model.fit(X_train, y_train)
lr_predict = lr_best_model.predict(X_test)

model_score(y_test, lr_predict)
plot_learning_curve(lr_best_model,
                    'Logistic Regression Learning Curve', X_train, y_train, cv=5)

coefs = np.abs(lr_best_model.coef_[0])
pd.DataFrame(coefs, X_train.columns, columns=['important']).sort_values(by = 'important', ascending=False).plot(kind='bar', figsize = (36, 10), rot = 45, title = 'Important Feature')
plt.show()

xgb_model = xgb.XGBClassifier(random_state = 87)

params_grid = [{}]

gridsearch = GridSearchCV(
    estimator=xgb_model, param_grid=params_grid, cv=5, n_jobs=-1, return_train_score=True)
gridsearch.fit(X_train, y_train)

result.append({'Estimator': str(gridsearch.best_estimator_).split('(')[0],
           'mean_train_score': float(gridsearch.cv_results_['mean_train_score']),
           'mean_test_score': float(gridsearch.cv_results_['mean_test_score'])})

xgb_best_model = gridsearch.best_estimator_
xgb_best_model.fit(X_train, y_train)
xgb_predict = xgb_best_model.predict(X_test)

model_score(y_test, xgb_predict)
plot_learning_curve(xgb_best_model,
                    'XGBoosting Learning Curve', X_train, y_train, cv=5)

important = xgb_best_model.feature_importances_

pd.DataFrame(important, X_train.columns,columns=['important']).sort_values(by = 'important', ascending=False).plot(kind='bar', figsize = (36, 10), rot = 45, title = 'Important Feature')
plt.show()

lgbm_model = LGBMClassifier(random_state = 87)
params_grid = [{}]

gridsearch = GridSearchCV(
    estimator=lgbm_model, param_grid=params_grid, cv=5, n_jobs=-1, return_train_score=True)
gridsearch.fit(X_train, y_train)

result.append({'Estimator': str(gridsearch.best_estimator_).split('(')[0],
           'mean_train_score': float(gridsearch.cv_results_['mean_train_score']),
           'mean_test_score': float(gridsearch.cv_results_['mean_test_score'])})