Chapter_4_sec_7.1_7.7.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 4.7 Lab: Classification Methods"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4.7.1 The Stock Market Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import pandas as pd \n",
    "import math\n",
    "from patsy import dmatrices\n",
    "import statsmodels.discrete.discrete_model as sm\n",
    "import statsmodels.formula.api as smf\n",
    "import statsmodels.api as sma\n",
    "from statsmodels.graphics.regressionplots import *\n",
    "from sklearn import datasets, linear_model\n",
    "from sklearn.metrics import confusion_matrix\n",
    "from sklearn.discriminant_analysis import LinearDiscriminantAnalysis as LDA\n",
    "from sklearn.discriminant_analysis import QuadraticDiscriminantAnalysis as QDA\n",
    "from sklearn.naive_bayes import GaussianNB as NB\n",
    "from sklearn.neighbors import KNeighborsClassifier as KNN\n",
    "from sklearn import preprocessing\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "Smarket = pd.read_csv('data/Smarket.csv', header=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "Smarket.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "Smarket.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "Smarket.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# for panda data frame, there is a method corr to compute pairwise correlation between numerical variables\n",
    "Smarket.corr()\n",
    "# as one would expect, the correlations between the lag variables and today’s returns are close to zero. \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# take a look at volume column\n",
    "plt.plot(Smarket.iloc[:, 6])\n",
    "# or plt.plot(Smarket[['Volume']])\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4.7.2 Logistic Regression\n",
    "There are some known complications that in Sklearn about applying parameter regularization. This can be aviod to set the tuning parameter 'C' to a large number. Here to be consistent with R output, I decieded to use Statsmodels."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "y, X = dmatrices('Direction~Lag1+Lag2+Lag3+Lag4+Lag5+Volume', Smarket, return_type = 'dataframe')\n",
    "print(y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# since we are more interested in stock marketing up, we take the second column of y as our response variable \n",
    "# we build a model to predict whether the direction will be up. \n",
    "logit = sm.Logit(y.iloc[:,1], X)\n",
    "logit.fit().summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# to extract the parameters directly\n",
    "logit.fit().params"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# to extract the probability of the market going up for the first 10 instances\n",
    "logit.fit().predict()[0:10] "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# in order to make a prediction as to whether the market will go up or down on a particular day, \n",
    "# we must convert these predicted probabilities into class labels, Up (1) or Down (0).\n",
    "# we will do this by threshold the probability by a predefined threshold \n",
    "threshold = 0.5 \n",
    "predict_label = pd.DataFrame(np.zeros(shape=(1250,1)), columns = ['label'])\n",
    "predict_label.iloc[logit.fit().predict()>threshold] = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# we can evalue the TRAINING result by constructing a confusion matrix \n",
    "confusion_matrix(y.iloc[:,1], predict_label.iloc[:,0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# the diagonal elements of the confusion matrix indicate correct predictions, while the off-diagonals represent incorrect predictions. \n",
    "# in this case, logistic regression correctly predicted the movement of the market 52.2% of the time.\n",
    "print(np.mean(y.iloc[:,1] == predict_label.iloc[:,0]))\n",
    "# or use the confusion matrix to compute the accuracy \n",
    "print(confusion_matrix(y.iloc[:,1], predict_label.iloc[:,0]).diagonal().sum()* 1.0 /confusion_matrix(y.iloc[:,1], predict_label.iloc[:,0]).sum())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Train-Validation Split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# in order to better assess the accuracy of the logistic regression model in this setting, \n",
    "# we can fit the model using part of the data, and then examine how well it predicts the hold out data. \n",
    "# this will yield a more realistic error rate, in the sense that in practice we will be interested in our \n",
    "# model’s performance not on the data that we used to fit the model, but rather on days in the future for which the market’s movements are unknown.\n",
    "Smarket_2005 = Smarket.query('Year >= 2005')\n",
    "Smarket_train = Smarket.query('Year < 2005')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# we will use the training dataset to build the logistic regression model \n",
    "y_train, X_train = dmatrices('Direction~Lag1+Lag2+Lag3+Lag4+Lag5+Volume', Smarket_train, return_type = 'dataframe')\n",
    "y_test, X_test = dmatrices('Direction~Lag1+Lag2+Lag3+Lag4+Lag5+Volume', Smarket_2005, return_type = 'dataframe')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "logit = sm.Logit(y_train.iloc[:,1], X_train)\n",
    "print(logit.fit().summary())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "preds = logit.fit().predict(X_test)\n",
    "predict_label = pd.DataFrame(np.zeros(shape=(X_test.shape[0],1)), columns = ['label'])\n",
    "threshold = 0.5\n",
    "mark = (preds > threshold).reset_index(drop=True)\n",
    "predict_label.loc[mark] = 1\n",
    "confusion_matrix(y_test.iloc[:,1], predict_label.iloc[:,0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# to get accuracy\n",
    "np.mean(y_test.iloc[:,1].reset_index(drop=True)==predict_label.iloc[:,0].reset_index(drop=True)) \n",
    "\n",
    "# note: we have trained and tested our model on two completely separate data sets: \n",
    "# training was performed using only the dates before 2005, and testing was performed \n",
    "# using only the dates in 2005. Finally, we compute the predictions for 2005 and compare \n",
    "# them to the actual movements of the market over that time period. The results are rather \n",
    "# disappointing: the test error rate is 1 - 48% = 52 %, which is worse than random guessing \n",
    "# for a balanced data. Of course this result is not all that surprising, given that one \n",
    "# would not generally expect to be able to use previous days’ returns to predict future market performance."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# the retrain of the model with Lag1 and Lag2 will be similar to previous steps (I will be brief here). \n",
    "y_train, X_train = dmatrices('Direction~Lag1+Lag2', Smarket_train, return_type = 'dataframe')\n",
    "y_test, X_test = dmatrices('Direction~Lag1+Lag2', Smarket_2005, return_type = 'dataframe')\n",
    "logit = sm.Logit(y_train.iloc[:,1], X_train)\n",
    "preds = logit.fit().predict(X_test)\n",
    "predict_label = pd.DataFrame(np.zeros(shape=(X_test.shape[0],1)), columns = ['label'])\n",
    "threshold = 0.5\n",
    "confusion_matrix(y_test.iloc[:,1], predict_label.iloc[:,0])\n",
    "np.mean(y_test.iloc[:,1].reset_index(drop=True)==predict_label.iloc[:,0].reset_index(drop=True)) # to get accuracy on validation set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# another way to deal with logistics regression is to change the threshold value from 0.5 to others. \n",
    "# there is an example below with threshold 0.45. \n",
    "preds = logit.fit().predict(X_test)\n",
    "predict_label = pd.DataFrame(np.zeros(shape=(X_test.shape[0],1)), columns = ['label'])\n",
    "threshold = 0.45\n",
    "predict_label.loc[(preds > threshold).reset_index(drop=True)] = 1\n",
    "confusion_matrix(y_test.iloc[:,1], predict_label.iloc[:,0])\n",
    "\n",
    "# to get accuracy on validation set, we did see an improvment of the accuracy from 0.48 to 0.56\n",
    "np.mean(y_test.iloc[:,1].reset_index(drop=True)==predict_label.iloc[:,0].reset_index(drop=True)) "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4.7.3 Linear Discriminant Analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# we will use sklearn's implementation of LDA\n",
    "# from sklearn.discriminant_analysis import LinearDiscriminantAnalysis as LDA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train.iloc[:,1].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# the training process \n",
    "sklearn_lda = LDA(n_components=1) #creating a LDA object\n",
    "lda = sklearn_lda.fit(X_train.iloc[:,1:3], y_train.iloc[:,1]) #learning the projection matrix\n",
    "X_lda = lda.transform(X_train.iloc[:,1:3]) #using the model to project X \n",
    "X_labels = lda.predict(X_train.iloc[:,1:3]) #gives you the predicted label for each sample\n",
    "X_prob = lda.predict_proba(X_train.iloc[:,1:3]) #the probability of each sample to belong to each class"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# testing step \n",
    "X_test_labels =lda.predict(X_test.iloc[:,1:3])\n",
    "X_test_prob = lda.predict_proba(X_test.iloc[:,1:3]) \n",
    "print(X_test_prob[0:5,:])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# get the accuracy of the test set using default threshold\n",
    "np.mean(y_test.iloc[:,1]==X_test_labels) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# let's change the threshod a bit to see whether we can improve the accuracy. \n",
    "# the 2nd column of X_test_prob is the probability belongs to UP group. \n",
    "# the default value is 0.5, let us first check that. \n",
    "threshold = 0.5 \n",
    "np.mean(y_test.iloc[:,1]==(X_test_prob[:,1]>=threshold))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "threshold = 0.48\n",
    "np.mean(y_test.iloc[:,1]==(X_test_prob[:,1]>=threshold))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4.7.4 Quadratic Discriminant Analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# it is a little bit of annoying that QDA and LDA have minor difference in their parameter \n",
    "# set-up and function names. \n",
    "# from sklearn.discriminant_analysis import QuadraticDiscriminantAnalysis as QDA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sklearn_qda = QDA(priors=None,store_covariance=True) #creating a QDA object\n",
    "qda = sklearn_qda.fit(X_train.iloc[:,1:3], y_train.iloc[:,1]) #learning the projection matrix\n",
    "X_labels = qda.predict(X_train.iloc[:,1:3]) #gives you the predicted label for each sample\n",
    "X_prob = qda.predict_proba(X_train.iloc[:,1:3]) #the probability of each sample to belong to each class\n",
    "\n",
    "X_test_labels=qda.predict(X_test.iloc[:,1:3])\n",
    "X_test_prob = qda.predict_proba(X_test.iloc[:,1:3]) \n",
    "\n",
    "print(np.mean(y_test.iloc[:,1]==X_test_labels) )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# again, use dir() to explore all the information stored in lda and qda.\n",
    "dir(qda)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "print(qda.means_)\n",
    "print(qda.covariance_)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4.7.5 Naive Bayes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# from sklearn.naive_bayes import GaussianNB as NB"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "NB_class = NB()\n",
    "NB_class.fit(X_train.iloc[:,1:3], y_train.iloc[:,1])\n",
    "X_test_labels=NB_class.predict(X_test.iloc[:,1:3])\n",
    "X_test_prob = NB_class.predict_proba(X_test.iloc[:,1:3]) \n",
    "print(np.mean(y_test.iloc[:,1]==X_test_labels))\n",
    "\n",
    "dir(NB_class) # use dir command to check what Naive Bayes classifier has"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4.7.6 K-Nearest Neighbors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# from sklearn.neighbors import KNeighborsClassifier as KNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "neigh = KNN(n_neighbors= 4) # use n_neighbors to change the # of tune the performance of KNN\n",
    "KNN_fit = neigh.fit(X_train.iloc[:,1:3], y_train.iloc[:,1]) #learning the projection matrix\n",
    "X_test_labels=KNN_fit.predict(X_test.iloc[:,1:3])\n",
    "X_test_prob = KNN_fit.predict_proba(X_test.iloc[:,1:3]) \n",
    "print(np.mean(y_test.iloc[:,1]==X_test_labels))\n",
    "\n",
    "dir(neigh) # use dir command to check what KNN offers"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4.7.7 Possion Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "Bikeshare = pd.read_csv('data/Bikeshare.csv', header=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(Bikeshare.head())\n",
    "print(Bikeshare.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# first build a linear regression model\n",
    "lm_bikeshare = smf.ols('bikers ~ mnth + hr + workingday + temp + weathersit', data = Bikeshare).fit()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# check the summary of the model, we may see the coefficients are different from the R output.\n",
    "# the diff in the coefficients is due to the difference in the way we chose the baseline for the catergotical variables.\n",
    "# here Python used April as the baseline month - probably due to the alphabetical order of the name of the month.\n",
    "lm_bikeshare.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# after building the model, we could do other things (i.e. plots, other statistics, RMSE etc.) to further explore the results. \n",
    "# here let us get a sense of the RMSE\n",
    "np.sqrt(((lm_bikeshare.fittedvalues - Bikeshare.bikers)**2).sum()/len(Bikeshare.bikers))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# let us go ahead and build a possion regression model \n",
    "# instead of use .ols(), we use .glm()\n",
    "glm_bikeshare = smf.glm('bikers ~ mnth + hr + workingday + temp + weathersit', data = Bikeshare, family=sma.families.Poisson()).fit()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "glm_bikeshare.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# here we do another quick look at the training RMSE \n",
    "# to judge whether model is better, we would do train/validation split and check the model performance on the validation set.\n",
    "np.sqrt(((glm_bikeshare.fittedvalues - Bikeshare.bikers)**2).sum()/len(Bikeshare.bikers))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4.7.8 An Application to Caravan Insurance Data \n",
    "This section is removed from the 2nd edition, but keep it as a reference."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "Caravan = pd.read_csv('data/Caravan.csv', header=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "Caravan.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "Caravan.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "Caravan.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "Scale of the variables matters in KNN ! The core question in KNN is how to define proper distance metric. \n",
    "Because the KNN classifier predicts the class of a given test observation by identifying the observations \n",
    "that are nearest to it, the scale of the variables matters. Any variables that are on a large scale will \n",
    "have a much larger effect on the distance between the observations, and hence on the KNN classifier, \n",
    "than variables that are on a small scale. For instance, imagine a data set that contains two variables, \n",
    "salary and age (measured in dollars and years, respectively). As far as KNN is concerned, \n",
    "a difference of 1,000 in salary is enormous compared to a difference of 50 years in age. \n",
    "Consequently, salary will drive the KNN classification results, and age will have almost no effect. \n",
    "This is contrary to our intuition that a salary difference of 1, 000 is quite small compared to an age difference of 50 years. \n",
    "Furthermore, the importance of scale to the KNN classifier leads to another issue: if we measured salary in Japanese yen, \n",
    "or if we measured age in minutes, then we’d get quite different classification results from what we get \n",
    "if these two variables are measured in dollars and years. \n",
    "\n",
    "A good (debatable) way to handle this problem is to standardize the data so that all standardize \n",
    "variables are given a mean of zero and a standard deviation of one. Then all variables will be on a comparable scale.\n",
    "The scale() function does just scale() this. In standardizing the data, we exclude column 86, \n",
    "because that is the qualitative Purchase variable.\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "predict_label = pd.DataFrame(np.zeros(shape=(Caravan.shape[0],1)), columns = ['label'])\n",
    "predict_label[Caravan['Purchase'] == 'Yes'] = 1\n",
    "Caravan_drop = Caravan.drop(labels='Purchase', axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "I took a slightly different approach from the book. \n",
    "The training and testing data were splited by index. \n",
    "The normalization was done on the train set. \n",
    "Afterwards, the same normalization was applied to validate test. \n",
    "The code might seem wordy, but it helps clear the logical flow. \n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# I took a slightly different approach from the book. The training and testing data were splited by index. \n",
    "# the normalization was done on the train set. Afterwards, the same normalization was applied to validate test.  \n",
    "# the code might seem wordy, but it helps clear the logical flow. \n",
    "train_size = 1000\n",
    "train_index = range(0, train_size)\n",
    "X_validate = Caravan_drop.iloc[train_index, ]\n",
    "Y_validate = predict_label.iloc[train_index, ]\n",
    "X_train = Caravan_drop.iloc[train_size:, ]\n",
    "Y_train = predict_label.iloc[train_size:, ]\n",
    "\n",
    "\n",
    "X_train_scaled = preprocessing.scale(X_train)\n",
    "scaler = preprocessing.StandardScaler().fit(X_train)\n",
    "X_validate_scaled = scaler.transform(X_validate)   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# train with 1 neighbor \n",
    "n_neighbors = 1\n",
    "neigh = KNN(n_neighbors= n_neighbors) # use n_neighbors to change the # of tune the performance of KNN\n",
    "KNN_fit = neigh.fit(X_train_scaled, Y_train.iloc[:,0]) #learning the projection matrix\n",
    "X_validate_labels=KNN_fit.predict(X_validate_scaled)\n",
    "X_validate_prob = KNN_fit.predict_proba(X_validate_scaled) \n",
    "print(np.mean(Y_validate.iloc[:,0]==X_validate_labels))\n",
    "print(confusion_matrix(Y_validate.iloc[:,0], X_validate_labels))\n",
    "\n",
    "# the rest of this exercise considers all the trade-off between False postive and False negative.  \n",
    "# the concept of accuracy is NOT always the golden metric for classification problems. \n",
    "# precision and recall, sensitivity and specificity, F1 score... are all reasonable metrics to consider. \n",
    "# we will discuss more on the concept of trainning, validation and test. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# End of Chapter 4"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "interpreter": {
   "hash": "4548a0e672c5b3a287feee7b2962606840aa548749d1830ef724408652b0c250"
  },
  "kernelspec": {
   "display_name": "Python 2.7.16 64-bit ('base': conda)",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}