{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Lecture: Decision Tree"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn import tree\n",
    "from sklearn.datasets import load_wine\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn import metrics #evaluation model.\n",
    "import matplotlib"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = load_wine()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data.target"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y=data.target\n",
    "\n",
    "y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(data.DESCR)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data.data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame(data.data)\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "list(data.feature_names)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = load_wine()\n",
    "x=data.feature_names\n",
    "df = pd.DataFrame(data.data, columns=x)\n",
    "df.head()\n",
    "df.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X=df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data.target"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data.data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# feature matrix\n",
    "X = data.data\n",
    "\n",
    "# target vector\n",
    "y = data.target\n",
    "\n",
    "# class labels\n",
    "labels = data.feature_names\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Split dataset into training set and test set\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2,random_state=999) # 80% tra"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Step1: Split dataset into training set and test set\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=100) # 80% training and 20% test\n",
    "\n",
    "# Step2: Select model: Create Decision Tree classifer object\n",
    "c0=DecisionTreeClassifier()\n",
    "# Create Decision Tree classifer object\n",
    "c1 =  DecisionTreeClassifier(criterion= \"gini\", max_depth=None)\n",
    "c2 = DecisionTreeClassifier(criterion= \"entropy\", max_depth=None)\n",
    "\n",
    "# Step 3: Train Decision Tree Classifer\n",
    "c0 = c0.fit(X_train,y_train)\n",
    "c1 = c1.fit(X_train,y_train)\n",
    "#Predict the response for test dataset\n",
    "y_pred = c1.predict(X_test)\n",
    "y_score = c1.score(X_test,y_test)\n",
    "y_score "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import classification_report,confusion_matrix\n",
    "\n",
    "print(classification_report(y_test,c1.predict(X_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "a=list((y_test,y_pred))\n",
    "a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "a=list((y_test,y_pred))\n",
    "b=np.transpose(a)\n",
    "b\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "a=pd.DataFrame(b,columns=['y_test','y_pred'])\n",
    "a\n",
    "a['compare']= a['y_test']==a['y_pred']\n",
    "a"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Visualize Decision Trees using Matplotlib"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tree.plot_tree(c1);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.tree import export_graphviz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "feature_name=data.feature_names\n",
    "feature_name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "\n",
    "fn=data.feature_names\n",
    "\n",
    "cn=data.target_names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "type(fn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig, axes = plt.subplots(nrows = 1,ncols = 1,figsize = (50,50), dpi=60)\n",
    "tree.plot_tree(c1,\n",
    "               feature_names =fn, \n",
    "               class_names=cn,\n",
    "               filled = True);\n",
    "fig.savefig('Example1.png')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Model Evaluation\n",
    "\n",
    "Before we get too far into additional classification algorithms, let’s take a\n",
    "step back and think about how to evaluate the performance of a classification\n",
    "model."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Accuracy\n",
    "Perhaps the most intuitive classification metric is accuracy, which is the fraction of correct predictions.\n",
    "\n",
    "For a scikit-learn classifier, this can be computed using the score method."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "c1.score(X_train,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_acc = c1.score(X_train, y_train) #In-sample\n",
    "test_acc = c1.score(X_test, y_test) #Out-of-sample\n",
    "\n",
    "train_acc, test_acc"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "When the testing accuracy is similar to or higher than the training\n",
    "accuracy (as it is here), the model might be underfitting.\n",
    "Thus, we should consider either using a more powerful model or adding additional\n",
    "features.\n",
    "\n",
    "In many contexts, this would be an appropriate way to evaluate a model, but in\n",
    "others, this is insufficient.\n",
    "\n",
    "For example, suppose we want to use a classification model to predict the\n",
    "likelihood of someone having a rare, but serious health condition.\n",
    "\n",
    "If the condition is very rare (say it appears in 0.01% of the population), then\n",
    "a model that always predicts false would have 99.99% accuracy, but the false\n",
    "negatives could have large consequences."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Precision and Recall (contingency matrix)\n",
    "\n",
    "In order to capture situations like that, data scientists often use two other\n",
    "very common metrics:\n",
    "\n",
    "- *Precision*: The number of true positives over the number of positive\n",
    "  predictions. Precision tells us how often the model was correct when it\n",
    "  predicted true.  \n",
    "- *Recall*: The number of true positives over the number of actual positives.\n",
    "  Recall answers the question, “What fraction of the positives did we get\n",
    "  correct?”  \n",
    "\n",
    "\n",
    "In the rare health condition example, you may prefer\n",
    "a model with high recall (never misses an at-risk patient), even if the\n",
    "precision is a bit low (sometimes you have false positives).\n",
    "\n",
    "On the other hand, if your algorithm filters spam emails out of an inbox,\n",
    "you may prefer a model with high precision so that when an email is\n",
    "classified as spam, it is very likely to actually be spam (i.e. non-spam\n",
    "messages don’t get sent to spam folder).\n",
    "\n",
    "In many settings, both precision and recall are equally important and a\n",
    "compound metric known as the F1-score is used:\n",
    "\n",
    "$$\n",
    "F1 = 2 \\frac{\\text{precision} \\cdot \\text{recall}}{\\text{precision} + \\text{recall}}\n",
    "$$\n",
    "\n",
    "The F1 score is bounded between 0 and 1. It will only achieve a value of 1 if\n",
    "both precision and recall are exactly 1.\n",
    "\n",
    "We can have scikit-learn produce a textual report with precision and recall.\n",
    "\n",
    "Scikit-learn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import classification_report,confusion_matrix\n",
    "report = metrics.classification_report(\n",
    "    y_test, clf.predict(X_test),\n",
    "    target_names=[\"wine0\", \"wine1\",\"wine2\"]\n",
    ")\n",
    "print(report)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sklearn.metrics as m"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.sqrt(m.mean_squared_error(y_test,y_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "m.mean_squared_error(y_test,y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print('mean absolute error is',m.mean_absolute_error(y_test,y_pred))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Workshop"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.datasets import load_iris\n",
    "from sklearn import tree\n",
    "iris= load_iris()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(iris.DESCR)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from matplotlib import pyplot as plt\n",
    "import numpy as np\n",
    "\n",
    "\n",
    "# We load the data with load_iris from sklearn\n",
    "from sklearn.datasets import load_iris\n",
    "data = load_iris()\n",
    "# load_iris returns an object with several fields\n",
    "features = data.data\n",
    "feature_names = data.feature_names\n",
    "target = data.target\n",
    "target_names = data.target_names\n",
    "for t in range(3):\n",
    "    \n",
    " if t == 0:\n",
    "     c = 'r'\n",
    "     marker = '>'\n",
    " elif t == 1:\n",
    "     c = 'g'\n",
    "     marker = 'o'\n",
    " elif t == 2:\n",
    "     c = 'b'\n",
    "     marker = 'x'\n",
    " plt.scatter(features[target == t, 2],\n",
    "            features[target == t, 3],\n",
    "            marker=marker,\n",
    "            c=c)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "fig,axes = plt.subplots(1, 2)\n",
    "pairs = [(1, 3), (2, 3)]\n",
    "\n",
    "# Set up 3 different pairs of (color, marker)\n",
    "color_markers = [\n",
    "        ('r', '>'),\n",
    "        ('g', 'o'),\n",
    "        ('b', 'x'),\n",
    "        ]\n",
    "for i, (p0, p1) in enumerate(pairs):\n",
    "    ax = axes.flat[i]\n",
    "\n",
    "    for t in range(3):\n",
    "        # Use a different color/marker for each class `t`\n",
    "        c,marker = color_markers[t]\n",
    "        ax.scatter(features[target == t, p0], features[\n",
    "                    target == t, p1], marker=marker, c=c)\n",
    "    ax.set_xlabel(feature_names[p0])\n",
    "    ax.set_ylabel(feature_names[p1])\n",
    "    ax.set_xticks([])\n",
    "    ax.set_yticks([])\n",
    "fig.tight_layout()\n",
    "fig.savefig('figure1.png')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig,axes = plt.subplots(2, 3)\n",
    "pairs = [(0, 1), (0, 2), (0, 3), (1, 2), (1, 3), (2, 3)]\n",
    "\n",
    "# Set up 3 different pairs of (color, marker)\n",
    "color_markers = [\n",
    "        ('r', '>'),\n",
    "        ('g', 'o'),\n",
    "        ('b', 'x'),\n",
    "        ]\n",
    "for i, (p0, p1) in enumerate(pairs):\n",
    "    ax = axes.flat[i]\n",
    "\n",
    "    for t in range(3):\n",
    "        # Use a different color/marker for each class `t`\n",
    "        c,marker = color_markers[t]\n",
    "        ax.scatter(features[target == t, p0], features[\n",
    "                    target == t, p1], marker=marker, c=c)\n",
    "    ax.set_xlabel(feature_names[p0])\n",
    "    ax.set_ylabel(feature_names[p1])\n",
    "    ax.set_xticks([])\n",
    "    ax.set_yticks([])\n",
    "fig.tight_layout()\n",
    "fig.savefig('figure2.png')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}