LightGBM_Classification_HPTuning_and_Feature_Selection.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "LightGBM Classification.ipynb",
      "provenance": [],
      "authorship_tag": "ABX9TyPjK3AGWUwg7AQrqYZSSAJ6",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/PrettyCharity/Machine_Learning_Practice/blob/main/LightGBM_Classification.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#@title Installing and upgrading packages\n",
        "!pip install lightgbm --upgrade\n",
        "!pip install optuna"
      ],
      "metadata": {
        "cellView": "form",
        "id": "SP8rMWQ5rGcP"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "cellView": "form",
        "id": "931yqq2DShJx"
      },
      "outputs": [],
      "source": [
        "#@title Preparing the data\n",
        "# Importing libraries\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "pd.set_option(\"display.precision\", 4)\n",
        "# Metrics and tools\n",
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn.metrics import f1_score\n",
        "from sklearn.feature_selection import SelectKBest, chi2, f_classif\n",
        "# Optuna packages\n",
        "import optuna.integration.lightgbm as lgb\n",
        "from lightgbm import early_stopping\n",
        "from lightgbm import log_evaluation\n",
        "\n",
        "# Loading the data\n",
        "df = pd.read_csv('Data.csv')\n",
        "X = df.iloc[:, :-1].values\n",
        "y = df.iloc[:, -1].values\n",
        "\n",
        "# Encoding the Dependent Variable\n",
        "from sklearn.preprocessing import LabelEncoder\n",
        "le = LabelEncoder()\n",
        "y = le.fit_transform(y)"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#@title Optuna Lightgbm tuner\n",
        "def tuner(data, target):\n",
        "  # Splitting the data\n",
        "  X_train, X_test, y_train, y_test = train_test_split(data, target,\n",
        "                                                    test_size = 0.25,\n",
        "                                                    random_state = 42)\n",
        "  \n",
        "  # Setting up the data for the model\n",
        "  dtrain = lgb.Dataset(X_train, label = y_train)\n",
        "  dtest = lgb.Dataset(X_test, label = y_test)\n",
        "\n",
        "  # Parameters for classification\n",
        "  params = {\n",
        "      'objective' : 'binary',\n",
        "      'metric' : 'binary_logloss',\n",
        "      'verbosity' : -1,\n",
        "      'boosting_type' : 'gbdt'\n",
        "  }\n",
        "\n",
        "  # Training the model\n",
        "  model = lgb.train(\n",
        "      params,\n",
        "      dtrain,\n",
        "      valid_sets = [dtrain, dtest],\n",
        "      callbacks = [early_stopping(100), log_evaluation(100)]\n",
        "  )\n",
        "\n",
        "  # Results\n",
        "  y_pred = np.rint(model.predict(X_test, \n",
        "                                     num_iteration = model.best_iteration))\n",
        "  score = f1_score(y_test, y_pred)\n",
        "  return score"
      ],
      "metadata": {
        "cellView": "form",
        "id": "ztwqm9gDrUym"
      },
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#@title Feature Selection\n",
        "# Chi-squared stats works for non-negative feature and class\n",
        "score_func = [chi2, f_classif][(df.values < 0).any()]\n",
        "# Store scores and number of features\n",
        "results = {}\n",
        "for num in range(5, X.shape[1] + 1):\n",
        "  X_new = SelectKBest(score_func, k = num).fit_transform(X, y)\n",
        "  results[num] = tuner(X_new, y)\n"
      ],
      "metadata": {
        "cellView": "form",
        "id": "vk5Dx_35uH73"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#@title Results of HP tuning and Feature Selection\n",
        "scores = list(results.values())\n",
        "selection_results = pd.DataFrame(np.array([scores]).reshape(-1, 1),\n",
        "                                 columns = ['f1 score'],\n",
        "                                 index = list(results.keys()))\n",
        "print('LightGBM scores at different number of features:\\n')\n",
        "selection_results.style"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 272
        },
        "cellView": "form",
        "id": "uf45o88Z4zmu",
        "outputId": "e509c25e-601f-4a91-8740-66c1e93473ec"
      },
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "LightGBM scores at different number of features:\n",
            "\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<pandas.io.formats.style.Styler at 0x7f17134027d0>"
            ],
            "text/html": [
              "<style type=\"text/css\">\n",
              "</style>\n",
              "<table id=\"T_6a43e_\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr>\n",
              "      <th class=\"blank level0\" >&nbsp;</th>\n",
              "      <th class=\"col_heading level0 col0\" >f1 score</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th id=\"T_6a43e_level0_row0\" class=\"row_heading level0 row0\" >5</th>\n",
              "      <td id=\"T_6a43e_row0_col0\" class=\"data row0 col0\" >0.9624</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th id=\"T_6a43e_level0_row1\" class=\"row_heading level0 row1\" >6</th>\n",
              "      <td id=\"T_6a43e_row1_col0\" class=\"data row1 col0\" >0.9624</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th id=\"T_6a43e_level0_row2\" class=\"row_heading level0 row2\" >7</th>\n",
              "      <td id=\"T_6a43e_row2_col0\" class=\"data row2 col0\" >0.9624</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th id=\"T_6a43e_level0_row3\" class=\"row_heading level0 row3\" >8</th>\n",
              "      <td id=\"T_6a43e_row3_col0\" class=\"data row3 col0\" >0.9624</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th id=\"T_6a43e_level0_row4\" class=\"row_heading level0 row4\" >9</th>\n",
              "      <td id=\"T_6a43e_row4_col0\" class=\"data row4 col0\" >0.9701</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th id=\"T_6a43e_level0_row5\" class=\"row_heading level0 row5\" >10</th>\n",
              "      <td id=\"T_6a43e_row5_col0\" class=\"data row5 col0\" >0.9701</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n"
            ]
          },
          "metadata": {},
          "execution_count": 22
        }
      ]
    }
  ]
}