QLCF_v1_demo.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "tags": []
   },
   "source": [
    "> **Essential ML process for Intrusion Detection**\n",
    "<br>` python  3.7.13    scikit-learn  1.0.2 `\n",
    "<br>`numpy   1.19.5          pandas  1.3.5`"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "_S34U5S-i69d"
   },
   "source": [
    "**Import the main libraries**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy\n",
    "import pandas\n",
    "\n",
    "from time import time\n",
    "\n",
    "import os\n",
    "data_path = '../datasets'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "9IZetEZ8jQJm"
   },
   "source": [
    "**Import the Dataset**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_file = os.path.join(data_path, 'NSL_ppTrain.csv') \n",
    "train_df = pandas.read_csv(data_file)\n",
    "print('Train Dataset: {} rows, {} columns'.format(train_df.shape[0], train_df.shape[1]))\n",
    "\n",
    "data_file = os.path.join(data_path, 'NSL_ppTest.csv') \n",
    "test_df = pandas.read_csv(data_file)\n",
    "print('Test Dataset: {} rows, {} columns'.format(test_df.shape[0], test_df.shape[1]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "Cpzuyj7gxwCg"
   },
   "source": [
    "***\n",
    "**Data Preparation and EDA** (consistency checks)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "tags": []
   },
   "source": [
    "* _Check column names of numeric attributes_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "trnn = train_df.select_dtypes(include=['float64','int64']).columns\n",
    "tstn = test_df.select_dtypes(include=['float64','int64']).columns\n",
    "trndif = numpy.setdiff1d(trnn, tstn)\n",
    "tstdif = numpy.setdiff1d(tstn, trnn)\n",
    "\n",
    "print(\"Numeric features in the train_set that are not in the test_set: \",end='')\n",
    "if len(trndif) > 0:\n",
    "    print('\\n',trndif)\n",
    "else:\n",
    "    print('None')\n",
    "\n",
    "print(\"Numeric features in the test_set that are not in the train_set: \",end='')\n",
    "if len(tstdif) > 0:\n",
    "    print('\\n',tstdif)\n",
    "else:\n",
    "    print('None')\n",
    "\n",
    "print()\n",
    "# correct any differences here"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* _Check column names of categorical attributes_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "trnn = train_df.select_dtypes(include=['object']).columns\n",
    "tstn = test_df.select_dtypes(include=['object']).columns\n",
    "trndif = numpy.setdiff1d(trnn, tstn)\n",
    "tstdif = numpy.setdiff1d(tstn, trnn)\n",
    "\n",
    "print(\"Categorical features in the train_set that are not in the test_set: \",end='')\n",
    "if len(trndif) > 0:\n",
    "    print('\\n',trndif)\n",
    "else:\n",
    "    print('None')\n",
    "\n",
    "print(\"Categorical features in the test_set that are not in the train_set: \",end='')\n",
    "if len(tstdif) > 0:\n",
    "    print('\\n\\t',tstdif)\n",
    "else:\n",
    "    print('None')\n",
    "\n",
    "print()\n",
    "# correct any differences here"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* _Check categorical feature values:<br>\n",
    "differences will be resolved by one-hot encoding the combined test and train sets_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "trnn = train_df.select_dtypes(include=['object']).columns\n",
    "for col in trnn:\n",
    "    tr = train_df[col].unique()\n",
    "    ts = test_df[col].unique()\n",
    "    trd = numpy.setdiff1d(tr, ts)\n",
    "    tsd = numpy.setdiff1d(ts, tr)\n",
    "    \n",
    "    print(col,'::> ')\n",
    "    print(\"\\tUnique text values in the train_set that are not in the test_set: \",end='')\n",
    "    if len(trd) > 0:\n",
    "        print('\\n\\t',trd)\n",
    "    else:\n",
    "        print('None')\n",
    "    \n",
    "    print(\"\\tUnique text values in the test_set that are not in the train_set: \",end='')\n",
    "    if len(tsd) > 0:\n",
    "        print('\\n\\t',tsd)\n",
    "    else:\n",
    "        print('None')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* _Combine for processing classification target and text features_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "combined_df = pandas.concat([train_df, test_df])\n",
    "print('Combined Dataset: {} rows, {} columns'.format(\n",
    "    combined_df.shape[0], combined_df.shape[1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Classification Target feature:\n",
    "# two columns of labels are available \n",
    "#    * Two-class: labels     * Multiclass: atakcat\n",
    "\n",
    "# Two-class: Reduce the detailed attack labels to 'normal' or 'attack'\n",
    "labels_df = combined_df['label'].copy()\n",
    "labels_df[labels_df != 'normal'] = 'attack'\n",
    "\n",
    "# drop target features \n",
    "combined_df.drop(['label'], axis=1, inplace=True)\n",
    "combined_df.drop(['atakcat'], axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# one-Hot encoding the remaining text features\n",
    "categori = combined_df.select_dtypes(include=['object']).columns\n",
    "category_cols = categori.tolist()\n",
    "\n",
    "features_df = pandas.get_dummies(combined_df, columns=category_cols)\n",
    "features_df.info(verbose=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Restore the train // test split: slice 1 Dataframe into 2 \n",
    "# pandas has a lot of rules about returning a 'view' vs. a copy from slice\n",
    "# so we force it to create a new dataframe [avoiding SettingWithCopy Warning]\n",
    "features_train = features_df.iloc[:len(train_df),:].copy()    # X_train\n",
    "features_train.reset_index(inplace=True, drop=True)\n",
    "\n",
    "features_test = features_df.iloc[len(train_df):,:].copy()     # X_test\n",
    "features_test.reset_index(inplace=True, drop=True)\n",
    "\n",
    "# Restore the train // test split: slice 1 Series into 2 \n",
    "labels_train = labels_df[:len(train_df)]               # y_train\n",
    "labels_train.reset_index(inplace=True, drop=True)\n",
    "\n",
    "labels_test = labels_df[len(train_df):]                # y_test\n",
    "labels_test.reset_index(inplace=True, drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "***\n",
    "**QLCFF: Quick Layered Correlation-based Feature Filter**<br>\n",
    "> **_library requirements:_**<br>\n",
    "    * Dataframe of features (text values may be one-hot encoded)<br>\n",
    "    * Class labels in np.ndarray or pd.Series with shape (n,1)<br>\n",
    "    * Binary classification (not multiclass or multilabel)<br><br>\n",
    "> **_workflow:_**<br>\n",
    "Workflow: Correlation-based feature filtering has four steps: preprocessing, discretization, calculating correlations, and feature reduction.\n",
    "Here the first two steps are implemented in the Discretizer class, and the second two steps in the qlcfFilter class."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "***\n",
    "_**import the local library**_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# add parent folder path where lib folder is\n",
    "import sys\n",
    "if \"..\" not in sys.path:import sys; sys.path.insert(0, '..') "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from QLCFF import Discretizer, qlcfFilter"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "***\n",
    "_**the discretizer**_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dtzr = Discretizer(numjobs= -2, msglvl=5)   # Initialise\n",
    "# Requires : none\n",
    "# Optional : joblib Parallel(n_jobs=, verbose=)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dtzr.fit(features_test, labels_test)    # Calls the preprocessor\n",
    "\n",
    "# Requires : features as pd.dataframe, labels as array-like\n",
    "# Optional : none\n",
    "#  X : preprocessor\n",
    "#    1. selects only column dtypes np.number and pd or np boolean\n",
    "#    2. normalizes all columns with signed dtypes to positive numbers\n",
    "#    3. normalizes all columns with boolean dtypes to zero//one\n",
    "#  y : Text labels are converted with sklearn LabelEncoder()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# After fit(), the preprocessed dataframe is an attribute\n",
    "dtzr.prebin_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# the discretized dataframe is an attribute after transform()\n",
    "_ = dtzr.transform(mkbins='hgrm', detail=True)\n",
    "\n",
    "# Returns  : discretized df\n",
    "# Requires : none\n",
    "# Optional : binning strategy, default or one of\n",
    "#     'unif-ten'  'unif-log'  'unif-sqrt'\n",
    "#     'mdlp-ten'  'mdlp-log'  'mdlp-sqrt'\n",
    "#     'chim-ten'  'chim-log'  'chim-sqrt'\n",
    "# Optional : (boolean) print binning report\n",
    "\n",
    "# Binning Strategy\n",
    "# The default value mkbins=hgrm applies numpy.histogram(feature, bins='auto'), \n",
    "# and repeatedly folds lower bins into the next higher one until there are a \n",
    "# maximum of 12 for the feature.\n",
    "\n",
    "# Otherwise, the valid values combine an algorithm for calculating the bin  \n",
    "# edges (cutpoints) with a method for determining the maximum number of bins.\n",
    "#     calculate edges\n",
    "#         unif: uniform [numpy.linspace()]\n",
    "#         mdlp: MDLP algorithm \n",
    "#         chim: ChiMerge algorithm\n",
    "#     number of bins\n",
    "#         ten: always ten \n",
    "#         sqrt: sqrt(len(feature)) \n",
    "#         log: log10(len(feature))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## After transform():\n",
    "\n",
    "# the discretized dataframe is an attribute\n",
    "dtzr.binned_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# the dict of bin edges is an attribute\n",
    "dtzr.cutpoints"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# note: distribution of values within bins\n",
    "for col in dtzr.binned_df.columns:\n",
    "    print(col, numpy.bincount(dtzr.binned_df[col].values))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "***\n",
    "_**the feature filter**_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ffltr = qlcfFilter()   #Initialise\n",
    "# Requires : none\n",
    "# Optional : none"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create layered feature selection filters \n",
    "\n",
    "# most informative\n",
    "fltrs = ['FDR', 'FWE', 'Floor', 'FCBF-SU', 'FCBF-PC']\n",
    "\n",
    "# quick way to drop the most\n",
    "#fltrs = ['Floor', 'FCBF-PC']\n",
    "\n",
    "ffltr.fit(dtzr.binned_df, labels_test, fltrs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## ffltr.fit(X, y, filters, plvl=0.5, minpc=0.035, minsu=0.0025, hipc=0.82, hisu=0.7)\n",
    "\n",
    "# Requires : discretizer.binned_df, labels as array-like, list of one or more filters\n",
    "# Optional : *varies depending on filters selected\n",
    "\n",
    "# Filters\n",
    "# A list with one or more of \n",
    "#     'Floor', 'FDR', 'FWE', 'FCBF-SU', 'FCBF-PC'\n",
    "# The list is processed in order with progressive filtering\n",
    "\n",
    "## 'Floor': filters on the basis that low correlation with the target labels (f2y) \n",
    "#          means low utility for distinguishing class membership. Keeps features that have \n",
    "#          f2y correlation greater than a threshold value.\n",
    "#          Optional :\n",
    "#              minpc : threshold for pearson correlation\n",
    "#              minsu : threshold for symmetric uncertainty\n",
    "## 'FDR', 'FWE': sklearn univariate chi-square test; selects features to keep \n",
    "#          based on an upper bound on the expected false discovery rate. \n",
    "#          fwe will select more to drop than fdr, \n",
    "#              lower thresholds will also select more to drop. \n",
    "#          The floor filter will select all from either univariate test, and more.\n",
    "#          Optional :\n",
    "#              plvl : chi-square threshold (alpha), standard values are 0.01, 0.05, 0.1\n",
    "## 'FCBF-SU', 'FCBF-PC': FCBF-style, filter on feature-to-feature (f2f) correlations. \n",
    "#          Given a group of features with high cross-correlations, keep the one with \n",
    "#          the highest (f2y) as a proxy for the others. \n",
    "#          Optional :\n",
    "#              hipc : threshold for \"high\" f2f pearson correlation\n",
    "#              hisu : threshold for \"high\" f2f symmetric uncertainty"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# After fit() -\n",
    "\n",
    "# the consolidated drop list is an attribute\n",
    "ffltr.QLCFFilter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# reporting methods are available\n",
    "\n",
    "# print feature-to-label (f2y) correlations\n",
    "# Optional : kd = 'keep' or 'drop'\n",
    "ffltr.get_f2y_report(kd='drop')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# returns a dict of correlations for each filter\n",
    "# Optional : kd = 'keep' or 'drop'\n",
    "fyd = ffltr.get_f2y_dict(kd='drop')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# print feature to feature (f2f) correlations above threshold report\n",
    "# only available for 'FCBF-SU' or 'FCBF-PC'\n",
    "ffltr.get_f2f_report()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# returns a dict of f2f correlations checked by each filter\n",
    "# only available for 'FCBF-SU' or 'FCBF-PC'\n",
    "ffd = ffltr.get_f2f_dict()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# apply the consolidated drop list\n",
    "reduced_df = ffltr.transform(features_test)\n",
    "\n",
    "# Requires : actual pd.dataframe for clf.fit_predict()\n",
    "# Optional : none\n",
    "\n",
    "reduced_df.info(verbose=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "***\n",
    "_**fit_transform**_<br>\n",
    "> _instantiate separately if you want attributes & reports_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# fit_transform()\n",
    "dtzdf = Discretizer(numjobs= -2, msglvl=5).fit_transform(features_test,\n",
    "                                                         labels_test,\n",
    "                                                         mkbins='mdlp-log',\n",
    "                                                         detail=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fltrs = ['Floor']\n",
    "filtered_df = qlcfFilter().fit_transform(dtzdf,\n",
    "                                         labels_test,\n",
    "                                         fltrs,\n",
    "                                         features_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "filtered_df.info(verbose=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "Cpzuyj7gxwCg",
    "tags": []
   },
   "source": [
    " ***"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "Cpzuyj7gxwCg",
    "tags": []
   },
   "source": [
    " ***"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.15"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}