DataWrangling.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# S1 - S21 are different sensor values Column Names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "col_names = ['id','cycle','setting1','setting2','setting3',\n",
    "            's1','s2','s3','s4','s5','s6','s7','s8','s9','s10','s11','s12','s13','s14','s15','s16','s17','s18','s19','s20','s21']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Loading Raw Train/Test Data "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>...</th>\n",
       "      <th>18</th>\n",
       "      <th>19</th>\n",
       "      <th>20</th>\n",
       "      <th>21</th>\n",
       "      <th>22</th>\n",
       "      <th>23</th>\n",
       "      <th>24</th>\n",
       "      <th>25</th>\n",
       "      <th>26</th>\n",
       "      <th>27</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>-0.0007</td>\n",
       "      <td>-0.0004</td>\n",
       "      <td>100.0</td>\n",
       "      <td>518.67</td>\n",
       "      <td>641.82</td>\n",
       "      <td>1589.70</td>\n",
       "      <td>1400.60</td>\n",
       "      <td>14.62</td>\n",
       "      <td>...</td>\n",
       "      <td>8138.62</td>\n",
       "      <td>8.4195</td>\n",
       "      <td>0.03</td>\n",
       "      <td>392</td>\n",
       "      <td>2388</td>\n",
       "      <td>100.0</td>\n",
       "      <td>39.06</td>\n",
       "      <td>23.4190</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>0.0019</td>\n",
       "      <td>-0.0003</td>\n",
       "      <td>100.0</td>\n",
       "      <td>518.67</td>\n",
       "      <td>642.15</td>\n",
       "      <td>1591.82</td>\n",
       "      <td>1403.14</td>\n",
       "      <td>14.62</td>\n",
       "      <td>...</td>\n",
       "      <td>8131.49</td>\n",
       "      <td>8.4318</td>\n",
       "      <td>0.03</td>\n",
       "      <td>392</td>\n",
       "      <td>2388</td>\n",
       "      <td>100.0</td>\n",
       "      <td>39.00</td>\n",
       "      <td>23.4236</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>-0.0043</td>\n",
       "      <td>0.0003</td>\n",
       "      <td>100.0</td>\n",
       "      <td>518.67</td>\n",
       "      <td>642.35</td>\n",
       "      <td>1587.99</td>\n",
       "      <td>1404.20</td>\n",
       "      <td>14.62</td>\n",
       "      <td>...</td>\n",
       "      <td>8133.23</td>\n",
       "      <td>8.4178</td>\n",
       "      <td>0.03</td>\n",
       "      <td>390</td>\n",
       "      <td>2388</td>\n",
       "      <td>100.0</td>\n",
       "      <td>38.95</td>\n",
       "      <td>23.3442</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>0.0007</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>100.0</td>\n",
       "      <td>518.67</td>\n",
       "      <td>642.35</td>\n",
       "      <td>1582.79</td>\n",
       "      <td>1401.87</td>\n",
       "      <td>14.62</td>\n",
       "      <td>...</td>\n",
       "      <td>8133.83</td>\n",
       "      <td>8.3682</td>\n",
       "      <td>0.03</td>\n",
       "      <td>392</td>\n",
       "      <td>2388</td>\n",
       "      <td>100.0</td>\n",
       "      <td>38.88</td>\n",
       "      <td>23.3739</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>-0.0019</td>\n",
       "      <td>-0.0002</td>\n",
       "      <td>100.0</td>\n",
       "      <td>518.67</td>\n",
       "      <td>642.37</td>\n",
       "      <td>1582.85</td>\n",
       "      <td>1406.22</td>\n",
       "      <td>14.62</td>\n",
       "      <td>...</td>\n",
       "      <td>8133.80</td>\n",
       "      <td>8.4294</td>\n",
       "      <td>0.03</td>\n",
       "      <td>393</td>\n",
       "      <td>2388</td>\n",
       "      <td>100.0</td>\n",
       "      <td>38.90</td>\n",
       "      <td>23.4044</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 28 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   0   1       2       3      4       5       6        7        8      9   \\\n",
       "0   1   1 -0.0007 -0.0004  100.0  518.67  641.82  1589.70  1400.60  14.62   \n",
       "1   1   2  0.0019 -0.0003  100.0  518.67  642.15  1591.82  1403.14  14.62   \n",
       "2   1   3 -0.0043  0.0003  100.0  518.67  642.35  1587.99  1404.20  14.62   \n",
       "3   1   4  0.0007  0.0000  100.0  518.67  642.35  1582.79  1401.87  14.62   \n",
       "4   1   5 -0.0019 -0.0002  100.0  518.67  642.37  1582.85  1406.22  14.62   \n",
       "\n",
       "   ...       18      19    20   21    22     23     24       25  26  27  \n",
       "0  ...  8138.62  8.4195  0.03  392  2388  100.0  39.06  23.4190 NaN NaN  \n",
       "1  ...  8131.49  8.4318  0.03  392  2388  100.0  39.00  23.4236 NaN NaN  \n",
       "2  ...  8133.23  8.4178  0.03  390  2388  100.0  38.95  23.3442 NaN NaN  \n",
       "3  ...  8133.83  8.3682  0.03  392  2388  100.0  38.88  23.3739 NaN NaN  \n",
       "4  ...  8133.80  8.4294  0.03  393  2388  100.0  38.90  23.4044 NaN NaN  \n",
       "\n",
       "[5 rows x 28 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train_raw = pd.read_csv('data/PM_train.csv', sep = ' ', header=None)\n",
    "df_train_raw.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train_raw.drop([26,27],axis=1,inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>cycle</th>\n",
       "      <th>setting1</th>\n",
       "      <th>setting2</th>\n",
       "      <th>setting3</th>\n",
       "      <th>s1</th>\n",
       "      <th>s2</th>\n",
       "      <th>s3</th>\n",
       "      <th>s4</th>\n",
       "      <th>s5</th>\n",
       "      <th>...</th>\n",
       "      <th>s12</th>\n",
       "      <th>s13</th>\n",
       "      <th>s14</th>\n",
       "      <th>s15</th>\n",
       "      <th>s16</th>\n",
       "      <th>s17</th>\n",
       "      <th>s18</th>\n",
       "      <th>s19</th>\n",
       "      <th>s20</th>\n",
       "      <th>s21</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>-0.0007</td>\n",
       "      <td>-0.0004</td>\n",
       "      <td>100.0</td>\n",
       "      <td>518.67</td>\n",
       "      <td>641.82</td>\n",
       "      <td>1589.70</td>\n",
       "      <td>1400.60</td>\n",
       "      <td>14.62</td>\n",
       "      <td>...</td>\n",
       "      <td>521.66</td>\n",
       "      <td>2388.02</td>\n",
       "      <td>8138.62</td>\n",
       "      <td>8.4195</td>\n",
       "      <td>0.03</td>\n",
       "      <td>392</td>\n",
       "      <td>2388</td>\n",
       "      <td>100.0</td>\n",
       "      <td>39.06</td>\n",
       "      <td>23.4190</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>0.0019</td>\n",
       "      <td>-0.0003</td>\n",
       "      <td>100.0</td>\n",
       "      <td>518.67</td>\n",
       "      <td>642.15</td>\n",
       "      <td>1591.82</td>\n",
       "      <td>1403.14</td>\n",
       "      <td>14.62</td>\n",
       "      <td>...</td>\n",
       "      <td>522.28</td>\n",
       "      <td>2388.07</td>\n",
       "      <td>8131.49</td>\n",
       "      <td>8.4318</td>\n",
       "      <td>0.03</td>\n",
       "      <td>392</td>\n",
       "      <td>2388</td>\n",
       "      <td>100.0</td>\n",
       "      <td>39.00</td>\n",
       "      <td>23.4236</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>-0.0043</td>\n",
       "      <td>0.0003</td>\n",
       "      <td>100.0</td>\n",
       "      <td>518.67</td>\n",
       "      <td>642.35</td>\n",
       "      <td>1587.99</td>\n",
       "      <td>1404.20</td>\n",
       "      <td>14.62</td>\n",
       "      <td>...</td>\n",
       "      <td>522.42</td>\n",
       "      <td>2388.03</td>\n",
       "      <td>8133.23</td>\n",
       "      <td>8.4178</td>\n",
       "      <td>0.03</td>\n",
       "      <td>390</td>\n",
       "      <td>2388</td>\n",
       "      <td>100.0</td>\n",
       "      <td>38.95</td>\n",
       "      <td>23.3442</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>0.0007</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>100.0</td>\n",
       "      <td>518.67</td>\n",
       "      <td>642.35</td>\n",
       "      <td>1582.79</td>\n",
       "      <td>1401.87</td>\n",
       "      <td>14.62</td>\n",
       "      <td>...</td>\n",
       "      <td>522.86</td>\n",
       "      <td>2388.08</td>\n",
       "      <td>8133.83</td>\n",
       "      <td>8.3682</td>\n",
       "      <td>0.03</td>\n",
       "      <td>392</td>\n",
       "      <td>2388</td>\n",
       "      <td>100.0</td>\n",
       "      <td>38.88</td>\n",
       "      <td>23.3739</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>-0.0019</td>\n",
       "      <td>-0.0002</td>\n",
       "      <td>100.0</td>\n",
       "      <td>518.67</td>\n",
       "      <td>642.37</td>\n",
       "      <td>1582.85</td>\n",
       "      <td>1406.22</td>\n",
       "      <td>14.62</td>\n",
       "      <td>...</td>\n",
       "      <td>522.19</td>\n",
       "      <td>2388.04</td>\n",
       "      <td>8133.80</td>\n",
       "      <td>8.4294</td>\n",
       "      <td>0.03</td>\n",
       "      <td>393</td>\n",
       "      <td>2388</td>\n",
       "      <td>100.0</td>\n",
       "      <td>38.90</td>\n",
       "      <td>23.4044</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 26 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   id  cycle  setting1  setting2  setting3      s1      s2       s3       s4  \\\n",
       "0   1      1   -0.0007   -0.0004     100.0  518.67  641.82  1589.70  1400.60   \n",
       "1   1      2    0.0019   -0.0003     100.0  518.67  642.15  1591.82  1403.14   \n",
       "2   1      3   -0.0043    0.0003     100.0  518.67  642.35  1587.99  1404.20   \n",
       "3   1      4    0.0007    0.0000     100.0  518.67  642.35  1582.79  1401.87   \n",
       "4   1      5   -0.0019   -0.0002     100.0  518.67  642.37  1582.85  1406.22   \n",
       "\n",
       "      s5  ...     s12      s13      s14     s15   s16  s17   s18    s19  \\\n",
       "0  14.62  ...  521.66  2388.02  8138.62  8.4195  0.03  392  2388  100.0   \n",
       "1  14.62  ...  522.28  2388.07  8131.49  8.4318  0.03  392  2388  100.0   \n",
       "2  14.62  ...  522.42  2388.03  8133.23  8.4178  0.03  390  2388  100.0   \n",
       "3  14.62  ...  522.86  2388.08  8133.83  8.3682  0.03  392  2388  100.0   \n",
       "4  14.62  ...  522.19  2388.04  8133.80  8.4294  0.03  393  2388  100.0   \n",
       "\n",
       "     s20      s21  \n",
       "0  39.06  23.4190  \n",
       "1  39.00  23.4236  \n",
       "2  38.95  23.3442  \n",
       "3  38.88  23.3739  \n",
       "4  38.90  23.4044  \n",
       "\n",
       "[5 rows x 26 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train_raw.columns = col_names\n",
    "df_train_raw.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>cycle</th>\n",
       "      <th>setting1</th>\n",
       "      <th>setting2</th>\n",
       "      <th>setting3</th>\n",
       "      <th>s1</th>\n",
       "      <th>s2</th>\n",
       "      <th>s3</th>\n",
       "      <th>s4</th>\n",
       "      <th>s5</th>\n",
       "      <th>...</th>\n",
       "      <th>s12</th>\n",
       "      <th>s13</th>\n",
       "      <th>s14</th>\n",
       "      <th>s15</th>\n",
       "      <th>s16</th>\n",
       "      <th>s17</th>\n",
       "      <th>s18</th>\n",
       "      <th>s19</th>\n",
       "      <th>s20</th>\n",
       "      <th>s21</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>count</td>\n",
       "      <td>20631.000000</td>\n",
       "      <td>20631.000000</td>\n",
       "      <td>20631.000000</td>\n",
       "      <td>20631.000000</td>\n",
       "      <td>20631.0</td>\n",
       "      <td>2.063100e+04</td>\n",
       "      <td>20631.000000</td>\n",
       "      <td>20631.000000</td>\n",
       "      <td>20631.000000</td>\n",
       "      <td>2.063100e+04</td>\n",
       "      <td>...</td>\n",
       "      <td>20631.000000</td>\n",
       "      <td>20631.000000</td>\n",
       "      <td>20631.000000</td>\n",
       "      <td>20631.000000</td>\n",
       "      <td>2.063100e+04</td>\n",
       "      <td>20631.000000</td>\n",
       "      <td>20631.0</td>\n",
       "      <td>20631.0</td>\n",
       "      <td>20631.000000</td>\n",
       "      <td>20631.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>mean</td>\n",
       "      <td>51.506568</td>\n",
       "      <td>108.807862</td>\n",
       "      <td>-0.000009</td>\n",
       "      <td>0.000002</td>\n",
       "      <td>100.0</td>\n",
       "      <td>5.186700e+02</td>\n",
       "      <td>642.680934</td>\n",
       "      <td>1590.523119</td>\n",
       "      <td>1408.933782</td>\n",
       "      <td>1.462000e+01</td>\n",
       "      <td>...</td>\n",
       "      <td>521.413470</td>\n",
       "      <td>2388.096152</td>\n",
       "      <td>8143.752722</td>\n",
       "      <td>8.442146</td>\n",
       "      <td>3.000000e-02</td>\n",
       "      <td>393.210654</td>\n",
       "      <td>2388.0</td>\n",
       "      <td>100.0</td>\n",
       "      <td>38.816271</td>\n",
       "      <td>23.289705</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>std</td>\n",
       "      <td>29.227633</td>\n",
       "      <td>68.880990</td>\n",
       "      <td>0.002187</td>\n",
       "      <td>0.000293</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6.537152e-11</td>\n",
       "      <td>0.500053</td>\n",
       "      <td>6.131150</td>\n",
       "      <td>9.000605</td>\n",
       "      <td>3.394700e-12</td>\n",
       "      <td>...</td>\n",
       "      <td>0.737553</td>\n",
       "      <td>0.071919</td>\n",
       "      <td>19.076176</td>\n",
       "      <td>0.037505</td>\n",
       "      <td>1.556432e-14</td>\n",
       "      <td>1.548763</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.180746</td>\n",
       "      <td>0.108251</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>min</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>-0.008700</td>\n",
       "      <td>-0.000600</td>\n",
       "      <td>100.0</td>\n",
       "      <td>5.186700e+02</td>\n",
       "      <td>641.210000</td>\n",
       "      <td>1571.040000</td>\n",
       "      <td>1382.250000</td>\n",
       "      <td>1.462000e+01</td>\n",
       "      <td>...</td>\n",
       "      <td>518.690000</td>\n",
       "      <td>2387.880000</td>\n",
       "      <td>8099.940000</td>\n",
       "      <td>8.324900</td>\n",
       "      <td>3.000000e-02</td>\n",
       "      <td>388.000000</td>\n",
       "      <td>2388.0</td>\n",
       "      <td>100.0</td>\n",
       "      <td>38.140000</td>\n",
       "      <td>22.894200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>25%</td>\n",
       "      <td>26.000000</td>\n",
       "      <td>52.000000</td>\n",
       "      <td>-0.001500</td>\n",
       "      <td>-0.000200</td>\n",
       "      <td>100.0</td>\n",
       "      <td>5.186700e+02</td>\n",
       "      <td>642.325000</td>\n",
       "      <td>1586.260000</td>\n",
       "      <td>1402.360000</td>\n",
       "      <td>1.462000e+01</td>\n",
       "      <td>...</td>\n",
       "      <td>520.960000</td>\n",
       "      <td>2388.040000</td>\n",
       "      <td>8133.245000</td>\n",
       "      <td>8.414900</td>\n",
       "      <td>3.000000e-02</td>\n",
       "      <td>392.000000</td>\n",
       "      <td>2388.0</td>\n",
       "      <td>100.0</td>\n",
       "      <td>38.700000</td>\n",
       "      <td>23.221800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>50%</td>\n",
       "      <td>52.000000</td>\n",
       "      <td>104.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>100.0</td>\n",
       "      <td>5.186700e+02</td>\n",
       "      <td>642.640000</td>\n",
       "      <td>1590.100000</td>\n",
       "      <td>1408.040000</td>\n",
       "      <td>1.462000e+01</td>\n",
       "      <td>...</td>\n",
       "      <td>521.480000</td>\n",
       "      <td>2388.090000</td>\n",
       "      <td>8140.540000</td>\n",
       "      <td>8.438900</td>\n",
       "      <td>3.000000e-02</td>\n",
       "      <td>393.000000</td>\n",
       "      <td>2388.0</td>\n",
       "      <td>100.0</td>\n",
       "      <td>38.830000</td>\n",
       "      <td>23.297900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>75%</td>\n",
       "      <td>77.000000</td>\n",
       "      <td>156.000000</td>\n",
       "      <td>0.001500</td>\n",
       "      <td>0.000300</td>\n",
       "      <td>100.0</td>\n",
       "      <td>5.186700e+02</td>\n",
       "      <td>643.000000</td>\n",
       "      <td>1594.380000</td>\n",
       "      <td>1414.555000</td>\n",
       "      <td>1.462000e+01</td>\n",
       "      <td>...</td>\n",
       "      <td>521.950000</td>\n",
       "      <td>2388.140000</td>\n",
       "      <td>8148.310000</td>\n",
       "      <td>8.465600</td>\n",
       "      <td>3.000000e-02</td>\n",
       "      <td>394.000000</td>\n",
       "      <td>2388.0</td>\n",
       "      <td>100.0</td>\n",
       "      <td>38.950000</td>\n",
       "      <td>23.366800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>max</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>362.000000</td>\n",
       "      <td>0.008700</td>\n",
       "      <td>0.000600</td>\n",
       "      <td>100.0</td>\n",
       "      <td>5.186700e+02</td>\n",
       "      <td>644.530000</td>\n",
       "      <td>1616.910000</td>\n",
       "      <td>1441.490000</td>\n",
       "      <td>1.462000e+01</td>\n",
       "      <td>...</td>\n",
       "      <td>523.380000</td>\n",
       "      <td>2388.560000</td>\n",
       "      <td>8293.720000</td>\n",
       "      <td>8.584800</td>\n",
       "      <td>3.000000e-02</td>\n",
       "      <td>400.000000</td>\n",
       "      <td>2388.0</td>\n",
       "      <td>100.0</td>\n",
       "      <td>39.430000</td>\n",
       "      <td>23.618400</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8 rows × 26 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                 id         cycle      setting1      setting2  setting3  \\\n",
       "count  20631.000000  20631.000000  20631.000000  20631.000000   20631.0   \n",
       "mean      51.506568    108.807862     -0.000009      0.000002     100.0   \n",
       "std       29.227633     68.880990      0.002187      0.000293       0.0   \n",
       "min        1.000000      1.000000     -0.008700     -0.000600     100.0   \n",
       "25%       26.000000     52.000000     -0.001500     -0.000200     100.0   \n",
       "50%       52.000000    104.000000      0.000000      0.000000     100.0   \n",
       "75%       77.000000    156.000000      0.001500      0.000300     100.0   \n",
       "max      100.000000    362.000000      0.008700      0.000600     100.0   \n",
       "\n",
       "                 s1            s2            s3            s4            s5  \\\n",
       "count  2.063100e+04  20631.000000  20631.000000  20631.000000  2.063100e+04   \n",
       "mean   5.186700e+02    642.680934   1590.523119   1408.933782  1.462000e+01   \n",
       "std    6.537152e-11      0.500053      6.131150      9.000605  3.394700e-12   \n",
       "min    5.186700e+02    641.210000   1571.040000   1382.250000  1.462000e+01   \n",
       "25%    5.186700e+02    642.325000   1586.260000   1402.360000  1.462000e+01   \n",
       "50%    5.186700e+02    642.640000   1590.100000   1408.040000  1.462000e+01   \n",
       "75%    5.186700e+02    643.000000   1594.380000   1414.555000  1.462000e+01   \n",
       "max    5.186700e+02    644.530000   1616.910000   1441.490000  1.462000e+01   \n",
       "\n",
       "       ...           s12           s13           s14           s15  \\\n",
       "count  ...  20631.000000  20631.000000  20631.000000  20631.000000   \n",
       "mean   ...    521.413470   2388.096152   8143.752722      8.442146   \n",
       "std    ...      0.737553      0.071919     19.076176      0.037505   \n",
       "min    ...    518.690000   2387.880000   8099.940000      8.324900   \n",
       "25%    ...    520.960000   2388.040000   8133.245000      8.414900   \n",
       "50%    ...    521.480000   2388.090000   8140.540000      8.438900   \n",
       "75%    ...    521.950000   2388.140000   8148.310000      8.465600   \n",
       "max    ...    523.380000   2388.560000   8293.720000      8.584800   \n",
       "\n",
       "                s16           s17      s18      s19           s20  \\\n",
       "count  2.063100e+04  20631.000000  20631.0  20631.0  20631.000000   \n",
       "mean   3.000000e-02    393.210654   2388.0    100.0     38.816271   \n",
       "std    1.556432e-14      1.548763      0.0      0.0      0.180746   \n",
       "min    3.000000e-02    388.000000   2388.0    100.0     38.140000   \n",
       "25%    3.000000e-02    392.000000   2388.0    100.0     38.700000   \n",
       "50%    3.000000e-02    393.000000   2388.0    100.0     38.830000   \n",
       "75%    3.000000e-02    394.000000   2388.0    100.0     38.950000   \n",
       "max    3.000000e-02    400.000000   2388.0    100.0     39.430000   \n",
       "\n",
       "                s21  \n",
       "count  20631.000000  \n",
       "mean      23.289705  \n",
       "std        0.108251  \n",
       "min       22.894200  \n",
       "25%       23.221800  \n",
       "50%       23.297900  \n",
       "75%       23.366800  \n",
       "max       23.618400  \n",
       "\n",
       "[8 rows x 26 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train_raw.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "id            int64\n",
       "cycle         int64\n",
       "setting1    float64\n",
       "setting2    float64\n",
       "setting3    float64\n",
       "s1          float64\n",
       "s2          float64\n",
       "s3          float64\n",
       "s4          float64\n",
       "s5          float64\n",
       "s6          float64\n",
       "s7          float64\n",
       "s8          float64\n",
       "s9          float64\n",
       "s10         float64\n",
       "s11         float64\n",
       "s12         float64\n",
       "s13         float64\n",
       "s14         float64\n",
       "s15         float64\n",
       "s16         float64\n",
       "s17           int64\n",
       "s18           int64\n",
       "s19         float64\n",
       "s20         float64\n",
       "s21         float64\n",
       "dtype: object"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train_raw.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "id          0\n",
       "cycle       0\n",
       "setting1    0\n",
       "setting2    0\n",
       "setting3    0\n",
       "s1          0\n",
       "s2          0\n",
       "s3          0\n",
       "s4          0\n",
       "s5          0\n",
       "s6          0\n",
       "s7          0\n",
       "s8          0\n",
       "s9          0\n",
       "s10         0\n",
       "s11         0\n",
       "s12         0\n",
       "s13         0\n",
       "s14         0\n",
       "s15         0\n",
       "s16         0\n",
       "s17         0\n",
       "s18         0\n",
       "s19         0\n",
       "s20         0\n",
       "s21         0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train_raw.isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>...</th>\n",
       "      <th>18</th>\n",
       "      <th>19</th>\n",
       "      <th>20</th>\n",
       "      <th>21</th>\n",
       "      <th>22</th>\n",
       "      <th>23</th>\n",
       "      <th>24</th>\n",
       "      <th>25</th>\n",
       "      <th>26</th>\n",
       "      <th>27</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0023</td>\n",
       "      <td>0.0003</td>\n",
       "      <td>100.0</td>\n",
       "      <td>518.67</td>\n",
       "      <td>643.02</td>\n",
       "      <td>1585.29</td>\n",
       "      <td>1398.21</td>\n",
       "      <td>14.62</td>\n",
       "      <td>...</td>\n",
       "      <td>8125.55</td>\n",
       "      <td>8.4052</td>\n",
       "      <td>0.03</td>\n",
       "      <td>392</td>\n",
       "      <td>2388</td>\n",
       "      <td>100.0</td>\n",
       "      <td>38.86</td>\n",
       "      <td>23.3735</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>-0.0027</td>\n",
       "      <td>-0.0003</td>\n",
       "      <td>100.0</td>\n",
       "      <td>518.67</td>\n",
       "      <td>641.71</td>\n",
       "      <td>1588.45</td>\n",
       "      <td>1395.42</td>\n",
       "      <td>14.62</td>\n",
       "      <td>...</td>\n",
       "      <td>8139.62</td>\n",
       "      <td>8.3803</td>\n",
       "      <td>0.03</td>\n",
       "      <td>393</td>\n",
       "      <td>2388</td>\n",
       "      <td>100.0</td>\n",
       "      <td>39.02</td>\n",
       "      <td>23.3916</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0.0003</td>\n",
       "      <td>0.0001</td>\n",
       "      <td>100.0</td>\n",
       "      <td>518.67</td>\n",
       "      <td>642.46</td>\n",
       "      <td>1586.94</td>\n",
       "      <td>1401.34</td>\n",
       "      <td>14.62</td>\n",
       "      <td>...</td>\n",
       "      <td>8130.10</td>\n",
       "      <td>8.4441</td>\n",
       "      <td>0.03</td>\n",
       "      <td>393</td>\n",
       "      <td>2388</td>\n",
       "      <td>100.0</td>\n",
       "      <td>39.08</td>\n",
       "      <td>23.4166</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>0.0042</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>100.0</td>\n",
       "      <td>518.67</td>\n",
       "      <td>642.44</td>\n",
       "      <td>1584.12</td>\n",
       "      <td>1406.42</td>\n",
       "      <td>14.62</td>\n",
       "      <td>...</td>\n",
       "      <td>8132.90</td>\n",
       "      <td>8.3917</td>\n",
       "      <td>0.03</td>\n",
       "      <td>391</td>\n",
       "      <td>2388</td>\n",
       "      <td>100.0</td>\n",
       "      <td>39.00</td>\n",
       "      <td>23.3737</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>0.0014</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>100.0</td>\n",
       "      <td>518.67</td>\n",
       "      <td>642.51</td>\n",
       "      <td>1587.19</td>\n",
       "      <td>1401.92</td>\n",
       "      <td>14.62</td>\n",
       "      <td>...</td>\n",
       "      <td>8129.54</td>\n",
       "      <td>8.4031</td>\n",
       "      <td>0.03</td>\n",
       "      <td>390</td>\n",
       "      <td>2388</td>\n",
       "      <td>100.0</td>\n",
       "      <td>38.99</td>\n",
       "      <td>23.4130</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 28 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   0   1       2       3      4       5       6        7        8      9   \\\n",
       "0   1   1  0.0023  0.0003  100.0  518.67  643.02  1585.29  1398.21  14.62   \n",
       "1   1   2 -0.0027 -0.0003  100.0  518.67  641.71  1588.45  1395.42  14.62   \n",
       "2   1   3  0.0003  0.0001  100.0  518.67  642.46  1586.94  1401.34  14.62   \n",
       "3   1   4  0.0042  0.0000  100.0  518.67  642.44  1584.12  1406.42  14.62   \n",
       "4   1   5  0.0014  0.0000  100.0  518.67  642.51  1587.19  1401.92  14.62   \n",
       "\n",
       "   ...       18      19    20   21    22     23     24       25  26  27  \n",
       "0  ...  8125.55  8.4052  0.03  392  2388  100.0  38.86  23.3735 NaN NaN  \n",
       "1  ...  8139.62  8.3803  0.03  393  2388  100.0  39.02  23.3916 NaN NaN  \n",
       "2  ...  8130.10  8.4441  0.03  393  2388  100.0  39.08  23.4166 NaN NaN  \n",
       "3  ...  8132.90  8.3917  0.03  391  2388  100.0  39.00  23.3737 NaN NaN  \n",
       "4  ...  8129.54  8.4031  0.03  390  2388  100.0  38.99  23.4130 NaN NaN  \n",
       "\n",
       "[5 rows x 28 columns]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_test_raw = pd.read_csv('data/PM_test.csv', sep = ' ', header=None)\n",
    "df_test_raw.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>cycle</th>\n",
       "      <th>setting1</th>\n",
       "      <th>setting2</th>\n",
       "      <th>setting3</th>\n",
       "      <th>s1</th>\n",
       "      <th>s2</th>\n",
       "      <th>s3</th>\n",
       "      <th>s4</th>\n",
       "      <th>s5</th>\n",
       "      <th>...</th>\n",
       "      <th>s12</th>\n",
       "      <th>s13</th>\n",
       "      <th>s14</th>\n",
       "      <th>s15</th>\n",
       "      <th>s16</th>\n",
       "      <th>s17</th>\n",
       "      <th>s18</th>\n",
       "      <th>s19</th>\n",
       "      <th>s20</th>\n",
       "      <th>s21</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0023</td>\n",
       "      <td>0.0003</td>\n",
       "      <td>100.0</td>\n",
       "      <td>518.67</td>\n",
       "      <td>643.02</td>\n",
       "      <td>1585.29</td>\n",
       "      <td>1398.21</td>\n",
       "      <td>14.62</td>\n",
       "      <td>...</td>\n",
       "      <td>521.72</td>\n",
       "      <td>2388.03</td>\n",
       "      <td>8125.55</td>\n",
       "      <td>8.4052</td>\n",
       "      <td>0.03</td>\n",
       "      <td>392</td>\n",
       "      <td>2388</td>\n",
       "      <td>100.0</td>\n",
       "      <td>38.86</td>\n",
       "      <td>23.3735</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>-0.0027</td>\n",
       "      <td>-0.0003</td>\n",
       "      <td>100.0</td>\n",
       "      <td>518.67</td>\n",
       "      <td>641.71</td>\n",
       "      <td>1588.45</td>\n",
       "      <td>1395.42</td>\n",
       "      <td>14.62</td>\n",
       "      <td>...</td>\n",
       "      <td>522.16</td>\n",
       "      <td>2388.06</td>\n",
       "      <td>8139.62</td>\n",
       "      <td>8.3803</td>\n",
       "      <td>0.03</td>\n",
       "      <td>393</td>\n",
       "      <td>2388</td>\n",
       "      <td>100.0</td>\n",
       "      <td>39.02</td>\n",
       "      <td>23.3916</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0.0003</td>\n",
       "      <td>0.0001</td>\n",
       "      <td>100.0</td>\n",
       "      <td>518.67</td>\n",
       "      <td>642.46</td>\n",
       "      <td>1586.94</td>\n",
       "      <td>1401.34</td>\n",
       "      <td>14.62</td>\n",
       "      <td>...</td>\n",
       "      <td>521.97</td>\n",
       "      <td>2388.03</td>\n",
       "      <td>8130.10</td>\n",
       "      <td>8.4441</td>\n",
       "      <td>0.03</td>\n",
       "      <td>393</td>\n",
       "      <td>2388</td>\n",
       "      <td>100.0</td>\n",
       "      <td>39.08</td>\n",
       "      <td>23.4166</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>0.0042</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>100.0</td>\n",
       "      <td>518.67</td>\n",
       "      <td>642.44</td>\n",
       "      <td>1584.12</td>\n",
       "      <td>1406.42</td>\n",
       "      <td>14.62</td>\n",
       "      <td>...</td>\n",
       "      <td>521.38</td>\n",
       "      <td>2388.05</td>\n",
       "      <td>8132.90</td>\n",
       "      <td>8.3917</td>\n",
       "      <td>0.03</td>\n",
       "      <td>391</td>\n",
       "      <td>2388</td>\n",
       "      <td>100.0</td>\n",
       "      <td>39.00</td>\n",
       "      <td>23.3737</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>0.0014</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>100.0</td>\n",
       "      <td>518.67</td>\n",
       "      <td>642.51</td>\n",
       "      <td>1587.19</td>\n",
       "      <td>1401.92</td>\n",
       "      <td>14.62</td>\n",
       "      <td>...</td>\n",
       "      <td>522.15</td>\n",
       "      <td>2388.03</td>\n",
       "      <td>8129.54</td>\n",
       "      <td>8.4031</td>\n",
       "      <td>0.03</td>\n",
       "      <td>390</td>\n",
       "      <td>2388</td>\n",
       "      <td>100.0</td>\n",
       "      <td>38.99</td>\n",
       "      <td>23.4130</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 26 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   id  cycle  setting1  setting2  setting3      s1      s2       s3       s4  \\\n",
       "0   1      1    0.0023    0.0003     100.0  518.67  643.02  1585.29  1398.21   \n",
       "1   1      2   -0.0027   -0.0003     100.0  518.67  641.71  1588.45  1395.42   \n",
       "2   1      3    0.0003    0.0001     100.0  518.67  642.46  1586.94  1401.34   \n",
       "3   1      4    0.0042    0.0000     100.0  518.67  642.44  1584.12  1406.42   \n",
       "4   1      5    0.0014    0.0000     100.0  518.67  642.51  1587.19  1401.92   \n",
       "\n",
       "      s5  ...     s12      s13      s14     s15   s16  s17   s18    s19  \\\n",
       "0  14.62  ...  521.72  2388.03  8125.55  8.4052  0.03  392  2388  100.0   \n",
       "1  14.62  ...  522.16  2388.06  8139.62  8.3803  0.03  393  2388  100.0   \n",
       "2  14.62  ...  521.97  2388.03  8130.10  8.4441  0.03  393  2388  100.0   \n",
       "3  14.62  ...  521.38  2388.05  8132.90  8.3917  0.03  391  2388  100.0   \n",
       "4  14.62  ...  522.15  2388.03  8129.54  8.4031  0.03  390  2388  100.0   \n",
       "\n",
       "     s20      s21  \n",
       "0  38.86  23.3735  \n",
       "1  39.02  23.3916  \n",
       "2  39.08  23.4166  \n",
       "3  39.00  23.3737  \n",
       "4  38.99  23.4130  \n",
       "\n",
       "[5 rows x 26 columns]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_test_raw.drop([26,27],axis=1,inplace=True)\n",
    "df_test_raw.columns =col_names\n",
    "df_test_raw.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>cycle</th>\n",
       "      <th>setting1</th>\n",
       "      <th>setting2</th>\n",
       "      <th>setting3</th>\n",
       "      <th>s1</th>\n",
       "      <th>s2</th>\n",
       "      <th>s3</th>\n",
       "      <th>s4</th>\n",
       "      <th>s5</th>\n",
       "      <th>...</th>\n",
       "      <th>s12</th>\n",
       "      <th>s13</th>\n",
       "      <th>s14</th>\n",
       "      <th>s15</th>\n",
       "      <th>s16</th>\n",
       "      <th>s17</th>\n",
       "      <th>s18</th>\n",
       "      <th>s19</th>\n",
       "      <th>s20</th>\n",
       "      <th>s21</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>count</td>\n",
       "      <td>13096.000000</td>\n",
       "      <td>13096.000000</td>\n",
       "      <td>13096.000000</td>\n",
       "      <td>13096.000000</td>\n",
       "      <td>13096.0</td>\n",
       "      <td>1.309600e+04</td>\n",
       "      <td>13096.000000</td>\n",
       "      <td>13096.000000</td>\n",
       "      <td>13096.000000</td>\n",
       "      <td>1.309600e+04</td>\n",
       "      <td>...</td>\n",
       "      <td>13096.000000</td>\n",
       "      <td>13096.000000</td>\n",
       "      <td>13096.000000</td>\n",
       "      <td>13096.000000</td>\n",
       "      <td>1.309600e+04</td>\n",
       "      <td>13096.000000</td>\n",
       "      <td>13096.0</td>\n",
       "      <td>13096.0</td>\n",
       "      <td>13096.000000</td>\n",
       "      <td>13096.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>mean</td>\n",
       "      <td>51.543907</td>\n",
       "      <td>76.836515</td>\n",
       "      <td>-0.000011</td>\n",
       "      <td>0.000004</td>\n",
       "      <td>100.0</td>\n",
       "      <td>5.186700e+02</td>\n",
       "      <td>642.475088</td>\n",
       "      <td>1588.099204</td>\n",
       "      <td>1404.735362</td>\n",
       "      <td>1.462000e+01</td>\n",
       "      <td>...</td>\n",
       "      <td>521.747724</td>\n",
       "      <td>2388.071025</td>\n",
       "      <td>8138.947817</td>\n",
       "      <td>8.425844</td>\n",
       "      <td>3.000000e-02</td>\n",
       "      <td>392.571625</td>\n",
       "      <td>2388.0</td>\n",
       "      <td>100.0</td>\n",
       "      <td>38.892502</td>\n",
       "      <td>23.335743</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>std</td>\n",
       "      <td>28.289423</td>\n",
       "      <td>53.057749</td>\n",
       "      <td>0.002203</td>\n",
       "      <td>0.000294</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6.014263e-11</td>\n",
       "      <td>0.400899</td>\n",
       "      <td>5.003274</td>\n",
       "      <td>6.688309</td>\n",
       "      <td>2.671743e-12</td>\n",
       "      <td>...</td>\n",
       "      <td>0.559627</td>\n",
       "      <td>0.056934</td>\n",
       "      <td>10.188605</td>\n",
       "      <td>0.029009</td>\n",
       "      <td>8.823140e-15</td>\n",
       "      <td>1.233577</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.141681</td>\n",
       "      <td>0.084120</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>min</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>-0.008200</td>\n",
       "      <td>-0.000600</td>\n",
       "      <td>100.0</td>\n",
       "      <td>5.186700e+02</td>\n",
       "      <td>641.130000</td>\n",
       "      <td>1569.040000</td>\n",
       "      <td>1384.390000</td>\n",
       "      <td>1.462000e+01</td>\n",
       "      <td>...</td>\n",
       "      <td>519.380000</td>\n",
       "      <td>2387.890000</td>\n",
       "      <td>8108.500000</td>\n",
       "      <td>8.332800</td>\n",
       "      <td>3.000000e-02</td>\n",
       "      <td>389.000000</td>\n",
       "      <td>2388.0</td>\n",
       "      <td>100.0</td>\n",
       "      <td>38.310000</td>\n",
       "      <td>22.935400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>25%</td>\n",
       "      <td>28.000000</td>\n",
       "      <td>33.000000</td>\n",
       "      <td>-0.001500</td>\n",
       "      <td>-0.000200</td>\n",
       "      <td>100.0</td>\n",
       "      <td>5.186700e+02</td>\n",
       "      <td>642.197500</td>\n",
       "      <td>1584.600000</td>\n",
       "      <td>1399.950000</td>\n",
       "      <td>1.462000e+01</td>\n",
       "      <td>...</td>\n",
       "      <td>521.380000</td>\n",
       "      <td>2388.030000</td>\n",
       "      <td>8132.310000</td>\n",
       "      <td>8.405600</td>\n",
       "      <td>3.000000e-02</td>\n",
       "      <td>392.000000</td>\n",
       "      <td>2388.0</td>\n",
       "      <td>100.0</td>\n",
       "      <td>38.800000</td>\n",
       "      <td>23.281600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>50%</td>\n",
       "      <td>52.000000</td>\n",
       "      <td>69.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>100.0</td>\n",
       "      <td>5.186700e+02</td>\n",
       "      <td>642.460000</td>\n",
       "      <td>1587.990000</td>\n",
       "      <td>1404.440000</td>\n",
       "      <td>1.462000e+01</td>\n",
       "      <td>...</td>\n",
       "      <td>521.780000</td>\n",
       "      <td>2388.070000</td>\n",
       "      <td>8138.390000</td>\n",
       "      <td>8.424900</td>\n",
       "      <td>3.000000e-02</td>\n",
       "      <td>393.000000</td>\n",
       "      <td>2388.0</td>\n",
       "      <td>100.0</td>\n",
       "      <td>38.900000</td>\n",
       "      <td>23.339200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>75%</td>\n",
       "      <td>76.000000</td>\n",
       "      <td>113.000000</td>\n",
       "      <td>0.001500</td>\n",
       "      <td>0.000300</td>\n",
       "      <td>100.0</td>\n",
       "      <td>5.186700e+02</td>\n",
       "      <td>642.740000</td>\n",
       "      <td>1591.362500</td>\n",
       "      <td>1409.050000</td>\n",
       "      <td>1.462000e+01</td>\n",
       "      <td>...</td>\n",
       "      <td>522.150000</td>\n",
       "      <td>2388.110000</td>\n",
       "      <td>8144.360000</td>\n",
       "      <td>8.444300</td>\n",
       "      <td>3.000000e-02</td>\n",
       "      <td>393.000000</td>\n",
       "      <td>2388.0</td>\n",
       "      <td>100.0</td>\n",
       "      <td>38.990000</td>\n",
       "      <td>23.393600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>max</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>303.000000</td>\n",
       "      <td>0.007800</td>\n",
       "      <td>0.000700</td>\n",
       "      <td>100.0</td>\n",
       "      <td>5.186700e+02</td>\n",
       "      <td>644.300000</td>\n",
       "      <td>1607.550000</td>\n",
       "      <td>1433.360000</td>\n",
       "      <td>1.462000e+01</td>\n",
       "      <td>...</td>\n",
       "      <td>523.760000</td>\n",
       "      <td>2388.320000</td>\n",
       "      <td>8220.480000</td>\n",
       "      <td>8.541400</td>\n",
       "      <td>3.000000e-02</td>\n",
       "      <td>397.000000</td>\n",
       "      <td>2388.0</td>\n",
       "      <td>100.0</td>\n",
       "      <td>39.410000</td>\n",
       "      <td>23.641900</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8 rows × 26 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                 id         cycle      setting1      setting2  setting3  \\\n",
       "count  13096.000000  13096.000000  13096.000000  13096.000000   13096.0   \n",
       "mean      51.543907     76.836515     -0.000011      0.000004     100.0   \n",
       "std       28.289423     53.057749      0.002203      0.000294       0.0   \n",
       "min        1.000000      1.000000     -0.008200     -0.000600     100.0   \n",
       "25%       28.000000     33.000000     -0.001500     -0.000200     100.0   \n",
       "50%       52.000000     69.000000      0.000000      0.000000     100.0   \n",
       "75%       76.000000    113.000000      0.001500      0.000300     100.0   \n",
       "max      100.000000    303.000000      0.007800      0.000700     100.0   \n",
       "\n",
       "                 s1            s2            s3            s4            s5  \\\n",
       "count  1.309600e+04  13096.000000  13096.000000  13096.000000  1.309600e+04   \n",
       "mean   5.186700e+02    642.475088   1588.099204   1404.735362  1.462000e+01   \n",
       "std    6.014263e-11      0.400899      5.003274      6.688309  2.671743e-12   \n",
       "min    5.186700e+02    641.130000   1569.040000   1384.390000  1.462000e+01   \n",
       "25%    5.186700e+02    642.197500   1584.600000   1399.950000  1.462000e+01   \n",
       "50%    5.186700e+02    642.460000   1587.990000   1404.440000  1.462000e+01   \n",
       "75%    5.186700e+02    642.740000   1591.362500   1409.050000  1.462000e+01   \n",
       "max    5.186700e+02    644.300000   1607.550000   1433.360000  1.462000e+01   \n",
       "\n",
       "       ...           s12           s13           s14           s15  \\\n",
       "count  ...  13096.000000  13096.000000  13096.000000  13096.000000   \n",
       "mean   ...    521.747724   2388.071025   8138.947817      8.425844   \n",
       "std    ...      0.559627      0.056934     10.188605      0.029009   \n",
       "min    ...    519.380000   2387.890000   8108.500000      8.332800   \n",
       "25%    ...    521.380000   2388.030000   8132.310000      8.405600   \n",
       "50%    ...    521.780000   2388.070000   8138.390000      8.424900   \n",
       "75%    ...    522.150000   2388.110000   8144.360000      8.444300   \n",
       "max    ...    523.760000   2388.320000   8220.480000      8.541400   \n",
       "\n",
       "                s16           s17      s18      s19           s20  \\\n",
       "count  1.309600e+04  13096.000000  13096.0  13096.0  13096.000000   \n",
       "mean   3.000000e-02    392.571625   2388.0    100.0     38.892502   \n",
       "std    8.823140e-15      1.233577      0.0      0.0      0.141681   \n",
       "min    3.000000e-02    389.000000   2388.0    100.0     38.310000   \n",
       "25%    3.000000e-02    392.000000   2388.0    100.0     38.800000   \n",
       "50%    3.000000e-02    393.000000   2388.0    100.0     38.900000   \n",
       "75%    3.000000e-02    393.000000   2388.0    100.0     38.990000   \n",
       "max    3.000000e-02    397.000000   2388.0    100.0     39.410000   \n",
       "\n",
       "                s21  \n",
       "count  13096.000000  \n",
       "mean      23.335743  \n",
       "std        0.084120  \n",
       "min       22.935400  \n",
       "25%       23.281600  \n",
       "50%       23.339200  \n",
       "75%       23.393600  \n",
       "max       23.641900  \n",
       "\n",
       "[8 rows x 26 columns]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_test_raw.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>112</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>98</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>69</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>82</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>91</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     0   1\n",
       "0  112 NaN\n",
       "1   98 NaN\n",
       "2   69 NaN\n",
       "3   82 NaN\n",
       "4   91 NaN"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_truth = pd.read_csv('data/PM_truth.csv',sep=' ',header=None)\n",
    "df_truth.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ttf</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>112</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>98</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>69</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>82</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>91</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   ttf\n",
       "0  112\n",
       "1   98\n",
       "2   69\n",
       "3   82\n",
       "4   91"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_truth.drop([1],axis=1,inplace=True)\n",
    "df_truth.columns = ['ttf']\n",
    "df_truth.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ttf</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>count</td>\n",
       "      <td>100.00000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>mean</td>\n",
       "      <td>75.52000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>std</td>\n",
       "      <td>41.76497</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>min</td>\n",
       "      <td>7.00000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>25%</td>\n",
       "      <td>32.75000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>50%</td>\n",
       "      <td>86.00000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>75%</td>\n",
       "      <td>112.25000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>max</td>\n",
       "      <td>145.00000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             ttf\n",
       "count  100.00000\n",
       "mean    75.52000\n",
       "std     41.76497\n",
       "min      7.00000\n",
       "25%     32.75000\n",
       "50%     86.00000\n",
       "75%    112.25000\n",
       "max    145.00000"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_truth.describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# This method calculates the moving average and standard deviation of the"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def add_features(df_in,rolling_win_size):\n",
    "    sensor_cols = ['s1','s2','s3','s4','s5','s6','s7','s8','s9','s10','s11','s12','s13','s14','s15','s16','s17','s18','s19','s20','s21']\n",
    "    \n",
    "    sensor_av_cols = [nm.replace('s','av') for nm in sensor_cols]\n",
    "    sensor_sd_cols = [nm.replace('s','sd') for nm in sensor_cols]\n",
    "    \n",
    "    df_out = pd.DataFrame()\n",
    "    \n",
    "    ws = rolling_win_size\n",
    "    \n",
    "    for m_id in pd.unique(df_in.id):\n",
    "        \n",
    "        df_engine =df_in[df_in['id'] == m_id]\n",
    "        df_sub = df_engine[sensor_cols]\n",
    "        \n",
    "        av = df_sub.rolling(ws, min_periods = 1).mean()\n",
    "        av.columns = sensor_av_cols\n",
    "        \n",
    "        sd = df_sub.rolling(ws,min_periods =1).std().fillna(0)\n",
    "        sd.columns = sensor_sd_cols\n",
    "        new_ftrs = pd.concat([df_engine,av,sd], axis=1)\n",
    "    \n",
    "        # add the new features rows to the output dataframe\n",
    "        df_out = pd.concat([df_out,new_ftrs])\n",
    "    \n",
    "    \n",
    "    return df_out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def prepare_train_data(df_in,period):\n",
    "    df_max_cycle = pd.DataFrame(df_in.groupby('id')['cycle'].max())\n",
    "    df_max_cycle.reset_index(level=0,inplace=True)\n",
    "    df_max_cycle.columns = ['id','last_cycle']\n",
    "    \n",
    "    df_in =pd.merge(df_in,df_max_cycle,on='id')\n",
    "    df_in['ttf'] = df_in['last_cycle'] - df_in['cycle']\n",
    "    df_in.drop(['last_cycle'], axis=1, inplace=True)\n",
    "\n",
    "    df_in['label_bnc'] =df_in['ttf'].apply(lambda x: 1 if x<=period else 0)\n",
    "    df_in['label_mcc'] = df_in['ttf'].apply(lambda x: 2 if x<=period/2 else 1 if x<= period else 0)\n",
    "    \n",
    "    return df_in"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "def prepare_test_data(df_test_in,df_truth_in,period):\n",
    "    df_tst_last_cycle = pd.DataFrame(df_test_in.groupby('id')['cycle'].max())\n",
    "    df_tst_last_cycle.reset_index(level=0,inplace=True)\n",
    "    \n",
    "    df_tst_last_cycle.columns = ['id','last_cycle']\n",
    "    \n",
    "    df_test_in =pd.merge(df_test_in,df_tst_last_cycle,on='id')\n",
    "    df_test_in = df_test_in[df_test_in['cycle'] == df_test_in['last_cycle']]\n",
    "\n",
    "    df_test_in.drop(['last_cycle'], axis=1, inplace=True)\n",
    "    \n",
    "    df_test_in.reset_index(drop=True, inplace=True)\n",
    "    df_test_in = pd.concat([df_test_in, df_truth], axis=1)\n",
    "    df_test_in['label_bnc'] = df_test_in['ttf'].apply(lambda x: 1 if x <= period else 0)\n",
    "    df_test_in['label_mcc'] = df_test_in['ttf'].apply(lambda x: 2 if x <= period/2 else 1 if x <= period else 0)\n",
    "\n",
    "    return df_test_in"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>cycle</th>\n",
       "      <th>setting1</th>\n",
       "      <th>setting2</th>\n",
       "      <th>setting3</th>\n",
       "      <th>s1</th>\n",
       "      <th>s2</th>\n",
       "      <th>s3</th>\n",
       "      <th>s4</th>\n",
       "      <th>s5</th>\n",
       "      <th>...</th>\n",
       "      <th>sd12</th>\n",
       "      <th>sd13</th>\n",
       "      <th>sd14</th>\n",
       "      <th>sd15</th>\n",
       "      <th>sd16</th>\n",
       "      <th>sd17</th>\n",
       "      <th>sd18</th>\n",
       "      <th>sd19</th>\n",
       "      <th>sd20</th>\n",
       "      <th>sd21</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>-0.0007</td>\n",
       "      <td>-0.0004</td>\n",
       "      <td>100.0</td>\n",
       "      <td>518.67</td>\n",
       "      <td>641.82</td>\n",
       "      <td>1589.70</td>\n",
       "      <td>1400.60</td>\n",
       "      <td>14.62</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>0.0019</td>\n",
       "      <td>-0.0003</td>\n",
       "      <td>100.0</td>\n",
       "      <td>518.67</td>\n",
       "      <td>642.15</td>\n",
       "      <td>1591.82</td>\n",
       "      <td>1403.14</td>\n",
       "      <td>14.62</td>\n",
       "      <td>...</td>\n",
       "      <td>0.438406</td>\n",
       "      <td>0.035355</td>\n",
       "      <td>5.041671</td>\n",
       "      <td>0.008697</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.042426</td>\n",
       "      <td>0.003253</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>-0.0043</td>\n",
       "      <td>0.0003</td>\n",
       "      <td>100.0</td>\n",
       "      <td>518.67</td>\n",
       "      <td>642.35</td>\n",
       "      <td>1587.99</td>\n",
       "      <td>1404.20</td>\n",
       "      <td>14.62</td>\n",
       "      <td>...</td>\n",
       "      <td>0.404475</td>\n",
       "      <td>0.026458</td>\n",
       "      <td>3.717450</td>\n",
       "      <td>0.007640</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.154701</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.055076</td>\n",
       "      <td>0.044573</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>0.0007</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>100.0</td>\n",
       "      <td>518.67</td>\n",
       "      <td>642.35</td>\n",
       "      <td>1582.79</td>\n",
       "      <td>1401.87</td>\n",
       "      <td>14.62</td>\n",
       "      <td>...</td>\n",
       "      <td>0.495950</td>\n",
       "      <td>0.029439</td>\n",
       "      <td>3.050906</td>\n",
       "      <td>0.028117</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.076322</td>\n",
       "      <td>0.037977</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>-0.0019</td>\n",
       "      <td>-0.0002</td>\n",
       "      <td>100.0</td>\n",
       "      <td>518.67</td>\n",
       "      <td>642.37</td>\n",
       "      <td>1582.85</td>\n",
       "      <td>1406.22</td>\n",
       "      <td>14.62</td>\n",
       "      <td>...</td>\n",
       "      <td>0.432574</td>\n",
       "      <td>0.025884</td>\n",
       "      <td>2.651326</td>\n",
       "      <td>0.025953</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.095445</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.073621</td>\n",
       "      <td>0.033498</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 68 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   id  cycle  setting1  setting2  setting3      s1      s2       s3       s4  \\\n",
       "0   1      1   -0.0007   -0.0004     100.0  518.67  641.82  1589.70  1400.60   \n",
       "1   1      2    0.0019   -0.0003     100.0  518.67  642.15  1591.82  1403.14   \n",
       "2   1      3   -0.0043    0.0003     100.0  518.67  642.35  1587.99  1404.20   \n",
       "3   1      4    0.0007    0.0000     100.0  518.67  642.35  1582.79  1401.87   \n",
       "4   1      5   -0.0019   -0.0002     100.0  518.67  642.37  1582.85  1406.22   \n",
       "\n",
       "      s5  ...      sd12      sd13      sd14      sd15  sd16      sd17  sd18  \\\n",
       "0  14.62  ...  0.000000  0.000000  0.000000  0.000000   0.0  0.000000   0.0   \n",
       "1  14.62  ...  0.438406  0.035355  5.041671  0.008697   0.0  0.000000   0.0   \n",
       "2  14.62  ...  0.404475  0.026458  3.717450  0.007640   0.0  1.154701   0.0   \n",
       "3  14.62  ...  0.495950  0.029439  3.050906  0.028117   0.0  1.000000   0.0   \n",
       "4  14.62  ...  0.432574  0.025884  2.651326  0.025953   0.0  1.095445   0.0   \n",
       "\n",
       "   sd19      sd20      sd21  \n",
       "0   0.0  0.000000  0.000000  \n",
       "1   0.0  0.042426  0.003253  \n",
       "2   0.0  0.055076  0.044573  \n",
       "3   0.0  0.076322  0.037977  \n",
       "4   0.0  0.073621  0.033498  \n",
       "\n",
       "[5 rows x 68 columns]"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train_fx = add_features(df_train_raw,5)\n",
    "df_train_fx.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>cycle</th>\n",
       "      <th>setting1</th>\n",
       "      <th>setting2</th>\n",
       "      <th>setting3</th>\n",
       "      <th>s1</th>\n",
       "      <th>s2</th>\n",
       "      <th>s3</th>\n",
       "      <th>s4</th>\n",
       "      <th>s5</th>\n",
       "      <th>...</th>\n",
       "      <th>sd15</th>\n",
       "      <th>sd16</th>\n",
       "      <th>sd17</th>\n",
       "      <th>sd18</th>\n",
       "      <th>sd19</th>\n",
       "      <th>sd20</th>\n",
       "      <th>sd21</th>\n",
       "      <th>ttf</th>\n",
       "      <th>label_bnc</th>\n",
       "      <th>label_mcc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>-0.0007</td>\n",
       "      <td>-0.0004</td>\n",
       "      <td>100.0</td>\n",
       "      <td>518.67</td>\n",
       "      <td>641.82</td>\n",
       "      <td>1589.70</td>\n",
       "      <td>1400.60</td>\n",
       "      <td>14.62</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>191</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>0.0019</td>\n",
       "      <td>-0.0003</td>\n",
       "      <td>100.0</td>\n",
       "      <td>518.67</td>\n",
       "      <td>642.15</td>\n",
       "      <td>1591.82</td>\n",
       "      <td>1403.14</td>\n",
       "      <td>14.62</td>\n",
       "      <td>...</td>\n",
       "      <td>0.008697</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.042426</td>\n",
       "      <td>0.003253</td>\n",
       "      <td>190</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>-0.0043</td>\n",
       "      <td>0.0003</td>\n",
       "      <td>100.0</td>\n",
       "      <td>518.67</td>\n",
       "      <td>642.35</td>\n",
       "      <td>1587.99</td>\n",
       "      <td>1404.20</td>\n",
       "      <td>14.62</td>\n",
       "      <td>...</td>\n",
       "      <td>0.007640</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.154701</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.055076</td>\n",
       "      <td>0.044573</td>\n",
       "      <td>189</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>0.0007</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>100.0</td>\n",
       "      <td>518.67</td>\n",
       "      <td>642.35</td>\n",
       "      <td>1582.79</td>\n",
       "      <td>1401.87</td>\n",
       "      <td>14.62</td>\n",
       "      <td>...</td>\n",
       "      <td>0.028117</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.076322</td>\n",
       "      <td>0.037977</td>\n",
       "      <td>188</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>-0.0019</td>\n",
       "      <td>-0.0002</td>\n",
       "      <td>100.0</td>\n",
       "      <td>518.67</td>\n",
       "      <td>642.37</td>\n",
       "      <td>1582.85</td>\n",
       "      <td>1406.22</td>\n",
       "      <td>14.62</td>\n",
       "      <td>...</td>\n",
       "      <td>0.025953</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.095445</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.073621</td>\n",
       "      <td>0.033498</td>\n",
       "      <td>187</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 71 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   id  cycle  setting1  setting2  setting3      s1      s2       s3       s4  \\\n",
       "0   1      1   -0.0007   -0.0004     100.0  518.67  641.82  1589.70  1400.60   \n",
       "1   1      2    0.0019   -0.0003     100.0  518.67  642.15  1591.82  1403.14   \n",
       "2   1      3   -0.0043    0.0003     100.0  518.67  642.35  1587.99  1404.20   \n",
       "3   1      4    0.0007    0.0000     100.0  518.67  642.35  1582.79  1401.87   \n",
       "4   1      5   -0.0019   -0.0002     100.0  518.67  642.37  1582.85  1406.22   \n",
       "\n",
       "      s5  ...      sd15  sd16      sd17  sd18  sd19      sd20      sd21  ttf  \\\n",
       "0  14.62  ...  0.000000   0.0  0.000000   0.0   0.0  0.000000  0.000000  191   \n",
       "1  14.62  ...  0.008697   0.0  0.000000   0.0   0.0  0.042426  0.003253  190   \n",
       "2  14.62  ...  0.007640   0.0  1.154701   0.0   0.0  0.055076  0.044573  189   \n",
       "3  14.62  ...  0.028117   0.0  1.000000   0.0   0.0  0.076322  0.037977  188   \n",
       "4  14.62  ...  0.025953   0.0  1.095445   0.0   0.0  0.073621  0.033498  187   \n",
       "\n",
       "   label_bnc  label_mcc  \n",
       "0          0          0  \n",
       "1          0          0  \n",
       "2          0          0  \n",
       "3          0          0  \n",
       "4          0          0  \n",
       "\n",
       "[5 rows x 71 columns]"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train = prepare_train_data(df_train_fx, 30)\n",
    "df_train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "id             int64\n",
       "cycle          int64\n",
       "setting1     float64\n",
       "setting2     float64\n",
       "setting3     float64\n",
       "              ...   \n",
       "sd20         float64\n",
       "sd21         float64\n",
       "ttf            int64\n",
       "label_bnc      int64\n",
       "label_mcc      int64\n",
       "Length: 71, dtype: object"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train.to_csv('data/train.csv',index=False)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>cycle</th>\n",
       "      <th>setting1</th>\n",
       "      <th>setting2</th>\n",
       "      <th>setting3</th>\n",
       "      <th>s1</th>\n",
       "      <th>s2</th>\n",
       "      <th>s3</th>\n",
       "      <th>s4</th>\n",
       "      <th>s5</th>\n",
       "      <th>...</th>\n",
       "      <th>sd12</th>\n",
       "      <th>sd13</th>\n",
       "      <th>sd14</th>\n",
       "      <th>sd15</th>\n",
       "      <th>sd16</th>\n",
       "      <th>sd17</th>\n",
       "      <th>sd18</th>\n",
       "      <th>sd19</th>\n",
       "      <th>sd20</th>\n",
       "      <th>sd21</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0023</td>\n",
       "      <td>0.0003</td>\n",
       "      <td>100.0</td>\n",
       "      <td>518.67</td>\n",
       "      <td>643.02</td>\n",
       "      <td>1585.29</td>\n",
       "      <td>1398.21</td>\n",
       "      <td>14.62</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>-0.0027</td>\n",
       "      <td>-0.0003</td>\n",
       "      <td>100.0</td>\n",
       "      <td>518.67</td>\n",
       "      <td>641.71</td>\n",
       "      <td>1588.45</td>\n",
       "      <td>1395.42</td>\n",
       "      <td>14.62</td>\n",
       "      <td>...</td>\n",
       "      <td>0.311127</td>\n",
       "      <td>0.021213</td>\n",
       "      <td>9.948992</td>\n",
       "      <td>0.017607</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.707107</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.113137</td>\n",
       "      <td>0.012799</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0.0003</td>\n",
       "      <td>0.0001</td>\n",
       "      <td>100.0</td>\n",
       "      <td>518.67</td>\n",
       "      <td>642.46</td>\n",
       "      <td>1586.94</td>\n",
       "      <td>1401.34</td>\n",
       "      <td>14.62</td>\n",
       "      <td>...</td>\n",
       "      <td>0.220681</td>\n",
       "      <td>0.017321</td>\n",
       "      <td>7.179807</td>\n",
       "      <td>0.032155</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.577350</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.113725</td>\n",
       "      <td>0.021642</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>0.0042</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>100.0</td>\n",
       "      <td>518.67</td>\n",
       "      <td>642.44</td>\n",
       "      <td>1584.12</td>\n",
       "      <td>1406.42</td>\n",
       "      <td>14.62</td>\n",
       "      <td>...</td>\n",
       "      <td>0.337182</td>\n",
       "      <td>0.015000</td>\n",
       "      <td>5.890096</td>\n",
       "      <td>0.027781</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.957427</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.093095</td>\n",
       "      <td>0.020353</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>0.0014</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>100.0</td>\n",
       "      <td>518.67</td>\n",
       "      <td>642.51</td>\n",
       "      <td>1587.19</td>\n",
       "      <td>1401.92</td>\n",
       "      <td>14.62</td>\n",
       "      <td>...</td>\n",
       "      <td>0.329742</td>\n",
       "      <td>0.014142</td>\n",
       "      <td>5.222300</td>\n",
       "      <td>0.024080</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.303840</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.080623</td>\n",
       "      <td>0.020672</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 68 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   id  cycle  setting1  setting2  setting3      s1      s2       s3       s4  \\\n",
       "0   1      1    0.0023    0.0003     100.0  518.67  643.02  1585.29  1398.21   \n",
       "1   1      2   -0.0027   -0.0003     100.0  518.67  641.71  1588.45  1395.42   \n",
       "2   1      3    0.0003    0.0001     100.0  518.67  642.46  1586.94  1401.34   \n",
       "3   1      4    0.0042    0.0000     100.0  518.67  642.44  1584.12  1406.42   \n",
       "4   1      5    0.0014    0.0000     100.0  518.67  642.51  1587.19  1401.92   \n",
       "\n",
       "      s5  ...      sd12      sd13      sd14      sd15  sd16      sd17  sd18  \\\n",
       "0  14.62  ...  0.000000  0.000000  0.000000  0.000000   0.0  0.000000   0.0   \n",
       "1  14.62  ...  0.311127  0.021213  9.948992  0.017607   0.0  0.707107   0.0   \n",
       "2  14.62  ...  0.220681  0.017321  7.179807  0.032155   0.0  0.577350   0.0   \n",
       "3  14.62  ...  0.337182  0.015000  5.890096  0.027781   0.0  0.957427   0.0   \n",
       "4  14.62  ...  0.329742  0.014142  5.222300  0.024080   0.0  1.303840   0.0   \n",
       "\n",
       "   sd19      sd20      sd21  \n",
       "0   0.0  0.000000  0.000000  \n",
       "1   0.0  0.113137  0.012799  \n",
       "2   0.0  0.113725  0.021642  \n",
       "3   0.0  0.093095  0.020353  \n",
       "4   0.0  0.080623  0.020672  \n",
       "\n",
       "[5 rows x 68 columns]"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_test_fx = add_features(df_test_raw, 5)\n",
    "df_test_fx.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>cycle</th>\n",
       "      <th>setting1</th>\n",
       "      <th>setting2</th>\n",
       "      <th>setting3</th>\n",
       "      <th>s1</th>\n",
       "      <th>s2</th>\n",
       "      <th>s3</th>\n",
       "      <th>s4</th>\n",
       "      <th>s5</th>\n",
       "      <th>...</th>\n",
       "      <th>sd15</th>\n",
       "      <th>sd16</th>\n",
       "      <th>sd17</th>\n",
       "      <th>sd18</th>\n",
       "      <th>sd19</th>\n",
       "      <th>sd20</th>\n",
       "      <th>sd21</th>\n",
       "      <th>ttf</th>\n",
       "      <th>label_bnc</th>\n",
       "      <th>label_mcc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>31</td>\n",
       "      <td>-0.0006</td>\n",
       "      <td>0.0004</td>\n",
       "      <td>100.0</td>\n",
       "      <td>518.67</td>\n",
       "      <td>642.58</td>\n",
       "      <td>1581.22</td>\n",
       "      <td>1398.91</td>\n",
       "      <td>14.62</td>\n",
       "      <td>...</td>\n",
       "      <td>0.021174</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.707107</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.124016</td>\n",
       "      <td>0.025037</td>\n",
       "      <td>112</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>49</td>\n",
       "      <td>0.0018</td>\n",
       "      <td>-0.0001</td>\n",
       "      <td>100.0</td>\n",
       "      <td>518.67</td>\n",
       "      <td>642.55</td>\n",
       "      <td>1586.59</td>\n",
       "      <td>1410.83</td>\n",
       "      <td>14.62</td>\n",
       "      <td>...</td>\n",
       "      <td>0.011429</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.095445</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.069785</td>\n",
       "      <td>0.027820</td>\n",
       "      <td>98</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>126</td>\n",
       "      <td>-0.0016</td>\n",
       "      <td>0.0004</td>\n",
       "      <td>100.0</td>\n",
       "      <td>518.67</td>\n",
       "      <td>642.88</td>\n",
       "      <td>1589.75</td>\n",
       "      <td>1418.89</td>\n",
       "      <td>14.62</td>\n",
       "      <td>...</td>\n",
       "      <td>0.030924</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.836660</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.143631</td>\n",
       "      <td>0.018966</td>\n",
       "      <td>69</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>106</td>\n",
       "      <td>0.0012</td>\n",
       "      <td>0.0004</td>\n",
       "      <td>100.0</td>\n",
       "      <td>518.67</td>\n",
       "      <td>642.78</td>\n",
       "      <td>1594.53</td>\n",
       "      <td>1406.88</td>\n",
       "      <td>14.62</td>\n",
       "      <td>...</td>\n",
       "      <td>0.021249</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.341641</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.126372</td>\n",
       "      <td>0.021106</td>\n",
       "      <td>82</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>98</td>\n",
       "      <td>-0.0013</td>\n",
       "      <td>-0.0004</td>\n",
       "      <td>100.0</td>\n",
       "      <td>518.67</td>\n",
       "      <td>642.27</td>\n",
       "      <td>1589.94</td>\n",
       "      <td>1419.36</td>\n",
       "      <td>14.62</td>\n",
       "      <td>...</td>\n",
       "      <td>0.009627</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.547723</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.091924</td>\n",
       "      <td>0.101100</td>\n",
       "      <td>91</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 71 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   id  cycle  setting1  setting2  setting3      s1      s2       s3       s4  \\\n",
       "0   1     31   -0.0006    0.0004     100.0  518.67  642.58  1581.22  1398.91   \n",
       "1   2     49    0.0018   -0.0001     100.0  518.67  642.55  1586.59  1410.83   \n",
       "2   3    126   -0.0016    0.0004     100.0  518.67  642.88  1589.75  1418.89   \n",
       "3   4    106    0.0012    0.0004     100.0  518.67  642.78  1594.53  1406.88   \n",
       "4   5     98   -0.0013   -0.0004     100.0  518.67  642.27  1589.94  1419.36   \n",
       "\n",
       "      s5  ...      sd15  sd16      sd17  sd18  sd19      sd20      sd21  ttf  \\\n",
       "0  14.62  ...  0.021174   0.0  0.707107   0.0   0.0  0.124016  0.025037  112   \n",
       "1  14.62  ...  0.011429   0.0  1.095445   0.0   0.0  0.069785  0.027820   98   \n",
       "2  14.62  ...  0.030924   0.0  0.836660   0.0   0.0  0.143631  0.018966   69   \n",
       "3  14.62  ...  0.021249   0.0  1.341641   0.0   0.0  0.126372  0.021106   82   \n",
       "4  14.62  ...  0.009627   0.0  0.547723   0.0   0.0  0.091924  0.101100   91   \n",
       "\n",
       "   label_bnc  label_mcc  \n",
       "0          0          0  \n",
       "1          0          0  \n",
       "2          0          0  \n",
       "3          0          0  \n",
       "4          0          0  \n",
       "\n",
       "[5 rows x 71 columns]"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_test = prepare_test_data(df_test_fx, df_truth, 30)\n",
    "df_test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "id             int64\n",
       "cycle          int64\n",
       "setting1     float64\n",
       "setting2     float64\n",
       "setting3     float64\n",
       "              ...   \n",
       "sd20         float64\n",
       "sd21         float64\n",
       "ttf            int64\n",
       "label_bnc      int64\n",
       "label_mcc      int64\n",
       "Length: 71, dtype: object"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_test.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_test.to_csv('data/test.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}