Visualize_LOO_Experiments-Margin.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": 0.142914,
     "end_time": "2018-11-19T22:34:24.491068",
     "exception": false,
     "start_time": "2018-11-19T22:34:24.348154",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": 0.464582,
     "end_time": "2018-11-19T22:34:24.955749",
     "exception": false,
     "start_time": "2018-11-19T22:34:24.491167",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import pandas as pd\n",
    "import sklearn.metrics\n",
    "import scipy\n",
    "import string\n",
    "import matplotlib.ticker"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": 0.031144,
     "end_time": "2018-11-19T22:34:24.986943",
     "exception": false,
     "start_time": "2018-11-19T22:34:24.955799",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Set the default plot style\n",
    "#default_plt_width = 15\n",
    "#default_plt_height = 10\n",
    "#plt.rcParams['figure.figsize'] = [default_plt_width, default_plt_height]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": 0.091717,
     "end_time": "2018-11-19T22:34:25.078748",
     "exception": false,
     "start_time": "2018-11-19T22:34:24.987031",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "sns.set_style(\"whitegrid\")\n",
    "sns.set_context(\"paper\")\n",
    "sns.set(font_scale=1.1)\n",
    "sns.despine(left=True)\n",
    "sns.set_style(\"ticks\", {\"xtick.major.size\": 8, \"ytick.major.size\": 8})\n",
    "#cmap = sns.color_palette(\"dark\")\n",
    "cmap = sns.color_palette(\"dark\", 10)\n",
    "sns.palplot(cmap)\n",
    "sns.set_palette(cmap)\n",
    "plt_y_axis_fmt_string = '%.3f'\n",
    "#plt.rcParams[\"axes.labelsize\"] = 12"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": 0.021456,
     "end_time": "2018-11-19T22:34:25.100269",
     "exception": false,
     "start_time": "2018-11-19T22:34:25.078813",
     "status": "completed"
    },
    "tags": [
     "parameters"
    ]
   },
   "outputs": [],
   "source": [
    "filename_prefix = \"aug_results_margin_CIFAR10_0_vs_1_crop_10\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": 0.022792,
     "end_time": "2018-11-19T22:34:25.140586",
     "exception": false,
     "start_time": "2018-11-19T22:34:25.117794",
     "status": "completed"
    },
    "tags": [
     "injected-parameters"
    ]
   },
   "outputs": [],
   "source": [
    "# Parameters\n",
    "filename_prefix = \"aug_results_margin_NORB_0_vs_1_crop_10_loss\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": 0.065698,
     "end_time": "2018-11-19T22:34:25.219371",
     "exception": true,
     "start_time": "2018-11-19T22:34:25.153673",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "runs_data = np.load(\"{}.npz\".format(filename_prefix))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": null,
     "end_time": null,
     "exception": null,
     "start_time": null,
     "status": "pending"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "runs_data.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": null,
     "end_time": null,
     "exception": null,
     "start_time": null,
     "status": "pending"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "for k in runs_data:\n",
    "    try:\n",
    "        v = runs_data[k]\n",
    "        print(\"{}: {}\".format(k, v))\n",
    "    except AttributeError as ex:\n",
    "        print(ex)\n",
    "        raise ex"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": null,
     "end_time": null,
     "exception": null,
     "start_time": null,
     "status": "pending"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "hist_bins=None\n",
    "hist_color=cmap.as_hex()[0]\n",
    "#hist_color=\"r\"\n",
    "hist_kws=dict(alpha=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": null,
     "end_time": null,
     "exception": null,
     "start_time": null,
     "status": "pending"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "hist_color"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": null,
     "end_time": null,
     "exception": null,
     "start_time": null,
     "status": "pending"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "initial_aug_scores = pd.Series(runs_data[\"initial_aug_scores\"],\n",
    "                               name=\"scores\").abs()\n",
    "hist = sns.distplot(initial_aug_scores,\n",
    "                    kde=False,\n",
    "                    bins=hist_bins,\n",
    "                    color=hist_color,\n",
    "                    hist_kws=hist_kws)\n",
    "hist.set_xlabel('')\n",
    "hist.set_yscale('log')\n",
    "hist.get_figure().savefig(filename_prefix + \"_abs_initial_aug_scores_histogram.pdf\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": null,
     "end_time": null,
     "exception": null,
     "start_time": null,
     "status": "pending"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "fig, ax = plt.subplots()\n",
    "after_abs_aug_scores = pd.Series(runs_data[\"after_aug_scores\"],\n",
    "                                 name=\"scores\").abs()\n",
    "hist = sns.distplot(after_abs_aug_scores,\n",
    "                    kde=False,\n",
    "                    bins=hist_bins,\n",
    "                    color=hist_color,\n",
    "                    hist_kws=hist_kws)\n",
    "hist.set_xlabel('')\n",
    "hist.set_yscale('log')\n",
    "hist.get_figure().savefig(filename_prefix + \"_after_abs_aug_scores_histogram.pdf\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": null,
     "end_time": null,
     "exception": null,
     "start_time": null,
     "status": "pending"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "after_abs_aug_scores_cond = after_abs_aug_scores[:len(initial_aug_scores)]\n",
    "hist = sns.distplot(after_abs_aug_scores_cond,\n",
    "                    kde=False,\n",
    "                    bins=hist_bins,\n",
    "                    color=hist_color,\n",
    "                    hist_kws=hist_kws)\n",
    "hist.set_xlabel('')\n",
    "hist.set_yscale('log')\n",
    "hist.get_figure().savefig(filename_prefix + \"_after_abs_aug_scores_cond_histogram.pdf\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": null,
     "end_time": null,
     "exception": null,
     "start_time": null,
     "status": "pending"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "joint = sns.jointplot(x=initial_aug_scores,\n",
    "                      y=after_abs_aug_scores_cond,\n",
    "                      marginal_kws=dict(bins=10,\n",
    "                                        rug=False,\n",
    "                                        hist=False,\n",
    "                                        kde=True,\n",
    "                                        kde_kws=dict(bw=.001,\n",
    "                                                     alpha=1.0,\n",
    "                                                    ))\n",
    "             );\n",
    "joint.ax_joint.set_xlabel('')\n",
    "joint.ax_joint.set_ylabel('')\n",
    "joint.ax_joint.get_figure().savefig(filename_prefix + \"_init_after_joint_histogram.pdf\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": null,
     "end_time": null,
     "exception": null,
     "start_time": null,
     "status": "pending"
    },
    "tags": []
   },
   "source": [
    "## Correlation in Scores"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": null,
     "end_time": null,
     "exception": null,
     "start_time": null,
     "status": "pending"
    },
    "tags": []
   },
   "source": [
    "Let's see how correlated the influences are before and after augmentation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": null,
     "end_time": null,
     "exception": null,
     "start_time": null,
     "status": "pending"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "scipy.stats.spearmanr(initial_aug_scores, after_abs_aug_scores_cond)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": null,
     "end_time": null,
     "exception": null,
     "start_time": null,
     "status": "pending"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "small_scores_idxs = initial_aug_scores < initial_aug_scores.mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": null,
     "end_time": null,
     "exception": null,
     "start_time": null,
     "status": "pending"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "scipy.stats.spearmanr(initial_aug_scores[small_scores_idxs],\n",
    "                      after_abs_aug_scores_cond[small_scores_idxs])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": null,
     "end_time": null,
     "exception": null,
     "start_time": null,
     "status": "pending"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "initial_aug_scores.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": null,
     "end_time": null,
     "exception": null,
     "start_time": null,
     "status": "pending"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "after_abs_aug_scores.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": null,
     "end_time": null,
     "exception": null,
     "start_time": null,
     "status": "pending"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "initial_aug_scores[small_scores_idxs].describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": null,
     "end_time": null,
     "exception": null,
     "start_time": null,
     "status": "pending"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "after_abs_aug_scores_cond[small_scores_idxs].describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": null,
     "end_time": null,
     "exception": null,
     "start_time": null,
     "status": "pending"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "sns.jointplot(x=initial_aug_scores[small_scores_idxs],\n",
    "              y=after_abs_aug_scores_cond[small_scores_idxs],\n",
    "              marginal_kws=dict(bins=10,\n",
    "                                rug=False,\n",
    "                                hist=False,\n",
    "                                kde=True,\n",
    "                                kde_kws=dict(bw=.0005))\n",
    "             );"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": null,
     "end_time": null,
     "exception": null,
     "start_time": null,
     "status": "pending"
    },
    "tags": []
   },
   "source": [
    "## Support Vectors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": null,
     "end_time": null,
     "exception": null,
     "start_time": null,
     "status": "pending"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "is_SV = runs_data[\"is_SV\"].astype(np.int)\n",
    "print(\"There are {} support vectors\".format(np.sum(is_SV)))\n",
    "hist = sns.distplot(is_SV, kde=False, bins=hist_bins)\n",
    "hist.set_yscale('log')\n",
    "hist.get_figure().savefig(\"is_SV_histogram.pdf\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": null,
     "end_time": null,
     "exception": null,
     "start_time": null,
     "status": "pending"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "VSV_acc = runs_data[\"VSV_acc\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": null,
     "end_time": null,
     "exception": null,
     "start_time": null,
     "status": "pending"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "VSV_acc"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": null,
     "end_time": null,
     "exception": null,
     "start_time": null,
     "status": "pending"
    },
    "tags": []
   },
   "source": [
    "## Parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": null,
     "end_time": null,
     "exception": null,
     "start_time": null,
     "status": "pending"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "runs_data[\"run_parameters\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": null,
     "end_time": null,
     "exception": null,
     "start_time": null,
     "status": "pending"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "baseline_acc = runs_data[\"no_aug_no_poison_acc\"]\n",
    "poisoned_acc = runs_data[\"poisoned_acc\"]\n",
    "all_aug_train_poisoned_acc = runs_data[\"all_aug_train_poisoned_acc\"]\n",
    "n_aug_sample_points = runs_data[\"n_aug_sample_points\"]\n",
    "n_train = runs_data[\"n_train\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": null,
     "end_time": null,
     "exception": null,
     "start_time": null,
     "status": "pending"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "baseline_acc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": null,
     "end_time": null,
     "exception": null,
     "start_time": null,
     "status": "pending"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "poisoned_acc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": null,
     "end_time": null,
     "exception": null,
     "start_time": null,
     "status": "pending"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "all_aug_train_poisoned_acc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": null,
     "end_time": null,
     "exception": null,
     "start_time": null,
     "status": "pending"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "n_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": null,
     "end_time": null,
     "exception": null,
     "start_time": null,
     "status": "pending"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "runs_data[\"experiment_results\"].item()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": null,
     "end_time": null,
     "exception": null,
     "start_time": null,
     "status": "pending"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "labels = list(runs_data[\"experiment_results\"].item().keys())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": null,
     "end_time": null,
     "exception": null,
     "start_time": null,
     "status": "pending"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "run_matrix = np.array([\n",
    "    np.array(runs_data[\"experiment_results\"].item()[k]) for k in labels  \n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": null,
     "end_time": null,
     "exception": null,
     "start_time": null,
     "status": "pending"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "run_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": null,
     "end_time": null,
     "exception": null,
     "start_time": null,
     "status": "pending"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "run_df_rows = []\n",
    "for i, label in enumerate(labels):\n",
    "    for test in range(run_matrix[i].shape[0]):\n",
    "        run_df_row = pd.Series()\n",
    "        run_df_row[\"test_i\"] = test\n",
    "        run_df_row[\"n_auged\"] = 0\n",
    "        run_df_row[\"test_type\"] = label\n",
    "        run_df_row[\"test_accuracy\"] = float(poisoned_acc)\n",
    "        run_df_rows.append(run_df_row)\n",
    "        for step in range(run_matrix[i].shape[1]):\n",
    "            run_df_row = pd.Series()\n",
    "            run_df_row[\"test_i\"] = test\n",
    "            run_df_row[\"n_auged\"] = n_aug_sample_points[step]\n",
    "            run_df_row[\"test_type\"] = label\n",
    "            run_df_row[\"test_accuracy\"] = run_matrix[i][test, step]\n",
    "            run_df_rows.append(run_df_row)\n",
    "run_df = pd.DataFrame(run_df_rows)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": null,
     "end_time": null,
     "exception": null,
     "start_time": null,
     "status": "pending"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "run_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": null,
     "end_time": null,
     "exception": null,
     "start_time": null,
     "status": "pending"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "#old_filename_prefix = filename_prefix + \"_old\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": null,
     "end_time": null,
     "exception": null,
     "start_time": null,
     "status": "pending"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "#old_run_df = pd.read_pickle(\"{}.pkl\".format(old_filename_prefix))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": null,
     "end_time": null,
     "exception": null,
     "start_time": null,
     "status": "pending"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "#old_run_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": null,
     "end_time": null,
     "exception": null,
     "start_time": null,
     "status": "pending"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "#to_remove = [i for i, x in enumerate(old_run_df[\"test_type\"]) if \"update\" in x]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": null,
     "end_time": null,
     "exception": null,
     "start_time": null,
     "status": "pending"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "#to_remove"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": null,
     "end_time": null,
     "exception": null,
     "start_time": null,
     "status": "pending"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "#old_run_df.drop(index=to_remove,\n",
    "#                inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": null,
     "end_time": null,
     "exception": null,
     "start_time": null,
     "status": "pending"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "#run_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": null,
     "end_time": null,
     "exception": null,
     "start_time": null,
     "status": "pending"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "#to_not_remove = [i for i, x in enumerate(run_df[\"test_type\"]) if \"update\" not in x]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": null,
     "end_time": null,
     "exception": null,
     "start_time": null,
     "status": "pending"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "#run_df.drop(index=to_not_remove,\n",
    "#            inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": null,
     "end_time": null,
     "exception": null,
     "start_time": null,
     "status": "pending"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "#run_df = pd.concat([old_run_df, run_df])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": null,
     "end_time": null,
     "exception": null,
     "start_time": null,
     "status": "pending"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "run_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": null,
     "end_time": null,
     "exception": null,
     "start_time": null,
     "status": "pending"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "run_df.to_pickle(filename_prefix + \".pkl\")\n",
    "run_df.to_csv(filename_prefix + \".csv\", encoding='utf-8')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": null,
     "end_time": null,
     "exception": null,
     "start_time": null,
     "status": "pending"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "n_aug_sample_points = run_df[\"n_auged\"].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": null,
     "end_time": null,
     "exception": null,
     "start_time": null,
     "status": "pending"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "n_aug_sample_points"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": null,
     "end_time": null,
     "exception": null,
     "start_time": null,
     "status": "pending"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "all_samples_points = np.unique(np.concatenate([[0], n_aug_sample_points]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": null,
     "end_time": null,
     "exception": null,
     "start_time": null,
     "status": "pending"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "all_samples_points"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": null,
     "end_time": null,
     "exception": null,
     "start_time": null,
     "status": "pending"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "aucs = (run_df\n",
    "        .sort_values(\"n_auged\", ascending=True)\n",
    "        .groupby([\"test_type\", \"test_i\"])[\"test_accuracy\"]\n",
    "        .apply(\n",
    "            lambda x: sklearn.metrics.auc(all_samples_points, x)\n",
    "        )\n",
    "       )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": null,
     "end_time": null,
     "exception": null,
     "start_time": null,
     "status": "pending"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "aucs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": null,
     "end_time": null,
     "exception": null,
     "start_time": null,
     "status": "pending"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "aucs.groupby(\"test_type\").mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": null,
     "end_time": null,
     "exception": null,
     "start_time": null,
     "status": "pending"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "aucs.groupby(\"test_type\").var()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": null,
     "end_time": null,
     "exception": null,
     "start_time": null,
     "status": "pending"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "auc_means = aucs.groupby(\"test_type\").mean().sort_values(ascending=False).rename(\"AUC Mean\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": null,
     "end_time": null,
     "exception": null,
     "start_time": null,
     "status": "pending"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "auc_std = aucs.groupby(\"test_type\").std().sort_values(ascending=False).rename(\"AUC Std.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": null,
     "end_time": null,
     "exception": null,
     "start_time": null,
     "status": "pending"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "auc_mean_std = pd.concat([auc_means, auc_std], axis=1).sort_values(ascending=False, by=\"AUC Mean\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": null,
     "end_time": null,
     "exception": null,
     "start_time": null,
     "status": "pending"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "#auc_mean_std.index.name = \"Test Type\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": null,
     "end_time": null,
     "exception": null,
     "start_time": null,
     "status": "pending"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "auc_mean_std"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": null,
     "end_time": null,
     "exception": null,
     "start_time": null,
     "status": "pending"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "replace_dict = {x: string.capwords(x.replace(\"_\", \" \")) for x in auc_mean_std.index.unique()}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": null,
     "end_time": null,
     "exception": null,
     "start_time": null,
     "status": "pending"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "replace_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": null,
     "end_time": null,
     "exception": null,
     "start_time": null,
     "status": "pending"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "formatted_auc_mean_std = auc_mean_std.copy(deep=True)\n",
    "formatted_auc_mean_std = formatted_auc_mean_std.replace(np.nan, \"{\\textemdash}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": null,
     "end_time": null,
     "exception": null,
     "start_time": null,
     "status": "pending"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "formatted_auc_mean_std.index.name = \"Policy\"\n",
    "formatted_auc_mean_std = formatted_auc_mean_std.reset_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": null,
     "end_time": null,
     "exception": null,
     "start_time": null,
     "status": "pending"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "allowed_columns = {\"baseline\",\n",
    "                   \"random_proportional\",\n",
    "                   \"random_proportional_update\",\n",
    "                   \"random_proportional_downweight\",\n",
    "                   \"random_proportional_update_downweight\",\n",
    "                   \"random_inverse_proportional\",\n",
    "                   \"deterministic_proportional\",\n",
    "                   \"deterministic_proportional_update\",\n",
    "                   \"deterministic_proportional_downweight\",\n",
    "                   \"deterministic_proportional_update_downweight\",\n",
    "                   \"deterministic_inverse_proportional\"\n",
    "                  }\n",
    "removed_idxs = [i for i, x in\n",
    "                enumerate(formatted_auc_mean_std[\"Policy\"])\n",
    "                if x not in allowed_columns]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": null,
     "end_time": null,
     "exception": null,
     "start_time": null,
     "status": "pending"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "removed_idxs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": null,
     "end_time": null,
     "exception": null,
     "start_time": null,
     "status": "pending"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "formatted_auc_mean_std.drop(index=removed_idxs, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": null,
     "end_time": null,
     "exception": null,
     "start_time": null,
     "status": "pending"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "formatted_auc_mean_std[\"Policy\"].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": null,
     "end_time": null,
     "exception": null,
     "start_time": null,
     "status": "pending"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "#formatted_auc_mean_std[\"Policy\"] = formatted_auc_mean_std[\"Policy\"].str.replace(r\"_\", r\"\\_\")\n",
    "formatted_auc_mean_std[\"Policy\"] = formatted_auc_mean_std[\"Policy\"].replace(replace_dict, regex=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": null,
     "end_time": null,
     "exception": null,
     "start_time": null,
     "status": "pending"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "formatted_auc_mean_std"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": null,
     "end_time": null,
     "exception": null,
     "start_time": null,
     "status": "pending"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "header_names=[\"{{{}}}\".format(c) for c in formatted_auc_mean_std.columns]\n",
    "with open(filename_prefix + '_auc_mean_std.tex', 'w') as f:\n",
    "    f.write(formatted_auc_mean_std.to_latex(\n",
    "        na_rep=\"---\",\n",
    "        escape=False,\n",
    "        header=header_names,\n",
    "        index=False,\n",
    "        column_format=\"l*{{{right_cols}}}{{S}}\".format(right_cols=len(formatted_auc_mean_std.columns)-1)\n",
    "    ))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": null,
     "end_time": null,
     "exception": null,
     "start_time": null,
     "status": "pending"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "best_at_n = run_df.groupby(\"n_auged\", as_index=False).max()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": null,
     "end_time": null,
     "exception": null,
     "start_time": null,
     "status": "pending"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "best_at_n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": null,
     "end_time": null,
     "exception": null,
     "start_time": null,
     "status": "pending"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "baseline_perf = run_df.query(\"test_type == 'baseline'\").groupby(\"n_auged\", as_index=False).mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": null,
     "end_time": null,
     "exception": null,
     "start_time": null,
     "status": "pending"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "baseline_perf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": null,
     "end_time": null,
     "exception": null,
     "start_time": null,
     "status": "pending"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "run_df.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": null,
     "end_time": null,
     "exception": null,
     "start_time": null,
     "status": "pending"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "run_plot = sns.lineplot(x=\"n_auged\",\n",
    "                        y=\"test_accuracy\",\n",
    "                        ci=95,\n",
    "                        data=run_df.query(\"test_type == 'baseline'\"))\n",
    "run_plot.axhline(y=baseline_acc,\n",
    "                 color=\"b\",\n",
    "                 linestyle=\"--\",\n",
    "                 label=\"baseline_acc\")\n",
    "run_plot.axhline(y=poisoned_acc,\n",
    "                 color=\"r\",\n",
    "                 linestyle=\"--\",\n",
    "                 label=\"poisoned_acc\")\n",
    "run_plot.axhline(y=all_aug_train_poisoned_acc,\n",
    "                 color=\"g\",\n",
    "                 linestyle=\"--\",\n",
    "                 label=\"all_aug_train_poisoned_acc\")\n",
    "run_plot = sns.lineplot(x=\"n_auged\",\n",
    "                        y=\"test_accuracy\",\n",
    "                        ci=95,\n",
    "                        data=best_at_n,\n",
    "                        ax=run_plot)\n",
    "\"\"\"\n",
    "run_plot = sns.relplot(x=\"n_auged\",\n",
    "                       y=\"test_accuracy\",\n",
    "                       hue=\"test_i\",\n",
    "                       col=\"test_type\",\n",
    "                       col_wrap=4,\n",
    "                       ci=0,\n",
    "                       markers=True,\n",
    "                       kind=\"line\",\n",
    "                       data=run_df,\n",
    "                       ax=run_plot,\n",
    "                       alpha=0.1)\n",
    "                       \"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": null,
     "end_time": null,
     "exception": null,
     "start_time": null,
     "status": "pending"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "run_plot = sns.lineplot(x=\"n_auged\",\n",
    "                        y=\"test_accuracy\",\n",
    "                        hue=\"test_type\",\n",
    "                        ci=95,\n",
    "                        data=run_df)\n",
    "run_plot.axhline(y=baseline_acc,\n",
    "                 color=\"b\",\n",
    "                 linestyle=\"--\",\n",
    "                 label=\"baseline_acc\")\n",
    "run_plot.axhline(y=poisoned_acc,\n",
    "                 color=\"r\",\n",
    "                 linestyle=\"--\",\n",
    "                 label=\"poisoned_acc\")\n",
    "run_plot.axhline(y=all_aug_train_poisoned_acc,\n",
    "                 color=\"g\",\n",
    "                 linestyle=\"--\",\n",
    "                 label=\"all_aug_train_poisoned_acc\")\n",
    "#run_plot.get_figure().savefig(\"test_accuracy_summary.pdf\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": null,
     "end_time": null,
     "exception": null,
     "start_time": null,
     "status": "pending"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "run_plot = sns.relplot(x=\"n_auged\",\n",
    "                       y=\"test_accuracy\",\n",
    "                       hue=\"test_i\",\n",
    "                       col=\"test_type\",\n",
    "                       col_wrap=4,\n",
    "                       ci=95,\n",
    "                       markers=True,\n",
    "                       kind=\"line\",\n",
    "                       data=run_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": null,
     "end_time": null,
     "exception": null,
     "start_time": null,
     "status": "pending"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "run_plot = sns.relplot(x=\"n_auged\",\n",
    "                       y=\"test_accuracy\",\n",
    "                       hue=\"test_type\",\n",
    "                       col=\"test_type\",\n",
    "                       col_wrap=4,\n",
    "                       ci=95,\n",
    "                       markers=True,\n",
    "                       kind=\"line\",\n",
    "                       palette=sns.color_palette(\"Set2\", n_colors=len(run_df[\"test_type\"].unique())),\n",
    "                       data=run_df.query(\"n_auged <= 10\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": null,
     "end_time": null,
     "exception": null,
     "start_time": null,
     "status": "pending"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "sns.set_style(\"whitegrid\")\n",
    "sns.set_context(\"paper\")\n",
    "sns.set(font_scale=1.1)\n",
    "sns.despine(left=True)\n",
    "sns.set_style(\"ticks\", {\"xtick.major.size\": 8, \"ytick.major.size\": 8})\n",
    "cmap = sns.color_palette(\"Set1\")\n",
    "sns.palplot(cmap)\n",
    "sns.set_palette(cmap)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": null,
     "end_time": null,
     "exception": null,
     "start_time": null,
     "status": "pending"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "cmap = sns.color_palette(\"Set1\")\n",
    "yellow = cmap[5]\n",
    "cmap[5] = cmap[6]\n",
    "cmap[6] = yellow\n",
    "sns.palplot(cmap)\n",
    "sns.set_palette(cmap)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": null,
     "end_time": null,
     "exception": null,
     "start_time": null,
     "status": "pending"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "update_downweight_run_df = (run_df.query(\"test_type == 'random_proportional'\"\n",
    "                                          \"| test_type == 'random_inverse_proportional'\"\n",
    "                                          \"| test_type == 'baseline'\")\n",
    "                            .query(\"n_auged < 1100\"))\n",
    "update_downweight_run_df = update_downweight_run_df.rename(\n",
    "    index=str,\n",
    "    columns={\n",
    "            \"test_accuracy\": \"Test Accuracy\",\n",
    "            \"n_auged\": \"Number of Augmented Points\",\n",
    "\n",
    "            },\n",
    ")\n",
    "update_downweight_run_df[\"test_type\"] = update_downweight_run_df[\"test_type\"].replace(replace_dict, regex=False)\n",
    "fig, ax = plt.subplots()\n",
    "run_plot = sns.lineplot(x=\"Number of Augmented Points\",\n",
    "                        y=\"Test Accuracy\",\n",
    "                        hue=\"test_type\",\n",
    "                        style=\"test_type\",\n",
    "                        ci=95,\n",
    "                        data=update_downweight_run_df,\n",
    "                        markers=True,\n",
    "                        dashes=True,\n",
    "                        ax=ax)\n",
    "l = ax.legend()\n",
    "#l.texts[0].set_text(\"\")\n",
    "#l.set_title('Whatever you want')\n",
    "handles, labels = ax.get_legend_handles_labels()\n",
    "ax.legend(handles=handles[1:], labels=labels[1:])\n",
    "ax.yaxis.set_major_formatter(matplotlib.ticker.FormatStrFormatter(plt_y_axis_fmt_string))\n",
    "plt.setp(ax.get_legend().get_texts(), fontsize='11.5') # for legend text \n",
    "#run_plot.axhline(y=baseline_acc,\n",
    "#                 color=\"b\",\n",
    "#                 linestyle=\"--\",\n",
    "#                 label=\"baseline_acc\")\n",
    "run_plot.axhline(y=poisoned_acc,\n",
    "                 color=\"r\",\n",
    "                 linestyle=\"--\",\n",
    "                 label=\"poisoned_acc\")\n",
    "run_plot.axhline(y=all_aug_train_poisoned_acc,\n",
    "                 color=\"g\",\n",
    "                 linestyle=\"--\",\n",
    "                 label=\"all_aug_train_poisoned_acc\")\n",
    "run_plot.get_figure().savefig(filename_prefix + \"_modifications_accuracy.pdf\",\n",
    "                              bbox_inches=\"tight\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": null,
     "end_time": null,
     "exception": null,
     "start_time": null,
     "status": "pending"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "update_downweight_run_df = (run_df.query(\"test_type == 'deterministic_proportional'\"\n",
    "                                          \"| test_type == 'deterministic_proportional_update'\"\n",
    "                                          \"| test_type == 'deterministic_proportional_downweight'\"\n",
    "                                          \"| test_type == 'deterministic_proportional_update_downweight'\"\n",
    "                                          \"| test_type == 'deterministic_inverse_proportional'\"\n",
    "                                          \"| test_type == 'baseline'\")\n",
    "                            .query(\"n_auged < 1100\"))\n",
    "update_downweight_run_df = update_downweight_run_df.rename(\n",
    "    index=str,\n",
    "    columns={\"test_accuracy\": \"Test Accuracy\",\n",
    "             \"n_auged\": \"Number of Augmented Points\",\n",
    "            },\n",
    ")\n",
    "update_downweight_run_df[\"test_type\"] = update_downweight_run_df[\"test_type\"].replace(replace_dict, regex=False)\n",
    "fig, ax = plt.subplots()\n",
    "run_plot = sns.lineplot(x=\"Number of Augmented Points\",\n",
    "                        y=\"Test Accuracy\",\n",
    "                        hue=\"test_type\",\n",
    "                        style=\"test_type\",\n",
    "                        ci=95,\n",
    "                        data=update_downweight_run_df,\n",
    "                        markers=True,\n",
    "                        dashes=True,\n",
    "                        ax=ax)\n",
    "l = ax.legend()\n",
    "#l.texts[0].set_text(\"\")\n",
    "#l.set_title('Whatever you want')\n",
    "handles, labels = ax.get_legend_handles_labels()\n",
    "ax.legend(handles=handles[1:], labels=labels[1:])\n",
    "ax.yaxis.set_major_formatter(matplotlib.ticker.FormatStrFormatter(plt_y_axis_fmt_string))\n",
    "plt.setp(ax.get_legend().get_texts(), fontsize='10') # for legend text \n",
    "#run_plot.axhline(y=baseline_acc,\n",
    "#                 color=\"b\",\n",
    "#                 linestyle=\"--\",\n",
    "#                 label=\"baseline_acc\")\n",
    "run_plot.axhline(y=poisoned_acc,\n",
    "                 color=\"r\",\n",
    "                 linestyle=\"--\",\n",
    "                 label=\"poisoned_acc\")\n",
    "run_plot.axhline(y=all_aug_train_poisoned_acc,\n",
    "                 color=\"g\",\n",
    "                 linestyle=\"--\",\n",
    "                 label=\"all_aug_train_poisoned_acc\")\n",
    "run_plot.get_figure().savefig(filename_prefix + \"_deterministic_accuracy.pdf\",\n",
    "                              bbox_inches=\"tight\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": null,
     "end_time": null,
     "exception": null,
     "start_time": null,
     "status": "pending"
    },
    "tags": []
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": null,
     "end_time": null,
     "exception": null,
     "start_time": null,
     "status": "pending"
    },
    "tags": []
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  },
  "papermill": {
   "duration": 2.31089,
   "end_time": "2018-11-19T22:34:25.860896",
   "environment_variables": {},
   "exception": true,
   "output_path": "Visualize_LOO_Experiments-Margin.ipynb",
   "parameters": {
    "filename_prefix": "aug_results_magin_NORB_0_vs_1_crop_10_loss"
   },
   "start_time": "2018-11-19T22:34:23.550006",
   "version": "0.15.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}