Visualize_LOO_Experiments-Combine_Inf_DPP_Random.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": 0.149119,
     "end_time": "2018-11-19T22:35:22.640106",
     "exception": false,
     "start_time": "2018-11-19T22:35:22.490987",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": 0.447586,
     "end_time": "2018-11-19T22:35:23.087794",
     "exception": false,
     "start_time": "2018-11-19T22:35:22.640208",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import pandas as pd\n",
    "import sklearn.metrics\n",
    "import scipy\n",
    "import adjustText\n",
    "import matplotlib.ticker"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": 0.01528,
     "end_time": "2018-11-19T22:35:23.103125",
     "exception": false,
     "start_time": "2018-11-19T22:35:23.087845",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Set the default plot style\n",
    "#default_plt_width = 15\n",
    "#default_plt_height = 10\n",
    "#plt.rcParams['figure.figsize'] = [default_plt_width, default_plt_height]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": 0.076541,
     "end_time": "2018-11-19T22:35:23.179763",
     "exception": false,
     "start_time": "2018-11-19T22:35:23.103222",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "sns.set_style(\"whitegrid\")\n",
    "sns.set_context(\"paper\")\n",
    "sns.set(font_scale=1.1)\n",
    "sns.despine(left=True)\n",
    "sns.set_style(\"ticks\", {\"xtick.major.size\": 8, \"ytick.major.size\": 8})\n",
    "cmap = sns.color_palette(\"Set1\")\n",
    "sns.palplot(cmap)\n",
    "sns.set_palette(cmap)\n",
    "plt_y_axis_fmt_string = '%.3f'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": 0.024345,
     "end_time": "2018-11-19T22:35:23.204171",
     "exception": false,
     "start_time": "2018-11-19T22:35:23.179826",
     "status": "completed"
    },
    "tags": [
     "parameters"
    ]
   },
   "outputs": [],
   "source": [
    "filename_prefix = \"aug_results_MNIST_3_vs_8_translate_10\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": 0.0243,
     "end_time": "2018-11-19T22:35:23.228568",
     "exception": false,
     "start_time": "2018-11-19T22:35:23.204268",
     "status": "completed"
    },
    "tags": [
     "injected-parameters"
    ]
   },
   "outputs": [],
   "source": [
    "# Parameters\n",
    "filename_prefix = \"aug_results_CIFAR10_0_vs_1_translate_10\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": 0.014605,
     "end_time": "2018-11-19T22:35:23.243217",
     "exception": false,
     "start_time": "2018-11-19T22:35:23.228612",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "filename_prefix_margin = \"aug_results_dpp_norm_\" + filename_prefix.split(\"aug_results_\", 1)[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": 0.01525,
     "end_time": "2018-11-19T22:35:23.258508",
     "exception": false,
     "start_time": "2018-11-19T22:35:23.243258",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "filename_prefix_margin"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": 0.135824,
     "end_time": "2018-11-19T22:35:23.394374",
     "exception": false,
     "start_time": "2018-11-19T22:35:23.258550",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "%ls"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": 0.042828,
     "end_time": "2018-11-19T22:35:23.437627",
     "exception": false,
     "start_time": "2018-11-19T22:35:23.394799",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "runs_data = np.load(\"{}.npz\".format(filename_prefix))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": 0.022904,
     "end_time": "2018-11-19T22:35:23.471061",
     "exception": false,
     "start_time": "2018-11-19T22:35:23.448157",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "baseline_acc = runs_data[\"no_aug_no_poison_acc\"]\n",
    "poisoned_acc = runs_data[\"poisoned_acc\"]\n",
    "all_aug_train_poisoned_acc = runs_data[\"all_aug_train_poisoned_acc\"]\n",
    "n_aug_sample_points = runs_data[\"n_aug_sample_points\"]\n",
    "n_train = runs_data[\"n_train\"]\n",
    "VSV_acc = runs_data[\"VSV_acc\"]\n",
    "is_SV = runs_data[\"is_SV\"].astype(np.int)\n",
    "n_SV = np.sum(is_SV)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": 0.094296,
     "end_time": "2018-11-19T22:35:23.565476",
     "exception": true,
     "start_time": "2018-11-19T22:35:23.471180",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "runs_data_inf = pd.read_pickle(\"{}.pkl\".format(filename_prefix))\n",
    "runs_data_loss = pd.read_pickle(\"{}.pkl\".format(filename_prefix_margin))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": null,
     "end_time": null,
     "exception": null,
     "start_time": null,
     "status": "pending"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "runs_data_inf[\"score\"] = \"influence\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": null,
     "end_time": null,
     "exception": null,
     "start_time": null,
     "status": "pending"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "runs_data_loss[\"score\"] = \"loss\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": null,
     "end_time": null,
     "exception": null,
     "start_time": null,
     "status": "pending"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "run_df_unprocessed = pd.concat([\n",
    "    runs_data_inf,\n",
    "    runs_data_loss,\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": null,
     "end_time": null,
     "exception": null,
     "start_time": null,
     "status": "pending"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "run_df_unprocessed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": null,
     "end_time": null,
     "exception": null,
     "start_time": null,
     "status": "pending"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "baseline = run_df_unprocessed.query(\"test_type == 'baseline'\").reset_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": null,
     "end_time": null,
     "exception": null,
     "start_time": null,
     "status": "pending"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "baseline[\"score\"] = \"baseline\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": null,
     "end_time": null,
     "exception": null,
     "start_time": null,
     "status": "pending"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "baseline[\"test_type\"] = \"Baseline\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": null,
     "end_time": null,
     "exception": null,
     "start_time": null,
     "status": "pending"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "baseline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": null,
     "end_time": null,
     "exception": null,
     "start_time": null,
     "status": "pending"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "prop_inf = run_df_unprocessed.query(\"test_type == 'random_proportional' & score == 'influence'\")\n",
    "prop_inf[\"test_type\"] = \"Random Proportional Influence\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": null,
     "end_time": null,
     "exception": null,
     "start_time": null,
     "status": "pending"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "prop_loss = run_df_unprocessed.query(\"test_type == 'random_proportional' & score == 'loss'\")\n",
    "prop_loss[\"test_type\"] = \"Influence DPP\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": null,
     "end_time": null,
     "exception": null,
     "start_time": null,
     "status": "pending"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "inv_prop_loss = run_df_unprocessed.query(\"test_type == 'random_inverse_proportional' & score == 'loss'\")\n",
    "inv_prop_loss[\"test_type\"] = \"Random Inverse Proportional Margin\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": null,
     "end_time": null,
     "exception": null,
     "start_time": null,
     "status": "pending"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "run_df = pd.concat([\n",
    "    baseline,\n",
    "    prop_inf,\n",
    "    prop_loss,\n",
    "    inv_prop_loss,\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": null,
     "end_time": null,
     "exception": null,
     "start_time": null,
     "status": "pending"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "run_df = run_df.rename(\n",
    "    index=str,\n",
    "    columns={\"test_accuracy\": \"Test Accuracy\",\n",
    "             \"n_auged\": \"Number of Augmented Points\",\n",
    "            },\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": null,
     "end_time": null,
     "exception": null,
     "start_time": null,
     "status": "pending"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "run_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": null,
     "end_time": null,
     "exception": null,
     "start_time": null,
     "status": "pending"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "VSV_x = n_SV\n",
    "VSV_y = VSV_acc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": null,
     "end_time": null,
     "exception": null,
     "start_time": null,
     "status": "pending"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "fig, ax = plt.subplots()\n",
    "run_plot = sns.lineplot(x=\"Number of Augmented Points\",\n",
    "                        y=\"Test Accuracy\",\n",
    "                        hue=\"test_type\",\n",
    "                        style=\"test_type\",\n",
    "                        ci=95,\n",
    "                        data=run_df,\n",
    "                        markers=True,\n",
    "                        dashes=True,\n",
    "                        ax=ax)\n",
    "#run_plot.scatter(VSV_x, VSV_y, marker=\"x\", color=\"k\", s=20)\n",
    "# text = run_plot.annotate(\"VSV\", (VSV_x, VSV_y))\n",
    "#text = run_plot.text(VSV_x, VSV_y, \"VSV\", fontsize=12)\n",
    "l = ax.legend()\n",
    "#l.texts[0].set_text(\"\")\n",
    "#l.set_title('Whatever you want')\n",
    "handles, labels = ax.get_legend_handles_labels()\n",
    "ax.legend(handles=handles[1:], labels=labels[1:])\n",
    "ax.yaxis.set_major_formatter(matplotlib.ticker.FormatStrFormatter(plt_y_axis_fmt_string))\n",
    "plt.setp(ax.get_legend().get_texts(), fontsize='11.5') # for legend text \n",
    "#run_plot.axhline(y=baseline_acc,\n",
    "#                 color=\"b\",\n",
    "#                 linestyle=\"--\",\n",
    "#                 label=\"baseline_acc\")\n",
    "run_plot.axhline(y=poisoned_acc,\n",
    "                 color=\"r\",\n",
    "                 linestyle=\"--\",\n",
    "                 label=\"poisoned_acc\")\n",
    "run_plot.axhline(y=all_aug_train_poisoned_acc,\n",
    "                 color=\"g\",\n",
    "                 linestyle=\"--\",\n",
    "                 label=\"all_aug_train_poisoned_acc\")\n",
    "#adjustText.adjust_text([text],\n",
    "#                       x=[VSV_x],\n",
    "#                       y=[VSV_y],\n",
    "#                       add_objects=[run_plot],\n",
    "#                       expand_points=(0.5, 0.3),\n",
    "#                       expand_objects=(0.3, 0.3),\n",
    "#                       ax=ax,\n",
    "#                       force_objects=(0.2, 0.2))\n",
    "run_plot.get_figure().savefig(filename_prefix + \"_joined_dpp.pdf\",\n",
    "                              bbox_inches=\"tight\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": null,
     "end_time": null,
     "exception": null,
     "start_time": null,
     "status": "pending"
    },
    "tags": []
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": null,
     "end_time": null,
     "exception": null,
     "start_time": null,
     "status": "pending"
    },
    "tags": []
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": null,
     "end_time": null,
     "exception": null,
     "start_time": null,
     "status": "pending"
    },
    "tags": []
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  },
  "papermill": {
   "duration": 2.310896,
   "end_time": "2018-11-19T22:35:24.178023",
   "environment_variables": {},
   "exception": true,
   "output_path": "Visualize_LOO_Experiments-Combine_Inf_Margin_Random.ipynb",
   "parameters": {
    "filename_prefix": "aug_results_NORB_0_vs_1_crop_10_loss"
   },
   "start_time": "2018-11-19T22:35:21.867127",
   "version": "0.15.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}