lcm_simulate.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Imports and registering variables with orca "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Juan\\Anaconda3\\envs\\semcog_py2\\lib\\site-packages\\matplotlib\\colors.py:298: MatplotlibDeprecationWarning: The is_string_like function was deprecated in version 2.1.\n",
      "  if cbook.is_string_like(arg):\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "using seed 271828\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Juan\\Anaconda3\\envs\\semcog_py2\\lib\\site-packages\\matplotlib\\colors.py:351: MatplotlibDeprecationWarning: The is_string_like function was deprecated in version 2.1.\n",
      "  if not cbook.is_string_like(arg) and cbook.iterable(arg):\n",
      "C:\\Users\\Juan\\Anaconda3\\envs\\semcog_py2\\lib\\site-packages\\matplotlib\\colors.py:765: MatplotlibDeprecationWarning: The is_string_like function was deprecated in version 2.1.\n",
      "  not cbook.is_string_like(colors[0]):\n",
      "\n",
      "Bad key \"patch.force_edgecolor\" on line 33 in\n",
      "C:\\Users\\Juan\\Anaconda3\\envs\\semcog_py2\\lib\\site-packages\\matplotlib\\mpl-data\\stylelib\\_classic_test.mplstyle.\n",
      "You probably need to get an updated matplotlibrc file from\n",
      "http://github.com/matplotlib/matplotlib/blob/master/matplotlibrc.template\n",
      "or from the matplotlib source distribution\n",
      "\n",
      "Bad key \"lines.dashed_pattern\" on line 18 in\n",
      "C:\\Users\\Juan\\Anaconda3\\envs\\semcog_py2\\lib\\site-packages\\matplotlib\\mpl-data\\stylelib\\_classic_test.mplstyle.\n",
      "You probably need to get an updated matplotlibrc file from\n",
      "http://github.com/matplotlib/matplotlib/blob/master/matplotlibrc.template\n",
      "or from the matplotlib source distribution\n",
      "\n",
      "Bad key \"boxplot.meanprops.markeredgecolor\" on line 362 in\n",
      "C:\\Users\\Juan\\Anaconda3\\envs\\semcog_py2\\lib\\site-packages\\matplotlib\\mpl-data\\stylelib\\_classic_test.mplstyle.\n",
      "You probably need to get an updated matplotlibrc file from\n",
      "http://github.com/matplotlib/matplotlib/blob/master/matplotlibrc.template\n",
      "or from the matplotlib source distribution\n",
      "\n",
      "Bad key \"date.autoformatter.second\" on line 228 in\n",
      "C:\\Users\\Juan\\Anaconda3\\envs\\semcog_py2\\lib\\site-packages\\matplotlib\\mpl-data\\stylelib\\_classic_test.mplstyle.\n",
      "You probably need to get an updated matplotlibrc file from\n",
      "http://github.com/matplotlib/matplotlib/blob/master/matplotlibrc.template\n",
      "or from the matplotlib source distribution\n",
      "\n",
      "Bad key \"ytick.major.right\" on line 264 in\n",
      "C:\\Users\\Juan\\Anaconda3\\envs\\semcog_py2\\lib\\site-packages\\matplotlib\\mpl-data\\stylelib\\_classic_test.mplstyle.\n",
      "You probably need to get an updated matplotlibrc file from\n",
      "http://github.com/matplotlib/matplotlib/blob/master/matplotlibrc.template\n",
      "or from the matplotlib source distribution\n",
      "\n",
      "Bad key \"date.autoformatter.month\" on line 224 in\n",
      "C:\\Users\\Juan\\Anaconda3\\envs\\semcog_py2\\lib\\site-packages\\matplotlib\\mpl-data\\stylelib\\_classic_test.mplstyle.\n",
      "You probably need to get an updated matplotlibrc file from\n",
      "http://github.com/matplotlib/matplotlib/blob/master/matplotlibrc.template\n",
      "or from the matplotlib source distribution\n",
      "\n",
      "Bad key \"scatter.marker\" on line 338 in\n",
      "C:\\Users\\Juan\\Anaconda3\\envs\\semcog_py2\\lib\\site-packages\\matplotlib\\mpl-data\\stylelib\\_classic_test.mplstyle.\n",
      "You probably need to get an updated matplotlibrc file from\n",
      "http://github.com/matplotlib/matplotlib/blob/master/matplotlibrc.template\n",
      "or from the matplotlib source distribution\n",
      "\n",
      "Bad key \"ytick.minor.right\" on line 266 in\n",
      "C:\\Users\\Juan\\Anaconda3\\envs\\semcog_py2\\lib\\site-packages\\matplotlib\\mpl-data\\stylelib\\_classic_test.mplstyle.\n",
      "You probably need to get an updated matplotlibrc file from\n",
      "http://github.com/matplotlib/matplotlib/blob/master/matplotlibrc.template\n",
      "or from the matplotlib source distribution\n",
      "\n",
      "Bad key \"ytick.major.left\" on line 263 in\n",
      "C:\\Users\\Juan\\Anaconda3\\envs\\semcog_py2\\lib\\site-packages\\matplotlib\\mpl-data\\stylelib\\_classic_test.mplstyle.\n",
      "You probably need to get an updated matplotlibrc file from\n",
      "http://github.com/matplotlib/matplotlib/blob/master/matplotlibrc.template\n",
      "or from the matplotlib source distribution\n",
      "\n",
      "Bad key \"ytick.left\" on line 251 in\n",
      "C:\\Users\\Juan\\Anaconda3\\envs\\semcog_py2\\lib\\site-packages\\matplotlib\\mpl-data\\stylelib\\_classic_test.mplstyle.\n",
      "You probably need to get an updated matplotlibrc file from\n",
      "http://github.com/matplotlib/matplotlib/blob/master/matplotlibrc.template\n",
      "or from the matplotlib source distribution\n",
      "\n",
      "Bad key \"xtick.minor.bottom\" on line 249 in\n",
      "C:\\Users\\Juan\\Anaconda3\\envs\\semcog_py2\\lib\\site-packages\\matplotlib\\mpl-data\\stylelib\\_classic_test.mplstyle.\n",
      "You probably need to get an updated matplotlibrc file from\n",
      "http://github.com/matplotlib/matplotlib/blob/master/matplotlibrc.template\n",
      "or from the matplotlib source distribution\n",
      "\n",
      "Bad key \"axes.autolimit_mode\" on line 213 in\n",
      "C:\\Users\\Juan\\Anaconda3\\envs\\semcog_py2\\lib\\site-packages\\matplotlib\\mpl-data\\stylelib\\_classic_test.mplstyle.\n",
      "You probably need to get an updated matplotlibrc file from\n",
      "http://github.com/matplotlib/matplotlib/blob/master/matplotlibrc.template\n",
      "or from the matplotlib source distribution\n",
      "\n",
      "Bad key \"hatch.color\" on line 37 in\n",
      "C:\\Users\\Juan\\Anaconda3\\envs\\semcog_py2\\lib\\site-packages\\matplotlib\\mpl-data\\stylelib\\_classic_test.mplstyle.\n",
      "You probably need to get an updated matplotlibrc file from\n",
      "http://github.com/matplotlib/matplotlib/blob/master/matplotlibrc.template\n",
      "or from the matplotlib source distribution\n",
      "\n",
      "Bad key \"boxplot.meanprops.marker\" on line 360 in\n",
      "C:\\Users\\Juan\\Anaconda3\\envs\\semcog_py2\\lib\\site-packages\\matplotlib\\mpl-data\\stylelib\\_classic_test.mplstyle.\n",
      "You probably need to get an updated matplotlibrc file from\n",
      "http://github.com/matplotlib/matplotlib/blob/master/matplotlibrc.template\n",
      "or from the matplotlib source distribution\n",
      "\n",
      "Bad key \"date.autoformatter.minute\" on line 227 in\n",
      "C:\\Users\\Juan\\Anaconda3\\envs\\semcog_py2\\lib\\site-packages\\matplotlib\\mpl-data\\stylelib\\_classic_test.mplstyle.\n",
      "You probably need to get an updated matplotlibrc file from\n",
      "http://github.com/matplotlib/matplotlib/blob/master/matplotlibrc.template\n",
      "or from the matplotlib source distribution\n",
      "\n",
      "Bad key \"xtick.major.top\" on line 246 in\n",
      "C:\\Users\\Juan\\Anaconda3\\envs\\semcog_py2\\lib\\site-packages\\matplotlib\\mpl-data\\stylelib\\_classic_test.mplstyle.\n",
      "You probably need to get an updated matplotlibrc file from\n",
      "http://github.com/matplotlib/matplotlib/blob/master/matplotlibrc.template\n",
      "or from the matplotlib source distribution\n",
      "\n",
      "Bad key \"date.autoformatter.day\" on line 225 in\n",
      "C:\\Users\\Juan\\Anaconda3\\envs\\semcog_py2\\lib\\site-packages\\matplotlib\\mpl-data\\stylelib\\_classic_test.mplstyle.\n",
      "You probably need to get an updated matplotlibrc file from\n",
      "http://github.com/matplotlib/matplotlib/blob/master/matplotlibrc.template\n",
      "or from the matplotlib source distribution\n",
      "\n",
      "Bad key \"ytick.right\" on line 252 in\n",
      "C:\\Users\\Juan\\Anaconda3\\envs\\semcog_py2\\lib\\site-packages\\matplotlib\\mpl-data\\stylelib\\_classic_test.mplstyle.\n",
      "You probably need to get an updated matplotlibrc file from\n",
      "http://github.com/matplotlib/matplotlib/blob/master/matplotlibrc.template\n",
      "or from the matplotlib source distribution\n",
      "\n",
      "Bad key \"xtick.bottom\" on line 235 in\n",
      "C:\\Users\\Juan\\Anaconda3\\envs\\semcog_py2\\lib\\site-packages\\matplotlib\\mpl-data\\stylelib\\_classic_test.mplstyle.\n",
      "You probably need to get an updated matplotlibrc file from\n",
      "http://github.com/matplotlib/matplotlib/blob/master/matplotlibrc.template\n",
      "or from the matplotlib source distribution\n",
      "\n",
      "Bad key \"date.autoformatter.microsecond\" on line 229 in\n",
      "C:\\Users\\Juan\\Anaconda3\\envs\\semcog_py2\\lib\\site-packages\\matplotlib\\mpl-data\\stylelib\\_classic_test.mplstyle.\n",
      "You probably need to get an updated matplotlibrc file from\n",
      "http://github.com/matplotlib/matplotlib/blob/master/matplotlibrc.template\n",
      "or from the matplotlib source distribution\n",
      "\n",
      "Bad key \"hist.bins\" on line 40 in\n",
      "C:\\Users\\Juan\\Anaconda3\\envs\\semcog_py2\\lib\\site-packages\\matplotlib\\mpl-data\\stylelib\\_classic_test.mplstyle.\n",
      "You probably need to get an updated matplotlibrc file from\n",
      "http://github.com/matplotlib/matplotlib/blob/master/matplotlibrc.template\n",
      "or from the matplotlib source distribution\n",
      "\n",
      "Bad key \"lines.scale_dashes\" on line 21 in\n",
      "C:\\Users\\Juan\\Anaconda3\\envs\\semcog_py2\\lib\\site-packages\\matplotlib\\mpl-data\\stylelib\\_classic_test.mplstyle.\n",
      "You probably need to get an updated matplotlibrc file from\n",
      "http://github.com/matplotlib/matplotlib/blob/master/matplotlibrc.template\n",
      "or from the matplotlib source distribution\n",
      "\n",
      "Bad key \"axes.titlepad\" on line 177 in\n",
      "C:\\Users\\Juan\\Anaconda3\\envs\\semcog_py2\\lib\\site-packages\\matplotlib\\mpl-data\\stylelib\\_classic_test.mplstyle.\n",
      "You probably need to get an updated matplotlibrc file from\n",
      "http://github.com/matplotlib/matplotlib/blob/master/matplotlibrc.template\n",
      "or from the matplotlib source distribution\n",
      "\n",
      "Bad key \"_internal.classic_mode\" on line 513 in\n",
      "C:\\Users\\Juan\\Anaconda3\\envs\\semcog_py2\\lib\\site-packages\\matplotlib\\mpl-data\\stylelib\\_classic_test.mplstyle.\n",
      "You probably need to get an updated matplotlibrc file from\n",
      "http://github.com/matplotlib/matplotlib/blob/master/matplotlibrc.template\n",
      "or from the matplotlib source distribution\n",
      "\n",
      "Bad key \"date.autoformatter.hour\" on line 226 in\n",
      "C:\\Users\\Juan\\Anaconda3\\envs\\semcog_py2\\lib\\site-packages\\matplotlib\\mpl-data\\stylelib\\_classic_test.mplstyle.\n",
      "You probably need to get an updated matplotlibrc file from\n",
      "http://github.com/matplotlib/matplotlib/blob/master/matplotlibrc.template\n",
      "or from the matplotlib source distribution\n",
      "\n",
      "Bad key \"boxplot.meanprops.markerfacecolor\" on line 361 in\n",
      "C:\\Users\\Juan\\Anaconda3\\envs\\semcog_py2\\lib\\site-packages\\matplotlib\\mpl-data\\stylelib\\_classic_test.mplstyle.\n",
      "You probably need to get an updated matplotlibrc file from\n",
      "http://github.com/matplotlib/matplotlib/blob/master/matplotlibrc.template\n",
      "or from the matplotlib source distribution\n",
      "\n",
      "Bad key \"date.autoformatter.year\" on line 223 in\n",
      "C:\\Users\\Juan\\Anaconda3\\envs\\semcog_py2\\lib\\site-packages\\matplotlib\\mpl-data\\stylelib\\_classic_test.mplstyle.\n",
      "You probably need to get an updated matplotlibrc file from\n",
      "http://github.com/matplotlib/matplotlib/blob/master/matplotlibrc.template\n",
      "or from the matplotlib source distribution\n",
      "C:\\Users\\Juan\\Anaconda3\\envs\\semcog_py2\\lib\\site-packages\\matplotlib\\__init__.py:1069: UserWarning: Bad val \"auto\" on line #353\n",
      "\t\"boxplot.flierprops.markerfacecolor: auto\n",
      "\"\n",
      "\tin file \"C:\\Users\\Juan\\Anaconda3\\envs\\semcog_py2\\lib\\site-packages\\matplotlib\\mpl-data\\stylelib\\_classic_test.mplstyle\"\n",
      "\tKey boxplot.flierprops.markerfacecolor: auto does not look like a color arg\n",
      "  (val, error_details, msg))\n",
      "\n",
      "Bad key \"lines.dotted_pattern\" on line 20 in\n",
      "C:\\Users\\Juan\\Anaconda3\\envs\\semcog_py2\\lib\\site-packages\\matplotlib\\mpl-data\\stylelib\\_classic_test.mplstyle.\n",
      "You probably need to get an updated matplotlibrc file from\n",
      "http://github.com/matplotlib/matplotlib/blob/master/matplotlibrc.template\n",
      "or from the matplotlib source distribution\n",
      "\n",
      "Bad key \"axes.formatter.offset_threshold\" on line 200 in\n",
      "C:\\Users\\Juan\\Anaconda3\\envs\\semcog_py2\\lib\\site-packages\\matplotlib\\mpl-data\\stylelib\\_classic_test.mplstyle.\n",
      "You probably need to get an updated matplotlibrc file from\n",
      "http://github.com/matplotlib/matplotlib/blob/master/matplotlibrc.template\n",
      "or from the matplotlib source distribution\n",
      "\n",
      "Bad key \"xtick.minor.top\" on line 248 in\n",
      "C:\\Users\\Juan\\Anaconda3\\envs\\semcog_py2\\lib\\site-packages\\matplotlib\\mpl-data\\stylelib\\_classic_test.mplstyle.\n",
      "You probably need to get an updated matplotlibrc file from\n",
      "http://github.com/matplotlib/matplotlib/blob/master/matplotlibrc.template\n",
      "or from the matplotlib source distribution\n",
      "\n",
      "Bad key \"ytick.minor.left\" on line 265 in\n",
      "C:\\Users\\Juan\\Anaconda3\\envs\\semcog_py2\\lib\\site-packages\\matplotlib\\mpl-data\\stylelib\\_classic_test.mplstyle.\n",
      "You probably need to get an updated matplotlibrc file from\n",
      "http://github.com/matplotlib/matplotlib/blob/master/matplotlibrc.template\n",
      "or from the matplotlib source distribution\n",
      "\n",
      "Bad key \"lines.dashdot_pattern\" on line 19 in\n",
      "C:\\Users\\Juan\\Anaconda3\\envs\\semcog_py2\\lib\\site-packages\\matplotlib\\mpl-data\\stylelib\\_classic_test.mplstyle.\n",
      "You probably need to get an updated matplotlibrc file from\n",
      "http://github.com/matplotlib/matplotlib/blob/master/matplotlibrc.template\n",
      "or from the matplotlib source distribution\n",
      "\n",
      "Bad key \"xtick.major.bottom\" on line 247 in\n",
      "C:\\Users\\Juan\\Anaconda3\\envs\\semcog_py2\\lib\\site-packages\\matplotlib\\mpl-data\\stylelib\\_classic_test.mplstyle.\n",
      "You probably need to get an updated matplotlibrc file from\n",
      "http://github.com/matplotlib/matplotlib/blob/master/matplotlibrc.template\n",
      "or from the matplotlib source distribution\n",
      "\n",
      "Bad key \"boxplot.meanprops.markersize\" on line 363 in\n",
      "C:\\Users\\Juan\\Anaconda3\\envs\\semcog_py2\\lib\\site-packages\\matplotlib\\mpl-data\\stylelib\\_classic_test.mplstyle.\n",
      "You probably need to get an updated matplotlibrc file from\n",
      "http://github.com/matplotlib/matplotlib/blob/master/matplotlibrc.template\n",
      "or from the matplotlib source distribution\n",
      "\n",
      "Bad key \"hatch.linewidth\" on line 38 in\n",
      "C:\\Users\\Juan\\Anaconda3\\envs\\semcog_py2\\lib\\site-packages\\matplotlib\\mpl-data\\stylelib\\_classic_test.mplstyle.\n",
      "You probably need to get an updated matplotlibrc file from\n",
      "http://github.com/matplotlib/matplotlib/blob/master/matplotlibrc.template\n",
      "or from the matplotlib source distribution\n",
      "\n",
      "Bad key \"xtick.top\" on line 234 in\n",
      "C:\\Users\\Juan\\Anaconda3\\envs\\semcog_py2\\lib\\site-packages\\matplotlib\\mpl-data\\stylelib\\_classic_test.mplstyle.\n",
      "You probably need to get an updated matplotlibrc file from\n",
      "http://github.com/matplotlib/matplotlib/blob/master/matplotlibrc.template\n",
      "or from the matplotlib source distribution\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Juan\\Anaconda3\\envs\\semcog_py2\\lib\\site-packages\\urbansim\\utils\\yamlio.py:162: YAMLLoadWarning: calling yaml.load() without Loader=... is deprecated, as the default Loader is unsafe. Please read https://msg.pyyaml.org/load for full details.\n",
      "  d = yaml.load(f)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running step 'refiner'\n",
      "Time to execute step 'refiner': 17.56 s\n",
      "Total time to execute iteration 1 with iteration value None: 17.56 s\n",
      "Running step 'build_networks'\n",
      "Time to execute step 'build_networks': 110.36 s\n",
      "Total time to execute iteration 1 with iteration value None: 110.38 s\n",
      "Running step 'neighborhood_vars'\n",
      "Computing accessibility variables\n",
      "Computing log_sum_residential_units\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Juan\\Anaconda3\\envs\\semcog_py2\\lib\\site-packages\\urbansim\\utils\\networks.py:18: YAMLLoadWarning: calling yaml.load() without Loader=... is deprecated, as the default Loader is unsafe. Please read https://msg.pyyaml.org/load for full details.\n",
      "  cfg = yaml.load(open(misc.config(cfgname)))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Removed 8 rows because they contain missing values\n",
      "Computing ave_lot_sqft\n",
      "Removed 8 rows because they contain missing values\n",
      "Computing log_sum_nonresidential_units\n",
      "Removed 8 rows because they contain missing values\n",
      "Computing population\n",
      "Computing households\n",
      "Computing midinc_hhs\n",
      "Computing lowinc_hhs\n",
      "Computing highinc_hhs\n",
      "Computing race_1_hhs\n",
      "Computing race_2_hhs\n",
      "Computing race_3_hhs\n",
      "Computing race_4_hhs\n",
      "Computing hhs_with_children\n",
      "Computing hhsize\n",
      "Computing ave_income\n",
      "Computing max_industrial_far\n",
      "Computing max_office_far\n",
      "Computing max_retail_far\n",
      "Computing max_medical_far\n",
      "Computing jobs\n",
      "Computing retail_jobs\n",
      "Computing node_r1500_sqft\n",
      "Computing residential\n",
      "Computing ave_nonres_sqft_price\n",
      "Removed 1 rows because they contain missing values\n",
      "Computing retail\n",
      "Computing office\n",
      "Removed 1 rows because they contain missing values\n",
      "Computing industrial\n",
      "Computing medical\n",
      "Computing quarter_mile_to_transit\n",
      "Computing log_15min_retail_jobs\n",
      "Computing log_15min_shopping_jobs\n",
      "Computing log_30min_retail_jobs\n",
      "Computing log_30min_shopping_jobs\n",
      "Computing accessibility variables\n",
      "Computing log_sum_30min_jobs\n",
      "Computing log_sum_45min_jobs\n",
      "Computing log_sum_60min_jobs\n",
      "Computing log_sum_10min_pop\n",
      "Computing log_sum_20min_pop\n",
      "Computing log_sum_15min_retail_jobs\n",
      "Computing log_sum_15min_shopping_jobs\n",
      "Computing log_sum_30min_retail_jobs\n",
      "Computing log_sum_30min_shopping_jobs\n",
      "Computing elem_school_perf\n",
      "Time to execute step 'neighborhood_vars': 1958.03 s\n",
      "Total time to execute iteration 1 with iteration value None: 1958.03 s\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "import orca\n",
    "from urbansim.utils import misc\n",
    "\n",
    "import dataset\n",
    "import variables\n",
    "import models\n",
    "import utils\n",
    "orca.run(['refiner'])\n",
    "orca.run([\"build_networks\"])\n",
    "orca.run([\"neighborhood_vars\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def estimate_repm(yaml_config):\n",
    "    model = RegressionModel.from_yaml(str_or_buffer=misc.config(yaml_config))\n",
    "\n",
    "    b = orca.get_table('buildings').to_frame(model.columns_used()).fillna(0)\n",
    "\n",
    "    print model.fit(b)\n",
    "    \n",
    "    model.to_yaml(str_or_buffer=misc.config(yaml_config)) #  .replace('.yaml', '_new.yaml')\n",
    "\n",
    "    return model.fit_parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(0, 'repm/nonres_repm11521.yaml'),\n",
       " (1, 'repm/nonres_repm11522.yaml'),\n",
       " (2, 'repm/nonres_repm11523.yaml'),\n",
       " (3, 'repm/nonres_repm11525.yaml'),\n",
       " (4, 'repm/nonres_repm11526.yaml'),\n",
       " (5, 'repm/nonres_repm11531.yaml'),\n",
       " (6, 'repm/nonres_repm11533.yaml'),\n",
       " (7, 'repm/nonres_repm11541.yaml'),\n",
       " (8, 'repm/nonres_repm11551.yaml'),\n",
       " (9, 'repm/nonres_repm11571.yaml'),\n",
       " (10, 'repm/nonres_repm12521.yaml'),\n",
       " (11, 'repm/nonres_repm12522.yaml'),\n",
       " (12, 'repm/nonres_repm12523.yaml'),\n",
       " (13, 'repm/nonres_repm12525.yaml'),\n",
       " (14, 'repm/nonres_repm12526.yaml'),\n",
       " (15, 'repm/nonres_repm12531.yaml'),\n",
       " (16, 'repm/nonres_repm12533.yaml'),\n",
       " (17, 'repm/nonres_repm12541.yaml'),\n",
       " (18, 'repm/nonres_repm12551.yaml'),\n",
       " (19, 'repm/nonres_repm12571.yaml'),\n",
       " (20, 'repm/nonres_repm14721.yaml'),\n",
       " (21, 'repm/nonres_repm14722.yaml'),\n",
       " (22, 'repm/nonres_repm14723.yaml'),\n",
       " (23, 'repm/nonres_repm14725.yaml'),\n",
       " (24, 'repm/nonres_repm14726.yaml'),\n",
       " (25, 'repm/nonres_repm14731.yaml'),\n",
       " (26, 'repm/nonres_repm14733.yaml'),\n",
       " (27, 'repm/nonres_repm14741.yaml'),\n",
       " (28, 'repm/nonres_repm14751.yaml'),\n",
       " (29, 'repm/nonres_repm14771.yaml'),\n",
       " (30, 'repm/nonres_repm16121.yaml'),\n",
       " (31, 'repm/nonres_repm16122.yaml'),\n",
       " (32, 'repm/nonres_repm16123.yaml'),\n",
       " (33, 'repm/nonres_repm16125.yaml'),\n",
       " (34, 'repm/nonres_repm16126.yaml'),\n",
       " (35, 'repm/nonres_repm16131.yaml'),\n",
       " (36, 'repm/nonres_repm16133.yaml'),\n",
       " (37, 'repm/nonres_repm16141.yaml'),\n",
       " (38, 'repm/nonres_repm16151.yaml'),\n",
       " (39, 'repm/nonres_repm16171.yaml'),\n",
       " (40, 'repm/nonres_repm24.yaml'),\n",
       " (41, 'repm/nonres_repm32.yaml'),\n",
       " (42, 'repm/nonres_repm321.yaml'),\n",
       " (43, 'repm/nonres_repm322.yaml'),\n",
       " (44, 'repm/nonres_repm323.yaml'),\n",
       " (45, 'repm/nonres_repm325.yaml'),\n",
       " (46, 'repm/nonres_repm326.yaml'),\n",
       " (47, 'repm/nonres_repm331.yaml'),\n",
       " (48, 'repm/nonres_repm333.yaml'),\n",
       " (49, 'repm/nonres_repm341.yaml'),\n",
       " (50, 'repm/nonres_repm351.yaml'),\n",
       " (51, 'repm/nonres_repm371.yaml'),\n",
       " (52, 'repm/nonres_repm42.yaml'),\n",
       " (53, 'repm/nonres_repm43.yaml'),\n",
       " (54, 'repm/nonres_repm52.yaml'),\n",
       " (55, 'repm/nonres_repm521.yaml'),\n",
       " (56, 'repm/nonres_repm522.yaml'),\n",
       " (57, 'repm/nonres_repm523.yaml'),\n",
       " (58, 'repm/nonres_repm525.yaml'),\n",
       " (59, 'repm/nonres_repm526.yaml'),\n",
       " (60, 'repm/nonres_repm53.yaml'),\n",
       " (61, 'repm/nonres_repm531.yaml'),\n",
       " (62, 'repm/nonres_repm533.yaml'),\n",
       " (63, 'repm/nonres_repm541.yaml'),\n",
       " (64, 'repm/nonres_repm551.yaml'),\n",
       " (65, 'repm/nonres_repm61.yaml'),\n",
       " (66, 'repm/nonres_repm62.yaml'),\n",
       " (67, 'repm/nonres_repm9321.yaml'),\n",
       " (68, 'repm/nonres_repm9322.yaml'),\n",
       " (69, 'repm/nonres_repm9323.yaml'),\n",
       " (70, 'repm/nonres_repm9325.yaml'),\n",
       " (71, 'repm/nonres_repm9326.yaml'),\n",
       " (72, 'repm/nonres_repm9331.yaml'),\n",
       " (73, 'repm/nonres_repm9333.yaml'),\n",
       " (74, 'repm/nonres_repm9341.yaml'),\n",
       " (75, 'repm/nonres_repm9351.yaml'),\n",
       " (76, 'repm/nonres_repm9371.yaml'),\n",
       " (77, 'repm/nonres_repm9921.yaml'),\n",
       " (78, 'repm/nonres_repm9922.yaml'),\n",
       " (79, 'repm/nonres_repm9923.yaml'),\n",
       " (80, 'repm/nonres_repm9925.yaml'),\n",
       " (81, 'repm/nonres_repm9926.yaml'),\n",
       " (82, 'repm/nonres_repm9931.yaml'),\n",
       " (83, 'repm/nonres_repm9933.yaml'),\n",
       " (84, 'repm/nonres_repm9941.yaml'),\n",
       " (85, 'repm/nonres_repm9951.yaml'),\n",
       " (86, 'repm/nonres_repm9971.yaml'),\n",
       " (87, 'repm/res_repm11581.yaml'),\n",
       " (88, 'repm/res_repm11582.yaml'),\n",
       " (89, 'repm/res_repm11583.yaml'),\n",
       " (90, 'repm/res_repm11584.yaml'),\n",
       " (91, 'repm/res_repm12581.yaml'),\n",
       " (92, 'repm/res_repm12582.yaml'),\n",
       " (93, 'repm/res_repm12583.yaml'),\n",
       " (94, 'repm/res_repm12584.yaml'),\n",
       " (95, 'repm/res_repm14781.yaml'),\n",
       " (96, 'repm/res_repm14782.yaml'),\n",
       " (97, 'repm/res_repm14783.yaml'),\n",
       " (98, 'repm/res_repm14784.yaml'),\n",
       " (99, 'repm/res_repm16181.yaml'),\n",
       " (100, 'repm/res_repm16182.yaml'),\n",
       " (101, 'repm/res_repm16183.yaml'),\n",
       " (102, 'repm/res_repm16184.yaml'),\n",
       " (103, 'repm/res_repm381.yaml'),\n",
       " (104, 'repm/res_repm382.yaml'),\n",
       " (105, 'repm/res_repm383.yaml'),\n",
       " (106, 'repm/res_repm384.yaml'),\n",
       " (107, 'repm/res_repm581.yaml'),\n",
       " (108, 'repm/res_repm582.yaml'),\n",
       " (109, 'repm/res_repm583.yaml'),\n",
       " (110, 'repm/res_repm9381.yaml'),\n",
       " (111, 'repm/res_repm9382.yaml'),\n",
       " (112, 'repm/res_repm9383.yaml'),\n",
       " (113, 'repm/res_repm9384.yaml'),\n",
       " (114, 'repm/res_repm9981.yaml'),\n",
       " (115, 'repm/res_repm9982.yaml'),\n",
       " (116, 'repm/res_repm9983.yaml'),\n",
       " (117, 'repm/res_repm9984.yaml')]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "repm_configs=[]\n",
    "path = './configs/repm/'\n",
    "for filename in os.listdir(path):\n",
    "    if not filename.endswith('.yaml'): continue\n",
    "    repm_configs.append(os.path.join(path, filename).replace('./configs/', ''))\n",
    "repm_configs.sort()\n",
    "list(enumerate(repm_configs))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from urbansim.models.regression import RegressionModel"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# cols = ['zones_logsum_job_high_income',\n",
    "#    'zones_logsum_job_low_income',\n",
    "#    'b_is_newerthan2010',\n",
    "#    'zones_empden',\n",
    "#    'walk_nearest_library',\n",
    "#    'crime_ucr_rate',\n",
    "#    'walk_nearest_hospital',\n",
    "#    'nodes_walk_log_sum_residential_units',\n",
    "#    'nodes_walk_race_2_hhs',\n",
    "#    'nodes_walk_ave_lot_sqft',\n",
    "#    'stories',\n",
    "#    'drv_nearest_park',\n",
    "#    'nodes_drv_log_sum_20min_pop',\n",
    "#    'nodes_walk_ave_unit_sqft',\n",
    "#    'zones_a_ln_retail_emp_15min_drive_alone']\n",
    "# cols1 = ['nodes_walk_ave_unit_sqft']\n",
    "# orca.get_table('buildings').to_frame(cols1).fillna(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# b = orca.get_table('buildings').to_frame(['sqft_per_unit', 'x', 'y'])\n",
    "# b = b[(b.sqft_per_unit > 0) & (b.x.notnull())]\n",
    "# net = orca.get_injectable('net_walk')\n",
    "# net.set_pois('build', 15000, 5, b.x, b.y)\n",
    "# ner = net.nearest_pois(10560, 'build', num_pois=5, include_poi_ids=True)\n",
    "# st = ner[ner.columns[ner.columns.str.startswith('poi') > 0]].unstack()\n",
    "# st = st[st > 0].astype(int)\n",
    "# lu = misc.reindex(orca.get_table('buildings').sqft_per_unit, st)\n",
    "# lu = lu[lu > 500]\n",
    "# print(lu.groupby(level=1).mean().reindex(net.index).fillna(2000))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Disaggregating logsum_pop_high_income to buildings from zones\n"
     ]
    },
    {
     "ename": "MemoryError",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mMemoryError\u001b[0m                               Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-15-601633635d3b>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[1;31m# for i in range(88, 117):\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      2\u001b[0m \u001b[1;31m#     print(repm_configs[i])\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 3\u001b[1;33m \u001b[0mestimate_repm\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mrepm_configs\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m88\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32m<ipython-input-2-ba9bdafc3afe>\u001b[0m in \u001b[0;36mestimate_repm\u001b[1;34m(yaml_config)\u001b[0m\n\u001b[0;32m      2\u001b[0m     \u001b[0mmodel\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mRegressionModel\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfrom_yaml\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mstr_or_buffer\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mmisc\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mconfig\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0myaml_config\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      3\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 4\u001b[1;33m     \u001b[0mb\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0morca\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget_table\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'buildings'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mto_frame\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcolumns_used\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfillna\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      5\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      6\u001b[0m     \u001b[1;32mprint\u001b[0m \u001b[0mmodel\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mb\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Users\\Juan\\Anaconda3\\envs\\semcog_py2\\lib\\site-packages\\orca\\orca.pyc\u001b[0m in \u001b[0;36mto_frame\u001b[1;34m(self, columns)\u001b[0m\n\u001b[0;32m    233\u001b[0m                             name, self.name),\n\u001b[0;32m    234\u001b[0m                         logger):\n\u001b[1;32m--> 235\u001b[1;33m                     \u001b[0mdf\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mname\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mcol\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    236\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    237\u001b[0m         \u001b[1;32mreturn\u001b[0m \u001b[0mdf\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Users\\Juan\\Anaconda3\\envs\\semcog_py2\\lib\\site-packages\\orca\\orca.pyc\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m    637\u001b[0m             kwargs = _collect_variables(names=self._argspec.args,\n\u001b[0;32m    638\u001b[0m                                         expressions=self._argspec.defaults)\n\u001b[1;32m--> 639\u001b[1;33m             \u001b[0mcol\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_func\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    640\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    641\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcache\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Users\\Juan\\Documents\\GitHub\\semcog_urbansim\\semcog_urbansim\\variables\\variables_building.pyc\u001b[0m in \u001b[0;36mfunc\u001b[1;34m()\u001b[0m\n\u001b[0;32m    380\u001b[0m         \u001b[0mfrom_geog\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0morca\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget_table\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfrom_geog_name\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    381\u001b[0m         \u001b[0mto_geog\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0morca\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget_table\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mto_geog_name\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 382\u001b[1;33m         \u001b[1;32mreturn\u001b[0m \u001b[0mmisc\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mreindex\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfrom_geog\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mvar_to_disaggregate\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mto_geog\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mfrom_geog_id_name\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfillna\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    383\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    384\u001b[0m     \u001b[1;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Users\\Juan\\Anaconda3\\envs\\semcog_py2\\lib\\site-packages\\orca\\orca.pyc\u001b[0m in \u001b[0;36m__getitem__\u001b[1;34m(self, key)\u001b[0m\n\u001b[0;32m    288\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    289\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0m__getitem__\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mkey\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 290\u001b[1;33m         \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget_column\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    291\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    292\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0m__getattr__\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mkey\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Users\\Juan\\Anaconda3\\envs\\semcog_py2\\lib\\site-packages\\orca\\orca.pyc\u001b[0m in \u001b[0;36mget_column\u001b[1;34m(self, column_name)\u001b[0m\n\u001b[0;32m    279\u001b[0m                             column_name, self.name),\n\u001b[0;32m    280\u001b[0m                         logger):\n\u001b[1;32m--> 281\u001b[1;33m                     \u001b[0mcolumn\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mextra_cols\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mcolumn_name\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    282\u001b[0m             \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    283\u001b[0m                 \u001b[0mcolumn\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mlocal\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mcolumn_name\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Users\\Juan\\Anaconda3\\envs\\semcog_py2\\lib\\site-packages\\orca\\orca.pyc\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m    637\u001b[0m             kwargs = _collect_variables(names=self._argspec.args,\n\u001b[0;32m    638\u001b[0m                                         expressions=self._argspec.defaults)\n\u001b[1;32m--> 639\u001b[1;33m             \u001b[0mcol\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_func\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    640\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    641\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcache\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Users\\Juan\\Documents\\GitHub\\semcog_urbansim\\semcog_urbansim\\variables\\variables_zone.pyc\u001b[0m in \u001b[0;36mlogsum_pop_high_income\u001b[1;34m(zones, travel_data)\u001b[0m\n\u001b[0;32m     97\u001b[0m     \u001b[0mname_attribute\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;34m'am_work_highinc_logsum'\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     98\u001b[0m     \u001b[0mspatial_var\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;34m'population'\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 99\u001b[1;33m     \u001b[1;32mreturn\u001b[0m \u001b[0mlogsum_based_accessibility\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtravel_data\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mzones\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mname_attribute\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mspatial_var\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    100\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    101\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Users\\Juan\\Documents\\GitHub\\semcog_urbansim\\semcog_urbansim\\variables\\variables_zone.pyc\u001b[0m in \u001b[0;36mlogsum_based_accessibility\u001b[1;34m(travel_data, zones, name_attribute, spatial_var)\u001b[0m\n\u001b[0;32m     61\u001b[0m     \u001b[0mzones\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mzones\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mto_frame\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'population'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m'employment'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     62\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 63\u001b[1;33m     \u001b[0mtd\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtd\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mreset_index\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     64\u001b[0m     \u001b[0mzones\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mzones\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mreset_index\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     65\u001b[0m     \u001b[0munique_zone_ids\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0munique\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mzones\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mzone_id\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Users\\Juan\\Anaconda3\\envs\\semcog_py2\\lib\\site-packages\\pandas\\core\\frame.pyc\u001b[0m in \u001b[0;36mreset_index\u001b[1;34m(self, level, drop, inplace, col_level, col_fill)\u001b[0m\n\u001b[0;32m   4071\u001b[0m             \u001b[0mnew_obj\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   4072\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 4073\u001b[1;33m             \u001b[0mnew_obj\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcopy\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   4074\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   4075\u001b[0m         \u001b[1;32mdef\u001b[0m \u001b[0m_maybe_casted_values\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mindex\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlabels\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mNone\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Users\\Juan\\Anaconda3\\envs\\semcog_py2\\lib\\site-packages\\pandas\\core\\generic.pyc\u001b[0m in \u001b[0;36mcopy\u001b[1;34m(self, deep)\u001b[0m\n\u001b[0;32m   5112\u001b[0m         \u001b[0mdtype\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0mobject\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   5113\u001b[0m         \"\"\"\n\u001b[1;32m-> 5114\u001b[1;33m         \u001b[0mdata\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_data\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcopy\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdeep\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mdeep\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   5115\u001b[0m         \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_constructor\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m__finalize__\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   5116\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Users\\Juan\\Anaconda3\\envs\\semcog_py2\\lib\\site-packages\\pandas\\core\\internals.pyc\u001b[0m in \u001b[0;36mcopy\u001b[1;34m(self, deep, mgr)\u001b[0m\n\u001b[0;32m   3918\u001b[0m             \u001b[0mnew_axes\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mlist\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0maxes\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   3919\u001b[0m         return self.apply('copy', axes=new_axes, deep=deep,\n\u001b[1;32m-> 3920\u001b[1;33m                           do_integrity_check=False)\n\u001b[0m\u001b[0;32m   3921\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   3922\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mas_array\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtranspose\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mFalse\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mitems\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mNone\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Users\\Juan\\Anaconda3\\envs\\semcog_py2\\lib\\site-packages\\pandas\\core\\internals.pyc\u001b[0m in \u001b[0;36mapply\u001b[1;34m(self, f, axes, filter, do_integrity_check, consolidate, **kwargs)\u001b[0m\n\u001b[0;32m   3579\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   3580\u001b[0m             \u001b[0mkwargs\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'mgr'\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 3581\u001b[1;33m             \u001b[0mapplied\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mgetattr\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mb\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mf\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   3582\u001b[0m             \u001b[0mresult_blocks\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0m_extend_blocks\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mapplied\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mresult_blocks\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   3583\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Users\\Juan\\Anaconda3\\envs\\semcog_py2\\lib\\site-packages\\pandas\\core\\internals.pyc\u001b[0m in \u001b[0;36mcopy\u001b[1;34m(self, deep, mgr)\u001b[0m\n\u001b[0;32m    776\u001b[0m         \u001b[0mvalues\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    777\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mdeep\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 778\u001b[1;33m             \u001b[0mvalues\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mvalues\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcopy\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    779\u001b[0m         \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmake_block_same_class\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mvalues\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    780\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mMemoryError\u001b[0m: "
     ]
    }
   ],
   "source": [
    "# for i in range(88, 117):\n",
    "#     print(repm_configs[i])\n",
    "estimate_repm(repm_configs[88])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Simulation Example"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Simple transition model step\n",
    "@orca.step('simple_jobs_transition')\n",
    "def simple_transition_model(jobs):\n",
    "    return utils.simple_transition(jobs, .005, 'building_id')\n",
    "\n",
    "# Create the choice model object for sector_id 1 and register model as inectable\n",
    "elcm1_model = utils.SimulationChoiceModel.from_yaml(str_or_buffer=misc.config('elcm/categories/elcm1.yaml'))\n",
    "elcm1_model.set_simulation_params('elcm1', 'job_spaces','vacant_job_spaces', 'jobs', 'buildings')\n",
    "orca.add_injectable('elcm1_model', elcm1_model)\n",
    "\n",
    "# ELCM model step for sector 1\n",
    "@orca.step('elcm1')\n",
    "def choice_model_simulate(elcm1_model, jobs):\n",
    "    choices = elcm1_model.simulate(choice_function=utils.unit_choices)\n",
    "\n",
    "    print 'There are %s unplaced agents.' % choices.isnull().sum()\n",
    "\n",
    "    jobs.update_col_from_series(elcm1_model.choice_column, choices, cast=True)\n",
    "\n",
    "    \n",
    "# Simulate\n",
    "orca.run(['simple_jobs_transition', 'elcm1'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Scoring Example"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "lids = orca.get_table('buildings').large_area_id.fillna(0).astype('int')\n",
    "\n",
    "model = utils.SimulationChoiceModel.from_yaml(str_or_buffer=misc.config('hlcm/hlcm1.yaml'))\n",
    "model.set_simulation_params('hlcm1', 'residential_units',\n",
    "                            'vacant_residential_units', 'households', 'buildings', lids)\n",
    "\n",
    "print '** Score is %s' % model.score()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Plotting probabilities example"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from bokeh.plotting import figure, output_notebook\n",
    "output_notebook()\n",
    "\n",
    "import datashader as ds\n",
    "from datashader import transfer_functions as tf\n",
    "from datashader.bokeh_ext import InteractiveImage\n",
    "from functools import partial\n",
    "from datashader.utils import export_image\n",
    "from datashader.colors import colormap_select\n",
    "\n",
    "def plot_by_high_low_proba(df):\n",
    "    df['ones'] = 1\n",
    "    region = x_range, y_range = ((df.x.min(),df.x.max()), (df.y.min(),df.y.max()))\n",
    "\n",
    "    plot_width  = int(750)\n",
    "    plot_height = int(plot_width//1.2)\n",
    "\n",
    "    def base_plot(tools='pan,wheel_zoom,reset',plot_width=plot_width, plot_height=plot_height, **plot_args):\n",
    "        p = figure(tools=tools, plot_width=plot_width, plot_height=plot_height,\n",
    "            x_range=x_range, y_range=y_range, outline_line_color=None,\n",
    "            min_border=0, min_border_left=0, min_border_right=0,\n",
    "            min_border_top=0, min_border_bottom=0, **plot_args)\n",
    "\n",
    "        p.axis.visible = False\n",
    "        p.xgrid.grid_line_color = None\n",
    "        p.ygrid.grid_line_color = None\n",
    "        return p\n",
    "\n",
    "    options = dict(line_color=None, fill_color='blue', size=5)\n",
    "\n",
    "    background = \"black\"\n",
    "    export = partial(export_image, export_path=\"export\", background=background)\n",
    "    cm = partial(colormap_select, reverse=(background==\"black\"))\n",
    "\n",
    "    def create_image(x_range, y_range, w=plot_width, h=plot_height):\n",
    "        cvs = ds.Canvas(plot_width=w, plot_height=h, x_range=x_range, y_range=y_range)\n",
    "        color_key = color_key = {'low':'aqua', 'high':'red'}\n",
    "\n",
    "        agg = cvs.points(df, 'x', 'y',  ds.count_cat('proba_class'))\n",
    "\n",
    "        img = tf.shade(agg, color_key=color_key, how='eq_hist')\n",
    "        return tf.dynspread(img, threshold=0.5, max_px=4)\n",
    "\n",
    "    p = base_plot(background_fill_color=background)\n",
    "\n",
    "    export(create_image(*region), 'probas')\n",
    "    return InteractiveImage(p, create_image)\n",
    "\n",
    "\n",
    "def elcm_probabilities(sector_id):\n",
    "    model_name = 'elcm%s' % sector_id\n",
    "    config_filename = 'elcm/categories/elcm%s.yaml' % sector_id\n",
    "\n",
    "    model = utils.SimulationChoiceModel.from_yaml(str_or_buffer=misc.config(config_filename))\n",
    "\n",
    "    model.set_simulation_params(model_name,\n",
    "                                'job_spaces',\n",
    "                                'vacant_job_spaces',\n",
    "                                'jobs',\n",
    "                                'buildings')\n",
    "\n",
    "    choosers, alternatives = model.calculate_model_variables()\n",
    "\n",
    "    return model.calculate_probabilities(choosers, alternatives)\n",
    "\n",
    "\n",
    "def hlcm_probabilities(segment_id):\n",
    "    model_name = 'hlcm%s' % segment_id\n",
    "    config_filename = 'hlcm/hlcm%s.yaml' % segment_id\n",
    "\n",
    "    model = utils.SimulationChoiceModel.from_yaml(str_or_buffer=misc.config(config_filename))\n",
    "\n",
    "    model.set_simulation_params(model_name,\n",
    "                                'residential_units',\n",
    "                                'vacant_residential_units',\n",
    "                                'households',\n",
    "                                'buildings')\n",
    "\n",
    "    choosers, alternatives = model.calculate_model_variables()\n",
    "\n",
    "    return model.calculate_probabilities(choosers, alternatives)\n",
    "\n",
    "\n",
    "def plot_building_level_probas(probas):\n",
    "    buildings = orca.get_table('buildings')\n",
    "    b_proba = pd.DataFrame({'proba':probas, 'x':buildings.parcels_centroid_x, 'y':buildings.parcels_centroid_y})\n",
    "\n",
    "    b_proba = b_proba[(b_proba.x > 0) & (b_proba.y > 0) & (b_proba.x < 13640484) & (b_proba.y <  600000)]\n",
    "    b_proba.proba = b_proba.proba*10000\n",
    "\n",
    "    mean_proba = b_proba.proba.median()\n",
    "    b_proba['proba_class'] = ''\n",
    "    b_proba.proba_class[b_proba.proba > mean_proba] = 'high'\n",
    "    b_proba.proba_class[b_proba.proba <= mean_proba] = 'low'\n",
    "    b_proba.proba_class = b_proba.proba_class.astype('category')\n",
    "\n",
    "    return plot_by_high_low_proba(b_proba)\n",
    "\n",
    "\n",
    "def plot_elcm_probabilities(sector_id):\n",
    "    probabilities = elcm_probabilities(sector_id)\n",
    "    return plot_building_level_probas(probabilities)\n",
    "\n",
    "def plot_hlcm_probabilities(segment_id):\n",
    "    probabilities = hlcm_probabilities(segment_id)\n",
    "    return plot_building_level_probas(probabilities)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_hlcm_probabilities(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_hlcm_probabilities(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_hlcm_probabilities(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_hlcm_probabilities(4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_elcm_probabilities(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_elcm_probabilities(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_elcm_probabilities(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_elcm_probabilities(4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_elcm_probabilities(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_elcm_probabilities(6)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_elcm_probabilities(7)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_elcm_probabilities(8)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_elcm_probabilities(9)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_elcm_probabilities(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_elcm_probabilities(11)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_elcm_probabilities(12)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_elcm_probabilities(13)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_elcm_probabilities(14)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_elcm_probabilities(15)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_elcm_probabilities(16)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_elcm_probabilities(17)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_elcm_probabilities(18)"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "semcog(py2)",
   "language": "python",
   "name": "semcog_py2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.15"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}