feature_extraction.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# スマートホームにおけるセンサデータを使った日常行動認識実験"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from progressbar import ProgressBar"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### データの読み込み\n",
    "\n",
    "データは\n",
    "http://ailab.wsu.edu/casas/datasets/\n",
    "からダウンロード\n",
    "\n",
    "今回は，このリストの17番 (http://ailab.wsu.edu/casas/datasets/aruba.zip)\n",
    "\n",
    "を落として， \"./wsu_dataset/aruba\"に展開した中身を配置しております"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>date</th>\n",
       "      <th>time</th>\n",
       "      <th>sensor_id</th>\n",
       "      <th>val</th>\n",
       "      <th>activity_name</th>\n",
       "      <th>state</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2010-11-04</td>\n",
       "      <td>00:03:50.209589</td>\n",
       "      <td>M003</td>\n",
       "      <td>ON</td>\n",
       "      <td>Sleeping</td>\n",
       "      <td>begin</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2010-11-04</td>\n",
       "      <td>00:03:57.399391</td>\n",
       "      <td>M003</td>\n",
       "      <td>OFF</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2010-11-04</td>\n",
       "      <td>00:15:08.984841</td>\n",
       "      <td>T002</td>\n",
       "      <td>21.5</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2010-11-04</td>\n",
       "      <td>00:30:19.185547</td>\n",
       "      <td>T003</td>\n",
       "      <td>21</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2010-11-04</td>\n",
       "      <td>00:30:19.385336</td>\n",
       "      <td>T004</td>\n",
       "      <td>21</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         date             time sensor_id   val activity_name  state\n",
       "0  2010-11-04  00:03:50.209589      M003    ON      Sleeping  begin\n",
       "1  2010-11-04  00:03:57.399391      M003   OFF           NaN    NaN\n",
       "2  2010-11-04  00:15:08.984841      T002  21.5           NaN    NaN\n",
       "3  2010-11-04  00:30:19.185547      T003    21           NaN    NaN\n",
       "4  2010-11-04  00:30:19.385336      T004    21           NaN    NaN"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "col_names = [ 'date','time', 'sensor_id', 'val', 'activity_name', 'state']\n",
    "\n",
    "aruba_df = pd.read_csv('./wsu_dataset/aruba/data', sep=\"\\s+\", names=col_names)\n",
    "\n",
    "aruba_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['M003', 'T002', 'T003', 'T004', 'T005', 'T001', 'M002', 'M007',\n",
       "       'M005', 'M004', 'M006', 'M008', 'M020', 'M010', 'M011', 'M012',\n",
       "       'M013', 'M014', 'M009', 'M018', 'M019', 'M015', 'M016', 'M017',\n",
       "       'M021', 'M022', 'M023', 'M001', 'M024', 'D002', 'M031', 'D004',\n",
       "       'M030', 'M029', 'M028', 'D001', 'M026', 'M027', 'M025', 'c',\n",
       "       'LEAVEHOME', 'ENTERHOME'], dtype=object)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "aruba_df.sensor_id.unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['Sleeping', nan, 'Bed_to_Toilet', 'Meal_Preparation', 'Relax',\n",
       "       'Housekeeping', 'Eating', 'Wash_Dishes', 'Leave_Home', 'Enter_Home',\n",
       "       'Work', 'Resperate'], dtype=object)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "aruba_df.activity_name.unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['Bed_to_Toilet', 'Eating', 'Enter_Home', 'Housekeeping', 'Leave_Home', 'Meal_Preparation', 'Relax', 'Resperate', 'Sleeping', 'Wash_Dishes', 'Work']\n"
     ]
    }
   ],
   "source": [
    "# acitivity_name_list = sorted(aruba_df.activity_name.unique().tolist())\n",
    "activity_name_list = sorted(aruba_df.activity_name.value_counts().index.tolist())\n",
    "print (activity_name_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 4.09 s, sys: 55 ms, total: 4.14 s\n",
      "Wall time: 4.13 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "## センサーのオン，オフの情報から各時間帯ごとにラベルを作成．\n",
    "\n",
    "labels = pd.Series(np.array([''] * aruba_df.shape[0]))\n",
    "\n",
    "for activity_id, activity_name in enumerate(activity_name_list):\n",
    "    target_idxes = aruba_df[aruba_df.activity_name == activity_name].index\n",
    "\n",
    "    target_start_idxes = target_idxes[::2]\n",
    "    target_end_idxes = target_idxes[1::2]\n",
    "\n",
    "    assert len(target_start_idxes) == len(target_end_idxes)\n",
    "    \n",
    "    for start_idx, end_idx in zip(target_start_idxes, target_end_idxes):\n",
    "        \n",
    "        labels[start_idx:end_idx+1] += activity_name + '-'\n",
    "\n",
    "aruba_df['label'] = labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "                                871275\n",
       "Bed_to_Toilet-                    1330\n",
       "Eating-                          16037\n",
       "Eating-Meal_Preparation-          2019\n",
       "Eating-Relax-                      178\n",
       "Eating-Wash_Dishes-                  4\n",
       "Enter_Home-                       2018\n",
       "Enter_Home-Housekeeping-             4\n",
       "Enter_Home-Meal_Preparation-         4\n",
       "Housekeeping-                    10579\n",
       "Housekeeping-Leave_Home-             6\n",
       "Leave_Home-                       1916\n",
       "Leave_Home-Meal_Preparation-        10\n",
       "Meal_Preparation-               283115\n",
       "Meal_Preparation-Relax-           8185\n",
       "Relax-                          346222\n",
       "Resperate-                         542\n",
       "Sleeping-                        32755\n",
       "Wash_Dishes-                     10460\n",
       "Work-                            16321\n",
       "Name: label, dtype: int64"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "activity_count_series = aruba_df.label.value_counts()\n",
    "\n",
    "activity_count_series = activity_count_series.sort_index()\n",
    "\n",
    "target_sensor_id_list = aruba_df.sensor_id.unique()\n",
    "target_sensor_id_list = [x  for x in target_sensor_id_list if (\"M0\" in x or \"D0\" in x)]\n",
    "\n",
    "aruba_df = aruba_df[aruba_df.sensor_id.isin(target_sensor_id_list)]\n",
    "\n",
    "aruba_df.label.value_counts().sort_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "OFF      798005\n",
       "ON       797981\n",
       "CLOSE      3419\n",
       "OPEN       3415\n",
       "Name: val, dtype: int64"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "label2id = {label:i for i, label in enumerate(aruba_df.label.unique().tolist())}\n",
    "\n",
    "all_sensor_id_list = aruba_df.sensor_id.unique()\n",
    "\n",
    "ok_values = [\"ON\", \"OFF\", \"OPEN\", \"CLOSE\"] # \"O5cc\"のようなノイズラベルを除く\n",
    "\n",
    "aruba_df = aruba_df[aruba_df.val.isin(ok_values)]\n",
    "\n",
    "aruba_df.val.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "aruba_df = aruba_df.applymap(lambda x: 1 if (x=='ON' or x=='OPEN') else x)\n",
    "aruba_df = aruba_df.applymap(lambda x: 0 if (x=='OFF' or x=='CLOSE') else x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>date</th>\n",
       "      <th>time</th>\n",
       "      <th>sensor_id</th>\n",
       "      <th>val</th>\n",
       "      <th>activity_name</th>\n",
       "      <th>state</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2010-11-04</td>\n",
       "      <td>00:03:50.209589</td>\n",
       "      <td>M003</td>\n",
       "      <td>1</td>\n",
       "      <td>Sleeping</td>\n",
       "      <td>begin</td>\n",
       "      <td>Sleeping-</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2010-11-04</td>\n",
       "      <td>00:03:57.399391</td>\n",
       "      <td>M003</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Sleeping-</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>2010-11-04</td>\n",
       "      <td>02:32:33.351906</td>\n",
       "      <td>M003</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Sleeping-</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>2010-11-04</td>\n",
       "      <td>02:32:38.895958</td>\n",
       "      <td>M003</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Sleeping-</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>2010-11-04</td>\n",
       "      <td>03:42:21.82365</td>\n",
       "      <td>M003</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Sleeping-</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          date             time sensor_id  val activity_name  state      label\n",
       "0   2010-11-04  00:03:50.209589      M003    1      Sleeping  begin  Sleeping-\n",
       "1   2010-11-04  00:03:57.399391      M003    0           NaN    NaN  Sleeping-\n",
       "15  2010-11-04  02:32:33.351906      M003    1           NaN    NaN  Sleeping-\n",
       "16  2010-11-04  02:32:38.895958      M003    0           NaN    NaN  Sleeping-\n",
       "20  2010-11-04   03:42:21.82365      M003    1           NaN    NaN  Sleeping-"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "aruba_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# dateカラムとtimeカラムを統合\n",
    "\n",
    "aruba_df['datetime'] = pd.to_datetime(aruba_df['date'] + \" \" +  aruba_df['time'])\n",
    "\n",
    "del aruba_df['date']\n",
    "del aruba_df['time']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(268697, 6) (54456, 6)\n"
     ]
    }
   ],
   "source": [
    "# 訓練データとテストデータの作成．\n",
    "## 訓練 6週間，　テスト 1週間\n",
    "import datetime as dt\n",
    "\n",
    "time_span = dt.timedelta(weeks=6)\n",
    "\n",
    "first_date = aruba_df.datetime[0]\n",
    "end_date = first_date = first_date + time_span\n",
    "\n",
    "train_aruba_df = aruba_df[aruba_df.datetime < end_date]\n",
    "test_aruba_df = aruba_df[(aruba_df.datetime >=end_date) & (aruba_df.datetime <= end_date+dt.timedelta(weeks=1))]\n",
    "print (train_aruba_df.shape, test_aruba_df.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def extract_last_state_sensor_feature(df, window_size=10):\n",
    "    nrow, ncol = df.shape\n",
    "    \n",
    "    res_df = pd.DataFrame(index=all_sensor_id_list)\n",
    "    labels = []\n",
    "    \n",
    "    p = ProgressBar(max_value=nrow - window_size) \n",
    "    for i in range(nrow - window_size):\n",
    "        target_df = df.iloc[i:i+window_size, :]\n",
    "        changed_sensor_id_list = target_df.sensor_id.unique()\n",
    "        sensor2val_dic = {}\n",
    "        for one_sensor_id in changed_sensor_id_list:\n",
    "            last_state = target_df[target_df.sensor_id == one_sensor_id].val.tolist()[-1]\n",
    "            sensor2val_dic[one_sensor_id] = last_state\n",
    "        labels.append(target_df.label.values[-1])\n",
    "\n",
    "        res_df[i] = pd.Series(sensor2val_dic)\n",
    "        \n",
    "        p.update(i+1)\n",
    "    res_df = res_df.T\n",
    "\n",
    "    res_df = res_df.fillna(0)\n",
    "    \n",
    "    \n",
    "    res_df['label'] = labels\n",
    "    res_df.label = res_df.label.apply(lambda x: label2id[x])\n",
    "    return res_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def extract_last_state_sensor_feature(df, window_size=10):\n",
    "    nrow, ncol = df.shape\n",
    "    \n",
    "    res_df = pd.DataFrame(index=all_sensor_id_list)\n",
    "    labels = []\n",
    "    \n",
    "    p = ProgressBar(max_value=nrow - window_size) \n",
    "    for i in range(nrow - window_size):\n",
    "        target_df = df.iloc[i:i+window_size, :]\n",
    "        changed_sensor_id_list = target_df.sensor_id.unique()\n",
    "        sensor2val_dic = {}\n",
    "        for one_sensor_id in changed_sensor_id_list:\n",
    "            last_state = target_df[target_df.sensor_id == one_sensor_id].val.tolist()[-1]\n",
    "            sensor2val_dic[one_sensor_id] = last_state\n",
    "        labels.append(target_df.label.values[-1])\n",
    "\n",
    "        res_df[i] = pd.Series(sensor2val_dic)\n",
    "        \n",
    "        p.update(i+1)\n",
    "        \n",
    "    res_df = res_df.T\n",
    "    res_df = res_df.fillna(0)\n",
    "    \n",
    "    res_df['label'] = labels\n",
    "    res_df.label = res_df.label.apply(lambda x: label2id[x])\n",
    "    return res_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def extract_baseline_feature(df, window_size=10):\n",
    "    nrow, ncol = df.shape\n",
    "    \n",
    "    start_features = ['S' + str(i) for i in range(24)]\n",
    "    end_features = ['E' + str(i) for i in range(24)]\n",
    "    res_df = pd.DataFrame(index=all_sensor_id_list.tolist() + start_features + end_features +  ['duration'])\n",
    "\n",
    "    labels = []\n",
    "    p = ProgressBar(max_value=nrow - window_size) \n",
    "    for i in range(nrow - window_size):\n",
    "        target_df = df.iloc[i:i+window_size, :] \n",
    "        label_series = target_df.groupby('sensor_id').count().label\n",
    "        start = target_df.iloc[0,:].datetime\n",
    "        end =  target_df.iloc[-1,:].datetime\n",
    "\n",
    "        def make_time_series(time_hour, prefix):\n",
    "            vec = np.zeros(24)\n",
    "            vec[time_hour] = 1\n",
    "            ser = pd.Series(vec)\n",
    "            ser.index = [prefix + str(x) for x in ser.index]\n",
    "            return ser\n",
    "\n",
    "        start_series = make_time_series(start.time().hour, 'S')\n",
    "        end_series = make_time_series(end.time().hour, 'E')\n",
    "\n",
    "        one_series = pd.concat([label_series, start_series, end_series])\n",
    "        one_series['duration'] = (end - start).total_seconds() / 60\n",
    "\n",
    "        labels.append(target_df.label.values[-1])\n",
    "        res_df[i] = one_series\n",
    "        p.update(i+1)\n",
    "        \n",
    "    res_df = res_df.T\n",
    "    res_df = res_df.fillna(0)\n",
    "    \n",
    "    res_df['label'] = labels\n",
    "    res_df.label = res_df.label.apply(lambda x: label2id[x])\n",
    "    return res_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "label2id_df = pd.DataFrame(label2id, index=[\"id\"]).T\n",
    "label2id_df.index.name = 'label'\n",
    "label2id_df.to_csv(\"label2id.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 95% (255888 of 268687) |################# | Elapsed Time: 0:51:28 ETA: 0:04:18  0% (26 of 268687) |                      | Elapsed Time: 0:00:00 ETA: 0:17:24"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "train_fet1_df = extract_baseline_feature(train_aruba_df)\n",
    "\n",
    "train_fet1_df.to_csv('train_fet1_df.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "%%time\n",
    "test_fet1_df = extract_baseline_feature(test_aruba_df)\n",
    "\n",
    "test_fet1_df.to_csv('test_fet1_df.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [conda root]",
   "language": "python",
   "name": "conda-root-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}