Instacart_Competition_Main_Script.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# My script in Instacart Market Basket Analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "import seaborn as sns\n",
    "import jnius_config\n",
    "jnius_config.set_classpath('.', 'JavaLibrary2/build/classes/')\n",
    "\n",
    "import gc\n",
    "import time\n",
    "import numpy as np # linear algebra\n",
    "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
    "import matplotlib as plt\n",
    "import timeit\n",
    "import xgboost as xgb\n",
    "from sklearn.model_selection import GroupKFold\n",
    "from sklearn.feature_selection import SelectKBest\n",
    "from sklearn.feature_selection import chi2\n",
    "from scipy.stats import linregress\n",
    "from scipy.spatial.distance import cosine\n",
    "from scipy.spatial.distance import euclidean\n",
    "\n",
    "from scipy.stats import linregress\n",
    "from sklearn.linear_model import LinearRegression\n",
    "from sklearn.metrics import f1_score\n",
    "import matplotlib.pyplot as plt\n",
    "import lightgbm as lgb\n",
    "\n",
    "sys.path.insert(0, '/src')\n",
    "\n",
    "from extra_panda_functions import *\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# This Python 3 environment comes with many helpful analytics libraries installed\n",
    "# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n",
    "# For example, here's several helpful packages to load in \n",
    "\n",
    "\n",
    "np.random.seed(98)\n",
    "\n",
    "def load_data(path_data):\n",
    "    '''\n",
    "    --------------------------------order_product--------------------------------\n",
    "    * Unique in order_id + product_id\n",
    "    '''\n",
    "    priors = pd.read_csv(path_data + 'order_products__prior.csv', \n",
    "                     dtype={\n",
    "                            'order_id': np.int32,\n",
    "                            'product_id': np.uint16,\n",
    "                            'add_to_cart_order': np.int16,\n",
    "                            'reordered': np.int8})\n",
    "    train = pd.read_csv(path_data + 'order_products__train.csv', \n",
    "                    dtype={\n",
    "                            'order_id': np.int32,\n",
    "                            'product_id': np.uint16,\n",
    "                            'add_to_cart_order': np.int16,\n",
    "                            'reordered': np.int8})\n",
    "    '''\n",
    "    --------------------------------order--------------------------------\n",
    "    * This file tells us which set (prior, train, test) an order belongs\n",
    "    * Unique in order_id\n",
    "    * order_id in train, prior, test has no intersection\n",
    "    * this is the #order_number order of this user\n",
    "    '''\n",
    "    orders = pd.read_csv(path_data + 'orders.csv', \n",
    "                         dtype={\n",
    "                                'order_id': np.int32,\n",
    "                                'user_id': np.int64,\n",
    "                                'eval_set': 'category',\n",
    "                                'order_number': np.int16,\n",
    "                                'order_dow': np.int8,\n",
    "                                'order_hour_of_day': np.int8,\n",
    "                                'days_since_prior_order': np.float32})\n",
    "\n",
    "    #  order in prior, train, test has no duplicate\n",
    "    #  order_ids_pri = priors.order_id.unique()\n",
    "    #  order_ids_trn = train.order_id.unique()\n",
    "    #  order_ids_tst = orders[orders.eval_set == 'test']['order_id'].unique()\n",
    "    #  print(set(order_ids_pri).intersection(set(order_ids_trn)))\n",
    "    #  print(set(order_ids_pri).intersection(set(order_ids_tst)))\n",
    "    #  print(set(order_ids_trn).intersection(set(order_ids_tst)))\n",
    "\n",
    "    '''\n",
    "    --------------------------------product--------------------------------\n",
    "    * Unique in product_id\n",
    "    '''\n",
    "    products = pd.read_csv(path_data + 'products.csv', \n",
    "                           dtype={\n",
    "                                'product_id': np.uint32,\n",
    "                                'order_id': np.int32,\n",
    "                                'aisle_id': np.uint8,\n",
    "                                'department_id': np.uint8},\n",
    "                                usecols=['product_id', 'aisle_id', 'department_id'])\n",
    "    aisles = pd.read_csv(path_data + \"aisles.csv\")\n",
    "    departments = pd.read_csv(path_data + \"departments.csv\")\n",
    "    sample_submission = pd.read_csv(path_data + \"sample_submission.csv\")\n",
    "    \n",
    "    return priors, train, orders, products, aisles, departments, sample_submission\n",
    "\n",
    "path_data = ''\n",
    "priors, train, orders, products, aisles, departments, sample_submission = load_data(path_data)\n",
    "\n",
    "print('priors:',priors.shape)\n",
    "print('train:',train.shape)\n",
    "print('orders:',orders.shape)\n",
    "print('products:',products.shape)\n",
    "print('aisles:',aisles.shape)\n",
    "print('deparments:',departments.shape)\n",
    "print('sample submission',sample_submission.shape)\n",
    "orders.days_since_prior_order = orders.days_since_prior_order.fillna(60)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Select part or all of the data for feature engineering"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = train.merge(right=orders[['order_id','user_id']],on='order_id')\n",
    "# LET'S FOCUS ANALYSIS ON THE FIRST 20000 USERS SHALL WE\n",
    "# JUST DELETE THIS PART IN ORDER TO GET ALL USERS\n",
    "\n",
    "priors = priors.merge(right=orders[['order_id','user_id']],on='order_id')\n",
    "\n",
    "# This is for all data\n",
    "\n",
    "priors.drop('user_id',axis=1,inplace=True)\n",
    "priors.head()\n",
    "\n",
    "# This is for performance\n",
    "\n",
    "\n",
    "orders=orders[orders.user_id>186000]\n",
    "train=train[train.user_id>186000]\n",
    "priors=priors[priors.user_id>186000]\n",
    "priors.drop('user_id',axis=1,inplace=True)\n",
    "priors.head()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Engineer product specific features :"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "priors_orders_detail = orders.merge(right=priors, how='inner', on='order_id')\n",
    "priors_orders_detail.days_since_prior_order = priors_orders_detail.days_since_prior_order.fillna(60)\n",
    "\n",
    "print(priors_orders_detail[:5])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "temp = priors_orders_detail.groupby('order_id').agg({'add_to_cart_order' : 'max'}).reset_index()\n",
    "temp.rename(index=str,columns={'add_to_cart_order':'max_cart'},inplace=True)\n",
    "print(temp[:5])\n",
    "priors_orders_detail = priors_orders_detail.merge(temp,on='order_id',how='left')\n",
    "priors_orders_detail['relative_cart']=priors_orders_detail.add_to_cart_order/priors_orders_detail.max_cart\n",
    "priors_orders_detail.drop('max_cart',axis=1,inplace=True)\n",
    "priors_orders_detail.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "#print(priors.head())\n",
    "# Products information ----------------------------------------------------------------\n",
    "# add order information to priors set\n",
    "\n",
    "\n",
    "# create new variables\n",
    "# _user_buy_product_times: The user is buying the item several times\n",
    "priors_orders_detail.loc[:,'_user_buy_product_times'] = priors_orders_detail.groupby(['user_id', 'product_id']).cumcount() + 1\n",
    "\n",
    "#print(priors_orders_detail[:5])\n",
    "#print(priors_orders_detail.shape)\n",
    "\n",
    "\n",
    "# _prod_tot_cnts: The total number of times the item was purchased, indicating the degree of liking\n",
    "# _reorder_tot_cnts_of_this_prod: The total number of times this item was purchased again\n",
    "### I think the following two are very difficult to understand, consider changing++++++++++++++++++++++++++\n",
    "# _prod_order_once: The total number of times the item was purchased once\n",
    "# _prod_order_more_than_once: The total number of times the item has been purchased more than once\n",
    "agg_dict = {'user_id':{'_prod_tot_cnts':'count',\n",
    "                       '_prod_unique_users': lambda x: x.nunique()}, \n",
    "            'reordered':{'_prod_reorder_tot_cnts':'sum',\n",
    "                         '_prod_average_reorder':'mean',\n",
    "                         '_prod_average_reorder_excl_first': \n",
    "                         lambda x: sum(priors_orders_detail.ix[x.index,'reordered']==1)/\n",
    "                         (sum(priors_orders_detail.ix[x.index,'order_number'] > 1))},\n",
    "            'days_since_prior_order':{'_prod_average_need_time':'mean'},\n",
    "            'order_dow':{'_prod_average_order_dow':'mean',\n",
    "                                     '_prod_std_order_dow':'std'},\n",
    "            'order_number':{'_prod_average_order_number':'mean'}, \n",
    "            'order_hour_of_day':{'_prod_average_order_hour_of_day':'mean',\n",
    "                         '_prod_std_order_hour_of_day':'std'},\n",
    "            'add_to_cart_order':{'_prod_average_add_to_cart_order':'mean',\n",
    "                         '_prod_std_add_to_cart_order':'std'},\n",
    "            'relative_cart': {'_prod_relative_cart':'mean'},\n",
    "            '_user_buy_product_times': {'_prod_average_user_buy_product_times':'mean',\n",
    "                                        '_prod_std_user_buy_product_times':'std',\n",
    "                                        '_prod_buy_first_time_total_cnt':lambda x: sum(x==1),\n",
    "                                        '_prod_buy_second_time_total_cnt':lambda x: sum(x==2),\n",
    "                                        '_prod_buy_more_than_once_total_cnt':lambda x: sum(x>=1)}}\n",
    "prd = ka_add_groupby_features_1_vs_n(priors_orders_detail, ['product_id'], agg_dict)\n",
    "\n",
    "\n",
    "\n",
    "# _prod_reorder_prob: This indicator is not well understood\n",
    "# _prod_reorder_ratio: Commodity Repurchase Rate\n",
    "prd['_prod_reorder_prob'] = prd._prod_buy_second_time_total_cnt / prd._prod_buy_first_time_total_cnt\n",
    "prd['_prod_reorder_ratio'] = prd._prod_reorder_tot_cnts / prd._prod_tot_cnts\n",
    "prd['_prod_reorder_times'] = 1 + prd._prod_reorder_tot_cnts / prd._prod_buy_first_time_total_cnt\n",
    "prd['_prod_conversion'] = prd._prod_buy_first_time_total_cnt / prd._prod_buy_more_than_once_total_cnt\n",
    "\n",
    "print(prd.shape)\n",
    "print(prd.head())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(priors_orders_detail.shape)\n",
    "agg_dict ={'order_number':['max']}\n",
    "priors_orders_detail_tmp = ka_add_groupby_features_1_vs_n(priors_orders_detail, ['user_id'], agg_dict)\n",
    "priors_orders_detail_tmp.rename(index=str,columns={'max':'us_last_order_number'},inplace=True)\n",
    "priors_orders_detail = priors_orders_detail.merge(right=priors_orders_detail_tmp,on='user_id')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(prd.shape)\n",
    "prd = prd.merge(right=products, how='inner', on='product_id')\n",
    "print(prd.shape)\n",
    "prd.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Add word2vec features:\n",
    "product_vector_df = pd.read_csv('product_vectors_25.csv')\n",
    "product_vector_df.head()\n",
    "\n",
    "prd = prd.merge(right=product_vector_df,on=['product_id'],how='left')\n",
    "print(prd.shape)\n",
    "prd.head()\n",
    "del product_vector_df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# User features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# _user_total_orders: The total number of orders for the user\n",
    "# May consider adding other statistical indicators ++++++++++++++++++++++++++\n",
    "# _user_sum_days_since_prior_order: From the last purchase time (and), this can only be calculated inside the orders table，\n",
    "# priors_orders_detail is not at\n",
    "# order level上面unique\n",
    "# _user_mean_days_since_prior_order: From the last purchase time (mean)\n",
    "agg_dict_2 = {'order_number':{'_user_total_orders':'max'},\n",
    "              'order_dow':{'_user_average_order_dow':'mean',\n",
    "                                     '_user_std_order_dow':'std'},\n",
    "              'order_hour_of_day':{'_user_average_order_hour_of_day':'mean',\n",
    "                         '_user_std_order_hour_of_day':'std'},\n",
    "              'days_since_prior_order':{'_user_sum_days_since_prior_order':'sum', \n",
    "                                        '_user_average_days_since_prior_order':\n",
    "                                       lambda x: (sum(x)+1)/(len(x)-1)}}\n",
    "            \n",
    "#sum(orders[orders.eval_set == 'prior'].ix[x.index,'order_number']>1)\n",
    "\n",
    "users = ka_add_groupby_features_1_vs_n(orders[orders.eval_set == 'prior'], ['user_id'], agg_dict_2)\n",
    "\n",
    "# _user_reorder_ratio: reorder The total number of times / the first single after the total number of post-purchase\n",
    "# _user_total_products: The total number of items purchased by the user\n",
    "# _user_distinct_products: The number of unique merchandise purchased by the user\n",
    "agg_dict_3 = {'reordered':\n",
    "              {'_user_reorder_ratio': \n",
    "               lambda x: sum(priors_orders_detail.ix[x.index,'reordered']==1)/\n",
    "                         sum(priors_orders_detail.ix[x.index,'order_number'] > 1)},\n",
    "              '_user_buy_product_times': {'_user_average_user_buy_product_times':'mean'},\n",
    "              'product_id':{'_user_total_products':'count', \n",
    "                            '_user_distinct_products': lambda x: x.nunique()}}\n",
    "\n",
    "\n",
    "us = ka_add_groupby_features_1_vs_n(priors_orders_detail, ['user_id'], agg_dict_3)\n",
    "users = users.merge(us, how='inner')\n",
    "\n",
    "# The average number of items per single\n",
    "# The largest number of goods per list, the least number of commodities ++++++++++++++\n",
    "users['_user_average_basket'] = users._user_total_products / users._user_total_orders\n",
    "\n",
    "us = orders[orders.eval_set != \"prior\"][['user_id', 'order_id', 'eval_set', 'days_since_prior_order']]\n",
    "us.rename(index=str, columns={'days_since_prior_order': 'time_since_last_order'}, inplace=True)\n",
    "\n",
    "users = users.merge(us, how='inner')\n",
    "\n",
    "print(users.shape)\n",
    "users[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(orders[orders.eval_set!='prior'].shape)\n",
    "users = users.merge(right=orders[orders.eval_set!='prior'][['user_id',\n",
    "                                  'order_dow','order_hour_of_day',\n",
    "                                  'days_since_prior_order']],on='user_id',how='left')\n",
    "users.rename(columns={'order_dow':'user_predicted_order_dow',\n",
    "                          'order_hour_of_day':'user_predicted_order_hour_of_day',\n",
    "                          'days_since_prior_order':'user_predicted_days_since_prior_order'}, inplace=True)\n",
    "\n",
    "\n",
    "print(users.shape)\n",
    "\n",
    "users.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Add word2vec features:\n",
    "user_vector_df = pd.read_csv('user_vectors_20.csv')\n",
    "print(user_vector_df[:5])\n",
    "\n",
    "users = users.merge(right=user_vector_df,on=['user_id'],how='left')\n",
    "print(users.shape)\n",
    "users.head()\n",
    "del user_vector_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# This feature indicates whether this user's general liking of instacart is going up or down\n",
    "# I think this is a good feature\n",
    "\n",
    "temp2 = orders.sort_values(['user_id','order_number'])\n",
    "\n",
    "temp2 = temp2.groupby('user_id').agg({'days_since_prior_order':\n",
    "                                      lambda x:linregress(list(range(len(x[1:]))),x[1:])[0]})\n",
    "\n",
    "temp2.rename(index=str,columns={'days_since_prior_order':'order_trend'},inplace=True)\n",
    "temp2.reset_index(inplace=True)\n",
    "temp2.user_id = temp2.user_id.astype(int)\n",
    "users = users.merge(right=temp2,on='user_id',how='left')\n",
    "del temp2\n",
    "print(users.columns)\n",
    "users.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # What is this user's none order rate?\n",
    "temp5 = priors_orders_detail[priors_orders_detail.order_number!=1].groupby(['user_id','order_id']).agg({'reordered': 'max'}).reset_index()\n",
    "temp5 =  temp5.groupby(['user_id']).agg({'reordered':lambda x: 1-np.mean(x)})\n",
    "temp5.reset_index(inplace=True)\n",
    "temp5.rename(index=str,columns={'reordered':'user_none_rate'},inplace=True)\n",
    "users = users.merge(right=temp5,on=['user_id'],how='left')\n",
    "users.head()\n",
    "del temp5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "users['user_none_rate'].hist(bins=20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# How does a typical non-user look like and what's the similarity of our user to this user?\n",
    "\n",
    "start_time = timeit.default_timer()\n",
    "# What does the average user who orders this product look like and whatis its similarity to our user:\n",
    "none_users  = users[users.user_none_rate>0.3]\n",
    "none_users.to_csv('none_users.csv')\n",
    "none_user_means = none_users[['uv_1',\n",
    "                            'uv_2',\n",
    "                            'uv_3', \n",
    "                            'uv_4',\n",
    "                            'uv_5', \n",
    "                            'uv_6', \n",
    "                            'uv_7', \n",
    "                            'uv_8', \n",
    "                            'uv_9', \n",
    "                            'uv_10',\n",
    "                            'uv_11',\n",
    "                            'uv_12', \n",
    "                            'uv_13', \n",
    "                            'uv_14',\n",
    "                            'uv_15', \n",
    "                            'uv_16', \n",
    "                            'uv_17', \n",
    "                            'uv_18', \n",
    "                            'uv_19', \n",
    "                            'uv_20']].mean(axis=0)\n",
    "#none_user_means.reset_index(inplace=True)\n",
    "print(none_user_means.tolist())\n",
    "print(type(none_user_means))\n",
    "\n",
    "temp2 = pd.DataFrame(users[['user_id','uv_1', 'uv_2', 'uv_3', 'uv_4',\n",
    "                            'uv_5', 'uv_6', 'uv_7', 'uv_8', 'uv_9', 'uv_10','uv_11', \n",
    "                            'uv_12', 'uv_13', 'uv_14','uv_15', 'uv_16', 'uv_17', \n",
    "                            'uv_18', 'uv_19', 'uv_20']].apply(lambda x: cosine(x[1:21],none_user_means.tolist()),axis=1,raw=True),columns=['user_none_user_sim'])\n",
    "\n",
    "temp2.head()\n",
    "users['user_none_user_sim'] = temp2['user_none_user_sim']\n",
    "del temp2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Database"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# There should be a lot of variables that can be added here\n",
    "# _up_order_count: The number of times a user buys the item\n",
    "# _up_first_order_number: The number of orders the user purchased for the first time\n",
    "# _up_last_order_number: The last time the user purchased the order\n",
    "# _up_average_cart_position: The item is added to the average position in the shopping basket\n",
    "agg_dict_4 = {'order_number':{'_up_order_count': 'count', \n",
    "                              '_up_first_order_number': 'min', \n",
    "                              '_up_last_order_number':'max'}, \n",
    "              'order_dow':{'_up_average_order_dow':'mean'},\n",
    "              'reordered':{'_up_average_reordered':'mean'},\n",
    "              'order_hour_of_day':{'_up_average_order_hour_of_day':'mean'},\n",
    "              'add_to_cart_order':{'_up_average_cart_position': 'mean'},\n",
    "              'relative_cart':{'_up_relative_cart': 'mean'},\n",
    "             'days_since_prior_order':{'_up_average_days_since_prior_order':\n",
    "                                       'mean'}}\n",
    "\n",
    "data = ka_add_groupby_features_1_vs_n(df=priors_orders_detail, \n",
    "                                                      group_columns_list=['user_id', 'product_id'], \n",
    "                                                      agg_dict=agg_dict_4,only_new_feature=True)\n",
    "\n",
    "print(priors_orders_detail.shape)\n",
    "print(data.shape)\n",
    "\n",
    "## MERGE ALL PRODUCTS USERS AND DATA\n",
    "data = data.merge(prd, how='inner', on='product_id').merge(users, how='inner', on='user_id')\n",
    "\n",
    "\n",
    "# The number of times the product was purchased / the total number of orders\n",
    "# The last time a product was purchased - the last time the item was purchased\n",
    "# The number of times the item was purchased / the first purchase of the item to the last purchase of the order\n",
    "data['_up_order_rate'] = data._up_order_count / data._user_total_orders\n",
    "data['_up_order_since_last_order'] = data._user_total_orders - data._up_last_order_number\n",
    "data['_up_order_rate_since_first_order'] = data._up_order_count / (data._user_total_orders - data._up_first_order_number + 1)\n",
    "\n",
    "#USER PRODUCT SIMILARITY\n",
    "\n",
    "\n",
    "print(data[:5])\n",
    "print(data.columns)\n",
    "# add user_id to train set\n",
    "# ADD PREDICTION LABEL TO DATA SET\n",
    "data = data.merge(train[['user_id', 'product_id', 'reordered']], on=['user_id', 'product_id'], how='left')\n",
    "data.reordered.fillna(0,inplace=True)\n",
    "\n",
    "# release Memory\n",
    "# del train, prd, usersre\n",
    "# gc.collect()\n",
    "# release Memory\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "start_time = timeit.default_timer()\n",
    "# What does the average user who orders this product look like and whatis its similarity to our user:\n",
    "typical_user_means  = data[data._up_order_rate>0.65].groupby('product_id').agg({'uv_1':'mean',\n",
    "                                                        'uv_2':'mean', \n",
    "                                                        'uv_3':'mean', \n",
    "                                                        'uv_4':'mean',\n",
    "                                                        'uv_5':'mean', \n",
    "                                                        'uv_6':'mean', \n",
    "                                                        'uv_7':'mean', \n",
    "                                                        'uv_8':'mean', \n",
    "                                                        'uv_9':'mean', \n",
    "                                                        'uv_10':'mean',\n",
    "                                                        'uv_11':'mean',\n",
    "                                                        'uv_12':'mean', \n",
    "                                                        'uv_13':'mean', \n",
    "                                                        'uv_14':'mean',\n",
    "                                                        'uv_15':'mean', \n",
    "                                                        'uv_16':'mean', \n",
    "                                                        'uv_17':'mean', \n",
    "                                                        'uv_18':'mean', \n",
    "                                                        'uv_19':'mean', \n",
    "                                                        'uv_20':'mean'})\n",
    "typical_user_means.reset_index(inplace=True)\n",
    "print(type(typical_user_means))\n",
    "print(typical_user_means.shape)\n",
    "#print(typical_user_means[:5])\n",
    "\n",
    "temp = data[['user_id','product_id','uv_1', 'uv_2', 'uv_3', 'uv_4',\n",
    "       'uv_5', 'uv_6', 'uv_7', 'uv_8', 'uv_9', 'uv_10','uv_11', 'uv_12', 'uv_13', 'uv_14',\n",
    "       'uv_15', 'uv_16', 'uv_17', 'uv_18', 'uv_19', 'uv_20']].merge(right = typical_user_means,on='product_id',how='left')\n",
    "temp = temp[['user_id','product_id','uv_1_x', 'uv_2_x', 'uv_3_x', 'uv_4_x',\n",
    "       'uv_5_x', 'uv_6_x', 'uv_7_x', 'uv_8_x', 'uv_9_x', 'uv_10_x','uv_11_x', 'uv_12_x', 'uv_13_x', 'uv_14_x',\n",
    "       'uv_15_x', 'uv_16_x', 'uv_17_x', 'uv_18_x', 'uv_19_x', 'uv_20_x','uv_1_y', 'uv_2_y', 'uv_3_y', 'uv_4_y',\n",
    "       'uv_5_y', 'uv_6_y', 'uv_7_y', 'uv_8_y', 'uv_9_y', 'uv_10_y','uv_11_y', 'uv_12_y', 'uv_13_y', 'uv_14_y',\n",
    "       'uv_15_y', 'uv_16_y', 'uv_17_y', 'uv_18_y', 'uv_19_y', 'uv_20_y']]\n",
    "print(temp.shape)\n",
    "print(temp[:5])\n",
    "\n",
    "temp_2 = pd.DataFrame(temp.apply(lambda x: cosine(x[2:22],x[22:42]),axis=1,raw=True),columns=['user_typical_user_sim'])\n",
    "\n",
    "temp_2.user_typical_user_sim.fillna(1,inplace=True)\n",
    "\n",
    "data['user_typical_user_sim'] = temp_2['user_typical_user_sim']\n",
    "\n",
    "\n",
    "print(timeit.default_timer() - start_time )\n",
    "\n",
    "del temp_2,temp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Add order streaks as a feature:\n",
    "order_streaks = pd.read_csv('order_streaks.csv')\n",
    "order_streaks.sort_values(['user_id','product_id'],inplace=True)\n",
    "order_streaks[order_streaks.user_id==196002].head()\n",
    "data = data.merge(right=order_streaks[['user_id','product_id','order_streak']],on=['user_id','product_id'],how='left')\n",
    "\n",
    "del order_streaks\n",
    "print(data.shape)\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def hour_diff(month1, month2):\n",
    "    m_min = min(month1, month2)\n",
    "    m_max = max(month1, month2)\n",
    "    diff = m_max - m_min\n",
    "    return diff if diff <= 12 else m_min + 24 - m_max\n",
    "\n",
    "\n",
    "def day_diff(month1, month2):\n",
    "    return(min(abs(month1 - month2), month1 - month2 + 7, month2 - month1 + 7))\n",
    "\n",
    "def time_diff(day1,hour1, day2,hour2):\n",
    "    time1 = day1*24+hour1\n",
    "    \n",
    "    time2 = day2*24+hour2\n",
    "    \n",
    "    return(min(abs(time1 - time2), time1 - time2 + 168, time2 - time1 + 168))\n",
    "\n",
    "\n",
    "time_diff(4,16,5,23)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # Is this a typical order for this user? and is this a typical time for this product user\n",
    "start_time = timeit.default_timer()\n",
    "\n",
    "\n",
    "data['user_product_hour_diff'] = data[['_prod_average_order_hour_of_day',\n",
    "                                  'user_predicted_order_hour_of_day',]].apply(lambda x: hour_diff(x[0],x[1]),axis=1,raw=True)\n",
    "\n",
    "\n",
    "data['user_product_day_diff'] = data[['_prod_average_order_dow',\n",
    "                                  'user_predicted_order_dow']].apply(lambda x: day_diff(x[0],x[1]),axis=1,raw=True)\n",
    "\n",
    "data['user_product_time_diff'] = data[['_prod_average_order_dow',\n",
    "                                  '_prod_average_order_hour_of_day',\n",
    "                                  'user_predicted_order_dow',\n",
    "                                  'user_predicted_order_hour_of_day',]].apply(lambda x: time_diff(x[0],x[1],x[2],x[3]),axis=1,raw=True)\n",
    "\n",
    "\n",
    "data['up_predicted_order_hour_diff'] = data[['_up_average_order_hour_of_day', \n",
    "                                      'user_predicted_order_hour_of_day']].apply(lambda x: hour_diff(x[0],x[1]),axis=1,raw=True)\n",
    "\n",
    "data['up_predicted_order_day_diff'] = data[['_up_average_order_dow', \n",
    "                                      'user_predicted_order_dow']].apply(lambda x: day_diff(x[0],x[1]),axis=1,raw=True)\n",
    "\n",
    "data['up_predicted_order_time_diff'] = data[['_up_average_order_dow',\n",
    "                                       '_up_average_order_hour_of_day', \n",
    "                                      'user_predicted_order_dow',\n",
    "                                      'user_predicted_order_hour_of_day']].apply(lambda x: time_diff(x[0],x[1],x[2],x[3]),axis=1,raw=True)\n",
    "\n",
    "\n",
    "\n",
    "data['user_predicted_hour_diff'] = data[['_user_average_order_hour_of_day',\n",
    "                                  'user_predicted_order_hour_of_day',]].apply(lambda x: hour_diff(x[0],x[1]),axis=1,raw=True)\n",
    "\n",
    "data['user_predicted_day_diff'] = data[['_user_average_order_dow',\n",
    "                                  'user_predicted_order_dow']].apply(lambda x: day_diff(x[0],x[1]),axis=1,raw=True)\n",
    "\n",
    "data['user_predicted_time_diff'] = data[['_user_average_order_dow',\n",
    "                                  '_user_average_order_hour_of_day',\n",
    "                                  'user_predicted_order_dow',\n",
    "                                  'user_predicted_order_hour_of_day']].apply(lambda x: time_diff(x[0],x[1],x[2],x[3]),axis=1,raw=True)\n",
    "\n",
    "\n",
    "print(timeit.default_timer() - start_time )\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# User product trends\n",
    "\n",
    "\n",
    "tmp1 = orders[orders.eval_set=='prior'].sort_values(['user_id','order_number'])\n",
    "tmp1 = tmp1.groupby('user_id').agg({'days_since_prior_order': lambda x : x.tolist()})\n",
    "tmp1.reset_index(inplace=True)\n",
    "\n",
    "tmp2 =  priors_orders_detail.sort_values(['user_id','order_number'])\n",
    "tmp2 =  tmp2.groupby(['user_id', \n",
    "                    'product_id']).apply(lambda x: [1 if item in x.order_number.tolist() else 0 for item in range(1,x.us_last_order_number.iloc[0]+1)])\n",
    "tmp2= pd.DataFrame(tmp2).reset_index()\n",
    "tmp2.columns.values[2] = 'order_binary'\n",
    "\n",
    "temp3 = tmp2.merge(right=tmp1,on='user_id',how='left')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sumgaps(binarylist,gaplist):\n",
    "    sumgap = []\n",
    "    temp = 0\n",
    "    for binary,gap in zip(binarylist,gaplist):\n",
    "        if binary==0:\n",
    "            temp+=gap\n",
    "        else:\n",
    "            sumgap.append(temp+gap)\n",
    "            temp=0   \n",
    "    return sumgap\n",
    "\n",
    "print('test_sumgaps:',sumgaps([0,0,0,1,0,0],[-1,2,3,5,4,4]))\n",
    "#del tmp1,tmp2\n",
    "\n",
    "def lastgap(binarylist,gaplist):\n",
    "    sumgap = []\n",
    "    temp = 0\n",
    "    for binary,gap in zip(binarylist,gaplist):\n",
    "        if binary==0:\n",
    "            temp+=gap\n",
    "        else:\n",
    "            temp=0   \n",
    "    return temp\n",
    "\n",
    "print('test_sumgaps:',lastgap([0,0,0,1,1,0],[-1,2,3,5,6,7]))\n",
    "\n",
    "\n",
    "temp3['up_gaps'] = temp3[['order_binary','days_since_prior_order']].apply(lambda x: sumgaps(x[0],x[1]),axis=1)\n",
    "temp3['up_average_need_time'] = temp3['up_gaps'].apply(lambda x: np.mean(x[1:]))\n",
    "temp3['up_trend'] = temp3['up_gaps'].apply(lambda x:linregress(list(range(len(x)-1)),x[1:])[0] if len(x)>2 else float('nan'))\n",
    "temp3['up_time_first_order'] = temp3['up_gaps'].apply(lambda x: x[0])\n",
    "temp3['up_ordered_since'] = temp3[['order_binary','days_since_prior_order']].apply(lambda x: lastgap(x[0],x[1]),axis=1)\n",
    "\n",
    "\n",
    "\n",
    "data = data.merge(right=temp3[['user_id','product_id','up_trend','up_time_first_order','up_ordered_since','up_average_need_time']],on=['user_id','product_id'],how='left')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Inject recursivity from binary labels\n",
    "\n",
    "def window_transform_series(series,window_size):\n",
    "    # containers for input/output pairs\n",
    "    X = []\n",
    "    y = []\n",
    "    z = []\n",
    "    # go through the time series in single steps and add input output pairs\n",
    "    for i in range(0,len(series)-window_size):\n",
    "        y.append(series[i+window_size])\n",
    "        X.append(series[i:(i+window_size)])\n",
    "    z.append(series[-window_size:])\n",
    "    #reshape each \n",
    "    X = np.asarray(X)\n",
    "    X.shape = (np.shape(X)[0:2])\n",
    "    y = np.asarray(y)\n",
    "    y.shape = (len(y),1)\n",
    "    z = np.asarray(z)\n",
    "    z.shape = (np.shape(z)[0:2])\n",
    "    \n",
    "    return X,y,z\n",
    "\n",
    "def calculate_recursive_prob(binary_list,window):\n",
    "    x,y,z = window_transform_series(binary_list,window) \n",
    "    lg = LinearRegression(fit_intercept=True, normalize=False, copy_X=True, n_jobs=-1)\n",
    "    lg.fit(x,y)\n",
    "\n",
    "    return np.log10(lg.predict(z)[0][0])\n",
    "\n",
    "temp3['recursive_prob_2'] = temp3['order_binary'].apply(lambda x: calculate_recursive_prob(x,2) if len(x)>2 else float('nan'))\n",
    "temp3['recursive_prob_3'] = temp3['order_binary'].apply(lambda x: calculate_recursive_prob(x,3) if len(x)>4 else float('nan'))\n",
    "\n",
    "\n",
    "data = data.merge(right=temp3[['user_id','product_id','recursive_prob_2','recursive_prob_3']],on=['user_id','product_id'],how='left')\n",
    "#temp3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Product trends\n",
    "\n",
    "temp4 = temp3.groupby('product_id').agg({'up_average_need_time':'mean',\n",
    "                                         'up_trend':'mean',\n",
    "                                         'up_time_first_order':'mean'})\n",
    "temp4.reset_index(inplace=True)\n",
    "temp4.rename(index=str,columns={'up_average_need_time':'product_specific_need_time',\n",
    "                                'up_trend':'product_trend',\n",
    "                                'up_time_first_order':'product_time_first_order'},inplace=True)\n",
    "\n",
    "data = data.merge(right=temp4[['product_id',\n",
    "                               'product_trend',\n",
    "                               'product_specific_need_time',\n",
    "                               'product_time_first_order']],on=['product_id'],how='left')\n",
    "\n",
    "data['up_late']=(data._user_sum_days_since_prior_order>data.product_time_first_order)*1\n",
    "data.head()\n",
    "\n",
    "del temp3,temp4,tmp1,tmp2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "temp = data.groupby('user_id').agg({'aisle_id': lambda x :x.nunique(),\n",
    "                             'department_id':lambda x :x.nunique(),\n",
    "                           'product_id': lambda x :x.nunique()}).reset_index().rename(index=str,columns={'aisle_id':'unique_aisles',\n",
    "                                                                    'department_id':'unique_departments',\n",
    "                                                                    'product_id':'unique_products'})\n",
    "temp['user_uniuqe_aisle_ratio'] = temp.unique_departments/temp.unique_products\n",
    "temp['user_uniuqe_depart_ratio'] = temp.unique_aisles/temp.unique_products\n",
    "data = data.merge(right=temp[['user_id','user_uniuqe_aisle_ratio','user_uniuqe_depart_ratio']],on='user_id',how='left')\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "temp = priors_orders_detail.merge(right=prd[['product_id',\n",
    "                                                 'aisle_id',\n",
    "                                                 'department_id']],on='product_id',how='left')\n",
    "\n",
    "temp2 = temp[temp.order_number==temp.us_last_order_number].groupby('user_id').agg({'aisle_id': lambda x :x.nunique(),\n",
    "                            'department_id':lambda x :x.nunique(),\n",
    "                           'product_id': lambda x :x.nunique()}).reset_index().rename(index=str,columns={'aisle_id':'unique_aisles',\n",
    "                                                                    'department_id':'unique_departments',\n",
    "                                                                    'product_id':'unique_products'})\n",
    "temp2['user_last_unique_aisle_ratio'] = temp2.unique_aisles/temp2.unique_products\n",
    "temp2['user_last_unique_depart_ratio'] = temp2.unique_departments/temp2.unique_products\n",
    "\n",
    "data = data.merge(right=temp2[['user_id','user_last_unique_aisle_ratio','user_last_unique_depart_ratio']],on='user_id',how='left')\n",
    "del temp,temp2\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "temp = data[['user_id','product_id','wv_1', 'wv_2', 'wv_3', 'wv_4',\n",
    "       'wv_5', 'wv_6', 'wv_7', 'wv_8', 'wv_9', 'wv_10','wv_11', 'wv_12', 'wv_13', 'wv_14',\n",
    "       'wv_15', 'wv_16', 'wv_17', 'wv_18', 'wv_19', 'wv_20']].groupby('user_id').mean().reset_index()\n",
    "\n",
    "temp = data[['user_id','product_id','wv_1', 'wv_2', 'wv_3', 'wv_4',\n",
    "       'wv_5', 'wv_6', 'wv_7', 'wv_8', 'wv_9', 'wv_10','wv_11', 'wv_12', 'wv_13', 'wv_14',\n",
    "       'wv_15', 'wv_16', 'wv_17', 'wv_18', 'wv_19', 'wv_20']].merge(right=temp,on='user_id',how='left')\n",
    "\n",
    "temp_2 = pd.DataFrame(temp.apply(lambda x: cosine(x[2:22],x[22:42]),axis=1,raw=True),columns=['prod_typical_prod_sim'])\n",
    "\n",
    "data['prod_typical_prod_sim'] = temp_2['prod_typical_prod_sim']\n",
    "del temp,temp_2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "temp = data[data.reordered==0]\n",
    "plt.hist(temp['_prod_tot_cnts'], normed=True, bins=100)\n",
    "plt.ylabel('prob');\n",
    "axes = plt.gca()\n",
    "axes.set_xlim([0,10000])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Seperate training and testing data and write to file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "del priors_orders_detail,orders,prd,users\n",
    "\n",
    "all_train_X = data.loc[data.eval_set == \"train\",:]\n",
    "test_X = data.loc[data.eval_set == \"test\",:]\n",
    "del data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "all_train_X.to_csv('all_train_X.csv')\n",
    "test_X.to_csv('test_X.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load Test and Train Data From File"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(8474661, 139)\n",
      "(4833292, 139)\n"
     ]
    }
   ],
   "source": [
    "all_train_X = pd.read_csv('all_train_X_AH_old.csv')\n",
    "all_train_X.drop('Unnamed: 0',axis=1,inplace=True)\n",
    "print(all_train_X.shape)\n",
    "test_X = pd.read_csv('test_X_AH_old.csv')\n",
    "test_X.drop('Unnamed: 0',axis=1,inplace=True)\n",
    "print(test_X.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Combine unique sh1ng feautures\n",
    "all_train_X = pd.read_csv('all_train_X_old.csv')\n",
    "all_train_X.drop('Unnamed: 0',axis=1,inplace=True)\n",
    "test_X = pd.read_csv('test_X_old.csv')\n",
    "test_X.drop('Unnamed: 0',axis=1,inplace=True)\n",
    "print(all_train_X.shape)\n",
    "print(test_X.shape)\n",
    "\n",
    "ah_train_features = pd.read_csv('AH_train_features_old.csv')\n",
    "ah_test_features = pd.read_csv('AH_test_features_old.csv')\n",
    "ah_train_features.drop('Unnamed: 0',axis=1,inplace=True)\n",
    "ah_test_features.drop('Unnamed: 0',axis=1,inplace=True)\n",
    "\n",
    "print(ah_train_features.shape)\n",
    "print(ah_test_features.shape)\n",
    "ah_train_features.head()\n",
    "ah_test_features.head()\n",
    "\n",
    "all_train_X = all_train_X.merge(ah_train_features,on=['user_id','product_id'],how='left')\n",
    "test_X = test_X.merge(ah_test_features,on=['user_id','product_id'],how='left')\n",
    "del ah_train_features,ah_test_features\n",
    "print(all_train_X.shape)\n",
    "print(test_X.shape)\n",
    "\n",
    "all_train_X.to_csv('all_train_X_AH_old.csv')\n",
    "test_X.to_csv('test_X_AH_old.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# XGBoost training starts here:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_index, valid_index = list(GroupKFold(n_splits=2).split(all_train_X.drop(['eval_set', \n",
    "                                                                               'user_id', \n",
    "                                                                               'product_id', \n",
    "                                                                               'order_id',\n",
    "                                                                               'reordered'], axis=1),\n",
    "                                                  all_train_X.reordered,groups=all_train_X.user_id))[0]\n",
    "all_train_X = all_train_X.iloc[valid_index,:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "train_index, valid_index = list(GroupKFold(n_splits=5).split(all_train_X.drop(['eval_set', \n",
    "                                                                               'user_id', \n",
    "                                                                               'product_id', \n",
    "                                                                               'order_id',\n",
    "                                                                               'reordered'], axis=1),\n",
    "                                                  all_train_X.reordered,groups=all_train_X.user_id))[0]\n",
    "\n",
    "    \n",
    "train_X = all_train_X.iloc[train_index,:]\n",
    "valid_X = all_train_X.iloc[valid_index,:]\n",
    "\n",
    "print(train_X.shape)\n",
    "print(valid_X.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print('the class imbalance:')\n",
    "print(np.mean(train_X.reordered))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "d_train = xgb.DMatrix(train_X.drop(['eval_set', 'user_id', \n",
    "                                        'product_id', 'order_id',\n",
    "                                        'reordered'], axis=1), train_X.reordered)\n",
    "\n",
    "d_valid = xgb.DMatrix(valid_X.drop(['eval_set', 'user_id', \n",
    "                                        'product_id', 'order_id',\n",
    "                                        'reordered'], axis=1), valid_X.reordered)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "start_time = timeit.default_timer()\n",
    "\n",
    "xgb_params = {\n",
    "    \"objective\"         : \"reg:logistic\"\n",
    "    ,\"eval_metric\"      : \"logloss\"\n",
    "    ,\"eta\"              : 0.1\n",
    "    ,\"max_depth\"        : 7\n",
    "    ,\"min_child_weight\" : 6 \n",
    "    ,\"gamma\"            :0.1\n",
    "    ,\"subsample\"        :0.76\n",
    "    ,\"colsample_bytree\" :0.95\n",
    "    ,\"alpha\"            :2e-05\n",
    "    ,\"lambda\"           :20\n",
    "}\n",
    "\n",
    "\n",
    "watchlist= [(d_valid, 'valid')]\n",
    "\n",
    "bst = xgb.train(params=xgb_params, \n",
    "                    dtrain=d_train, \n",
    "                    num_boost_round=800, evals=watchlist, \n",
    "                    early_stopping_rounds=20, \n",
    "                    verbose_eval=10)\n",
    "\n",
    "\n",
    "valid_probs = bst.predict(d_valid)\n",
    "\n",
    "\n",
    "print(timeit.default_timer() - start_time )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sns.set(context='poster',font_scale = 0.5)\n",
    "xgb.plot_importance(bst,height=0.3,importance_type='gain')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Training for None orders prediction:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "data_tmp = train_X.groupby('user_id').agg({'reordered' : lambda x: 1!=max(x) }).reset_index()\n",
    "data_tmp.rename(index=str,columns={'reordered':'None_order'},inplace=True)\n",
    "train_none_X = train_X.merge(right=data_tmp,on='user_id',how='left').groupby('user_id').mean().reset_index(inplace=False)\n",
    "\n",
    "data_tmp = valid_X.groupby('user_id').agg({'reordered' : lambda x: 1!=max(x) }).reset_index()\n",
    "data_tmp.rename(index=str,columns={'reordered':'None_order'},inplace=True)\n",
    "valid_none_X = valid_X.merge(right=data_tmp,on='user_id',how='left').groupby('user_id').mean().reset_index(inplace=False)\n",
    "\n",
    "\n",
    "del data_tmp \n",
    "print('Percent None orders')\n",
    "print(np.mean(train_none_X.None_order))\n",
    "\n",
    "print('train_none_X size', train_none_X.shape)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "d_train_none = xgb.DMatrix(train_none_X.drop([ 'user_id', \n",
    "                                    'None_order','order_id','reordered'], axis=1), train_none_X.None_order)\n",
    "\n",
    "d_valid_none = xgb.DMatrix(valid_none_X.drop(['user_id', \n",
    "                                        'None_order','order_id','reordered'], axis=1), valid_none_X.None_order)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "start_time = timeit.default_timer()\n",
    "\n",
    "\n",
    "xgb_none_params = {\n",
    "    \"objective\"         : \"reg:logistic\"\n",
    "    ,\"eval_metric\"      : \"logloss\"\n",
    "    ,\"eta\"              : 0.05\n",
    "    ,\"max_depth\"        : 5\n",
    "    ,\"min_child_weight\" :14\n",
    "    ,\"gamma\"            :0.1\n",
    "    ,\"subsample\"        :0.85\n",
    "    ,\"colsample_bytree\" :0.75\n",
    "    ,\"alpha\"            :2e-05\n",
    "    ,\"lambda\"           :10\n",
    "}\n",
    "\n",
    "\n",
    "watchlist= [(d_train_none, 'train'),(d_valid_none, 'valid')]\n",
    "\n",
    "bst_none = xgb.train(params=xgb_none_params, \n",
    "                    dtrain=d_train_none, \n",
    "                    num_boost_round=811, evals=watchlist, \n",
    "                    early_stopping_rounds=5, \n",
    "                    verbose_eval=10)\n",
    "\n",
    "print(f1_score(valid_none_X.None_order,bst_none.predict(d_valid_none)>0.20))\n",
    "\n",
    "print(timeit.default_timer() - start_time )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(f1_score(valid_none_X.None_order,bst_none.predict(d_valid_none)>0.21))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sns.set(context='poster',font_scale = 0.5)\n",
    "xgb.plot_importance(bst_none,height=0.3, importance_type='gain')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Light GBM training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "start_time = timeit.default_timer()\n",
    "\n",
    "#X_new = SelectKBest(chi2, k=2).fit_transform(X, y)\n",
    "\n",
    "\n",
    "lgb_train_data = lgb.Dataset(train_X.drop(['eval_set', 'user_id', \n",
    "                                        'product_id', 'order_id',\n",
    "                                        'reordered'], axis=1), train_X.reordered)\n",
    "\n",
    "lgb_valid_data = lgb.Dataset(valid_X.drop(['eval_set', 'user_id', \n",
    "                                        'product_id', 'order_id',\n",
    "                                        'reordered'], axis=1), valid_X.reordered)\n",
    "\n",
    "params = {\n",
    "    'task': 'train',\n",
    "    'boosting_type': 'gbdt',\n",
    "    'objective': 'binary',\n",
    "     'early_stopping_rounds':10,\n",
    "    'metric': {'binary_logloss'},\n",
    "    'num_leaves': 256,\n",
    "    'min_sum_hessian_in_leaf': 20,\n",
    "    'max_depth': -1,\n",
    "    'learning_rate': 0.05,\n",
    "    'feature_fraction': 0.6,\n",
    "    # 'bagging_fraction': 0.9,\n",
    "    # 'bagging_freq': 3,\n",
    "    #'evals_result': {'train':lgb_train_data, 'valid':lgb_valid_data},\n",
    "    'verbose': 1\n",
    "    \n",
    "}\n",
    "\n",
    "\n",
    "\n",
    "num_round = 200\n",
    "lgb_bst = lgb.train(params, lgb_train_data, num_round, valid_sets=[lgb_valid_data],verbose_eval=20)\n",
    "\n",
    "valid_probs = lgb_bst.predict(valid_X.drop(['eval_set', 'user_id', \n",
    "                                        'product_id', 'order_id',\n",
    "                                        'reordered'], axis=1))\n",
    "\n",
    "print(timeit.default_timer() - start_time )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ax = lgb.plot_importance(lgb_bst, height=0.2, xlim=None, ylim=None, title='Feature importance', \n",
    "                         xlabel='Feature importance', ylabel='Features', importance_type='gain', \n",
    "                         max_num_features=None, ignore_zero=True, figsize=(40,40), grid=False)\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# XGBoost predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train = pd.read_csv('order_products__train.csv', \n",
    "                dtype={\n",
    "                        'order_id': np.int32,\n",
    "                        'product_id': np.uint16,\n",
    "                        'add_to_cart_order': np.int16,\n",
    "                        'reordered': np.int8})\n",
    "\n",
    "orders = pd.read_csv('orders.csv', \n",
    "                     dtype={\n",
    "                            'order_id': np.int32,\n",
    "                            'user_id': np.int64,\n",
    "                            'eval_set': 'category',\n",
    "                            'order_number': np.int16,\n",
    "                            'order_dow': np.int8,\n",
    "                            'order_hour_of_day': np.int8,\n",
    "                            'days_since_prior_order': np.float32})\n",
    "\n",
    "train = train.merge(right=orders[['order_id','user_id']],on='order_id')\n",
    "del orders"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Prepare None predictions\n",
    "\n",
    "valid_predicted_none_X = valid_none_X[['user_id','order_id']].copy()\n",
    "valid_predicted_none_X.loc[:,'predicted_none'] = (bst_none.predict(d_valid_none) > 0.21).astype(int)\n",
    "valid_predicted_none_X.sort_values('order_id',inplace=True)\n",
    "valid_predicted_none_X.set_index('order_id', inplace=True)\n",
    "print(valid_predicted_none_X.shape)\n",
    "valid_predicted_none_X.head()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from jnius import autoclass\n",
    "\n",
    "FScore = autoclass('FScore')\n",
    "\n",
    "fscore = FScore()\n",
    "\n",
    "# f1 score based on maximum f1 score optimization\n",
    "\n",
    "valid_predicted_X = valid_X[['user_id','order_id','product_id']].copy()\n",
    "valid_predicted_X.loc[:,'reordered'] = valid_probs\n",
    "# Predictions with catboost\n",
    "#valid_predicted_X.loc[:,'reordered'] = cb_model.predict(valid_XX,prediction_type='RawFormulaVal')\n",
    "\n",
    "a = valid_predicted_X.groupby('order_id').apply(lambda x: fscore.max_expected_fscore_preds_cube(x.reordered.tolist(),1))\n",
    "a = pd.DataFrame(a,columns=['labels'])\n",
    "b = pd.DataFrame(valid_predicted_X.groupby('order_id').apply(lambda x: x.product_id.tolist()),columns=['products'])\n",
    "b['labels'] = a['labels']\n",
    "valid_submit_pros = pd.DataFrame(b.apply(lambda x: ' '.join([str(item) for item,label in zip(x['products'],x['labels']) if label==1]),axis=1),columns=['products'])\n",
    "valid_submit_pros.index = valid_submit_pros.index.astype(int)\n",
    "valid_submit_pros.head()\n",
    "\n",
    "# #ADD NONE TO ORDER WITH NO PREDICTED PRODUCTS\n",
    "valid_submit_nones = valid_predicted_X.groupby('order_id').agg({'reordered':'max','product_id':'min'})\n",
    "valid_submit_nones = valid_submit_nones[valid_submit_nones.reordered==0]\n",
    "valid_submit_nones.drop('reordered',axis=1,inplace=True)\n",
    "valid_submit_nones.rename(index=str, columns={'product_id':'products'},inplace=True)\n",
    "valid_submit_nones.index = valid_submit_nones.index.astype(int)\n",
    "valid_submit_nones.products=''\n",
    "\n",
    "valid_submit = pd.concat([valid_submit_pros,valid_submit_nones])\n",
    "valid_submit.sort_index(inplace=True)\n",
    "valid_submit.products = valid_submit.products.astype(str)\n",
    "\n",
    "\n",
    "# Add extra none order prediction\n",
    "\n",
    "valid_submit.loc[valid_predicted_none_X['predicted_none']==1,'products'] = 'None ' + valid_submit['products']\n",
    "\n",
    "\n",
    "valid_submit.loc[valid_submit['products']=='','products'] = 'None'\n",
    "\n",
    "valid_submit['products'] = valid_submit['products'].str.strip()\n",
    "\n",
    "print(valid_submit.shape)\n",
    "print(valid_submit[:20])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#PROCESS GROUND TRUTH\n",
    "\n",
    "valid_ordered_products = train[train.isin({'user_id': valid_X.user_id.tolist()}).user_id==True]\n",
    "\n",
    "try:\n",
    "    df_train_gt = pd.read_csv('valid.csv', index_col='order_id')\n",
    "    print('reading_from_file:')\n",
    "except:\n",
    "    print('regenerating')\n",
    "    train_gtl = []\n",
    "\n",
    "    for uid, subset in valid_ordered_products.groupby('user_id'):\n",
    "        subset1 = subset[subset.reordered == 1]\n",
    "        oid = subset.order_id.values[0]\n",
    "\n",
    "        if len(subset1) == 0:\n",
    "            train_gtl.append((oid, 'None'))\n",
    "            continue\n",
    "\n",
    "        ostr = ' '.join([str(int(e)) for e in subset1.product_id.values])\n",
    "        # .strip is needed because join can have a padding space at the end\n",
    "        train_gtl.append((oid, ostr.strip()))\n",
    "\n",
    "    print(len(train_gtl))\n",
    "    df_valid_gt = pd.DataFrame(train_gtl)\n",
    "\n",
    "    df_valid_gt.columns = ['order_id', 'products']\n",
    "    df_valid_gt.set_index('order_id', inplace=True)\n",
    "    df_valid_gt.sort_index(inplace=True)\n",
    "    \n",
    "    #df_valid_gt.to_csv('train.csv')\n",
    "print(df_valid_gt.shape)\n",
    "df_valid_gt.sort_index(inplace=True)\n",
    "df_valid_gt.products = df_valid_gt.products.astype(str)\n",
    "df_valid_gt.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "f1 = []\n",
    "for gt, pred in zip(df_valid_gt.sort_index().products, valid_submit.sort_index().products):\n",
    "    lgt = gt.replace(\"None\", \"-1\").split(' ')\n",
    "    lpred = pred.replace(\"None\", \"-1\").split(' ')\n",
    "    \n",
    "    rr = (np.intersect1d(lgt, lpred))\n",
    "    precision = np.float(len(rr)) / len(lpred)\n",
    "    recall = np.float(len(rr)) / len(lgt)\n",
    "\n",
    "    denom = precision + recall\n",
    "    f1.append(((2 * precision * recall) / denom) if denom > 0 else 0)\n",
    "\n",
    "print(np.mean(f1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "%matplotlib inline\n",
    "plt.hist(f1, normed=True, bins=40)\n",
    "plt.ylabel('prob');\n",
    "print(np.mean([item for item in f1 if item>-1]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# F1 score in training data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Prepare None predictions\n",
    "\n",
    "valid_predicted_none_X = train_none_X.copy()\n",
    "valid_predicted_none_X.loc[:,'predicted_none'] = (bst_none.predict(d_train_none) > 0.19).astype(int)\n",
    "valid_predicted_none_X.sort_values('order_id',inplace=True)\n",
    "valid_predicted_none_X.set_index('order_id', inplace=True)\n",
    "print(valid_predicted_none_X.shape)\n",
    "#valid_predicted_none_X.head()\n",
    "\n",
    "\n",
    "from jnius import autoclass\n",
    "\n",
    "FScore = autoclass('FScore')\n",
    "\n",
    "fscore = FScore()\n",
    "\n",
    "# f1 score based on maximum f1 score optimization\n",
    "\n",
    "valid_predicted_X = train_X.copy()\n",
    "valid_predicted_X.loc[:,'reordered'] = bst.predict(d_train)\n",
    "\n",
    "a = valid_predicted_X.groupby('order_id').apply(lambda x: fscore.max_expected_fscore_preds_cube(x.reordered.tolist(),1))\n",
    "a = pd.DataFrame(a,columns=['labels'])\n",
    "b = pd.DataFrame(valid_predicted_X.groupby('order_id').apply(lambda x: x.product_id.tolist()),columns=['products'])\n",
    "b['labels'] = a['labels']\n",
    "valid_submit_pros = pd.DataFrame(b.apply(lambda x: ' '.join([str(item) for item,label in zip(x['products'],x['labels']) if label==1]),axis=1),columns=['products'])\n",
    "valid_submit_pros.index = valid_submit_pros.index.astype(int)\n",
    "valid_submit_pros.head()\n",
    "\n",
    "#ADD NONE TO ORDER WITH NO PREDICTED PRODUCTS\n",
    "valid_submit_nones = valid_predicted_X.groupby('order_id').agg({'reordered':'max','product_id':'min'})\n",
    "valid_submit_nones = valid_submit_nones[valid_submit_nones.reordered==0]\n",
    "valid_submit_nones.drop('reordered',axis=1,inplace=True)\n",
    "valid_submit_nones.rename(index=str, columns={'product_id':'products'},inplace=True)\n",
    "valid_submit_nones.index = valid_submit_nones.index.astype(int)\n",
    "valid_submit_nones.products=''\n",
    "\n",
    "valid_submit = pd.concat([valid_submit_pros,valid_submit_nones])\n",
    "valid_submit.sort_index(inplace=True)\n",
    "valid_submit.products = valid_submit.products.astype(str)\n",
    "\n",
    "\n",
    "# Add extra none order prediction\n",
    "\n",
    "valid_submit.loc[valid_predicted_none_X['predicted_none']==1,'products'] = 'None ' + valid_submit['products']\n",
    "\n",
    "\n",
    "valid_submit.loc[valid_submit['products']=='','products'] = 'None'\n",
    "\n",
    "valid_submit['products'] = valid_submit['products'].str.strip()\n",
    "\n",
    "#print(valid_submit.shape)\n",
    "#print(valid_submit[:20])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#PROCESS GROUND TRUTH\n",
    "\n",
    "valid_ordered_products = train[train.isin({'user_id': train_users.user_id.tolist()}).user_id==True]\n",
    "\n",
    "try:\n",
    "    df_train_gt = pd.read_csv('valid.csv', index_col='order_id')\n",
    "    print('reading_from_file:')\n",
    "except:\n",
    "    print('regenerating')\n",
    "    train_gtl = []\n",
    "\n",
    "    for uid, subset in valid_ordered_products.groupby('user_id'):\n",
    "        subset1 = subset[subset.reordered == 1]\n",
    "        oid = subset.order_id.values[0]\n",
    "\n",
    "        if len(subset1) == 0:\n",
    "            train_gtl.append((oid, 'None'))\n",
    "            continue\n",
    "\n",
    "        ostr = ' '.join([str(int(e)) for e in subset1.product_id.values])\n",
    "        # .strip is needed because join can have a padding space at the end\n",
    "        train_gtl.append((oid, ostr.strip()))\n",
    "\n",
    "    print(len(train_gtl))\n",
    "    df_valid_gt = pd.DataFrame(train_gtl)\n",
    "\n",
    "    df_valid_gt.columns = ['order_id', 'products']\n",
    "    df_valid_gt.set_index('order_id', inplace=True)\n",
    "    df_valid_gt.sort_index(inplace=True)\n",
    "    \n",
    "    #df_valid_gt.to_csv('train.csv')\n",
    "print(df_valid_gt.shape)\n",
    "df_valid_gt.sort_index(inplace=True)\n",
    "df_valid_gt.products = df_valid_gt.products.astype(str)\n",
    "#df_valid_gt.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "f1 = []\n",
    "for gt, pred in zip(df_valid_gt.sort_index().products, valid_submit.sort_index().products):\n",
    "    lgt = gt.replace(\"None\", \"-1\").split(' ')\n",
    "    lpred = pred.replace(\"None\", \"-1\").split(' ')\n",
    "    \n",
    "    rr = (np.intersect1d(lgt, lpred))\n",
    "    precision = np.float(len(rr)) / len(lpred)\n",
    "    recall = np.float(len(rr)) / len(lgt)\n",
    "\n",
    "    denom = precision + recall\n",
    "    f1.append(((2 * precision * recall) / denom) if denom > 0 else 0)\n",
    "\n",
    "print(np.mean(f1))\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "%matplotlib inline\n",
    "plt.hist(f1, normed=True, bins=40)\n",
    "plt.ylabel('prob');\n",
    "print(np.mean([item for item in f1 if item>-1]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Prepare test data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# TEST DATA FOR xgboost\n",
    "# F1 SCORE MAXIMIZATION THREHSOLD PREDICTIONS\n",
    "# None PREDICTIONS\n",
    "valid_predicted_none_X = test_none_X.copy()\n",
    "valid_predicted_none_X.loc[:,'predicted_none'] = (bst_none.predict(d_test_none) > 0.16).astype(int)\n",
    "valid_predicted_none_X.sort_values('order_id',inplace=True)\n",
    "valid_predicted_none_X.set_index('order_id', inplace=True)\n",
    "print(valid_predicted_none_X.shape)\n",
    "\n",
    "\n",
    "\n",
    "d_test = xgb.DMatrix(test_X.drop(['eval_set', 'user_id', \n",
    "                                        'product_id', 'order_id',\n",
    "                                        'reordered'], axis=1), test_X.reordered)\n",
    "\n",
    "\n",
    "valid_predicted_X = test_X.copy()\n",
    "valid_predicted_X.loc[:,'reordered'] = bst.predict(d_test)\n",
    "\n",
    "a = valid_predicted_X.groupby('order_id').apply(lambda x: fscore.max_expected_fscore_preds_cube(x.reordered.tolist(),1))\n",
    "a = pd.DataFrame(a,columns=['labels'])\n",
    "b = pd.DataFrame(valid_predicted_X.groupby('order_id').apply(lambda x: x.product_id.tolist()),columns=['products'])\n",
    "b['labels'] = a['labels']\n",
    "valid_submit_pros = pd.DataFrame(b.apply(lambda x: ' '.join([str(item) for item,label in zip(x['products'],x['labels']) if label==1]),axis=1),columns=['products'])\n",
    "valid_submit_pros.index = valid_submit_pros.index.astype(int)\n",
    "valid_submit_pros.head()\n",
    "\n",
    "\n",
    "\n",
    "#ADD NONE TO ORDER WITH NO PREDICTED PRODUCTS\n",
    "valid_submit_nones = valid_predicted_X.groupby('order_id').agg({'reordered':'max','product_id':'min'})\n",
    "valid_submit_nones = valid_submit_nones[valid_submit_nones.reordered==0]\n",
    "valid_submit_nones.drop('reordered',axis=1,inplace=True)\n",
    "valid_submit_nones.rename(index=str, columns={'product_id':'products'},inplace=True)\n",
    "valid_submit_nones.index = valid_submit_nones.index.astype(int)\n",
    "valid_submit_nones.products=''\n",
    "\n",
    "valid_submit = pd.concat([valid_submit_pros,valid_submit_nones])\n",
    "valid_submit.sort_index(inplace=True)\n",
    "valid_submit.products = valid_submit.products.astype(str)\n",
    "\n",
    "\n",
    "# Add extra none order prediction\n",
    "\n",
    "valid_submit.loc[valid_predicted_none_X['predicted_none']==1,'products'] = 'None ' + valid_submit['products']\n",
    "\n",
    "valid_submit.loc[valid_submit['products']=='','products'] = 'None'\n",
    "\n",
    "valid_submit['products'] = valid_submit['products'].str.strip()\n",
    "\n",
    "print(valid_submit.shape)\n",
    "valid_submit.to_csv('test_submit_single.csv')\n",
    "print(valid_submit[:20])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#  XGB Optimization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Optimization STEP1: The goal here is to determine the number of boosting rounds or the number of estimators, I think\n",
    "# they are the same thing, however perhaps because of the leak the number I got was huge. probably more than 500 is \n",
    "# overkill. In this competition we cannot do this step because of shared user id between samples\n",
    "from xgboost.sklearn import XGBClassifier\n",
    "from sklearn import cross_validation, metrics   #Additional scklearn functions\n",
    "from sklearn.grid_search import GridSearchCV   #Perforing grid search\n",
    "\n",
    "\n",
    "xgb1 = XGBClassifier(\n",
    " learning_rate =0.1,\n",
    " n_estimators=200,\n",
    " max_depth=5,\n",
    " min_child_weight=1,\n",
    " gamma=0,\n",
    " subsample=0.8,\n",
    " colsample_bytree=0.8,\n",
    " objective= 'binary:logistic',\n",
    " nthread=6,\n",
    " scale_pos_weight=1,\n",
    " seed=27)\n",
    "\n",
    "\n",
    "\n",
    "xgb_param = xgb1.get_xgb_params()\n",
    "cvresult = xgb.cv(xgb_param, \n",
    "                  d_train, \n",
    "                  num_boost_round=xgb1.get_params()['n_estimators'],\n",
    "                  metrics='logloss',\n",
    "                  early_stopping_rounds=50, \n",
    "                  verbose_eval=100)\n",
    "\n",
    "cvresult.tail(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_X_opt = train_X.sample.drop(['eval_set', 'user_id', 'product_id', 'order_id',\n",
    "                                        'reordered'], axis=1)\n",
    "train_Y_opt = train_X.reordered"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "start_time = timeit.default_timer()\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "# OPTIMIZATION STEP 2:\n",
    "# Tune max_depth and min_child_weight, try to increase these next time and maybe try a finer round\n",
    "from sklearn.model_selection import GroupKFold\n",
    "from sklearn.grid_search import GridSearchCV\n",
    "from xgboost.sklearn import XGBClassifier\n",
    "\n",
    "gkf = list(GroupKFold(n_splits=3).split(train_X_opt,train_Y_opt,train_X.user_id))\n",
    "\n",
    "\n",
    "param_test1 = {\n",
    " 'max_depth':[5,6,7,8],\n",
    " 'min_child_weight':[6,8,10,12,14]\n",
    "}\n",
    "\n",
    "gsearch1 = GridSearchCV(estimator = XGBClassifier( learning_rate =0.1, n_estimators=123, max_depth=5,\n",
    " min_child_weight=10, gamma=0.70, subsample=0.76, colsample_bytree=0.95,\n",
    " objective= 'binary:logistic', nthread=-1,scale_pos_weight=1, reg_lambda=20,seed=27), \n",
    " param_grid = param_test1, scoring='neg_log_loss',n_jobs=-1,iid=False, cv=gkf)\n",
    "\n",
    "gsearch1.fit(train_X_opt,train_Y_opt)\n",
    "\n",
    "print(timeit.default_timer() - start_time )\n",
    "\n",
    "gsearch1.grid_scores_, gsearch1.best_params_, gsearch1.best_score_\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# OPTINAL OPTIMIZATION STEP 2b for finer determination of tree depth and :"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Step 3: Tune gamma\n",
    "# Now lets tune gamma value using the parameters already tuned above. Gamma can take various values \n",
    "# but I’ll check for 5 values here. You can go into more precise values as.\n",
    "\n",
    "start_time = timeit.default_timer()\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "# OPTIMIZATION STEP 2:\n",
    "# Tune max_depth and min_child_weight, try to increase these next time and maybe try a finer round\n",
    "from sklearn.model_selection import GroupKFold\n",
    "from sklearn.grid_search import GridSearchCV\n",
    "from xgboost.sklearn import XGBClassifier\n",
    "\n",
    "gkf = list(GroupKFold(n_splits=3).split(train_X_opt,train_Y_opt,train_X.user_id))\n",
    "\n",
    "\n",
    "param_test1 = {\n",
    "    'gamma':[i/10.0 for i in range(0,15)]\n",
    "}\n",
    "\n",
    "gsearch1 = GridSearchCV(estimator = XGBClassifier( learning_rate =0.1, n_estimators=91, max_depth=7,\n",
    " min_child_weight=6, gamma=0.70, subsample=0.76, colsample_bytree=0.95,\n",
    " objective= 'binary:logistic', nthread=-1,scale_pos_weight=1, reg_lambda=20,seed=27), \n",
    " param_grid = param_test1, scoring='neg_log_loss',n_jobs=-1,iid=False, cv=gkf)\n",
    "\n",
    "gsearch1.fit(train_X_opt,train_Y_opt)\n",
    "\n",
    "print(timeit.default_timer() - start_time )\n",
    "\n",
    "gsearch1.grid_scores_, gsearch1.best_params_, gsearch1.best_score_\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# This shows that our original value of gamma, i.e. 0 is the optimum one. \n",
    "# Before proceeding, a good idea would be to re-calibrate the number of boosting rounds for the updated parameters.\n",
    "\n",
    "xgb1 = XGBClassifier(\n",
    " learning_rate =0.1,\n",
    " n_estimators=1000,\n",
    " max_depth=5,\n",
    " min_child_weight=3,\n",
    " gamma=0,\n",
    " subsample=0.8,\n",
    " colsample_bytree=0.8,\n",
    " objective= 'binary:logistic',\n",
    " nthread=6,\n",
    " scale_pos_weight=1,\n",
    " seed=27)\n",
    "\n",
    "xgb_param = xgb1.get_xgb_params()\n",
    "cvresult = xgb.cv(xgb_param, \n",
    "                  d_train_opt, \n",
    "                  num_boost_round=xgb1.get_params()['n_estimators'], \n",
    "                  nfold=5,\n",
    "                  metrics='logloss',\n",
    "                  early_stopping_rounds=50, \n",
    "                  verbose_eval=100)\n",
    "\n",
    "cvresult.tail(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# The next step would be try different subsample and colsample_bytree values. \n",
    "# Lets do this in 2 stages as well and take values 0.6,0.7,0.8,0.9 for both to start with.\n",
    "# BE CAREFUL ABOUT MAX AND MIN ALLOWED LEVELS OF EACH OF THE PARAMETERS\n",
    "\n",
    "start_time = timeit.default_timer()\n",
    "\n",
    "from sklearn.model_selection import GroupKFold\n",
    "from sklearn.grid_search import GridSearchCV\n",
    "from xgboost.sklearn import XGBClassifier\n",
    "\n",
    "gkf = list(GroupKFold(n_splits=3).split(train_X_opt,train_Y_opt,train_X.user_id))\n",
    "\n",
    "\n",
    "\n",
    "param_test1 = {\n",
    " 'subsample':[i/10.0 for i in range(6,10)],\n",
    " 'colsample_bytree':[i/10.0 for i in range(6,10)]\n",
    "}\n",
    "\n",
    "gsearch1 = GridSearchCV(estimator = XGBClassifier( learning_rate =0.1, n_estimators=126, max_depth=7,\n",
    " min_child_weight=6, gamma=0.1, subsample=0.76, colsample_bytree=0.95,\n",
    " objective= 'binary:logistic', nthread=-1,scale_pos_weight=1, reg_lambda=20,seed=27), \n",
    " param_grid = param_test1, scoring='neg_log_loss',n_jobs=-1,iid=False, cv=gkf)\n",
    "\n",
    "gsearch1.fit(train_X_opt,train_Y_opt)\n",
    "\n",
    "print(timeit.default_timer() - start_time )\n",
    "\n",
    "gsearch1.grid_scores_, gsearch1.best_params_, gsearch1.best_score_\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Here, we found 0.8 as the optimum value for both subsample and colsample_bytree. \n",
    "# Now we should try values in 0.05 interval around these.\n",
    "\n",
    "param_test5 = {\n",
    " 'subsample':[i/100.0 for i in range(75,90,5)],\n",
    " 'colsample_bytree':[i/100.0 for i in range(75,90,5)]\n",
    "}\n",
    "gsearch5 = GridSearchCV(estimator = XGBClassifier( learning_rate =0.1, n_estimators=177, max_depth=5,\n",
    " min_child_weight=3, gamma=0, subsample=0.8, colsample_bytree=0.8,\n",
    " objective= 'binary:logistic', nthread=-1, scale_pos_weight=1,seed=27), \n",
    " param_grid = param_test5, scoring='roc_auc',n_jobs=-1,iid=False, cv=3)\n",
    "gsearch5.fit(train_X_opt,train_Y_opt)\n",
    "gsearch5.grid_scores_, gsearch5.best_params_, gsearch5.best_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "param_test6 = {\n",
    " 'reg_alpha':[1e-5, 1e-2, 0.1, 1, 100]\n",
    "}\n",
    "gsearch6 = GridSearchCV(estimator = XGBClassifier( learning_rate =0.1, n_estimators=177, max_depth=4,\n",
    " min_child_weight=6, gamma=0.1, subsample=0.8, colsample_bytree=0.8,\n",
    " objective= 'binary:logistic', nthread=4, scale_pos_weight=1,seed=27), \n",
    " param_grid = param_test6, scoring='roc_auc',n_jobs=4,iid=False, cv=5)\n",
    "gsearch6.fit(train_X_opt,train_Y_opt)\n",
    "gsearch6.grid_scores_, gsearch6.best_params_, gsearch6.best_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "param_test7 = {\n",
    " 'reg_alpha':[0, 0.001, 0.005, 0.01, 0.05]\n",
    "}\n",
    "gsearch7 = GridSearchCV(estimator = XGBClassifier( learning_rate =0.1, n_estimators=177, max_depth=4,\n",
    " min_child_weight=6, gamma=0.1, subsample=0.8, colsample_bytree=0.8,\n",
    " objective= 'binary:logistic', nthread=4, scale_pos_weight=1,seed=27), \n",
    " param_grid = param_test7, scoring='roc_auc',n_jobs=4,iid=False, cv=5)\n",
    "gsearch7.fit(train_X_opt,train_Y_opt)\n",
    "gsearch7.grid_scores_, gsearch7.best_params_, gsearch7.best_score_\n",
    "\n",
    "# FINAL RESULT:\n",
    "# YOU should lower the learning rate and add more trees. with the optimum number of parameters"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# None prediction tuning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "start_time = timeit.default_timer()\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "# OPTIMIZATION STEP 2:\n",
    "# Tune max_depth and min_child_weight, try to increase these next time and maybe try a finer round\n",
    "from sklearn.model_selection import GroupKFold\n",
    "from sklearn.grid_search import GridSearchCV\n",
    "from xgboost.sklearn import XGBClassifier\n",
    "\n",
    "\n",
    "gkf = list(GroupKFold(n_splits=3).split(train_none_X.drop([ 'user_id', \n",
    "                                        'None_order','order_id','reordered'], axis=1), \n",
    "                                        train_none_X.None_order,\n",
    "                                        train_none_X.user_id))\n",
    "\n",
    "\n",
    "\n",
    "param_test1 = {\n",
    " 'max_depth':[5,6,7,8,9],\n",
    " 'min_child_weight':[6,8,10,12,14]\n",
    "}\n",
    "\n",
    "gsearch1 = GridSearchCV(estimator = XGBClassifier( learning_rate =0.1, n_estimators=158, max_depth=5,\n",
    " min_child_weight=14, gamma=0.1, subsample=0.76, colsample_bytree=0.95,\n",
    " objective= 'binary:logistic', nthread=-1,scale_pos_weight=1, reg_lambda=10,seed=27), \n",
    " param_grid = param_test1, scoring='neg_log_loss',n_jobs=-1,iid=False, cv=gkf)\n",
    "\n",
    "gsearch1.fit(train_none_X.drop([ 'user_id', \n",
    "                                        'None_order','order_id','reordered'], axis=1), \n",
    "                                        train_none_X.None_order)\n",
    "\n",
    "print(timeit.default_timer() - start_time )\n",
    "\n",
    "gsearch1.grid_scores_, gsearch1.best_params_, gsearch1.best_score_\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "start_time = timeit.default_timer()\n",
    "\n",
    "\n",
    "# OPTIMIZATION STEP 2:\n",
    "# Tune max_depth and min_child_weight, try to increase these next time and maybe try a finer round\n",
    "from sklearn.model_selection import GroupKFold\n",
    "from sklearn.grid_search import GridSearchCV\n",
    "from xgboost.sklearn import XGBClassifier\n",
    "\n",
    "\n",
    "gkf = list(GroupKFold(n_splits=3).split(train_none_X.drop([ 'user_id', \n",
    "                                        'None_order','order_id','reordered'], axis=1), \n",
    "                                        train_none_X.None_order,\n",
    "                                        train_none_X.user_id))\n",
    "\n",
    "\n",
    "\n",
    "param_test1 = {\n",
    "    'gamma':[i/10.0 for i in range(0,15)]\n",
    "}\n",
    "\n",
    "gsearch1 = GridSearchCV(estimator = XGBClassifier( learning_rate =0.1, n_estimators=158, max_depth=5,\n",
    " min_child_weight=14, gamma=0.1, subsample=0.76, colsample_bytree=0.95,\n",
    " objective= 'binary:logistic', nthread=-1,scale_pos_weight=1, reg_lambda=10,seed=27), \n",
    " param_grid = param_test1, scoring='neg_log_loss',n_jobs=-1,iid=False, cv=gkf)\n",
    "\n",
    "gsearch1.fit(train_none_X.drop([ 'user_id', \n",
    "                                        'None_order','order_id','reordered'], axis=1), \n",
    "                                        train_none_X.None_order)\n",
    "\n",
    "print(timeit.default_timer() - start_time )\n",
    "\n",
    "gsearch1.grid_scores_, gsearch1.best_params_, gsearch1.best_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "start_time = timeit.default_timer()\n",
    "\n",
    "\n",
    "# OPTIMIZATION STEP 2:\n",
    "# Tune max_depth and min_child_weight, try to increase these next time and maybe try a finer round\n",
    "from sklearn.model_selection import GroupKFold\n",
    "from sklearn.grid_search import GridSearchCV\n",
    "from xgboost.sklearn import XGBClassifier\n",
    "\n",
    "\n",
    "gkf = list(GroupKFold(n_splits=3).split(train_none_X.drop([ 'user_id', \n",
    "                                        'None_order','order_id','reordered'], axis=1), \n",
    "                                        train_none_X.None_order,\n",
    "                                        train_none_X.user_id))\n",
    "\n",
    "\n",
    "\n",
    "param_test1 = {\n",
    "     'colsample_bytree':[i/100.0 for i in range(55,85,10)]\n",
    "}\n",
    "\n",
    "gsearch1 = GridSearchCV(estimator = XGBClassifier( learning_rate =0.1, n_estimators=158, max_depth=5,\n",
    " min_child_weight=14, gamma=0.1, subsample=0.85, colsample_bytree=0.75,\n",
    " objective= 'binary:logistic', nthread=-1,scale_pos_weight=1, reg_lambda=10,seed=27), \n",
    " param_grid = param_test1, scoring='neg_log_loss',n_jobs=-1,iid=False, cv=gkf)\n",
    "\n",
    "gsearch1.fit(train_none_X.drop([ 'user_id', \n",
    "                                        'None_order','order_id','reordered'], axis=1), \n",
    "                                        train_none_X.None_order)\n",
    "\n",
    "print(timeit.default_timer() - start_time )\n",
    "\n",
    "gsearch1.grid_scores_, gsearch1.best_params_, gsearch1.best_score_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Light GBM optimization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_train_X = pd.read_csv('all_train_X_AH.csv')\n",
    "all_train_X.drop('Unnamed: 0',axis=1,inplace=True)\n",
    "#test_X = pd.read_csv('test_X_AH.csv')\n",
    "#test_X.drop('Unnamed: 0',axis=1,inplace=True)\n",
    "print(all_train_X.shape)\n",
    "#print(test_X.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_index, valid_index = list(GroupKFold(n_splits=4).split(all_train_X.drop(['eval_set', \n",
    "                                                                               'user_id', \n",
    "                                                                               'product_id', \n",
    "                                                                               'order_id',\n",
    "                                                                               'reordered'], axis=1),\n",
    "                                                  all_train_X.reordered,groups=all_train_X.user_id))[0]\n",
    "all_train_X = all_train_X.iloc[valid_index,:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "start_time = timeit.default_timer()\n",
    "\n",
    "from lightgbm import LGBMClassifier\n",
    "from sklearn.model_selection import GroupKFold\n",
    "from sklearn.grid_search import GridSearchCV \n",
    "\n",
    "gkf = list(GroupKFold(n_splits=3).split(all_train_X.drop(['eval_set', \n",
    "                                                            'user_id', \n",
    "                                                            'product_id', \n",
    "                                                            'order_id',\n",
    "                                                            'reordered'],axis=1), all_train_X.reordered,groups=all_train_X.user_id))\n",
    "           \n",
    "           \n",
    "param_test={'num_leaves':[216, 236, 256, 276,296],\n",
    "              'max_depth':[-1, 10, 12, 14],\n",
    "              'min_child_weight':[ 15, 20, 25]}\n",
    "           \n",
    "\n",
    "gsearch1 = GridSearchCV(estimator = LGBMClassifier( boosting_type='gbdt', num_leaves=256, feature_fraction=0.6,\n",
    "                                                   max_depth=12, learning_rate=0.1, \n",
    "n_estimators=260, max_bin=255, subsample_for_bin=50000, objective='binary', min_split_gain=0, min_child_weight=20, \n",
    "min_child_samples=10, subsample=1, subsample_freq=1, colsample_bytree=1, reg_alpha=0, reg_lambda=0, seed=0, \n",
    "                                              nthread=-1, silent=True), \n",
    "param_grid = param_test, scoring='neg_log_loss',n_jobs=-1,iid=False, cv=gkf)\n",
    "\n",
    "gsearch1.fit(all_train_X.drop(['eval_set', 'user_id', 'product_id', 'order_id','reordered'],axis=1), all_train_X.reordered)\n",
    "\n",
    "print(timeit.default_timer() - start_time )\n",
    "gsearch1.grid_scores_, gsearch1.best_params_, gsearch1.best_score_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "# 5 fold CV Functions"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Once the features are engineered, we can do 5-fold CV using the functions below"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# -*- coding: utf-8 -*-\n",
    "\"\"\"\n",
    "@author: Faron\n",
    "\"\"\"\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pylab as plt\n",
    "from datetime import datetime\n",
    "\n",
    "'''\n",
    "This kernel implements the O(nÂ²) F1-Score expectation maximization algorithm presented in\n",
    "\"Ye, N., Chai, K., Lee, W., and Chieu, H.  Optimizing F-measures: A Tale of Two Approaches. In ICML, 2012.\"\n",
    "\n",
    "It solves argmax_(0 <= k <= n,[[None]]) E[F1(P,k,[[None]])]\n",
    "with [[None]] being the indicator for predicting label \"None\"\n",
    "given posteriors P = [p_1, p_2, ... , p_n], where p_1 > p_2 > ... > p_n\n",
    "under label independence assumption by means of dynamic programming in O(nÂ²).\n",
    "'''\n",
    "\n",
    "class F1Optimizer():\n",
    "    def __init__(self):\n",
    "        pass\n",
    "\n",
    "    @staticmethod\n",
    "    def get_expectations(P, pNone=None):\n",
    "        expectations = []\n",
    "        P = np.sort(P)[::-1]\n",
    "\n",
    "        n = np.array(P).shape[0]\n",
    "        DP_C = np.zeros((n + 2, n + 1))\n",
    "        if pNone is None:\n",
    "            pNone = (1.0 - P).prod()\n",
    "\n",
    "        DP_C[0][0] = 1.0\n",
    "        for j in range(1, n):\n",
    "            DP_C[0][j] = (1.0 - P[j - 1]) * DP_C[0, j - 1]\n",
    "\n",
    "        for i in range(1, n + 1):\n",
    "            DP_C[i, i] = DP_C[i - 1, i - 1] * P[i - 1]\n",
    "            for j in range(i + 1, n + 1):\n",
    "                DP_C[i, j] = P[j - 1] * DP_C[i - 1, j - 1] + (1.0 - P[j - 1]) * DP_C[i, j - 1]\n",
    "\n",
    "        DP_S = np.zeros((2 * n + 1,))\n",
    "        DP_SNone = np.zeros((2 * n + 1,))\n",
    "        for i in range(1, 2 * n + 1):\n",
    "            DP_S[i] = 1. / (1. * i)\n",
    "            DP_SNone[i] = 1. / (1. * i + 1)\n",
    "        for k in range(n + 1)[::-1]:\n",
    "            f1 = 0\n",
    "            f1None = 0\n",
    "            for k1 in range(n + 1):\n",
    "                f1 += 2 * k1 * DP_C[k1][k] * DP_S[k + k1]\n",
    "                f1None += 2 * k1 * DP_C[k1][k] * DP_SNone[k + k1]\n",
    "            for i in range(1, 2 * k - 1):\n",
    "                DP_S[i] = (1 - P[k - 1]) * DP_S[i] + P[k - 1] * DP_S[i + 1]\n",
    "                DP_SNone[i] = (1 - P[k - 1]) * DP_SNone[i] + P[k - 1] * DP_SNone[i + 1]\n",
    "            expectations.append([f1None + 2 * pNone / (2 + k), f1])\n",
    "\n",
    "        return np.array(expectations[::-1]).T\n",
    "\n",
    "    @staticmethod\n",
    "    def maximize_expectation(P, pNone=None):\n",
    "        expectations = F1Optimizer.get_expectations(P, pNone)\n",
    "\n",
    "        ix_max = np.unravel_index(expectations.argmax(), expectations.shape)\n",
    "        max_f1 = expectations[ix_max]\n",
    "\n",
    "        predNone = True if ix_max[0] == 0 else False\n",
    "        best_k = ix_max[1]\n",
    "\n",
    "        return best_k, predNone, max_f1\n",
    "\n",
    "    @staticmethod\n",
    "    def _F1(tp, fp, fn):\n",
    "        return 2 * tp / (2 * tp + fp + fn)\n",
    "\n",
    "    @staticmethod\n",
    "    def _Fbeta(tp, fp, fn, beta=1.0):\n",
    "        beta_squared = beta ** 2\n",
    "        return (1.0 + beta_squared) * tp / ((1.0 + beta_squared) * tp + fp + beta_squared * fn)\n",
    "\n",
    "\n",
    "def print_best_prediction(P, pNone=None):\n",
    "    print(\"Maximize F1-Expectation\")\n",
    "    print(\"=\" * 23)\n",
    "    P = np.sort(P)[::-1]\n",
    "    n = P.shape[0]\n",
    "    L = ['L{}'.format(i + 1) for i in range(n)]\n",
    "\n",
    "    if pNone is None:\n",
    "        print(\"Estimate p(None|x) as (1-p_1)*(1-p_2)*...*(1-p_n)\")\n",
    "        pNone = (1.0 - P).prod()\n",
    "\n",
    "    PL = ['p({}|x)={}'.format(l, p) for l, p in zip(L, P)]\n",
    "    print(\"Posteriors: {} (n={})\".format(PL, n))\n",
    "    print(\"p(None|x)={}\".format(pNone))\n",
    "\n",
    "    opt = F1Optimizer.maximize_expectation(P, pNone)\n",
    "    best_prediction = ['None'] if opt[1] else []\n",
    "    best_prediction += (L[:opt[0]])\n",
    "    f1_max = opt[2]\n",
    "\n",
    "    print(\"Prediction {} yields best E[F1] of {}\\n\".format(best_prediction, f1_max))\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\n",
    "def prepare_valid_preds_faron(valid_X,valid_probs,valid_predicted_none_X,threshold):\n",
    "\n",
    "    valid_predicted_X = valid_X.copy()\n",
    "    valid_predicted_X.loc[:,'reordered'] = valid_probs\n",
    "\n",
    "    valid_predicted_X = valid_predicted_X.merge(valid_predicted_none_X[['order_id','predicted_none']],on='order_id',how='left')\n",
    "    valid_predicted_X.sort_values(['order_id','reordered'],ascending=False,inplace=True)\n",
    "\n",
    "\n",
    "    a = valid_predicted_X.groupby('order_id').apply(lambda x: F1Optimizer.maximize_expectation([prob if prob>threshold else 0 for prob in x.reordered.tolist()])[0])\n",
    "    \n",
    "    # a = valid_predicted_X.groupby('order_id').apply(lambda x: F1Optimizer.maximize_expectation(x.reordered.tolist(), \n",
    "    #                                                                                       x.predicted_none.tolist()[0])[0])\n",
    "    a = pd.DataFrame(a,columns=['top_k'])\n",
    "\n",
    "    #b = valid_predicted_X.groupby('order_id').apply(lambda x: F1Optimizer.maximize_expectation(x.reordered.tolist(), \n",
    "    #                                                                                           x.predicted_none.tolist()[0])[1])\n",
    "    \n",
    "    b = valid_predicted_X.groupby('order_id').apply(lambda x: F1Optimizer.maximize_expectation([prob if prob>threshold else 0 for prob in x.reordered.tolist()])[1])\n",
    "    b = pd.DataFrame(b,columns=['None_product'])\n",
    "\n",
    "    c = pd.DataFrame(valid_predicted_X.groupby('order_id').apply(lambda x: x.product_id.tolist()),columns=['products'])\n",
    "\n",
    "    c['top_k'] = a['top_k']\n",
    "    c['None_product'] = b['None_product']\n",
    "\n",
    "    valid_submit = pd.DataFrame(c.apply(lambda x:' '.join([str(item) for index,item in enumerate(x['products']) if index<x['top_k']]),axis=1),columns=['products'])\n",
    "    valid_submit.index = valid_submit.index.astype(int)\n",
    "\n",
    "    valid_submit.loc[valid_predicted_none_X['predicted_none']==1,'products'] = 'None ' + valid_submit['products']\n",
    "\n",
    "\n",
    "    valid_submit.loc[valid_submit['products']==' ','products'] = 'None'\n",
    "\n",
    "    valid_submit['products'] = valid_submit['products'].str.strip()\n",
    "\n",
    "    print('valid_submit:',valid_submit.shape)\n",
    "    print('valid_submit:',valid_submit[:20])\n",
    "    \n",
    "    return valid_submit\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def prepare_none_data(train_X,valid_X,test_X):\n",
    "    \n",
    "    data_tmp = train_X.groupby('user_id').agg({'reordered' : lambda x: 1!=max(x) }).reset_index()\n",
    "    data_tmp.rename(index=str,columns={'reordered':'None_order'},inplace=True)\n",
    "    train_none_X = train_X.merge(right=data_tmp,on='user_id',how='left').groupby('user_id').mean().reset_index(inplace=False)\n",
    "\n",
    "    data_tmp = valid_X.groupby('user_id').agg({'reordered' : lambda x: 1!=max(x) }).reset_index()\n",
    "    data_tmp.rename(index=str,columns={'reordered':'None_order'},inplace=True)\n",
    "    valid_none_X = valid_X.merge(right=data_tmp,on='user_id',how='left').groupby('user_id').mean().reset_index(inplace=False)\n",
    "\n",
    "    data_tmp = test_X.groupby('user_id').agg({'reordered' : lambda x: 1!=max(x) }).reset_index()\n",
    "    data_tmp.rename(index=str,columns={'reordered':'None_order'},inplace=True)\n",
    "    test_none_X = test_X.merge(right=data_tmp,on='user_id',how='left').groupby('user_id').mean().reset_index(inplace=False)\n",
    "    \n",
    "\n",
    "    print('Percent None orders')\n",
    "    print(np.mean(train_none_X.None_order))\n",
    "#     print(train_none_X.columns)\n",
    "#     print(valid_none_X.columns)\n",
    "#     print(test_none_X.columns)\n",
    "\n",
    "    print('train_none_X size', train_none_X.shape)\n",
    "    print('valid_none_X size',valid_none_X.shape)\n",
    "    print('test_none_X size',test_none_X.shape)\n",
    "    \n",
    "    return train_none_X,valid_none_X,test_none_X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def prepare_none_data_test(test_X):\n",
    "    \n",
    "\n",
    "    data_tmp = test_X.groupby('user_id').agg({'reordered' : lambda x: 1!=max(x) }).reset_index()\n",
    "    data_tmp.rename(index=str,columns={'reordered':'None_order'},inplace=True)\n",
    "    test_none_X = test_X.merge(right=data_tmp,on='user_id',how='left').groupby('user_id').mean().reset_index(inplace=False)\n",
    "    \n",
    "    print('test_none_X size',test_none_X.shape)\n",
    "    \n",
    "    return test_none_X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "from jnius import autoclass\n",
    "\n",
    "\n",
    "def prepare_valid_preds(valid_X,valid_probs,valid_predicted_none_X,threshold):\n",
    "\n",
    "    \n",
    "\n",
    "    FScore = autoclass('FScore')\n",
    "\n",
    "    fscore = FScore()\n",
    "\n",
    "    # f1 score based on maximum f1 score optimization\n",
    "\n",
    "    valid_predicted_X = valid_X[['user_id','order_id','product_id']].copy()\n",
    "    valid_predicted_X.loc[:,'reordered'] = valid_probs\n",
    "    # Predictions with catboost\n",
    "    #valid_predicted_X.loc[:,'reordered'] = cb_model.predict(valid_XX,prediction_type='RawFormulaVal')\n",
    "\n",
    "    a = valid_predicted_X.groupby('order_id').apply(lambda x: \n",
    "                                                    fscore.max_expected_fscore_preds_cube([prob if prob>threshold else 0 for prob in x.reordered.tolist()],1))\n",
    "    a = pd.DataFrame(a,columns=['labels'])\n",
    "    b = pd.DataFrame(valid_predicted_X.groupby('order_id').apply(lambda x: x.product_id.tolist()),columns=['products'])\n",
    "    b['labels'] = a['labels']\n",
    "    valid_submit_pros = pd.DataFrame(b.apply(lambda x: ' '.join([str(item) for item,label in zip(x['products'],x['labels']) if label==1]),axis=1),columns=['products'])\n",
    "    valid_submit_pros.index = valid_submit_pros.index.astype(int)\n",
    "    valid_submit_pros.head()\n",
    "\n",
    "    #ADD NONE TO ORDER WITH NO PREDICTED PRODUCTS\n",
    "    valid_submit_nones = valid_predicted_X.groupby('order_id').agg({'reordered':'max','product_id':'min'})\n",
    "    valid_submit_nones = valid_submit_nones[valid_submit_nones.reordered==0]\n",
    "    valid_submit_nones.drop('reordered',axis=1,inplace=True)\n",
    "    valid_submit_nones.rename(index=str, columns={'product_id':'products'},inplace=True)\n",
    "    valid_submit_nones.index = valid_submit_nones.index.astype(int)\n",
    "    valid_submit_nones.products=''\n",
    "\n",
    "    valid_submit = pd.concat([valid_submit_pros,valid_submit_nones])\n",
    "    valid_submit.sort_index(inplace=True)\n",
    "    valid_submit.products = valid_submit.products.astype(str)\n",
    "\n",
    "\n",
    "    # Add extra none order prediction\n",
    "\n",
    "    valid_submit.loc[valid_predicted_none_X['predicted_none']==1,'products'] = 'None ' + valid_submit['products']\n",
    "\n",
    "\n",
    "    valid_submit.loc[valid_submit['products']=='','products'] = 'None'\n",
    "\n",
    "    valid_submit['products'] = valid_submit['products'].str.strip()\n",
    "\n",
    "    print('valid_submit:',valid_submit.shape)\n",
    "    print('valid_submit:',valid_submit[:20])\n",
    "    \n",
    "    return valid_submit"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def prepare_ground_truth_simple(valid_ordered_products):\n",
    "    \n",
    "    #PROCESS GROUND TRUTH\n",
    "\n",
    "    #valid_ordered_products = train[train.isin({'user_id': valid_X.user_id.tolist()}).user_id==True]\n",
    "\n",
    "    print('Reordered distribution:')\n",
    "    print(np.mean(valid_ordered_products.reordered))\n",
    "    train_gtl = []\n",
    "\n",
    "    for uid, subset in valid_ordered_products.groupby('user_id'):\n",
    "        subset1 = subset[subset.reordered == 1]\n",
    "        oid = subset.order_id.values[0]\n",
    "\n",
    "        if len(subset1) == 0:\n",
    "            train_gtl.append((oid, 'None'))\n",
    "            continue\n",
    "\n",
    "        ostr = ' '.join([str(int(e)) for e in subset1.product_id.values])\n",
    "        # .strip is needed because join can have a padding space at the end\n",
    "        train_gtl.append((oid, ostr.strip()))\n",
    "\n",
    "    print(len(train_gtl))\n",
    "    df_valid_gt = pd.DataFrame(train_gtl)\n",
    "\n",
    "    df_valid_gt.columns = ['order_id', 'products']\n",
    "    df_valid_gt.set_index('order_id', inplace=True)\n",
    "    df_valid_gt.sort_index(inplace=True)\n",
    "\n",
    "\n",
    "    print(df_valid_gt.shape)\n",
    "    df_valid_gt.sort_index(inplace=True)\n",
    "    df_valid_gt.products = df_valid_gt.products.astype(str)\n",
    "    \n",
    "    print('df_valid_gt:',df_valid_gt.shape)\n",
    "    print('df_valid_gt:',df_valid_gt[:5])\n",
    "    \n",
    "    return df_valid_gt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def prepare_ground_truth(train,valid_X):\n",
    "    \n",
    "    #PROCESS GROUND TRUTH\n",
    "\n",
    "    valid_ordered_products = train[train.isin({'user_id': valid_X.user_id.tolist()}).user_id==True]\n",
    "\n",
    "    print('regenerating')\n",
    "    train_gtl = []\n",
    "\n",
    "    for uid, subset in valid_ordered_products.groupby('user_id'):\n",
    "        subset1 = subset[subset.reordered == 1]\n",
    "        oid = subset.order_id.values[0]\n",
    "\n",
    "        if len(subset1) == 0:\n",
    "            train_gtl.append((oid, 'None'))\n",
    "            continue\n",
    "\n",
    "        ostr = ' '.join([str(int(e)) for e in subset1.product_id.values])\n",
    "        # .strip is needed because join can have a padding space at the end\n",
    "        train_gtl.append((oid, ostr.strip()))\n",
    "\n",
    "    print(len(train_gtl))\n",
    "    df_valid_gt = pd.DataFrame(train_gtl)\n",
    "\n",
    "    df_valid_gt.columns = ['order_id', 'products']\n",
    "    df_valid_gt.set_index('order_id', inplace=True)\n",
    "    df_valid_gt.sort_index(inplace=True)\n",
    "\n",
    "\n",
    "    print(df_valid_gt.shape)\n",
    "    df_valid_gt.sort_index(inplace=True)\n",
    "    df_valid_gt.products = df_valid_gt.products.astype(str)\n",
    "    \n",
    "    print('df_valid_gt:',df_valid_gt.shape)\n",
    "    print('df_valid_gt:',df_valid_gt[:5])\n",
    "    \n",
    "    return df_valid_gt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def print_F1_score(valid_submit,df_valid_gt):\n",
    "\n",
    "    f1 = []\n",
    "    for gt, pred in zip(df_valid_gt.sort_index().products, valid_submit.sort_index().products):\n",
    "        lgt = gt.replace(\"None\", \"-1\").split(' ')\n",
    "        lpred = pred.replace(\"None\", \"-1\").split(' ')\n",
    "\n",
    "        rr = (np.intersect1d(lgt, lpred))\n",
    "        precision = np.float(len(rr)) / len(lpred)\n",
    "        recall = np.float(len(rr)) / len(lgt)\n",
    "\n",
    "        denom = precision + recall\n",
    "        f1.append(((2 * precision * recall) / denom) if denom > 0 else 0)\n",
    "\n",
    "    valid_score = np.mean(f1)\n",
    "    print('validation score:',valid_score)\n",
    "\n",
    "    return"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Prepare None predictions\n",
    "\n",
    "def prepare_None_predictions(valid_none_X,valid_none_probs,none_threshold):\n",
    "\n",
    "    valid_predicted_none_X = valid_none_X[['user_id','order_id']].copy()\n",
    "    valid_predicted_none_X.loc[:,'predicted_none'] = ( valid_none_probs > none_threshold).astype(int)\n",
    "    valid_predicted_none_X.sort_values('order_id',inplace=True)\n",
    "    valid_predicted_none_X.set_index('order_id', inplace=True)\n",
    "    print('valid_predicted_none_X.shape:',valid_predicted_none_X.shape)\n",
    "    valid_predicted_none_X.head()\n",
    "    \n",
    "    return valid_predicted_none_X\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Prepare None predictions\n",
    "\n",
    "def prepare_None_predictions_faron(valid_none_X,valid_none_probs,none_threshold):\n",
    "\n",
    "    valid_predicted_none_X = valid_none_X[['user_id','order_id']].copy()\n",
    "    valid_predicted_none_X.loc[:,'predicted_none'] = ( valid_none_probs > none_threshold).astype(int)\n",
    "    valid_predicted_none_X.sort_values('order_id',inplace=True)\n",
    "    #valid_predicted_none_X.set_index('order_id', inplace=True)\n",
    "    print('valid_predicted_none_X.shape:',valid_predicted_none_X.shape)\n",
    "    valid_predicted_none_X.head()\n",
    "    \n",
    "    return valid_predicted_none_X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# TEST DATA FOR xgboost\n",
    "# F1 SCORE MAXIMIZATION THREHSOLD PREDICTIONS\n",
    "# None PREDICTIONS\n",
    "\n",
    "from jnius import autoclass\n",
    "\n",
    " \n",
    "\n",
    "def prepare_test_predictions(test_X,text_none_X,test_probs,test_none_probs,none_threshold,threshold):\n",
    "    valid_predicted_none_X = test_none_X.copy()\n",
    "    valid_predicted_none_X.loc[:,'predicted_none'] = ( test_none_probs> none_threshold).astype(int)\n",
    "    valid_predicted_none_X.sort_values('order_id',inplace=True)\n",
    "    valid_predicted_none_X.set_index('order_id', inplace=True)\n",
    "    print(valid_predicted_none_X.shape)\n",
    "\n",
    "    FScore = autoclass('FScore')\n",
    "\n",
    "    fscore = FScore()\n",
    "\n",
    "    valid_predicted_X = test_X[['user_id','order_id','product_id']].copy()\n",
    "    valid_predicted_X.loc[:,'reordered'] = test_probs\n",
    "\n",
    "    a = valid_predicted_X.groupby('order_id').apply(lambda x: \n",
    "                                                    fscore.max_expected_fscore_preds_cube([prob if prob>threshold else 0 for prob in x.reordered.tolist()],1))\n",
    "    a = pd.DataFrame(a,columns=['labels'])\n",
    "    b = pd.DataFrame(valid_predicted_X.groupby('order_id').apply(lambda x: x.product_id.tolist()),columns=['products'])\n",
    "    b['labels'] = a['labels']\n",
    "    valid_submit_pros = pd.DataFrame(b.apply(lambda x: ' '.join([str(item) for item,label in zip(x['products'],x['labels']) if label==1]),axis=1),columns=['products'])\n",
    "    valid_submit_pros.index = valid_submit_pros.index.astype(int)\n",
    "    valid_submit_pros.head()\n",
    "\n",
    "\n",
    "\n",
    "    #ADD NONE TO ORDER WITH NO PREDICTED PRODUCTS\n",
    "    valid_submit_nones = valid_predicted_X.groupby('order_id').agg({'reordered':'max','product_id':'min'})\n",
    "    valid_submit_nones = valid_submit_nones[valid_submit_nones.reordered==0]\n",
    "    valid_submit_nones.drop('reordered',axis=1,inplace=True)\n",
    "    valid_submit_nones.rename(index=str, columns={'product_id':'products'},inplace=True)\n",
    "    valid_submit_nones.index = valid_submit_nones.index.astype(int)\n",
    "    valid_submit_nones.products=''\n",
    "\n",
    "    valid_submit = pd.concat([valid_submit_pros,valid_submit_nones])\n",
    "    valid_submit.sort_index(inplace=True)\n",
    "    valid_submit.products = valid_submit.products.astype(str)\n",
    "\n",
    "\n",
    "    # Add extra none order prediction\n",
    "\n",
    "    valid_submit.loc[valid_predicted_none_X['predicted_none']==1,'products'] = 'None ' + valid_submit['products']\n",
    "\n",
    "    valid_submit.loc[valid_submit['products']=='','products'] = 'None'\n",
    "\n",
    "    valid_submit['products'] = valid_submit['products'].str.strip()\n",
    "\n",
    "    print(valid_submit.shape)\n",
    "    valid_submit.to_csv('test_submit.csv')\n",
    "    print(valid_submit[:5])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Predictions with lightGBM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_index, valid_index = list(GroupKFold(n_splits=3).split(all_train_X.drop(['eval_set', \n",
    "                                                                               'user_id', \n",
    "                                                                               'product_id', \n",
    "                                                                               'order_id',\n",
    "                                                                               'reordered'], axis=1),\n",
    "                                                  all_train_X.reordered,groups=all_train_X.user_id))[0]\n",
    "all_train_X = all_train_X.iloc[valid_index,:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#\n",
    "\n",
    "import lightgbm as lgb\n",
    "\n",
    "#################################\n",
    "\n",
    "xgb_none_params = {\n",
    "    \"objective\"         : \"reg:logistic\"\n",
    "    ,\"eval_metric\"      : \"logloss\"\n",
    "    ,\"eta\"              : 0.05\n",
    "    ,\"max_depth\"        : 5\n",
    "    ,\"min_child_weight\" :14\n",
    "    ,\"gamma\"            :0.05\n",
    "    ,\"subsample\"        :0.85\n",
    "    ,\"colsample_bytree\" :0.75\n",
    "    ,\"alpha\"            :2e-05\n",
    "    ,\"lambda\"           :10\n",
    "}\n",
    "\n",
    "num_round = 620\n",
    "learning_rates= [0.05 if x<350 else 0.01 for x in range(num_round)]\n",
    "\n",
    "light_params = {\n",
    "    'task': 'train',\n",
    "    'boosting_type': 'gbdt',\n",
    "    'objective': 'binary',\n",
    "     'early_stopping_rounds':10,\n",
    "    'metric': {'binary_logloss'},\n",
    "    'num_leaves': 256,\n",
    "    'min_sum_hessian_in_leaf': 20,\n",
    "    'max_depth': -1,\n",
    "    'feature_fraction': 0.50,\n",
    "    # 'bagging_fraction': 0.9,\n",
    "    # 'bagging_freq': 3,\n",
    "    'verbose': 1,\n",
    "    'min_gain_to_split':0,\n",
    "    'max_bin':255\n",
    "}\n",
    "\n",
    "\n",
    "none_threshold = 0.18\n",
    "\n",
    "n_splits = 4\n",
    "group_kfold = GroupKFold(n_splits=n_splits)\n",
    "\n",
    "test_probs = np.ndarray(shape=(n_splits,test_X.shape[0]))\n",
    "test_none_probs = np.ndarray(shape=(n_splits,test_X.groupby('user_id').size().shape[0]))\n",
    "\n",
    "\n",
    "split_index = 0\n",
    "#stopping_rounds = [400,500,400,500]\n",
    "\n",
    "for train_index, valid_index in group_kfold.split(all_train_X.drop(['eval_set', 'user_id', \n",
    "                                            'product_id', 'order_id',\n",
    "                                            'reordered'], axis=1),\n",
    "                                                  all_train_X.reordered,groups=all_train_X.user_id):\n",
    "    \n",
    "    #train_X = all_train_X.iloc[train_index,:]\n",
    "    valid_X = all_train_X.iloc[valid_index,:]\n",
    "    \n",
    "    train_none_X,valid_none_X,test_none_X = prepare_none_data(all_train_X.iloc[train_index,:],valid_X,test_X)\n",
    "    \n",
    "    #####################################################################\n",
    "    #Start training for none_orders\n",
    "    d_train_none = xgb.DMatrix(train_none_X.drop([ 'user_id', \n",
    "                                        'None_order','order_id','reordered'], axis=1), train_none_X.None_order)\n",
    "\n",
    "    d_valid_none = xgb.DMatrix(valid_none_X.drop(['user_id', \n",
    "                                            'None_order','order_id','reordered'], axis=1), valid_none_X.None_order)\n",
    "\n",
    "\n",
    "    \n",
    "\n",
    "    watchlist= [(d_valid_none, 'valid')]\n",
    "\n",
    "    bst_none = xgb.train(params=xgb_none_params, \n",
    "                        dtrain=d_train_none, \n",
    "                        num_boost_round=811, evals=watchlist, \n",
    "                        early_stopping_rounds=10, \n",
    "                        verbose_eval=100)\n",
    "    \n",
    "    d_test_none = xgb.DMatrix(test_none_X.drop(['user_id', \n",
    "                                        'None_order','order_id','reordered'], axis=1))\n",
    "    \n",
    "    valid_none_probs = bst_none.predict(d_valid_none)\n",
    "    \n",
    "    \n",
    "    \n",
    "    ####################################################################\n",
    "    # Start training for LightGBM\n",
    "    lgb_train_data = lgb.Dataset(all_train_X.iloc[train_index,:].drop(['eval_set', 'user_id', \n",
    "                                            'product_id', 'order_id',\n",
    "                                            'reordered'], axis=1), label=all_train_X.iloc[train_index,:].reordered)\n",
    "\n",
    "    lgb_valid_data = lgb.Dataset(valid_X.drop(['eval_set', 'user_id', \n",
    "                                            'product_id', 'order_id',\n",
    "                                            'reordered'], axis=1), label=valid_X.reordered)\n",
    "\n",
    "\n",
    "    lgb_bst = lgb.train(light_params, \n",
    "                        lgb_train_data, \n",
    "                        num_round, \n",
    "                        valid_sets=[lgb_valid_data],\n",
    "                        verbose_eval=20,learning_rates=learning_rates)\n",
    "\n",
    "    valid_probs = lgb_bst.predict(valid_X.drop(['eval_set', 'user_id', \n",
    "                                            'product_id', 'order_id',\n",
    "                                            'reordered'], axis=1))\n",
    "\n",
    "    ####################################################################################\n",
    "    \n",
    "    valid_predicted_none_X = prepare_None_predictions(valid_none_X,valid_none_probs,none_threshold)\n",
    "    valid_submit = prepare_valid_preds(valid_X,valid_probs,valid_predicted_none_X,0)\n",
    "    df_valid_gt = prepare_ground_truth_simple(valid_X)\n",
    "    print_F1_score(valid_submit,df_valid_gt)\n",
    "    \n",
    "    \n",
    "    test_probs[split_index,:] = lgb_bst.predict(test_X.drop(['eval_set', 'user_id', \n",
    "                                            'product_id', 'order_id',\n",
    "                                            'reordered'], axis=1))\n",
    "    test_none_probs[split_index,:] = bst_none.predict(d_test_none)\n",
    "    np.savez('10CV_light_probs.npz',  test_probs=test_probs,test_none_probs=test_none_probs)\n",
    "\n",
    "    print('finished '+str(split_index)+' split')\n",
    "    print()\n",
    "    \n",
    "    split_index+=1\n",
    "    \n",
    "    if split_index==1:\n",
    "        test_probs = test_probs[0]\n",
    "        test_none_probs = test_none_probs[0]\n",
    "        break\n",
    "        \n",
    "    \n",
    "test_none_probs = np.mean(test_none_probs,axis=0)\n",
    "test_probs = np.mean(test_probs,axis=0)\n",
    "    \n",
    "#prepare_test_predictions(test_X,test_none_X,test_probs,test_none_probs,none_threshold) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(75000, 139)\n",
      "(75000, 1)\n",
      "                                                   products\n",
      "order_id                                                   \n",
      "17                      21709 47766 38777 21463 26429 13107\n",
      "34        13176 47766 47792 21137 43504 39180 39475 1608...\n",
      "137                 41787 24852 5134 38689 25890 2326 23794\n",
      "182       47209 11520 39275 13629 47672 5479 33000 41149...\n",
      "257       49235 24852 27966 37646 21137 24838 27104 4501...\n"
     ]
    }
   ],
   "source": [
    "# Predict with a different None threshold\n",
    "probs_dict = np.load('10CV_light_probs.npz')\n",
    "test_probs=probs_dict['test_probs']\n",
    "test_none_probs=probs_dict['test_none_probs']\n",
    "test_probs = test_probs[0]\n",
    "test_none_probs = test_none_probs[0]\n",
    "none_threshold = 0.19\n",
    "prepare_test_predictions(test_X,test_none_X,test_probs,test_none_probs,none_threshold,0.005) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ax = lgb.plot_importance(lgb_bst, height=0.2, xlim=None, ylim=None, title='Feature importance', \n",
    "                         xlabel='Feature importance', ylabel='Features', importance_type='gain', \n",
    "                         max_num_features=None, ignore_zero=True, figsize=(20,20), grid=False)\n",
    "\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}