From 4ac06866071f24bbd5b840498fb03fffa6a8d215 Mon Sep 17 00:00:00 2001
From: benjas <909336740@qq.com>
Date: Mon, 25 Jan 2021 11:17:42 +0800
Subject: [PATCH] Update. Create the Item table

---
 .../数据清洗.ipynb                        | 1743 ++++++++++++++++-
 1 file changed, 1712 insertions(+), 31 deletions(-)

diff --git a/机器学习竞赛实战_优胜解决方案/京东用户购买意向预测/数据清洗.ipynb b/机器学习竞赛实战_优胜解决方案/京东用户购买意向预测/数据清洗.ipynb
index fc69279..2224c2f 100644
--- a/机器学习竞赛实战_优胜解决方案/京东用户购买意向预测/数据清洗.ipynb
+++ b/机器学习竞赛实战_优胜解决方案/京东用户购买意向预测/数据清洗.ipynb
@@ -1488,7 +1488,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 3,
+   "execution_count": 11,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -1500,7 +1500,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 4,
+   "execution_count": 12,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -1516,14 +1516,14 @@
     "    group['favor_num'] = type_cnt[5]\n",
     "    group['click_num'] = type_cnt[6]\n",
     " \n",
-    "    return group[['user_id', 'browse_num', 'addcart_num',\n",
+    "    return group[['sku_id', 'browse_num', 'addcart_num',\n",
     "                  'delcart_num', 'buy_num', 'favor_num',\n",
     "                  'click_num']]"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 5,
+   "execution_count": 13,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -1534,22 +1534,22 @@
     "    loop = True\n",
     "    while loop:\n",
     "        try:\n",
-    "            chunk = reader.get_chunk(chunk_size)[[\"user_id\", \"type\"]]\n",
+    "            chunk = reader.get_chunk(chunk_size)[[\"sku_id\", \"type\"]]\n",
     "            chunks.append(chunk)\n",
     "        except StopIteration:\n",
     "            loop = False\n",
     "            print(\"Iteration is stopped\")\n",
     "            \n",
     "    df_ac = pd.concat(chunks, ignore_index=True)\n",
-    "    df_ac = df_ac.groupby(['user_id'], as_index=False).apply(add_type_count)\n",
-    "    df_ac = df_ac.drop_duplicates('user_id')\n",
+    "    df_ac = df_ac.groupby(['sku_id'], as_index=False).apply(add_type_count)\n",
+    "    df_ac = df_ac.drop_duplicates('sku_id')\n",
     " \n",
     "    return df_ac"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 6,
+   "execution_count": 14,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -1567,7 +1567,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 7,
+   "execution_count": 15,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -1578,7 +1578,7 @@
     "    df_ac.append(get_from_action_data(fname=ACTION_201604_FILE))\n",
     " \n",
     "    df_ac = pd.concat(df_ac, ignore_index=True)\n",
-    "    df_ac = df_ac.groupby(['user_id'], as_index=False).sum()\n",
+    "    df_ac = df_ac.groupby(['sku_id'], as_index=False).sum()\n",
     "\n",
     "    df_ac['buy_addcart_ratio'] = df_ac['buy_num'] / df_ac['addcart_num']\n",
     "    df_ac['buy_browse_ratio'] = df_ac['buy_num'] / df_ac['browse_num']\n",
@@ -1595,45 +1595,1726 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 16,
    "metadata": {},
    "outputs": [
     {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Iteration is stopped\n"
-     ]
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>sku_id</th>\n",
+       "      <th>a1</th>\n",
+       "      <th>a2</th>\n",
+       "      <th>a3</th>\n",
+       "      <th>cate</th>\n",
+       "      <th>brand</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>10</td>\n",
+       "      <td>3</td>\n",
+       "      <td>1</td>\n",
+       "      <td>1</td>\n",
+       "      <td>8</td>\n",
+       "      <td>489</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>100002</td>\n",
+       "      <td>3</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "      <td>8</td>\n",
+       "      <td>489</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>100003</td>\n",
+       "      <td>1</td>\n",
+       "      <td>-1</td>\n",
+       "      <td>-1</td>\n",
+       "      <td>8</td>\n",
+       "      <td>30</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>100006</td>\n",
+       "      <td>1</td>\n",
+       "      <td>2</td>\n",
+       "      <td>1</td>\n",
+       "      <td>8</td>\n",
+       "      <td>545</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>10001</td>\n",
+       "      <td>-1</td>\n",
+       "      <td>1</td>\n",
+       "      <td>2</td>\n",
+       "      <td>8</td>\n",
+       "      <td>244</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "   sku_id  a1  a2  a3  cate  brand\n",
+       "0      10   3   1   1     8    489\n",
+       "1  100002   3   2   2     8    489\n",
+       "2  100003   1  -1  -1     8     30\n",
+       "3  100006   1   2   1     8    545\n",
+       "4   10001  -1   1   2     8    244"
+      ]
+     },
+     "execution_count": 16,
+     "metadata": {},
+     "output_type": "execute_result"
     }
    ],
    "source": [
     "item_base = get_from_jdata_product()\n",
-    "item_behavior = merge_action_data()\n",
-    "item_comment = get_from_jdata_comment()\n",
-    "\n",
-    "# SQL: left join\n",
-    "item_behavior = pd.merge(item_base, item_behavior, on=['sku_id'], how='left')\n",
-    "item_behavior = pd.merge(item_behavior, item_comment, on=['sku_id'], how='left')\n",
-    " \n",
-    "item_behavior.to_csv(ITEM_TABLE_FILE, index=False)"
+    "item_base.head()"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 17,
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Iteration is stopped\n",
+      "Iteration is stopped\n",
+      "Iteration is stopped\n"
+     ]
+    },
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>sku_id</th>\n",
+       "      <th>browse_num</th>\n",
+       "      <th>addcart_num</th>\n",
+       "      <th>delcart_num</th>\n",
+       "      <th>buy_num</th>\n",
+       "      <th>favor_num</th>\n",
+       "      <th>click_num</th>\n",
+       "      <th>buy_addcart_ratio</th>\n",
+       "      <th>buy_browse_ratio</th>\n",
+       "      <th>buy_click_ratio</th>\n",
+       "      <th>buy_favor_ratio</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>2</td>\n",
+       "      <td>55</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>79</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>NaN</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>18</td>\n",
+       "      <td>2</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>2</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>NaN</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>36</td>\n",
+       "      <td>107</td>\n",
+       "      <td>4</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>1</td>\n",
+       "      <td>186</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>0.0</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>37</td>\n",
+       "      <td>5</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>10</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>NaN</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>40</td>\n",
+       "      <td>79</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0</td>\n",
+       "      <td>179</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>0.0</td>\n",
+       "      <td>NaN</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "   sku_id  browse_num  addcart_num  delcart_num  buy_num  favor_num  \\\n",
+       "0       2          55            0            0        0          0   \n",
+       "1      18           2            0            0        0          0   \n",
+       "2      36         107            4            0        0          1   \n",
+       "3      37           5            0            0        0          0   \n",
+       "4      40          79            2            2        0          0   \n",
+       "\n",
+       "   click_num  buy_addcart_ratio  buy_browse_ratio  buy_click_ratio  \\\n",
+       "0         79                NaN               0.0              0.0   \n",
+       "1          2                NaN               0.0              0.0   \n",
+       "2        186                0.0               0.0              0.0   \n",
+       "3         10                NaN               0.0              0.0   \n",
+       "4        179                0.0               0.0              0.0   \n",
+       "\n",
+       "   buy_favor_ratio  \n",
+       "0              NaN  \n",
+       "1              NaN  \n",
+       "2              0.0  \n",
+       "3              NaN  \n",
+       "4              NaN  "
+      ]
+     },
+     "execution_count": 17,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
    "source": [
-    "item_table = pd.read_csv(ITEM_TABLE_FILE)\n",
-    "item_table = haed()"
+    "item_behavior = merge_action_data()\n",
+    "item_behavior.head()"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 18,
    "metadata": {},
-   "outputs": [],
-   "source": []
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>sku_id</th>\n",
+       "      <th>comment_num</th>\n",
+       "      <th>has_bad_comment</th>\n",
+       "      <th>bad_comment_rate</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>512006</th>\n",
+       "      <td>1000</td>\n",
+       "      <td>3</td>\n",
+       "      <td>1</td>\n",
+       "      <td>0.0417</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>512007</th>\n",
+       "      <td>10000</td>\n",
+       "      <td>2</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0.0000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>512008</th>\n",
+       "      <td>100011</td>\n",
+       "      <td>4</td>\n",
+       "      <td>1</td>\n",
+       "      <td>0.0376</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>512009</th>\n",
+       "      <td>100018</td>\n",
+       "      <td>3</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0.0000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>512010</th>\n",
+       "      <td>100020</td>\n",
+       "      <td>3</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0.0000</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "        sku_id  comment_num  has_bad_comment  bad_comment_rate\n",
+       "512006    1000            3                1            0.0417\n",
+       "512007   10000            2                0            0.0000\n",
+       "512008  100011            4                1            0.0376\n",
+       "512009  100018            3                0            0.0000\n",
+       "512010  100020            3                0            0.0000"
+      ]
+     },
+     "execution_count": 18,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "item_comment = get_from_jdata_comment()\n",
+    "item_comment.head()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 19,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# SQL: left join\n",
+    "item_behavior = pd.merge(item_base, item_behavior, on=['sku_id'], how='left')\n",
+    "item_behavior = pd.merge(item_behavior, item_comment, on=['sku_id'], how='left')\n",
+    " \n",
+    "item_behavior.to_csv(ITEM_TABLE_FILE, index=False)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 23,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>sku_id</th>\n",
+       "      <th>a1</th>\n",
+       "      <th>a2</th>\n",
+       "      <th>a3</th>\n",
+       "      <th>cate</th>\n",
+       "      <th>brand</th>\n",
+       "      <th>browse_num</th>\n",
+       "      <th>addcart_num</th>\n",
+       "      <th>delcart_num</th>\n",
+       "      <th>buy_num</th>\n",
+       "      <th>favor_num</th>\n",
+       "      <th>click_num</th>\n",
+       "      <th>buy_addcart_ratio</th>\n",
+       "      <th>buy_browse_ratio</th>\n",
+       "      <th>buy_click_ratio</th>\n",
+       "      <th>buy_favor_ratio</th>\n",
+       "      <th>comment_num</th>\n",
+       "      <th>has_bad_comment</th>\n",
+       "      <th>bad_comment_rate</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>10</td>\n",
+       "      <td>3</td>\n",
+       "      <td>1</td>\n",
+       "      <td>1</td>\n",
+       "      <td>8</td>\n",
+       "      <td>489</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>100002</td>\n",
+       "      <td>3</td>\n",
+       "      <td>2</td>\n",
+       "      <td>2</td>\n",
+       "      <td>8</td>\n",
+       "      <td>489</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>100003</td>\n",
+       "      <td>1</td>\n",
+       "      <td>-1</td>\n",
+       "      <td>-1</td>\n",
+       "      <td>8</td>\n",
+       "      <td>30</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>100006</td>\n",
+       "      <td>1</td>\n",
+       "      <td>2</td>\n",
+       "      <td>1</td>\n",
+       "      <td>8</td>\n",
+       "      <td>545</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>10001</td>\n",
+       "      <td>-1</td>\n",
+       "      <td>1</td>\n",
+       "      <td>2</td>\n",
+       "      <td>8</td>\n",
+       "      <td>244</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "   sku_id  a1  a2  a3  cate  brand  browse_num  addcart_num  delcart_num  \\\n",
+       "0      10   3   1   1     8    489         NaN          NaN          NaN   \n",
+       "1  100002   3   2   2     8    489         NaN          NaN          NaN   \n",
+       "2  100003   1  -1  -1     8     30         NaN          NaN          NaN   \n",
+       "3  100006   1   2   1     8    545         NaN          NaN          NaN   \n",
+       "4   10001  -1   1   2     8    244         NaN          NaN          NaN   \n",
+       "\n",
+       "   buy_num  favor_num  click_num  buy_addcart_ratio  buy_browse_ratio  \\\n",
+       "0      NaN        NaN        NaN                NaN               NaN   \n",
+       "1      NaN        NaN        NaN                NaN               NaN   \n",
+       "2      NaN        NaN        NaN                NaN               NaN   \n",
+       "3      NaN        NaN        NaN                NaN               NaN   \n",
+       "4      NaN        NaN        NaN                NaN               NaN   \n",
+       "\n",
+       "   buy_click_ratio  buy_favor_ratio  comment_num  has_bad_comment  \\\n",
+       "0              NaN              NaN          NaN              NaN   \n",
+       "1              NaN              NaN          NaN              NaN   \n",
+       "2              NaN              NaN          NaN              NaN   \n",
+       "3              NaN              NaN          NaN              NaN   \n",
+       "4              NaN              NaN          NaN              NaN   \n",
+       "\n",
+       "   bad_comment_rate  \n",
+       "0               NaN  \n",
+       "1               NaN  \n",
+       "2               NaN  \n",
+       "3               NaN  \n",
+       "4               NaN  "
+      ]
+     },
+     "execution_count": 23,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "item_table = pd.read_csv(ITEM_TABLE_FILE)\n",
+    "item_table.head()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### 数据清洗\n",
+    "\n",
+    "用户清洗"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 24,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>user_id</th>\n",
+       "      <th>age</th>\n",
+       "      <th>sex</th>\n",
+       "      <th>user_lv_cd</th>\n",
+       "      <th>browse_num</th>\n",
+       "      <th>addcart_num</th>\n",
+       "      <th>delcart_num</th>\n",
+       "      <th>buy_num</th>\n",
+       "      <th>favor_num</th>\n",
+       "      <th>click_num</th>\n",
+       "      <th>buy_addcart_ratio</th>\n",
+       "      <th>buy_browse_ratio</th>\n",
+       "      <th>buy_click_ratio</th>\n",
+       "      <th>buy_favor_ratio</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>count</th>\n",
+       "      <td>105,321.000</td>\n",
+       "      <td>105,318.000</td>\n",
+       "      <td>105,318.000</td>\n",
+       "      <td>105,321.000</td>\n",
+       "      <td>105,180.000</td>\n",
+       "      <td>105,180.000</td>\n",
+       "      <td>105,180.000</td>\n",
+       "      <td>105,180.000</td>\n",
+       "      <td>105,180.000</td>\n",
+       "      <td>105,180.000</td>\n",
+       "      <td>72,129.000</td>\n",
+       "      <td>105,172.000</td>\n",
+       "      <td>103,197.000</td>\n",
+       "      <td>45,986.000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>mean</th>\n",
+       "      <td>252,661.000</td>\n",
+       "      <td>2.773</td>\n",
+       "      <td>1.113</td>\n",
+       "      <td>3.850</td>\n",
+       "      <td>180.466</td>\n",
+       "      <td>5.471</td>\n",
+       "      <td>2.434</td>\n",
+       "      <td>0.459</td>\n",
+       "      <td>1.045</td>\n",
+       "      <td>291.222</td>\n",
+       "      <td>0.147</td>\n",
+       "      <td>0.005</td>\n",
+       "      <td>0.009</td>\n",
+       "      <td>0.552</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>std</th>\n",
+       "      <td>30,403.698</td>\n",
+       "      <td>1.672</td>\n",
+       "      <td>0.956</td>\n",
+       "      <td>1.072</td>\n",
+       "      <td>273.437</td>\n",
+       "      <td>10.618</td>\n",
+       "      <td>5.600</td>\n",
+       "      <td>1.048</td>\n",
+       "      <td>3.442</td>\n",
+       "      <td>460.031</td>\n",
+       "      <td>0.270</td>\n",
+       "      <td>0.022</td>\n",
+       "      <td>0.074</td>\n",
+       "      <td>0.473</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>min</th>\n",
+       "      <td>200,001.000</td>\n",
+       "      <td>-1.000</td>\n",
+       "      <td>0.000</td>\n",
+       "      <td>1.000</td>\n",
+       "      <td>0.000</td>\n",
+       "      <td>0.000</td>\n",
+       "      <td>0.000</td>\n",
+       "      <td>0.000</td>\n",
+       "      <td>0.000</td>\n",
+       "      <td>0.000</td>\n",
+       "      <td>0.000</td>\n",
+       "      <td>0.000</td>\n",
+       "      <td>0.000</td>\n",
+       "      <td>0.000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>25%</th>\n",
+       "      <td>226,331.000</td>\n",
+       "      <td>3.000</td>\n",
+       "      <td>0.000</td>\n",
+       "      <td>3.000</td>\n",
+       "      <td>40.000</td>\n",
+       "      <td>0.000</td>\n",
+       "      <td>0.000</td>\n",
+       "      <td>0.000</td>\n",
+       "      <td>0.000</td>\n",
+       "      <td>59.000</td>\n",
+       "      <td>0.000</td>\n",
+       "      <td>0.000</td>\n",
+       "      <td>0.000</td>\n",
+       "      <td>0.000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>50%</th>\n",
+       "      <td>252,661.000</td>\n",
+       "      <td>3.000</td>\n",
+       "      <td>2.000</td>\n",
+       "      <td>4.000</td>\n",
+       "      <td>94.000</td>\n",
+       "      <td>2.000</td>\n",
+       "      <td>0.000</td>\n",
+       "      <td>0.000</td>\n",
+       "      <td>0.000</td>\n",
+       "      <td>148.000</td>\n",
+       "      <td>0.000</td>\n",
+       "      <td>0.000</td>\n",
+       "      <td>0.000</td>\n",
+       "      <td>1.000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>75%</th>\n",
+       "      <td>278,991.000</td>\n",
+       "      <td>4.000</td>\n",
+       "      <td>2.000</td>\n",
+       "      <td>5.000</td>\n",
+       "      <td>212.000</td>\n",
+       "      <td>6.000</td>\n",
+       "      <td>3.000</td>\n",
+       "      <td>1.000</td>\n",
+       "      <td>0.000</td>\n",
+       "      <td>342.000</td>\n",
+       "      <td>0.167</td>\n",
+       "      <td>0.002</td>\n",
+       "      <td>0.001</td>\n",
+       "      <td>1.000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>max</th>\n",
+       "      <td>305,321.000</td>\n",
+       "      <td>6.000</td>\n",
+       "      <td>2.000</td>\n",
+       "      <td>5.000</td>\n",
+       "      <td>7,605.000</td>\n",
+       "      <td>369.000</td>\n",
+       "      <td>231.000</td>\n",
+       "      <td>50.000</td>\n",
+       "      <td>99.000</td>\n",
+       "      <td>15,302.000</td>\n",
+       "      <td>1.000</td>\n",
+       "      <td>1.000</td>\n",
+       "      <td>1.000</td>\n",
+       "      <td>1.000</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "          user_id         age         sex  user_lv_cd  browse_num  \\\n",
+       "count 105,321.000 105,318.000 105,318.000 105,321.000 105,180.000   \n",
+       "mean  252,661.000       2.773       1.113       3.850     180.466   \n",
+       "std    30,403.698       1.672       0.956       1.072     273.437   \n",
+       "min   200,001.000      -1.000       0.000       1.000       0.000   \n",
+       "25%   226,331.000       3.000       0.000       3.000      40.000   \n",
+       "50%   252,661.000       3.000       2.000       4.000      94.000   \n",
+       "75%   278,991.000       4.000       2.000       5.000     212.000   \n",
+       "max   305,321.000       6.000       2.000       5.000   7,605.000   \n",
+       "\n",
+       "       addcart_num  delcart_num     buy_num   favor_num   click_num  \\\n",
+       "count  105,180.000  105,180.000 105,180.000 105,180.000 105,180.000   \n",
+       "mean         5.471        2.434       0.459       1.045     291.222   \n",
+       "std         10.618        5.600       1.048       3.442     460.031   \n",
+       "min          0.000        0.000       0.000       0.000       0.000   \n",
+       "25%          0.000        0.000       0.000       0.000      59.000   \n",
+       "50%          2.000        0.000       0.000       0.000     148.000   \n",
+       "75%          6.000        3.000       1.000       0.000     342.000   \n",
+       "max        369.000      231.000      50.000      99.000  15,302.000   \n",
+       "\n",
+       "       buy_addcart_ratio  buy_browse_ratio  buy_click_ratio  buy_favor_ratio  \n",
+       "count         72,129.000       105,172.000      103,197.000       45,986.000  \n",
+       "mean               0.147             0.005            0.009            0.552  \n",
+       "std                0.270             0.022            0.074            0.473  \n",
+       "min                0.000             0.000            0.000            0.000  \n",
+       "25%                0.000             0.000            0.000            0.000  \n",
+       "50%                0.000             0.000            0.000            1.000  \n",
+       "75%                0.167             0.002            0.001            1.000  \n",
+       "max                1.000             1.000            1.000            1.000  "
+      ]
+     },
+     "execution_count": 24,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "df_user = pd.read_csv('data/User_table.csv',header=0)\n",
+    "pd.options.display.float_format = '{:,.3f}'.format  #输出格式设置，保留三位小数\n",
+    "df_user.describe()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "由上述统计信息发现： 第一行中根据User_id统计发现有105321个用户，发现有3个用户没有age,sex字段\n",
+    "\n",
+    "根据浏览、加购、删购、购买等记录却只有105180条记录，说明存在用户无任何交互记录，因此可以删除上述用户"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "删除没有age,sex字段的用户"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 25,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>user_id</th>\n",
+       "      <th>age</th>\n",
+       "      <th>sex</th>\n",
+       "      <th>user_lv_cd</th>\n",
+       "      <th>browse_num</th>\n",
+       "      <th>addcart_num</th>\n",
+       "      <th>delcart_num</th>\n",
+       "      <th>buy_num</th>\n",
+       "      <th>favor_num</th>\n",
+       "      <th>click_num</th>\n",
+       "      <th>buy_addcart_ratio</th>\n",
+       "      <th>buy_browse_ratio</th>\n",
+       "      <th>buy_click_ratio</th>\n",
+       "      <th>buy_favor_ratio</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>34072</th>\n",
+       "      <td>234073</td>\n",
+       "      <td>nan</td>\n",
+       "      <td>nan</td>\n",
+       "      <td>1</td>\n",
+       "      <td>32.000</td>\n",
+       "      <td>6.000</td>\n",
+       "      <td>4.000</td>\n",
+       "      <td>1.000</td>\n",
+       "      <td>0.000</td>\n",
+       "      <td>41.000</td>\n",
+       "      <td>0.167</td>\n",
+       "      <td>0.031</td>\n",
+       "      <td>0.024</td>\n",
+       "      <td>1.000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>38905</th>\n",
+       "      <td>238906</td>\n",
+       "      <td>nan</td>\n",
+       "      <td>nan</td>\n",
+       "      <td>1</td>\n",
+       "      <td>171.000</td>\n",
+       "      <td>3.000</td>\n",
+       "      <td>2.000</td>\n",
+       "      <td>2.000</td>\n",
+       "      <td>3.000</td>\n",
+       "      <td>464.000</td>\n",
+       "      <td>0.667</td>\n",
+       "      <td>0.012</td>\n",
+       "      <td>0.004</td>\n",
+       "      <td>0.667</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>67704</th>\n",
+       "      <td>267705</td>\n",
+       "      <td>nan</td>\n",
+       "      <td>nan</td>\n",
+       "      <td>1</td>\n",
+       "      <td>342.000</td>\n",
+       "      <td>18.000</td>\n",
+       "      <td>8.000</td>\n",
+       "      <td>0.000</td>\n",
+       "      <td>0.000</td>\n",
+       "      <td>743.000</td>\n",
+       "      <td>0.000</td>\n",
+       "      <td>0.000</td>\n",
+       "      <td>0.000</td>\n",
+       "      <td>nan</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "       user_id  age  sex  user_lv_cd  browse_num  addcart_num  delcart_num  \\\n",
+       "34072   234073  nan  nan           1      32.000        6.000        4.000   \n",
+       "38905   238906  nan  nan           1     171.000        3.000        2.000   \n",
+       "67704   267705  nan  nan           1     342.000       18.000        8.000   \n",
+       "\n",
+       "       buy_num  favor_num  click_num  buy_addcart_ratio  buy_browse_ratio  \\\n",
+       "34072    1.000      0.000     41.000              0.167             0.031   \n",
+       "38905    2.000      3.000    464.000              0.667             0.012   \n",
+       "67704    0.000      0.000    743.000              0.000             0.000   \n",
+       "\n",
+       "       buy_click_ratio  buy_favor_ratio  \n",
+       "34072            0.024            1.000  \n",
+       "38905            0.004            0.667  \n",
+       "67704            0.000              nan  "
+      ]
+     },
+     "execution_count": 25,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "df_user[df_user['age'].isnull()]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 26,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>user_id</th>\n",
+       "      <th>age</th>\n",
+       "      <th>sex</th>\n",
+       "      <th>user_lv_cd</th>\n",
+       "      <th>browse_num</th>\n",
+       "      <th>addcart_num</th>\n",
+       "      <th>delcart_num</th>\n",
+       "      <th>buy_num</th>\n",
+       "      <th>favor_num</th>\n",
+       "      <th>click_num</th>\n",
+       "      <th>buy_addcart_ratio</th>\n",
+       "      <th>buy_browse_ratio</th>\n",
+       "      <th>buy_click_ratio</th>\n",
+       "      <th>buy_favor_ratio</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>count</th>\n",
+       "      <td>105,318.000</td>\n",
+       "      <td>105,318.000</td>\n",
+       "      <td>105,318.000</td>\n",
+       "      <td>105,318.000</td>\n",
+       "      <td>105,177.000</td>\n",
+       "      <td>105,177.000</td>\n",
+       "      <td>105,177.000</td>\n",
+       "      <td>105,177.000</td>\n",
+       "      <td>105,177.000</td>\n",
+       "      <td>105,177.000</td>\n",
+       "      <td>72,126.000</td>\n",
+       "      <td>105,169.000</td>\n",
+       "      <td>103,194.000</td>\n",
+       "      <td>45,984.000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>mean</th>\n",
+       "      <td>252,661.164</td>\n",
+       "      <td>2.773</td>\n",
+       "      <td>1.113</td>\n",
+       "      <td>3.850</td>\n",
+       "      <td>180.466</td>\n",
+       "      <td>5.471</td>\n",
+       "      <td>2.434</td>\n",
+       "      <td>0.459</td>\n",
+       "      <td>1.045</td>\n",
+       "      <td>291.219</td>\n",
+       "      <td>0.147</td>\n",
+       "      <td>0.005</td>\n",
+       "      <td>0.009</td>\n",
+       "      <td>0.552</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>std</th>\n",
+       "      <td>30,404.012</td>\n",
+       "      <td>1.672</td>\n",
+       "      <td>0.956</td>\n",
+       "      <td>1.071</td>\n",
+       "      <td>273.440</td>\n",
+       "      <td>10.618</td>\n",
+       "      <td>5.600</td>\n",
+       "      <td>1.048</td>\n",
+       "      <td>3.442</td>\n",
+       "      <td>460.034</td>\n",
+       "      <td>0.270</td>\n",
+       "      <td>0.022</td>\n",
+       "      <td>0.074</td>\n",
+       "      <td>0.473</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>min</th>\n",
+       "      <td>200,001.000</td>\n",
+       "      <td>-1.000</td>\n",
+       "      <td>0.000</td>\n",
+       "      <td>1.000</td>\n",
+       "      <td>0.000</td>\n",
+       "      <td>0.000</td>\n",
+       "      <td>0.000</td>\n",
+       "      <td>0.000</td>\n",
+       "      <td>0.000</td>\n",
+       "      <td>0.000</td>\n",
+       "      <td>0.000</td>\n",
+       "      <td>0.000</td>\n",
+       "      <td>0.000</td>\n",
+       "      <td>0.000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>25%</th>\n",
+       "      <td>226,330.250</td>\n",
+       "      <td>3.000</td>\n",
+       "      <td>0.000</td>\n",
+       "      <td>3.000</td>\n",
+       "      <td>40.000</td>\n",
+       "      <td>0.000</td>\n",
+       "      <td>0.000</td>\n",
+       "      <td>0.000</td>\n",
+       "      <td>0.000</td>\n",
+       "      <td>59.000</td>\n",
+       "      <td>0.000</td>\n",
+       "      <td>0.000</td>\n",
+       "      <td>0.000</td>\n",
+       "      <td>0.000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>50%</th>\n",
+       "      <td>252,661.500</td>\n",
+       "      <td>3.000</td>\n",
+       "      <td>2.000</td>\n",
+       "      <td>4.000</td>\n",
+       "      <td>94.000</td>\n",
+       "      <td>2.000</td>\n",
+       "      <td>0.000</td>\n",
+       "      <td>0.000</td>\n",
+       "      <td>0.000</td>\n",
+       "      <td>148.000</td>\n",
+       "      <td>0.000</td>\n",
+       "      <td>0.000</td>\n",
+       "      <td>0.000</td>\n",
+       "      <td>1.000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>75%</th>\n",
+       "      <td>278,991.750</td>\n",
+       "      <td>4.000</td>\n",
+       "      <td>2.000</td>\n",
+       "      <td>5.000</td>\n",
+       "      <td>212.000</td>\n",
+       "      <td>6.000</td>\n",
+       "      <td>3.000</td>\n",
+       "      <td>1.000</td>\n",
+       "      <td>0.000</td>\n",
+       "      <td>342.000</td>\n",
+       "      <td>0.167</td>\n",
+       "      <td>0.002</td>\n",
+       "      <td>0.001</td>\n",
+       "      <td>1.000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>max</th>\n",
+       "      <td>305,321.000</td>\n",
+       "      <td>6.000</td>\n",
+       "      <td>2.000</td>\n",
+       "      <td>5.000</td>\n",
+       "      <td>7,605.000</td>\n",
+       "      <td>369.000</td>\n",
+       "      <td>231.000</td>\n",
+       "      <td>50.000</td>\n",
+       "      <td>99.000</td>\n",
+       "      <td>15,302.000</td>\n",
+       "      <td>1.000</td>\n",
+       "      <td>1.000</td>\n",
+       "      <td>1.000</td>\n",
+       "      <td>1.000</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "          user_id         age         sex  user_lv_cd  browse_num  \\\n",
+       "count 105,318.000 105,318.000 105,318.000 105,318.000 105,177.000   \n",
+       "mean  252,661.164       2.773       1.113       3.850     180.466   \n",
+       "std    30,404.012       1.672       0.956       1.071     273.440   \n",
+       "min   200,001.000      -1.000       0.000       1.000       0.000   \n",
+       "25%   226,330.250       3.000       0.000       3.000      40.000   \n",
+       "50%   252,661.500       3.000       2.000       4.000      94.000   \n",
+       "75%   278,991.750       4.000       2.000       5.000     212.000   \n",
+       "max   305,321.000       6.000       2.000       5.000   7,605.000   \n",
+       "\n",
+       "       addcart_num  delcart_num     buy_num   favor_num   click_num  \\\n",
+       "count  105,177.000  105,177.000 105,177.000 105,177.000 105,177.000   \n",
+       "mean         5.471        2.434       0.459       1.045     291.219   \n",
+       "std         10.618        5.600       1.048       3.442     460.034   \n",
+       "min          0.000        0.000       0.000       0.000       0.000   \n",
+       "25%          0.000        0.000       0.000       0.000      59.000   \n",
+       "50%          2.000        0.000       0.000       0.000     148.000   \n",
+       "75%          6.000        3.000       1.000       0.000     342.000   \n",
+       "max        369.000      231.000      50.000      99.000  15,302.000   \n",
+       "\n",
+       "       buy_addcart_ratio  buy_browse_ratio  buy_click_ratio  buy_favor_ratio  \n",
+       "count         72,126.000       105,169.000      103,194.000       45,984.000  \n",
+       "mean               0.147             0.005            0.009            0.552  \n",
+       "std                0.270             0.022            0.074            0.473  \n",
+       "min                0.000             0.000            0.000            0.000  \n",
+       "25%                0.000             0.000            0.000            0.000  \n",
+       "50%                0.000             0.000            0.000            1.000  \n",
+       "75%                0.167             0.002            0.001            1.000  \n",
+       "max                1.000             1.000            1.000            1.000  "
+      ]
+     },
+     "execution_count": 26,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "delete_list = df_user[df_user['age'].isnull()].index\n",
+    "df_user.drop(delete_list,axis=0,inplace=True)\n",
+    "df_user.describe()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "删除无交互记录的用户"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 27,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "105177\n"
+     ]
+    }
+   ],
+   "source": [
+    "df_naction = df_user[(df_user['browse_num'].isnull()) & (df_user['addcart_num'].isnull()) & (df_user['delcart_num'].isnull()) & (df_user['buy_num'].isnull()) & (df_user['favor_num'].isnull()) & (df_user['click_num'].isnull())]\n",
+    "df_user.drop(df_naction.index,axis=0,inplace=True)\n",
+    "print(len(df_user))"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "统计并删除无购买记录的用户"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 28,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "75694\n"
+     ]
+    }
+   ],
+   "source": [
+    "df_bzero = df_user[df_user['buy_num']==0]\n",
+    "# 输出购买数为0的总记录数\n",
+    "print(len(df_bzero))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 29,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>user_id</th>\n",
+       "      <th>age</th>\n",
+       "      <th>sex</th>\n",
+       "      <th>user_lv_cd</th>\n",
+       "      <th>browse_num</th>\n",
+       "      <th>addcart_num</th>\n",
+       "      <th>delcart_num</th>\n",
+       "      <th>buy_num</th>\n",
+       "      <th>favor_num</th>\n",
+       "      <th>click_num</th>\n",
+       "      <th>buy_addcart_ratio</th>\n",
+       "      <th>buy_browse_ratio</th>\n",
+       "      <th>buy_click_ratio</th>\n",
+       "      <th>buy_favor_ratio</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>count</th>\n",
+       "      <td>29,483.000</td>\n",
+       "      <td>29,483.000</td>\n",
+       "      <td>29,483.000</td>\n",
+       "      <td>29,483.000</td>\n",
+       "      <td>29,483.000</td>\n",
+       "      <td>29,483.000</td>\n",
+       "      <td>29,483.000</td>\n",
+       "      <td>29,483.000</td>\n",
+       "      <td>29,483.000</td>\n",
+       "      <td>29,483.000</td>\n",
+       "      <td>29,483.000</td>\n",
+       "      <td>29,483.000</td>\n",
+       "      <td>29,483.000</td>\n",
+       "      <td>29,483.000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>mean</th>\n",
+       "      <td>250,746.445</td>\n",
+       "      <td>2.914</td>\n",
+       "      <td>1.025</td>\n",
+       "      <td>4.272</td>\n",
+       "      <td>302.488</td>\n",
+       "      <td>10.525</td>\n",
+       "      <td>4.673</td>\n",
+       "      <td>1.637</td>\n",
+       "      <td>1.677</td>\n",
+       "      <td>486.653</td>\n",
+       "      <td>0.360</td>\n",
+       "      <td>0.018</td>\n",
+       "      <td>0.030</td>\n",
+       "      <td>0.862</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>std</th>\n",
+       "      <td>29,979.676</td>\n",
+       "      <td>1.490</td>\n",
+       "      <td>0.959</td>\n",
+       "      <td>0.808</td>\n",
+       "      <td>391.535</td>\n",
+       "      <td>14.301</td>\n",
+       "      <td>7.568</td>\n",
+       "      <td>1.412</td>\n",
+       "      <td>4.584</td>\n",
+       "      <td>658.671</td>\n",
+       "      <td>0.320</td>\n",
+       "      <td>0.038</td>\n",
+       "      <td>0.136</td>\n",
+       "      <td>0.287</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>min</th>\n",
+       "      <td>200,001.000</td>\n",
+       "      <td>-1.000</td>\n",
+       "      <td>0.000</td>\n",
+       "      <td>2.000</td>\n",
+       "      <td>1.000</td>\n",
+       "      <td>0.000</td>\n",
+       "      <td>0.000</td>\n",
+       "      <td>1.000</td>\n",
+       "      <td>0.000</td>\n",
+       "      <td>0.000</td>\n",
+       "      <td>0.004</td>\n",
+       "      <td>0.000</td>\n",
+       "      <td>0.000</td>\n",
+       "      <td>0.010</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>25%</th>\n",
+       "      <td>225,058.500</td>\n",
+       "      <td>3.000</td>\n",
+       "      <td>0.000</td>\n",
+       "      <td>4.000</td>\n",
+       "      <td>76.000</td>\n",
+       "      <td>3.000</td>\n",
+       "      <td>0.000</td>\n",
+       "      <td>1.000</td>\n",
+       "      <td>0.000</td>\n",
+       "      <td>116.000</td>\n",
+       "      <td>0.118</td>\n",
+       "      <td>0.004</td>\n",
+       "      <td>0.002</td>\n",
+       "      <td>1.000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>50%</th>\n",
+       "      <td>249,144.000</td>\n",
+       "      <td>3.000</td>\n",
+       "      <td>1.000</td>\n",
+       "      <td>4.000</td>\n",
+       "      <td>178.000</td>\n",
+       "      <td>6.000</td>\n",
+       "      <td>2.000</td>\n",
+       "      <td>1.000</td>\n",
+       "      <td>0.000</td>\n",
+       "      <td>282.000</td>\n",
+       "      <td>0.250</td>\n",
+       "      <td>0.008</td>\n",
+       "      <td>0.005</td>\n",
+       "      <td>1.000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>75%</th>\n",
+       "      <td>276,252.500</td>\n",
+       "      <td>4.000</td>\n",
+       "      <td>2.000</td>\n",
+       "      <td>5.000</td>\n",
+       "      <td>381.000</td>\n",
+       "      <td>13.000</td>\n",
+       "      <td>6.000</td>\n",
+       "      <td>2.000</td>\n",
+       "      <td>1.000</td>\n",
+       "      <td>604.000</td>\n",
+       "      <td>0.500</td>\n",
+       "      <td>0.018</td>\n",
+       "      <td>0.012</td>\n",
+       "      <td>1.000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>max</th>\n",
+       "      <td>305,318.000</td>\n",
+       "      <td>6.000</td>\n",
+       "      <td>2.000</td>\n",
+       "      <td>5.000</td>\n",
+       "      <td>7,605.000</td>\n",
+       "      <td>288.000</td>\n",
+       "      <td>178.000</td>\n",
+       "      <td>50.000</td>\n",
+       "      <td>96.000</td>\n",
+       "      <td>15,302.000</td>\n",
+       "      <td>1.000</td>\n",
+       "      <td>1.000</td>\n",
+       "      <td>1.000</td>\n",
+       "      <td>1.000</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "          user_id        age        sex  user_lv_cd  browse_num  addcart_num  \\\n",
+       "count  29,483.000 29,483.000 29,483.000  29,483.000  29,483.000   29,483.000   \n",
+       "mean  250,746.445      2.914      1.025       4.272     302.488       10.525   \n",
+       "std    29,979.676      1.490      0.959       0.808     391.535       14.301   \n",
+       "min   200,001.000     -1.000      0.000       2.000       1.000        0.000   \n",
+       "25%   225,058.500      3.000      0.000       4.000      76.000        3.000   \n",
+       "50%   249,144.000      3.000      1.000       4.000     178.000        6.000   \n",
+       "75%   276,252.500      4.000      2.000       5.000     381.000       13.000   \n",
+       "max   305,318.000      6.000      2.000       5.000   7,605.000      288.000   \n",
+       "\n",
+       "       delcart_num    buy_num  favor_num  click_num  buy_addcart_ratio  \\\n",
+       "count   29,483.000 29,483.000 29,483.000 29,483.000         29,483.000   \n",
+       "mean         4.673      1.637      1.677    486.653              0.360   \n",
+       "std          7.568      1.412      4.584    658.671              0.320   \n",
+       "min          0.000      1.000      0.000      0.000              0.004   \n",
+       "25%          0.000      1.000      0.000    116.000              0.118   \n",
+       "50%          2.000      1.000      0.000    282.000              0.250   \n",
+       "75%          6.000      2.000      1.000    604.000              0.500   \n",
+       "max        178.000     50.000     96.000 15,302.000              1.000   \n",
+       "\n",
+       "       buy_browse_ratio  buy_click_ratio  buy_favor_ratio  \n",
+       "count        29,483.000       29,483.000       29,483.000  \n",
+       "mean              0.018            0.030            0.862  \n",
+       "std               0.038            0.136            0.287  \n",
+       "min               0.000            0.000            0.010  \n",
+       "25%               0.004            0.002            1.000  \n",
+       "50%               0.008            0.005            1.000  \n",
+       "75%               0.018            0.012            1.000  \n",
+       "max               1.000            1.000            1.000  "
+      ]
+     },
+     "execution_count": 29,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "df_user = df_user[df_user['buy_num']!=0]  # 只要有购买记录的\n",
+    "df_user.describe()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "删除爬虫及惰性用户\n",
+    "\n",
+    "由上表所知，浏览购买转换比和点击购买转换比均值为0.018,0.030，因此这里认为浏览购买转换比和点击购买转换比小于0.0005的用户为惰性用户"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 30,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "90\n"
+     ]
+    }
+   ],
+   "source": [
+    "bindex = df_user[df_user['buy_browse_ratio']<0.0005].index\n",
+    "print (len(bindex))\n",
+    "df_user.drop(bindex,axis=0,inplace=True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 31,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "323\n"
+     ]
+    }
+   ],
+   "source": [
+    "cindex = df_user[df_user['buy_click_ratio']<0.0005].index\n",
+    "print (len(cindex))\n",
+    "df_user.drop(cindex,axis=0,inplace=True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 32,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>user_id</th>\n",
+       "      <th>age</th>\n",
+       "      <th>sex</th>\n",
+       "      <th>user_lv_cd</th>\n",
+       "      <th>browse_num</th>\n",
+       "      <th>addcart_num</th>\n",
+       "      <th>delcart_num</th>\n",
+       "      <th>buy_num</th>\n",
+       "      <th>favor_num</th>\n",
+       "      <th>click_num</th>\n",
+       "      <th>buy_addcart_ratio</th>\n",
+       "      <th>buy_browse_ratio</th>\n",
+       "      <th>buy_click_ratio</th>\n",
+       "      <th>buy_favor_ratio</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>count</th>\n",
+       "      <td>29,070.000</td>\n",
+       "      <td>29,070.000</td>\n",
+       "      <td>29,070.000</td>\n",
+       "      <td>29,070.000</td>\n",
+       "      <td>29,070.000</td>\n",
+       "      <td>29,070.000</td>\n",
+       "      <td>29,070.000</td>\n",
+       "      <td>29,070.000</td>\n",
+       "      <td>29,070.000</td>\n",
+       "      <td>29,070.000</td>\n",
+       "      <td>29,070.000</td>\n",
+       "      <td>29,070.000</td>\n",
+       "      <td>29,070.000</td>\n",
+       "      <td>29,070.000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>mean</th>\n",
+       "      <td>250,767.099</td>\n",
+       "      <td>2.910</td>\n",
+       "      <td>1.028</td>\n",
+       "      <td>4.268</td>\n",
+       "      <td>280.260</td>\n",
+       "      <td>10.145</td>\n",
+       "      <td>4.457</td>\n",
+       "      <td>1.644</td>\n",
+       "      <td>1.589</td>\n",
+       "      <td>447.113</td>\n",
+       "      <td>0.364</td>\n",
+       "      <td>0.019</td>\n",
+       "      <td>0.031</td>\n",
+       "      <td>0.866</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>std</th>\n",
+       "      <td>29,998.870</td>\n",
+       "      <td>1.492</td>\n",
+       "      <td>0.959</td>\n",
+       "      <td>0.809</td>\n",
+       "      <td>325.129</td>\n",
+       "      <td>13.443</td>\n",
+       "      <td>6.998</td>\n",
+       "      <td>1.420</td>\n",
+       "      <td>4.294</td>\n",
+       "      <td>530.994</td>\n",
+       "      <td>0.320</td>\n",
+       "      <td>0.038</td>\n",
+       "      <td>0.137</td>\n",
+       "      <td>0.282</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>min</th>\n",
+       "      <td>200,001.000</td>\n",
+       "      <td>-1.000</td>\n",
+       "      <td>0.000</td>\n",
+       "      <td>2.000</td>\n",
+       "      <td>1.000</td>\n",
+       "      <td>0.000</td>\n",
+       "      <td>0.000</td>\n",
+       "      <td>1.000</td>\n",
+       "      <td>0.000</td>\n",
+       "      <td>0.000</td>\n",
+       "      <td>0.004</td>\n",
+       "      <td>0.001</td>\n",
+       "      <td>0.001</td>\n",
+       "      <td>0.018</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>25%</th>\n",
+       "      <td>225,036.000</td>\n",
+       "      <td>3.000</td>\n",
+       "      <td>0.000</td>\n",
+       "      <td>4.000</td>\n",
+       "      <td>75.000</td>\n",
+       "      <td>3.000</td>\n",
+       "      <td>0.000</td>\n",
+       "      <td>1.000</td>\n",
+       "      <td>0.000</td>\n",
+       "      <td>114.000</td>\n",
+       "      <td>0.125</td>\n",
+       "      <td>0.004</td>\n",
+       "      <td>0.002</td>\n",
+       "      <td>1.000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>50%</th>\n",
+       "      <td>249,200.500</td>\n",
+       "      <td>3.000</td>\n",
+       "      <td>1.000</td>\n",
+       "      <td>4.000</td>\n",
+       "      <td>174.000</td>\n",
+       "      <td>6.000</td>\n",
+       "      <td>2.000</td>\n",
+       "      <td>1.000</td>\n",
+       "      <td>0.000</td>\n",
+       "      <td>275.000</td>\n",
+       "      <td>0.250</td>\n",
+       "      <td>0.008</td>\n",
+       "      <td>0.005</td>\n",
+       "      <td>1.000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>75%</th>\n",
+       "      <td>276,284.000</td>\n",
+       "      <td>4.000</td>\n",
+       "      <td>2.000</td>\n",
+       "      <td>5.000</td>\n",
+       "      <td>366.000</td>\n",
+       "      <td>13.000</td>\n",
+       "      <td>6.000</td>\n",
+       "      <td>2.000</td>\n",
+       "      <td>1.000</td>\n",
+       "      <td>585.000</td>\n",
+       "      <td>0.500</td>\n",
+       "      <td>0.018</td>\n",
+       "      <td>0.012</td>\n",
+       "      <td>1.000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>max</th>\n",
+       "      <td>305,318.000</td>\n",
+       "      <td>6.000</td>\n",
+       "      <td>2.000</td>\n",
+       "      <td>5.000</td>\n",
+       "      <td>5,007.000</td>\n",
+       "      <td>288.000</td>\n",
+       "      <td>158.000</td>\n",
+       "      <td>50.000</td>\n",
+       "      <td>69.000</td>\n",
+       "      <td>8,156.000</td>\n",
+       "      <td>1.000</td>\n",
+       "      <td>1.000</td>\n",
+       "      <td>1.000</td>\n",
+       "      <td>1.000</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "          user_id        age        sex  user_lv_cd  browse_num  addcart_num  \\\n",
+       "count  29,070.000 29,070.000 29,070.000  29,070.000  29,070.000   29,070.000   \n",
+       "mean  250,767.099      2.910      1.028       4.268     280.260       10.145   \n",
+       "std    29,998.870      1.492      0.959       0.809     325.129       13.443   \n",
+       "min   200,001.000     -1.000      0.000       2.000       1.000        0.000   \n",
+       "25%   225,036.000      3.000      0.000       4.000      75.000        3.000   \n",
+       "50%   249,200.500      3.000      1.000       4.000     174.000        6.000   \n",
+       "75%   276,284.000      4.000      2.000       5.000     366.000       13.000   \n",
+       "max   305,318.000      6.000      2.000       5.000   5,007.000      288.000   \n",
+       "\n",
+       "       delcart_num    buy_num  favor_num  click_num  buy_addcart_ratio  \\\n",
+       "count   29,070.000 29,070.000 29,070.000 29,070.000         29,070.000   \n",
+       "mean         4.457      1.644      1.589    447.113              0.364   \n",
+       "std          6.998      1.420      4.294    530.994              0.320   \n",
+       "min          0.000      1.000      0.000      0.000              0.004   \n",
+       "25%          0.000      1.000      0.000    114.000              0.125   \n",
+       "50%          2.000      1.000      0.000    275.000              0.250   \n",
+       "75%          6.000      2.000      1.000    585.000              0.500   \n",
+       "max        158.000     50.000     69.000  8,156.000              1.000   \n",
+       "\n",
+       "       buy_browse_ratio  buy_click_ratio  buy_favor_ratio  \n",
+       "count        29,070.000       29,070.000       29,070.000  \n",
+       "mean              0.019            0.031            0.866  \n",
+       "std               0.038            0.137            0.282  \n",
+       "min               0.001            0.001            0.018  \n",
+       "25%               0.004            0.002            1.000  \n",
+       "50%               0.008            0.005            1.000  \n",
+       "75%               0.018            0.012            1.000  \n",
+       "max               1.000            1.000            1.000  "
+      ]
+     },
+     "execution_count": 32,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "df_user.describe()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 33,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df_user.to_csv(\"data/JData_FUser.csv\", index=None)"
+   ]
   }
  ],
  "metadata": {