You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
ML-For-Beginners/6-NLP/5-Hotel-Reviews-2/solution/3-notebook.ipynb

247 lines
8.0 KiB

{
"metadata": {
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.7.0"
},
"orig_nbformat": 4,
"kernelspec": {
"name": "python3",
"display_name": "Python 3.7.0 64-bit ('3.7')"
},
"interpreter": {
"hash": "70b38d7a306a849643e446cd70466270a13445e5987dfa1344ef2b127438fa4d"
}
},
"nbformat": 4,
"nbformat_minor": 2,
"cells": [
{
"cell_type": "code",
"execution_count": 9,
"metadata": {},
"outputs": [
{
"output_type": "stream",
"name": "stderr",
"text": [
"[nltk_data] Downloading package vader_lexicon to\n[nltk_data] /Users/jenlooper/nltk_data...\n"
]
},
{
"output_type": "execute_result",
"data": {
"text/plain": [
"True"
]
},
"metadata": {},
"execution_count": 9
}
],
"source": [
"import time\n",
"import pandas as pd\n",
"import nltk as nltk\n",
"from nltk.corpus import stopwords\n",
"from nltk.sentiment.vader import SentimentIntensityAnalyzer\n",
"nltk.download('vader_lexicon')\n"
]
},
{
"cell_type": "code",
"execution_count": 10,
"metadata": {},
"outputs": [],
"source": [
"vader_sentiment = SentimentIntensityAnalyzer()\n",
"\n"
]
},
{
"cell_type": "code",
"execution_count": 11,
"metadata": {},
"outputs": [],
"source": [
"# There are 3 possibilities of input for a review:\n",
"# It could be \"No Negative\", in which case, return 0\n",
"# It could be \"No Positive\", in which case, return 0\n",
"# It could be a review, in which case calculate the sentiment\n",
"def calc_sentiment(review): \n",
" if review == \"No Negative\" or review == \"No Positive\":\n",
" return 0\n",
" return vader_sentiment.polarity_scores(review)[\"compound\"] \n"
]
},
{
"cell_type": "code",
"execution_count": 12,
"metadata": {},
"outputs": [],
"source": [
"# Load the hotel reviews from CSV\n",
"df = pd.read_csv(\"../../data/Hotel_Reviews_Filtered.csv\")\n"
]
},
{
"cell_type": "code",
"execution_count": 13,
"metadata": {},
"outputs": [],
"source": [
"# Remove stop words - can be slow for a lot of text!\n",
"# Ryan Han (ryanxjhan on Kaggle) has a great post measuring performance of different stop words removal approaches\n",
"# https://www.kaggle.com/ryanxjhan/fast-stop-words-removal # using the approach that Ryan recommends\n",
"start = time.time()\n",
"cache = set(stopwords.words(\"english\"))\n",
"def remove_stopwords(review):\n",
" text = \" \".join([word for word in review.split() if word not in cache])\n",
" return text\n"
]
},
{
"cell_type": "code",
"execution_count": 14,
"metadata": {},
"outputs": [],
"source": [
"# Remove the stop words from both columns\n",
"df.Negative_Review = df.Negative_Review.apply(remove_stopwords) \n",
"df.Positive_Review = df.Positive_Review.apply(remove_stopwords)\n"
]
},
{
"cell_type": "code",
"execution_count": 15,
"metadata": {},
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"Removing stop words took 5.77 seconds\n"
]
}
],
"source": [
"end = time.time()\n",
"print(\"Removing stop words took \" + str(round(end - start, 2)) + \" seconds\")\n"
]
},
{
"cell_type": "code",
"execution_count": 16,
"metadata": {},
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"Calculating sentiment columns for both positive and negative reviews\n",
"Calculating sentiment took 201.07 seconds\n"
]
}
],
"source": [
"# Add a negative sentiment and positive sentiment column\n",
"print(\"Calculating sentiment columns for both positive and negative reviews\")\n",
"start = time.time()\n",
"df[\"Negative_Sentiment\"] = df.Negative_Review.apply(calc_sentiment)\n",
"df[\"Positive_Sentiment\"] = df.Positive_Review.apply(calc_sentiment)\n",
"end = time.time()\n",
"print(\"Calculating sentiment took \" + str(round(end - start, 2)) + \" seconds\")\n"
]
},
{
"cell_type": "code",
"execution_count": 17,
"metadata": {},
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
" Negative_Review Negative_Sentiment\n",
"186584 So bad experience memories I hotel The first n... -0.9920\n",
"129503 First charged twice room booked booking second... -0.9896\n",
"307286 The staff Had bad experience even booking Janu... -0.9889\n",
"452092 No WLAN room Incredibly rude restaurant staff ... -0.9884\n",
"201293 We usually traveling Paris 2 3 times year busi... -0.9873\n",
"... ... ...\n",
"26899 I would say however one night expensive even d... 0.9933\n",
"138365 Wifi terribly slow I speed test network upload... 0.9938\n",
"79215 I find anything hotel first I walked past hote... 0.9938\n",
"278506 The property great location There bakery next ... 0.9945\n",
"339189 Guys I like hotel I wish return next year Howe... 0.9948\n",
"\n",
"[515738 rows x 2 columns]\n",
" Positive_Review Positive_Sentiment\n",
"137893 Bathroom Shower We going stay twice hotel 2 ni... -0.9820\n",
"5839 I completely disappointed mad since reception ... -0.9780\n",
"64158 get everything extra internet parking breakfas... -0.9751\n",
"124178 I didnt like anythig Room small Asked upgrade ... -0.9721\n",
"489137 Very rude manager abusive staff reception Dirt... -0.9703\n",
"... ... ...\n",
"331570 Everything This recently renovated hotel class... 0.9984\n",
"322920 From moment stepped doors Guesthouse Hotel sta... 0.9985\n",
"293710 This place surprise expected good actually gre... 0.9985\n",
"417442 We celebrated wedding night Langham I commend ... 0.9985\n",
"132492 We arrived super cute boutique hotel area expl... 0.9987\n",
"\n",
"[515738 rows x 2 columns]\n"
]
}
],
"source": [
"df = df.sort_values(by=[\"Negative_Sentiment\"], ascending=True)\n",
"print(df[[\"Negative_Review\", \"Negative_Sentiment\"]])\n",
"df = df.sort_values(by=[\"Positive_Sentiment\"], ascending=True)\n",
"print(df[[\"Positive_Review\", \"Positive_Sentiment\"]])\n"
]
},
{
"cell_type": "code",
"execution_count": 18,
"metadata": {},
"outputs": [],
"source": [
"# Reorder the columns (This is cosmetic, but to make it easier to explore the data later)\n",
"df = df.reindex([\"Hotel_Name\", \"Hotel_Address\", \"Total_Number_of_Reviews\", \"Average_Score\", \"Reviewer_Score\", \"Negative_Sentiment\", \"Positive_Sentiment\", \"Reviewer_Nationality\", \"Leisure_trip\", \"Couple\", \"Solo_traveler\", \"Business_trip\", \"Group\", \"Family_with_young_children\", \"Family_with_older_children\", \"With_a_pet\", \"Negative_Review\", \"Positive_Review\"], axis=1)\n"
]
},
{
"cell_type": "code",
"execution_count": 19,
"metadata": {},
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"Saving results to Hotel_Reviews_NLP.csv\n"
]
}
],
"source": [
"print(\"Saving results to Hotel_Reviews_NLP.csv\")\n",
"df.to_csv(r\"../../data/Hotel_Reviews_NLP.csv\", index = False)\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
]
}