{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [],
"source": [
"import re\n",
"from sklearn import feature_extraction \n",
"stop_words = feature_extraction.text.ENGLISH_STOP_WORDS\n",
"from nltk.stem import PorterStemmer\n",
"from nltk.stem import WordNetLemmatizer\n",
"import pandas as pd\n",
"from gensim.models.word2vec import Word2Vec\n",
"import matplotlib.pyplot as plt\n",
"import numpy as np\n",
"from sklearn.manifold import TSNE"
]
},
{
"cell_type": "code",
"execution_count": 15,
"metadata": {},
"outputs": [],
"source": [
"\n",
"def preprocess(text):\n",
" text = text.lower() #lowercase\n",
" text = re.sub(r'[^\\w\\s]', '', text) #remove punctuations\n",
" text = re.sub(r'\\d+', '', text) #remove numbers\n",
" text = \" \".join(text.split()) #stripWhitespace\n",
" text = text.split()\n",
" text = [x for x in text if x not in stop_words] #remove stopwords\n",
" text = [x for x in text if x not in [\"dr\", \"doctor\"]] #remove task specific stopwords\n",
" text = \" \".join(text)\n",
" stemmer_ps = PorterStemmer() \n",
" text = [stemmer_ps.stem(word) for word in text.split()] #stemming\n",
" text = \" \".join(text)\n",
" lemmatizer = WordNetLemmatizer()\n",
" text = [lemmatizer.lemmatize(word) for word in text.split()] #lemmatization\n",
" text = \" \".join(text)\n",
" return(text)"
]
},
{
"cell_type": "code",
"execution_count": 16,
"metadata": {},
"outputs": [],
"source": [
"data = pd.read_csv(\"winemag-data-130k-v2.csv\")"
]
},
{
"cell_type": "code",
"execution_count": 17,
"metadata": {},
"outputs": [],
"source": [
"data['description_processed']=data['description'].apply(lambda x:preprocess(x))\n",
"data['description_processed']=data['description_processed'].apply(lambda x:x.split())"
]
},
{
"cell_type": "code",
"execution_count": 18,
"metadata": {},
"outputs": [],
"source": [
"model = Word2Vec(sentences=data['description_processed'].tolist(), vector_size=200, sg=1,min_count=5,window=10,workers=-1,seed=10,epochs=400)"
]
},
{
"cell_type": "code",
"execution_count": 19,
"metadata": {},
"outputs": [
{
"data": {
   "text/plain": [
   "10135"
   ]
},
"execution_count": 19,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"vocab = model.wv.index_to_key\n",
"len(vocab)"
]
},
{
"cell_type": "code",
"execution_count": 20,
"metadata": {},
"outputs": [],
"source": [
"outdata=pd.DataFrame(model.wv.vectors)"
]
},
{
"cell_type": "code",
"execution_count": 21,
"metadata": {},
"outputs": [],
"source": [
"outdata.index = model.wv.index_to_key"
]
},
{
"cell_type": "code",
"execution_count": 22,
"metadata": {},
"outputs": [
{
"data": {
   "text/html": [
   "<div>\n",
   "<style scoped>\n",
   " .dataframe tbody tr th:only-of-type {\n",
   "    vertical-align: middle;\n",
   " }\n",
   "\n",
   " .dataframe tbody tr th {\n",
   "    vertical-align: top;\n",
   " }\n",
   "\n",
   " .dataframe thead th {\n",
   "    text-align: right;\n",
   " }\n",
   "</style>\n",
   "<table border=\"1\" class=\"dataframe\">\n",
   " <thead>\n",
   " <tr style=\"text-align: right;\">\n",
   "    <th></th>\n",
   "    <th>0</th>\n",
   "    <th>1</th>\n",
   "    <th>2</th>\n",
   "    <th>3</th>\n",
   "    <th>4</th>\n",
   "    <th>5</th>\n",
   "    <th>6</th>\n",
   "    <th>7</th>\n",
   "    <th>8</th>\n",
   "    <th>9</th>\n",
   "    <th>...</th>\n",
   "    <th>190</th>\n",
   "    <th>191</th>\n",
   "    <th>192</th>\n",
   "    <th>193</th>\n",
   "    <th>194</th>\n",
   "    <th>195</th>\n",
   "    <th>196</th>\n",
   "    <th>197</th>\n",
   "    <th>198</th>\n",
   "    <th>199</th>\n",
   " </tr>\n",
   " </thead>\n",
   " <tbody>\n",
   " <tr>\n",
   "    <th>wine</th>\n",
   "    <td>0.002765</td>\n",
   "    <td>0.004560</td>\n",
   "    <td>-0.002360</td>\n",
   "    <td>-0.002923</td>\n",
   "    <td>0.002926</td>\n",
   "    <td>0.003284</td>\n",
   "    <td>0.000147</td>\n",
   "    <td>-0.003507</td>\n",
   "    <td>0.003329</td>\n",
   "    <td>0.000128</td>\n",
   "    <td>...</td>\n",
   "    <td>0.002686</td>\n",
   "    <td>-0.000727</td>\n",
   "    <td>0.002859</td>\n",
   "    <td>0.002905</td>\n",
   "    <td>0.000998</td>\n",
   "    <td>0.000316</td>\n",
   "    <td>-0.000280</td>\n",
   "    <td>-0.004870</td>\n",
   "    <td>0.001276</td>\n",
   "    <td>-0.004801</td>\n",
   " </tr>\n",
   " <tr>\n",
   "    <th>flavor</th>\n",
   "    <td>0.003476</td>\n",
   "    <td>-0.001096</td>\n",
   "    <td>0.001377</td>\n",
   "    <td>0.003953</td>\n",
   "    <td>0.003949</td>\n",
   "    <td>-0.002684</td>\n",
   "    <td>-0.001951</td>\n",
   "    <td>-0.000556</td>\n",
   "    <td>-0.000888</td>\n",
   "    <td>-0.003559</td>\n",
   "    <td>...</td>\n",
   "    <td>-0.003644</td>\n",

Part A: Random Numbers This is a small exercise in random numbers...