# data manipulation
import pandas as pd
import numpy as np
import time
import re
from tqdm import tqdm

# text preprocessing
import nltk
from nltk.corpus import stopwords
from nltk.stem import WordNetLemmatizer
nltk.downloader.download('vader_lexicon')

# visualizing
import matplotlib.pyplot as plt
import altair as alt
import seaborn as sns
sns.set_style('darkgrid')

# clusters
from sklearn import metrics
from sklearn.cluster import MiniBatchKMeans, KMeans
from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer
from sklearn.datasets import fetch_20newsgroups
# PCA Decomposition
from sklearn.decomposition import PCA
# silhouette method
from sklearn.metrics import silhouette_samples, silhouette_score

# annoying error messages =P
import warnings
warnings.filterwarnings("ignore")

# https://scikit-learn.org/0.19/datasets/twenty_newsgroups.html
newsgroups_train = fetch_20newsgroups(subset='train')

#show topics
print(list(newsgroups_train.target_names))

categories = [
 'comp.graphics',
 'comp.os.ms-windows.misc',
 'rec.sport.baseball',
 'talk.politics.guns',
 'talk.politics.mideast',
 'talk.politics.misc',
 'rec.sport.hockey',
 'alt.atheism',
 'soc.religion.christian',
]
dataset = fetch_20newsgroups(subset='train', categories=categories, shuffle=True, remove=('headers', 'footers', 'quotes'))

df1 = pd.DataFrame(dataset['data'], columns = ['content'])

news = df1.copy()

import string

def text_process(text):
    stemmer = WordNetLemmatizer()
    # remove punctuation
    nopunc = [char for char in text if char not in string.punctuation] # '!"#$%&\'()*+,-./:;<=>?@[\\]^_`{|}~'
    # remove digits
    nopunc = ''.join([i for i in nopunc if not i.isdigit()])
    # split each review into words, lowecase them and remove stopwords
    nopunc = [word.lower() for word in nopunc.split() if word not in stopwords.words('english')]
    # lemmatize the list of words created
    return [stemmer.lemmatize(word) for word in nopunc]

t0 = time.time()
news['filtered_content'] = news['content'].apply(text_process).apply(lambda x : " ".join(x))
print(f'Done! Elapsed time: {round(time.time() - t0, 1)} seconds')

news.shape

news.head()

stop_words = stopwords.words('english')
vectorizer = TfidfVectorizer(
                             min_df = 5, 
                             max_df = 0.95,
                             max_features = None,
                             analyzer = 'word',
                             stop_words = stop_words)


tfidf_text = vectorizer.fit_transform(news.filtered_content)

print(f'n_samples:{tfidf_text.shape[0]}, num_features: {tfidf_text.shape[1]}')

vec = tfidf_text

def optimise_kmeans(data, max_k):
    means = []
    inertias = []
    
    silhouette_avg = []
    
    for k in tqdm(range(2, max_k + 1)):
        kmeans = KMeans( n_clusters = k)
        kmeans.fit(data)
        means.append(k)
        inertias.append(kmeans.inertia_)
        
        # silhouette
        cluster_labels = kmeans.labels_
        silhouette_avg.append(silhouette_score(data, cluster_labels))
        #labels.append(k)

    
    fig, ax =  plt.subplots(1, 2, figsize = (10,3))
    fig.tight_layout(w_pad = 4)
    
    # Elbow method
    ax[0].plot(means, inertias)
    ax[0].set_title('Elbow Method')
    ax[0].set_xlabel('Values of K')
    #plt.xticks(np.arange(1, k, 1))
    ax[0].set_ylabel('Sum squared Error')
    plt.grid(True)
    
    # Silhouette
    ax[1].plot(means, silhouette_avg , 'bv-', color = 'green')
    ax[1].set_title('Silhouette Score')
    ax[1].set_xlabel('Values of K')
    ax[1].set_ylabel('Score')
    plt.grid(True)

    plt.show()
    
    
optimise_kmeans(vec, 10)

kmeans = KMeans(init = 'random', 
                n_clusters = 4,
                n_init = 10,
                random_state = 42)
kmeans.fit(vec)

news['clusters'] = kmeans.labels_

words = vectorizer.get_feature_names_out()[0:10]
words

# Greater TFIDF values
review_groups = news.clusters.value_counts()
review_groups

words = vectorizer.get_feature_names_out()

# 14 most common words from each group
common_words = kmeans.cluster_centers_.argsort()[:,-1:-15:-1]
for num, centroid in enumerate(common_words):
    print(str(num) + " : " + ', '.join(words[word] for word in centroid))

# Renaming clusters
def cluster_names(cluster):
    if cluster == 0:
        return 'sports'
    elif cluster == 1:
        return "religion"
    elif cluster == 2:
        return 'technology'
    elif cluster == 3:
        return 'politics'
    
news.clusters = news.clusters.apply(cluster_names)

# show random samples of each group
def sample_reviews(data, amount = 1):
    
    print('')
    for i in data.clusters.unique():
        print(f'cluster: {i}\n')
        for j in range(0, amount):
            print(data['content'][data['clusters'] == i].sample(1))
            print('')
        print(f"{'-' * 90}")
        

sample_reviews(news, 1)

sns.countplot(x = news.clusters)
plt.title('Topics Distributions')
plt.show()

# num words
pd.set_option('max_colwidth', 100)
news['num_words'] = news["filtered_content"].apply(lambda x: x.split()).apply(len)
news.sample(2)

# number of unique words
news['num_vocab'] = news['filtered_content'].apply(lambda x: x.split()).apply(set).apply(len)
news.sample(2)

news['lexical_div'] = news['num_words'] / news['num_vocab']
news.sample(2)

news["num_char"]=news["filtered_content"].str.len()
news['avg_word_length'] = news['num_char'] / news['num_words']
news.sample(2)

centroids = kmeans.cluster_centers_.argsort()[:, ::-1]

# PCA Decomposition
pca = PCA(n_components = 2, random_state = 42)
pca_vecs = pca.fit_transform(vec.toarray())
x0 = pca_vecs[:, 0]
x1 = pca_vecs[:, 1]

news['x0'] = x0
news['x1'] = x1

import seaborn as sns

plt.figure(figsize = (12, 7))
plt.title("TFIDF + KMeans clustering")
plt.xlabel("X0", fontdict = {'fontsize': 16})
plt.ylabel("X1", fontdict = {'fontsize': 16})
sns.scatterplot(data = news, x = 'x0', y = 'x1', hue = 'clusters', palette = 'tab10')
plt.show()

# transforming the words vocabulary into a dataframe
clusters = vectorizer.vocabulary_

data  = {'Words': list(clusters.keys()),
         'Count': list(clusters.values())}
df2 = pd.DataFrame(data)

# Visualizing groups

alt.Chart(news.sample(4000)).mark_circle(
    size = 100
).encode(
    x = 'x0',
    y = 'x1',
    color = 'clusters:N',
    tooltip = 'content'
).properties(
    width=750,
    height=500
).interactive()