# Data handling
import numpy as np
import pandas as pd
from collections import Counter
import time, math

# Parallel Tasking
from joblib import Parallel, delayed

# Web Crawling
from bs4 import BeautifulSoup
import requests

import matplotlib.pyplot as plt

# For the recommender system
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.metrics.pairwise import cosine_similarity
from sklearn.preprocessing import MultiLabelBinarizer 

netflix_data = pd.read_csv("Data/netflix_titles.csv")
netflix_data.head(2)

netflix_data.describe(include='all').head(4)

netflix_data.groupby('type').count()

movies_data = netflix_data.loc[netflix_data["type"]=="Movie",].copy()

movies_data["title"] = movies_data['title'].str.strip().str.lower()
temp = movies_data['title'].value_counts()
movies_data.loc[movies_data["title"].isin(list(temp.index[temp>1])),]

movies_data = movies_data.drop([6529,6705,7345])

print("Rows with missing values in the data: "+
      str(round(100*sum(movies_data.isnull().any(axis=1))/movies_data.shape[0],2))+"%")
movies_data.isna().sum()

movies_data.dropna(subset=["rating","duration"], how='any', inplace=True)
print("Rows with missing values in the data: "+str(round(100*sum(movies_data.isnull().any(axis=1))/movies_data.shape[0],2))+"%")
movies_data.isna().sum()

nan_rows_df = movies_data[movies_data.isnull().any(axis=1)]
nan_rows_df.head(2)

# Functions to extract the director, cast, country of origin data from a page source

def get_director(soup):
    """
    Extract the director information from the HTML source data
    Args:
        soup: object (page source) obtained from scraping the website using BeautifulSoup() function
    Returns:
        director: returns a string containing directors of a movie separated by a comma
    """ 
    try:
        director = ""
        temp = soup.find("section",{"data-testid":"title-cast"}).find_all("li",{"class","ipc-metadata-list__item"})
        if len(temp)==4: #if the section on the page in found
            director_soups = temp[0].find_all("a")
            for director_soup in director_soups:
                name = director_soup.get_text().strip()
                director = director + name + ", "
            director = director[:-2]
            return director
        else:
            return director
    except:
        return director

def get_cast(soup):
    """
    Extract the cast information from the HTML source data
    Args:
        soup: object (page source) obtained from scraping the website using BeautifulSoup() function
    Returns:
        cast: returns a string containing all the cast members of a movie separated by a comma
    """ 
    try:
        cast = ""
        cast_soups = soup.find("section",{"data-testid":"title-cast"}).find_all("a",{"data-testid":"title-cast-item__actor"})
        for cast_soup in cast_soups:
            name = cast_soup.get_text().strip()
            cast = cast + name + ", "
        cast = cast[:-2]
        return cast
    except:
        return cast
    
def get_country(soup):
    """
    Extract the country information from the HTML source data
    Args:
        soup: object (page source) obtained from scraping the website using BeautifulSoup() function
    Returns:
        country: returns a string containing all the countries of origin of a movie separated by a comma
    """ 
    try:
        country = ""
        countries_soups = soup.find("div",{"data-testid":"title-details-section"}).find("li",{"data-testid":"title-details-origin"}).find_all("a")
        for countries_soup in countries_soups:
            name = countries_soup.get_text().strip()
            country = country + name + ", "
        country = country[:-2]
        return country
    except:
        return country

%%time

def imdb_requests(row):
    """
    Main Function that extracts the director, cast, and countries of origin information for each row with NaN value
    Args:
        row: dataframe row containing atleast one NaN value
    Returns:
        main_dict: returns a dictionary containing title, show_id, director, cast, country as keys and their
        corresponding values as dictionary values
    """ 
    main_dict = {}
    main_dict["title"] = row["title"]
    main_dict["show_id"] = row["show_id"]
    
    try:
        source = requests.get("https://www.imdb.com/find?ref_=nv_sr_fn&q="+str(main_dict["title"]))
        source.raise_for_status()
        soup = BeautifulSoup(source.text,'html.parser')

        #take the first URL on the results page and extract information from it
        title = soup.find("td",{"class":"result_text"}).find('a').get("href")
        
        new_url = "https://www.imdb.com"+title
        source = requests.get(new_url)
        source.raise_for_status()

        soup = BeautifulSoup(source.text,'html.parser')
        main_dict["director"] = get_director(soup)
        main_dict["cast"] = get_cast(soup)
        main_dict["country"] = get_country(soup)        
    except:
        pass
    
    return main_dict

"""
The below four rows calls several URL requests using a parallel function, convert an array of dictionaries 
to a data frame, and replace empty string with NaN values (to impute them later).
This code takes close to 5 minutes to run. I ran this already and generated the intermediate file, 
which I will use in the subsequent sections.
"""
# nan_rows_search_results = Parallel(n_jobs=-1)(delayed(imdb_requests)(row) for index, row in nan_rows_df.iterrows())
# nan_rows_search_results_df = pd.DataFrame(nan_rows_search_results)
# nan_rows_search_results_df = nan_rows_search_results_df.replace('',np.nan,regex=True)
# nan_rows_search_results_df.to_csv("IMDB_intermediate_data.csv",index=False)
nan_rows_search_results_df = pd.read_csv("IMDB_intermediate_data.csv")

movies_data.cast = np.where(movies_data.cast.isnull(),movies_data.show_id.map(nan_rows_search_results_df.set_index('show_id').cast),movies_data.cast)
movies_data.country = np.where(movies_data.country.isnull(),movies_data.show_id.map(nan_rows_search_results_df.set_index('show_id').country),movies_data.country)
movies_data.director = np.where(movies_data.director.isnull(),movies_data.show_id.map(nan_rows_search_results_df.set_index('show_id').director),movies_data.director)

print("Rows with missing values in the data: "+str(round(100*sum(movies_data.isnull().any(axis=1))/movies_data.shape[0],2))+"%")
movies_data.isna().sum()

movies_data = movies_data.replace(np.nan,'',regex=True)
movies_data.reset_index(drop=True,inplace=True)
print("Rows with missing values in the data: "+str(round(100*sum(movies_data.isnull().any(axis=1))/movies_data.shape[0],2))+"%")
movies_data.isna().sum()

# Year added column
movies_data['year_added'] = movies_data['date_added'].apply(lambda x: x.split(" ")[-1])
movies_data['year_added'] = movies_data["year_added"].astype("int")
# Month added column
movies_data['month_added'] = movies_data['date_added'].apply(lambda x: x.split(" ")[0])
movies_data['date_added'] = pd.to_datetime(movies_data['date_added'])
movies_data['day_of_week'] = movies_data['date_added'].dt.day_name()
movies_data[['month_added','year_added','day_of_week']].head()

movies_data['duration']=movies_data['duration'].str.replace(' min','')
movies_data['duration']=movies_data['duration'].astype(str).astype(int)

features = ['title','director','cast','listed_in']

def clean_data(df,features):
    df_subset = df[features].copy()
    df_subset['main_column'] = ""
    for feature in features:
        if feature!="description":
            df_subset[feature] = df_subset[feature].apply(lambda x: str.lower(x.replace(" ", "")))
        df_subset["main_column"] = df_subset["main_column"] + ' ' + df_subset[feature]
    return df_subset

movies_data_subset = clean_data(movies_data,features)
movies_data_subset.head(2)

tfidf = TfidfVectorizer(stop_words='english')
tfidf_matrix = tfidf.fit_transform(movies_data_subset['main_column'])
cosine_sim = cosine_similarity(tfidf_matrix, tfidf_matrix)

movie_titles_df = pd.DataFrame(movies_data['title']).reset_index()
movie_titles_df.columns = ["row_id","Title"]

cosine_sim_df = pd.DataFrame(cosine_sim).reset_index()
cosine_sim_df_melted = pd.melt(cosine_sim_df, id_vars=['index'], value_vars=list(cosine_sim_df.columns[1:]))
cosine_sim_df_melted.columns = ["row_id1","row_id2","similarity"]
cosine_sim_df_melted = cosine_sim_df_melted.sort_values("similarity",ascending=False)
cosine_sim_df_melted = cosine_sim_df_melted.loc[cosine_sim_df_melted["row_id1"]<cosine_sim_df_melted["row_id2"],].reset_index(drop=True)

thres = 0.9
filtered_df = cosine_sim_df_melted.loc[cosine_sim_df_melted["similarity"]>thres,].copy()
filtered_df = filtered_df.merge(movie_titles_df,left_on="row_id1",right_on="row_id")
filtered_df = filtered_df.merge(movie_titles_df,left_on="row_id2",right_on="row_id")
filtered_df = filtered_df[["Title_x","Title_y","similarity"]].copy()
filtered_df.columns = ["Movie1","Movie2","Similarity"]
filtered_df["Similarity"] = round(filtered_df["Similarity"],2)
filtered_df

movies_data_subset=movies_data_subset.reset_index()
indices = pd.Series(movies_data_subset.index, index=movies_data_subset['title'])

def get_recommendations_new(title, cosine_sim, n):
    """
    Find the similar movies to a given movie
    Args:
        title: movie title to which we find recommendations
        cosine_sim: cosine similarity matrix for finding similar movies
        n: number of movies to recommend
    Returns:
        results_df: returns a dataframe containing the list of recommended movies with rowids
        and their similarity score
    """ 
    title = title.replace(' ','').lower()
    idx = indices[title]

    #pairwsie similarity scores of all movies with that movie
    sim_scores = list(enumerate(cosine_sim[idx]))
    #sort the movies based on cosine similarity scores
    sim_scores = sorted(sim_scores, key=lambda x: x[1], reverse=True)

    # Get the scores of the top n most similar movies
    sim_scores = sim_scores[1:(n+1)]
    # Get their movie indices
    movie_indices = [i[0] for i in sim_scores]

    # Return the top 10 most similar movies
    results_df = pd.DataFrame(movies_data['title'].iloc[movie_indices])
    results_df["score"] = np.round(np.array(sim_scores)[:,1],2)
    results_df = results_df.reset_index(drop=False)
    results_df.columns = ["RowID","Recommended Movie","Similarity Score"]
    return results_df

movie_title = "pk"
recommendations_df = get_recommendations_new(movie_title,cosine_sim,5)

temp_df = movies_data.loc[movies_data.title.isin([movie_title]+list(recommendations_df["Recommended Movie"]))]
temp_df = temp_df[features].reset_index(drop=True)
temp_df = temp_df.merge(recommendations_df,left_on="title",right_on = "Recommended Movie",how="outer")
temp_df = temp_df.sort_values("Similarity Score",ascending=False)
temp_df = temp_df[["title","director","cast","listed_in","Similarity Score"]]
temp_df["new"] = range(1,len(temp_df)+1)
temp_df.loc[temp_df.title==movie_title,'new'] = 0
temp_df = temp_df.sort_values("new").drop('new', axis=1)
temp_df