from warnings import filterwarnings
filterwarnings("ignore")

from sklearn.feature_extraction.text import ENGLISH_STOP_WORDS
from nltk.tokenize import word_tokenize, sent_tokenize
from nltk.corpus import stopwords as nltk_stopwords
import matplotlib.pyplot as plt
from wordcloud import WordCloud
from string import punctuation
from bs4 import BeautifulSoup
from datetime import datetime
import seaborn as sns
import pandas as pd
import numpy as np
import requests
import random
import docx
import nltk
import io
import os
import re

output_dir = "Scraped Blogs"
os.makedirs(output_dir, exist_ok=True)

for index, row in input_df.iterrows():
    url = row['URL']
    url_id = row['URL_ID']

    try:
        response = requests.get(url)
        soup = BeautifulSoup(response.text, 'html.parser')
        
        title = soup.title.string.replace(' | Blackcoffer Insights', '')

        if title != "Page not found":

            # Removing the specified title elements from the HTML tree
            title_classes = ['tdb-entry-crumb', 'tdb-bred-no-url-last', 'tdb-title-text']
            for title_class in title_classes:
                title_elements = soup.find_all(True, {'class': title_class})
                for title_element in title_elements:
                    title_element.decompose()
            
            # Removing the div elements that we don't want from the HTML tree
            div_classes = [
                'td_block_wrap tdb_single_next_prev tdi_139 td-animation-stack td-pb-border-top td_block_template_1',
                'tdb-author-box td_block_wrap tdb_single_author_box tdi_140 tdb-content-vert-top td-pb-border-top td_block_template_1',
                'td_block_wrap tdb_single_author tdi_124 td-pb-border-top td_block_template_1 tdb-post-meta',
                'td_block_wrap tdb_single_date tdi_125 td-pb-border-top td_block_template_1 tdb-post-meta',
                'td_block_wrap tdb_single_comments_count tdi_126 td-pb-border-top td_block_template_1 tdb-post-meta',
                'td_block_wrap tdb_single_post_views tdi_127 td-pb-border-top td_block_template_1 tdb-post-meta'
            ]
            for div_class in div_classes:
                div_elements = soup.find_all('div', {'class': div_class})
                for div_element in div_elements:
                    div_element.decompose()
            
            # Removing the ul element from the HTML tree
            ul_element = soup.find('ul', {'class': 'tdb-tags'})
            if ul_element:
                ul_element.decompose()
            
            # Removing the preformatted block from the HTML tree
            preformatted_block = soup.find('pre', {'class': 'wp-block-preformatted'})
            if preformatted_block:
                preformatted_block.decompose()
            
            content_div = soup.find('div', {'id': 'tdb-autoload-article'})
            if content_div:
                content = ''
                for div in content_div.find_all('div', {'class': 'tdb-block-inner td-fix-index'}):
                    content += div.text + '\n'
            else:
                content_div = soup.find('div', {'class': 'td-post-content tagdiv-type'})
                if content_div:
                    content = content_div.text
        
            output_file_path = os.path.join(output_dir, f"{url_id}.txt")
            
            with open(output_file_path, 'w', encoding='utf-8') as f:
                f.write(f"Title: {title}\n\nContent:\n{content.strip()}")
            
    except requests.exceptions.RequestException as e:
        print(f"Error processing URL_ID {url_id}: {e}")

def compute_sentiment_scores(text):

    text = ''.join(char for char in text if char not in punctuation)
    
    words = word_tokenize(text)
    
    words = [word.lower() for word in words if word.isalpha() and word.lower() not in stop_words]
    
    positive_score = sum(1 for word in words if word in positive_words)
    negative_score = sum(1 for word in words if word in negative_words)
    
    polarity_score = round((positive_score - negative_score) / ((positive_score + negative_score) + 0.000001), 2)
    
    subjectivity_score = round((positive_score + negative_score) / ((len(words)) + 0.000001), 2)
    
    return positive_score, negative_score, polarity_score, subjectivity_score

input_dir = "Scraped Blogs"

sentiment_scores = {}

for file_name in os.listdir(input_dir):
    url_id = os.path.splitext(file_name)[0]
    
    with open(os.path.join(input_dir, file_name), 'r', encoding='utf-8') as f:
        content = f.read()
    
    positive_score, negative_score, polarity_score, subjectivity_score = compute_sentiment_scores(content)
    
    sentiment_scores[url_id] = {
        'positive_score': positive_score,
        'negative_score': negative_score,
        'polarity_score': polarity_score,
        'subjectivity_score': subjectivity_score
    }

def count_syllables(word):

    count = 0
    vowels = "aeiouAEIOU"
    if word[0] in vowels:
        count += 1
    for index in range(1, len(word)):
        if word[index] in vowels and word[index - 1] not in vowels:
            count += 1
    
    if word.endswith("es") or word.endswith("ed"):
        count -= 1
    if count == 0:
        count += 1
    
    return count

def compute_variables(text):

    sentences = sent_tokenize(text)
    text = ''.join(char for char in text if char not in punctuation)
    
    stop_words = set(nltk_stopwords.words('english'))
    words = [word.lower() for word in word_tokenize(text) if word.isalpha() and word.lower() not in stop_words]
    
    avg_sentence_length = avg_number_of_words = int(round(len(words) / len(sentences), 0))

    num_syllables = 0
    num_complex_words = 0
    num_chars = 0
    
    for word in words:
        num_syllables_in_word = count_syllables(word)
        if num_syllables_in_word > 2:
            num_complex_words += 1
        
        num_syllables += num_syllables_in_word
        num_chars += len(word)

    complex_word_count = num_complex_words
    total_word_count = len(words)
    percentage_complex_words = round((complex_word_count / total_word_count) * 100, 2)
    avg_syllable_per_word = int(round(num_syllables / total_word_count, 0))
    avg_word_length = int(round(num_chars / total_word_count, 0))
    
    fog_index = round(0.4 * (avg_sentence_length + percentage_complex_words), 2)
    
    personal_pronouns_pattern = r'\bI\b|\bwe\b|\bmy\b|\bours\b|\bus\b(?!A)'
    personal_pronouns_count = len(re.findall(personal_pronouns_pattern, text, re.IGNORECASE))
    
    return avg_sentence_length, percentage_complex_words, fog_index, avg_number_of_words, complex_word_count,\
            total_word_count, avg_syllable_per_word, personal_pronouns_count, avg_word_length

input_dir = "Scraped Blogs"

text_analysis_vars = {}

for file_name in os.listdir(input_dir):

    url_id = os.path.splitext(file_name)[0]
    
    with open(os.path.join(input_dir, file_name), 'r', encoding='utf-8') as f:
        content = f.read()
    
    avg_sentence_length, percentage_complex_words, fog_index, avg_number_of_words, complex_word_count,\
            total_word_count, avg_syllable_per_word, personal_pronouns_count, avg_word_length = compute_variables(content)
    
    text_analysis_vars[url_id] = {
                                'avg_sentence_length': avg_sentence_length,
                                'percentage_of_complex_words': percentage_complex_words,
                                'fog_index': fog_index,
                                'avg_number_of_words': avg_number_of_words,
                                'complex_word_count': complex_word_count,
                                'word_count': total_word_count,
                                'syllable_per_word': avg_syllable_per_word,
                                'personal_pronouns': personal_pronouns_count,
                                'avg_word_length': avg_word_length
                                }

sentiment_scores_df = pd.DataFrame.from_dict(sentiment_scores, orient='index').reset_index()
sentiment_scores_df = sentiment_scores_df.rename(columns={'index': 'URL_ID'})
sentiment_scores_df['URL_ID'] = sentiment_scores_df['URL_ID'].astype('int64')

text_analysis_vars_df = pd.DataFrame.from_dict(text_analysis_vars, orient='index').reset_index()
text_analysis_vars_df = text_analysis_vars_df.rename(columns={'index': 'URL_ID'})
text_analysis_vars_df['URL_ID'] = text_analysis_vars_df['URL_ID'].astype('int64')

output_df.to_excel('Output Data.xlsx', engine ='xlsxwriter', sheet_name = 'Output', index = False)

output_df.drop('avg_sentence_length', inplace = True, axis = 1)

output_df.url_id = output_df.url_id.astype('O')

num_cols = [col for col in output_df.columns if col not in ['url_id', 'url', 'title']]

# Defining the ranges for different correlation labels
ranges = {
    'Highly positively correlated': (0.7, 1.0),
    'Low Positive Correlation': (0.3, 0.7),
    'No Correlation': (-0.3, 0.3),
    'Low Negative Correlation': (-0.7, -0.3),
    'High Negatively Correlated': (-1.0, -0.7)
}

labeled_corr_matrix = pd.DataFrame(index=corr_matrix.index, columns=corr_matrix.columns)

for i in range(corr_matrix.shape[0]):
    for j in range(corr_matrix.shape[1]):
        # Getting the value of the current cell
        value = corr_matrix.iloc[i, j]
        
        # Finding the label for the current value
        label = None
        for k, v in ranges.items():
            if v[0] <= value < v[1]:
                label = k
                break
        
        # Setting the label for the current cell in the labeled correlation matrix
        labeled_corr_matrix.iloc[i, j] = label