main.py

import json
import pandas as pd

import plotly.io as pio
from bertopic import BERTopic
import matplotlib.pyplot as plt

from controller import df

documents = df["content"].tolist()
timestamps = df["year"].tolist()

# Training
#topic_model = BERTopic(verbose=True, calculate_probabilities=True)
#topics, probs = topic_model.fit_transform(documents)
#topic_model.save("topic_model")

#probs_df = pd.DataFrame(probs)
#probs_df.to_csv("topic_probabilities.csv", index=False)

topic_model = BERTopic.load("topic_model")
#topic_model.reduce_topics(documents, nr_topics = 44)

#topic_info = topic_model.get_topic_info()
#topic_info.to_csv("topic_info.csv", index = False)
#print(topic_info)

# To view the Intertopic Distance Map
#fig = topic_model.visualize_topics()
#pio.show(fig)

# To view the Hierarchical Dendrogram
#hierarchical_topics = topic_model.hierarchical_topics(documents)
#fig = topic_model.visualize_hierarchy(hierarchical_topics=hierarchical_topics)
#pio.show(fig)

# To perform dynamic topic modeling
topics_over_time = topic_model.topics_over_time(documents, timestamps)
topics_over_time.to_csv("topics_over_time.csv", index = False)
print(topics_over_time)

# To view the topics over time graph
fig = topic_model.visualize_topics_over_time(topics_over_time)
fig.write_html("topics_over_time.html")
pio.show(fig)