import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import gensim
from gensim.corpora import Dictionary
from gensim.models import LdaModel, CoherenceModel

读取数据

data = pd.read_excel('data_perplexity.xlsx')
texts = data.apply(lambda x:str(x).split())

构建词典

dictionary = Dictionary(texts)
corpus = [dictionary.doc2bow(text) for text in texts]

定义最大主题数

max_topics = 16

训练LDA模型并计算困惑度

perplexity = []
for num_topics in range(1, max_topics+1):
lda_model = LdaModel(corpus=corpus, id2word=dictionary, num_topics=num_topics)
perplexity.append(np.exp2(-lda_model.log_perplexity(corpus))) # 将困惑度转换为正值

绘制主题数与困惑度曲线

plt.plot(range(1, max_topics+1), perplexity)
plt.xlabel("Number of Topics")
plt.ylabel("Perplexity")
plt.title("Number of Topics vs Perplexity")
plt.show()

计算一致性

coherence = []
for num_topics in range(1, max_topics+1):
lda_model = LdaModel(corpus=corpus, id2word=dictionary, num_topics=num_topics)
if coherence == 'c_v' and texts is None:
raise ValueError("'texts' should be provided for c_v coherence.")
else:
coherence_model_lda = CoherenceModel(model=lda_model, corpus=corpus, dictionary=dictionary, coherence='c_v', texts=texts)
coherence_lda = coherence_model_lda.get_coherence()
coherence.append(coherence_lda)

绘制主题数与一致性曲线

plt.plot(range(1, max_topics+1), coherence)
plt.xlabel("Number of Topics")
plt.ylabel("Coherence Score")
plt.title("Number of Topics vs Coherence Score")
plt.show()