Python文本摘要生成：TF-IDF算法实现

{ /'title/': /'# 导入必要的库/', /'description/': /'本代码示例展示了如何使用Python实现TF-IDF算法来生成文本摘要。/', /'keywords/': /'文本摘要, TF-IDF, Python/', /'content/': /'import re/nimport math/nfrom collections import Counter/n/n# 定义一个函数，用于对文本进行预处理/ndef preprocess_text(text):/n '将文本中的标点符号、数字、空格等非文字字符替换为空格'/n text = re.sub(r'[^/w/s]','',text)/n text = re.sub(r'//d+','',text)/n text = re.sub(r'//s+',' ',text)/n '将文本转换为小写字母'/n text = text.lower()/n return text/n/n# 定义一个函数，用于计算TF-IDF权重/ndef compute_tf_idf(text):/n '对文本进行预处理'/n text = preprocess_text(text)/n '将文本分词'/n words = text.split()/n '计算每个单词在文本中出现的次数'/n word_counts = Counter(words)/n '计算文本中单词的总数'/n total_words = len(words)/n '计算每个单词在文本中出现的频率'/n word_freqs = {word: count/total_words for word, count in word_counts.items()}/n '定义一个空字典，用于存储每个单词的TF-IDF权重'/n tf_idf = {}/n '计算每个单词的TF-IDF权重'/n for word in word_freqs:/n '计算单词在所有文本中出现的次数'/n word_in_docs = sum(1 for doc in documents if word in doc)/n '如果单词在所有文本中都没有出现，则将其IDF值设为0'/n if word_in_docs == 0:/n idf = 0/n else:/n '计算单词的IDF值'/n idf = math.log(len(documents)/word_in_docs)/n '计算单词的TF-IDF权重'/n tf_idf[word] = word_freqs[word] * idf/n return tf_idf/n/n# 定义一个函数，用于生成文本摘要/ndef generate_summary(text, num_sentences=3):/n '对文本进行预处理'/n text = preprocess_text(text)/n '将文本分句'/n sentences = re.split(r'(?<!//w//.//w.)(?<![A-Z][a-z]//.)(?<=//.|'r'//?)//s', text)/n '计算每个句子的TF-IDF权重'/n sentence_scores = {}/n for sentence in sentences:/n '将句子分词'/n words = sentence.split()/n '计算句子中单词的TF-IDF权重之和'/n score = sum(tf_idf[word] for word in words if word in tf_idf)/n '存储每个句子的TF-IDF权重'/n sentence_scores[sentence] = score/n '获取TF-IDF权重最高的前num_sentences个句子'/n summary_sentences = sorted(sentence_scores, key=sentence_scores.get, reverse=True)[:num_sentences]/n '将摘要句子按照原文本的顺序重新排序'/n summary_sentences = sorted(summary_sentences, key=lambda sentence: sentences.index(sentence))/n '将摘要句子合并成一个字符串'/n summary = ' '.join(summary_sentences)/n return summary/n/n# 定义一个包含多篇文本的列表/ndocuments = [/n 'Python is a popular programming language. It was created in 1991 by Guido van Rossum.',/n 'Python is used for web development, data analysis, artificial intelligence, and more.',/n 'Python is easy to learn and has a simple syntax, making it a popular choice for beginners.',/n 'Python is open-source software, which means it is free to use and distribute.',/n 'Python has a large and active community, which provides support and contributes to its development.'/n]/n/n# 计算TF-IDF权重/ntf_idf = compute_tf_idf(' '.join(documents))/n/n# 生成文本摘要/nsummary = generate_summary(' '.join(documents))/nprint(summary)/n