import math import numpy as np import jieba import jieba.posseg as psg from gensim import corpora, models from jieba import analyse import functools

步骤2:定义停用词表加载函数。停用词表加载方法

def get_stopword_list(): # 停用词表存储路径，每一行为一个词，按行读取进行加载 # 进行编码转换确保匹配准确率 stop_word_path = 'd:/Users/Administrator/Desktop/data/stop_words.utf8' stopword_list = [sw.replace(' ', '') for sw in open(stop_word_path, encoding='utf-8').readlines()] return stopword_list

步骤3:定义分词函数。分词方法，调用jieba接口

def seg_to_list(sentence, pos=False): if not pos: # 不进行词性标注的分词方法 seg_list = jieba.cut(sentence) else: # 进行词性标注的分词方法 seg_list = psg.cut(sentence) return seg_list

去除干扰词

def word_filter(seg_list, pos=False): stopword_list = get_stopword_list() filter_list = [] # 根据 pos参数选择是否进行词性过滤 # 不进行词性过滤，则将词性都标记为n，表示全部保留 for seg in seg_list: if not pos: word = seg flag = 'n' else: word = seg.word flag = seg.flag if not flag.startswith('n'): continue # 过滤停用词表中的词，以及长度小于2的词 if not word in stopword_list and len(word) > 1: filter_list.append(word) return filter_list

数据加载

def load_data(pos=False, corpus_path='d:/Users/Administrator/Desktop/data/corpus.txt'): doc_list = [] for line in open(corpus_path, 'r', encoding='utf-8'): content = line.strip() seg_list = seg_to_list(content, pos) filter_list = word_filter(seg_list, pos) doc_list.append(filter_list) return doc_list

IDF值统计方法

def train_idf(doc_list): idf_dic = {} # 总文档数 tt_count = len(doc_list) # 每个词出现的文档数 for doc in doc_list: for word in set(doc): idf_dic[word] = idf_dic.get(word, 0.0) + 1.0 # 按公式转换为IDF值，分母加1进行平滑处理 for k, v in idf_dic.items(): idf_dic[k] = math.log(tt_count / (1.0 + v)) # 对于没有在字典中的词，默认其仅在一个文档中出现，得到默认IDF值 default_idf = math.log(tt_count / (1.0)) return idf_dic, default_idf

比较函数，用于topK关键词的按值排序

def cmp(e1, e2): res = np.sign(e1[1] - e2[1]) if res != 0: return res else: a = e1[0] + e2[0] b = e2[0] + e1[0] if a > b: return 1 elif a == b: return 0 else: return -1

TF-IDF类

class TfIdf(object): # 四个参数分别是训练好的IDF字典、默认IDF值、处理后的待提取文本、关键词数量 def init(self, idf_dic, default_idf, word_list, keyword_num): self.word_list = word_list self.idf_dic, self.default_idf = idf_dic, default_idf self.tf_dic = self.get_tf_dic() self.keyword_num = keyword_num

# 统计TF值
def get_tf_dic(self):
    tf_dic = {}
    for word in self.word_list:
        tf_dic[word] = tf_dic.get(word, 0.0) + 1.0
    tt_count = len(self.word_list)
    for k, v in tf_dic.items():
        tf_dic[k] = float(v) / tt_count
    return tf_dic

# 按公式计算TF-IDF值
def get_tfidf(self):
    tfidf_dic = {}
    for word in self.word_list:
        idf = self.idf_dic.get(word, self.default_idf)
        tf = self.tf_dic.get(word, 0)
        tfidf = tf * idf
        tfidf_dic[word] = tfidf
    tfidf_dic.items()
    # 根据TF-IDF值排序，去排序前keyword_num的词作为关键词
    for k, v in sorted(tfidf_dic.items(), key=functools.cmp_to_key(cmp), reverse=True)[:self.keyword_num]:
        print(k + '/', end='')
    print()

def tfidf_extract(word_list, pos=False, keyword_num=10): doc_list = load_data(pos) idf_dic, default_idf = train_idf(doc_list) tfidf_model = TfIdf(idf_dic, default_idf, word_list, keyword_num) tfidf_model.get_tfidf()

def textrank_extract(text, pos=False, keyword_num=10): textrank = analyse.textrank keywords = textrank(text, keyword_num) # 输出提取的关键词 for keyword in keywords: print(keyword + '/', end='') print()

修改的部分

with open('d:/Users/Administrator/Desktop/data/corpus.txt', 'r', encoding='utf-8') as f: # 打开文本 text = f.read() # 读取文本

seg_list = seg_to_list(text, pos) filter_list = word_filter(seg_list, pos)

print('TF-IDF模型结果:') tfidf_extract(filter_list)

print('TextRank模型结果: ') textrank_extract(text)

result = '''TF-IDF模型结果: ''' tfidf_extract(filter_list)

result += 'TextRank模型结果: ' textrank_extract(text)

with open('d:/Users/Administrator/Desktop/data/results_textrank.txt', 'w', encoding='utf-8') as f: f.write(result)