导入必要的库

import jieba import pandas as pd from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer from gensim.models import Word2Vec

加载停用词表

stopwords = pd.read_csv('stopwords.txt', index_col=False, quoting=3, sep='\t', names=['stopword'], encoding='utf-8') stopwords = stopwords['stopword'].values

加载数据集

corpus = pd.read_csv('corpus.txt', sep='\t', names=['category', 'text'], encoding='utf-8')

分词

corpus['text'] = corpus['text'].apply(lambda x: ' '.join(jieba.cut(x)))

停用词处理

corpus['text'] = corpus['text'].apply(lambda x: ' '.join([word for word in x.split() if word not in stopwords]))

文本向量化 - One-hot

one_hot_vectorizer = CountVectorizer(binary=True) one_hot = one_hot_vectorizer.fit_transform(corpus['text'])

文本向量化 - TF-IDF

tfidf_vectorizer = TfidfVectorizer() tfidf = tfidf_vectorizer.fit_transform(corpus['text'])

文本向量化 - Word2Vec

sentences = [text.split() for text in corpus['text'].values] word2vec = Word2Vec(sentences, size=100, window=5, min_count=1, workers=4)

查看结果

print(one_hot.toarray()) print(tfidf.toarray()) print(word2vec.wv['文本'])