Python KNN 文本分类详解：使用哈工大停用词表和交叉验证优化

对于文本分类任务，我们首先需要准备好数据集和停用词表。这里我们选择使用 20 Newsgroups 数据集和哈工大停用词表。

准备数据集和停用词表

20 Newsgroups 数据集可以在 sklearn 库中直接下载，也可以在官网上下载：http://qwone.com/~jason/20Newsgroups/。我们选择使用 sklearn 库中的数据集。

from sklearn.datasets import fetch_20newsgroups

newsgroups_train = fetch_20newsgroups(subset='train', remove=('headers', 'footers', 'quotes'))
newsgroups_test = fetch_20newsgroups(subset='test', remove=('headers', 'footers', 'quotes'))

X_train = newsgroups_train.data
y_train = newsgroups_train.target
X_test = newsgroups_test.data
y_test = newsgroups_test.target

# 加载停用词表
stopwords_file = open('哈工大停用词表.txt', 'r', encoding='utf-8')
stopwords = stopwords_file.readlines()
stopwords = [word.strip() for word in stopwords]
stopwords_file.close()

数据预处理

我们需要对文本进行预处理，包括分词、去停用词、提取特征等。

import jieba

def preprocess(text, stopwords):
    # 分词
    words = jieba.cut(text)
    # 去停用词
    words = [word for word in words if word not in stopwords]
    # 提取特征
    features = {}
    for word in words:
        features[word] = features.get(word, 0) + 1
    return features

X_train_preprocessed = [preprocess(text, stopwords) for text in X_train]
X_test_preprocessed = [preprocess(text, stopwords) for text in X_test]

特征提取

我们使用 TF-IDF 进行特征提取，同时使用交叉验证选择最佳的 K 值。

from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.model_selection import cross_val_score
from sklearn.neighbors import KNeighborsClassifier

vectorizer = TfidfVectorizer()

X_train_tfidf = vectorizer.fit_transform(X_train)
X_test_tfidf = vectorizer.transform(X_test)

k_range = range(1, 10)
k_scores = []
for k in k_range:
    knn = KNeighborsClassifier(n_neighbors=k)
    scores = cross_val_score(knn, X_train_tfidf, y_train, cv=5, scoring='accuracy')
    k_scores.append(scores.mean())

best_k = k_range[k_scores.index(max(k_scores))]

print('Best K:', best_k)

训练模型

我们使用 KNN 算法进行分类，选择最佳的 K 值。

knn = KNeighborsClassifier(n_neighbors=best_k)
knn.fit(X_train_tfidf, y_train)

y_pred = knn.predict(X_test_tfidf)

print('Accuracy:', knn.score(X_test_tfidf, y_test))

完整代码如下：

import jieba
from sklearn.datasets import fetch_20newsgroups
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.model_selection import cross_val_score
from sklearn.neighbors import KNeighborsClassifier

# 加载数据集和停用词表
newsgroups_train = fetch_20newsgroups(subset='train', remove=('headers', 'footers', 'quotes'))
newsgroups_test = fetch_20newsgroups(subset='test', remove=('headers', 'footers', 'quotes'))

X_train = newsgroups_train.data
y_train = newsgroups_train.target
X_test = newsgroups_test.data
y_test = newsgroups_test.target

stopwords_file = open('哈工大停用词表.txt', 'r', encoding='utf-8')
stopwords = stopwords_file.readlines()
stopwords = [word.strip() for word in stopwords]
stopwords_file.close()

# 数据预处理
def preprocess(text, stopwords):
    # 分词
    words = jieba.cut(text)
    # 去停用词
    words = [word for word in words if word not in stopwords]
    # 提取特征
    features = {}
    for word in words:
        features[word] = features.get(word, 0) + 1
    return features

X_train_preprocessed = [preprocess(text, stopwords) for text in X_train]
X_test_preprocessed = [preprocess(text, stopwords) for text in X_test]

# 特征提取
vectorizer = TfidfVectorizer()

X_train_tfidf = vectorizer.fit_transform(X_train)
X_test_tfidf = vectorizer.transform(X_test)

# 交叉验证选择最佳K值
k_range = range(1, 10)
k_scores = []
for k in k_range:
    knn = KNeighborsClassifier(n_neighbors=k)
    scores = cross_val_score(knn, X_train_tfidf, y_train, cv=5, scoring='accuracy')
    k_scores.append(scores.mean())

best_k = k_range[k_scores.index(max(k_scores))]

print('Best K:', best_k)

# 训练模型
knn = KNeighborsClassifier(n_neighbors=best_k)
knn.fit(X_train_tfidf, y_train)

y_pred = knn.predict(X_test_tfidf)

print('Accuracy:', knn.score(X_test_tfidf, y_test))