import pandas as pd
from nltk.tokenize import word_tokenize
from nltk.corpus import stopwords
from nltk.stem import PorterStemmer
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.neighbors import KNeighborsClassifier
from sklearn.metrics import accuracy_score

# 读取训练集和测试集
train_df = pd.read_csv('BBC News Train.csv')
test_df = pd.read_csv('BBC News Test.csv')

# 读取样本解决方案
solution_df = pd.read_csv('BBC News Sample Solution.csv')
solution_dict = dict(zip(solution_df['ArticleId'], solution_df['Category']))

# 分词、去停用词、词干化
stop_words = set(stopwords.words('english'))
stemmer = PorterStemmer()

train_texts = []
train_labels = []
for i, row in train_df.iterrows():
    text = row['Text'].lower()
    tokens = word_tokenize(text)
    tokens = [token for token in tokens if token.isalpha() and token not in stop_words]
    tokens = [stemmer.stem(token) for token in tokens]
    train_texts.append(' '.join(tokens))
    train_labels.append(row['Category'])

test_texts = []
test_ids = []
for i, row in test_df.iterrows():
    text = row['Text'].lower()
    tokens = word_tokenize(text)
    tokens = [token for token in tokens if token.isalpha() and token not in stop_words]
    tokens = [stemmer.stem(token) for token in tokens]
    test_texts.append(' '.join(tokens))
    test_ids.append(row['ArticleId'])

# 转化为TF-IDF模型
vectorizer = TfidfVectorizer()
train_features = vectorizer.fit_transform(train_texts)
test_features = vectorizer.transform(test_texts)

# 训练KNN分类器
k = 5
knn_classifier = KNeighborsClassifier(n_neighbors=k)
knn_classifier.fit(train_features, train_labels)

# 对测试集进行分类
test_predictions = knn_classifier.predict(test_features)

# 保存预测结果
predictions_df = pd.DataFrame({'ArticleId': test_ids, 'Category': test_predictions})
predictions_df.to_csv('BBC News Test Predictions.csv', index=False)

# 对测试集进行分类并计算准确率
test_labels = [solution_dict[id] for id in test_ids]
accuracy = accuracy_score(test_labels, test_predictions)
print('Accuracy: {:.2f}%'.format(accuracy * 100))
BBC News分类: 使用KNN和TF-IDF进行文本分类

原文地址: https://www.cveoy.top/t/topic/owjA 著作权归作者所有。请勿转载和采集!

免费AI点我,无需注册和登录