文本预处理：分词、停用词处理、向量化 (Python 代码)

文本预处理是自然语言处理中必不可少的一步，它可以将原始文本数据转换为机器可以理解的形式，方便进行后续的分析和建模。常见的文本预处理方法包括分词、停用词处理、文本向量化等。

1. 文本分词

文本分词是指将文本划分为一个个单词或词组的过程。

2. 停用词处理

停用词是指在文本中出现频率很高但语义价值很低的词，如“的”、“了”、“是”等。停用词处理是指去除文本中的停用词，以提高文本的质量和效率。

3. 文本向量化

文本向量化是指将文本转换成向量，以便进行机器学习或深度学习等操作。常用的文本向量化方法包括 one-hot、TF-IDF、Word2Vec 等。

代码实现

首先，需要安装以下库：

!pip install jieba scikit-learn pandas numpy

其中，jieba用于中文分词，scikit-learn用于TF-IDF文本向量化，pandas用于数据处理，numpy用于科学计算。

文本分词

import jieba
import os

# 读取文本文件并进行分词
def read_file(file_path):
    with open(file_path, 'r', encoding='utf-8') as f:
        text = f.read()
        seg_list = jieba.cut(text)
        return seg_list

# 读取整个数据集并进行分词
def read_corpus(corpus_path):
    corpus = []
    labels = []
    for root, dirs, files in os.walk(corpus_path):
        for file in files:
            label = root.split('\')[-1]
            labels.append(label)
            seg_list = read_file(os.path.join(root, file))
            corpus.append(' '.join(seg_list))
    return corpus, labels

停用词处理

import jieba
import os

# 读取停用词文件
def read_stopwords(stopwords_path):
    with open(stopwords_path, 'r', encoding='utf-8') as f:
        stopwords = [line.strip() for line in f]
        return stopwords

# 去除停用词
def remove_stopwords(seg_list, stopwords):
    return [word for word in seg_list if word not in stopwords]

# 读取文本文件并进行分词和停用词处理
def read_file(file_path, stopwords):
    with open(file_path, 'r', encoding='utf-8') as f:
        text = f.read()
        seg_list = jieba.cut(text)
        seg_list = remove_stopwords(seg_list, stopwords)
        return seg_list

# 读取整个数据集并进行分词和停用词处理
def read_corpus(corpus_path, stopwords_path):
    stopwords = read_stopwords(stopwords_path)
    corpus = []
    labels = []
    for root, dirs, files in os.walk(corpus_path):
        for file in files:
            label = root.split('\')[-1]
            labels.append(label)
            seg_list = read_file(os.path.join(root, file), stopwords)
            corpus.append(' '.join(seg_list))
    return corpus, labels

文本向量化

from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer

# one-hot向量化
def one_hot_vectorizer(corpus):
    vectorizer = CountVectorizer()
    X = vectorizer.fit_transform(corpus)
    return X.toarray()

# TF-IDF向量化
def tfidf_vectorizer(corpus):
    vectorizer = TfidfVectorizer()
    X = vectorizer.fit_transform(corpus)
    return X.toarray()

完整代码

import jieba
import os
from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer
import pandas as pd
import numpy as np

# 读取停用词文件
def read_stopwords(stopwords_path):
    with open(stopwords_path, 'r', encoding='utf-8') as f:
        stopwords = [line.strip() for line in f]
        return stopwords

# 去除停用词
def remove_stopwords(seg_list, stopwords):
    return [word for word in seg_list if word not in stopwords]

# 读取文本文件并进行分词和停用词处理
def read_file(file_path, stopwords):
    with open(file_path, 'r', encoding='utf-8') as f:
        text = f.read()
        seg_list = jieba.cut(text)
        seg_list = remove_stopwords(seg_list, stopwords)
        return seg_list

# 读取整个数据集并进行分词和停用词处理
def read_corpus(corpus_path, stopwords_path):
    stopwords = read_stopwords(stopwords_path)
    corpus = []
    labels = []
    for root, dirs, files in os.walk(corpus_path):
        for file in files:
            label = root.split('\')[-1]
            labels.append(label)
            seg_list = read_file(os.path.join(root, file), stopwords)
            corpus.append(' '.join(seg_list))
    return corpus, labels

# one-hot向量化
def one_hot_vectorizer(corpus):
    vectorizer = CountVectorizer()
    X = vectorizer.fit_transform(corpus)
    return X.toarray()

# TF-IDF向量化
def tfidf_vectorizer(corpus):
    vectorizer = TfidfVectorizer()
    X = vectorizer.fit_transform(corpus)
    return X.toarray()

# Word2Vec向量化
def word2vec_vectorizer(corpus):
    pass

if __name__ == "__main__":
    corpus_path = './corpus'
    stopwords_path = './stopwords.txt'
    corpus, labels = read_corpus(corpus_path, stopwords_path)
    X_one_hot = one_hot_vectorizer(corpus)
    X_tfidf = tfidf_vectorizer(corpus)
    print(X_one_hot.shape)
    print(X_tfidf.shape)