IMDB情感分析：基于NLTK的朴素贝叶斯分类器实现

import\u0020os\nimport\u0020string\nfrom\u0020nltk.corpus\u0020import\u0020stopwords\nfrom\u0020nltk.stem\u0020import\u0020WordNetLemmatizer\nfrom\u0020nltk.tokenize\u0020import\u0020word_tokenize\nfrom\u0020sklearn.feature_extraction.text\u0020import\u0020CountVectorizer\nfrom\u0020sklearn.naive_bayes\u0020import\u0020MultinomialNB\nfrom\u0020sklearn.metrics\u0020import\u0020accuracy_score\n\n#\u0020Download\u0020necessary\u0020NLTK\u0020resources\nimport\u0020nltk\nnltk.download('punkt')\nnltk.download('stopwords')\nnltk.download('wordnet')\n\n#\u0020Preprocess\u0020the\u0020text\ndef\u0020preprocess_text(text):\n\t#\u0020Tokenize\u0020the\u0020text\n\ttokens\u0020=\u0020word_tokenize(text.lower())\n\t\n\t#\u0020Remove\u0020punctuation\n\ttokens\u0020=\u0020[token\u0020for\u0020token\u0020in\u0020tokens\u0020if\u0020token\u0020not\u0020in\u0020string.punctuation]\n\t\n\t#\u0020Remove\u0020stopwords\n\tstop_words\u0020=\u0020set(stopwords.words('english'))\n\ttokens\u0020=\u0020[token\u0020for\u0020token\u0020in\u0020tokens\u0020if\u0020token\u0020not\u0020in\u0020stop_words]\n\t\n\t#\u0020Lemmatize\u0020words\n\tlemmatizer\u0020=\u0020WordNetLemmatizer()\n\ttokens\u0020=\u0020[lemmatizer.lemmatize(token)\u0020for\u0020token\u0020in\u0020tokens]\n\t\n\t#\u0020Join\u0020tokens\u0020back\u0020to\u0020text\n\tpreprocessed_text\u0020=\u0020' \u0027.join(tokens)\n\treturn\u0020preprocessed_text\n\n#\u0020Load\u0020the\u0020IMDB\u0020dataset\ndef\u0020load_dataset(path):\n\tdocuments\u0020=\u0020[]\n\tlabels\u0020=\u0020[]\n\tfor\u0020label_type\u0020in\u0020['neg',\u0020'pos']: \n\t\tdir_path\u0020=\u0020os.path.join(path,\u0020label_type)\n\t\tfor\u0020filename\u0020in\u0020os.listdir(dir_path):\n\t\t\tfile_path\u0020=\u0020os.path.join(dir_path,\u0020filename)\n\t\t\twith\u0020open(file_path,\u0020'r')\u0020as\u0020file:\n\t\t\t\ttext\u0020=\u0020file.read()\n\t\t\t\tpreprocessed_text\u0020=\u0020preprocess_text(text)\n\t\t\t\tdocuments.append(preprocessed_text)\n\t\t\t\tlabels.append(label_type)\n\treturn\u0020documents,\u0020labels\n\n#\u0020Train\u0020the\u0020classifier\u0020and\u0020predict\ndef\u0020train_and_predict(X_train,\u0020y_train,\u0020X_test):\n\t#\u0020Convert\u0020text\u0020to\u0020vectors\n\tvectorizer\u0020=\u0020CountVectorizer()\n\tX_train\u0020=\u0020vectorizer.fit_transform(X_train)\n\tX_test\u0020=\u0020vectorizer.transform(X_test)\n\t\n\t#\u0020Train\u0020the\u0020classifier\n\tclassifier\u0020=\u0020MultinomialNB()\n\tclassifier.fit(X_train,\u0020y_train)\n\t\n\t#\u0020Predict\u0020labels\n\ty_pred\u0020=\u0020classifier.predict(X_test)\n\treturn\u0020y_pred\n\n#\u0020Calculate\u0020accuracy\ndef\u0020calculate_accuracy(y_pred,\u0020y_test):\n\taccuracy\u0020=\u0020accuracy_score(y_test,\u0020y_pred)\n\treturn\u0020accuracy\n\n#\u0020Define\u0020the\u0020path\u0020to\u0020the\u0020IMDB\u0020dataset\ndataset_path\u0020=\u0020'path/to/dataset'\n\n#\u0020Load\u0020the\u0020dataset\nX,\u0020y\u0020=\u0020load_dataset(dataset_path)\n\n#\u0020Split\u0020the\u0020dataset\u0020into\u0020training\u0020and\u0020testing\u0020sets\nsplit_ratio\u0020=\u00200.8\nsplit_index\u0020=\u0020int(len(X)\u0020*\u0020split_ratio)\nX_train\u0020=\u0020X[:split_index]\ny_train\u0020=\u0020y[:split_index]\nX_test\u0020=\u0020X[split_index:]\ny_test\u0020=\u0020y[split_index:]\n\n#\u0020Train\u0020and\u0020predict\ny_pred\u0020=\u0020train_and_predict(X_train,\u0020y_train,\u0020X_test)\n\n#\u0020Calculate\u0020accuracy\naccuracy\u0020=\u0020calculate_accuracy(y_pred,\u0020y_test)\n\n#\u0020Print\u0020the\u0020predicted\u0020labels\u0020and\u0020accuracy\nprint("Predicted\u0020labels:",\u0020y_pred)\nprint("Accuracy:",\u0020accuracy)