BiLSTM with Attention and Pre-trained Embeddings for Sentiment Analysis

{/'title/': /'BiLSTM with Attention and Pre-trained Embeddings for Sentiment Analysis/', /'description/': /'This code implements a sentiment analysis model using a BiLSTM with attention mechanism and pre-trained GloVe embeddings. The model is trained on the SST dataset and achieves improved accuracy compared to a basic LSTM model./', /'keywords/': /'sentiment analysis, BiLSTM, attention, pre-trained embeddings, GloVe, SST dataset, natural language processing, NLP/', /'content/': /'import//u0020copy//nimport//u0020torch//nfrom//u0020torch//u0020import//u0020nn//nfrom//u0020torch//u0020import//u0020optim//nimport//u0020torchtext//nfrom//u0020torchtext//u0020import//u0020data//nfrom//u0020torchtext//u0020import//u0020datasets//n//nTEXT//u0020=//u0020data.Field(sequential=True, batch_first=True, lower=True)//nLABEL//u0020=//u0020data.LabelField()//n//n#//u0020load//u0020data//u0020splits//ntrain_data, val_data, test_data//u0020=//u0020datasets.SST.splits(TEXT, LABEL)//n//n#//u0020build//u0020dictionary//nTEXT.build_vocab(train_data, vectors='glove.6B.100d')//u0020#//u0020Use//u0020pre-trained//u0020GloVe//u0020embeddings//nLABEL.build_vocab(train_data)//n//n#//u0020hyperparameters//nvocab_size//u0020=//u0020len(TEXT.vocab)//nlabel_size//u0020=//u0020len(LABEL.vocab)//npadding_idx//u0020=//u0020TEXT.vocab.stoi['']//nembedding_dim//u0020=//u0020100//u0020#//u0020Match//u0020the//u0020dimension//u0020of//u0020the//u0020pre-trained//u0020GloVe//u0020embeddings//nhidden_dim//u0020=//u0020128//n//n#//u0020build//u0020iterators//ntrain_iter, val_iter, test_iter//u0020=//u0020data.BucketIterator.splits(//n (train_data, val_data, test_data),//n batch_size=32)//n#//u0020your//u0020code//u0020here//nuse_cuda//u0020=//u0020torch.cuda.is_available()//ndevice//u0020=//u0020torch.device(/'cuda/'//u0020if//u0020use_cuda//u0020else//u0020/'cpu/')//n//nclass//u0020Attention(nn.Module)://n def//u0020__init__(self, hidden_dim)://n super(Attention, self).init()//n self.hidden_dim//u0020=//u0020hidden_dim//n self.attention_weights//u0020=//u0020nn.Linear(hidden_dim, hidden_dim)//n self.softmax//u0020=//u0020nn.Softmax(dim=1)//n //n def//u0020forward(self, encoder_outputs)://n energy//u0020=//u0020self.attention_weights(encoder_outputs)//n attention_scores//u0020=//u0020self.softmax(energy)//n context_vector//u0020=//u0020torch.bmm(attention_scores.unsqueeze(1), encoder_outputs)//n return//u0020context_vector//n//nclass//u0020BiLSTMAttentionClassifier(nn.Module)://n def//u0020__init__(self, vocab_size, embedding_dim, hidden_dim, label_size, padding_idx)://n super(BiLSTMAttentionClassifier, self).init()//n self.vocab_size//u0020=//u0020vocab_size//n self.embedding_dim//u0020=//u0020embedding_dim//n self.hidden_dim//u0020=//u0020hidden_dim//n self.label_size//u0020=//u0020label_size//n self.padding_idx//u0020=//u0020padding_idx//n //n self.embedding//u0020=//u0020nn.Embedding(vocab_size, embedding_dim, padding_idx=padding_idx)//n self.lstm//u0020=//u0020nn.LSTM(embedding_dim, hidden_dim, num_layers=2, batch_first=True, bidirectional=True)//n self.attention//u0020=//u0020Attention(hidden_dim * 2)//n self.fc//u0020=//u0020nn.Linear(hidden_dim * 2, label_size)//n self.dropout//u0020=//u0020nn.Dropout(0.5)//n self.softmax//u0020=//u0020nn.LogSoftmax(dim=1)//n //n def//u0020forward(self, text)://n embedded//u0020=//u0020self.embedding(text)//n lstm_output, _//u0020=//u0020self.lstm(embedded)//n attention_output//u0020=//u0020self.attention(lstm_output)//n attention_output//u0020=//u0020attention_output.squeeze(1)//n output//u0020=//u0020self.fc(attention_output)//n output//u0020=//u0020self.dropout(output)//n output//u0020=//u0020self.softmax(output)//n return//u0020output//n//nmodel//u0020=//u0020BiLSTMAttentionClassifier(vocab_size, embedding_dim, hidden_dim, label_size, padding_idx).to(device)//n#optimizer//u0020=//u0020optim.SGD(model.parameters(), lr=0.01)//u0020#//u0020Use//u0020SGD//u0020optimizer//u0020for//u0020now//noptimizer//u0020=//u0020optim.Adam(model.parameters(), lr=0.001)//n//n#criterion//u0020=//u0020nn.NLLLoss()//ncriterion//u0020=//u0020nn.CrossEntropyLoss()//nnum_epochs//u0020=//u002010//n#Define//u0020the//u0020loss//u0020value//u0020and//u0020precision//u0020value//u0020in//u0020the//u0020process//u0020of//u0020model//u0020training//u0020and//u0020verification, which is convenient to use Plot output later//ntrain_losses//u0020=//u0020[]//nval_losses//u0020=//u0020[]//ntrain_accs//u0020=//u0020[]//nval_accs//u0020=//u0020[]//n//ndef//u0020train(model, train_loader, optimizer, criterion)://n model.train()//n total_loss//u0020=//u00200.0//n total_correct//u0020=//u00200//n for//u0020batch//u0020in//u0020train_loader://n text, labels//u0020=//u0020batch.text.to(device), batch.label.to(device)//n optimizer.zero_grad()//n logits//u0020=//u0020model(text)//n loss//u0020=//u0020criterion(logits, labels)//n loss.backward()//n optimizer.step()//n//n total_loss//u0020+=//u0020loss.item() * text.size(0)//n preds//u0020=//u0020logits.argmax(dim=1)//n total_correct//u0020+=//u0020(preds//u0020==//u0020labels).sum().item()//n avg_loss//u0020=//u0020total_loss / len(train_loader.dataset)//n accuracy//u0020=//u0020total_correct / len(train_loader.dataset)//n return//u0020avg_loss, accuracy//n//ndef//u0020evaluate(model, iterator, criterion)://n epoch_loss//u0020=//u00200//n epoch_acc//u0020=//u00200//n//n model.eval()//n//n with//u0020torch.no_grad()://n for//u0020batch//u0020in//u0020iterator://n text, labels//u0020=//u0020batch.text.to(device), batch.label.to(device)//n predictions//u0020=//u0020model(text)//n//n loss//u0020=//u0020criterion(predictions, batch.label)//n acc//u0020=//u0020accuracy(predictions, batch.label)//n//n epoch_loss//u0020+=//u0020loss.item()//n epoch_acc//u0020+=//u0020acc.item()//n//n return//u0020epoch_loss / len(iterator), epoch_acc / len(iterator)//n//ndef//u0020accuracy(predictions, labels)://n _, predicted_labels//u0020=//u0020torch.max(predictions, 1)//n correct//u0020=//u0020(predicted_labels//u0020==//u0020labels).float()//n accuracy//u0020=//u0020correct.sum() / len(correct)//n return//u0020accuracy//n//ndef//u0020model_running(model, train_iter, val_iter, optimizer, criterion)://n best_val_acc//u0020=//u00200.0//n best_model//u0020=//u0020None//n for//u0020epoch//u0020in//u0020range(1, num_epochs+1)://n train_loss, train_acc//u0020=//u0020train(model, train_iter, optimizer, criterion)//n val_loss, val_acc//u0020=//u0020evaluate(model, val_iter, criterion)//n //n train_losses.append(train_loss)//n val_losses.append(val_loss)//n train_accs.append(train_acc)//n val_accs.append(val_acc)//n //n print('Epoch [{}/{}], Train Loss: {:.4f}, Val Loss: {:.4f}, Train Acc: {:.4f}, Val Acc: {:.4f}'//n .format(epoch, num_epochs, train_loss, val_loss, train_acc, val_acc))//n //n if//u0020val_acc//u0020>//u0020best_val_acc://n best_val_acc//u0020=//u0020val_acc//n best_model//u0020=//u0020copy.deepcopy(model.state_dict())//n //n return//u0020best_model//n//nbest_model//u0020=//u0020model_running(model, train_iter, val_iter, optimizer, criterion)//nmodel.load_state_dict(best_model)//n//ntest_loss, test_acc//u0020=//u0020evaluate(model, test_iter, criterion)//nprint(f/'Test loss: {test_loss:.4f}, Test accuracy: {test_acc:.4f}/')//n/