Sentiment Analysis with Bi-LSTM, Attention, FastText, and Mini Batch: A Comprehensive Approach

import\u0020copy import\u0020torch from\u0020torch\u0020import\u0020nn from\u0020torch\u0020import\u0020optim import\u0020torchtext from\u0020torchtext\u0020import\u0020data from\u0020torchtext\u0020import\u0020datasets

TEXT\u0020=\u0020data.Field(sequential=True,\u0020batch_first=True,\u0020lower=True) LABEL\u0020=\u0020data.LabelField()

#\u0020load\u0020data\u0020splits train_data,\u0020val_data,\u0020test_data\u0020=\u0020datasets.SST.splits(TEXT,\u0020LABEL)

#\u0020build\u0020dictionary TEXT.build_vocab(train_data) LABEL.build_vocab(train_data)

#\u0020hyperparameters vocab_size\u0020=\u0020len(TEXT.vocab) label_size\u0020=\u0020len(LABEL.vocab) padding_idx\u0020=\u0020TEXT.vocab.stoi[''] embedding_dim\u0020=\u0020128 hidden_dim\u0020=\u0020128

#\u0020build\u0020iterators train_iter,\u0020val_iter,\u0020test_iter\u0020=\u0020data.BucketIterator.splits( (train_data,\u0020val_data,\u0020test_data),\u0020 batch_size=32) #\u0020your\u0020code\u0020here use_cuda\u0020=\u0020torch.cuda.is_available() device\u0020=\u0020torch.device("cuda"\u0020if\u0020use_cuda\u0020else\u0020"cpu")\u0020

#\u0020Training\u0020function\u0020 def\u0020train(model,\u0020train_loader,\u0020optimizer,\u0020criterion):\u0020 model.train()\u0020 total_loss\u0020=\u00200.0\u0020 total_correct\u0020=\u00200\u0020 for\u0020batch\u0020in\u0020train_loader:\u0020 text,\u0020labels\u0020=\u0020batch.text.to(device),\u0020batch.label.to(device)\u0020 optimizer.zero_grad()\u0020 logits,\u0020attention_weights\u0020=\u0020model(text) loss\u0020=\u0020criterion(logits,\u0020labels)\u0020

    loss.backward()\u0020
    optimizer.step()\u0020
    
    total_loss\u0020+=\u0020loss.item()\u0020*\u0020text.size(0)\u0020
    preds\u0020=\u0020logits.argmax(dim=1)\u0020
    total_correct\u0020+=\u0020(preds\u0020==\u0020labels).sum().item()\u0020
avg_loss\u0020=\u0020total_loss\u0020/\u0020len(train_loader.dataset)\u0020
accuracy\u0020=\u0020total_correct\u0020/\u0020len(train_loader.dataset)\u0020
return\u0020avg_loss,\u0020accuracy\u0020

def\u0020evaluate(model,\u0020iterator,\u0020criterion): epoch_loss\u0020=\u00200 epoch_acc\u0020=\u00200

model.eval()

with\u0020torch.no_grad():
    for\u0020batch\u0020in\u0020iterator:
        text,\u0020labels\u0020=\u0020batch.text.to(device),\u0020batch.label.to(device)\u0020
        predictions,\u0020_\u0020=\u0020model(text)

        loss\u0020=\u0020criterion(predictions,\u0020batch.label)
        acc\u0020=\u0020accuracy(predictions,\u0020batch.label)

        epoch_loss\u0020+=\u0020loss.item()
        epoch_acc\u0020+=\u0020acc.item()

return\u0020epoch_loss\u0020/\u0020len(iterator),\u0020epoch_acc\u0020/\u0020len(iterator)

def\u0020accuracy(predictions,\u0020labels): _,\u0020predicted_labels\u0020=\u0020torch.max(predictions,\u00201) correct\u0020=\u0020(predicted_labels\u0020==\u0020labels).float() accuracy\u0020=\u0020correct.sum()\u0020/\u0020len(correct) return\u0020accuracy

class\u0020Attention(nn.Module): def\u0020__init__(self,\u0020hidden_dim): super(Attention,\u0020self).init() self.hidden_dim\u0020=\u0020hidden_dim self.attention_weights\u0020=\u0020nn.Linear(hidden_dim,\u00201)

def\u0020forward(self,\u0020lstm_output):
    attention_scores\u0020=\u0020self.attention_weights(lstm_output).squeeze(2)
    attention_weights\u0020=\u0020torch.softmax(attention_scores,\u0020dim=1)
    attention_output\u0020=\u0020torch.bmm(lstm_output.transpose(1,\u00202),\u0020attention_weights.unsqueeze(2)).squeeze(2)
    return\u0020attention_output,\u0020attention_weights

class\u0020RNNClassifier(nn.Module):\u0020 def\u0020__init__(self,\u0020vocab_size,\u0020embedding_dim,\u0020hidden_dim,\u0020label_size,\u0020padding_idx):\u0020 super(RNNClassifier,\u0020self).init()\u0020 self.vocab_size\u0020=\u0020vocab_size\u0020 self.embedding_dim\u0020=\u0020embedding_dim\u0020 self.hidden_dim\u0020=\u0020hidden_dim\u0020 self.label_size\u0020=\u0020label_size\u0020 self.num_layers\u0020=\u00202 self.dropout_num\u0020=\u00200.5 self.bidirectional=True

    #\u0020Embedding\u0020Layer\u0020
    self.embedding\u0020=\u0020nn.Embedding(vocab_size,\u0020embedding_dim,\u0020padding_idx=padding_idx)\u0020
    self.embedding_dropout\u0020=\u0020nn.Dropout(self.dropout_num)\u0020

    #\u0020LSTM\u0020Layer\u0020
    self.lstm\u0020=\u0020nn.LSTM(embedding_dim,\u0020hidden_dim,\u0020num_layers=self.num_layers,\u0020batch_first=True,\u0020bidirectional=self.bidirectional)\u0020
    self.lstm_dropout\u0020=\u0020nn.Dropout(self.dropout_num)\u0020

    #\u0020Attention\u0020Layer
    self.attention\u0020=\u0020Attention(hidden_dim\u0020*\u00202\u0020if\u0020self.bidirectional\u0020else\u0020hidden_dim)
    
    #\u0020Fully\u0020Connected\u0020Layer\u0020
    self.fc\u0020=\u0020nn.Linear(hidden_dim\u0020*\u00202\u0020if\u0020self.bidirectional\u0020else\u0020hidden_dim,\u0020label_size)\u0020
    self.fc_dropout\u0020=\u0020nn.Dropout(self.dropout_num)\u0020
    self.softmax\u0020=\u0020nn.LogSoftmax(dim=1)

def\u0020zero_state(self,\u0020batch_size):\u0020
    hidden\u0020=\u0020torch.zeros(self.num_layers\u0020*\u0020(2\u0020if\u0020self.bidirectional\u0020else\u00201),\u0020batch_size,\u0020self.hidden_dim).to(device)\u0020
    cell\u0020=\u0020torch.zeros(self.num_layers\u0020*\u0020(2\u0020if\u0020self.bidirectional\u0020else\u00201),\u0020batch_size,\u0020self.hidden_dim).to(device)\u0020
    return\u0020hidden,\u0020cell\u0020

def\u0020forward(self,\u0020text):\u0020
    emb\u0020=\u0020self.embedding(text)\u0020
    emb\u0020=\u0020self.embedding_dropout(emb)\u0020
    
    h0,\u0020c0\u0020=\u0020self.zero_state(text.size(0))\u0020
    output,\u0020(hn,\u0020cn)\u0020=\u0020self.lstm(emb,\u0020(h0,\u0020c0))
    output\u0020=\u0020self.lstm_dropout(output)
    
    attention_output,\u0020attention_weights\u0020=\u0020self.attention(output)
    
    output\u0020=\u0020self.fc(attention_output)\u0020
    output\u0020=\u0020self.fc_dropout(output)\u0020
    
    return\u0020output,\u0020attention_weights

model\u0020=\u0020RNNClassifier(vocab_size,\u0020embedding_dim,\u0020hidden_dim,\u0020label_size,padding_idx).to(device)\u0020 optimizer\u0020=\u0020optim.Adam(model.parameters(),\u0020lr=0.001)\u0020 criterion\u0020=\u0020nn.CrossEntropyLoss() num_epochs\u0020=\u002010

train_losses\u0020=\u0020[] val_losses\u0020=\u0020[] train_accs\u0020=\u0020[] val_accs\u0020=\u0020[]

def\u0020model_running(model,train_iter,\u0020val_iter,optimizer,\u0020criterion): best_val_acc\u0020=\u00200.0\u0020 for\u0020epoch\u0020in\u0020range(1,num_epochs+1): train_acc\u0020=\u00200.0 train_loss\u0020=\u00200.0 val_acc\u0020=\u00200.0 val_loss\u0020=\u00200.0 test_acc\u0020=\u00200.0 test_loss\u0020=\u00200.0

    #Call\u0020the\u0020training\u0020function,train\u0020the\u0020model,\u0020and\u0020return\u0020train_loss,train_acc
    train_loss,\u0020train_acc\u0020=\u0020train(model,\u0020train_iter,\u0020optimizer,\u0020criterion)
    val_loss,\u0020val_acc\u0020=\u0020evaluate(model,\u0020val_iter,\u0020criterion)
    
    #\u0020Print\u0020progress
    print('Epoch\u0020[{}/{}],\u0020Train\u0020Loss:\u0020{:.4f},\u0020Val\u0020Loss:\u0020{:.4f},\u0020Train\u0020Acc:\u0020{:.4f},\u0020Val\u0020Acc:\u0020{:.4f}'\n          .format(epoch,\u0020num_epochs,\u0020train_loss,\u0020val_loss,\u0020train_acc,\u0020val_acc))
    
    #\u0020Save\u0020loss\u0020and\u0020accuracy\u0020to\u0020list
    train_losses.append(train_loss)
    val_losses.append(val_loss)
    train_accs.append(train_acc)
    val_accs.append(val_acc)
    
if\u0020val_acc\u0020>\u0020best_val_acc:\u0020
    best_val_acc\u0020=\u0020val_acc\u0020
    best_model\u0020=\u0020model.state_dict()\u0020
    
return\u0020best_model

best_model\u0020=\u0020model_running(model,train_iter,\u0020val_iter,optimizer,\u0020criterion) #\u0020Load\u0020the\u0020best\u0020model\u0020 model.load_state_dict(best_model)\u0020

#\u0020Test\u0020the\u0020model\u0020on\u0020the\u0020test\u0020set\u0020 test_loss,\u0020test_acc\u0020=\u0020evaluate(model,\u0020test_iter,\u0020criterion)\u0020 print(f"Test\u0020loss:\u0020{test_loss:.4f},\u0020Test\u0020accuracy:\u0020{test_acc:.4f}")