PyTorch 分布式训练转单卡训练：简单修改指南

将原先的分布式训练代码中的分布式相关代码删除或注释掉即可。同时，需要将 batch size、learning rate 等参数适当调整，以适应单卡训练的场景。例如：

# 分布式训练代码
import torch
import torch.distributed as dist

# 初始化进程组
dist.init_process_group(backend='nccl', init_method='...')
# 定义模型、损失函数和优化器
model = ...
criterion = ...
optimizer = torch.optim.SGD(model.parameters(), lr=0.01)
optimizer = torch.nn.parallel.DistributedDataParallel(optimizer, device_ids=[dist.get_rank()])

# 训练循环
for epoch in range(num_epochs):
    for i, (inputs, labels) in enumerate(train_loader):
        inputs, labels = inputs.cuda(), labels.cuda()
        optimizer.zero_grad()
        outputs = model(inputs)
        loss = criterion(outputs, labels)
        loss.backward()
        optimizer.step()

# 单卡训练代码
import torch

# 定义模型、损失函数和优化器
model = ...
criterion = ...
optimizer = torch.optim.SGD(model.parameters(), lr=0.001)

# 训练循环
for epoch in range(num_epochs):
    for i, (inputs, labels) in enumerate(train_loader):
        inputs, labels = inputs.cuda(), labels.cuda()
        optimizer.zero_grad()
        outputs = model(inputs)
        loss = criterion(outputs, labels)
        loss.backward()
        optimizer.step()