PyTorch Error Database

# Bug: nn.Linear(256, 10) but flatten output is 512
# Fix: match in_features to actual flatten output
self.fc = nn.Linear(512, 10)

# Or calculate dynamically:
dummy = torch.zeros(1, 3, 32, 32)
dummy = self.features(dummy)
self.fc = nn.Linear(dummy.view(1, -1).shape[1], 10)

# Bug: model expects 3 channels, input has 1
# Fix option 1: change first conv layer
self.conv1 = nn.Conv2d(1, 64, kernel_size=7, stride=2, padding=3)

# Fix option 2: convert grayscale to 3-channel
x = x.repeat(1, 3, 1, 1)  # repeat grayscale across 3 channels

# Bug: output has batch 32, target has batch 16
# Check shapes before loss computation:
print(f"Output: {output.shape}, Target: {target.shape}")

# Common fix: ensure data loader returns matching pairs
for inputs, targets in dataloader:
    outputs = model(inputs)
    assert outputs.shape[0] == targets.shape[0]
    loss = criterion(outputs, targets)

# Bug: hardcoded reshape dimensions
x = x.view(32, 784)  # fails if batch != 32 or features != 784

# Fix: use -1 for automatic dimension inference
x = x.view(x.size(0), -1)  # auto-compute feature dim
# Or use nn.Flatten()
self.flatten = nn.Flatten()

# Bug: too many downsampling layers for input size
# Input: 32x32 -> Conv(s=2) -> 16x16 -> Pool(2) -> 8x8 -> Conv(s=2) -> 4x4 -> Pool(2) -> 2x2 -> Conv(k=3) -> ERROR

# Fix: reduce kernel size, add padding, or use fewer downsampling layers
self.conv3 = nn.Conv2d(128, 256, kernel_size=1)  # 1x1 conv instead of 3x3
# Or add padding:
self.conv3 = nn.Conv2d(128, 256, kernel_size=3, padding=1)

# Bug: hidden state initialized with wrong batch size
h0 = torch.zeros(2, 1, 256)  # batch=1
output, _ = self.lstm(x, (h0, c0))  # x has batch=32

# Fix: match hidden batch to input batch
batch_size = x.size(0)  # or x.size(1) if batch_first=False
h0 = torch.zeros(2, batch_size, 256).to(x.device)
c0 = torch.zeros(2, batch_size, 256).to(x.device)

# Bug: incompatible shapes for addition
a = torch.randn(32, 10)
b = torch.randn(32, 5)
c = a + b  # RuntimeError!

# Fix: ensure shapes are compatible
b = torch.randn(32, 10)  # match dimension
# Or reshape/project:
proj = nn.Linear(5, 10)
c = a + proj(b)

# Bug: passing single image without batch dim
img = torch.randn(3, 224, 224)
output = model(img)  # RuntimeError!

# Fix: add batch dimension
img = img.unsqueeze(0)  # shape: [1, 3, 224, 224]
output = model(img)

# Bug: 3-channel normalize on 1-channel image
transform = transforms.Compose([
    transforms.ToTensor(),
    transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225))
])

# Fix: use single-channel normalization
transform = transforms.Compose([
    transforms.ToTensor(),
    transforms.Normalize((0.5,), (0.5,))
])
# Or convert to RGB first:
transforms.Grayscale(num_output_channels=3)

# Bug: dimension mismatch between Linear layers
self.fc1 = nn.Linear(784, 512)
self.fc2 = nn.Linear(256, 10)  # expects 256 but fc1 outputs 512

# Fix: match in_features to previous out_features
self.fc1 = nn.Linear(784, 512)
self.fc2 = nn.Linear(512, 10)  # 512 matches fc1 output

# Bug: MSELoss with class indices instead of one-hot
output = model(x)  # shape: [32, 10]
loss = F.mse_loss(output, targets)  # targets shape: [32]

# Fix option 1: use CrossEntropyLoss (accepts class indices)
loss = F.cross_entropy(output, targets)

# Fix option 2: one-hot encode targets
targets_onehot = F.one_hot(targets, num_classes=10).float()
loss = F.mse_loss(output, targets_onehot)

# Bug: feeding flattened data to Conv2d
x = x.view(x.size(0), -1)  # [32, 784] -- flattened too early
x = self.conv(x)  # expects [32, C, H, W]

# Fix: reshape back to image format
x = x.view(x.size(0), 1, 28, 28)  # [32, 1, 28, 28]
x = self.conv(x)

# Or: don't flatten before conv layers

# Bug: arguments swapped
loss = F.cross_entropy(targets, output)  # wrong order!

# Fix: correct argument order (predictions first, targets second)
loss = F.cross_entropy(output, targets)

# Bug: LSTM expects input_size=128 but got features=64
self.lstm = nn.LSTM(input_size=128, hidden_size=256)
x = torch.randn(10, 32, 64)  # seq=10, batch=32, features=64

# Fix: match input_size to actual feature dimension
self.lstm = nn.LSTM(input_size=64, hidden_size=256)

# Bug: 512 / 6 = 85.33 (not integer)
attn = nn.MultiheadAttention(embed_dim=512, num_heads=6)

# Fix: choose num_heads that divides embed_dim evenly
attn = nn.MultiheadAttention(embed_dim=512, num_heads=8)  # 512/8 = 64

# Bug: forgot to flatten CNN output before Linear
x = self.conv_layers(x)  # shape: [32, 64, 7, 7]
x = self.fc(x)  # Linear expects 2D!

# Fix: add flatten between conv and linear
x = self.conv_layers(x)  # [32, 64, 7, 7]
x = x.view(x.size(0), -1)  # [32, 3136]
x = self.fc(x)  # Linear(3136, 10)

# Bug: BatchNorm features don't match conv output channels
self.conv = nn.Conv2d(3, 64, kernel_size=3)
self.bn = nn.BatchNorm2d(128)  # wrong: 128 != 64

# Fix: match num_features to conv out_channels
self.conv = nn.Conv2d(3, 64, kernel_size=3)
self.bn = nn.BatchNorm2d(64)  # correct

# Bug: view after transpose on non-contiguous tensor
x = torch.randn(32, 10, 64)
x = x.transpose(1, 2)  # now non-contiguous
x = x.view(32, -1)  # RuntimeError!

# Fix option 1: make contiguous first
x = x.transpose(1, 2).contiguous().view(32, -1)

# Fix option 2: use reshape (handles non-contiguous)
x = x.transpose(1, 2).reshape(32, -1)

# Fix 1: Reduce batch size
train_loader = DataLoader(dataset, batch_size=16)  # was 64

# Fix 2: Enable gradient checkpointing
model.gradient_checkpointing_enable()

# Fix 3: Use mixed precision (halves activation memory)
scaler = torch.cuda.amp.GradScaler()
with torch.cuda.amp.autocast():
    output = model(input)
    loss = criterion(output, target)

# Fix 4: Clear cache between operations
torch.cuda.empty_cache()

# Fix 5: Accumulate gradients over smaller batches
for i, (x, y) in enumerate(loader):
    loss = model(x, y) / accumulation_steps
    loss.backward()
    if (i + 1) % accumulation_steps == 0:
        optimizer.step()
        optimizer.zero_grad()

# Fix: wrap inference in no_grad to prevent gradient storage
with torch.no_grad():
    output = model(input)

# Also useful: delete intermediate tensors
del intermediate_tensor
torch.cuda.empty_cache()

# For inference, use torch.inference_mode() (faster than no_grad)
with torch.inference_mode():
    output = model(input)

# Common cause: embedding index exceeds num_embeddings
embed = nn.Embedding(1000, 128)  # indices 0-999 valid
x = torch.tensor([1500])  # out of range!

# Fix: clamp indices to valid range
x = x.clamp(0, embed.num_embeddings - 1)

# Debug: set CUDA_LAUNCH_BLOCKING=1 for synchronous errors
import os
os.environ['CUDA_LAUNCH_BLOCKING'] = '1'
# Now errors will point to the exact line

# Bug: accidentally huge Linear layer
self.fc = nn.Linear(50000, 50000)  # 50000*50000*4 bytes = 9.3 GB!

# Fix: review your architecture dimensions
# If this is an output projection, you likely meant:
self.fc = nn.Linear(512, 50000)

# For large language models: use quantization
from bitsandbytes import nn as bnb
self.fc = bnb.Linear8bitLt(in_features, out_features)

# Or use device_map="auto" for model parallelism
model = AutoModel.from_pretrained("large-model", device_map="auto")

# Bug: no torch.no_grad() during validation
model.eval()
for x, y in val_loader:
    output = model(x)  # still tracking gradients!
    val_loss += criterion(output, y)

# Fix: disable gradient tracking
model.eval()
with torch.no_grad():
    for x, y in val_loader:
        output = model(x)
        val_loss += criterion(output, y).item()  # .item() returns Python scalar

# Bug: storing tensor losses in a list (retains computation graph)
losses = []
for x, y in train_loader:
    loss = criterion(model(x), y)
    losses.append(loss)  # keeps graph alive!

# Fix: store scalar values with .item()
losses = []
for x, y in train_loader:
    loss = criterion(model(x), y)
    loss.backward()
    losses.append(loss.item())  # Python float, no graph reference
    optimizer.step()
    optimizer.zero_grad()

# Fix 1: Set max_split_size_mb to reduce fragmentation
import os
os.environ['PYTORCH_CUDA_ALLOC_CONF'] = 'max_split_size_mb:128'

# Fix 2: Periodically clear cache
torch.cuda.empty_cache()

# Fix 3: Use expandable_segments (PyTorch 2.0+)
os.environ['PYTORCH_CUDA_ALLOC_CONF'] = 'expandable_segments:True'

# Fix: call flatten_parameters() after loading or in forward()
class MyModel(nn.Module):
    def __init__(self):
        super().__init__()
        self.lstm = nn.LSTM(128, 256, batch_first=True)

    def forward(self, x):
        self.lstm.flatten_parameters()
        output, (hn, cn) = self.lstm(x)
        return output

# Fix 1: Reduce number of workers
loader = DataLoader(dataset, num_workers=2)  # was 8

# Fix 2: Reduce prefetch factor
loader = DataLoader(dataset, num_workers=4, prefetch_factor=1)

# Fix 3: Use pin_memory=False if RAM is tight
loader = DataLoader(dataset, pin_memory=False)

# Fix 4: Use smaller images or reduce data augmentation memory usage

# Fix 1: Load model with memory mapping
model = torch.load('model.pt', map_location='cpu', mmap=True)

# Fix 2: Process data in chunks
for chunk in torch.split(large_tensor, 1000):
    process(chunk)

# Fix 3: Use float16 to halve memory
model = model.half()

# Fix 4: Use memory-mapped datasets
from torch.utils.data import Dataset
class MMapDataset(Dataset):
    def __init__(self, path):
        self.data = np.memmap(path, dtype='float32', mode='r')

# Bug: model on GPU, input on CPU
model = model.cuda()
output = model(input)  # input is on CPU!

# Fix: move all inputs to the same device as the model
device = next(model.parameters()).device
input = input.to(device)
output = model(input)

# Or explicitly:
input = input.cuda()
target = target.cuda()

# Bug: loading GPU model on CPU-only machine
model = torch.load('model_gpu.pt')  # fails without CUDA

# Fix: map to CPU when loading
model = torch.load('model_gpu.pt', map_location='cpu')

# Or map to a specific device
model = torch.load('model_gpu.pt', map_location=torch.device('cpu'))

# Best practice when saving: save state_dict (device-agnostic)
torch.save(model.state_dict(), 'model.pt')
# Then load:
model.load_state_dict(torch.load('model.pt', map_location='cpu'))

# Diagnostic steps:
import torch
print(torch.version.cuda)       # CUDA toolkit version
print(torch.cuda.is_available()) # should be True
print(torch.backends.cudnn.version())  # cuDNN version

# Fix 1: Check driver compatibility
# nvidia-smi shows driver CUDA version; it must be >= PyTorch CUDA version

# Fix 2: Reinstall PyTorch with matching CUDA version
# pip install torch --index-url https://download.pytorch.org/whl/cu121

# Fix 3: Restart Python/notebook (CUDA state is corrupted)
# A previous CUDA error may have left the GPU in a bad state

# Common cause: labels out of range for CrossEntropyLoss
# CrossEntropyLoss expects labels in [0, num_classes-1]
output = model(x)  # shape [32, 10] (10 classes)
target = torch.tensor([10])  # label 10 is out of range (max is 9)!

# Fix: validate labels
assert target.max() < num_classes
assert target.min() >= 0

# Debug: run on CPU first to get a clear error message
model = model.cpu()
output = model(x.cpu())
loss = criterion(output, target.cpu())  # will show clear IndexError

# Bug: input on GPU, model on CPU
model = MyModel()  # CPU by default
x = x.cuda()
output = model(x)  # RuntimeError!

# Fix: move model to same device as input
model = model.cuda()
output = model(x)

# Best practice: use a device variable
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
model = model.to(device)
x = x.to(device)

# Bug: creating tensors in loss function without specifying device
def custom_loss(pred, target):
    weights = torch.tensor([1.0, 2.0, 3.0])  # CPU!
    return (weights * (pred - target) ** 2).mean()

# Fix: create tensors on the correct device
def custom_loss(pred, target):
    weights = torch.tensor([1.0, 2.0, 3.0], device=pred.device)
    return (weights * (pred - target) ** 2).mean()

# Fix 1: Disable flash attention
# For Hugging Face Transformers:
model = AutoModel.from_pretrained("model", attn_implementation="eager")

# Fix 2: Use a compatible attention backend
import torch.backends.cuda
torch.backends.cuda.enable_flash_sdp(False)

# Fix 3: Use math attention fallback
with torch.backends.cuda.sdp_kernel(
    enable_flash=False, enable_math=True, enable_mem_efficient=True
):
    output = model(input)

# Debug: temporarily remove DataParallel to see the real error
# model = nn.DataParallel(model)  # comment out
model = model.cuda()  # single GPU
output = model(input)  # now error message is clear

# Better alternative: use DistributedDataParallel
model = torch.nn.parallel.DistributedDataParallel(
    model, device_ids=[local_rank]
)

# Bug: in-place operations break autograd
x = self.linear(input)
x += self.bias  # in-place add!
x = x.relu_()   # in-place relu!

# Fix: use out-of-place operations
x = self.linear(input)
x = x + self.bias    # creates new tensor
x = x.relu()         # creates new tensor (no underscore)
# Or:
x = torch.relu(x)    # functional form, always out-of-place

# Bug: two backward calls on same graph
loss1 = criterion(model(x), y)
loss1.backward()
loss2 = some_regularization(model)
loss2.backward()  # graph from loss1 already freed!

# Fix option 1: retain graph for first backward
loss1.backward(retain_graph=True)
loss2.backward()

# Fix option 2 (better): combine losses before backward
loss = criterion(model(x), y) + some_regularization(model)
loss.backward()  # single backward pass

# Fix option 3: recompute forward pass
loss1 = criterion(model(x), y)
loss1.backward()
optimizer.step()
optimizer.zero_grad()
loss2 = criterion(model(x), y)  # fresh forward pass
loss2.backward()

# Bug: calling backward on non-differentiable tensor
x = torch.tensor([1.0, 2.0, 3.0])  # requires_grad=False by default
loss = x.sum()
loss.backward()  # RuntimeError!

# Fix: enable gradient tracking
x = torch.tensor([1.0, 2.0, 3.0], requires_grad=True)
loss = x.sum()
loss.backward()

# Common mistake: using .data or .detach() too early
pred = model(x).detach()  # breaks gradient chain!
loss = criterion(pred, target)
loss.backward()  # error: no grad_fn

# Bug: custom backward returning wrong shape
class MyFunc(torch.autograd.Function):
    @staticmethod
    def forward(ctx, input, weight):
        ctx.save_for_backward(input, weight)
        return input @ weight.T

    @staticmethod
    def backward(ctx, grad_output):
        input, weight = ctx.saved_tensors
        grad_input = grad_output @ weight  # shape must match input
        grad_weight = input.T @ grad_output  # shape must match weight
        return grad_input, grad_weight  # wrong if shapes don't match

# Fix: verify gradient shapes match input shapes
# grad_input.shape == input.shape
# grad_weight.shape == weight.shape

# Bug: in-place modification of model parameters
model.weight.data.fill_(1.0)  # .data bypasses autograd (OK but fragile)
model.weight.fill_(1.0)  # in-place on leaf variable, RuntimeError!

# Fix: use .data or torch.no_grad() for parameter modification
with torch.no_grad():
    model.weight.fill_(1.0)

# Or use .data (less safe but works)
model.weight.data.fill_(1.0)

# For custom initialization:
def init_weights(m):
    if isinstance(m, nn.Linear):
        torch.nn.init.xavier_uniform_(m.weight)  # uses .data internally
model.apply(init_weights)

# Fix 1: set find_unused_parameters=True
model = torch.nn.parallel.DistributedDataParallel(
    model, device_ids=[local_rank],
    find_unused_parameters=True
)

# Fix 2: if parameters are truly unused, remove them
# Audit your forward() to ensure all parameters are used

# Fix 3: for shared parameters, use static_graph
model = torch.nn.parallel.DistributedDataParallel(
    model, device_ids=[local_rank],
    static_graph=True
)

# Bug: inplace ReLU in residual connection
class ResBlock(nn.Module):
    def forward(self, x):
        identity = x
        out = self.conv1(x)
        out = self.bn1(out)
        out = F.relu(out, inplace=True)  # overwrites out
        out = self.conv2(out)
        out += identity  # in-place add on top of inplace relu
        return out

# Fix: use inplace=False
out = F.relu(out, inplace=False)
# And use out-of-place addition:
out = out + identity  # creates new tensor

# Bug: backward on non-scalar
loss = model(x) - y  # shape: [32, 10], not scalar!
loss.backward()  # RuntimeError!

# Fix: reduce to scalar
loss = (model(x) - y).pow(2).mean()  # scalar
loss.backward()

# Or pass gradient argument for non-scalar backward:
output = model(x)  # [32, 10]
output.backward(torch.ones_like(output))  # provides gradient shape

# Bug: in-place operation on parameter slice
model.weight[:, 0] = 0  # in-place modification via view!

# Fix: use torch.no_grad() context
with torch.no_grad():
    model.weight[:, 0] = 0

# Or create a new tensor with the modification
mask = torch.ones_like(model.weight)
mask[:, 0] = 0
# Use mask in forward pass instead of modifying weight

# Bug: shape mismatch causes silent broadcasting
pred = model(x)  # shape: [32, 1]
target = labels   # shape: [32]
loss = F.binary_cross_entropy(pred, target)  # broadcasts incorrectly

# Fix: match shapes explicitly
target = labels.unsqueeze(1)  # [32] -> [32, 1]
# Or squeeze prediction:
pred = model(x).squeeze(1)  # [32, 1] -> [32]

# Bug: manually casting to half without autocast
model = model.half()
x = x.float()  # float32
output = model(x)  # half model, float input -> error

# Fix option 1: use autocast (recommended)
with torch.cuda.amp.autocast():
    output = model(x)  # automatic dtype management

# Fix option 2: match dtypes manually
x = x.half()  # or model = model.float()

# Fix option 3: cast specific layers
model.layer_norm = model.layer_norm.float()  # keep in float32

# Bug: float targets for CrossEntropyLoss
targets = torch.tensor([0.0, 1.0, 2.0])  # float!
loss = F.cross_entropy(output, targets)  # expects Long

# Fix: cast targets to long
targets = targets.long()
# Or create with correct dtype:
targets = torch.tensor([0, 1, 2], dtype=torch.long)

# For Embedding:
indices = torch.tensor([1, 5, 3], dtype=torch.long)
output = embedding(indices)

# Bug: numpy default is float64
import numpy as np
data = np.array([1.0, 2.0, 3.0])  # float64
x = torch.from_numpy(data)  # torch.float64 (Double)
output = model(x)  # model expects float32!

# Fix: explicitly convert to float32
x = torch.from_numpy(data).float()  # cast to float32
# Or:
x = torch.tensor(data, dtype=torch.float32)

# Bug: int32 targets instead of int64
targets = torch.tensor([0, 1, 2], dtype=torch.int32)
loss = F.cross_entropy(output, targets)  # needs Long!

# Fix: cast to long
targets = targets.long()

# In DataLoader: ensure labels are long
class MyDataset(Dataset):
    def __getitem__(self, idx):
        return self.data[idx], torch.tensor(self.labels[idx], dtype=torch.long)

# Bug: float32 CPU input, half GPU model
model = model.half().cuda()
x = torch.randn(1, 3, 224, 224)  # float32, CPU

# Fix: match both device and dtype
x = x.half().cuda()
output = model(x)

# Better: use autocast for automatic dtype handling
model = model.cuda()  # keep float32
with torch.cuda.amp.autocast():
    output = model(x.cuda())  # autocast handles half precision

# Bug: torch.where with mixed types
mask = torch.tensor([True, False, True])
a = torch.tensor([1.5, 2.5, 3.5])  # float
b = torch.tensor([0, 0, 0])        # int/long
result = torch.where(mask, a, b)    # can't cast float to long

# Fix: ensure both tensors have the same dtype
b = torch.tensor([0.0, 0.0, 0.0])  # float
result = torch.where(mask, a, b)

# Or cast explicitly:
result = torch.where(mask, a, b.float())