Network Architecture: Depth vs Width

import torch
import torch.nn as nn
import matplotlib.pyplot as plt
import numpy as np
from tqdm import tqdm

from torchvision import datasets
from torchvision import transforms

from pathlib import Path

from torch.utils.data import DataLoader, TensorDataset, random_split
import pandas as pd
import owncloud

def train(model: nn.Module, features_train: torch.Tensor, labels_train: torch.Tensor, learning_rate = 0.01, nepochs = 50) -> None:
    """Train a neural network model

    Args:
        model: Model to train.
        features_train: Input features tensor of shape (n_samples, n_features).
        labels_train: Target labels tensor of shape (n_samples,).
        learning_rate: Learning rate for the optimizer
        nepochs: Number of training epochs

    Returns:
        A tuple of (losses, accuracies). Each is a list of values recorded
        at each epoch during training.
    """

    # create dataloader for batching if needed
    dataset_train = TensorDataset(features_train, labels_train)
    dataloader_train = DataLoader(dataset_train, batch_size=len(dataset_train))

    loss_function = nn.CrossEntropyLoss()

    optimizer = torch.optim.Adam(model.parameters(), lr = learning_rate)

    losses = []
    accuracies = []
    for epoch in range(nepochs):
        for features_train, labels_train in dataloader_train:
            optimizer.zero_grad()

            output = model.forward(features_train)

            loss = loss_function(output, labels_train)

            loss.backward()

            optimizer.step()

            losses.append(loss.item())
            accuracy = utils.calculate_accuracy(model, features_train, labels_train, verbose = False)
            accuracies.append(accuracy)
            if epoch % 10 == 0:
                print(f"Epoch: {epoch}, Loss: {loss:.2f}, Train accuracy: {accuracy:.2f}")


    return losses, accuracies

def calculate_accuracy(model: nn.Module, features: torch.Tensor, labels: torch.Tensor, verbose = True) -> torch.Tensor:
    """Calculate classification accuracy of a model

    Args:
        model: Model to evaluate.
        features: Input features tensor of shape (n_samples, n_features).
        labels: Ground truth labels tensor of shape (n_samples,).
        verbose: If True, prints the accuracy. Defaults to True.

    Returns:
        The accuracy as a percentage (0-100).
    """
    output = model(features)
    accuracy = (labels == torch.argmax(output, dim=1)).float().mean()
    if verbose:
        print(f'Accuracy: {(accuracy*100).item()}')
    return (accuracy*100).item()

def plot_performance_training(losses, accuracies) -> None:
    """Plot training loss and accuracy curves side by side.

    Args:
        losses: List or array of loss values recorded during training.
        accuracies: List or array of accuracy values recorded during training.

    Returns:
        None. Displays a matplotlib figure with two subplots.
    """
    fig, axes = plt.subplots(ncols = 2, figsize = (6, 3))

    axes[0].plot(losses)
    axes[0].set_title('Loss throughout training')
    axes[0].set_xlabel('Epoch')
    axes[0].set_ylabel('Cross-entropy loss')

    axes[1].plot(accuracies)
    axes[1].set_title('Accuracy on training data')
    axes[1].set_xlabel('Epoch')
    axes[1].set_ylabel('Accuracy (%)')

    fig.tight_layout()

class utils:
    train = train
    calculate_accuracy = calculate_accuracy
    plot_performance_training = plot_performance_training

mean_cifar10 = (0.4914, 0.4822, 0.4465)
std_cifar10 = (0.2023, 0.1994, 0.2010)

download = True
if Path("data/cifar-10-batches-py").exists():
    download = False

train_data = datasets.CIFAR10(
    download=download,
    root="data",
    train=True,
    transform = transforms.Compose([
        transforms.ToTensor(),
        transforms.Normalize(mean_cifar10, std_cifar10)
    ])
)

test_data = datasets.CIFAR10(
    download=download,
    root="data",
    train=False,
    transform = transforms.Compose([
        transforms.ToTensor(),
        transforms.Normalize(mean_cifar10, std_cifar10)
    ])
)

# extract train data
features_train = torch.tensor(train_data.data) # get images used for training and put them in tensor
features_train = features_train.flatten(start_dim=1) # flatten images (n_images, 32, 32, 3) -> (n_images, 3072)
features_train = features_train / 255.0 # normalize pixel values from [0, 255] to [0, 1]

fraction_to_use = 0.2
n_to_use = int(len(features_train)*fraction_to_use)

features_train = features_train[:n_to_use]
labels_train = torch.tensor(train_data.targets)[:n_to_use]

# extract test data
features_test = torch.tensor(test_data.data) # get images used for training and put them in tensor
features_test = features_test.flatten(start_dim=1) # flatten images (n_images, 32, 32, 3) -> (n_images, 3072)
features_test = features_test / 255.0 # normalize pixel values from [0, 255] to [0, 1]

n_to_use = int(len(features_test)*fraction_to_use)

features_test = features_test[:n_to_use]
labels_test = torch.tensor(test_data.targets)[:n_to_use]


print(f"features_train shape: {features_train.shape}")
print(f"features_test shape: {features_test.shape}")
print(f"features_train min/max: {features_train.min():.2f} / {features_train.max():.2f}")

# CIFAR-10 class names
class_names = ['airplane', 'automobile', 'bird', 'cat', 'deer', 
               'dog', 'frog', 'horse', 'ship', 'truck']

# Plot two sample images
fig, axes = plt.subplots(1, 2, figsize=(4, 2))

for i, ax in enumerate(axes):
    ax.imshow(train_data.data[i])
    ax.set_title(f"Label: {class_names[train_data.targets[i]]}")
    ax.axis('off')

fig.tight_layout()

Path('data').mkdir(exist_ok=True, parents=True)
Path('models').mkdir(exist_ok=True, parents=True)

owncloud.Client.from_public_link('https://uni-bonn.sciebo.de/s/ijRPs9KpSNetqBL', folder_password="ibots").get_file('/', f'data/train_perfomance.csv')

owncloud.Client.from_public_link('https://uni-bonn.sciebo.de/s/yFGi2YMk9jTkEC3', folder_password="ibots").get_file('/', f'models/model_4layers_80hidden_neurons_1000epochs.pth')
owncloud.Client.from_public_link('https://uni-bonn.sciebo.de/s/w4MEp7jFFZeDkRG', folder_password="ibots").get_file('/', f'models/model_with_batchnorm_4layers_100hidden_neurons_81epochs.pth')

df = pd.read_csv('data/train_perfomance.csv')
df

n_in_features = 32 * 32 * 3  # CIFAR-10 images flattened
n_classes = 10  # CIFAR-10 has 10 classes
SEED = 2026

hidden_layer_size = 1

torch.manual_seed(SEED)
# create model
model = nn.Sequential(
    nn.Linear(n_in_features, hidden_layer_size),
    nn.ReLU(),
    nn.Linear(hidden_layer_size, n_classes)
)
model

losses, accuracies = utils.train(model, features_train, labels_train, nepochs=50)

utils.plot_performance_training(losses, accuracies)

utils.plot_performance_training(losses, accuracies)

utils.plot_performance_training(losses, accuracies)

losses = df[df.source == '100hidden_1000epochs']['losses']
accuracies = df[df.source == '100hidden_1000epochs']['accuracies']

n_in_features = 32 * 32 * 3  # CIFAR-10 images flattened
n_classes = 10  # CIFAR-10 has 10 classes
SEED = 2026

hidden_layer_size = 10

# create model
torch.manual_seed(SEED)

model = nn.Sequential(
    nn.Linear(n_in_features, hidden_layer_size),
    nn.ReLU(),
    nn.Linear(hidden_layer_size, hidden_layer_size),
    nn.ReLU(),
    nn.Linear(hidden_layer_size, n_classes)
)
model

losses, accuracies = utils.train(model, features_train, labels_train, nepochs=50)

utils.plot_performance_training(losses, accuracies)

utils.plot_performance_training(losses, accuracies)

losses = df[df.source == '4layers_10hidden_1000epochs']['losses']
accuracies = df[df.source == '4layers_10hidden_1000epochs']['accuracies']

losses = df[df.source == '8layers_10hidden_1000epochs']['losses']
accuracies = df[df.source == '8layers_10hidden_1000epochs']['accuracies']

# Load model
model = torch.load('models/model_4layers_80hidden_neurons_1000epochs.pth', weights_only=False)
print("Model loaded successfully")

losses = df[df.source == '4layers_80hidden_1000epochs']['losses']
accuracies = df[df.source == '4layers_80hidden_1000epochs']['accuracies']

n_in_features = 32 * 32 * 3  # CIFAR-10 images flattened
n_classes = 10  # CIFAR-10 has 10 classes
SEED = 2026

hidden_layer_size = 100

# create model with batch normalization
torch.manual_seed(SEED)

model_bn = nn.Sequential(
    nn.Linear(n_in_features, hidden_layer_size),
    nn.BatchNorm1d(hidden_layer_size),
    nn.ReLU(),
    nn.Linear(hidden_layer_size, n_classes)
)
model_bn

# load model
model_bn = torch.load('models/model_with_batchnorm_4layers_100hidden_neurons_81epochs.pth', weights_only=False)
print("Model loaded successfully")

# load results
losses_bn = df[df.source == 'with_batchnorm_4layers_100hidden_81epochs']['losses'].values
accuracies_bn = df[df.source == 'with_batchnorm_4layers_100hidden_81epochs']['accuracies'].values
model_bn

losses_no_bn = df[df.source == '4layers_100hidden_81epochs']['losses'].values
accuracies_no_bn = df[df.source == '4layers_100hidden_81epochs']['accuracies'].values

fig, axes = plt.subplots(ncols=2, figsize=(10, 4))

axes[0].plot(losses_no_bn, label='Without BatchNorm')
axes[0].plot(losses_bn, label='With BatchNorm')
axes[0].set_title('Loss throughout training')
axes[0].set_xlabel('Epoch')
axes[0].set_ylabel('Cross-entropy loss')

axes[1].plot(accuracies_no_bn, label='Without BatchNorm')
axes[1].plot(accuracies_bn, label='With BatchNorm')
axes[1].set_title('Accuracy on training data')
axes[1].set_xlabel('Epoch')
axes[1].set_ylabel('Accuracy (%)')
axes[1].legend()

fig.tight_layout()