21_Gradient Norm Clipping

Implement gradient norm clipping — a training stability technique.

Signature

def clip_grad_norm(parameters, max_norm: float) -> float:
    # Clip gradients in-place so total norm <= max_norm
    # Returns the original (unclipped) total norm

Algorithm

  1. Compute total norm: sqrt(sum(p.grad.norm()^2 for p in parameters))
  2. If total > max_norm: scale all grads by max_norm / total
  3. Return original total norm
import torch

# ✏️ YOUR IMPLEMENTATION HERE

def clip_grad_norm(parameters, max_norm):
    # compute total norm, clip if needed, return original norm
    grads = [p.grad for p in parameters if p.grad is not None]
    total = torch.sqrt(sum(g.norm()**2 for g in grads))
    if total > max_norm:
        for g in grads:
            g.mul_(max_norm / total)
    return total.item()

梯度裁剪将梯度的第二范数上线卡到max_norm

返回原来的梯度范数作为监控信息提示供训练者判断情况

这里注意一定要使用原地改的写法

22_2D Convolution

Implement 2D convolution from scratch.

Signature

def my_conv2d(x, weight, bias=None, stride=1, padding=0):
    # x: (B, C_in, H, W), weight: (C_out, C_in, kH, kW)
    # Returns: (B, C_out, H_out, W_out)

Rules

  • Do NOT use F.conv2d or nn.Conv2d
  • Support stride and padding parameters
  • F.pad for zero-padding is allowed
import torch
import torch.nn.functional as F

# ✏️ YOUR IMPLEMENTATION HERE

def my_conv2d(x, weight, bias=None, stride=1, padding=0):
    # extract patches, apply kernel, handle stride/padding
    B, C_in, H, W = x.shape
    C_out, _, kH, kW = weight.shape
    H_out = (H + 2 * padding - kH) // stride + 1
    W_out = (W + 2 * padding - kW) // stride + 1
    out = torch.zeros(B, C_out, H_out, W_out)
    x = F.pad(x, (padding,padding,padding,padding) )
    for b in range(B):
        for co in range(C_out):
            for i in range(H_out):
                for j in range(W_out):
                    patch = x[b, :, i*stride : i*stride + kH, j*stride : j*stride +kW]
                    out[b, co, i, j] = (patch * weight[co]).sum() + (bias[co] if bias is not None else 0)
    return out

Cin是输入通道数,比如单通道灰度和三通道RGB。

Cout是输出特征数量,可以自定义。

23_Multi-Head Cross-Attention

Implement multi-head cross-attention (encoder-decoder attention).

Signature

class MultiHeadCrossAttention(nn.Module):
    def __init__(self, d_model: int, num_heads: int): ...
    def forward(self, x_q: Tensor, x_kv: Tensor) -> Tensor:
        # x_q: (B, S_q, D) — decoder queries
        # x_kv: (B, S_kv, D) — encoder keys/values

Key Differences from Self-Attention

  • Q comes from the decoder, K and V come from the encoder
  • No causal mask (all encoder positions visible)
import torch
import torch.nn as nn
import math

# ✏️ YOUR IMPLEMENTATION HERE

class MultiHeadCrossAttention(nn.Module):
    def __init__(self, d_model, num_heads):
        super().__init__()
        # W_q, W_k, W_v, W_o
        self.W_q = nn.Linear(d_model, d_model)
        self.W_k = nn.Linear(d_model, d_model)
        self.W_v = nn.Linear(d_model, d_model)
        self.W_o = nn.Linear(d_model, d_model)
        self.d_k = d_model // num_heads
        self.num_heads = num_heads
        
    def forward(self, x_q, x_kv):
        # Q from x_q, K/V from x_kv, no causal mask
        B, sq, D = x_q.shape
        _, skv,D = x_kv.shape 
        q = self.W_q(x_q).view(B, sq, self.num_heads, self.d_k).transpose(1,2)
        k = self.W_k(x_kv).view(B, skv, self.num_heads, self.d_k).transpose(1,2)
        v = self.W_v(x_kv).view(B, skv, self.num_heads, self.d_k).transpose(1,2)
        score = q @ k.transpose(-1,-2) / math.sqrt(self.d_k)
        atten = torch.softmax(score, dim = -1) @ v
        atten = atten.transpose(1,2).contiguous().view(B,sq,D)
        return atten

24_Rotary Position Embedding (RoPE)

mplement RoPE — the position encoding used in LLaMA, GPT-NeoX, and most modern LLMs.

Signature

def apply_rope(q: Tensor, k: Tensor) -> tuple[Tensor, Tensor]:
    # q, k: (B, S, D) where D is even
    # Returns rotated (q, k) with same shape

Key Idea

Split each vector into consecutive pairs. Rotate each pair by θ = pos / 10000^(2i/D):

[x_0, x_1] → [x_0*cosθ - x_1*sinθ, x_0*sinθ + x_1*cosθ]

This makes dot(q_rot[i], k_rot[j]) depend only on i - j (relative position).

import torch
import math

# ✏️ YOUR IMPLEMENTATION HERE

def apply_rope(q, k):
    B, S, D = q.shape

    # ① 角度表 (S, D//2)
    pos = torch.arange(S)
    freq = 10000 ** (-2 * torch.arange(D // 2) / D)
    angles = pos[:, None] * freq[None, :]     # (S, D//2)
    cos = torch.cos(angles)
    sin = torch.sin(angles)

    def rotate(x):
        # ② 拆对
        x = x.reshape(B, S, D // 2, 2)        # (B, S, D//2, 2)
        x_even, x_odd = x[..., 0], x[..., 1]  # 各 (B, S, D//2)
        # ③ 旋转(复数乘法展开)
        rot_even = x_even * cos - x_odd * sin
        rot_odd  = x_even * sin + x_odd * cos
        # ④ 合并
        return torch.stack([rot_even, rot_odd], dim=-1).reshape(B, S, D)

    return rotate(q), rotate(k)

将向量维度两两配对分组,并分别使用不用的周期进行旋转操作,实现快慢指针高精度表征位置关系。而且可以实现相对位置关系的计算。

25_Flash Attention (Tiled)

Implement tiled attention with online softmax — the core idea behind Flash Attention.

Signature

def flash_attention(Q, K, V, block_size=32) -> Tensor:
    # Q, K, V: (B, S, D)
    # Returns: (B, S, D) — same as standard attention

Key Insight

Instead of materializing the full S×S attention matrix, process in blocks:

  1. For each Q-block, iterate over K/V blocks
  2. Use online softmax: track running max and sum
  3. Rescale accumulator when max changes: acc *= exp(old_max - new_max)
  4. Final: output = acc / row_sum

Must give identical results to standard softmax attention.

import torch
import math

def flash_attention(Q, K, V, block_size=32):
    B, S, D = Q.shape
    scale = 1.0 / math.sqrt(D)
    m = torch.full((B, S, 1), float('-inf'))   # running max,初始 -inf
    l = torch.zeros(B, S, 1)                   # running sum
    O = torch.zeros(B, S, D)                   # running 输出
    for i in range(0, S, block_size):
        K_b = K[:, i:i+block_size]             # (B, bs, D) 切片,天然处理余数
        V_b = V[:, i:i+block_size]
        score = Q @ K_b.transpose(-1, -2) * scale     # (B, S, bs) ④ 补上缩放
        m2 = torch.amax(score, dim=-1, keepdim=True)  # (B, S, 1) ②③ 块内 max
        m_new = torch.maximum(m, m2)                  # 新 running max
        P = torch.exp(score - m_new)                  # (B, S, bs) 未归一化权重
        alpha = torch.exp(m - m_new)                  # (B, S, 1) 旧块衰减因子
        l = alpha * l + P.sum(dim=-1, keepdim=True)   # ③ keepdim
        O = alpha * O + P @ V_b                       # ⑥ P 直接乘 V
        m = m_new
    return O / l                                       # 最后整体归一化

Online softmax还是有点绕的

26_LoRA (Low-Rank Adaptation)

Implement LoRA — parameter-efficient fine-tuning for large models.

h = W_0 x + \frac{\alpha}{r} B A x

Signature

class LoRALinear(nn.Module):
    def __init__(self, in_features, out_features, rank, alpha=1.0): ...
    def forward(self, x: Tensor) -> Tensor: ...

Requirements

  • self.linear: frozen nn.Linear (weight & bias requires_grad=False)
  • self.lora_A: nn.Parameter(rank, in_features) — random init
  • self.lora_B: nn.Parameter(out_features, rank) — zero init
  • Scaling: alpha / rank
import torch
import torch.nn as nn

# ✏️ YOUR IMPLEMENTATION HERE

class LoRALinear(nn.Module):
    def __init__(self, in_features, out_features, rank, alpha=1.0):
        super().__init__()
        self.alpha = alpha
        self.rank = rank
        self.linear = nn.Linear(in_features, out_features)
        self.linear.weight.requires_grad_(False)
        self.linear.bias.requires_grad_(False)
        self.lora_A = nn.Parameter(torch.randn(rank, in_features))
        self.lora_B = nn.Parameter(torch.zeros(out_features, rank))
        # frozen linear + lora_A + lora_B

    def forward(self, x):
        return self.linear(x) + (self.alpha / self.rank) * (x @ self.lora_A.T @ self.lora_B.T)
        # base + lora

LoRA是参数高效微调(PEFT)中最主流的一种,其关键是并不改变原有参数,而是添加低秩参数矩阵作为修正。

其中A矩阵为随机初始化,B矩阵为零初始化。

如果AB都为随机初始化,则训练刚开始就被从预训练收敛点加上偏移。

如果AB都为零初始化,则AB矩阵的梯度将永远为零,旁路死亡。

27_ViT Patch Embedding

Implement the patch embedding layer from Vision Transformer (ViT).

Signature

class PatchEmbedding(nn.Module):
    def __init__(self, img_size, patch_size, in_channels, embed_dim): ...
    def forward(self, x: Tensor) -> Tensor:
        # x: (B, C, H, W)
        # Returns: (B, num_patches, embed_dim)

Algorithm

  1. Reshape image into non-overlapping patches: (B, C, H, W) → (B, N, C*P*P)
  2. Project each patch: nn.Linear(C*P*P, embed_dim)
  3. num_patches = (img_size // patch_size) ** 2
import torch
import torch.nn as nn

# ✏️ YOUR IMPLEMENTATION HERE

class PatchEmbedding(nn.Module):
    def __init__(self, img_size, patch_size, in_channels, embed_dim):
        super().__init__()
        # self.num_patches, self.proj
        self.patch_size = patch_size
        self.num_patches = (img_size // patch_size) ** 2
        self.proj = nn.Linear(in_channels*patch_size*patch_size, embed_dim)
    def forward(self, x):
        # reshape to patches, project
        B, C, H, W = x.shape
        x = x.view(B, C, H//self.patch_size, self.patch_size, W//self.patch_size,self.patch_size)
        x = x.permute(0,2,4,1,3,5)
        x = x.contiguous().view(B, self.num_patches, self.patch_size * self.patch_size * C)
        return self.proj(x)

将图像进行分割再embedding向量化,作为输入transformer的token。

一个常见误区是直接进行形状变换,那样的结果是行采样而不是window采样

28_Mixture of Experts (MoE)

Implement a Mixture of Experts layer (Mixtral / Switch Transformer style).

Signature

class MixtureOfExperts(nn.Module):
    def __init__(self, d_model, d_ff, num_experts, top_k=2): ...
    def forward(self, x: Tensor) -> Tensor:
        # x: (B, S, D) -> (B, S, D)

Architecture

  • self.router: nn.Linear(d_model, num_experts) — gating network
  • self.experts: nn.ModuleList of MLPs (Linear→ReLU→Linear)
  • For each token: select top-k experts, compute weighted sum of their outputs
import torch
import torch.nn as nn

# ✏️ YOUR IMPLEMENTATION HERE

class MixtureOfExperts(nn.Module):
    def __init__(self, d_model, d_ff, num_experts, top_k=2):
        super().__init__()
        self.top_k = top_k
        self.router = nn.Linear(d_model, num_experts)
        self.experts = nn.ModuleList([
            nn.Sequential(nn.Linear(d_model, d_ff), nn.ReLU(), nn.Linear(d_ff, d_model))
            for _ in range(num_experts)
        ])
        # router + experts

    def forward(self, x):                       # x: (B, T, d_model)
        logits = self.router(x)                                 # (B,T,E) 每个 token 的专家分
        topk_logits, topk_idx = logits.topk(self.top_k, dim=-1) # 各 (B,T,k)
        weights = torch.softmax(topk_logits, dim=-1)            # (B,T,k) 只在入选者之间分权重
        out = torch.zeros_like(x)                               # 累加器
        for e in range(len(self.experts)):
            chosen = (topk_idx == e)                            # (B,T,k)
            w = (weights * chosen).sum(dim=-1)                  # (B,T)
            out = out + self.experts[e](x) * w.unsqueeze(-1)    # 广播 (B,T,1)×(B,T,D)
        return out

MoE模型使用门控网络来计算专家的优先级。并选择前top_k个专家网络,对专家网络给出的输出按照门控网络评分的softmax加权作为输出。这里的softmax只包含被选择的专家,而不是全部专家。

这种写法即使是没被选择的专家也参与了计算,self.experts[e](x)把token扔给所有专家前向了一遍。

for e in range(len(self.experts)):
    chosen = (topk_idx == e)        # mask 还是要的:位置级开关
    if not chosen.any():            # guard:没有任何 token 选 e → 白算,跳过
        continue
    w = (weights * chosen).sum(dim=-1)
    out = out + self.experts[e](x) * w.unsqueeze(-1)

加一个判断就好了,这里要注意(topk_idx == e)是一个tensor mask, 不是bool

29_Adam Optimizer

Implement the Adam optimizer from scratch.

Signature

class MyAdam:
    def __init__(self, params, lr=1e-3, betas=(0.9, 0.999), eps=1e-8): ...
    def step(self): ...
    def zero_grad(self): ...

Algorithm (per parameter)

m = β1 * m + (1-β1) * grad
v = β2 * v + (1-β2) * grad²
m̂ = m / (1 - β1ᵗ)    # bias correction
v̂ = v / (1 - β2ᵗ)
p -= lr * m̂ / (√v̂ + ε)

params : 要优化的参数清单,在真实使用时是model.parameters(),枚举了模型中登记过的nn.Parameter

p : 模型里的参数

m : 一阶动量 / 一阶矩估计(first moment,即梯度 EMA)

v :二阶动量 / 二阶矩估计(second moment,即 g² 的 EMA)

Adam优化器优化的内容:

m = β1·m + (1−β1)·g          # 一阶矩:梯度的滑动平均
v = β2·v + (1−β2)·g²         # 二阶矩:梯度平方的滑动平均
p -= lr · m̂ / (√v̂ + ε)       # 更新

Adam = Adaptive Moment Estimation 自适应矩估计

其核心是Momentum + RMSProp

Momentum 即 x=βx+(1−β)∇(x) x = \beta x + (1 – \beta) \grad(x) 记录梯度的惯性,减小参数的震荡,β\beta 是质数衰减率也被成为momentum系数

RMSProp即 使用根号下二阶矩来约束梯度值,相对于SGD在梯度较小的时候走大步,在梯度较大的时候走小步

import torch

# ✏️ YOUR IMPLEMENTATION HERE

class MyAdam:
    def __init__(self, params, lr=1e-3, betas=(0.9, 0.999), eps=1e-8):
        # store params, init m and v to zeros
        self.params = list(params)
        self.lr = lr
        self.beta1, self.beta2 = betas
        self.eps = eps
        self.state = {}                            # 账本:key = 参数张量
        for p in self.params:
            self.state[p] = {"step": 0,
                             "m": torch.zeros_like(p),   # 一阶矩
                             "v": torch.zeros_like(p)}   # 二阶矩
    def step(self):
        # update params using Adam rule
        for p in self.params:
            if p.grad is None:                     # 这步没收到梯度的参数跳过
                continue
            st = self.state[p]
            st["step"] += 1
            t = st["step"]
            st["m"].mul_(self.beta1).add_(p.grad, alpha=1 - self.beta1)
            st["v"].mul_(self.beta2).add_(p.grad * p.grad, alpha=1 - self.beta2)
            m_hat = st["m"] / (1 - self.beta1 ** t)
            v_hat = st["v"] / (1 - self.beta2 ** t)
            p.data.add_(m_hat / (v_hat.sqrt() + self.eps), alpha=-self.lr)

    def zero_grad(self):
        # zero all gradients
        for p in self.params:
            if p.grad is not None:
                p.grad.zero_()

30_Cosine LR Scheduler with Warmup

Implement a cosine learning rate schedule with linear warmup.

Signature

def cosine_lr_schedule(step, total_steps, warmup_steps, max_lr, min_lr=0.0) -> float:

Schedule

step < warmup:  lr = max_lr * step / warmup_steps  (linear ramp)
step >= warmup: lr = min_lr + 0.5*(max_lr-min_lr)*(1 + cos(π * progress))

where progress = (step - warmup) / (total - warmup)

import math
# ✏️ YOUR IMPLEMENTATION HERE

def cosine_lr_schedule(step, total_steps, warmup_steps, max_lr, min_lr=0.0):
    # warmup then cosine decay
    progress = (step - warmup_steps) / (total_steps - warmup_steps)
    if step < warmup_steps:
        lr = max_lr * step / warmup_steps
    else:
        lr = min_lr + 0.5 * (max_lr - min_lr) * (1 + math.cos(math.pi * progress))
    return lr

预热区使用线性上升的学习率,预热结束后让学习率先快后慢,直至训练末期达到最小值。

Related Posts