21_Gradient Norm Clipping
Implement gradient norm clipping — a training stability technique.
Signature
def clip_grad_norm(parameters, max_norm: float) -> float:
# Clip gradients in-place so total norm <= max_norm
# Returns the original (unclipped) total norm
Algorithm
- Compute total norm:
sqrt(sum(p.grad.norm()^2 for p in parameters)) - If total > max_norm: scale all grads by
max_norm / total - Return original total norm
import torch
# ✏️ YOUR IMPLEMENTATION HERE
def clip_grad_norm(parameters, max_norm):
# compute total norm, clip if needed, return original norm
grads = [p.grad for p in parameters if p.grad is not None]
total = torch.sqrt(sum(g.norm()**2 for g in grads))
if total > max_norm:
for g in grads:
g.mul_(max_norm / total)
return total.item()梯度裁剪将梯度的第二范数上线卡到max_norm
返回原来的梯度范数作为监控信息提示供训练者判断情况
这里注意一定要使用原地改的写法
22_2D Convolution
Implement 2D convolution from scratch.
Signature
def my_conv2d(x, weight, bias=None, stride=1, padding=0):
# x: (B, C_in, H, W), weight: (C_out, C_in, kH, kW)
# Returns: (B, C_out, H_out, W_out)
Rules
- Do NOT use
F.conv2dornn.Conv2d - Support
strideandpaddingparameters F.padfor zero-padding is allowed
import torch
import torch.nn.functional as F
# ✏️ YOUR IMPLEMENTATION HERE
def my_conv2d(x, weight, bias=None, stride=1, padding=0):
# extract patches, apply kernel, handle stride/padding
B, C_in, H, W = x.shape
C_out, _, kH, kW = weight.shape
H_out = (H + 2 * padding - kH) // stride + 1
W_out = (W + 2 * padding - kW) // stride + 1
out = torch.zeros(B, C_out, H_out, W_out)
x = F.pad(x, (padding,padding,padding,padding) )
for b in range(B):
for co in range(C_out):
for i in range(H_out):
for j in range(W_out):
patch = x[b, :, i*stride : i*stride + kH, j*stride : j*stride +kW]
out[b, co, i, j] = (patch * weight[co]).sum() + (bias[co] if bias is not None else 0)
return outCin是输入通道数,比如单通道灰度和三通道RGB。
Cout是输出特征数量,可以自定义。
23_Multi-Head Cross-Attention
Implement multi-head cross-attention (encoder-decoder attention).
Signature¶
class MultiHeadCrossAttention(nn.Module):
def __init__(self, d_model: int, num_heads: int): ...
def forward(self, x_q: Tensor, x_kv: Tensor) -> Tensor:
# x_q: (B, S_q, D) — decoder queries
# x_kv: (B, S_kv, D) — encoder keys/values
Key Differences from Self-Attention
- Q comes from the decoder, K and V come from the encoder
- No causal mask (all encoder positions visible)
import torch
import torch.nn as nn
import math
# ✏️ YOUR IMPLEMENTATION HERE
class MultiHeadCrossAttention(nn.Module):
def __init__(self, d_model, num_heads):
super().__init__()
# W_q, W_k, W_v, W_o
self.W_q = nn.Linear(d_model, d_model)
self.W_k = nn.Linear(d_model, d_model)
self.W_v = nn.Linear(d_model, d_model)
self.W_o = nn.Linear(d_model, d_model)
self.d_k = d_model // num_heads
self.num_heads = num_heads
def forward(self, x_q, x_kv):
# Q from x_q, K/V from x_kv, no causal mask
B, sq, D = x_q.shape
_, skv,D = x_kv.shape
q = self.W_q(x_q).view(B, sq, self.num_heads, self.d_k).transpose(1,2)
k = self.W_k(x_kv).view(B, skv, self.num_heads, self.d_k).transpose(1,2)
v = self.W_v(x_kv).view(B, skv, self.num_heads, self.d_k).transpose(1,2)
score = q @ k.transpose(-1,-2) / math.sqrt(self.d_k)
atten = torch.softmax(score, dim = -1) @ v
atten = atten.transpose(1,2).contiguous().view(B,sq,D)
return atten24_Rotary Position Embedding (RoPE)
mplement RoPE — the position encoding used in LLaMA, GPT-NeoX, and most modern LLMs.
Signature
def apply_rope(q: Tensor, k: Tensor) -> tuple[Tensor, Tensor]:
# q, k: (B, S, D) where D is even
# Returns rotated (q, k) with same shape
Key Idea
Split each vector into consecutive pairs. Rotate each pair by θ = pos / 10000^(2i/D):
[x_0, x_1] → [x_0*cosθ - x_1*sinθ, x_0*sinθ + x_1*cosθ]
This makes dot(q_rot[i], k_rot[j]) depend only on i - j (relative position).
import torch
import math
# ✏️ YOUR IMPLEMENTATION HERE
def apply_rope(q, k):
B, S, D = q.shape
# ① 角度表 (S, D//2)
pos = torch.arange(S)
freq = 10000 ** (-2 * torch.arange(D // 2) / D)
angles = pos[:, None] * freq[None, :] # (S, D//2)
cos = torch.cos(angles)
sin = torch.sin(angles)
def rotate(x):
# ② 拆对
x = x.reshape(B, S, D // 2, 2) # (B, S, D//2, 2)
x_even, x_odd = x[..., 0], x[..., 1] # 各 (B, S, D//2)
# ③ 旋转(复数乘法展开)
rot_even = x_even * cos - x_odd * sin
rot_odd = x_even * sin + x_odd * cos
# ④ 合并
return torch.stack([rot_even, rot_odd], dim=-1).reshape(B, S, D)
return rotate(q), rotate(k)将向量维度两两配对分组,并分别使用不用的周期进行旋转操作,实现快慢指针高精度表征位置关系。而且可以实现相对位置关系的计算。
25_Flash Attention (Tiled)
Implement tiled attention with online softmax — the core idea behind Flash Attention.
Signature
def flash_attention(Q, K, V, block_size=32) -> Tensor:
# Q, K, V: (B, S, D)
# Returns: (B, S, D) — same as standard attention
Key Insight
Instead of materializing the full S×S attention matrix, process in blocks:
- For each Q-block, iterate over K/V blocks
- Use online softmax: track running
maxandsum - Rescale accumulator when max changes:
acc *= exp(old_max - new_max) - Final:
output = acc / row_sum
Must give identical results to standard softmax attention.
import torch
import math
def flash_attention(Q, K, V, block_size=32):
B, S, D = Q.shape
scale = 1.0 / math.sqrt(D)
m = torch.full((B, S, 1), float('-inf')) # running max,初始 -inf
l = torch.zeros(B, S, 1) # running sum
O = torch.zeros(B, S, D) # running 输出
for i in range(0, S, block_size):
K_b = K[:, i:i+block_size] # (B, bs, D) 切片,天然处理余数
V_b = V[:, i:i+block_size]
score = Q @ K_b.transpose(-1, -2) * scale # (B, S, bs) ④ 补上缩放
m2 = torch.amax(score, dim=-1, keepdim=True) # (B, S, 1) ②③ 块内 max
m_new = torch.maximum(m, m2) # 新 running max
P = torch.exp(score - m_new) # (B, S, bs) 未归一化权重
alpha = torch.exp(m - m_new) # (B, S, 1) 旧块衰减因子
l = alpha * l + P.sum(dim=-1, keepdim=True) # ③ keepdim
O = alpha * O + P @ V_b # ⑥ P 直接乘 V
m = m_new
return O / l # 最后整体归一化Online softmax还是有点绕的
26_LoRA (Low-Rank Adaptation)
Implement LoRA — parameter-efficient fine-tuning for large models.
h = W_0 x + \frac{\alpha}{r} B A xSignature
class LoRALinear(nn.Module):
def __init__(self, in_features, out_features, rank, alpha=1.0): ...
def forward(self, x: Tensor) -> Tensor: ...
Requirements
self.linear: frozennn.Linear(weight & biasrequires_grad=False)self.lora_A:nn.Parameter(rank, in_features)— random initself.lora_B:nn.Parameter(out_features, rank)— zero init- Scaling:
alpha / rank
import torch
import torch.nn as nn
# ✏️ YOUR IMPLEMENTATION HERE
class LoRALinear(nn.Module):
def __init__(self, in_features, out_features, rank, alpha=1.0):
super().__init__()
self.alpha = alpha
self.rank = rank
self.linear = nn.Linear(in_features, out_features)
self.linear.weight.requires_grad_(False)
self.linear.bias.requires_grad_(False)
self.lora_A = nn.Parameter(torch.randn(rank, in_features))
self.lora_B = nn.Parameter(torch.zeros(out_features, rank))
# frozen linear + lora_A + lora_B
def forward(self, x):
return self.linear(x) + (self.alpha / self.rank) * (x @ self.lora_A.T @ self.lora_B.T)
# base + loraLoRA是参数高效微调(PEFT)中最主流的一种,其关键是并不改变原有参数,而是添加低秩参数矩阵作为修正。
其中A矩阵为随机初始化,B矩阵为零初始化。
如果AB都为随机初始化,则训练刚开始就被从预训练收敛点加上偏移。
如果AB都为零初始化,则AB矩阵的梯度将永远为零,旁路死亡。
27_ViT Patch Embedding
Implement the patch embedding layer from Vision Transformer (ViT).
Signature
class PatchEmbedding(nn.Module):
def __init__(self, img_size, patch_size, in_channels, embed_dim): ...
def forward(self, x: Tensor) -> Tensor:
# x: (B, C, H, W)
# Returns: (B, num_patches, embed_dim)
Algorithm
- Reshape image into non-overlapping patches:
(B, C, H, W)→(B, N, C*P*P) - Project each patch:
nn.Linear(C*P*P, embed_dim) num_patches = (img_size // patch_size) ** 2
import torch
import torch.nn as nn
# ✏️ YOUR IMPLEMENTATION HERE
class PatchEmbedding(nn.Module):
def __init__(self, img_size, patch_size, in_channels, embed_dim):
super().__init__()
# self.num_patches, self.proj
self.patch_size = patch_size
self.num_patches = (img_size // patch_size) ** 2
self.proj = nn.Linear(in_channels*patch_size*patch_size, embed_dim)
def forward(self, x):
# reshape to patches, project
B, C, H, W = x.shape
x = x.view(B, C, H//self.patch_size, self.patch_size, W//self.patch_size,self.patch_size)
x = x.permute(0,2,4,1,3,5)
x = x.contiguous().view(B, self.num_patches, self.patch_size * self.patch_size * C)
return self.proj(x)将图像进行分割再embedding向量化,作为输入transformer的token。
一个常见误区是直接进行形状变换,那样的结果是行采样而不是window采样
28_Mixture of Experts (MoE)
Implement a Mixture of Experts layer (Mixtral / Switch Transformer style).
Signature
class MixtureOfExperts(nn.Module):
def __init__(self, d_model, d_ff, num_experts, top_k=2): ...
def forward(self, x: Tensor) -> Tensor:
# x: (B, S, D) -> (B, S, D)
Architecture
self.router:nn.Linear(d_model, num_experts)— gating networkself.experts:nn.ModuleListof MLPs(Linear→ReLU→Linear)- For each token: select top-k experts, compute weighted sum of their outputs
import torch
import torch.nn as nn
# ✏️ YOUR IMPLEMENTATION HERE
class MixtureOfExperts(nn.Module):
def __init__(self, d_model, d_ff, num_experts, top_k=2):
super().__init__()
self.top_k = top_k
self.router = nn.Linear(d_model, num_experts)
self.experts = nn.ModuleList([
nn.Sequential(nn.Linear(d_model, d_ff), nn.ReLU(), nn.Linear(d_ff, d_model))
for _ in range(num_experts)
])
# router + experts
def forward(self, x): # x: (B, T, d_model)
logits = self.router(x) # (B,T,E) 每个 token 的专家分
topk_logits, topk_idx = logits.topk(self.top_k, dim=-1) # 各 (B,T,k)
weights = torch.softmax(topk_logits, dim=-1) # (B,T,k) 只在入选者之间分权重
out = torch.zeros_like(x) # 累加器
for e in range(len(self.experts)):
chosen = (topk_idx == e) # (B,T,k)
w = (weights * chosen).sum(dim=-1) # (B,T)
out = out + self.experts[e](x) * w.unsqueeze(-1) # 广播 (B,T,1)×(B,T,D)
return outMoE模型使用门控网络来计算专家的优先级。并选择前top_k个专家网络,对专家网络给出的输出按照门控网络评分的softmax加权作为输出。这里的softmax只包含被选择的专家,而不是全部专家。
这种写法即使是没被选择的专家也参与了计算,self.experts[e](x)把token扔给所有专家前向了一遍。
for e in range(len(self.experts)):
chosen = (topk_idx == e) # mask 还是要的:位置级开关
if not chosen.any(): # guard:没有任何 token 选 e → 白算,跳过
continue
w = (weights * chosen).sum(dim=-1)
out = out + self.experts[e](x) * w.unsqueeze(-1)加一个判断就好了,这里要注意(topk_idx == e)是一个tensor mask, 不是bool
29_Adam Optimizer
Implement the Adam optimizer from scratch.
Signature
class MyAdam:
def __init__(self, params, lr=1e-3, betas=(0.9, 0.999), eps=1e-8): ...
def step(self): ...
def zero_grad(self): ...
Algorithm (per parameter)
m = β1 * m + (1-β1) * grad
v = β2 * v + (1-β2) * grad²
m̂ = m / (1 - β1ᵗ) # bias correction
v̂ = v / (1 - β2ᵗ)
p -= lr * m̂ / (√v̂ + ε)
params : 要优化的参数清单,在真实使用时是model.parameters(),枚举了模型中登记过的nn.Parameter
p : 模型里的参数
m : 一阶动量 / 一阶矩估计(first moment,即梯度 EMA)
v :二阶动量 / 二阶矩估计(second moment,即 g² 的 EMA)
Adam优化器优化的内容:
m = β1·m + (1−β1)·g # 一阶矩:梯度的滑动平均
v = β2·v + (1−β2)·g² # 二阶矩:梯度平方的滑动平均
p -= lr · m̂ / (√v̂ + ε) # 更新Adam = Adaptive Moment Estimation 自适应矩估计
其核心是Momentum + RMSProp
Momentum 即 记录梯度的惯性,减小参数的震荡, 是质数衰减率也被成为momentum系数
RMSProp即 使用根号下二阶矩来约束梯度值,相对于SGD在梯度较小的时候走大步,在梯度较大的时候走小步
import torch
# ✏️ YOUR IMPLEMENTATION HERE
class MyAdam:
def __init__(self, params, lr=1e-3, betas=(0.9, 0.999), eps=1e-8):
# store params, init m and v to zeros
self.params = list(params)
self.lr = lr
self.beta1, self.beta2 = betas
self.eps = eps
self.state = {} # 账本:key = 参数张量
for p in self.params:
self.state[p] = {"step": 0,
"m": torch.zeros_like(p), # 一阶矩
"v": torch.zeros_like(p)} # 二阶矩
def step(self):
# update params using Adam rule
for p in self.params:
if p.grad is None: # 这步没收到梯度的参数跳过
continue
st = self.state[p]
st["step"] += 1
t = st["step"]
st["m"].mul_(self.beta1).add_(p.grad, alpha=1 - self.beta1)
st["v"].mul_(self.beta2).add_(p.grad * p.grad, alpha=1 - self.beta2)
m_hat = st["m"] / (1 - self.beta1 ** t)
v_hat = st["v"] / (1 - self.beta2 ** t)
p.data.add_(m_hat / (v_hat.sqrt() + self.eps), alpha=-self.lr)
def zero_grad(self):
# zero all gradients
for p in self.params:
if p.grad is not None:
p.grad.zero_()30_Cosine LR Scheduler with Warmup
Implement a cosine learning rate schedule with linear warmup.
Signature
def cosine_lr_schedule(step, total_steps, warmup_steps, max_lr, min_lr=0.0) -> float:
Schedule
step < warmup: lr = max_lr * step / warmup_steps (linear ramp)
step >= warmup: lr = min_lr + 0.5*(max_lr-min_lr)*(1 + cos(π * progress))
where progress = (step - warmup) / (total - warmup)
import math
# ✏️ YOUR IMPLEMENTATION HERE
def cosine_lr_schedule(step, total_steps, warmup_steps, max_lr, min_lr=0.0):
# warmup then cosine decay
progress = (step - warmup_steps) / (total_steps - warmup_steps)
if step < warmup_steps:
lr = max_lr * step / warmup_steps
else:
lr = min_lr + 0.5 * (max_lr - min_lr) * (1 + math.cos(math.pi * progress))
return lr预热区使用线性上升的学习率,预热结束后让学习率先快后慢,直至训练末期达到最小值。