Part 13 · Transformer 深入

import torchimport torch.nn.functional as F# 3 个词，每个 4 维X = torch.tensor([    [1.0, 0.0, 0.0, 0.0],  # 词A "我"    [0.0, 1.0, 0.0, 0.0],  # 词B "爱"    [0.0, 0.0, 1.0, 0.0],  # 词C "你"])# 简化：直接用 X 当 Q、K、V（真实模型会用可学习权重变换）Q = K = V = X# 1. 算注意力分数：Q 和所有 K 的点积scores = torch.mm(Q, K.T)          # 3x3print("注意力分数:\n", scores)# 2. 归一化成权重（softmax，按行）weights = F.softmax(scores, dim=-1)print("\n注意力权重:\n", weights)# 3. 加权求和 Vout = torch.mm(weights, V)print("\n输出（每个词融合了全句信息）:\n", out)

多头 = 用多组 Wq/Wk/Wv 同时算多次注意力，每组关注不同关系例：头1 关注"语法搭配"，头2 关注"指代关系"，头3 关注"数字大小"最后把头拼起来 → 模型看得更全面

import torchimport torch.nn as nnclass MultiHeadAttention(nn.Module):    def __init__(self, d_model=4, n_heads=2):        super().__init__()        self.n_heads = n_heads        self.head_dim = d_model // n_heads        # 每组头用自己的权重        self.wq = nn.Linear(d_model, d_model)        self.wk = nn.Linear(d_model, d_model)        self.wv = nn.Linear(d_model, d_model)        self.out = nn.Linear(d_model, d_model)    def forward(self, x):        B, T, D = x.shape        Q = self.wq(x).view(B, T, self.n_heads, self.head_dim).transpose(1, 2)        K = self.wk(x).view(B, T, self.n_heads, self.head_dim).transpose(1, 2)        V = self.wv(x).view(B, T, self.n_heads, self.head_dim).transpose(1, 2)        scores = Q @ K.transpose(-2, -1) / (self.head_dim ** 0.5)        weights = torch.softmax(scores, dim=-1)        out = (weights @ V).transpose(1, 2).reshape(B, T, D)        return self.out(out)x = torch.randn(1, 3, 4)   # 1句话3个词，4维print(MultiHeadAttention()(x).shape)   # torch.Size([1, 3, 4])

注意力不分前后顺序（"我打你"和"你打我"权重一样）→ 需要位置编码位置编码 = 给每个词的位置加一组特殊向量，让模型知道"谁在前谁在后"经典做法：sin/cos 公式（原论文）或 可学习的位置向量（GPT 用）

import torchimport mathdef position_encoding(seq_len=4, d_model=8):    pe = torch.zeros(seq_len, d_model)    for pos in range(seq_len):        for i in range(0, d_model, 2):            pe[pos, i] = math.sin(pos / (10000 ** (i / d_model)))            pe[pos, i + 1] = math.cos(pos / (10000 ** (i / d_model)))    return pepe = position_encoding()print("位置0的编码:", pe[0])print("位置1的编码:", pe[1])print("不同位置编码不同 → 模型能区分顺序")

Encoder-only   双向看全句（BERT）→ 适合理解/分类/嵌入Decoder-only   只往前看（GPT）  → 适合生成（所有现代 LLM 都用它）Encoder-Decoder 编码+解码（T5/机器翻译）→ 复杂生成任务

import torchscores = torch.rand(4, 4)   # 4个词的注意力分数# 掩码：只允许看自己和之前（上三角置为 -inf）mask = torch.triu(torch.full((4, 4), float("-inf")), diagonal=1)masked = scores + maskprint("掩码后（下三角可见，上三角=无效）:\n", torch.round(masked, decimals=2))

import torchimport torch.nn.functional as F# 假设模型给出 5 个候选词的分数logits = torch.tensor([2.0, 1.5, 0.1, -1.0, 3.5])# 1. 贪心：直接取分数最高的print("贪心:", logits.argmax().item())# 2. 采样（temperature）：分数缩放后随机抽probs = F.softmax(logits / 0.8, dim=-1)sample = torch.multinomial(probs, 1).item()print("采样(温度0.8):", sample, "| 概率分布:", [round(p, 2) for p in probs])# 3. Top-P（核采样）：只从累计概率到 p 的词里抽def top_p_sampling(logits, p=0.9):    probs = F.softmax(logits, dim=-1)    sorted_p, idx = torch.sort(probs, descending=True)    cum = torch.cumsum(sorted_p, dim=-1)    mask = cum - sorted_p > p    sorted_p[mask] = 0    probs = sorted_p / sorted_p.sum()    return torch.multinomial(probs, 1)print("Top-P 采样:", top_p_sampling(logits).item())

import torchimport torch.nn as nnimport torch.nn.functional as Fclass SelfAttention(nn.Module):    def __init__(self, d_model=8):        super().__init__()        self.wq = nn.Linear(d_model, d_model)        self.wk = nn.Linear(d_model, d_model)        self.wv = nn.Linear(d_model, d_model)    def forward(self, x):        Q, K, V = self.wq(x), self.wk(x), self.wv(x)        d = Q.shape[-1]        scores = torch.mm(Q, K.T) / (d ** 0.5)        weights = F.softmax(scores, dim=-1)        return torch.mm(weights, V)x = torch.randn(4, 8)          # 4个词，8维out = SelfAttention()(x)print("输出形状:", out.shape)   # torch.Size([4, 8])

import torchimport torch.nn as nnclass MiniGPT(nn.Module):    def __init__(self, vocab=100, d_model=32, n_heads=4, n_layers=2):        super().__init__()        self.embed = nn.Embedding(vocab, d_model)        self.pos = nn.Parameter(torch.randn(1, 50, d_model))        self.blocks = nn.ModuleList([            nn.TransformerEncoderLayer(d_model, n_heads, dim_feedforward=128, batch_first=True)            for _ in range(n_layers)        ])        self.head = nn.Linear(d_model, vocab)    def forward(self, x):        x = self.embed(x) + self.pos[:, :x.shape[1]]        for b in self.blocks:            x = b(x)        return self.head(x)   # 每个位置预测下一个词的分数model = MiniGPT()x = torch.randint(0, 100, (2, 10))     # 2句话，每句10个词logits = model(x)print("输出形状:", logits.shape)         # torch.Size([2, 10, 100])print("第一句第一个位置预测的词表分数前3:", logits[0, 0, :3].detach().numpy())

from transformers import pipeline# 加载一个很小的中文生成模型gen = pipeline("text-generation", model="uer/gpt2-chinese-cluecorpussmall")result = gen("今天天气", max_new_tokens=20, do_sample=True, temperature=0.8)print(result[0]["generated_text"])