第 9 章 LoRA 参数高效微调

class LoRALayerImpl : public torch::nn::Module {
public:
    LoRALayerImpl(int64_t in_dim, int64_t out_dim, int64_t rank,
                  double alpha, double dropout = 0.0) {
        A_ = register_parameter("A", torch::empty({in_dim, rank}));
        torch::nn::init::kaiming_uniform_(A_, std::sqrt(5.0));
        B_ = register_parameter("B", torch::zeros({rank, out_dim}));  // 全 0！
        if (dropout > 0.0)
            dropout_ = register_module("dropout", torch::nn::Dropout(dropout));
    }
    torch::Tensor forward(torch::Tensor x) {
        const double scale = alpha_ / static_cast<double>(rank_);
        return (x.matmul(A_)).matmul(B_) * scale;    // (x@A)@B · α/r
    }
};

class LinearWithLoRAImpl : public torch::nn::Module {
public:
    LinearWithLoRAImpl(torch::nn::Linear linear, int64_t rank,
                       double alpha, double dropout = 0.0) {
        linear_ = register_module("linear", linear);
        lora_ = register_module("lora",
            LoRALayer(linear->weight.size(1),   // in_features
                      linear->weight.size(0),   // out_features
                      rank, alpha, dropout));
        for (auto& p : linear_->parameters())     // 冻结原权重
            p.set_requires_grad(false);
    }
    torch::Tensor forward(torch::Tensor x) {
        return linear_->forward(x) + lora_->forward(x);
    }
};

inline int64_t freeze_non_lora_parameters(torch::nn::Module& model) {
    int64_t trainable = 0;
    for (auto& pair : model.named_parameters(/*recurse=*/true)) {
        const std::string& name = pair.key();
        const bool is_lora = (name.find(".lora.A") != std::string::npos) ||
                             (name.find(".lora.B") != std::string::npos);
        if (is_lora) { pair.value().set_requires_grad(true); trainable += pair.value().numel(); }
        else         { pair.value().set_requires_grad(false); }
    }
    return trainable;
}

auto h = model->forward_hidden(x);            // [1, seq, emb]
auto last = h.select(1, h.size(1) - 1);       // [1, emb] 最后一个词元
auto logits = model->out_head->forward(last); // [1, num_classes]

                随机基线    LoRA 5 epoch
Train 准确率      50%         100%
Val   准确率      50%         96.64%
Test  准确率      50%         98.0%      ← 对比第 6 章冻结微调的 95.7%
loss: ep0 0.76 → ep2 0.31 → ep4 0.10（train/val 全程贴合，无过拟合分叉）

for (auto& pair : model->named_parameters(/*recurse=*/true)) {
    if (n.find(".lora.A") == std::string::npos &&
        n.find(".lora.B") == std::string::npos) continue;   // 只存 LoRA
    // 写入 名字 + 形状 + float 数据
}

// W: [out, in]，A@B: [in, out] —— 必须转置才能相加
torch::Tensor W_merged = W + (alpha / rank) * (A.matmul(B)).t();
// 展开写法（注意乘法顺序也反了）：(A@B)^T = B^T @ A^T
// 偏置 b 不变（LoRA 分支没有偏置）

# 独立构建（在代码仓库根目录执行）
cmake -S chapters/chapter09_lora -B chapters/chapter09_lora/build \
      -DCMAKE_CXX_COMPILER=/usr/bin/g++-11
cmake --build chapters/chapter09_lora/build -j

# 运行（直接执行需自带 LibTorch 运行库路径）
export LD_LIBRARY_PATH=/opt/libtorch/lib:${LD_LIBRARY_PATH:-}
./chapters/chapter09_lora/build/chapter09_lora          # CPU 可跑
./chapters/chapter09_lora/build/chapter09_lora --cuda

[appendix-E] using CPU
[appendix-E] tokenizer vocab = 50257
[appendix-E] LoRA only        : trainable=73728  total=7239938  (1.0186%)
[appendix-E] dataset size = 400
[appendix-E] train batches = 35, val = 5, test = 10

=== E.7 LoRA training ===
Ep 1 | train_loss 0.69 | val_acc 50% | 2.1 s
Ep 5 | train_loss 0.10 | val_acc 100% | 2.0 s
[appendix-E] LoRA test_acc = 100%  (trainable=73728 / 7239938)

=== E.7 ablation: full fine-tuning (unfreeze all) ===
[appendix-E] Full FT         : trainable=7239938  total=7239938 (100%)
Ep 5 | train_loss ... | val_acc ...
[appendix-E] Full FT test_acc = 100%

=== summary ===
LoRA-weights file size  : 295396 bytes      ← 仅 LoRA 矩阵 ≈ 0.3 MB