test_gpt2作业中遇到的一些问题

[zhouyibetter]

// 测试类
class GPT2TrainingTest : public ::testing::Test {
protected:
    void SetUp() override {
        llmc_filepath = "../../Data/gpt2_124M.bin";
        input_bin = "../../Data/tinyshakespeare/tiny_shakespeare_train.bin";
        tokenizer_bin = "../../Data/gpt2_tokenizer.bin";
        logits_reference = "../../Data/gpt2_logits_reference.bin";

        device_flag = "cuda";
        model_name = "gpt2";
        batch_size = 2;
        sequence_length = 64;
        total_batch_size = 256;
        num_iteration = 10;    // 迭代次数
        text_length = 64;    // 生成文本长度
        learning_rate = 1e-4;    //学习率

        Initialize();

        LOG(INFO)<< "Initialize() finished!";
    }

这里提到的logits_reference = "../../Data/gpt2_logits_reference.bin";找不到，download_starter_pack.sh中也没有找到适合的连接，难道是tiny_shakespeare_val.bin吗？

其次，在

        /* tokenizer */
        if ((step + 1) % freq_generate_txt == 0) {
            if (!tokenizer) {
                continue;
            }
            tokenizer->GenerateText(*model, batch_size, sequence_length, text_length, device);
        }

文本生成的长度过长会导致爆内存的问题，报错信息如下：

Breakpoint 6 at 0x5555556817ac: file /home/eq/TinyInfiniTrain/example/common/tokenizer.cc, line 170.
(gdb) c
Continuing.
The meaning of life is that deriving pleasure from things being desired: if two people voluntarily premise that an ideal related to an ideal is desirable, the ideal relates to the human enviroment.)<|endoftext|>Quickunknown file: Failure
C++ exception with description "parallel_for failed: cudaErrorInvalidDevice: invalid device ordinal" thrown in the test body.

[  FAILED  ] GPT2TrainingTest.LogitsConsistency (555723 ms)
[----------] 1 test from GPT2TrainingTest (555723 ms total)

[----------] Global test environment tear-down
[==========] 1 test from 1 test suite ran. (555723 ms total)

[zhouyibetter]

我忘记贴上我的实现了：

void Tokenizer::GenerateText(infini_train::nn::Module &model, uint32_t batch_size, uint32_t sequence_length,
                             uint32_t text_length, Device device) const {
    std::vector<int64_t> dims;
    dims.assign({batch_size, sequence_length});
    // x_tensor (FLAGS_batch_size, FLAGS_sequence_length) eq:(4, 64)
    infini_train::Tensor x_tensor = infini_train::Tensor(dims, DataType::kINT64);
    int64_t *x_buff = static_cast<int64_t *>(x_tensor.DataPtr());
    for (int i = 0; i < batch_size * sequence_length; ++i) { x_buff[i] = eot_token_; }

    // Give some contexts: "The meaning of life is "
    auto prompt = kPromptMap.at(magic_number_);
    auto prompt_len = prompt.size();
    for (int i = 0; i < prompt_len; ++i) { x_buff[i] = prompt[i]; }
    std::cout << "The meaning of life is";

    auto x = std::make_shared<infini_train::Tensor>(x_tensor.To(device));
    uint64_t kRngState = kRngState;
    LOG(INFO) << "start generate text:";

    uint32_t max_safe_length = 16;  // 留一些缓冲
    infini_train::Device cpu_device = infini_train::Device(infini_train::DeviceType::kCPU, 0);
    for (int t = prompt_len; t < max_safe_length; t++) {
        /* ===================================== 作业 =====================================
        TODO：实现单步文本生成逻辑
        HINT：调用model.Forward推理获取logits，根据推理结果进行随机采样，调用Decode获取文本结果
        ===================================== 作业 ===================================== */
        x = std::make_shared<infini_train::Tensor>(x->To(device)); // CPU->calc device
        // TODO(jym): use no_grad forward later
        auto logits = model.Forward({x})[0];
        auto logits_orignal = nn::function::Softmax(logits, -1);
        auto logits_cpu = logits_orignal->To(cpu_device);
        auto data = logits_cpu.DataPtr();
        auto vocab_size = logits->Dims()[2];
        float *probs = static_cast<float *>(data) + (t - 1) * vocab_size;
        float coin = RandomF32(kRngState);
        int next_token = SampleMult(probs, vocab_size, coin);

        x = std::make_shared<infini_train::Tensor>(x->To(cpu_device)); // calc device->CPU
        auto data_temp = static_cast<int64_t *>(x->DataPtr());
        data_temp[t] = next_token;
        std::cout << Decode(next_token);
    }
    std::cout << std::endl;
}

为了测试完整性质，我截短了输出的长度uint32_t max_safe_length = 16; // 留一些缓冲，我觉得这里的问题应该是auto logits = model.Forward({x})[0];没有做到清除auto_grad，生成长度太长后直接爆显存了，请大家帮帮我。

[JYMiracle305]

我忘记贴上我的实现了：

void Tokenizer::GenerateText(infini_train::nn::Module &model, uint32_t batch_size, uint32_t sequence_length,
uint32_t text_length, Device device) const {
std::vector<int64_t> dims;
dims.assign({batch_size, sequence_length});
// x_tensor (FLAGS_batch_size, FLAGS_sequence_length) eq:(4, 64)
infini_train::Tensor x_tensor = infini_train::Tensor(dims, DataType::kINT64);
int64_t *x_buff = static_cast<int64_t *>(x_tensor.DataPtr());
for (int i = 0; i < batch_size * sequence_length; ++i) { x_buff[i] = eot_token_; }

// Give some contexts: "The meaning of life is "
auto prompt = kPromptMap.at(magic_number_);
auto prompt_len = prompt.size();
for (int i = 0; i < prompt_len; ++i) { x_buff[i] = prompt[i]; }
std::cout << "The meaning of life is";

auto x = std::make_shared<infini_train::Tensor>(x_tensor.To(device));
uint64_t kRngState = kRngState;
LOG(INFO) << "start generate text:";

uint32_t max_safe_length = 16;  // 留一些缓冲
infini_train::Device cpu_device = infini_train::Device(infini_train::DeviceType::kCPU, 0);
for (int t = prompt_len; t < max_safe_length; t++) {
    /* ===================================== 作业 =====================================
    TODO：实现单步文本生成逻辑
    HINT：调用model.Forward推理获取logits，根据推理结果进行随机采样，调用Decode获取文本结果
    ===================================== 作业 ===================================== */
    x = std::make_shared<infini_train::Tensor>(x->To(device)); // CPU->calc device
    // TODO(jym): use no_grad forward later
    auto logits = model.Forward({x})[0];
    auto logits_orignal = nn::function::Softmax(logits, -1);
    auto logits_cpu = logits_orignal->To(cpu_device);
    auto data = logits_cpu.DataPtr();
    auto vocab_size = logits->Dims()[2];
    float *probs = static_cast<float *>(data) + (t - 1) * vocab_size;
    float coin = RandomF32(kRngState);
    int next_token = SampleMult(probs, vocab_size, coin);

    x = std::make_shared<infini_train::Tensor>(x->To(cpu_device)); // calc device->CPU
    auto data_temp = static_cast<int64_t *>(x->DataPtr());
    data_temp[t] = next_token;
    std::cout << Decode(next_token);
}
std::cout << std::endl;

}
为了测试完整性质，我截短了输出的长度uint32_t max_safe_length = 16; // 留一些缓冲，我觉得这里的问题应该是auto logits = model.Forward({x})[0];没有做到清除auto_grad，生成长度太长后直接爆显存了，请大家帮帮我。

针对以上三个问题

1. gpt2_logits_reference.bin文件提传到仓库内
2. 其他二进制文件确实需要从huggingface下载，如果下载失败，可以登录训练营提供的服务器，在公共目录下放置了这些文件
3. 请使用最新代码（修复内存泄漏问题）、训练营服务器（显存足够）重试