From ca4e1167212bd142a60fa08e6ae747106f4abe13 Mon Sep 17 00:00:00 2001 From: lichunyou <2695928308@qq.com> Date: Mon, 21 Jun 2021 14:23:36 +0800 Subject: [PATCH 1/2] align bert --- LanguageModeling/bert-oneflow/README.md | 23 +++ .../bert-oneflow/dataset/dataset.py | 127 ++++++++++++ .../bert-oneflow/dataset/vocab.py | 186 ++++++++++++++++++ .../VS002/oneflow.INFO | 1 + ....lichunyou.log.INFO.20210615-153653.216106 | 40 ++++ ....lichunyou.log.INFO.20210615-153942.217036 | 19 ++ ....lichunyou.log.INFO.20210615-155552.219680 | 40 ++++ ....lichunyou.log.INFO.20210615-160705.221626 | 19 ++ ....lichunyou.log.INFO.20210615-160849.222566 | 19 ++ ....lichunyou.log.INFO.20210615-161144.223722 | 40 ++++ ....lichunyou.log.INFO.20210615-161246.224493 | 19 ++ ....lichunyou.log.INFO.20210615-162501.226843 | 19 ++ ...2.lichunyou.log.INFO.20210616-121348.68726 | 40 ++++ ...2.lichunyou.log.INFO.20210616-121504.69423 | 40 ++++ ...2.lichunyou.log.INFO.20210616-121548.70085 | 40 ++++ ...2.lichunyou.log.INFO.20210616-143323.80849 | 40 ++++ ...2.lichunyou.log.INFO.20210616-143407.81483 | 40 ++++ ...2.lichunyou.log.INFO.20210616-143528.82120 | 19 ++ ...2.lichunyou.log.INFO.20210616-144741.83066 | 40 ++++ ...2.lichunyou.log.INFO.20210616-144906.83746 | 19 ++ ...2.lichunyou.log.INFO.20210616-145207.84540 | 19 ++ ...2.lichunyou.log.INFO.20210616-150059.85434 | 40 ++++ ...2.lichunyou.log.INFO.20210616-155156.93096 | 19 ++ ...2.lichunyou.log.INFO.20210617-132416.22441 | 19 ++ ...2.lichunyou.log.INFO.20210617-132529.23158 | 19 ++ ...2.lichunyou.log.INFO.20210621-105503.13951 | 40 ++++ ....lichunyou.log.INFO.20210621-124230.196962 | 19 ++ ....lichunyou.log.INFO.20210621-124337.198163 | 19 ++ ....lichunyou.log.INFO.20210621-130530.205914 | 19 ++ ....lichunyou.log.INFO.20210621-140937.227330 | 19 ++ ...02.lichunyou.log.INFO.20210621-141631.1046 | 19 ++ ...02.lichunyou.log.INFO.20210621-141811.2328 | 19 ++ LanguageModeling/bert-oneflow/main.py | 83 ++++++++ .../model/attention/multi_head.py | 40 ++++ .../bert-oneflow/model/attention/single.py | 29 +++ LanguageModeling/bert-oneflow/model/bert.py | 50 +++++ .../bert-oneflow/model/embedding/bert.py | 68 +++++++ .../bert-oneflow/model/language_model.py | 62 ++++++ .../bert-oneflow/model/transformer.py | 32 +++ .../bert-oneflow/model/utils/feed_forward.py | 19 ++ .../bert-oneflow/model/utils/gelu.py | 14 ++ .../bert-oneflow/model/utils/layer_norm.py | 19 ++ .../bert-oneflow/model/utils/sublayer.py | 19 ++ .../bert-oneflow/requirements.txt | 3 + LanguageModeling/bert-oneflow/test.py | 73 +++++++ LanguageModeling/bert-oneflow/test.sh | 1 + LanguageModeling/bert-oneflow/train.py | 79 ++++++++ LanguageModeling/bert-oneflow/train.sh | 1 + .../bert-oneflow/trainer/optim_schedule.py | 35 ++++ .../bert-oneflow/trainer/pretrain.py | 168 ++++++++++++++++ 50 files changed, 1895 insertions(+) create mode 100644 LanguageModeling/bert-oneflow/README.md create mode 100644 LanguageModeling/bert-oneflow/dataset/dataset.py create mode 100644 LanguageModeling/bert-oneflow/dataset/vocab.py create mode 120000 LanguageModeling/bert-oneflow/log/default_physical_env_log/VS002/oneflow.INFO create mode 100644 LanguageModeling/bert-oneflow/log/default_physical_env_log/VS002/oneflow.VS002.lichunyou.log.INFO.20210615-153653.216106 create mode 100644 LanguageModeling/bert-oneflow/log/default_physical_env_log/VS002/oneflow.VS002.lichunyou.log.INFO.20210615-153942.217036 create mode 100644 LanguageModeling/bert-oneflow/log/default_physical_env_log/VS002/oneflow.VS002.lichunyou.log.INFO.20210615-155552.219680 create mode 100644 LanguageModeling/bert-oneflow/log/default_physical_env_log/VS002/oneflow.VS002.lichunyou.log.INFO.20210615-160705.221626 create mode 100644 LanguageModeling/bert-oneflow/log/default_physical_env_log/VS002/oneflow.VS002.lichunyou.log.INFO.20210615-160849.222566 create mode 100644 LanguageModeling/bert-oneflow/log/default_physical_env_log/VS002/oneflow.VS002.lichunyou.log.INFO.20210615-161144.223722 create mode 100644 LanguageModeling/bert-oneflow/log/default_physical_env_log/VS002/oneflow.VS002.lichunyou.log.INFO.20210615-161246.224493 create mode 100644 LanguageModeling/bert-oneflow/log/default_physical_env_log/VS002/oneflow.VS002.lichunyou.log.INFO.20210615-162501.226843 create mode 100644 LanguageModeling/bert-oneflow/log/default_physical_env_log/VS002/oneflow.VS002.lichunyou.log.INFO.20210616-121348.68726 create mode 100644 LanguageModeling/bert-oneflow/log/default_physical_env_log/VS002/oneflow.VS002.lichunyou.log.INFO.20210616-121504.69423 create mode 100644 LanguageModeling/bert-oneflow/log/default_physical_env_log/VS002/oneflow.VS002.lichunyou.log.INFO.20210616-121548.70085 create mode 100644 LanguageModeling/bert-oneflow/log/default_physical_env_log/VS002/oneflow.VS002.lichunyou.log.INFO.20210616-143323.80849 create mode 100644 LanguageModeling/bert-oneflow/log/default_physical_env_log/VS002/oneflow.VS002.lichunyou.log.INFO.20210616-143407.81483 create mode 100644 LanguageModeling/bert-oneflow/log/default_physical_env_log/VS002/oneflow.VS002.lichunyou.log.INFO.20210616-143528.82120 create mode 100644 LanguageModeling/bert-oneflow/log/default_physical_env_log/VS002/oneflow.VS002.lichunyou.log.INFO.20210616-144741.83066 create mode 100644 LanguageModeling/bert-oneflow/log/default_physical_env_log/VS002/oneflow.VS002.lichunyou.log.INFO.20210616-144906.83746 create mode 100644 LanguageModeling/bert-oneflow/log/default_physical_env_log/VS002/oneflow.VS002.lichunyou.log.INFO.20210616-145207.84540 create mode 100644 LanguageModeling/bert-oneflow/log/default_physical_env_log/VS002/oneflow.VS002.lichunyou.log.INFO.20210616-150059.85434 create mode 100644 LanguageModeling/bert-oneflow/log/default_physical_env_log/VS002/oneflow.VS002.lichunyou.log.INFO.20210616-155156.93096 create mode 100644 LanguageModeling/bert-oneflow/log/default_physical_env_log/VS002/oneflow.VS002.lichunyou.log.INFO.20210617-132416.22441 create mode 100644 LanguageModeling/bert-oneflow/log/default_physical_env_log/VS002/oneflow.VS002.lichunyou.log.INFO.20210617-132529.23158 create mode 100644 LanguageModeling/bert-oneflow/log/default_physical_env_log/VS002/oneflow.VS002.lichunyou.log.INFO.20210621-105503.13951 create mode 100644 LanguageModeling/bert-oneflow/log/default_physical_env_log/VS002/oneflow.VS002.lichunyou.log.INFO.20210621-124230.196962 create mode 100644 LanguageModeling/bert-oneflow/log/default_physical_env_log/VS002/oneflow.VS002.lichunyou.log.INFO.20210621-124337.198163 create mode 100644 LanguageModeling/bert-oneflow/log/default_physical_env_log/VS002/oneflow.VS002.lichunyou.log.INFO.20210621-130530.205914 create mode 100644 LanguageModeling/bert-oneflow/log/default_physical_env_log/VS002/oneflow.VS002.lichunyou.log.INFO.20210621-140937.227330 create mode 100644 LanguageModeling/bert-oneflow/log/default_physical_env_log/VS002/oneflow.VS002.lichunyou.log.INFO.20210621-141631.1046 create mode 100644 LanguageModeling/bert-oneflow/log/default_physical_env_log/VS002/oneflow.VS002.lichunyou.log.INFO.20210621-141811.2328 create mode 100644 LanguageModeling/bert-oneflow/main.py create mode 100644 LanguageModeling/bert-oneflow/model/attention/multi_head.py create mode 100644 LanguageModeling/bert-oneflow/model/attention/single.py create mode 100644 LanguageModeling/bert-oneflow/model/bert.py create mode 100644 LanguageModeling/bert-oneflow/model/embedding/bert.py create mode 100644 LanguageModeling/bert-oneflow/model/language_model.py create mode 100644 LanguageModeling/bert-oneflow/model/transformer.py create mode 100644 LanguageModeling/bert-oneflow/model/utils/feed_forward.py create mode 100644 LanguageModeling/bert-oneflow/model/utils/gelu.py create mode 100644 LanguageModeling/bert-oneflow/model/utils/layer_norm.py create mode 100644 LanguageModeling/bert-oneflow/model/utils/sublayer.py create mode 100644 LanguageModeling/bert-oneflow/requirements.txt create mode 100644 LanguageModeling/bert-oneflow/test.py create mode 100644 LanguageModeling/bert-oneflow/test.sh create mode 100644 LanguageModeling/bert-oneflow/train.py create mode 100644 LanguageModeling/bert-oneflow/train.sh create mode 100644 LanguageModeling/bert-oneflow/trainer/optim_schedule.py create mode 100644 LanguageModeling/bert-oneflow/trainer/pretrain.py diff --git a/LanguageModeling/bert-oneflow/README.md b/LanguageModeling/bert-oneflow/README.md new file mode 100644 index 0000000..f2b23ae --- /dev/null +++ b/LanguageModeling/bert-oneflow/README.md @@ -0,0 +1,23 @@ +Oneflow-Module版BERT实现，origin form:https://github.com/codertimo/BERT-pytorch + +### dataset + +示例数据集下载：[data.zip](https://oneflow-public.oss-cn-beijing.aliyuncs.com/datasets/BERT-pytorch/sampledataset/data.zip) 并解压至本文件夹根目录 + +### requirements + +1.oneflow版本直接用master分支上的oneflow即可：https://github.com/Oneflow-Inc/oneflow/ + +2.同时，由于数据集加载部分的功能尚未对齐，需要依赖torch.utils.data.DataLoader，故依赖torch，需安装torch（版本不限）. + +3.需手动将bert_pytorch.zip解压并复制到site-packages下，如： + +`cp -r bert_pytorch ~/anaconda3/envs/oneflow/lib/python3.7/site-packages/` + +### demo + + +- train: `bash train.sh` +- test: `bash test.sh` + + diff --git a/LanguageModeling/bert-oneflow/dataset/dataset.py b/LanguageModeling/bert-oneflow/dataset/dataset.py new file mode 100644 index 0000000..597d4a6 --- /dev/null +++ b/LanguageModeling/bert-oneflow/dataset/dataset.py @@ -0,0 +1,127 @@ +# from oneflow.utils.data import Dataset +import tqdm +import oneflow as flow +import torch +import oneflow.nn as nn +import random + + +class BERTDataset(nn.Module): + def __init__(self, corpus_path, vocab, seq_len, encoding="utf-8", corpus_lines=None, on_memory=True): + self.vocab = vocab + self.seq_len = seq_len + + self.on_memory = on_memory + self.corpus_lines = corpus_lines + self.corpus_path = corpus_path + self.encoding = encoding + + with open(corpus_path, "r", encoding=encoding) as f: + if self.corpus_lines is None and not on_memory: + for _ in tqdm.tqdm(f, desc="Loading Dataset", total=corpus_lines): + self.corpus_lines += 1 + + if on_memory: + self.lines = [line[:-1].split("\t") + for line in tqdm.tqdm(f, desc="Loading Dataset", total=corpus_lines)] + self.corpus_lines = len(self.lines) + + if not on_memory: + self.file = open(corpus_path, "r", encoding=encoding) + self.random_file = open(corpus_path, "r", encoding=encoding) + + for _ in range(random.randint(self.corpus_lines if self.corpus_lines < 1000 else 1000)): + self.random_file.__next__() + + def __len__(self): + return self.corpus_lines + + def __getitem__(self, item): + t1, t2, is_next_label = self.random_sent(item) + t1_random, t1_label = self.random_word(t1) + t2_random, t2_label = self.random_word(t2) + + # [CLS] tag = SOS tag, [SEP] tag = EOS tag + t1 = [self.vocab.sos_index] + t1_random + [self.vocab.eos_index] + t2 = t2_random + [self.vocab.eos_index] + + t1_label = [self.vocab.pad_index] + t1_label + [self.vocab.pad_index] + t2_label = t2_label + [self.vocab.pad_index] + + segment_label = ([1 for _ in range(len(t1))] + [2 for _ in range(len(t2))])[:self.seq_len] + bert_input = (t1 + t2)[:self.seq_len] + bert_label = (t1_label + t2_label)[:self.seq_len] + + padding = [self.vocab.pad_index for _ in range(self.seq_len - len(bert_input))] + bert_input.extend(padding), bert_label.extend(padding), segment_label.extend(padding) + + output = {"bert_input": bert_input, + "bert_label": bert_label, + "segment_label": segment_label, + "is_next": is_next_label} + + return {key: torch.tensor(value) for key, value in output.items()} + + def random_word(self, sentence): + tokens = sentence.split() + output_label = [] + + for i, token in enumerate(tokens): + prob = random.random() + if prob < 0.15: + prob /= 0.15 + + # 80% randomly change token to mask token + if prob < 0.8: + tokens[i] = self.vocab.mask_index + + # 10% randomly change token to random token + elif prob < 0.9: + tokens[i] = random.randrange(len(self.vocab)) + + # 10% randomly change token to current token + else: + tokens[i] = self.vocab.stoi.get(token, self.vocab.unk_index) + + output_label.append(self.vocab.stoi.get(token, self.vocab.unk_index)) + + else: + tokens[i] = self.vocab.stoi.get(token, self.vocab.unk_index) + output_label.append(0) + + return tokens, output_label + + def random_sent(self, index): + t1, t2 = self.get_corpus_line(index) + + # output_text, label(isNotNext:0, isNext:1) + if random.random() > 0.5: + return t1, t2, 1 + else: + return t1, self.get_random_line(), 0 + + def get_corpus_line(self, item): + if self.on_memory: + return self.lines[item][0], self.lines[item][1] + else: + line = self.file.__next__() + if line is None: + self.file.close() + self.file = open(self.corpus_path, "r", encoding=self.encoding) + line = self.file.__next__() + + t1, t2 = line[:-1].split("\t") + return t1, t2 + + def get_random_line(self): + if self.on_memory: + return self.lines[random.randrange(len(self.lines))][1] + + line = self.file.__next__() + if line is None: + self.file.close() + self.file = open(self.corpus_path, "r", encoding=self.encoding) + for _ in range(random.randint(self.corpus_lines if self.corpus_lines < 1000 else 1000)): + self.random_file.__next__() + line = self.random_file.__next__() + return line[:-1].split("\t")[1] diff --git a/LanguageModeling/bert-oneflow/dataset/vocab.py b/LanguageModeling/bert-oneflow/dataset/vocab.py new file mode 100644 index 0000000..eb61d72 --- /dev/null +++ b/LanguageModeling/bert-oneflow/dataset/vocab.py @@ -0,0 +1,186 @@ +import pickle +import tqdm +from collections import Counter + + +class FlowVocab(object): + """Defines a vocabulary object that will be used to numericalize a field. + Attributes: + freqs: A collections.Counter object holding the frequencies of tokens + in the data used to build the Vocab. + stoi: A collections.defaultdict instance mapping token strings to + numerical identifiers. + itos: A list of token strings indexed by their numerical identifiers. + """ + + def __init__(self, counter, max_size=None, min_freq=1, specials=['', ''], + vectors=None, unk_init=None, vectors_cache=None): + """Create a Vocab object from a collections.Counter. + Arguments: + counter: collections.Counter object holding the frequencies of + each value found in the data. + max_size: The maximum size of the vocabulary, or None for no + maximum. Default: None. + min_freq: The minimum frequency needed to include a token in the + vocabulary. Values less than 1 will be set to 1. Default: 1. + specials: The list of special tokens (e.g., padding or eos) that + will be prepended to the vocabulary in addition to an + token. Default: [''] + vectors: One of either the available pretrained vectors + or custom pretrained vectors (see Vocab.load_vectors); + or a list of aforementioned vectors + unk_init (callback): by default, initialize out-of-vocabulary word vectors + to zero vectors; can be any function that takes in a Tensor and + returns a Tensor of the same size. Default: torch.Tensor.zero_ + vectors_cache: directory for cached vectors. Default: '.vector_cache' + """ + self.freqs = counter + counter = counter.copy() + min_freq = max(min_freq, 1) + + self.itos = list(specials) + # frequencies of special tokens are not counted when building vocabulary + # in frequency order + for tok in specials: + del counter[tok] + + max_size = None if max_size is None else max_size + len(self.itos) + + # sort by frequency, then alphabetically + words_and_frequencies = sorted(counter.items(), key=lambda tup: tup[0]) + words_and_frequencies.sort(key=lambda tup: tup[1], reverse=True) + + for word, freq in words_and_frequencies: + if freq < min_freq or len(self.itos) == max_size: + break + self.itos.append(word) + + # stoi is simply a reverse dict for itos + self.stoi = {tok: i for i, tok in enumerate(self.itos)} + + self.vectors = None + if vectors is not None: + self.load_vectors(vectors, unk_init=unk_init, cache=vectors_cache) + else: + assert unk_init is None and vectors_cache is None + + def __eq__(self, other): + if self.freqs != other.freqs: + return False + if self.stoi != other.stoi: + return False + if self.itos != other.itos: + return False + if self.vectors != other.vectors: + return False + return True + + def __len__(self): + return len(self.itos) + + def vocab_rerank(self): + self.stoi = {word: i for i, word in enumerate(self.itos)} + + def extend(self, v, sort=False): + words = sorted(v.itos) if sort else v.itos + for w in words: + if w not in self.stoi: + self.itos.append(w) + self.stoi[w] = len(self.itos) - 1 + + +class Vocab(FlowVocab): + def __init__(self, counter, max_size=None, min_freq=1): + self.pad_index = 0 + self.unk_index = 1 + self.eos_index = 2 + self.sos_index = 3 + self.mask_index = 4 + super().__init__(counter, specials=["", "", "", "", ""], + max_size=max_size, min_freq=min_freq) + + def to_seq(self, sentece, seq_len, with_eos=False, with_sos=False) -> list: + pass + + def from_seq(self, seq, join=False, with_pad=False): + pass + + @staticmethod + def load_vocab(vocab_path: str) -> 'Vocab': + with open(vocab_path, "rb") as f: + return pickle.load(f) + + def save_vocab(self, vocab_path): + with open(vocab_path, "wb") as f: + pickle.dump(self, f) + + +# Building Vocab with text files +class WordVocab(Vocab): + def __init__(self, texts, max_size=None, min_freq=1): + print("Building Vocab") + counter = Counter() + for line in tqdm.tqdm(texts): + if isinstance(line, list): + words = line + else: + words = line.replace("\n", "").replace("\t", "").split() + + for word in words: + counter[word] += 1 + super().__init__(counter, max_size=max_size, min_freq=min_freq) + + def to_seq(self, sentence, seq_len=None, with_eos=False, with_sos=False, with_len=False): + if isinstance(sentence, str): + sentence = sentence.split() + + seq = [self.stoi.get(word, self.unk_index) for word in sentence] + + if with_eos: + seq += [self.eos_index] # this would be index 1 + if with_sos: + seq = [self.sos_index] + seq + + origin_seq_len = len(seq) + + if seq_len is None: + pass + elif len(seq) <= seq_len: + seq += [self.pad_index for _ in range(seq_len - len(seq))] + else: + seq = seq[:seq_len] + + return (seq, origin_seq_len) if with_len else seq + + def from_seq(self, seq, join=False, with_pad=False): + words = [self.itos[idx] + if idx < len(self.itos) + else "<%d>" % idx + for idx in seq + if not with_pad or idx != self.pad_index] + + return " ".join(words) if join else words + + @staticmethod + def load_vocab(vocab_path: str) -> 'WordVocab': + with open(vocab_path, "rb") as f: + print("vocab_path >>>>>>>>>>>>>>>>>>>>>> ", vocab_path) + return pickle.load(f) + + +def build(): + import argparse + + parser = argparse.ArgumentParser() + parser.add_argument("-c", "--corpus_path", required=True, type=str) + parser.add_argument("-o", "--output_path", required=True, type=str) + parser.add_argument("-s", "--vocab_size", type=int, default=None) + parser.add_argument("-e", "--encoding", type=str, default="utf-8") + parser.add_argument("-m", "--min_freq", type=int, default=1) + args = parser.parse_args() + + with open(args.corpus_path, "r", encoding=args.encoding) as f: + vocab = WordVocab(f, max_size=args.vocab_size, min_freq=args.min_freq) + + print("VOCAB SIZE:", len(vocab)) + vocab.save_vocab(args.output_path) diff --git a/LanguageModeling/bert-oneflow/log/default_physical_env_log/VS002/oneflow.INFO b/LanguageModeling/bert-oneflow/log/default_physical_env_log/VS002/oneflow.INFO new file mode 120000 index 0000000..06d2785 --- /dev/null +++ b/LanguageModeling/bert-oneflow/log/default_physical_env_log/VS002/oneflow.INFO @@ -0,0 +1 @@ +oneflow.VS002.lichunyou.log.INFO.20210621-141811.2328 \ No newline at end of file diff --git a/LanguageModeling/bert-oneflow/log/default_physical_env_log/VS002/oneflow.VS002.lichunyou.log.INFO.20210615-153653.216106 b/LanguageModeling/bert-oneflow/log/default_physical_env_log/VS002/oneflow.VS002.lichunyou.log.INFO.20210615-153653.216106 new file mode 100644 index 0000000..678624d --- /dev/null +++ b/LanguageModeling/bert-oneflow/log/default_physical_env_log/VS002/oneflow.VS002.lichunyou.log.INFO.20210615-153653.216106 @@ -0,0 +1,40 @@ +Log file created at: 2021/06/15 15:36:53 +Running on machine: VS002 +Log line format: [IWEF]mmdd hh:mm:ss.uuuuuu threadid file:line] msg +I0615 15:36:53.881130 216106 global.h:36] NewGlobal 14cudaDeviceProp +I0615 15:36:53.946734 216106 global.h:36] NewGlobal N7oneflow7EnvDescE +I0615 15:36:53.946768 216106 global.h:36] NewGlobal N7oneflow10ProcessCtxE +I0615 15:36:53.946776 216106 env_global_objects_scope.cpp:112] using rpc backend: local +I0615 15:36:53.946787 216106 global.h:36] NewGlobal N7oneflow12ResourceDescE +I0615 15:36:53.946794 216106 global.h:36] NewGlobal N7oneflow12ResourceDescE +I0615 15:36:53.953372 216106 global.h:36] NewGlobal N7oneflow10ThreadPoolE +I0615 15:36:53.954262 216106 global.h:36] NewGlobal N7oneflow16EagerNcclCommMgrE +I0615 15:36:53.954270 216106 global.h:36] NewGlobal N7oneflow18CudnnConvAlgoCacheE +I0615 15:36:53.954275 216106 global.h:36] NewGlobal N7oneflow2vm19VirtualMachineScopeE +I0615 15:36:53.954282 216106 global.h:36] NewGlobal N7oneflow9OneflowVME +I0615 15:36:53.962713 216106 global.h:36] NewGlobal N7oneflow27EagerJobBuildAndInferCtxMgrE +I0615 15:36:53.962735 216106 global.h:36] NewGlobal N7oneflow12EpollCommNetE +I0615 15:36:53.962810 216106 epoll_comm_network.cpp:63] CommNet:Epoll listening on 0.0.0.0:42334 +I0615 15:36:53.962831 216106 epoll_comm_network.cpp:198] machine 0 sockfd -1 +I0615 15:36:53.962906 216106 global.h:36] NewGlobal N7oneflow9TransportE +I0615 15:36:54.047060 216106 global.h:43] DeleteGlobal N7oneflow21EnvGlobalObjectsScopeE +I0615 15:36:54.047080 216106 global.h:43] DeleteGlobal N7oneflow9TransportE +I0615 15:36:54.047129 216106 global.h:43] DeleteGlobal N7oneflow12EpollCommNetE +I0615 15:36:54.047137 216106 epoll_comm_network.cpp:87] CommNet Thread 0 finish +I0615 15:36:54.047158 216106 epoll_comm_network.cpp:87] CommNet Thread 1 finish +I0615 15:36:54.047178 216106 epoll_comm_network.cpp:87] CommNet Thread 2 finish +I0615 15:36:54.047197 216106 epoll_comm_network.cpp:87] CommNet Thread 3 finish +I0615 15:36:54.047350 216106 global.h:43] DeleteGlobal N7oneflow27EagerJobBuildAndInferCtxMgrE +I0615 15:36:54.047358 216106 global.h:43] DeleteGlobal N7oneflow2vm19VirtualMachineScopeE +I0615 15:36:54.047363 216106 global.h:43] DeleteGlobal N7oneflow9OneflowVME +I0615 15:36:54.052028 216106 global.h:43] DeleteGlobal N7oneflow18CudnnConvAlgoCacheE +I0615 15:36:54.052049 216106 global.h:43] DeleteGlobal N7oneflow16EagerNcclCommMgrE +I0615 15:36:54.052057 216106 global.h:43] DeleteGlobal N7oneflow10ThreadPoolE +I0615 15:36:54.053153 216106 global.h:43] DeleteGlobal N7oneflow12ResourceDescE +I0615 15:36:54.053164 216106 global.h:43] DeleteGlobal N7oneflow12ResourceDescE +I0615 15:36:54.053169 216106 global.h:43] DeleteGlobal N7oneflow6device27NodeDeviceDescriptorManagerE +I0615 15:36:54.053181 216106 global.h:43] DeleteGlobal N7oneflow10RpcManagerE +I0615 15:36:54.053187 216106 global.h:43] DeleteGlobal N7oneflow10CtrlClientE +I0615 15:36:54.053194 216106 global.h:43] DeleteGlobal N7oneflow10ProcessCtxE +I0615 15:36:54.053201 216106 global.h:43] DeleteGlobal N7oneflow7EnvDescE +I0615 15:36:54.053208 216106 global.h:43] DeleteGlobal 14cudaDeviceProp diff --git a/LanguageModeling/bert-oneflow/log/default_physical_env_log/VS002/oneflow.VS002.lichunyou.log.INFO.20210615-153942.217036 b/LanguageModeling/bert-oneflow/log/default_physical_env_log/VS002/oneflow.VS002.lichunyou.log.INFO.20210615-153942.217036 new file mode 100644 index 0000000..d700f4b --- /dev/null +++ b/LanguageModeling/bert-oneflow/log/default_physical_env_log/VS002/oneflow.VS002.lichunyou.log.INFO.20210615-153942.217036 @@ -0,0 +1,19 @@ +Log file created at: 2021/06/15 15:39:42 +Running on machine: VS002 +Log line format: [IWEF]mmdd hh:mm:ss.uuuuuu threadid file:line] msg +I0615 15:39:42.711254 217036 global.h:36] NewGlobal 14cudaDeviceProp +I0615 15:39:42.772075 217036 global.h:36] NewGlobal N7oneflow7EnvDescE +I0615 15:39:42.772105 217036 global.h:36] NewGlobal N7oneflow10ProcessCtxE +I0615 15:39:42.772114 217036 env_global_objects_scope.cpp:112] using rpc backend: local +I0615 15:39:42.772123 217036 global.h:36] NewGlobal N7oneflow12ResourceDescE +I0615 15:39:42.772130 217036 global.h:36] NewGlobal N7oneflow12ResourceDescE +I0615 15:39:42.778702 217036 global.h:36] NewGlobal N7oneflow10ThreadPoolE +I0615 15:39:42.779572 217036 global.h:36] NewGlobal N7oneflow16EagerNcclCommMgrE +I0615 15:39:42.779580 217036 global.h:36] NewGlobal N7oneflow18CudnnConvAlgoCacheE +I0615 15:39:42.779584 217036 global.h:36] NewGlobal N7oneflow2vm19VirtualMachineScopeE +I0615 15:39:42.779589 217036 global.h:36] NewGlobal N7oneflow9OneflowVME +I0615 15:39:42.787849 217036 global.h:36] NewGlobal N7oneflow27EagerJobBuildAndInferCtxMgrE +I0615 15:39:42.787869 217036 global.h:36] NewGlobal N7oneflow12EpollCommNetE +I0615 15:39:42.787940 217036 epoll_comm_network.cpp:63] CommNet:Epoll listening on 0.0.0.0:40496 +I0615 15:39:42.787958 217036 epoll_comm_network.cpp:198] machine 0 sockfd -1 +I0615 15:39:42.788033 217036 global.h:36] NewGlobal N7oneflow9TransportE diff --git a/LanguageModeling/bert-oneflow/log/default_physical_env_log/VS002/oneflow.VS002.lichunyou.log.INFO.20210615-155552.219680 b/LanguageModeling/bert-oneflow/log/default_physical_env_log/VS002/oneflow.VS002.lichunyou.log.INFO.20210615-155552.219680 new file mode 100644 index 0000000..81cb4aa --- /dev/null +++ b/LanguageModeling/bert-oneflow/log/default_physical_env_log/VS002/oneflow.VS002.lichunyou.log.INFO.20210615-155552.219680 @@ -0,0 +1,40 @@ +Log file created at: 2021/06/15 15:55:52 +Running on machine: VS002 +Log line format: [IWEF]mmdd hh:mm:ss.uuuuuu threadid file:line] msg +I0615 15:55:52.643743 219680 global.h:36] NewGlobal 14cudaDeviceProp +I0615 15:55:52.703361 219680 global.h:36] NewGlobal N7oneflow7EnvDescE +I0615 15:55:52.703390 219680 global.h:36] NewGlobal N7oneflow10ProcessCtxE +I0615 15:55:52.703397 219680 env_global_objects_scope.cpp:112] using rpc backend: local +I0615 15:55:52.703408 219680 global.h:36] NewGlobal N7oneflow12ResourceDescE +I0615 15:55:52.703413 219680 global.h:36] NewGlobal N7oneflow12ResourceDescE +I0615 15:55:52.709794 219680 global.h:36] NewGlobal N7oneflow10ThreadPoolE +I0615 15:55:52.710682 219680 global.h:36] NewGlobal N7oneflow16EagerNcclCommMgrE +I0615 15:55:52.710691 219680 global.h:36] NewGlobal N7oneflow18CudnnConvAlgoCacheE +I0615 15:55:52.710695 219680 global.h:36] NewGlobal N7oneflow2vm19VirtualMachineScopeE +I0615 15:55:52.710700 219680 global.h:36] NewGlobal N7oneflow9OneflowVME +I0615 15:55:52.718988 219680 global.h:36] NewGlobal N7oneflow27EagerJobBuildAndInferCtxMgrE +I0615 15:55:52.719008 219680 global.h:36] NewGlobal N7oneflow12EpollCommNetE +I0615 15:55:52.719079 219680 epoll_comm_network.cpp:63] CommNet:Epoll listening on 0.0.0.0:35038 +I0615 15:55:52.719099 219680 epoll_comm_network.cpp:198] machine 0 sockfd -1 +I0615 15:55:52.719175 219680 global.h:36] NewGlobal N7oneflow9TransportE +I0615 15:58:40.093083 219680 global.h:43] DeleteGlobal N7oneflow21EnvGlobalObjectsScopeE +I0615 15:58:40.093164 219680 global.h:43] DeleteGlobal N7oneflow9TransportE +I0615 15:58:40.093287 219680 global.h:43] DeleteGlobal N7oneflow12EpollCommNetE +I0615 15:58:40.093299 219680 epoll_comm_network.cpp:87] CommNet Thread 0 finish +I0615 15:58:40.093397 219680 epoll_comm_network.cpp:87] CommNet Thread 1 finish +I0615 15:58:40.093466 219680 epoll_comm_network.cpp:87] CommNet Thread 2 finish +I0615 15:58:40.093623 219680 epoll_comm_network.cpp:87] CommNet Thread 3 finish +I0615 15:58:40.093811 219680 global.h:43] DeleteGlobal N7oneflow27EagerJobBuildAndInferCtxMgrE +I0615 15:58:40.093822 219680 global.h:43] DeleteGlobal N7oneflow2vm19VirtualMachineScopeE +I0615 15:58:40.093827 219680 global.h:43] DeleteGlobal N7oneflow9OneflowVME +I0615 15:58:40.123791 219680 global.h:43] DeleteGlobal N7oneflow18CudnnConvAlgoCacheE +I0615 15:58:40.123827 219680 global.h:43] DeleteGlobal N7oneflow16EagerNcclCommMgrE +I0615 15:58:40.123838 219680 global.h:43] DeleteGlobal N7oneflow10ThreadPoolE +I0615 15:58:40.126832 219680 global.h:43] DeleteGlobal N7oneflow12ResourceDescE +I0615 15:58:40.126843 219680 global.h:43] DeleteGlobal N7oneflow12ResourceDescE +I0615 15:58:40.126853 219680 global.h:43] DeleteGlobal N7oneflow6device27NodeDeviceDescriptorManagerE +I0615 15:58:40.126874 219680 global.h:43] DeleteGlobal N7oneflow10RpcManagerE +I0615 15:58:40.126884 219680 global.h:43] DeleteGlobal N7oneflow10CtrlClientE +I0615 15:58:40.126891 219680 global.h:43] DeleteGlobal N7oneflow10ProcessCtxE +I0615 15:58:40.126904 219680 global.h:43] DeleteGlobal N7oneflow7EnvDescE +I0615 15:58:40.126916 219680 global.h:43] DeleteGlobal 14cudaDeviceProp diff --git a/LanguageModeling/bert-oneflow/log/default_physical_env_log/VS002/oneflow.VS002.lichunyou.log.INFO.20210615-160705.221626 b/LanguageModeling/bert-oneflow/log/default_physical_env_log/VS002/oneflow.VS002.lichunyou.log.INFO.20210615-160705.221626 new file mode 100644 index 0000000..525ca3c --- /dev/null +++ b/LanguageModeling/bert-oneflow/log/default_physical_env_log/VS002/oneflow.VS002.lichunyou.log.INFO.20210615-160705.221626 @@ -0,0 +1,19 @@ +Log file created at: 2021/06/15 16:07:05 +Running on machine: VS002 +Log line format: [IWEF]mmdd hh:mm:ss.uuuuuu threadid file:line] msg +I0615 16:07:05.230154 221626 global.h:36] NewGlobal 14cudaDeviceProp +I0615 16:07:05.301681 221626 global.h:36] NewGlobal N7oneflow7EnvDescE +I0615 16:07:05.301717 221626 global.h:36] NewGlobal N7oneflow10ProcessCtxE +I0615 16:07:05.301725 221626 env_global_objects_scope.cpp:112] using rpc backend: local +I0615 16:07:05.301738 221626 global.h:36] NewGlobal N7oneflow12ResourceDescE +I0615 16:07:05.301744 221626 global.h:36] NewGlobal N7oneflow12ResourceDescE +I0615 16:07:05.309193 221626 global.h:36] NewGlobal N7oneflow10ThreadPoolE +I0615 16:07:05.310214 221626 global.h:36] NewGlobal N7oneflow16EagerNcclCommMgrE +I0615 16:07:05.310225 221626 global.h:36] NewGlobal N7oneflow18CudnnConvAlgoCacheE +I0615 16:07:05.310235 221626 global.h:36] NewGlobal N7oneflow2vm19VirtualMachineScopeE +I0615 16:07:05.310242 221626 global.h:36] NewGlobal N7oneflow9OneflowVME +I0615 16:07:05.319964 221626 global.h:36] NewGlobal N7oneflow27EagerJobBuildAndInferCtxMgrE +I0615 16:07:05.319988 221626 global.h:36] NewGlobal N7oneflow12EpollCommNetE +I0615 16:07:05.320072 221626 epoll_comm_network.cpp:63] CommNet:Epoll listening on 0.0.0.0:33728 +I0615 16:07:05.320096 221626 epoll_comm_network.cpp:198] machine 0 sockfd -1 +I0615 16:07:05.320184 221626 global.h:36] NewGlobal N7oneflow9TransportE diff --git a/LanguageModeling/bert-oneflow/log/default_physical_env_log/VS002/oneflow.VS002.lichunyou.log.INFO.20210615-160849.222566 b/LanguageModeling/bert-oneflow/log/default_physical_env_log/VS002/oneflow.VS002.lichunyou.log.INFO.20210615-160849.222566 new file mode 100644 index 0000000..a5d9977 --- /dev/null +++ b/LanguageModeling/bert-oneflow/log/default_physical_env_log/VS002/oneflow.VS002.lichunyou.log.INFO.20210615-160849.222566 @@ -0,0 +1,19 @@ +Log file created at: 2021/06/15 16:08:49 +Running on machine: VS002 +Log line format: [IWEF]mmdd hh:mm:ss.uuuuuu threadid file:line] msg +I0615 16:08:49.703088 222566 global.h:36] NewGlobal 14cudaDeviceProp +I0615 16:08:49.766950 222566 global.h:36] NewGlobal N7oneflow7EnvDescE +I0615 16:08:49.766980 222566 global.h:36] NewGlobal N7oneflow10ProcessCtxE +I0615 16:08:49.766988 222566 env_global_objects_scope.cpp:112] using rpc backend: local +I0615 16:08:49.766997 222566 global.h:36] NewGlobal N7oneflow12ResourceDescE +I0615 16:08:49.767004 222566 global.h:36] NewGlobal N7oneflow12ResourceDescE +I0615 16:08:49.773383 222566 global.h:36] NewGlobal N7oneflow10ThreadPoolE +I0615 16:08:49.774260 222566 global.h:36] NewGlobal N7oneflow16EagerNcclCommMgrE +I0615 16:08:49.774269 222566 global.h:36] NewGlobal N7oneflow18CudnnConvAlgoCacheE +I0615 16:08:49.774273 222566 global.h:36] NewGlobal N7oneflow2vm19VirtualMachineScopeE +I0615 16:08:49.774279 222566 global.h:36] NewGlobal N7oneflow9OneflowVME +I0615 16:08:49.782550 222566 global.h:36] NewGlobal N7oneflow27EagerJobBuildAndInferCtxMgrE +I0615 16:08:49.782570 222566 global.h:36] NewGlobal N7oneflow12EpollCommNetE +I0615 16:08:49.782644 222566 epoll_comm_network.cpp:63] CommNet:Epoll listening on 0.0.0.0:44700 +I0615 16:08:49.782673 222566 epoll_comm_network.cpp:198] machine 0 sockfd -1 +I0615 16:08:49.782743 222566 global.h:36] NewGlobal N7oneflow9TransportE diff --git a/LanguageModeling/bert-oneflow/log/default_physical_env_log/VS002/oneflow.VS002.lichunyou.log.INFO.20210615-161144.223722 b/LanguageModeling/bert-oneflow/log/default_physical_env_log/VS002/oneflow.VS002.lichunyou.log.INFO.20210615-161144.223722 new file mode 100644 index 0000000..a27e077 --- /dev/null +++ b/LanguageModeling/bert-oneflow/log/default_physical_env_log/VS002/oneflow.VS002.lichunyou.log.INFO.20210615-161144.223722 @@ -0,0 +1,40 @@ +Log file created at: 2021/06/15 16:11:44 +Running on machine: VS002 +Log line format: [IWEF]mmdd hh:mm:ss.uuuuuu threadid file:line] msg +I0615 16:11:44.690667 223722 global.h:36] NewGlobal 14cudaDeviceProp +I0615 16:11:44.762032 223722 global.h:36] NewGlobal N7oneflow7EnvDescE +I0615 16:11:44.762066 223722 global.h:36] NewGlobal N7oneflow10ProcessCtxE +I0615 16:11:44.762075 223722 env_global_objects_scope.cpp:112] using rpc backend: local +I0615 16:11:44.762087 223722 global.h:36] NewGlobal N7oneflow12ResourceDescE +I0615 16:11:44.762095 223722 global.h:36] NewGlobal N7oneflow12ResourceDescE +I0615 16:11:44.769475 223722 global.h:36] NewGlobal N7oneflow10ThreadPoolE +I0615 16:11:44.770489 223722 global.h:36] NewGlobal N7oneflow16EagerNcclCommMgrE +I0615 16:11:44.770499 223722 global.h:36] NewGlobal N7oneflow18CudnnConvAlgoCacheE +I0615 16:11:44.770505 223722 global.h:36] NewGlobal N7oneflow2vm19VirtualMachineScopeE +I0615 16:11:44.770512 223722 global.h:36] NewGlobal N7oneflow9OneflowVME +I0615 16:11:44.780143 223722 global.h:36] NewGlobal N7oneflow27EagerJobBuildAndInferCtxMgrE +I0615 16:11:44.780164 223722 global.h:36] NewGlobal N7oneflow12EpollCommNetE +I0615 16:11:44.780247 223722 epoll_comm_network.cpp:63] CommNet:Epoll listening on 0.0.0.0:33194 +I0615 16:11:44.780270 223722 epoll_comm_network.cpp:198] machine 0 sockfd -1 +I0615 16:11:44.780357 223722 global.h:36] NewGlobal N7oneflow9TransportE +I0615 16:12:06.573643 223722 global.h:43] DeleteGlobal N7oneflow21EnvGlobalObjectsScopeE +I0615 16:12:06.573709 223722 global.h:43] DeleteGlobal N7oneflow9TransportE +I0615 16:12:06.573839 223722 global.h:43] DeleteGlobal N7oneflow12EpollCommNetE +I0615 16:12:06.573853 223722 epoll_comm_network.cpp:87] CommNet Thread 0 finish +I0615 16:12:06.573976 223722 epoll_comm_network.cpp:87] CommNet Thread 1 finish +I0615 16:12:06.574168 223722 epoll_comm_network.cpp:87] CommNet Thread 2 finish +I0615 16:12:06.574260 223722 epoll_comm_network.cpp:87] CommNet Thread 3 finish +I0615 16:12:06.574414 223722 global.h:43] DeleteGlobal N7oneflow27EagerJobBuildAndInferCtxMgrE +I0615 16:12:06.574424 223722 global.h:43] DeleteGlobal N7oneflow2vm19VirtualMachineScopeE +I0615 16:12:06.574431 223722 global.h:43] DeleteGlobal N7oneflow9OneflowVME +I0615 16:12:06.606727 223722 global.h:43] DeleteGlobal N7oneflow18CudnnConvAlgoCacheE +I0615 16:12:06.606770 223722 global.h:43] DeleteGlobal N7oneflow16EagerNcclCommMgrE +I0615 16:12:06.606781 223722 global.h:43] DeleteGlobal N7oneflow10ThreadPoolE +I0615 16:12:06.609259 223722 global.h:43] DeleteGlobal N7oneflow12ResourceDescE +I0615 16:12:06.609272 223722 global.h:43] DeleteGlobal N7oneflow12ResourceDescE +I0615 16:12:06.609282 223722 global.h:43] DeleteGlobal N7oneflow6device27NodeDeviceDescriptorManagerE +I0615 16:12:06.609308 223722 global.h:43] DeleteGlobal N7oneflow10RpcManagerE +I0615 16:12:06.609314 223722 global.h:43] DeleteGlobal N7oneflow10CtrlClientE +I0615 16:12:06.609323 223722 global.h:43] DeleteGlobal N7oneflow10ProcessCtxE +I0615 16:12:06.609333 223722 global.h:43] DeleteGlobal N7oneflow7EnvDescE +I0615 16:12:06.609344 223722 global.h:43] DeleteGlobal 14cudaDeviceProp diff --git a/LanguageModeling/bert-oneflow/log/default_physical_env_log/VS002/oneflow.VS002.lichunyou.log.INFO.20210615-161246.224493 b/LanguageModeling/bert-oneflow/log/default_physical_env_log/VS002/oneflow.VS002.lichunyou.log.INFO.20210615-161246.224493 new file mode 100644 index 0000000..846a4d8 --- /dev/null +++ b/LanguageModeling/bert-oneflow/log/default_physical_env_log/VS002/oneflow.VS002.lichunyou.log.INFO.20210615-161246.224493 @@ -0,0 +1,19 @@ +Log file created at: 2021/06/15 16:12:46 +Running on machine: VS002 +Log line format: [IWEF]mmdd hh:mm:ss.uuuuuu threadid file:line] msg +I0615 16:12:46.772022 224493 global.h:36] NewGlobal 14cudaDeviceProp +I0615 16:12:46.844890 224493 global.h:36] NewGlobal N7oneflow7EnvDescE +I0615 16:12:46.844928 224493 global.h:36] NewGlobal N7oneflow10ProcessCtxE +I0615 16:12:46.844936 224493 env_global_objects_scope.cpp:112] using rpc backend: local +I0615 16:12:46.844949 224493 global.h:36] NewGlobal N7oneflow12ResourceDescE +I0615 16:12:46.844956 224493 global.h:36] NewGlobal N7oneflow12ResourceDescE +I0615 16:12:46.852439 224493 global.h:36] NewGlobal N7oneflow10ThreadPoolE +I0615 16:12:46.853443 224493 global.h:36] NewGlobal N7oneflow16EagerNcclCommMgrE +I0615 16:12:46.853453 224493 global.h:36] NewGlobal N7oneflow18CudnnConvAlgoCacheE +I0615 16:12:46.853459 224493 global.h:36] NewGlobal N7oneflow2vm19VirtualMachineScopeE +I0615 16:12:46.853466 224493 global.h:36] NewGlobal N7oneflow9OneflowVME +I0615 16:12:46.863137 224493 global.h:36] NewGlobal N7oneflow27EagerJobBuildAndInferCtxMgrE +I0615 16:12:46.863163 224493 global.h:36] NewGlobal N7oneflow12EpollCommNetE +I0615 16:12:46.863245 224493 epoll_comm_network.cpp:63] CommNet:Epoll listening on 0.0.0.0:46335 +I0615 16:12:46.863268 224493 epoll_comm_network.cpp:198] machine 0 sockfd -1 +I0615 16:12:46.863353 224493 global.h:36] NewGlobal N7oneflow9TransportE diff --git a/LanguageModeling/bert-oneflow/log/default_physical_env_log/VS002/oneflow.VS002.lichunyou.log.INFO.20210615-162501.226843 b/LanguageModeling/bert-oneflow/log/default_physical_env_log/VS002/oneflow.VS002.lichunyou.log.INFO.20210615-162501.226843 new file mode 100644 index 0000000..6df7d63 --- /dev/null +++ b/LanguageModeling/bert-oneflow/log/default_physical_env_log/VS002/oneflow.VS002.lichunyou.log.INFO.20210615-162501.226843 @@ -0,0 +1,19 @@ +Log file created at: 2021/06/15 16:25:01 +Running on machine: VS002 +Log line format: [IWEF]mmdd hh:mm:ss.uuuuuu threadid file:line] msg +I0615 16:25:01.193742 226843 global.h:36] NewGlobal 14cudaDeviceProp +I0615 16:25:01.254277 226843 global.h:36] NewGlobal N7oneflow7EnvDescE +I0615 16:25:01.254308 226843 global.h:36] NewGlobal N7oneflow10ProcessCtxE +I0615 16:25:01.254315 226843 env_global_objects_scope.cpp:112] using rpc backend: local +I0615 16:25:01.254325 226843 global.h:36] NewGlobal N7oneflow12ResourceDescE +I0615 16:25:01.254331 226843 global.h:36] NewGlobal N7oneflow12ResourceDescE +I0615 16:25:01.260859 226843 global.h:36] NewGlobal N7oneflow10ThreadPoolE +I0615 16:25:01.261742 226843 global.h:36] NewGlobal N7oneflow16EagerNcclCommMgrE +I0615 16:25:01.261749 226843 global.h:36] NewGlobal N7oneflow18CudnnConvAlgoCacheE +I0615 16:25:01.261754 226843 global.h:36] NewGlobal N7oneflow2vm19VirtualMachineScopeE +I0615 16:25:01.261759 226843 global.h:36] NewGlobal N7oneflow9OneflowVME +I0615 16:25:01.270251 226843 global.h:36] NewGlobal N7oneflow27EagerJobBuildAndInferCtxMgrE +I0615 16:25:01.270270 226843 global.h:36] NewGlobal N7oneflow12EpollCommNetE +I0615 16:25:01.270340 226843 epoll_comm_network.cpp:63] CommNet:Epoll listening on 0.0.0.0:44430 +I0615 16:25:01.270360 226843 epoll_comm_network.cpp:198] machine 0 sockfd -1 +I0615 16:25:01.270432 226843 global.h:36] NewGlobal N7oneflow9TransportE diff --git a/LanguageModeling/bert-oneflow/log/default_physical_env_log/VS002/oneflow.VS002.lichunyou.log.INFO.20210616-121348.68726 b/LanguageModeling/bert-oneflow/log/default_physical_env_log/VS002/oneflow.VS002.lichunyou.log.INFO.20210616-121348.68726 new file mode 100644 index 0000000..62369fc --- /dev/null +++ b/LanguageModeling/bert-oneflow/log/default_physical_env_log/VS002/oneflow.VS002.lichunyou.log.INFO.20210616-121348.68726 @@ -0,0 +1,40 @@ +Log file created at: 2021/06/16 12:13:48 +Running on machine: VS002 +Log line format: [IWEF]mmdd hh:mm:ss.uuuuuu threadid file:line] msg +I0616 12:13:48.738842 68726 global.h:36] NewGlobal 14cudaDeviceProp +I0616 12:13:49.115579 68726 global.h:36] NewGlobal N7oneflow7EnvDescE +I0616 12:13:49.115646 68726 global.h:36] NewGlobal N7oneflow10ProcessCtxE +I0616 12:13:49.115658 68726 env_global_objects_scope.cpp:112] using rpc backend: local +I0616 12:13:49.115706 68726 global.h:36] NewGlobal N7oneflow12ResourceDescE +I0616 12:13:49.115717 68726 global.h:36] NewGlobal N7oneflow12ResourceDescE +I0616 12:13:49.124030 68726 global.h:36] NewGlobal N7oneflow10ThreadPoolE +I0616 12:13:49.125347 68726 global.h:36] NewGlobal N7oneflow16EagerNcclCommMgrE +I0616 12:13:49.125360 68726 global.h:36] NewGlobal N7oneflow18CudnnConvAlgoCacheE +I0616 12:13:49.125367 68726 global.h:36] NewGlobal N7oneflow2vm19VirtualMachineScopeE +I0616 12:13:49.125375 68726 global.h:36] NewGlobal N7oneflow9OneflowVME +I0616 12:13:49.134821 68726 global.h:36] NewGlobal N7oneflow27EagerJobBuildAndInferCtxMgrE +I0616 12:13:49.134842 68726 global.h:36] NewGlobal N7oneflow12EpollCommNetE +I0616 12:13:49.134929 68726 epoll_comm_network.cpp:63] CommNet:Epoll listening on 0.0.0.0:36657 +I0616 12:13:49.134959 68726 epoll_comm_network.cpp:198] machine 0 sockfd -1 +I0616 12:13:49.135046 68726 global.h:36] NewGlobal N7oneflow9TransportE +I0616 12:13:49.205109 68726 global.h:43] DeleteGlobal N7oneflow21EnvGlobalObjectsScopeE +I0616 12:13:49.205139 68726 global.h:43] DeleteGlobal N7oneflow9TransportE +I0616 12:13:49.205246 68726 global.h:43] DeleteGlobal N7oneflow12EpollCommNetE +I0616 12:13:49.205256 68726 epoll_comm_network.cpp:87] CommNet Thread 0 finish +I0616 12:13:49.205293 68726 epoll_comm_network.cpp:87] CommNet Thread 1 finish +I0616 12:13:49.205324 68726 epoll_comm_network.cpp:87] CommNet Thread 2 finish +I0616 12:13:49.205354 68726 epoll_comm_network.cpp:87] CommNet Thread 3 finish +I0616 12:13:49.205478 68726 global.h:43] DeleteGlobal N7oneflow27EagerJobBuildAndInferCtxMgrE +I0616 12:13:49.205489 68726 global.h:43] DeleteGlobal N7oneflow2vm19VirtualMachineScopeE +I0616 12:13:49.205497 68726 global.h:43] DeleteGlobal N7oneflow9OneflowVME +I0616 12:13:49.212520 68726 global.h:43] DeleteGlobal N7oneflow18CudnnConvAlgoCacheE +I0616 12:13:49.212568 68726 global.h:43] DeleteGlobal N7oneflow16EagerNcclCommMgrE +I0616 12:13:49.212575 68726 global.h:43] DeleteGlobal N7oneflow10ThreadPoolE +I0616 12:13:49.214356 68726 global.h:43] DeleteGlobal N7oneflow12ResourceDescE +I0616 12:13:49.214373 68726 global.h:43] DeleteGlobal N7oneflow12ResourceDescE +I0616 12:13:49.214380 68726 global.h:43] DeleteGlobal N7oneflow6device27NodeDeviceDescriptorManagerE +I0616 12:13:49.214404 68726 global.h:43] DeleteGlobal N7oneflow10RpcManagerE +I0616 12:13:49.214411 68726 global.h:43] DeleteGlobal N7oneflow10CtrlClientE +I0616 12:13:49.214426 68726 global.h:43] DeleteGlobal N7oneflow10ProcessCtxE +I0616 12:13:49.214434 68726 global.h:43] DeleteGlobal N7oneflow7EnvDescE +I0616 12:13:49.214442 68726 global.h:43] DeleteGlobal 14cudaDeviceProp diff --git a/LanguageModeling/bert-oneflow/log/default_physical_env_log/VS002/oneflow.VS002.lichunyou.log.INFO.20210616-121504.69423 b/LanguageModeling/bert-oneflow/log/default_physical_env_log/VS002/oneflow.VS002.lichunyou.log.INFO.20210616-121504.69423 new file mode 100644 index 0000000..4d4b251 --- /dev/null +++ b/LanguageModeling/bert-oneflow/log/default_physical_env_log/VS002/oneflow.VS002.lichunyou.log.INFO.20210616-121504.69423 @@ -0,0 +1,40 @@ +Log file created at: 2021/06/16 12:15:04 +Running on machine: VS002 +Log line format: [IWEF]mmdd hh:mm:ss.uuuuuu threadid file:line] msg +I0616 12:15:04.468799 69423 global.h:36] NewGlobal 14cudaDeviceProp +I0616 12:15:04.864488 69423 global.h:36] NewGlobal N7oneflow7EnvDescE +I0616 12:15:04.864612 69423 global.h:36] NewGlobal N7oneflow10ProcessCtxE +I0616 12:15:04.864631 69423 env_global_objects_scope.cpp:112] using rpc backend: local +I0616 12:15:04.864665 69423 global.h:36] NewGlobal N7oneflow12ResourceDescE +I0616 12:15:04.864681 69423 global.h:36] NewGlobal N7oneflow12ResourceDescE +I0616 12:15:04.874059 69423 global.h:36] NewGlobal N7oneflow10ThreadPoolE +I0616 12:15:04.875102 69423 global.h:36] NewGlobal N7oneflow16EagerNcclCommMgrE +I0616 12:15:04.875111 69423 global.h:36] NewGlobal N7oneflow18CudnnConvAlgoCacheE +I0616 12:15:04.875115 69423 global.h:36] NewGlobal N7oneflow2vm19VirtualMachineScopeE +I0616 12:15:04.875121 69423 global.h:36] NewGlobal N7oneflow9OneflowVME +I0616 12:15:04.884797 69423 global.h:36] NewGlobal N7oneflow27EagerJobBuildAndInferCtxMgrE +I0616 12:15:04.884819 69423 global.h:36] NewGlobal N7oneflow12EpollCommNetE +I0616 12:15:04.884923 69423 epoll_comm_network.cpp:63] CommNet:Epoll listening on 0.0.0.0:45076 +I0616 12:15:04.884955 69423 epoll_comm_network.cpp:198] machine 0 sockfd -1 +I0616 12:15:04.885033 69423 global.h:36] NewGlobal N7oneflow9TransportE +I0616 12:15:04.990201 69423 global.h:43] DeleteGlobal N7oneflow21EnvGlobalObjectsScopeE +I0616 12:15:04.990232 69423 global.h:43] DeleteGlobal N7oneflow9TransportE +I0616 12:15:04.990332 69423 global.h:43] DeleteGlobal N7oneflow12EpollCommNetE +I0616 12:15:04.990340 69423 epoll_comm_network.cpp:87] CommNet Thread 0 finish +I0616 12:15:04.990376 69423 epoll_comm_network.cpp:87] CommNet Thread 1 finish +I0616 12:15:04.990406 69423 epoll_comm_network.cpp:87] CommNet Thread 2 finish +I0616 12:15:04.990435 69423 epoll_comm_network.cpp:87] CommNet Thread 3 finish +I0616 12:15:04.990547 69423 global.h:43] DeleteGlobal N7oneflow27EagerJobBuildAndInferCtxMgrE +I0616 12:15:04.990556 69423 global.h:43] DeleteGlobal N7oneflow2vm19VirtualMachineScopeE +I0616 12:15:04.990561 69423 global.h:43] DeleteGlobal N7oneflow9OneflowVME +I0616 12:15:04.995379 69423 global.h:43] DeleteGlobal N7oneflow18CudnnConvAlgoCacheE +I0616 12:15:04.995400 69423 global.h:43] DeleteGlobal N7oneflow16EagerNcclCommMgrE +I0616 12:15:04.995407 69423 global.h:43] DeleteGlobal N7oneflow10ThreadPoolE +I0616 12:15:04.996918 69423 global.h:43] DeleteGlobal N7oneflow12ResourceDescE +I0616 12:15:04.996929 69423 global.h:43] DeleteGlobal N7oneflow12ResourceDescE +I0616 12:15:04.996937 69423 global.h:43] DeleteGlobal N7oneflow6device27NodeDeviceDescriptorManagerE +I0616 12:15:04.996951 69423 global.h:43] DeleteGlobal N7oneflow10RpcManagerE +I0616 12:15:04.996958 69423 global.h:43] DeleteGlobal N7oneflow10CtrlClientE +I0616 12:15:04.996963 69423 global.h:43] DeleteGlobal N7oneflow10ProcessCtxE +I0616 12:15:04.996971 69423 global.h:43] DeleteGlobal N7oneflow7EnvDescE +I0616 12:15:04.996978 69423 global.h:43] DeleteGlobal 14cudaDeviceProp diff --git a/LanguageModeling/bert-oneflow/log/default_physical_env_log/VS002/oneflow.VS002.lichunyou.log.INFO.20210616-121548.70085 b/LanguageModeling/bert-oneflow/log/default_physical_env_log/VS002/oneflow.VS002.lichunyou.log.INFO.20210616-121548.70085 new file mode 100644 index 0000000..14a37e4 --- /dev/null +++ b/LanguageModeling/bert-oneflow/log/default_physical_env_log/VS002/oneflow.VS002.lichunyou.log.INFO.20210616-121548.70085 @@ -0,0 +1,40 @@ +Log file created at: 2021/06/16 12:15:48 +Running on machine: VS002 +Log line format: [IWEF]mmdd hh:mm:ss.uuuuuu threadid file:line] msg +I0616 12:15:48.501327 70085 global.h:36] NewGlobal 14cudaDeviceProp +I0616 12:15:48.884912 70085 global.h:36] NewGlobal N7oneflow7EnvDescE +I0616 12:15:48.885016 70085 global.h:36] NewGlobal N7oneflow10ProcessCtxE +I0616 12:15:48.885042 70085 env_global_objects_scope.cpp:112] using rpc backend: local +I0616 12:15:48.885076 70085 global.h:36] NewGlobal N7oneflow12ResourceDescE +I0616 12:15:48.885089 70085 global.h:36] NewGlobal N7oneflow12ResourceDescE +I0616 12:15:48.897119 70085 global.h:36] NewGlobal N7oneflow10ThreadPoolE +I0616 12:15:48.898788 70085 global.h:36] NewGlobal N7oneflow16EagerNcclCommMgrE +I0616 12:15:48.898846 70085 global.h:36] NewGlobal N7oneflow18CudnnConvAlgoCacheE +I0616 12:15:48.898854 70085 global.h:36] NewGlobal N7oneflow2vm19VirtualMachineScopeE +I0616 12:15:48.898861 70085 global.h:36] NewGlobal N7oneflow9OneflowVME +I0616 12:15:48.909502 70085 global.h:36] NewGlobal N7oneflow27EagerJobBuildAndInferCtxMgrE +I0616 12:15:48.909530 70085 global.h:36] NewGlobal N7oneflow12EpollCommNetE +I0616 12:15:48.909626 70085 epoll_comm_network.cpp:63] CommNet:Epoll listening on 0.0.0.0:46610 +I0616 12:15:48.909657 70085 epoll_comm_network.cpp:198] machine 0 sockfd -1 +I0616 12:15:48.909744 70085 global.h:36] NewGlobal N7oneflow9TransportE +I0616 12:15:49.243652 70085 global.h:43] DeleteGlobal N7oneflow21EnvGlobalObjectsScopeE +I0616 12:15:49.243695 70085 global.h:43] DeleteGlobal N7oneflow9TransportE +I0616 12:15:49.243834 70085 global.h:43] DeleteGlobal N7oneflow12EpollCommNetE +I0616 12:15:49.243842 70085 epoll_comm_network.cpp:87] CommNet Thread 0 finish +I0616 12:15:49.243896 70085 epoll_comm_network.cpp:87] CommNet Thread 1 finish +I0616 12:15:49.243925 70085 epoll_comm_network.cpp:87] CommNet Thread 2 finish +I0616 12:15:49.243984 70085 epoll_comm_network.cpp:87] CommNet Thread 3 finish +I0616 12:15:49.244272 70085 global.h:43] DeleteGlobal N7oneflow27EagerJobBuildAndInferCtxMgrE +I0616 12:15:49.244333 70085 global.h:43] DeleteGlobal N7oneflow2vm19VirtualMachineScopeE +I0616 12:15:49.244349 70085 global.h:43] DeleteGlobal N7oneflow9OneflowVME +I0616 12:15:49.252218 70085 global.h:43] DeleteGlobal N7oneflow18CudnnConvAlgoCacheE +I0616 12:15:49.252254 70085 global.h:43] DeleteGlobal N7oneflow16EagerNcclCommMgrE +I0616 12:15:49.252269 70085 global.h:43] DeleteGlobal N7oneflow10ThreadPoolE +I0616 12:15:49.255165 70085 global.h:43] DeleteGlobal N7oneflow12ResourceDescE +I0616 12:15:49.255223 70085 global.h:43] DeleteGlobal N7oneflow12ResourceDescE +I0616 12:15:49.255239 70085 global.h:43] DeleteGlobal N7oneflow6device27NodeDeviceDescriptorManagerE +I0616 12:15:49.255266 70085 global.h:43] DeleteGlobal N7oneflow10RpcManagerE +I0616 12:15:49.255280 70085 global.h:43] DeleteGlobal N7oneflow10CtrlClientE +I0616 12:15:49.255295 70085 global.h:43] DeleteGlobal N7oneflow10ProcessCtxE +I0616 12:15:49.255311 70085 global.h:43] DeleteGlobal N7oneflow7EnvDescE +I0616 12:15:49.255329 70085 global.h:43] DeleteGlobal 14cudaDeviceProp diff --git a/LanguageModeling/bert-oneflow/log/default_physical_env_log/VS002/oneflow.VS002.lichunyou.log.INFO.20210616-143323.80849 b/LanguageModeling/bert-oneflow/log/default_physical_env_log/VS002/oneflow.VS002.lichunyou.log.INFO.20210616-143323.80849 new file mode 100644 index 0000000..c4c3e70 --- /dev/null +++ b/LanguageModeling/bert-oneflow/log/default_physical_env_log/VS002/oneflow.VS002.lichunyou.log.INFO.20210616-143323.80849 @@ -0,0 +1,40 @@ +Log file created at: 2021/06/16 14:33:23 +Running on machine: VS002 +Log line format: [IWEF]mmdd hh:mm:ss.uuuuuu threadid file:line] msg +I0616 14:33:23.703843 80849 global.h:36] NewGlobal 14cudaDeviceProp +I0616 14:33:24.055442 80849 global.h:36] NewGlobal N7oneflow7EnvDescE +I0616 14:33:24.055500 80849 global.h:36] NewGlobal N7oneflow10ProcessCtxE +I0616 14:33:24.055507 80849 env_global_objects_scope.cpp:112] using rpc backend: local +I0616 14:33:24.055541 80849 global.h:36] NewGlobal N7oneflow12ResourceDescE +I0616 14:33:24.055548 80849 global.h:36] NewGlobal N7oneflow12ResourceDescE +I0616 14:33:24.061887 80849 global.h:36] NewGlobal N7oneflow10ThreadPoolE +I0616 14:33:24.062867 80849 global.h:36] NewGlobal N7oneflow16EagerNcclCommMgrE +I0616 14:33:24.062876 80849 global.h:36] NewGlobal N7oneflow18CudnnConvAlgoCacheE +I0616 14:33:24.062880 80849 global.h:36] NewGlobal N7oneflow2vm19VirtualMachineScopeE +I0616 14:33:24.062885 80849 global.h:36] NewGlobal N7oneflow9OneflowVME +I0616 14:33:24.073797 80849 global.h:36] NewGlobal N7oneflow27EagerJobBuildAndInferCtxMgrE +I0616 14:33:24.073866 80849 global.h:36] NewGlobal N7oneflow12EpollCommNetE +I0616 14:33:24.073969 80849 epoll_comm_network.cpp:63] CommNet:Epoll listening on 0.0.0.0:35633 +I0616 14:33:24.074003 80849 epoll_comm_network.cpp:198] machine 0 sockfd -1 +I0616 14:33:24.074086 80849 global.h:36] NewGlobal N7oneflow9TransportE +I0616 14:33:29.385749 80849 global.h:43] DeleteGlobal N7oneflow21EnvGlobalObjectsScopeE +I0616 14:33:29.385855 80849 global.h:43] DeleteGlobal N7oneflow9TransportE +I0616 14:33:29.386006 80849 global.h:43] DeleteGlobal N7oneflow12EpollCommNetE +I0616 14:33:29.386024 80849 epoll_comm_network.cpp:87] CommNet Thread 0 finish +I0616 14:33:29.386140 80849 epoll_comm_network.cpp:87] CommNet Thread 1 finish +I0616 14:33:29.386188 80849 epoll_comm_network.cpp:87] CommNet Thread 2 finish +I0616 14:33:29.386268 80849 epoll_comm_network.cpp:87] CommNet Thread 3 finish +I0616 14:33:29.386445 80849 global.h:43] DeleteGlobal N7oneflow27EagerJobBuildAndInferCtxMgrE +I0616 14:33:29.386462 80849 global.h:43] DeleteGlobal N7oneflow2vm19VirtualMachineScopeE +I0616 14:33:29.386474 80849 global.h:43] DeleteGlobal N7oneflow9OneflowVME +I0616 14:33:29.399066 80849 global.h:43] DeleteGlobal N7oneflow18CudnnConvAlgoCacheE +I0616 14:33:29.399102 80849 global.h:43] DeleteGlobal N7oneflow16EagerNcclCommMgrE +I0616 14:33:29.399111 80849 global.h:43] DeleteGlobal N7oneflow10ThreadPoolE +I0616 14:33:29.402797 80849 global.h:43] DeleteGlobal N7oneflow12ResourceDescE +I0616 14:33:29.402856 80849 global.h:43] DeleteGlobal N7oneflow12ResourceDescE +I0616 14:33:29.402871 80849 global.h:43] DeleteGlobal N7oneflow6device27NodeDeviceDescriptorManagerE +I0616 14:33:29.402897 80849 global.h:43] DeleteGlobal N7oneflow10RpcManagerE +I0616 14:33:29.402912 80849 global.h:43] DeleteGlobal N7oneflow10CtrlClientE +I0616 14:33:29.402927 80849 global.h:43] DeleteGlobal N7oneflow10ProcessCtxE +I0616 14:33:29.402942 80849 global.h:43] DeleteGlobal N7oneflow7EnvDescE +I0616 14:33:29.402956 80849 global.h:43] DeleteGlobal 14cudaDeviceProp diff --git a/LanguageModeling/bert-oneflow/log/default_physical_env_log/VS002/oneflow.VS002.lichunyou.log.INFO.20210616-143407.81483 b/LanguageModeling/bert-oneflow/log/default_physical_env_log/VS002/oneflow.VS002.lichunyou.log.INFO.20210616-143407.81483 new file mode 100644 index 0000000..088b6de --- /dev/null +++ b/LanguageModeling/bert-oneflow/log/default_physical_env_log/VS002/oneflow.VS002.lichunyou.log.INFO.20210616-143407.81483 @@ -0,0 +1,40 @@ +Log file created at: 2021/06/16 14:34:07 +Running on machine: VS002 +Log line format: [IWEF]mmdd hh:mm:ss.uuuuuu threadid file:line] msg +I0616 14:34:07.170157 81483 global.h:36] NewGlobal 14cudaDeviceProp +I0616 14:34:07.558320 81483 global.h:36] NewGlobal N7oneflow7EnvDescE +I0616 14:34:07.558429 81483 global.h:36] NewGlobal N7oneflow10ProcessCtxE +I0616 14:34:07.558437 81483 env_global_objects_scope.cpp:112] using rpc backend: local +I0616 14:34:07.558460 81483 global.h:36] NewGlobal N7oneflow12ResourceDescE +I0616 14:34:07.558468 81483 global.h:36] NewGlobal N7oneflow12ResourceDescE +I0616 14:34:07.565513 81483 global.h:36] NewGlobal N7oneflow10ThreadPoolE +I0616 14:34:07.566704 81483 global.h:36] NewGlobal N7oneflow16EagerNcclCommMgrE +I0616 14:34:07.566715 81483 global.h:36] NewGlobal N7oneflow18CudnnConvAlgoCacheE +I0616 14:34:07.566720 81483 global.h:36] NewGlobal N7oneflow2vm19VirtualMachineScopeE +I0616 14:34:07.566725 81483 global.h:36] NewGlobal N7oneflow9OneflowVME +I0616 14:34:07.577455 81483 global.h:36] NewGlobal N7oneflow27EagerJobBuildAndInferCtxMgrE +I0616 14:34:07.577481 81483 global.h:36] NewGlobal N7oneflow12EpollCommNetE +I0616 14:34:07.577610 81483 epoll_comm_network.cpp:63] CommNet:Epoll listening on 0.0.0.0:37443 +I0616 14:34:07.577634 81483 epoll_comm_network.cpp:198] machine 0 sockfd -1 +I0616 14:34:07.577718 81483 global.h:36] NewGlobal N7oneflow9TransportE +I0616 14:34:12.959990 81483 global.h:43] DeleteGlobal N7oneflow21EnvGlobalObjectsScopeE +I0616 14:34:12.960086 81483 global.h:43] DeleteGlobal N7oneflow9TransportE +I0616 14:34:12.960222 81483 global.h:43] DeleteGlobal N7oneflow12EpollCommNetE +I0616 14:34:12.960237 81483 epoll_comm_network.cpp:87] CommNet Thread 0 finish +I0616 14:34:12.960294 81483 epoll_comm_network.cpp:87] CommNet Thread 1 finish +I0616 14:34:12.960341 81483 epoll_comm_network.cpp:87] CommNet Thread 2 finish +I0616 14:34:12.960440 81483 epoll_comm_network.cpp:87] CommNet Thread 3 finish +I0616 14:34:12.960605 81483 global.h:43] DeleteGlobal N7oneflow27EagerJobBuildAndInferCtxMgrE +I0616 14:34:12.960628 81483 global.h:43] DeleteGlobal N7oneflow2vm19VirtualMachineScopeE +I0616 14:34:12.960640 81483 global.h:43] DeleteGlobal N7oneflow9OneflowVME +I0616 14:34:12.973390 81483 global.h:43] DeleteGlobal N7oneflow18CudnnConvAlgoCacheE +I0616 14:34:12.973443 81483 global.h:43] DeleteGlobal N7oneflow16EagerNcclCommMgrE +I0616 14:34:12.973456 81483 global.h:43] DeleteGlobal N7oneflow10ThreadPoolE +I0616 14:34:12.975657 81483 global.h:43] DeleteGlobal N7oneflow12ResourceDescE +I0616 14:34:12.975674 81483 global.h:43] DeleteGlobal N7oneflow12ResourceDescE +I0616 14:34:12.975682 81483 global.h:43] DeleteGlobal N7oneflow6device27NodeDeviceDescriptorManagerE +I0616 14:34:12.975708 81483 global.h:43] DeleteGlobal N7oneflow10RpcManagerE +I0616 14:34:12.975716 81483 global.h:43] DeleteGlobal N7oneflow10CtrlClientE +I0616 14:34:12.975725 81483 global.h:43] DeleteGlobal N7oneflow10ProcessCtxE +I0616 14:34:12.975734 81483 global.h:43] DeleteGlobal N7oneflow7EnvDescE +I0616 14:34:12.975744 81483 global.h:43] DeleteGlobal 14cudaDeviceProp diff --git a/LanguageModeling/bert-oneflow/log/default_physical_env_log/VS002/oneflow.VS002.lichunyou.log.INFO.20210616-143528.82120 b/LanguageModeling/bert-oneflow/log/default_physical_env_log/VS002/oneflow.VS002.lichunyou.log.INFO.20210616-143528.82120 new file mode 100644 index 0000000..b73089b --- /dev/null +++ b/LanguageModeling/bert-oneflow/log/default_physical_env_log/VS002/oneflow.VS002.lichunyou.log.INFO.20210616-143528.82120 @@ -0,0 +1,19 @@ +Log file created at: 2021/06/16 14:35:28 +Running on machine: VS002 +Log line format: [IWEF]mmdd hh:mm:ss.uuuuuu threadid file:line] msg +I0616 14:35:28.097849 82120 global.h:36] NewGlobal 14cudaDeviceProp +I0616 14:35:28.478919 82120 global.h:36] NewGlobal N7oneflow7EnvDescE +I0616 14:35:28.479009 82120 global.h:36] NewGlobal N7oneflow10ProcessCtxE +I0616 14:35:28.479018 82120 env_global_objects_scope.cpp:112] using rpc backend: local +I0616 14:35:28.479033 82120 global.h:36] NewGlobal N7oneflow12ResourceDescE +I0616 14:35:28.479040 82120 global.h:36] NewGlobal N7oneflow12ResourceDescE +I0616 14:35:28.485635 82120 global.h:36] NewGlobal N7oneflow10ThreadPoolE +I0616 14:35:28.486738 82120 global.h:36] NewGlobal N7oneflow16EagerNcclCommMgrE +I0616 14:35:28.486748 82120 global.h:36] NewGlobal N7oneflow18CudnnConvAlgoCacheE +I0616 14:35:28.486753 82120 global.h:36] NewGlobal N7oneflow2vm19VirtualMachineScopeE +I0616 14:35:28.486759 82120 global.h:36] NewGlobal N7oneflow9OneflowVME +I0616 14:35:28.496556 82120 global.h:36] NewGlobal N7oneflow27EagerJobBuildAndInferCtxMgrE +I0616 14:35:28.496577 82120 global.h:36] NewGlobal N7oneflow12EpollCommNetE +I0616 14:35:28.496685 82120 epoll_comm_network.cpp:63] CommNet:Epoll listening on 0.0.0.0:40155 +I0616 14:35:28.496708 82120 epoll_comm_network.cpp:198] machine 0 sockfd -1 +I0616 14:35:28.496790 82120 global.h:36] NewGlobal N7oneflow9TransportE diff --git a/LanguageModeling/bert-oneflow/log/default_physical_env_log/VS002/oneflow.VS002.lichunyou.log.INFO.20210616-144741.83066 b/LanguageModeling/bert-oneflow/log/default_physical_env_log/VS002/oneflow.VS002.lichunyou.log.INFO.20210616-144741.83066 new file mode 100644 index 0000000..d1bd5ea --- /dev/null +++ b/LanguageModeling/bert-oneflow/log/default_physical_env_log/VS002/oneflow.VS002.lichunyou.log.INFO.20210616-144741.83066 @@ -0,0 +1,40 @@ +Log file created at: 2021/06/16 14:47:41 +Running on machine: VS002 +Log line format: [IWEF]mmdd hh:mm:ss.uuuuuu threadid file:line] msg +I0616 14:47:41.597259 83066 global.h:36] NewGlobal 14cudaDeviceProp +I0616 14:47:41.975654 83066 global.h:36] NewGlobal N7oneflow7EnvDescE +I0616 14:47:41.975700 83066 global.h:36] NewGlobal N7oneflow10ProcessCtxE +I0616 14:47:41.975708 83066 env_global_objects_scope.cpp:112] using rpc backend: local +I0616 14:47:41.975719 83066 global.h:36] NewGlobal N7oneflow12ResourceDescE +I0616 14:47:41.975726 83066 global.h:36] NewGlobal N7oneflow12ResourceDescE +I0616 14:47:41.982234 83066 global.h:36] NewGlobal N7oneflow10ThreadPoolE +I0616 14:47:41.983127 83066 global.h:36] NewGlobal N7oneflow16EagerNcclCommMgrE +I0616 14:47:41.983135 83066 global.h:36] NewGlobal N7oneflow18CudnnConvAlgoCacheE +I0616 14:47:41.983140 83066 global.h:36] NewGlobal N7oneflow2vm19VirtualMachineScopeE +I0616 14:47:41.983145 83066 global.h:36] NewGlobal N7oneflow9OneflowVME +I0616 14:47:41.991276 83066 global.h:36] NewGlobal N7oneflow27EagerJobBuildAndInferCtxMgrE +I0616 14:47:41.991298 83066 global.h:36] NewGlobal N7oneflow12EpollCommNetE +I0616 14:47:41.991374 83066 epoll_comm_network.cpp:63] CommNet:Epoll listening on 0.0.0.0:36373 +I0616 14:47:41.991396 83066 epoll_comm_network.cpp:198] machine 0 sockfd -1 +I0616 14:47:41.991464 83066 global.h:36] NewGlobal N7oneflow9TransportE +I0616 14:47:49.087057 83066 global.h:43] DeleteGlobal N7oneflow21EnvGlobalObjectsScopeE +I0616 14:47:49.087123 83066 global.h:43] DeleteGlobal N7oneflow9TransportE +I0616 14:47:49.087374 83066 global.h:43] DeleteGlobal N7oneflow12EpollCommNetE +I0616 14:47:49.087451 83066 epoll_comm_network.cpp:87] CommNet Thread 0 finish +I0616 14:47:49.087553 83066 epoll_comm_network.cpp:87] CommNet Thread 1 finish +I0616 14:47:49.087625 83066 epoll_comm_network.cpp:87] CommNet Thread 2 finish +I0616 14:47:49.087733 83066 epoll_comm_network.cpp:87] CommNet Thread 3 finish +I0616 14:47:49.087904 83066 global.h:43] DeleteGlobal N7oneflow27EagerJobBuildAndInferCtxMgrE +I0616 14:47:49.087924 83066 global.h:43] DeleteGlobal N7oneflow2vm19VirtualMachineScopeE +I0616 14:47:49.087937 83066 global.h:43] DeleteGlobal N7oneflow9OneflowVME +I0616 14:47:49.114552 83066 global.h:43] DeleteGlobal N7oneflow18CudnnConvAlgoCacheE +I0616 14:47:49.114586 83066 global.h:43] DeleteGlobal N7oneflow16EagerNcclCommMgrE +I0616 14:47:49.114600 83066 global.h:43] DeleteGlobal N7oneflow10ThreadPoolE +I0616 14:47:49.116340 83066 global.h:43] DeleteGlobal N7oneflow12ResourceDescE +I0616 14:47:49.116353 83066 global.h:43] DeleteGlobal N7oneflow12ResourceDescE +I0616 14:47:49.116362 83066 global.h:43] DeleteGlobal N7oneflow6device27NodeDeviceDescriptorManagerE +I0616 14:47:49.116387 83066 global.h:43] DeleteGlobal N7oneflow10RpcManagerE +I0616 14:47:49.116397 83066 global.h:43] DeleteGlobal N7oneflow10CtrlClientE +I0616 14:47:49.116405 83066 global.h:43] DeleteGlobal N7oneflow10ProcessCtxE +I0616 14:47:49.116420 83066 global.h:43] DeleteGlobal N7oneflow7EnvDescE +I0616 14:47:49.116434 83066 global.h:43] DeleteGlobal 14cudaDeviceProp diff --git a/LanguageModeling/bert-oneflow/log/default_physical_env_log/VS002/oneflow.VS002.lichunyou.log.INFO.20210616-144906.83746 b/LanguageModeling/bert-oneflow/log/default_physical_env_log/VS002/oneflow.VS002.lichunyou.log.INFO.20210616-144906.83746 new file mode 100644 index 0000000..a388a16 --- /dev/null +++ b/LanguageModeling/bert-oneflow/log/default_physical_env_log/VS002/oneflow.VS002.lichunyou.log.INFO.20210616-144906.83746 @@ -0,0 +1,19 @@ +Log file created at: 2021/06/16 14:49:06 +Running on machine: VS002 +Log line format: [IWEF]mmdd hh:mm:ss.uuuuuu threadid file:line] msg +I0616 14:49:06.940276 83746 global.h:36] NewGlobal 14cudaDeviceProp +I0616 14:49:07.297325 83746 global.h:36] NewGlobal N7oneflow7EnvDescE +I0616 14:49:07.297399 83746 global.h:36] NewGlobal N7oneflow10ProcessCtxE +I0616 14:49:07.297408 83746 env_global_objects_scope.cpp:112] using rpc backend: local +I0616 14:49:07.297451 83746 global.h:36] NewGlobal N7oneflow12ResourceDescE +I0616 14:49:07.297459 83746 global.h:36] NewGlobal N7oneflow12ResourceDescE +I0616 14:49:07.304072 83746 global.h:36] NewGlobal N7oneflow10ThreadPoolE +I0616 14:49:07.305037 83746 global.h:36] NewGlobal N7oneflow16EagerNcclCommMgrE +I0616 14:49:07.305047 83746 global.h:36] NewGlobal N7oneflow18CudnnConvAlgoCacheE +I0616 14:49:07.305053 83746 global.h:36] NewGlobal N7oneflow2vm19VirtualMachineScopeE +I0616 14:49:07.305059 83746 global.h:36] NewGlobal N7oneflow9OneflowVME +I0616 14:49:07.313510 83746 global.h:36] NewGlobal N7oneflow27EagerJobBuildAndInferCtxMgrE +I0616 14:49:07.313535 83746 global.h:36] NewGlobal N7oneflow12EpollCommNetE +I0616 14:49:07.313640 83746 epoll_comm_network.cpp:63] CommNet:Epoll listening on 0.0.0.0:36376 +I0616 14:49:07.313675 83746 epoll_comm_network.cpp:198] machine 0 sockfd -1 +I0616 14:49:07.313750 83746 global.h:36] NewGlobal N7oneflow9TransportE diff --git a/LanguageModeling/bert-oneflow/log/default_physical_env_log/VS002/oneflow.VS002.lichunyou.log.INFO.20210616-145207.84540 b/LanguageModeling/bert-oneflow/log/default_physical_env_log/VS002/oneflow.VS002.lichunyou.log.INFO.20210616-145207.84540 new file mode 100644 index 0000000..9c21c06 --- /dev/null +++ b/LanguageModeling/bert-oneflow/log/default_physical_env_log/VS002/oneflow.VS002.lichunyou.log.INFO.20210616-145207.84540 @@ -0,0 +1,19 @@ +Log file created at: 2021/06/16 14:52:07 +Running on machine: VS002 +Log line format: [IWEF]mmdd hh:mm:ss.uuuuuu threadid file:line] msg +I0616 14:52:07.388607 84540 global.h:36] NewGlobal 14cudaDeviceProp +I0616 14:52:07.754192 84540 global.h:36] NewGlobal N7oneflow7EnvDescE +I0616 14:52:07.754243 84540 global.h:36] NewGlobal N7oneflow10ProcessCtxE +I0616 14:52:07.754253 84540 env_global_objects_scope.cpp:112] using rpc backend: local +I0616 14:52:07.754266 84540 global.h:36] NewGlobal N7oneflow12ResourceDescE +I0616 14:52:07.754274 84540 global.h:36] NewGlobal N7oneflow12ResourceDescE +I0616 14:52:07.761760 84540 global.h:36] NewGlobal N7oneflow10ThreadPoolE +I0616 14:52:07.762856 84540 global.h:36] NewGlobal N7oneflow16EagerNcclCommMgrE +I0616 14:52:07.762867 84540 global.h:36] NewGlobal N7oneflow18CudnnConvAlgoCacheE +I0616 14:52:07.762873 84540 global.h:36] NewGlobal N7oneflow2vm19VirtualMachineScopeE +I0616 14:52:07.762879 84540 global.h:36] NewGlobal N7oneflow9OneflowVME +I0616 14:52:07.772663 84540 global.h:36] NewGlobal N7oneflow27EagerJobBuildAndInferCtxMgrE +I0616 14:52:07.772688 84540 global.h:36] NewGlobal N7oneflow12EpollCommNetE +I0616 14:52:07.772779 84540 epoll_comm_network.cpp:63] CommNet:Epoll listening on 0.0.0.0:45780 +I0616 14:52:07.772812 84540 epoll_comm_network.cpp:198] machine 0 sockfd -1 +I0616 14:52:07.772897 84540 global.h:36] NewGlobal N7oneflow9TransportE diff --git a/LanguageModeling/bert-oneflow/log/default_physical_env_log/VS002/oneflow.VS002.lichunyou.log.INFO.20210616-150059.85434 b/LanguageModeling/bert-oneflow/log/default_physical_env_log/VS002/oneflow.VS002.lichunyou.log.INFO.20210616-150059.85434 new file mode 100644 index 0000000..2c6fcf8 --- /dev/null +++ b/LanguageModeling/bert-oneflow/log/default_physical_env_log/VS002/oneflow.VS002.lichunyou.log.INFO.20210616-150059.85434 @@ -0,0 +1,40 @@ +Log file created at: 2021/06/16 15:00:59 +Running on machine: VS002 +Log line format: [IWEF]mmdd hh:mm:ss.uuuuuu threadid file:line] msg +I0616 15:00:59.435274 85434 global.h:36] NewGlobal 14cudaDeviceProp +I0616 15:00:59.813540 85434 global.h:36] NewGlobal N7oneflow7EnvDescE +I0616 15:00:59.813589 85434 global.h:36] NewGlobal N7oneflow10ProcessCtxE +I0616 15:00:59.813598 85434 env_global_objects_scope.cpp:112] using rpc backend: local +I0616 15:00:59.813613 85434 global.h:36] NewGlobal N7oneflow12ResourceDescE +I0616 15:00:59.813621 85434 global.h:36] NewGlobal N7oneflow12ResourceDescE +I0616 15:00:59.821141 85434 global.h:36] NewGlobal N7oneflow10ThreadPoolE +I0616 15:00:59.822273 85434 global.h:36] NewGlobal N7oneflow16EagerNcclCommMgrE +I0616 15:00:59.822283 85434 global.h:36] NewGlobal N7oneflow18CudnnConvAlgoCacheE +I0616 15:00:59.822289 85434 global.h:36] NewGlobal N7oneflow2vm19VirtualMachineScopeE +I0616 15:00:59.822296 85434 global.h:36] NewGlobal N7oneflow9OneflowVME +I0616 15:00:59.833077 85434 global.h:36] NewGlobal N7oneflow27EagerJobBuildAndInferCtxMgrE +I0616 15:00:59.833104 85434 global.h:36] NewGlobal N7oneflow12EpollCommNetE +I0616 15:00:59.833190 85434 epoll_comm_network.cpp:63] CommNet:Epoll listening on 0.0.0.0:33941 +I0616 15:00:59.833214 85434 epoll_comm_network.cpp:198] machine 0 sockfd -1 +I0616 15:00:59.833305 85434 global.h:36] NewGlobal N7oneflow9TransportE +I0616 15:34:10.653270 85434 global.h:43] DeleteGlobal N7oneflow21EnvGlobalObjectsScopeE +I0616 15:34:10.653389 85434 global.h:43] DeleteGlobal N7oneflow9TransportE +I0616 15:34:10.653527 85434 global.h:43] DeleteGlobal N7oneflow12EpollCommNetE +I0616 15:34:10.653549 85434 epoll_comm_network.cpp:87] CommNet Thread 0 finish +I0616 15:34:10.653633 85434 epoll_comm_network.cpp:87] CommNet Thread 1 finish +I0616 15:34:10.653703 85434 epoll_comm_network.cpp:87] CommNet Thread 2 finish +I0616 15:34:10.653772 85434 epoll_comm_network.cpp:87] CommNet Thread 3 finish +I0616 15:34:10.653913 85434 global.h:43] DeleteGlobal N7oneflow27EagerJobBuildAndInferCtxMgrE +I0616 15:34:10.653932 85434 global.h:43] DeleteGlobal N7oneflow2vm19VirtualMachineScopeE +I0616 15:34:10.653986 85434 global.h:43] DeleteGlobal N7oneflow9OneflowVME +I0616 15:34:10.679136 85434 global.h:43] DeleteGlobal N7oneflow18CudnnConvAlgoCacheE +I0616 15:34:10.679169 85434 global.h:43] DeleteGlobal N7oneflow16EagerNcclCommMgrE +I0616 15:34:10.679176 85434 global.h:43] DeleteGlobal N7oneflow10ThreadPoolE +I0616 15:34:10.683662 85434 global.h:43] DeleteGlobal N7oneflow12ResourceDescE +I0616 15:34:10.683728 85434 global.h:43] DeleteGlobal N7oneflow12ResourceDescE +I0616 15:34:10.683749 85434 global.h:43] DeleteGlobal N7oneflow6device27NodeDeviceDescriptorManagerE +I0616 15:34:10.683794 85434 global.h:43] DeleteGlobal N7oneflow10RpcManagerE +I0616 15:34:10.683810 85434 global.h:43] DeleteGlobal N7oneflow10CtrlClientE +I0616 15:34:10.683827 85434 global.h:43] DeleteGlobal N7oneflow10ProcessCtxE +I0616 15:34:10.683843 85434 global.h:43] DeleteGlobal N7oneflow7EnvDescE +I0616 15:34:10.683861 85434 global.h:43] DeleteGlobal 14cudaDeviceProp diff --git a/LanguageModeling/bert-oneflow/log/default_physical_env_log/VS002/oneflow.VS002.lichunyou.log.INFO.20210616-155156.93096 b/LanguageModeling/bert-oneflow/log/default_physical_env_log/VS002/oneflow.VS002.lichunyou.log.INFO.20210616-155156.93096 new file mode 100644 index 0000000..31194ff --- /dev/null +++ b/LanguageModeling/bert-oneflow/log/default_physical_env_log/VS002/oneflow.VS002.lichunyou.log.INFO.20210616-155156.93096 @@ -0,0 +1,19 @@ +Log file created at: 2021/06/16 15:51:56 +Running on machine: VS002 +Log line format: [IWEF]mmdd hh:mm:ss.uuuuuu threadid file:line] msg +I0616 15:51:56.472827 93096 global.h:36] NewGlobal 14cudaDeviceProp +I0616 15:51:56.800246 93096 global.h:36] NewGlobal N7oneflow7EnvDescE +I0616 15:51:56.800299 93096 global.h:36] NewGlobal N7oneflow10ProcessCtxE +I0616 15:51:56.800307 93096 env_global_objects_scope.cpp:112] using rpc backend: local +I0616 15:51:56.800321 93096 global.h:36] NewGlobal N7oneflow12ResourceDescE +I0616 15:51:56.800329 93096 global.h:36] NewGlobal N7oneflow12ResourceDescE +I0616 15:51:56.806735 93096 global.h:36] NewGlobal N7oneflow10ThreadPoolE +I0616 15:51:56.807636 93096 global.h:36] NewGlobal N7oneflow16EagerNcclCommMgrE +I0616 15:51:56.807644 93096 global.h:36] NewGlobal N7oneflow18CudnnConvAlgoCacheE +I0616 15:51:56.807651 93096 global.h:36] NewGlobal N7oneflow2vm19VirtualMachineScopeE +I0616 15:51:56.807657 93096 global.h:36] NewGlobal N7oneflow9OneflowVME +I0616 15:51:56.816476 93096 global.h:36] NewGlobal N7oneflow27EagerJobBuildAndInferCtxMgrE +I0616 15:51:56.816498 93096 global.h:36] NewGlobal N7oneflow12EpollCommNetE +I0616 15:51:56.816570 93096 epoll_comm_network.cpp:63] CommNet:Epoll listening on 0.0.0.0:43218 +I0616 15:51:56.816593 93096 epoll_comm_network.cpp:198] machine 0 sockfd -1 +I0616 15:51:56.816663 93096 global.h:36] NewGlobal N7oneflow9TransportE diff --git a/LanguageModeling/bert-oneflow/log/default_physical_env_log/VS002/oneflow.VS002.lichunyou.log.INFO.20210617-132416.22441 b/LanguageModeling/bert-oneflow/log/default_physical_env_log/VS002/oneflow.VS002.lichunyou.log.INFO.20210617-132416.22441 new file mode 100644 index 0000000..d3cb554 --- /dev/null +++ b/LanguageModeling/bert-oneflow/log/default_physical_env_log/VS002/oneflow.VS002.lichunyou.log.INFO.20210617-132416.22441 @@ -0,0 +1,19 @@ +Log file created at: 2021/06/17 13:24:16 +Running on machine: VS002 +Log line format: [IWEF]mmdd hh:mm:ss.uuuuuu threadid file:line] msg +I0617 13:24:16.577369 22441 global.h:36] NewGlobal 14cudaDeviceProp +I0617 13:24:16.954938 22441 global.h:36] NewGlobal N7oneflow7EnvDescE +I0617 13:24:16.955024 22441 global.h:36] NewGlobal N7oneflow10ProcessCtxE +I0617 13:24:16.955044 22441 env_global_objects_scope.cpp:112] using rpc backend: local +I0617 13:24:16.955061 22441 global.h:36] NewGlobal N7oneflow12ResourceDescE +I0617 13:24:16.955068 22441 global.h:36] NewGlobal N7oneflow12ResourceDescE +I0617 13:24:16.961655 22441 global.h:36] NewGlobal N7oneflow10ThreadPoolE +I0617 13:24:16.962766 22441 global.h:36] NewGlobal N7oneflow16EagerNcclCommMgrE +I0617 13:24:16.962776 22441 global.h:36] NewGlobal N7oneflow18CudnnConvAlgoCacheE +I0617 13:24:16.962781 22441 global.h:36] NewGlobal N7oneflow2vm19VirtualMachineScopeE +I0617 13:24:16.962787 22441 global.h:36] NewGlobal N7oneflow9OneflowVME +I0617 13:24:16.972095 22441 global.h:36] NewGlobal N7oneflow27EagerJobBuildAndInferCtxMgrE +I0617 13:24:16.972115 22441 global.h:36] NewGlobal N7oneflow12EpollCommNetE +I0617 13:24:16.972209 22441 epoll_comm_network.cpp:63] CommNet:Epoll listening on 0.0.0.0:38143 +I0617 13:24:16.972234 22441 epoll_comm_network.cpp:198] machine 0 sockfd -1 +I0617 13:24:16.972308 22441 global.h:36] NewGlobal N7oneflow9TransportE diff --git a/LanguageModeling/bert-oneflow/log/default_physical_env_log/VS002/oneflow.VS002.lichunyou.log.INFO.20210617-132529.23158 b/LanguageModeling/bert-oneflow/log/default_physical_env_log/VS002/oneflow.VS002.lichunyou.log.INFO.20210617-132529.23158 new file mode 100644 index 0000000..d4b7253 --- /dev/null +++ b/LanguageModeling/bert-oneflow/log/default_physical_env_log/VS002/oneflow.VS002.lichunyou.log.INFO.20210617-132529.23158 @@ -0,0 +1,19 @@ +Log file created at: 2021/06/17 13:25:29 +Running on machine: VS002 +Log line format: [IWEF]mmdd hh:mm:ss.uuuuuu threadid file:line] msg +I0617 13:25:29.002583 23158 global.h:36] NewGlobal 14cudaDeviceProp +I0617 13:25:29.371557 23158 global.h:36] NewGlobal N7oneflow7EnvDescE +I0617 13:25:29.371663 23158 global.h:36] NewGlobal N7oneflow10ProcessCtxE +I0617 13:25:29.371693 23158 env_global_objects_scope.cpp:112] using rpc backend: local +I0617 13:25:29.371752 23158 global.h:36] NewGlobal N7oneflow12ResourceDescE +I0617 13:25:29.371768 23158 global.h:36] NewGlobal N7oneflow12ResourceDescE +I0617 13:25:29.378367 23158 global.h:36] NewGlobal N7oneflow10ThreadPoolE +I0617 13:25:29.379439 23158 global.h:36] NewGlobal N7oneflow16EagerNcclCommMgrE +I0617 13:25:29.379448 23158 global.h:36] NewGlobal N7oneflow18CudnnConvAlgoCacheE +I0617 13:25:29.379453 23158 global.h:36] NewGlobal N7oneflow2vm19VirtualMachineScopeE +I0617 13:25:29.379462 23158 global.h:36] NewGlobal N7oneflow9OneflowVME +I0617 13:25:29.388799 23158 global.h:36] NewGlobal N7oneflow27EagerJobBuildAndInferCtxMgrE +I0617 13:25:29.388821 23158 global.h:36] NewGlobal N7oneflow12EpollCommNetE +I0617 13:25:29.388970 23158 epoll_comm_network.cpp:63] CommNet:Epoll listening on 0.0.0.0:43347 +I0617 13:25:29.388994 23158 epoll_comm_network.cpp:198] machine 0 sockfd -1 +I0617 13:25:29.389086 23158 global.h:36] NewGlobal N7oneflow9TransportE diff --git a/LanguageModeling/bert-oneflow/log/default_physical_env_log/VS002/oneflow.VS002.lichunyou.log.INFO.20210621-105503.13951 b/LanguageModeling/bert-oneflow/log/default_physical_env_log/VS002/oneflow.VS002.lichunyou.log.INFO.20210621-105503.13951 new file mode 100644 index 0000000..3d5891a --- /dev/null +++ b/LanguageModeling/bert-oneflow/log/default_physical_env_log/VS002/oneflow.VS002.lichunyou.log.INFO.20210621-105503.13951 @@ -0,0 +1,40 @@ +Log file created at: 2021/06/21 10:55:03 +Running on machine: VS002 +Log line format: [IWEF]mmdd hh:mm:ss.uuuuuu threadid file:line] msg +I0621 10:55:03.954901 13951 global.h:36] NewGlobal 14cudaDeviceProp +I0621 10:55:04.140427 13951 global.h:36] NewGlobal N7oneflow7EnvDescE +I0621 10:55:04.140476 13951 global.h:36] NewGlobal N7oneflow10ProcessCtxE +I0621 10:55:04.140486 13951 env_global_objects_scope.cpp:112] using rpc backend: local +I0621 10:55:04.140527 13951 global.h:36] NewGlobal N7oneflow12ResourceDescE +I0621 10:55:04.140534 13951 global.h:36] NewGlobal N7oneflow12ResourceDescE +I0621 10:55:04.160535 13951 global.h:36] NewGlobal N7oneflow10ThreadPoolE +I0621 10:55:04.161613 13951 global.h:36] NewGlobal N7oneflow16EagerNcclCommMgrE +I0621 10:55:04.161623 13951 global.h:36] NewGlobal N7oneflow18CudnnConvAlgoCacheE +I0621 10:55:04.161628 13951 global.h:36] NewGlobal N7oneflow2vm19VirtualMachineScopeE +I0621 10:55:04.161634 13951 global.h:36] NewGlobal N7oneflow9OneflowVME +I0621 10:55:04.171347 13951 global.h:36] NewGlobal N7oneflow27EagerJobBuildAndInferCtxMgrE +I0621 10:55:04.171376 13951 global.h:36] NewGlobal N7oneflow12EpollCommNetE +I0621 10:55:04.171459 13951 epoll_comm_network.cpp:63] CommNet:Epoll listening on 0.0.0.0:34373 +I0621 10:55:04.171485 13951 epoll_comm_network.cpp:198] machine 0 sockfd -1 +I0621 10:55:04.171561 13951 global.h:36] NewGlobal N7oneflow9TransportE +I0621 10:55:07.442209 13951 global.h:43] DeleteGlobal N7oneflow21EnvGlobalObjectsScopeE +I0621 10:55:07.442253 13951 global.h:43] DeleteGlobal N7oneflow9TransportE +I0621 10:55:07.442427 13951 global.h:43] DeleteGlobal N7oneflow12EpollCommNetE +I0621 10:55:07.442502 13951 epoll_comm_network.cpp:87] CommNet Thread 0 finish +I0621 10:55:07.442584 13951 epoll_comm_network.cpp:87] CommNet Thread 1 finish +I0621 10:55:07.442634 13951 epoll_comm_network.cpp:87] CommNet Thread 2 finish +I0621 10:55:07.442726 13951 epoll_comm_network.cpp:87] CommNet Thread 3 finish +I0621 10:55:07.442898 13951 global.h:43] DeleteGlobal N7oneflow27EagerJobBuildAndInferCtxMgrE +I0621 10:55:07.442915 13951 global.h:43] DeleteGlobal N7oneflow2vm19VirtualMachineScopeE +I0621 10:55:07.442929 13951 global.h:43] DeleteGlobal N7oneflow9OneflowVME +I0621 10:55:07.450901 13951 global.h:43] DeleteGlobal N7oneflow18CudnnConvAlgoCacheE +I0621 10:55:07.450938 13951 global.h:43] DeleteGlobal N7oneflow16EagerNcclCommMgrE +I0621 10:55:07.450947 13951 global.h:43] DeleteGlobal N7oneflow10ThreadPoolE +I0621 10:55:07.453130 13951 global.h:43] DeleteGlobal N7oneflow12ResourceDescE +I0621 10:55:07.453140 13951 global.h:43] DeleteGlobal N7oneflow12ResourceDescE +I0621 10:55:07.453146 13951 global.h:43] DeleteGlobal N7oneflow6device27NodeDeviceDescriptorManagerE +I0621 10:55:07.453163 13951 global.h:43] DeleteGlobal N7oneflow10RpcManagerE +I0621 10:55:07.453171 13951 global.h:43] DeleteGlobal N7oneflow10CtrlClientE +I0621 10:55:07.453177 13951 global.h:43] DeleteGlobal N7oneflow10ProcessCtxE +I0621 10:55:07.453184 13951 global.h:43] DeleteGlobal N7oneflow7EnvDescE +I0621 10:55:07.453193 13951 global.h:43] DeleteGlobal 14cudaDeviceProp diff --git a/LanguageModeling/bert-oneflow/log/default_physical_env_log/VS002/oneflow.VS002.lichunyou.log.INFO.20210621-124230.196962 b/LanguageModeling/bert-oneflow/log/default_physical_env_log/VS002/oneflow.VS002.lichunyou.log.INFO.20210621-124230.196962 new file mode 100644 index 0000000..1b4a1e5 --- /dev/null +++ b/LanguageModeling/bert-oneflow/log/default_physical_env_log/VS002/oneflow.VS002.lichunyou.log.INFO.20210621-124230.196962 @@ -0,0 +1,19 @@ +Log file created at: 2021/06/21 12:42:30 +Running on machine: VS002 +Log line format: [IWEF]mmdd hh:mm:ss.uuuuuu threadid file:line] msg +I0621 12:42:30.981474 196962 global.h:36] NewGlobal 14cudaDeviceProp +I0621 12:42:31.055187 196962 global.h:36] NewGlobal N7oneflow7EnvDescE +I0621 12:42:31.055233 196962 global.h:36] NewGlobal N7oneflow10ProcessCtxE +I0621 12:42:31.055241 196962 env_global_objects_scope.cpp:112] using rpc backend: local +I0621 12:42:31.055275 196962 global.h:36] NewGlobal N7oneflow12ResourceDescE +I0621 12:42:31.055284 196962 global.h:36] NewGlobal N7oneflow12ResourceDescE +I0621 12:42:31.064765 196962 global.h:36] NewGlobal N7oneflow10ThreadPoolE +I0621 12:42:31.066110 196962 global.h:36] NewGlobal N7oneflow16EagerNcclCommMgrE +I0621 12:42:31.066120 196962 global.h:36] NewGlobal N7oneflow18CudnnConvAlgoCacheE +I0621 12:42:31.066126 196962 global.h:36] NewGlobal N7oneflow2vm19VirtualMachineScopeE +I0621 12:42:31.066133 196962 global.h:36] NewGlobal N7oneflow9OneflowVME +I0621 12:42:31.078323 196962 global.h:36] NewGlobal N7oneflow27EagerJobBuildAndInferCtxMgrE +I0621 12:42:31.078349 196962 global.h:36] NewGlobal N7oneflow12EpollCommNetE +I0621 12:42:31.078440 196962 epoll_comm_network.cpp:63] CommNet:Epoll listening on 0.0.0.0:41359 +I0621 12:42:31.078474 196962 epoll_comm_network.cpp:198] machine 0 sockfd -1 +I0621 12:42:31.078570 196962 global.h:36] NewGlobal N7oneflow9TransportE diff --git a/LanguageModeling/bert-oneflow/log/default_physical_env_log/VS002/oneflow.VS002.lichunyou.log.INFO.20210621-124337.198163 b/LanguageModeling/bert-oneflow/log/default_physical_env_log/VS002/oneflow.VS002.lichunyou.log.INFO.20210621-124337.198163 new file mode 100644 index 0000000..fdf5b2a --- /dev/null +++ b/LanguageModeling/bert-oneflow/log/default_physical_env_log/VS002/oneflow.VS002.lichunyou.log.INFO.20210621-124337.198163 @@ -0,0 +1,19 @@ +Log file created at: 2021/06/21 12:43:37 +Running on machine: VS002 +Log line format: [IWEF]mmdd hh:mm:ss.uuuuuu threadid file:line] msg +I0621 12:43:37.827342 198163 global.h:36] NewGlobal 14cudaDeviceProp +I0621 12:43:37.901626 198163 global.h:36] NewGlobal N7oneflow7EnvDescE +I0621 12:43:37.901664 198163 global.h:36] NewGlobal N7oneflow10ProcessCtxE +I0621 12:43:37.901672 198163 env_global_objects_scope.cpp:112] using rpc backend: local +I0621 12:43:37.901686 198163 global.h:36] NewGlobal N7oneflow12ResourceDescE +I0621 12:43:37.901695 198163 global.h:36] NewGlobal N7oneflow12ResourceDescE +I0621 12:43:37.909188 198163 global.h:36] NewGlobal N7oneflow10ThreadPoolE +I0621 12:43:37.910650 198163 global.h:36] NewGlobal N7oneflow16EagerNcclCommMgrE +I0621 12:43:37.910661 198163 global.h:36] NewGlobal N7oneflow18CudnnConvAlgoCacheE +I0621 12:43:37.910673 198163 global.h:36] NewGlobal N7oneflow2vm19VirtualMachineScopeE +I0621 12:43:37.910681 198163 global.h:36] NewGlobal N7oneflow9OneflowVME +I0621 12:43:37.924012 198163 global.h:36] NewGlobal N7oneflow27EagerJobBuildAndInferCtxMgrE +I0621 12:43:37.924034 198163 global.h:36] NewGlobal N7oneflow12EpollCommNetE +I0621 12:43:37.924126 198163 epoll_comm_network.cpp:63] CommNet:Epoll listening on 0.0.0.0:44248 +I0621 12:43:37.924149 198163 epoll_comm_network.cpp:198] machine 0 sockfd -1 +I0621 12:43:37.924260 198163 global.h:36] NewGlobal N7oneflow9TransportE diff --git a/LanguageModeling/bert-oneflow/log/default_physical_env_log/VS002/oneflow.VS002.lichunyou.log.INFO.20210621-130530.205914 b/LanguageModeling/bert-oneflow/log/default_physical_env_log/VS002/oneflow.VS002.lichunyou.log.INFO.20210621-130530.205914 new file mode 100644 index 0000000..c10c95f --- /dev/null +++ b/LanguageModeling/bert-oneflow/log/default_physical_env_log/VS002/oneflow.VS002.lichunyou.log.INFO.20210621-130530.205914 @@ -0,0 +1,19 @@ +Log file created at: 2021/06/21 13:05:30 +Running on machine: VS002 +Log line format: [IWEF]mmdd hh:mm:ss.uuuuuu threadid file:line] msg +I0621 13:05:30.001822 205914 global.h:36] NewGlobal 14cudaDeviceProp +I0621 13:05:30.438489 205914 global.h:36] NewGlobal N7oneflow7EnvDescE +I0621 13:05:30.438539 205914 global.h:36] NewGlobal N7oneflow10ProcessCtxE +I0621 13:05:30.438547 205914 env_global_objects_scope.cpp:112] using rpc backend: local +I0621 13:05:30.438560 205914 global.h:36] NewGlobal N7oneflow12ResourceDescE +I0621 13:05:30.438567 205914 global.h:36] NewGlobal N7oneflow12ResourceDescE +I0621 13:05:30.445070 205914 global.h:36] NewGlobal N7oneflow10ThreadPoolE +I0621 13:05:30.446085 205914 global.h:36] NewGlobal N7oneflow16EagerNcclCommMgrE +I0621 13:05:30.446094 205914 global.h:36] NewGlobal N7oneflow18CudnnConvAlgoCacheE +I0621 13:05:30.446099 205914 global.h:36] NewGlobal N7oneflow2vm19VirtualMachineScopeE +I0621 13:05:30.446105 205914 global.h:36] NewGlobal N7oneflow9OneflowVME +I0621 13:05:30.454943 205914 global.h:36] NewGlobal N7oneflow27EagerJobBuildAndInferCtxMgrE +I0621 13:05:30.454964 205914 global.h:36] NewGlobal N7oneflow12EpollCommNetE +I0621 13:05:30.455040 205914 epoll_comm_network.cpp:63] CommNet:Epoll listening on 0.0.0.0:36618 +I0621 13:05:30.455072 205914 epoll_comm_network.cpp:198] machine 0 sockfd -1 +I0621 13:05:30.455154 205914 global.h:36] NewGlobal N7oneflow9TransportE diff --git a/LanguageModeling/bert-oneflow/log/default_physical_env_log/VS002/oneflow.VS002.lichunyou.log.INFO.20210621-140937.227330 b/LanguageModeling/bert-oneflow/log/default_physical_env_log/VS002/oneflow.VS002.lichunyou.log.INFO.20210621-140937.227330 new file mode 100644 index 0000000..f6f8ba4 --- /dev/null +++ b/LanguageModeling/bert-oneflow/log/default_physical_env_log/VS002/oneflow.VS002.lichunyou.log.INFO.20210621-140937.227330 @@ -0,0 +1,19 @@ +Log file created at: 2021/06/21 14:09:37 +Running on machine: VS002 +Log line format: [IWEF]mmdd hh:mm:ss.uuuuuu threadid file:line] msg +I0621 14:09:37.440506 227330 global.h:36] NewGlobal 14cudaDeviceProp +I0621 14:09:37.502467 227330 global.h:36] NewGlobal N7oneflow7EnvDescE +I0621 14:09:37.502503 227330 global.h:36] NewGlobal N7oneflow10ProcessCtxE +I0621 14:09:37.502511 227330 env_global_objects_scope.cpp:112] using rpc backend: local +I0621 14:09:37.502522 227330 global.h:36] NewGlobal N7oneflow12ResourceDescE +I0621 14:09:37.502528 227330 global.h:36] NewGlobal N7oneflow12ResourceDescE +I0621 14:09:37.508935 227330 global.h:36] NewGlobal N7oneflow10ThreadPoolE +I0621 14:09:37.510071 227330 global.h:36] NewGlobal N7oneflow16EagerNcclCommMgrE +I0621 14:09:37.510080 227330 global.h:36] NewGlobal N7oneflow18CudnnConvAlgoCacheE +I0621 14:09:37.510084 227330 global.h:36] NewGlobal N7oneflow2vm19VirtualMachineScopeE +I0621 14:09:37.510090 227330 global.h:36] NewGlobal N7oneflow9OneflowVME +I0621 14:09:37.520928 227330 global.h:36] NewGlobal N7oneflow27EagerJobBuildAndInferCtxMgrE +I0621 14:09:37.520952 227330 global.h:36] NewGlobal N7oneflow12EpollCommNetE +I0621 14:09:37.521034 227330 epoll_comm_network.cpp:63] CommNet:Epoll listening on 0.0.0.0:35867 +I0621 14:09:37.521054 227330 epoll_comm_network.cpp:198] machine 0 sockfd -1 +I0621 14:09:37.521147 227330 global.h:36] NewGlobal N7oneflow9TransportE diff --git a/LanguageModeling/bert-oneflow/log/default_physical_env_log/VS002/oneflow.VS002.lichunyou.log.INFO.20210621-141631.1046 b/LanguageModeling/bert-oneflow/log/default_physical_env_log/VS002/oneflow.VS002.lichunyou.log.INFO.20210621-141631.1046 new file mode 100644 index 0000000..d317af8 --- /dev/null +++ b/LanguageModeling/bert-oneflow/log/default_physical_env_log/VS002/oneflow.VS002.lichunyou.log.INFO.20210621-141631.1046 @@ -0,0 +1,19 @@ +Log file created at: 2021/06/21 14:16:31 +Running on machine: VS002 +Log line format: [IWEF]mmdd hh:mm:ss.uuuuuu threadid file:line] msg +I0621 14:16:31.812388 1046 global.h:36] NewGlobal 14cudaDeviceProp +I0621 14:16:32.319845 1046 global.h:36] NewGlobal N7oneflow7EnvDescE +I0621 14:16:32.319926 1046 global.h:36] NewGlobal N7oneflow10ProcessCtxE +I0621 14:16:32.319934 1046 env_global_objects_scope.cpp:112] using rpc backend: local +I0621 14:16:32.319977 1046 global.h:36] NewGlobal N7oneflow12ResourceDescE +I0621 14:16:32.319989 1046 global.h:36] NewGlobal N7oneflow12ResourceDescE +I0621 14:16:32.326699 1046 global.h:36] NewGlobal N7oneflow10ThreadPoolE +I0621 14:16:32.327786 1046 global.h:36] NewGlobal N7oneflow16EagerNcclCommMgrE +I0621 14:16:32.327795 1046 global.h:36] NewGlobal N7oneflow18CudnnConvAlgoCacheE +I0621 14:16:32.327802 1046 global.h:36] NewGlobal N7oneflow2vm19VirtualMachineScopeE +I0621 14:16:32.327808 1046 global.h:36] NewGlobal N7oneflow9OneflowVME +I0621 14:16:32.337846 1046 global.h:36] NewGlobal N7oneflow27EagerJobBuildAndInferCtxMgrE +I0621 14:16:32.337867 1046 global.h:36] NewGlobal N7oneflow12EpollCommNetE +I0621 14:16:32.338035 1046 epoll_comm_network.cpp:63] CommNet:Epoll listening on 0.0.0.0:39423 +I0621 14:16:32.338060 1046 epoll_comm_network.cpp:198] machine 0 sockfd -1 +I0621 14:16:32.338157 1046 global.h:36] NewGlobal N7oneflow9TransportE diff --git a/LanguageModeling/bert-oneflow/log/default_physical_env_log/VS002/oneflow.VS002.lichunyou.log.INFO.20210621-141811.2328 b/LanguageModeling/bert-oneflow/log/default_physical_env_log/VS002/oneflow.VS002.lichunyou.log.INFO.20210621-141811.2328 new file mode 100644 index 0000000..7648ecf --- /dev/null +++ b/LanguageModeling/bert-oneflow/log/default_physical_env_log/VS002/oneflow.VS002.lichunyou.log.INFO.20210621-141811.2328 @@ -0,0 +1,19 @@ +Log file created at: 2021/06/21 14:18:11 +Running on machine: VS002 +Log line format: [IWEF]mmdd hh:mm:ss.uuuuuu threadid file:line] msg +I0621 14:18:11.731659 2328 global.h:36] NewGlobal 14cudaDeviceProp +I0621 14:18:12.142030 2328 global.h:36] NewGlobal N7oneflow7EnvDescE +I0621 14:18:12.142112 2328 global.h:36] NewGlobal N7oneflow10ProcessCtxE +I0621 14:18:12.142119 2328 env_global_objects_scope.cpp:112] using rpc backend: local +I0621 14:18:12.142175 2328 global.h:36] NewGlobal N7oneflow12ResourceDescE +I0621 14:18:12.142189 2328 global.h:36] NewGlobal N7oneflow12ResourceDescE +I0621 14:18:12.148756 2328 global.h:36] NewGlobal N7oneflow10ThreadPoolE +I0621 14:18:12.149766 2328 global.h:36] NewGlobal N7oneflow16EagerNcclCommMgrE +I0621 14:18:12.149775 2328 global.h:36] NewGlobal N7oneflow18CudnnConvAlgoCacheE +I0621 14:18:12.149778 2328 global.h:36] NewGlobal N7oneflow2vm19VirtualMachineScopeE +I0621 14:18:12.149785 2328 global.h:36] NewGlobal N7oneflow9OneflowVME +I0621 14:18:12.159430 2328 global.h:36] NewGlobal N7oneflow27EagerJobBuildAndInferCtxMgrE +I0621 14:18:12.159451 2328 global.h:36] NewGlobal N7oneflow12EpollCommNetE +I0621 14:18:12.159600 2328 epoll_comm_network.cpp:63] CommNet:Epoll listening on 0.0.0.0:44480 +I0621 14:18:12.159623 2328 epoll_comm_network.cpp:198] machine 0 sockfd -1 +I0621 14:18:12.159706 2328 global.h:36] NewGlobal N7oneflow9TransportE diff --git a/LanguageModeling/bert-oneflow/main.py b/LanguageModeling/bert-oneflow/main.py new file mode 100644 index 0000000..5d7d81f --- /dev/null +++ b/LanguageModeling/bert-oneflow/main.py @@ -0,0 +1,83 @@ +import argparse + +from torch.utils.data import DataLoader +# from oneflow.utils.data import DataLoader +import oneflow.experimental as flow +from model import BERT +from trainer import BERTTrainer +from dataset import BERTDataset, WordVocab + +import numpy as np + +# eager mode +flow.env.init() +flow.enable_eager_execution() + +def train(): + + parser = argparse.ArgumentParser() + + parser.add_argument("-c", "--train_dataset", required=False, type=str, default='data/corpus.small', help="train dataset for train bert") + parser.add_argument("-t", "--test_dataset", type=str, default='data/corpus.small', help="test set for evaluate train set") + parser.add_argument("-v", "--vocab_path", required=False, default='data/vocab.small', type=str, help="built vocab model path with bert-vocab") + parser.add_argument("-o", "--output_path", required=False, default='output/bert.model', type=str, help="ex)output/bert.model") + + parser.add_argument("-hs", "--hidden", type=int, default=256, help="hidden size of transformer model") + parser.add_argument("-l", "--layers", type=int, default=8, help="number of layers") + parser.add_argument("-a", "--attn_heads", type=int, default=8, help="number of attention heads") + parser.add_argument("-s", "--seq_len", type=int, default=20, help="maximum sequence len") + + parser.add_argument("-b", "--batch_size", type=int, default=32, help="number of batch_size") + parser.add_argument("-e", "--epochs", type=int, default=10, help="number of epochs") + parser.add_argument("-w", "--num_workers", type=int, default=5, help="dataloader worker size") + + parser.add_argument("--with_cuda", type=bool, default=True, help="training with CUDA: true, or false") + parser.add_argument("--corpus_lines", type=int, default=None, help="total number of lines in corpus") + parser.add_argument("--cuda_devices", type=int, nargs='+', default=None, help="CUDA device ids") + parser.add_argument("--on_memory", type=bool, default=True, help="Loading on memory: true or false") + + parser.add_argument("--lr", type=float, default=1e-3, help="learning rate of adam") + parser.add_argument("--adam_weight_decay", type=float, default=0.01, help="weight_decay of adam") + parser.add_argument("--adam_beta1", type=float, default=0.9, help="adam first beta value") + parser.add_argument("--adam_beta2", type=float, default=0.999, help="adam first beta value") + + args = parser.parse_args() + + print("Loading Vocab", args.vocab_path) + vocab = WordVocab.load_vocab(args.vocab_path) + print("Vocab Size: ", len(vocab)) + + print("Loading Train Dataset", args.train_dataset) + train_dataset = BERTDataset(args.train_dataset, vocab, seq_len=args.seq_len, + corpus_lines=args.corpus_lines, on_memory=args.on_memory) + + print("Loading Test Dataset", args.test_dataset) + test_dataset = BERTDataset(args.test_dataset, vocab, seq_len=args.seq_len, on_memory=args.on_memory) \ + if args.test_dataset is not None else None + + print("Creating Dataloader") + train_data_loader = DataLoader(train_dataset, batch_size=args.batch_size, num_workers=args.num_workers) + test_data_loader = DataLoader(test_dataset, batch_size=args.batch_size, num_workers=args.num_workers) \ + if test_dataset is not None else None + + print("Building BERT model") + bert = BERT(len(vocab), hidden=args.hidden, n_layers=args.layers, attn_heads=args.attn_heads) + + print("Creating BERT Trainer") + trainer = BERTTrainer(bert, len(vocab), train_dataloader=train_data_loader, test_dataloader=test_data_loader, + lr=args.lr, betas=(args.adam_beta1, args.adam_beta2), weight_decay=args.adam_weight_decay, + with_cuda=args.with_cuda, cuda_devices=args.cuda_devices, log_freq=10) + + print("Trainer build finished!!!\n") + + print("Training Start") + for epoch in range(args.epochs): + trainer.train(epoch) + trainer.save(epoch, args.output_path) + + if test_data_loader is not None: + trainer.test(epoch) + + + +train() diff --git a/LanguageModeling/bert-oneflow/model/attention/multi_head.py b/LanguageModeling/bert-oneflow/model/attention/multi_head.py new file mode 100644 index 0000000..4fc0023 --- /dev/null +++ b/LanguageModeling/bert-oneflow/model/attention/multi_head.py @@ -0,0 +1,40 @@ +import oneflow.experimental.nn as nn +import oneflow as flow +from model.attention.single import Attention + + +class MultiHeadedAttention(nn.Module): + """ + Take in model size and number of heads. + """ + + def __init__(self, h, d_model, dropout=0.1): + super().__init__() + assert d_model % h == 0 + + # We assume d_v always equals d_k + self.d_k = d_model // h + self.h = h + self.linear_layers = nn.ModuleList([nn.Linear(d_model, d_model) for _ in range(3)]) + self.output_linear = nn.Linear(d_model, d_model) + self.attention = Attention() + + self.dropout = nn.Dropout(p=dropout) + + def forward(self, query, key, value, mask=None): + batch_size = query.size(0) # 16 + + + query, key, value = [l(x).reshape(shape=[batch_size, -1, self.h, self.d_k]).permute(0, 2, 1, 3) + for l, x in zip(self.linear_layers, (query, key, value))] + # # query,key,value shape >> flow.Size([16, 8, 20, 32]) + + + # 2) Apply attention on all the projected vectors in batch. + x, attn = self.attention(query, key, value, mask, self.dropout) + + # 3) "Concat" using a view and apply a final linear. + res = x.transpose(1, 2).reshape(shape = [batch_size, -1, self.h * self.d_k]) + res = self.output_linear(res) + return res + diff --git a/LanguageModeling/bert-oneflow/model/attention/single.py b/LanguageModeling/bert-oneflow/model/attention/single.py new file mode 100644 index 0000000..c028a15 --- /dev/null +++ b/LanguageModeling/bert-oneflow/model/attention/single.py @@ -0,0 +1,29 @@ +import oneflow.experimental.nn as nn +import oneflow.experimental as flow +import math +import numpy as np + +class Attention(nn.Module): + """ + Compute 'Scaled Dot Product Attention + """ + + def __init__(self): + super().__init__() + self.softmax = flow.nn.Softmax(dim = -1) + + def forward(self, query, key, value, mask=None, dropout=None): # q k v shape >> flow.Size([16, 8, 20, 32]) + x = flow.matmul(query, key.transpose(-2, -1)) + scores = x / math.sqrt(query.size(-1)) + + if mask is not None: + # scores = scores.masked_fill(mask == 0, -1e9) + mask = flow.Tensor((mask.numpy() == 0).astype(np.int8), dtype=flow.int, device=scores.device) + scores = scores.masked_fill(mask, -1e9) + + p_attn = self.softmax(scores) + + if dropout is not None: + p_attn = dropout(p_attn) + + return flow.matmul(p_attn, value), p_attn diff --git a/LanguageModeling/bert-oneflow/model/bert.py b/LanguageModeling/bert-oneflow/model/bert.py new file mode 100644 index 0000000..ef5b60d --- /dev/null +++ b/LanguageModeling/bert-oneflow/model/bert.py @@ -0,0 +1,50 @@ +import oneflow.experimental.nn as nn + +from model.transformer import TransformerBlock +from model.embedding.bert import BERTEmbedding +import numpy as np +import oneflow.experimental as flow + + +class BERT(nn.Module): + """ + BERT model : Bidirectional Encoder Representations from Transformers. + """ + + def __init__(self, vocab_size, hidden=768, n_layers=12, attn_heads=12, dropout=0.1): + """ + :param vocab_size: vocab_size of total words + :param hidden: BERT model hidden size + :param n_layers: numbers of Transformer blocks(layers) + :param attn_heads: number of attention heads + :param dropout: dropout rate + """ + + super().__init__() + self.hidden = hidden + self.n_layers = n_layers + self.attn_heads = attn_heads + + # paper noted they used 4*hidden_size for ff_network_hidden_size + self.feed_forward_hidden = hidden * 4 + + # embedding for BERT, sum of positional, segment, token embeddings + self.embedding = BERTEmbedding(vocab_size=vocab_size, embed_size=hidden) + + # multi-layers transformer blocks, deep network + self.transformer_blocks = nn.ModuleList( + [TransformerBlock(hidden, attn_heads, hidden * 4, dropout) for _ in range(n_layers)]) + + def forward(self, x, segment_info): # x.shape >> flow.Size([16, 20]) + # attention masking for padded token + + mask = (x > 0).unsqueeze(1).repeat(sizes=(1, x.shape[1], 1)).unsqueeze(1).repeat(sizes=(1, 8, 1, 1)) + + # embedding the indexed sequence to sequence of vectors + x = self.embedding(x, segment_info) + + # running over multiple transformer blocks + for transformer in self.transformer_blocks: + x = transformer.forward(x, mask) + + return x diff --git a/LanguageModeling/bert-oneflow/model/embedding/bert.py b/LanguageModeling/bert-oneflow/model/embedding/bert.py new file mode 100644 index 0000000..8cdd737 --- /dev/null +++ b/LanguageModeling/bert-oneflow/model/embedding/bert.py @@ -0,0 +1,68 @@ +import oneflow.experimental as flow +import oneflow.experimental.nn as nn +import numpy as np +import math + + +class TokenEmbedding(nn.Embedding): + def __init__(self, vocab_size, embed_size=512): + super().__init__(vocab_size, embed_size, padding_idx=0) + +class PositionalEmbedding(nn.Module): + + def __init__(self, d_model, max_len=512): + super().__init__() + + # Compute the positional encodings once in log space. + pe = flow.zeros(size=(max_len, d_model), requires_grad=False) + + position = flow.arange(0, max_len, dtype=flow.float).unsqueeze(1) + div_term = (flow.arange(0, d_model, 2, dtype=flow.float)* -(math.log(10000.0) / d_model)).exp() + + pe[:, 0::2] = flow.sin(position * div_term) + pe[:, 1::2] = flow.cos(position * div_term) + + pe = pe.unsqueeze(0) + self.register_buffer('pe', flow.Tensor(pe)) + + def forward(self, x): + return self.pe[:, :x.size(1)] + + +class SegmentEmbedding(nn.Embedding): + def __init__(self, embed_size=512): + super().__init__(3, embed_size, padding_idx=0) + + +class BERTEmbedding(nn.Module): + """ + BERT Embedding which is consisted with under features + 1. TokenEmbedding : normal embedding matrix + 2. PositionalEmbedding : adding positional information using sin, cos + 2. SegmentEmbedding : adding sentence segment info, (sent_A:1, sent_B:2) + + sum of all these features are output of BERTEmbedding + """ + + def __init__(self, vocab_size, embed_size, dropout=0): + """ + :param vocab_size: total vocab size + :param embed_size: embedding size of token embedding + :param dropout: dropout rate + """ + super().__init__() + self.token = TokenEmbedding(vocab_size=vocab_size, embed_size=embed_size) # shape >> flow.Size([16, 20, 256]) + self.position = PositionalEmbedding(d_model=self.token.embedding_dim) # shape >> flow.Size([1, 20, 256]) + self.segment = SegmentEmbedding(embed_size=self.token.embedding_dim) # shape >> flow.Size([16, 20, 256]) + + + self.dropout = flow.nn.Dropout(p=dropout) + self.embed_size = embed_size + + def forward(self, sequence, segment_label): # sequence/segment_label .shape >>> flow.Size([16, 20]) + + + x = self.segment(segment_label) + self.token(sequence) + self.position(sequence) + + + return self.dropout(x) diff --git a/LanguageModeling/bert-oneflow/model/language_model.py b/LanguageModeling/bert-oneflow/model/language_model.py new file mode 100644 index 0000000..3012865 --- /dev/null +++ b/LanguageModeling/bert-oneflow/model/language_model.py @@ -0,0 +1,62 @@ +import oneflow.experimental as flow +import oneflow.experimental.nn as nn +from model.bert import BERT +import numpy as np + + +class BERTLM(nn.Module): + """ + BERT Language Model + Next Sentence Prediction Model + Masked Language Model + """ + + def __init__(self, bert: BERT, vocab_size): + """ + :param bert: BERT model which should be trained + :param vocab_size: total vocab size for masked_lm + """ + + super().__init__() + self.bert = bert + self.next_sentence = NextSentencePrediction(self.bert.hidden) + self.mask_lm = MaskedLanguageModel(self.bert.hidden, vocab_size) + + def forward(self, x, segment_label): + + x = self.bert(x, segment_label) + return self.next_sentence(x), self.mask_lm(x) + + +class NextSentencePrediction(nn.Module): + """ + 2-class classification model : is_next, is_not_next + """ + + def __init__(self, hidden): + """ + :param hidden: BERT model output size + """ + super().__init__() + self.linear = nn.Linear(hidden, 2) + self.softmax = nn.LogSoftmax(dim=-1) + + def forward(self, x): # x.shape >> flow.Size([16, 20, 256]) + return self.softmax(self.linear(x[:, 0])) + +class MaskedLanguageModel(nn.Module): + """ + predicting origin token from masked input sequence + n-class classification problem, n-class = vocab_size + """ + + def __init__(self, hidden, vocab_size): + """ + :param hidden: output size of BERT model + :param vocab_size: total vocab size + """ + super().__init__() + self.linear = nn.Linear(hidden, vocab_size) + self.softmax = nn.LogSoftmax(dim=-1) + + def forward(self, x): + return self.softmax(self.linear(x)) diff --git a/LanguageModeling/bert-oneflow/model/transformer.py b/LanguageModeling/bert-oneflow/model/transformer.py new file mode 100644 index 0000000..c0ac0a5 --- /dev/null +++ b/LanguageModeling/bert-oneflow/model/transformer.py @@ -0,0 +1,32 @@ +import oneflow.experimental.nn as nn +import oneflow.experimental as flow + +from model.attention.multi_head import MultiHeadedAttention +from model.utils.sublayer import SublayerConnection +from model.utils.feed_forward import PositionwiseFeedForward + +class TransformerBlock(nn.Module): + """ + Bidirectional Encoder = Transformer (self-attention) + Transformer = MultiHead_Attention + Feed_Forward with sublayer connection + """ + + def __init__(self, hidden, attn_heads, feed_forward_hidden, dropout): + """ + :param hidden: hidden size of transformer + :param attn_heads: head sizes of multi-head attention + :param feed_forward_hidden: feed_forward_hidden, usually 4*hidden_size + :param dropout: dropout rate + """ + super().__init__() + self.attention = MultiHeadedAttention(h=attn_heads, d_model=hidden) + self.feed_forward = PositionwiseFeedForward(d_model=hidden, d_ff=feed_forward_hidden, dropout=dropout) + self.input_sublayer = SublayerConnection(size=hidden, dropout=dropout) + self.output_sublayer = SublayerConnection(size=hidden, dropout=dropout) + self.dropout = nn.Dropout(p=dropout) + + def forward(self, x, mask): + + x = self.input_sublayer(x, lambda _x: self.attention.forward(_x, _x, _x, mask=mask)) + x = self.output_sublayer(x, self.feed_forward) + return self.dropout(x) diff --git a/LanguageModeling/bert-oneflow/model/utils/feed_forward.py b/LanguageModeling/bert-oneflow/model/utils/feed_forward.py new file mode 100644 index 0000000..1bb84d9 --- /dev/null +++ b/LanguageModeling/bert-oneflow/model/utils/feed_forward.py @@ -0,0 +1,19 @@ +import oneflow.experimental.nn as nn +from model.utils.gelu import GELU + + +class PositionwiseFeedForward(nn.Module): + "Implements FFN equation." + + def __init__(self, d_model, d_ff, dropout=0.1): + super(PositionwiseFeedForward, self).__init__() + # d_model,d_ff >> 256,1024 + self.w_1 = nn.Linear(d_model, d_ff) + self.w_2 = nn.Linear(d_ff, d_model) + self.dropout = nn.Dropout(dropout) + self.activation = GELU() + # self.activation = nn.GELU() # NOTE: torch原实现为调用GELU()、此处直接可用已有module(nn.GELU)代替 + + def forward(self, x): # input/output shape >> flow.Size([16, 20, 256]) + return self.w_2(self.dropout(self.activation(self.w_1(x)))) + diff --git a/LanguageModeling/bert-oneflow/model/utils/gelu.py b/LanguageModeling/bert-oneflow/model/utils/gelu.py new file mode 100644 index 0000000..d08507c --- /dev/null +++ b/LanguageModeling/bert-oneflow/model/utils/gelu.py @@ -0,0 +1,14 @@ +import oneflow.experimental.nn as nn +import oneflow.experimental as flow +import math + +class GELU(nn.Module): + """ + Paper Section 3.4, last paragraph notice that BERT used the GELU instead of RELU + """ + def __init__(self): + super().__init__() + + def forward(self, x): + tmp = flow.Tensor([math.sqrt(2 / math.pi)], device=x.device) + return 0.5 * x * (1 + flow.tanh(tmp) * (x + 0.044715 * x.pow(3.0))) diff --git a/LanguageModeling/bert-oneflow/model/utils/layer_norm.py b/LanguageModeling/bert-oneflow/model/utils/layer_norm.py new file mode 100644 index 0000000..fee8e2e --- /dev/null +++ b/LanguageModeling/bert-oneflow/model/utils/layer_norm.py @@ -0,0 +1,19 @@ +import oneflow.experimental.nn as nn +import oneflow.experimental as flow + + +class LayerNorm(nn.Module): + "Construct a layernorm module (See citation for details)." + + def __init__(self, features, eps=1e-6): + super(LayerNorm, self).__init__() + self.eps = eps + self.a_2 = nn.Parameter(flow.Tensor(flow.ones(features, dtype=flow.float32))) + self.b_2 = nn.Parameter(flow.Tensor(flow.zeros(features, dtype=flow.float32))) + + + def forward(self, x): # x input/output >> shape flow.Size([16, 20, 256]) + mean = x.mean(-1, keepdim=True) + + std = x.std(dim=-1, keepdim=True) + return self.a_2 * (x - mean) / (std + self.eps) + self.b_2 diff --git a/LanguageModeling/bert-oneflow/model/utils/sublayer.py b/LanguageModeling/bert-oneflow/model/utils/sublayer.py new file mode 100644 index 0000000..85529dd --- /dev/null +++ b/LanguageModeling/bert-oneflow/model/utils/sublayer.py @@ -0,0 +1,19 @@ +import oneflow.experimental.nn as nn +from model.utils.layer_norm import LayerNorm + + +class SublayerConnection(nn.Module): + """ + A residual connection followed by a layer norm. + Note for code simplicity the norm is first as opposed to last. + """ + + def __init__(self, size, dropout): + super(SublayerConnection, self).__init__() + self.norm = LayerNorm(size) + self.dropout = nn.Dropout(dropout) + + def forward(self, x, sublayer): + + return x + self.dropout(sublayer(self.norm(x))) + # return x + self.dropout(sublayer(x)) diff --git a/LanguageModeling/bert-oneflow/requirements.txt b/LanguageModeling/bert-oneflow/requirements.txt new file mode 100644 index 0000000..888a50f --- /dev/null +++ b/LanguageModeling/bert-oneflow/requirements.txt @@ -0,0 +1,3 @@ +tqdm +numpy +torch==0.4.1.post2 \ No newline at end of file diff --git a/LanguageModeling/bert-oneflow/test.py b/LanguageModeling/bert-oneflow/test.py new file mode 100644 index 0000000..8083a55 --- /dev/null +++ b/LanguageModeling/bert-oneflow/test.py @@ -0,0 +1,73 @@ +import argparse + +from torch.utils.data import DataLoader +# from oneflow.utils.data import DataLoader +import oneflow as flow +from model import BERT +from trainer import BERTTrainer +from dataset import BERTDataset, WordVocab + +# eager mode +flow.enable_eager_execution() + +def test(): + + parser = argparse.ArgumentParser() + + parser.add_argument("-c", "--train_dataset", required=False, type=str, default='data/corpus.small', help="train dataset for train bert") + parser.add_argument("-t", "--test_dataset", type=str, default='data/corpus.small', help="test set for evaluate train set") + parser.add_argument("-v", "--vocab_path", required=False, default='data/vocab.small', type=str, help="built vocab model path with bert-vocab") + parser.add_argument("-o", "--output_path", required=False, default='output/bert.model', type=str, help="ex)output/bert.model") + + parser.add_argument("-hs", "--hidden", type=int, default=256, help="hidden size of transformer model") + parser.add_argument("-l", "--layers", type=int, default=8, help="number of layers") + parser.add_argument("-a", "--attn_heads", type=int, default=8, help="number of attention heads") + parser.add_argument("-s", "--seq_len", type=int, default=20, help="maximum sequence len") + + parser.add_argument("-b", "--batch_size", type=int, default=16, help="number of batch_size") + parser.add_argument("-e", "--epochs", type=int, default=10, help="number of epochs") + parser.add_argument("-w", "--num_workers", type=int, default=5, help="dataloader worker size") + + parser.add_argument("--with_cuda", type=bool, default=True, help="training with CUDA: true, or false") + parser.add_argument("--corpus_lines", type=int, default=None, help="total number of lines in corpus") + parser.add_argument("--cuda_devices", type=int, nargs='+', default=None, help="CUDA device ids") + parser.add_argument("--on_memory", type=bool, default=True, help="Loading on memory: true or false") + + parser.add_argument("--lr", type=float, default=1e-3, help="learning rate of adam") + parser.add_argument("--adam_weight_decay", type=float, default=0.01, help="weight_decay of adam") + parser.add_argument("--adam_beta1", type=float, default=0.9, help="adam first beta value") + parser.add_argument("--adam_beta2", type=float, default=0.999, help="adam first beta value") + + args = parser.parse_args() + + print("Loading Vocab", args.vocab_path) + vocab = WordVocab.load_vocab(args.vocab_path) + print("Vocab Size: ", len(vocab)) + + print("Loading Train Dataset", args.train_dataset) + train_dataset = BERTDataset(args.train_dataset, vocab, seq_len=args.seq_len, + corpus_lines=args.corpus_lines, on_memory=args.on_memory) + + print("Loading Test Dataset", args.test_dataset) + test_dataset = BERTDataset(args.test_dataset, vocab, seq_len=args.seq_len, on_memory=args.on_memory) \ + if args.test_dataset is not None else None + + print("Creating Dataloader") + train_data_loader = DataLoader(train_dataset, batch_size=args.batch_size, num_workers=args.num_workers) + test_data_loader = DataLoader(test_dataset, batch_size=args.batch_size, num_workers=args.num_workers) \ + if test_dataset is not None else None + + print("Building BERT model") + bert = BERT(len(vocab), hidden=args.hidden, n_layers=args.layers, attn_heads=args.attn_heads) + + print("Creating BERT Trainer") + trainer = BERTTrainer(bert, len(vocab), train_dataloader=train_data_loader, test_dataloader=test_data_loader, + lr=args.lr, betas=(args.adam_beta1, args.adam_beta2), weight_decay=args.adam_weight_decay, + with_cuda=args.with_cuda, cuda_devices=args.cuda_devices, log_freq=10) + + print("Trainer build finished!!!\n") + print("Testing Start......") + trainer.test(1) + + +test() \ No newline at end of file diff --git a/LanguageModeling/bert-oneflow/test.sh b/LanguageModeling/bert-oneflow/test.sh new file mode 100644 index 0000000..baf8317 --- /dev/null +++ b/LanguageModeling/bert-oneflow/test.sh @@ -0,0 +1 @@ +python3 test.py \ No newline at end of file diff --git a/LanguageModeling/bert-oneflow/train.py b/LanguageModeling/bert-oneflow/train.py new file mode 100644 index 0000000..48cafa3 --- /dev/null +++ b/LanguageModeling/bert-oneflow/train.py @@ -0,0 +1,79 @@ +import argparse + +from torch.utils.data import DataLoader +# from oneflow.utils.data import DataLoader +import oneflow as flow +from model.bert import BERT +from trainer.pretrain import BERTTrainer +from dataset.dataset import BERTDataset +from dataset.vocab import WordVocab + +# eager mode +flow.enable_eager_execution() + +def main(): + + parser = argparse.ArgumentParser() + + parser.add_argument("-c", "--train_dataset", required=False, type=str, default='data/corpus.small', help="train dataset for train bert") + parser.add_argument("-t", "--test_dataset", type=str, default='data/corpus.small', help="test set for evaluate train set") + parser.add_argument("-v", "--vocab_path", required=False, default='data/vocab.small', type=str, help="built vocab model path with bert-vocab") + parser.add_argument("-o", "--output_path", required=False, default='output/', type=str, help="ex)output/bert.model") + + parser.add_argument("-hs", "--hidden", type=int, default=256, help="hidden size of transformer model") + parser.add_argument("-l", "--layers", type=int, default=8, help="number of layers") + parser.add_argument("-a", "--attn_heads", type=int, default=8, help="number of attention heads") + parser.add_argument("-s", "--seq_len", type=int, default=20, help="maximum sequence len") + + parser.add_argument("-b", "--batch_size", type=int, default=16, help="number of batch_size") + parser.add_argument("-e", "--epochs", type=int, default=10, help="number of epochs") + parser.add_argument("-w", "--num_workers", type=int, default=5, help="dataloader worker size") + + parser.add_argument("--with_cuda", type=bool, default=True, help="training with CUDA: true, or false") + parser.add_argument("--corpus_lines", type=int, default=None, help="total number of lines in corpus") + parser.add_argument("--cuda_devices", type=int, nargs='+', default=None, help="CUDA device ids") + parser.add_argument("--on_memory", type=bool, default=True, help="Loading on memory: true or false") + + parser.add_argument("--lr", type=float, default=1e-3, help="learning rate of adam") + parser.add_argument("--adam_weight_decay", type=float, default=0.01, help="weight_decay of adam") + parser.add_argument("--adam_beta1", type=float, default=0.9, help="adam first beta value") + parser.add_argument("--adam_beta2", type=float, default=0.999, help="adam first beta value") + + args = parser.parse_args() + + print("Loading Vocab", args.vocab_path) + vocab = WordVocab.load_vocab(args.vocab_path) + print("Vocab Size: ", len(vocab)) + + print("Loading Train Dataset", args.train_dataset) + train_dataset = BERTDataset(args.train_dataset, vocab, seq_len=args.seq_len, + corpus_lines=args.corpus_lines, on_memory=args.on_memory) + + print("Loading Test Dataset", args.test_dataset) + test_dataset = BERTDataset(args.test_dataset, vocab, seq_len=args.seq_len, on_memory=args.on_memory) \ + if args.test_dataset is not None else None + + print("Creating Dataloader") + train_data_loader = DataLoader(train_dataset, batch_size=args.batch_size, num_workers=args.num_workers) + test_data_loader = DataLoader(test_dataset, batch_size=args.batch_size, num_workers=args.num_workers) \ + if test_dataset is not None else None + + print("Building BERT model") + bert = BERT(len(vocab), hidden=args.hidden, n_layers=args.layers, attn_heads=args.attn_heads) + + print("Creating BERT Trainer") + trainer = BERTTrainer(bert, len(vocab), train_dataloader=train_data_loader, test_dataloader=test_data_loader, + lr=args.lr, betas=(args.adam_beta1, args.adam_beta2), weight_decay=args.adam_weight_decay, + with_cuda=args.with_cuda, cuda_devices=args.cuda_devices, log_freq=10) + + print("Trainer build finished!") + print("Training Start......") + for epoch in range(args.epochs): + trainer.train(epoch) + # print("Saving model...") + trainer.save(epoch, args.output_path) + if test_data_loader is not None: + print("Running testing...") + trainer.test(epoch) + +main() diff --git a/LanguageModeling/bert-oneflow/train.sh b/LanguageModeling/bert-oneflow/train.sh new file mode 100644 index 0000000..ef69436 --- /dev/null +++ b/LanguageModeling/bert-oneflow/train.sh @@ -0,0 +1 @@ +python3 train.py \ No newline at end of file diff --git a/LanguageModeling/bert-oneflow/trainer/optim_schedule.py b/LanguageModeling/bert-oneflow/trainer/optim_schedule.py new file mode 100644 index 0000000..5ccd222 --- /dev/null +++ b/LanguageModeling/bert-oneflow/trainer/optim_schedule.py @@ -0,0 +1,35 @@ +'''A wrapper class for optimizer ''' +import numpy as np + + +class ScheduledOptim(): + '''A simple wrapper class for learning rate scheduling''' + + def __init__(self, optimizer, d_model, n_warmup_steps): + self._optimizer = optimizer + self.n_warmup_steps = n_warmup_steps + self.n_current_steps = 0 + self.init_lr = np.power(d_model, -0.5) + + def step_and_update_lr(self): + "Step with the inner optimizer" + self._update_learning_rate() + self._optimizer.step() + + def zero_grad(self): + "Zero out the gradients by the inner optimizer" + self._optimizer.zero_grad() + + def _get_lr_scale(self): + return np.min([ + np.power(self.n_current_steps, -0.5), + np.power(self.n_warmup_steps, -1.5) * self.n_current_steps]) + + def _update_learning_rate(self): + ''' Learning rate scheduling per step ''' + + self.n_current_steps += 1 + lr = self.init_lr * self._get_lr_scale() + + for param_group in self._optimizer.param_groups: + param_group['lr'] = lr diff --git a/LanguageModeling/bert-oneflow/trainer/pretrain.py b/LanguageModeling/bert-oneflow/trainer/pretrain.py new file mode 100644 index 0000000..b890173 --- /dev/null +++ b/LanguageModeling/bert-oneflow/trainer/pretrain.py @@ -0,0 +1,168 @@ + +import oneflow.experimental as flow +import oneflow.experimental.nn as nn +# from flow.utils.data import DataLoader +from torch.utils.data import DataLoader + +from model.language_model import BERTLM +from model.bert import BERT +from trainer.optim_schedule import ScheduledOptim +flow.enable_eager_execution() + + +import tqdm +import numpy as np + + +class BERTTrainer: + """ + BERTTrainer make the pretrained BERT model with two LM training method. + + 1. Masked Language Model : 3.3.1 Task #1: Masked LM + 2. Next Sentence prediction : 3.3.2 Task #2: Next Sentence Prediction + + please check the details on README.md with simple example. + + """ + + def __init__(self, bert: BERT, vocab_size: int, + train_dataloader = None, test_dataloader = None, + lr: float = 1e-4, betas=(0.9, 0.999), weight_decay: float = 0.01, warmup_steps=10000, + with_cuda: bool = True, cuda_devices=None, log_freq: int = 10): + """ + :param bert: BERT model which you want to train + :param vocab_size: total word vocab size + :param train_dataloader: train dataset data loader + :param test_dataloader: test dataset data loader [can be None] + :param lr: learning rate of optimizer + :param betas: Adam optimizer betas + :param weight_decay: Adam optimizer weight decay param + :param with_cuda: traning with cuda + :param log_freq: logging frequency of the batch iteration + """ + + # Setup cuda device for BERT training, argument -c, --cuda should be true + # cuda_condition = flow.cuda.is_available() and with_cuda + cuda_condition=with_cuda + self.device = flow.device("cuda:0" if cuda_condition else "cpu") + + # This BERT model will be saved every epoch + self.bert = bert.to(self.device) + # Initialize the BERT Language Model, with BERT model + self.model = BERTLM(bert, vocab_size).to(self.device) + #self.model.load_state_dict(flow.load("output/init")) + + # # Distributed GPU training if CUDA can detect more than 1 GPU + # if with_cuda and flow.cuda.device_count() > 1: + # print("Using %d GPUS for BERT" % flow.cuda.device_count()) + # self.model = nn.DataParallel(self.model, device_ids=cuda_devices) + + # Setting the train and test data loader + self.train_data = train_dataloader + self.test_data = test_dataloader + + # # Setting the Adam optimizer with hyper-param + self.optim = flow.optim.Adam(self.model.parameters(), lr=lr, betas=betas, weight_decay=weight_decay) + self.optim_schedule = ScheduledOptim(self.optim, self.bert.hidden, n_warmup_steps=warmup_steps) + + # Using Negative Log Likelihood Loss function for predicting the masked_token + # self.criterion = nn.NLLLoss(ignore_index=0) + self.criterion = nn.NLLLoss(ignore_index=0) + self.criterion = self.criterion.to(self.device) + + self.log_freq = log_freq + print("Total Parameters:", sum([p.nelement() for p in self.model.parameters()])) + + def train(self, epoch): + self.iteration(epoch, self.train_data) + + def test(self, epoch): + self.iteration(epoch, self.test_data, train=False) + + def iteration(self, epoch, data_loader, train=True): + """ + loop over the data_loader for training or testing + if on train status, backward operation is activated + and also auto save the model every peoch + + :param epoch: current epoch index + :param data_loader: torch.utils.data.DataLoader for iteration + :param train: boolean value of is train or test + :return: None + """ + str_code = "train" if train else "test" + + # Setting the tqdm progress bar + data_iter = tqdm.tqdm(enumerate(data_loader), + desc="EP_%s:%d" % (str_code, epoch), + total=len(data_loader), + bar_format="{l_bar}{r_bar}") + + avg_loss = 0.0 + total_correct = 0 + total_element = 0 + + + for i, data in data_iter: + for key, value in data.items(): + data[str(key)] = flow.Tensor(value.numpy(), dtype=flow.int64, device=self.device) + # # 0. batch_data will be sent into the device(GPU or cpu) + data = {key: value.to(device=self.device) for key, value in data.items()} + + # 1. forward the next_sentence_prediction and masked_lm model + next_sent_output, mask_lm_output = self.model.forward(data["bert_input"], data["segment_label"]) + + # 2-1. NLL(negative log likelihood) loss of is_next classification result + next_loss = self.criterion(next_sent_output, data["is_next"]) + + # 2-2. NLLLoss of predicting masked token word + mask_loss = self.criterion(mask_lm_output.transpose(1, 2), data["bert_label"]) + + # 2-3. Adding next_loss and mask_loss : 3.4 Pre-training Procedure + loss = next_loss + mask_loss + + # 3. backward and optimization only in train + if train: + loss.backward() + self.optim_schedule.step_and_update_lr() + self.optim_schedule.zero_grad() + + #flow.save(self.bert.state_dict(), "checkpoints/bert_%d_loss_%f" % (i, loss.numpy().item())) + + # next sentence prediction accuracy + # correct = next_sent_output.argmax(dim=-1).eq(data["is_next"]).sum().item() + correct = next_sent_output.argmax(dim=-1).eq(data["is_next"]).sum().numpy().item() + avg_loss += loss.numpy().item() + total_correct += correct + total_element += data["is_next"].nelement() + + post_fix = { + "epoch": epoch, + "iter": i, + "avg_loss": avg_loss / (i + 1), + "avg_acc": total_correct / total_element * 100, + "loss": loss.numpy().item() + } + + + if i % self.log_freq == 0: + data_iter.write(str(post_fix)) + + print("total_correct >>>>>>>>>>>>>> ", total_correct) + print("total_element >>>>>>>>>>>>>> ", total_element) + print("EP%d_%s, avg_loss=" % (epoch, str_code), avg_loss / len(data_iter), "total_acc=", + total_correct * 100.0 / total_element) + + + def save(self, epoch, file_path="checkpoints"): + """ + Saving the current BERT model on file_path + + :param epoch: current epoch number + :param file_path: model output path which gonna be file_path+"ep%d" % epoch + :return: final_output_path + """ + output_path = file_path + "epoch%d" % epoch + flow.save(self.bert.state_dict(), output_path) + print("EP:%d Model Saved on:" % epoch, output_path) + return output_path From 39c7cf06b63b646dd2ed547f5b4ea30e735a2834 Mon Sep 17 00:00:00 2001 From: lichunyou <2695928308@qq.com> Date: Mon, 21 Jun 2021 15:45:21 +0800 Subject: [PATCH 2/2] align last --- LanguageModeling/bert_oneflow/README.md | 23 --- .../bert_oneflow/bert_pytorch.zip | Bin 47550 -> 0 bytes .../bert_oneflow/dataset/__init__.py | 2 - .../bert_oneflow/dataset/dataset.py | 127 ------------ .../bert_oneflow/dataset/vocab.py | 186 ------------------ LanguageModeling/bert_oneflow/main.py | 83 -------- .../bert_oneflow/model/__init__.py | 2 - .../bert_oneflow/model/attention/__init__.py | 2 - .../model/attention/multi_head.py | 40 ---- .../bert_oneflow/model/attention/single.py | 29 --- LanguageModeling/bert_oneflow/model/bert.py | 49 ----- .../bert_oneflow/model/embedding/__init__.py | 1 - .../bert_oneflow/model/embedding/bert.py | 65 ------ .../bert_oneflow/model/language_model.py | 61 ------ .../bert_oneflow/model/transformer.py | 32 --- .../bert_oneflow/model/utils/__init__.py | 4 - .../bert_oneflow/model/utils/feed_forward.py | 19 -- .../bert_oneflow/model/utils/gelu.py | 14 -- .../bert_oneflow/model/utils/layer_norm.py | 18 -- .../bert_oneflow/model/utils/sublayer.py | 19 -- .../bert_oneflow/requirements.txt | 3 - LanguageModeling/bert_oneflow/test.py | 73 ------- LanguageModeling/bert_oneflow/test.sh | 1 - LanguageModeling/bert_oneflow/train.py | 78 -------- LanguageModeling/bert_oneflow/train.sh | 1 - .../bert_oneflow/trainer/__init__.py | 1 - .../bert_oneflow/trainer/optim_schedule.py | 35 ---- .../bert_oneflow/trainer/pretrain.py | 159 --------------- 28 files changed, 1127 deletions(-) delete mode 100644 LanguageModeling/bert_oneflow/README.md delete mode 100644 LanguageModeling/bert_oneflow/bert_pytorch.zip delete mode 100644 LanguageModeling/bert_oneflow/dataset/__init__.py delete mode 100644 LanguageModeling/bert_oneflow/dataset/dataset.py delete mode 100644 LanguageModeling/bert_oneflow/dataset/vocab.py delete mode 100644 LanguageModeling/bert_oneflow/main.py delete mode 100644 LanguageModeling/bert_oneflow/model/__init__.py delete mode 100644 LanguageModeling/bert_oneflow/model/attention/__init__.py delete mode 100644 LanguageModeling/bert_oneflow/model/attention/multi_head.py delete mode 100644 LanguageModeling/bert_oneflow/model/attention/single.py delete mode 100644 LanguageModeling/bert_oneflow/model/bert.py delete mode 100644 LanguageModeling/bert_oneflow/model/embedding/__init__.py delete mode 100644 LanguageModeling/bert_oneflow/model/embedding/bert.py delete mode 100644 LanguageModeling/bert_oneflow/model/language_model.py delete mode 100644 LanguageModeling/bert_oneflow/model/transformer.py delete mode 100644 LanguageModeling/bert_oneflow/model/utils/__init__.py delete mode 100644 LanguageModeling/bert_oneflow/model/utils/feed_forward.py delete mode 100644 LanguageModeling/bert_oneflow/model/utils/gelu.py delete mode 100644 LanguageModeling/bert_oneflow/model/utils/layer_norm.py delete mode 100644 LanguageModeling/bert_oneflow/model/utils/sublayer.py delete mode 100644 LanguageModeling/bert_oneflow/requirements.txt delete mode 100644 LanguageModeling/bert_oneflow/test.py delete mode 100644 LanguageModeling/bert_oneflow/test.sh delete mode 100644 LanguageModeling/bert_oneflow/train.py delete mode 100644 LanguageModeling/bert_oneflow/train.sh delete mode 100644 LanguageModeling/bert_oneflow/trainer/__init__.py delete mode 100644 LanguageModeling/bert_oneflow/trainer/optim_schedule.py delete mode 100644 LanguageModeling/bert_oneflow/trainer/pretrain.py diff --git a/LanguageModeling/bert_oneflow/README.md b/LanguageModeling/bert_oneflow/README.md deleted file mode 100644 index f2b23ae..0000000 --- a/LanguageModeling/bert_oneflow/README.md +++ /dev/null @@ -1,23 +0,0 @@ -Oneflow-Module版BERT实现，origin form:https://github.com/codertimo/BERT-pytorch - -### dataset - -示例数据集下载：[data.zip](https://oneflow-public.oss-cn-beijing.aliyuncs.com/datasets/BERT-pytorch/sampledataset/data.zip) 并解压至本文件夹根目录 - -### requirements - -1.oneflow版本直接用master分支上的oneflow即可：https://github.com/Oneflow-Inc/oneflow/ - -2.同时，由于数据集加载部分的功能尚未对齐，需要依赖torch.utils.data.DataLoader，故依赖torch，需安装torch（版本不限）. - -3.需手动将bert_pytorch.zip解压并复制到site-packages下，如： - -`cp -r bert_pytorch ~/anaconda3/envs/oneflow/lib/python3.7/site-packages/` - -### demo - - -- train: `bash train.sh` -- test: `bash test.sh` - - diff --git a/LanguageModeling/bert_oneflow/bert_pytorch.zip b/LanguageModeling/bert_oneflow/bert_pytorch.zip deleted file mode 100644 index 460c2a8b6fac322a1dc93ad84e42e3c6ee6e164e..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 47550 zcmbSzV|Zm-mvwC0sMxlXN-DNdv2EL^*tTukR>i2;s@Pxd?f%~GxB8-Q=Xv&dPX4UD z&fH^-IpHnxmVf7W+;kDmf`FDMPa$BThpZ8eIB4Dr#zcj5;)Gk*c_TKo_{aj+yK`+-{;? zbROVuvHq{Xn4j^-B!K|{>_Y!-U@Ub_t^PbP@RAz82F6f%ENbmDR@1su)9T#yc}hES zTr_{2OR53TDOh5A&>}6!=UQxC$du#CR4bX6!}u~X%sG3ke4cOpT|I%&=x-tGJg1hP z8n#%FkUgLj+uip{w|&cJ3o4aQp77qcJf-a%d>0<~k9~BJbY?5?>PN-^>p-I$!M)`wUXvP`A@h1R1jsA{&~t43W}N(UOPha9`$POuufW<7$B?;Dg~AuKn| zj{@t2K_2Nz;;)Es>3znzEgnzBve!r764|7rA`#abL{0S$_>42PKzEbMa%_ISu z&!~tm8R`nuR{`inBFY?F#z(UfV0ktKXoBuy>7?B}&Ci<8&YqBzuzgN=^L?ucSOeIV zdQygbL?*DFOC#AuBWRKkh5eZgowk8_#B~c?jJ0(-KGM=@4dA81EHdaz%fY%1G})d` z_rt=VyBVS$q+W89qyCPH;Fh*rSr#fw(OvO<)#WCUeN}g87o!w1ma^>YlxLv6T;S1eHvauVrCGkr{R zMbI~j>^Xkp#+kf2{sSM)Z1lq~!IPb^srRziM>Mv0)6n2giFc0;uTR4puC|I_;Kpeq zkGtE5@6~aipLl-IQG`}~8`|Ux$PDOwdiZMPvhFc+4RR;NuhhmpEHl=?^|fu4!Wnf| zSF@fnaAJ8!cNmjKvW873aM^eUz#`NG(4t0CQgZLWW=}ezS_=s_R7c+&poy{_4p()a ze3i6M)aF#8x;~AK+ZvsV6$4>riq0`emsm5j*;ZU__tQBqq%MC|JiSTbWBWmY5CaBO z!HR*5hS|DE4{S+>O1oF$YUUz5h2cmHSdrLyX<%9)hy9%84dY1nMY8R!G(CQf{O3E2 zsYr_(M+(eA9rQg^7(tK@GC^X2EX343G9yNx+Rp)|+mwl0hOX<^jCbrI-BF~e7K6K{ z81VWK(>sI`t>d6*qE6Em!683oTFf-OohQ0*8=b(3d~N zM)9{2spB<=PKAHQJE95&5ocXtR8%Ex9Ualpi7j`pl~S*>>hB4XE4(Hgj=ea0Na zg-}|7Zp_jA84irsJ1D>_UI!!Yd&R?ME7bD1)9q=h_Tl?n7h3%i#uO=WmG_jMu&X5mv6i)%orI+f_$uPr4HjFMZ zM3y;o5Hb2{%#HeTEv29uFsPK)dfdV})G>!P)gRN?6qG(qHO28a$Zy4`npi#|l`yv) zTY8&1xs|9qhtDyI0tO=5ZuZ(3dgZ12dTArDzkQ{(AkPEb z!lXj8y567<3VH)cYBxa;2WA`$6DM_%^DU(tmrfN4Y=Auv>5XY6Pee^Th1yKPb!-Pi zyv23kjy93&IUVP7{(Mb2_?1-0GBz+*j@OmLE9^?29cQSge+wkFvUKTZT!>EBwZhMD zbwJb;RgYuuD&sHAlo>r+ZC#4 zO}n(N&}SWfnbcE_SP{sr%>*Vy{c4cTYk|6PkLouUUTWl=n>n66}?A1s*&74=}~RO1!3Ssv4FXI zNVsa!-``Qmx5Advddu?0qT+l@dm43V)LRk}vOa{4n`t&!@@b{d1W}h|e2P5>Se)5& zSGRw`0W%P5tzkgXjknZ6FtCG`xcfGG`dy6ib;X{$!u{jHbR8TFtsG3Pt^SY0QvBVp|F;6E z|C@Ou4hR6i(nl>I_)l(O>1g3#s%>JZYw! z4Dt{Klww7YVAiBhglZK*6O&6{-8)Xk%jZ^V=`8XCgy3nuq|_nbU7G6-)Z&QU=VEU)icAj6tIDHf7t6Epg5}3^4}Pf#2AfEzqFi49 z)AvjLu2kS*IP%OjzmL?GKkT6v#Ls*c4B%igROD6EF+D{YQ*)YtHjYVj3}B4Dw#IBtVwEo(Xs*yP z7%3nbT&NP+0%{rc>nzb*>54da-{>6FRe(wnRv}%fIGOPfdPkFvso@+<=Fuy64yu{c z6#<$35SP2m!3A((vABK!NeJAoH;KQQ9hPMD?h>b|u=HmmVw@DB8L~+F`N)W52#i zYia%pkIW9KPnP5Wz`o@u}?pX#Idz}tJ^arXRnA+3nX@2T^&8EBA;c+^dSIQ z$Fd3&G(?28I$&emfG5BuCZG{NcQODC)8Vo(U-(@`2hfyL-yWjf7x$gdNU7`AwOJSJ zvR5aC*(BvnA41dHa{_1?-JshE(3&LCpo-ER`S`Lfr#rh2%7)84`q3`=`3EOuwt&7T z)_+}*S?>MHZtt!mp}~oX&7SRXA1jLfiITo_m3Rjl`>nktL$^p}W13RiNZG{Ey5PGE zAg1)dHRCV}0;Ueivf+Zy&T8$fO`Q4skdp43KRg95^!Mx7Z4F^z`QmI$G7$j>R27r> zIdlT&i+&mZ&hE@bE{u!vlq&yat+A-dJFDH+w8MkWM79$69~CjyLBfII#4OuKZK{{B z(||as*J|twO}3Ad;xJD{Y1_S`$|0lfke`w^SWOyWi1rXLdxIX(IQNIGKB!{M}hMJItu%h(}ELm6uuWftb zv|(>~#WuW?T9&?mQW=wk0blQTq<}NcNfyOT4V?UpVQAEU&&_Vni#PxYqp(Weqd7=H zFRV7yK%&!D)jH!)GwbAWUrEl#m!(v^r^u05;X2NG2|Z{vBV%p*EYldBRR2Y;t)WLY z_CY5n_b0G6mYX%=_#v(+8P`R;1;Dy{fUYF3!|Tg2Qp8uoW%tGLO2Xp+bpO+hT`Zz< zdScs4zgsBS;Y|&Gh%*NyJ=TcT9qd#qn2miEJ>HXeZ{UBA?X!d!s2vahK*2{F&ihYd zYj0|0Y+?8h_})@s!fKu#Au?lgy|LJWqsP$N1m1#QIGysd5(t^HsIXvdKm@kCcWr2f zV|uUhvVz;50Q)rvH@|kbiu5xSj_PSOjd1U*2j*&=b?4Pmu)F)ZHU5UT2RQCQu)Vp< z;8zT)C>Ysbzi}}@pbBt>=+J0-DEMh_OC&lZbe3`+*`_n}3ouledg!*THY1Q@&=d4tgNql*a7IF@Uj+VhV#uK+we;%nfPApuZLYK!( z*P^eFi?f>~$D68^^j9IL9#Gxey(VA;q{cS)mSbp{KToG=@RqJEHKaHo&zDEZl_w`h zCR7Xc141r%Q2BBf5t$X|LTQI)EYYLiEB1+w&X3IWAqdybU7_M#<<^D*_#Lq>+noP>~p8%)Jr6!+Wb6XzgqA zEU)kItxg-`VRmr>@2;lM^Y#REM6*5;nVEkxnYu+)Va?^qY?ghw|@`{@*GZ8`pmcKYyTac*aXur_-YZ-oJ#E zl~69Dk}!?XA|>P+%?Qsb70~8FaM)n3T`l!Tx$Z9BV_{%0npPlU@I^*V8Aq)(4!T_q zCQ22uK!B^0HLz$5r_Y-Q55WuNC#5~XR05u^FaX zv}t+x!=jD~Gw}T}6S3jKFGP}h=?MDy;1Ce>Pib2Yd4y;hWP$6LBn$csIh65Ce#<{cjd3s^z`+`F+gm5?-^?(er zM#A(eR3zq(GbnicQ1BHdRcH*{EfXlkswj^kUfbhEpBSvLr0mqS*g37cW9=-Clqe60 z;%XXH{>;6BE3QmI7_4ZNxKS> z+M*3hK=&%ss+V=LzbseAu$^(|eE-;VWf@5yWp8`Py8^XWoxBvfF)bp;Q1#bk;#4+m zyAAq5f!!ZX9^>B+{F{9L50bo=qRekiUDool)~W^< z2o@9yq-7Z%J(80gLTNd9_H+m|h$5wuItpxDMor!1o2}7;r9KDQ7jaq~cESg+bdD>f zyHUVBs+)fIwv#0TX$q!h0TIeq~kZj z>n=eIGCE$3r|#p3NDSj6dsjJrFlGVBf#k+@$1>~GI*J9=M=U{EEJsg3Be5sxAT%R< zr?PP1eBQX$B|ITZQLN#+y#Iu0in=eC7`C)2BNT1^o|0-}sF(`fPX8ur9NLvwc9f7= z2}!iu3On-hJap0g_QvRNg0s%49rOssZ&wS@j}{P1WiH*!V$8pc3TUP+fT}Uf(ZW?^ zQ|l#=A+muZWSDd_i^VvS?>RHx!J#o-WFUZ!!h9|Yeryj)AUzsy14}9x-tV0fxE+#a zQV3?=o(22LJ(jbil9IUJIAcp(AP#;)p!5m6yHiJf7OoheMmgZ5T|g)0ZI@`w$?f&>X#C2~*}G$#Xm#It zm`1LWCH>julgQ`|U_$z0>uW7hK`|SL`_5T^%t#+~$|-L@XSv&(Ks- zz!zqeJ^;F@OF#7diX^j~0R zDN-mfDz#}yC65t0qW68mqxAGaU^UZhi|mXte&RI2W)atUB7D_VUJ9ir8^~b6VOh|O zL4r2mb9?u zCNq1I8pKXbLxhUMF?;Z*+GixP3*9*olGQSsE-r@gyN})1SP7D7aoM%V?0Peph91C45 zV@F+ML+#&nw|_uA6$!snPq*=1)y|4(njjQ&{TQqevXDQ2n0f$zh=iU}K9UIuvxU>( z9cNqJ`Dv?E1P4hmStp>j57VYmRr*iV=5F^ob7S0ep@Pc#Ixg$CcD9SxjkI;Aw#P*o zzL!(3fEOvVD9{U)OCy<_t$?}Yh+>(Ro{1$<$<6#-u3j3eEMzNa&e}=IO|Z~Dt?1^2 zen2t*Dqn+&*yeP+Sm|pZ#~WJmdR%o?ZSUvTy30~Gk6K(45WsR%usGFMwoXT?;ZWTM(1qxbp;YnOP@NzpIZM^=4aF) z+}vk}ojFA(V9bD$-5x8IVRSWTkJKVc9Iv0tMe83FaZaDm#+Bt`|&opBKnru7i?6>UhYRu|fT~obtTot@{R+ZjBgS{L6HuU_BeDd(EOjZH7NP#?=BD4;STEXVM->MrRCT?VqP684vS!Hq82?pU zv6L~0<;;73*JHH5%cTxOuOAVW-Swt5Q+!vo$?-uBp9PPH7XbT*H&a*WQ3xbJ}P>`**> zp1B+z^%a&^4%oMO^wW>3xm^dsm_qY-$63saxE-|2lsM&@<4@g9^502pJN1ba_~ zD#1!^wtud!$$$r&^2zwc^-hdtMdobDnm1rjhfHgD(!+z)q(Yqde?M}Cv|i`9!`Z0d zXvb0~zuaYmP~LP&N(XFJ{pKdFVk|^>aP0M1Wx1+~;b@B4JKyuIft9nv`Su-wd!!~io*V)f)qSxvnOTD zB~ZoaVp9Gtygl~EhITCd`U!XgpInqUz5?fgEhztxSL!}0E{?w^KnFWr zD|;hrJ4-{mf53i@iW63=A_yCMS>#H!8Cjf=(?HxJrZj~%)Bal+mMeuF{p8WPpb#Ld zsLo0=Ut^DHmA?47#R4A4@`+A)Vt;iX&7qUMnZ}HbHDoKOu1nvV^kCYYH0fGAdAT|c z#(%ekc#N~)&W@YXNfAQugD4$_{8n5YP*FY~X(9e&T~7K~#OhcWk3<`J3+f^rLS{B@ zVK&l3Ks(m!gl3S-OG4nz)b)Nq5p{$u7Mg;W4k`szEc3SF=iS(m>{ZIV_O2SKetqJ7 z=Y*7>wQ*?7Q~}DAz2{&<2pdyF+B!D~A>^*;CyD9dPJ%qjO?y7O0HQT}q!x5p)m;JQ zuPxUZ`ze@voC!}UqdRkt))@eq#-XAP82~ChDr)J_CH_S{gI7OtV$AA?7LQaS)4(&q zP8gS4M}Oo2gzHG5_Nd860MEta_MfztnbfS#@3BIkFmL*k4u2bd#!&xvq^wm(~+G)-`;55R_X$k5DcQDGoB*m`4 zs3-1ALm=VggpJ@qoQFD2`B$Kc@G_xkdk4)%5Q!~6rOgFi8|aK)5u)k^Je$Fu;Z`j+ zx(uBAwgU4)nd2G0wtd%3$2FjVwHX4;!G4{AOcgsQtGf(}E%&Hu>xE=mERyj)|a$1R6Iq^4+u%RZ4 zKM>XU0ImT2+>~P4L{DGe{m+}KIhZoV2S?%h;Ib$^E*S5aV5F}L29k&M4>vcUa!y$+ zktGU#^qq4OmTqbv_)*j=w>(7ZJ2p{ibm2ZK1;f-zZVtu`TkznX*NeuN3})lOUkE2C z?zsfoe>>qkBWm?KR@G|NfB!`ca2u#Pnfnj|06yAo*1xBRf5`#=+?=E{@ZZfzWB+DO zLP(Z+TtaekN8nb;RxXO*oT;ydyMHT%GSe+KE4P_=k51d^HyzKPx3k4GF(W zMbs&&YEFZjyvKP)q?E#JTT)!vG~ozg!RWE{JoF;8>K=?>O;yy=(wg5 z83FwiR;x6xEV|k!-qMgn-c(%M?lPJ5qIlajk1IYB=G-yV362H`+Ki=sQ=) zi7F1*l&u&$FVZB--j`6(U^?Ri6B!kLG<|r??WX7cnK0ii^9Al6KI%S>K`gd(YXNRS zv1Gdm&C_2k1%O9IMWwH*D?l0AB0InzWmab|U%B1aT`6Y08`ueqhH~gNFfhjjWCYEA zaEqWCY%o7IvSwaS;o!2A%F966O9zgO0;Kfy-kQ%&wtVX%UbpKx71&w22H6hrYv4@A z`TASq(zj&&wKobbD;I1Mx;&=CR0yZ}p)}TBD}SIc#&f>pZgA@kvUHN-%m@GD8i0ewmtyNd+$~4(R|svv8oz$elvgM-I-&$APX03bKomIeEV6Y7PywDnw8s zlh$hxjRJ6NHt365tlh@pEC3w5ieRQ}N;0aG+`8F2)Y@dlAA%&i8iRVox(1`SaK z+#n$K-AMyKRyu8ej)_)2f1%E|>4y{HT@d|bjn@9^P>A|QQXz2w@Xl@7g<#r^Os9lr z%Wkd0iZH}a8Jpxb=n%_)N2bx}nm*V(o|GaRt5K1+*B;5dZs(o}HfKvo4fGsi|L}&p zfyN5H2%8@-{j6*1-FFKF}iMt{y(&(OKt6RZEUZnD_ApOFzSQE&=fMiuXU4gb3guh@b z7@yn_&i13mD@pOYSiYuM?@pLUm}wBmWE{FLsKMF>W;9KEWC3ESf>D7Jpp zjxtYwEa)K9)HoN@urBk0KU?Doc7-qJbZcn<3o!G%du0bR)~>}JMtl+)|=t# zj{>{!^{UJAL#n3Dh=q0uWn&jc>B4nk&bMw3cltX)nzGVTg?Ww#%Mz#FCErzd!&dZs z$7>T+*>)?ckf|ew)CKpE{EQOC9{6?4L7E>l0MR?*YY8tWBev!~Dm3|`zEjbRL>w&5 zqMQC>>e*C!;+%6qwtxYmG(3a~YfLw>40tRshNlM09YX?kc-GJoB|f)TEIik|M&A&a_E!$GZ*il*XYWaP(sNs>{7O4?+2K z9$IsYBv{qs$8Q@yw|a+ZO4H;g-Cj|l+b~5v*u#s5R%4kM`1q^*Wlpu(EFBFI_| zi3<@Pv^)AGWUTwdc)(;v>9_p3{N!&h-?z3$f`2A%{BU_qySN6&QrTfVQgg(+Il%-w z{>pmo{w)N!hg_YKlbZ=@cjdfwnOYX9B9YDZ3Z!bY&-o|vtCoTa+n8tdlL&e1!uMbL z?ub`R;`LAOOXkbLCWEiv5d0`(p@wuoTw9MiAzB~{sfnn^tI^;0je2VJj!^6=@ z6R{X&WN1*L?9AWj)u-Pt!a7rd5F$@z0lfy?E^QzJqEC( zOye^qwIWs~#O^@vA)fn})$M+s!a?VgvaH&hOg9m&liE4gLEM4`u?qe9MjP<>A^SqP zAA#OM>lETh7<=WEZ?>1VL++p_pn0&WD|DQ>ov_qlKc8VfAYWQn@vdJv0TqBNK1}`C7-||U2ky9 zM|y!qp+%-Y+REj7jxon4yf~}*$8O_#z#&vGBR*i;dzNgc^&ZmAQVA98TnlZwC9cKko8{AZGB@{ z;6Np6e=>=?WnKxP$+hytP?}B`#p#%YO9KZ3O-u5!St2TcuyhA{884j-g>AcG!HS-Z zm6HD%GHD?&wqfpYt80e&^22^*GjzRN4e{E?_kBkXTB!sv+{QkG3YK9M_jD{~TL)6< zM;Ghc)<#wH{;};qMuqgtP|1($gRs(8gDzSz6ep=h7fm1r`(y>YgS0x}LL3v@LF>hz z3H0wR30Y}Ccd)-DsXu;9B!7&R|A^@yZe;xJBlE^GC%XQ4T#FxtAN&7ycw<8g$A8dO zTgFaUAu}Khz1D~&O#=ms64+j`Abks$zYQP7-t|*m)KiWoJ38epKOgZ!ne(X@a+>cJ9{3LU}7yR(8O{7^FTLJH=jT{k9`m=1%DSyhAks+6oo7myg5L62gTQ+ooGRk;^5oEzzjQmB{<*NX6YXNkUo`8#kTeM{kLs&@ zSNeq@d+geW&rA3mZS9HZaY>qm?qI>j)WGUnEO6ySaH`66P*c-;bkcyD!_WfE6L>pq z`$mO<7vY!Ied86^8s4!Y<)hds54i=8{4uqz)ZhyHlgfMrHhtvO$sZ?YOKop%`VgDM zsJ+7_XrCMc07lCrPAmy&jcYY8EFg0%Xam}SBS>FiTX0EcSFchY>iQoPPi{EqNL1IeL_)oqVabhH3_8Vp=X4=Z&C>Y5hzZA{H+Z}Psry&19*I^r6bp5P*YVQOZ#Fgff) z5D5iUUIl#vn%bg5uw20M&M`%W8)p3?OL87eRZv8xpM#gFSquc=HG6cqc!|+*ReRYK= z%vV=r+Le}p?^f{z;CF7xV`m%wqD3NWcy+rW9f+Kq z2&2=K%*9mXAazT}q( zwu_UaCD${)*kGAc*7^KG(bk1u zS;9kAT8VlL#Ok+7K~te98}uwIk5x%8!5efP7b}`u5zH4B0tj8FP%c&ku^rvDxHZ}= zD!mZ(49Q3=)Qw1NAjcaic(SGKX2y8_1>eS48-H7cV+Nb zv~?MV`274ZlTeZfauYPP2V5 z+39Lc$d&!GBbv-V&=yRIi^e!rd^gqfvmv0y2jTCpiw2srU{%QX%J?dv@5jI_x!8)G zLOy_%QLLs|9mJ=pw=27CUc z6Wm!miE;lRDBO=u@PGYK>>c%f=cs=`PnHT&QFHVN4c9Y95^aK-{5DrHqSHQ3TOc9d zKiT?zz4JkVOks914KoK{H5B`mihoq^M%{oa zd%pq+37uuZuHAPcn+${rsR6V=g1uX;6mr6O4r7D0Xuk*XPrl>RixKgA(YZN(wtA(m zG^Mq*ur&(jbE1aB`Llzuy$DK+qX6+r00yUExiI&fE3lgh!e~{~oXMc(0Wq*DF0dEO zA1kO1>ZN@Q!6ERA>Fl79oP9;cxuu{f36(^NlaA$Xa=mZ7>-{(72KA-JG1fR{3!=UO z5WR-oC!I2m0up-W_&5G8S5_ta%utb$@@P<#zOcNAF>BH?>@o)qPrDapffi1Rf#oE{{LF>w?-A z-ezcmY>&e(1XO()nd+ek^omm2F5EH_>12-8nc1*l3sGtbxx{zdYZa{XDE22;Nbn1N z(M6pz#B(-3_D+}=l6uw)00DkEsBmapR z_``xlSbmnnxjzs9f(+C@q%2w>CL})QxSk-(R*n3it$-o_tj(OZa^zLf@GC`hIG1L1+5;w*pW*J{o^l^ST?) zKSkpQ`<7y?A89<-TEb+%l%{wi(|M#|b1zqB9{5k3^p ze<|6&Hm4(1pdb}gGSX*WSx3M(rWl0v_Iivsd(Io@Qdzg0;09EQ30pAOLYefFm2U}D znOjj~XVLA%8e!NO+~+TE341ZrEw`3g zxE$=Do$MlW7}iizLxVn5s+sBn6TR#r?8?rwKp1~P(1KT+e{y#;_c#bj#>5RA!$ISH z>RDZsyvNy`iekdw05BQ|UB7B9yfCLaW~F&QsLHptf6!^_9wAaoTF*e+IO?{}-N)v=EV?`;0M zJS)!czbM)y&ABtvAN?rJN3JFKtLEnShUTyPP$`)i@qT&)k0})xPw|HSOcg{A6gX{^ z-FbM`m3a`iv}B7GuTE2*0bfJs#0<5;EIm;m$P(uu%=n)2m##B9ju`d=MLxMn7fx<( z6QgUSAlKSke?b8yB@OQa6XXpGI$rv_LUY;Aexjkg$im=o+wRb|#4g-8n+(QWC}wQ> zE|Yqu{1~>>FH;`aU|f1IN0O$~%hLM(dq2H}YZ=Kz%+GBC3*tAno{(fhGQR33za)I# z+%|$X%&l7NE5K0E2PgcA%=I69@UNuz z*U_?!ma_7vM+m%6*XtiGj)9jg2`|=aT;2s3?(x?faVf8n6O`HPK{(7}^KreHV4H%C zVHYAjujB-whZHP!D+H`(L$<>X0k;W1_3Y#r5mr*lV58`9ZAV$A3iQ!(LQ#Zx@;^PS zajoQ+)>_bIqJ@Dt=QtU8+H z5VwdsP9Y+mm@*EepuFj7Z$J;;^2BedzcJ2(A)pPbLeM={Cu)rk%PALm00&))n%;kaI?U zX=97>>WsjN9gZD)^!qWuYgP~$(YzYPDs3cis3GGlaQBiY^zj$zSxHH$FpQX`^gJv@D22o8-^t>ke_3k52K+z3J^%#% z0iZvzy#5D({##r3*T2Ctib3YX^7`QkhdL<)S;^Md;Xm$&f`GiEtgj5Iw523N4b+a( zICn#x-6E`sy+6Hwr2jk_O~i(u2%!n|X^*W+yrj^Xq9^ZJ&?^#$ImR>WY$>lrg`mMb zB%U!!3Jm;}nkYslQ-wd6HY%b7 z07t%I=;wK}c_lO4u-k+}_bG!DuI{AxV|8U`pm_CZxIQ|4_2`(usiPfkVt*RcOLovD1w zm^A#O*f9Rk(f_2v{^MKxL+h-i;zZQyXOyg~FQ>^9wX#s@r5vm#*mYX{gn}ON%IT`8 z>X0KySfrUFgGWo!#)IqUl6+E)p{$G+Y^$k(K5WNmuo?d|vbo-Z50 zp4=G3-Y*Y*GVdS|uYpdH2{TD|MfmEyh(kydwm~gJo<|8@`}zit{9uO3fsycrRS~ri zdDaLQSBFSfqCZ^Qf7bC1$xPFG0o%%K=tu-NuY0TWA%D-a`@~LHs!-N)+q@7+aU3?H zb=!k%s~$G$(E36u&Hh9r0a6Pgw{M&N$n6$bnEcG$d}<5`?|Ok`xJ7jmeG4GP<#KuW zT%;v?mobol;Lb0_$-|L``WS(dy6ac~Z3#kV{Baz)a9c2@M5j#X4s#OWh%@n4A>Ozsw;0#t2kQ~@zpi7uwb_bVEoj?;8P_usg8H_R>eGDBHwFLsx14n}6B z7sae;dW@rjooYM6wK;cpKo!b~>5kK-ifL*4lap7LcsXGKC!{pk3cj5==@AHpf*G}a z-*nh7{kr58QH7D4yljNwW?36JOS-BklrEf}$bxPcqU8M=^kZPeW;&=<4dVM?Jera& zxzcnCs{=oZ10%MB4LRK?Yo|Ios-Izfio=PL5*m-@j#(aTw_{~CY<;u>sz#6LJeN6y zr?~w{ec_GsO_ivzBZa){R~3q~WKVo{E@|Og!J=?`g))!3E3z*o)T9KH$Hy)=re-lA z?$C*R`bC08$T=|UONEsy^?m2GU&8Ghj1`33U}26)T=4OijPbSGy)Ao0RB%ULGuDCL z%r865L)S6t`>EOp?P1>WeFin$x?fNGcxhfyJ<={m=w*S)J@AKDwV;7+X_*-owq1cQ zq*#~gRAP%HS||G0Yp<%rxm^Tu}9 zM|#`;xc;Q$_+tY2SIYa}cO3u!Ju=zd|J|zbSMQN&|83u-vPG&`1OUf%rkwe6=V*DG0HX17AlXP{7ZDpXRX6V~NwwQ4m!Z3rA>1e#h~C z>fG_*oVbwg%6tX$aeuHTrsZv^w$_r*P7Z=5T|$?Q3A5%O=%>v+QSA@Q;E%C*&DiO+Xrp|nQT2#q)3qv2>0xy&q`kIr(Rzl%&$?rt175%Zi=p2um zxfPYgIzmCJW~MaT_z!K{Wrd`bY~QwKhie(cCpl6s5}VRGk4C>g`@?IhKzC`f81ZuL zHXa{oQH@#Ki6zn%pV*e~Ip&GlyE%-UP2QESJJ5+lu_ha-xKC*?y3bB)% z_7i?_IBmGQRtZTmFL5UG<&yj&Lt=V%$GDkK&0He!%6(6Pl|@4?s-}t_D(Zie!XRW2 zF`Ri5aP3{J^)s80XT|h!7|+!8eZ2CX5)M!6kl1zN4B8)^^`qu4H zIFk@?T7Bl|;Wzd~MjhB10yKnN{xonj2k)IA}gfsuaM3{jyS&0yyQ4ANGFu-hVkUsYioxgHKG~Y#n+T z2@}qmugGp~1hdG7V0dwyYfKr@jAIreJcYaBbmF6gqcipyLkv(3V!yciMh|f&rz_t$ zXYoST=37_gX`bI^g8ES4$>csLFr#oDLMv5svKFg?=(Rxh#PVC)m3m|w2yZr7_R+M4 z@zh2-53_EN(r~Y~8YZ}neI3mIO6J~5Ku0}`Mvj3}3LYw+t`~Xm;0H4tgp0+Y=v38a zFoW=gj0rHw{mxHjUWgstY+W78U(+)Z#oM5!EWoAF6}ybHGCX133Mz#QLSnkiINDX^8a=S82=z zYL2da?}#UXPGNH*3=1!mw?8_OAc8`+mT?v9RUI9ryeMc7)Q0Y~P!3v^`hTpwbC@M> zvMpS;ZQHhOv&**4F1xyH+v>7yyQ<5!tIN1GGv_yRhWFg_ef!z(v;WK7nHd>x#9AvN z_F5dvEi=`aZP$a3E{g5iET11X@S1<~$fZPQ$Ue5#c-3p5j~meIvHsGq z#hbk4^UP_=uw}vcBIn6cYCe3DP7-|Q2Ag}_gr()Gq7w6Bky@>=?~atRYmzq9kXU8F z_>=0;mL0XF0TrhW`F{AreI{Q{OQxr!jU=3xIr(#u56OEfy~A5#6H)A6HGkua75~=j z`wu}$uWH-ew=Z)T`Z9-qBUt#`-uhP$`k(XGFSgtAORx}hHxAa3mmok1mgvTdoC@w)^IGu<#dSDb0E~d@esOKQpJ1<=B4TL- zVS647w~^FhGlZ6^sLV{{t-pL$e-#C^u%H&0kmZB|1lqifl!8$s*feb&u8*Kf?^N{# zw-{M0G&_l>VvOj!m!CR!3nS40ZooKz-@-3s0RU;*05XQA&~4*Lo(kMPPbm)(JKzie zCZ$nsBNG|+DWppWe#@BrM40%CVcDl=zhA;seA#J-Y-?>=0++ zNBuUq_lwn5LGX6Mvxa(<1=Q3}uFHdtaPs!u?U%DS-z<|ASSKwL2m{HINAa13J;=-L zL~Jb2^=55)%YJg2vub6B?AX`I#5-;#PrcxuEH==DR!m-Fh|8N5 zFzY)>D1N-Gqjdj0sr+So{!3ra?&51F{5P1%-=5w7K)4$H#m%e{AauT}M>+PDo7g;< zkA_ya4NFzHUWKLJfF;4B+av#8Mt=irVh?qcFbx?tI=rD-(60oyvv#X$1UnQs%WoDF zyqKh4ATS;9lF?lRg+G-+4ND?s)0?s0Bs{FUS&eG3 z@^NxMDb)n7o!IN2TIa|mHOUgAy;H;Rbnx-2g&0j}h*7n0a@Uv9lYAW!4pKJZ7-QSg zEn%cYKkaF1%YQc}{AqE>KdRRF>w0l{>o5rBLW$3h*{A1-q@DcfV~9p?V({$yYT7B=>FUq3>kN-Cm%z%WpMZ?yrcaMoU5XRrNQ|3=R9 zH>>^2rt_aqZxh`M)yIG!a`ha+ONI~tr5l9z4tyn@ZtYUH5oC*UbJi&8d7EXbkJqnE z>lCPV=nPGSfGQHjXc1LzPw5tax1c;lqfU*~v?~YJWsC(77mS^+iUkfo9d8&jW_K$v z?mfBT@coGs`lXab3!Uw;`yg-VHymeoTNd*_h}z!0u@_vwt}5$mk^kSe)<1HS|9ow- z6S9B0wve6oOv7Zz#BF%nAUa-ua12A{D&R>LW1_H<zkyz9HjCSlwsGd&3fd+#YvEo#P*|(>V$P6(>hxCFW~u`O=J2NpKG7@ z_%CY*u9zh~AGqP|{{Df1k?RPFez_P7_^q=S0nVIGSL~2RjuM|IK;NNISV-W-az90<1nF&8mpMnINN9=SaHb|V(*OI zj@vsqK+;Ii0gq${z)vZ5@I zmkMGnSgpygtMN`IQu>%@G!%aD;?>uG6j4|sECeqCS?WePhK3WR9J(j2f-1`@$=;rZ zs_zxDdG>e`T%n98Em{O8&6T`z{%p>IMNq}7u#f12ku8>RL*eF4RvuQ&pSi>_ zUpuyGJjzdh=#eU^Blej`$8fju!3S0r^=i-2AgPA^noB0`Ot8 z&B@p6X}Ni4YlGAt=D~hOVmo}DUwZMr->=HQbh;RuE^Hy9GGsiy>Y#!Fvp6=%2B=Ky zhZrkQ8*+&J0`Zc-_JSlac`F8c-=uKuNeK|R4_hk3%PP_;YLwJ$quj*9@b>HXU6D8G zbAS?GgQ9xN7NM*%YTV6pJ-Zfj@ib}pK+O?ujLN7N^mF-kw>6%pxpq3YQ$ z!>sMxqFC@})!Key3A^9C^ueo9u=uWAi@i4A)$+RlPm&YAz!N{B8ig`*1I?gxi9aNO zvwbR}lZ32k?l+;UXGC)lT|FOz;NxzljvKA1O4&Ovn-QLdOBU?C!~e-G{cR`vafz1q1E6QnaC;(8vMS6Q& zX}q+lK4a57gceWybB8X-?B zoU9rt*C0)Ae@)gqt*k}MYDWx!bsPrEWk-9Rc@cmRO(x3G^!do1Kj%LLnb$(X2L^@A$lcZ0>P!#bSel8fJ?Y@!@rA&;F$_PrE!FWXo ziD^=n01ps+usAK+1s6|WmB@2UJIgC7JV2fpy2KNDnT@!)Vl zRt;&f@X~=xlqm*0QLw14Mf1`9oju23+)Rw!9VfbR^37V8jfG6D9OG8!$t-fCV|=zKbaNMpZ4Nw zb01eydlhNT{#M)&8T1NpwaVR|Tbung-q4*S*06ir2aENL!KeKo)x{VdqWa_~8pV{TNCHuLmyBf;BoVE8Hbnx>hu*cnr3qcvd+35p#lHYnpW9VhlRE<(KD6a$ z65{?Oth%%TH%K`Y7lf~v8&QDDTDs{~ocGvv``mcJ7%@H6#YR&9LL4?ic=LB%P_zC--`_-JLh zp{nD{6>Wd5e-e=YtJnLlVfDYk^Z&(*{Z~AH;D3teum1_pCln1&^C%Pj6Pk|$Dq2QL zV2e27_f=py@uvbysfGV_fhF?1a^&v>rAVpWJj>tG1j(nv!)Avj$pgb@s4APgwWE{r zl#`N;z^&C7`>_82-&R^aDE5AN@mA#j;oY)#H2L#Q{CBYJzqnh`8XI=l!-(CT@VD-7 z2&B@9=Y_qCW$8@el;2S(d&pdOS6t@D$`LssJgXcyH>yhuE(3pYOQAV==li2jfD|5E zefW3m7k!HTdfb$rz3)Z%mKy{5HaB&CBOQ+~72wre&)i;L&wQ70P-VIJG@l^P&#jSE z_+7#YAJq4%pZ)^|i%M}HVGJFMczY>RO$;`=-yr`TphNZ;% zzKvIW{!Q=HvL_x=2@*}4wGE?&E9RF9-CfYjH@v7(Jid@GmMRa)T#hEVnkmVm5@3tS z+of1>k2)*a;vmg5$vkN(TQXS?-5fXDq)iO{WRBTT_1L6MMt!?V%4Ft6N=-}^&uJJM z)2$1f3X{zHFwAuOoSAvm$n>k_&P*u=dteyT7SC-wFEeMv#AIkO#zWr^%vxjmd|F|d zkQ{8 zCSE+0C7k#!)VXnT+fy5{m-W>Ebi>c8b_J5fwQcVcQowr!JsTAoeJ*Y3#eUD&r{Do4VsG;3z7(*r41BeWG!_if~ExhB~>n@ zC2PsCq=O+@mW(z>M4L&_nN0)K3CG{qo_N~e9>A2MUu^F&-x=;!Jy|^BT!N5cwfhY5KJvIt^Za-@j6)_fMLLl4)U|cxvDx-_ zVu&GNqn!kFU6m8eqi%CLZVNNwjl-w@7~KbvsQQFZrxh7Lk`0s z7MjwFOT(LnXAiM_N+7T0BHjUH%&2E|2x8!wd%IblN;p2gnkD{&0Kar?ZaMYxckDBb z9|)*BL36cP;qcd^G}q<8_EPL&d)uQ}w-*|Y7CK%3rN3j)G0QDSba|=sc@qu0wbpy7 zi?V6iS&tZKQ~evq4L{zp!yiyT#)JLVd6Jzht$mT(*P!Ff%VAec@ajQFMGdXh=(?aa zaqFi+zpmk0m75O)0Dsp;I;0KjGLx)~U70(Oe!+?>9AP9EH{Uo^Ymo=;V_{z20m&E( zViW-XGh>S)zYJ5=Ft7WFfw?LI^=x(5^xH%#sJdFZ7__)7a{+r}R}p0MKGz(y7hVy~ zgk^x~u#rq-gjPmIh9Zs`X-WaIKlF_%io&k%Or@#i#M_4Uny7VhZEFy0Hc@vJ>`Ry` z=N>++VBh=NbG9L4g(@-$4?BFPi)+IhtF}sx23p#l`m7V0m9Zw}$oLqNe~l6Ws7W+S zV!C!L_&7Z`iaOts6VX}0W-KM?#MuUqbO1BkC|ZlA1s3mHXg=bb8R%2>Mn}<7O&lUyVF#YFXZHD}U;I zzONWGhVx<_Qp@k%6*LDZ#55}H={no67E;nMhaf6HIb(HOiEx@1h2yMJ;b^oFg-laH`4r`g?0ZrV!#6d&})K*SfJpLp(})b76JafpCC5RKUw1ayLl>m zBmfyEa#0ImEAsVD(CSFAM+Ni{?!SGLwW^r22XuZ%bHBH>AVH+mImUWIN&K0c*pAvJ zbw*yOg*z?P))*HQ#!(5}Mz_5tUr4xZ{VmjtanY&ER%gbA;|J2UpkfPEClO%EM}!D^ zN|AOvolg-g`aDu)%Sx=dgWjQ=a#eexwiG&|UAVLTqcWvtaf&`3YE=qakMFOr9-48# z3u*@B368$6vMTY;Ly^0db8*G#C^$e)kycO#>zH2H`MU=F0XGh1_be&Tc8#QiQY>=0 zxtJ|52Kz@sI7)!Il>%9MHr&!{I6#0Cl_khBNDpeJEItC?bDTp|Yh5IU8L-+&|2ym> zK*=Bx?Aiz?iCK?jl+{n3qjj}fV?XbeQ*s5VCi`60iuXZ2`o3jv%vm+B$xvVyR{P=PS>22qM9gr%%Kv!e(?rIEH!bn)^0Q0~aH(2PE) zcse?ernMh~~YFgg;*1tL8uZN?L(D&i%kN!-L-e9($Mu!4v27lL!uL z3WP*CfbT6n&>o~Tm?hBo`@ybVlH08^d75#qXIgHv|+ z>Ovfb$e!jrtST$UW6z9Gdur~^J=7oe0*NO_>!+1|0iNJ1X%E4ia!crbz%ObNnXu}@ zAZmE!DerWJ)ei6ZqbE);cYIweBZWFb5JrihtrF>JOxjVnHGd(ARa9LCV>UdmLyYRW zlA0+F(osDTOEZ3XME{`Z@?qDp+Co<~n!cMhPE^|jhvbw8s(_4Adq#OKmY|*5-3o{y z5s^~~R8jMOn{8onfVW)0WFI$%5+W(ETejbGjdNf>tjsDt?y#CI_RWX2R=X(RPzUgHGAYb%4MV`H~qv??N>0f?y;D5XIfOIrsiSW>qH9r36Z|F zN4YbpJx8rzx47-q^s`5`JJ|yh^xAV{p_$|MPR}q9-9|U?@CHo};RB)>`oqb)!oUGt z$ii|h&0tMml7fGy({K&kXBa;Q#U?|n+U71mB1+-YQ558eZa_D_3<9JhxI%{6Qy zBpIowL*ZK?siM=}BiUKEOr6Yk_U2xjpahKQVxxh$2<_0E>bE~59e*n-HqW;r5QhK& za6tbTBJMvDjK7A-{}&)uV=E?;1IhQQyN?J9k`5*<`&>GfgVHTct7;(u+lEYDF=&1J zvI&dJR27kT{(Wm>eJF+fqJTpQ*6iT~2~`}#)cy&b4|aKoe= z(G}Dzv~S(?bGL&#GsX*XXsE2(brUIR3~Wl0`7UhW_#MxS8<@su4#+%t@2Qx%XUaSW zvn#K(kKv1E-3D)gfhMjd<-O#=Y)IlxWAqBIU5@_g5B1?6_@M12CyZYBoiR=4h+?Z5 zHauY)@J@Eb51Ji*;`wv=V!6 zN?+bD_ex&dODCeGX+dD}DRntMwA2~+oD(8}rIqc~O3Jn(b`gGdnmfzHL9S|=c;__T z@)Mc+XkeUR18{UoSXzB*w-?p)PH|68!C=`LH& zQHU0G7?zistgSAyH*ZdD6kn&oQ!#4#M;hignJhclE{%+lE|U(R6!Z43>BVhRK_)g8 zI@slaGT;I!FemiLIk6!S{DN2yZuG3nE9($UW0nyKUq+vG9SBHG@dLjaG(MO$Xg6_` z!4f#CE7Z>3mtRh>*c6Qb=ygyVllXX>0==iz#jL5L1g;9vl0kUNLPHrw>Ov;*PH96m7+Q)jNWR%)^dDsj`;#l z#C7;x6apmtqAU(E9UcNLv5bZdzaZf&cLSsjYq-UIB+vGaud?0mn%94E<6S|*gp*=$r4+f%s86bOpk&1KE`)>=nsvz!Yw{cs zLU}2`9EBb?I2{vn;9{9+BI^mMUEDG?`TX?s9M&*BFmVEPzAM+#iIia|CL9g{I^B5e zVU+X*BjSk@J8{U2p_qkoybz9Iyr-?Ck7llN5kiEsT!;}h^W5{kO>g*sFGinAQ%ERw zmqk0BU!YFDe}@NGM<>?$yP#@LejX#)Gdj46^TLmLoDISxsc-jhp{7JNEdpO-uPloJ zIAJ5PkCa-#H2S=1!EJS({IghGepyy?OYHMbXA{&omSS3A0}mX=;+P#zJTsu5VRa}* zxe16Objh9~#A4vofbKqZD96VzHbfTpf{DdMF`3bwv>?&LLBNsl}?)Dg~+jyh;1Ge|-;xggsdl|t; zDoXx~GIXFY<+;SD%kDdklKb19necZy)a@f;3Xsomd-xg)Nd}~NVqzvf>^cnr;(mmS zZc1#^G-^?YUkqD4LLotEXiN(53*ZEW=agZou&&NCP;Vx`Xl?GEpGo`_mFOlpfi!{p z^JH1A5QIb^5q=Mwnxf!<8-OL@RO|KT(b`<((YpL_)38TNjdT<yRZMjGE*P3ASa+j%qaL*}-^W``DW$ z&2qQNa*(BPg}S^+t?GJbz#aRZO_76s_`(MSgjk$g?EC8mtGs*SC#d}SGO#y43l(tw zOjtOV)j{deMX5>G^{Y9EoAy*ZQ9Wikl`f-diJ4S~r*R~@Nw<(W(Y6K^FB{KeU@#9B z>Tj~jsG1XD+k}8d3w2>@(QhmtMEQx@7h~v%*@7uMt7js<1IZpzqW0xf&yhLfOs$(> zaZfwbPBcPtND8#mI2hOGHikGlacKeA5Wm(?^YD%Y%uNIPLyP_Wcot3sy9jt_4bKpF zz3PfE#vLZ?F_Ni;JFOUa+@F6uBgNaa&%sLI30(~`hi*-JzhbXd`>5WWe@n^d5LOtG zhn%z-V#(vDL3TBVnPpaEc@XZ7`G!Elak9~`3b(cE}hIO+2{3Y^54?{lN60@7oIEOVdwL&B1xD(bKp&q!CWHEW{D3#GokDY&L!P!yHX zZhGhx$HG7^?%Y!35;_BkQ&>HxHz0VAa!!GbYcX-zb4q|H9Ux@=mPhwYGo0#RARu}J zw%bPC~`Jaf=q{3!D;dXAiPH zUE_V`Fx(;~E=-Za^-n5=qFfJRYfbD+9;{`_P-F^Be9slwdrH%9{`T{coE$Y(q8DEG zo>1nn6$B-+rsZHZ>s~&ZW;LTTI;B0qbnZ4#Q2rV=IR$uXUI7Bc@1Sp>f|k(*b(N-_ zAIT^22ZXB}9w-uM`+IS?Z52P$8%fI=n!FmPt3&*B0Zg=U> zp^nDTd~94KekOA9-9PT!-_T7KjdQ&xFlI9$vWfVW3^L*mhoBo}W0uf{Jc47x?TAB| zvwmZ28kTYt15LRDt=;Tuehz>0H&uT*XZolehhXq>CHN&fozCtlImz}djb#QFa( zUh$8^{qK0ikIFeAR5$>DE8MTL#eZZ#|J9*g?Ticz{~eD0FQ(BJX-+z=4WSP2MiNMj z{-pUngj-OCoKYpZ>OdxGGuUYKRihqtK7hwde*3 zl8wjJ*|7Wp{LXD}E#LJp6DaRvcO$3}&zu#4LsyHx^7zoX^$J8UFPrxL`_s01PIi3Z zcgds}sP41tNIw4%sl+EG^7eQ*|BCZ*-l7786W(wt6(1VKq_TD?@xU%Im*X>;z%ID= zuQ54x47e|>-%1XR%8V2o)h$`G39Am!B(2!{1reAp)zW65$@qdoBqdsimFE;oqeRM@ z$dqq%7R%(UVkE_jz^eKmuvjgCtGR6EC|j0}4Lrl9dY}Tw1l-wJuY%Wgy7AklBJN=0 z3bm{kp*4&-m(YH|dWT+V?2Lgq8By!H8J@)*y)Hr3gCQ2E;e6}O9d0?|>S8y(UML^- z{>4W7v>s-3Sp{Q=ri1A7R_WM;sZ(dGVSD^qTY?Uz$j$F0Xv%wEhF%HCNwx&9vF+KT z)B<(iMiVx+alHp>Na$i%vTtFs{j35L8Q3@up3iarpj2YLi220$+phs#qz<`d;*|5f zYa0-<&j^PL@t#Z~MZB43EMO(D{?1%xB2w5vWh-(w{@Ib9KGzz1MQ2q|jQ2yQ{VPhI zln}P5yPC1GhwQJis*WotfaHhuWk)2O&XbfNTAi_t)v*vUY9E6;I8W?w_P5LUxS8*8)<&TG*X+RsXUUy--W4#SW2 z9Jn%PMC6w*Z&`mfI$whLVQCx-QW3T2cbW&u{pyLF&A8fTMfnLGp~A_1|HL#wYtsr_ z$$)OKq&yokEPIus&!E;a5$XFvF>Lvsg@9HO^o|c~v2Cn(^6U>-M0Z=Rt$@MJ{fThI zD4|!-N;Qv_z&c<*cC}U5F0gM9yHJKVd!}dAZ{JI^R-pJN9G0tShw*!)R)W+R*RwS? zoSH5mV9p|3>~IBJFMfmZLW_gs>RVeEp;yLgjT`u70Hyc&7~c)AEmgtXs1Ka-f^_zX zT{;Em55V3OEDdy~bRtWHwpW}& zHKCKfPjkw=?DR&lS6N%;&Ef{LMniq zL%1jXZ(0^2;>t-r%(SPb1Fl5o3^ildIOL-Y%vs$Zh@ei@`FEqa7+#^)6+g}Lb)#YZ#%w5 z2aiOqdgH4cR@Rvt1mCPoEz{*L1Kw#Nv{^p7DH+YWc~UR5K43WpTf^yLnnBuV`Smod zMe^OhAAf7th^cg4o%(d;YUGfQwC`-0PRC+v`Ecy*=}ewDvzN8Xv=P)WCWR%Hp96_D znsFC6*kKVNWQr$j8~q;3zSy3$p(RDtOkX?zToXIVz-pc*5 zeOixiK3=(iAwdPX%??+JHSTv|Id*_Es(pW1glS57Dq&&7bbt!@^=Os)S1ja7{a_h5L0SHGH~^LWx6|;4{?M*y*~AoBi^kVax(LG)~bUWYSsvMAh*^6 z`RKm7Tl`!UIKla}bO7F*zPbYQo;cVgQ}sTr@?HD#-5EU1Vll!p%*1`og(bx-jDbYi z!JvmkCUMgP3&*GtV=+#$z4b~S9!I}Q9ZvP;%ukV@PSOIFz{Nq1{$AInKz^3zqkINsk6`F0DEY$>Nu+v1^Ui#zB=xnKH9- z$&1M$s6YivoqMp#-dad9%=DO&f+V_P!f(JfjZDX;@$}c`XS}r;YjRXl(QHQ!0UmR` za>FU#ij+i4^k_0oT?%5Tkhng845h2%@ioTbY4)AL1Z_zz14$gONMb0K|zLLTI4p+5vg3mZND}F3vIlLj*Ntr8jLrpY#mB7Tegt;qd1G zTF=fj+I&zcamC0&XW$#nXD2>t)I}t!Rv=ogFsQ7OFxOzh3NxZGvMVl?KeGyLj{kH$ zl(t9;Q!0Q1y)-CRdZ$EetOq+ducW|`5I18r6?orpr;40I6WJ`AxA26B(7uyiX`2UN z#Fu1sc#-*4{f$t2+#gp5-PI2@p_I$7p%!o^yGjAJqDW&ci7gkb?-$mYrDe4Rq?&3p zIo0U>>O_7c!5xgl1I(ju$cDpWNE)#=0r8-85V}v@k%-Q~ea!U@zTq8{Wq0z@4y6`K zxCJjXDFjJ2+zDOk|3{}t@2;CcdFhOi=ez`z2!jLwu)Uqmam53EX0NJ$jHEb8h z62EV6Cgh+372DSI%@`Tk(}xi{jX?=CTVgSKhT1$Z^HG$p64k+~Mu<5@kP$R*b7Eb5 zD}4ER4WhQsXczf+0AivqjZ%)z^BEb+J_s?A=W|RCq^}Z3kjf!-Y(}X8{=9$Jk*g2H zBgPgX5)Gt!ZCi>wGxX<3uwJHSeDAGUIViwpG9ZeB1zKfidYu@ERlULqvDNm4YhJdv zd}C;@cn`rRr_y))vI^&!v;MFi49Ai0bz7u`ru)`)QQ?#E;!KAQJUX7D9u|^s>E3|N ztznZzK>6Nq8Q3cwk+19pE8)D&!vu>(xf`iY-m*h>d;2PI&MD8nfDb%J+BE(#nulg1 zSVPo-GRtNt%5kHWRHksQn(pg!CbF_El_$5j>x4IGKb%>>E}=4$Ag9U&Gx{_!`y>|y ze==LV13o0nlux13&QcKLA4?u550!V0O&Cdf&tcp--g_`O{FLnt}GYG(u1A0Ik zLGDf11hs0FA85L80C^M`&k=BLq*NDj@@qW|fSZUjgj(b2YFH9`g`rwHi52axnMst5 zs*jhMP`fm)))R4b<*LvD$<38!eu7>ddSKTdk}fG=d^JFtdGF^K_Kw{TPO0$D82k@T zmaQmEi^V5;2Mi~^4@x%v!NZ6|Gqp0X)lDpF1kN--~oycIM)*4 z%PPUyMDXf>R%TM{-ZJFdk}C)PSq8u%(Xa`<3g7q0C;Ss#qP#(xL+Jf^S9l zq?$?OxnN?Wdc{zQyi{OuBJXg&>Cpmwa6zQgm7Il2@heDC8EJC`+P00lkzD2^DFqVa zj9Q_-8JE&^2{8(K)W_CiXvDXk2@+}eRgz&?cN5~0t*BIrEzYPE++D!DAs2mrG zO1Xi{u|(T_3MDBQ9ZeC_Y=n?Q$TE0xT(SMr+}a?rXr4Q z>rYQ}1aY*znGWnwEC|hGJw8W1I722TttN-MmwjSAjw2tFz%`?CNSGQ_Hb+nGB{>~p zspJX*R4zR;PcDK#@NnZ`<-ZE);13gdafR1g-`0YSVTS6pESc6<4XfxAT*n_UM1>LU zKlFZ3Q|=H46jf#k{PqEdqE?&7$q)ZQGC;t&0Cscwrp2({zq4BuIb$uwQ8+7B*MP+n z8vWXAx9L9I_Ue1IK^;qg0jGDy&0~1td;>p-dw$z+qF~AWoT|W=YK%})tnq_6uE(@Q zk&{-BT4088*lcw7EO2hZE^?S^7~D!*9LSTupekYmaSvD4x}N>_{J^LEcj328gu^gh z`=&RnTO3YWd1O2yzO%IF?axY!f?Iy+Wd7sTZkqfs6pJ){JsuAQ!gQ=iO&u>-)NFEB zu$I8IS7!Q2?GO6D7r*r``&T1;aZ_*z008X&lN|KF^3wl@EHq19+73q(wX4SSp&xH} zMUKoLp^stGd|i?UnFwVU5Nsvtn<0|Hf|QE#rIxK*Yg@UFD({LKYX)2ZC~?pVfDp|} zFIqT%kLwd~qkx12gtYZa9t7l6>u-rQhu9F?#nzbewxji{%)1R`t*YerS8{Irw>wm! z4=FTXplY{msZ0M@kvYF8iyWo%dcWUkmL543YWIW9Vf#uvq)P72Q?Kd9@RH)(gN^F0p`ihV>iQdRT;x8&l8t$S{WxPeT4Rp~Zh4h7#wC1c9<6sSK)?;W5A=cxh z=CZ9cdj3mFRP+bYH9mXq;zeC%zgRZB?sTtadnjDK**9*#aGk!hheeqT5MqK$GUJkr zv&8Zg5QvfqSe9igN-`6!iocP6OpJuVWAiiY5zJ?h!2QsbhTR&08D*A@hWZR)$t%Q_Z;oSlA-FO1SFN8tq1-^=p4V*>2?_!B43Z333fQtR$lW|a2nn(Lk||15p!*YGyNq{O4Cgx~ zH%hMDz2U$oW~1CEFuy`2X@Y)`)XHI%Hb}wDD{XknbsBuz>+H%I*s$V?i?AC)@Z83! z=T4}kn`5TfjsEA7R&#r=IeO@;VoRe(b4HZX)nqs~hkdB-+{W+S1%R2}A+z)7f+r3;xO{geS)6r^c=oZ`0og|*+=c_9^d4v;-o!m^YP{W+pcQOT z-OaEK1u-LWSQZSJPHp-_N8mNZ)WhPvpiRWol{&{b_g>u&FzR7y3c5T2(VG&5%6zG| z6lHNi?J$!}mI|MAhykl(t#ak86P5Q>CY@p{=U_;Ts##WZgOH6v@(Of+)nRa2`Xp;} z@F0Q^+zXddCdl;ZrDPuVVS5CL;a0`g;F$zzGuEMeE)nHqEE)0 zI?NnMjCmNK8^#O^W}WC3EKFu(v56z$bc-{Q--EH$37F$i!HDWUu-S9tB`mBpW`}(v zi7HGZw!Ewg*ruDAK#Xcm9qC%3Z-I|8GI433I0MrYD?K8IX=y|7qj;dE#`2RW7wiVC zUUKXbb(@uGt6fQ~{@hq5R;U+vB_#FrVy1vv=G{p3YP|xQO#wD;%aoHai#a8V_DxW22IE{V4Tj?tJ zmBm-8L8UWCAnvCOs8;=uOwvMD7s{2sGFRM6lc;9xde=4*-)}N3ROCo2HsQB->m;xv z-LyXsKB%>eLQ0xzj5g0H6=;Dn3}UMmx}U?89v_)Ho<_3~?E>38J1avcQ(WH}Tgb$B z9~RKCW%xZs2;xj39tiYNA~kwk{Z<=W~nPa ztLP0(Hz-WMTwnpUNy$LNTaHmyS9>>2%*BpEO*_wKa|ip&)P19fx+iY=M0JSbL)P`@ z2hzReA(Pah63R2963OpToE0;OOsD%$m1nf6t1bWLK^f_^>BAAOwnF={Ppmbo!PMR- zUY_4RfZjxx3Du@G4oOOZc_ z*FGwUNZcAMr(%{172hQ>XP)RJ&|oLqe@eNL)e}y|9dfI)HNX(`3v6nVh09Oom>n8E zB~4sBmE697p3I@aE1SjEe}&A%Mz~ZGR0|ODyKTUhY#SNdit52#zCz3FCv4 z_CgMrMa;qLD5)~}BL#E(uqU!z@>RU=rRB=^E&CiwtQp6!k)WSHbr9+T;*O*Ng6kFR zC~zWI=WsGF!ieG;i0(G~KsUSm=-UbK3W|z9GHEF7Po zV&Ih`xe;mv8%Ux&OdsMsq@wt)F$qDVq71_fVHgXBHdfBVf7h{~nq(m*3kWIBn|KSf ztg4U63n~wosAk+q6=sbBn^~)(gQ6#PLltobl_Q8isv8w-pr}yR%%H;ktruF0qW0RI zGwcfCts0O-4JC|=ArN-fXkHb-DLT8O4dRING0Wf+Yi>&=R#zN@u*QR`Hq=9>rvhNn z8y7&ygahEkh}DQf;D#EYD^cKAJhcGpldE};lx(V%B%pU)4DXXG!tZ8azO5lv!ftZz zLad}<-<<$bXF&tNK3_ZsRa*LN$FuQGBz;qv?k%L{bfC(+jZ ze5CCP3JH&i0`}^K%@#-=TfOa=!GRq{Q~N8$VKvrf?gt$5W?VCzQ47EK;yoO>jocRG zEfE`cm4QbpDn-enfc#pM6CmQ*HFq+S7FdQ$D1}MXyL5{*T9=xF$CSm2l-!Jpw< z%ys2{YziHU{iAyMgzQ)7X#BaVp6t?cum2E$Fnyj`Eq?*_e`>wJaD3o$X%6XP`DG0W7Y9fJl5897RS z?y=*-IKui*!;fSouo1kzNVFS70Dym*6#U1H{3Ck%&kWk6x|Urg2a0cwo&lmNP;hO( zMAFZKDL!X*J5|@l6Kq^UCU+!iKT0;6g7VMKVf0#9ym-Y&gCoK^&gLs^P8{xU#B%NZ zMqiVq>k4Vy{UtW$eiU?XPcZ16P=9JK?P-R=q*R+reXCKc5LRUf+1z@2Rg4UI`QD&I z_>}Y%cALo(bmTE&?-DiZ?`kuB+q50S_#y;9=o5wUk*VGij%W+lKr-1+cl>;F%7w8? zJ*VH)UIhCC(dHIwxxs7G=iIZBHav$RRwBu~M5Ud?1}yBQ!SSTPa2sQqE)o{J<-8(A zvBlzPVA2E|2P}KXT)>BvKt^jlNNy&8u9k=uhe(CNnWYzj-|;?|3?`g|XoS#`f<3$K z=dN2J?iiwlF)7KeJOm~UBlmd#d_!!HOCKOoNk^$`Q?%j>e7%L@W1+%KO z#W!owAH{WW0&Bo#M+EB@^K|x#jy!IcGv8TJS;@43It#C zq%^p=CSI%4u$w^f8OU*#_ZX1Sr9tHhr?K595?9MxhLdp6!O;4*e{WY{FNodu_w-8&U$TBf! zK^b`%)FA>S*L0)AE)e~*>K%-+=h9)g*LKY*55Esq*Cx4DoF(}+P?>pD5#??E3bJNd zd4KBlQfi6-m^H#1jzr<6g;LmtGJ>|nDIIx@aNyV6x-PTeThnE2#S^yt94YIP^k;!nI>FR z=y=va3*5FEIE?N1Gu&_FCu`4{$pc%i@onjf^fSL-qL=h|3;GKvBw=HeRnX5RH%#N3 zHmp8xbG64td{a}(k%N>Lq4;=Cknnit-@@!oj9%VNWYjb!5f}vuj;*~zyUuc{Q}nyResiU>w*AE9KCg#Mw)5>| zm#~^2Gys61aIyl1EFtW~IJZybWsF}E>a(Tn4hCOq)DYMo{f05(@T>M1o?=AO4({@p zmkc-jPC=`Z*!-dt1eV#?Mu@>>w4D#uYAZZ?Nq1)7G#T>Qr)hKeF7(6~uw8rI%~Z5$ zStJ7YB|{A9@7v|*1Q&KQS26O} z+tpPr-qnQ^S(17-Di7IsG~|ZN%ugo9&&aC`Wz}1t_NjzsDG@$jpuf*fQQvws8NN0L zzOVI9!h}D3+Fv(>|LoNMtD~qSjUMdJFsSB9;0C zfc`RT=5HEKESAww!>>G93+t=L`oGJQ|H9Ay=f6>~rRZ3@T-;T58X=`pyxmo71->h(Bgf|vyYQ%T1y}z! zO}-^V!ix-#a=A$161hqh7g{M|)q=D&@0ypPFxxtfMAqeW9|awcw^FGt`Na=2f+;m< zux=7bUlj0Fh)E9aJzu@M%RU-JW^vn<`bw)N^vdOYL$SJI{&2bEj^u6#4wu;cYN*|Z zsZ_a3FXD!SAKk*_+4m8hk&QH&p1Ko?i%Hga0+|Fax-3N?Neh7cyb)sX>(}JiM3pT& zfpV|9OQ^I7af9{(*rX+M)DGJ|HdK)LCIzr|QoV8=Hj>~d5W`&Vv)61$ys1;hq*%ng z4|3oJa}-RC$-iL6;5&?!!_gJoz6*+)JYunjjR)Wfs5h;RiluMRE#~v>>qBwUgD2^^ zVg=NGgj#}hjG2aRYSB9tYbi?Up8Y;NtUg$Lmz}b`Oyt_yvc8KGz2^=zQz^mC>GB}) zzg)Drl+wkkDy0!YzxUpPv}HgXx#KHLOnv({6WG4#0>AA)B(fU7Z?<*%XfiRu1?XAN zT;r7sLZ+I@KN$Aw)oG9qhmlZszl@(zJ5RswTWM@&*^fMCKS~j$$$_>#(AQbhwTM!3 zAf16s0edc}`v+|eViA)&EjE)4^<$F^<)x6cnOwbIzWmYjMW8YK(-wpAy!b|j5ob)w zW*n=-yJ5@2yVa_;eB&|-yogLeUqzZ8c%hRh0i)tH^bOa1)aZK4b0roiO@kf>bd|0^ z?T(xqqMNu&zTswKsg!2w2W8ofd%$%P)<{WcwILs4$o`UE+02u zTFH=BYTMnMMc$-@iHWFEj?F7Anl0^jBP*ZytwQJU^Y{4fD`o>JnSaK+x);gFohiiY z1V4`=RuZe1v^3s}y^@F%r7p@dh;^pckX!@BlQ2cpOd|c+Lf2z9Uls{YFhF+IK+ zFCiAT4`(PTuU5PD54D5#*~2(;q_2{9zk^tw^utNY;~f)eZ9)L}#zj5X-fDwQyvQymwTPt~k~HvkD`)_|`MJ`HuWk6ur*ere*t2bzUzV@gxWs z4flzUd(&PJuMI3sb~@De-ap-R^TjNMgrT^?c@*vU+q%!BXuog1dZ%WO$ZMG)=Z$JE zD~EAT*G$LH+chsd)Y~PW`Mp-~Ee`W^?vVW0d<|uZ)5j=9G`yc-iu_|^53z;Hn*YAA zi#l$yyx`o84Y#+bnhb~!j&AHDXtsRzYb4spCrSyqMwy>$FTI=QG&kDg;|~kj_#+Q= za0M2=GM!2BYlu=Siz4UF$1^VL(tQ%wL}+Hv=uNs|+P$ic--M2tRs8w5bx!m1ViBBr z%j-;$>fY5OhgaLI>^ZBfL06D!k{Epxci7?B0L>>NBi4y79)DLjwC|=1g7Zp(x z-1Ow}E;|PsQa^~%+G0Z88+ulkZH}%nXmIk@aceJiuj@$LPY&t0{_fG3^V#E`VlMR7 zMmdokUrBe|>mHJHYMgXsq@0eNqwBSfc7^oNgWA7dE_!||+luO10m)e_U$>Wj{{j(|LZ>Fi0=6yFp{OcLIE%Dpit7M-EsV|YZxOQt5 z@j}@19}A*Fw#b_LhGIJ;UiPoj@0^`d)39lkTCmHPW%)ws*;c1Sk8N`40_{|KsLldQn^1D8IZQ#If{y zx}HMami4FOTAE(@bcLOvl@uLYB$HvQ9956zG6wb=}A#Tbw!{0 z7jIMVM80r(Y=ZyuwMdUvZMIz3&Oj%t6m#nFTlAbExf_Y|i!1wd!q!IpwgvaY{;a1g zBOlwy-M47mxR)Y_jmc=3~1vM9ZAUa;`ZjjVc(D<1(WAy9Bz3m)fKxs6Of*j%v?bzLxE} zokHnaAea}^?3P2VjMGZIpHOK=h#jH17*r(}Q5bc>JISeq4D43h zTJ_3+If@S=72dpVvfH&XRjTZ-&i(IUn`kBJ@$pyP zIIRn$Px%Q+V|oo`ac8QGA04ncCH5oY_WucH9KBjZOVTQP3l z)zFu{b9>#;by+!Q=PX-i$71sULPyRrWk!050in{uY27m~vHS6BMHlFPAr$s%cH5VV z)d{Ou@2#?q>a9@w@VH~Rq1MZL^O+>FpuOELM-g9f?EU_%BM&UYbfsn;y=vA?w#K&i z201s;j-(T@Bf{M=+FjE6x`v9o=MDYRdR}j(fnE5w58um_o6CRvxL96@PinpFP3+5l zw@8tC+;;9;?|!ANRswPD<9LbZTx|I)p;50vg6g4{E?YbXlY2Ccmw5Jt z;M6+<=%`+OcnWiKa5r`;l?vUP0Q>|uiakMTCRFaxk!9P;jLr?{aucAkRvPjXqXmFq zfmvJy?+pxuhV4QFR{2T;Z2;kbgn)aHw?&+^peK2i%!lr9G($mJ&&Dy+qHHEOpCIo* zPlAX0-hF@j5er?S-2L|GK_`@mN>Ezh0X=M?)0|vF#_=rN?0=Z9V^)^J{(>0HC8$W( z0a{#x27RH{A^;k+&t*mpI!|bSdYYh|#X#A#3c&xe7!l9T zT>ebg9Qzb|0m@+JN1NhLHmgPhHMp`T0*#cp9M^(7$eHK+C(MMB%3YhQe7HN&W56}m za7_GCu9zu#gPsKLHinbhpq0m5OVN|$8Au>Z>;dN0$hem~cmd_MoGXL1dIoKwgKLc8 zCY10J53EN+au|1G7PGUP*}KG{n|E9s_V>kXXbl6G{jv;WCS2}OPqZkmtABY#jg#u& z7FamFeg#THVatQ~D`#-~CR~SBeUv)nZT7$*fawr*pvDB z(*j%y0oTI8W(HbtUlD<>Pl1aFSSql(;r=ZMDzJ@5s%lQJ0!Qb=X=>XLXcKvlMB)@T zIB*?KvbRMbvGW^=#<2?=t`4WQAkjFZ*3ktFoOTXJQFkC1Hd!+0?E+_z!(k`vrohbCQI0qH3!<9YLcoA+YJKOoAgL6gU^kxDI`sBa?K0ndG5l!&?#ClCbXA{xqDL7{V zPALeOhQjtL5`~jX;6w#DMJsR`%9Q*;VsR=De5@bNnhkAI0Ip6^Tf&h(S<}pSu&zU= zN}oIo!n@$Oiw5iZ!E~bG$<7+j%`9dPvJ zSQO~VlY;zJGkB^K4%a$~0?y7A{^;OQNO)HIotlO|X)aRgb3zNAO@c$rQm28i%|cHB j&mF-j4bxFLr;w&g6k+IAAoHgSJVph&|9YIc?*#J~R&web diff --git a/LanguageModeling/bert_oneflow/dataset/__init__.py b/LanguageModeling/bert_oneflow/dataset/__init__.py deleted file mode 100644 index 90e9036..0000000 --- a/LanguageModeling/bert_oneflow/dataset/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -from .dataset import BERTDataset -from .vocab import WordVocab diff --git a/LanguageModeling/bert_oneflow/dataset/dataset.py b/LanguageModeling/bert_oneflow/dataset/dataset.py deleted file mode 100644 index 597d4a6..0000000 --- a/LanguageModeling/bert_oneflow/dataset/dataset.py +++ /dev/null @@ -1,127 +0,0 @@ -# from oneflow.utils.data import Dataset -import tqdm -import oneflow as flow -import torch -import oneflow.nn as nn -import random - - -class BERTDataset(nn.Module): - def __init__(self, corpus_path, vocab, seq_len, encoding="utf-8", corpus_lines=None, on_memory=True): - self.vocab = vocab - self.seq_len = seq_len - - self.on_memory = on_memory - self.corpus_lines = corpus_lines - self.corpus_path = corpus_path - self.encoding = encoding - - with open(corpus_path, "r", encoding=encoding) as f: - if self.corpus_lines is None and not on_memory: - for _ in tqdm.tqdm(f, desc="Loading Dataset", total=corpus_lines): - self.corpus_lines += 1 - - if on_memory: - self.lines = [line[:-1].split("\t") - for line in tqdm.tqdm(f, desc="Loading Dataset", total=corpus_lines)] - self.corpus_lines = len(self.lines) - - if not on_memory: - self.file = open(corpus_path, "r", encoding=encoding) - self.random_file = open(corpus_path, "r", encoding=encoding) - - for _ in range(random.randint(self.corpus_lines if self.corpus_lines < 1000 else 1000)): - self.random_file.__next__() - - def __len__(self): - return self.corpus_lines - - def __getitem__(self, item): - t1, t2, is_next_label = self.random_sent(item) - t1_random, t1_label = self.random_word(t1) - t2_random, t2_label = self.random_word(t2) - - # [CLS] tag = SOS tag, [SEP] tag = EOS tag - t1 = [self.vocab.sos_index] + t1_random + [self.vocab.eos_index] - t2 = t2_random + [self.vocab.eos_index] - - t1_label = [self.vocab.pad_index] + t1_label + [self.vocab.pad_index] - t2_label = t2_label + [self.vocab.pad_index] - - segment_label = ([1 for _ in range(len(t1))] + [2 for _ in range(len(t2))])[:self.seq_len] - bert_input = (t1 + t2)[:self.seq_len] - bert_label = (t1_label + t2_label)[:self.seq_len] - - padding = [self.vocab.pad_index for _ in range(self.seq_len - len(bert_input))] - bert_input.extend(padding), bert_label.extend(padding), segment_label.extend(padding) - - output = {"bert_input": bert_input, - "bert_label": bert_label, - "segment_label": segment_label, - "is_next": is_next_label} - - return {key: torch.tensor(value) for key, value in output.items()} - - def random_word(self, sentence): - tokens = sentence.split() - output_label = [] - - for i, token in enumerate(tokens): - prob = random.random() - if prob < 0.15: - prob /= 0.15 - - # 80% randomly change token to mask token - if prob < 0.8: - tokens[i] = self.vocab.mask_index - - # 10% randomly change token to random token - elif prob < 0.9: - tokens[i] = random.randrange(len(self.vocab)) - - # 10% randomly change token to current token - else: - tokens[i] = self.vocab.stoi.get(token, self.vocab.unk_index) - - output_label.append(self.vocab.stoi.get(token, self.vocab.unk_index)) - - else: - tokens[i] = self.vocab.stoi.get(token, self.vocab.unk_index) - output_label.append(0) - - return tokens, output_label - - def random_sent(self, index): - t1, t2 = self.get_corpus_line(index) - - # output_text, label(isNotNext:0, isNext:1) - if random.random() > 0.5: - return t1, t2, 1 - else: - return t1, self.get_random_line(), 0 - - def get_corpus_line(self, item): - if self.on_memory: - return self.lines[item][0], self.lines[item][1] - else: - line = self.file.__next__() - if line is None: - self.file.close() - self.file = open(self.corpus_path, "r", encoding=self.encoding) - line = self.file.__next__() - - t1, t2 = line[:-1].split("\t") - return t1, t2 - - def get_random_line(self): - if self.on_memory: - return self.lines[random.randrange(len(self.lines))][1] - - line = self.file.__next__() - if line is None: - self.file.close() - self.file = open(self.corpus_path, "r", encoding=self.encoding) - for _ in range(random.randint(self.corpus_lines if self.corpus_lines < 1000 else 1000)): - self.random_file.__next__() - line = self.random_file.__next__() - return line[:-1].split("\t")[1] diff --git a/LanguageModeling/bert_oneflow/dataset/vocab.py b/LanguageModeling/bert_oneflow/dataset/vocab.py deleted file mode 100644 index eb61d72..0000000 --- a/LanguageModeling/bert_oneflow/dataset/vocab.py +++ /dev/null @@ -1,186 +0,0 @@ -import pickle -import tqdm -from collections import Counter - - -class FlowVocab(object): - """Defines a vocabulary object that will be used to numericalize a field. - Attributes: - freqs: A collections.Counter object holding the frequencies of tokens - in the data used to build the Vocab. - stoi: A collections.defaultdict instance mapping token strings to - numerical identifiers. - itos: A list of token strings indexed by their numerical identifiers. - """ - - def __init__(self, counter, max_size=None, min_freq=1, specials=['', ''], - vectors=None, unk_init=None, vectors_cache=None): - """Create a Vocab object from a collections.Counter. - Arguments: - counter: collections.Counter object holding the frequencies of - each value found in the data. - max_size: The maximum size of the vocabulary, or None for no - maximum. Default: None. - min_freq: The minimum frequency needed to include a token in the - vocabulary. Values less than 1 will be set to 1. Default: 1. - specials: The list of special tokens (e.g., padding or eos) that - will be prepended to the vocabulary in addition to an - token. Default: [''] - vectors: One of either the available pretrained vectors - or custom pretrained vectors (see Vocab.load_vectors); - or a list of aforementioned vectors - unk_init (callback): by default, initialize out-of-vocabulary word vectors - to zero vectors; can be any function that takes in a Tensor and - returns a Tensor of the same size. Default: torch.Tensor.zero_ - vectors_cache: directory for cached vectors. Default: '.vector_cache' - """ - self.freqs = counter - counter = counter.copy() - min_freq = max(min_freq, 1) - - self.itos = list(specials) - # frequencies of special tokens are not counted when building vocabulary - # in frequency order - for tok in specials: - del counter[tok] - - max_size = None if max_size is None else max_size + len(self.itos) - - # sort by frequency, then alphabetically - words_and_frequencies = sorted(counter.items(), key=lambda tup: tup[0]) - words_and_frequencies.sort(key=lambda tup: tup[1], reverse=True) - - for word, freq in words_and_frequencies: - if freq < min_freq or len(self.itos) == max_size: - break - self.itos.append(word) - - # stoi is simply a reverse dict for itos - self.stoi = {tok: i for i, tok in enumerate(self.itos)} - - self.vectors = None - if vectors is not None: - self.load_vectors(vectors, unk_init=unk_init, cache=vectors_cache) - else: - assert unk_init is None and vectors_cache is None - - def __eq__(self, other): - if self.freqs != other.freqs: - return False - if self.stoi != other.stoi: - return False - if self.itos != other.itos: - return False - if self.vectors != other.vectors: - return False - return True - - def __len__(self): - return len(self.itos) - - def vocab_rerank(self): - self.stoi = {word: i for i, word in enumerate(self.itos)} - - def extend(self, v, sort=False): - words = sorted(v.itos) if sort else v.itos - for w in words: - if w not in self.stoi: - self.itos.append(w) - self.stoi[w] = len(self.itos) - 1 - - -class Vocab(FlowVocab): - def __init__(self, counter, max_size=None, min_freq=1): - self.pad_index = 0 - self.unk_index = 1 - self.eos_index = 2 - self.sos_index = 3 - self.mask_index = 4 - super().__init__(counter, specials=["", "", "", "", ""], - max_size=max_size, min_freq=min_freq) - - def to_seq(self, sentece, seq_len, with_eos=False, with_sos=False) -> list: - pass - - def from_seq(self, seq, join=False, with_pad=False): - pass - - @staticmethod - def load_vocab(vocab_path: str) -> 'Vocab': - with open(vocab_path, "rb") as f: - return pickle.load(f) - - def save_vocab(self, vocab_path): - with open(vocab_path, "wb") as f: - pickle.dump(self, f) - - -# Building Vocab with text files -class WordVocab(Vocab): - def __init__(self, texts, max_size=None, min_freq=1): - print("Building Vocab") - counter = Counter() - for line in tqdm.tqdm(texts): - if isinstance(line, list): - words = line - else: - words = line.replace("\n", "").replace("\t", "").split() - - for word in words: - counter[word] += 1 - super().__init__(counter, max_size=max_size, min_freq=min_freq) - - def to_seq(self, sentence, seq_len=None, with_eos=False, with_sos=False, with_len=False): - if isinstance(sentence, str): - sentence = sentence.split() - - seq = [self.stoi.get(word, self.unk_index) for word in sentence] - - if with_eos: - seq += [self.eos_index] # this would be index 1 - if with_sos: - seq = [self.sos_index] + seq - - origin_seq_len = len(seq) - - if seq_len is None: - pass - elif len(seq) <= seq_len: - seq += [self.pad_index for _ in range(seq_len - len(seq))] - else: - seq = seq[:seq_len] - - return (seq, origin_seq_len) if with_len else seq - - def from_seq(self, seq, join=False, with_pad=False): - words = [self.itos[idx] - if idx < len(self.itos) - else "<%d>" % idx - for idx in seq - if not with_pad or idx != self.pad_index] - - return " ".join(words) if join else words - - @staticmethod - def load_vocab(vocab_path: str) -> 'WordVocab': - with open(vocab_path, "rb") as f: - print("vocab_path >>>>>>>>>>>>>>>>>>>>>> ", vocab_path) - return pickle.load(f) - - -def build(): - import argparse - - parser = argparse.ArgumentParser() - parser.add_argument("-c", "--corpus_path", required=True, type=str) - parser.add_argument("-o", "--output_path", required=True, type=str) - parser.add_argument("-s", "--vocab_size", type=int, default=None) - parser.add_argument("-e", "--encoding", type=str, default="utf-8") - parser.add_argument("-m", "--min_freq", type=int, default=1) - args = parser.parse_args() - - with open(args.corpus_path, "r", encoding=args.encoding) as f: - vocab = WordVocab(f, max_size=args.vocab_size, min_freq=args.min_freq) - - print("VOCAB SIZE:", len(vocab)) - vocab.save_vocab(args.output_path) diff --git a/LanguageModeling/bert_oneflow/main.py b/LanguageModeling/bert_oneflow/main.py deleted file mode 100644 index 0ab3c30..0000000 --- a/LanguageModeling/bert_oneflow/main.py +++ /dev/null @@ -1,83 +0,0 @@ -import argparse - -from torch.utils.data import DataLoader -# from oneflow.utils.data import DataLoader -import oneflow.experimental as flow -from model import BERT -from trainer import BERTTrainer -from dataset import BERTDataset, WordVocab - -import numpy as np - -# eager mode -flow.env.init() -flow.enable_eager_execution() - -def train(): - - parser = argparse.ArgumentParser() - - parser.add_argument("-c", "--train_dataset", required=False, type=str, default='data/corpus.small', help="train dataset for train bert") - parser.add_argument("-t", "--test_dataset", type=str, default='data/corpus.small', help="test set for evaluate train set") - parser.add_argument("-v", "--vocab_path", required=False, default='data/vocab.small', type=str, help="built vocab model path with bert-vocab") - parser.add_argument("-o", "--output_path", required=False, default='output/bert.model', type=str, help="ex)output/bert.model") - - parser.add_argument("-hs", "--hidden", type=int, default=256, help="hidden size of transformer model") - parser.add_argument("-l", "--layers", type=int, default=8, help="number of layers") - parser.add_argument("-a", "--attn_heads", type=int, default=8, help="number of attention heads") - parser.add_argument("-s", "--seq_len", type=int, default=20, help="maximum sequence len") - - parser.add_argument("-b", "--batch_size", type=int, default=16, help="number of batch_size") - parser.add_argument("-e", "--epochs", type=int, default=10, help="number of epochs") - parser.add_argument("-w", "--num_workers", type=int, default=5, help="dataloader worker size") - - parser.add_argument("--with_cuda", type=bool, default=True, help="training with CUDA: true, or false") - parser.add_argument("--corpus_lines", type=int, default=None, help="total number of lines in corpus") - parser.add_argument("--cuda_devices", type=int, nargs='+', default=None, help="CUDA device ids") - parser.add_argument("--on_memory", type=bool, default=True, help="Loading on memory: true or false") - - parser.add_argument("--lr", type=float, default=1e-3, help="learning rate of adam") - parser.add_argument("--adam_weight_decay", type=float, default=0.01, help="weight_decay of adam") - parser.add_argument("--adam_beta1", type=float, default=0.9, help="adam first beta value") - parser.add_argument("--adam_beta2", type=float, default=0.999, help="adam first beta value") - - args = parser.parse_args() - - print("Loading Vocab", args.vocab_path) - vocab = WordVocab.load_vocab(args.vocab_path) - print("Vocab Size: ", len(vocab)) - - print("Loading Train Dataset", args.train_dataset) - train_dataset = BERTDataset(args.train_dataset, vocab, seq_len=args.seq_len, - corpus_lines=args.corpus_lines, on_memory=args.on_memory) - - print("Loading Test Dataset", args.test_dataset) - test_dataset = BERTDataset(args.test_dataset, vocab, seq_len=args.seq_len, on_memory=args.on_memory) \ - if args.test_dataset is not None else None - - print("Creating Dataloader") - train_data_loader = DataLoader(train_dataset, batch_size=args.batch_size, num_workers=args.num_workers) - test_data_loader = DataLoader(test_dataset, batch_size=args.batch_size, num_workers=args.num_workers) \ - if test_dataset is not None else None - - print("Building BERT model") - bert = BERT(len(vocab), hidden=args.hidden, n_layers=args.layers, attn_heads=args.attn_heads) - - print("Creating BERT Trainer") - trainer = BERTTrainer(bert, len(vocab), train_dataloader=train_data_loader, test_dataloader=test_data_loader, - lr=args.lr, betas=(args.adam_beta1, args.adam_beta2), weight_decay=args.adam_weight_decay, - with_cuda=args.with_cuda, cuda_devices=args.cuda_devices, log_freq=10) - - print("Trainer build finished!!!\n") - - print("Training Start") - for epoch in range(args.epochs): - trainer.train(epoch) - trainer.save(epoch, args.output_path) - - if test_data_loader is not None: - trainer.test(epoch) - - - -train() \ No newline at end of file diff --git a/LanguageModeling/bert_oneflow/model/__init__.py b/LanguageModeling/bert_oneflow/model/__init__.py deleted file mode 100644 index aa318cb..0000000 --- a/LanguageModeling/bert_oneflow/model/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -from .bert import BERT -from .language_model import BERTLM diff --git a/LanguageModeling/bert_oneflow/model/attention/__init__.py b/LanguageModeling/bert_oneflow/model/attention/__init__.py deleted file mode 100644 index 6a39ec1..0000000 --- a/LanguageModeling/bert_oneflow/model/attention/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -from .multi_head import MultiHeadedAttention -from .single import Attention diff --git a/LanguageModeling/bert_oneflow/model/attention/multi_head.py b/LanguageModeling/bert_oneflow/model/attention/multi_head.py deleted file mode 100644 index 6252432..0000000 --- a/LanguageModeling/bert_oneflow/model/attention/multi_head.py +++ /dev/null @@ -1,40 +0,0 @@ -import oneflow.experimental.nn as nn -import oneflow as flow -from .single import Attention - - -class MultiHeadedAttention(nn.Module): - """ - Take in model size and number of heads. - """ - - def __init__(self, h, d_model, dropout=0.1): - super().__init__() - assert d_model % h == 0 - - # We assume d_v always equals d_k - self.d_k = d_model // h - self.h = h - self.linear_layers = nn.ModuleList([nn.Linear(d_model, d_model) for _ in range(3)]) - self.output_linear = nn.Linear(d_model, d_model) - self.attention = Attention() - - self.dropout = nn.Dropout(p=dropout) - - def forward(self, query, key, value, mask=None): - batch_size = query.size(0) # 16 - - # 1) Do all the linear projections in batch from d_model => h x d_k - query, key, value = [l(x).reshape(shape=[batch_size, -1, self.h, self.d_k]).permute(0, 2, 1, 3) - for l, x in zip(self.linear_layers, (query, key, value))] - # # query,key,value shape >> flow.Size([16, 8, 20, 32]) - - - # 2) Apply attention on all the projected vectors in batch. - x, attn = self.attention(query, key, value, mask, self.dropout) - - # 3) "Concat" using a view and apply a final linear. - res = x.transpose(1, 2).reshape(shape = [batch_size, -1, self.h * self.d_k]) - res = self.output_linear(res) - return res - diff --git a/LanguageModeling/bert_oneflow/model/attention/single.py b/LanguageModeling/bert_oneflow/model/attention/single.py deleted file mode 100644 index c028a15..0000000 --- a/LanguageModeling/bert_oneflow/model/attention/single.py +++ /dev/null @@ -1,29 +0,0 @@ -import oneflow.experimental.nn as nn -import oneflow.experimental as flow -import math -import numpy as np - -class Attention(nn.Module): - """ - Compute 'Scaled Dot Product Attention - """ - - def __init__(self): - super().__init__() - self.softmax = flow.nn.Softmax(dim = -1) - - def forward(self, query, key, value, mask=None, dropout=None): # q k v shape >> flow.Size([16, 8, 20, 32]) - x = flow.matmul(query, key.transpose(-2, -1)) - scores = x / math.sqrt(query.size(-1)) - - if mask is not None: - # scores = scores.masked_fill(mask == 0, -1e9) - mask = flow.Tensor((mask.numpy() == 0).astype(np.int8), dtype=flow.int, device=scores.device) - scores = scores.masked_fill(mask, -1e9) - - p_attn = self.softmax(scores) - - if dropout is not None: - p_attn = dropout(p_attn) - - return flow.matmul(p_attn, value), p_attn diff --git a/LanguageModeling/bert_oneflow/model/bert.py b/LanguageModeling/bert_oneflow/model/bert.py deleted file mode 100644 index 0d7f54c..0000000 --- a/LanguageModeling/bert_oneflow/model/bert.py +++ /dev/null @@ -1,49 +0,0 @@ -import oneflow.experimental.nn as nn - -from .transformer import TransformerBlock -from .embedding import BERTEmbedding -import numpy as np -import oneflow.experimental as flow - - -class BERT(nn.Module): - """ - BERT model : Bidirectional Encoder Representations from Transformers. - """ - - def __init__(self, vocab_size, hidden=768, n_layers=12, attn_heads=12, dropout=0.1): - """ - :param vocab_size: vocab_size of total words - :param hidden: BERT model hidden size - :param n_layers: numbers of Transformer blocks(layers) - :param attn_heads: number of attention heads - :param dropout: dropout rate - """ - - super().__init__() - self.hidden = hidden - self.n_layers = n_layers - self.attn_heads = attn_heads - - # paper noted they used 4*hidden_size for ff_network_hidden_size - self.feed_forward_hidden = hidden * 4 - - # embedding for BERT, sum of positional, segment, token embeddings - self.embedding = BERTEmbedding(vocab_size=vocab_size, embed_size=hidden) - - # multi-layers transformer blocks, deep network - self.transformer_blocks = nn.ModuleList( - [TransformerBlock(hidden, attn_heads, hidden * 4, dropout) for _ in range(n_layers)]) - - def forward(self, x, segment_info): # x.shape >> flow.Size([16, 20]) - # attention masking for padded token - - mask = (x > 0).unsqueeze(1).repeat(sizes=(1, x.shape[1], 1)).unsqueeze(1).repeat(sizes=(1, 8, 1, 1)) - - # embedding the indexed sequence to sequence of vectors - x = self.embedding(x, segment_info) - - # running over multiple transformer blocks - for transformer in self.transformer_blocks: - x = transformer.forward(x, mask) - return x diff --git a/LanguageModeling/bert_oneflow/model/embedding/__init__.py b/LanguageModeling/bert_oneflow/model/embedding/__init__.py deleted file mode 100644 index 0eb5843..0000000 --- a/LanguageModeling/bert_oneflow/model/embedding/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .bert import BERTEmbedding diff --git a/LanguageModeling/bert_oneflow/model/embedding/bert.py b/LanguageModeling/bert_oneflow/model/embedding/bert.py deleted file mode 100644 index 178aab2..0000000 --- a/LanguageModeling/bert_oneflow/model/embedding/bert.py +++ /dev/null @@ -1,65 +0,0 @@ -import oneflow.experimental as flow -import oneflow.experimental.nn as nn -import numpy as np -import math - - -class TokenEmbedding(nn.Embedding): - def __init__(self, vocab_size, embed_size=512): - super().__init__(vocab_size, embed_size, padding_idx=0) - -class PositionalEmbedding(nn.Module): - - def __init__(self, d_model, max_len=512): - super().__init__() - - # Compute the positional encodings once in log space. - pe = flow.zeros(size=(max_len, d_model), requires_grad=False) - - position = flow.arange(0, max_len, dtype=flow.float).unsqueeze(1) - div_term = (flow.arange(0, d_model, 2, dtype=flow.float)* -(math.log(10000.0) / d_model)).exp() - - pe[:, 0::2] = flow.sin(position * div_term) - pe[:, 1::2] = flow.cos(position * div_term) - - pe = pe.unsqueeze(0) - self.register_buffer('pe', flow.Tensor(pe)) - - def forward(self, x): - return self.pe[:, :x.size(1)] - - -class SegmentEmbedding(nn.Embedding): - def __init__(self, embed_size=512): - super().__init__(3, embed_size, padding_idx=0) - - -class BERTEmbedding(nn.Module): - """ - BERT Embedding which is consisted with under features - 1. TokenEmbedding : normal embedding matrix - 2. PositionalEmbedding : adding positional information using sin, cos - 2. SegmentEmbedding : adding sentence segment info, (sent_A:1, sent_B:2) - - sum of all these features are output of BERTEmbedding - """ - - def __init__(self, vocab_size, embed_size, dropout=0.1): - """ - :param vocab_size: total vocab size - :param embed_size: embedding size of token embedding - :param dropout: dropout rate - """ - super().__init__() - self.token = TokenEmbedding(vocab_size=vocab_size, embed_size=embed_size) # shape >> flow.Size([16, 20, 256]) - self.position = PositionalEmbedding(d_model=self.token.embedding_dim) # shape >> flow.Size([1, 20, 256]) - self.segment = SegmentEmbedding(embed_size=self.token.embedding_dim) # shape >> flow.Size([16, 20, 256]) - - - self.dropout = flow.nn.Dropout(p=dropout) - self.embed_size = embed_size - - def forward(self, sequence, segment_label): # sequence/segment_label .shape >>> flow.Size([16, 20]) - sequence = sequence.to(dtype=flow.int) - x = self.segment(segment_label) + self.token(sequence) + self.position(sequence) - return self.dropout(x) diff --git a/LanguageModeling/bert_oneflow/model/language_model.py b/LanguageModeling/bert_oneflow/model/language_model.py deleted file mode 100644 index cf8462c..0000000 --- a/LanguageModeling/bert_oneflow/model/language_model.py +++ /dev/null @@ -1,61 +0,0 @@ -import oneflow.experimental as flow -import oneflow.experimental.nn as nn -from .bert import BERT -import numpy as np - - -class BERTLM(nn.Module): - """ - BERT Language Model - Next Sentence Prediction Model + Masked Language Model - """ - - def __init__(self, bert: BERT, vocab_size): - """ - :param bert: BERT model which should be trained - :param vocab_size: total vocab size for masked_lm - """ - - super().__init__() - self.bert = bert - self.next_sentence = NextSentencePrediction(self.bert.hidden) - self.mask_lm = MaskedLanguageModel(self.bert.hidden, vocab_size) - - def forward(self, x, segment_label): - x = self.bert(x, segment_label) - return self.next_sentence(x), self.mask_lm(x) - - -class NextSentencePrediction(nn.Module): - """ - 2-class classification model : is_next, is_not_next - """ - - def __init__(self, hidden): - """ - :param hidden: BERT model output size - """ - super().__init__() - self.linear = nn.Linear(hidden, 2) - self.softmax = nn.LogSoftmax(dim=-1) - - def forward(self, x): # x.shape >> flow.Size([16, 20, 256]) - return self.softmax(self.linear(x[:, 0])) - -class MaskedLanguageModel(nn.Module): - """ - predicting origin token from masked input sequence - n-class classification problem, n-class = vocab_size - """ - - def __init__(self, hidden, vocab_size): - """ - :param hidden: output size of BERT model - :param vocab_size: total vocab size - """ - super().__init__() - self.linear = nn.Linear(hidden, vocab_size) - self.softmax = nn.LogSoftmax(dim=-1) - - def forward(self, x): - return self.softmax(self.linear(x)) diff --git a/LanguageModeling/bert_oneflow/model/transformer.py b/LanguageModeling/bert_oneflow/model/transformer.py deleted file mode 100644 index 8b41eca..0000000 --- a/LanguageModeling/bert_oneflow/model/transformer.py +++ /dev/null @@ -1,32 +0,0 @@ -import oneflow.experimental.nn as nn -import oneflow.experimental as flow - -from .attention import MultiHeadedAttention -from .utils import SublayerConnection, PositionwiseFeedForward - - -class TransformerBlock(nn.Module): - """ - Bidirectional Encoder = Transformer (self-attention) - Transformer = MultiHead_Attention + Feed_Forward with sublayer connection - """ - - def __init__(self, hidden, attn_heads, feed_forward_hidden, dropout): - """ - :param hidden: hidden size of transformer - :param attn_heads: head sizes of multi-head attention - :param feed_forward_hidden: feed_forward_hidden, usually 4*hidden_size - :param dropout: dropout rate - """ - super().__init__() - self.multihead_attention = MultiHeadedAttention(h=attn_heads, d_model=hidden) - self.feed_forward = PositionwiseFeedForward(d_model=hidden, d_ff=feed_forward_hidden, dropout=dropout) - self.input_sublayer = SublayerConnection(size=hidden, dropout=dropout) - self.output_sublayer = SublayerConnection(size=hidden, dropout=dropout) - self.dropout = nn.Dropout(p=dropout) - - def forward(self, x, mask): - # input/output shape >> flow.Size([16, 20, 256]) - x = self.input_sublayer(x, lambda _x: self.multihead_attention.forward(_x, _x, _x, mask=mask)) - x = self.output_sublayer(x, self.feed_forward) - return self.dropout(x) diff --git a/LanguageModeling/bert_oneflow/model/utils/__init__.py b/LanguageModeling/bert_oneflow/model/utils/__init__.py deleted file mode 100644 index e7bddc6..0000000 --- a/LanguageModeling/bert_oneflow/model/utils/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -from .feed_forward import PositionwiseFeedForward -from .layer_norm import LayerNorm -from .sublayer import SublayerConnection -from .gelu import GELU diff --git a/LanguageModeling/bert_oneflow/model/utils/feed_forward.py b/LanguageModeling/bert_oneflow/model/utils/feed_forward.py deleted file mode 100644 index 8ae290d..0000000 --- a/LanguageModeling/bert_oneflow/model/utils/feed_forward.py +++ /dev/null @@ -1,19 +0,0 @@ -import oneflow.experimental.nn as nn -from .gelu import GELU - - -class PositionwiseFeedForward(nn.Module): - "Implements FFN equation." - - def __init__(self, d_model, d_ff, dropout=0.1): - super(PositionwiseFeedForward, self).__init__() - # d_model,d_ff >> 256,1024 - self.w_1 = nn.Linear(d_model, d_ff) - self.w_2 = nn.Linear(d_ff, d_model) - self.dropout = nn.Dropout(dropout) - self.activation = GELU() - # self.activation = nn.GELU() # NOTE: torch原实现为调用GELU()、此处直接可用已有module(nn.GELU)代替 - - def forward(self, x): # input/output shape >> flow.Size([16, 20, 256]) - return self.w_2(self.dropout(self.activation(self.w_1(x)))) - diff --git a/LanguageModeling/bert_oneflow/model/utils/gelu.py b/LanguageModeling/bert_oneflow/model/utils/gelu.py deleted file mode 100644 index 3fbc7e6..0000000 --- a/LanguageModeling/bert_oneflow/model/utils/gelu.py +++ /dev/null @@ -1,14 +0,0 @@ -import oneflow.experimental.nn as nn -import oneflow.experimental as flow -import math - -class GELU(nn.Module): - """ - Paper Section 3.4, last paragraph notice that BERT used the GELU instead of RELU - """ - def __init__(self): - super().__init__() - - def forward(self, x): - tmp = flow.Tensor([math.sqrt(2 / math.pi)], device=x.device) - return 0.5 * x * (1 + flow.tanh(tmp) * (x + 0.044715 * x.pow(3.0))) \ No newline at end of file diff --git a/LanguageModeling/bert_oneflow/model/utils/layer_norm.py b/LanguageModeling/bert_oneflow/model/utils/layer_norm.py deleted file mode 100644 index 711c257..0000000 --- a/LanguageModeling/bert_oneflow/model/utils/layer_norm.py +++ /dev/null @@ -1,18 +0,0 @@ -import oneflow.experimental.nn as nn -import oneflow.experimental as flow - - -class LayerNorm(nn.Module): - "Construct a layernorm module (See citation for details)." - - def __init__(self, features, eps=1e-6): - super(LayerNorm, self).__init__() - self.eps = eps - self.a_2 = nn.Parameter(flow.Tensor(flow.ones(features, dtype=flow.float32))) - self.b_2 = nn.Parameter(flow.Tensor(flow.zeros(features, dtype=flow.float32))) - - - def forward(self, x): # x input/output >> shape flow.Size([16, 20, 256]) - mean = x.mean(-1, keepdim=True) - std = x.std(dim=-1, keepdim=True) - return self.a_2 * (x - mean) / (std + self.eps) + self.b_2 diff --git a/LanguageModeling/bert_oneflow/model/utils/sublayer.py b/LanguageModeling/bert_oneflow/model/utils/sublayer.py deleted file mode 100644 index 810881d..0000000 --- a/LanguageModeling/bert_oneflow/model/utils/sublayer.py +++ /dev/null @@ -1,19 +0,0 @@ -import oneflow.experimental.nn as nn -from .layer_norm import LayerNorm - - -class SublayerConnection(nn.Module): - """ - A residual connection followed by a layer norm. - Note for code simplicity the norm is first as opposed to last. - """ - - def __init__(self, size, dropout): - super(SublayerConnection, self).__init__() - self.norm = LayerNorm(size) - self.dropout = nn.Dropout(dropout) - - def forward(self, x, sublayer): - "Apply residual connection to any sublayer with the same size." - return x + self.dropout(sublayer(self.norm(x))) - # return x + self.dropout(sublayer(x)) diff --git a/LanguageModeling/bert_oneflow/requirements.txt b/LanguageModeling/bert_oneflow/requirements.txt deleted file mode 100644 index 888a50f..0000000 --- a/LanguageModeling/bert_oneflow/requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -tqdm -numpy -torch==0.4.1.post2 \ No newline at end of file diff --git a/LanguageModeling/bert_oneflow/test.py b/LanguageModeling/bert_oneflow/test.py deleted file mode 100644 index 8083a55..0000000 --- a/LanguageModeling/bert_oneflow/test.py +++ /dev/null @@ -1,73 +0,0 @@ -import argparse - -from torch.utils.data import DataLoader -# from oneflow.utils.data import DataLoader -import oneflow as flow -from model import BERT -from trainer import BERTTrainer -from dataset import BERTDataset, WordVocab - -# eager mode -flow.enable_eager_execution() - -def test(): - - parser = argparse.ArgumentParser() - - parser.add_argument("-c", "--train_dataset", required=False, type=str, default='data/corpus.small', help="train dataset for train bert") - parser.add_argument("-t", "--test_dataset", type=str, default='data/corpus.small', help="test set for evaluate train set") - parser.add_argument("-v", "--vocab_path", required=False, default='data/vocab.small', type=str, help="built vocab model path with bert-vocab") - parser.add_argument("-o", "--output_path", required=False, default='output/bert.model', type=str, help="ex)output/bert.model") - - parser.add_argument("-hs", "--hidden", type=int, default=256, help="hidden size of transformer model") - parser.add_argument("-l", "--layers", type=int, default=8, help="number of layers") - parser.add_argument("-a", "--attn_heads", type=int, default=8, help="number of attention heads") - parser.add_argument("-s", "--seq_len", type=int, default=20, help="maximum sequence len") - - parser.add_argument("-b", "--batch_size", type=int, default=16, help="number of batch_size") - parser.add_argument("-e", "--epochs", type=int, default=10, help="number of epochs") - parser.add_argument("-w", "--num_workers", type=int, default=5, help="dataloader worker size") - - parser.add_argument("--with_cuda", type=bool, default=True, help="training with CUDA: true, or false") - parser.add_argument("--corpus_lines", type=int, default=None, help="total number of lines in corpus") - parser.add_argument("--cuda_devices", type=int, nargs='+', default=None, help="CUDA device ids") - parser.add_argument("--on_memory", type=bool, default=True, help="Loading on memory: true or false") - - parser.add_argument("--lr", type=float, default=1e-3, help="learning rate of adam") - parser.add_argument("--adam_weight_decay", type=float, default=0.01, help="weight_decay of adam") - parser.add_argument("--adam_beta1", type=float, default=0.9, help="adam first beta value") - parser.add_argument("--adam_beta2", type=float, default=0.999, help="adam first beta value") - - args = parser.parse_args() - - print("Loading Vocab", args.vocab_path) - vocab = WordVocab.load_vocab(args.vocab_path) - print("Vocab Size: ", len(vocab)) - - print("Loading Train Dataset", args.train_dataset) - train_dataset = BERTDataset(args.train_dataset, vocab, seq_len=args.seq_len, - corpus_lines=args.corpus_lines, on_memory=args.on_memory) - - print("Loading Test Dataset", args.test_dataset) - test_dataset = BERTDataset(args.test_dataset, vocab, seq_len=args.seq_len, on_memory=args.on_memory) \ - if args.test_dataset is not None else None - - print("Creating Dataloader") - train_data_loader = DataLoader(train_dataset, batch_size=args.batch_size, num_workers=args.num_workers) - test_data_loader = DataLoader(test_dataset, batch_size=args.batch_size, num_workers=args.num_workers) \ - if test_dataset is not None else None - - print("Building BERT model") - bert = BERT(len(vocab), hidden=args.hidden, n_layers=args.layers, attn_heads=args.attn_heads) - - print("Creating BERT Trainer") - trainer = BERTTrainer(bert, len(vocab), train_dataloader=train_data_loader, test_dataloader=test_data_loader, - lr=args.lr, betas=(args.adam_beta1, args.adam_beta2), weight_decay=args.adam_weight_decay, - with_cuda=args.with_cuda, cuda_devices=args.cuda_devices, log_freq=10) - - print("Trainer build finished!!!\n") - print("Testing Start......") - trainer.test(1) - - -test() \ No newline at end of file diff --git a/LanguageModeling/bert_oneflow/test.sh b/LanguageModeling/bert_oneflow/test.sh deleted file mode 100644 index baf8317..0000000 --- a/LanguageModeling/bert_oneflow/test.sh +++ /dev/null @@ -1 +0,0 @@ -python3 test.py \ No newline at end of file diff --git a/LanguageModeling/bert_oneflow/train.py b/LanguageModeling/bert_oneflow/train.py deleted file mode 100644 index 3fd325f..0000000 --- a/LanguageModeling/bert_oneflow/train.py +++ /dev/null @@ -1,78 +0,0 @@ -import argparse - -from torch.utils.data import DataLoader -# from oneflow.utils.data import DataLoader -import oneflow as flow -from model import BERT -from trainer import BERTTrainer -from dataset import BERTDataset, WordVocab - -# eager mode -flow.enable_eager_execution() - -def main(): - - parser = argparse.ArgumentParser() - - parser.add_argument("-c", "--train_dataset", required=False, type=str, default='data/corpus.small', help="train dataset for train bert") - parser.add_argument("-t", "--test_dataset", type=str, default='data/corpus.small', help="test set for evaluate train set") - parser.add_argument("-v", "--vocab_path", required=False, default='data/vocab.small', type=str, help="built vocab model path with bert-vocab") - parser.add_argument("-o", "--output_path", required=False, default='output/bert.model', type=str, help="ex)output/bert.model") - - parser.add_argument("-hs", "--hidden", type=int, default=256, help="hidden size of transformer model") - parser.add_argument("-l", "--layers", type=int, default=8, help="number of layers") - parser.add_argument("-a", "--attn_heads", type=int, default=8, help="number of attention heads") - parser.add_argument("-s", "--seq_len", type=int, default=20, help="maximum sequence len") - - parser.add_argument("-b", "--batch_size", type=int, default=16, help="number of batch_size") - parser.add_argument("-e", "--epochs", type=int, default=10, help="number of epochs") - parser.add_argument("-w", "--num_workers", type=int, default=5, help="dataloader worker size") - - parser.add_argument("--with_cuda", type=bool, default=True, help="training with CUDA: true, or false") - parser.add_argument("--corpus_lines", type=int, default=None, help="total number of lines in corpus") - parser.add_argument("--cuda_devices", type=int, nargs='+', default=None, help="CUDA device ids") - parser.add_argument("--on_memory", type=bool, default=True, help="Loading on memory: true or false") - - parser.add_argument("--lr", type=float, default=1e-3, help="learning rate of adam") - parser.add_argument("--adam_weight_decay", type=float, default=0.01, help="weight_decay of adam") - parser.add_argument("--adam_beta1", type=float, default=0.9, help="adam first beta value") - parser.add_argument("--adam_beta2", type=float, default=0.999, help="adam first beta value") - - args = parser.parse_args() - - print("Loading Vocab", args.vocab_path) - vocab = WordVocab.load_vocab(args.vocab_path) - print("Vocab Size: ", len(vocab)) - - print("Loading Train Dataset", args.train_dataset) - train_dataset = BERTDataset(args.train_dataset, vocab, seq_len=args.seq_len, - corpus_lines=args.corpus_lines, on_memory=args.on_memory) - - print("Loading Test Dataset", args.test_dataset) - test_dataset = BERTDataset(args.test_dataset, vocab, seq_len=args.seq_len, on_memory=args.on_memory) \ - if args.test_dataset is not None else None - - print("Creating Dataloader") - train_data_loader = DataLoader(train_dataset, batch_size=args.batch_size, num_workers=args.num_workers) - test_data_loader = DataLoader(test_dataset, batch_size=args.batch_size, num_workers=args.num_workers) \ - if test_dataset is not None else None - - print("Building BERT model") - bert = BERT(len(vocab), hidden=args.hidden, n_layers=args.layers, attn_heads=args.attn_heads) - - print("Creating BERT Trainer") - trainer = BERTTrainer(bert, len(vocab), train_dataloader=train_data_loader, test_dataloader=test_data_loader, - lr=args.lr, betas=(args.adam_beta1, args.adam_beta2), weight_decay=args.adam_weight_decay, - with_cuda=args.with_cuda, cuda_devices=args.cuda_devices, log_freq=10) - - print("Trainer build finished!") - print("Training Start......") - for epoch in range(args.epochs): - trainer.train(epoch) - # print("Saving model...") - # trainer.save(epoch, args.output_path) - if test_data_loader is not None: - print("Running testing...") - trainer.test(epoch) - -main() \ No newline at end of file diff --git a/LanguageModeling/bert_oneflow/train.sh b/LanguageModeling/bert_oneflow/train.sh deleted file mode 100644 index ef69436..0000000 --- a/LanguageModeling/bert_oneflow/train.sh +++ /dev/null @@ -1 +0,0 @@ -python3 train.py \ No newline at end of file diff --git a/LanguageModeling/bert_oneflow/trainer/__init__.py b/LanguageModeling/bert_oneflow/trainer/__init__.py deleted file mode 100644 index 6a0eb37..0000000 --- a/LanguageModeling/bert_oneflow/trainer/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .pretrain import BERTTrainer diff --git a/LanguageModeling/bert_oneflow/trainer/optim_schedule.py b/LanguageModeling/bert_oneflow/trainer/optim_schedule.py deleted file mode 100644 index 5ccd222..0000000 --- a/LanguageModeling/bert_oneflow/trainer/optim_schedule.py +++ /dev/null @@ -1,35 +0,0 @@ -'''A wrapper class for optimizer ''' -import numpy as np - - -class ScheduledOptim(): - '''A simple wrapper class for learning rate scheduling''' - - def __init__(self, optimizer, d_model, n_warmup_steps): - self._optimizer = optimizer - self.n_warmup_steps = n_warmup_steps - self.n_current_steps = 0 - self.init_lr = np.power(d_model, -0.5) - - def step_and_update_lr(self): - "Step with the inner optimizer" - self._update_learning_rate() - self._optimizer.step() - - def zero_grad(self): - "Zero out the gradients by the inner optimizer" - self._optimizer.zero_grad() - - def _get_lr_scale(self): - return np.min([ - np.power(self.n_current_steps, -0.5), - np.power(self.n_warmup_steps, -1.5) * self.n_current_steps]) - - def _update_learning_rate(self): - ''' Learning rate scheduling per step ''' - - self.n_current_steps += 1 - lr = self.init_lr * self._get_lr_scale() - - for param_group in self._optimizer.param_groups: - param_group['lr'] = lr diff --git a/LanguageModeling/bert_oneflow/trainer/pretrain.py b/LanguageModeling/bert_oneflow/trainer/pretrain.py deleted file mode 100644 index 74fb8ee..0000000 --- a/LanguageModeling/bert_oneflow/trainer/pretrain.py +++ /dev/null @@ -1,159 +0,0 @@ - -import oneflow.experimental as flow -import oneflow.experimental.nn as nn -# from flow.utils.data import DataLoader -from torch.utils.data import DataLoader - -from model import BERTLM, BERT -from .optim_schedule import ScheduledOptim - -import tqdm -import numpy as np - - -class BERTTrainer: - """ - BERTTrainer make the pretrained BERT model with two LM training method. - - 1. Masked Language Model : 3.3.1 Task #1: Masked LM - 2. Next Sentence prediction : 3.3.2 Task #2: Next Sentence Prediction - - please check the details on README.md with simple example. - - """ - - def __init__(self, bert: BERT, vocab_size: int, - train_dataloader = None, test_dataloader = None, - lr: float = 1e-4, betas=(0.9, 0.999), weight_decay: float = 0.01, warmup_steps=10000, - with_cuda: bool = True, cuda_devices=None, log_freq: int = 10): - """ - :param bert: BERT model which you want to train - :param vocab_size: total word vocab size - :param train_dataloader: train dataset data loader - :param test_dataloader: test dataset data loader [can be None] - :param lr: learning rate of optimizer - :param betas: Adam optimizer betas - :param weight_decay: Adam optimizer weight decay param - :param with_cuda: traning with cuda - :param log_freq: logging frequency of the batch iteration - """ - - # Setup cuda device for BERT training, argument -c, --cuda should be true - # cuda_condition = flow.cuda.is_available() and with_cuda - cuda_condition=with_cuda - self.device = flow.device("cuda:0" if cuda_condition else "cpu") - - # This BERT model will be saved every epoch - self.bert = bert.to(self.device) - # Initialize the BERT Language Model, with BERT model - self.model = BERTLM(bert, vocab_size).to(self.device) - - # # Distributed GPU training if CUDA can detect more than 1 GPU - # if with_cuda and flow.cuda.device_count() > 1: - # print("Using %d GPUS for BERT" % flow.cuda.device_count()) - # self.model = nn.DataParallel(self.model, device_ids=cuda_devices) - - # Setting the train and test data loader - self.train_data = train_dataloader - self.test_data = test_dataloader - - # # Setting the Adam optimizer with hyper-param - self.optim = flow.optim.Adam(self.model.parameters(), lr=lr, betas=betas, weight_decay=weight_decay) - self.optim_schedule = ScheduledOptim(self.optim, self.bert.hidden, n_warmup_steps=warmup_steps) - - # Using Negative Log Likelihood Loss function for predicting the masked_token - self.criterion = nn.NLLLoss(ignore_index=0) - self.criterion = self.criterion.to(self.device) - - self.log_freq = log_freq - print("Total Parameters:", sum([p.nelement() for p in self.model.parameters()])) - - def train(self, epoch): - self.iteration(epoch, self.train_data) - - def test(self, epoch): - self.iteration(epoch, self.test_data, train=False) - - def iteration(self, epoch, data_loader, train=True): - """ - loop over the data_loader for training or testing - if on train status, backward operation is activated - and also auto save the model every peoch - - :param epoch: current epoch index - :param data_loader: torch.utils.data.DataLoader for iteration - :param train: boolean value of is train or test - :return: None - """ - str_code = "train" if train else "test" - - # Setting the tqdm progress bar - data_iter = tqdm.tqdm(enumerate(data_loader), - desc="EP_%s:%d" % (str_code, epoch), - total=len(data_loader), - bar_format="{l_bar}{r_bar}") - - avg_loss = 0.0 - total_correct = 0 - total_element = 0 - - - for i, data in data_iter: - for key, value in data.items(): - data[str(key)] = flow.Tensor(value.numpy(), dtype=flow.int64, device=self.device) - # # 0. batch_data will be sent into the device(GPU or cpu) - data = {key: value.to(device=self.device) for key, value in data.items()} - - # 1. forward the next_sentence_prediction and masked_lm model - next_sent_output, mask_lm_output = self.model.forward(data["bert_input"], data["segment_label"]) - - # 2-1. NLL(negative log likelihood) loss of is_next classification result - next_loss = self.criterion(next_sent_output, data["is_next"]) - - # 2-2. NLLLoss of predicting masked token word - mask_loss = self.criterion(mask_lm_output.transpose(1, 2), data["bert_label"]) - - # 2-3. Adding next_loss and mask_loss : 3.4 Pre-training Procedure - loss = next_loss + mask_loss - - # 3. backward and optimization only in train - if train: - loss.backward() - self.optim_schedule.step_and_update_lr() - self.optim_schedule.zero_grad() - - flow.save(self.bert.state_dict(), "checkpoints/bert_%d_loss_%f" % (i, loss.numpy().item())) - - # next sentence prediction accuracy - correct = next_sent_output.argmax(dim=-1).eq(data["is_next"]).sum().numpy().item() - avg_loss += loss.numpy().item() - total_correct += correct - total_element += data["is_next"].nelement() - - post_fix = { - "epoch": epoch, - "iter": i, - "avg_loss": avg_loss / (i + 1), - "avg_acc": total_correct / total_element * 100, - "loss": loss.numpy().item() - } - - if i % self.log_freq == 0: - data_iter.write(str(post_fix)) - - print("EP%d_%s, avg_loss=" % (epoch, str_code), avg_loss / len(data_iter), "total_acc=", - total_correct * 100.0 / total_element) - - - def save(self, epoch, file_path="checkpoints"): - """ - Saving the current BERT model on file_path - - :param epoch: current epoch number - :param file_path: model output path which gonna be file_path+"ep%d" % epoch - :return: final_output_path - """ - output_path = file_path + ".ep%d" % epoch - flow.save(self.bert.state_dict(), output_path) - print("EP:%d Model Saved on:" % epoch, output_path) - return output_path