diff --git a/model.py b/model.py index a4f2507..aadfb1a 100644 --- a/model.py +++ b/model.py @@ -223,7 +223,7 @@ def __init__(self, config): # "UserWarning: functional_call was passed multiple values for tied weights. # This behavior is deprecated and will be an error in future versions" # not 100% sure what this is, so far seems to be harmless. TODO investigate - # *we don't use it becuase in the nGPT paper there was no weight tying of weights* + # *we don't use it because in the nGPT paper there was no weight tying of weights* # self.transformer.wte.weight = self.lm_head.weight # https://paperswithcode.com/method/weight-tying # init all weights