Add get_vocab method to PretrainedTokenizer
This commit is contained in:
@@ -147,6 +147,9 @@ class CTRLTokenizer(PreTrainedTokenizer):
|
||||
def vocab_size(self):
|
||||
return len(self.encoder)
|
||||
|
||||
def get_vocab(self):
|
||||
return dict(self.encoder, **self.added_tokens_encoder)
|
||||
|
||||
def bpe(self, token):
|
||||
if token in self.cache:
|
||||
return self.cache[token]
|
||||
|
||||
Reference in New Issue
Block a user