From d61602245566b1e42dca9238b3b8a0f23f3fdad1 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Mon, 15 Apr 2019 16:07:45 +0200 Subject: [PATCH] fix openai special tokens loading --- pytorch_pretrained_bert/tokenization_openai.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pytorch_pretrained_bert/tokenization_openai.py b/pytorch_pretrained_bert/tokenization_openai.py index d9713e51eb..7a10271175 100644 --- a/pytorch_pretrained_bert/tokenization_openai.py +++ b/pytorch_pretrained_bert/tokenization_openai.py @@ -87,6 +87,7 @@ class OpenAIGPTTokenizer(object): if pretrained_model_name_or_path in PRETRAINED_VOCAB_ARCHIVE_MAP: vocab_file = PRETRAINED_VOCAB_ARCHIVE_MAP[pretrained_model_name_or_path] merges_file = PRETRAINED_MERGES_ARCHIVE_MAP[pretrained_model_name_or_path] + special_tokens_file = None else: vocab_file = os.path.join(pretrained_model_name_or_path, VOCAB_NAME) merges_file = os.path.join(pretrained_model_name_or_path, MERGES_NAME)