SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

TransfoXL is deprecated, don't keep it in tested examples! (#37707)

Browse Source 
* TransfoXL is deprecated, so we should remove it from examples that get tested

* Remove the tokenizer too

* Trigger tests
This commit is contained in:
Matt
2025-04-23 14:59:38 +01:00
committed by GitHub
parent be9b0e8521
commit 9ec8be56dd
1 changed files with 1 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
examples/pytorch/text-generation/run_generation.py
View File

@@ -38,8 +38,6 @@ from transformers import (
OpenAIGPTLMHeadModel, OpenAIGPTLMHeadModel,
OpenAIGPTTokenizer, OpenAIGPTTokenizer,
OPTForCausalLM, OPTForCausalLM,
TransfoXLLMHeadModel,
TransfoXLTokenizer,
XLMTokenizer, XLMTokenizer,
XLMWithLMHeadModel, XLMWithLMHeadModel,
XLNetLMHeadModel, XLNetLMHeadModel,
@@ -62,7 +60,6 @@ MODEL_CLASSES = {
"ctrl": (CTRLLMHeadModel, CTRLTokenizer), "ctrl": (CTRLLMHeadModel, CTRLTokenizer),
"openai-gpt": (OpenAIGPTLMHeadModel, OpenAIGPTTokenizer), "openai-gpt": (OpenAIGPTLMHeadModel, OpenAIGPTTokenizer),
"xlnet": (XLNetLMHeadModel, XLNetTokenizer), "xlnet": (XLNetLMHeadModel, XLNetTokenizer),
"transfo-xl": (TransfoXLLMHeadModel, TransfoXLTokenizer),
"xlm": (XLMWithLMHeadModel, XLMTokenizer), "xlm": (XLMWithLMHeadModel, XLMTokenizer),
"gptj": (GPTJForCausalLM, AutoTokenizer), "gptj": (GPTJForCausalLM, AutoTokenizer),
"bloom": (BloomForCausalLM, BloomTokenizerFast), "bloom": (BloomForCausalLM, BloomTokenizerFast),
@@ -368,9 +365,6 @@ def main():
prepare_input = PREPROCESSING_FUNCTIONS.get(args.model_type) prepare_input = PREPROCESSING_FUNCTIONS.get(args.model_type)
preprocessed_prompt_text = prepare_input(args, model, tokenizer, prompt_text) preprocessed_prompt_text = prepare_input(args, model, tokenizer, prompt_text)
if model.__class__.__name__ in ["TransfoXLLMHeadModel"]:
tokenizer_kwargs = {"add_space_before_punct_symbol": True}
else:
tokenizer_kwargs = {} tokenizer_kwargs = {}
encoded_prompt = tokenizer.encode( encoded_prompt = tokenizer.encode(