SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

Make sure dynamic objects can be saved and reloaded (#21008)

Browse Source 
* Make sure dynamic objects can be saved and reloaded

* Remove processor test
This commit is contained in:
Sylvain Gugger
2023-01-05 07:30:25 -05:00
committed by GitHub
parent bf82c9b74f
commit 12313838d3
12 changed files with 64 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
tests/models/auto/test_processor_auto.py
View File

@@ -157,12 +157,12 @@ class AutoFeatureExtractorTest(unittest.TestCase):
self.assertEqual(tokenizer.__class__.__name__, "NewTokenizerFast")
# Test we can also load the slow version
processor = AutoProcessor.from_pretrained(
new_processor = AutoProcessor.from_pretrained(
"hf-internal-testing/test_dynamic_processor", trust_remote_code=True, use_fast=False
)
tokenizer = processor.tokenizer
self.assertTrue(tokenizer.special_attribute_present)
self.assertEqual(tokenizer.__class__.__name__, "NewTokenizer")
new_tokenizer = new_processor.tokenizer
self.assertTrue(new_tokenizer.special_attribute_present)
self.assertEqual(new_tokenizer.__class__.__name__, "NewTokenizer")
else:
self.assertEqual(tokenizer.__class__.__name__, "NewTokenizer")