SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

Remove unused arguments in Multiple Choice example (#4853)

Browse Source 
* Remove unused arguments

* Formatting

* Remove second todo comment
This commit is contained in:
Sylvain Gugger
2020-06-09 20:05:09 -04:00
committed by GitHub
parent 29c36e9f36
commit e8db8b845a
1 changed files with 4 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files

32
examples/multiple-choice/utils_multiple_choice.py
View File

@@ -121,16 +121,7 @@ if is_torch_available():
else:
examples = processor.get_train_examples(data_dir)
logger.info("Training examples: %s", len(examples))
# TODO clean up all this to leverage built-in features of tokenizers
self.features = convert_examples_to_features(
examples,
label_list,
max_seq_length,
tokenizer,
pad_on_left=bool(tokenizer.padding_side == "left"),
pad_token=tokenizer.pad_token_id,
pad_token_segment_id=tokenizer.pad_token_type_id,
)
self.features = convert_examples_to_features(examples, label_list, max_seq_length, tokenizer,)
logger.info("Saving features into cached file %s", cached_features_file)
torch.save(self.features, cached_features_file)
@@ -172,16 +163,8 @@ if is_tf_available():
else:
examples = processor.get_train_examples(data_dir)
logger.info("Training examples: %s", len(examples))
# TODO clean up all this to leverage built-in features of tokenizers
self.features = convert_examples_to_features(
examples,
label_list,
max_seq_length,
tokenizer,
pad_on_left=bool(tokenizer.padding_side == "left"),
pad_token=tokenizer.pad_token_id,
pad_token_segment_id=tokenizer.pad_token_type_id,
)
self.features = convert_examples_to_features(examples, label_list, max_seq_length, tokenizer,)
def gen():
for (ex_index, ex) in tqdm.tqdm(enumerate(self.features), desc="convert examples to features"):
@@ -506,14 +489,7 @@ class ArcProcessor(DataProcessor):
def convert_examples_to_features(
examples: List[InputExample],
label_list: List[str],
max_length: int,
tokenizer: PreTrainedTokenizer,
pad_token_segment_id=0,
pad_on_left=False,
pad_token=0,
mask_padding_with_zero=True,
examples: List[InputExample], label_list: List[str], max_length: int, tokenizer: PreTrainedTokenizer,
) -> List[InputFeatures]:
"""
Loads a data file into a list of `InputFeatures`