Remove unused arguments in Multiple Choice example (#4853)
* Remove unused arguments * Formatting * Remove second todo comment
This commit is contained in:
@@ -121,16 +121,7 @@ if is_torch_available():
|
|||||||
else:
|
else:
|
||||||
examples = processor.get_train_examples(data_dir)
|
examples = processor.get_train_examples(data_dir)
|
||||||
logger.info("Training examples: %s", len(examples))
|
logger.info("Training examples: %s", len(examples))
|
||||||
# TODO clean up all this to leverage built-in features of tokenizers
|
self.features = convert_examples_to_features(examples, label_list, max_seq_length, tokenizer,)
|
||||||
self.features = convert_examples_to_features(
|
|
||||||
examples,
|
|
||||||
label_list,
|
|
||||||
max_seq_length,
|
|
||||||
tokenizer,
|
|
||||||
pad_on_left=bool(tokenizer.padding_side == "left"),
|
|
||||||
pad_token=tokenizer.pad_token_id,
|
|
||||||
pad_token_segment_id=tokenizer.pad_token_type_id,
|
|
||||||
)
|
|
||||||
logger.info("Saving features into cached file %s", cached_features_file)
|
logger.info("Saving features into cached file %s", cached_features_file)
|
||||||
torch.save(self.features, cached_features_file)
|
torch.save(self.features, cached_features_file)
|
||||||
|
|
||||||
@@ -172,16 +163,8 @@ if is_tf_available():
|
|||||||
else:
|
else:
|
||||||
examples = processor.get_train_examples(data_dir)
|
examples = processor.get_train_examples(data_dir)
|
||||||
logger.info("Training examples: %s", len(examples))
|
logger.info("Training examples: %s", len(examples))
|
||||||
# TODO clean up all this to leverage built-in features of tokenizers
|
|
||||||
self.features = convert_examples_to_features(
|
self.features = convert_examples_to_features(examples, label_list, max_seq_length, tokenizer,)
|
||||||
examples,
|
|
||||||
label_list,
|
|
||||||
max_seq_length,
|
|
||||||
tokenizer,
|
|
||||||
pad_on_left=bool(tokenizer.padding_side == "left"),
|
|
||||||
pad_token=tokenizer.pad_token_id,
|
|
||||||
pad_token_segment_id=tokenizer.pad_token_type_id,
|
|
||||||
)
|
|
||||||
|
|
||||||
def gen():
|
def gen():
|
||||||
for (ex_index, ex) in tqdm.tqdm(enumerate(self.features), desc="convert examples to features"):
|
for (ex_index, ex) in tqdm.tqdm(enumerate(self.features), desc="convert examples to features"):
|
||||||
@@ -506,14 +489,7 @@ class ArcProcessor(DataProcessor):
|
|||||||
|
|
||||||
|
|
||||||
def convert_examples_to_features(
|
def convert_examples_to_features(
|
||||||
examples: List[InputExample],
|
examples: List[InputExample], label_list: List[str], max_length: int, tokenizer: PreTrainedTokenizer,
|
||||||
label_list: List[str],
|
|
||||||
max_length: int,
|
|
||||||
tokenizer: PreTrainedTokenizer,
|
|
||||||
pad_token_segment_id=0,
|
|
||||||
pad_on_left=False,
|
|
||||||
pad_token=0,
|
|
||||||
mask_padding_with_zero=True,
|
|
||||||
) -> List[InputFeatures]:
|
) -> List[InputFeatures]:
|
||||||
"""
|
"""
|
||||||
Loads a data file into a list of `InputFeatures`
|
Loads a data file into a list of `InputFeatures`
|
||||||
|
|||||||
Reference in New Issue
Block a user