SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

Remove only_first

Browse Source
This commit is contained in:
LysandreJik
2019-12-04 16:26:45 -05:00
parent f7e4a7cdfa
commit 33508ae310
1 changed files with 5 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
transformers/data/processors/squad.py
View File

@@ -300,29 +300,29 @@ class SquadProcessor(DataProcessor):
return examples return examples
def get_train_examples(self, data_dir, only_first=None): def get_train_examples(self, data_dir):
"""See base class.""" """See base class."""
if self.train_file is None: if self.train_file is None:
raise ValueError("SquadProcessor should be instantiated via SquadV1Processor or SquadV2Processor") raise ValueError("SquadProcessor should be instantiated via SquadV1Processor or SquadV2Processor")
with open(os.path.join(data_dir, self.train_file), "r", encoding='utf-8') as reader: with open(os.path.join(data_dir, self.train_file), "r", encoding='utf-8') as reader:
input_data = json.load(reader)["data"] input_data = json.load(reader)["data"]
return self._create_examples(input_data, "train", only_first) return self._create_examples(input_data, "train")
def get_dev_examples(self, data_dir, only_first=None): def get_dev_examples(self, data_dir):
"""See base class.""" """See base class."""
if self.dev_file is None: if self.dev_file is None:
raise ValueError("SquadProcessor should be instantiated via SquadV1Processor or SquadV2Processor") raise ValueError("SquadProcessor should be instantiated via SquadV1Processor or SquadV2Processor")
with open(os.path.join(data_dir, self.dev_file), "r", encoding='utf-8') as reader: with open(os.path.join(data_dir, self.dev_file), "r", encoding='utf-8') as reader:
input_data = json.load(reader)["data"] input_data = json.load(reader)["data"]
return self._create_examples(input_data, "dev", only_first) return self._create_examples(input_data, "dev")
def get_labels(self): def get_labels(self):
"""See base class.""" """See base class."""
return ["0", "1"] return ["0", "1"]
def _create_examples(self, input_data, set_type, only_first=None): def _create_examples(self, input_data, set_type):
"""Creates examples for the training and dev sets.""" """Creates examples for the training and dev sets."""
is_training = set_type == "train" is_training = set_type == "train"
@@ -363,9 +363,6 @@ class SquadProcessor(DataProcessor):
) )
examples.append(example) examples.append(example)
if only_first is not None and len(examples) > only_first:
return examples
return examples return examples
class SquadV1Processor(SquadProcessor): class SquadV1Processor(SquadProcessor):