SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

allowing to load small number of examples

Browse Source
This commit is contained in:
thomwolf
2018-11-05 13:21:24 +01:00
parent a725db4f6c
commit 5622d8320f
1 changed files with 3 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
tensorflow_code/run_squad.py
View File

@@ -207,7 +207,7 @@ class InputFeatures(object):
self.end_position = end_position self.end_position = end_position
def read_squad_examples(input_file, is_training): def read_squad_examples(input_file, is_training, max_num=-1):
"""Read a SQuAD json file into a list of SquadExample.""" """Read a SQuAD json file into a list of SquadExample."""
with tf.gfile.Open(input_file, "r") as reader: with tf.gfile.Open(input_file, "r") as reader:
input_data = json.load(reader)["data"] input_data = json.load(reader)["data"]
@@ -219,6 +219,8 @@ def read_squad_examples(input_file, is_training):
examples = [] examples = []
for entry in input_data: for entry in input_data:
if max_num != -1 and len(examples) > max_num:
break
for paragraph in entry["paragraphs"]: for paragraph in entry["paragraphs"]:
paragraph_text = paragraph["context"] paragraph_text = paragraph["context"]
doc_tokens = [] doc_tokens = []