From bd41e8292a4bd7db10eb036112019d93c50adcf5 Mon Sep 17 00:00:00 2001 From: Lysandre Date: Thu, 28 Nov 2019 16:03:56 -0500 Subject: [PATCH] Cleanup & Evaluation now works --- examples/run_squad.py | 44 +++++++++++---------------- transformers/data/processors/squad.py | 14 ++------- 2 files changed, 20 insertions(+), 38 deletions(-) diff --git a/examples/run_squad.py b/examples/run_squad.py index 634b566a46..545c3ad55a 100644 --- a/examples/run_squad.py +++ b/examples/run_squad.py @@ -16,7 +16,7 @@ """ Finetuning the library models for question-answering on SQuAD (DistilBERT, Bert, XLM, XLNet).""" from __future__ import absolute_import, division, print_function -from transformers.data.processors.squad import SquadV1Processor +from transformers.data.processors.squad import SquadV1Processor, SquadV2Processor import argparse import logging @@ -45,9 +45,9 @@ from transformers import (WEIGHTS_NAME, BertConfig, XLNetTokenizer, DistilBertConfig, DistilBertForQuestionAnswering, DistilBertTokenizer) -from transformers import AdamW, get_linear_schedule_with_warmup, squad_convert_examples_to_features, read_squad_examples as sread_squad_examples +from transformers import AdamW, get_linear_schedule_with_warmup, squad_convert_examples_to_features -from utils_squad import (RawResult, write_predictions, +from utils_squad import (convert_examples_to_features as old_convert, read_squad_examples as old_read, RawResult, write_predictions, RawResultExtended, write_predictions_extended) # The follwing import is the official SQuAD evaluation script (2.0). @@ -304,28 +304,20 @@ def load_and_cache_examples(args, tokenizer, evaluate=False, output_examples=Fal features = torch.load(cached_features_file) else: logger.info("Creating features from dataset file at %s", input_file) - examples = read_squad_examples(input_file=input_file, - is_training=not evaluate, - version_2_with_negative=args.version_2_with_negative) - keep_n_examples = 1000 - processor = SquadV1Processor() - values = processor.get_dev_examples("examples/squad") - examples = values[:keep_n_examples] - features = squad_convert_examples_to_features(examples=exampless, - tokenizer=tokenizer, - max_seq_length=args.max_seq_length, - doc_stride=args.doc_stride, - max_query_length=args.max_query_length, - is_training=not evaluate, - cls_token_segment_id=2 if args.model_type in ['xlnet'] else 0, - pad_token_segment_id=3 if args.model_type in ['xlnet'] else 0, - cls_token_at_end=True if args.model_type in ['xlnet'] else False, - sequence_a_is_doc=True if args.model_type in ['xlnet'] else False) - print("DONE") - import sys - sys.exit() - + processor = SquadV2Processor() + examples = processor.get_dev_examples("examples/squad") if evaluate else processor.get_train_examples("examples/squad") + features = squad_convert_examples_to_features( + examples=examples, + tokenizer=tokenizer, + max_seq_length=args.max_seq_length, + doc_stride=args.doc_stride, + max_query_length=args.max_query_length, + is_training=not evaluate, + sequence_a_is_doc=True if args.model_type in ['xlnet'] else False + ) + + if args.local_rank in [-1, 0]: logger.info("Saving features into cached file %s", cached_features_file) torch.save(features, cached_features_file) @@ -335,8 +327,8 @@ def load_and_cache_examples(args, tokenizer, evaluate=False, output_examples=Fal # Convert to Tensors and build dataset all_input_ids = torch.tensor([f.input_ids for f in features], dtype=torch.long) - all_input_mask = torch.tensor([f.input_mask for f in features], dtype=torch.long) - all_segment_ids = torch.tensor([f.segment_ids for f in features], dtype=torch.long) + all_input_mask = torch.tensor([f.attention_mask for f in features], dtype=torch.long) + all_segment_ids = torch.tensor([f.token_type_ids for f in features], dtype=torch.long) all_cls_index = torch.tensor([f.cls_index for f in features], dtype=torch.long) all_p_mask = torch.tensor([f.p_mask for f in features], dtype=torch.float) if evaluate: diff --git a/transformers/data/processors/squad.py b/transformers/data/processors/squad.py index 39ee00ae56..3d5a3eca80 100644 --- a/transformers/data/processors/squad.py +++ b/transformers/data/processors/squad.py @@ -74,26 +74,16 @@ def _is_whitespace(c): def squad_convert_examples_to_features(examples, tokenizer, max_seq_length, doc_stride, max_query_length, is_training, - cls_token_at_end=True, - cls_token='[CLS]', sep_token='[SEP]', pad_token=0, - sequence_a_segment_id=0, sequence_b_segment_id=1, - cls_token_segment_id=0, pad_token_segment_id=0, - mask_padding_with_zero=True, sequence_a_is_doc=False): """Loads a data file into a list of `InputBatch`s.""" - cls_token = tokenizer.cls_token - sep_token = tokenizer.sep_token - # Defining helper methods unique_id = 1000000000 features = [] - new_features = [] for (example_index, example) in enumerate(tqdm(examples)): if is_training and not example.is_impossible: # Get start and end position - answer_length = len(example.answer_text) start_position = example.start_position end_position = example.end_position @@ -227,7 +217,7 @@ def squad_convert_examples_to_features(examples, tokenizer, max_seq_length, end_position = tok_end_position - doc_start + doc_offset - new_features.append(NewSquadFeatures( + features.append(NewSquadFeatures( span['input_ids'], span['attention_mask'], span['token_type_ids'], @@ -247,7 +237,7 @@ def squad_convert_examples_to_features(examples, tokenizer, max_seq_length, unique_id += 1 - return new_features + return features class SquadProcessor(DataProcessor):