SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

Complete warning + cleanup

Browse Source
This commit is contained in:
LysandreJik
2019-12-10 14:33:24 -05:00
parent dc4e9e5cb3
commit 6a73382706
2 changed files with 2 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
examples/run_squad.py
View File

@@ -299,7 +299,6 @@ def evaluate(args, model, tokenizer, prefix=""):
# XLNet and XLM use a more complex post-processing procedure # XLNet and XLM use a more complex post-processing procedure
if args.model_type in ['xlnet', 'xlm']: if args.model_type in ['xlnet', 'xlm']:
start_n_top = model.config.start_n_top if hasattr(model, "config") else model.module.config.start_n_top start_n_top = model.config.start_n_top if hasattr(model, "config") else model.module.config.start_n_top
end_n_top = model.config.end_n_top if hasattr(model, "config") else model.module.config.end_n_top end_n_top = model.config.end_n_top if hasattr(model, "config") else model.module.config.end_n_top

4
transformers/tokenization_utils.py
View File

@@ -226,7 +226,7 @@ class PreTrainedTokenizer(object):
self.max_len = max_len if max_len is not None else int(1e12) self.max_len = max_len if max_len is not None else int(1e12)
# Padding side is right by default and over-riden in subclsses. If specified in the kwargs, it is changed. # Padding side is right by default and over-riden in subclasses. If specified in the kwargs, it is changed.
self.padding_side = kwargs.pop('padding_side', self.padding_side) self.padding_side = kwargs.pop('padding_side', self.padding_side)
# Added tokens # Added tokens
@@ -1003,7 +1003,7 @@ class PreTrainedTokenizer(object):
) )
if pad_to_max_length and max_length is None and self.max_len > 10000: if pad_to_max_length and max_length is None and self.max_len > 10000:
logger.warning("Sequence can't be padded as the maximum ") logger.warning("Sequence can't be padded as no maximum length is specified and the model maximum length is too high.")
if needs_to_be_padded: if needs_to_be_padded:
difference = (max_length if max_length is not None else self.max_len) - len(encoded_inputs["input_ids"]) difference = (max_length if max_length is not None else self.max_len) - len(encoded_inputs["input_ids"])