SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

Add option to set max_len in run_ner (#12929)

Browse Source
This commit is contained in:
Sylvain Gugger
2021-07-28 09:38:12 -04:00
committed by GitHub
parent 1486fb8108
commit fd85734e0e
1 changed files with 8 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
examples/pytorch/token-classification/run_ner.py
View File

@@ -123,6 +123,13 @@ class DataTrainingArguments:
default=None, default=None,
metadata={"help": "The number of processes to use for the preprocessing."}, metadata={"help": "The number of processes to use for the preprocessing."},
) )
max_seq_length: int = field(
default=None,
metadata={
"help": "The maximum total input sequence length after tokenization. If set, sequences longer "
"than this will be truncated, sequences shorter will be padded."
},
)
pad_to_max_length: bool = field( pad_to_max_length: bool = field(
default=False, default=False,
metadata={ metadata={
@@ -358,6 +365,7 @@ def main():
examples[text_column_name], examples[text_column_name],
padding=padding, padding=padding,
truncation=True, truncation=True,
max_length=data_args.max_seq_length,
# We use this argument because the texts in our dataset are lists of words (with a label for each word). # We use this argument because the texts in our dataset are lists of words (with a label for each word).
is_split_into_words=True, is_split_into_words=True,
) )