SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

Removed an old irrelevant comment

Browse Source
This commit is contained in:
Matthew Carrigan
2019-03-21 13:36:41 +00:00
parent 29a392fbcf
commit 6a9038ba53
1 changed files with 1 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
examples/lm_finetuning/pregenerate_training_data.py
View File

@@ -151,11 +151,7 @@ def create_instances_from_document(
is_random_next = True is_random_next = True
target_b_length = target_seq_length - len(tokens_a) target_b_length = target_seq_length - len(tokens_a)
# This should rarely go for more than one iteration for large # Sample a random document, with longer docs being sampled more frequently
# corpora. However, just to be careful, we try to make sure that
# the random document is not the same as the document
# we're processing.
# random_document = get_random_doc(all_documents, document, doc_weights)
random_document = doc_database.sample_doc(current_idx=doc_idx, sentence_weighted=True) random_document = doc_database.sample_doc(current_idx=doc_idx, sentence_weighted=True)
random_start = randint(0, len(random_document) - 1) random_start = randint(0, len(random_document) - 1)