SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

[distil] fix once for all general logger for scripts

Browse Source
This commit is contained in:
VictorSanh
2019-09-11 14:19:07 +00:00
parent b62abe87c9
commit 32e1332acf
2 changed files with 11 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
examples/distillation/scripts/binarized_data.py
View File

@@ -21,8 +21,12 @@ import random
import time import time
import numpy as np import numpy as np
from pytorch_transformers import BertTokenizer from pytorch_transformers import BertTokenizer
import logging
from examples.distillation.utils import logger logging.basicConfig(format = '%(asctime)s - %(levelname)s - %(name)s - %(message)s',
datefmt = '%m/%d/%Y %H:%M:%S',
level = logging.INFO)
logger = logging.getLogger(__name__)
def main(): def main():
parser = argparse.ArgumentParser(description="Preprocess the data to avoid re-doing it several times by (tokenization + token_to_ids).") parser = argparse.ArgumentParser(description="Preprocess the data to avoid re-doing it several times by (tokenization + token_to_ids).")

6
examples/distillation/scripts/token_counts.py
View File

@@ -18,8 +18,12 @@ Preprocessing script before training DistilBERT.
from collections import Counter from collections import Counter
import argparse import argparse
import pickle import pickle
import logging
from examples.distillation.utils import logger logging.basicConfig(format = '%(asctime)s - %(levelname)s - %(name)s - %(message)s',
datefmt = '%m/%d/%Y %H:%M:%S',
level = logging.INFO)
logger = logging.getLogger(__name__)
if __name__ == '__main__': if __name__ == '__main__':
parser = argparse.ArgumentParser(description="Token Counts for smoothing the masking probabilities in MLM (cf XLM/word2vec)") parser = argparse.ArgumentParser(description="Token Counts for smoothing the masking probabilities in MLM (cf XLM/word2vec)")