SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

Update no_trainer examples to use new logger (#17044)

Browse Source 
* Propagate and fix imports
This commit is contained in:
Zachary Mueller
2022-05-02 11:56:15 -04:00
committed by GitHub
parent daecae1f1c
commit 35d48db881
13 changed files with 38 additions and 76 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
examples/pytorch/language-modeling/run_clm_no_trainer.py
View File

@@ -39,6 +39,7 @@ from tqdm.auto import tqdm
import transformers
from accelerate import Accelerator, DistributedType
from accelerate.logging import get_logger
from accelerate.utils import set_seed
from huggingface_hub import Repository
from transformers import (
@@ -56,7 +57,7 @@ from transformers.utils import get_full_repo_name
from transformers.utils.versions import require_version
logger = logging.getLogger(__name__)
logger = get_logger(__name__)
require_version("datasets>=1.8.0", "To fix: pip install -r examples/pytorch/language-modeling/requirements.txt")
@@ -234,11 +235,7 @@ def main():
datefmt="%m/%d/%Y %H:%M:%S",
level=logging.INFO,
)
logger.info(accelerator.state)
# Setup logging, we only want one process per machine to log things on the screen.
# accelerator.is_local_main_process is only True for one process per machine.
logger.setLevel(logging.INFO if accelerator.is_local_main_process else logging.ERROR)
logger.info(accelerator.state, main_process_only=False)
if accelerator.is_local_main_process:
datasets.utils.logging.set_verbosity_warning()
transformers.utils.logging.set_verbosity_info()

9
examples/pytorch/language-modeling/run_mlm_no_trainer.py
View File

@@ -39,6 +39,7 @@ from tqdm.auto import tqdm
import transformers
from accelerate import Accelerator, DistributedType
from accelerate.logging import get_logger
from accelerate.utils import set_seed
from huggingface_hub import Repository
from transformers import (
@@ -56,7 +57,7 @@ from transformers.utils import get_full_repo_name
from transformers.utils.versions import require_version
logger = logging.getLogger(__name__)
logger = get_logger(__name__)
require_version("datasets>=1.8.0", "To fix: pip install -r examples/pytorch/language-modeling/requirements.txt")
MODEL_CONFIG_CLASSES = list(MODEL_MAPPING.keys())
MODEL_TYPES = tuple(conf.model_type for conf in MODEL_CONFIG_CLASSES)
@@ -245,11 +246,7 @@ def main():
datefmt="%m/%d/%Y %H:%M:%S",
level=logging.INFO,
)
logger.info(accelerator.state)
# Setup logging, we only want one process per machine to log things on the screen.
# accelerator.is_local_main_process is only True for one process per machine.
logger.setLevel(logging.INFO if accelerator.is_local_main_process else logging.ERROR)
logger.info(accelerator.state, main_process_only=False)
if accelerator.is_local_main_process:
datasets.utils.logging.set_verbosity_warning()
transformers.utils.logging.set_verbosity_info()