SUMIN/HuggingFace_transformer
1
0
Fork 0
Code Issues Pull Requests Actions 7 Packages Projects Releases Wiki Activity

update conversion scripts

Browse Source
This commit is contained in:
thomwolf
2019-09-05 02:17:50 +02:00
parent d77abd4d08
commit 121f88cae3
9 changed files with 20 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
pytorch_transformers/__init__.py
View File

@@ -1,4 +1,15 @@
__version__ = "1.2.0"
# Work around to update TensorFlow's absl.logging threshold which alters the
# default Python logging output behavior when present.
# see: https://github.com/abseil/abseil-py/issues/99
# and: https://github.com/tensorflow/tensorflow/issues/26691#issuecomment-500369493
try:
import absl.logging
absl.logging.set_verbosity('info')
absl.logging.set_stderrthreshold('info')
absl.logging._warn_preinit_stderr = False
except:
pass
# Tokenizer
from .tokenization_utils import (PreTrainedTokenizer)

2
pytorch_transformers/convert_gpt2_checkpoint_to_pytorch.py
View File

@@ -21,7 +21,7 @@ from io import open
import torch
from pytorch_transformers.modeling_gpt2 import (CONFIG_NAME, WEIGHTS_NAME,
from pytorch_transformers import (CONFIG_NAME, WEIGHTS_NAME,
GPT2Config,
GPT2Model,
load_tf_weights_in_gpt2)

2
pytorch_transformers/convert_openai_checkpoint_to_pytorch.py
View File

@@ -21,7 +21,7 @@ from io import open
import torch
from pytorch_transformers.modeling_openai import (CONFIG_NAME, WEIGHTS_NAME,
from pytorch_transformers import (CONFIG_NAME, WEIGHTS_NAME,
OpenAIGPTConfig,
OpenAIGPTModel,
load_tf_weights_in_openai_gpt)

2
pytorch_transformers/convert_pytorch_checkpoint_to_tf.py
View File

@@ -20,7 +20,7 @@ import argparse
import torch
import numpy as np
import tensorflow as tf
from pytorch_transformers.modeling import BertModel
from pytorch_transformers import BertModel
def convert_pytorch_checkpoint_to_tf(model:BertModel, ckpt_dir:str, model_name:str):

4
pytorch_transformers/convert_roberta_checkpoint_to_pytorch.py
View File

@@ -23,12 +23,12 @@ import torch
from fairseq.models.roberta import RobertaModel as FairseqRobertaModel
from fairseq.modules import TransformerSentenceEncoderLayer
from pytorch_transformers.modeling_bert import (BertConfig, BertEncoder,
from pytorch_transformers import (BertConfig, BertEncoder,
BertIntermediate, BertLayer,
BertModel, BertOutput,
BertSelfAttention,
BertSelfOutput)
from pytorch_transformers.modeling_roberta import (RobertaEmbeddings,
from pytorch_transformers import (RobertaEmbeddings,
RobertaForMaskedLM,
RobertaForSequenceClassification,
RobertaModel)

2
pytorch_transformers/convert_tf_checkpoint_to_pytorch.py
View File

@@ -21,7 +21,7 @@ from __future__ import print_function
import argparse
import torch
from pytorch_transformers.modeling_bert import BertConfig, BertForPreTraining, load_tf_weights_in_bert
from pytorch_transformers import BertConfig, BertForPreTraining, load_tf_weights_in_bert
import logging
logging.basicConfig(level=logging.INFO)

2
pytorch_transformers/convert_transfo_xl_checkpoint_to_pytorch.py
View File

@@ -26,7 +26,7 @@ import torch
import pytorch_transformers.tokenization_transfo_xl as data_utils
from pytorch_transformers import CONFIG_NAME, WEIGHTS_NAME
from pytorch_transformers.modeling_transfo_xl import (TransfoXLConfig, TransfoXLLMHeadModel,
from pytorch_transformers import (TransfoXLConfig, TransfoXLLMHeadModel,
load_tf_weights_in_transfo_xl)
from pytorch_transformers.tokenization_transfo_xl import (CORPUS_NAME, VOCAB_FILES_NAMES)

2
pytorch_transformers/convert_xlm_checkpoint_to_pytorch.py
View File

@@ -23,7 +23,7 @@ from io import open
import torch
import numpy
from pytorch_transformers.modeling_utils import CONFIG_NAME, WEIGHTS_NAME
from pytorch_transformers import CONFIG_NAME, WEIGHTS_NAME
from pytorch_transformers.tokenization_xlm import VOCAB_FILES_NAMES
import logging

2
pytorch_transformers/convert_xlnet_checkpoint_to_pytorch.py
View File

@@ -22,7 +22,7 @@ import os
import argparse
import torch
from pytorch_transformers.modeling_xlnet import (CONFIG_NAME, WEIGHTS_NAME,
from pytorch_transformers import (CONFIG_NAME, WEIGHTS_NAME,
XLNetConfig,
XLNetLMHeadModel, XLNetForQuestionAnswering,
XLNetForSequenceClassification,