309 lines
7.9 KiB
Python
309 lines
7.9 KiB
Python
# This file is autogenerated by the command `make fix-copies`, do not edit.
|
|
from ..file_utils import requires_backends
|
|
|
|
|
|
class AlbertTokenizerFast:
|
|
def __init__(self, *args, **kwargs):
|
|
requires_backends(self, ["tokenizers"])
|
|
|
|
@classmethod
|
|
def from_pretrained(self, *args, **kwargs):
|
|
requires_backends(self, ["tokenizers"])
|
|
|
|
|
|
class BartTokenizerFast:
|
|
def __init__(self, *args, **kwargs):
|
|
requires_backends(self, ["tokenizers"])
|
|
|
|
@classmethod
|
|
def from_pretrained(self, *args, **kwargs):
|
|
requires_backends(self, ["tokenizers"])
|
|
|
|
|
|
class BarthezTokenizerFast:
|
|
def __init__(self, *args, **kwargs):
|
|
requires_backends(self, ["tokenizers"])
|
|
|
|
@classmethod
|
|
def from_pretrained(self, *args, **kwargs):
|
|
requires_backends(self, ["tokenizers"])
|
|
|
|
|
|
class BertTokenizerFast:
|
|
def __init__(self, *args, **kwargs):
|
|
requires_backends(self, ["tokenizers"])
|
|
|
|
@classmethod
|
|
def from_pretrained(self, *args, **kwargs):
|
|
requires_backends(self, ["tokenizers"])
|
|
|
|
|
|
class CamembertTokenizerFast:
|
|
def __init__(self, *args, **kwargs):
|
|
requires_backends(self, ["tokenizers"])
|
|
|
|
@classmethod
|
|
def from_pretrained(self, *args, **kwargs):
|
|
requires_backends(self, ["tokenizers"])
|
|
|
|
|
|
class ConvBertTokenizerFast:
|
|
def __init__(self, *args, **kwargs):
|
|
requires_backends(self, ["tokenizers"])
|
|
|
|
@classmethod
|
|
def from_pretrained(self, *args, **kwargs):
|
|
requires_backends(self, ["tokenizers"])
|
|
|
|
|
|
class DebertaTokenizerFast:
|
|
def __init__(self, *args, **kwargs):
|
|
requires_backends(self, ["tokenizers"])
|
|
|
|
@classmethod
|
|
def from_pretrained(self, *args, **kwargs):
|
|
requires_backends(self, ["tokenizers"])
|
|
|
|
|
|
class DistilBertTokenizerFast:
|
|
def __init__(self, *args, **kwargs):
|
|
requires_backends(self, ["tokenizers"])
|
|
|
|
@classmethod
|
|
def from_pretrained(self, *args, **kwargs):
|
|
requires_backends(self, ["tokenizers"])
|
|
|
|
|
|
class DPRContextEncoderTokenizerFast:
|
|
def __init__(self, *args, **kwargs):
|
|
requires_backends(self, ["tokenizers"])
|
|
|
|
@classmethod
|
|
def from_pretrained(self, *args, **kwargs):
|
|
requires_backends(self, ["tokenizers"])
|
|
|
|
|
|
class DPRQuestionEncoderTokenizerFast:
|
|
def __init__(self, *args, **kwargs):
|
|
requires_backends(self, ["tokenizers"])
|
|
|
|
@classmethod
|
|
def from_pretrained(self, *args, **kwargs):
|
|
requires_backends(self, ["tokenizers"])
|
|
|
|
|
|
class DPRReaderTokenizerFast:
|
|
def __init__(self, *args, **kwargs):
|
|
requires_backends(self, ["tokenizers"])
|
|
|
|
@classmethod
|
|
def from_pretrained(self, *args, **kwargs):
|
|
requires_backends(self, ["tokenizers"])
|
|
|
|
|
|
class ElectraTokenizerFast:
|
|
def __init__(self, *args, **kwargs):
|
|
requires_backends(self, ["tokenizers"])
|
|
|
|
@classmethod
|
|
def from_pretrained(self, *args, **kwargs):
|
|
requires_backends(self, ["tokenizers"])
|
|
|
|
|
|
class FunnelTokenizerFast:
|
|
def __init__(self, *args, **kwargs):
|
|
requires_backends(self, ["tokenizers"])
|
|
|
|
@classmethod
|
|
def from_pretrained(self, *args, **kwargs):
|
|
requires_backends(self, ["tokenizers"])
|
|
|
|
|
|
class GPT2TokenizerFast:
|
|
def __init__(self, *args, **kwargs):
|
|
requires_backends(self, ["tokenizers"])
|
|
|
|
@classmethod
|
|
def from_pretrained(self, *args, **kwargs):
|
|
requires_backends(self, ["tokenizers"])
|
|
|
|
|
|
class HerbertTokenizerFast:
|
|
def __init__(self, *args, **kwargs):
|
|
requires_backends(self, ["tokenizers"])
|
|
|
|
@classmethod
|
|
def from_pretrained(self, *args, **kwargs):
|
|
requires_backends(self, ["tokenizers"])
|
|
|
|
|
|
class LayoutLMTokenizerFast:
|
|
def __init__(self, *args, **kwargs):
|
|
requires_backends(self, ["tokenizers"])
|
|
|
|
@classmethod
|
|
def from_pretrained(self, *args, **kwargs):
|
|
requires_backends(self, ["tokenizers"])
|
|
|
|
|
|
class LEDTokenizerFast:
|
|
def __init__(self, *args, **kwargs):
|
|
requires_backends(self, ["tokenizers"])
|
|
|
|
@classmethod
|
|
def from_pretrained(self, *args, **kwargs):
|
|
requires_backends(self, ["tokenizers"])
|
|
|
|
|
|
class LongformerTokenizerFast:
|
|
def __init__(self, *args, **kwargs):
|
|
requires_backends(self, ["tokenizers"])
|
|
|
|
@classmethod
|
|
def from_pretrained(self, *args, **kwargs):
|
|
requires_backends(self, ["tokenizers"])
|
|
|
|
|
|
class LxmertTokenizerFast:
|
|
def __init__(self, *args, **kwargs):
|
|
requires_backends(self, ["tokenizers"])
|
|
|
|
@classmethod
|
|
def from_pretrained(self, *args, **kwargs):
|
|
requires_backends(self, ["tokenizers"])
|
|
|
|
|
|
class MBart50TokenizerFast:
|
|
def __init__(self, *args, **kwargs):
|
|
requires_backends(self, ["tokenizers"])
|
|
|
|
@classmethod
|
|
def from_pretrained(self, *args, **kwargs):
|
|
requires_backends(self, ["tokenizers"])
|
|
|
|
|
|
class MBartTokenizerFast:
|
|
def __init__(self, *args, **kwargs):
|
|
requires_backends(self, ["tokenizers"])
|
|
|
|
@classmethod
|
|
def from_pretrained(self, *args, **kwargs):
|
|
requires_backends(self, ["tokenizers"])
|
|
|
|
|
|
class MobileBertTokenizerFast:
|
|
def __init__(self, *args, **kwargs):
|
|
requires_backends(self, ["tokenizers"])
|
|
|
|
@classmethod
|
|
def from_pretrained(self, *args, **kwargs):
|
|
requires_backends(self, ["tokenizers"])
|
|
|
|
|
|
class MPNetTokenizerFast:
|
|
def __init__(self, *args, **kwargs):
|
|
requires_backends(self, ["tokenizers"])
|
|
|
|
@classmethod
|
|
def from_pretrained(self, *args, **kwargs):
|
|
requires_backends(self, ["tokenizers"])
|
|
|
|
|
|
class MT5TokenizerFast:
|
|
def __init__(self, *args, **kwargs):
|
|
requires_backends(self, ["tokenizers"])
|
|
|
|
@classmethod
|
|
def from_pretrained(self, *args, **kwargs):
|
|
requires_backends(self, ["tokenizers"])
|
|
|
|
|
|
class OpenAIGPTTokenizerFast:
|
|
def __init__(self, *args, **kwargs):
|
|
requires_backends(self, ["tokenizers"])
|
|
|
|
@classmethod
|
|
def from_pretrained(self, *args, **kwargs):
|
|
requires_backends(self, ["tokenizers"])
|
|
|
|
|
|
class PegasusTokenizerFast:
|
|
def __init__(self, *args, **kwargs):
|
|
requires_backends(self, ["tokenizers"])
|
|
|
|
@classmethod
|
|
def from_pretrained(self, *args, **kwargs):
|
|
requires_backends(self, ["tokenizers"])
|
|
|
|
|
|
class ReformerTokenizerFast:
|
|
def __init__(self, *args, **kwargs):
|
|
requires_backends(self, ["tokenizers"])
|
|
|
|
@classmethod
|
|
def from_pretrained(self, *args, **kwargs):
|
|
requires_backends(self, ["tokenizers"])
|
|
|
|
|
|
class RetriBertTokenizerFast:
|
|
def __init__(self, *args, **kwargs):
|
|
requires_backends(self, ["tokenizers"])
|
|
|
|
@classmethod
|
|
def from_pretrained(self, *args, **kwargs):
|
|
requires_backends(self, ["tokenizers"])
|
|
|
|
|
|
class RobertaTokenizerFast:
|
|
def __init__(self, *args, **kwargs):
|
|
requires_backends(self, ["tokenizers"])
|
|
|
|
@classmethod
|
|
def from_pretrained(self, *args, **kwargs):
|
|
requires_backends(self, ["tokenizers"])
|
|
|
|
|
|
class SqueezeBertTokenizerFast:
|
|
def __init__(self, *args, **kwargs):
|
|
requires_backends(self, ["tokenizers"])
|
|
|
|
@classmethod
|
|
def from_pretrained(self, *args, **kwargs):
|
|
requires_backends(self, ["tokenizers"])
|
|
|
|
|
|
class T5TokenizerFast:
|
|
def __init__(self, *args, **kwargs):
|
|
requires_backends(self, ["tokenizers"])
|
|
|
|
@classmethod
|
|
def from_pretrained(self, *args, **kwargs):
|
|
requires_backends(self, ["tokenizers"])
|
|
|
|
|
|
class XLMRobertaTokenizerFast:
|
|
def __init__(self, *args, **kwargs):
|
|
requires_backends(self, ["tokenizers"])
|
|
|
|
@classmethod
|
|
def from_pretrained(self, *args, **kwargs):
|
|
requires_backends(self, ["tokenizers"])
|
|
|
|
|
|
class XLNetTokenizerFast:
|
|
def __init__(self, *args, **kwargs):
|
|
requires_backends(self, ["tokenizers"])
|
|
|
|
@classmethod
|
|
def from_pretrained(self, *args, **kwargs):
|
|
requires_backends(self, ["tokenizers"])
|
|
|
|
|
|
class PreTrainedTokenizerFast:
|
|
def __init__(self, *args, **kwargs):
|
|
requires_backends(self, ["tokenizers"])
|
|
|
|
@classmethod
|
|
def from_pretrained(self, *args, **kwargs):
|
|
requires_backends(self, ["tokenizers"])
|